<commit_before>package shell

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"reflect"
	"strings"
	"sync"
	"syscall"

	"github.com/zetamatta/go-findfile"

	"github.com/zetamatta/nyagos/defined"
	"github.com/zetamatta/nyagos/dos"
)

var WildCardExpansionAlways = false

type CommandNotFound struct {
	Name string
	Err  error
}

func (this CommandNotFound) Stringer() string {
	return fmt.Sprintf("'%s' is not recognized as an internal or external command,\noperable program or batch file", this.Name)
}

func (this CommandNotFound) Error() string {
	return this.Stringer()
}

type session struct {
	unreadline []string
}

type CloneCloser interface {
	Clone(context.Context) (context.Context, CloneCloser, error)
	Close() error
}

type Shell struct {
	*session
	Stdout       *os.File
	Stderr       *os.File
	Stdin        *os.File
	Console      io.Writer
	tag          CloneCloser
	IsBackGround bool
}

func (sh *Shell) In() io.Reader          { return sh.Stdin }
func (sh *Shell) Out() io.Writer         { return sh.Stdout }
func (sh *Shell) Err() io.Writer         { return sh.Stderr }
func (sh *Shell) Term() io.Writer        { return sh.Console }
func (sh *Shell) Tag() CloneCloser       { return sh.tag }
func (sh *Shell) SetTag(tag CloneCloser) { sh.tag = tag }

type Cmd struct {
	Shell
	args            []string
	rawArgs         []string
	fullPath        string
	UseShellExecute bool
	Closers         []io.Closer
}

func (cmd *Cmd) Arg(n int) string      { return cmd.args[n] }
func (cmd *Cmd) Args() []string        { return cmd.args }
func (cmd *Cmd) SetArgs(s []string)    { cmd.args = s }
func (cmd *Cmd) RawArg(n int) string   { return cmd.rawArgs[n] }
func (cmd *Cmd) RawArgs() []string     { return cmd.rawArgs }
func (cmd *Cmd) SetRawArgs(s []string) { cmd.rawArgs = s }

var LookCurdirOrder = dos.LookCurdirFirst

func (cmd *Cmd) FullPath() string {
	if cmd.args == nil || len(cmd.args) <= 0 {
		return ""
	}
	if cmd.fullPath == "" {
		cmd.fullPath = dos.LookPath(LookCurdirOrder, cmd.args[0], "NYAGOSPATH")
	}
	return cmd.fullPath
}

func (cmd *Cmd) Close() {
	if cmd.Closers != nil {
		for _, c := range cmd.Closers {
			c.Close()
		}
		cmd.Closers = nil
	}
}

func (sh *Shell) Close() {}

func New() *Shell {
	return &Shell{
		Stdin:   os.Stdin,
		Stdout:  os.Stdout,
		Stderr:  os.Stderr,
		session: &session{},
	}
}

func (sh *Shell) Command() *Cmd {
	cmd := &Cmd{
		Shell: Shell{
			Stdin:   sh.Stdin,
			Stdout:  sh.Stdout,
			Stderr:  sh.Stderr,
			Console: sh.Console,
			tag:     sh.tag,
		},
	}
	if sh.session != nil {
		cmd.session = sh.session
	} else {
		cmd.session = &session{}
	}
	return cmd
}

type ArgsHookT func(ctx context.Context, sh *Shell, args []string) ([]string, error)

var argsHook = func(ctx context.Context, sh *Shell, args []string) ([]string, error) {
	return args, nil
}

func SetArgsHook(argsHook_ ArgsHookT) (rv ArgsHookT) {
	rv, argsHook = argsHook, argsHook_
	return
}

type HookT func(context.Context, *Cmd) (int, bool, error)

var hook = func(context.Context, *Cmd) (int, bool, error) {
	return 0, false, nil
}

func SetHook(hook_ HookT) (rv HookT) {
	rv, hook = hook, hook_
	return
}

var OnCommandNotFound = func(ctx context.Context, cmd *Cmd, err error) error {
	err = &CommandNotFound{cmd.args[0], err}
	return err
}

var LastErrorLevel int

func makeCmdline(args, rawargs []string) string {
	var buffer strings.Builder
	for i, s := range args {
		if i > 0 {
			buffer.WriteRune(' ')
		}
		if (len(rawargs) > i && len(rawargs[i]) > 0 && rawargs[i][0] == '"') || strings.ContainsAny(s, " &|<>\t\"") {
			fmt.Fprintf(&buffer, `"%s"`, strings.Replace(s, `"`, `\"`, -1))
		} else {
			buffer.WriteString(s)
		}
	}
	return buffer.String()
}

var UseSourceRunBatch = true

func encloseWithQuote(fullpath string) string {
	if strings.ContainsRune(fullpath, ' ') {
		var f strings.Builder
		f.WriteRune('"')
		f.WriteString(fullpath)
		f.WriteRune('"')
		return f.String()
	} else {
		return fullpath
	}
}

func (cmd *Cmd) spawnvpSilent(ctx context.Context) (int, error) {
	// command is empty.
	if len(cmd.args) <= 0 {
		return 0, nil
	}
	if defined.DBG {
		print("spawnvpSilent('", cmd.args[0], "')\n")
	}

	// aliases and lua-commands
	if errorlevel, done, err := hook(ctx, cmd); done || err != nil {
		return errorlevel, err
	}

	// command not found hook
	fullpath := cmd.FullPath()
	if fullpath == "" {
		return 255, OnCommandNotFound(ctx, cmd, os.ErrNotExist)
	}
	cmd.args[0] = fullpath

	if defined.DBG {
		print("exec.LookPath(", cmd.args[0], ")==", fullpath, "\n")
	}
	if WildCardExpansionAlways {
		cmd.args = findfile.Globs(cmd.args)
	}
	if cmd.UseShellExecute {
		// GUI Application
		cmdline := makeCmdline(cmd.args[1:], cmd.rawArgs[1:])
		err := dos.ShellExecute("open", fullpath, cmdline, "")
		return 0, err
	}
	if UseSourceRunBatch {
		lowerName := strings.ToLower(cmd.args[0])
		if strings.HasSuffix(lowerName, ".cmd") || strings.HasSuffix(lowerName, ".bat") {
			rawargs := cmd.RawArgs()
			args := make([]string, len(rawargs))
			args[0] = encloseWithQuote(fullpath)
			for i, end := 1, len(rawargs); i < end; i++ {
				args[i] = rawargs[i]
			}
			// Batch files
			return RawSource(args, nil, false, cmd.Stdin, cmd.Stdout, cmd.Stderr)
		}
	}
	// Do not use exec.CommandContext because it cancels background process.
	xcmd := exec.Command(cmd.args[0], cmd.args[1:]...)
	xcmd.Stdin = cmd.Stdin
	xcmd.Stdout = cmd.Stdout
	xcmd.Stderr = cmd.Stderr

	if xcmd.SysProcAttr == nil {
		xcmd.SysProcAttr = new(syscall.SysProcAttr)
	}
	cmdline := makeCmdline(xcmd.Args, cmd.rawArgs)
	if defined.DBG {
		println(cmdline)
	}
	xcmd.SysProcAttr.CmdLine = cmdline
	err := xcmd.Run()
	errorlevel, errorlevelOk := dos.GetErrorLevel(xcmd)
	if errorlevelOk {
		return errorlevel, err
	} else {
		return 255, err
	}
}

type AlreadyReportedError struct {
	Err error
}

func (_ AlreadyReportedError) Error() string {
	return ""
}

func IsAlreadyReported(err error) bool {
	_, ok := err.(AlreadyReportedError)
	return ok
}

func (cmd *Cmd) Spawnvp(ctx context.Context) (int, error) {
	errorlevel, err := cmd.spawnvpSilent(ctx)
	if err != nil && err != io.EOF && !IsAlreadyReported(err) {
		if defined.DBG {
			val := reflect.ValueOf(err)
			fmt.Fprintf(cmd.Stderr, "error-type=%s\n", val.Type())
		}
		fmt.Fprintln(cmd.Stderr, err.Error())
		err = AlreadyReportedError{err}
	}
	return errorlevel, err
}

func (sh *Shell) Spawnlp(ctx context.Context, args, rawargs []string) (int, error) {
	cmd := sh.Command()
	defer cmd.Close()
	cmd.SetArgs(args)
	cmd.SetRawArgs(rawargs)
	return cmd.Spawnvp(ctx)
}

func (sh *Shell) Interpret(ctx context.Context, text string) (errorlevel int, finalerr error) {
	if defined.DBG {
		print("Interpret('", text, "')\n")
	}
	if sh == nil {
		return 255, errors.New("Fatal Error: Interpret: instance is nil")
	}
	errorlevel = 0
	finalerr = nil

	statements, statementsErr := Parse(text)
	if statementsErr != nil {
		if defined.DBG {
			print("Parse Error:", statementsErr.Error(), "\n")
		}
		return 0, statementsErr
	}
	if argsHook != nil {
		if defined.DBG {
			print("call argsHook\n")
		}
		for _, pipeline := range statements {
			for _, state := range pipeline {
				var err error
				state.Args, err = argsHook(ctx, sh, state.Args)
				if err != nil {
					return 255, err
				}
			}
		}
		if defined.DBG {
			print("done argsHook\n")
		}
	}
	for _, pipeline := range statements {
		for i, state := range pipeline {
			if state.Term == "|" && (i+1 >= len(pipeline) || len(pipeline[i+1].Args) <= 0) {
				return 255, errors.New("The syntax of the command is incorrect.")
			}
		}
	}

	for _, pipeline := range statements {

		var pipeIn *os.File = nil
		isBackGround := sh.IsBackGround
		for _, state := range pipeline {
			if state.Term == "&" {
				isBackGround = true
				break
			}
		}
		var wg sync.WaitGroup
		shutdown_immediately := false
		for i, state := range pipeline {
			if defined.DBG {
				print(i, ": pipeline loop(", state.Args[0], ")\n")
			}
			cmd := sh.Command()
			cmd.IsBackGround = isBackGround

			if pipeIn != nil {
				cmd.Stdin = pipeIn
				cmd.Closers = append(cmd.Closers, pipeIn)
				pipeIn = nil
			}

			var err error
			if state.Term[0] == '|' {
				var pipeOut *os.File
				pipeIn, pipeOut, err = os.Pipe()
				cmd.Stdout = pipeOut
				if state.Term == "|&" {
					cmd.Stderr = pipeOut
				}
				cmd.Closers = append(cmd.Closers, pipeOut)
			}

			for _, red := range state.Redirect {
				var fd *os.File
				fd, err = red.OpenOn(cmd)
				if err != nil {
					return 0, err
				}
				defer fd.Close()
			}

			cmd.args = state.Args
			cmd.rawArgs = state.RawArgs
			if i > 0 {
				cmd.IsBackGround = true
			}
			if len(pipeline) == 1 && dos.IsGui(cmd.FullPath()) {
				cmd.UseShellExecute = true
			}
			if i == len(pipeline)-1 && state.Term != "&" {
				// foreground execution.
				errorlevel, finalerr = cmd.Spawnvp(ctx)
				LastErrorLevel = errorlevel
				cmd.Close()
			} else {
				// background
				if !isBackGround {
					wg.Add(1)
				}
				newctx := ctx
				if tag := cmd.Tag(); tag != nil {
					var newtag CloneCloser
					if newctx, newtag, err = tag.Clone(ctx); err != nil {
						fmt.Fprintln(os.Stderr, err.Error())
						return -1, err
					} else {
						cmd.SetTag(newtag)
					}
				}
				go func(ctx1 context.Context, cmd1 *Cmd) {
					if !isBackGround {
						defer wg.Done()
					}
					cmd1.Spawnvp(ctx1)
					if tag := cmd1.Tag(); tag != nil {
						if err := tag.Close(); err != nil {
							fmt.Fprintln(os.Stderr, err.Error())
						}
					}
					cmd1.Close()
				}(newctx, cmd)
			}
		}
		if !isBackGround {
			wg.Wait()
			if shutdown_immediately {
				return errorlevel, nil
			}
			if len(pipeline) > 0 {
				switch pipeline[len(pipeline)-1].Term {
				case "&&":
					if errorlevel != 0 {
						return errorlevel, nil
					}
				case "||":
					if errorlevel == 0 {
						return errorlevel, nil
					}
				}
			}
		}
	}
	return
}
<commit_msg>Fix: it sometimes failed to execute GUI application on symblic linked folder<commit_after>package shell

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"reflect"
	"strings"
	"sync"
	"syscall"

	"github.com/zetamatta/go-findfile"

	"github.com/zetamatta/nyagos/defined"
	"github.com/zetamatta/nyagos/dos"
)

const haveToEvalSymlinkError = syscall.Errno(4294967294)

var WildCardExpansionAlways = false

type CommandNotFound struct {
	Name string
	Err  error
}

func (this CommandNotFound) Stringer() string {
	return fmt.Sprintf("'%s' is not recognized as an internal or external command,\noperable program or batch file", this.Name)
}

func (this CommandNotFound) Error() string {
	return this.Stringer()
}

type session struct {
	unreadline []string
}

type CloneCloser interface {
	Clone(context.Context) (context.Context, CloneCloser, error)
	Close() error
}

type Shell struct {
	*session
	Stdout       *os.File
	Stderr       *os.File
	Stdin        *os.File
	Console      io.Writer
	tag          CloneCloser
	IsBackGround bool
}

func (sh *Shell) In() io.Reader          { return sh.Stdin }
func (sh *Shell) Out() io.Writer         { return sh.Stdout }
func (sh *Shell) Err() io.Writer         { return sh.Stderr }
func (sh *Shell) Term() io.Writer        { return sh.Console }
func (sh *Shell) Tag() CloneCloser       { return sh.tag }
func (sh *Shell) SetTag(tag CloneCloser) { sh.tag = tag }

type Cmd struct {
	Shell
	args            []string
	rawArgs         []string
	fullPath        string
	UseShellExecute bool
	Closers         []io.Closer
}

func (cmd *Cmd) Arg(n int) string      { return cmd.args[n] }
func (cmd *Cmd) Args() []string        { return cmd.args }
func (cmd *Cmd) SetArgs(s []string)    { cmd.args = s }
func (cmd *Cmd) RawArg(n int) string   { return cmd.rawArgs[n] }
func (cmd *Cmd) RawArgs() []string     { return cmd.rawArgs }
func (cmd *Cmd) SetRawArgs(s []string) { cmd.rawArgs = s }

var LookCurdirOrder = dos.LookCurdirFirst

func (cmd *Cmd) FullPath() string {
	if cmd.args == nil || len(cmd.args) <= 0 {
		return ""
	}
	if cmd.fullPath == "" {
		cmd.fullPath = dos.LookPath(LookCurdirOrder, cmd.args[0], "NYAGOSPATH")
	}
	return cmd.fullPath
}

func (cmd *Cmd) Close() {
	if cmd.Closers != nil {
		for _, c := range cmd.Closers {
			c.Close()
		}
		cmd.Closers = nil
	}
}

func (sh *Shell) Close() {}

func New() *Shell {
	return &Shell{
		Stdin:   os.Stdin,
		Stdout:  os.Stdout,
		Stderr:  os.Stderr,
		session: &session{},
	}
}

func (sh *Shell) Command() *Cmd {
	cmd := &Cmd{
		Shell: Shell{
			Stdin:   sh.Stdin,
			Stdout:  sh.Stdout,
			Stderr:  sh.Stderr,
			Console: sh.Console,
			tag:     sh.tag,
		},
	}
	if sh.session != nil {
		cmd.session = sh.session
	} else {
		cmd.session = &session{}
	}
	return cmd
}

type ArgsHookT func(ctx context.Context, sh *Shell, args []string) ([]string, error)

var argsHook = func(ctx context.Context, sh *Shell, args []string) ([]string, error) {
	return args, nil
}

func SetArgsHook(argsHook_ ArgsHookT) (rv ArgsHookT) {
	rv, argsHook = argsHook, argsHook_
	return
}

type HookT func(context.Context, *Cmd) (int, bool, error)

var hook = func(context.Context, *Cmd) (int, bool, error) {
	return 0, false, nil
}

func SetHook(hook_ HookT) (rv HookT) {
	rv, hook = hook, hook_
	return
}

var OnCommandNotFound = func(ctx context.Context, cmd *Cmd, err error) error {
	err = &CommandNotFound{cmd.args[0], err}
	return err
}

var LastErrorLevel int

func makeCmdline(args, rawargs []string) string {
	var buffer strings.Builder
	for i, s := range args {
		if i > 0 {
			buffer.WriteRune(' ')
		}
		if (len(rawargs) > i && len(rawargs[i]) > 0 && rawargs[i][0] == '"') || strings.ContainsAny(s, " &|<>\t\"") {
			fmt.Fprintf(&buffer, `"%s"`, strings.Replace(s, `"`, `\"`, -1))
		} else {
			buffer.WriteString(s)
		}
	}
	return buffer.String()
}

var UseSourceRunBatch = true

func encloseWithQuote(fullpath string) string {
	if strings.ContainsRune(fullpath, ' ') {
		var f strings.Builder
		f.WriteRune('"')
		f.WriteString(fullpath)
		f.WriteRune('"')
		return f.String()
	} else {
		return fullpath
	}
}

func (cmd *Cmd) spawnvpSilent(ctx context.Context) (int, error) {
	// command is empty.
	if len(cmd.args) <= 0 {
		return 0, nil
	}
	if defined.DBG {
		print("spawnvpSilent('", cmd.args[0], "')\n")
	}

	// aliases and lua-commands
	if errorlevel, done, err := hook(ctx, cmd); done || err != nil {
		return errorlevel, err
	}

	// command not found hook
	fullpath := cmd.FullPath()
	if fullpath == "" {
		return 255, OnCommandNotFound(ctx, cmd, os.ErrNotExist)
	}
	cmd.args[0] = fullpath

	if defined.DBG {
		print("exec.LookPath(", cmd.args[0], ")==", fullpath, "\n")
	}
	if WildCardExpansionAlways {
		cmd.args = findfile.Globs(cmd.args)
	}
	if cmd.UseShellExecute {
		// GUI Application
		cmdline := makeCmdline(cmd.args[1:], cmd.rawArgs[1:])
		err := dos.ShellExecute("open", fullpath, cmdline, "")
		if err == haveToEvalSymlinkError {
			fullpath, err = filepath.EvalSymlinks(fullpath)
			if err == nil {
				err = dos.ShellExecute("open", fullpath, cmdline, "")
			}
		}
		return 0, err
	}
	if UseSourceRunBatch {
		lowerName := strings.ToLower(cmd.args[0])
		if strings.HasSuffix(lowerName, ".cmd") || strings.HasSuffix(lowerName, ".bat") {
			rawargs := cmd.RawArgs()
			args := make([]string, len(rawargs))
			args[0] = encloseWithQuote(fullpath)
			for i, end := 1, len(rawargs); i < end; i++ {
				args[i] = rawargs[i]
			}
			// Batch files
			return RawSource(args, nil, false, cmd.Stdin, cmd.Stdout, cmd.Stderr)
		}
	}
	// Do not use exec.CommandContext because it cancels background process.
	xcmd := exec.Command(cmd.args[0], cmd.args[1:]...)
	xcmd.Stdin = cmd.Stdin
	xcmd.Stdout = cmd.Stdout
	xcmd.Stderr = cmd.Stderr

	if xcmd.SysProcAttr == nil {
		xcmd.SysProcAttr = new(syscall.SysProcAttr)
	}
	cmdline := makeCmdline(xcmd.Args, cmd.rawArgs)
	if defined.DBG {
		println(cmdline)
	}
	xcmd.SysProcAttr.CmdLine = cmdline
	err := xcmd.Run()
	errorlevel, errorlevelOk := dos.GetErrorLevel(xcmd)
	if errorlevelOk {
		return errorlevel, err
	} else {
		return 255, err
	}
}

type AlreadyReportedError struct {
	Err error
}

func (_ AlreadyReportedError) Error() string {
	return ""
}

func IsAlreadyReported(err error) bool {
	_, ok := err.(AlreadyReportedError)
	return ok
}

func (cmd *Cmd) Spawnvp(ctx context.Context) (int, error) {
	errorlevel, err := cmd.spawnvpSilent(ctx)
	if err != nil && err != io.EOF && !IsAlreadyReported(err) {
		if defined.DBG {
			val := reflect.ValueOf(err)
			fmt.Fprintf(cmd.Stderr, "error-type=%s\n", val.Type())
		}
		fmt.Fprintln(cmd.Stderr, err.Error())
		err = AlreadyReportedError{err}
	}
	return errorlevel, err
}

func (sh *Shell) Spawnlp(ctx context.Context, args, rawargs []string) (int, error) {
	cmd := sh.Command()
	defer cmd.Close()
	cmd.SetArgs(args)
	cmd.SetRawArgs(rawargs)
	return cmd.Spawnvp(ctx)
}

func (sh *Shell) Interpret(ctx context.Context, text string) (errorlevel int, finalerr error) {
	if defined.DBG {
		print("Interpret('", text, "')\n")
	}
	if sh == nil {
		return 255, errors.New("Fatal Error: Interpret: instance is nil")
	}
	errorlevel = 0
	finalerr = nil

	statements, statementsErr := Parse(text)
	if statementsErr != nil {
		if defined.DBG {
			print("Parse Error:", statementsErr.Error(), "\n")
		}
		return 0, statementsErr
	}
	if argsHook != nil {
		if defined.DBG {
			print("call argsHook\n")
		}
		for _, pipeline := range statements {
			for _, state := range pipeline {
				var err error
				state.Args, err = argsHook(ctx, sh, state.Args)
				if err != nil {
					return 255, err
				}
			}
		}
		if defined.DBG {
			print("done argsHook\n")
		}
	}
	for _, pipeline := range statements {
		for i, state := range pipeline {
			if state.Term == "|" && (i+1 >= len(pipeline) || len(pipeline[i+1].Args) <= 0) {
				return 255, errors.New("The syntax of the command is incorrect.")
			}
		}
	}

	for _, pipeline := range statements {

		var pipeIn *os.File = nil
		isBackGround := sh.IsBackGround
		for _, state := range pipeline {
			if state.Term == "&" {
				isBackGround = true
				break
			}
		}
		var wg sync.WaitGroup
		shutdown_immediately := false
		for i, state := range pipeline {
			if defined.DBG {
				print(i, ": pipeline loop(", state.Args[0], ")\n")
			}
			cmd := sh.Command()
			cmd.IsBackGround = isBackGround

			if pipeIn != nil {
				cmd.Stdin = pipeIn
				cmd.Closers = append(cmd.Closers, pipeIn)
				pipeIn = nil
			}

			var err error
			if state.Term[0] == '|' {
				var pipeOut *os.File
				pipeIn, pipeOut, err = os.Pipe()
				cmd.Stdout = pipeOut
				if state.Term == "|&" {
					cmd.Stderr = pipeOut
				}
				cmd.Closers = append(cmd.Closers, pipeOut)
			}

			for _, red := range state.Redirect {
				var fd *os.File
				fd, err = red.OpenOn(cmd)
				if err != nil {
					return 0, err
				}
				defer fd.Close()
			}

			cmd.args = state.Args
			cmd.rawArgs = state.RawArgs
			if i > 0 {
				cmd.IsBackGround = true
			}
			if len(pipeline) == 1 && dos.IsGui(cmd.FullPath()) {
				cmd.UseShellExecute = true
			}
			if i == len(pipeline)-1 && state.Term != "&" {
				// foreground execution.
				errorlevel, finalerr = cmd.Spawnvp(ctx)
				LastErrorLevel = errorlevel
				cmd.Close()
			} else {
				// background
				if !isBackGround {
					wg.Add(1)
				}
				newctx := ctx
				if tag := cmd.Tag(); tag != nil {
					var newtag CloneCloser
					if newctx, newtag, err = tag.Clone(ctx); err != nil {
						fmt.Fprintln(os.Stderr, err.Error())
						return -1, err
					} else {
						cmd.SetTag(newtag)
					}
				}
				go func(ctx1 context.Context, cmd1 *Cmd) {
					if !isBackGround {
						defer wg.Done()
					}
					cmd1.Spawnvp(ctx1)
					if tag := cmd1.Tag(); tag != nil {
						if err := tag.Close(); err != nil {
							fmt.Fprintln(os.Stderr, err.Error())
						}
					}
					cmd1.Close()
				}(newctx, cmd)
			}
		}
		if !isBackGround {
			wg.Wait()
			if shutdown_immediately {
				return errorlevel, nil
			}
			if len(pipeline) > 0 {
				switch pipeline[len(pipeline)-1].Term {
				case "&&":
					if errorlevel != 0 {
						return errorlevel, nil
					}
				case "||":
					if errorlevel == 0 {
						return errorlevel, nil
					}
				}
			}
		}
	}
	return
}
<|endoftext|>


<commit_before>package shells

import (
	"bytes"
	"fmt"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
	"strings"
)

type PowerShell struct {
	AbstractShell
}

type PsWriter struct {
	bytes.Buffer
}

func psQuote(text string) string {
	// taken from: http://www.robvanderwoude.com/escapechars.php
	text = strings.Replace(text, "`", "``", -1)
	// text = strings.Replace(text, "\0", "`0", -1)
	text = strings.Replace(text, "\a", "`a", -1)
	text = strings.Replace(text, "\b", "`b", -1)
	text = strings.Replace(text, "\f", "^f", -1)
	text = strings.Replace(text, "\r", "`r", -1)
	text = strings.Replace(text, "\n", "`n", -1)
	text = strings.Replace(text, "\t", "^t", -1)
	text = strings.Replace(text, "\v", "^v", -1)
	text = strings.Replace(text, "#", "`#", -1)
	text = strings.Replace(text, "'", "`'", -1)
	text = strings.Replace(text, "\"", "`\"", -1)
	return "\"" + text + "\""
}

func psQuoteVariable(text string) string {
	text = strings.Replace(text, "$", "`$", -1)
	text = psQuote(text)
	return text
}

func (b *PsWriter) Line(text string) {
	b.WriteString(text + "\r\n")
}

func (b *PsWriter) checkErrorLevel() {
	b.Line("if (!$?) { Exit $LASTEXITCODE }")
}

func (b *PsWriter) Command(command string, arguments ...string) {
	list := []string{
		psQuote(command),
	}

	for _, argument := range arguments {
		list = append(list, psQuote(argument))
	}

	b.Line(strings.Join(list, " "))
	b.checkErrorLevel()
}

func (b *PsWriter) Variable(variable common.BuildVariable) {
	b.Line("$env:" + variable.Key + "=" + psQuoteVariable(variable.Value))
}

func (b *PsWriter) IfDirectory(path string) {
	b.Line("if(Test-Path " + psQuote(helpers.ToBackslash(path)) + " {")
}

func (b *PsWriter) IfFile(path string) {
	b.Line("if(Test-Path " + psQuote(helpers.ToBackslash(path)) + " {")
}

func (b *PsWriter) Else() {
	b.Line("} else {")
}

func (b *PsWriter) EndIf() {
	b.Line("}")
}

func (b *PsWriter) Cd(path string) {
	b.Line("cd " + psQuote(helpers.ToBackslash(path)))
	b.checkErrorLevel()
}

func (b *PsWriter) RmDir(path string) {
	path = psQuote(helpers.ToBackslash(path))
	b.Line("if( (Get-Command -Name Remove-Item2 -Module NTFSSecurity -ErrorAction SilentlyContinue) -and (Test-Path " + path + ") ) {")
	b.Line("Remove-Item2 -Force -Recurse " + path)
	b.Line("} elseif(Test-Path " + path + ") {")
	b.Line("Remove-Item -Force -Recurse " + path)
	b.Line("}")
}

func (b *PsWriter) RmFile(path string) {
	path = psQuote(helpers.ToBackslash(path))
	b.Line("if( (Get-Command -Name Remove-Item2 -Module NTFSSecurity -ErrorAction SilentlyContinue) -and (Test-Path " + path + ") ) {")
	b.Line("Remove-Item2 -Force " + path)
	b.Line("} elseif(Test-Path " + path + ") {")
	b.Line("Remove-Item -Force " + path)
	b.Line("}")
}

func (b *PsWriter) Print(format string, arguments ...interface{}) {
	coloredText := fmt.Sprintf(format, arguments...)
	b.Line("echo " + psQuoteVariable(coloredText))
}

func (b *PsWriter) Notice(format string, arguments ...interface{}) {
	coloredText := fmt.Sprintf(format, arguments...)
	b.Line("echo " + psQuoteVariable(coloredText))
}

func (b *PsWriter) Warning(format string, arguments ...interface{}) {
	coloredText := fmt.Sprintf(format, arguments...)
	b.Line("echo " + psQuoteVariable(coloredText))
}

func (b *PsWriter) Error(format string, arguments ...interface{}) {
	coloredText := fmt.Sprintf(format, arguments...)
	b.Line("echo " + psQuoteVariable(coloredText))
}

func (b *PsWriter) EmptyLine() {
	b.Line("echo \"\"")
}

func (b *PowerShell) GetName() string {
	return "powershell"
}

func (b *PowerShell) GenerateScript(info common.ShellScriptInfo) (*common.ShellScript, error) {
	w := &PsWriter{}
	w.Line("$ErrorActionPreference = \"Stop\"")
	w.EmptyLine()

	if len(info.Build.Hostname) != 0 {
		w.Line("echo Running on $env:computername via " + psQuoteVariable(info.Build.Hostname) + "...")
	} else {
		w.Line("echo Running on  $env:computername...")
	}

	w.Line("& {")
	b.GeneratePreBuild(w, info)
	w.Line("}")
	w.checkErrorLevel()

	w.Line("& {")
	b.GenerateCommands(w, info)
	w.Line("}")
	w.checkErrorLevel()

	w.Line("& {")
	b.GeneratePostBuild(w, info)
	w.Line("}")
	w.checkErrorLevel()

	script := common.ShellScript{
		BuildScript: w.String(),
		Command:     "powershell",
		Arguments:   []string{"-noprofile", "-noninteractive", "-executionpolicy", "Bypass", "-command"},
		PassFile:    true,
		Extension:   "ps1",
	}
	return &script, nil
}

func (b *PowerShell) IsDefault() bool {
	return false
}

func init() {
	common.RegisterShell(&PowerShell{})
}
<commit_msg>added missing closing brackets<commit_after>package shells

import (
	"bytes"
	"fmt"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
	"strings"
)

type PowerShell struct {
	AbstractShell
}

type PsWriter struct {
	bytes.Buffer
}

func psQuote(text string) string {
	// taken from: http://www.robvanderwoude.com/escapechars.php
	text = strings.Replace(text, "`", "``", -1)
	// text = strings.Replace(text, "\0", "`0", -1)
	text = strings.Replace(text, "\a", "`a", -1)
	text = strings.Replace(text, "\b", "`b", -1)
	text = strings.Replace(text, "\f", "^f", -1)
	text = strings.Replace(text, "\r", "`r", -1)
	text = strings.Replace(text, "\n", "`n", -1)
	text = strings.Replace(text, "\t", "^t", -1)
	text = strings.Replace(text, "\v", "^v", -1)
	text = strings.Replace(text, "#", "`#", -1)
	text = strings.Replace(text, "'", "`'", -1)
	text = strings.Replace(text, "\"", "`\"", -1)
	return "\"" + text + "\""
}

func psQuoteVariable(text string) string {
	text = strings.Replace(text, "$", "`$", -1)
	text = psQuote(text)
	return text
}

func (b *PsWriter) Line(text string) {
	b.WriteString(text + "\r\n")
}

func (b *PsWriter) checkErrorLevel() {
	b.Line("if (!$?) { Exit $LASTEXITCODE }")
}

func (b *PsWriter) Command(command string, arguments ...string) {
	list := []string{
		psQuote(command),
	}

	for _, argument := range arguments {
		list = append(list, psQuote(argument))
	}

	b.Line(strings.Join(list, " "))
	b.checkErrorLevel()
}

func (b *PsWriter) Variable(variable common.BuildVariable) {
	b.Line("$env:" + variable.Key + "=" + psQuoteVariable(variable.Value))
}

func (b *PsWriter) IfDirectory(path string) {
	b.Line("if(Test-Path " + psQuote(helpers.ToBackslash(path)) + ") {")
}

func (b *PsWriter) IfFile(path string) {
	b.Line("if(Test-Path " + psQuote(helpers.ToBackslash(path)) + ") {")
}

func (b *PsWriter) Else() {
	b.Line("} else {")
}

func (b *PsWriter) EndIf() {
	b.Line("}")
}

func (b *PsWriter) Cd(path string) {
	b.Line("cd " + psQuote(helpers.ToBackslash(path)))
	b.checkErrorLevel()
}

func (b *PsWriter) RmDir(path string) {
	path = psQuote(helpers.ToBackslash(path))
	b.Line("if( (Get-Command -Name Remove-Item2 -Module NTFSSecurity -ErrorAction SilentlyContinue) -and (Test-Path " + path + ") ) {")
	b.Line("Remove-Item2 -Force -Recurse " + path)
	b.Line("} elseif(Test-Path " + path + ") {")
	b.Line("Remove-Item -Force -Recurse " + path)
	b.Line("}")
}

func (b *PsWriter) RmFile(path string) {
	path = psQuote(helpers.ToBackslash(path))
	b.Line("if( (Get-Command -Name Remove-Item2 -Module NTFSSecurity -ErrorAction SilentlyContinue) -and (Test-Path " + path + ") ) {")
	b.Line("Remove-Item2 -Force " + path)
	b.Line("} elseif(Test-Path " + path + ") {")
	b.Line("Remove-Item -Force " + path)
	b.Line("}")
}

func (b *PsWriter) Print(format string, arguments ...interface{}) {
	coloredText := fmt.Sprintf(format, arguments...)
	b.Line("echo " + psQuoteVariable(coloredText))
}

func (b *PsWriter) Notice(format string, arguments ...interface{}) {
	coloredText := fmt.Sprintf(format, arguments...)
	b.Line("echo " + psQuoteVariable(coloredText))
}

func (b *PsWriter) Warning(format string, arguments ...interface{}) {
	coloredText := fmt.Sprintf(format, arguments...)
	b.Line("echo " + psQuoteVariable(coloredText))
}

func (b *PsWriter) Error(format string, arguments ...interface{}) {
	coloredText := fmt.Sprintf(format, arguments...)
	b.Line("echo " + psQuoteVariable(coloredText))
}

func (b *PsWriter) EmptyLine() {
	b.Line("echo \"\"")
}

func (b *PowerShell) GetName() string {
	return "powershell"
}

func (b *PowerShell) GenerateScript(info common.ShellScriptInfo) (*common.ShellScript, error) {
	w := &PsWriter{}
	w.Line("$ErrorActionPreference = \"Stop\"")
	w.EmptyLine()

	if len(info.Build.Hostname) != 0 {
		w.Line("echo Running on $env:computername via " + psQuoteVariable(info.Build.Hostname) + "...")
	} else {
		w.Line("echo Running on  $env:computername...")
	}

	w.Line("& {")
	b.GeneratePreBuild(w, info)
	w.Line("}")
	w.checkErrorLevel()

	w.Line("& {")
	b.GenerateCommands(w, info)
	w.Line("}")
	w.checkErrorLevel()

	w.Line("& {")
	b.GeneratePostBuild(w, info)
	w.Line("}")
	w.checkErrorLevel()

	script := common.ShellScript{
		BuildScript: w.String(),
		Command:     "powershell",
		Arguments:   []string{"-noprofile", "-noninteractive", "-executionpolicy", "Bypass", "-command"},
		PassFile:    true,
		Extension:   "ps1",
	}
	return &script, nil
}

func (b *PowerShell) IsDefault() bool {
	return false
}

func init() {
	common.RegisterShell(&PowerShell{})
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/codegangsta/negroni"
	"github.com/julienschmidt/httprouter"
	"github.com/libgit2/git2go"
	"github.com/unrolled/render"

	"github.com/wantedly/risu/registry"
	"github.com/wantedly/risu/schema"
)

const (
	DefaultCloneBasePath = "/var/risu/src/"
)

var ren = render.New()

func create(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	defer r.Body.Close()
	var opts schema.BuildCreateOpts
	err := json.NewDecoder(r.Body).Decode(&opts)
	if err != nil {
		log.Fatal(err)
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}

	if opts.SourceBranch == "" {
		opts.SourceBranch = "master"
	}

	if opts.Dockerfile == "" {
		opts.Dockerfile = "Dockerfile"
	}

	currentTime := time.Now()
	build := schema.Build{
		ID:           uuid.NewUUID(),
		SourceRepo:   opts.SourceRepo,
		SourceBranch: opts.SourceBranch,
		Name:         opts.Name,
		Dockerfile:   opts.Dockerfile,
		Status:       "building",
		CreatedAt:    currentTime,
		UpdatedAt:    currentTime,
	}

	reg := registry.NewRegistry("localfs", "")
	err = reg.Set(build)
	if err != nil {
		log.Fatal(err)
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}
	ren.JSON(w, http.StatusCreated, build)

	// git clone
	err = gitClone(build)
	if err != nil {
		log.Fatal(err)
	}
}

func root(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	ren.JSON(w, http.StatusOK, map[string]string{"status": "ok"})
}

func index(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	reg := registry.NewRegistry("localfs", "")
	builds, err := reg.List()
	if err != nil {
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}
	ren.JSON(w, http.StatusOK, builds)
}

func show(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	id := ps.ByName("id")
	uuid := uuid.Parse(id)
	reg := registry.NewRegistry("localfs", "")
	build, err := reg.Get(uuid)
	if err != nil {
		ren.JSON(w, http.StatusNotFound, map[string]string{"status": "not found"})
		return
	}
	ren.JSON(w, http.StatusOK, build)
}

// Clone run "git clone <repository_URL>" and "git checkout branch"
func gitClone(build schema.Build) error {
	basePath := DefaultCloneBasePath
	if _, err := os.Stat(basePath); err != nil {
		os.MkdirAll(basePath, 0755)
	}

	// htpps://<token>@github.com/<SourceRepo>.git
	cloneURL := "https://" + os.Getenv("GITHUB_ACCESS_TOKEN") + "@github.com/" + build.SourceRepo + ".git"

	// debug
	fmt.Println(cloneURL)

	clonePath := basePath + "github.com/" + build.SourceRepo

	// debug
	fmt.Println(clonePath)

	_, err := git.Clone(cloneURL, clonePath, &git.CloneOptions{CheckoutBranch: build.SourceBranch})
	if err != nil {
		return err
	}
	return nil
}

func setUpServer() *negroni.Negroni {
	router := httprouter.New()
	router.GET("/", root)
	router.GET("/builds", index)
	router.GET("/builds/:id", show)
	router.POST("/builds", create)

	n := negroni.Classic()
	n.UseHandler(router)
	return n
}

func main() {
	if os.Getenv("GITHUB_ACCESS_TOKEN") == "" {
		log.Fatal("Please provide 'GITHUB_ACCESS_TOKEN' through environment")
		os.Exit(1)
	}
	n := setUpServer()
	n.Run(":8080")
}
<commit_msg>add const SourceBasePath and CacheBasePath<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/codegangsta/negroni"
	"github.com/julienschmidt/httprouter"
	"github.com/libgit2/git2go"
	"github.com/unrolled/render"

	"github.com/wantedly/risu/registry"
	"github.com/wantedly/risu/schema"
)

const (
	SourceBasePath = "/var/risu/src/github.com/"
	CacheBasePath  = "/var/risu/cache"
)

var ren = render.New()

func create(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	defer r.Body.Close()
	var opts schema.BuildCreateOpts
	err := json.NewDecoder(r.Body).Decode(&opts)
	if err != nil {
		log.Fatal(err)
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}

	if opts.SourceBranch == "" {
		opts.SourceBranch = "master"
	}

	if opts.Dockerfile == "" {
		opts.Dockerfile = "Dockerfile"
	}

	currentTime := time.Now()
	build := schema.Build{
		ID:           uuid.NewUUID(),
		SourceRepo:   opts.SourceRepo,
		SourceBranch: opts.SourceBranch,
		Name:         opts.Name,
		Dockerfile:   opts.Dockerfile,
		Status:       "building",
		CreatedAt:    currentTime,
		UpdatedAt:    currentTime,
	}

	reg := registry.NewRegistry("localfs", "")
	err = reg.Set(build)
	if err != nil {
		log.Fatal(err)
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}
	ren.JSON(w, http.StatusCreated, build)

	// git clone
	err = gitClone(build)
	if err != nil {
		log.Fatal(err)
	}
}

func root(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	ren.JSON(w, http.StatusOK, map[string]string{"status": "ok"})
}

func index(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	reg := registry.NewRegistry("localfs", "")
	builds, err := reg.List()
	if err != nil {
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}
	ren.JSON(w, http.StatusOK, builds)
}

func show(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	id := ps.ByName("id")
	uuid := uuid.Parse(id)
	reg := registry.NewRegistry("localfs", "")
	build, err := reg.Get(uuid)
	if err != nil {
		ren.JSON(w, http.StatusNotFound, map[string]string{"status": "not found"})
		return
	}
	ren.JSON(w, http.StatusOK, build)
}

// Clone run "git clone <repository_URL>" and "git checkout branch"
func gitClone(build schema.Build) error {
	if _, err := os.Stat(SourceBasePath); err != nil {
		os.MkdirAll(SourceBasePath, 0755)
	}

	// htpps://<token>@github.com/<SourceRepo>.git
	cloneURL := "https://" + os.Getenv("GITHUB_ACCESS_TOKEN") + "@github.com/" + build.SourceRepo + ".git"

	// debug
	fmt.Println(cloneURL)

	clonePath := SourceBasePath + build.SourceRepo

	// debug
	fmt.Println(clonePath)

	_, err := git.Clone(cloneURL, clonePath, &git.CloneOptions{CheckoutBranch: build.SourceBranch})
	if err != nil {
		return err
	}
	return nil
}

func setUpServer() *negroni.Negroni {
	router := httprouter.New()
	router.GET("/", root)
	router.GET("/builds", index)
	router.GET("/builds/:id", show)
	router.POST("/builds", create)

	n := negroni.Classic()
	n.UseHandler(router)
	return n
}

func main() {
	if os.Getenv("GITHUB_ACCESS_TOKEN") == "" {
		log.Fatal("Please provide 'GITHUB_ACCESS_TOKEN' through environment")
		os.Exit(1)
	}
	n := setUpServer()
	n.Run(":8080")
}
<|endoftext|>


<commit_before>package command

import (
	"io/ioutil"
	"path/filepath"
)

func init() {
	cmdScaffold.Run = runScaffold // break init cycle
}

var cmdScaffold = &Command{
	UsageLine: "scaffold -config=[filer|notification|replication|security]",
	Short:     "generate basic configuration files",
	Long: `Generate filer.toml with all possible configurations for you to customize.

  `,
}

var (
	outputPath = cmdScaffold.Flag.String("output", "", "if not empty, save the configuration file to this directory")
	config     = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication|security] the configuration file to generate")
)

func runScaffold(cmd *Command, args []string) bool {

	content := ""
	switch *config {
	case "filer":
		content = FILER_TOML_EXAMPLE
	case "notification":
		content = NOTIFICATION_TOML_EXAMPLE
	case "replication":
		content = REPLICATION_TOML_EXAMPLE
	case "security":
		content = SECURITY_TOML_EXAMPLE
	}
	if content == "" {
		println("need a valid -config option")
		return false
	}

	if *outputPath != "" {
		ioutil.WriteFile(filepath.Join(*outputPath, *config+".toml"), []byte(content), 0644)
	} else {
		println(content)
	}
	return true
}

const (
	FILER_TOML_EXAMPLE = `
# A sample TOML config file for SeaweedFS filer store
# Used with "weed filer" or "weed server -filer"
# Put this file to one of the location, with descending priority
#    ./filer.toml
#    $HOME/.seaweedfs/filer.toml
#    /etc/seaweedfs/filer.toml

[memory]
# local in memory, mostly for testing purpose
enabled = false

[leveldb]
# local on disk, mostly for simple single-machine setup, fairly scalable
enabled = true
dir = "."					# directory to store level db files

####################################################
# multiple filers on shared storage, fairly scalable
####################################################

[mysql]
# CREATE TABLE IF NOT EXISTS filemeta (
#   dirhash     BIGINT         COMMENT 'first 64 bits of MD5 hash value of directory field',
#   name        VARCHAR(1000)  COMMENT 'directory or file name',
#   directory   TEXT           COMMENT 'full path to parent directory',
#   meta        BLOB,
#   PRIMARY KEY (dirhash, name)
# ) DEFAULT CHARSET=utf8;

enabled = false
hostname = "localhost"
port = 3306
username = "root"
password = ""
database = ""              # create or use an existing database
connection_max_idle = 2
connection_max_open = 100

[postgres]
# CREATE TABLE IF NOT EXISTS filemeta (
#   dirhash     BIGINT,
#   name        VARCHAR(65535),
#   directory   VARCHAR(65535),
#   meta        bytea,
#   PRIMARY KEY (dirhash, name)
# );
enabled = false
hostname = "localhost"
port = 5432
username = "postgres"
password = ""
database = ""              # create or use an existing database
sslmode = "disable"
connection_max_idle = 100
connection_max_open = 100

[cassandra]
# CREATE TABLE filemeta (
#    directory varchar,
#    name varchar,
#    meta blob,
#    PRIMARY KEY (directory, name)
# ) WITH CLUSTERING ORDER BY (name ASC);
enabled = false
keyspace="seaweedfs"
hosts=[
	"localhost:9042",
]

[redis]
enabled = false
address  = "localhost:6379"
password = ""
db = 0

[redis_cluster]
enabled = false
addresses = [
    "localhost:30001",
    "localhost:30002",
    "localhost:30003",
    "localhost:30004",
    "localhost:30005",
    "localhost:30006",
]

`

	NOTIFICATION_TOML_EXAMPLE = `
# A sample TOML config file for SeaweedFS filer store
# Used by both "weed filer" or "weed server -filer" and "weed filer.replicate"
# Put this file to one of the location, with descending priority
#    ./notification.toml
#    $HOME/.seaweedfs/notification.toml
#    /etc/seaweedfs/notification.toml

####################################################
# notification
# send and receive filer updates for each file to an external message queue
####################################################
[notification.log]
# this is only for debugging perpose and does not work with "weed filer.replicate"
enabled = false

[notification.kafka]
enabled = false
hosts = [
  "localhost:9092"
]
topic = "seaweedfs_filer"
offsetFile = "./last.offset"
offsetSaveIntervalSeconds = 10

[notification.aws_sqs]
# experimental, let me know if it works
enabled = false
aws_access_key_id     = ""        # if empty, loads from the shared credentials file (~/.aws/credentials).
aws_secret_access_key = ""        # if empty, loads from the shared credentials file (~/.aws/credentials).
region = "us-east-2"
sqs_queue_name = "my_filer_queue" # an existing queue name

[notification.google_pub_sub]
# read credentials doc at https://cloud.google.com/docs/authentication/getting-started
enabled = false
google_application_credentials = "/path/to/x.json" # path to json credential file
project_id = ""                       # an existing project id
topic = "seaweedfs_filer_topic"       # a topic, auto created if does not exists

[notification.gocdk_pub_sub]
# The Go Cloud Development Kit (https://gocloud.dev).
# PubSub API (https://godoc.org/gocloud.dev/pubsub).
# Supports AWS SNS/SQS, Azure Service Bus, Google PubSub, NATS and RabbitMQ.
enabled = false
# This URL will Dial the RabbitMQ server at the URL in the environment
# variable RABBIT_SERVER_URL and open the exchange "myexchange".
# The exchange must have already been created by some other means, like
# the RabbitMQ management plugin.
topic_url = "rabbit://myexchange"
sub_url = "rabbit://myqueue"
`

	REPLICATION_TOML_EXAMPLE = `
# A sample TOML config file for replicating SeaweedFS filer
# Used with "weed filer.replicate"
# Put this file to one of the location, with descending priority
#    ./replication.toml
#    $HOME/.seaweedfs/replication.toml
#    /etc/seaweedfs/replication.toml

[source.filer]
enabled = true
grpcAddress = "localhost:18888"
# all files under this directory tree are replicated.
# this is not a directory on your hard drive, but on your filer.
# i.e., all files with this "prefix" are sent to notification message queue.
directory = "/buckets"

[sink.filer]
enabled = false
grpcAddress = "localhost:18888"
# all replicated files are under this directory tree
# this is not a directory on your hard drive, but on your filer.
# i.e., all received files will be "prefixed" to this directory.
directory = "/backup"
replication = ""
collection = ""
ttlSec = 0

[sink.s3]
# read credentials doc at https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sessions.html
# default loads credentials from the shared credentials file (~/.aws/credentials).
enabled = false
aws_access_key_id     = ""     # if empty, loads from the shared credentials file (~/.aws/credentials).
aws_secret_access_key = ""     # if empty, loads from the shared credentials file (~/.aws/credentials).
region = "us-east-2"
bucket = "your_bucket_name"    # an existing bucket
directory = "/"                # destination directory

[sink.google_cloud_storage]
# read credentials doc at https://cloud.google.com/docs/authentication/getting-started
enabled = false
google_application_credentials = "/path/to/x.json" # path to json credential file
bucket = "your_bucket_seaweedfs"    # an existing bucket
directory = "/"                     # destination directory

[sink.azure]
# experimental, let me know if it works
enabled = false
account_name = ""
account_key  = ""
container = "mycontainer"      # an existing container
directory = "/"                # destination directory

[sink.backblaze]
enabled = false
b2_account_id = ""
b2_master_application_key  = ""
bucket = "mybucket"            # an existing bucket
directory = "/"                # destination directory

`

	SECURITY_TOML_EXAMPLE = `
# Put this file to one of the location, with descending priority
#    ./security.toml
#    $HOME/.seaweedfs/security.toml
#    /etc/seaweedfs/security.toml
# this file is read by master, volume server, and filer

# the jwt signing key is read by master and volume server.
# a jwt defaults to expire after 10 seconds.
[jwt.signing]
key = ""
expires_after_seconds = 10           # seconds

# all grpc tls authentications are mutual
# the values for the following ca, cert, and key are paths to the PERM files.
[grpc]
ca = ""

[grpc.volume]
cert = ""
key  = ""

[grpc.master]
cert = ""
key  = ""

[grpc.filer]
cert = ""
key  = ""

# use this for any place needs a grpc client
# i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
[grpc.client]
cert = ""
key  = ""

# volume server https options
# Note: work in progress!
#     this does not work with other clients, e.g., "weed filer|mount" etc, yet.
[https.client]
enabled = true
[https.volume]
cert = ""
key  = ""

`
)
<commit_msg>filer: adjust recommended mysql meta data type to LONGBLOB<commit_after>package command

import (
	"io/ioutil"
	"path/filepath"
)

func init() {
	cmdScaffold.Run = runScaffold // break init cycle
}

var cmdScaffold = &Command{
	UsageLine: "scaffold -config=[filer|notification|replication|security]",
	Short:     "generate basic configuration files",
	Long: `Generate filer.toml with all possible configurations for you to customize.

  `,
}

var (
	outputPath = cmdScaffold.Flag.String("output", "", "if not empty, save the configuration file to this directory")
	config     = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication|security] the configuration file to generate")
)

func runScaffold(cmd *Command, args []string) bool {

	content := ""
	switch *config {
	case "filer":
		content = FILER_TOML_EXAMPLE
	case "notification":
		content = NOTIFICATION_TOML_EXAMPLE
	case "replication":
		content = REPLICATION_TOML_EXAMPLE
	case "security":
		content = SECURITY_TOML_EXAMPLE
	}
	if content == "" {
		println("need a valid -config option")
		return false
	}

	if *outputPath != "" {
		ioutil.WriteFile(filepath.Join(*outputPath, *config+".toml"), []byte(content), 0644)
	} else {
		println(content)
	}
	return true
}

const (
	FILER_TOML_EXAMPLE = `
# A sample TOML config file for SeaweedFS filer store
# Used with "weed filer" or "weed server -filer"
# Put this file to one of the location, with descending priority
#    ./filer.toml
#    $HOME/.seaweedfs/filer.toml
#    /etc/seaweedfs/filer.toml

[memory]
# local in memory, mostly for testing purpose
enabled = false

[leveldb]
# local on disk, mostly for simple single-machine setup, fairly scalable
enabled = true
dir = "."					# directory to store level db files

####################################################
# multiple filers on shared storage, fairly scalable
####################################################

[mysql]
# CREATE TABLE IF NOT EXISTS filemeta (
#   dirhash     BIGINT         COMMENT 'first 64 bits of MD5 hash value of directory field',
#   name        VARCHAR(1000)  COMMENT 'directory or file name',
#   directory   TEXT           COMMENT 'full path to parent directory',
#   meta        LONGBLOB,
#   PRIMARY KEY (dirhash, name)
# ) DEFAULT CHARSET=utf8;

enabled = false
hostname = "localhost"
port = 3306
username = "root"
password = ""
database = ""              # create or use an existing database
connection_max_idle = 2
connection_max_open = 100

[postgres]
# CREATE TABLE IF NOT EXISTS filemeta (
#   dirhash     BIGINT,
#   name        VARCHAR(65535),
#   directory   VARCHAR(65535),
#   meta        bytea,
#   PRIMARY KEY (dirhash, name)
# );
enabled = false
hostname = "localhost"
port = 5432
username = "postgres"
password = ""
database = ""              # create or use an existing database
sslmode = "disable"
connection_max_idle = 100
connection_max_open = 100

[cassandra]
# CREATE TABLE filemeta (
#    directory varchar,
#    name varchar,
#    meta blob,
#    PRIMARY KEY (directory, name)
# ) WITH CLUSTERING ORDER BY (name ASC);
enabled = false
keyspace="seaweedfs"
hosts=[
	"localhost:9042",
]

[redis]
enabled = false
address  = "localhost:6379"
password = ""
db = 0

[redis_cluster]
enabled = false
addresses = [
    "localhost:30001",
    "localhost:30002",
    "localhost:30003",
    "localhost:30004",
    "localhost:30005",
    "localhost:30006",
]

`

	NOTIFICATION_TOML_EXAMPLE = `
# A sample TOML config file for SeaweedFS filer store
# Used by both "weed filer" or "weed server -filer" and "weed filer.replicate"
# Put this file to one of the location, with descending priority
#    ./notification.toml
#    $HOME/.seaweedfs/notification.toml
#    /etc/seaweedfs/notification.toml

####################################################
# notification
# send and receive filer updates for each file to an external message queue
####################################################
[notification.log]
# this is only for debugging perpose and does not work with "weed filer.replicate"
enabled = false

[notification.kafka]
enabled = false
hosts = [
  "localhost:9092"
]
topic = "seaweedfs_filer"
offsetFile = "./last.offset"
offsetSaveIntervalSeconds = 10

[notification.aws_sqs]
# experimental, let me know if it works
enabled = false
aws_access_key_id     = ""        # if empty, loads from the shared credentials file (~/.aws/credentials).
aws_secret_access_key = ""        # if empty, loads from the shared credentials file (~/.aws/credentials).
region = "us-east-2"
sqs_queue_name = "my_filer_queue" # an existing queue name

[notification.google_pub_sub]
# read credentials doc at https://cloud.google.com/docs/authentication/getting-started
enabled = false
google_application_credentials = "/path/to/x.json" # path to json credential file
project_id = ""                       # an existing project id
topic = "seaweedfs_filer_topic"       # a topic, auto created if does not exists

[notification.gocdk_pub_sub]
# The Go Cloud Development Kit (https://gocloud.dev).
# PubSub API (https://godoc.org/gocloud.dev/pubsub).
# Supports AWS SNS/SQS, Azure Service Bus, Google PubSub, NATS and RabbitMQ.
enabled = false
# This URL will Dial the RabbitMQ server at the URL in the environment
# variable RABBIT_SERVER_URL and open the exchange "myexchange".
# The exchange must have already been created by some other means, like
# the RabbitMQ management plugin.
topic_url = "rabbit://myexchange"
sub_url = "rabbit://myqueue"
`

	REPLICATION_TOML_EXAMPLE = `
# A sample TOML config file for replicating SeaweedFS filer
# Used with "weed filer.replicate"
# Put this file to one of the location, with descending priority
#    ./replication.toml
#    $HOME/.seaweedfs/replication.toml
#    /etc/seaweedfs/replication.toml

[source.filer]
enabled = true
grpcAddress = "localhost:18888"
# all files under this directory tree are replicated.
# this is not a directory on your hard drive, but on your filer.
# i.e., all files with this "prefix" are sent to notification message queue.
directory = "/buckets"

[sink.filer]
enabled = false
grpcAddress = "localhost:18888"
# all replicated files are under this directory tree
# this is not a directory on your hard drive, but on your filer.
# i.e., all received files will be "prefixed" to this directory.
directory = "/backup"
replication = ""
collection = ""
ttlSec = 0

[sink.s3]
# read credentials doc at https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sessions.html
# default loads credentials from the shared credentials file (~/.aws/credentials).
enabled = false
aws_access_key_id     = ""     # if empty, loads from the shared credentials file (~/.aws/credentials).
aws_secret_access_key = ""     # if empty, loads from the shared credentials file (~/.aws/credentials).
region = "us-east-2"
bucket = "your_bucket_name"    # an existing bucket
directory = "/"                # destination directory

[sink.google_cloud_storage]
# read credentials doc at https://cloud.google.com/docs/authentication/getting-started
enabled = false
google_application_credentials = "/path/to/x.json" # path to json credential file
bucket = "your_bucket_seaweedfs"    # an existing bucket
directory = "/"                     # destination directory

[sink.azure]
# experimental, let me know if it works
enabled = false
account_name = ""
account_key  = ""
container = "mycontainer"      # an existing container
directory = "/"                # destination directory

[sink.backblaze]
enabled = false
b2_account_id = ""
b2_master_application_key  = ""
bucket = "mybucket"            # an existing bucket
directory = "/"                # destination directory

`

	SECURITY_TOML_EXAMPLE = `
# Put this file to one of the location, with descending priority
#    ./security.toml
#    $HOME/.seaweedfs/security.toml
#    /etc/seaweedfs/security.toml
# this file is read by master, volume server, and filer

# the jwt signing key is read by master and volume server.
# a jwt defaults to expire after 10 seconds.
[jwt.signing]
key = ""
expires_after_seconds = 10           # seconds

# all grpc tls authentications are mutual
# the values for the following ca, cert, and key are paths to the PERM files.
[grpc]
ca = ""

[grpc.volume]
cert = ""
key  = ""

[grpc.master]
cert = ""
key  = ""

[grpc.filer]
cert = ""
key  = ""

# use this for any place needs a grpc client
# i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
[grpc.client]
cert = ""
key  = ""

# volume server https options
# Note: work in progress!
#     this does not work with other clients, e.g., "weed filer|mount" etc, yet.
[https.client]
enabled = true
[https.volume]
cert = ""
key  = ""

`
)
<|endoftext|>


<commit_before>// Copyright 2014 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package ra

import (
	"crypto/x509"
	"fmt"
	"math/big"
	"net/url"
	"regexp"
	"strconv"
	"time"

	"github.com/letsencrypt/boulder/core"
	blog "github.com/letsencrypt/boulder/log"
	"github.com/letsencrypt/boulder/policy"
)

// All of the fields in RegistrationAuthorityImpl need to be
// populated, or there is a risk of panic.
type RegistrationAuthorityImpl struct {
	CA  core.CertificateAuthority
	VA  core.ValidationAuthority
	SA  core.StorageAuthority
	PA  core.PolicyAuthority
	log *blog.AuditLogger

	AuthzBase string
}

func NewRegistrationAuthorityImpl() RegistrationAuthorityImpl {
	logger := blog.GetAuditLogger()
	logger.Notice("Registration Authority Starting")

	ra := RegistrationAuthorityImpl{log: logger}
	ra.PA = policy.NewPolicyAuthorityImpl()
	return ra
}

var allButLastPathSegment = regexp.MustCompile("^.*/")

func lastPathSegment(url core.AcmeURL) string {
	return allButLastPathSegment.ReplaceAllString(url.Path, "")
}

type certificateRequestEvent struct {
	ID                  string    `json:",omitempty"`
	Requester           int64     `json:",omitempty"`
	SerialNumber        *big.Int  `json:",omitempty"`
	RequestMethod       string    `json:",omitempty"`
	VerificationMethods []string  `json:",omitempty"`
	VerifiedFields      []string  `json:",omitempty"`
	CommonName          string    `json:",omitempty"`
	Names               []string  `json:",omitempty"`
	NotBefore           time.Time `json:",omitempty"`
	NotAfter            time.Time `json:",omitempty"`
	RequestTime         time.Time `json:",omitempty"`
	ResponseTime        time.Time `json:",omitempty"`
	Error               string    `json:",omitempty"`
}

func (ra *RegistrationAuthorityImpl) NewRegistration(init core.Registration) (reg core.Registration, err error) {
	if !core.GoodKey(init.Key.Key) {
		return core.Registration{}, core.UnauthorizedError("Invalid public key.")
	}
	reg = core.Registration{
		RecoveryToken: core.NewToken(),
		Key:           init.Key,
	}
	reg.MergeUpdate(init)

	// Store the authorization object, then return it
	reg, err = ra.SA.NewRegistration(reg)
	if err != nil {
		err = core.InternalServerError(err.Error())
	}
	return
}

func (ra *RegistrationAuthorityImpl) NewAuthorization(request core.Authorization, regID int64) (authz core.Authorization, err error) {
	if regID <= 0 {
		err = core.InternalServerError("Invalid registration ID")
		return authz, err
	}

	identifier := request.Identifier

	// Check that the identifier is present and appropriate
	if err = ra.PA.WillingToIssue(identifier); err != nil {
		err = core.UnauthorizedError(err.Error())
		return authz, err
	}

	// Create validations, but we have to update them with URIs later
	challenges, combinations := ra.PA.ChallengesFor(identifier)

	// Partially-filled object
	authz = core.Authorization{
		Identifier:     identifier,
		RegistrationID: regID,
		Status:         core.StatusPending,
		Combinations:   combinations,
	}

	// Get a pending Auth first so we can get our ID back, then update with challenges
	authz, err = ra.SA.NewPendingAuthorization(authz)
	if err != nil {
		err = core.InternalServerError(err.Error())
		return authz, err
	}

	// Construct all the challenge URIs
	for i := range challenges {
		// Ignoring these errors because we construct the URLs to be correct
		challengeURI, _ := url.Parse(ra.AuthzBase + authz.ID + "?challenge=" + strconv.Itoa(i))
		challenges[i].URI = core.AcmeURL(*challengeURI)

		if !challenges[i].IsSane(false) {
			err = core.InternalServerError(fmt.Sprintf("Challenge didn't pass sanity check: %+v", challenges[i]))
			return authz, err
		}
	}

	// Update object
	authz.Challenges = challenges

	// Store the authorization object, then return it
	err = ra.SA.UpdatePendingAuthorization(authz)
	if err != nil {
		err = core.InternalServerError(err.Error())
	}
	return authz, err
}

func (ra *RegistrationAuthorityImpl) NewCertificate(req core.CertificateRequest, regID int64) (cert core.Certificate, err error) {
	emptyCert := core.Certificate{}
	var logEventResult string

	// Assume the worst
	logEventResult = "error"

	// Construct the log event
	logEvent := certificateRequestEvent{
		ID:            core.NewToken(),
		Requester:     regID,
		RequestMethod: "online",
		RequestTime:   time.Now(),
	}

	// No matter what, log the request
	defer func() {
		// AUDIT[ Certificate Requests ] 11917fa4-10ef-4e0d-9105-bacbe7836a3c
		ra.log.AuditObject(fmt.Sprintf("Certificate request - %s", logEventResult), logEvent)
	}()

	if regID <= 0 {
		err = core.InternalServerError("Invalid registration ID")
		return emptyCert, err
	}

	registration, err := ra.SA.GetRegistration(regID)
	if err != nil {
		logEvent.Error = err.Error()
		return emptyCert, err
	}

	// Verify the CSR
	csr := req.CSR
	if err = core.VerifyCSR(csr); err != nil {
		logEvent.Error = err.Error()
		err = core.UnauthorizedError("Invalid signature on CSR")
		return emptyCert, err
	}

	logEvent.CommonName = csr.Subject.CommonName
	logEvent.Names = csr.DNSNames

	// Validate that authorization key is authorized for all domains
	names := make([]string, len(csr.DNSNames))
	copy(names, csr.DNSNames)
	if len(csr.Subject.CommonName) > 0 {
		names = append(names, csr.Subject.CommonName)
	}

	if len(names) == 0 {
		err = core.UnauthorizedError("CSR has no names in it")
		logEvent.Error = err.Error()
		return emptyCert, err
	}

	csrPreviousDenied, err := ra.SA.AlreadyDeniedCSR(names)
	if err != nil {
		logEvent.Error = err.Error()
		return emptyCert, err
	}
	if csrPreviousDenied {
		err = core.UnauthorizedError("CSR has already been revoked/denied")
		logEvent.Error = err.Error()
		return emptyCert, err
	}

	if core.KeyDigestEquals(csr.PublicKey, registration.Key) {
		err = core.MalformedRequestError("Certificate public key must be different than account key")
		return emptyCert, err
	}

	// Gather authorized domains from the referenced authorizations
	authorizedDomains := map[string]bool{}
	verificationMethodSet := map[string]bool{}
	earliestExpiry := time.Date(2100, 01, 01, 0, 0, 0, 0, time.UTC)
	now := time.Now()
	for _, url := range req.Authorizations {
		id := lastPathSegment(url)
		authz, err := ra.SA.GetAuthorization(id)
		if err != nil || // Couldn't find authorization
			authz.RegistrationID != registration.ID ||
			authz.Status != core.StatusValid || // Not finalized or not successful
			authz.Expires.Before(now) || // Expired
			authz.Identifier.Type != core.IdentifierDNS {
			// XXX: It may be good to fail here instead of ignoring invalid authorizations.
			//      However, it seems like this treatment is more in the spirit of Postel's
			//      law, and it hides information from attackers.
			continue
		}

		if authz.Expires.Before(earliestExpiry) {
			earliestExpiry = authz.Expires
		}

		for _, challenge := range authz.Challenges {
			if challenge.Status == core.StatusValid {
				verificationMethodSet[challenge.Type] = true
			}
		}

		authorizedDomains[authz.Identifier.Value] = true
	}
	verificationMethods := []string{}
	for method, _ := range verificationMethodSet {
		verificationMethods = append(verificationMethods, method)
	}
	logEvent.VerificationMethods = verificationMethods

	// Validate all domains
	for _, name := range names {
		if !authorizedDomains[name] {
			err = core.UnauthorizedError(fmt.Sprintf("Key not authorized for name %s", name))
			logEvent.Error = err.Error()
			return emptyCert, err
		}
	}

	// Mark that we verified the CN and SANs
	logEvent.VerifiedFields = []string{"subject.commonName", "subjectAltName"}

	// Create the certificate and log the result
	if cert, err = ra.CA.IssueCertificate(*csr, regID, earliestExpiry); err != nil {
		err = core.InternalServerError(err.Error())
		logEvent.Error = err.Error()
		return emptyCert, err
	}

	parsedCertificate, err := x509.ParseCertificate([]byte(cert.DER))
	if err != nil {
		err = core.InternalServerError(err.Error())
		logEvent.Error = err.Error()
		return emptyCert, err
	}

	logEvent.SerialNumber = parsedCertificate.SerialNumber
	logEvent.CommonName = parsedCertificate.Subject.CommonName
	logEvent.NotBefore = parsedCertificate.NotBefore
	logEvent.NotAfter = parsedCertificate.NotAfter
	logEvent.ResponseTime = time.Now()

	logEventResult = "successful"
	return cert, nil
}

func (ra *RegistrationAuthorityImpl) UpdateRegistration(base core.Registration, update core.Registration) (reg core.Registration, err error) {
	base.MergeUpdate(update)
	reg = base
	err = ra.SA.UpdateRegistration(base)
	if err != nil {
		err = core.InternalServerError(err.Error())
	}
	return
}

func (ra *RegistrationAuthorityImpl) UpdateAuthorization(base core.Authorization, challengeIndex int, response core.Challenge) (authz core.Authorization, err error) {
	// Copy information over that the client is allowed to supply
	authz = base
	if challengeIndex >= len(authz.Challenges) {
		err = core.MalformedRequestError("Invalid challenge index")
		return
	}
	authz.Challenges[challengeIndex] = authz.Challenges[challengeIndex].MergeResponse(response)

	// Store the updated version
	if err = ra.SA.UpdatePendingAuthorization(authz); err != nil {
		err = core.InternalServerError(err.Error())
		return
	}

	// Dispatch to the VA for service
	ra.VA.UpdateValidations(authz, challengeIndex)

	return
}

func (ra *RegistrationAuthorityImpl) RevokeCertificate(cert x509.Certificate) (err error) {
	serialString := core.SerialToString(cert.SerialNumber)
	err = ra.CA.RevokeCertificate(serialString, 0)

	// AUDIT[ Revocation Requests ] 4e85d791-09c0-4ab3-a837-d3d67e945134
	if err != nil {
		ra.log.Audit(fmt.Sprintf("Revocation error - %s - %s", serialString, err))
		err = core.InternalServerError(err.Error())
		return
	}

	ra.log.Audit(fmt.Sprintf("Revocation - %s", serialString))
	return
}

func (ra *RegistrationAuthorityImpl) OnValidationUpdate(authz core.Authorization) error {
	// Check to see whether the updated validations are sufficient
	// Current policy is to accept if any validation succeeded
	for _, val := range authz.Challenges {
		if val.Status == core.StatusValid {
			authz.Status = core.StatusValid
			break
		}
	}

	// If no validation succeeded, then the authorization is invalid
	// NOTE: This only works because we only ever do one validation
	if authz.Status != core.StatusValid {
		authz.Status = core.StatusInvalid
	} else {
		// TODO: Enable configuration of expiry time
		authz.Expires = time.Now().Add(365 * 24 * time.Hour)
	}

	// Finalize the authorization (error ignored)
	return ra.SA.FinalizeAuthorization(authz)
}
<commit_msg>Aesthetic fixes to ra.go<commit_after>// Copyright 2014 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package ra

import (
	"crypto/x509"
	"fmt"
	"math/big"
	"net/url"
	"regexp"
	"strconv"
	"time"

	"github.com/letsencrypt/boulder/core"
	blog "github.com/letsencrypt/boulder/log"
	"github.com/letsencrypt/boulder/policy"
)

// All of the fields in RegistrationAuthorityImpl need to be
// populated, or there is a risk of panic.
type RegistrationAuthorityImpl struct {
	CA  core.CertificateAuthority
	VA  core.ValidationAuthority
	SA  core.StorageAuthority
	PA  core.PolicyAuthority
	log *blog.AuditLogger

	AuthzBase string
}

func NewRegistrationAuthorityImpl() RegistrationAuthorityImpl {
	logger := blog.GetAuditLogger()
	logger.Notice("Registration Authority Starting")

	ra := RegistrationAuthorityImpl{log: logger}
	ra.PA = policy.NewPolicyAuthorityImpl()
	return ra
}

var allButLastPathSegment = regexp.MustCompile("^.*/")

func lastPathSegment(url core.AcmeURL) string {
	return allButLastPathSegment.ReplaceAllString(url.Path, "")
}

type certificateRequestEvent struct {
	ID                  string    `json:",omitempty"`
	Requester           int64     `json:",omitempty"`
	SerialNumber        *big.Int  `json:",omitempty"`
	RequestMethod       string    `json:",omitempty"`
	VerificationMethods []string  `json:",omitempty"`
	VerifiedFields      []string  `json:",omitempty"`
	CommonName          string    `json:",omitempty"`
	Names               []string  `json:",omitempty"`
	NotBefore           time.Time `json:",omitempty"`
	NotAfter            time.Time `json:",omitempty"`
	RequestTime         time.Time `json:",omitempty"`
	ResponseTime        time.Time `json:",omitempty"`
	Error               string    `json:",omitempty"`
}

func (ra *RegistrationAuthorityImpl) NewRegistration(init core.Registration) (reg core.Registration, err error) {
	if !core.GoodKey(init.Key.Key) {
		return core.Registration{}, core.UnauthorizedError("Invalid public key.")
	}
	reg = core.Registration{
		RecoveryToken: core.NewToken(),
		Key:           init.Key,
	}
	reg.MergeUpdate(init)

	// Store the authorization object, then return it
	reg, err = ra.SA.NewRegistration(reg)
	if err != nil {
		err = core.InternalServerError(err.Error())
	}
	return
}

func (ra *RegistrationAuthorityImpl) NewAuthorization(request core.Authorization, regID int64) (authz core.Authorization, err error) {
	if regID <= 0 {
		err = core.InternalServerError("Invalid registration ID")
		return authz, err
	}

	identifier := request.Identifier

	// Check that the identifier is present and appropriate
	if err = ra.PA.WillingToIssue(identifier); err != nil {
		err = core.UnauthorizedError(err.Error())
		return authz, err
	}

	// Create validations, but we have to update them with URIs later
	challenges, combinations := ra.PA.ChallengesFor(identifier)

	// Partially-filled object
	authz = core.Authorization{
		Identifier:     identifier,
		RegistrationID: regID,
		Status:         core.StatusPending,
		Combinations:   combinations,
	}

	// Get a pending Auth first so we can get our ID back, then update with challenges
	authz, err = ra.SA.NewPendingAuthorization(authz)
	if err != nil {
		err = core.InternalServerError(err.Error())
		return authz, err
	}

	// Construct all the challenge URIs
	for i := range challenges {
		// Ignoring these errors because we construct the URLs to be correct
		challengeURI, _ := url.Parse(ra.AuthzBase + authz.ID + "?challenge=" + strconv.Itoa(i))
		challenges[i].URI = core.AcmeURL(*challengeURI)

		if !challenges[i].IsSane(false) {
			err = core.InternalServerError(fmt.Sprintf("Challenge didn't pass sanity check: %+v", challenges[i]))
			return authz, err
		}
	}

	// Update object
	authz.Challenges = challenges

	// Store the authorization object, then return it
	err = ra.SA.UpdatePendingAuthorization(authz)
	if err != nil {
		err = core.InternalServerError(err.Error())
	}
	return authz, err
}

func (ra *RegistrationAuthorityImpl) NewCertificate(req core.CertificateRequest, regID int64) (cert core.Certificate, err error) {
	emptyCert := core.Certificate{}
	var logEventResult string

	// Assume the worst
	logEventResult = "error"

	// Construct the log event
	logEvent := certificateRequestEvent{
		ID:            core.NewToken(),
		Requester:     regID,
		RequestMethod: "online",
		RequestTime:   time.Now(),
	}

	// No matter what, log the request
	defer func() {
		// AUDIT[ Certificate Requests ] 11917fa4-10ef-4e0d-9105-bacbe7836a3c
		ra.log.AuditObject(fmt.Sprintf("Certificate request - %s", logEventResult), logEvent)
	}()

	if regID <= 0 {
		err = core.InternalServerError("Invalid registration ID")
		return emptyCert, err
	}

	registration, err := ra.SA.GetRegistration(regID)
	if err != nil {
		logEvent.Error = err.Error()
		return emptyCert, err
	}

	// Verify the CSR
	csr := req.CSR
	if err = core.VerifyCSR(csr); err != nil {
		logEvent.Error = err.Error()
		err = core.UnauthorizedError("Invalid signature on CSR")
		return emptyCert, err
	}

	logEvent.CommonName = csr.Subject.CommonName
	logEvent.Names = csr.DNSNames

	// Validate that authorization key is authorized for all domains
	names := make([]string, len(csr.DNSNames))
	copy(names, csr.DNSNames)
	if len(csr.Subject.CommonName) > 0 {
		names = append(names, csr.Subject.CommonName)
	}

	if len(names) == 0 {
		err = core.UnauthorizedError("CSR has no names in it")
		logEvent.Error = err.Error()
		return emptyCert, err
	}

	csrPreviousDenied, err := ra.SA.AlreadyDeniedCSR(names)
	if err != nil {
		logEvent.Error = err.Error()
		return emptyCert, err
	}
	if csrPreviousDenied {
		err = core.UnauthorizedError("CSR has already been revoked/denied")
		logEvent.Error = err.Error()
		return emptyCert, err
	}

	if core.KeyDigestEquals(csr.PublicKey, registration.Key) {
		err = core.MalformedRequestError("Certificate public key must be different than account key")
		return emptyCert, err
	}

	// Gather authorized domains from the referenced authorizations
	authorizedDomains := map[string]bool{}
	verificationMethodSet := map[string]bool{}
	earliestExpiry := time.Date(2100, 01, 01, 0, 0, 0, 0, time.UTC)
	now := time.Now()
	for _, url := range req.Authorizations {
		id := lastPathSegment(url)
		authz, err := ra.SA.GetAuthorization(id)
		if err != nil || // Couldn't find authorization
			authz.RegistrationID != registration.ID || // Not for this account
			authz.Status != core.StatusValid || // Not finalized or not successful
			authz.Expires.Before(now) || // Expired
			authz.Identifier.Type != core.IdentifierDNS {
			// XXX: It may be good to fail here instead of ignoring invalid authorizations.
			//      However, it seems like this treatment is more in the spirit of Postel's
			//      law, and it hides information from attackers.
			continue
		}

		if authz.Expires.Before(earliestExpiry) {
			earliestExpiry = authz.Expires
		}

		for _, challenge := range authz.Challenges {
			if challenge.Status == core.StatusValid {
				verificationMethodSet[challenge.Type] = true
			}
		}

		authorizedDomains[authz.Identifier.Value] = true
	}
	verificationMethods := []string{}
	for method, _ := range verificationMethodSet {
		verificationMethods = append(verificationMethods, method)
	}
	logEvent.VerificationMethods = verificationMethods

	// Validate all domains
	for _, name := range names {
		if !authorizedDomains[name] {
			err = core.UnauthorizedError(fmt.Sprintf("Key not authorized for name %s", name))
			logEvent.Error = err.Error()
			return emptyCert, err
		}
	}

	// Mark that we verified the CN and SANs
	logEvent.VerifiedFields = []string{"subject.commonName", "subjectAltName"}

	// Create the certificate and log the result
	if cert, err = ra.CA.IssueCertificate(*csr, regID, earliestExpiry); err != nil {
		err = core.InternalServerError(err.Error())
		logEvent.Error = err.Error()
		return emptyCert, err
	}

	parsedCertificate, err := x509.ParseCertificate([]byte(cert.DER))
	if err != nil {
		err = core.InternalServerError(err.Error())
		logEvent.Error = err.Error()
		return emptyCert, err
	}

	logEvent.SerialNumber = parsedCertificate.SerialNumber
	logEvent.CommonName = parsedCertificate.Subject.CommonName
	logEvent.NotBefore = parsedCertificate.NotBefore
	logEvent.NotAfter = parsedCertificate.NotAfter
	logEvent.ResponseTime = time.Now()

	logEventResult = "successful"
	return cert, nil
}

func (ra *RegistrationAuthorityImpl) UpdateRegistration(base core.Registration, update core.Registration) (reg core.Registration, err error) {
	base.MergeUpdate(update)
	reg = base
	err = ra.SA.UpdateRegistration(base)
	if err != nil {
		err = core.InternalServerError(err.Error())
	}
	return
}

func (ra *RegistrationAuthorityImpl) UpdateAuthorization(base core.Authorization, challengeIndex int, response core.Challenge) (authz core.Authorization, err error) {
	// Copy information over that the client is allowed to supply
	authz = base
	if challengeIndex >= len(authz.Challenges) {
		err = core.MalformedRequestError("Invalid challenge index")
		return
	}
	authz.Challenges[challengeIndex] = authz.Challenges[challengeIndex].MergeResponse(response)

	// Store the updated version
	if err = ra.SA.UpdatePendingAuthorization(authz); err != nil {
		err = core.InternalServerError(err.Error())
		return
	}

	// Dispatch to the VA for service
	ra.VA.UpdateValidations(authz, challengeIndex)

	return
}

func (ra *RegistrationAuthorityImpl) RevokeCertificate(cert x509.Certificate) (err error) {
	serialString := core.SerialToString(cert.SerialNumber)
	err = ra.CA.RevokeCertificate(serialString, 0)

	// AUDIT[ Revocation Requests ] 4e85d791-09c0-4ab3-a837-d3d67e945134
	if err != nil {
		ra.log.Audit(fmt.Sprintf("Revocation error - %s - %s", serialString, err))
		return
	}

	ra.log.Audit(fmt.Sprintf("Revocation - %s", serialString))
	return err
}

func (ra *RegistrationAuthorityImpl) OnValidationUpdate(authz core.Authorization) error {
	// Check to see whether the updated validations are sufficient
	// Current policy is to accept if any validation succeeded
	for _, val := range authz.Challenges {
		if val.Status == core.StatusValid {
			authz.Status = core.StatusValid
			break
		}
	}

	// If no validation succeeded, then the authorization is invalid
	// NOTE: This only works because we only ever do one validation
	if authz.Status != core.StatusValid {
		authz.Status = core.StatusInvalid
	} else {
		// TODO: Enable configuration of expiry time
		authz.Expires = time.Now().Add(365 * 24 * time.Hour)
	}

	// Finalize the authorization (error ignored)
	return ra.SA.FinalizeAuthorization(authz)
}
<|endoftext|>


<commit_before>/*
 * http://www.apache.org/licenses/LICENSE-2.0.txt
 *
 * Copyright 2017 OpsVision Solutions
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 	http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package signalfx

import (
	"bytes"
	"fmt"
	"github.com/intelsdi-x/snap-plugin-lib-go/v1/plugin"
	"github.com/signalfx/golib/datapoint"
	"github.com/signalfx/golib/sfxclient"
	"golang.org/x/net/context"
	"log"
	"os"
	"strings"
)

const (
	NS_VENDOR = "opsvision"
	NS_PLUGIN = "signalfx"
	VERSION   = 1
)

var fileHandle *os.File

type SignalFx struct {
	token     string
	hostname  string
	namespace string
}

// Constructor
func New() *SignalFx {
	return new(SignalFx)
}

/**
 * Returns the configPolicy for the plugin
 */
func (s *SignalFx) GetConfigPolicy() (plugin.ConfigPolicy, error) {
	policy := plugin.NewConfigPolicy()

	// The SignalFx token
	policy.AddNewStringRule([]string{NS_VENDOR, NS_PLUGIN},
		"token",
		true)

	// The hostname to use (defaults to local hostname)
	policy.AddNewStringRule([]string{NS_VENDOR, NS_PLUGIN},
		"hostname",
		false)

	// The file name to use when debugging
	policy.AddNewStringRule([]string{NS_VENDOR, NS_PLUGIN},
		"debug-file",
		false)

	return *policy, nil
}

/**
 * Publish metrics to SignalFx using the TOKEN found in the config
 */
func (s *SignalFx) Publish(mts []plugin.Metric, cfg plugin.Config) error {
	// Enable debugging if the debug-file config property was set
	fileName, err := cfg.GetString("debug-file")
	if err != nil {
		// Open the output file
		f, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			return err
		}
		defer f.Close()

		// Set logging output for debugging
		log.SetOutput(f)
	}

	// Fetch the token
	token, err := cfg.GetString("token")
	if err != nil {
		return err
	}
	s.token = token

	// Attempt to set the hostname
	hostname, err := cfg.GetString("hostname")
	if err != nil {
		hostname, err = os.Hostname()
		if err != nil {
			hostname = "localhost"
		}
	}
	s.hostname = hostname

	// Iterate over the supplied metrics
	for _, m := range mts {
		var buffer bytes.Buffer

		// Convert the namespace to dot notation
		fmt.Fprintf(&buffer, "snap.%s", strings.Join(m.Namespace.Strings(), "."))
		s.namespace = buffer.String()

		// Do some type conversion and send the data
		switch v := m.Data.(type) {
		case uint:
			s.sendIntValue(int64(v))
		case uint32:
			s.sendIntValue(int64(v))
		case uint64:
			s.sendIntValue(int64(v))
		case int:
			s.sendIntValue(int64(v))
		case int32:
			s.sendIntValue(int64(v))
		case int64:
			s.sendIntValue(int64(v))
		case float32:
			s.sendFloatValue(float64(v))
		case float64:
			s.sendFloatValue(float64(v))
		default:
			log.Printf("Ignoring %T: %v\n", v, v)
			log.Printf("Contact the plugin author if you think this is an error")
		}
	}

	return nil
}

/**
 * Method for sending int64 values to SignalFx
 */
func (s *SignalFx) sendIntValue(value int64) {
	log.Printf("Sending [int64] %s -> %v", s.namespace, value)

	client := sfxclient.NewHTTPDatapointSink()
	client.AuthToken = s.token
	ctx := context.Background()
	client.AddDatapoints(ctx, []*datapoint.Datapoint{
		sfxclient.Gauge(s.namespace, map[string]string{
			"host": s.hostname,
		}, value),
	})
}

/**
 * Method for sending float64 values to SignalFx
 */
func (s *SignalFx) sendFloatValue(value float64) {
	log.Printf("Sending [float64] %s -> %v", s.namespace, value)

	client := sfxclient.NewHTTPDatapointSink()
	client.AuthToken = s.token
	ctx := context.Background()
	client.AddDatapoints(ctx, []*datapoint.Datapoint{
		sfxclient.GaugeF(s.namespace, map[string]string{
			"host": s.hostname,
		}, value),
	})
}
<commit_msg>Performed cleanup and added comments<commit_after>/*
 * http://www.apache.org/licenses/LICENSE-2.0.txt
 *
 * Copyright 2017 OpsVision Solutions
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 	http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package signalfx

import (
	"bytes"
	"fmt"
	"github.com/intelsdi-x/snap-plugin-lib-go/v1/plugin"
	"github.com/signalfx/golib/datapoint"
	"github.com/signalfx/golib/sfxclient"
	"golang.org/x/net/context"
	"log"
	"os"
	"strings"
)

const (
	NS_VENDOR = "opsvision"		// plugin vendor
	NS_PLUGIN = "signalfx"		// plugin name
	VERSION   = 1			// plugin version
)

// Our SignalFx object
type SignalFx struct {
	token     string		// SignalFx API token
	hostname  string		// Hostname
	namespace string		// Metric namespace
}

// Constructor
func New() *SignalFx {
	return new(SignalFx)
}

/**
 * Returns the configPolicy for the plugin
 */
func (s *SignalFx) GetConfigPolicy() (plugin.ConfigPolicy, error) {
	policy := plugin.NewConfigPolicy()

	// The SignalFx token
	policy.AddNewStringRule([]string{NS_VENDOR, NS_PLUGIN},
		"token",
		true)

	// The hostname to use (defaults to local hostname)
	policy.AddNewStringRule([]string{NS_VENDOR, NS_PLUGIN},
		"hostname",
		false)

	// The file name to use when debugging
	policy.AddNewStringRule([]string{NS_VENDOR, NS_PLUGIN},
		"debug-file",
		false)

	return *policy, nil
}

/**
 * Publish metrics to SignalFx using the TOKEN found in the config
 */
func (s *SignalFx) Publish(mts []plugin.Metric, cfg plugin.Config) error {
	// Enable debugging if the debug-file config property was set
	fileName, err := cfg.GetString("debug-file")
	if err != nil {
		// Open the output file
		f, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			return err
		}
		defer f.Close()

		// Set logging output for debugging
		log.SetOutput(f)
	}

	// Fetch the token
	token, err := cfg.GetString("token")
	if err != nil {
		return err
	}
	s.token = token

	// Attempt to set the hostname
	hostname, err := cfg.GetString("hostname")
	if err != nil {
		hostname, err = os.Hostname()
		if err != nil {
			hostname = "localhost"
		}
	}
	s.hostname = hostname

	// Iterate over the supplied metrics
	for _, m := range mts {
		var buffer bytes.Buffer

		// Convert the namespace to dot notation
		fmt.Fprintf(&buffer, "snap.%s", strings.Join(m.Namespace.Strings(), "."))
		s.namespace = buffer.String()

		// Do some type conversion and send the data
		switch v := m.Data.(type) {
		case uint:
			s.sendIntValue(int64(v))
		case uint32:
			s.sendIntValue(int64(v))
		case uint64:
			s.sendIntValue(int64(v))
		case int:
			s.sendIntValue(int64(v))
		case int32:
			s.sendIntValue(int64(v))
		case int64:
			s.sendIntValue(int64(v))
		case float32:
			s.sendFloatValue(float64(v))
		case float64:
			s.sendFloatValue(float64(v))
		default:
			log.Printf("Ignoring %T: %v\n", v, v)
			log.Printf("Contact the plugin author if you think this is an error")
		}
	}

	return nil
}

/**
 * Method for sending int64 values to SignalFx
 */
func (s *SignalFx) sendIntValue(value int64) {
	log.Printf("Sending [int64] %s -> %v", s.namespace, value)

	client := sfxclient.NewHTTPDatapointSink()
	client.AuthToken = s.token
	ctx := context.Background()
	client.AddDatapoints(ctx, []*datapoint.Datapoint{
		sfxclient.Gauge(s.namespace, map[string]string{
			"host": s.hostname,
		}, value),
	})
}

/**
 * Method for sending float64 values to SignalFx
 */
func (s *SignalFx) sendFloatValue(value float64) {
	log.Printf("Sending [float64] %s -> %v", s.namespace, value)

	client := sfxclient.NewHTTPDatapointSink()
	client.AuthToken = s.token
	ctx := context.Background()
	client.AddDatapoints(ctx, []*datapoint.Datapoint{
		sfxclient.GaugeF(s.namespace, map[string]string{
			"host": s.hostname,
		}, value),
	})
}
<|endoftext|>


<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to you under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package avatica

import (
	"context"
	"database/sql/driver"
	"io"
	"time"

	"github.com/apache/calcite-avatica-go/v5/internal"
	"github.com/apache/calcite-avatica-go/v5/message"
)

type resultSet struct {
	columns    []*internal.Column
	done       bool
	offset     uint64
	data       [][]*message.TypedValue
	currentRow int
}

type rows struct {
	conn             *conn
	statementID      uint32
	resultSets       []*resultSet
	currentResultSet int
}

// Columns returns the names of the columns. The number of
// columns of the result is inferred from the length of the
// slice.  If a particular column name isn't known, an empty
// string should be returned for that entry.
func (r *rows) Columns() []string {

	var cols []string

	for _, column := range r.resultSets[r.currentResultSet].columns {
		cols = append(cols, column.Name)
	}

	return cols
}

// Close closes the rows iterator.
func (r *rows) Close() error {

	r.conn = nil
	return nil
}

// Next is called to populate the next row of data into
// the provided slice. The provided slice will be the same
// size as the Columns() are wide.
//
// The dest slice may be populated only with
// a driver Value type, but excluding string.
// All string values must be converted to []byte.
//
// Next should return io.EOF when there are no more rows.
func (r *rows) Next(dest []driver.Value) error {

	resultSet := r.resultSets[r.currentResultSet]

	if resultSet.currentRow >= len(resultSet.data) {

		if resultSet.done {
			// Finished iterating through all results
			return io.EOF
		}

		// Fetch more results from the server
		res, err := r.conn.httpClient.post(context.Background(), &message.FetchRequest{
			ConnectionId: r.conn.connectionId,
			StatementId:  r.statementID,
			Offset:       resultSet.offset,
			FrameMaxSize: r.conn.config.frameMaxSize,
		})

		if err != nil {
			return r.conn.avaticaErrorToResponseErrorOrError(err)
		}

		frame := res.(*message.FetchResponse).Frame

		var data [][]*message.TypedValue

		// In some cases the server does not return done as true
		// until it returns a result with no rows
		if len(frame.Rows) == 0 {
			return io.EOF
		}

		for _, row := range frame.Rows {
			var rowData []*message.TypedValue

			for _, col := range row.Value {
				rowData = append(rowData, col.ScalarValue)
			}

			data = append(data, rowData)
		}

		resultSet.done = frame.Done
		resultSet.data = data
		resultSet.currentRow = 0

	}

	for i, val := range resultSet.data[resultSet.currentRow] {
		dest[i] = typedValueToNative(resultSet.columns[i].Rep, val, r.conn.config)
	}

	resultSet.currentRow++

	return nil
}

// newRows create a new set of rows from a result set.
func newRows(conn *conn, statementID uint32, resultSets []*message.ResultSetResponse) *rows {

	var rsets []*resultSet

	for _, result := range resultSets {
		if result.Signature == nil {
			break
		}

		var columns []*internal.Column

		for _, col := range result.Signature.Columns {
			column := conn.adapter.GetColumnTypeDefinition(col)
			columns = append(columns, column)
		}

		frame := result.FirstFrame

		var data [][]*message.TypedValue

		for _, row := range frame.Rows {
			var rowData []*message.TypedValue

			for _, col := range row.Value {
				rowData = append(rowData, col.ScalarValue)
			}

			data = append(data, rowData)
		}

		rsets = append(rsets, &resultSet{
			columns: columns,
			done:    frame.Done,
			offset:  frame.Offset,
			data:    data,
		})
	}

	return &rows{
		conn:             conn,
		statementID:      statementID,
		resultSets:       rsets,
		currentResultSet: 0,
	}
}

// typedValueToNative converts values from avatica's types to Go's native types
func typedValueToNative(rep message.Rep, v *message.TypedValue, config *Config) interface{} {

	if v.Type == message.Rep_NULL {
		return nil
	}

	switch rep {

	case message.Rep_BOOLEAN, message.Rep_PRIMITIVE_BOOLEAN:
		return v.BoolValue

	case message.Rep_STRING, message.Rep_PRIMITIVE_CHAR, message.Rep_CHARACTER, message.Rep_BIG_DECIMAL:
		return v.StringValue

	case message.Rep_FLOAT, message.Rep_PRIMITIVE_FLOAT:
		return float32(v.DoubleValue)

	case message.Rep_LONG,
		message.Rep_PRIMITIVE_LONG,
		message.Rep_INTEGER,
		message.Rep_PRIMITIVE_INT,
		message.Rep_BIG_INTEGER,
		message.Rep_NUMBER,
		message.Rep_BYTE,
		message.Rep_PRIMITIVE_BYTE,
		message.Rep_SHORT,
		message.Rep_PRIMITIVE_SHORT:
		return v.NumberValue

	case message.Rep_BYTE_STRING:
		return v.BytesValue

	case message.Rep_DOUBLE, message.Rep_PRIMITIVE_DOUBLE:
		return v.DoubleValue

	case message.Rep_JAVA_SQL_DATE, message.Rep_JAVA_UTIL_DATE:

		// We receive the number of days since 1970/1/1 from the server
		// Because a location can have multiple time zones due to daylight savings,
		// we first do all our calculations in UTC and then force the timezone to
		// the one the user has chosen.
		t, _ := time.ParseInLocation("2006-Jan-02", "1970-Jan-01", time.UTC)
		days := time.Hour * 24 * time.Duration(v.NumberValue)
		t = t.Add(days)

		return forceTimezone(t, config.location)

	case message.Rep_JAVA_SQL_TIME:

		// We receive the number of milliseconds since 00:00:00.000 from the server
		// Because a location can have multiple time zones due to daylight savings,
		// we first do all our calculations in UTC and then force the timezone to
		// the one the user has chosen.
		t, _ := time.ParseInLocation("15:04:05", "00:00:00", time.UTC)
		ms := time.Millisecond * time.Duration(v.NumberValue)
		t = t.Add(ms)
		return forceTimezone(t, config.location)

	case message.Rep_JAVA_SQL_TIMESTAMP:

		// We receive the number of milliseconds since 1970-01-01 00:00:00.000 from the server
		// Force to UTC for consistency because time.Unix uses the local timezone
		t := time.Unix(0, v.NumberValue*int64(time.Millisecond)).In(time.UTC)
		return forceTimezone(t, config.location)

	default:
		return nil
	}
}

// forceTimezone takes a time.Time and changes its location without shifting the timezone.
func forceTimezone(t time.Time, loc *time.Location) time.Time {
	return time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), loc)
}
<commit_msg>[CALCITE-5072] Index out of range when calling rows.Next()<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to you under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package avatica

import (
	"context"
	"database/sql/driver"
	"io"
	"time"

	"github.com/apache/calcite-avatica-go/v5/internal"
	"github.com/apache/calcite-avatica-go/v5/message"
)

type resultSet struct {
	columns    []*internal.Column
	done       bool
	offset     uint64
	data       [][]*message.TypedValue
	currentRow int
}

type rows struct {
	conn             *conn
	statementID      uint32
	resultSets       []*resultSet
	currentResultSet int
	columnNames      []string
}

// Columns returns the names of the columns. The number of
// columns of the result is inferred from the length of the
// slice.  If a particular column name isn't known, an empty
// string should be returned for that entry.
func (r *rows) Columns() []string {
	if r.columnNames != nil {
		return r.columnNames
	}
	var cols []string
	if len(r.resultSets) == 0 {
		return cols
	}
	for _, column := range r.resultSets[r.currentResultSet].columns {
		cols = append(cols, column.Name)
	}
	r.columnNames = cols
	return cols
}

// Close closes the rows iterator.
func (r *rows) Close() error {

	r.conn = nil
	return nil
}

// Next is called to populate the next row of data into
// the provided slice. The provided slice will be the same
// size as the Columns() are wide.
//
// The dest slice may be populated only with
// a driver Value type, but excluding string.
// All string values must be converted to []byte.
//
// Next should return io.EOF when there are no more rows.
func (r *rows) Next(dest []driver.Value) error {
	if len(r.resultSets) == 0 {
		return io.EOF
	}
	resultSet := r.resultSets[r.currentResultSet]

	if resultSet.currentRow >= len(resultSet.data) {

		if resultSet.done {
			// Finished iterating through all results
			return io.EOF
		}

		// Fetch more results from the server
		res, err := r.conn.httpClient.post(context.Background(), &message.FetchRequest{
			ConnectionId: r.conn.connectionId,
			StatementId:  r.statementID,
			Offset:       resultSet.offset,
			FrameMaxSize: r.conn.config.frameMaxSize,
		})

		if err != nil {
			return r.conn.avaticaErrorToResponseErrorOrError(err)
		}

		frame := res.(*message.FetchResponse).Frame

		var data [][]*message.TypedValue

		// In some cases the server does not return done as true
		// until it returns a result with no rows
		if len(frame.Rows) == 0 {
			return io.EOF
		}

		for _, row := range frame.Rows {
			var rowData []*message.TypedValue

			for _, col := range row.Value {
				rowData = append(rowData, col.ScalarValue)
			}

			data = append(data, rowData)
		}

		resultSet.done = frame.Done
		resultSet.data = data
		resultSet.currentRow = 0

	}

	for i, val := range resultSet.data[resultSet.currentRow] {
		dest[i] = typedValueToNative(resultSet.columns[i].Rep, val, r.conn.config)
	}

	resultSet.currentRow++

	return nil
}

// newRows create a new set of rows from a result set.
func newRows(conn *conn, statementID uint32, resultSets []*message.ResultSetResponse) *rows {

	var rsets []*resultSet

	for _, result := range resultSets {
		if result.Signature == nil {
			break
		}

		var columns []*internal.Column

		for _, col := range result.Signature.Columns {
			column := conn.adapter.GetColumnTypeDefinition(col)
			columns = append(columns, column)
		}

		frame := result.FirstFrame

		var data [][]*message.TypedValue

		for _, row := range frame.Rows {
			var rowData []*message.TypedValue

			for _, col := range row.Value {
				rowData = append(rowData, col.ScalarValue)
			}

			data = append(data, rowData)
		}

		rsets = append(rsets, &resultSet{
			columns: columns,
			done:    frame.Done,
			offset:  frame.Offset,
			data:    data,
		})
	}

	return &rows{
		conn:             conn,
		statementID:      statementID,
		resultSets:       rsets,
		currentResultSet: 0,
	}
}

// typedValueToNative converts values from avatica's types to Go's native types
func typedValueToNative(rep message.Rep, v *message.TypedValue, config *Config) interface{} {

	if v.Type == message.Rep_NULL {
		return nil
	}

	switch rep {

	case message.Rep_BOOLEAN, message.Rep_PRIMITIVE_BOOLEAN:
		return v.BoolValue

	case message.Rep_STRING, message.Rep_PRIMITIVE_CHAR, message.Rep_CHARACTER, message.Rep_BIG_DECIMAL:
		return v.StringValue

	case message.Rep_FLOAT, message.Rep_PRIMITIVE_FLOAT:
		return float32(v.DoubleValue)

	case message.Rep_LONG,
		message.Rep_PRIMITIVE_LONG,
		message.Rep_INTEGER,
		message.Rep_PRIMITIVE_INT,
		message.Rep_BIG_INTEGER,
		message.Rep_NUMBER,
		message.Rep_BYTE,
		message.Rep_PRIMITIVE_BYTE,
		message.Rep_SHORT,
		message.Rep_PRIMITIVE_SHORT:
		return v.NumberValue

	case message.Rep_BYTE_STRING:
		return v.BytesValue

	case message.Rep_DOUBLE, message.Rep_PRIMITIVE_DOUBLE:
		return v.DoubleValue

	case message.Rep_JAVA_SQL_DATE, message.Rep_JAVA_UTIL_DATE:

		// We receive the number of days since 1970/1/1 from the server
		// Because a location can have multiple time zones due to daylight savings,
		// we first do all our calculations in UTC and then force the timezone to
		// the one the user has chosen.
		t, _ := time.ParseInLocation("2006-Jan-02", "1970-Jan-01", time.UTC)
		days := time.Hour * 24 * time.Duration(v.NumberValue)
		t = t.Add(days)

		return forceTimezone(t, config.location)

	case message.Rep_JAVA_SQL_TIME:

		// We receive the number of milliseconds since 00:00:00.000 from the server
		// Because a location can have multiple time zones due to daylight savings,
		// we first do all our calculations in UTC and then force the timezone to
		// the one the user has chosen.
		t, _ := time.ParseInLocation("15:04:05", "00:00:00", time.UTC)
		ms := time.Millisecond * time.Duration(v.NumberValue)
		t = t.Add(ms)
		return forceTimezone(t, config.location)

	case message.Rep_JAVA_SQL_TIMESTAMP:

		// We receive the number of milliseconds since 1970-01-01 00:00:00.000 from the server
		// Force to UTC for consistency because time.Unix uses the local timezone
		t := time.Unix(0, v.NumberValue*int64(time.Millisecond)).In(time.UTC)
		return forceTimezone(t, config.location)

	default:
		return nil
	}
}

// forceTimezone takes a time.Time and changes its location without shifting the timezone.
func forceTimezone(t time.Time, loc *time.Location) time.Time {
	return time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), loc)
}
<|endoftext|>