<commit_before>package shell

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"reflect"
	"strings"
	"sync"
	"syscall"

	"github.com/zetamatta/go-findfile"

	"github.com/zetamatta/nyagos/defined"
	"github.com/zetamatta/nyagos/dos"
)

var WildCardExpansionAlways = false

type CommandNotFound struct {
	Name string
	Err  error
}

func (this CommandNotFound) Stringer() string {
	return fmt.Sprintf("'%s' is not recognized as an internal or external command,\noperable program or batch file", this.Name)
}

func (this CommandNotFound) Error() string {
	return this.Stringer()
}

type session struct {
	unreadline []string
}

type CloneCloser interface {
	Clone(context.Context) (context.Context, CloneCloser, error)
	Close() error
}

type Shell struct {
	*session
	Stdout       *os.File
	Stderr       *os.File
	Stdin        *os.File
	Console      io.Writer
	tag          CloneCloser
	IsBackGround bool
}

func (sh *Shell) In() io.Reader          { return sh.Stdin }
func (sh *Shell) Out() io.Writer         { return sh.Stdout }
func (sh *Shell) Err() io.Writer         { return sh.Stderr }
func (sh *Shell) Term() io.Writer        { return sh.Console }
func (sh *Shell) Tag() CloneCloser       { return sh.tag }
func (sh *Shell) SetTag(tag CloneCloser) { sh.tag = tag }

type Cmd struct {
	Shell
	args            []string
	rawArgs         []string
	fullPath        string
	UseShellExecute bool
	Closers         []io.Closer
}

func (cmd *Cmd) Arg(n int) string      { return cmd.args[n] }
func (cmd *Cmd) Args() []string        { return cmd.args }
func (cmd *Cmd) SetArgs(s []string)    { cmd.args = s }
func (cmd *Cmd) RawArg(n int) string   { return cmd.rawArgs[n] }
func (cmd *Cmd) RawArgs() []string     { return cmd.rawArgs }
func (cmd *Cmd) SetRawArgs(s []string) { cmd.rawArgs = s }

var LookCurdirOrder = dos.LookCurdirFirst

func (cmd *Cmd) FullPath() string {
	if cmd.args == nil || len(cmd.args) <= 0 {
		return ""
	}
	if cmd.fullPath == "" {
		cmd.fullPath = dos.LookPath(LookCurdirOrder, cmd.args[0], "NYAGOSPATH")
	}
	return cmd.fullPath
}

func (cmd *Cmd) Close() {
	if cmd.Closers != nil {
		for _, c := range cmd.Closers {
			c.Close()
		}
		cmd.Closers = nil
	}
}

func (sh *Shell) Close() {}

func New() *Shell {
	return &Shell{
		Stdin:   os.Stdin,
		Stdout:  os.Stdout,
		Stderr:  os.Stderr,
		session: &session{},
	}
}

func (sh *Shell) Command() *Cmd {
	cmd := &Cmd{
		Shell: Shell{
			Stdin:   sh.Stdin,
			Stdout:  sh.Stdout,
			Stderr:  sh.Stderr,
			Console: sh.Console,
			tag:     sh.tag,
		},
	}
	if sh.session != nil {
		cmd.session = sh.session
	} else {
		cmd.session = &session{}
	}
	return cmd
}

type ArgsHookT func(ctx context.Context, sh *Shell, args []string) ([]string, error)

var argsHook = func(ctx context.Context, sh *Shell, args []string) ([]string, error) {
	return args, nil
}

func SetArgsHook(argsHook_ ArgsHookT) (rv ArgsHookT) {
	rv, argsHook = argsHook, argsHook_
	return
}

type HookT func(context.Context, *Cmd) (int, bool, error)

var hook = func(context.Context, *Cmd) (int, bool, error) {
	return 0, false, nil
}

func SetHook(hook_ HookT) (rv HookT) {
	rv, hook = hook, hook_
	return
}

var OnCommandNotFound = func(ctx context.Context, cmd *Cmd, err error) error {
	err = &CommandNotFound{cmd.args[0], err}
	return err
}

var LastErrorLevel int

func makeCmdline(args, rawargs []string) string {
	var buffer strings.Builder
	for i, s := range args {
		if i > 0 {
			buffer.WriteRune(' ')
		}
		if (len(rawargs) > i && len(rawargs[i]) > 0 && rawargs[i][0] == '"') || strings.ContainsAny(s, " &|<>\t\"") {
			fmt.Fprintf(&buffer, `"%s"`, strings.Replace(s, `"`, `\"`, -1))
		} else {
			buffer.WriteString(s)
		}
	}
	return buffer.String()
}

var UseSourceRunBatch = true

func encloseWithQuote(fullpath string) string {
	if strings.ContainsRune(fullpath, ' ') {
		var f strings.Builder
		f.WriteRune('"')
		f.WriteString(fullpath)
		f.WriteRune('"')
		return f.String()
	} else {
		return fullpath
	}
}

func (cmd *Cmd) spawnvpSilent(ctx context.Context) (int, error) {
	// command is empty.
	if len(cmd.args) <= 0 {
		return 0, nil
	}
	if defined.DBG {
		print("spawnvpSilent('", cmd.args[0], "')\n")
	}

	// aliases and lua-commands
	if errorlevel, done, err := hook(ctx, cmd); done || err != nil {
		return errorlevel, err
	}

	// command not found hook
	fullpath := cmd.FullPath()
	if fullpath == "" {
		return 255, OnCommandNotFound(ctx, cmd, os.ErrNotExist)
	}
	cmd.args[0] = fullpath

	if defined.DBG {
		print("exec.LookPath(", cmd.args[0], ")==", fullpath, "\n")
	}
	if WildCardExpansionAlways {
		cmd.args = findfile.Globs(cmd.args)
	}
	if cmd.UseShellExecute {
		// GUI Application
		cmdline := makeCmdline(cmd.args[1:], cmd.rawArgs[1:])
		return 0, dos.ShellExecute("open", fullpath, cmdline, "")
	}
	if UseSourceRunBatch {
		lowerName := strings.ToLower(cmd.args[0])
		if strings.HasSuffix(lowerName, ".cmd") || strings.HasSuffix(lowerName, ".bat") {
			rawargs := cmd.RawArgs()
			args := make([]string, len(rawargs))
			args[0] = encloseWithQuote(fullpath)
			for i, end := 1, len(rawargs); i < end; i++ {
				args[i] = rawargs[i]
			}
			// Batch files
			return RawSource(args, nil, false, cmd.Stdin, cmd.Stdout, cmd.Stderr)
		}
	}
	// Do not use exec.CommandContext because it cancels background process.
	xcmd := exec.Command(cmd.args[0], cmd.args[1:]...)
	xcmd.Stdin = cmd.Stdin
	xcmd.Stdout = cmd.Stdout
	xcmd.Stderr = cmd.Stderr

	if xcmd.SysProcAttr == nil {
		xcmd.SysProcAttr = new(syscall.SysProcAttr)
	}
	cmdline := makeCmdline(xcmd.Args, cmd.rawArgs)
	if defined.DBG {
		println(cmdline)
	}
	xcmd.SysProcAttr.CmdLine = cmdline
	err := xcmd.Run()
	errorlevel, errorlevelOk := dos.GetErrorLevel(xcmd)
	if errorlevelOk {
		return errorlevel, err
	} else {
		return 255, err
	}
}

type AlreadyReportedError struct {
	Err error
}

func (_ AlreadyReportedError) Error() string {
	return ""
}

func IsAlreadyReported(err error) bool {
	_, ok := err.(AlreadyReportedError)
	return ok
}

func (cmd *Cmd) Spawnvp(ctx context.Context) (int, error) {
	errorlevel, err := cmd.spawnvpSilent(ctx)
	if err != nil && err != io.EOF && !IsAlreadyReported(err) {
		if defined.DBG {
			val := reflect.ValueOf(err)
			fmt.Fprintf(cmd.Stderr, "error-type=%s\n", val.Type())
		}
		fmt.Fprintln(cmd.Stderr, err.Error())
		err = AlreadyReportedError{err}
	}
	return errorlevel, err
}

func (sh *Shell) Spawnlp(ctx context.Context, args, rawargs []string) (int, error) {
	cmd := sh.Command()
	defer cmd.Close()
	cmd.SetArgs(args)
	cmd.SetRawArgs(rawargs)
	return cmd.Spawnvp(ctx)
}

func (sh *Shell) Interpret(ctx context.Context, text string) (errorlevel int, finalerr error) {
	if defined.DBG {
		print("Interpret('", text, "')\n")
	}
	if sh == nil {
		return 255, errors.New("Fatal Error: Interpret: instance is nil")
	}
	errorlevel = 0
	finalerr = nil

	statements, statementsErr := Parse(text)
	if statementsErr != nil {
		if defined.DBG {
			print("Parse Error:", statementsErr.Error(), "\n")
		}
		return 0, statementsErr
	}
	if argsHook != nil {
		if defined.DBG {
			print("call argsHook\n")
		}
		for _, pipeline := range statements {
			for _, state := range pipeline {
				var err error
				state.Args, err = argsHook(ctx, sh, state.Args)
				if err != nil {
					return 255, err
				}
			}
		}
		if defined.DBG {
			print("done argsHook\n")
		}
	}
	for _, pipeline := range statements {
		for i, state := range pipeline {
			if state.Term == "|" && (i+1 >= len(pipeline) || len(pipeline[i+1].Args) <= 0) {
				return 255, errors.New("The syntax of the command is incorrect.")
			}
		}
	}

	for _, pipeline := range statements {

		var pipeIn *os.File = nil
		isBackGround := sh.IsBackGround
		for _, state := range pipeline {
			if state.Term == "&" {
				isBackGround = true
				break
			}
		}
		var wg sync.WaitGroup
		shutdown_immediately := false
		for i, state := range pipeline {
			if defined.DBG {
				print(i, ": pipeline loop(", state.Args[0], ")\n")
			}
			cmd := sh.Command()
			cmd.IsBackGround = isBackGround

			if pipeIn != nil {
				cmd.Stdin = pipeIn
				cmd.Closers = append(cmd.Closers, pipeIn)
				pipeIn = nil
			}

			var err error
			if state.Term[0] == '|' {
				var pipeOut *os.File
				pipeIn, pipeOut, err = os.Pipe()
				cmd.Stdout = pipeOut
				if state.Term == "|&" {
					cmd.Stderr = pipeOut
				}
				cmd.Closers = append(cmd.Closers, pipeOut)
			}

			for _, red := range state.Redirect {
				var fd *os.File
				fd, err = red.OpenOn(cmd)
				if err != nil {
					return 0, err
				}
				defer fd.Close()
			}

			cmd.args = state.Args
			cmd.rawArgs = state.RawArgs
			if i > 0 {
				cmd.IsBackGround = true
			}
			if len(pipeline) == 1 && dos.IsGui(cmd.FullPath()) {
				cmd.UseShellExecute = true
			}
			if i == len(pipeline)-1 && state.Term != "&" {
				// foreground execution.
				errorlevel, finalerr = cmd.Spawnvp(ctx)
				LastErrorLevel = errorlevel
				cmd.Close()
			} else {
				// background
				if !isBackGround {
					wg.Add(1)
				}
				newctx := ctx
				if tag := cmd.Tag(); tag != nil {
					var newtag CloneCloser
					if newctx, newtag, err = tag.Clone(ctx); err != nil {
						fmt.Fprintln(os.Stderr, err.Error())
						return -1, err
					} else {
						cmd.SetTag(newtag)
					}
				}
				go func(ctx1 context.Context, cmd1 *Cmd) {
					if !isBackGround {
						defer wg.Done()
					}
					cmd1.Spawnvp(ctx1)
					if tag := cmd1.Tag(); tag != nil {
						if err := tag.Close(); err != nil {
							fmt.Fprintln(os.Stderr, err.Error())
						}
					}
					cmd1.Close()
				}(newctx, cmd)
			}
		}
		if !isBackGround {
			wg.Wait()
			if shutdown_immediately {
				return errorlevel, nil
			}
			if len(pipeline) > 0 {
				switch pipeline[len(pipeline)-1].Term {
				case "&&":
					if errorlevel != 0 {
						return errorlevel, nil
					}
				case "||":
					if errorlevel == 0 {
						return errorlevel, nil
					}
				}
			}
		}
	}
	return
}
<commit_msg>Fix: Commands with redirect (not pipeline) could not run on background<commit_after>package shell

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"reflect"
	"strings"
	"sync"
	"syscall"

	"github.com/zetamatta/go-findfile"

	"github.com/zetamatta/nyagos/defined"
	"github.com/zetamatta/nyagos/dos"
)

var WildCardExpansionAlways = false

type CommandNotFound struct {
	Name string
	Err  error
}

func (this CommandNotFound) Stringer() string {
	return fmt.Sprintf("'%s' is not recognized as an internal or external command,\noperable program or batch file", this.Name)
}

func (this CommandNotFound) Error() string {
	return this.Stringer()
}

type session struct {
	unreadline []string
}

type CloneCloser interface {
	Clone(context.Context) (context.Context, CloneCloser, error)
	Close() error
}

type Shell struct {
	*session
	Stdout       *os.File
	Stderr       *os.File
	Stdin        *os.File
	Console      io.Writer
	tag          CloneCloser
	IsBackGround bool
}

func (sh *Shell) In() io.Reader          { return sh.Stdin }
func (sh *Shell) Out() io.Writer         { return sh.Stdout }
func (sh *Shell) Err() io.Writer         { return sh.Stderr }
func (sh *Shell) Term() io.Writer        { return sh.Console }
func (sh *Shell) Tag() CloneCloser       { return sh.tag }
func (sh *Shell) SetTag(tag CloneCloser) { sh.tag = tag }

type Cmd struct {
	Shell
	args            []string
	rawArgs         []string
	fullPath        string
	UseShellExecute bool
	Closers         []io.Closer
}

func (cmd *Cmd) Arg(n int) string      { return cmd.args[n] }
func (cmd *Cmd) Args() []string        { return cmd.args }
func (cmd *Cmd) SetArgs(s []string)    { cmd.args = s }
func (cmd *Cmd) RawArg(n int) string   { return cmd.rawArgs[n] }
func (cmd *Cmd) RawArgs() []string     { return cmd.rawArgs }
func (cmd *Cmd) SetRawArgs(s []string) { cmd.rawArgs = s }

var LookCurdirOrder = dos.LookCurdirFirst

func (cmd *Cmd) FullPath() string {
	if cmd.args == nil || len(cmd.args) <= 0 {
		return ""
	}
	if cmd.fullPath == "" {
		cmd.fullPath = dos.LookPath(LookCurdirOrder, cmd.args[0], "NYAGOSPATH")
	}
	return cmd.fullPath
}

func (cmd *Cmd) Close() {
	if cmd.Closers != nil {
		for _, c := range cmd.Closers {
			c.Close()
		}
		cmd.Closers = nil
	}
}

func (sh *Shell) Close() {}

func New() *Shell {
	return &Shell{
		Stdin:   os.Stdin,
		Stdout:  os.Stdout,
		Stderr:  os.Stderr,
		session: &session{},
	}
}

func (sh *Shell) Command() *Cmd {
	cmd := &Cmd{
		Shell: Shell{
			Stdin:   sh.Stdin,
			Stdout:  sh.Stdout,
			Stderr:  sh.Stderr,
			Console: sh.Console,
			tag:     sh.tag,
		},
	}
	if sh.session != nil {
		cmd.session = sh.session
	} else {
		cmd.session = &session{}
	}
	return cmd
}

type ArgsHookT func(ctx context.Context, sh *Shell, args []string) ([]string, error)

var argsHook = func(ctx context.Context, sh *Shell, args []string) ([]string, error) {
	return args, nil
}

func SetArgsHook(argsHook_ ArgsHookT) (rv ArgsHookT) {
	rv, argsHook = argsHook, argsHook_
	return
}

type HookT func(context.Context, *Cmd) (int, bool, error)

var hook = func(context.Context, *Cmd) (int, bool, error) {
	return 0, false, nil
}

func SetHook(hook_ HookT) (rv HookT) {
	rv, hook = hook, hook_
	return
}

var OnCommandNotFound = func(ctx context.Context, cmd *Cmd, err error) error {
	err = &CommandNotFound{cmd.args[0], err}
	return err
}

var LastErrorLevel int

func makeCmdline(args, rawargs []string) string {
	var buffer strings.Builder
	for i, s := range args {
		if i > 0 {
			buffer.WriteRune(' ')
		}
		if (len(rawargs) > i && len(rawargs[i]) > 0 && rawargs[i][0] == '"') || strings.ContainsAny(s, " &|<>\t\"") {
			fmt.Fprintf(&buffer, `"%s"`, strings.Replace(s, `"`, `\"`, -1))
		} else {
			buffer.WriteString(s)
		}
	}
	return buffer.String()
}

var UseSourceRunBatch = true

func encloseWithQuote(fullpath string) string {
	if strings.ContainsRune(fullpath, ' ') {
		var f strings.Builder
		f.WriteRune('"')
		f.WriteString(fullpath)
		f.WriteRune('"')
		return f.String()
	} else {
		return fullpath
	}
}

func (cmd *Cmd) spawnvpSilent(ctx context.Context) (int, error) {
	// command is empty.
	if len(cmd.args) <= 0 {
		return 0, nil
	}
	if defined.DBG {
		print("spawnvpSilent('", cmd.args[0], "')\n")
	}

	// aliases and lua-commands
	if errorlevel, done, err := hook(ctx, cmd); done || err != nil {
		return errorlevel, err
	}

	// command not found hook
	fullpath := cmd.FullPath()
	if fullpath == "" {
		return 255, OnCommandNotFound(ctx, cmd, os.ErrNotExist)
	}
	cmd.args[0] = fullpath

	if defined.DBG {
		print("exec.LookPath(", cmd.args[0], ")==", fullpath, "\n")
	}
	if WildCardExpansionAlways {
		cmd.args = findfile.Globs(cmd.args)
	}
	if cmd.UseShellExecute {
		// GUI Application
		cmdline := makeCmdline(cmd.args[1:], cmd.rawArgs[1:])
		return 0, dos.ShellExecute("open", fullpath, cmdline, "")
	}
	if UseSourceRunBatch {
		lowerName := strings.ToLower(cmd.args[0])
		if strings.HasSuffix(lowerName, ".cmd") || strings.HasSuffix(lowerName, ".bat") {
			rawargs := cmd.RawArgs()
			args := make([]string, len(rawargs))
			args[0] = encloseWithQuote(fullpath)
			for i, end := 1, len(rawargs); i < end; i++ {
				args[i] = rawargs[i]
			}
			// Batch files
			return RawSource(args, nil, false, cmd.Stdin, cmd.Stdout, cmd.Stderr)
		}
	}
	// Do not use exec.CommandContext because it cancels background process.
	xcmd := exec.Command(cmd.args[0], cmd.args[1:]...)
	xcmd.Stdin = cmd.Stdin
	xcmd.Stdout = cmd.Stdout
	xcmd.Stderr = cmd.Stderr

	if xcmd.SysProcAttr == nil {
		xcmd.SysProcAttr = new(syscall.SysProcAttr)
	}
	cmdline := makeCmdline(xcmd.Args, cmd.rawArgs)
	if defined.DBG {
		println(cmdline)
	}
	xcmd.SysProcAttr.CmdLine = cmdline
	err := xcmd.Run()
	errorlevel, errorlevelOk := dos.GetErrorLevel(xcmd)
	if errorlevelOk {
		return errorlevel, err
	} else {
		return 255, err
	}
}

type AlreadyReportedError struct {
	Err error
}

func (_ AlreadyReportedError) Error() string {
	return ""
}

func IsAlreadyReported(err error) bool {
	_, ok := err.(AlreadyReportedError)
	return ok
}

func (cmd *Cmd) Spawnvp(ctx context.Context) (int, error) {
	errorlevel, err := cmd.spawnvpSilent(ctx)
	if err != nil && err != io.EOF && !IsAlreadyReported(err) {
		if defined.DBG {
			val := reflect.ValueOf(err)
			fmt.Fprintf(cmd.Stderr, "error-type=%s\n", val.Type())
		}
		fmt.Fprintln(cmd.Stderr, err.Error())
		err = AlreadyReportedError{err}
	}
	return errorlevel, err
}

func (sh *Shell) Spawnlp(ctx context.Context, args, rawargs []string) (int, error) {
	cmd := sh.Command()
	defer cmd.Close()
	cmd.SetArgs(args)
	cmd.SetRawArgs(rawargs)
	return cmd.Spawnvp(ctx)
}

func (sh *Shell) Interpret(ctx context.Context, text string) (errorlevel int, finalerr error) {
	if defined.DBG {
		print("Interpret('", text, "')\n")
	}
	if sh == nil {
		return 255, errors.New("Fatal Error: Interpret: instance is nil")
	}
	errorlevel = 0
	finalerr = nil

	statements, statementsErr := Parse(text)
	if statementsErr != nil {
		if defined.DBG {
			print("Parse Error:", statementsErr.Error(), "\n")
		}
		return 0, statementsErr
	}
	if argsHook != nil {
		if defined.DBG {
			print("call argsHook\n")
		}
		for _, pipeline := range statements {
			for _, state := range pipeline {
				var err error
				state.Args, err = argsHook(ctx, sh, state.Args)
				if err != nil {
					return 255, err
				}
			}
		}
		if defined.DBG {
			print("done argsHook\n")
		}
	}
	for _, pipeline := range statements {
		for i, state := range pipeline {
			if state.Term == "|" && (i+1 >= len(pipeline) || len(pipeline[i+1].Args) <= 0) {
				return 255, errors.New("The syntax of the command is incorrect.")
			}
		}
	}

	for _, pipeline := range statements {

		var pipeIn *os.File = nil
		isBackGround := sh.IsBackGround
		for _, state := range pipeline {
			if state.Term == "&" {
				isBackGround = true
				break
			}
		}
		var wg sync.WaitGroup
		shutdown_immediately := false
		for i, state := range pipeline {
			if defined.DBG {
				print(i, ": pipeline loop(", state.Args[0], ")\n")
			}
			cmd := sh.Command()
			cmd.IsBackGround = isBackGround

			if pipeIn != nil {
				cmd.Stdin = pipeIn
				cmd.Closers = append(cmd.Closers, pipeIn)
				pipeIn = nil
			}

			var err error
			if state.Term[0] == '|' {
				var pipeOut *os.File
				pipeIn, pipeOut, err = os.Pipe()
				cmd.Stdout = pipeOut
				if state.Term == "|&" {
					cmd.Stderr = pipeOut
				}
				cmd.Closers = append(cmd.Closers, pipeOut)
			}

			for _, red := range state.Redirect {
				var fd *os.File
				fd, err = red.OpenOn(cmd)
				if err != nil {
					return 0, err
				}
				cmd.Closers = append(cmd.Closers, fd)
			}

			cmd.args = state.Args
			cmd.rawArgs = state.RawArgs
			if i > 0 {
				cmd.IsBackGround = true
			}
			if len(pipeline) == 1 && dos.IsGui(cmd.FullPath()) {
				cmd.UseShellExecute = true
			}
			if i == len(pipeline)-1 && state.Term != "&" {
				// foreground execution.
				errorlevel, finalerr = cmd.Spawnvp(ctx)
				LastErrorLevel = errorlevel
				cmd.Close()
			} else {
				// background
				if !isBackGround {
					wg.Add(1)
				}
				newctx := ctx
				if tag := cmd.Tag(); tag != nil {
					var newtag CloneCloser
					if newctx, newtag, err = tag.Clone(ctx); err != nil {
						fmt.Fprintln(os.Stderr, err.Error())
						return -1, err
					} else {
						cmd.SetTag(newtag)
					}
				}
				go func(ctx1 context.Context, cmd1 *Cmd) {
					if !isBackGround {
						defer wg.Done()
					}
					cmd1.Spawnvp(ctx1)
					if tag := cmd1.Tag(); tag != nil {
						if err := tag.Close(); err != nil {
							fmt.Fprintln(os.Stderr, err.Error())
						}
					}
					cmd1.Close()
				}(newctx, cmd)
			}
		}
		if !isBackGround {
			wg.Wait()
			if shutdown_immediately {
				return errorlevel, nil
			}
			if len(pipeline) > 0 {
				switch pipeline[len(pipeline)-1].Term {
				case "&&":
					if errorlevel != 0 {
						return errorlevel, nil
					}
				case "||":
					if errorlevel == 0 {
						return errorlevel, nil
					}
				}
			}
		}
	}
	return
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package debuggingsnapshot

import (
	"context"
	"net/http"
	"sync"
	"time"

	v1 "k8s.io/api/core/v1"
	"k8s.io/klog/v2"
	"k8s.io/kubernetes/pkg/scheduler/framework"
)

// DebuggingSnapshotterState is the type for the debugging snapshot State machine
// The states guide the workflow of the snapshot.
type DebuggingSnapshotterState int

// DebuggingSnapshotterState help navigate the different workflows of the snapshot capture.
const (
	// SNAPSHOTTER_DISABLED is when debuggingSnapshot is disabled on the cluster and no action can be taken
	SNAPSHOTTER_DISABLED DebuggingSnapshotterState = iota + 1
	// LISTENING is set when snapshotter is enabled on the cluster and is ready to listen to a
	// snapshot request. Used by ResponseHandler to wait on to listen to request
	LISTENING
	// TRIGGER_ENABLED is set by ResponseHandler if a valid snapshot request is received
	// it states that a snapshot request needs to be processed
	TRIGGER_ENABLED
	// START_DATA_COLLECTION is used to synchronise the collection of data.
	// Since the trigger is an asynchronous process, data collection could be started mid-loop
	// leading to incomplete data. So setter methods wait for START_DATA_COLLECTION before collecting data
	// which is set at the start of the next loop after receiving the trigger
	START_DATA_COLLECTION
	// DATA_COLLECTED is set by setter func (also used by setter func for data collection)
	// This is set to let Flush know that at least some data collected and there isn't
	// an error State leading to no data collection
	DATA_COLLECTED
)

// DebuggingSnapshotterImpl is the impl for DebuggingSnapshotter
type DebuggingSnapshotterImpl struct {
	// State captures the internal state of the snapshotter
	State *DebuggingSnapshotterState
	// DebuggingSnapshot is the data bean for the snapshot
	DebuggingSnapshot DebuggingSnapshot
	// Mutex is the synchronisation used to the methods/states in the critical section
	Mutex *sync.Mutex
	// Trigger is the channel on which the Response Handler waits on to know
	// when there is data to be flushed back to the channel from the Snapshot
	Trigger chan struct{}
	// CancelRequest is the cancel function for the snapshot request. It is used to
	// terminate any ongoing request when CA is shutting down
	CancelRequest context.CancelFunc
}

// DebuggingSnapshotter is the interface for debugging snapshot
type DebuggingSnapshotter interface {

	// StartDataCollection will check the State(s) and enable data
	// collection for the loop if applicable
	StartDataCollection()
	// SetClusterNodes is a setter to capture all the ClusterNode
	SetClusterNodes([]*framework.NodeInfo)
	// SetUnscheduledPodsCanBeScheduled is a setter for all pods which are unscheduled
	// but they can be scheduled. i.e. pods which aren't triggering scale-up
	SetUnscheduledPodsCanBeScheduled([]*v1.Pod)
	// SetTemplateNodes is a setter for all the TemplateNodes present in the cluster
	// incl. templates for which there are no nodes
	SetTemplateNodes(map[string]*framework.NodeInfo)
	// ResponseHandler is the http response handler to manage incoming requests
	ResponseHandler(http.ResponseWriter, *http.Request)
	// IsDataCollectionAllowed checks the internal State of the snapshotter
	// to find if data can be collected. This can be used before preprocessing
	// for the snapshot
	IsDataCollectionAllowed() bool
	// Flush triggers the flushing of the snapshot
	Flush()
	// Cleanup clears the internal data beans of the snapshot, readying for next request
	Cleanup()
}

// NewDebuggingSnapshotter returns a new instance of DebuggingSnapshotter
func NewDebuggingSnapshotter(isDebuggerEnabled bool) DebuggingSnapshotter {
	state := SNAPSHOTTER_DISABLED
	if isDebuggerEnabled {
		klog.Infof("Debugging Snapshot is enabled")
		state = LISTENING
	}
	return &DebuggingSnapshotterImpl{
		State:             &state,
		Mutex:             &sync.Mutex{},
		DebuggingSnapshot: &DebuggingSnapshotImpl{},
		Trigger:           make(chan struct{}, 1),
	}
}

// ResponseHandler is the impl for request handler
func (d *DebuggingSnapshotterImpl) ResponseHandler(w http.ResponseWriter, r *http.Request) {

	d.Mutex.Lock()
	// checks if the handler is in the correct State to accept a new snapshot request
	if *d.State != LISTENING {
		defer d.Mutex.Unlock()
		klog.Errorf("Debugging Snapshot is currently being processed. Another snapshot can't be processed")
		w.WriteHeader(http.StatusTooManyRequests)
		w.Write([]byte("Another debugging snapshot request is being processed. Concurrent requests not supported"))
		return
	}

	ctx, cancel := context.WithCancel(r.Context())
	d.CancelRequest = cancel

	klog.Infof("Received a new snapshot, that is accepted")
	// set the State to trigger enabled, to allow workflow to collect data
	*d.State = TRIGGER_ENABLED
	d.Mutex.Unlock()

	select {
	case <-d.Trigger:
		d.Mutex.Lock()
		d.DebuggingSnapshot.SetEndTimestamp(time.Now().In(time.UTC))
		body, isErrorMessage := d.DebuggingSnapshot.GetOutputBytes()
		if isErrorMessage {
			w.WriteHeader(http.StatusInternalServerError)
		} else {
			w.WriteHeader(http.StatusOK)
		}
		w.Write(body)

		// reset the debugging State to receive a new snapshot request
		*d.State = LISTENING
		d.CancelRequest = nil
		d.DebuggingSnapshot.Cleanup()

		d.Mutex.Unlock()
	case <-ctx.Done():
		d.Mutex.Lock()
		klog.Infof("Received terminate trigger, aborting ongoing snapshot request")
		w.WriteHeader(http.StatusServiceUnavailable)

		d.DebuggingSnapshot.Cleanup()
		*d.State = LISTENING
		d.CancelRequest = nil
		select {
		case <-d.Trigger:
		default:
		}
		d.Mutex.Unlock()
	}
}

// IsDataCollectionAllowed encapsulate the check to know if data collection is currently active
// This should be used by setters and by any function that is contingent on data collection State
// before doing extra processing.
// e.g. If you want to pre-process a particular State in cloud-provider for snapshot
// you should check this func in the loop before doing that extra processing
func (d *DebuggingSnapshotterImpl) IsDataCollectionAllowed() bool {
	d.Mutex.Lock()
	defer d.Mutex.Unlock()
	return *d.State == DATA_COLLECTED || *d.State == START_DATA_COLLECTION
}

// StartDataCollection changes the State when the trigger has been enabled
// to start data collection. To be done at the start of the runLoop to allow for consistency
// as the trigger can be called mid-loop leading to partial data collection
func (d *DebuggingSnapshotterImpl) StartDataCollection() {
	d.Mutex.Lock()
	defer d.Mutex.Unlock()
	if *d.State == TRIGGER_ENABLED {
		*d.State = START_DATA_COLLECTION
		klog.Infof("Trigger Enabled for Debugging Snapshot, starting data collection")
		d.DebuggingSnapshot.SetStartTimestamp(time.Now().In(time.UTC))
	}
}

// Flush is the impl for DebuggingSnapshotter.Flush
// It checks if any data has been collected or data collection failed
func (d *DebuggingSnapshotterImpl) Flush() {
	d.Mutex.Lock()
	defer d.Mutex.Unlock()

	// Case where Data Collection was started but no data was collected, needs to
	// be stated as an error and reset to pre-trigger State
	if *d.State == START_DATA_COLLECTION {
		klog.Errorf("No data was collected for the snapshot in this loop. So no snapshot can be generated.")
		d.DebuggingSnapshot.SetErrorMessage("Unable to collect any data")
		d.Trigger <- struct{}{}
		return
	}

	if *d.State == DATA_COLLECTED {
		d.Trigger <- struct{}{}
	}
}

// SetClusterNodes is the setter for Node Group Info
// All filtering/prettifying of data should be done here.
func (d *DebuggingSnapshotterImpl) SetClusterNodes(nodeInfos []*framework.NodeInfo) {
	if !d.IsDataCollectionAllowed() {
		return
	}
	d.Mutex.Lock()
	defer d.Mutex.Unlock()
	klog.V(4).Infof("NodeGroupInfo is being set for the debugging snapshot")
	d.DebuggingSnapshot.SetClusterNodes(nodeInfos)
	*d.State = DATA_COLLECTED
}

// SetUnscheduledPodsCanBeScheduled is the setter for UnscheduledPodsCanBeScheduled
func (d *DebuggingSnapshotterImpl) SetUnscheduledPodsCanBeScheduled(podList []*v1.Pod) {
	if !d.IsDataCollectionAllowed() {
		return
	}
	d.Mutex.Lock()
	defer d.Mutex.Unlock()
	klog.V(4).Infof("UnscheduledPodsCanBeScheduled is being set for the debugging snapshot")
	d.DebuggingSnapshot.SetUnscheduledPodsCanBeScheduled(podList)
	*d.State = DATA_COLLECTED
}

// SetTemplateNodes is the setter for TemplateNodes
func (d *DebuggingSnapshotterImpl) SetTemplateNodes(templates map[string]*framework.NodeInfo) {
	if !d.IsDataCollectionAllowed() {
		return
	}
	klog.V(4).Infof("TemplateNodes is being set for the debugging snapshot")
	d.Mutex.Lock()
	defer d.Mutex.Unlock()
	d.DebuggingSnapshot.SetTemplateNodes(templates)
}

// Cleanup clears the internal data sets of the cluster
func (d *DebuggingSnapshotterImpl) Cleanup() {
	if d.CancelRequest != nil {
		d.CancelRequest()
	}
}
<commit_msg>CA: Debugging snapshotter locking optimisation for better transactions<commit_after>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package debuggingsnapshot

import (
	"context"
	"net/http"
	"sync"
	"time"

	v1 "k8s.io/api/core/v1"
	"k8s.io/klog/v2"
	"k8s.io/kubernetes/pkg/scheduler/framework"
)

// DebuggingSnapshotterState is the type for the debugging snapshot State machine
// The states guide the workflow of the snapshot.
type DebuggingSnapshotterState int

// DebuggingSnapshotterState help navigate the different workflows of the snapshot capture.
const (
	// SNAPSHOTTER_DISABLED is when debuggingSnapshot is disabled on the cluster and no action can be taken
	SNAPSHOTTER_DISABLED DebuggingSnapshotterState = iota + 1
	// LISTENING is set when snapshotter is enabled on the cluster and is ready to listen to a
	// snapshot request. Used by ResponseHandler to wait on to listen to request
	LISTENING
	// TRIGGER_ENABLED is set by ResponseHandler if a valid snapshot request is received
	// it states that a snapshot request needs to be processed
	TRIGGER_ENABLED
	// START_DATA_COLLECTION is used to synchronise the collection of data.
	// Since the trigger is an asynchronous process, data collection could be started mid-loop
	// leading to incomplete data. So setter methods wait for START_DATA_COLLECTION before collecting data
	// which is set at the start of the next loop after receiving the trigger
	START_DATA_COLLECTION
	// DATA_COLLECTED is set by setter func (also used by setter func for data collection)
	// This is set to let Flush know that at least some data collected and there isn't
	// an error State leading to no data collection
	DATA_COLLECTED
)

// DebuggingSnapshotterImpl is the impl for DebuggingSnapshotter
type DebuggingSnapshotterImpl struct {
	// State captures the internal state of the snapshotter
	State *DebuggingSnapshotterState
	// DebuggingSnapshot is the data bean for the snapshot
	DebuggingSnapshot DebuggingSnapshot
	// Mutex is the synchronisation used to the methods/states in the critical section
	Mutex *sync.Mutex
	// Trigger is the channel on which the Response Handler waits on to know
	// when there is data to be flushed back to the channel from the Snapshot
	Trigger chan struct{}
	// CancelRequest is the cancel function for the snapshot request. It is used to
	// terminate any ongoing request when CA is shutting down
	CancelRequest context.CancelFunc
}

// DebuggingSnapshotter is the interface for debugging snapshot
type DebuggingSnapshotter interface {

	// StartDataCollection will check the State(s) and enable data
	// collection for the loop if applicable
	StartDataCollection()
	// SetClusterNodes is a setter to capture all the ClusterNode
	SetClusterNodes([]*framework.NodeInfo)
	// SetUnscheduledPodsCanBeScheduled is a setter for all pods which are unscheduled
	// but they can be scheduled. i.e. pods which aren't triggering scale-up
	SetUnscheduledPodsCanBeScheduled([]*v1.Pod)
	// SetTemplateNodes is a setter for all the TemplateNodes present in the cluster
	// incl. templates for which there are no nodes
	SetTemplateNodes(map[string]*framework.NodeInfo)
	// ResponseHandler is the http response handler to manage incoming requests
	ResponseHandler(http.ResponseWriter, *http.Request)
	// IsDataCollectionAllowed checks the internal State of the snapshotter
	// to find if data can be collected. This can be used before preprocessing
	// for the snapshot
	IsDataCollectionAllowed() bool
	// Flush triggers the flushing of the snapshot
	Flush()
	// Cleanup clears the internal data beans of the snapshot, readying for next request
	Cleanup()
}

// NewDebuggingSnapshotter returns a new instance of DebuggingSnapshotter
func NewDebuggingSnapshotter(isDebuggerEnabled bool) DebuggingSnapshotter {
	state := SNAPSHOTTER_DISABLED
	if isDebuggerEnabled {
		klog.Infof("Debugging Snapshot is enabled")
		state = LISTENING
	}
	return &DebuggingSnapshotterImpl{
		State:             &state,
		Mutex:             &sync.Mutex{},
		DebuggingSnapshot: &DebuggingSnapshotImpl{},
		Trigger:           make(chan struct{}, 1),
	}
}

// ResponseHandler is the impl for request handler
func (d *DebuggingSnapshotterImpl) ResponseHandler(w http.ResponseWriter, r *http.Request) {

	d.Mutex.Lock()
	// checks if the handler is in the correct State to accept a new snapshot request
	if *d.State != LISTENING {
		defer d.Mutex.Unlock()
		klog.Errorf("Debugging Snapshot is currently being processed. Another snapshot can't be processed")
		w.WriteHeader(http.StatusTooManyRequests)
		w.Write([]byte("Another debugging snapshot request is being processed. Concurrent requests not supported"))
		return
	}

	ctx, cancel := context.WithCancel(r.Context())
	d.CancelRequest = cancel

	klog.Infof("Received a new snapshot, that is accepted")
	// set the State to trigger enabled, to allow workflow to collect data
	*d.State = TRIGGER_ENABLED
	d.Mutex.Unlock()

	select {
	case <-d.Trigger:
		d.Mutex.Lock()
		d.DebuggingSnapshot.SetEndTimestamp(time.Now().In(time.UTC))
		body, isErrorMessage := d.DebuggingSnapshot.GetOutputBytes()
		if isErrorMessage {
			w.WriteHeader(http.StatusInternalServerError)
		} else {
			w.WriteHeader(http.StatusOK)
		}
		w.Write(body)

		// reset the debugging State to receive a new snapshot request
		*d.State = LISTENING
		d.CancelRequest = nil
		d.DebuggingSnapshot.Cleanup()

		d.Mutex.Unlock()
	case <-ctx.Done():
		d.Mutex.Lock()
		klog.Infof("Received terminate trigger, aborting ongoing snapshot request")
		w.WriteHeader(http.StatusServiceUnavailable)

		d.DebuggingSnapshot.Cleanup()
		*d.State = LISTENING
		d.CancelRequest = nil
		select {
		case <-d.Trigger:
		default:
		}
		d.Mutex.Unlock()
	}
}

// IsDataCollectionAllowed encapsulate the check to know if data collection is currently active
// This should be used by setters and by any function that is contingent on data collection State
// before doing extra processing.
// e.g. If you want to pre-process a particular State in cloud-provider for snapshot
// you should check this func in the loop before doing that extra processing
func (d *DebuggingSnapshotterImpl) IsDataCollectionAllowed() bool {
	d.Mutex.Lock()
	defer d.Mutex.Unlock()
	return d.IsDataCollectionAllowedNoLock()
}

// IsDataCollectionAllowedNoLock encapsulated the check to know if data collection is currently active
// The need for NoLock implementation is for cases when the caller funcs have procured the lock
// for a single transactional execution
func (d *DebuggingSnapshotterImpl) IsDataCollectionAllowedNoLock() bool {
	return *d.State == DATA_COLLECTED || *d.State == START_DATA_COLLECTION
}

// StartDataCollection changes the State when the trigger has been enabled
// to start data collection. To be done at the start of the runLoop to allow for consistency
// as the trigger can be called mid-loop leading to partial data collection
func (d *DebuggingSnapshotterImpl) StartDataCollection() {
	d.Mutex.Lock()
	defer d.Mutex.Unlock()
	if *d.State == TRIGGER_ENABLED {
		*d.State = START_DATA_COLLECTION
		klog.Infof("Trigger Enabled for Debugging Snapshot, starting data collection")
		d.DebuggingSnapshot.SetStartTimestamp(time.Now().In(time.UTC))
	}
}

// Flush is the impl for DebuggingSnapshotter.Flush
// It checks if any data has been collected or data collection failed
func (d *DebuggingSnapshotterImpl) Flush() {
	d.Mutex.Lock()
	defer d.Mutex.Unlock()

	// Case where Data Collection was started but no data was collected, needs to
	// be stated as an error and reset to pre-trigger State
	if *d.State == START_DATA_COLLECTION {
		klog.Errorf("No data was collected for the snapshot in this loop. So no snapshot can be generated.")
		d.DebuggingSnapshot.SetErrorMessage("Unable to collect any data")
		d.Trigger <- struct{}{}
		return
	}

	if *d.State == DATA_COLLECTED {
		d.Trigger <- struct{}{}
	}
}

// SetClusterNodes is the setter for Node Group Info
// All filtering/prettifying of data should be done here.
func (d *DebuggingSnapshotterImpl) SetClusterNodes(nodeInfos []*framework.NodeInfo) {
	d.Mutex.Lock()
	defer d.Mutex.Unlock()
	if !d.IsDataCollectionAllowedNoLock() {
		return
	}
	klog.V(4).Infof("NodeGroupInfo is being set for the debugging snapshot")
	d.DebuggingSnapshot.SetClusterNodes(nodeInfos)
	*d.State = DATA_COLLECTED
}

// SetUnscheduledPodsCanBeScheduled is the setter for UnscheduledPodsCanBeScheduled
func (d *DebuggingSnapshotterImpl) SetUnscheduledPodsCanBeScheduled(podList []*v1.Pod) {
	d.Mutex.Lock()
	defer d.Mutex.Unlock()
	if !d.IsDataCollectionAllowedNoLock() {
		return
	}
	klog.V(4).Infof("UnscheduledPodsCanBeScheduled is being set for the debugging snapshot")
	d.DebuggingSnapshot.SetUnscheduledPodsCanBeScheduled(podList)
	*d.State = DATA_COLLECTED
}

// SetTemplateNodes is the setter for TemplateNodes
func (d *DebuggingSnapshotterImpl) SetTemplateNodes(templates map[string]*framework.NodeInfo) {
	d.Mutex.Lock()
	defer d.Mutex.Unlock()
	if !d.IsDataCollectionAllowedNoLock() {
		return
	}
	klog.V(4).Infof("TemplateNodes is being set for the debugging snapshot")
	d.DebuggingSnapshot.SetTemplateNodes(templates)
}

// Cleanup clears the internal data sets of the cluster
func (d *DebuggingSnapshotterImpl) Cleanup() {
	if d.CancelRequest != nil {
		d.CancelRequest()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/MG-RAST/Shock/shock-server/auth"
	"github.com/MG-RAST/Shock/shock-server/conf"
	"github.com/MG-RAST/Shock/shock-server/controller"
	"github.com/MG-RAST/Shock/shock-server/db"
	"github.com/MG-RAST/Shock/shock-server/logger"
	"github.com/MG-RAST/Shock/shock-server/node"
	"github.com/MG-RAST/Shock/shock-server/preauth"
	"github.com/MG-RAST/Shock/shock-server/user"
	"github.com/MG-RAST/golib/goweb"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"syscall"
	"time"
)

func launchSite(control chan int) {
	goweb.ConfigureDefaultFormatters()
	r := &goweb.RouteManager{}
	r.MapFunc("/raw", RawDir)
	r.MapFunc("/assets", AssetsDir)
	r.MapFunc("*", Site)
	if conf.Bool(conf.Conf["ssl"]) {
		err := goweb.ListenAndServeRoutesTLS(":"+conf.Conf["site-port"], conf.Conf["ssl-cert"], conf.Conf["ssl-key"], r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: site: %v\n", err)
			logger.Error("ERROR: site: " + err.Error())
		}
	} else {
		err := goweb.ListenAndServeRoutes(":"+conf.Conf["site-port"], r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: site: %v\n", err)
			logger.Error("ERROR: site: " + err.Error())
		}
	}
	control <- 1 //we are ending
}

func launchAPI(control chan int) {
	c := controller.New()
	goweb.ConfigureDefaultFormatters()
	r := &goweb.RouteManager{}
	r.MapFunc("/preauth/{id}", c.Preauth, goweb.GetMethod)
	r.Map("/node/{nid}/acl/{type}", c.Acl["typed"])
	r.Map("/node/{nid}/acl", c.Acl["base"])
	r.Map("/node/{nid}/index/{type}", c.Index)
	r.Map("/node/{nid}/index", c.Index)
	r.MapRest("/node", c.Node)
	r.MapFunc("*", ResourceDescription, goweb.GetMethod)
	r.MapFunc("*", RespondOk, goweb.OptionsMethod)
	if conf.Bool(conf.Conf["ssl"]) {
		err := goweb.ListenAndServeRoutesTLS(":"+conf.Conf["api-port"], conf.Conf["ssl-cert"], conf.Conf["ssl-key"], r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: api: %v\n", err)
			logger.Error("ERROR: api: " + err.Error())
		}
	} else {
		err := goweb.ListenAndServeRoutes(":"+conf.Conf["api-port"], r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: api: %v\n", err)
			logger.Error("ERROR: api: " + err.Error())
		}
	}
	control <- 1 //we are ending
}

func main() {
	// init(s)
	conf.Initialize()
	logger.Initialize()
	if err := db.Initialize(); err != nil {
		logger.Error(err.Error())
	}
	user.Initialize()
	node.Initialize()
	preauth.Initialize()
	auth.Initialize()

	// print conf
	printLogo()
	conf.Print()

	if _, err := os.Stat(conf.Conf["data-path"] + "/temp"); err != nil && os.IsNotExist(err) {
		if err := os.Mkdir(conf.Conf["data-path"]+"/temp", 0777); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
			logger.Error("ERROR: " + err.Error())
		}
	}

	// reload
	if conf.RELOAD != "" {
		fmt.Println("####### Reloading #######")
		err := reload(conf.RELOAD)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
			logger.Error("ERROR: " + err.Error())
		}
		fmt.Println("Done")
	}

	// setting GOMAXPROCS
	var procs int
	avail := runtime.NumCPU()
	if avail <= 2 {
		procs = 1
	} else if avail == 3 {
		procs = 2
	} else {
		procs = avail - 2
	}

	fmt.Println("##### Procs #####")
	fmt.Printf("Number of available CPUs = %d\n", avail)
	if conf.Conf["GOMAXPROCS"] != "" {
		if setting, err := strconv.Atoi(conf.Conf["GOMAXPROCS"]); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: could not interpret configured GOMAXPROCS value as integer.")
		} else {
			procs = setting
		}
	}

	if procs <= avail {
		fmt.Printf("Running Shock server with GOMAXPROCS = %d\n", procs)
		runtime.GOMAXPROCS(procs)
	} else {
		fmt.Println("GOMAXPROCS config value is greater than available number of CPUs.")
		fmt.Printf("Running Shock server with GOMAXPROCS = %d\n", avail)
		runtime.GOMAXPROCS(avail)
	}

	//launch server
	control := make(chan int)
	go launchSite(control)
	go launchAPI(control)

	//checking to make sure that server has launched
	connect := false
	for i := 0; i < 10; i++ {
		time.Sleep(100 * time.Millisecond)
		_, err := http.Get("http://localhost:" + conf.Conf["api-port"])
		if err == nil {
			connect = true
			break
		}
		time.Sleep(100 * time.Millisecond)
	}

	//exiting if server could not be reached after 1 second
	if connect != true {
		fmt.Fprintln(os.Stderr, "ERROR: server could not be reached at "+"http://localhost:"+conf.Conf["api-port"])
		fmt.Fprintln(os.Stderr, "Exiting!")
		os.Exit(1)
	}

	if conf.Conf["uid"] != "" && conf.Conf["gid"] != "" {
		var uid, gid int
		var err error
		if uid, err = strconv.Atoi(conf.Conf["uid"]); err != nil {
			fmt.Fprint(os.Stderr, "ERROR: uid in config file must be numeric.")
			os.Exit(1)
		}
		if gid, err = strconv.Atoi(conf.Conf["gid"]); err != nil {
			fmt.Fprint(os.Stderr, "ERROR: gid in config file must be numeric.")
			os.Exit(1)
		}
		err = syscall.Setgid(gid)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Setgid error: %v\n", err)
			os.Exit(1)
		}
		err = syscall.Setuid(uid)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Setuid error: %v\n", err)
			os.Exit(1)
		}
	}
	fmt.Println("\nReady to receive requests...")
	<-control //block till something dies
}
<commit_msg>Reordering starup process<commit_after>package main

import (
	"fmt"
	"github.com/MG-RAST/Shock/shock-server/auth"
	"github.com/MG-RAST/Shock/shock-server/conf"
	"github.com/MG-RAST/Shock/shock-server/controller"
	"github.com/MG-RAST/Shock/shock-server/db"
	"github.com/MG-RAST/Shock/shock-server/logger"
	"github.com/MG-RAST/Shock/shock-server/node"
	"github.com/MG-RAST/Shock/shock-server/preauth"
	"github.com/MG-RAST/Shock/shock-server/user"
	"github.com/MG-RAST/golib/goweb"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"syscall"
	"time"
)

func launchSite(control chan int) {
	goweb.ConfigureDefaultFormatters()
	r := &goweb.RouteManager{}
	r.MapFunc("/raw", RawDir)
	r.MapFunc("/assets", AssetsDir)
	r.MapFunc("*", Site)
	if conf.Bool(conf.Conf["ssl"]) {
		err := goweb.ListenAndServeRoutesTLS(":"+conf.Conf["site-port"], conf.Conf["ssl-cert"], conf.Conf["ssl-key"], r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: site: %v\n", err)
			logger.Error("ERROR: site: " + err.Error())
		}
	} else {
		err := goweb.ListenAndServeRoutes(":"+conf.Conf["site-port"], r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: site: %v\n", err)
			logger.Error("ERROR: site: " + err.Error())
		}
	}
	control <- 1 //we are ending
}

func launchAPI(control chan int) {
	c := controller.New()
	goweb.ConfigureDefaultFormatters()
	r := &goweb.RouteManager{}
	r.MapFunc("/preauth/{id}", c.Preauth, goweb.GetMethod)
	r.Map("/node/{nid}/acl/{type}", c.Acl["typed"])
	r.Map("/node/{nid}/acl", c.Acl["base"])
	r.Map("/node/{nid}/index/{type}", c.Index)
	r.Map("/node/{nid}/index", c.Index)
	r.MapRest("/node", c.Node)
	r.MapFunc("*", ResourceDescription, goweb.GetMethod)
	r.MapFunc("*", RespondOk, goweb.OptionsMethod)
	if conf.Bool(conf.Conf["ssl"]) {
		err := goweb.ListenAndServeRoutesTLS(":"+conf.Conf["api-port"], conf.Conf["ssl-cert"], conf.Conf["ssl-key"], r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: api: %v\n", err)
			logger.Error("ERROR: api: " + err.Error())
		}
	} else {
		err := goweb.ListenAndServeRoutes(":"+conf.Conf["api-port"], r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: api: %v\n", err)
			logger.Error("ERROR: api: " + err.Error())
		}
	}
	control <- 1 //we are ending
}

func main() {
	conf.Initialize()

	//launch server
	control := make(chan int)
	go launchSite(control)
	go launchAPI(control)

	time.Sleep(10 * time.Second)

	if conf.Conf["uid"] != "" && conf.Conf["gid"] != "" {
		var uid, gid int
		var err error
		if uid, err = strconv.Atoi(conf.Conf["uid"]); err != nil {
			fmt.Fprint(os.Stderr, "ERROR: uid in config file must be numeric.")
			os.Exit(1)
		}
		if gid, err = strconv.Atoi(conf.Conf["gid"]); err != nil {
			fmt.Fprint(os.Stderr, "ERROR: gid in config file must be numeric.")
			os.Exit(1)
		}
		err = syscall.Setgid(gid)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Setgid error: %v\n", err)
			os.Exit(1)
		}
		err = syscall.Setuid(uid)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Setuid error: %v\n", err)
			os.Exit(1)
		}
	}

	// init(s)
	logger.Initialize()
	if err := db.Initialize(); err != nil {
		logger.Error(err.Error())
	}
	user.Initialize()
	node.Initialize()
	preauth.Initialize()
	auth.Initialize()

	// print conf
	printLogo()
	conf.Print()

	if _, err := os.Stat(conf.Conf["data-path"] + "/temp"); err != nil && os.IsNotExist(err) {
		if err := os.Mkdir(conf.Conf["data-path"]+"/temp", 0777); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
			logger.Error("ERROR: " + err.Error())
		}
	}

	// reload
	if conf.RELOAD != "" {
		fmt.Println("####### Reloading #######")
		err := reload(conf.RELOAD)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
			logger.Error("ERROR: " + err.Error())
		}
		fmt.Println("Done")
	}

	// setting GOMAXPROCS
	var procs int
	avail := runtime.NumCPU()
	if avail <= 2 {
		procs = 1
	} else if avail == 3 {
		procs = 2
	} else {
		procs = avail - 2
	}

	fmt.Println("##### Procs #####")
	fmt.Printf("Number of available CPUs = %d\n", avail)
	if conf.Conf["GOMAXPROCS"] != "" {
		if setting, err := strconv.Atoi(conf.Conf["GOMAXPROCS"]); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: could not interpret configured GOMAXPROCS value as integer.")
		} else {
			procs = setting
		}
	}

	if procs <= avail {
		fmt.Printf("Running Shock server with GOMAXPROCS = %d\n", procs)
		runtime.GOMAXPROCS(procs)
	} else {
		fmt.Println("GOMAXPROCS config value is greater than available number of CPUs.")
		fmt.Printf("Running Shock server with GOMAXPROCS = %d\n", avail)
		runtime.GOMAXPROCS(avail)
	}

	//checking to make sure that server has launched
	connect := false
	for i := 0; i < 10; i++ {
		time.Sleep(100 * time.Millisecond)
		_, err := http.Get("http://localhost:" + conf.Conf["api-port"])
		if err == nil {
			connect = true
			break
		}
		time.Sleep(100 * time.Millisecond)
	}

	//exiting if server could not be reached after 1 second
	if connect != true {
		fmt.Fprintln(os.Stderr, "ERROR: server could not be reached at "+"http://localhost:"+conf.Conf["api-port"])
		fmt.Fprintln(os.Stderr, "Exiting!")
		os.Exit(1)
	}

	fmt.Println("\nReady to receive requests...")
	<-control //block till something dies
}
<|endoftext|>


<commit_before>package captain // import "github.com/harbur/captain/captain"

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestGitIsDirty(t *testing.T) {
	assert.Equal(t, false, isDirty(), "they should be equal")
}
<commit_msg>Added git tests<commit_after>package captain // import "github.com/harbur/captain/captain"

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestGitGetRevision(t *testing.T) {
	assert.Equal(t, 7, len(getRevision()), "they should be equal")
}

func TestGitGetBranch(t *testing.T) {
	assert.Equal(t, "master", getBranch(), "they should be equal")
}

func TestGitIsDirty(t *testing.T) {
	assert.Equal(t, false, isDirty(), "they should be equal")
}

func TestGitIsGit(t *testing.T) {
	assert.Equal(t, true, isGit(), "they should be equal")
}
<|endoftext|>


<commit_before>// Package log implements basic but useful request logging middleware.
package log

import (
	"log"
	"net/http"
	"os"

	"github.com/mholt/caddy/middleware"
)

// New instantiates a new instance of logging middleware.
func New(c middleware.Controller) (middleware.Middleware, error) {
	var logWhat, outputFile, format string
	var logger *log.Logger

	for c.Next() {
		c.Args(&logWhat, &outputFile, &format)

		if logWhat == "" {
			return nil, c.ArgErr()
		}
		if outputFile == "" {
			outputFile = defaultLogFilename
		}
		switch format {
		case "":
			format = defaultReqLogFormat
		case "{common}":
			format = commonLogFormat
		case "{combined}":
			format = combinedLogFormat
		}
	}

	// Open the log file for writing when the server starts
	c.Startup(func() error {
		var err error
		var file *os.File

		if outputFile == "stdout" {
			file = os.Stdout
		} else if outputFile == "stderr" {
			file = os.Stderr
		} else {
			file, err = os.OpenFile(outputFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
			if err != nil {
				return err
			}
		}

		logger = log.New(file, "", 0)
		return nil
	})

	return func(next http.HandlerFunc) http.HandlerFunc {
		return func(w http.ResponseWriter, r *http.Request) {
			sw := middleware.NewResponseRecorder(w)
			next(sw, r)
			rep := middleware.NewReplacer(r, sw)
			logger.Println(rep.Replace(format))
		}
	}, nil
}

const (
	defaultLogFilename  = "access.log"
	commonLogFormat     = `{remote} ` + middleware.EmptyStringReplacer + ` [{when}] "{method} {uri} {proto}" {status} {size}`
	combinedLogFormat   = commonLogFormat + ` "{>Referer}" "{>User-Agent}"`
	defaultReqLogFormat = commonLogFormat
)
<commit_msg>Rewrote access log middleware<commit_after>// Package log implements basic but useful request (access) logging middleware.
package log

import (
	"log"
	"net/http"
	"os"

	"github.com/mholt/caddy/middleware"
)

// New instantiates a new instance of logging middleware.
func New(c middleware.Controller) (middleware.Middleware, error) {
	rules, err := parse(c)
	if err != nil {
		return nil, err
	}

	// Open the log files for writing when the server starts
	c.Startup(func() error {
		for i := 0; i < len(rules); i++ {
			var err error
			var file *os.File

			if rules[i].OutputFile == "stdout" {
				file = os.Stdout
			} else if rules[i].OutputFile == "stderr" {
				file = os.Stderr
			} else {
				file, err = os.OpenFile(rules[i].OutputFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
				if err != nil {
					return err
				}
			}

			rules[i].Log = log.New(file, "", 0)
		}

		return nil
	})

	return func(next middleware.HandlerFunc) middleware.HandlerFunc {
		return Logger{Next: next, Rules: rules}.ServeHTTP
	}, nil
}

func (l Logger) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
	for _, rule := range l.Rules {
		if middleware.Path(r.URL.Path).Matches(rule.PathScope) {
			responseRecorder := middleware.NewResponseRecorder(w)
			status, err := l.Next(responseRecorder, r)
			rep := middleware.NewReplacer(r, responseRecorder)
			rule.Log.Println(rep.Replace(rule.Format))
			return status, err
		}
	}
	return l.Next(w, r)
}

func parse(c middleware.Controller) ([]LogRule, error) {
	var rules []LogRule

	for c.Next() {
		args := c.RemainingArgs()

		if len(args) == 0 {
			// Nothing specified; use defaults
			rules = append(rules, LogRule{
				PathScope:  "/",
				OutputFile: defaultLogFilename,
				Format:     defaultLogFormat,
			})
		} else if len(args) == 1 {
			// Only an output file specified
			rules = append(rules, LogRule{
				PathScope:  "/",
				OutputFile: args[0],
				Format:     defaultLogFormat,
			})
		} else {
			// Path scope, output file, and maybe a format specified

			format := defaultLogFormat

			if len(args) > 2 {
				switch args[2] {
				case "{common}":
					format = commonLogFormat
				case "{combined}":
					format = combinedLogFormat
				}
			}

			rules = append(rules, LogRule{
				PathScope:  args[0],
				OutputFile: args[1],
				Format:     format,
			})
		}
	}

	return rules, nil
}

type Logger struct {
	Next  middleware.HandlerFunc
	Rules []LogRule
}

type LogRule struct {
	PathScope  string
	OutputFile string
	Format     string
	Log        *log.Logger
}

const (
	defaultLogFilename = "access.log"
	commonLogFormat    = `{remote} ` + middleware.EmptyStringReplacer + ` [{when}] "{method} {uri} {proto}" {status} {size}`
	combinedLogFormat  = commonLogFormat + ` "{>Referer}" "{>User-Agent}"`
	defaultLogFormat   = commonLogFormat
)
<|endoftext|>


<commit_before>package midi

import (
	"time"

	"github.com/emicklei/melrose/core"
	"github.com/emicklei/melrose/midi/transport"
	"github.com/emicklei/melrose/notify"
)

type OutputDevice struct {
	id             int
	stream         transport.MIDIOut
	defaultChannel int

	echo     bool
	timeline *core.Timeline
}

func NewOutputDevice(id int, out transport.MIDIOut, ch int, line *core.Timeline) *OutputDevice {
	return &OutputDevice{
		id:             id,
		stream:         out,
		defaultChannel: ch,
		echo:           false,
		timeline:       line,
	}
}

func (d *OutputDevice) Start() {
	go d.timeline.Play()
}

func (d *OutputDevice) Reset() {
	d.timeline.Reset()
	if core.IsDebug() {
		notify.Debugf("device.%d: sending Note OFF to all 16 channels", d.id)
	}
	if d.stream != nil {
		// send note off all to all channels for current device
		for c := 1; c <= 16; c++ {
			if err := d.stream.WriteShort(controlChange|int64(c-1), noteAllOff, 0); err != nil {
				notify.Console.Errorf("device.%d: portmidi write error:%v", d.id, err)
			}
		}
	}
}

func (d *OutputDevice) handledPedalChange(condition core.Condition, channel int, timeline *core.Timeline, moment time.Time, group []core.Note) bool {
	if len(group) == 0 || len(group) > 1 {
		return false
	}
	note := group[0]
	switch {
	case note.IsPedalUp():
		timeline.Schedule(pedalEvent{
			goingDown:  false,
			channel:    channel,
			out:        d.stream,
			mustHandle: condition}, moment)
		return true
	case note.IsPedalUpDown():
		timeline.Schedule(pedalEvent{
			goingDown:  false,
			channel:    channel,
			out:        d.stream,
			mustHandle: condition}, moment)
		timeline.Schedule(pedalEvent{
			goingDown:  true,
			channel:    channel,
			out:        d.stream,
			mustHandle: condition}, moment)
		return true
	case note.IsPedalDown():
		timeline.Schedule(pedalEvent{
			goingDown:  true,
			channel:    channel,
			out:        d.stream,
			mustHandle: condition}, moment)
	}
	return false
}

func (d *OutputDevice) Play(condition core.Condition, seq core.Sequenceable, bpm float64, beginAt time.Time) time.Time {
	// which channel?
	channel := d.defaultChannel
	if sel, ok := seq.(core.ChannelSelector); ok {
		channel = sel.Channel()
		seq = sel.Unwrap()
	}

	// schedule all notes of the sequenceable
	wholeNoteDuration := core.WholeNoteDuration(bpm)
	moment := beginAt
	for _, eachGroup := range seq.S().Notes {
		if len(eachGroup) == 0 {
			continue
		}
		// pedal
		if d.handledPedalChange(condition, channel, d.timeline, moment, eachGroup) {
			continue
		}
		// one note
		if len(eachGroup) == 1 {
			moment = scheduleOneNote(d, condition, channel, eachGroup[0], wholeNoteDuration, moment)
			continue
		}
		//  more than one note
		if canCombineEvent(eachGroup) {
			event := combinedMidiEvent(d.id, channel, eachGroup, d.stream)
			if d.echo {
				event.echoString = core.StringFromNoteGroup(eachGroup)
			}
			actualDuration := durationOfGroup(eachGroup, wholeNoteDuration)
			event.mustHandle = condition
			moment = scheduleOnOffEvents(d, event, actualDuration, moment)
			continue
		}
		//  not combinable group of more than one note
		earliest := moment.Add(1 * time.Hour)
		for _, each := range eachGroup {
			endTime := scheduleOneNote(d, condition, channel, each, wholeNoteDuration, moment)
			if endTime.Before(earliest) {
				earliest = endTime
			}
		}
		moment = earliest
	}
	return moment
}

// returns the longest TODO in core?
func durationOfGroup(notes []core.Note, whole time.Duration) time.Duration {
	longest := time.Duration(0)
	for _, each := range notes {
		eachDuration := time.Duration(float32(whole) * each.DurationFactor())
		if eachDuration > longest {
			longest = eachDuration
		}
	}
	return longest
}

func scheduleOneNote(device *OutputDevice, condition core.Condition, channel int, note core.Note, whole time.Duration, moment time.Time) time.Time {
	if note.IsRest() {
		event := restEvent{mustHandle: condition}
		if device.echo {
			event.echoString = note.String()
		}
		device.timeline.Schedule(event, moment)
		actualDuration := time.Duration(float32(whole) * note.DurationFactor())
		return moment.Add(actualDuration)
	}
	// midi variable length note?
	if fixed, ok := note.NonFractionBasedDuration(); ok {
		event := midiEvent{
			which:      []int64{int64(note.MIDI())},
			onoff:      noteOn,
			device:     device.id,
			channel:    channel,
			velocity:   int64(note.Velocity),
			out:        device.stream,
			mustHandle: condition,
		}
		if device.echo {
			event.echoString = note.String()
		}
		return scheduleOnOffEvents(device, event, fixed, moment)
	}
	// normal note
	event := midiEvent{
		which:      []int64{int64(note.MIDI())},
		onoff:      noteOn,
		device:     device.id,
		channel:    channel,
		velocity:   int64(note.Velocity),
		out:        device.stream,
		mustHandle: condition,
	}
	if device.echo {
		event.echoString = note.String()
	}
	actualDuration := time.Duration(float32(whole) * note.DurationFactor())
	return scheduleOnOffEvents(device, event, actualDuration, moment)

}

func scheduleOnOffEvents(device *OutputDevice, event midiEvent, duration time.Duration, at time.Time) time.Time {
	device.timeline.Schedule(event, at)
	moment := at.Add(duration)
	device.timeline.Schedule(event.asNoteoff(), moment)
	return moment
}

func canCombineEvent(notes []core.Note) bool {
	if len(notes) <= 1 {
		return true
	}
	dur, vel := notes[0].DurationFactor(), notes[0].Velocity
	for n := 1; n < len(notes); n++ {
		d, v := notes[n].DurationFactor(), notes[n].Velocity
		if d != dur || v != vel {
			return false
		}
	}
	return true
}

// Pre: notes not empty
func combinedMidiEvent(deviceID int, channel int, notes []core.Note, stream transport.MIDIOut) midiEvent {
	// first note makes fraction and velocity
	velocity := notes[0].Velocity
	if velocity > 127 {
		velocity = 127
	}
	if velocity < 1 {
		velocity = core.Normal
	}
	nrs := []int64{}
	for _, each := range notes {
		nrs = append(nrs, int64(each.MIDI()))
	}
	return midiEvent{
		which:    nrs,
		onoff:    noteOn,
		device:   deviceID,
		channel:  channel,
		velocity: int64(velocity),
		out:      stream,
	}
}
<commit_msg>fix pedal up<commit_after>package midi

import (
	"time"

	"github.com/emicklei/melrose/core"
	"github.com/emicklei/melrose/midi/transport"
	"github.com/emicklei/melrose/notify"
)

type OutputDevice struct {
	id             int
	stream         transport.MIDIOut
	defaultChannel int

	echo     bool
	timeline *core.Timeline
}

func NewOutputDevice(id int, out transport.MIDIOut, ch int, line *core.Timeline) *OutputDevice {
	return &OutputDevice{
		id:             id,
		stream:         out,
		defaultChannel: ch,
		echo:           false,
		timeline:       line,
	}
}

func (d *OutputDevice) Start() {
	go d.timeline.Play()
}

func (d *OutputDevice) Reset() {
	d.timeline.Reset()
	if core.IsDebug() {
		notify.Debugf("device.%d: sending Note OFF to all 16 channels", d.id)
	}
	if d.stream != nil {
		// send note off all to all channels for current device
		for c := 1; c <= 16; c++ {
			if err := d.stream.WriteShort(controlChange|int64(c-1), noteAllOff, 0); err != nil {
				notify.Console.Errorf("device.%d: portmidi write error:%v", d.id, err)
			}
		}
	}
}

func (d *OutputDevice) handledPedalChange(condition core.Condition, channel int, timeline *core.Timeline, moment time.Time, group []core.Note) bool {
	if len(group) == 0 || len(group) > 1 {
		return false
	}
	note := group[0]
	switch {
	case note.IsPedalUp():
		timeline.Schedule(pedalEvent{
			goingDown:  false,
			channel:    channel,
			out:        d.stream,
			mustHandle: condition}, moment)
		return true
	case note.IsPedalUpDown():
		timeline.Schedule(pedalEvent{
			goingDown:  false,
			channel:    channel,
			out:        d.stream,
			mustHandle: condition}, moment)
		timeline.Schedule(pedalEvent{
			goingDown:  true,
			channel:    channel,
			out:        d.stream,
			mustHandle: condition}, moment)
		return true
	case note.IsPedalDown():
		timeline.Schedule(pedalEvent{
			goingDown:  true,
			channel:    channel,
			out:        d.stream,
			mustHandle: condition}, moment)
		return true
	}
	return false
}

func (d *OutputDevice) Play(condition core.Condition, seq core.Sequenceable, bpm float64, beginAt time.Time) time.Time {
	// which channel?
	channel := d.defaultChannel
	if sel, ok := seq.(core.ChannelSelector); ok {
		channel = sel.Channel()
		seq = sel.Unwrap()
	}

	// schedule all notes of the sequenceable
	wholeNoteDuration := core.WholeNoteDuration(bpm)
	moment := beginAt
	for _, eachGroup := range seq.S().Notes {
		if len(eachGroup) == 0 {
			continue
		}
		// pedal
		if d.handledPedalChange(condition, channel, d.timeline, moment, eachGroup) {
			continue
		}
		// one note
		if len(eachGroup) == 1 {
			moment = scheduleOneNote(d, condition, channel, eachGroup[0], wholeNoteDuration, moment)
			continue
		}
		//  more than one note
		if canCombineEvent(eachGroup) {
			event := combinedMidiEvent(d.id, channel, eachGroup, d.stream)
			if d.echo {
				event.echoString = core.StringFromNoteGroup(eachGroup)
			}
			actualDuration := durationOfGroup(eachGroup, wholeNoteDuration)
			event.mustHandle = condition
			moment = scheduleOnOffEvents(d, event, actualDuration, moment)
			continue
		}
		//  not combinable group of more than one note
		earliest := moment.Add(1 * time.Hour)
		for _, each := range eachGroup {
			endTime := scheduleOneNote(d, condition, channel, each, wholeNoteDuration, moment)
			if endTime.Before(earliest) {
				earliest = endTime
			}
		}
		moment = earliest
	}
	return moment
}

// returns the longest TODO in core?
func durationOfGroup(notes []core.Note, whole time.Duration) time.Duration {
	longest := time.Duration(0)
	for _, each := range notes {
		eachDuration := time.Duration(float32(whole) * each.DurationFactor())
		if eachDuration > longest {
			longest = eachDuration
		}
	}
	return longest
}

func scheduleOneNote(device *OutputDevice, condition core.Condition, channel int, note core.Note, whole time.Duration, moment time.Time) time.Time {
	if note.IsRest() {
		event := restEvent{mustHandle: condition}
		if device.echo {
			event.echoString = note.String()
		}
		device.timeline.Schedule(event, moment)
		actualDuration := time.Duration(float32(whole) * note.DurationFactor())
		return moment.Add(actualDuration)
	}
	// midi variable length note?
	if fixed, ok := note.NonFractionBasedDuration(); ok {
		event := midiEvent{
			which:      []int64{int64(note.MIDI())},
			onoff:      noteOn,
			device:     device.id,
			channel:    channel,
			velocity:   int64(note.Velocity),
			out:        device.stream,
			mustHandle: condition,
		}
		if device.echo {
			event.echoString = note.String()
		}
		return scheduleOnOffEvents(device, event, fixed, moment)
	}
	// normal note
	event := midiEvent{
		which:      []int64{int64(note.MIDI())},
		onoff:      noteOn,
		device:     device.id,
		channel:    channel,
		velocity:   int64(note.Velocity),
		out:        device.stream,
		mustHandle: condition,
	}
	if device.echo {
		event.echoString = note.String()
	}
	actualDuration := time.Duration(float32(whole) * note.DurationFactor())
	return scheduleOnOffEvents(device, event, actualDuration, moment)

}

func scheduleOnOffEvents(device *OutputDevice, event midiEvent, duration time.Duration, at time.Time) time.Time {
	device.timeline.Schedule(event, at)
	moment := at.Add(duration)
	device.timeline.Schedule(event.asNoteoff(), moment)
	return moment
}

func canCombineEvent(notes []core.Note) bool {
	if len(notes) <= 1 {
		return true
	}
	dur, vel := notes[0].DurationFactor(), notes[0].Velocity
	for n := 1; n < len(notes); n++ {
		d, v := notes[n].DurationFactor(), notes[n].Velocity
		if d != dur || v != vel {
			return false
		}
	}
	return true
}

// Pre: notes not empty
func combinedMidiEvent(deviceID int, channel int, notes []core.Note, stream transport.MIDIOut) midiEvent {
	// first note makes fraction and velocity
	velocity := notes[0].Velocity
	if velocity > 127 {
		velocity = 127
	}
	if velocity < 1 {
		velocity = core.Normal
	}
	nrs := []int64{}
	for _, each := range notes {
		nrs = append(nrs, int64(each.MIDI()))
	}
	return midiEvent{
		which:    nrs,
		onoff:    noteOn,
		device:   deviceID,
		channel:  channel,
		velocity: int64(velocity),
		out:      stream,
	}
}
<|endoftext|>


<commit_before>/**
 * Copyright (c) 2016 Magicshui
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */
/**
 * Copyright (c) 2017 yunify
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

package qingcloud

import (
	"fmt"

	"github.com/hashicorp/terraform/helper/schema"
	qc "github.com/yunify/qingcloud-sdk-go/service"
)

func modifyVxnetAttributes(d *schema.ResourceData, meta interface{}) error {
	clt := meta.(*QingCloudClient).vxnet
	input := new(qc.ModifyVxNetAttributesInput)
	input.VxNet = qc.String(d.Id())
	nameUpdate := false
	descriptionUpdate := false
	input.VxNetName, nameUpdate = getNamePointer(d)
	input.Description, descriptionUpdate = getDescriptionPointer(d)
	if nameUpdate || descriptionUpdate {
		var output *qc.ModifyVxNetAttributesOutput
		var err error
		simpleRetry(func() error {
			output, err = clt.ModifyVxNetAttributes(input)
			return isServerBusy(err)
		})
		if err != nil {
			return err
		}
	}
	return nil
}

func vxnetJoinRouter(d *schema.ResourceData, meta interface{}) error {
	clt := meta.(*QingCloudClient).router
	if _, err := RouterTransitionStateRefresh(meta.(*QingCloudClient).router, d.Get(resourceVxnetVpcID).(string)); err != nil {
		return err
	}
	input := new(qc.JoinRouterInput)
	input.VxNet = qc.String(d.Id())
	input.Router = qc.String(d.Get(resourceVxnetVpcID).(string))
	input.IPNetwork = qc.String(d.Get(resourceVxnetVpcIPNetwork).(string))
	var output *qc.JoinRouterOutput
	var err error
	simpleRetry(func() error {
		output, err = clt.JoinRouter(input)
		return isServerBusy(err)
	})
	if err != nil {
		return err
	}
	if _, err := RouterTransitionStateRefresh(meta.(*QingCloudClient).router, d.Get(resourceVxnetVpcID).(string)); err != nil {
		return err
	}
	return nil
}

func vxnetLeaverRouter(d *schema.ResourceData, meta interface{}) error {
	oldVPC, _ := d.GetChange(resourceVxnetVpcID)
	if _, err := RouterTransitionStateRefresh(meta.(*QingCloudClient).router, oldVPC.(string)); err != nil {
		return err
	}
	clt := meta.(*QingCloudClient).router
	input := new(qc.LeaveRouterInput)
	input.VxNets = []*string{qc.String(d.Id())}
	input.Router = qc.String(oldVPC.(string))
	var output *qc.LeaveRouterOutput
	var err error
	simpleRetry(func() error {
		output, err = clt.LeaveRouter(input)
		return isServerBusy(err)
	})
	if err != nil {
		return err
	}
	if _, err := VxnetLeaveRouterTransitionStateRefresh(meta.(*QingCloudClient).vxnet, d.Id()); err != nil {
		return err
	}
	if _, err := RouterTransitionStateRefresh(meta.(*QingCloudClient).router, d.Get(resourceVxnetVpcID).(string)); err != nil {
		return err
	}
	return nil
}

func isVxnetSelfManaged(vxnetId string, clt *qc.VxNetService) (bool, error) {

	input := new(qc.DescribeVxNetsInput)
	input.VxNets = []*string{qc.String(vxnetId)}
	output, err := clt.DescribeVxNets(input)
	if err != nil {
		return false, err
	}
	if len(output.VxNetSet) == 0 {
		return false, fmt.Errorf("Error can not find vxnet ")
	}
	if qc.IntValue(output.VxNetSet[0].VxNetType) == 0 {
		return true, nil
	} else {
		return false, nil
	}

}
<commit_msg>fix vxnet check<commit_after>/**
 * Copyright (c) 2016 Magicshui
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */
/**
 * Copyright (c) 2017 yunify
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

package qingcloud

import (
	"fmt"

	"github.com/hashicorp/terraform/helper/schema"
	qc "github.com/yunify/qingcloud-sdk-go/service"
)

func modifyVxnetAttributes(d *schema.ResourceData, meta interface{}) error {
	clt := meta.(*QingCloudClient).vxnet
	input := new(qc.ModifyVxNetAttributesInput)
	input.VxNet = qc.String(d.Id())
	nameUpdate := false
	descriptionUpdate := false
	input.VxNetName, nameUpdate = getNamePointer(d)
	input.Description, descriptionUpdate = getDescriptionPointer(d)
	if nameUpdate || descriptionUpdate {
		var output *qc.ModifyVxNetAttributesOutput
		var err error
		simpleRetry(func() error {
			output, err = clt.ModifyVxNetAttributes(input)
			return isServerBusy(err)
		})
		if err != nil {
			return err
		}
	}
	return nil
}

func vxnetJoinRouter(d *schema.ResourceData, meta interface{}) error {
	clt := meta.(*QingCloudClient).router
	if _, err := RouterTransitionStateRefresh(meta.(*QingCloudClient).router, d.Get(resourceVxnetVpcID).(string)); err != nil {
		return err
	}
	input := new(qc.JoinRouterInput)
	input.VxNet = qc.String(d.Id())
	input.Router = qc.String(d.Get(resourceVxnetVpcID).(string))
	input.IPNetwork = qc.String(d.Get(resourceVxnetVpcIPNetwork).(string))
	var output *qc.JoinRouterOutput
	var err error
	simpleRetry(func() error {
		output, err = clt.JoinRouter(input)
		return isServerBusy(err)
	})
	if err != nil {
		return err
	}
	if _, err := RouterTransitionStateRefresh(meta.(*QingCloudClient).router, d.Get(resourceVxnetVpcID).(string)); err != nil {
		return err
	}
	return nil
}

func vxnetLeaverRouter(d *schema.ResourceData, meta interface{}) error {
	oldVPC, _ := d.GetChange(resourceVxnetVpcID)
	if _, err := RouterTransitionStateRefresh(meta.(*QingCloudClient).router, oldVPC.(string)); err != nil {
		return err
	}
	clt := meta.(*QingCloudClient).router
	input := new(qc.LeaveRouterInput)
	input.VxNets = []*string{qc.String(d.Id())}
	input.Router = qc.String(oldVPC.(string))
	var output *qc.LeaveRouterOutput
	var err error
	simpleRetry(func() error {
		output, err = clt.LeaveRouter(input)
		return isServerBusy(err)
	})
	if err != nil {
		return err
	}
	if _, err := VxnetLeaveRouterTransitionStateRefresh(meta.(*QingCloudClient).vxnet, d.Id()); err != nil {
		return err
	}
	if _, err := RouterTransitionStateRefresh(meta.(*QingCloudClient).router, d.Get(resourceVxnetVpcID).(string)); err != nil {
		return err
	}
	return nil
}

func isVxnetSelfManaged(vxnetId string, clt *qc.VxNetService) (bool, error) {
	if vxnetId == BasicNetworkID {
		return false, nil
	}
	input := new(qc.DescribeVxNetsInput)
	input.VxNets = []*string{qc.String(vxnetId)}
	output, err := clt.DescribeVxNets(input)
	if err != nil {
		return false, err
	}
	if len(output.VxNetSet) == 0 {
		return false, fmt.Errorf("Error can not find vxnet ")
	}
	if qc.IntValue(output.VxNetSet[0].VxNetType) == 0 {
		return true, nil
	} else {
		return false, nil
	}

}
<|endoftext|>


<commit_before>package sisparse

import (
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/yhat/scrape"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

const sisUrl = "https://is.cuni.cz/studium/predmety/index.php?do=predmet&kod=%s"

// Returns a two-dimensional array containing groups of events.
// Each group is a slice of events which must be enrolled together,
// the groups represent different times/teachers of the same course.
// Also, lectures and seminars/practicals are in separate groups.
func GetCourseEvents(courseCode string) ([][]Event, error) {
	resp, err := http.Get(fmt.Sprintf(sisUrl, courseCode))
	if err != nil {
		return nil, err
	}
	// It is difficult to directly convert an event code to a schedule link,
	// because SIS requires the faculty number. Therefore we first open the course
	// in the "Subjects" SIS module and then go to a link which takes
	// us to the schedule.
	relativeScheduleUrl, err := getRelativeScheduleUrl(resp.Body)
	if err != nil {
		return nil, err
	}
	scheduleUrl := getAbsoluteUrl(sisUrl, relativeScheduleUrl)

	resp, err = http.Get(scheduleUrl)
	if err != nil {
		return nil, err
	}
	return parseCourseEvents(resp.Body), nil
}

func getRelativeScheduleUrl(body io.ReadCloser) (string, error) {
	const scheduleLinkText = "Rozvrh"

	root, err := html.Parse(body)
	if err != nil {
		panic(err)
	}

	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A {
			return scrape.Text(n) == scheduleLinkText
		}
		return false
	}

	scheduleLink, ok := scrape.Find(root, matcher)
	if !ok {
		return "", errors.New("Couldn't find schedule URL")
	}
	return scrape.Attr(scheduleLink, "href"), nil
}

func parseCourseEvents(body io.ReadCloser) [][]Event {
	root, err := html.Parse(body)
	if err != nil {
		panic(err)
	}

	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Tr && n.Parent != nil && n.Parent.Parent != nil {
			return scrape.Attr(n.Parent.Parent, "id") == "table1" &&
				scrape.Attr(n, "class") != "head1" // ignore table header
		}
		return false
	}

	eventsTable := scrape.FindAll(root, matcher)
	if len(eventsTable) == 0 {
		// The event table is not present at all (possibly SIS returned an error message)
		return [][]Event{}
	}

	res := [][]Event{}
	group := []Event{}
	for _, row := range eventsTable {
		event := parseEvent(row)
		// A non-empty name means the start of a new group;
		// names are omitted in all but the first event of a group.
		if event.Name != "" {
			if len(group) > 0 {
				res = append(res, group)
			}
			group = []Event{}
		} else {
			// Add the missing fields based on the group's first event
			event.Name = group[0].Name
			event.Teacher = group[0].Teacher
		}
		group = append(group, event)
	}
	if len(group) > 0 {
		res = append(res, group)
	}
	return res
}

func parseEvent(event *html.Node) Event {
	var cols []string
	for col := event.FirstChild; col != nil; col = col.NextSibling {
		// For some reason we also get siblings with no tag and no data?
		if len(strings.TrimSpace(col.Data)) > 0 {
			cols = append(cols, scrape.Text(col))
		}
	}

	e := Event{
		Type:    cols[1],
		Name:    cols[2],
		Teacher: cols[3],
	}

	addEventScheduling(&e, cols[4], cols[6])
	return e
}

func addEventScheduling(e *Event, daytime string, dur string) {
	// For strings such as "Út 12:20"
	daytimeRunes := []rune(daytime)
	e.Day = parseDay(string(daytimeRunes[:2]))

	timeFrom, err := time.Parse("15:04", string(daytimeRunes[3:]))
	if err != nil {
		panic(fmt.Sprintf("Unable to parse time: %s", string(daytimeRunes[3:])))
	}

	d, parity := parseDurationAndWeekParity(dur)

	e.TimeFrom = timeFrom
	e.TimeTo = timeFrom.Add(time.Minute * time.Duration(d))
	e.WeekParity = parity
}

func parseDurationAndWeekParity(dur string) (int, int) {
	// Strings like "90" or "240 Sudé týdny (liché kalendářní)"
	w := strings.Fields(dur)
	d, err := strconv.Atoi(w[0])
	if err != nil {
		panic(fmt.Sprintf("Unable to parse duration: %s", err))
	}
	parity := 0
	if len(w) > 1 {
		if w[1] == "Liché" {
			parity = 1
		} else {
			parity = 2
		}
	}
	return d, parity
}

func parseDay(day string) int {
	days := []string{"Po", "Út", "St", "Čt", "Pá"}
	for i, d := range days {
		if d == day {
			return i
		}
	}
	panic(fmt.Sprintf("Unknown day \"%s\"", day))
}

func getAbsoluteUrl(base, relative string) string {
	baseUrl, err := url.Parse(base)
	if err != nil {
		panic(err)
	}
	relativeUrl, err := url.Parse(relative)
	if err != nil {
		panic(err)
	}
	return baseUrl.ResolveReference(relativeUrl).String()
}
<commit_msg>Hack: force the display of the timetable for the current semester<commit_after>package sisparse

import (
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/yhat/scrape"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

const sisUrl = "https://is.cuni.cz/studium/predmety/index.php?do=predmet&kod=%s&skr=2018&sem=1"

// Returns a two-dimensional array containing groups of events.
// Each group is a slice of events which must be enrolled together,
// the groups represent different times/teachers of the same course.
// Also, lectures and seminars/practicals are in separate groups.
func GetCourseEvents(courseCode string) ([][]Event, error) {
	resp, err := http.Get(fmt.Sprintf(sisUrl, courseCode))
	if err != nil {
		return nil, err
	}
	// It is difficult to directly convert an event code to a schedule link,
	// because SIS requires the faculty number. Therefore we first open the course
	// in the "Subjects" SIS module and then go to a link which takes
	// us to the schedule.
	relativeScheduleUrl, err := getRelativeScheduleUrl(resp.Body)
	if err != nil {
		return nil, err
	}
	scheduleUrl := getAbsoluteUrl(sisUrl, relativeScheduleUrl)

	resp, err = http.Get(scheduleUrl)
	if err != nil {
		return nil, err
	}
	return parseCourseEvents(resp.Body), nil
}

func getRelativeScheduleUrl(body io.ReadCloser) (string, error) {
	const scheduleLinkText = "Rozvrh"

	root, err := html.Parse(body)
	if err != nil {
		panic(err)
	}

	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A {
			return scrape.Text(n) == scheduleLinkText
		}
		return false
	}

	scheduleLink, ok := scrape.Find(root, matcher)
	if !ok {
		return "", errors.New("Couldn't find schedule URL")
	}
	return scrape.Attr(scheduleLink, "href"), nil
}

func parseCourseEvents(body io.ReadCloser) [][]Event {
	root, err := html.Parse(body)
	if err != nil {
		panic(err)
	}

	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Tr && n.Parent != nil && n.Parent.Parent != nil {
			return scrape.Attr(n.Parent.Parent, "id") == "table1" &&
				scrape.Attr(n, "class") != "head1" // ignore table header
		}
		return false
	}

	eventsTable := scrape.FindAll(root, matcher)
	if len(eventsTable) == 0 {
		// The event table is not present at all (possibly SIS returned an error message)
		return [][]Event{}
	}

	res := [][]Event{}
	group := []Event{}
	for _, row := range eventsTable {
		event := parseEvent(row)
		// A non-empty name means the start of a new group;
		// names are omitted in all but the first event of a group.
		if event.Name != "" {
			if len(group) > 0 {
				res = append(res, group)
			}
			group = []Event{}
		} else {
			// Add the missing fields based on the group's first event
			event.Name = group[0].Name
			event.Teacher = group[0].Teacher
		}
		group = append(group, event)
	}
	if len(group) > 0 {
		res = append(res, group)
	}
	return res
}

func parseEvent(event *html.Node) Event {
	var cols []string
	for col := event.FirstChild; col != nil; col = col.NextSibling {
		// For some reason we also get siblings with no tag and no data?
		if len(strings.TrimSpace(col.Data)) > 0 {
			cols = append(cols, scrape.Text(col))
		}
	}

	e := Event{
		Type:    cols[1],
		Name:    cols[2],
		Teacher: cols[3],
	}

	addEventScheduling(&e, cols[4], cols[6])
	return e
}

func addEventScheduling(e *Event, daytime string, dur string) {
	// For strings such as "Út 12:20"
	daytimeRunes := []rune(daytime)
	e.Day = parseDay(string(daytimeRunes[:2]))

	timeFrom, err := time.Parse("15:04", string(daytimeRunes[3:]))
	if err != nil {
		panic(fmt.Sprintf("Unable to parse time: %s", string(daytimeRunes[3:])))
	}

	d, parity := parseDurationAndWeekParity(dur)

	e.TimeFrom = timeFrom
	e.TimeTo = timeFrom.Add(time.Minute * time.Duration(d))
	e.WeekParity = parity
}

func parseDurationAndWeekParity(dur string) (int, int) {
	// Strings like "90" or "240 Sudé týdny (liché kalendářní)"
	w := strings.Fields(dur)
	d, err := strconv.Atoi(w[0])
	if err != nil {
		panic(fmt.Sprintf("Unable to parse duration: %s", err))
	}
	parity := 0
	if len(w) > 1 {
		if w[1] == "Liché" {
			parity = 1
		} else {
			parity = 2
		}
	}
	return d, parity
}

func parseDay(day string) int {
	days := []string{"Po", "Út", "St", "Čt", "Pá"}
	for i, d := range days {
		if d == day {
			return i
		}
	}
	panic(fmt.Sprintf("Unknown day \"%s\"", day))
}

func getAbsoluteUrl(base, relative string) string {
	baseUrl, err := url.Parse(base)
	if err != nil {
		panic(err)
	}
	relativeUrl, err := url.Parse(relative)
	if err != nil {
		panic(err)
	}
	return baseUrl.ResolveReference(relativeUrl).String()
}
<|endoftext|>


<commit_before>// Copyright 2017 Xiaomi, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"github.com/open-falcon/falcon-plus/modules/agent/cron"
	"github.com/open-falcon/falcon-plus/modules/agent/funcs"
	"github.com/open-falcon/falcon-plus/modules/agent/g"
	"github.com/open-falcon/falcon-plus/modules/agent/http"
	"os"
)

func main() {

	cfg := flag.String("c", "cfg.json", "configuration file")
	version := flag.Bool("v", false, "show version")
	check := flag.Bool("check", false, "check collector")

	flag.Parse()

	if *version {
		fmt.Println(g.VERSION)
		os.Exit(0)
	}

	if *check {
		funcs.CheckCollector()
		os.Exit(0)
	}

	g.ParseConfig(*cfg)

	if g.Config().Debug {
		g.InitLog("debug")
	} else {
		g.InitLog("info")
	}

	g.InitRootDir()
	g.InitLocalIp()
	g.InitRpcClients()

	funcs.BuildMappers()

	go cron.InitDataHistory()

	cron.ReportAgentStatus()
	cron.SyncMinePlugins()
	cron.SyncBuiltinMetrics()
	cron.SyncTrustableIps()
	cron.Collect()

	go http.Start()

	select {}

}
<commit_msg>[agent] SET ENV FALCON_AGENT_RUNTIME<commit_after>// Copyright 2017 Xiaomi, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"github.com/open-falcon/falcon-plus/modules/agent/cron"
	"github.com/open-falcon/falcon-plus/modules/agent/funcs"
	"github.com/open-falcon/falcon-plus/modules/agent/g"
	"github.com/open-falcon/falcon-plus/modules/agent/http"
	"os"
)

func main() {

	cfg := flag.String("c", "cfg.json", "configuration file")
	version := flag.Bool("v", false, "show version")
	check := flag.Bool("check", false, "check collector")

	flag.Parse()

	if *version {
		fmt.Println(g.VERSION)
		os.Exit(0)
	}

	if *check {
		funcs.CheckCollector()
		os.Exit(0)
	}

	g.ParseConfig(*cfg)

	if g.Config().Debug {
		g.InitLog("debug")
	} else {
		g.InitLog("info")
	}

	os.Setenv("FALCON_AGENT_RUNTIME", g.VERSION)
	g.InitRootDir()
	g.InitLocalIp()
	g.InitRpcClients()

	funcs.BuildMappers()

	go cron.InitDataHistory()

	cron.ReportAgentStatus()
	cron.SyncMinePlugins()
	cron.SyncBuiltinMetrics()
	cron.SyncTrustableIps()
	cron.Collect()

	go http.Start()

	select {}

}
<|endoftext|>


<commit_before>package snapshot

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"sort"
	"strings"

	"github.com/desal/git"
	"github.com/desal/gocmd"
	"github.com/desal/richtext"
)

//go:generate stringer -type Flag

type (
	empty     struct{}
	Flag      int
	flagSet   map[Flag]empty
	stringSet map[string]empty

	Context struct {
		startPkg string
		doneDirs stringSet
		format   richtext.Format
		goPath   []string
		goCtx    *gocmd.Context
		gitCtx   *git.Context
		gitFlags []git.Flag
		flags    flagSet
	}

	DepsFile struct {
		Deps     []PkgDep
		TestDeps []PkgDep
	}

	PkgDep struct {
		ImportPath string
		GitRemote  string //Blank for standard packages
		SHA        string //Blank for standard packages
		Tags       []string
		Error      error `json:"-"`
	}

	PkgDepsByImport []PkgDep
)

const (
	_          Flag = iota
	MustExit        //
	MustPanic       //
	Warn            //
	Verbose         // show pkgname\n as it goes
	CmdVerbose      // Also displays commands being executed
	SkipVendor      //
)

var (
	gitFlags = map[Flag]git.Flag{
		MustExit:   git.MustExit,
		MustPanic:  git.MustPanic,
		Warn:       git.Warn,
		CmdVerbose: git.Verbose,
	}
)

func (fs flagSet) Checked(flag Flag) bool {
	_, ok := fs[flag]

	return ok
}

func (ss stringSet) Sorted() []string {
	r := []string{}
	for s, _ := range ss {
		r = append(r, s)
	}
	sort.Strings(r)
	return r
}

func (d *DepsFile) Sort() {
	sort.Sort(PkgDepsByImport(d.Deps))
	sort.Sort(PkgDepsByImport(d.TestDeps))
}

func (a PkgDepsByImport) Len() int           { return len(a) }
func (a PkgDepsByImport) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a PkgDepsByImport) Less(i, j int) bool { return a[i].ImportPath < a[j].ImportPath }

func New(format richtext.Format, goPath []string, flags ...Flag) *Context {
	c := &Context{
		doneDirs: stringSet{},
		format:   format,
		goPath:   goPath,
		goCtx:    gocmd.New(format, goPath, ""),
		flags:    flagSet{},
	}

	for _, flag := range flags {
		if gitFlag, ok := gitFlags[flag]; ok {
			c.gitFlags = append(c.gitFlags, gitFlag)
		}
		c.flags[flag] = empty{}
	}

	c.gitCtx = git.New(format, c.gitFlags...)

	return c
}

func (c *Context) errorf(s string, a ...interface{}) error {
	if c.flags.Checked(MustExit) {
		c.format.ErrorLine(s, a...)
		os.Exit(1)
	} else if c.flags.Checked(MustPanic) {
		panic(fmt.Errorf(s, a...))
	} else if c.flags.Checked(Warn) || c.flags.Checked(Verbose) {
		c.format.WarningLine(s, a...)
	}
	return fmt.Errorf(s, a...)
}

func (c *Context) warnf(s string, a ...interface{}) {
	if c.flags.Checked(Warn) {
		c.format.WarningLine(s, a...)
	}
}

func (c *Context) verbosef(s string, a ...interface{}) {
	if c.flags.Checked(Verbose) {
		c.format.PrintLine(s, a...)
	}
}

func ReadJson(filename string) (DepsFile, error) {
	var result DepsFile

	file, err := ioutil.ReadFile(filename)
	if err != nil {
		return result, err
	}

	err = json.Unmarshal(file, &result)
	return result, err
}

func WriteJson(filename string, depsFile DepsFile) error {
	jsonOutput, err := json.MarshalIndent(&depsFile, "", "  ")
	if err != nil {
		return err
	}

	if filename == "stdout" {
		_, err := os.Stdout.Write(jsonOutput)
		return err
	} else {
		return ioutil.WriteFile(filename, jsonOutput, 0644)
	}
}

func pkgContains(parent, child string) bool {
	if parent == child || strings.HasPrefix(child, parent+"/") {
		return true
	}
	return false
}
<commit_msg>alias 'stdin' for filename to read from stdin<commit_after>package snapshot

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"sort"
	"strings"

	"github.com/desal/git"
	"github.com/desal/gocmd"
	"github.com/desal/richtext"
)

//go:generate stringer -type Flag

type (
	empty     struct{}
	Flag      int
	flagSet   map[Flag]empty
	stringSet map[string]empty

	Context struct {
		startPkg string
		doneDirs stringSet
		format   richtext.Format
		goPath   []string
		goCtx    *gocmd.Context
		gitCtx   *git.Context
		gitFlags []git.Flag
		flags    flagSet
	}

	DepsFile struct {
		Deps     []PkgDep
		TestDeps []PkgDep
	}

	PkgDep struct {
		ImportPath string
		GitRemote  string //Blank for standard packages
		SHA        string //Blank for standard packages
		Tags       []string
		Error      error `json:"-"`
	}

	PkgDepsByImport []PkgDep
)

const (
	_          Flag = iota
	MustExit        //
	MustPanic       //
	Warn            //
	Verbose         // show pkgname\n as it goes
	CmdVerbose      // Also displays commands being executed
	SkipVendor      //
)

var (
	gitFlags = map[Flag]git.Flag{
		MustExit:   git.MustExit,
		MustPanic:  git.MustPanic,
		Warn:       git.Warn,
		CmdVerbose: git.Verbose,
	}
)

func (fs flagSet) Checked(flag Flag) bool {
	_, ok := fs[flag]

	return ok
}

func (ss stringSet) Sorted() []string {
	r := []string{}
	for s, _ := range ss {
		r = append(r, s)
	}
	sort.Strings(r)
	return r
}

func (d *DepsFile) Sort() {
	sort.Sort(PkgDepsByImport(d.Deps))
	sort.Sort(PkgDepsByImport(d.TestDeps))
}

func (a PkgDepsByImport) Len() int           { return len(a) }
func (a PkgDepsByImport) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a PkgDepsByImport) Less(i, j int) bool { return a[i].ImportPath < a[j].ImportPath }

func New(format richtext.Format, goPath []string, flags ...Flag) *Context {
	c := &Context{
		doneDirs: stringSet{},
		format:   format,
		goPath:   goPath,
		goCtx:    gocmd.New(format, goPath, ""),
		flags:    flagSet{},
	}

	for _, flag := range flags {
		if gitFlag, ok := gitFlags[flag]; ok {
			c.gitFlags = append(c.gitFlags, gitFlag)
		}
		c.flags[flag] = empty{}
	}

	c.gitCtx = git.New(format, c.gitFlags...)

	return c
}

func (c *Context) errorf(s string, a ...interface{}) error {
	if c.flags.Checked(MustExit) {
		c.format.ErrorLine(s, a...)
		os.Exit(1)
	} else if c.flags.Checked(MustPanic) {
		panic(fmt.Errorf(s, a...))
	} else if c.flags.Checked(Warn) || c.flags.Checked(Verbose) {
		c.format.WarningLine(s, a...)
	}
	return fmt.Errorf(s, a...)
}

func (c *Context) warnf(s string, a ...interface{}) {
	if c.flags.Checked(Warn) {
		c.format.WarningLine(s, a...)
	}
}

func (c *Context) verbosef(s string, a ...interface{}) {
	if c.flags.Checked(Verbose) {
		c.format.PrintLine(s, a...)
	}
}

func ReadJson(filename string) (DepsFile, error) {
	var result DepsFile

	var r io.Reader
	var err error
	if filename == "stdin" {
		r = os.Stdin
	} else {
		r, err = os.Open(filename)
		if err != nil {
			return result, err
		}
	}

	dec := json.NewDecoder(r)
	err = dec.Decode(&result)
	return result, err
}

func WriteJson(filename string, depsFile DepsFile) error {
	jsonOutput, err := json.MarshalIndent(&depsFile, "", "  ")
	if err != nil {
		return err
	}

	if filename == "stdout" {
		_, err := os.Stdout.Write(jsonOutput)
		return err
	} else {
		return ioutil.WriteFile(filename, jsonOutput, 0644)
	}
}

func pkgContains(parent, child string) bool {
	if parent == child || strings.HasPrefix(child, parent+"/") {
		return true
	}
	return false
}
<|endoftext|>