<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"runtime"

	"github.com/fatih/color"
)

func main() {

	// test flags
	var (
		compile            bool
		codegen            bool
		optimize           bool
		optimizeStandalone bool

		clean bool

		invertFlags bool
	)
	test := flag.NewFlagSet("test", flag.ExitOnError)
	test.BoolVar(&clean, "clean", false,
		"Clean out the output directories before running tests")

	test.BoolVar(&codegen, "codegen", false,
		"Generate asm but DON'T optimize")
	test.BoolVar(&compile, "compile", false,
		"Generate asm and optimize")
	test.BoolVar(&optimize, "optimize", false,
		"Optimize asm from -codegen.\n"+
			"\tDifferent to -compile:\n"+
			"\tCode is read back in, after being written to a file")
	test.BoolVar(&optimizeStandalone, "optimizeStandalone", false,
		"Optimize asm written explicitly for testing")

	test.BoolVar(&invertFlags, "invert", false,
		"Inverts all flags, making them subtractive instead of additive")

	// view flags
	var (
		diff  bool
		run   bool
		asm   bool
		build bool
	)
	view := flag.NewFlagSet("view", flag.ExitOnError)
	view.BoolVar(&diff, "diff", false,
		"view as a diff, instead of result and expectation separately")
	view.BoolVar(&run, "run", false,
		"compare results of the run phase of testing")
	view.BoolVar(&asm, "asm", false,
		"compare asm generated by building phase of testing")
	view.BoolVar(&build, "build", false,
		"compare results of the build phase of testing")

	// accept flags
	var all bool
	accept := flag.NewFlagSet("accept", flag.ExitOnError)
	accept.BoolVar(&all, "all", false,
		"accept every test result run in the last testing round")

	if len(os.Args) == 1 {
		helpMessage()
		os.Exit(0)
	}

	command := os.Args[1]
	switch command {
	case "test":
		test.Parse(os.Args[2:])
		testCommand(
			clean,
			codegen, optimize,
			compile,
			optimizeStandalone,
			invertFlags)
	case "view":
		view.Parse(os.Args[2:])
	case "create":
		// do creation tasks
	case "accept":
		accept.Parse(os.Args[2:])
	case "init":
		initDirs()
	case "help":
		fallthrough
	case "-help":
		fallthrough
	case "--help":
		helpMessage()
		os.Exit(0)
	default:
		color.Red("invalid command " + command)
		helpMessage()
		os.Exit(1)
	}

	os.Exit(0)
}

func testCommand(clean, codegen, optimize, compile, optimizeStandalone, invertFlags bool) {
	if invertFlags {
		compile = !compile
		codegen = !codegen
		optimize = !optimize
		optimizeStandalone = !optimizeStandalone

		clean = !clean
	}

	cores := runtime.NumCPU()
	runtime.GOMAXPROCS(cores + 1)

	if clean {
		fmt.Print("CLEANING... ")
		cleanDirs()
		fmt.Println(" done")
	}
	if codegen {
		color.Cyan("GENERATING CODE...")
		batchCodeGen(cores)
		color.Yellow("RUNNING...")
		batchRunUnoptimized(cores)
	}
	if optimize {
		color.Cyan("OPTIMIZING...")
		batchOptimize(cores)
		color.Yellow("RUNNING...")
		batchRunOptimized(cores)
	}
	if compile {
		color.Cyan("GENERATING CODE + OPTIMIZING...")
		batchCompile(cores)
		color.Yellow("RUNNING...")
		batchRunCompiled(cores)
	}
	if optimizeStandalone {
		color.Cyan("OPTIMIZING HAND WRITTEN ASM...")
		batchOptimizeStandalone(cores)
		color.YellowString("RUNNING...")
		batchRunOptimizedStandalone(cores)
	}
}
<commit_msg>make ouput on compile routine consistent with flag<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"runtime"

	"github.com/fatih/color"
)

func main() {

	// test flags
	var (
		compile            bool
		codegen            bool
		optimize           bool
		optimizeStandalone bool

		clean bool

		invertFlags bool
	)
	test := flag.NewFlagSet("test", flag.ExitOnError)
	test.BoolVar(&clean, "clean", false,
		"Clean out the output directories before running tests")

	test.BoolVar(&codegen, "codegen", false,
		"Generate asm but DON'T optimize")
	test.BoolVar(&compile, "compile", false,
		"Generate asm and optimize")
	test.BoolVar(&optimize, "optimize", false,
		"Optimize asm from -codegen.\n"+
			"\tDifferent to -compile:\n"+
			"\tCode is read back in, after being written to a file")
	test.BoolVar(&optimizeStandalone, "optimizeStandalone", false,
		"Optimize asm written explicitly for testing")

	test.BoolVar(&invertFlags, "invert", false,
		"Inverts all flags, making them subtractive instead of additive")

	// view flags
	var (
		diff  bool
		run   bool
		asm   bool
		build bool
	)
	view := flag.NewFlagSet("view", flag.ExitOnError)
	view.BoolVar(&diff, "diff", false,
		"view as a diff, instead of result and expectation separately")
	view.BoolVar(&run, "run", false,
		"compare results of the run phase of testing")
	view.BoolVar(&asm, "asm", false,
		"compare asm generated by building phase of testing")
	view.BoolVar(&build, "build", false,
		"compare results of the build phase of testing")

	// accept flags
	var all bool
	accept := flag.NewFlagSet("accept", flag.ExitOnError)
	accept.BoolVar(&all, "all", false,
		"accept every test result run in the last testing round")

	if len(os.Args) == 1 {
		helpMessage()
		os.Exit(0)
	}

	command := os.Args[1]
	switch command {
	case "test":
		test.Parse(os.Args[2:])
		testCommand(
			clean,
			codegen, optimize,
			compile,
			optimizeStandalone,
			invertFlags)
	case "view":
		view.Parse(os.Args[2:])
	case "create":
		// do creation tasks
	case "accept":
		accept.Parse(os.Args[2:])
	case "init":
		initDirs()
	case "help":
		fallthrough
	case "-help":
		fallthrough
	case "--help":
		helpMessage()
		os.Exit(0)
	default:
		color.Red("invalid command " + command)
		helpMessage()
		os.Exit(1)
	}

	os.Exit(0)
}
func createCommand(testName string) {

}

func testCommand(clean, codegen, optimize, compile, optimizeStandalone, invertFlags bool) {
	if invertFlags {
		compile = !compile
		codegen = !codegen
		optimize = !optimize
		optimizeStandalone = !optimizeStandalone

		clean = !clean
	}

	cores := runtime.NumCPU()
	runtime.GOMAXPROCS(cores + 1)

	if clean {
		fmt.Print("CLEANING... ")
		cleanDirs()
		fmt.Println(" done")
	}
	if codegen {
		color.Cyan("GENERATING CODE...")
		batchCodeGen(cores)
		color.Yellow("RUNNING...")
		batchRunUnoptimized(cores)
	}
	if optimize {
		color.Cyan("OPTIMIZING...")
		batchOptimize(cores)
		color.Yellow("RUNNING...")
		batchRunOptimized(cores)
	}
	if compile {
		color.Cyan("COMPILING...")
		batchCompile(cores)
		color.Yellow("RUNNING...")
		batchRunCompiled(cores)
	}
	if optimizeStandalone {
		color.Cyan("OPTIMIZING HAND WRITTEN ASM...")
		batchOptimizeStandalone(cores)
		color.YellowString("RUNNING...")
		batchRunOptimizedStandalone(cores)
	}
}
<|endoftext|>


<commit_before>package hot

import (
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"os"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"

	"gopkg.in/fsnotify.v1"
)

type Config struct {
	Watch          bool
	BaseName       string
	Dir            string
	Funcs          template.FuncMap
	LeftDelim      string
	RightDelim     string
	FilesExtension []string
}

type Template struct {
	tpl *template.Template
	cfg *Config
	Out io.Writer
}

func New(cfg *Config) (*Template, error) {
	leftDelim := "{{"
	rightDelim := "}}"
	if cfg.LeftDelim != "" {
		leftDelim = cfg.LeftDelim
	}
	if cfg.RightDelin != "" {
		rightDelim = cfg.RightDelin
	}
	tmpl := template.New(cfg.BaseName).Delims(leftDelim, rightDelim)
	if cfg.Funcs != nil {
		tmpl = template.New(cfg.BaseName).Funcs(cfg.Funcs).Delims(leftDelim, rightDelim)
	}
	tpl := &Template{
		tpl: tmpl,
		cfg: cfg,
		Out: os.Stdout,
	}
	tpl.Init()
	err := tpl.Load(cfg.Dir)
	if err != nil {
		return nil, err
	}
	return tpl, nil
}

func (t *Template) Init() {
	if t.cfg.Watch {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGTERM, syscall.SIGINT)
		go func() {
			watcher, err := fsnotify.NewWatcher()
			if err != nil {
				fmt.Fprintln(t.Out, err)
				return
			}
			err = filepath.Walk(t.cfg.Dir, func(fPath string, info os.FileInfo, ferr error) error {
				if ferr != nil {
					return ferr
				}
				if info.IsDir() {
					fmt.Fprintln(t.Out, "start watching ", fPath)
					return watcher.Add(fPath)
				}
				return nil
			})
			if err != nil {
				fmt.Fprintln(t.Out, err)
				return
			}

			for {
				select {
				case <-c:
					watcher.Close()
					fmt.Println("shutting down hot templates... done")
					os.Exit(0)
				case evt := <-watcher.Events:
					fmt.Fprintf(t.Out, "%s:  reloading... \n", evt.String())
					t.Reload()
				}
			}
		}()
	}
}

func (t *Template) Reload() {
	tpl := *t.tpl
	t.tpl = template.New(t.cfg.BaseName)
	if t.cfg.Funcs != nil {
		t.tpl = template.New(t.cfg.BaseName).Funcs(t.cfg.Funcs)
	}
	err := t.Load(t.cfg.Dir)
	if err != nil {
		fmt.Fprintln(t.Out, err.Error())
		t.tpl = &tpl
	}
}

func (t *Template) Load(dir string) error {
	fmt.Fprintln(t.Out, "loading ", dir)
	return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}

		extension := filepath.Ext(path)
		found := false
		for _, ext := range t.cfg.FilesExtension {
			if ext == extension {
				found = true
				break
			}
		}
		if !found {
			return nil
		}

		data, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}

		// We remove the directory name from the path
		// this means if we have directory foo, with file bar.tpl
		// full path for bar file foo/bar.tpl
		// we trim the foo part and remain with /bar.tpl
		name := path[len(dir):]

		name = filepath.ToSlash(name)

		name = strings.TrimPrefix(name, "/") // case  we missed the opening slash

		tpl := t.tpl.New(name)
		_, err = tpl.Parse(string(data))
		if err != nil {
			return err
		}
		return nil
	})
}

func (t *Template) Execute(w io.Writer, name string, ctx interface{}) error {
	return t.tpl.ExecuteTemplate(w, name, ctx)
}
<commit_msg>Fix typo<commit_after>package hot

import (
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"os"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"

	"gopkg.in/fsnotify.v1"
)

type Config struct {
	Watch          bool
	BaseName       string
	Dir            string
	Funcs          template.FuncMap
	LeftDelim      string
	RightDelim     string
	FilesExtension []string
}

type Template struct {
	tpl *template.Template
	cfg *Config
	Out io.Writer
}

func New(cfg *Config) (*Template, error) {
	leftDelim := "{{"
	rightDelim := "}}"
	if cfg.LeftDelim != "" {
		leftDelim = cfg.LeftDelim
	}
	if cfg.RightDelim != "" {
		rightDelim = cfg.RightDelim
	}
	tmpl := template.New(cfg.BaseName).Delims(leftDelim, rightDelim)
	if cfg.Funcs != nil {
		tmpl = template.New(cfg.BaseName).Funcs(cfg.Funcs).Delims(leftDelim, rightDelim)
	}
	tpl := &Template{
		tpl: tmpl,
		cfg: cfg,
		Out: os.Stdout,
	}
	tpl.Init()
	err := tpl.Load(cfg.Dir)
	if err != nil {
		return nil, err
	}
	return tpl, nil
}

func (t *Template) Init() {
	if t.cfg.Watch {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGTERM, syscall.SIGINT)
		go func() {
			watcher, err := fsnotify.NewWatcher()
			if err != nil {
				fmt.Fprintln(t.Out, err)
				return
			}
			err = filepath.Walk(t.cfg.Dir, func(fPath string, info os.FileInfo, ferr error) error {
				if ferr != nil {
					return ferr
				}
				if info.IsDir() {
					fmt.Fprintln(t.Out, "start watching ", fPath)
					return watcher.Add(fPath)
				}
				return nil
			})
			if err != nil {
				fmt.Fprintln(t.Out, err)
				return
			}

			for {
				select {
				case <-c:
					watcher.Close()
					fmt.Println("shutting down hot templates... done")
					os.Exit(0)
				case evt := <-watcher.Events:
					fmt.Fprintf(t.Out, "%s:  reloading... \n", evt.String())
					t.Reload()
				}
			}
		}()
	}
}

func (t *Template) Reload() {
	tpl := *t.tpl
	t.tpl = template.New(t.cfg.BaseName)
	if t.cfg.Funcs != nil {
		t.tpl = template.New(t.cfg.BaseName).Funcs(t.cfg.Funcs)
	}
	err := t.Load(t.cfg.Dir)
	if err != nil {
		fmt.Fprintln(t.Out, err.Error())
		t.tpl = &tpl
	}
}

func (t *Template) Load(dir string) error {
	fmt.Fprintln(t.Out, "loading ", dir)
	return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}

		extension := filepath.Ext(path)
		found := false
		for _, ext := range t.cfg.FilesExtension {
			if ext == extension {
				found = true
				break
			}
		}
		if !found {
			return nil
		}

		data, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}

		// We remove the directory name from the path
		// this means if we have directory foo, with file bar.tpl
		// full path for bar file foo/bar.tpl
		// we trim the foo part and remain with /bar.tpl
		name := path[len(dir):]

		name = filepath.ToSlash(name)

		name = strings.TrimPrefix(name, "/") // case  we missed the opening slash

		tpl := t.tpl.New(name)
		_, err = tpl.Parse(string(data))
		if err != nil {
			return err
		}
		return nil
	})
}

func (t *Template) Execute(w io.Writer, name string, ctx interface{}) error {
	return t.tpl.ExecuteTemplate(w, name, ctx)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"regexp"
	"strconv"

	ircevent "github.com/thoj/go-ircevent"
)

//irc client options struct
type ircConfig struct {
	Server   string
	Channels []string
	Nickname string
	Verbose  bool
	Debug    bool
}

var (
	ircConn *ircevent.Connection //irc connection
	ircCfg  *ircConfig           //irc config
	//regex parse string for newEpisode()
	releaseWatch = regexp.MustCompile(
		`(?i)release:.+\[horriblesubs\] (.+) - ([0-9]{1,3}) \[(1080p|720p|480p)\]`)
)

//Starts the irc client
func ircConnStart(c *ircConfig) {
	ircCfg = c

	ircConn = ircevent.IRC(c.Nickname, c.Nickname)
	ircConn.Debug = c.Debug
	ircConn.VerboseCallbackHandler = c.Verbose
	ircConn.AddCallback("PRIVMSG", ircMsgHandler)
	ircConn.AddCallback("001", ircWelcomeHandler)

	err := ircConn.Connect(c.Server)
	if err != nil {
		log.Fatal("ircConnStart() => Connect() error:\t", err)
	}
	ircConn.Loop()
}

//irc client incomming message handler function
func ircMsgHandler(e *ircevent.Event) {
	//Samples new anime string:
	//Release: [Anime] [HorribleSubs] Ushio to Tora - 22 [480p].mkv
	//Release: [Anime] [HorribleSubs] Hackadoll the Animation - 09 [720p].mkv

	if releaseWatch.MatchString(e.Message()) == true {
		newEpisode(releaseWatch.FindStringSubmatch(e.Message()))
	} else if e.Arguments[0] == ircClient.Nickname && e.Nick != ircClient.Nickname {
		ircConn.Privmsg(e.Nick, "Discord BOT - https://github.com/gagizagi/Axolotl-GO")
		ircConn.Privmsg(e.Nick, "Current uptime is "+getUptime())
	}
}

//irc client onWelcome handler function
func ircWelcomeHandler(e *ircevent.Event) {
	for _, channel := range ircCfg.Channels {
		ircConn.Join(channel)
	}
}

//new episode handler function
//args
// 0 = full message
// 1 = anime name
// 2 = episode (1-3 length integers only)
// 3 = resolution (1080p | 720p | 480p)
func newEpisode(args []string) {
	epnum, _ := strconv.Atoi(args[2])
	episode := anime{Name: args[1], Episode: epnum}
	if episode.Exists() {
		if episode.NewEpisode() {
			episode.UpdateEp()
			if len(episode.Subs) > 0 {
				resultstr := fmt.Sprintf(
					"**New episode of %s released - Episode %d**\n",
					episode.Name, episode.Episode)
				for _, person := range episode.Subs {
					resultstr += fmt.Sprintf("<@%s>", person)
				}
				resultstr += fmt.Sprintf("\nDownload at %s\n", episode.Href)
				resultstr += fmt.Sprintf(
					"To subscribe to this anime type \"!sub %s\"",
					episode.ID)

				//TODO make a function
				for _, channel := range discordCfg.AnimeChannels {
					discord.ChannelMessageSend(channel, resultstr)
				}
			}
		}
	} else {
		//Insert series into DB
		episode.Insert()

		//Announce new series to all anime channels
		resultstr := fmt.Sprintf("**New series started: %s - Episode %d**\n",
			episode.Name, episode.Episode)
		resultstr += fmt.Sprintf("To subscribe to this anime type \"!sub %s\"",
			episode.ID)

		//TODO make a function
		for _, channel := range discordCfg.AnimeChannels {
			discord.ChannelMessageSend(channel, resultstr)
		}
	}
}
<commit_msg>Don't add 'Download at' line from response if href is not set<commit_after>package main

import (
	"fmt"
	"log"
	"regexp"
	"strconv"

	ircevent "github.com/thoj/go-ircevent"
)

//irc client options struct
type ircConfig struct {
	Server   string
	Channels []string
	Nickname string
	Verbose  bool
	Debug    bool
}

var (
	ircConn *ircevent.Connection //irc connection
	ircCfg  *ircConfig           //irc config
	//regex parse string for newEpisode()
	releaseWatch = regexp.MustCompile(
		`(?i)release:.+\[horriblesubs\] (.+) - ([0-9]{1,3}) \[(1080p|720p|480p)\]`)
)

//Starts the irc client
func ircConnStart(c *ircConfig) {
	ircCfg = c

	ircConn = ircevent.IRC(c.Nickname, c.Nickname)
	ircConn.Debug = c.Debug
	ircConn.VerboseCallbackHandler = c.Verbose
	ircConn.AddCallback("PRIVMSG", ircMsgHandler)
	ircConn.AddCallback("001", ircWelcomeHandler)

	err := ircConn.Connect(c.Server)
	if err != nil {
		log.Fatal("ircConnStart() => Connect() error:\t", err)
	}
	ircConn.Loop()
}

//irc client incomming message handler function
func ircMsgHandler(e *ircevent.Event) {
	//Samples new anime string:
	//Release: [Anime] [HorribleSubs] Ushio to Tora - 22 [480p].mkv
	//Release: [Anime] [HorribleSubs] Hackadoll the Animation - 09 [720p].mkv

	if releaseWatch.MatchString(e.Message()) == true {
		newEpisode(releaseWatch.FindStringSubmatch(e.Message()))
	} else if e.Arguments[0] == ircClient.Nickname && e.Nick != ircClient.Nickname {
		ircConn.Privmsg(e.Nick, "Discord BOT - https://github.com/gagizagi/Axolotl-GO")
		ircConn.Privmsg(e.Nick, "Current uptime is "+getUptime())
	}
}

//irc client onWelcome handler function
func ircWelcomeHandler(e *ircevent.Event) {
	for _, channel := range ircCfg.Channels {
		ircConn.Join(channel)
	}
}

//new episode handler function
//args
// 0 = full message
// 1 = anime name
// 2 = episode (1-3 length integers only)
// 3 = resolution (1080p | 720p | 480p)
func newEpisode(args []string) {
	epnum, _ := strconv.Atoi(args[2])
	episode := anime{Name: args[1], Episode: epnum}
	if episode.Exists() {
		if episode.NewEpisode() {
			episode.UpdateEp()
			if len(episode.Subs) > 0 {
				resultstr := fmt.Sprintf(
					"**New episode of %s released - Episode %d**\n",
					episode.Name, episode.Episode)
				for _, person := range episode.Subs {
					resultstr += fmt.Sprintf("<@%s>", person)
				}
				if episode.Href != "" {
					resultstr += fmt.Sprintf("\nDownload at %s\n", episode.Href)
				}
				resultstr += fmt.Sprintf(
					"To subscribe to this anime type \"!sub %s\"",
					episode.ID)

				//TODO make a function
				for _, channel := range discordCfg.AnimeChannels {
					discord.ChannelMessageSend(channel, resultstr)
				}
			}
		}
	} else {
		//Insert series into DB
		episode.Insert()

		//Announce new series to all anime channels
		resultstr := fmt.Sprintf("**New series started: %s - Episode %d**\n",
			episode.Name, episode.Episode)
		resultstr += fmt.Sprintf("To subscribe to this anime type \"!sub %s\"",
			episode.ID)

		//TODO make a function
		for _, channel := range discordCfg.AnimeChannels {
			discord.ChannelMessageSend(channel, resultstr)
		}
	}
}
<|endoftext|>


<commit_before>package gocron

import (
	"errors"
	"fmt"
	"log"
	"reflect"
	"time"
)

var (
	ErrTimeFormat           = errors.New("time format error")
	ErrParamsNotAdapted     = errors.New("the number of params is not adapted")
	ErrNotAFunction         = errors.New("only functions can be schedule into the job queue")
	ErrPeriodNotSpecified   = errors.New("unspecified job period")
	ErrParameterCannotBeNil = errors.New("nil paramaters cannot be used with reflection")
)

// Job struct keeping information about job
type Job struct {
	interval uint64                   // pause interval * unit between runs
	jobFunc  string                   // the job jobFunc to run, func[jobFunc]
	unit     timeUnit                 // time units, ,e.g. 'minutes', 'hours'...
	atTime   time.Duration            // optional time at which this job runs
	err      error                    // error related to job
	loc      *time.Location           // optional timezone that the atTime is in
	lastRun  time.Time                // datetime of last run
	nextRun  time.Time                // datetime of next run
	startDay time.Weekday             // Specific day of the week to start on
	funcs    map[string]interface{}   // Map for the function task store
	fparams  map[string][]interface{} // Map for function and  params of function
	lock     bool                     // lock the job from running at same time form multiple instances
	tags     []string                 // allow the user to tag jobs with certain labels
}

// NewJob creates a new job with the time interval.
func NewJob(interval uint64) *Job {
	return &Job{
		interval: interval,
		loc:      loc,
		lastRun:  time.Unix(0, 0),
		nextRun:  time.Unix(0, 0),
		startDay: time.Sunday,
		funcs:    make(map[string]interface{}),
		fparams:  make(map[string][]interface{}),
		tags:     []string{},
	}
}

// True if the job should be run now
func (j *Job) shouldRun() bool {
	return time.Now().Unix() >= j.nextRun.Unix()
}

//Run the job and immediately reschedule it
func (j *Job) run() ([]reflect.Value, error) {
	if j.lock {
		if locker == nil {
			return nil, fmt.Errorf("trying to lock %s with nil locker", j.jobFunc)
		}
		key := getFunctionKey(j.jobFunc)

		locker.Lock(key)
		defer locker.Unlock(key)
	}

	j.lastRun = time.Now()
	if err := j.scheduleNextRun(); err != nil {
		return nil, err
	}
	result, err := callJobFuncWithParams(j.funcs[j.jobFunc], j.fparams[j.jobFunc])
	if err != nil {
		return nil, err
	}
	return result, nil
}

// Err should be checked to ensure an error didn't occur creating the job
func (j *Job) Err() error {
	return j.err
}

// Do specifies the jobFunc that should be called every time the job runs
func (j *Job) Do(jobFun interface{}, params ...interface{}) error {
	if j.err != nil {
		return j.err
	}

	typ := reflect.TypeOf(jobFun)
	if typ.Kind() != reflect.Func {
		return ErrNotAFunction
	}
	fname := getFunctionName(jobFun)
	j.funcs[fname] = jobFun
	j.fparams[fname] = params
	j.jobFunc = fname
	j.scheduleNextRun()

	return nil
}

// DoSafely does the same thing as Do, but logs unexpected panics, instead of unwinding them up the chain
// Deprecated: DoSafely exists due to historical compatibility and will be removed soon. Use Do instead
func (j *Job) DoSafely(jobFun interface{}, params ...interface{}) error {
	recoveryWrapperFunc := func() {
		defer func() {
			if r := recover(); r != nil {
				log.Printf("Internal panic occurred: %s", r)
			}
		}()

		_, _ = callJobFuncWithParams(jobFun, params)
	}

	return j.Do(recoveryWrapperFunc)
}

// At schedules job at specific time of day
//	s.Every(1).Day().At("10:30:01").Do(task)
//	s.Every(1).Monday().At("10:30:01").Do(task)
func (j *Job) At(t string) *Job {
	hour, min, sec, err := formatTime(t)
	if err != nil {
		j.err = ErrTimeFormat
		return j
	}
	// save atTime start as duration from midnight
	j.atTime = time.Duration(hour)*time.Hour + time.Duration(min)*time.Minute + time.Duration(sec)*time.Second
	return j
}

// GetAt returns the specific time of day the job will run at
//	s.Every(1).Day().At("10:30").GetAt() == "10:30"
func (j *Job) GetAt() string {
	return fmt.Sprintf("%d:%d", j.atTime/time.Hour, (j.atTime%time.Hour)/time.Minute)
}

// Loc sets the location for which to interpret "At"
//	s.Every(1).Day().At("10:30").Loc(time.UTC).Do(task)
func (j *Job) Loc(loc *time.Location) *Job {
	j.loc = loc
	return j
}

// Tag allows you to add labels to a job
// they don't impact the functionality of the job.
func (j *Job) Tag(t string, others ...string) {
	j.tags = append(j.tags, t)
	for _, tag := range others {
		j.tags = append(j.tags, tag)
	}
}

// Untag removes a tag from a job
func (j *Job) Untag(t string) {
	newTags := []string{}
	for _, tag := range j.tags {
		if t != tag {
			newTags = append(newTags, tag)
		}
	}

	j.tags = newTags
}

// Tags returns the tags attached to the job
func (j *Job) Tags() []string {
	return j.tags
}

func (j *Job) periodDuration() (time.Duration, error) {
	interval := time.Duration(j.interval)
	switch j.unit {
	case seconds:
		return interval * time.Second, nil
	case minutes:
		return interval * time.Minute, nil
	case hours:
		return interval * time.Hour, nil
	case days:
		return interval * time.Hour * 24, nil
	case weeks:
		return interval * time.Hour * 24 * 7, nil
	}
	return 0, ErrPeriodNotSpecified
}

// roundToMidnight truncate time to midnight
func (j *Job) roundToMidnight(t time.Time) time.Time {
	return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, j.loc)
}

// scheduleNextRun Compute the instant when this job should run next
func (j *Job) scheduleNextRun() error {
	now := time.Now()
	if j.lastRun == time.Unix(0, 0) {
		j.lastRun = now
	}

	if j.nextRun.After(now) {
		return nil
	}

	periodDuration, err := j.periodDuration()
	if err != nil {
		return err
	}

	switch j.unit {
	case seconds, minutes, hours:
		j.nextRun = j.lastRun.Add(periodDuration)
	case days:
		j.nextRun = j.roundToMidnight(j.lastRun)
		j.nextRun = j.nextRun.Add(j.atTime)
	case weeks:
		j.nextRun = j.roundToMidnight(j.lastRun)
		dayDiff := int(j.startDay)
		dayDiff -= int(j.nextRun.Weekday())
		if dayDiff != 0 {
			j.nextRun = j.nextRun.Add(time.Duration(dayDiff) * 24 * time.Hour)
		}
		j.nextRun = j.nextRun.Add(j.atTime)
	}

	// advance to next possible schedule
	for j.nextRun.Before(now) || j.nextRun.Before(j.lastRun) {
		j.nextRun = j.nextRun.Add(periodDuration)
	}

	return nil
}

// NextScheduledTime returns the time of when this job is to run next
func (j *Job) NextScheduledTime() time.Time {
	return j.nextRun
}

// set the job's unit with seconds,minutes,hours...
func (j *Job) mustInterval(i uint64) error {
	if j.interval != i {
		return fmt.Errorf("interval must be %d", i)
	}
	return nil
}

// From schedules the next run of the job
func (j *Job) From(t *time.Time) *Job {
	j.nextRun = *t
	return j
}

// setUnit sets unit type
func (j *Job) setUnit(unit timeUnit) *Job {
	j.unit = unit
	return j
}

// Seconds set the unit with seconds
func (j *Job) Seconds() *Job {
	return j.setUnit(seconds)
}

// Minutes set the unit with minute
func (j *Job) Minutes() *Job {
	return j.setUnit(minutes)
}

// Hours set the unit with hours
func (j *Job) Hours() *Job {
	return j.setUnit(hours)
}

// Days set the job's unit with days
func (j *Job) Days() *Job {
	return j.setUnit(days)
}

// Weeks sets the units as weeks
func (j *Job) Weeks() *Job {
	return j.setUnit(weeks)
}

// Second sets the unit with second
func (j *Job) Second() *Job {
	j.mustInterval(1)
	return j.Seconds()
}

// Minute sets the unit  with minute, which interval is 1
func (j *Job) Minute() *Job {
	j.mustInterval(1)
	return j.Minutes()
}

// Hour sets the unit with hour, which interval is 1
func (j *Job) Hour() *Job {
	j.mustInterval(1)
	return j.Hours()
}

// Day sets the job's unit with day, which interval is 1
func (j *Job) Day() *Job {
	j.mustInterval(1)
	return j.Days()
}

// Week sets the job's unit with week, which interval is 1
func (j *Job) Week() *Job {
	j.mustInterval(1)
	return j.Weeks()
}

// Weekday start job on specific Weekday
func (j *Job) Weekday(startDay time.Weekday) *Job {
	j.mustInterval(1)
	j.startDay = startDay
	return j.Weeks()
}

// GetWeekday returns which day of the week the job will run on
// This should only be used when .Weekday(...) was called on the job.
func (j *Job) GetWeekday() time.Weekday {
	return j.startDay
}

// Monday set the start day with Monday
// - s.Every(1).Monday().Do(task)
func (j *Job) Monday() (job *Job) {
	return j.Weekday(time.Monday)
}

// Tuesday sets the job start day Tuesday
func (j *Job) Tuesday() *Job {
	return j.Weekday(time.Tuesday)
}

// Wednesday sets the job start day Wednesday
func (j *Job) Wednesday() *Job {
	return j.Weekday(time.Wednesday)
}

// Thursday sets the job start day Thursday
func (j *Job) Thursday() *Job {
	return j.Weekday(time.Thursday)
}

// Friday sets the job start day Friday
func (j *Job) Friday() *Job {
	return j.Weekday(time.Friday)
}

// Saturday sets the job start day Saturday
func (j *Job) Saturday() *Job {
	return j.Weekday(time.Saturday)
}

// Sunday sets the job start day Sunday
func (j *Job) Sunday() *Job {
	return j.Weekday(time.Sunday)
}

// Lock prevents job to run from multiple instances of gocron
func (j *Job) Lock() *Job {
	j.lock = true
	return j
}
<commit_msg>refactoring (j *Job) periodDuration() (time.Duration, error)<commit_after>package gocron

import (
	"errors"
	"fmt"
	"log"
	"reflect"
	"time"
)

var (
	ErrTimeFormat           = errors.New("time format error")
	ErrParamsNotAdapted     = errors.New("the number of params is not adapted")
	ErrNotAFunction         = errors.New("only functions can be schedule into the job queue")
	ErrPeriodNotSpecified   = errors.New("unspecified job period")
	ErrParameterCannotBeNil = errors.New("nil paramaters cannot be used with reflection")
)

// Job struct keeping information about job
type Job struct {
	interval uint64                   // pause interval * unit between runs
	jobFunc  string                   // the job jobFunc to run, func[jobFunc]
	unit     timeUnit                 // time units, ,e.g. 'minutes', 'hours'...
	atTime   time.Duration            // optional time at which this job runs
	err      error                    // error related to job
	loc      *time.Location           // optional timezone that the atTime is in
	lastRun  time.Time                // datetime of last run
	nextRun  time.Time                // datetime of next run
	startDay time.Weekday             // Specific day of the week to start on
	funcs    map[string]interface{}   // Map for the function task store
	fparams  map[string][]interface{} // Map for function and  params of function
	lock     bool                     // lock the job from running at same time form multiple instances
	tags     []string                 // allow the user to tag jobs with certain labels
}

// NewJob creates a new job with the time interval.
func NewJob(interval uint64) *Job {
	return &Job{
		interval: interval,
		loc:      loc,
		lastRun:  time.Unix(0, 0),
		nextRun:  time.Unix(0, 0),
		startDay: time.Sunday,
		funcs:    make(map[string]interface{}),
		fparams:  make(map[string][]interface{}),
		tags:     []string{},
	}
}

// True if the job should be run now
func (j *Job) shouldRun() bool {
	return time.Now().Unix() >= j.nextRun.Unix()
}

//Run the job and immediately reschedule it
func (j *Job) run() ([]reflect.Value, error) {
	if j.lock {
		if locker == nil {
			return nil, fmt.Errorf("trying to lock %s with nil locker", j.jobFunc)
		}
		key := getFunctionKey(j.jobFunc)

		locker.Lock(key)
		defer locker.Unlock(key)
	}

	j.lastRun = time.Now()
	if err := j.scheduleNextRun(); err != nil {
		return nil, err
	}
	result, err := callJobFuncWithParams(j.funcs[j.jobFunc], j.fparams[j.jobFunc])
	if err != nil {
		return nil, err
	}
	return result, nil
}

// Err should be checked to ensure an error didn't occur creating the job
func (j *Job) Err() error {
	return j.err
}

// Do specifies the jobFunc that should be called every time the job runs
func (j *Job) Do(jobFun interface{}, params ...interface{}) error {
	if j.err != nil {
		return j.err
	}

	typ := reflect.TypeOf(jobFun)
	if typ.Kind() != reflect.Func {
		return ErrNotAFunction
	}
	fname := getFunctionName(jobFun)
	j.funcs[fname] = jobFun
	j.fparams[fname] = params
	j.jobFunc = fname
	j.scheduleNextRun()

	return nil
}

// DoSafely does the same thing as Do, but logs unexpected panics, instead of unwinding them up the chain
// Deprecated: DoSafely exists due to historical compatibility and will be removed soon. Use Do instead
func (j *Job) DoSafely(jobFun interface{}, params ...interface{}) error {
	recoveryWrapperFunc := func() {
		defer func() {
			if r := recover(); r != nil {
				log.Printf("Internal panic occurred: %s", r)
			}
		}()

		_, _ = callJobFuncWithParams(jobFun, params)
	}

	return j.Do(recoveryWrapperFunc)
}

// At schedules job at specific time of day
//	s.Every(1).Day().At("10:30:01").Do(task)
//	s.Every(1).Monday().At("10:30:01").Do(task)
func (j *Job) At(t string) *Job {
	hour, min, sec, err := formatTime(t)
	if err != nil {
		j.err = ErrTimeFormat
		return j
	}
	// save atTime start as duration from midnight
	j.atTime = time.Duration(hour)*time.Hour + time.Duration(min)*time.Minute + time.Duration(sec)*time.Second
	return j
}

// GetAt returns the specific time of day the job will run at
//	s.Every(1).Day().At("10:30").GetAt() == "10:30"
func (j *Job) GetAt() string {
	return fmt.Sprintf("%d:%d", j.atTime/time.Hour, (j.atTime%time.Hour)/time.Minute)
}

// Loc sets the location for which to interpret "At"
//	s.Every(1).Day().At("10:30").Loc(time.UTC).Do(task)
func (j *Job) Loc(loc *time.Location) *Job {
	j.loc = loc
	return j
}

// Tag allows you to add labels to a job
// they don't impact the functionality of the job.
func (j *Job) Tag(t string, others ...string) {
	j.tags = append(j.tags, t)
	for _, tag := range others {
		j.tags = append(j.tags, tag)
	}
}

// Untag removes a tag from a job
func (j *Job) Untag(t string) {
	newTags := []string{}
	for _, tag := range j.tags {
		if t != tag {
			newTags = append(newTags, tag)
		}
	}

	j.tags = newTags
}

// Tags returns the tags attached to the job
func (j *Job) Tags() []string {
	return j.tags
}

func (j *Job) periodDuration() (time.Duration, error) {
	interval := time.Duration(j.interval)
	var periodDuration time.Duration

	switch j.unit {
	case seconds:
		periodDuration = interval * time.Second
	case minutes:
		periodDuration = interval * time.Minute
	case hours:
		periodDuration = interval * time.Hour
	case days:
		periodDuration = interval * time.Hour * 24
	case weeks:
		periodDuration = interval * time.Hour * 24 * 7
	default:
		return 0, ErrPeriodNotSpecified
	}
	return periodDuration, nil
}

// roundToMidnight truncate time to midnight
func (j *Job) roundToMidnight(t time.Time) time.Time {
	return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, j.loc)
}

// scheduleNextRun Compute the instant when this job should run next
func (j *Job) scheduleNextRun() error {
	now := time.Now()
	if j.lastRun == time.Unix(0, 0) {
		j.lastRun = now
	}

	if j.nextRun.After(now) {
		return nil
	}

	periodDuration, err := j.periodDuration()
	if err != nil {
		return err
	}

	switch j.unit {
	case seconds, minutes, hours:
		j.nextRun = j.lastRun.Add(periodDuration)
	case days:
		j.nextRun = j.roundToMidnight(j.lastRun)
		j.nextRun = j.nextRun.Add(j.atTime)
	case weeks:
		j.nextRun = j.roundToMidnight(j.lastRun)
		dayDiff := int(j.startDay)
		dayDiff -= int(j.nextRun.Weekday())
		if dayDiff != 0 {
			j.nextRun = j.nextRun.Add(time.Duration(dayDiff) * 24 * time.Hour)
		}
		j.nextRun = j.nextRun.Add(j.atTime)
	}

	// advance to next possible schedule
	for j.nextRun.Before(now) || j.nextRun.Before(j.lastRun) {
		j.nextRun = j.nextRun.Add(periodDuration)
	}

	return nil
}

// NextScheduledTime returns the time of when this job is to run next
func (j *Job) NextScheduledTime() time.Time {
	return j.nextRun
}

// set the job's unit with seconds,minutes,hours...
func (j *Job) mustInterval(i uint64) error {
	if j.interval != i {
		return fmt.Errorf("interval must be %d", i)
	}
	return nil
}

// From schedules the next run of the job
func (j *Job) From(t *time.Time) *Job {
	j.nextRun = *t
	return j
}

// setUnit sets unit type
func (j *Job) setUnit(unit timeUnit) *Job {
	j.unit = unit
	return j
}

// Seconds set the unit with seconds
func (j *Job) Seconds() *Job {
	return j.setUnit(seconds)
}

// Minutes set the unit with minute
func (j *Job) Minutes() *Job {
	return j.setUnit(minutes)
}

// Hours set the unit with hours
func (j *Job) Hours() *Job {
	return j.setUnit(hours)
}

// Days set the job's unit with days
func (j *Job) Days() *Job {
	return j.setUnit(days)
}

// Weeks sets the units as weeks
func (j *Job) Weeks() *Job {
	return j.setUnit(weeks)
}

// Second sets the unit with second
func (j *Job) Second() *Job {
	j.mustInterval(1)
	return j.Seconds()
}

// Minute sets the unit  with minute, which interval is 1
func (j *Job) Minute() *Job {
	j.mustInterval(1)
	return j.Minutes()
}

// Hour sets the unit with hour, which interval is 1
func (j *Job) Hour() *Job {
	j.mustInterval(1)
	return j.Hours()
}

// Day sets the job's unit with day, which interval is 1
func (j *Job) Day() *Job {
	j.mustInterval(1)
	return j.Days()
}

// Week sets the job's unit with week, which interval is 1
func (j *Job) Week() *Job {
	j.mustInterval(1)
	return j.Weeks()
}

// Weekday start job on specific Weekday
func (j *Job) Weekday(startDay time.Weekday) *Job {
	j.mustInterval(1)
	j.startDay = startDay
	return j.Weeks()
}

// GetWeekday returns which day of the week the job will run on
// This should only be used when .Weekday(...) was called on the job.
func (j *Job) GetWeekday() time.Weekday {
	return j.startDay
}

// Monday set the start day with Monday
// - s.Every(1).Monday().Do(task)
func (j *Job) Monday() (job *Job) {
	return j.Weekday(time.Monday)
}

// Tuesday sets the job start day Tuesday
func (j *Job) Tuesday() *Job {
	return j.Weekday(time.Tuesday)
}

// Wednesday sets the job start day Wednesday
func (j *Job) Wednesday() *Job {
	return j.Weekday(time.Wednesday)
}

// Thursday sets the job start day Thursday
func (j *Job) Thursday() *Job {
	return j.Weekday(time.Thursday)
}

// Friday sets the job start day Friday
func (j *Job) Friday() *Job {
	return j.Weekday(time.Friday)
}

// Saturday sets the job start day Saturday
func (j *Job) Saturday() *Job {
	return j.Weekday(time.Saturday)
}

// Sunday sets the job start day Sunday
func (j *Job) Sunday() *Job {
	return j.Weekday(time.Sunday)
}

// Lock prevents job to run from multiple instances of gocron
func (j *Job) Lock() *Job {
	j.lock = true
	return j
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"golang.org/x/net/context"

	pubsub "google.golang.org/api/pubsub/v1"
)

type (
	CommandConfig struct {
		Template []string            `json:"-"`
		Options  map[string][]string `json:"options,omitempty"`
		Dryrun   bool                `json:"dryrun,omitempty"`
	}

	Job struct {
		config *CommandConfig
		// https://godoc.org/google.golang.org/genproto/googleapis/pubsub/v1#ReceivedMessage
		message      *pubsub.ReceivedMessage
		notification *ProgressNotification
		storage      Storage

		// These are set at at setupWorkspace
		workspace     string
		downloads_dir string
		uploads_dir   string

		// These are set at setupDownloadFiles
		downloadFileMap     map[string]string
		remoteDownloadFiles interface{}
		localDownloadFiles  interface{}
	}
)

func (job *Job) run(ctx context.Context) error {
	job.notification.notify(PROCESSING, job.message.Message.MessageId, "info")
	err := job.setupWorkspace(ctx, func() error {
		err := job.withNotify(PREPARING, job.setupDownloadFiles)()
		if err != nil {
			return err
		}

		err = job.withNotify(DOWNLOADING, job.downloadFiles)()
		if err != nil {
			return err
		}

		err = job.withNotify(EXECUTING, job.execute)()
		if err != nil {
			return err
		}

		err = job.withNotify(UPLOADING, job.uploadFiles)()
		if err != nil {
			return err
		}

		return nil
	})
	job.notification.notify(CLEANUP, job.message.Message.MessageId, "info")
	return err
}

func (job *Job) withNotify(progress int, f func() error) func() error {
	msg_id := job.message.Message.MessageId
	return func() error {
		job.notification.notify(progress, msg_id, "info")
		err := f()
		if err != nil {
			job.notification.notify(progress+2, msg_id, "error")
			return err
		}
		job.notification.notify(progress+1, msg_id, "info")
		return nil
	}
}

func (job *Job) setupWorkspace(ctx context.Context, f func() error) error {
	dir, err := ioutil.TempDir("", "workspace")
	if err != nil {
		log.Fatal(err)
		return err
	}
	defer os.RemoveAll(dir) // clean up

	subdirs := []string{
		filepath.Join(dir, "downloads"),
		filepath.Join(dir, "uploads"),
	}
	for _, subdir := range subdirs {
		err := os.MkdirAll(subdir, 0700)
		if err != nil {
			return err
		}
	}
	job.workspace = dir
	job.downloads_dir = subdirs[0]
	job.uploads_dir = subdirs[1]
	return f()
}

func (job *Job) setupDownloadFiles() error {
	job.downloadFileMap = map[string]string{}
	job.remoteDownloadFiles = job.parseJson(job.message.Message.Attributes["download_files"])
	objects := job.flatten(job.remoteDownloadFiles)
	remoteUrls := []string{}
	for _, obj := range objects {
		switch obj.(type) {
		case string:
			remoteUrls = append(remoteUrls, obj.(string))
		default:
			log.Printf("Invalid download file URL: %v [%T]", obj, obj)
		}
	}
	for _, remote_url := range remoteUrls {
		url, err := url.Parse(remote_url)
		if err != nil {
			log.Fatalf("Invalid URL: %v because of %v\n", remote_url, err)
			return err
		}
		urlstr := fmt.Sprintf("gs://%v%v", url.Host, url.Path)
		destPath := filepath.Join(job.downloads_dir, url.Host, url.Path)
		job.downloadFileMap[urlstr] = destPath
	}
	job.localDownloadFiles = job.copyWithFileMap(job.remoteDownloadFiles)
	return nil
}

func (job *Job) copyWithFileMap(obj interface{}) interface{} {
	switch obj.(type) {
	case map[string]interface{}:
		result := map[string]interface{}{}
		for k, v := range obj.(map[string]interface{}) {
			result[k] = job.copyWithFileMap(v)
		}
		return result
	case []interface{}:
		result := []interface{}{}
		for _, v := range obj.([]interface{}) {
			result = append(result, job.copyWithFileMap(v))
		}
		return result
	case string:
		return job.downloadFileMap[obj.(string)]
	default:
		return obj
	}
}

func (job *Job) buildVariable() *Variable {
	return &Variable{
		data: map[string]interface{}{
			"workspace":             job.workspace,
			"downloads_dir":         job.downloads_dir,
			"uploads_dir":           job.uploads_dir,
			"download_files":        job.localDownloadFiles,
			"local_download_files":  job.localDownloadFiles,
			"remote_download_files": job.remoteDownloadFiles,
			"attrs":                 job.message.Message.Attributes,
			"attributes":            job.message.Message.Attributes,
			"data":                  job.message.Message.Data,
		},
	}
}

func (job *Job) build() (*exec.Cmd, error) {
	v := job.buildVariable()

	values, err := job.extract(v, job.config.Template)
	if err != nil {
		return nil, err
	}
	if len(job.config.Options) > 0 {
		key := strings.Join(values, " ")
		t := job.config.Options[key]
		if t == nil {
			t = job.config.Options["default"]
		}
		if t != nil {
			values, err = job.extract(v, t)
			if err != nil {
				return nil, err
			}
		}
	}
	cmd := exec.Command(values[0], values[1:]...)
	return cmd, nil
}

func (job *Job) extract(v *Variable, values []string) ([]string, error) {
	result := []string{}
	for _, src := range values {
		extracted, err := v.expand(src)
		if err != nil {
			return nil, err
		}
		vals := strings.Split(extracted, v.separator)
		for _, val := range vals {
			result = append(result, val)
		}
	}
	return result, nil
}

func (job *Job) downloadFiles() error {
	for remoteURL, destPath := range job.downloadFileMap {
		url, err := url.Parse(remoteURL)
		if err != nil {
			log.Fatalf("Invalid URL: %v because of %v\n", remoteURL, err)
			return err
		}

		dir := path.Dir(destPath)
		err = os.MkdirAll(dir, 0700)
		if err != nil {
			return err
		}

		err = job.storage.Download(url.Host, url.Path[1:], destPath)
		if err != nil {
			return err
		}
	}
	return nil
}

func (job *Job) execute() error {
	cmd, err := job.build()
	if err != nil {
		log.Printf("Command build Error template: %v msg: %v cause of %v\n", job.config.Template, job.message, err)
		return err
	}
	err = cmd.Run()
	if err != nil {
		log.Printf("Command Error: cmd: %v cause of %v\n", cmd, err)
		// return err // Don't return this err
	}
	return nil
}

func (job *Job) uploadFiles() error {
	localPaths, err := job.listFiles(job.uploads_dir)
	if err != nil {
		return err
	}
	for _, localPath := range localPaths {
		relPath, err := filepath.Rel(job.uploads_dir, localPath)
		if err != nil {
			log.Fatalf("Error getting relative path of %v: %v\n", localPath, err)
			return err
		}
		sep := string([]rune{os.PathSeparator})
		parts := strings.Split(relPath, sep)
		bucket := parts[0]
		object := strings.Join(parts[1:], sep)
		err = job.storage.Upload(bucket, object, localPath)
		if err != nil {
			log.Fatalf("Error uploading %v to gs://%v/%v: %v\n", localPath, bucket, object, err)
			return err
		}
	}
	return nil
}

func (job *Job) listFiles(dir string) ([]string, error) {
	result := []string{}
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			result = append(result, path)
		}
		return nil
	})
	if err != nil {
		log.Fatalf("Error listing upload files: %v\n", err)
		return nil, err
	}
	return result, nil
}

func (job *Job) parseJson(str string) interface{} {
	matched, err := regexp.MatchString(`\A\[.*\]\z|\A\{.*\}\z|`, str)
	if err != nil {
		return str
	}
	if !matched {
		return str
	}
	var dest interface{}
	err = json.Unmarshal([]byte(str), &dest)
	if err != nil {
		return str
	}
	return dest
}

func (job *Job) flatten(obj interface{}) []interface{} {
	// Support only unmarshalled object from JSON
	// See https://golang.org/pkg/encoding/json/#Unmarshal also
	switch obj.(type) {
	case []interface{}:
		res := []interface{}{}
		for _, i := range obj.([]interface{}) {
			switch i.(type) {
			case bool, float64, string, nil:
				res = append(res, i)
			default:
				for _, j := range job.flatten(i) {
					res = append(res, j)
				}
			}
		}
		return res
	case map[string]interface{}:
		values := []interface{}{}
		for _, val := range obj.(map[string]interface{}) {
			values = append(values, val)
		}
		return job.flatten(values)
	default:
		return []interface{}{obj}
	}
}
<commit_msg>:bug: Change error level<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"golang.org/x/net/context"

	pubsub "google.golang.org/api/pubsub/v1"
)

type (
	CommandConfig struct {
		Template []string            `json:"-"`
		Options  map[string][]string `json:"options,omitempty"`
		Dryrun   bool                `json:"dryrun,omitempty"`
	}

	Job struct {
		config *CommandConfig
		// https://godoc.org/google.golang.org/genproto/googleapis/pubsub/v1#ReceivedMessage
		message      *pubsub.ReceivedMessage
		notification *ProgressNotification
		storage      Storage

		// These are set at at setupWorkspace
		workspace     string
		downloads_dir string
		uploads_dir   string

		// These are set at setupDownloadFiles
		downloadFileMap     map[string]string
		remoteDownloadFiles interface{}
		localDownloadFiles  interface{}
	}
)

func (job *Job) run(ctx context.Context) error {
	job.notification.notify(PROCESSING, job.message.Message.MessageId, "info")
	err := job.setupWorkspace(ctx, func() error {
		err := job.withNotify(PREPARING, job.setupDownloadFiles)()
		if err != nil {
			return err
		}

		err = job.withNotify(DOWNLOADING, job.downloadFiles)()
		if err != nil {
			return err
		}

		err = job.withNotify(EXECUTING, job.execute)()
		if err != nil {
			return err
		}

		err = job.withNotify(UPLOADING, job.uploadFiles)()
		if err != nil {
			return err
		}

		return nil
	})
	job.notification.notify(CLEANUP, job.message.Message.MessageId, "info")
	return err
}

func (job *Job) withNotify(progress int, f func() error) func() error {
	msg_id := job.message.Message.MessageId
	return func() error {
		job.notification.notify(progress, msg_id, "info")
		err := f()
		if err != nil {
			job.notification.notify(progress+2, msg_id, "error")
			return err
		}
		job.notification.notify(progress+1, msg_id, "info")
		return nil
	}
}

func (job *Job) setupWorkspace(ctx context.Context, f func() error) error {
	dir, err := ioutil.TempDir("", "workspace")
	if err != nil {
		log.Fatal(err)
		return err
	}
	defer os.RemoveAll(dir) // clean up

	subdirs := []string{
		filepath.Join(dir, "downloads"),
		filepath.Join(dir, "uploads"),
	}
	for _, subdir := range subdirs {
		err := os.MkdirAll(subdir, 0700)
		if err != nil {
			return err
		}
	}
	job.workspace = dir
	job.downloads_dir = subdirs[0]
	job.uploads_dir = subdirs[1]
	return f()
}

func (job *Job) setupDownloadFiles() error {
	job.downloadFileMap = map[string]string{}
	job.remoteDownloadFiles = job.parseJson(job.message.Message.Attributes["download_files"])
	objects := job.flatten(job.remoteDownloadFiles)
	remoteUrls := []string{}
	for _, obj := range objects {
		switch obj.(type) {
		case string:
			remoteUrls = append(remoteUrls, obj.(string))
		default:
			log.Printf("Invalid download file URL: %v [%T]", obj, obj)
		}
	}
	for _, remote_url := range remoteUrls {
		url, err := url.Parse(remote_url)
		if err != nil {
			log.Fatalf("Invalid URL: %v because of %v\n", remote_url, err)
			return err
		}
		urlstr := fmt.Sprintf("gs://%v%v", url.Host, url.Path)
		destPath := filepath.Join(job.downloads_dir, url.Host, url.Path)
		job.downloadFileMap[urlstr] = destPath
	}
	job.localDownloadFiles = job.copyWithFileMap(job.remoteDownloadFiles)
	return nil
}

func (job *Job) copyWithFileMap(obj interface{}) interface{} {
	switch obj.(type) {
	case map[string]interface{}:
		result := map[string]interface{}{}
		for k, v := range obj.(map[string]interface{}) {
			result[k] = job.copyWithFileMap(v)
		}
		return result
	case []interface{}:
		result := []interface{}{}
		for _, v := range obj.([]interface{}) {
			result = append(result, job.copyWithFileMap(v))
		}
		return result
	case string:
		return job.downloadFileMap[obj.(string)]
	default:
		return obj
	}
}

func (job *Job) buildVariable() *Variable {
	return &Variable{
		data: map[string]interface{}{
			"workspace":             job.workspace,
			"downloads_dir":         job.downloads_dir,
			"uploads_dir":           job.uploads_dir,
			"download_files":        job.localDownloadFiles,
			"local_download_files":  job.localDownloadFiles,
			"remote_download_files": job.remoteDownloadFiles,
			"attrs":                 job.message.Message.Attributes,
			"attributes":            job.message.Message.Attributes,
			"data":                  job.message.Message.Data,
		},
	}
}

func (job *Job) build() (*exec.Cmd, error) {
	v := job.buildVariable()

	values, err := job.extract(v, job.config.Template)
	if err != nil {
		return nil, err
	}
	if len(job.config.Options) > 0 {
		key := strings.Join(values, " ")
		t := job.config.Options[key]
		if t == nil {
			t = job.config.Options["default"]
		}
		if t != nil {
			values, err = job.extract(v, t)
			if err != nil {
				return nil, err
			}
		}
	}
	cmd := exec.Command(values[0], values[1:]...)
	return cmd, nil
}

func (job *Job) extract(v *Variable, values []string) ([]string, error) {
	result := []string{}
	for _, src := range values {
		extracted, err := v.expand(src)
		if err != nil {
			return nil, err
		}
		vals := strings.Split(extracted, v.separator)
		for _, val := range vals {
			result = append(result, val)
		}
	}
	return result, nil
}

func (job *Job) downloadFiles() error {
	for remoteURL, destPath := range job.downloadFileMap {
		url, err := url.Parse(remoteURL)
		if err != nil {
			log.Fatalf("Invalid URL: %v because of %v\n", remoteURL, err)
			return err
		}

		dir := path.Dir(destPath)
		err = os.MkdirAll(dir, 0700)
		if err != nil {
			return err
		}

		err = job.storage.Download(url.Host, url.Path[1:], destPath)
		if err != nil {
			return err
		}
	}
	return nil
}

func (job *Job) execute() error {
	cmd, err := job.build()
	if err != nil {
		log.Fatalf("Command build Error template: %v msg: %v cause of %v\n", job.config.Template, job.message, err)
		return err
	}
	err = cmd.Run()
	if err != nil {
		log.Printf("Command Error: cmd: %v cause of %v\n", cmd, err)
		// return err // Don't return this err
	}
	return nil
}

func (job *Job) uploadFiles() error {
	localPaths, err := job.listFiles(job.uploads_dir)
	if err != nil {
		return err
	}
	for _, localPath := range localPaths {
		relPath, err := filepath.Rel(job.uploads_dir, localPath)
		if err != nil {
			log.Fatalf("Error getting relative path of %v: %v\n", localPath, err)
			return err
		}
		sep := string([]rune{os.PathSeparator})
		parts := strings.Split(relPath, sep)
		bucket := parts[0]
		object := strings.Join(parts[1:], sep)
		err = job.storage.Upload(bucket, object, localPath)
		if err != nil {
			log.Fatalf("Error uploading %v to gs://%v/%v: %v\n", localPath, bucket, object, err)
			return err
		}
	}
	return nil
}

func (job *Job) listFiles(dir string) ([]string, error) {
	result := []string{}
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			result = append(result, path)
		}
		return nil
	})
	if err != nil {
		log.Fatalf("Error listing upload files: %v\n", err)
		return nil, err
	}
	return result, nil
}

func (job *Job) parseJson(str string) interface{} {
	matched, err := regexp.MatchString(`\A\[.*\]\z|\A\{.*\}\z|`, str)
	if err != nil {
		return str
	}
	if !matched {
		return str
	}
	var dest interface{}
	err = json.Unmarshal([]byte(str), &dest)
	if err != nil {
		return str
	}
	return dest
}

func (job *Job) flatten(obj interface{}) []interface{} {
	// Support only unmarshalled object from JSON
	// See https://golang.org/pkg/encoding/json/#Unmarshal also
	switch obj.(type) {
	case []interface{}:
		res := []interface{}{}
		for _, i := range obj.([]interface{}) {
			switch i.(type) {
			case bool, float64, string, nil:
				res = append(res, i)
			default:
				for _, j := range job.flatten(i) {
					res = append(res, j)
				}
			}
		}
		return res
	case map[string]interface{}:
		values := []interface{}{}
		for _, val := range obj.(map[string]interface{}) {
			values = append(values, val)
		}
		return job.flatten(values)
	default:
		return []interface{}{obj}
	}
}
<|endoftext|>


<commit_before>package jwt

import (
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"time"

	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3iface"
	"github.com/pmylund/go-cache"
	"github.com/square/go-jose"
	"github.com/stripe/aws-go/aws"
)

// JWKSet represents JWK Set.
// See https://tools.ietf.org/html/rfc7517#section-5
type JWKSet struct {
	Keys []*jose.JsonWebKey `json:"keys"`
}

// JWKSetResponse represents a response of JWK Set.
// This contains a TTL (Time to Live) for caching purpose.
type JWKSetResponse struct {
	Keys []*jose.JsonWebKey

	TTL time.Duration // This would be used as TTL for caching.
}

// JWKsFetcher is an interface that represents JWKs fetcher.
type JWKsFetcher interface {
	// FetchJWKs retrieves JWKSet from path.
	FetchJWKs(path string) (*JWKSetResponse, error)
}

// JWKsHTTPFetcher fetches JWKs via HTTP.
type JWKsHTTPFetcher struct {
	Client *http.Client
}

// FetchJWKs implements JWKsFetcher interface by using http.Client.
// FetchJWKs tries to retrieve JWKSet from uri.
func (f *JWKsHTTPFetcher) FetchJWKs(uri string) (*JWKSetResponse, error) {
	resp, err := f.Client.Get(uri)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	jwks, err := DecodeJWKSet(resp.Body)
	return &JWKSetResponse{
		Keys: jwks,
	}, err
}

// JWKsS3Fetcher fetches JWKs via S3.
type JWKsS3Fetcher struct {
	S3Svc s3iface.S3API
}

// FetchJWKs implements JWKsS3Fetcher by using S3. It tries to retrieve an S3 object from path.
// path must be in s3://<bucket>/<key>.
func (f *JWKsS3Fetcher) FetchJWKs(path string) (*JWKSetResponse, error) {
	s3url, err := url.Parse(path)
	if err != nil {
		return nil, err
	}

	params := &s3.GetObjectInput{
		Bucket: aws.String(s3url.Host),
		Key:    aws.String(s3url.Path),
	}
	resp, err := f.S3Svc.GetObject(params)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	jwks, err := DecodeJWKSet(resp.Body)
	return &JWKSetResponse{
		Keys: jwks,
	}, err
}

// JWKsCacher fetches JWKs via Cache if available.
type JWKsCacher struct {
	fetcher JWKsFetcher
	cache   *cache.Cache

	defaultExpiration time.Duration
	cleanupInterval   time.Duration
}

// NewCacher returns JWKsCacher with initializing cache store.
func NewCacher(defaultExpiration, cleanupInterval time.Duration, f JWKsFetcher) *JWKsCacher {
	c := cache.New(defaultExpiration, cleanupInterval)
	return &JWKsCacher{
		fetcher: f,
		cache:   c,

		defaultExpiration: defaultExpiration,
		cleanupInterval:   cleanupInterval,
	}
}

// FetchJWKs tries to retrieve JWKs from Cache. If the cache is not available,
// it will call Fetcher.FetchJWKs and cache the result for future request.
func (c *JWKsCacher) FetchJWKs(cacheKey string) (*JWKSetResponse, error) {
	if keys, found := c.cache.Get(cacheKey); found {
		return &JWKSetResponse{Keys: keys.([]*jose.JsonWebKey)}, nil
	}
	jwksresp, err := c.fetcher.FetchJWKs(cacheKey)
	if err != nil {
		return nil, err
	}

	ttl := jwksresp.TTL

	// If TTL is larger than cleanupInterval, we should subtract cleanupInterval from TTL to
	// make sure that the latest jwks is obtained.
	if ttl > c.cleanupInterval {
		ttl -= c.cleanupInterval
	}

	c.cache.Set(cacheKey, jwksresp.Keys, ttl)
	return jwksresp, nil
}

// DecodeJWKSet decodes the data with reading from r into JWKs.
func DecodeJWKSet(r io.Reader) ([]*jose.JsonWebKey, error) {
	keyset := JWKSet{}
	if err := json.NewDecoder(r).Decode(&keyset); err != nil && err != io.EOF {
		return nil, err
	}

	return keyset.Keys, nil
}
<commit_msg>jwk: Fix import path<commit_after>package jwt

import (
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3iface"
	"github.com/pmylund/go-cache"
	"github.com/square/go-jose"
)

// JWKSet represents JWK Set.
// See https://tools.ietf.org/html/rfc7517#section-5
type JWKSet struct {
	Keys []*jose.JsonWebKey `json:"keys"`
}

// JWKSetResponse represents a response of JWK Set.
// This contains a TTL (Time to Live) for caching purpose.
type JWKSetResponse struct {
	Keys []*jose.JsonWebKey

	TTL time.Duration // This would be used as TTL for caching.
}

// JWKsFetcher is an interface that represents JWKs fetcher.
type JWKsFetcher interface {
	// FetchJWKs retrieves JWKSet from path.
	FetchJWKs(path string) (*JWKSetResponse, error)
}

// JWKsHTTPFetcher fetches JWKs via HTTP.
type JWKsHTTPFetcher struct {
	Client *http.Client
}

// FetchJWKs implements JWKsFetcher interface by using http.Client.
// FetchJWKs tries to retrieve JWKSet from uri.
func (f *JWKsHTTPFetcher) FetchJWKs(uri string) (*JWKSetResponse, error) {
	resp, err := f.Client.Get(uri)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	jwks, err := DecodeJWKSet(resp.Body)
	return &JWKSetResponse{
		Keys: jwks,
	}, err
}

// JWKsS3Fetcher fetches JWKs via S3.
type JWKsS3Fetcher struct {
	S3Svc s3iface.S3API
}

// FetchJWKs implements JWKsS3Fetcher by using S3. It tries to retrieve an S3 object from path.
// path must be in s3://<bucket>/<key>.
func (f *JWKsS3Fetcher) FetchJWKs(path string) (*JWKSetResponse, error) {
	s3url, err := url.Parse(path)
	if err != nil {
		return nil, err
	}

	params := &s3.GetObjectInput{
		Bucket: aws.String(s3url.Host),
		Key:    aws.String(s3url.Path),
	}
	resp, err := f.S3Svc.GetObject(params)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	jwks, err := DecodeJWKSet(resp.Body)
	return &JWKSetResponse{
		Keys: jwks,
	}, err
}

// JWKsCacher fetches JWKs via Cache if available.
type JWKsCacher struct {
	fetcher JWKsFetcher
	cache   *cache.Cache

	defaultExpiration time.Duration
	cleanupInterval   time.Duration
}

// NewCacher returns JWKsCacher with initializing cache store.
func NewCacher(defaultExpiration, cleanupInterval time.Duration, f JWKsFetcher) *JWKsCacher {
	c := cache.New(defaultExpiration, cleanupInterval)
	return &JWKsCacher{
		fetcher: f,
		cache:   c,

		defaultExpiration: defaultExpiration,
		cleanupInterval:   cleanupInterval,
	}
}

// FetchJWKs tries to retrieve JWKs from Cache. If the cache is not available,
// it will call Fetcher.FetchJWKs and cache the result for future request.
func (c *JWKsCacher) FetchJWKs(cacheKey string) (*JWKSetResponse, error) {
	if keys, found := c.cache.Get(cacheKey); found {
		return &JWKSetResponse{Keys: keys.([]*jose.JsonWebKey)}, nil
	}
	jwksresp, err := c.fetcher.FetchJWKs(cacheKey)
	if err != nil {
		return nil, err
	}

	ttl := jwksresp.TTL

	// If TTL is larger than cleanupInterval, we should subtract cleanupInterval from TTL to
	// make sure that the latest jwks is obtained.
	if ttl > c.cleanupInterval {
		ttl -= c.cleanupInterval
	}

	c.cache.Set(cacheKey, jwksresp.Keys, ttl)
	return jwksresp, nil
}

// DecodeJWKSet decodes the data with reading from r into JWKs.
func DecodeJWKSet(r io.Reader) ([]*jose.JsonWebKey, error) {
	keyset := JWKSet{}
	if err := json.NewDecoder(r).Decode(&keyset); err != nil && err != io.EOF {
		return nil, err
	}

	return keyset.Keys, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"github.com/bgentry/heroku-go"
	"log"
	"net/http"
	"os"
)

var (
	lines  int
	source string
	dyno   string
)

var cmdLog = &Command{
	Run:      runLog,
	Usage:    "log [-n <lines>] [-s <source>] [-d <dyno>]",
	NeedsApp: true,
	Category: "app",
	Short:    "stream app log lines",
	Long: `
Log prints the streaming application log.

Options:

    -n <N>       print at most N log lines
    -s <source>  filter log source
    -d <dyno>    filter dyno or process type

Examples:

    $ hk log
    2013-10-17T00:17:35.066089+00:00 app[web.1]: Completed 302 Found in 0ms
    2013-10-17T00:17:35.079095+00:00 heroku[router]: at=info method=GET path=/ host=www.heroku.com fwd="1.2.3.4" dyno=web.1 connect=1ms service=6ms status=302 bytes=95
    2013-10-17T00:17:35.505389+00:00 heroku[nginx]: 1.2.3.4 - - [17/Oct/2013:00:17:35 +0000] "GET / HTTP/1.1" 301 5 "-" "Amazon Route 53 Health Check Service" www.heroku.com
		...

    $ hk log -n 2 -s app -d web
		2013-10-17T00:17:34.288521+00:00 app[web.1]: Completed 200 OK in 10ms (Views: 10.0ms)
    2013-10-17T00:17:33.918946+00:00 heroku[web.5]: Started GET "/" for 1.2.3.4 at 2013-10-17 00:17:32 +0000
		2013-10-17T00:17:34.667654+00:00 heroku[router]: at=info method=GET path=/ host=www.heroku.com fwd="1.2.3.4" dyno=web.5 connect=3ms service=8ms status=301 bytes=0
    2013-10-17T00:17:35.079095+00:00 heroku[router]: at=info method=GET path=/ host=www.heroku.com fwd="1.2.3.4" dyno=web.1 connect=1ms service=6ms status=302 bytes=95
		...

    $ hk log -d web.5
    2013-10-17T00:17:33.918946+00:00 app[web.5]: Started GET "/" for 1.2.3.4 at 2013-10-17 00:17:32 +0000
    2013-10-17T00:17:33.918658+00:00 app[web.5]: Processing by PagesController#root as HTML
		...
`,
}

func init() {
	cmdLog.Flag.IntVar(&lines, "n", -1, "max number of log lines to request")
	cmdLog.Flag.StringVar(&source, "s", "", "only display logs from the given source")
	cmdLog.Flag.StringVar(&dyno, "d", "", "only display logs from the given dyno or process type")
}

func runLog(cmd *Command, args []string) {
	opts := heroku.LogSessionCreateOpts{}
	if dyno != "" {
		opts.Dyno = &dyno
	}
	if source != "" {
		opts.Source = &source
	}

	if lines != -1 {
		opts.Lines = &lines
	} else {
		tailopt := true
		lineopt := 10
		opts.Tail = &tailopt
		opts.Lines = &lineopt
	}

	session, err := client.LogSessionCreate(mustApp(), opts)
	if err != nil {
		log.Fatal(err)
	}
	resp, err := http.Get(session.LogplexURL)
	if err != nil {
		log.Fatal(err)
	}
	if resp.StatusCode/100 != 2 {
		if resp.StatusCode/100 == 4 {
			log.Fatal("Unauthorized")
		} else {
			log.Fatal("Unexpected error: " + resp.Status)
		}
	}

	scanner := bufio.NewScanner(resp.Body)
	scanner.Split(bufio.ScanLines)

	for scanner.Scan() {
		if _, err = fmt.Fprintln(os.Stdout, scanner.Text()); err != nil {
			log.Fatal(err)
		}
	}

	resp.Body.Close()
}
<commit_msg>Extract local variable<commit_after>package main

import (
	"bufio"
	"fmt"
	"github.com/bgentry/heroku-go"
	"io"
	"log"
	"net/http"
	"os"
)

var (
	lines  int
	source string
	dyno   string
)

var cmdLog = &Command{
	Run:      runLog,
	Usage:    "log [-n <lines>] [-s <source>] [-d <dyno>]",
	NeedsApp: true,
	Category: "app",
	Short:    "stream app log lines",
	Long: `
Log prints the streaming application log.

Options:

    -n <N>       print at most N log lines
    -s <source>  filter log source
    -d <dyno>    filter dyno or process type

Examples:

    $ hk log
    2013-10-17T00:17:35.066089+00:00 app[web.1]: Completed 302 Found in 0ms
    2013-10-17T00:17:35.079095+00:00 heroku[router]: at=info method=GET path=/ host=www.heroku.com fwd="1.2.3.4" dyno=web.1 connect=1ms service=6ms status=302 bytes=95
    2013-10-17T00:17:35.505389+00:00 heroku[nginx]: 1.2.3.4 - - [17/Oct/2013:00:17:35 +0000] "GET / HTTP/1.1" 301 5 "-" "Amazon Route 53 Health Check Service" www.heroku.com
		...

    $ hk log -n 2 -s app -d web
		2013-10-17T00:17:34.288521+00:00 app[web.1]: Completed 200 OK in 10ms (Views: 10.0ms)
    2013-10-17T00:17:33.918946+00:00 heroku[web.5]: Started GET "/" for 1.2.3.4 at 2013-10-17 00:17:32 +0000
		2013-10-17T00:17:34.667654+00:00 heroku[router]: at=info method=GET path=/ host=www.heroku.com fwd="1.2.3.4" dyno=web.5 connect=3ms service=8ms status=301 bytes=0
    2013-10-17T00:17:35.079095+00:00 heroku[router]: at=info method=GET path=/ host=www.heroku.com fwd="1.2.3.4" dyno=web.1 connect=1ms service=6ms status=302 bytes=95
		...

    $ hk log -d web.5
    2013-10-17T00:17:33.918946+00:00 app[web.5]: Started GET "/" for 1.2.3.4 at 2013-10-17 00:17:32 +0000
    2013-10-17T00:17:33.918658+00:00 app[web.5]: Processing by PagesController#root as HTML
		...
`,
}

func init() {
	cmdLog.Flag.IntVar(&lines, "n", -1, "max number of log lines to request")
	cmdLog.Flag.StringVar(&source, "s", "", "only display logs from the given source")
	cmdLog.Flag.StringVar(&dyno, "d", "", "only display logs from the given dyno or process type")
}

func runLog(cmd *Command, args []string) {
	opts := heroku.LogSessionCreateOpts{}
	if dyno != "" {
		opts.Dyno = &dyno
	}
	if source != "" {
		opts.Source = &source
	}

	if lines != -1 {
		opts.Lines = &lines
	} else {
		tailopt := true
		lineopt := 10
		opts.Tail = &tailopt
		opts.Lines = &lineopt
	}

	session, err := client.LogSessionCreate(mustApp(), opts)
	if err != nil {
		log.Fatal(err)
	}
	resp, err := http.Get(session.LogplexURL)
	if err != nil {
		log.Fatal(err)
	}
	if resp.StatusCode/100 != 2 {
		if resp.StatusCode/100 == 4 {
			log.Fatal("Unauthorized")
		} else {
			log.Fatal("Unexpected error: " + resp.Status)
		}
	}

	writer := io.Writer(os.Stdout)

	scanner := bufio.NewScanner(resp.Body)
	scanner.Split(bufio.ScanLines)

	for scanner.Scan() {
		if _, err = fmt.Fprintln(writer, scanner.Text()); err != nil {
			log.Fatal(err)
		}
	}

	resp.Body.Close()
}
<|endoftext|>


<commit_before>// Copyright (C) 2015 Thomas de Zeeuw.
//
// Licensed under the MIT license that can be found in the LICENSE file.

package logger

import (
	"fmt"
	"runtime"
	"time"
)

const (
	defaultStackSize        = 8192
	defaultEventChannelSize = 1024
	maxNWriteErrors         = 5
)

// EventWriter is the backend of the logger package. It takes events and writes
// them to a storage, this can for example be a file, a database or the
// terminal.
type EventWriter interface {
	// Write is called every time a Log operation is called. This call should
	// block until the event is written. Advised is to buffer the events. All
	// calls to Write are synchronous.
	//
	// If an error is returned the event is expected to NOT have been written. The
	// event will written again after the error handler is called. If the
	// EventWriter returns 5 errors in a row the EventWriter is consided to be bad
	// and will be removed from the list of event writers. HandlerError will be
	// called with ErrBadEventWriter if this happens.
	Write(Event) error

	// HandleError is called every time Write returns an error.
	HandleError(error)

	// Close is called on the EventWriter once Close() (on the package) is called.
	Close() error
}

var (
	eventChannel       = make(chan Event, defaultEventChannelSize)
	eventChannelClosed = make(chan struct{}, 1) // Can't block.
	eventWriters       []EventWriter
	started            bool
)

// Start starts the logger package and enables writing to the given
// EventWriters.
func Start(ews ...EventWriter) {
	if started {
		panic("logger: can only Start once")
	}
	started = true

	if len(ews) < 1 {
		panic("logger: need atleast a single EventWriter to write to")
	}

	eventWriters = ews
	go writeEvents()
}

var ErrBadEventWriter = fmt.Errorf("EventWriter is bad, more then %d faulty writes, EventWriter will be dropped", maxNWriteErrors)

// Needs to be run in it's own goroutine, it blocks until eventChannel is
// closed. After eventChannel is closed it sends a signal to eventChannelClosed.
func writeEvents() {
	l := len(eventWriters)

	// Create sub channels for each EventWriter and start each EventWriter.
	closeSubChannel := make(chan struct{}, l)
	var eventSubChannels = make([]chan Event, l)
	for i, ew := range eventWriters {
		eventSubChannels[i] = make(chan Event, defaultEventChannelSize)
		go startEventWriter(ew, eventSubChannels[i], closeSubChannel)
	}

	// Fanout the events to all the sub channels.
	for event := range eventChannel {
		for _, eventSubChannel := range eventSubChannels {
			eventSubChannel <- event
		}
	}

	// Close each sub channel and wait for their close signal.
	for _, eventSubChannel := range eventSubChannels {
		close(eventSubChannel)
	}

	// Wait for all the EventWriters to be closed.
	for n := 0; n < l; n++ {
		<-closeSubChannel
	}

	eventChannelClosed <- struct{}{}
}

// Must run in it's own go routine, it blocks until the events channel is
// closed. After the events channels is closed it sends a signal to the closed
// channel.
func startEventWriter(ew EventWriter, events <-chan Event, closed chan<- struct{}) {
	var badEventWriter = false

	for event := range events {
		if badEventWriter {
			// Simply drain the channel.
			// todo: improve this, don't send to the channel anymore if the writer is
			// bad.
			continue
		}

		if err := writeEvent(ew, event); err != nil {
			// At this point the EventWriter is bad and we won't write to it anymore.
			ew.HandleError(err)
			badEventWriter = true
		}
	}

	closed <- struct{}{}
}

// WriteEvent tries to write the event to the given EventWriter, it tries it up
// to maxNWriteErrors times. If EventWriter.Write returns an error it gets
// passed to the error handler of the EventWriter. This function either returns
// ErrBadEventWriter or nil as an error.
func writeEvent(ew EventWriter, event Event) error {
	for n := 1; n <= maxNWriteErrors; n++ {
		err := ew.Write(event)
		if err == nil {
			return nil
		}

		// Handle the error and try again.
		ew.HandleError(err)
	}

	return ErrBadEventWriter
}

// Close stops all the Log Operations from being usable, and they will panic if
// used after close is called. It also closes all EventWriters and returns the
// first returned error. The EventWriters are closed in the order they are
// passed to Start.
func Close() error {
	close(eventChannel)
	<-eventChannelClosed

	var err error
	for _, eventWriter := range eventWriters {
		er := eventWriter.Close()
		if er != nil && err == nil {
			err = er
		}
	}
	return err
}

// Subbed for testing.
var now = func() time.Time {
	return time.Now()
}

// Debug logs a debug message.
func Debug(tags Tags, msg string) {
	eventChannel <- Event{DebugEvent, now(), tags, msg, nil}
}

// Debugf is a formatted function of Debug.
func Debugf(tags Tags, format string, v ...interface{}) {
	Debug(tags, fmt.Sprintf(format, v...))
}

// Info logs an informational message.
func Info(tags Tags, msg string) {
	eventChannel <- Event{InfoEvent, now(), tags, msg, nil}
}

// Infof is a formatted function of Info.
func Infof(tags Tags, format string, v ...interface{}) {
	Info(tags, fmt.Sprintf(format, v...))
}

// Warn logs a warning message.
func Warn(tags Tags, msg string) {
	eventChannel <- Event{WarnEvent, now(), tags, msg, nil}
}

// Warnf is a formatted function of Warn.
func Warnf(tags Tags, format string, v ...interface{}) {
	Warn(tags, fmt.Sprintf(format, v...))
}

// Error logs an error message.
func Error(tags Tags, err error) {
	eventChannel <- Event{ErrorEvent, now(), tags, err.Error(), nil}
}

// Errorf is a formatted function of Error.
func Errorf(tags Tags, format string, v ...interface{}) {
	Error(tags, fmt.Errorf(format, v...))
}

// Fatal logs a recovered error which could have killed the application. Fatal
// adds a stack trace (type []byte) as Event.Data.
func Fatal(tags Tags, recv interface{}) {
	stackTrace := make([]byte, defaultStackSize)
	n := runtime.Stack(stackTrace, false)
	stackTrace = stackTrace[:n]

	msg := interfaceToString(recv)
	eventChannel <- Event{FatalEvent, now(), tags, msg, stackTrace}
}

// Thumbstone indicates a function is still used in production. When developing
// software it's possible to introduce dead code with updates and new features.
// If a function is being suspected of being dead (not used) in production, add
// a call to Thumbstone and check the production logs to see if you're right.
//
// The caller of the (possibly) dead function will be put in the message, using
// the following format:
//	Function functionName called by callerFunctionName, from file /path/to/file on line lineNumber
// For example:
//	Function myFunction called by main.main, from file /main.go on line 20
func Thumbstone(tags Tags, functionName string) {
	var msg string
	if pc, file, line, ok := runtime.Caller(2); ok {
		fn := runtime.FuncForPC(pc)
		msg = fmt.Sprintf("Function %s called by %s, from file %s on line %d",
			functionName, fn.Name(), file, line)
	} else {
		msg = "Function " + functionName + " called from unkown location"
	}

	eventChannel <- Event{ThumbEvent, now(), tags, msg, nil}
}

// Log logs a custom created event.
//
// Note: the timestamp doesn't need to be set, because it will be set by Log.
func Log(event Event) {
	event.Timestamp = now()
	eventChannel <- event
}

// interfaceToString converts a interface{} variable to a string.
func interfaceToString(value interface{}) string {
	switch v := value.(type) {
	case string:
		return v
	case fmt.Stringer:
		return v.String()
	case []byte:
		return string(v)
	case error:
		return v.Error()
	}
	return fmt.Sprintf("%v", value)
}
<commit_msg>Small cleanup in Start<commit_after>// Copyright (C) 2015 Thomas de Zeeuw.
//
// Licensed under the MIT license that can be found in the LICENSE file.

package logger

import (
	"fmt"
	"runtime"
	"time"
)

const (
	defaultStackSize        = 8192
	defaultEventChannelSize = 1024
	maxNWriteErrors         = 5
)

// EventWriter is the backend of the logger package. It takes events and writes
// them to a storage, this can for example be a file, a database or the
// terminal.
type EventWriter interface {
	// Write is called every time a Log operation is called. This call should
	// block until the event is written. Advised is to buffer the events. All
	// calls to Write are synchronous.
	//
	// If an error is returned the event is expected to NOT have been written. The
	// event will written again after the error handler is called. If the
	// EventWriter returns 5 errors in a row the EventWriter is consided to be bad
	// and will be removed from the list of event writers. HandlerError will be
	// called with ErrBadEventWriter if this happens.
	Write(Event) error

	// HandleError is called every time Write returns an error.
	HandleError(error)

	// Close is called on the EventWriter once Close() (on the package) is called.
	Close() error
}

var (
	eventChannel       = make(chan Event, defaultEventChannelSize)
	eventChannelClosed = make(chan struct{}, 1) // Can't block.
	eventWriters       []EventWriter
	started            bool
)

// Start starts the logger package and enables writing to the given
// EventWriters.
func Start(ews ...EventWriter) {
	if started {
		panic("logger: can only Start once")
	} else if len(ews) < 1 {
		panic("logger: need atleast a single EventWriter to write to")
	}

	started = true
	eventWriters = ews

	go writeEvents()
}

var ErrBadEventWriter = fmt.Errorf("EventWriter is bad, more then %d faulty writes, EventWriter will be dropped", maxNWriteErrors)

// Needs to be run in it's own goroutine, it blocks until eventChannel is
// closed. After eventChannel is closed it sends a signal to eventChannelClosed.
func writeEvents() {
	l := len(eventWriters)

	// Create sub channels for each EventWriter and start each EventWriter.
	closeSubChannel := make(chan struct{}, l)
	var eventSubChannels = make([]chan Event, l)
	for i, ew := range eventWriters {
		eventSubChannels[i] = make(chan Event, defaultEventChannelSize)
		go startEventWriter(ew, eventSubChannels[i], closeSubChannel)
	}

	// Fanout the events to all the sub channels.
	for event := range eventChannel {
		for _, eventSubChannel := range eventSubChannels {
			eventSubChannel <- event
		}
	}

	// Close each sub channel and wait for their close signal.
	for _, eventSubChannel := range eventSubChannels {
		close(eventSubChannel)
	}

	// Wait for all the EventWriters to be closed.
	for n := 0; n < l; n++ {
		<-closeSubChannel
	}

	eventChannelClosed <- struct{}{}
}

// Must run in it's own go routine, it blocks until the events channel is
// closed. After the events channels is closed it sends a signal to the closed
// channel.
func startEventWriter(ew EventWriter, events <-chan Event, closed chan<- struct{}) {
	var badEventWriter = false

	for event := range events {
		if badEventWriter {
			// Simply drain the channel.
			// todo: improve this, don't send to the channel anymore if the writer is
			// bad.
			continue
		}

		if err := writeEvent(ew, event); err != nil {
			// At this point the EventWriter is bad and we won't write to it anymore.
			ew.HandleError(err)
			badEventWriter = true
		}
	}

	closed <- struct{}{}
}

// WriteEvent tries to write the event to the given EventWriter, it tries it up
// to maxNWriteErrors times. If EventWriter.Write returns an error it gets
// passed to the error handler of the EventWriter. This function either returns
// ErrBadEventWriter or nil as an error.
func writeEvent(ew EventWriter, event Event) error {
	for n := 1; n <= maxNWriteErrors; n++ {
		err := ew.Write(event)
		if err == nil {
			return nil
		}

		// Handle the error and try again.
		ew.HandleError(err)
	}

	return ErrBadEventWriter
}

// Close stops all the Log Operations from being usable, and they will panic if
// used after close is called. It also closes all EventWriters and returns the
// first returned error. The EventWriters are closed in the order they are
// passed to Start.
func Close() error {
	close(eventChannel)
	<-eventChannelClosed

	var err error
	for _, eventWriter := range eventWriters {
		er := eventWriter.Close()
		if er != nil && err == nil {
			err = er
		}
	}
	return err
}

// Subbed for testing.
var now = func() time.Time {
	return time.Now()
}

// Debug logs a debug message.
func Debug(tags Tags, msg string) {
	eventChannel <- Event{DebugEvent, now(), tags, msg, nil}
}

// Debugf is a formatted function of Debug.
func Debugf(tags Tags, format string, v ...interface{}) {
	Debug(tags, fmt.Sprintf(format, v...))
}

// Info logs an informational message.
func Info(tags Tags, msg string) {
	eventChannel <- Event{InfoEvent, now(), tags, msg, nil}
}

// Infof is a formatted function of Info.
func Infof(tags Tags, format string, v ...interface{}) {
	Info(tags, fmt.Sprintf(format, v...))
}

// Warn logs a warning message.
func Warn(tags Tags, msg string) {
	eventChannel <- Event{WarnEvent, now(), tags, msg, nil}
}

// Warnf is a formatted function of Warn.
func Warnf(tags Tags, format string, v ...interface{}) {
	Warn(tags, fmt.Sprintf(format, v...))
}

// Error logs an error message.
func Error(tags Tags, err error) {
	eventChannel <- Event{ErrorEvent, now(), tags, err.Error(), nil}
}

// Errorf is a formatted function of Error.
func Errorf(tags Tags, format string, v ...interface{}) {
	Error(tags, fmt.Errorf(format, v...))
}

// Fatal logs a recovered error which could have killed the application. Fatal
// adds a stack trace (type []byte) as Event.Data.
func Fatal(tags Tags, recv interface{}) {
	stackTrace := make([]byte, defaultStackSize)
	n := runtime.Stack(stackTrace, false)
	stackTrace = stackTrace[:n]

	msg := interfaceToString(recv)
	eventChannel <- Event{FatalEvent, now(), tags, msg, stackTrace}
}

// Thumbstone indicates a function is still used in production. When developing
// software it's possible to introduce dead code with updates and new features.
// If a function is being suspected of being dead (not used) in production, add
// a call to Thumbstone and check the production logs to see if you're right.
//
// The caller of the (possibly) dead function will be put in the message, using
// the following format:
//	Function functionName called by callerFunctionName, from file /path/to/file on line lineNumber
// For example:
//	Function myFunction called by main.main, from file /main.go on line 20
func Thumbstone(tags Tags, functionName string) {
	var msg string
	if pc, file, line, ok := runtime.Caller(2); ok {
		fn := runtime.FuncForPC(pc)
		msg = fmt.Sprintf("Function %s called by %s, from file %s on line %d",
			functionName, fn.Name(), file, line)
	} else {
		msg = "Function " + functionName + " called from unkown location"
	}

	eventChannel <- Event{ThumbEvent, now(), tags, msg, nil}
}

// Log logs a custom created event.
//
// Note: the timestamp doesn't need to be set, because it will be set by Log.
func Log(event Event) {
	event.Timestamp = now()
	eventChannel <- event
}

// interfaceToString converts a interface{} variable to a string.
func interfaceToString(value interface{}) string {
	switch v := value.(type) {
	case string:
		return v
	case fmt.Stringer:
		return v.String()
	case []byte:
		return string(v)
	case error:
		return v.Error()
	}
	return fmt.Sprintf("%v", value)
}
<|endoftext|>


<commit_before>package neutrino

import (
	"github.com/btcsuite/btcd/blockchain"
	"github.com/btcsuite/btcd/txscript"
	"github.com/btcsuite/btclog"
)

// log is a logger that is initialized with no output filters.  This
// means the package will not perform any logging by default until the caller
// requests it.
var log btclog.Logger

// The default amount of logging is none.
func init() {
	DisableLog()
}

// DisableLog disables all library log output.  Logging output is disabled
// by default until either UseLogger or SetLogWriter are called.
func DisableLog() {
	log = btclog.Disabled
}

// UseLogger uses a specified Logger to output package logging info.
// This should be used in preference to SetLogWriter if the caller is also
// using btclog.
func UseLogger(logger btclog.Logger) {
	log = logger
	blockchain.UseLogger(logger)
	txscript.UseLogger(logger)
}
<commit_msg>log: add peer and addmgr to logger<commit_after>package neutrino

import (
	"github.com/btcsuite/btcd/addrmgr"
	"github.com/btcsuite/btcd/blockchain"
	"github.com/btcsuite/btcd/peer"
	"github.com/btcsuite/btcd/txscript"
	"github.com/btcsuite/btclog"
)

// log is a logger that is initialized with no output filters.  This
// means the package will not perform any logging by default until the caller
// requests it.
var log btclog.Logger

// The default amount of logging is none.
func init() {
	DisableLog()
}

// DisableLog disables all library log output.  Logging output is disabled
// by default until either UseLogger or SetLogWriter are called.
func DisableLog() {
	log = btclog.Disabled
}

// UseLogger uses a specified Logger to output package logging info.
// This should be used in preference to SetLogWriter if the caller is also
// using btclog.
func UseLogger(logger btclog.Logger) {
	log = logger
	blockchain.UseLogger(logger)
	txscript.UseLogger(logger)
	peer.UseLogger(logger)
	addrmgr.UseLogger(logger)
}
<|endoftext|>


<commit_before>package migration

import (
	"crypto/md5"
	"encoding/base64"

	"github.com/jmoiron/sqlx"
)

var registeredMigrationSets []MigrationSet

type Step struct {
	Migrate string
	Revert  string
}

type MigrationSet []Step

func Register(set MigrationSet) {
	registeredMigrationSets = append(registeredMigrationSets, set)
}

func Run(db *sqlx.DB) error {
	for _, set := range registeredMigrationSets {
		err := runSet(db, set)
		if err != nil {
			return err
		}
	}

	return nil
}

func runSet(db *sqlx.DB, steps []Step) error {
	err := checkMigrationTable(db)
	if err != nil {
		return err
	}

	rows, err := db.Query(`
	SELECT hash, down
	FROM   migration
	`)
	if err != nil {
		return err
	}

	recordedMigrations := map[string]string{}
	for rows.Next() {
		var hash, down string
		err := rows.Scan(&hash, &down)
		if err != nil {
			break
		}
		recordedMigrations[hash] = down
	}

	if err := rows.Err(); err != nil {
		return err
	}

	for len(steps) > 0 {
		hash := createHash(steps[0].Up)
		if _, ok := recordedMigrations[hash]; !ok {
			break //we've found the migration to rewind to!
		}
		delete(recordedMigrations, hash)
		steps = steps[1:]
	}

	// do the rewind process
	// TODO: do this in reverse chronology!
	for hash, down := range recordedMigrations {
		_, err := db.Exec(`
		DELETE from migration
		WHERE  hash = $1
		`, hash)
		if err != nil {
			return err
		}

		_, err = db.Exec(down)
		if err != nil {
			return err
		}
	}

	//execute the remaining steps
	for _, migration := range steps {
		tx, err := db.Begin()

		if err != nil {
			return err
		}

		_, err = tx.Exec(migration.Up)

		if err != nil {
			tx.Rollback()
			return err
		}

		hash := createHash(migration.Up)

		_, err = tx.Exec(`
		INSERT into migration (hash, down)
		VALUES ($1, $2);
		`, hash, migration.Down)

		if err != nil {
			tx.Rollback()
			return err
		}

		err = tx.Commit()
		if err != nil {
			return err
		}
	}

	return nil
}

func createHash(str string) string {
	//TODO: remove empty lines and leading whitespace to make this a bit more robust
	sum := md5.Sum([]byte(str))
	b64 := base64.StdEncoding.EncodeToString(sum[:])
	return string(b64[:])
}

func checkMigrationTable(db *sqlx.DB) error {
	_, err := db.Query(`select 1 from migration;`)
	if err == nil {
		return nil //it already exists
	}

	//TODO: timestamps and other audit trails
	_, err = db.Exec(`
	CREATE TABLE migration (
		hash TEXT,
		down TEXT
	)
	`)

	return err
}
<commit_msg>Fix the change from up/down to migrate/revert<commit_after>package migration

import (
	"crypto/md5"
	"encoding/base64"

	"github.com/jmoiron/sqlx"
)

var registeredMigrationSets []MigrationSet

type Step struct {
	Migrate string
	Revert  string
}

type MigrationSet []Step

func Register(set MigrationSet) {
	registeredMigrationSets = append(registeredMigrationSets, set)
}

func Run(db *sqlx.DB) error {
	for _, set := range registeredMigrationSets {
		err := runSet(db, set)
		if err != nil {
			return err
		}
	}

	return nil
}

func runSet(db *sqlx.DB, steps []Step) error {
	err := checkMigrationTable(db)
	if err != nil {
		return err
	}

	rows, err := db.Query(`
	SELECT hash, down
	FROM   migration
	`)
	if err != nil {
		return err
	}

	recordedMigrations := map[string]string{}
	for rows.Next() {
		var hash, down string
		err := rows.Scan(&hash, &down)
		if err != nil {
			break
		}
		recordedMigrations[hash] = down
	}

	if err := rows.Err(); err != nil {
		return err
	}

	for len(steps) > 0 {
		hash := createHash(steps[0].Migrate)
		if _, ok := recordedMigrations[hash]; !ok {
			break //we've found the migration to rewind to!
		}
		delete(recordedMigrations, hash)
		steps = steps[1:]
	}

	// do the rewind process
	// TODO: do this in reverse chronology!
	for hash, down := range recordedMigrations {
		_, err := db.Exec(`
		DELETE from migration
		WHERE  hash = $1
		`, hash)
		if err != nil {
			return err
		}

		_, err = db.Exec(down)
		if err != nil {
			return err
		}
	}

	//execute the remaining steps
	for _, migration := range steps {
		tx, err := db.Begin()

		if err != nil {
			return err
		}

		_, err = tx.Exec(migration.Migrate)

		if err != nil {
			tx.Rollback()
			return err
		}

		hash := createHash(migration.Migrate)

		_, err = tx.Exec(`
		INSERT into migration (hash, down)
		VALUES ($1, $2);
		`, hash, migration.Revert)

		if err != nil {
			tx.Rollback()
			return err
		}

		err = tx.Commit()
		if err != nil {
			return err
		}
	}

	return nil
}

func createHash(str string) string {
	//TODO: remove empty lines and leading whitespace to make this a bit more robust
	sum := md5.Sum([]byte(str))
	b64 := base64.StdEncoding.EncodeToString(sum[:])
	return string(b64[:])
}

func checkMigrationTable(db *sqlx.DB) error {
	_, err := db.Query(`select 1 from migration;`)
	if err == nil {
		return nil //it already exists
	}

	//TODO: timestamps and other audit trails
	_, err = db.Exec(`
	CREATE TABLE migration (
		hash TEXT,
		down TEXT
	)
	`)

	return err
}
<|endoftext|>


<commit_before>package rmux

import (
	"context"
	"encoding/json"
	"net/http"
	"strings"
)

const (
	nodeKindMethod = iota
	nodeKindStatic
	nodeKindComplete
	nodeKindNonStatic
)

// ServeMux is an HTTP request multiplexer.
// It matches the URL of each incoming request
// against a list of registered patterns and calls the handler for the pattern that most closely matches the URL.
//
// It support RESTful naming convention:
// 	/users
//	/users/action
// 	/users/:id
// 	/users/:id/comments
// 	/users/:id/comments/hide
type ServeMux struct {
	methods  map[string]*node
	notFound http.Handler
	inter    Interceptor
	ctx      context.Context
}

// ServeMuxOpts allow to pass extra options to the muxer.
type ServeMuxOpts struct {
	NotFound    http.Handler
	Interceptor Interceptor
	Context     context.Context
}

// Interceptor is a function that decorate handler with custom logic.
type Interceptor func(http.ResponseWriter, *http.Request, http.Handler)

// NewServeMux allocates and returns a new ServeMux with default not found handler and context.
func NewServeMux(opts ServeMuxOpts) *ServeMux {
	if opts.NotFound == nil {
		opts.NotFound = http.NotFoundHandler()
	}
	if opts.Context == nil {
		opts.Context = context.Background()
	}
	sm := &ServeMux{
		methods:  map[string]*node{},
		notFound: opts.NotFound,
		inter:    opts.Interceptor,
		ctx:      opts.Context,
	}

	return sm
}

// Handle registers the handler for the given pattern.
func (sm *ServeMux) Handle(p string, h http.Handler) {
	args := sm.split(p)

	if _, ok := sm.methods[args.vals[0]]; !ok {
		sm.methods[args.vals[0]] = newNode(args.vals[0], nodeKindMethod)
	}

	sm.methods[args.vals[0]].add(args.vals[1:], h, 0)
	return

}

// ServeHTTP dispatches the request to the handler whose pattern most closely matches the request URL.
func (sm *ServeMux) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	args := sm.split(r.URL.Path)

	var (
		h  http.Handler
		ok bool
	)

	if root := sm.methods[r.Method]; root != nil {
		if !ok {
			h, ok = sm.search(root, &args)
		}
		if !ok {
			sm.notFound.ServeHTTP(rw, r.WithContext(sm.ctx))
			return
		}

		if args.len > 0 {
			r = r.WithContext(newContext(sm.ctx, args))
		} else {
			r = r.WithContext(sm.ctx)
		}
		if sm.inter != nil {
			sm.inter(rw, r, h)
		} else {
			h.ServeHTTP(rw, r)
		}

		return
	}

	sm.notFound.ServeHTTP(rw, r.WithContext(sm.ctx))
}

func (sm *ServeMux) search(n *node, args *arguments) (http.Handler, bool) {
	if args.len == 0 {
		return nil, false
	}

	var (
		nn *node
		ok bool
	)

	for idx, res := range (*args).vals {
		if n.static != nil {
			// Find static or non-static node.
			if nn, ok = n.static[res]; ok {
				if nn.end && args.len-1 == idx {
					if nn.res == res {
						return nn.handler, true
					}

					return nil, false
				}
			}
		}

		if n.nonstatic != nil {
			ok = true
			nn = n.nonstatic
			// slice of keys, values are stored in path variable
			if args.keys == nil {
				args.keys = make([]string, args.len)
			}
			args.keys[idx] = nn.res

			if nn.end && args.len-1 == idx {
				return nn.handler, true
			}
		}

		if !ok {
			return nil, false
		}

		n = nn
	}

	return nil, false
}

func (sm *ServeMux) split(p string) arguments {
	if p == "" {
		return arguments{}
	}

	if strings.HasPrefix(p, "/") {
		p = p[1:]
	}

	if strings.HasSuffix(p, "/") {
		p = p[:len(p)-1]
	}

	vals := strings.Split(p, "/")
	return arguments{
		vals: vals,
		len:  len(vals),
	}
}

// GoString implements fmt GoStringer interface.
func (sm *ServeMux) GoString() string {
	b, err := json.MarshalIndent(sm.methods, "", "\t")
	if err != nil {
		return err.Error()
	}

	return string(b)
}

type node struct {
	res       string
	end       bool
	kind      int
	handler   http.Handler
	static    map[string]*node
	nonstatic *node
}

func (n *node) MarshalJSON() ([]byte, error) {
	d := struct {
		Resource  string
		End       bool
		Kind      int
		Handler   bool
		NonStatic *node
		Static    map[string]*node
	}{
		Resource:  n.res,
		End:       n.end,
		Kind:      n.kind,
		NonStatic: n.nonstatic,
		Static:    n.static,
		Handler:   n.handler != nil,
	}

	return json.Marshal(d)
}

func newNode(res string, kind int) *node {
	return &node{
		res:    strings.TrimLeft(res, ":"),
		kind:   kind,
		static: map[string]*node{},
	}
}

func (n *node) add(path []string, h http.Handler, idx int) *node {
	if n == nil {
		n = &node{}
	}

	// for example GET/
	if len(path) == 0 && idx == 0 {
		nn := &node{
			end:     true,
			handler: h,
			kind:    nodeKindStatic,
		}
		n.static[""] = nn
		return nn
	}

	if len(path) == 0 || idx >= len(path) {
		return nil
	}

	var (
		ok  bool
		nn  *node
		res string
	)

	if len(path) > 0 {
		res = path[idx]
	}

	if isStatic(res) {
		if n.static == nil {
			n.static = map[string]*node{}
		}

		if nn, ok = n.static[res]; !ok {
			nn = newNode(res, nodeKindStatic)
		}

		if idx == len(path)-1 {
			nn.handler = h
			nn.end = true
		}

		if !ok {
			n.static[res] = nn
		}

		return nn.add(path, h, idx+1)
	}

	if n.nonstatic == nil {
		n.nonstatic = newNode(res, nodeKindNonStatic)
	}

	if idx == len(path)-1 {
		n.nonstatic.handler = h
		n.nonstatic.end = true
	}

	return n.nonstatic.add(path, h, idx+1)

}

func isStatic(p string) bool {
	return !strings.HasPrefix(p, ":")
}
<commit_msg>context allocated only if at least one key value pair exists<commit_after>package rmux

import (
	"context"
	"encoding/json"
	"net/http"
	"strings"
)

const (
	nodeKindMethod = iota
	nodeKindStatic
	nodeKindComplete
	nodeKindNonStatic
)

// ServeMux is an HTTP request multiplexer.
// It matches the URL of each incoming request
// against a list of registered patterns and calls the handler for the pattern that most closely matches the URL.
//
// It support RESTful naming convention:
// 	/users
//	/users/action
// 	/users/:id
// 	/users/:id/comments
// 	/users/:id/comments/hide
type ServeMux struct {
	methods  map[string]*node
	notFound http.Handler
	inter    Interceptor
	ctx      context.Context
}

// ServeMuxOpts allow to pass extra options to the muxer.
type ServeMuxOpts struct {
	NotFound    http.Handler
	Interceptor Interceptor
	Context     context.Context
}

// Interceptor is a function that decorate handler with custom logic.
type Interceptor func(http.ResponseWriter, *http.Request, http.Handler)

// NewServeMux allocates and returns a new ServeMux with default not found handler and context.
func NewServeMux(opts ServeMuxOpts) *ServeMux {
	if opts.NotFound == nil {
		opts.NotFound = http.NotFoundHandler()
	}
	if opts.Context == nil {
		opts.Context = context.Background()
	}
	sm := &ServeMux{
		methods:  map[string]*node{},
		notFound: opts.NotFound,
		inter:    opts.Interceptor,
		ctx:      opts.Context,
	}

	return sm
}

// Handle registers the handler for the given pattern.
func (sm *ServeMux) Handle(p string, h http.Handler) {
	args := sm.split(p)

	if _, ok := sm.methods[args.vals[0]]; !ok {
		sm.methods[args.vals[0]] = newNode(args.vals[0], nodeKindMethod)
	}

	sm.methods[args.vals[0]].add(args.vals[1:], h, 0)
	return

}

// ServeHTTP dispatches the request to the handler whose pattern most closely matches the request URL.
func (sm *ServeMux) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	args := sm.split(r.URL.Path)

	var (
		h  http.Handler
		ok bool
	)

	if root := sm.methods[r.Method]; root != nil {
		if !ok {
			h, ok = sm.search(root, &args)
		}
		if !ok {
			sm.notFound.ServeHTTP(rw, r.WithContext(sm.ctx))
			return
		}

		if len(args.keys) > 0 {
			r = r.WithContext(newContext(sm.ctx, args))
		} else {
			r = r.WithContext(sm.ctx)
		}
		if sm.inter != nil {
			sm.inter(rw, r, h)
		} else {
			h.ServeHTTP(rw, r)
		}

		return
	}

	sm.notFound.ServeHTTP(rw, r.WithContext(sm.ctx))
}

func (sm *ServeMux) search(n *node, args *arguments) (http.Handler, bool) {
	if args.len == 0 {
		return nil, false
	}

	var (
		nn *node
		ok bool
	)

	for idx, res := range (*args).vals {
		if n.static != nil {
			// Find static or non-static node.
			if nn, ok = n.static[res]; ok {
				if nn.end && args.len-1 == idx {
					if nn.res == res {
						return nn.handler, true
					}

					return nil, false
				}
			}
		}

		if n.nonstatic != nil {
			ok = true
			nn = n.nonstatic
			// slice of keys, values are stored in path variable
			if args.keys == nil {
				args.keys = make([]string, args.len)
			}
			args.keys[idx] = nn.res

			if nn.end && args.len-1 == idx {
				return nn.handler, true
			}
		}

		if !ok {
			return nil, false
		}

		n = nn
	}

	return nil, false
}

func (sm *ServeMux) split(p string) arguments {
	if p == "" {
		return arguments{}
	}

	if strings.HasPrefix(p, "/") {
		p = p[1:]
	}

	if strings.HasSuffix(p, "/") {
		p = p[:len(p)-1]
	}

	vals := strings.Split(p, "/")
	return arguments{
		vals: vals,
		len:  len(vals),
	}
}

// GoString implements fmt GoStringer interface.
func (sm *ServeMux) GoString() string {
	b, err := json.MarshalIndent(sm.methods, "", "\t")
	if err != nil {
		return err.Error()
	}

	return string(b)
}

type node struct {
	res       string
	end       bool
	kind      int
	handler   http.Handler
	static    map[string]*node
	nonstatic *node
}

func (n *node) MarshalJSON() ([]byte, error) {
	d := struct {
		Resource  string
		End       bool
		Kind      int
		Handler   bool
		NonStatic *node
		Static    map[string]*node
	}{
		Resource:  n.res,
		End:       n.end,
		Kind:      n.kind,
		NonStatic: n.nonstatic,
		Static:    n.static,
		Handler:   n.handler != nil,
	}

	return json.Marshal(d)
}

func newNode(res string, kind int) *node {
	return &node{
		res:    strings.TrimLeft(res, ":"),
		kind:   kind,
		static: map[string]*node{},
	}
}

func (n *node) add(path []string, h http.Handler, idx int) *node {
	if n == nil {
		n = &node{}
	}

	// for example GET/
	if len(path) == 0 && idx == 0 {
		nn := &node{
			end:     true,
			handler: h,
			kind:    nodeKindStatic,
		}
		n.static[""] = nn
		return nn
	}

	if len(path) == 0 || idx >= len(path) {
		return nil
	}

	var (
		ok  bool
		nn  *node
		res string
	)

	if len(path) > 0 {
		res = path[idx]
	}

	if isStatic(res) {
		if n.static == nil {
			n.static = map[string]*node{}
		}

		if nn, ok = n.static[res]; !ok {
			nn = newNode(res, nodeKindStatic)
		}

		if idx == len(path)-1 {
			nn.handler = h
			nn.end = true
		}

		if !ok {
			n.static[res] = nn
		}

		return nn.add(path, h, idx+1)
	}

	if n.nonstatic == nil {
		n.nonstatic = newNode(res, nodeKindNonStatic)
	}

	if idx == len(path)-1 {
		n.nonstatic.handler = h
		n.nonstatic.end = true
	}

	return n.nonstatic.add(path, h, idx+1)

}

func isStatic(p string) bool {
	return !strings.HasPrefix(p, ":")
}
<|endoftext|>