<commit_before>package build

import (
	"fmt"
	anko_core "github.com/mattn/anko/builtins"
	"github.com/mattn/anko/parser"
	"github.com/mattn/anko/vm"
	zglob "github.com/mattn/go-zglob"
	"io/ioutil"
	"neon/util"
	"os"
	"reflect"
	"regexp"
	"runtime"
	"sort"
	"strings"
)

// Build context
type Context struct {
	VM          *vm.Env
	Properties  []string
	Environment map[string]string
	Index       *Index
	Stack       *Stack
}

// NewContext make a new build context
func NewContext(build *Build) (*Context, error) {
	v := vm.NewEnv()
	anko_core.LoadAllBuiltins(v)
	LoadBuiltins(v)
	properties := build.GetProperties()
	environment := build.GetEnvironment()
	context := &Context{
		VM:          v,
		Properties:  properties.Fields(),
		Environment: environment,
		Index:       NewIndex(),
		Stack:       NewStack(),
	}
	for _, script := range build.Scripts {
		source, err := ioutil.ReadFile(script)
		if err != nil {
			return nil, fmt.Errorf("reading script '%s': %v", script, err)
		}
		_, err = v.Execute(string(source))
		if err != nil {
			return nil, fmt.Errorf("evaluating script '%s': %v", script, FormatScriptError(err))
		}
	}
	err := context.setInitialProperties(build, properties)
	if err != nil {
		return nil, fmt.Errorf("evaluating properties: %v", err)
	}
	return context, nil
}

func (context *Context) Copy(index int, data interface{}) *Context {
	properties := make([]string, len(context.Properties))
	for i := 0; i < len(context.Properties); i++ {
		properties[i] = context.Properties[i]
	}
	environment := make(map[string]string)
	for name, value := range context.Environment {
		environment[name] = value
	}
	copy := Context{
		VM:          context.VM.NewEnv(),
		Properties:  properties,
		Environment: environment,
		Index:       context.Index.Copy(),
		Stack:       context.Stack.Copy(),
	}
	context.SetProperty("_data", index)
	return &copy
}

// Set initial build properties
func (context *Context) setInitialProperties(build *Build, object util.Object) error {
	context.SetProperty("_OS", runtime.GOOS)
	context.SetProperty("_ARCH", runtime.GOARCH)
	context.SetProperty("_CPUS", runtime.NumCPU())
	context.SetProperty("_BASE", build.Dir)
	context.SetProperty("_HERE", build.Here)
	todo := object.Fields()
	var crash error
	for len(todo) > 0 {
		var done []string
		for _, name := range todo {
			value := object[name]
			eval, err := context.EvaluateObject(value)
			if err == nil {
				context.SetProperty(name, eval)
				done = append(done, name)
			} else {
				crash = err
			}
		}
		if len(done) == 0 {
			return crash
		}
		var next []string
		for _, name := range todo {
			found := false
			for _, n := range done {
				if name == n {
					found = true
				}
			}
			if !found {
				next = append(next, name)
			}
		}
		todo = next
	}
	return nil
}

// Set property with given to given value
func (context *Context) SetProperty(name string, value interface{}) {
	context.VM.Define(name, value)
}

// Get property value with given name
func (context *Context) GetProperty(name string) (interface{}, error) {
	value, err := context.VM.Get(name)
	if err != nil {
		return nil, err
	}
	return util.ValueToInterface(value), nil
}

// Evaluate given expression in context and return its value
func (context *Context) EvaluateExpression(source string) (interface{}, error) {
	value, err := context.VM.Execute(source)
	if err != nil {
		return nil, FormatScriptError(err)
	}
	return util.ValueToInterface(value), nil
}

// Evaluate a given object, that is replace '#{foo}' in strings with the value
// of property foo
func (context *Context) EvaluateObject(object interface{}) (interface{}, error) {
	switch value := object.(type) {
	case string:
		evaluated, err := context.EvaluateString(value)
		if err != nil {
			return nil, err
		}
		return evaluated, nil
	case bool:
		return value, nil
	case int:
		return value, nil
	case int32:
		return value, nil
	case int64:
		return value, nil
	case float64:
		return value, nil
	default:
		if value == nil {
			return nil, nil
		}
		switch reflect.TypeOf(object).Kind() {
		case reflect.Slice:
			slice := reflect.ValueOf(object)
			elements := make([]interface{}, slice.Len())
			for index := 0; index < slice.Len(); index++ {
				val, err := context.EvaluateObject(slice.Index(index).Interface())
				if err != nil {
					return nil, err
				}
				elements[index] = val
			}
			return elements, nil
		case reflect.Map:
			dict := reflect.ValueOf(object)
			elements := make(map[interface{}]interface{})
			for _, key := range dict.MapKeys() {
				keyEval, err := context.EvaluateObject(key.Interface())
				if err != nil {
					return nil, err
				}
				valueEval, err := context.EvaluateObject(dict.MapIndex(key).Interface())
				if err != nil {
					return nil, err
				}
				elements[keyEval] = valueEval
			}
			return elements, nil
		default:
			return nil, fmt.Errorf("no serializer for type '%T'", object)
		}
	}
}

// Evaluate a string by replacing '#{foo}' with value of property foo
func (context *Context) EvaluateString(text string) (string, error) {
	r := regexp.MustCompile(`#{.*?}`)
	var errors []error
	replaced := r.ReplaceAllStringFunc(text, func(expression string) string {
		name := expression[2 : len(expression)-1]
		var value interface{}
		value, err := context.EvaluateExpression(name)
		if err != nil {
			errors = append(errors, err)
			return ""
		} else {
			var str string
			str, err = PropertyToString(value, false)
			if err != nil {
				errors = append(errors, err)
				return ""
			} else {
				return str
			}
		}
	})
	if len(errors) > 0 {
		return replaced, errors[0]
	} else {
		return replaced, nil
	}
}

// Evaluate environment in context and return it as a slice of strings
func (context *Context) EvaluateEnvironment(build *Build) ([]string, error) {
	environment := make(map[string]string)
	for _, line := range os.Environ() {
		index := strings.Index(line, "=")
		name := line[:index]
		value := line[index+1:]
		environment[name] = value
	}
	environment["_BASE"] = build.Dir
	environment["_HERE"] = build.Here
	var variables []string
	for name := range context.Environment {
		variables = append(variables, name)
	}
	sort.Strings(variables)
	for _, name := range variables {
		value := context.Environment[name]
		r := regexp.MustCompile(`[$#]{.*?}`)
		replaced := r.ReplaceAllStringFunc(value, func(expression string) string {
			name := expression[2 : len(expression)-1]
			if expression[0:1] == "$" {
				value, ok := environment[name]
				if !ok {
					return expression
				} else {
					return value
				}
			} else {
				value, err := context.EvaluateExpression(name)
				if err != nil {
					return expression
				} else {
					str, _ := PropertyToString(value, false)
					return str
				}
			}
		})
		environment[name] = replaced
	}
	var lines []string
	for name, value := range environment {
		line := name + "=" + value
		lines = append(lines, line)
	}
	return lines, nil
}

// Find files in the context:
// - dir: the search root directory
// - includes: the list of globs to include
// - excludes: the list of globs to exclude
// - folder: tells if we should include folders
// Return the list of files as a slice of strings
func (context *Context) FindFiles(dir string, includes, excludes []string, folder bool) ([]string, error) {
	eval, err := context.EvaluateString(dir)
	if err != nil {
		return nil, fmt.Errorf("evaluating source directory: %v", err)
	}
	dir = util.ExpandUserHome(eval)
	if dir != "" {
		oldDir, err := os.Getwd()
		if err != nil {
			return nil, fmt.Errorf("getting working directory: %v", err)
		}
		defer os.Chdir(oldDir)
		err = os.Chdir(dir)
		if err != nil {
			return nil, nil
		}
	}
	var included []string
	for _, include := range includes {
		pattern, err := context.EvaluateString(include)
		if err != nil {
			return nil, fmt.Errorf("evaluating pattern: %v", err)
		}
		included = append(included, pattern)
	}
	var excluded []string
	for _, exclude := range excludes {
		pattern, err := context.EvaluateString(exclude)
		if err != nil {
			return nil, fmt.Errorf("evaluating pattern: %v", err)
		}
		pattern = util.ExpandUserHome(pattern)
		excluded = append(excluded, pattern)
	}
	var candidates []string
	for _, include := range included {
		list, _ := zglob.Glob(util.ExpandUserHome(include))
		for _, file := range list {
			stat, err := os.Stat(file)
			if err != nil {
				return nil, fmt.Errorf("stating file: %v", err)
			}
			if stat.Mode().IsRegular() || folder {
				candidates = append(candidates, file)
			}
		}
	}
	var files []string
	if excluded != nil {
		for index, file := range candidates {
			for _, exclude := range excluded {
				match, err := zglob.Match(exclude, file)
				if match || err != nil {
					candidates[index] = ""
				}
			}
		}
		for _, file := range candidates {
			if file != "" {
				files = append(files, file)
			}
		}
	} else {
		files = candidates
	}
	sort.Strings(files)
	return files, nil
}

// FormatScriptError adds line and column numbers on parser or vm errors.
func FormatScriptError(err error) error {
	if e, ok := err.(*parser.Error); ok {
		return fmt.Errorf("%s (at line %d, column %d)", err, e.Pos.Line, e.Pos.Column)
	} else if e, ok := err.(*vm.Error); ok {
		return fmt.Errorf("%s (at line %d, column %d)", err, e.Pos.Line, e.Pos.Column)
	} else {
		return err
	}
}

// Message print a message on the console
func (context *Context) Message(text string, args ...interface{}) {
	Message(text, args...)
}
<commit_msg>Fixed _NCPU definition<commit_after>package build

import (
	"fmt"
	anko_core "github.com/mattn/anko/builtins"
	"github.com/mattn/anko/parser"
	"github.com/mattn/anko/vm"
	zglob "github.com/mattn/go-zglob"
	"io/ioutil"
	"neon/util"
	"os"
	"reflect"
	"regexp"
	"runtime"
	"sort"
	"strings"
)

// Build context
type Context struct {
	VM          *vm.Env
	Properties  []string
	Environment map[string]string
	Index       *Index
	Stack       *Stack
}

// NewContext make a new build context
func NewContext(build *Build) (*Context, error) {
	v := vm.NewEnv()
	anko_core.LoadAllBuiltins(v)
	LoadBuiltins(v)
	properties := build.GetProperties()
	environment := build.GetEnvironment()
	context := &Context{
		VM:          v,
		Properties:  properties.Fields(),
		Environment: environment,
		Index:       NewIndex(),
		Stack:       NewStack(),
	}
	for _, script := range build.Scripts {
		source, err := ioutil.ReadFile(script)
		if err != nil {
			return nil, fmt.Errorf("reading script '%s': %v", script, err)
		}
		_, err = v.Execute(string(source))
		if err != nil {
			return nil, fmt.Errorf("evaluating script '%s': %v", script, FormatScriptError(err))
		}
	}
	err := context.setInitialProperties(build, properties)
	if err != nil {
		return nil, fmt.Errorf("evaluating properties: %v", err)
	}
	return context, nil
}

func (context *Context) Copy(index int, data interface{}) *Context {
	properties := make([]string, len(context.Properties))
	for i := 0; i < len(context.Properties); i++ {
		properties[i] = context.Properties[i]
	}
	environment := make(map[string]string)
	for name, value := range context.Environment {
		environment[name] = value
	}
	copy := Context{
		VM:          context.VM.NewEnv(),
		Properties:  properties,
		Environment: environment,
		Index:       context.Index.Copy(),
		Stack:       context.Stack.Copy(),
	}
	context.SetProperty("_data", index)
	return &copy
}

// Set initial build properties
func (context *Context) setInitialProperties(build *Build, object util.Object) error {
	context.SetProperty("_OS", runtime.GOOS)
	context.SetProperty("_ARCH", runtime.GOARCH)
	context.SetProperty("_NCPU", runtime.NumCPU())
	context.SetProperty("_BASE", build.Dir)
	context.SetProperty("_HERE", build.Here)
	todo := object.Fields()
	var crash error
	for len(todo) > 0 {
		var done []string
		for _, name := range todo {
			value := object[name]
			eval, err := context.EvaluateObject(value)
			if err == nil {
				context.SetProperty(name, eval)
				done = append(done, name)
			} else {
				crash = err
			}
		}
		if len(done) == 0 {
			return crash
		}
		var next []string
		for _, name := range todo {
			found := false
			for _, n := range done {
				if name == n {
					found = true
				}
			}
			if !found {
				next = append(next, name)
			}
		}
		todo = next
	}
	return nil
}

// Set property with given to given value
func (context *Context) SetProperty(name string, value interface{}) {
	context.VM.Define(name, value)
}

// Get property value with given name
func (context *Context) GetProperty(name string) (interface{}, error) {
	value, err := context.VM.Get(name)
	if err != nil {
		return nil, err
	}
	return util.ValueToInterface(value), nil
}

// Evaluate given expression in context and return its value
func (context *Context) EvaluateExpression(source string) (interface{}, error) {
	value, err := context.VM.Execute(source)
	if err != nil {
		return nil, FormatScriptError(err)
	}
	return util.ValueToInterface(value), nil
}

// Evaluate a given object, that is replace '#{foo}' in strings with the value
// of property foo
func (context *Context) EvaluateObject(object interface{}) (interface{}, error) {
	switch value := object.(type) {
	case string:
		evaluated, err := context.EvaluateString(value)
		if err != nil {
			return nil, err
		}
		return evaluated, nil
	case bool:
		return value, nil
	case int:
		return value, nil
	case int32:
		return value, nil
	case int64:
		return value, nil
	case float64:
		return value, nil
	default:
		if value == nil {
			return nil, nil
		}
		switch reflect.TypeOf(object).Kind() {
		case reflect.Slice:
			slice := reflect.ValueOf(object)
			elements := make([]interface{}, slice.Len())
			for index := 0; index < slice.Len(); index++ {
				val, err := context.EvaluateObject(slice.Index(index).Interface())
				if err != nil {
					return nil, err
				}
				elements[index] = val
			}
			return elements, nil
		case reflect.Map:
			dict := reflect.ValueOf(object)
			elements := make(map[interface{}]interface{})
			for _, key := range dict.MapKeys() {
				keyEval, err := context.EvaluateObject(key.Interface())
				if err != nil {
					return nil, err
				}
				valueEval, err := context.EvaluateObject(dict.MapIndex(key).Interface())
				if err != nil {
					return nil, err
				}
				elements[keyEval] = valueEval
			}
			return elements, nil
		default:
			return nil, fmt.Errorf("no serializer for type '%T'", object)
		}
	}
}

// Evaluate a string by replacing '#{foo}' with value of property foo
func (context *Context) EvaluateString(text string) (string, error) {
	r := regexp.MustCompile(`#{.*?}`)
	var errors []error
	replaced := r.ReplaceAllStringFunc(text, func(expression string) string {
		name := expression[2 : len(expression)-1]
		var value interface{}
		value, err := context.EvaluateExpression(name)
		if err != nil {
			errors = append(errors, err)
			return ""
		} else {
			var str string
			str, err = PropertyToString(value, false)
			if err != nil {
				errors = append(errors, err)
				return ""
			} else {
				return str
			}
		}
	})
	if len(errors) > 0 {
		return replaced, errors[0]
	} else {
		return replaced, nil
	}
}

// Evaluate environment in context and return it as a slice of strings
func (context *Context) EvaluateEnvironment(build *Build) ([]string, error) {
	environment := make(map[string]string)
	for _, line := range os.Environ() {
		index := strings.Index(line, "=")
		name := line[:index]
		value := line[index+1:]
		environment[name] = value
	}
	environment["_BASE"] = build.Dir
	environment["_HERE"] = build.Here
	var variables []string
	for name := range context.Environment {
		variables = append(variables, name)
	}
	sort.Strings(variables)
	for _, name := range variables {
		value := context.Environment[name]
		r := regexp.MustCompile(`[$#]{.*?}`)
		replaced := r.ReplaceAllStringFunc(value, func(expression string) string {
			name := expression[2 : len(expression)-1]
			if expression[0:1] == "$" {
				value, ok := environment[name]
				if !ok {
					return expression
				} else {
					return value
				}
			} else {
				value, err := context.EvaluateExpression(name)
				if err != nil {
					return expression
				} else {
					str, _ := PropertyToString(value, false)
					return str
				}
			}
		})
		environment[name] = replaced
	}
	var lines []string
	for name, value := range environment {
		line := name + "=" + value
		lines = append(lines, line)
	}
	return lines, nil
}

// Find files in the context:
// - dir: the search root directory
// - includes: the list of globs to include
// - excludes: the list of globs to exclude
// - folder: tells if we should include folders
// Return the list of files as a slice of strings
func (context *Context) FindFiles(dir string, includes, excludes []string, folder bool) ([]string, error) {
	eval, err := context.EvaluateString(dir)
	if err != nil {
		return nil, fmt.Errorf("evaluating source directory: %v", err)
	}
	dir = util.ExpandUserHome(eval)
	if dir != "" {
		oldDir, err := os.Getwd()
		if err != nil {
			return nil, fmt.Errorf("getting working directory: %v", err)
		}
		defer os.Chdir(oldDir)
		err = os.Chdir(dir)
		if err != nil {
			return nil, nil
		}
	}
	var included []string
	for _, include := range includes {
		pattern, err := context.EvaluateString(include)
		if err != nil {
			return nil, fmt.Errorf("evaluating pattern: %v", err)
		}
		included = append(included, pattern)
	}
	var excluded []string
	for _, exclude := range excludes {
		pattern, err := context.EvaluateString(exclude)
		if err != nil {
			return nil, fmt.Errorf("evaluating pattern: %v", err)
		}
		pattern = util.ExpandUserHome(pattern)
		excluded = append(excluded, pattern)
	}
	var candidates []string
	for _, include := range included {
		list, _ := zglob.Glob(util.ExpandUserHome(include))
		for _, file := range list {
			stat, err := os.Stat(file)
			if err != nil {
				return nil, fmt.Errorf("stating file: %v", err)
			}
			if stat.Mode().IsRegular() || folder {
				candidates = append(candidates, file)
			}
		}
	}
	var files []string
	if excluded != nil {
		for index, file := range candidates {
			for _, exclude := range excluded {
				match, err := zglob.Match(exclude, file)
				if match || err != nil {
					candidates[index] = ""
				}
			}
		}
		for _, file := range candidates {
			if file != "" {
				files = append(files, file)
			}
		}
	} else {
		files = candidates
	}
	sort.Strings(files)
	return files, nil
}

// FormatScriptError adds line and column numbers on parser or vm errors.
func FormatScriptError(err error) error {
	if e, ok := err.(*parser.Error); ok {
		return fmt.Errorf("%s (at line %d, column %d)", err, e.Pos.Line, e.Pos.Column)
	} else if e, ok := err.(*vm.Error); ok {
		return fmt.Errorf("%s (at line %d, column %d)", err, e.Pos.Line, e.Pos.Column)
	} else {
		return err
	}
}

// Message print a message on the console
func (context *Context) Message(text string, args ...interface{}) {
	Message(text, args...)
}
<|endoftext|>


<commit_before>package main

import (
  "fmt"
  "net/http"
  "log"
  "os"
  "io/ioutil"
)

func main(){
  var url string
  fmt.Println(">> Insert url: (Make sure to use http://)")
  fmt.Scanf("%s", &url)
  if url == "" {
    fmt.Println("Please type in an url")
    os.Exit(1)
  }
  data, err := http.Get(url)
  if err != nil{
    log.Error(err)
    os.Exit(1)
  }else{
    fmt.Println("Fetching url...")
    defer data.Body.Close()
    contents, err := ioutil.ReadAll(data.Body)
    if err != nil{
      log.Error(err)
    }
    fmt.Println("%s\n", string(contents))
  }
}
<commit_msg>better error logging with log.Error rather than fmt<commit_after>package main

import (
  "fmt"
  "net/http"
  "log"
  "io/ioutil"
)

func main(){
  var url string
  fmt.Println(">> Insert url: (Make sure to use http://)")
  fmt.Scanf("%s", &url)
  if url == "" {
    fmt.Println("Please type in an url")
  }
  data, err := http.Get(url)
  if err != nil{
    log.Error(err)
  }else{
    fmt.Println("Fetching url...")
    defer data.Body.Close()
    contents, err := ioutil.ReadAll(data.Body)
    if err != nil{
      log.Error(err)
    }
    fmt.Println("%s\n", string(contents))
  }
}
<|endoftext|>


<commit_before>// Boringstreamer looks for mp3 files and broadcasts via http.
// $ boringstreamer -addr 4444 -max 42 /
// recursively looks for mp3 files starting from / and broadcasts on port 4444 for at most 42 concurrent streamer clients.
// Browse to listen (e.g. http://localhost:4444/)
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/tcolgate/mp3"
)

var (
	addr           = flag.String("addr", ":4444", "listen on address (format: :port or host:port)")
	maxConnections = flag.Int("max", 42, "set maximum number of streaming connections")
	recursively    = flag.Bool("r", true, "recursively look for music starting from path")
	verbose        = flag.Bool("v", false, "display verbose messages")
)

// like /dev/null
type nullWriter struct {
}

func (nw nullWriter) Write(p []byte) (n int, err error) {
	return len(p), nil
}

type streamFrame []byte

// client's event
type broadcastResult struct {
	qid int
	ok  bool
}

// After a start() mux broadcasts audio stream to it's listener clients.
// Clients subscribe() and unsubscribe by writing to result chanel.
type mux struct {
	sync.Mutex

	clients map[int]chan streamFrame // set of listener clients to be notified
	result  chan broadcastResult     // clients share broadcast success-failure here

	nextFile   chan string      // next file to be broadcast
	nextStream chan io.Reader   // next (ID3 stripped) raw audio stream
	nextFrame  chan streamFrame // next audio frame
}

// subscribe(ch) adds ch to the set of channels to be received on by the clients when a new audio frame is available.
// Returns uniq client id (qid) for ch and a broadcast result channel for the client.
// Returns -1, nil if too many clients are already listening.
// clients: qid, br := m.subscribe(ch)
func (m *mux) subscribe(ch chan streamFrame) (int, chan broadcastResult) {
	m.Lock()
	defer m.Unlock()
	// search for available qid
	qid := 0
	_, ok := m.clients[qid]
	for ; ok; _, ok = m.clients[qid] {
		if qid >= *maxConnections-1 {
			return -1, nil
		}
		qid++
	}
	m.clients[qid] = ch
	if *verbose {
		log.Printf("New connection (qid: %v), streaming to %v connections.", qid, len(m.clients))
	}

	return qid, m.result
}

// stripID3Header(r) reads file from r, strips id3v2 headers and returns the rest
// id3v2 tag details: id3.org
func stripID3Header(r io.Reader) io.Reader {
	buf, err := ioutil.ReadAll(r)
	if err != nil {
		log.Printf("Error: skipping file, stripID3Header(), err=%v", err)
		return bytes.NewReader(make([]byte, 0))
	}

	// TODO(fgergo) add ID3 v1 detection
	if string(buf[:3]) != "ID3" {
		return bytes.NewReader(buf) // no ID3 header
	}

	// The ID3v2 tag size is encoded in four bytes
	// where msb (bit 7) is set to zero in every byte,
	// ie. tag size is at most 2^28 (4*8-4=28).
	id3size := int32(buf[6])<<21 | int32(buf[7])<<14 | int32(buf[8])<<7 | int32(buf[9])
	id3size += 10 // calculated tag size is excluding the header => +10

	return bytes.NewReader(buf[id3size:])
}

// genFileList() periodically checks for files available from root and
// sends filenames down chan queue.
func genFileList(root string, queue chan string) {
	rand.Seed(time.Now().Unix()) // minimal randomness

	rescan := make(chan chan string)
	go func() {
		for {
			files := <-rescan
			filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
				if err != nil {
					return err
				}
				if !info.Mode().IsRegular() {
					return nil
				}
				ok := strings.HasSuffix(strings.ToLower(info.Name()), ".mp3") // probably file is mp3
				if !info.IsDir() && !ok {
					return nil
				}
				files <- path // found file

				return nil
			})
			close(files)
			time.Sleep(1 * time.Second) // poll at least with 1Hz
		}
	}()

	// buffer and shuffle
	go func() {
		for {
			files := make(chan string)
			rescan <- files

			shuffled := make([]string, 0) // randomized set of files

			for f := range files {
				select {
				case queue <- f: // start playing as soon as possible
					if *verbose {
						fmt.Printf("Next: %v\n", f)
					}
					continue
				default:
					// shuffle files for random playback
					// (random permutation)
					if len(shuffled) == 0 {
						shuffled = append(shuffled, f)
					} else {
						i := rand.Intn(len(shuffled))
						shuffled = append(shuffled, shuffled[i])
						shuffled[i] = f
					}
				}
			}

			// queue shuffled files
			for _, f := range shuffled {
				queue <- f
				if *verbose {
					fmt.Printf("Next: %v\n", f)
				}
			}
		}
	}()
}

// start() initializes a multiplexer for raw audio streams
// e.g: m := new(mux).start(path)
func (m *mux) start(path string) *mux {
	m.result = make(chan broadcastResult)
	m.clients = make(map[int]chan streamFrame)

	m.nextFile = make(chan string)
	m.nextStream = make(chan io.Reader)
	m.nextFrame = make(chan streamFrame)

	// generate randomized list of files available from path
	genFileList(path, m.nextFile)

	// read file, strip ID3 header
	go func() {
		for {
			filename := <-m.nextFile
			f, err := os.Open(filename)
			if err != nil {
				log.Printf("Skipped \"%v\", err=%v", filename, err)
				continue
			}
			m.nextStream <- stripID3Header(f)
			if *verbose {
				fmt.Printf("Now playing: %v\n", filename)
			}
		}
	}()

	// decode stream to frames
	go func() {
		nullwriter := new(nullWriter)
		for {
			streamReader := <-m.nextStream
			d := mp3.NewDecoder(streamReader)
			var f mp3.Frame
			//			sent := 0 // TODO(fgergo) remove later
			//			lastSent := time.Now().UTC()
			for {
				tmp := log.Prefix()
				if !*verbose {
					log.SetOutput(nullwriter) // hack to silence mp3 debug/log output
				} else {
					log.SetPrefix("info: mp3 decode msg: ")
				}
				err := d.Decode(&f)
				log.SetPrefix(tmp)
				if !*verbose {
					log.SetOutput(os.Stderr)
				}
				if err == io.EOF {
					break
				}
				if err != nil {
					if *verbose {
						log.Printf("Skipping frame, d.Decode() err=%v", err)
					}
					continue
				}
				buf, err := ioutil.ReadAll(f.Reader())
				if err != nil {
					if *verbose {
						log.Printf("Skipping frame, ioutil.ReadAll() err=%v", err)
					}
					continue
				}
				m.nextFrame <- buf
				/*
					sent += len(buf)
					if sent >= 1*1024*1024 {
						now := time.Now().UTC()
						dur := now.Sub(lastSent)
						kBps := int64(sent)*1e9/1024/dur.Nanoseconds()
						if *verbose {
							log.Printf("Info: sent %#v bytes in the last %v (%vkB/sec)", sent, dur, int(kBps))
						}
						lastSent = now
						sent = 0
					}
				*/
				time.Sleep(f.Duration()) // TODO(fgergo) streaming is not working cotinuously, probably because of too much Sleep()
			}
		}
	}()

	// notify receiving clients about new available audio stream
	go func() {
		for {
			f := <-m.nextFrame
			// notify clients of new audio frame or let them quit
			for _, ch := range m.clients {
				ch <- f
				br := <-m.result // handle quitting clients
				if !br.ok {
					m.Lock()
					close(m.clients[br.qid])
					delete(m.clients, br.qid)
					m.Unlock()
					if *verbose {
						log.Printf("Connection exited (qid: %v), streaming to %v connections.", br.qid, len(m.clients))
					}
				}
			}
		}
	}()

	return m
}

type streamHandler struct {
	stream mux
}

// chrome and firefox play mp3 audio stream directly
func (sh streamHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	now := time.Now().UTC()
	frames := make(chan streamFrame)
	qid, br := sh.stream.subscribe(frames)
	if qid < 0 {
		log.Printf("New connection request denied, already serving %v connections. See -h for details.", *maxConnections)
		w.WriteHeader(http.StatusTooManyRequests)
		return
	}

	w.WriteHeader(http.StatusOK)
	w.Header().Set("Date", now.Format(http.TimeFormat))
	w.Header().Set("Connection", "Keep-Alive")
	w.Header().Set("Cache-Control", "no-cache")
	w.Header().Set("Content-Type", "audio/mpeg")
	w.Header().Set("Server", "BoringStreamer/4.0")

	// browsers need ID3 tag to identify frames as media to be played
	// mp3 header to designate mp3 stream
	b := []byte{0x49, 0x44, 0x33, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
	_, err := io.Copy(w, bytes.NewReader(b))
	if err != nil {
		log.Printf("Error streaming id3 tag (qid: %v), err=%v", qid, err)
	} else {
		// write mp3 stream to w
		for {
			buf := <-frames
			_, err = io.Copy(w, bytes.NewReader(buf))
			if err != nil {
				break
			}
			br <- broadcastResult{qid, true}
			w.(http.Flusher).Flush()
		}
	}
	br <- broadcastResult{qid, false}
	if *verbose {
		log.Printf("Stopped connection (qid: %v), reason err=%v", qid, err)
	}
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [flags] [path]\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "Browse to listen (e.g. http://localhost:4444/)\n\nflags:\n")
		flag.PrintDefaults()
	}
	flag.Parse()
	if len(flag.Args()) > 1 {
		flag.Usage()
		os.Exit(1)
	}

	path := ""
	switch len(flag.Args()) {
	case 0:
		path = "."
		if *verbose {
			fmt.Printf("Using path %#v, see -h for details.\n", path)
		}
	case 1:
		path = flag.Args()[0]
	}

	if *verbose {
		fmt.Printf("Looking for files available from \"%v\" ...\n", path)
	}

	// check if path is available
	matches, err := filepath.Glob(path)
	if err != nil || len(matches) != 1 {
		fmt.Fprintf(os.Stderr, "Error: \"%v\" unavailable.\n", path)
		os.Exit(1)
	}

	// initialize and start mp3 streamer
	http.Handle("/", streamHandler{*new(mux).start(path)})
	if *verbose {
		fmt.Printf("Waiting for connections on %v\n", *addr)
	}

	err = http.ListenAndServe(*addr, nil)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>time.Sleep(duration) corrected with elapsed time, to avoid underflow during streaming<commit_after>// Boringstreamer looks for mp3 files and broadcasts via http.
// $ boringstreamer -addr 4444 -max 42 /
// recursively looks for mp3 files starting from / and broadcasts on port 4444 for at most 42 concurrent streamer clients.
// Browse to listen (e.g. http://localhost:4444/)
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/tcolgate/mp3"
)

var (
	addr           = flag.String("addr", ":4444", "listen on address (format: :port or host:port)")
	maxConnections = flag.Int("max", 42, "set maximum number of streaming connections")
	recursively    = flag.Bool("r", true, "recursively look for music starting from path")
	verbose        = flag.Bool("v", false, "display verbose messages")
)

// like /dev/null
type nullWriter struct {
}

func (nw nullWriter) Write(p []byte) (n int, err error) {
	return len(p), nil
}

type streamFrame []byte

// client's event
type broadcastResult struct {
	qid int
	ok  bool
}

// After a start() mux broadcasts audio stream to it's listener clients.
// Clients subscribe() and unsubscribe by writing to result chanel.
type mux struct {
	sync.Mutex

	clients map[int]chan streamFrame // set of listener clients to be notified
	result  chan broadcastResult     // clients share broadcast success-failure here

	nextFile   chan string      // next file to be broadcast
	nextStream chan io.Reader   // next (ID3 stripped) raw audio stream
	nextFrame  chan streamFrame // next audio frame
}

// subscribe(ch) adds ch to the set of channels to be received on by the clients when a new audio frame is available.
// Returns uniq client id (qid) for ch and a broadcast result channel for the client.
// Returns -1, nil if too many clients are already listening.
// clients: qid, br := m.subscribe(ch)
func (m *mux) subscribe(ch chan streamFrame) (int, chan broadcastResult) {
	m.Lock()
	defer m.Unlock()
	// search for available qid
	qid := 0
	_, ok := m.clients[qid]
	for ; ok; _, ok = m.clients[qid] {
		if qid >= *maxConnections-1 {
			return -1, nil
		}
		qid++
	}
	m.clients[qid] = ch
	if *verbose {
		log.Printf("New connection (qid: %v), streaming to %v connections.", qid, len(m.clients))
	}

	return qid, m.result
}

// stripID3Header(r) reads file from r, strips id3v2 headers and returns the rest
// id3v2 tag details: id3.org
func stripID3Header(r io.Reader) io.Reader {
	buf, err := ioutil.ReadAll(r)
	if err != nil {
		log.Printf("Error: skipping file, stripID3Header(), err=%v", err)
		return bytes.NewReader(make([]byte, 0))
	}

	// TODO(fgergo) add ID3 v1 detection
	if string(buf[:3]) != "ID3" {
		return bytes.NewReader(buf) // no ID3 header
	}

	// The ID3v2 tag size is encoded in four bytes
	// where msb (bit 7) is set to zero in every byte,
	// ie. tag size is at most 2^28 (4*8-4=28).
	id3size := int32(buf[6])<<21 | int32(buf[7])<<14 | int32(buf[8])<<7 | int32(buf[9])
	id3size += 10 // calculated tag size is excluding the header => +10

	return bytes.NewReader(buf[id3size:])
}

// genFileList() periodically checks for files available from root and
// sends filenames down chan queue.
func genFileList(root string, queue chan string) {
	rand.Seed(time.Now().Unix()) // minimal randomness

	rescan := make(chan chan string)
	go func() {
		for {
			files := <-rescan
			filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
				if err != nil {
					return err
				}
				if !info.Mode().IsRegular() {
					return nil
				}
				ok := strings.HasSuffix(strings.ToLower(info.Name()), ".mp3") // probably file is mp3
				if !info.IsDir() && !ok {
					return nil
				}
				files <- path // found file

				return nil
			})
			close(files)
			time.Sleep(1 * time.Second) // poll at least with 1Hz
		}
	}()

	// buffer and shuffle
	go func() {
		for {
			files := make(chan string)
			rescan <- files

			shuffled := make([]string, 0) // randomized set of files

			for f := range files {
				select {
				case queue <- f: // start playing as soon as possible
					if *verbose {
						fmt.Printf("Next: %v\n", f)
					}
					continue
				default:
					// shuffle files for random playback
					// (random permutation)
					if len(shuffled) == 0 {
						shuffled = append(shuffled, f)
					} else {
						i := rand.Intn(len(shuffled))
						shuffled = append(shuffled, shuffled[i])
						shuffled[i] = f
					}
				}
			}

			// queue shuffled files
			for _, f := range shuffled {
				queue <- f
				if *verbose {
					fmt.Printf("Next: %v\n", f)
				}
			}
		}
	}()
}

// start() initializes a multiplexer for raw audio streams
// e.g: m := new(mux).start(path)
func (m *mux) start(path string) *mux {
	m.result = make(chan broadcastResult)
	m.clients = make(map[int]chan streamFrame)

	m.nextFile = make(chan string)
	m.nextStream = make(chan io.Reader)
	m.nextFrame = make(chan streamFrame)

	// generate randomized list of files available from path
	genFileList(path, m.nextFile)

	// read file, strip ID3 header
	go func() {
		for {
			filename := <-m.nextFile
			f, err := os.Open(filename)
			if err != nil {
				log.Printf("Skipped \"%v\", err=%v", filename, err)
				continue
			}
			m.nextStream <- stripID3Header(f)
			if *verbose {
				fmt.Printf("Now playing: %v\n", filename)
			}
		}
	}()

	// decode stream to frames
	go func() {
		nullwriter := new(nullWriter)
		for {
			streamReader := <-m.nextStream
			d := mp3.NewDecoder(streamReader)
			var f mp3.Frame
			//			sent := 0 // TODO(fgergo) remove later
			//			lastSent := time.Now().UTC()
			for {
				t0:= time.Now()
				tmp := log.Prefix()
				if !*verbose {
					log.SetOutput(nullwriter) // hack to silence mp3 debug/log output
				} else {
					log.SetPrefix("info: mp3 decode msg: ")
				}
				err := d.Decode(&f)
				log.SetPrefix(tmp)
				if !*verbose {
					log.SetOutput(os.Stderr)
				}
				if err == io.EOF {
					break
				}
				if err != nil {
					if *verbose {
						log.Printf("Skipping frame, d.Decode() err=%v", err)
					}
					continue
				}
				buf, err := ioutil.ReadAll(f.Reader())
				if err != nil {
					if *verbose {
						log.Printf("Skipping frame, ioutil.ReadAll() err=%v", err)
					}
					continue
				}
				m.nextFrame <- buf

				/*
					sent += len(buf)
					if sent >= 1*1024*1024 {
						now := time.Now().UTC()
						dur := now.Sub(lastSent)
						kBps := int64(sent)*1e9/1024/dur.Nanoseconds()
						if *verbose {
							log.Printf("Info: sent %#v bytes in the last %v (%vkB/sec)", sent, dur, int(kBps))
						}
						lastSent = now
						sent = 0
					}
				*/
				towait := f.Duration() - time.Now().Sub(t0)
				if towait > 0 {
					time.Sleep(towait)
				}
			}
		}
	}()

	// broadcast frame to clients
	go func() {
		for {
			f := <-m.nextFrame
			// notify clients of new audio frame or let them quit
			for _, ch := range m.clients {
				ch <- f
				br := <-m.result // handle quitting clients
				if !br.ok {
					m.Lock()
					close(m.clients[br.qid])
					delete(m.clients, br.qid)
					m.Unlock()
					if *verbose {
						log.Printf("Connection exited (qid: %v), streaming to %v connections.", br.qid, len(m.clients))
					}
				}
			}
		}
	}()

	return m
}

type streamHandler struct {
	stream mux
}

// chrome and firefox play mp3 audio stream directly
func (sh streamHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	now := time.Now().UTC()
	frames := make(chan streamFrame)
	qid, br := sh.stream.subscribe(frames)
	if qid < 0 {
		log.Printf("New connection request denied, already serving %v connections. See -h for details.", *maxConnections)
		w.WriteHeader(http.StatusTooManyRequests)
		return
	}

	w.WriteHeader(http.StatusOK)
	w.Header().Set("Date", now.Format(http.TimeFormat))
	w.Header().Set("Connection", "Keep-Alive")
	w.Header().Set("Cache-Control", "no-cache")
	w.Header().Set("Content-Type", "audio/mpeg")
	w.Header().Set("Server", "BoringStreamer/4.0")

	// browsers need ID3 tag to identify frames as media to be played
	// mp3 header to designate mp3 stream
	b := []byte{0x49, 0x44, 0x33, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
	_, err := io.Copy(w, bytes.NewReader(b))
	if err != nil {
		log.Printf("Error streaming id3 tag (qid: %v), err=%v", qid, err)
	} else {
		// write mp3 stream to w
		for {
			buf := <-frames
			_, err = io.Copy(w, bytes.NewReader(buf))
			if err != nil {
				break
			}
			br <- broadcastResult{qid, true}
			w.(http.Flusher).Flush()
		}
	}
	br <- broadcastResult{qid, false}
	if *verbose {
		log.Printf("Stopped connection (qid: %v), reason err=%v", qid, err)
	}
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [flags] [path]\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "Browse to listen (e.g. http://localhost:4444/)\n\nflags:\n")
		flag.PrintDefaults()
	}
	flag.Parse()
	if len(flag.Args()) > 1 {
		flag.Usage()
		os.Exit(1)
	}

	path := ""
	switch len(flag.Args()) {
	case 0:
		path = "."
		if *verbose {
			fmt.Printf("Using path %#v, see -h for details.\n", path)
		}
	case 1:
		path = flag.Args()[0]
	}

	if *verbose {
		fmt.Printf("Looking for files available from \"%v\" ...\n", path)
	}

	// check if path is available
	matches, err := filepath.Glob(path)
	if err != nil || len(matches) != 1 {
		fmt.Fprintf(os.Stderr, "Error: \"%v\" unavailable.\n", path)
		os.Exit(1)
	}

	// initialize and start mp3 streamer
	http.Handle("/", streamHandler{*new(mux).start(path)})
	if *verbose {
		fmt.Printf("Waiting for connections on %v\n", *addr)
	}

	err = http.ListenAndServe(*addr, nil)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package images

//genmodules:config

import (
	"fmt"
	"log"
	"math/rand"
	"os"
	"strconv"
	"strings"

	"github.com/Necroforger/Boorudl/extractor"
	"github.com/Necroforger/Fantasia/system"
	"github.com/Necroforger/dream"
)

// Config ...
type Config struct {
	ImageCommands      [][]string
	BooruCommands      [][]string
	ConvolutionKernels [][]string
}

// ImageCommand ...
type ImageCommand struct {
	Name string
	URL  string
}

// NewConfig ...
func NewConfig() *Config {
	return &Config{
		// Default Image commands
		ImageCommands: [][]string{},

		// Default booru commands
		BooruCommands: [][]string{
			{"danbooru", "http://danbooru.donmai.us"},
			{"safebooru", "https://safebooru.org/"},
			{"googleimg", "http://google.com"},
		},
	}
}

// Module ...
type Module struct {
	Config *Config
}

// Build ...
func (m *Module) Build(s *system.System) {
	r := s.CommandRouter

	/////////////////////////////////
	// Convolution filters
	////////////////////////////////
	lastcategory := r.CurrentCategory
	r.CurrentCategory = "Effects"
	r.On("edgedetect", MakeConvolutionFunc(MatrixEdgeDetect, getDivisor(MatrixEdgeDetect), 1)).Set("", "`usage: edge [iteratins]` Detects the edges of the given image")
	r.On("blur", MakeConvolutionFunc(MatrixGaussian, getDivisor(MatrixGaussian), 1)).Set("", "`usage: blur [iterations]` Gaussian blurs the given image")
	r.On("motionblur", MakeConvolutionFunc(MatrixMotionBlur, getDivisor(MatrixMotionBlur), 1)).Set("", "`usage: motionblue [iterations]` Applies a motion blur to the given image")
	r.On("sharpen", MakeConvolutionFunc(MatrixSharpen, getDivisor(MatrixSharpen), 1)).Set("", "`usage: motionblue [iterations]`, sharpens the given image")

	r.CurrentCategory = lastcategory
	///////////////////////////////
	//   Booru commands
	///////////////////////////////
	for _, v := range m.Config.BooruCommands {
		if len(v) < 2 {
			log.Println("error creating booru command " + fmt.Sprint(v) + ", array must be in the form of [command name, booru url]")
			continue
		}
		AddBooru(r, v[0], v[1])
	}

	////////////////////////////////
	//  Custom image commands
	///////////////////////////////
	for _, v := range m.Config.ImageCommands {
		AddImageCommand(r, v)
	}

}

// AddImageCommand makes an image command from an array of strings in the format
// [command name, description, urls...]
func AddImageCommand(r *system.CommandRouter, cmd []string) {
	if len(cmd) < 3 {
		return
	}
	cmdName := cmd[0]

	r.On(cmdName, MakeImageCommand(cmd[2:], true)).Set("", cmd[1])
}

// MakeImageCommand makes an image command
func MakeImageCommand(urls []string, openFiles bool) func(*system.Context) {
	return func(ctx *system.Context) {
		index := int(rand.Float64() * float64(len(urls)))
		path := urls[index]

		// If the path is not a URL, it will check the file system for the image.
		if !strings.HasPrefix(path, "http://") &&
			!strings.HasPrefix(path, "https://") &&
			openFiles {

			f, err := os.Open(path)
			if err != nil {
				ctx.ReplyError(err)
				return
			}

			info, err := f.Stat()
			if err != nil {
				ctx.ReplyError(err)
				return
			}

			if info.IsDir() {
				randFile, err := system.RandomFileInDir(path)
				if err != nil {
					ctx.ReplyError(err)
					return
				}
				ctx.Ses.DG.ChannelFileSend(ctx.Msg.ChannelID, randFile.Name(), randFile)
			} else {
				ctx.Ses.DG.ChannelFileSend(ctx.Msg.ChannelID, info.Name(), f)
			}
			return
		}

		ctx.ReplyEmbed(dream.NewEmbed().
			SetImage(urls[index]).
			SetColor(system.StatusNotify).
			MessageEmbed)

	}
}

// AddBooru adds a booru command to the router
func AddBooru(r *system.CommandRouter, commandName string, booruURL string) {
	r.On(commandName, MakeBooruSearcher(booruURL)).
		Set("", "Returns an image result from ["+commandName+"]("+booruURL+")\n"+
			"Usage: `"+commandName+" [tags] [post index] [to post index]`\n"+
			"Enclose the tag list in quotes to include multiple tags")
}

// MakeBooruSearcher returns a command that searches the given booru link
func MakeBooruSearcher(booruURL string) func(*system.Context) {
	return func(ctx *system.Context) {
		index := 0
		if n, err := strconv.Atoi(ctx.Args.Get(1)); err == nil {
			index = n
		}

		indexTo := index + 1
		if n, err := strconv.Atoi(ctx.Args.Get(2)); err == nil {
			indexTo = n
		}

		if indexTo > index+10 {
			ctx.ReplyError("You cannot bulk view more than 10 images at a time")
		}

		posts, err := extractor.Search(booruURL, extractor.SearchQuery{
			Limit:  indexTo + 1,
			Page:   0,
			Tags:   ctx.Args.Get(0),
			Random: false,
		})
		if err != nil {
			ctx.ReplyError(err)
			return
		}

		for i := index; i < indexTo; i++ {
			if i >= 0 && i < len(posts) {
				post := posts[i]
				ctx.ReplyEmbed(dream.NewEmbed().
					SetColor(system.StatusNotify).
					SetImage(post.ImageURL).
					MessageEmbed)
			}
		}

	}
}
<commit_msg>Ability to change categories of image commands<commit_after>package images

//genmodules:config

import (
	"fmt"
	"log"
	"math/rand"
	"os"
	"strconv"
	"strings"

	"github.com/Necroforger/Boorudl/extractor"
	"github.com/Necroforger/Fantasia/system"
	"github.com/Necroforger/dream"
)

// Config ...
type Config struct {
	ImageCommandsCategory string
	ImageCommands         [][]string
	BooruCommandsCategory string
	BooruCommands         [][]string
	ImageFiltersCategory  string
}

// ImageCommand ...
type ImageCommand struct {
	Name string
	URL  string
}

// NewConfig ...
func NewConfig() *Config {
	return &Config{
		// Default Image commands
		ImageCommands: [][]string{},

		// Default booru commands
		BooruCommands: [][]string{
			{"danbooru", "http://danbooru.donmai.us"},
			{"safebooru", "https://safebooru.org/"},
			{"googleimg", "http://google.com"},
		},
	}
}

// Module ...
type Module struct {
	Config *Config
}

// Build ...
func (m *Module) Build(s *system.System) {
	r := s.CommandRouter
	maincategory := r.CurrentCategory

	setCategory := func(name string) {
		if name != "" {
			r.SetCategory(name)
		} else {
			r.SetCategory(maincategory)
		}
	}

	/////////////////////////////////
	// Convolution filters
	////////////////////////////////
	setCategory(m.Config.ImageFiltersCategory)
	r.On("edgedetect", MakeConvolutionFunc(MatrixEdgeDetect, getDivisor(MatrixEdgeDetect), 1)).Set("", "`usage: edge [iteratins]` Detects the edges of the given image")
	r.On("blur", MakeConvolutionFunc(MatrixGaussian, getDivisor(MatrixGaussian), 1)).Set("", "`usage: blur [iterations]` Gaussian blurs the given image")
	r.On("motionblur", MakeConvolutionFunc(MatrixMotionBlur, getDivisor(MatrixMotionBlur), 1)).Set("", "`usage: motionblue [iterations]` Applies a motion blur to the given image")
	r.On("sharpen", MakeConvolutionFunc(MatrixSharpen, getDivisor(MatrixSharpen), 1)).Set("", "`usage: motionblue [iterations]`, sharpens the given image")

	///////////////////////////////
	//   Booru commands
	///////////////////////////////
	setCategory(m.Config.BooruCommandsCategory)
	for _, v := range m.Config.BooruCommands {
		if len(v) < 2 {
			log.Println("error creating booru command " + fmt.Sprint(v) + ", array must be in the form of [command name, booru url]")
			continue
		}
		AddBooru(r, v[0], v[1])
	}

	////////////////////////////////
	//  Custom image commands
	///////////////////////////////
	setCategory(m.Config.ImageCommandsCategory)
	for _, v := range m.Config.ImageCommands {
		AddImageCommand(r, v)
	}

}

// AddImageCommand makes an image command from an array of strings in the format
// [command name, description, urls...]
func AddImageCommand(r *system.CommandRouter, cmd []string) {
	if len(cmd) < 3 {
		return
	}
	cmdName := cmd[0]

	r.On(cmdName, MakeImageCommand(cmd[2:], true)).Set("", cmd[1])
}

// MakeImageCommand makes an image command
func MakeImageCommand(urls []string, openFiles bool) func(*system.Context) {
	return func(ctx *system.Context) {
		index := int(rand.Float64() * float64(len(urls)))
		path := urls[index]

		// If the path is not a URL, it will check the file system for the image.
		if !strings.HasPrefix(path, "http://") &&
			!strings.HasPrefix(path, "https://") &&
			openFiles {

			f, err := os.Open(path)
			if err != nil {
				ctx.ReplyError(err)
				return
			}

			info, err := f.Stat()
			if err != nil {
				ctx.ReplyError(err)
				return
			}

			if info.IsDir() {
				randFile, err := system.RandomFileInDir(path)
				if err != nil {
					ctx.ReplyError(err)
					return
				}
				ctx.Ses.DG.ChannelFileSend(ctx.Msg.ChannelID, randFile.Name(), randFile)
			} else {
				ctx.Ses.DG.ChannelFileSend(ctx.Msg.ChannelID, info.Name(), f)
			}
			return
		}

		ctx.ReplyEmbed(dream.NewEmbed().
			SetImage(urls[index]).
			SetColor(system.StatusNotify).
			MessageEmbed)

	}
}

// AddBooru adds a booru command to the router
func AddBooru(r *system.CommandRouter, commandName string, booruURL string) {
	r.On(commandName, MakeBooruSearcher(booruURL)).
		Set("", "Returns an image result from ["+commandName+"]("+booruURL+")\n"+
			"Usage: `"+commandName+" [tags] [post index] [to post index]`\n"+
			"Enclose the tag list in quotes to include multiple tags")
}

// MakeBooruSearcher returns a command that searches the given booru link
func MakeBooruSearcher(booruURL string) func(*system.Context) {
	return func(ctx *system.Context) {
		index := 0
		if n, err := strconv.Atoi(ctx.Args.Get(1)); err == nil {
			index = n
		}

		indexTo := index + 1
		if n, err := strconv.Atoi(ctx.Args.Get(2)); err == nil {
			indexTo = n
		}

		if indexTo > index+10 {
			ctx.ReplyError("You cannot bulk view more than 10 images at a time")
		}

		posts, err := extractor.Search(booruURL, extractor.SearchQuery{
			Limit:  indexTo + 1,
			Page:   0,
			Tags:   ctx.Args.Get(0),
			Random: false,
		})
		if err != nil {
			ctx.ReplyError(err)
			return
		}

		for i := index; i < indexTo; i++ {
			if i >= 0 && i < len(posts) {
				post := posts[i]
				ctx.ReplyEmbed(dream.NewEmbed().
					SetColor(system.StatusNotify).
					SetImage(post.ImageURL).
					MessageEmbed)
			}
		}

	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package mailer

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/smtp"
	"strings"

	"github.com/gogits/gogs/modules/log"
	"github.com/gogits/gogs/modules/setting"
)

type Message struct {
	To      []string
	From    string
	Subject string
	Body    string
	User    string
	Type    string
	Massive bool
	Info    string
}

// create mail content
func (m Message) Content() string {
	// set mail type
	contentType := "text/plain; charset=UTF-8"
	if m.Type == "html" {
		contentType = "text/html; charset=UTF-8"
	}

	// create mail content
	content := "From: \"" + m.From + "\" <" + m.User +
		">\r\nSubject: " + m.Subject + "\r\nContent-Type: " + contentType + "\r\n\r\n" + m.Body
	return content
}

var mailQueue chan *Message

func NewMailerContext() {
	mailQueue = make(chan *Message, setting.Cfg.MustInt("mailer", "SEND_BUFFER_LEN", 10))
	go processMailQueue()
}

func processMailQueue() {
	for {
		select {
		case msg := <-mailQueue:
			num, err := Send(msg)
			tos := strings.Join(msg.To, "; ")
			info := ""
			if err != nil {
				if len(msg.Info) > 0 {
					info = ", info: " + msg.Info
				}
				log.Error(4, fmt.Sprintf("Async sent email %d succeed, not send emails: %s%s err: %s", num, tos, info, err))
			} else {
				log.Trace(fmt.Sprintf("Async sent email %d succeed, sent emails: %s%s", num, tos, info))
			}
		}
	}
}

// sendMail allows mail with self-signed certificates.
func sendMail(hostAddressWithPort string, auth smtp.Auth, from string, recipients []string, msgContent []byte) error {
	client, err := smtp.Dial(hostAddressWithPort)
	if err != nil {
		return err
	}

	host, _, _ := net.SplitHostPort(hostAddressWithPort)
	tlsConn := &tls.Config{
		InsecureSkipVerify: true,
		ServerName:         host,
	}
	if err = client.StartTLS(tlsConn); err != nil {
		return err
	}

	if ok, _ := client.Extension("AUTH"); ok && auth != nil {
		if err = client.Auth(auth); err != nil {
			return err
		}
	}

	if err = client.Mail(from); err != nil {
		return err
	}

	for _, rec := range recipients {
		if err = client.Rcpt(rec); err != nil {
			return err
		}
	}

	w, err := client.Data()
	if err != nil {
		return err
	}
	if _, err = w.Write([]byte(msgContent)); err != nil {
		return err
	}

	if err = w.Close(); err != nil {
		return err
	}

	return client.Quit()
}

// Direct Send mail message
func Send(msg *Message) (int, error) {
	log.Trace("Sending mails to: %s", strings.Join(msg.To, "; "))
	host := strings.Split(setting.MailService.Host, ":")

	// get message body
	content := msg.Content()

	if len(msg.To) == 0 {
		return 0, fmt.Errorf("empty receive emails")
	} else if len(msg.Body) == 0 {
		return 0, fmt.Errorf("empty email body")
	}

	auth := smtp.PlainAuth("", setting.MailService.User, setting.MailService.Passwd, host[0])

	if msg.Massive {
		// send mail to multiple emails one by one
		num := 0
		for _, to := range msg.To {
			body := []byte("To: " + to + "\r\n" + content)
			err := sendMail(setting.MailService.Host, auth, msg.From, []string{to}, body)
			if err != nil {
				return num, err
			}
			num++
		}
		return num, nil
	} else {
		body := []byte("To: " + strings.Join(msg.To, ";") + "\r\n" + content)

		// send to multiple emails in one message
		err := sendMail(setting.MailService.Host, auth, msg.From, msg.To, body)
		if err != nil {
			return 0, err
		} else {
			return 1, nil
		}
	}
}

// Async Send mail message
func SendAsync(msg *Message) {
	go func() {
		mailQueue <- msg
	}()
}

// Create html mail message
func NewHtmlMessage(To []string, From, Subject, Body string) Message {
	return Message{
		To:      To,
		From:    From,
		Subject: Subject,
		Body:    Body,
		Type:    "html",
	}
}
<commit_msg>Allow send mail without authentication if SMTP server allow this<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package mailer

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/smtp"
	"strings"

	"github.com/gogits/gogs/modules/log"
	"github.com/gogits/gogs/modules/setting"
)

type Message struct {
	To      []string
	From    string
	Subject string
	Body    string
	User    string
	Type    string
	Massive bool
	Info    string
}

// create mail content
func (m Message) Content() string {
	// set mail type
	contentType := "text/plain; charset=UTF-8"
	if m.Type == "html" {
		contentType = "text/html; charset=UTF-8"
	}

	// create mail content
	content := "From: \"" + m.From + "\" <" + m.User +
		">\r\nSubject: " + m.Subject + "\r\nContent-Type: " + contentType + "\r\n\r\n" + m.Body
	return content
}

var mailQueue chan *Message

func NewMailerContext() {
	mailQueue = make(chan *Message, setting.Cfg.MustInt("mailer", "SEND_BUFFER_LEN", 10))
	go processMailQueue()
}

func processMailQueue() {
	for {
		select {
		case msg := <-mailQueue:
			num, err := Send(msg)
			tos := strings.Join(msg.To, "; ")
			info := ""
			if err != nil {
				if len(msg.Info) > 0 {
					info = ", info: " + msg.Info
				}
				log.Error(4, fmt.Sprintf("Async sent email %d succeed, not send emails: %s%s err: %s", num, tos, info, err))
			} else {
				log.Trace(fmt.Sprintf("Async sent email %d succeed, sent emails: %s%s", num, tos, info))
			}
		}
	}
}

// sendMail allows mail with self-signed certificates.
func sendMail(hostAddressWithPort string, auth smtp.Auth, from string, recipients []string, msgContent []byte) error {
	client, err := smtp.Dial(hostAddressWithPort)
	if err != nil {
		return err
	}

	host, _, _ := net.SplitHostPort(hostAddressWithPort)
	tlsConn := &tls.Config{
		InsecureSkipVerify: true,
		ServerName:         host,
	}
	if err = client.StartTLS(tlsConn); err != nil {
		return err
	}

	if ok, _ := client.Extension("AUTH"); ok && auth != nil {
		if err = client.Auth(auth); err != nil {
			return err
		}
	}

	if err = client.Mail(from); err != nil {
		return err
	}

	for _, rec := range recipients {
		if err = client.Rcpt(rec); err != nil {
			return err
		}
	}

	w, err := client.Data()
	if err != nil {
		return err
	}
	if _, err = w.Write([]byte(msgContent)); err != nil {
		return err
	}

	if err = w.Close(); err != nil {
		return err
	}

	return client.Quit()
}

// Direct Send mail message
func Send(msg *Message) (int, error) {
	log.Trace("Sending mails to: %s", strings.Join(msg.To, "; "))
	host := strings.Split(setting.MailService.Host, ":")

	// get message body
	content := msg.Content()

	if len(msg.To) == 0 {
		return 0, fmt.Errorf("empty receive emails")
	} else if len(msg.Body) == 0 {
		return 0, fmt.Errorf("empty email body")
	}

	var auth smtp.Auth
	if len(setting.MailService.Passwd) > 0 {
		auth = smtp.PlainAuth("", setting.MailService.User, setting.MailService.Passwd, host[0])
	}

	if msg.Massive {
		// send mail to multiple emails one by one
		num := 0
		for _, to := range msg.To {
			body := []byte("To: " + to + "\r\n" + content)
			err := sendMail(setting.MailService.Host, auth, msg.From, []string{to}, body)
			if err != nil {
				return num, err
			}
			num++
		}
		return num, nil
	} else {
		body := []byte("To: " + strings.Join(msg.To, ";") + "\r\n" + content)

		// send to multiple emails in one message
		err := sendMail(setting.MailService.Host, auth, msg.From, msg.To, body)
		if err != nil {
			return 0, err
		} else {
			return 1, nil
		}
	}
}

// Async Send mail message
func SendAsync(msg *Message) {
	go func() {
		mailQueue <- msg
	}()
}

// Create html mail message
func NewHtmlMessage(To []string, From, Subject, Body string) Message {
	return Message{
		To:      To,
		From:    From,
		Subject: Subject,
		Body:    Body,
		Type:    "html",
	}
}
<|endoftext|>


<commit_before>package file

import (
	"os"
	"path/filepath"
	"syscall"
	"time"
)

// winExFileInfo stores windows specific file information.
// At the moment all the information we need is available
// through the Sys() interface.
type winExFileInfo struct {
	os.FileInfo
	fid  FID
	path string
}

// CTime returns the CreationTime from Win32FileAttributeData.
func (fi *winExFileInfo) CTime() time.Time {
	return time.Unix(0, fi.Sys().(syscall.Win32FileAttributeData).CreationTime)
}

// ATime returns the LastAccessTime from Win32FileAttributeData.
func (fi *winExFileInfo) ATime() time.Time {
	return time.Unix(0, fi.Sys().(syscall.Win32FileAttributeData).LastAccessTime)
}

// FID returns the windows version of a file id. The FID for Windows
// is the VolumeSerialNumber (IDHigh) and the FileIndexHigh/Low (IDLow)
func (fi *winExFileInfo) FID() FID {
	return fid
}

// Path returns the full path for the file.
func (fi *winExFileInfo) Path() string {
	return path
}

// newExFileInfo creates a new winExFileInfo from a os.FileInfo.
func newExFileInfo(fi os.FileInfo, path string) *winExFileInfo {
	fid, err := createFID(path)
	if err != nil {
		// do something
	}

	absolute, _ := filepath.Abs(path)
	return &winExFileInfo{
		FileInfo: fi,
		fid:      fid,
		path:     filepath.Clean(absolute),
	}
}

// createFID creates the file by making a windows specific system call
// to retrieve the VolumeSerialNumber and FileIndexHigh/Low. Unfortunately
// these values are not exposed through the Sys() in FileInfo. The code
// for making these calls is a slightly modified version of the code in
// the go os package types_windows.go file.
func createFID(path string) (FID, error) {
	fid := FID{}
	pathp, err := syscall.UTF16PtrFromString(path)
	if err != nil {
		return fid, err
	}
	h, err := syscall.CreateFile(pathp, 0, 0, nil, syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0)
	if err != nil {
		return fid, err
	}
	defer syscall.CloseHandle(h)
	var handleInfo syscall.ByHandleFileInformation
	err = syscall.GetFileInformationByHandle(syscall.Handle(h), &handleInfo)
	if err != nil {
		return fid, err
	}
	fid.IDHigh = uint64(handleInfo.VolumeSerialNumber)
	fid.IDLow = int64(handleInfo.FileIndexHigh)<<32 + int64(handleInfo.FileIndexLow)
	return fid, nil
}
<commit_msg>Fix up references to unsigned int64.<commit_after>package file

import (
	"os"
	"path/filepath"
	"syscall"
	"time"
)

// winExFileInfo stores windows specific file information.
// At the moment all the information we need is available
// through the Sys() interface.
type winExFileInfo struct {
	os.FileInfo
	fid  FID
	path string
}

// CTime returns the CreationTime from Win32FileAttributeData.
func (fi *winExFileInfo) CTime() time.Time {
	return time.Unix(0, fi.Sys().(syscall.Win32FileAttributeData).CreationTime)
}

// ATime returns the LastAccessTime from Win32FileAttributeData.
func (fi *winExFileInfo) ATime() time.Time {
	return time.Unix(0, fi.Sys().(syscall.Win32FileAttributeData).LastAccessTime)
}

// FID returns the windows version of a file id. The FID for Windows
// is the VolumeSerialNumber (IDHigh) and the FileIndexHigh/Low (IDLow)
func (fi *winExFileInfo) FID() FID {
	return fi.fid
}

// Path returns the full path for the file.
func (fi *winExFileInfo) Path() string {
	return fi.path
}

// newExFileInfo creates a new winExFileInfo from a os.FileInfo.
func newExFileInfo(fi os.FileInfo, path string) *winExFileInfo {
	fid, err := createFID(path)
	if err != nil {
		// do something
	}

	absolute, _ := filepath.Abs(path)
	return &winExFileInfo{
		FileInfo: fi,
		fid:      fid,
		path:     filepath.Clean(absolute),
	}
}

// createFID creates the file by making a windows specific system call
// to retrieve the VolumeSerialNumber and FileIndexHigh/Low. Unfortunately
// these values are not exposed through the Sys() in FileInfo. The code
// for making these calls is a slightly modified version of the code in
// the go os package types_windows.go file.
func createFID(path string) (FID, error) {
	fid := FID{}
	pathp, err := syscall.UTF16PtrFromString(path)
	if err != nil {
		return fid, err
	}
	h, err := syscall.CreateFile(pathp, 0, 0, nil, syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0)
	if err != nil {
		return fid, err
	}
	defer syscall.CloseHandle(h)
	var handleInfo syscall.ByHandleFileInformation
	err = syscall.GetFileInformationByHandle(syscall.Handle(h), &handleInfo)
	if err != nil {
		return fid, err
	}
	fid.IDHigh = uint64(handleInfo.VolumeSerialNumber)
	fid.IDLow = uint64(handleInfo.FileIndexHigh)<<32 + uint64(handleInfo.FileIndexLow)
	return fid, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"github.com/afex/hystrix-go/hystrix"
	"github.com/hashicorp/consul/api"
	"github.com/nsqio/go-nsq"
	"io"
	"log"
	"os"
	"strings"
	"time"
)

type message struct {
	topic      string
	body       [][]byte
	ResultChan chan error
}

type LogTask struct {
	Writer        *nsq.Producer
	LogStat       map[string]chan int
	CurrentConfig map[string]string
	Setting       map[string]string
	msgChan       chan *message
	client        *api.Client
	exitChan      chan int
}

func (m *LogTask) Run() {
	m.exitChan = make(chan int)
	m.msgChan = make(chan *message)
	ticker := time.Tick(time.Second * 600)
	config := api.DefaultConfig()
	config.Address = m.Setting["consul_address"]
	config.Datacenter = m.Setting["datacenter"]
	config.Token = m.Setting["consul_token"]
	var err error
	m.client, err = api.NewClient(config)
	if err != nil {
		fmt.Println("reload consul setting failed", err)
	}
	err = m.CheckReload()
	if err != nil {
		fmt.Println("reload consul setting failed", err)
	}
	for {
		select {
		case <-ticker:
			err = m.CheckReload()
			if err != nil {
				fmt.Println("reload consul setting failed", err)
			}
		case <-m.exitChan:
			return
		}
	}
}
func (m *LogTask) Stop() {
	close(m.exitChan)
	for _, v := range m.LogStat {
		close(v)
	}
	m.Writer.Stop()
}
func (m *LogTask) ReadConfigFromConsul() (map[string]string, error) {
	consulSetting := make(map[string]string)
	kv := m.client.KV()
	pairs, _, err := kv.List(m.Setting["cluster"], nil)
	if err != nil {
		return consulSetting, err
	}
	size := len(m.Setting["cluster"]) + 1
	for _, value := range pairs {
		if len(value.Key) > size {
			consulSetting[value.Key[size:]] = string(value.Value)
		}
	}
	return consulSetting, err

}
func (m *LogTask) CheckReload() error {
	newConf, err := m.ReadConfigFromConsul()
	if err != nil {
		return err
	}
	for k, _ := range newConf {
		if m.CurrentConfig[k] != newConf[k] {
			if len(m.CurrentConfig[k]) > 0 {
				close(m.LogStat[k])
				delete(m.LogStat, k)
				delete(m.CurrentConfig, k)
			}
			if len(newConf[k]) > 0 {
				fileNames := strings.Split(newConf[k], ",")
				m.LogStat[k] = make(chan int)
				for _, fileName := range fileNames {
					go m.WriteLoop(m.LogStat[k])
					go m.ReadLog(fileName, k, m.LogStat[k])
				}
			}
		}
	}
	for k, _ := range m.CurrentConfig {
		if m.CurrentConfig[k] != newConf[k] {
			if len(newConf[k]) == 0 {
				close(m.LogStat[k])
				delete(m.LogStat, k)
			}
		}
	}
	m.CurrentConfig = newConf
	return nil
}

func (m *LogTask) ReadLog(file string, topic string, exitchan chan int) {
	fd, err := os.Open(file)
	if err != nil {
		log.Println(err)
		return
	}
	defer fd.Close()
	_, err = fd.Seek(0, io.SeekStart)
	if err != nil {
		return
	}
	if len(m.Setting["read_all"]) == 0 {
		_, err = fd.Seek(0, io.SeekEnd)
		if err != nil {
			return
		}
		log.Println("reading from EOF")
	}
	log.Println("reading ", file)
	reader := bufio.NewReader(fd)
	var body [][]byte
	for {
		select {
		case <-exitchan:
			return
		default:
			line, err := reader.ReadString('\n')
			if err != nil {
				time.Sleep(time.Second)
				line, err = reader.ReadString('\n')
			}
			if err == io.EOF {
				log.Println(file, "READ EOF")
				size0, err := fd.Seek(0, io.SeekCurrent)
				if err != nil {
					return
				}
				fd, err = os.Open(file)
				if err != nil {
					log.Println("open failed", err)
					return
				}
				size1, err := fd.Seek(0, io.SeekEnd)
				if err != nil {
					log.Println(err)
				}
				if size1 < size0 {
					fd.Seek(0, io.SeekCurrent)
				} else {
					fd.Seek(size0, io.SeekStart)
				}
				reader = bufio.NewReader(fd)
				continue
			}
			if err != nil {
				log.Println(err)
				return
			}
			body = append(body, []byte(line))
			if len(body) > *batch {
				msg := &message{
					topic:      topic,
					body:       body,
					ResultChan: make(chan error),
				}
				m.msgChan <- msg
				for {
					err := <-msg.ResultChan
					if err == nil {
						break
					}
					time.Sleep(time.Second)
					m.msgChan <- msg
				}
				body = body[:0]
			}
		}
	}
}

func (m *LogTask) WriteLoop(exitchan chan int) {
	hystrix.ConfigureCommand("NSQWriter", hystrix.CommandConfig{
		Timeout:               1000,
		MaxConcurrentRequests: 1000,
		ErrorPercentThreshold: 25,
	})
	for {
		select {
		case <-m.exitChan:
			return
		case <-exitchan:
			return
		case msg := <-m.msgChan:
			resultChan := make(chan int, 1)
			var err error
			errChan := hystrix.Go("NSQWriter", func() error {
				err = m.Writer.MultiPublish(msg.topic, msg.body)
				if err != nil {
					return err
				}
				resultChan <- 1
				return nil
			}, nil)
			select {
			case <-resultChan:
			case err = <-errChan:
				log.Println("writeNSQ Error", err)
			}
			msg.ResultChan <- err
		}
	}
}
<commit_msg>fix key<commit_after>package main

import (
	"bufio"
	"fmt"
	"github.com/afex/hystrix-go/hystrix"
	"github.com/hashicorp/consul/api"
	"github.com/nsqio/go-nsq"
	"io"
	"log"
	"os"
	"strings"
	"time"
)

type message struct {
	topic      string
	body       [][]byte
	ResultChan chan error
}

type LogTask struct {
	Writer        *nsq.Producer
	LogStat       map[string]chan int
	CurrentConfig map[string]string
	Setting       map[string]string
	msgChan       chan *message
	client        *api.Client
	exitChan      chan int
}

func (m *LogTask) Run() {
	m.exitChan = make(chan int)
	m.msgChan = make(chan *message)
	ticker := time.Tick(time.Second * 600)
	config := api.DefaultConfig()
	config.Address = m.Setting["consul_address"]
	config.Datacenter = m.Setting["datacenter"]
	config.Token = m.Setting["consul_token"]
	var err error
	m.client, err = api.NewClient(config)
	if err != nil {
		fmt.Println("reload consul setting failed", err)
	}
	err = m.CheckReload()
	if err != nil {
		fmt.Println("reload consul setting failed", err)
	}
	for {
		select {
		case <-ticker:
			err = m.CheckReload()
			if err != nil {
				fmt.Println("reload consul setting failed", err)
			}
		case <-m.exitChan:
			return
		}
	}
}
func (m *LogTask) Stop() {
	close(m.exitChan)
	for _, v := range m.LogStat {
		close(v)
	}
	m.Writer.Stop()
}
func (m *LogTask) ReadConfigFromConsul() (map[string]string, error) {
	consulSetting := make(map[string]string)
	kv := m.client.KV()
	pairs, _, err := kv.List(m.Setting["cluster"], nil)
	if err != nil {
		return consulSetting, err
	}
	size := len(m.Setting["cluster"]) + 1
	for _, value := range pairs {
		if len(value.Key) > size && value.Key[size-1] == '/' {
			consulSetting[value.Key[size:]] = string(value.Value)
		}
	}
	return consulSetting, err

}
func (m *LogTask) CheckReload() error {
	newConf, err := m.ReadConfigFromConsul()
	if err != nil {
		return err
	}
	for k, _ := range newConf {
		if m.CurrentConfig[k] != newConf[k] {
			if len(m.CurrentConfig[k]) > 0 {
				close(m.LogStat[k])
				delete(m.LogStat, k)
				delete(m.CurrentConfig, k)
			}
			if len(newConf[k]) > 0 {
				fileNames := strings.Split(newConf[k], ",")
				m.LogStat[k] = make(chan int)
				for _, fileName := range fileNames {
					go m.WriteLoop(m.LogStat[k])
					go m.ReadLog(fileName, k, m.LogStat[k])
				}
			}
		}
	}
	for k, _ := range m.CurrentConfig {
		if m.CurrentConfig[k] != newConf[k] {
			if len(newConf[k]) == 0 {
				close(m.LogStat[k])
				delete(m.LogStat, k)
			}
		}
	}
	m.CurrentConfig = newConf
	return nil
}

func (m *LogTask) ReadLog(file string, topic string, exitchan chan int) {
	fd, err := os.Open(file)
	if err != nil {
		log.Println(err)
		return
	}
	defer fd.Close()
	_, err = fd.Seek(0, io.SeekStart)
	if err != nil {
		return
	}
	if len(m.Setting["read_all"]) == 0 {
		_, err = fd.Seek(0, io.SeekEnd)
		if err != nil {
			return
		}
		log.Println("reading from EOF")
	}
	log.Println("reading ", file)
	reader := bufio.NewReader(fd)
	var body [][]byte
	for {
		select {
		case <-exitchan:
			return
		default:
			line, err := reader.ReadString('\n')
			if err != nil {
				time.Sleep(time.Second)
				line, err = reader.ReadString('\n')
			}
			if err == io.EOF {
				log.Println(file, "READ EOF")
				size0, err := fd.Seek(0, io.SeekCurrent)
				if err != nil {
					return
				}
				fd, err = os.Open(file)
				if err != nil {
					log.Println("open failed", err)
					return
				}
				size1, err := fd.Seek(0, io.SeekEnd)
				if err != nil {
					log.Println(err)
				}
				if size1 < size0 {
					fd.Seek(0, io.SeekCurrent)
				} else {
					fd.Seek(size0, io.SeekStart)
				}
				reader = bufio.NewReader(fd)
				continue
			}
			if err != nil {
				log.Println(err)
				return
			}
			body = append(body, []byte(line))
			if len(body) > *batch {
				msg := &message{
					topic:      topic,
					body:       body,
					ResultChan: make(chan error),
				}
				m.msgChan <- msg
				for {
					err := <-msg.ResultChan
					if err == nil {
						break
					}
					time.Sleep(time.Second)
					m.msgChan <- msg
				}
				body = body[:0]
			}
		}
	}
}

func (m *LogTask) WriteLoop(exitchan chan int) {
	hystrix.ConfigureCommand("NSQWriter", hystrix.CommandConfig{
		Timeout:               1000,
		MaxConcurrentRequests: 1000,
		ErrorPercentThreshold: 25,
	})
	for {
		select {
		case <-m.exitChan:
			return
		case <-exitchan:
			return
		case msg := <-m.msgChan:
			resultChan := make(chan int, 1)
			var err error
			errChan := hystrix.Go("NSQWriter", func() error {
				err = m.Writer.MultiPublish(msg.topic, msg.body)
				if err != nil {
					return err
				}
				resultChan <- 1
				return nil
			}, nil)
			select {
			case <-resultChan:
			case err = <-errChan:
				log.Println("writeNSQ Error", err)
			}
			msg.ResultChan <- err
		}
	}
}
<|endoftext|>


<commit_before>// Copyright © 2017 Matt Tyler <me@matthewtyler.io>
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gke

import (
	"context"
	"encoding/base64"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/container/v1"
	k8s "k8s.io/client-go/kubernetes"
	rest "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
	"net/http"
	"time"
)

type GkeClient struct {
	Project string
	Zone    string
	client  *http.Client
}

func NewGkeClient(gkeClient *GkeClient, ctx context.Context, project, zone string) error {
	client, err := google.DefaultClient(ctx, container.CloudPlatformScope)
	if err != nil {
		return err
	}
	*gkeClient = GkeClient{project, zone, client}
	return nil
}

func (c *GkeClient) DeleteCluster(clusterId string) (string, error) {
	service, err := container.New(c.client)
	if err != nil {
		return "", err
	}

	projectsZonesClustersService := container.NewProjectsZonesClustersService(service)
	projectZonesClustersDeleteCall := projectsZonesClustersService.Delete(c.Project, c.Zone, clusterId)

	op, err := projectZonesClustersDeleteCall.Do()
	if err != nil {
		return "", err
	}

	return op.Name, nil
}

func (c *Cluster) Client() (*k8s.Clientset, error) {
	config, err := clientcmd.BuildConfigFromFlags(c.Endpoint, "")
	if err != nil {
		return nil, err
	}

	config.AuthProvider = &clientcmdapi.AuthProviderConfig{
		Name: "gcp",
	}

	cacert, _ := base64.StdEncoding.DecodeString(c.Auth.ClusterCaCertificate)

	config.TLSClientConfig = rest.TLSClientConfig{
		CAData: cacert,
	}

	return k8s.NewForConfig(config)
}

type Cluster struct {
	Auth     *container.MasterAuth
	Status   string
	Endpoint string
}

func (c *GkeClient) GetCluster(clusterId string) (*Cluster, error) {
	service, err := container.New(c.client)
	if err != nil {
		return nil, err
	}

	projectsZonesClustersService := container.NewProjectsZonesClustersService(service)
	projectZonesClustersGetCall := projectsZonesClustersService.Get(c.Project, c.Zone, clusterId)
	projectZonesClustersGetCall.Fields("status,endpoint,masterAuth")

	cluster, err := projectZonesClustersGetCall.Do()
	if err != nil {
		return nil, err
	}

	return &Cluster{cluster.MasterAuth, cluster.Status, cluster.Endpoint}, nil
}

func (c *GkeClient) CreateCluster(clusterId string) (string, error) {
	service, err := container.New(c.client)
	if err != nil {
		return "", err
	}

	projectsZonesClustersService := container.NewProjectsZonesClustersService(service)

	createClusterRequest := &container.CreateClusterRequest{
		Cluster: &container.Cluster{
			Name:                  clusterId,
			Description:           "A cluster for e2e testing of elasticsearch-operator",
			InitialClusterVersion: "1.7.2",
			InitialNodeCount:      3,
			EnableKubernetesAlpha: true,
			NodeConfig: &container.NodeConfig{
				DiskSizeGb:  40,
				ImageType:   "COS",
				MachineType: "f1-micro",
				OauthScopes: []string{
					"https://www.googleapis.com/auth/compute",
					"https://www.googleapis.com/auth/devstorage.read_only",
					"https://www.googleapis.com/auth/logging.write",
					"https://www.googleapis.com/auth/monitoring.write",
					"https://www.googleapis.com/auth/servicecontrol",
					"https://www.googleapis.com/auth/service.management.readonly",
					"https://www.googleapis.com/auth/trace.append",
				},
			},
		},
	}

	createCall := projectsZonesClustersService.Create(c.Project, c.Zone, createClusterRequest)
	op, err := createCall.Do()
	if err != nil {
		return "", err
	}

	return op.Name, nil
}

func (c *GkeClient) Done(operationId string) error {
	service, err := container.New(c.client)
	if err != nil {
		return err
	}

	projectsZonesOperationsService := container.NewProjectsZonesOperationsService(service)
	projectsZonesOperationsGetCall := projectsZonesOperationsService.Get(c.Project, c.Zone, operationId)
	projectsZonesOperationsGetCall.Fields("status")

	DoGetCall := func() (*container.Operation, error) {
		return projectsZonesOperationsGetCall.Do()
	}

	for op, err := DoGetCall(); op.Status != "DONE"; op, err = DoGetCall() {
		if err != nil {
			return err
		}

		time.Sleep(10 * time.Second)
	}

	return nil
}
<commit_msg>e2e/gke: bump cluster version to 1.7.3<commit_after>// Copyright © 2017 Matt Tyler <me@matthewtyler.io>
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gke

import (
	"context"
	"encoding/base64"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/container/v1"
	k8s "k8s.io/client-go/kubernetes"
	rest "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
	"net/http"
	"time"
)

type GkeClient struct {
	Project string
	Zone    string
	client  *http.Client
}

func NewGkeClient(gkeClient *GkeClient, ctx context.Context, project, zone string) error {
	client, err := google.DefaultClient(ctx, container.CloudPlatformScope)
	if err != nil {
		return err
	}
	*gkeClient = GkeClient{project, zone, client}
	return nil
}

func (c *GkeClient) DeleteCluster(clusterId string) (string, error) {
	service, err := container.New(c.client)
	if err != nil {
		return "", err
	}

	projectsZonesClustersService := container.NewProjectsZonesClustersService(service)
	projectZonesClustersDeleteCall := projectsZonesClustersService.Delete(c.Project, c.Zone, clusterId)

	op, err := projectZonesClustersDeleteCall.Do()
	if err != nil {
		return "", err
	}

	return op.Name, nil
}

func (c *Cluster) Client() (*k8s.Clientset, error) {
	config, err := clientcmd.BuildConfigFromFlags(c.Endpoint, "")
	if err != nil {
		return nil, err
	}

	config.AuthProvider = &clientcmdapi.AuthProviderConfig{
		Name: "gcp",
	}

	cacert, _ := base64.StdEncoding.DecodeString(c.Auth.ClusterCaCertificate)

	config.TLSClientConfig = rest.TLSClientConfig{
		CAData: cacert,
	}

	return k8s.NewForConfig(config)
}

type Cluster struct {
	Auth     *container.MasterAuth
	Status   string
	Endpoint string
}

func (c *GkeClient) GetCluster(clusterId string) (*Cluster, error) {
	service, err := container.New(c.client)
	if err != nil {
		return nil, err
	}

	projectsZonesClustersService := container.NewProjectsZonesClustersService(service)
	projectZonesClustersGetCall := projectsZonesClustersService.Get(c.Project, c.Zone, clusterId)
	projectZonesClustersGetCall.Fields("status,endpoint,masterAuth")

	cluster, err := projectZonesClustersGetCall.Do()
	if err != nil {
		return nil, err
	}

	return &Cluster{cluster.MasterAuth, cluster.Status, cluster.Endpoint}, nil
}

func (c *GkeClient) CreateCluster(clusterId string) (string, error) {
	service, err := container.New(c.client)
	if err != nil {
		return "", err
	}

	projectsZonesClustersService := container.NewProjectsZonesClustersService(service)

	createClusterRequest := &container.CreateClusterRequest{
		Cluster: &container.Cluster{
			Name:                  clusterId,
			Description:           "A cluster for e2e testing of elasticsearch-operator",
			InitialClusterVersion: "1.7.3",
			InitialNodeCount:      3,
			EnableKubernetesAlpha: true,
			NodeConfig: &container.NodeConfig{
				DiskSizeGb:  40,
				ImageType:   "COS",
				MachineType: "f1-micro",
				OauthScopes: []string{
					"https://www.googleapis.com/auth/compute",
					"https://www.googleapis.com/auth/devstorage.read_only",
					"https://www.googleapis.com/auth/logging.write",
					"https://www.googleapis.com/auth/monitoring.write",
					"https://www.googleapis.com/auth/servicecontrol",
					"https://www.googleapis.com/auth/service.management.readonly",
					"https://www.googleapis.com/auth/trace.append",
				},
			},
		},
	}

	createCall := projectsZonesClustersService.Create(c.Project, c.Zone, createClusterRequest)
	op, err := createCall.Do()
	if err != nil {
		return "", err
	}

	return op.Name, nil
}

func (c *GkeClient) Done(operationId string) error {
	service, err := container.New(c.client)
	if err != nil {
		return err
	}

	projectsZonesOperationsService := container.NewProjectsZonesOperationsService(service)
	projectsZonesOperationsGetCall := projectsZonesOperationsService.Get(c.Project, c.Zone, operationId)
	projectsZonesOperationsGetCall.Fields("status")

	DoGetCall := func() (*container.Operation, error) {
		return projectsZonesOperationsGetCall.Do()
	}

	for op, err := DoGetCall(); op.Status != "DONE"; op, err = DoGetCall() {
		if err != nil {
			return err
		}

		time.Sleep(10 * time.Second)
	}

	return nil
}
<|endoftext|>


<commit_before>package y

import (
	"database/sql"
	"log"

	sq "github.com/lann/squirrel"
)

// Execer decribes exec operation
type Execer interface {
	Exec(string, ...interface{}) (sql.Result, error)
}

// Queryer decribes query operation
type Queryer interface {
	Query(string, ...interface{}) (*sql.Rows, error)
	QueryRow(string, ...interface{}) *sql.Row
}

// Versionable mixins a version to a model
type Versionable struct {
	Version sql.NullInt64 `json:"-" y:"_version"`
}

// DB describes db operations
type DB interface {
	Execer
	Queryer
}

// Qualifier updates a select builder if you need
type Qualifier func(sq.SelectBuilder) sq.SelectBuilder

// ByEq returns the filter by squirrel.Eq
var ByEq = func(eq sq.Eq) Qualifier {
	return func(b sq.SelectBuilder) sq.SelectBuilder {
		return b.Where(eq)
	}
}

// ByID returns the filter by ID
var ByID = func(id interface{}) Qualifier {
	return ByEq(sq.Eq{"id": id})
}

// TxPipe run some db statement
type TxPipe func(db DB, v interface{}) error

// Tx executes statements in a transaction
func Tx(db *sql.DB, v interface{}, pipes ...TxPipe) (err error) {
	tx, err := db.Begin()
	if err != nil {
		return
	}
	for _, pipe := range pipes {
		err = pipe(tx, v)
		if err != nil {
			tx.Rollback()
			return
		}
	}
	tx.Commit()
	return
}

func sqlize(q sq.Sqlizer) (sql string, args []interface{}) {
	sql, args, _ = q.ToSql()
	if Debug {
		log.Printf("y/db: SQL: %s, args: %#v", sql, args)
	}
	return
}

func exec(q sq.Sqlizer, db DB) (sql.Result, error) {
	sql, args := sqlize(q)
	return db.Exec(sql, args...)
}

func query(q sq.Sqlizer, db DB) (*sql.Rows, error) {
	sql, args := sqlize(q)
	return db.Query(sql, args...)
}

func queryRow(q sq.Sqlizer, db DB) *sql.Row {
	sql, args := sqlize(q)
	return db.QueryRow(sql, args...)
}
<commit_msg>added MakeVersionable func<commit_after>package y

import (
	"database/sql"
	"log"

	sq "github.com/lann/squirrel"
)

// Execer decribes exec operation
type Execer interface {
	Exec(string, ...interface{}) (sql.Result, error)
}

// Queryer decribes query operation
type Queryer interface {
	Query(string, ...interface{}) (*sql.Rows, error)
	QueryRow(string, ...interface{}) *sql.Row
}

// Versionable mixins a version to a model
type Versionable struct {
	Version sql.NullInt64 `json:"-" y:"_version"`
}

// MakeVersionable inits a new version
func MakeVersionable(n int64) Versionable {
	return Versionable{
		sql.NullInt64{Int64: 1, Valid: true},
	}
}

// DB describes db operations
type DB interface {
	Execer
	Queryer
}

// Qualifier updates a select builder if you need
type Qualifier func(sq.SelectBuilder) sq.SelectBuilder

// ByEq returns the filter by squirrel.Eq
var ByEq = func(eq sq.Eq) Qualifier {
	return func(b sq.SelectBuilder) sq.SelectBuilder {
		return b.Where(eq)
	}
}

// ByID returns the filter by ID
var ByID = func(id interface{}) Qualifier {
	return ByEq(sq.Eq{"id": id})
}

// TxPipe run some db statement
type TxPipe func(db DB, v interface{}) error

// Tx executes statements in a transaction
func Tx(db *sql.DB, v interface{}, pipes ...TxPipe) (err error) {
	tx, err := db.Begin()
	if err != nil {
		return
	}
	for _, pipe := range pipes {
		err = pipe(tx, v)
		if err != nil {
			tx.Rollback()
			return
		}
	}
	tx.Commit()
	return
}

func sqlize(q sq.Sqlizer) (sql string, args []interface{}) {
	sql, args, _ = q.ToSql()
	if Debug {
		log.Printf("y/db: SQL: %s, args: %#v", sql, args)
	}
	return
}

func exec(q sq.Sqlizer, db DB) (sql.Result, error) {
	sql, args := sqlize(q)
	return db.Exec(sql, args...)
}

func query(q sq.Sqlizer, db DB) (*sql.Rows, error) {
	sql, args := sqlize(q)
	return db.Query(sql, args...)
}

func queryRow(q sq.Sqlizer, db DB) *sql.Row {
	sql, args := sqlize(q)
	return db.QueryRow(sql, args...)
}
<|endoftext|>