<commit_before>package main

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/kr/pty"
)

// A Reflex is a single watch + command to execute.
type Reflex struct {
	id           int
	source       string // Describes what config/line defines this Reflex
	startService bool
	backlog      Backlog
	matcher      Matcher
	onlyFiles    bool
	onlyDirs     bool
	command      []string
	subSymbol    string
	done         chan struct{}

	mu      *sync.Mutex // protects killed and running
	killed  bool
	running bool
	timeout time.Duration

	// Used for services (startService = true)
	cmd *exec.Cmd
	tty *os.File
}

// NewReflex prepares a Reflex from a Config, with sanity checking.
func NewReflex(c *Config) (*Reflex, error) {
	matcher, err := ParseMatchers(c.regexes, c.inverseRegexes, c.globs, c.inverseGlobs)
	if err != nil {
		return nil, fmt.Errorf("error parsing glob/regex: %s", err)
	}
	if !c.allFiles {
		matcher = multiMatcher{defaultExcludeMatcher, matcher}
	}
	if len(c.command) == 0 {
		return nil, errors.New("must give command to execute")
	}

	if c.subSymbol == "" {
		return nil, errors.New("substitution symbol must be non-empty")
	}

	substitution := false
	for _, part := range c.command {
		if strings.Contains(part, c.subSymbol) {
			substitution = true
			break
		}
	}

	var backlog Backlog
	if substitution {
		if c.startService {
			return nil, errors.New("using --start-service does not work with a command that has a substitution symbol")
		}
		backlog = NewUniqueFilesBacklog()
	} else {
		backlog = NewUnifiedBacklog()
	}

	if c.onlyFiles && c.onlyDirs {
		return nil, errors.New("cannot specify both --only-files and --only-dirs")
	}

	if c.shutdownTimeout <= 0 {
		return nil, errors.New("shutdown timeout cannot be <= 0")
	}

	reflex := &Reflex{
		id:           reflexID,
		source:       c.source,
		startService: c.startService,
		backlog:      backlog,
		matcher:      matcher,
		onlyFiles:    c.onlyFiles,
		onlyDirs:     c.onlyDirs,
		command:      c.command,
		subSymbol:    c.subSymbol,
		done:         make(chan struct{}),
		timeout:      c.shutdownTimeout,
		mu:           &sync.Mutex{},
	}
	reflexID++

	return reflex, nil
}

func (r *Reflex) String() string {
	var buf bytes.Buffer
	fmt.Fprintln(&buf, "Reflex from", r.source)
	fmt.Fprintln(&buf, "| ID:", r.id)
	for _, matcherInfo := range strings.Split(r.matcher.String(), "\n") {
		fmt.Fprintln(&buf, "|", matcherInfo)
	}
	if r.onlyFiles {
		fmt.Fprintln(&buf, "| Only matching files.")
	} else if r.onlyDirs {
		fmt.Fprintln(&buf, "| Only matching directories.")
	}
	if !r.startService {
		fmt.Fprintln(&buf, "| Substitution symbol", r.subSymbol)
	}
	replacer := strings.NewReplacer(r.subSymbol, "<filename>")
	command := make([]string, len(r.command))
	for i, part := range r.command {
		command[i] = replacer.Replace(part)
	}
	fmt.Fprintln(&buf, "| Command:", command)
	fmt.Fprintln(&buf, "+---------")
	return buf.String()
}

// filterMatching passes on messages matching the regex/glob.
func (r *Reflex) filterMatching(out chan<- string, in <-chan string) {
	for name := range in {
		if !r.matcher.Match(name) {
			continue
		}

		if r.onlyFiles || r.onlyDirs {
			stat, err := os.Stat(name)
			if err != nil {
				continue
			}
			if (r.onlyFiles && stat.IsDir()) || (r.onlyDirs && !stat.IsDir()) {
				continue
			}
		}
		out <- name
	}
}

// batch receives file notification events and batches them up. It's a bit
// tricky, but here's what it accomplishes:
// * When we initially get a message, wait a bit and batch messages before
//   trying to send anything. This is because the file events come in bursts.
// * Once it's time to send, don't do it until the out channel is unblocked.
//   In the meantime, keep batching. When we've sent off all the batched
//   messages, go back to the beginning.
func (r *Reflex) batch(out chan<- string, in <-chan string) {
	for name := range in {
		r.backlog.Add(name)
		timer := time.NewTimer(300 * time.Millisecond)
	outer:
		for {
			select {
			case name := <-in:
				r.backlog.Add(name)
			case <-timer.C:
				for {
					select {
					case name := <-in:
						r.backlog.Add(name)
					case out <- r.backlog.Next():
						if r.backlog.RemoveOne() {
							break outer
						}
					}
				}
			}
		}
	}
}

// runEach runs the command on each name that comes through the names channel.
// Each {} is replaced by the name of the file. The output of the command is
// passed line-by-line to the stdout chan.
func (r *Reflex) runEach(names <-chan string) {
	for name := range names {
		if r.startService {
			if r.Running() {
				infoPrintln(r.id, "Killing service")
				r.terminate()
			}
			infoPrintln(r.id, "Starting service")
			r.runCommand(name, stdout)
		} else {
			r.runCommand(name, stdout)
			<-r.done
			r.mu.Lock()
			r.running = false
			r.mu.Unlock()
		}
	}
}

func (r *Reflex) terminate() {
	r.mu.Lock()
	r.killed = true
	r.mu.Unlock()
	// Write ascii 3 (what you get from ^C) to the controlling pty.
	// (This won't do anything if the process already died as the write will
	// simply fail.)
	r.tty.Write([]byte{3})

	timer := time.NewTimer(r.timeout)
	sig := syscall.SIGINT
	for {
		select {
		case <-r.done:
			return
		case <-timer.C:
			if sig == syscall.SIGINT {
				infoPrintln(r.id, "Sending SIGINT signal...")
			} else {
				infoPrintln(r.id, "Sending SIGKILL signal...")
			}

			// Instead of killing the process, we want to kill its
			// whole pgroup in order to clean up any children the
			// process may have created.
			if err := syscall.Kill(-1*r.cmd.Process.Pid, sig); err != nil {
				infoPrintln(r.id, "Error killing:", err)
				if err.(syscall.Errno) == syscall.ESRCH { // no such process
					return
				}
			}
			// After SIGINT doesn't do anything, try SIGKILL next.
			timer.Reset(r.timeout)
			sig = syscall.SIGKILL
		}
	}
}

func replaceSubSymbol(command []string, subSymbol string, name string) []string {
	replacer := strings.NewReplacer(subSymbol, name)
	newCommand := make([]string, len(command))
	for i, c := range command {
		newCommand[i] = replacer.Replace(c)
	}
	return newCommand
}

var seqCommands = &sync.Mutex{}

// runCommand runs the given Command. All output is passed line-by-line to the
// stdout channel.
func (r *Reflex) runCommand(name string, stdout chan<- OutMsg) {
	command := replaceSubSymbol(r.command, r.subSymbol, name)
	cmd := exec.Command(command[0], command[1:]...)
	r.cmd = cmd

	if flagSequential {
		seqCommands.Lock()
	}

	tty, err := pty.Start(cmd)
	if err != nil {
		infoPrintln(r.id, err)
		return
	}
	r.tty = tty

	go func() {
		scanner := bufio.NewScanner(tty)
		for scanner.Scan() {
			stdout <- OutMsg{r.id, scanner.Text()}
		}
		// Intentionally ignoring scanner.Err() for now. Unfortunately,
		// the pty returns a read error when the child dies naturally,
		// so I'm just going to ignore errors here unless I can find a
		// better way to handle it.
	}()

	r.mu.Lock()
	r.running = true
	r.mu.Unlock()
	go func() {
		err := cmd.Wait()
		if !r.Killed() && err != nil {
			stdout <- OutMsg{r.id, fmt.Sprintf("(error exit: %s)", err)}
		}
		r.done <- struct{}{}
		if flagSequential {
			seqCommands.Unlock()
		}
	}()
}

func (r *Reflex) Start(changes <-chan string) {
	filtered := make(chan string)
	batched := make(chan string)
	go r.filterMatching(filtered, changes)
	go r.batch(batched, filtered)
	go r.runEach(batched)
	if r.startService {
		// Easy hack to kick off the initial start.
		infoPrintln(r.id, "Starting service")
		r.runCommand("", stdout)
	}
}

func (r *Reflex) Killed() bool {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.killed
}

func (r *Reflex) Running() bool {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.running
}
<commit_msg>Batch events until files stop changing<commit_after>package main

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/kr/pty"
)

// A Reflex is a single watch + command to execute.
type Reflex struct {
	id           int
	source       string // Describes what config/line defines this Reflex
	startService bool
	backlog      Backlog
	matcher      Matcher
	onlyFiles    bool
	onlyDirs     bool
	command      []string
	subSymbol    string
	done         chan struct{}

	mu      *sync.Mutex // protects killed and running
	killed  bool
	running bool
	timeout time.Duration

	// Used for services (startService = true)
	cmd *exec.Cmd
	tty *os.File
}

// NewReflex prepares a Reflex from a Config, with sanity checking.
func NewReflex(c *Config) (*Reflex, error) {
	matcher, err := ParseMatchers(c.regexes, c.inverseRegexes, c.globs, c.inverseGlobs)
	if err != nil {
		return nil, fmt.Errorf("error parsing glob/regex: %s", err)
	}
	if !c.allFiles {
		matcher = multiMatcher{defaultExcludeMatcher, matcher}
	}
	if len(c.command) == 0 {
		return nil, errors.New("must give command to execute")
	}

	if c.subSymbol == "" {
		return nil, errors.New("substitution symbol must be non-empty")
	}

	substitution := false
	for _, part := range c.command {
		if strings.Contains(part, c.subSymbol) {
			substitution = true
			break
		}
	}

	var backlog Backlog
	if substitution {
		if c.startService {
			return nil, errors.New("using --start-service does not work with a command that has a substitution symbol")
		}
		backlog = NewUniqueFilesBacklog()
	} else {
		backlog = NewUnifiedBacklog()
	}

	if c.onlyFiles && c.onlyDirs {
		return nil, errors.New("cannot specify both --only-files and --only-dirs")
	}

	if c.shutdownTimeout <= 0 {
		return nil, errors.New("shutdown timeout cannot be <= 0")
	}

	reflex := &Reflex{
		id:           reflexID,
		source:       c.source,
		startService: c.startService,
		backlog:      backlog,
		matcher:      matcher,
		onlyFiles:    c.onlyFiles,
		onlyDirs:     c.onlyDirs,
		command:      c.command,
		subSymbol:    c.subSymbol,
		done:         make(chan struct{}),
		timeout:      c.shutdownTimeout,
		mu:           &sync.Mutex{},
	}
	reflexID++

	return reflex, nil
}

func (r *Reflex) String() string {
	var buf bytes.Buffer
	fmt.Fprintln(&buf, "Reflex from", r.source)
	fmt.Fprintln(&buf, "| ID:", r.id)
	for _, matcherInfo := range strings.Split(r.matcher.String(), "\n") {
		fmt.Fprintln(&buf, "|", matcherInfo)
	}
	if r.onlyFiles {
		fmt.Fprintln(&buf, "| Only matching files.")
	} else if r.onlyDirs {
		fmt.Fprintln(&buf, "| Only matching directories.")
	}
	if !r.startService {
		fmt.Fprintln(&buf, "| Substitution symbol", r.subSymbol)
	}
	replacer := strings.NewReplacer(r.subSymbol, "<filename>")
	command := make([]string, len(r.command))
	for i, part := range r.command {
		command[i] = replacer.Replace(part)
	}
	fmt.Fprintln(&buf, "| Command:", command)
	fmt.Fprintln(&buf, "+---------")
	return buf.String()
}

// filterMatching passes on messages matching the regex/glob.
func (r *Reflex) filterMatching(out chan<- string, in <-chan string) {
	for name := range in {
		if !r.matcher.Match(name) {
			continue
		}

		if r.onlyFiles || r.onlyDirs {
			stat, err := os.Stat(name)
			if err != nil {
				continue
			}
			if (r.onlyFiles && stat.IsDir()) || (r.onlyDirs && !stat.IsDir()) {
				continue
			}
		}
		out <- name
	}
}

// batch receives file notification events and batches them up. It's a bit
// tricky, but here's what it accomplishes:
// * When we initially get a message, wait a bit and batch messages before
//   trying to send anything. This is because the file events come in bursts.
// * Once it's time to send, don't do it until the out channel is unblocked.
//   In the meantime, keep batching. When we've sent off all the batched
//   messages, go back to the beginning.
func (r *Reflex) batch(out chan<- string, in <-chan string) {
	for name := range in {
		r.backlog.Add(name)
		timer := time.NewTimer(300 * time.Millisecond)
	outer:
		for {
			select {
			case name := <-in:
				r.backlog.Add(name)
				if !timer.Stop() {
					<-timer.C
				}
				timer.Reset(300 * time.Millisecond)
			case <-timer.C:
				for {
					select {
					case name := <-in:
						r.backlog.Add(name)
					case out <- r.backlog.Next():
						if r.backlog.RemoveOne() {
							break outer
						}
					}
				}
			}
		}
	}
}

// runEach runs the command on each name that comes through the names channel.
// Each {} is replaced by the name of the file. The output of the command is
// passed line-by-line to the stdout chan.
func (r *Reflex) runEach(names <-chan string) {
	for name := range names {
		if r.startService {
			if r.Running() {
				infoPrintln(r.id, "Killing service")
				r.terminate()
			}
			infoPrintln(r.id, "Starting service")
			r.runCommand(name, stdout)
		} else {
			r.runCommand(name, stdout)
			<-r.done
			r.mu.Lock()
			r.running = false
			r.mu.Unlock()
		}
	}
}

func (r *Reflex) terminate() {
	r.mu.Lock()
	r.killed = true
	r.mu.Unlock()
	// Write ascii 3 (what you get from ^C) to the controlling pty.
	// (This won't do anything if the process already died as the write will
	// simply fail.)
	r.tty.Write([]byte{3})

	timer := time.NewTimer(r.timeout)
	sig := syscall.SIGINT
	for {
		select {
		case <-r.done:
			return
		case <-timer.C:
			if sig == syscall.SIGINT {
				infoPrintln(r.id, "Sending SIGINT signal...")
			} else {
				infoPrintln(r.id, "Sending SIGKILL signal...")
			}

			// Instead of killing the process, we want to kill its
			// whole pgroup in order to clean up any children the
			// process may have created.
			if err := syscall.Kill(-1*r.cmd.Process.Pid, sig); err != nil {
				infoPrintln(r.id, "Error killing:", err)
				if err.(syscall.Errno) == syscall.ESRCH { // no such process
					return
				}
			}
			// After SIGINT doesn't do anything, try SIGKILL next.
			timer.Reset(r.timeout)
			sig = syscall.SIGKILL
		}
	}
}

func replaceSubSymbol(command []string, subSymbol string, name string) []string {
	replacer := strings.NewReplacer(subSymbol, name)
	newCommand := make([]string, len(command))
	for i, c := range command {
		newCommand[i] = replacer.Replace(c)
	}
	return newCommand
}

var seqCommands = &sync.Mutex{}

// runCommand runs the given Command. All output is passed line-by-line to the
// stdout channel.
func (r *Reflex) runCommand(name string, stdout chan<- OutMsg) {
	command := replaceSubSymbol(r.command, r.subSymbol, name)
	cmd := exec.Command(command[0], command[1:]...)
	r.cmd = cmd

	if flagSequential {
		seqCommands.Lock()
	}

	tty, err := pty.Start(cmd)
	if err != nil {
		infoPrintln(r.id, err)
		return
	}
	r.tty = tty

	go func() {
		scanner := bufio.NewScanner(tty)
		for scanner.Scan() {
			stdout <- OutMsg{r.id, scanner.Text()}
		}
		// Intentionally ignoring scanner.Err() for now. Unfortunately,
		// the pty returns a read error when the child dies naturally,
		// so I'm just going to ignore errors here unless I can find a
		// better way to handle it.
	}()

	r.mu.Lock()
	r.running = true
	r.mu.Unlock()
	go func() {
		err := cmd.Wait()
		if !r.Killed() && err != nil {
			stdout <- OutMsg{r.id, fmt.Sprintf("(error exit: %s)", err)}
		}
		r.done <- struct{}{}
		if flagSequential {
			seqCommands.Unlock()
		}
	}()
}

func (r *Reflex) Start(changes <-chan string) {
	filtered := make(chan string)
	batched := make(chan string)
	go r.filterMatching(filtered, changes)
	go r.batch(batched, filtered)
	go r.runEach(batched)
	if r.startService {
		// Easy hack to kick off the initial start.
		infoPrintln(r.id, "Starting service")
		r.runCommand("", stdout)
	}
}

func (r *Reflex) Killed() bool {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.killed
}

func (r *Reflex) Running() bool {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.running
}
<|endoftext|>


<commit_before>// Copyright 2016 David Lavieri.  All rights reserved.
// Use of this source code is governed by a MIT License
// License that can be found in the LICENSE file.

package goradix

// ----------------------- Remove ------------------------ //

// Remove an item, require string
func (r *Radix) Remove(s string) bool {
	return r.RemoveBytes([]byte(s))
}

// RemoveBytes an item, require slice of byte
func (r *Radix) RemoveBytes(bs []byte) bool {
	succeed, _ := r.cRemove(bs)
	return succeed
}

// return (succeed, delete child)

func (r *Radix) cRemove(bs []byte) (bool, bool) {
	r.mu.Lock()

	lbs, matches, _ := r.match(bs)

	if matches == len(r.Path) {
		if len(lbs) > 0 {
			for i, c := range r.nodes {
				if sd, dc := c.cRemove(lbs); sd {
					if dc {
						r.removeChild(i)
					}

					r.mu.Unlock()

					return sd, false
				}
			}

			r.mu.Unlock()

			return false, false
		}

		r.set(nil)
		r.key = false
		isLeft := r.leaf

		if !isLeft {
			r.mergeChild()
		}

		r.mu.Unlock()

		return true, isLeft
	}

	r.mu.Unlock()

	return false, false
}

func (r *Radix) removeChild(i int) {
	copy(r.nodes[i:], r.nodes[i+1:])
	r.nodes[len(r.nodes)-1] = nil
	r.nodes = r.nodes[:len(r.nodes)-1]

	r.mergeChild()
}

func (r *Radix) mergeChild() {
	if len(r.nodes) == 1 {
		c := r.nodes[0]
		c.mu.RLock()
		r.Path = append(r.Path, c.Path...)
		c.mu.RUnlock()
	}
}
<commit_msg>Fix remove locks<commit_after>// Copyright 2016 David Lavieri.  All rights reserved.
// Use of this source code is governed by a MIT License
// License that can be found in the LICENSE file.

package goradix

// ----------------------- Remove ------------------------ //

// Remove an item, require string
func (r *Radix) Remove(s string) bool {
	return r.RemoveBytes([]byte(s))
}

// RemoveBytes an item, require slice of byte
func (r *Radix) RemoveBytes(bs []byte) bool {
	succeed, _ := r.cRemove(bs)
	return succeed
}

// return (succeed, delete child)

func (r *Radix) cRemove(bs []byte) (bool, bool) {
	r.lock()

	lbs, matches, _ := r.match(bs)

	if matches == len(r.Path) {
		if len(lbs) > 0 {
			for i, c := range r.nodes {
				if sd, dc := c.cRemove(lbs); sd {
					if dc {
						r.removeChild(i)
					}

					r.unlock()

					return sd, false
				}
			}

			r.unlock()

			return false, false
		}

		r.set(nil)
		r.key = false
		isLeft := r.leaf

		if !isLeft {
			r.mergeChild()
		}

		r.unlock()

		return true, isLeft
	}

	r.unlock()

	return false, false
}

func (r *Radix) removeChild(i int) {
	copy(r.nodes[i:], r.nodes[i+1:])
	r.nodes[len(r.nodes)-1] = nil
	r.nodes = r.nodes[:len(r.nodes)-1]

	r.mergeChild()
}

func (r *Radix) mergeChild() {
	if len(r.nodes) == 1 {
		c := r.nodes[0]
		c.mu.RLock()
		r.Path = append(r.Path, c.Path...)
		c.mu.RUnlock()
	}
}
<|endoftext|>


<commit_before>package acerender

import (
	"bytes"
	"io"
	"net/http"
	"strings"

	"github.com/go-martini/martini"
	"github.com/martini-contrib/render"
	"github.com/yosssi/ace"
	"github.com/yosssi/ace-proxy"
)

const defaultContentType = render.ContentHTML + "; charset=utf-8"

// Render is an interface for parsing Ace templates and redering HTML.
type Render interface {
	HTML(status int, name string, v interface{}, opts *ace.Options)
}

// render represents a renderer of Ace templates.
type renderer struct {
	http.ResponseWriter
	req *http.Request
	p   *proxy.Proxy
}

// HTML parses the Ace templates and renders HTML to the response writer.
func (r *renderer) HTML(status int, name string, v interface{}, opts *ace.Options) {
	var basePath, innerPath string

	paths := strings.Split(name, ":")

	basePath = paths[0]

	if len(paths) > 1 {
		innerPath = paths[1]
	}

	tpl, err := r.p.Load(basePath, innerPath, opts)

	if err != nil {
		http.Error(r, err.Error(), http.StatusInternalServerError)
		return
	}

	buf := new(bytes.Buffer)

	if err := tpl.Execute(buf, v); err != nil {
		http.Error(r, err.Error(), http.StatusInternalServerError)
		return
	}

	r.Header().Set(render.ContentType, defaultContentType)
	r.WriteHeader(status)
	io.Copy(r, buf)
}

// Renderer is a Martini middleware that maps a render.Render service into the Martini handler chain.
func Renderer(opts *Options) martini.Handler {
	opts = initializeOptions(opts)

	return func(res http.ResponseWriter, req *http.Request, c martini.Context) {
		c.MapTo(
			&renderer{
				ResponseWriter: res,
				req:            req,
				p:              proxy.New(opts.AceOptions),
			},
			(*Render)(nil),
		)
	}
}
<commit_msg>Update render.go<commit_after>package acerender

import (
	"bytes"
	"html/template"
	"io"
	"net/http"
	"strings"

	"github.com/go-martini/martini"
	"github.com/martini-contrib/render"
	"github.com/yosssi/ace"
	"github.com/yosssi/ace-proxy"
)

const defaultContentType = render.ContentHTML + "; charset=utf-8"

// Render is an interface for parsing Ace templates and redering HTML.
type Render interface {
	HTML(status int, name string, v interface{}, opts *ace.Options)
}

// render represents a renderer of Ace templates.
type renderer struct {
	http.ResponseWriter
	req *http.Request
	p   *proxy.Proxy
}

// HTML parses the Ace templates and renders HTML to the response writer.
func (r *renderer) HTML(status int, name string, v interface{}, opts *ace.Options) {
	var basePath, innerPath string

	paths := strings.Split(name, ":")

	basePath = paths[0]

	if len(paths) > 1 {
		innerPath = paths[1]
	}

	tplc, errc := r.p.Load(basePath, innerPath, opts)

	var tpl *template.Template

	select {
	case tpl = <-tplc:
	case err := <-errc:
		http.Error(r, err.Error(), http.StatusInternalServerError)
		return
	}

	buf := new(bytes.Buffer)

	if err := tpl.Execute(buf, v); err != nil {
		http.Error(r, err.Error(), http.StatusInternalServerError)
		return
	}

	r.Header().Set(render.ContentType, defaultContentType)
	r.WriteHeader(status)
	io.Copy(r, buf)
}

// Renderer is a Martini middleware that maps a render.Render service into the Martini handler chain.
func Renderer(opts *Options) martini.Handler {
	opts = initializeOptions(opts)

	return func(res http.ResponseWriter, req *http.Request, c martini.Context) {
		c.MapTo(
			&renderer{
				ResponseWriter: res,
				req:            req,
				p:              proxy.New(opts.AceOptions),
			},
			(*Render)(nil),
		)
	}
}
<|endoftext|>


<commit_before>package reopen

import (
	"bufio"
	"io"
	"io/ioutil"
	"os"
	"sync"
	"time"
)

// Reopener interface defines something that can be reopened
type Reopener interface {
	Reopen() error
}

// Writer is a writer that also can be reopened
type Writer interface {
	Reopener
	io.Writer
}

// WriteCloser is a io.WriteCloser that can also be reopened
type WriteCloser interface {
	Reopener
	io.WriteCloser
}

// FileWriter that can also be reopened
type FileWriter struct {
	mu   sync.Mutex // ensures close / reopen / write are not called at the same time, protects f
	f    *os.File
	mode os.FileMode
	name string
}

// Close calls the underlyding File.Close()
func (f *FileWriter) Close() error {
	f.mu.Lock()
	err := f.f.Close()
	f.mu.Unlock()
	return err
}

// mutex free version
func (f *FileWriter) reopen() error {
	if f.f != nil {
		f.f.Close()
		f.f = nil
	}
	newf, err := os.OpenFile(f.name, os.O_WRONLY|os.O_APPEND|os.O_CREATE, f.mode)
	if err != nil {
		f.f = nil
		return err
	}
	f.f = newf

	return nil
}

// Reopen the file
func (f *FileWriter) Reopen() error {
	f.mu.Lock()
	err := f.reopen()
	f.mu.Unlock()
	return err
}

// Write implements the stander io.Writer interface
func (f *FileWriter) Write(p []byte) (int, error) {
	f.mu.Lock()
	n, err := f.f.Write(p)
	f.mu.Unlock()
	return n, err
}

// NewFileWriter opens a file for appending and writing and can be reopened.
// it is a ReopenWriteCloser...
func NewFileWriter(name string) (*FileWriter, error) {
	// Standard default mode
	return NewFileWriterMode(name, 0666)
}

// NewFileWriterMode opens a Reopener file with a specific permission
func NewFileWriterMode(name string, mode os.FileMode) (*FileWriter, error) {
	writer := FileWriter{
		f:    nil,
		name: name,
		mode: mode,
	}
	err := writer.reopen()
	if err != nil {
		return nil, err
	}
	return &writer, nil
}

// BufferedFileWriter is buffer writer than can be reopned
type BufferedFileWriter struct {
	mu         sync.Mutex
	quitChan   chan bool
	done       bool
	OrigWriter *FileWriter
	BufWriter  *bufio.Writer
}

// Reopen implement Reopener
func (bw *BufferedFileWriter) Reopen() error {
	bw.mu.Lock()
	bw.BufWriter.Flush()

	// use non-mutex version since we are using this one
	err := bw.OrigWriter.reopen()

	bw.BufWriter.Reset(io.Writer(bw.OrigWriter))
	bw.mu.Unlock()

	return err
}

// Close flushes the internal buffer and closes the destination file
func (bw *BufferedFileWriter) Close() error {
	bw.quitChan <- true
	bw.mu.Lock()
	bw.done = true
	bw.BufWriter.Flush()
	bw.OrigWriter.f.Close()
	bw.mu.Unlock()
	return nil
}

// Write implements io.Writer (and reopen.Writer)
func (bw *BufferedFileWriter) Write(p []byte) (int, error) {
	bw.mu.Lock()
	n, err := bw.BufWriter.Write(p)

	// Special Case... if the used space in the buffer is LESS than
	// the input, then we did a flush in the middle of the line
	// and the full log line was not sent on its way.
	if bw.BufWriter.Buffered() < len(p) {
		bw.BufWriter.Flush()
	}

	bw.mu.Unlock()
	return n, err
}

// Flush flushes the buffer.
func (bw *BufferedFileWriter) Flush() {
	bw.mu.Lock()
	// could add check if bw.done already
	//  should never happen
	bw.BufWriter.Flush()
	bw.OrigWriter.f.Sync()
	bw.mu.Unlock()
}

// flushDaemon periodically flushes the log file buffers.
func (bw *BufferedFileWriter) flushDaemon(interval time.Duration) {
	ticker := time.Tick(interval)
	for {
		select {
		case <-bw.quitChan:
			return
		case <-ticker:
			bw.Flush()
		}
	}
}

const bufferSize = 256 * 1024
const flushInterval = 30 * time.Second

// NewBufferedFileWriter opens a buffered file that is periodically
//  flushed.
func NewBufferedFileWriter(w *FileWriter) *BufferedFileWriter {
	return NewBufferedFileWriterSize(w, bufferSize, flushInterval)
}

// NewBufferedFileWriterSize opens a buffered file with the given size that is periodically
//  flushed on the given interval.
func NewBufferedFileWriterSize(w *FileWriter, size int, flush time.Duration) *BufferedFileWriter {
	bw := BufferedFileWriter{
		quitChan:   make(chan bool, 1),
		OrigWriter: w,
		BufWriter:  bufio.NewWriterSize(w, size),
	}
	go bw.flushDaemon(flush)
	return &bw
}

type multiReopenWriter struct {
	writers []Writer
}

// Reopen reopens all child Reopeners
func (t *multiReopenWriter) Reopen() error {
	for _, w := range t.writers {
		err := w.Reopen()
		if err != nil {
			return err
		}
	}
	return nil
}

// Write implements standard io.Write and reopen.Write
func (t *multiReopenWriter) Write(p []byte) (int, error) {
	for _, w := range t.writers {
		n, err := w.Write(p)
		if err != nil {
			return n, err
		}
		if n != len(p) {
			return n, io.ErrShortWrite
		}
	}
	return len(p), nil
}

// MultiWriter creates a writer that duplicates its writes to all the
// provided writers, similar to the Unix tee(1) command.
//  Also allow reopen
func MultiWriter(writers ...Writer) Writer {
	w := make([]Writer, len(writers))
	copy(w, writers)
	return &multiReopenWriter{w}
}

type nopReopenWriteCloser struct {
	io.Writer
}

func (nopReopenWriteCloser) Reopen() error {
	return nil
}

func (nopReopenWriteCloser) Close() error {
	return nil
}

// NopWriter turns a normal writer into a ReopenWriter
//  by doing a NOP on Reopen.   See https://en.wikipedia.org/wiki/NOP
func NopWriter(w io.Writer) WriteCloser {
	return nopReopenWriteCloser{w}
}

// Reopenable versions of os.Stdout, os.Stderr, /dev/null (reopen does nothing)
var (
	Stdout  = NopWriter(os.Stdout)
	Stderr  = NopWriter(os.Stderr)
	Discard = NopWriter(ioutil.Discard)
)
<commit_msg>Close #6 goroutine leak due to time.Tick<commit_after>package reopen

import (
	"bufio"
	"io"
	"io/ioutil"
	"os"
	"sync"
	"time"
)

// Reopener interface defines something that can be reopened
type Reopener interface {
	Reopen() error
}

// Writer is a writer that also can be reopened
type Writer interface {
	Reopener
	io.Writer
}

// WriteCloser is a io.WriteCloser that can also be reopened
type WriteCloser interface {
	Reopener
	io.WriteCloser
}

// FileWriter that can also be reopened
type FileWriter struct {
	mu   sync.Mutex // ensures close / reopen / write are not called at the same time, protects f
	f    *os.File
	mode os.FileMode
	name string
}

// Close calls the underlyding File.Close()
func (f *FileWriter) Close() error {
	f.mu.Lock()
	err := f.f.Close()
	f.mu.Unlock()
	return err
}

// mutex free version
func (f *FileWriter) reopen() error {
	if f.f != nil {
		f.f.Close()
		f.f = nil
	}
	newf, err := os.OpenFile(f.name, os.O_WRONLY|os.O_APPEND|os.O_CREATE, f.mode)
	if err != nil {
		f.f = nil
		return err
	}
	f.f = newf

	return nil
}

// Reopen the file
func (f *FileWriter) Reopen() error {
	f.mu.Lock()
	err := f.reopen()
	f.mu.Unlock()
	return err
}

// Write implements the stander io.Writer interface
func (f *FileWriter) Write(p []byte) (int, error) {
	f.mu.Lock()
	n, err := f.f.Write(p)
	f.mu.Unlock()
	return n, err
}

// NewFileWriter opens a file for appending and writing and can be reopened.
// it is a ReopenWriteCloser...
func NewFileWriter(name string) (*FileWriter, error) {
	// Standard default mode
	return NewFileWriterMode(name, 0666)
}

// NewFileWriterMode opens a Reopener file with a specific permission
func NewFileWriterMode(name string, mode os.FileMode) (*FileWriter, error) {
	writer := FileWriter{
		f:    nil,
		name: name,
		mode: mode,
	}
	err := writer.reopen()
	if err != nil {
		return nil, err
	}
	return &writer, nil
}

// BufferedFileWriter is buffer writer than can be reopned
type BufferedFileWriter struct {
	mu         sync.Mutex
	quitChan   chan bool
	done       bool
	OrigWriter *FileWriter
	BufWriter  *bufio.Writer
}

// Reopen implement Reopener
func (bw *BufferedFileWriter) Reopen() error {
	bw.mu.Lock()
	bw.BufWriter.Flush()

	// use non-mutex version since we are using this one
	err := bw.OrigWriter.reopen()

	bw.BufWriter.Reset(io.Writer(bw.OrigWriter))
	bw.mu.Unlock()

	return err
}

// Close flushes the internal buffer and closes the destination file
func (bw *BufferedFileWriter) Close() error {
	bw.quitChan <- true
	bw.mu.Lock()
	bw.done = true
	bw.BufWriter.Flush()
	bw.OrigWriter.f.Close()
	bw.mu.Unlock()
	return nil
}

// Write implements io.Writer (and reopen.Writer)
func (bw *BufferedFileWriter) Write(p []byte) (int, error) {
	bw.mu.Lock()
	n, err := bw.BufWriter.Write(p)

	// Special Case... if the used space in the buffer is LESS than
	// the input, then we did a flush in the middle of the line
	// and the full log line was not sent on its way.
	if bw.BufWriter.Buffered() < len(p) {
		bw.BufWriter.Flush()
	}

	bw.mu.Unlock()
	return n, err
}

// Flush flushes the buffer.
func (bw *BufferedFileWriter) Flush() {
	bw.mu.Lock()
	// could add check if bw.done already
	//  should never happen
	bw.BufWriter.Flush()
	bw.OrigWriter.f.Sync()
	bw.mu.Unlock()
}

// flushDaemon periodically flushes the log file buffers.
func (bw *BufferedFileWriter) flushDaemon(interval time.Duration) {
	ticker := time.NewTicker(interval)
	for {
		select {
		case <-bw.quitChan:
			ticker.Stop()
			return
		case <-ticker.C:
			bw.Flush()
		}
	}
}

const bufferSize = 256 * 1024
const flushInterval = 30 * time.Second

// NewBufferedFileWriter opens a buffered file that is periodically
//  flushed.
func NewBufferedFileWriter(w *FileWriter) *BufferedFileWriter {
	return NewBufferedFileWriterSize(w, bufferSize, flushInterval)
}

// NewBufferedFileWriterSize opens a buffered file with the given size that is periodically
//  flushed on the given interval.
func NewBufferedFileWriterSize(w *FileWriter, size int, flush time.Duration) *BufferedFileWriter {
	bw := BufferedFileWriter{
		quitChan:   make(chan bool, 1),
		OrigWriter: w,
		BufWriter:  bufio.NewWriterSize(w, size),
	}
	go bw.flushDaemon(flush)
	return &bw
}

type multiReopenWriter struct {
	writers []Writer
}

// Reopen reopens all child Reopeners
func (t *multiReopenWriter) Reopen() error {
	for _, w := range t.writers {
		err := w.Reopen()
		if err != nil {
			return err
		}
	}
	return nil
}

// Write implements standard io.Write and reopen.Write
func (t *multiReopenWriter) Write(p []byte) (int, error) {
	for _, w := range t.writers {
		n, err := w.Write(p)
		if err != nil {
			return n, err
		}
		if n != len(p) {
			return n, io.ErrShortWrite
		}
	}
	return len(p), nil
}

// MultiWriter creates a writer that duplicates its writes to all the
// provided writers, similar to the Unix tee(1) command.
//  Also allow reopen
func MultiWriter(writers ...Writer) Writer {
	w := make([]Writer, len(writers))
	copy(w, writers)
	return &multiReopenWriter{w}
}

type nopReopenWriteCloser struct {
	io.Writer
}

func (nopReopenWriteCloser) Reopen() error {
	return nil
}

func (nopReopenWriteCloser) Close() error {
	return nil
}

// NopWriter turns a normal writer into a ReopenWriter
//  by doing a NOP on Reopen.   See https://en.wikipedia.org/wiki/NOP
func NopWriter(w io.Writer) WriteCloser {
	return nopReopenWriteCloser{w}
}

// Reopenable versions of os.Stdout, os.Stderr, /dev/null (reopen does nothing)
var (
	Stdout  = NopWriter(os.Stdout)
	Stderr  = NopWriter(os.Stderr)
	Discard = NopWriter(ioutil.Discard)
)
<|endoftext|>


<commit_before>package fwk

import (
	"bytes"
	"fmt"
	"reflect"
	"sort"

	"github.com/go-hep/fwk/utils/tarjan"
)

type node struct {
	in  map[string]reflect.Type
	out map[string]reflect.Type
}

func newNode() *node {
	return &node{
		in:  make(map[string]reflect.Type),
		out: make(map[string]reflect.Type),
	}
}

// dflowsvc models and describes the runtime data-flow and (data) dependencies between
// components as declared during configuration.
type dflowsvc struct {
	SvcBase
	nodes map[string]*node
	edges map[string]reflect.Type
}

func (svc *dflowsvc) Configure(ctx Context) error {
	return nil
}

func (svc *dflowsvc) StartSvc(ctx Context) error {
	var err error

	// sort node-names for reproducibility
	nodenames := make([]string, 0, len(svc.nodes))
	for n := range svc.nodes {
		nodenames = append(nodenames, n)
	}
	sort.Strings(nodenames)

	// - make sure all input keys of components are available
	//   as output keys of a task
	// - also detect whether a key is labeled as an out-port
	//   by 2 different components
	out := make(map[string]string) // outport-name -> producer-name
	for _, tsk := range nodenames {
		node := svc.nodes[tsk]
		for k := range node.out {
			n, dup := out[k]
			if dup {
				return Errorf("%s: component [%s] already declared port [%s] as its output (current=%s)",
					svc.Name(), n, k, tsk,
				)
			}
			out[k] = tsk
		}
	}

	for _, tsk := range nodenames {
		node := svc.nodes[tsk]
		for k := range node.in {
			_, ok := out[k]
			if !ok {
				return Errorf("%s: component [%s] declared port [%s] as input but NO KNOWN producer",
					svc.Name(), tsk, k,
				)
			}
		}
	}

	// detect cycles.
	graph := make(map[interface{}][]interface{})
	for _, n := range nodenames {
		node := svc.nodes[n]
		graph[n] = []interface{}{}
		for in := range node.in {
			for _, o := range nodenames {
				if o == n {
					continue
				}
				onode := svc.nodes[o]
				connected := false
				for out := range onode.out {
					if in == out {
						connected = true
						break
					}
				}
				if connected {
					graph[n] = append(graph[n], o)
				}
			}
		}
	}

	cycles := tarjan.Connections(graph)
	if len(cycles) > 0 {
		msg := ctx.Msg()
		ncycles := 0
		for _, cycle := range cycles {
			if len(cycle) > 1 {
				ncycles++
				msg.Errorf("cycle detected: %v\n", cycle)
			}
		}
		s := ""
		if ncycles > 1 {
			s = "s"
		}
		if ncycles > 0 {
			return Errorf("%s: cycle%s detected: %d", svc.Name(), s, ncycles)
		}
	}

	return err
}

func (svc *dflowsvc) StopSvc(ctx Context) error {
	return nil
}

func (svc *dflowsvc) keys() []string {
	keys := make([]string, 0, len(svc.edges))
	for k := range svc.edges {
		keys = append(keys, k)
	}
	return keys
}

func (svc *dflowsvc) addInNode(tsk string, name string, t reflect.Type) error {
	node, ok := svc.nodes[tsk]
	if !ok {
		node = newNode()
		svc.nodes[tsk] = node
	}
	_, ok = node.in[name]
	if ok {
		return Errorf(
			"fwk.DeclInPort: component [%s] already declared in-port with name [%s]",
			tsk,
			name,
		)
	}

	node.in[name] = t
	edgetyp, dup := svc.edges[name]
	if dup {
		// make sure types match
		if edgetyp != t {
			type elemT struct {
				port string // in/out
				task string // task which defined the port
				typ  reflect.Type
			}
			cont := []elemT{}
			nodenames := make([]string, 0, len(svc.nodes))
			for tskname := range svc.nodes {
				nodenames = append(nodenames, tskname)
			}
			sort.Strings(nodenames)
			for _, tskname := range nodenames {
				node := svc.nodes[tskname]
				for k, in := range node.in {
					if k != name {
						continue
					}
					cont = append(cont,
						elemT{
							port: "in ",
							task: tskname,
							typ:  in,
						},
					)
				}
				for k, out := range node.out {
					if k != name {
						continue
					}
					cont = append(cont,
						elemT{
							port: "out",
							task: tskname,
							typ:  out,
						},
					)
				}
			}
			var o bytes.Buffer
			fmt.Fprintf(&o, "fwk.DeclInPort: detected type inconsistency for port [%s]:\n", name)
			for _, c := range cont {
				fmt.Fprintf(&o, " component=%q port=%s type=%v\n", c.task, c.port, c.typ)
			}
			return Errorf(string(o.Bytes()))
		}
	}

	svc.edges[name] = t
	return nil
}

func (svc *dflowsvc) addOutNode(tsk string, name string, t reflect.Type) error {
	node, ok := svc.nodes[tsk]
	if !ok {
		node = newNode()
		svc.nodes[tsk] = node
	}
	_, ok = node.out[name]
	if ok {
		return Errorf(
			"fwk.DeclOutPort: component [%s] already declared out-port with name [%s]",
			tsk,
			name,
		)
	}

	node.out[name] = t

	edgetyp, dup := svc.edges[name]
	if dup {
		// edge already exists
		// loop over nodes, find out who already defined that edge
		nodenames := make([]string, 0, len(svc.nodes))
		for tskname := range svc.nodes {
			nodenames = append(nodenames, tskname)
		}
		sort.Strings(nodenames)
		for _, duptsk := range nodenames {
			dupnode := svc.nodes[duptsk]
			if duptsk == tsk {
				continue
			}
			for out := range dupnode.out {
				if out == name {
					return Errorf(
						"fwk.DeclOutPort: component [%s] already declared out-port with name [%s (type=%v)].\nfwk.DeclOutPort: component [%s] is trying to add a duplicate out-port [%s (type=%v)]",
						duptsk,
						name,
						edgetyp,
						tsk,
						name,
						t,
					)
				}
			}
		}
	}
	svc.edges[name] = t
	return nil
}

func init() {
	Register(reflect.TypeOf(dflowsvc{}),
		func(t, name string, mgr App) (Component, error) {
			svc := &dflowsvc{
				SvcBase: NewSvc(t, name, mgr),
				nodes:   make(map[string]*node),
				edges:   make(map[string]reflect.Type),
			}
			return svc, nil
		},
	)
}

// EOF
<commit_msg>dflowsvc: add generation of DOT-file from dataflow graph<commit_after>package fwk

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"reflect"
	"sort"

	"code.google.com/p/gographviz"
	"github.com/go-hep/fwk/utils/tarjan"
)

type node struct {
	in  map[string]reflect.Type
	out map[string]reflect.Type
}

func newNode() *node {
	return &node{
		in:  make(map[string]reflect.Type),
		out: make(map[string]reflect.Type),
	}
}

// dflowsvc models and describes the runtime data-flow and (data) dependencies between
// components as declared during configuration.
type dflowsvc struct {
	SvcBase
	nodes map[string]*node
	edges map[string]reflect.Type

	dotfile string // path to a DOT file where to dump the data dependency graph.
}

func (svc *dflowsvc) Configure(ctx Context) error {
	return nil
}

func (svc *dflowsvc) StartSvc(ctx Context) error {
	var err error

	// sort node-names for reproducibility
	nodenames := make([]string, 0, len(svc.nodes))
	for n := range svc.nodes {
		nodenames = append(nodenames, n)
	}
	sort.Strings(nodenames)

	// - make sure all input keys of components are available
	//   as output keys of a task
	// - also detect whether a key is labeled as an out-port
	//   by 2 different components
	out := make(map[string]string) // outport-name -> producer-name
	for _, tsk := range nodenames {
		node := svc.nodes[tsk]
		for k := range node.out {
			n, dup := out[k]
			if dup {
				return Errorf("%s: component [%s] already declared port [%s] as its output (current=%s)",
					svc.Name(), n, k, tsk,
				)
			}
			out[k] = tsk
		}
	}

	for _, tsk := range nodenames {
		node := svc.nodes[tsk]
		for k := range node.in {
			_, ok := out[k]
			if !ok {
				return Errorf("%s: component [%s] declared port [%s] as input but NO KNOWN producer",
					svc.Name(), tsk, k,
				)
			}
		}
	}

	// detect cycles.
	graph := make(map[interface{}][]interface{})
	for _, n := range nodenames {
		node := svc.nodes[n]
		graph[n] = []interface{}{}
		for in := range node.in {
			for _, o := range nodenames {
				if o == n {
					continue
				}
				onode := svc.nodes[o]
				connected := false
				for out := range onode.out {
					if in == out {
						connected = true
						break
					}
				}
				if connected {
					graph[n] = append(graph[n], o)
				}
			}
		}
	}

	cycles := tarjan.Connections(graph)
	if len(cycles) > 0 {
		msg := ctx.Msg()
		ncycles := 0
		for _, cycle := range cycles {
			if len(cycle) > 1 {
				ncycles++
				msg.Errorf("cycle detected: %v\n", cycle)
			}
		}
		s := ""
		if ncycles > 1 {
			s = "s"
		}
		if ncycles > 0 {
			return Errorf("%s: cycle%s detected: %d", svc.Name(), s, ncycles)
		}
	}

	if svc.dotfile != "" {
		err = svc.dumpgraph()
		if err != nil {
			return err
		}
	}
	return err
}

func (svc *dflowsvc) StopSvc(ctx Context) error {
	return nil
}

func (svc *dflowsvc) keys() []string {
	keys := make([]string, 0, len(svc.edges))
	for k := range svc.edges {
		keys = append(keys, k)
	}
	return keys
}

func (svc *dflowsvc) addInNode(tsk string, name string, t reflect.Type) error {
	node, ok := svc.nodes[tsk]
	if !ok {
		node = newNode()
		svc.nodes[tsk] = node
	}
	_, ok = node.in[name]
	if ok {
		return Errorf(
			"fwk.DeclInPort: component [%s] already declared in-port with name [%s]",
			tsk,
			name,
		)
	}

	node.in[name] = t
	edgetyp, dup := svc.edges[name]
	if dup {
		// make sure types match
		if edgetyp != t {
			type elemT struct {
				port string // in/out
				task string // task which defined the port
				typ  reflect.Type
			}
			cont := []elemT{}
			nodenames := make([]string, 0, len(svc.nodes))
			for tskname := range svc.nodes {
				nodenames = append(nodenames, tskname)
			}
			sort.Strings(nodenames)
			for _, tskname := range nodenames {
				node := svc.nodes[tskname]
				for k, in := range node.in {
					if k != name {
						continue
					}
					cont = append(cont,
						elemT{
							port: "in ",
							task: tskname,
							typ:  in,
						},
					)
				}
				for k, out := range node.out {
					if k != name {
						continue
					}
					cont = append(cont,
						elemT{
							port: "out",
							task: tskname,
							typ:  out,
						},
					)
				}
			}
			var o bytes.Buffer
			fmt.Fprintf(&o, "fwk.DeclInPort: detected type inconsistency for port [%s]:\n", name)
			for _, c := range cont {
				fmt.Fprintf(&o, " component=%q port=%s type=%v\n", c.task, c.port, c.typ)
			}
			return Errorf(string(o.Bytes()))
		}
	}

	svc.edges[name] = t
	return nil
}

func (svc *dflowsvc) addOutNode(tsk string, name string, t reflect.Type) error {
	node, ok := svc.nodes[tsk]
	if !ok {
		node = newNode()
		svc.nodes[tsk] = node
	}
	_, ok = node.out[name]
	if ok {
		return Errorf(
			"fwk.DeclOutPort: component [%s] already declared out-port with name [%s]",
			tsk,
			name,
		)
	}

	node.out[name] = t

	edgetyp, dup := svc.edges[name]
	if dup {
		// edge already exists
		// loop over nodes, find out who already defined that edge
		nodenames := make([]string, 0, len(svc.nodes))
		for tskname := range svc.nodes {
			nodenames = append(nodenames, tskname)
		}
		sort.Strings(nodenames)
		for _, duptsk := range nodenames {
			dupnode := svc.nodes[duptsk]
			if duptsk == tsk {
				continue
			}
			for out := range dupnode.out {
				if out == name {
					return Errorf(
						"fwk.DeclOutPort: component [%s] already declared out-port with name [%s (type=%v)].\nfwk.DeclOutPort: component [%s] is trying to add a duplicate out-port [%s (type=%v)]",
						duptsk,
						name,
						edgetyp,
						tsk,
						name,
						t,
					)
				}
			}
		}
	}
	svc.edges[name] = t
	return nil
}

func (svc *dflowsvc) dumpgraph() error {
	var err error
	g := gographviz.NewGraph()
	gname := "dataflow"
	g.SetName(gname)
	g.SetDir(true)

	quote := func(s string) string {
		return fmt.Sprintf("%q", s)
	}

	for name, typ := range svc.edges {
		typename := typ.String()
		attr_data := map[string]string{
			`"node"`: `"data"`,
			`"type"`: quote(typename),
		}
		g.AddNode(gname, quote(name), attr_data)
	}

	attr_task := map[string]string{
		`"node"`:  `"task"`,
		`"shape"`: `"component"`,
	}
	for name, node := range svc.nodes {
		g.AddNode(gname, quote(name), attr_task)

		for in := range node.in {
			g.AddEdge(quote(in), quote(name), true, nil)
		}

		for out := range node.out {
			g.AddEdge(quote(name), quote(out), true, nil)
		}
	}

	err = ioutil.WriteFile(svc.dotfile, []byte(g.String()), 0644)
	if err != nil {
		return Error(err)
	}

	return err
}

func newDataFlowSvc(typ, name string, mgr App) (Component, error) {
	var err error
	svc := &dflowsvc{
		SvcBase: NewSvc(typ, name, mgr),
		nodes:   make(map[string]*node),
		edges:   make(map[string]reflect.Type),
		dotfile: "", // empty: no dump
	}

	err = svc.DeclProp("DotFile", &svc.dotfile)
	if err != nil {
		return nil, err
	}

	return svc, err
}

func init() {
	Register(reflect.TypeOf(dflowsvc{}), newDataFlowSvc)
}

// EOF
<|endoftext|>


<commit_before>// Title：图片缩放
//
// Description:
//
// Author:black
//
// Createtime:2013-10-18 14:16
//
// Version:1.0
//
// 修改历史:版本号 修改日期 修改人 修改说明
//
// 1.0 2013-10-18 14:16 black 创建文档
package commonlib

import (
	"github.com/gosexy/canvas"
)

func Resample(sourceImgPath,distImgPaht string, w, h int) error {

	img := canvas.New()
	err := img.Open(sourceImgPath)

	if err != nil {
		return err
	}

	img.AutoOrientate()
	img.SetQuality(95)
	img.Thumbnail(uint(w), uint(h))
	img.Write(distImgPaht)

	return nil
}
<commit_msg>图片代码回退，先不用第三方的切图方案<commit_after>// Title：图片缩放
//
// Description:
//
// Author:black
//
// Createtime:2013-10-18 14:16
//
// Version:1.0
//
// 修改历史:版本号 修改日期 修改人 修改说明
//
// 1.0 2013-10-18 14:16 black 创建文档
package commonlib

import (
	"image"
	"image/color"
)

// average convert the sums to averages and returns the result.
func average(sum []uint64, w, h int, n uint64) *image.RGBA {
	ret := image.NewRGBA(image.Rect(0, 0, w, h))
	for y := 0; y < h; y++ {
		for x := 0; x < w; x++ {
			index := 4 * (y*w + x)
			pix := ret.Pix[y*ret.Stride+x*4:]
			pix[0] = uint8(sum[index+0] / n)
			pix[1] = uint8(sum[index+1] / n)
			pix[2] = uint8(sum[index+2] / n)
			pix[3] = uint8(sum[index+3] / n)
		}
	}
	return ret
}

// ResizeRGBA returns a scaled copy of the RGBA image slice r of m.
// The returned image has width w and height h.
func ResizeRGBA(m *image.RGBA, r image.Rectangle, w, h int) *image.RGBA {
	ww, hh := uint64(w), uint64(h)
	dx, dy := uint64(r.Dx()), uint64(r.Dy())
	// See comment in Resize.
	n, sum := dx*dy, make([]uint64, 4*w*h)
	for y := r.Min.Y; y < r.Max.Y; y++ {
		pix := m.Pix[(y-r.Min.Y)*m.Stride:]
		for x := r.Min.X; x < r.Max.X; x++ {
			// Get the source pixel.
			p := pix[(x-r.Min.X)*4:]
			r64 := uint64(p[0])
			g64 := uint64(p[1])
			b64 := uint64(p[2])
			a64 := uint64(p[3])
			// Spread the source pixel over 1 or more destination rows.
			py := uint64(y) * hh
			for remy := hh; remy > 0; {
				qy := dy - (py % dy)
				if qy > remy {
					qy = remy
				}
				// Spread the source pixel over 1 or more destination columns.
				px := uint64(x) * ww
				index := 4 * ((py/dy)*ww + (px / dx))
				for remx := ww; remx > 0; {
					qx := dx - (px % dx)
					if qx > remx {
						qx = remx
					}
					qxy := qx * qy
					sum[index+0] += r64 * qxy
					sum[index+1] += g64 * qxy
					sum[index+2] += b64 * qxy
					sum[index+3] += a64 * qxy
					index += 4
					px += qx
					remx -= qx
				}
				py += qy
				remy -= qy
			}
		}
	}
	return average(sum, w, h, n)
}

// ResizeNRGBA returns a scaled copy of the RGBA image slice r of m.
// The returned image has width w and height h.
func ResizeNRGBA(m *image.NRGBA, r image.Rectangle, w, h int) *image.RGBA {
	ww, hh := uint64(w), uint64(h)
	dx, dy := uint64(r.Dx()), uint64(r.Dy())
	// See comment in Resize.
	n, sum := dx*dy, make([]uint64, 4*w*h)
	for y := r.Min.Y; y < r.Max.Y; y++ {
		pix := m.Pix[(y-r.Min.Y)*m.Stride:]
		for x := r.Min.X; x < r.Max.X; x++ {
			// Get the source pixel.
			p := pix[(x-r.Min.X)*4:]
			r64 := uint64(p[0])
			g64 := uint64(p[1])
			b64 := uint64(p[2])
			a64 := uint64(p[3])
			r64 = (r64 * a64) / 255
			g64 = (g64 * a64) / 255
			b64 = (b64 * a64) / 255
			// Spread the source pixel over 1 or more destination rows.
			py := uint64(y) * hh
			for remy := hh; remy > 0; {
				qy := dy - (py % dy)
				if qy > remy {
					qy = remy
				}
				// Spread the source pixel over 1 or more destination columns.
				px := uint64(x) * ww
				index := 4 * ((py/dy)*ww + (px / dx))
				for remx := ww; remx > 0; {
					qx := dx - (px % dx)
					if qx > remx {
						qx = remx
					}
					qxy := qx * qy
					sum[index+0] += r64 * qxy
					sum[index+1] += g64 * qxy
					sum[index+2] += b64 * qxy
					sum[index+3] += a64 * qxy
					index += 4
					px += qx
					remx -= qx
				}
				py += qy
				remy -= qy
			}
		}
	}
	return average(sum, w, h, n)
}

// Resample returns a resampled copy of the image slice r of m.
// The returned image has width w and height h.
func Resample(m image.Image, r image.Rectangle, w, h int) *image.RGBA {
	if w < 0 || h < 0 {
		return nil
	}
	if w == 0 || h == 0 || r.Dx() <= 0 || r.Dy() <= 0 {
		return image.NewRGBA(image.Rect(0, 0, w, h))
	}
	curw, curh := r.Dx(), r.Dy()
	img := image.NewRGBA(image.Rect(0, 0, w, h))
	for y := 0; y < h; y++ {
		for x := 0; x < w; x++ {
			// Get a source pixel.
			subx := x * curw / w
			suby := y * curh / h
			r32, g32, b32, a32 := m.At(subx, suby).RGBA()
			r := uint8(r32 >> 8)
			g := uint8(g32 >> 8)
			b := uint8(b32 >> 8)
			a := uint8(a32 >> 8)
			img.SetRGBA(x, y, color.RGBA{r, g, b, a})
		}
	}
	return img
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"math"
	"regexp"

	"github.com/disintegration/imaging"
)

var width = flag.Int("width", math.MaxInt16, "Output image width")
var height = flag.Int("height", math.MaxInt16, "Output image height")
var output = flag.String("type", "png", "Output image format")

func main() {
	flag.Parse()

	extMatch := regexp.MustCompile(`\.[a-zA-Z0-9]+$`)

	for _, srcFilename := range flag.Args() {
		fmt.Printf("Processing [%s]... ", srcFilename)
		src, err := imaging.Open(srcFilename)
		if err != nil {
			log.Printf("error: %s\n", err)
			continue
		}
		dest := imaging.Fit(src, *width, *height, imaging.MitchellNetravali)
		destFilename := extMatch.ReplaceAllString(srcFilename, "") + fmt.Sprintf("-%dx%d.%s", dest.Bounds().Dx(), dest.Bounds().Dy(), *output)
		err = imaging.Save(dest, destFilename)
		if err != nil {
			log.Printf("error: %s\n", err)
			continue
		}
		fmt.Printf("OK; written to %s\n", destFilename)
	}
}
<commit_msg>Added globbing support (for Windows)<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"math"
	"path/filepath"
	"regexp"

	"github.com/disintegration/imaging"
)

var width = flag.Int("width", math.MaxInt16, "Output image width")
var height = flag.Int("height", math.MaxInt16, "Output image height")
var output = flag.String("type", "png", "Output image format")

func main() {
	flag.Parse()

	extMatch := regexp.MustCompile(`\.[a-zA-Z0-9]+$`)

	for _, srcArg := range flag.Args() {
		srcFilenames, err := filepath.Glob(srcArg)
		if err != nil {
			log.Printf("Glob error: %s\n", err)
			continue
		}
		for _, srcFilename := range srcFilenames {
			fmt.Printf("Processing [%s]... ", srcFilename)
			src, err := imaging.Open(srcFilename)
			if err != nil {
				log.Printf("error: %s\n", err)
				continue
			}
			dest := imaging.Fit(src, *width, *height, imaging.MitchellNetravali)
			destFilename := extMatch.ReplaceAllString(srcFilename, "") + fmt.Sprintf("-%dx%d.%s", dest.Bounds().Dx(), dest.Bounds().Dy(), *output)
			err = imaging.Save(dest, destFilename)
			if err != nil {
				log.Printf("error: %s\n", err)
				continue
			}
			fmt.Printf("OK; written to %s\n", destFilename)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"log"
	"net/http"
	"strings"
	"sync/atomic"
	"time"

	"github.com/heroku/lumbermill/Godeps/_workspace/src/github.com/bmizerany/lpx"
	"github.com/heroku/lumbermill/Godeps/_workspace/src/github.com/kr/logfmt"
	metrics "github.com/heroku/lumbermill/Godeps/_workspace/src/github.com/rcrowley/go-metrics"
)

var (
	// TokenPrefix contains the prefix for non-heroku tokens.
	TokenPrefix = []byte("t.")
	// Heroku contains the prefix for heroku tokens.
	Heroku = []byte("heroku")

	// go-metrics Instruments
	wrongMethodErrorCounter    = metrics.GetOrRegisterCounter("lumbermill.errors.drain.wrong.method", metrics.DefaultRegistry)
	authFailureCounter         = metrics.GetOrRegisterCounter("lumbermill.errors.auth.failure", metrics.DefaultRegistry)
	badRequestCounter          = metrics.GetOrRegisterCounter("lumbermill.errors.badrequest", metrics.DefaultRegistry)
	internalServerErrorCounter = metrics.GetOrRegisterCounter("lumbermill.errors.internalserver", metrics.DefaultRegistry)
	tokenMissingCounter        = metrics.GetOrRegisterCounter("lumbermill.errors.token.missing", metrics.DefaultRegistry)
	timeParsingErrorCounter    = metrics.GetOrRegisterCounter("lumbermill.errors.time.parse", metrics.DefaultRegistry)
	logfmtParsingErrorCounter  = metrics.GetOrRegisterCounter("lumbermill.errors.logfmt.parse", metrics.DefaultRegistry)
	droppedErrorCounter        = metrics.GetOrRegisterCounter("lumbermill.errors.dropped", metrics.DefaultRegistry)
	batchCounter               = metrics.GetOrRegisterCounter("lumbermill.batch", metrics.DefaultRegistry)
	linesCounter               = metrics.GetOrRegisterCounter("lumbermill.lines", metrics.DefaultRegistry)
	routerErrorLinesCounter    = metrics.GetOrRegisterCounter("lumbermill.lines.router.error", metrics.DefaultRegistry)
	routerLinesCounter         = metrics.GetOrRegisterCounter("lumbermill.lines.router", metrics.DefaultRegistry)
	routerBlankLinesCounter    = metrics.GetOrRegisterCounter("lumbermill.lines.router.blank", metrics.DefaultRegistry)
	dynoErrorLinesCounter      = metrics.GetOrRegisterCounter("lumbermill.lines.dyno.error", metrics.DefaultRegistry)
	dynoMemLinesCounter        = metrics.GetOrRegisterCounter("lumbermill.lines.dyno.mem", metrics.DefaultRegistry)
	dynoLoadLinesCounter       = metrics.GetOrRegisterCounter("lumbermill.lines.dyno.load", metrics.DefaultRegistry)
	unknownHerokuLinesCounter  = metrics.GetOrRegisterCounter("lumbermill.lines.unknown.heroku", metrics.DefaultRegistry)
	unknownUserLinesCounter    = metrics.GetOrRegisterCounter("lumbermill.lines.unknown.user", metrics.DefaultRegistry)
	parseTimer                 = metrics.GetOrRegisterTimer("lumbermill.batches.parse.time", metrics.DefaultRegistry)
	batchSizeHistogram         = metrics.GetOrRegisterHistogram("lumbermill.batches.sizes", metrics.DefaultRegistry, metrics.NewUniformSample(100))
)

// Dyno's are generally reported as "<type>.<#>"
// Extract the <type> and return it
func dynoType(what string) string {
	s := strings.Split(what, ".")
	return s[0]
}

// Lock, or don't do any work, but don't block.
// This, essentially, samples the incoming tokens for the purposes of health checking
// live tokens. Rather than use a random number generator, or a global counter, we
// let the scheduler do the sampling for us.
func (s *server) maybeUpdateRecentTokens(host, id string) {
	if atomic.CompareAndSwapInt32(s.tokenLock, 0, 1) {
		s.recentTokensLock.Lock()
		s.recentTokens[host] = id
		s.recentTokensLock.Unlock()
		atomic.StoreInt32(s.tokenLock, 0)
	}
}

func handleLogFmtParsingError(msg []byte, err error) {
	logfmtParsingErrorCounter.Inc(1)
	log.Printf("logfmt unmarshal error(%q): %q\n", string(msg), err)
}

// "Parse tree" from hell
func (s *server) serveDrain(w http.ResponseWriter, r *http.Request) {
	s.Add(1)
	defer s.Done()

	w.Header().Set("Content-Length", "0")

	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		wrongMethodErrorCounter.Inc(1)
		return
	}

	id := r.Header.Get("Logplex-Drain-Token")

	batchCounter.Inc(1)

	parseStart := time.Now()
	lp := lpx.NewReader(bufio.NewReader(r.Body))

	linesCounterInc := 0

	for lp.Next() {
		linesCounterInc++
		header := lp.Header()

		// If the syslog Name Header field contains what looks like a log token,
		// let's assume it's an override of the id and we're getting the data from the magic
		// channel
		if bytes.HasPrefix(header.Name, TokenPrefix) {
			id = string(header.Name)
		}

		// If we still don't have an id, throw an error and try the next line
		if id == "" {
			tokenMissingCounter.Inc(1)
			continue
		}

		destination := s.hashRing.Get(id)

		msg := lp.Bytes()
		switch {
		case bytes.Equal(header.Name, Heroku), bytes.HasPrefix(header.Name, TokenPrefix):
			timeStr := string(lp.Header().Time)
			t, e := time.Parse("2006-01-02T15:04:05.000000+00:00", timeStr)
			if e != nil {
				t, e = time.Parse("2006-01-02T15:04:05+00:00", timeStr)
				if e != nil {
					timeParsingErrorCounter.Inc(1)
					log.Printf("Error Parsing Time(%s): %q\n", string(lp.Header().Time), e)
					continue
				}
			}

			timestamp := t.UnixNano() / int64(time.Microsecond)

			pid := string(header.Procid)
			switch pid {
			case "router":

				switch {
				// router logs with a H error code in them
				case bytes.Contains(msg, keyCodeH):
					routerErrorLinesCounter.Inc(1)
					re := routerError{}
					err := logfmt.Unmarshal(msg, &re)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}

					// Track the breakout of different error types.
					metrics.GetOrRegisterCounter("lumbermill.lines.router.errors."+re.Code, metrics.DefaultRegistry).Inc(1)

					destination.PostPoint(point{id, routerEvent, []interface{}{timestamp, re.Code}})

					// If the app is blank (not pushed) we don't care
				// do nothing atm, increment a counter
				case bytes.Contains(msg, keyCodeBlank), bytes.Contains(msg, keyDescBlank):
					routerBlankLinesCounter.Inc(1)

				// likely a standard router log
				default:
					routerLinesCounter.Inc(1)
					rm := routerMsg{}
					err := logfmt.Unmarshal(msg, &rm)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}

					destination.PostPoint(point{id, routerRequest, []interface{}{timestamp, rm.Status, rm.Service}})
				}

				// Non router logs, so either dynos, runtime, etc
			default:
				switch {
				// Dyno error messages
				case bytes.HasPrefix(msg, dynoErrorSentinel):
					dynoErrorLinesCounter.Inc(1)
					de, err := parseBytesToDynoError(msg)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}

					what := string(lp.Header().Procid)
					destination.PostPoint(
						point{id, dynoEvents, []interface{}{timestamp, what, "R", de.Code, string(msg), dynoType(what)}},
					)

				// Dyno log-runtime-metrics memory messages
				case bytes.Contains(msg, dynoMemMsgSentinel):
					s.maybeUpdateRecentTokens(destination.Name, id)

					dynoMemLinesCounter.Inc(1)
					dm := dynoMemMsg{}
					err := logfmt.Unmarshal(msg, &dm)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}
					if dm.Source != "" {
						destination.PostPoint(
							point{
								id,
								dynoMem,
								[]interface{}{
									timestamp,
									dm.Source,
									dm.MemoryCache,
									dm.MemoryPgpgin,
									dm.MemoryPgpgout,
									dm.MemoryRSS,
									dm.MemorySwap,
									dm.MemoryTotal,
									dynoType(dm.Source),
								},
							},
						)
					}

					// Dyno log-runtime-metrics load messages
				case bytes.Contains(msg, dynoLoadMsgSentinel):
					s.maybeUpdateRecentTokens(destination.Name, id)

					dynoLoadLinesCounter.Inc(1)
					dm := dynoLoadMsg{}
					err := logfmt.Unmarshal(msg, &dm)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}
					if dm.Source != "" {
						destination.PostPoint(
							point{
								id,
								dynoLoad,
								[]interface{}{timestamp, dm.Source, dm.LoadAvg1Min, dm.LoadAvg5Min, dm.LoadAvg15Min, dynoType(dm.Source)},
							},
						)
					}

				// unknown
				default:
					unknownHerokuLinesCounter.Inc(1)
					if debug {
						log.Printf("Unknown Heroku Line - Header: PRI: %s, Time: %s, Hostname: %s, Name: %s, ProcId: %s, MsgId: %s - Body: %s",
							header.PrivalVersion,
							header.Time,
							header.Hostname,
							header.Name,
							header.Procid,
							header.Msgid,
							string(msg),
						)
					}
				}
			}

		// non heroku lines
		default:
			unknownUserLinesCounter.Inc(1)
			if debug {
				log.Printf("Unknown User Line - Header: PRI: %s, Time: %s, Hostname: %s, Name: %s, ProcId: %s, MsgId: %s - Body: %s",
					header.PrivalVersion,
					header.Time,
					header.Hostname,
					header.Name,
					header.Procid,
					header.Msgid,
					string(msg),
				)
			}
		}
	}

	linesCounter.Inc(int64(linesCounterInc))

	batchSizeHistogram.Update(int64(linesCounterInc))

	parseTimer.UpdateSince(parseStart)

	w.WriteHeader(http.StatusNoContent)
}
<commit_msg>Revert "Revert "Add H99 debug logging""<commit_after>package main

import (
	"bufio"
	"bytes"
	"log"
	"net/http"
	"strings"
	"sync/atomic"
	"time"

	"github.com/heroku/lumbermill/Godeps/_workspace/src/github.com/bmizerany/lpx"
	"github.com/heroku/lumbermill/Godeps/_workspace/src/github.com/kr/logfmt"
	metrics "github.com/heroku/lumbermill/Godeps/_workspace/src/github.com/rcrowley/go-metrics"
)

var (
	// TokenPrefix contains the prefix for non-heroku tokens.
	TokenPrefix = []byte("t.")
	// Heroku contains the prefix for heroku tokens.
	Heroku = []byte("heroku")

	// go-metrics Instruments
	wrongMethodErrorCounter    = metrics.GetOrRegisterCounter("lumbermill.errors.drain.wrong.method", metrics.DefaultRegistry)
	authFailureCounter         = metrics.GetOrRegisterCounter("lumbermill.errors.auth.failure", metrics.DefaultRegistry)
	badRequestCounter          = metrics.GetOrRegisterCounter("lumbermill.errors.badrequest", metrics.DefaultRegistry)
	internalServerErrorCounter = metrics.GetOrRegisterCounter("lumbermill.errors.internalserver", metrics.DefaultRegistry)
	tokenMissingCounter        = metrics.GetOrRegisterCounter("lumbermill.errors.token.missing", metrics.DefaultRegistry)
	timeParsingErrorCounter    = metrics.GetOrRegisterCounter("lumbermill.errors.time.parse", metrics.DefaultRegistry)
	logfmtParsingErrorCounter  = metrics.GetOrRegisterCounter("lumbermill.errors.logfmt.parse", metrics.DefaultRegistry)
	droppedErrorCounter        = metrics.GetOrRegisterCounter("lumbermill.errors.dropped", metrics.DefaultRegistry)
	batchCounter               = metrics.GetOrRegisterCounter("lumbermill.batch", metrics.DefaultRegistry)
	linesCounter               = metrics.GetOrRegisterCounter("lumbermill.lines", metrics.DefaultRegistry)
	routerErrorLinesCounter    = metrics.GetOrRegisterCounter("lumbermill.lines.router.error", metrics.DefaultRegistry)
	routerLinesCounter         = metrics.GetOrRegisterCounter("lumbermill.lines.router", metrics.DefaultRegistry)
	routerBlankLinesCounter    = metrics.GetOrRegisterCounter("lumbermill.lines.router.blank", metrics.DefaultRegistry)
	dynoErrorLinesCounter      = metrics.GetOrRegisterCounter("lumbermill.lines.dyno.error", metrics.DefaultRegistry)
	dynoMemLinesCounter        = metrics.GetOrRegisterCounter("lumbermill.lines.dyno.mem", metrics.DefaultRegistry)
	dynoLoadLinesCounter       = metrics.GetOrRegisterCounter("lumbermill.lines.dyno.load", metrics.DefaultRegistry)
	unknownHerokuLinesCounter  = metrics.GetOrRegisterCounter("lumbermill.lines.unknown.heroku", metrics.DefaultRegistry)
	unknownUserLinesCounter    = metrics.GetOrRegisterCounter("lumbermill.lines.unknown.user", metrics.DefaultRegistry)
	parseTimer                 = metrics.GetOrRegisterTimer("lumbermill.batches.parse.time", metrics.DefaultRegistry)
	batchSizeHistogram         = metrics.GetOrRegisterHistogram("lumbermill.batches.sizes", metrics.DefaultRegistry, metrics.NewUniformSample(100))
)

// Dyno's are generally reported as "<type>.<#>"
// Extract the <type> and return it
func dynoType(what string) string {
	s := strings.Split(what, ".")
	return s[0]
}

// Lock, or don't do any work, but don't block.
// This, essentially, samples the incoming tokens for the purposes of health checking
// live tokens. Rather than use a random number generator, or a global counter, we
// let the scheduler do the sampling for us.
func (s *server) maybeUpdateRecentTokens(host, id string) {
	if atomic.CompareAndSwapInt32(s.tokenLock, 0, 1) {
		s.recentTokensLock.Lock()
		s.recentTokens[host] = id
		s.recentTokensLock.Unlock()
		atomic.StoreInt32(s.tokenLock, 0)
	}
}

func handleLogFmtParsingError(msg []byte, err error) {
	logfmtParsingErrorCounter.Inc(1)
	log.Printf("logfmt unmarshal error(%q): %q\n", string(msg), err)
}

// "Parse tree" from hell
func (s *server) serveDrain(w http.ResponseWriter, r *http.Request) {
	s.Add(1)
	defer s.Done()

	w.Header().Set("Content-Length", "0")

	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		wrongMethodErrorCounter.Inc(1)
		return
	}

	id := r.Header.Get("Logplex-Drain-Token")

	batchCounter.Inc(1)

	parseStart := time.Now()
	lp := lpx.NewReader(bufio.NewReader(r.Body))

	linesCounterInc := 0

	for lp.Next() {
		linesCounterInc++
		header := lp.Header()

		// If the syslog Name Header field contains what looks like a log token,
		// let's assume it's an override of the id and we're getting the data from the magic
		// channel
		if bytes.HasPrefix(header.Name, TokenPrefix) {
			id = string(header.Name)
		}

		// If we still don't have an id, throw an error and try the next line
		if id == "" {
			tokenMissingCounter.Inc(1)
			continue
		}

		destination := s.hashRing.Get(id)

		msg := lp.Bytes()
		switch {
		case bytes.Equal(header.Name, Heroku), bytes.HasPrefix(header.Name, TokenPrefix):
			timeStr := string(lp.Header().Time)
			t, e := time.Parse("2006-01-02T15:04:05.000000+00:00", timeStr)
			if e != nil {
				t, e = time.Parse("2006-01-02T15:04:05+00:00", timeStr)
				if e != nil {
					timeParsingErrorCounter.Inc(1)
					log.Printf("Error Parsing Time(%s): %q\n", string(lp.Header().Time), e)
					continue
				}
			}

			timestamp := t.UnixNano() / int64(time.Microsecond)

			pid := string(header.Procid)
			switch pid {
			case "router":

				switch {
				// router logs with a H error code in them
				case bytes.Contains(msg, keyCodeH):
					routerErrorLinesCounter.Inc(1)
					re := routerError{}
					err := logfmt.Unmarshal(msg, &re)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}

					// Track the breakout of different error types.
					metrics.GetOrRegisterCounter("lumbermill.lines.router.errors."+re.Code, metrics.DefaultRegistry).Inc(1)
					if re.Code == "H99" {
						log.Printf("debug=error.H99 %s", msg)
					}

					destination.PostPoint(point{id, routerEvent, []interface{}{timestamp, re.Code}})

					// If the app is blank (not pushed) we don't care
				// do nothing atm, increment a counter
				case bytes.Contains(msg, keyCodeBlank), bytes.Contains(msg, keyDescBlank):
					routerBlankLinesCounter.Inc(1)

				// likely a standard router log
				default:
					routerLinesCounter.Inc(1)
					rm := routerMsg{}
					err := logfmt.Unmarshal(msg, &rm)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}

					destination.PostPoint(point{id, routerRequest, []interface{}{timestamp, rm.Status, rm.Service}})
				}

				// Non router logs, so either dynos, runtime, etc
			default:
				switch {
				// Dyno error messages
				case bytes.HasPrefix(msg, dynoErrorSentinel):
					dynoErrorLinesCounter.Inc(1)
					de, err := parseBytesToDynoError(msg)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}

					what := string(lp.Header().Procid)
					destination.PostPoint(
						point{id, dynoEvents, []interface{}{timestamp, what, "R", de.Code, string(msg), dynoType(what)}},
					)

				// Dyno log-runtime-metrics memory messages
				case bytes.Contains(msg, dynoMemMsgSentinel):
					s.maybeUpdateRecentTokens(destination.Name, id)

					dynoMemLinesCounter.Inc(1)
					dm := dynoMemMsg{}
					err := logfmt.Unmarshal(msg, &dm)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}
					if dm.Source != "" {
						destination.PostPoint(
							point{
								id,
								dynoMem,
								[]interface{}{
									timestamp,
									dm.Source,
									dm.MemoryCache,
									dm.MemoryPgpgin,
									dm.MemoryPgpgout,
									dm.MemoryRSS,
									dm.MemorySwap,
									dm.MemoryTotal,
									dynoType(dm.Source),
								},
							},
						)
					}

					// Dyno log-runtime-metrics load messages
				case bytes.Contains(msg, dynoLoadMsgSentinel):
					s.maybeUpdateRecentTokens(destination.Name, id)

					dynoLoadLinesCounter.Inc(1)
					dm := dynoLoadMsg{}
					err := logfmt.Unmarshal(msg, &dm)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}
					if dm.Source != "" {
						destination.PostPoint(
							point{
								id,
								dynoLoad,
								[]interface{}{timestamp, dm.Source, dm.LoadAvg1Min, dm.LoadAvg5Min, dm.LoadAvg15Min, dynoType(dm.Source)},
							},
						)
					}

				// unknown
				default:
					unknownHerokuLinesCounter.Inc(1)
					if debug {
						log.Printf("Unknown Heroku Line - Header: PRI: %s, Time: %s, Hostname: %s, Name: %s, ProcId: %s, MsgId: %s - Body: %s",
							header.PrivalVersion,
							header.Time,
							header.Hostname,
							header.Name,
							header.Procid,
							header.Msgid,
							string(msg),
						)
					}
				}
			}

		// non heroku lines
		default:
			unknownUserLinesCounter.Inc(1)
			if debug {
				log.Printf("Unknown User Line - Header: PRI: %s, Time: %s, Hostname: %s, Name: %s, ProcId: %s, MsgId: %s - Body: %s",
					header.PrivalVersion,
					header.Time,
					header.Hostname,
					header.Name,
					header.Procid,
					header.Msgid,
					string(msg),
				)
			}
		}
	}

	linesCounter.Inc(int64(linesCounterInc))

	batchSizeHistogram.Update(int64(linesCounterInc))

	parseTimer.UpdateSince(parseStart)

	w.WriteHeader(http.StatusNoContent)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"encoding/base64"
	"errors"
	"log"
	"net/http"
	"strings"
	"time"

	"github.com/bmizerany/lpx"
	"github.com/kr/logfmt"
	metrics "github.com/rcrowley/go-metrics"
)

var (
	TokenPrefix = []byte("t.")
	Heroku      = []byte("heroku")

	// go-metrics Instruments
	wrongMethodErrorCounter    = metrics.NewRegisteredCounter("lumbermill.errors.drain.wrong.method", metrics.DefaultRegistry)
	authFailureCounter         = metrics.NewRegisteredCounter("lumbermill.errors.auth.failure", metrics.DefaultRegistry)
	badRequestCounter          = metrics.NewRegisteredCounter("lumbermill.errors.badrequest", metrics.DefaultRegistry)
	internalServerErrorCounter = metrics.NewRegisteredCounter("lumbermill.errors.internalserver", metrics.DefaultRegistry)
	tokenMissingCounter        = metrics.NewRegisteredCounter("lumbermill.errors.token.missing", metrics.DefaultRegistry)
	timeParsingErrorCounter    = metrics.NewRegisteredCounter("lumbermill.errors.time.parse", metrics.DefaultRegistry)
	logfmtParsingErrorCounter  = metrics.NewRegisteredCounter("lumbermill.errors.logfmt.parse", metrics.DefaultRegistry)
	droppedErrorCounter        = metrics.NewRegisteredCounter("lumbermill.errors.dropped", metrics.DefaultRegistry)
	batchCounter               = metrics.NewRegisteredCounter("lumbermill.batch", metrics.DefaultRegistry)
	linesCounter               = metrics.NewRegisteredCounter("lumbermill.lines", metrics.DefaultRegistry)
	routerErrorLinesCounter    = metrics.NewRegisteredCounter("lumbermill.lines.router.error", metrics.DefaultRegistry)
	routerLinesCounter         = metrics.NewRegisteredCounter("lumbermill.lines.router", metrics.DefaultRegistry)
	dynoErrorLinesCounter      = metrics.NewRegisteredCounter("lumbermill.lines.dyno.error", metrics.DefaultRegistry)
	dynoMemLinesCounter        = metrics.NewRegisteredCounter("lumbermill.lines.dyno.mem", metrics.DefaultRegistry)
	dynoLoadLinesCounter       = metrics.NewRegisteredCounter("lumbermill.lines.dyno.load", metrics.DefaultRegistry)
	unknownHerokuLinesCounter  = metrics.NewRegisteredCounter("lumbermill.lines.unknown.heroku", metrics.DefaultRegistry)
	unknownUserLinesCounter    = metrics.NewRegisteredCounter("lumbermill.lines.unknown.user", metrics.DefaultRegistry)
	parseTimer                 = metrics.NewRegisteredTimer("lumbermill.batches.parse.time", metrics.DefaultRegistry)
	batchSizeHistogram         = metrics.NewRegisteredHistogram("lumbermill.batches.sizes", metrics.DefaultRegistry, metrics.NewUniformSample(100))
)

func checkAuth(r *http.Request) error {
	header := r.Header.Get("Authorization")
	if header == "" {
		return errors.New("Authorization required")
	}
	headerParts := strings.SplitN(header, " ", 2)
	if len(headerParts) != 2 {
		return errors.New("Authorization header is malformed")
	}

	method := headerParts[0]
	if method != "Basic" {
		return errors.New("Only Basic Authorization is accepted")
	}

	encodedUserPass := headerParts[1]
	decodedUserPass, err := base64.StdEncoding.DecodeString(encodedUserPass)
	if err != nil {
		return errors.New("Authorization header is malformed")
	}

	userPassParts := bytes.SplitN(decodedUserPass, []byte{':'}, 2)
	if len(userPassParts) != 2 {
		return errors.New("Authorization header is malformed")
	}

	user := userPassParts[0]
	pass := userPassParts[1]

	if string(user) != User {
		return errors.New("Unknown user")
	}
	if string(pass) != Password {
		return errors.New("Incorrect token")
	}

	return nil
}

// Dyno's are generally reported as "<type>.<#>"
// Extract the <type> and return it
func dynoType(what string) string {
	s := strings.Split(what, ".")
	return s[0]
}

func handleLogFmtParsingError(msg []byte, err error) {
	logfmtParsingErrorCounter.Inc(1)
	log.Printf("logfmt unmarshal error(%q): %q\n", string(msg), err)
}

// "Parse tree" from hell
func serveDrain(w http.ResponseWriter, r *http.Request) {

	w.Header().Set("Content-Length", "0")

	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		wrongMethodErrorCounter.Inc(1)
		return
	}

	id := r.Header.Get("Logplex-Drain-Token")

	if id == "" {
		if err := checkAuth(r); err != nil {
			w.WriteHeader(http.StatusForbidden)
			authFailureCounter.Inc(1)
			return
		}
	}

	batchCounter.Inc(1)

	parseStart := time.Now()
	lp := lpx.NewReader(bufio.NewReader(r.Body))

	linesCounterInc := 0

	for lp.Next() {
		linesCounterInc += 1
		header := lp.Header()

		// If the syslog Name Header field contains what looks like a log token,
		// let's assume it's an override of the id and we're getting the data from the magic
		// channel
		if bytes.HasPrefix(header.Name, TokenPrefix) {
			id = string(header.Name)
		}

		// If we still don't have an id, throw an error and try the next line
		if id == "" {
			tokenMissingCounter.Inc(1)
			continue
		}

		destination := hashRing.Get(id)

		msg := lp.Bytes()
		switch {
		case bytes.Equal(header.Name, Heroku), bytes.HasPrefix(header.Name, TokenPrefix):
			timeStr := string(lp.Header().Time)
			t, e := time.Parse("2006-01-02T15:04:05.000000+00:00", timeStr)
			if e != nil {
				t, e = time.Parse("2006-01-02T15:04:05+00:00", timeStr)
				if e != nil {
					timeParsingErrorCounter.Inc(1)
					log.Printf("Error Parsing Time(%s): %q\n", string(lp.Header().Time), e)
					continue
				}
			}

			timestamp := t.UnixNano() / int64(time.Microsecond)

			pid := string(header.Procid)
			switch pid {
			case "router":

				switch {
				// router logs with a H error code in them
				case bytes.Contains(msg, keyCodeH):
					routerErrorLinesCounter.Inc(1)
					re := routerError{}
					err := logfmt.Unmarshal(msg, &re)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}
					destination.PostPoint(Point{id, EventsRouter, []interface{}{timestamp, re.Code}})

				//If the app is blank (not pushed) we don't care
				case bytes.Contains(msg, keyCodeBlank), bytes.Contains(msg, keyDescBlank):
					// do nothing

				// likely a standard router log
				default:
					routerLinesCounter.Inc(1)
					rm := routerMsg{}
					err := logfmt.Unmarshal(msg, &rm)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}

					destination.PostPoint(Point{id, Router, []interface{}{timestamp, rm.Status, rm.Service}})
				}

				// Non router logs, so either dynos, runtime, etc
			default:
				switch {
				// Dyno error messages
				case bytes.HasPrefix(msg, dynoErrorSentinel):
					dynoErrorLinesCounter.Inc(1)
					de, err := parseBytesToDynoError(msg)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}

					what := string(lp.Header().Procid)
					destination.PostPoint(
						Point{id, EventsDyno, []interface{}{timestamp, what, "R", de.Code, string(msg), dynoType(what)}},
					)

				// Dyno log-runtime-metrics memory messages
				case bytes.Contains(msg, dynoMemMsgSentinel):
					dynoMemLinesCounter.Inc(1)
					dm := dynoMemMsg{}
					err := logfmt.Unmarshal(msg, &dm)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}
					if dm.Source != "" {
						destination.PostPoint(
							Point{
								id,
								DynoMem,
								[]interface{}{
									timestamp,
									dm.Source,
									dm.MemoryCache,
									dm.MemoryPgpgin,
									dm.MemoryPgpgout,
									dm.MemoryRSS,
									dm.MemorySwap,
									dm.MemoryTotal,
									dynoType(dm.Source),
								},
							},
						)
					}

					// Dyno log-runtime-metrics load messages
				case bytes.Contains(msg, dynoLoadMsgSentinel):
					dynoLoadLinesCounter.Inc(1)
					dm := dynoLoadMsg{}
					err := logfmt.Unmarshal(msg, &dm)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}
					if dm.Source != "" {
						destination.PostPoint(
							Point{
								id,
								DynoLoad,
								[]interface{}{timestamp, dm.Source, dm.LoadAvg1Min, dm.LoadAvg5Min, dm.LoadAvg15Min, dynoType(dm.Source)},
							},
						)
					}

				// unknown
				default:
					unknownHerokuLinesCounter.Inc(1)
					if Debug {
						log.Printf("Unknown Heroku Line - Header: PRI: %s, Time: %s, Hostname: %s, Name: %s, ProcId: %s, MsgId: %s - Body: %s",
							header.PrivalVersion,
							header.Time,
							header.Hostname,
							header.Name,
							header.Procid,
							header.Msgid,
							string(msg),
						)
					}
				}
			}

		// non heroku lines
		default:
			unknownUserLinesCounter.Inc(1)
			if Debug {
				log.Printf("Unknown User Line - Header: PRI: %s, Time: %s, Hostname: %s, Name: %s, ProcId: %s, MsgId: %s - Body: %s",
					header.PrivalVersion,
					header.Time,
					header.Hostname,
					header.Name,
					header.Procid,
					header.Msgid,
					string(msg),
				)
			}
		}
	}

	linesCounter.Inc(int64(linesCounterInc))

	batchSizeHistogram.Update(int64(linesCounterInc))

	parseTimer.UpdateSince(parseStart)

	// If we are told to close the connection after the reply, do so.
	select {
	case <-connectionCloser:
		w.Header().Set("Connection", "close")
	default:
		//Nothing
	}

	w.WriteHeader(http.StatusNoContent)
}
<commit_msg>Report a metric instead of doing nothing<commit_after>package main

import (
	"bufio"
	"bytes"
	"encoding/base64"
	"errors"
	"log"
	"net/http"
	"strings"
	"time"

	"github.com/bmizerany/lpx"
	"github.com/kr/logfmt"
	metrics "github.com/rcrowley/go-metrics"
)

var (
	TokenPrefix = []byte("t.")
	Heroku      = []byte("heroku")

	// go-metrics Instruments
	wrongMethodErrorCounter    = metrics.NewRegisteredCounter("lumbermill.errors.drain.wrong.method", metrics.DefaultRegistry)
	authFailureCounter         = metrics.NewRegisteredCounter("lumbermill.errors.auth.failure", metrics.DefaultRegistry)
	badRequestCounter          = metrics.NewRegisteredCounter("lumbermill.errors.badrequest", metrics.DefaultRegistry)
	internalServerErrorCounter = metrics.NewRegisteredCounter("lumbermill.errors.internalserver", metrics.DefaultRegistry)
	tokenMissingCounter        = metrics.NewRegisteredCounter("lumbermill.errors.token.missing", metrics.DefaultRegistry)
	timeParsingErrorCounter    = metrics.NewRegisteredCounter("lumbermill.errors.time.parse", metrics.DefaultRegistry)
	logfmtParsingErrorCounter  = metrics.NewRegisteredCounter("lumbermill.errors.logfmt.parse", metrics.DefaultRegistry)
	droppedErrorCounter        = metrics.NewRegisteredCounter("lumbermill.errors.dropped", metrics.DefaultRegistry)
	batchCounter               = metrics.NewRegisteredCounter("lumbermill.batch", metrics.DefaultRegistry)
	linesCounter               = metrics.NewRegisteredCounter("lumbermill.lines", metrics.DefaultRegistry)
	routerErrorLinesCounter    = metrics.NewRegisteredCounter("lumbermill.lines.router.error", metrics.DefaultRegistry)
	routerLinesCounter         = metrics.NewRegisteredCounter("lumbermill.lines.router", metrics.DefaultRegistry)
	routerBlankLinesCounter    = metrics.NewRegisteredCounter("lumbermill.lines.router.blank", metrics.DefaultRegistry)
	dynoErrorLinesCounter      = metrics.NewRegisteredCounter("lumbermill.lines.dyno.error", metrics.DefaultRegistry)
	dynoMemLinesCounter        = metrics.NewRegisteredCounter("lumbermill.lines.dyno.mem", metrics.DefaultRegistry)
	dynoLoadLinesCounter       = metrics.NewRegisteredCounter("lumbermill.lines.dyno.load", metrics.DefaultRegistry)
	unknownHerokuLinesCounter  = metrics.NewRegisteredCounter("lumbermill.lines.unknown.heroku", metrics.DefaultRegistry)
	unknownUserLinesCounter    = metrics.NewRegisteredCounter("lumbermill.lines.unknown.user", metrics.DefaultRegistry)
	parseTimer                 = metrics.NewRegisteredTimer("lumbermill.batches.parse.time", metrics.DefaultRegistry)
	batchSizeHistogram         = metrics.NewRegisteredHistogram("lumbermill.batches.sizes", metrics.DefaultRegistry, metrics.NewUniformSample(100))
)

func checkAuth(r *http.Request) error {
	header := r.Header.Get("Authorization")
	if header == "" {
		return errors.New("Authorization required")
	}
	headerParts := strings.SplitN(header, " ", 2)
	if len(headerParts) != 2 {
		return errors.New("Authorization header is malformed")
	}

	method := headerParts[0]
	if method != "Basic" {
		return errors.New("Only Basic Authorization is accepted")
	}

	encodedUserPass := headerParts[1]
	decodedUserPass, err := base64.StdEncoding.DecodeString(encodedUserPass)
	if err != nil {
		return errors.New("Authorization header is malformed")
	}

	userPassParts := bytes.SplitN(decodedUserPass, []byte{':'}, 2)
	if len(userPassParts) != 2 {
		return errors.New("Authorization header is malformed")
	}

	user := userPassParts[0]
	pass := userPassParts[1]

	if string(user) != User {
		return errors.New("Unknown user")
	}
	if string(pass) != Password {
		return errors.New("Incorrect token")
	}

	return nil
}

// Dyno's are generally reported as "<type>.<#>"
// Extract the <type> and return it
func dynoType(what string) string {
	s := strings.Split(what, ".")
	return s[0]
}

func handleLogFmtParsingError(msg []byte, err error) {
	logfmtParsingErrorCounter.Inc(1)
	log.Printf("logfmt unmarshal error(%q): %q\n", string(msg), err)
}

// "Parse tree" from hell
func serveDrain(w http.ResponseWriter, r *http.Request) {

	w.Header().Set("Content-Length", "0")

	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		wrongMethodErrorCounter.Inc(1)
		return
	}

	id := r.Header.Get("Logplex-Drain-Token")

	if id == "" {
		if err := checkAuth(r); err != nil {
			w.WriteHeader(http.StatusForbidden)
			authFailureCounter.Inc(1)
			return
		}
	}

	batchCounter.Inc(1)

	parseStart := time.Now()
	lp := lpx.NewReader(bufio.NewReader(r.Body))

	linesCounterInc := 0

	for lp.Next() {
		linesCounterInc += 1
		header := lp.Header()

		// If the syslog Name Header field contains what looks like a log token,
		// let's assume it's an override of the id and we're getting the data from the magic
		// channel
		if bytes.HasPrefix(header.Name, TokenPrefix) {
			id = string(header.Name)
		}

		// If we still don't have an id, throw an error and try the next line
		if id == "" {
			tokenMissingCounter.Inc(1)
			continue
		}

		destination := hashRing.Get(id)

		msg := lp.Bytes()
		switch {
		case bytes.Equal(header.Name, Heroku), bytes.HasPrefix(header.Name, TokenPrefix):
			timeStr := string(lp.Header().Time)
			t, e := time.Parse("2006-01-02T15:04:05.000000+00:00", timeStr)
			if e != nil {
				t, e = time.Parse("2006-01-02T15:04:05+00:00", timeStr)
				if e != nil {
					timeParsingErrorCounter.Inc(1)
					log.Printf("Error Parsing Time(%s): %q\n", string(lp.Header().Time), e)
					continue
				}
			}

			timestamp := t.UnixNano() / int64(time.Microsecond)

			pid := string(header.Procid)
			switch pid {
			case "router":

				switch {
				// router logs with a H error code in them
				case bytes.Contains(msg, keyCodeH):
					routerErrorLinesCounter.Inc(1)
					re := routerError{}
					err := logfmt.Unmarshal(msg, &re)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}
					destination.PostPoint(Point{id, EventsRouter, []interface{}{timestamp, re.Code}})

				// If the app is blank (not pushed) we don't care
				// do nothing atm, increment a counter
				case bytes.Contains(msg, keyCodeBlank), bytes.Contains(msg, keyDescBlank):
					routerBlankLinesCounter.Inc(1)

				// likely a standard router log
				default:
					routerLinesCounter.Inc(1)
					rm := routerMsg{}
					err := logfmt.Unmarshal(msg, &rm)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}

					destination.PostPoint(Point{id, Router, []interface{}{timestamp, rm.Status, rm.Service}})
				}

				// Non router logs, so either dynos, runtime, etc
			default:
				switch {
				// Dyno error messages
				case bytes.HasPrefix(msg, dynoErrorSentinel):
					dynoErrorLinesCounter.Inc(1)
					de, err := parseBytesToDynoError(msg)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}

					what := string(lp.Header().Procid)
					destination.PostPoint(
						Point{id, EventsDyno, []interface{}{timestamp, what, "R", de.Code, string(msg), dynoType(what)}},
					)

				// Dyno log-runtime-metrics memory messages
				case bytes.Contains(msg, dynoMemMsgSentinel):
					dynoMemLinesCounter.Inc(1)
					dm := dynoMemMsg{}
					err := logfmt.Unmarshal(msg, &dm)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}
					if dm.Source != "" {
						destination.PostPoint(
							Point{
								id,
								DynoMem,
								[]interface{}{
									timestamp,
									dm.Source,
									dm.MemoryCache,
									dm.MemoryPgpgin,
									dm.MemoryPgpgout,
									dm.MemoryRSS,
									dm.MemorySwap,
									dm.MemoryTotal,
									dynoType(dm.Source),
								},
							},
						)
					}

					// Dyno log-runtime-metrics load messages
				case bytes.Contains(msg, dynoLoadMsgSentinel):
					dynoLoadLinesCounter.Inc(1)
					dm := dynoLoadMsg{}
					err := logfmt.Unmarshal(msg, &dm)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}
					if dm.Source != "" {
						destination.PostPoint(
							Point{
								id,
								DynoLoad,
								[]interface{}{timestamp, dm.Source, dm.LoadAvg1Min, dm.LoadAvg5Min, dm.LoadAvg15Min, dynoType(dm.Source)},
							},
						)
					}

				// unknown
				default:
					unknownHerokuLinesCounter.Inc(1)
					if Debug {
						log.Printf("Unknown Heroku Line - Header: PRI: %s, Time: %s, Hostname: %s, Name: %s, ProcId: %s, MsgId: %s - Body: %s",
							header.PrivalVersion,
							header.Time,
							header.Hostname,
							header.Name,
							header.Procid,
							header.Msgid,
							string(msg),
						)
					}
				}
			}

		// non heroku lines
		default:
			unknownUserLinesCounter.Inc(1)
			if Debug {
				log.Printf("Unknown User Line - Header: PRI: %s, Time: %s, Hostname: %s, Name: %s, ProcId: %s, MsgId: %s - Body: %s",
					header.PrivalVersion,
					header.Time,
					header.Hostname,
					header.Name,
					header.Procid,
					header.Msgid,
					string(msg),
				)
			}
		}
	}

	linesCounter.Inc(int64(linesCounterInc))

	batchSizeHistogram.Update(int64(linesCounterInc))

	parseTimer.UpdateSince(parseStart)

	// If we are told to close the connection after the reply, do so.
	select {
	case <-connectionCloser:
		w.Header().Set("Connection", "close")
	default:
		//Nothing
	}

	w.WriteHeader(http.StatusNoContent)
}
<|endoftext|>


<commit_before>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
package main

import (
	"errors"
	"io"
	"strings"

	"github.com/mendersoftware/log"
)

// This will be run manually from command line ONLY
func doRootfs(device UInstaller, args runOptionsType) error {
	var image io.ReadCloser
	var imageSize int64
	var err error
	var client Updater

	if args == (runOptionsType{}) {
		return errors.New("rootfs called without needed parameters")
	}

	log.Debug("Starting device update.")

	updateLocation := *args.imageFile
	if strings.HasPrefix(updateLocation, "http:") ||
		strings.HasPrefix(updateLocation, "https:") {
		log.Infof("Perfroming remote update from: [%s].", updateLocation)

		// we are having remote update
		client, err = NewUpdateClient(args.httpsClientConfig)

		if err != nil {
			return errors.New("Can not initialize client for performing network update.")
		}

		log.Debug("Client initialized. Start downloading image.")

		image, imageSize, err = client.FetchUpdate(updateLocation)
		log.Debugf("Image downloaded: %d [%v] [%v]", imageSize, image, err)
	} else {
		// perform update from local file
		log.Infof("Start updating from local image file: [%s]", updateLocation)
		image, imageSize, err = FetchUpdateFromFile(updateLocation)

		log.Debugf("Feting update from file results: [%v], %d, %v", image, imageSize, err)
	}

	if image != nil {
		defer image.Close()
	}

	if err != nil {
		return errors.New("Error while updateing image from command line: " + err.Error())
	}

	if err = device.InstallUpdate(image, imageSize); err != nil {
		return err
	}
	log.Info("Image correctly installed to inactive partition. " +
		"Marking inactive partition as the new boot candidate.")

	return device.EnableUpdatedPartition()
}
<commit_msg>rootfs: update the code to use updated API client interface<commit_after>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
package main

import (
	"errors"
	"io"
	"strings"

	"github.com/mendersoftware/log"
)

// This will be run manually from command line ONLY
func doRootfs(device UInstaller, args runOptionsType) error {
	var image io.ReadCloser
	var imageSize int64
	var err error
	var client Updater

	if args == (runOptionsType{}) {
		return errors.New("rootfs called without needed parameters")
	}

	log.Debug("Starting device update.")

	updateLocation := *args.imageFile
	if strings.HasPrefix(updateLocation, "http:") ||
		strings.HasPrefix(updateLocation, "https:") {
		log.Infof("Perfroming remote update from: [%s].", updateLocation)

		// we are having remote update
		ac, err := NewApiClient(args.httpsClientConfig)
		if err != nil {
			return errors.New("Can not initialize client for performing network update.")
		}
		client = NewUpdateClient()

		log.Debug("Client initialized. Start downloading image.")

		image, imageSize, err = client.FetchUpdate(ac, updateLocation)
		log.Debugf("Image downloaded: %d [%v] [%v]", imageSize, image, err)
	} else {
		// perform update from local file
		log.Infof("Start updating from local image file: [%s]", updateLocation)
		image, imageSize, err = FetchUpdateFromFile(updateLocation)

		log.Debugf("Feting update from file results: [%v], %d, %v", image, imageSize, err)
	}

	if image != nil {
		defer image.Close()
	}

	if err != nil {
		return errors.New("Error while updateing image from command line: " + err.Error())
	}

	if err = device.InstallUpdate(image, imageSize); err != nil {
		return err
	}
	log.Info("Image correctly installed to inactive partition. " +
		"Marking inactive partition as the new boot candidate.")

	return device.EnableUpdatedPartition()
}
<|endoftext|>