<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package httplog

import (
	"bufio"
	"context"
	"fmt"
	"net"
	"net/http"
	"runtime"
	"time"

	"k8s.io/klog"
)

// StacktracePred returns true if a stacktrace should be logged for this status.
type StacktracePred func(httpStatus int) (logStacktrace bool)

type logger interface {
	Addf(format string, data ...interface{})
}

type respLoggerContextKeyType int

// respLoggerContextKey is used to store the respLogger pointer in the request context.
const respLoggerContextKey respLoggerContextKeyType = iota

// Add a layer on top of ResponseWriter, so we can track latency and error
// message sources.
//
// TODO now that we're using go-restful, we shouldn't need to be wrapping
// the http.ResponseWriter. We can recover panics from go-restful, and
// the logging value is questionable.
type respLogger struct {
	hijacked       bool
	statusRecorded bool
	status         int
	statusStack    string
	addedInfo      string
	startTime      time.Time

	captureErrorOutput bool

	req *http.Request
	w   http.ResponseWriter

	logStacktracePred StacktracePred
}

// Simple logger that logs immediately when Addf is called
type passthroughLogger struct{}

// Addf logs info immediately.
func (passthroughLogger) Addf(format string, data ...interface{}) {
	klog.V(2).Info(fmt.Sprintf(format, data...))
}

// DefaultStacktracePred is the default implementation of StacktracePred.
func DefaultStacktracePred(status int) bool {
	return (status < http.StatusOK || status >= http.StatusInternalServerError) && status != http.StatusSwitchingProtocols
}

// WithLogging wraps the handler with logging.
func WithLogging(handler http.Handler, pred StacktracePred) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		ctx := req.Context()
		if old := respLoggerFromContext(req); old != nil {
			panic("multiple WithLogging calls!")
		}
		rl := newLogged(req, w).StacktraceWhen(pred)
		req = req.WithContext(context.WithValue(ctx, respLoggerContextKey, rl))

		defer rl.Log()
		handler.ServeHTTP(rl, req)
	})
}

// respLoggerFromContext returns the respLogger or nil.
func respLoggerFromContext(req *http.Request) *respLogger {
	ctx := req.Context()
	val := ctx.Value(respLoggerContextKey)
	if rl, ok := val.(*respLogger); ok {
		return rl
	}
	return nil
}

// newLogged turns a normal response writer into a logged response writer.
func newLogged(req *http.Request, w http.ResponseWriter) *respLogger {
	return &respLogger{
		startTime:         time.Now(),
		req:               req,
		w:                 w,
		logStacktracePred: DefaultStacktracePred,
	}
}

// LogOf returns the logger hiding in w. If there is not an existing logger
// then a passthroughLogger will be created which will log to stdout immediately
// when Addf is called.
func LogOf(req *http.Request, w http.ResponseWriter) logger {
	if rl := respLoggerFromContext(req); rl != nil {
		return rl
	}
	return &passthroughLogger{}
}

// Unlogged returns the original ResponseWriter, or w if it is not our inserted logger.
func Unlogged(req *http.Request, w http.ResponseWriter) http.ResponseWriter {
	if rl := respLoggerFromContext(req); rl != nil {
		return rl.w
	}
	return w
}

// StacktraceWhen sets the stacktrace logging predicate, which decides when to log a stacktrace.
// There's a default, so you don't need to call this unless you don't like the default.
func (rl *respLogger) StacktraceWhen(pred StacktracePred) *respLogger {
	rl.logStacktracePred = pred
	return rl
}

// StatusIsNot returns a StacktracePred which will cause stacktraces to be logged
// for any status *not* in the given list.
func StatusIsNot(statuses ...int) StacktracePred {
	statusesNoTrace := map[int]bool{}
	for _, s := range statuses {
		statusesNoTrace[s] = true
	}
	return func(status int) bool {
		_, ok := statusesNoTrace[status]
		return !ok
	}
}

// Addf adds additional data to be logged with this request.
func (rl *respLogger) Addf(format string, data ...interface{}) {
	rl.addedInfo += "\n" + fmt.Sprintf(format, data...)
}

// Log is intended to be called once at the end of your request handler, via defer
func (rl *respLogger) Log() {
	latency := time.Since(rl.startTime)
	if klog.V(3) {
		if !rl.hijacked {
			klog.InfoDepth(1, fmt.Sprintf("%s %s: (%v) %v [%s %s]%v%v", rl.req.Method, rl.req.RequestURI, latency, rl.status, rl.req.UserAgent(), rl.req.RemoteAddr, rl.statusStack, rl.addedInfo))
		} else {
			klog.InfoDepth(1, fmt.Sprintf("%s %s: (%v) hijacked [%s %s]", rl.req.Method, rl.req.RequestURI, latency, rl.req.UserAgent(), rl.req.RemoteAddr))
		}
	}
}

// Header implements http.ResponseWriter.
func (rl *respLogger) Header() http.Header {
	return rl.w.Header()
}

// Write implements http.ResponseWriter.
func (rl *respLogger) Write(b []byte) (int, error) {
	if !rl.statusRecorded {
		rl.recordStatus(http.StatusOK) // Default if WriteHeader hasn't been called
	}
	if rl.captureErrorOutput {
		rl.Addf("logging error output: %q\n", string(b))
	}
	return rl.w.Write(b)
}

// Flush implements http.Flusher even if the underlying http.Writer doesn't implement it.
// Flush is used for streaming purposes and allows to flush buffered data to the client.
func (rl *respLogger) Flush() {
	if flusher, ok := rl.w.(http.Flusher); ok {
		flusher.Flush()
	} else if klog.V(2) {
		klog.InfoDepth(1, fmt.Sprintf("Unable to convert %+v into http.Flusher", rl.w))
	}
}

// WriteHeader implements http.ResponseWriter.
func (rl *respLogger) WriteHeader(status int) {
	rl.recordStatus(status)
	rl.w.WriteHeader(status)
}

// Hijack implements http.Hijacker.
func (rl *respLogger) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	rl.hijacked = true
	return rl.w.(http.Hijacker).Hijack()
}

// CloseNotify implements http.CloseNotifier
func (rl *respLogger) CloseNotify() <-chan bool {
	return rl.w.(http.CloseNotifier).CloseNotify()
}

func (rl *respLogger) recordStatus(status int) {
	rl.status = status
	rl.statusRecorded = true
	if rl.logStacktracePred(status) {
		// Only log stacks for errors
		stack := make([]byte, 50*1024)
		stack = stack[:runtime.Stack(stack, false)]
		rl.statusStack = "\n" + string(stack)
		rl.captureErrorOutput = true
	} else {
		rl.statusStack = ""
	}
}
<commit_msg>make request logs greppable<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package httplog

import (
	"bufio"
	"context"
	"fmt"
	"net"
	"net/http"
	"runtime"
	"time"

	"k8s.io/klog"
)

// StacktracePred returns true if a stacktrace should be logged for this status.
type StacktracePred func(httpStatus int) (logStacktrace bool)

type logger interface {
	Addf(format string, data ...interface{})
}

type respLoggerContextKeyType int

// respLoggerContextKey is used to store the respLogger pointer in the request context.
const respLoggerContextKey respLoggerContextKeyType = iota

// Add a layer on top of ResponseWriter, so we can track latency and error
// message sources.
//
// TODO now that we're using go-restful, we shouldn't need to be wrapping
// the http.ResponseWriter. We can recover panics from go-restful, and
// the logging value is questionable.
type respLogger struct {
	hijacked       bool
	statusRecorded bool
	status         int
	statusStack    string
	addedInfo      string
	startTime      time.Time

	captureErrorOutput bool

	req *http.Request
	w   http.ResponseWriter

	logStacktracePred StacktracePred
}

// Simple logger that logs immediately when Addf is called
type passthroughLogger struct{}

// Addf logs info immediately.
func (passthroughLogger) Addf(format string, data ...interface{}) {
	klog.V(2).Info(fmt.Sprintf(format, data...))
}

// DefaultStacktracePred is the default implementation of StacktracePred.
func DefaultStacktracePred(status int) bool {
	return (status < http.StatusOK || status >= http.StatusInternalServerError) && status != http.StatusSwitchingProtocols
}

// WithLogging wraps the handler with logging.
func WithLogging(handler http.Handler, pred StacktracePred) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		ctx := req.Context()
		if old := respLoggerFromContext(req); old != nil {
			panic("multiple WithLogging calls!")
		}
		rl := newLogged(req, w).StacktraceWhen(pred)
		req = req.WithContext(context.WithValue(ctx, respLoggerContextKey, rl))

		defer rl.Log()
		handler.ServeHTTP(rl, req)
	})
}

// respLoggerFromContext returns the respLogger or nil.
func respLoggerFromContext(req *http.Request) *respLogger {
	ctx := req.Context()
	val := ctx.Value(respLoggerContextKey)
	if rl, ok := val.(*respLogger); ok {
		return rl
	}
	return nil
}

// newLogged turns a normal response writer into a logged response writer.
func newLogged(req *http.Request, w http.ResponseWriter) *respLogger {
	return &respLogger{
		startTime:         time.Now(),
		req:               req,
		w:                 w,
		logStacktracePred: DefaultStacktracePred,
	}
}

// LogOf returns the logger hiding in w. If there is not an existing logger
// then a passthroughLogger will be created which will log to stdout immediately
// when Addf is called.
func LogOf(req *http.Request, w http.ResponseWriter) logger {
	if rl := respLoggerFromContext(req); rl != nil {
		return rl
	}
	return &passthroughLogger{}
}

// Unlogged returns the original ResponseWriter, or w if it is not our inserted logger.
func Unlogged(req *http.Request, w http.ResponseWriter) http.ResponseWriter {
	if rl := respLoggerFromContext(req); rl != nil {
		return rl.w
	}
	return w
}

// StacktraceWhen sets the stacktrace logging predicate, which decides when to log a stacktrace.
// There's a default, so you don't need to call this unless you don't like the default.
func (rl *respLogger) StacktraceWhen(pred StacktracePred) *respLogger {
	rl.logStacktracePred = pred
	return rl
}

// StatusIsNot returns a StacktracePred which will cause stacktraces to be logged
// for any status *not* in the given list.
func StatusIsNot(statuses ...int) StacktracePred {
	statusesNoTrace := map[int]bool{}
	for _, s := range statuses {
		statusesNoTrace[s] = true
	}
	return func(status int) bool {
		_, ok := statusesNoTrace[status]
		return !ok
	}
}

// Addf adds additional data to be logged with this request.
func (rl *respLogger) Addf(format string, data ...interface{}) {
	rl.addedInfo += "\n" + fmt.Sprintf(format, data...)
}

// Log is intended to be called once at the end of your request handler, via defer
func (rl *respLogger) Log() {
	latency := time.Since(rl.startTime)
	if klog.V(3) {
		if !rl.hijacked {
			klog.InfoDepth(1, fmt.Sprintf("verb=%q URI=%q latency=%v resp=%v UserAgent=%q srcIP=%q: %v%v",
				rl.req.Method, rl.req.RequestURI,
				latency, rl.status,
				rl.req.UserAgent(), rl.req.RemoteAddr,
				rl.statusStack, rl.addedInfo,
			))
		} else {
			klog.InfoDepth(1, fmt.Sprintf("verb=%q URI=%q latency=%v UserAgent=%q srcIP=%q: hijacked",
				rl.req.Method, rl.req.RequestURI,
				latency, rl.req.UserAgent(), rl.req.RemoteAddr,
			))
		}
	}
}

// Header implements http.ResponseWriter.
func (rl *respLogger) Header() http.Header {
	return rl.w.Header()
}

// Write implements http.ResponseWriter.
func (rl *respLogger) Write(b []byte) (int, error) {
	if !rl.statusRecorded {
		rl.recordStatus(http.StatusOK) // Default if WriteHeader hasn't been called
	}
	if rl.captureErrorOutput {
		rl.Addf("logging error output: %q\n", string(b))
	}
	return rl.w.Write(b)
}

// Flush implements http.Flusher even if the underlying http.Writer doesn't implement it.
// Flush is used for streaming purposes and allows to flush buffered data to the client.
func (rl *respLogger) Flush() {
	if flusher, ok := rl.w.(http.Flusher); ok {
		flusher.Flush()
	} else if klog.V(2) {
		klog.InfoDepth(1, fmt.Sprintf("Unable to convert %+v into http.Flusher", rl.w))
	}
}

// WriteHeader implements http.ResponseWriter.
func (rl *respLogger) WriteHeader(status int) {
	rl.recordStatus(status)
	rl.w.WriteHeader(status)
}

// Hijack implements http.Hijacker.
func (rl *respLogger) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	rl.hijacked = true
	return rl.w.(http.Hijacker).Hijack()
}

// CloseNotify implements http.CloseNotifier
func (rl *respLogger) CloseNotify() <-chan bool {
	return rl.w.(http.CloseNotifier).CloseNotify()
}

func (rl *respLogger) recordStatus(status int) {
	rl.status = status
	rl.statusRecorded = true
	if rl.logStacktracePred(status) {
		// Only log stacks for errors
		stack := make([]byte, 50*1024)
		stack = stack[:runtime.Stack(stack, false)]
		rl.statusStack = "\n" + string(stack)
		rl.captureErrorOutput = true
	} else {
		rl.statusStack = ""
	}
}
<|endoftext|>


<commit_before>package termite

import (
	"bytes"
	"fmt"
	"github.com/hanwen/termite/attr"
	"github.com/hanwen/termite/fs"
	"log"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"
)

type WorkerTask struct {
	req       *WorkRequest
	rep       *WorkResponse
	stdinConn net.Conn
	mirror    *Mirror
	cmd       *exec.Cmd
	taskInfo  string
}

func (me *WorkerTask) Kill() {
	if me.cmd.Process != nil {
		pid := me.cmd.Process.Pid
		err := syscall.Kill(pid, syscall.SIGQUIT)
		log.Printf("Killed pid %d, result %v", pid, err)
	}
}

func (me *WorkerTask) String() string {
	return me.taskInfo
}

func (me *WorkerTask) Run() error {
	fuseFs, err := me.mirror.newFs(me)

	if err == ShuttingDownError {
		// We can't return an error, since that would cause
		// the master to drop us directly before the other
		// jobs finished, opening a time window where there
		// might no worker running at all.
		//
		// TODO - some gentler signaling to the master?
		select {}
	}

	if err != nil {
		return err
	}

	me.mirror.worker.stats.Enter("fuse")
	err = me.runInFuse(fuseFs)
	me.mirror.worker.stats.Exit("fuse")

	defer me.mirror.returnFs(fuseFs)

	me.mirror.worker.stats.Enter("reap")
	if me.mirror.considerReap(fuseFs, me) {
		me.rep.FileSet, me.rep.TaskIds = me.mirror.reapFuse(fuseFs)
	}
	me.mirror.worker.stats.Exit("reap")

	return err
}

func (me *WorkerTask) runInFuse(fuseFs *workerFuseFs) error {
	fuseFs.SetDebug(me.req.Debug)
	stdout := &bytes.Buffer{}
	stderr := &bytes.Buffer{}

	// See /bin/true for the background of
	// /bin/true. http://code.google.com/p/go/issues/detail?id=2373
	me.cmd = &exec.Cmd{
		Path: me.req.Binary,
		Args: me.req.Argv,
	}
	cmd := me.cmd
	if os.Geteuid() == 0 {
		attr := &syscall.SysProcAttr{}
		attr.Credential = &syscall.Credential{
			Uid: uint32(me.mirror.worker.options.User.Uid),
			Gid: uint32(me.mirror.worker.options.User.Gid),
		}
		attr.Chroot = fuseFs.mount

		cmd.SysProcAttr = attr
		cmd.Dir = me.req.Dir
	} else {
		cmd.Path = filepath.Join(fuseFs.mount, me.req.Binary)
		cmd.Dir = filepath.Join(fuseFs.mount, me.req.Dir)
	}

	cmd.Env = me.req.Env
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	if me.stdinConn != nil {
		cmd.Stdin = me.stdinConn
	}

	if err := cmd.Start(); err != nil {
		return err
	}

	printCmd := fmt.Sprintf("%v", cmd.Args)
	if me.req.Debug {
		printCmd = fmt.Sprintf("%v", cmd)
	}
	me.taskInfo = fmt.Sprintf("%v, dir %v, fuse FS %v",
		printCmd, cmd.Dir, fuseFs.id)
	err := cmd.Wait()

	waitMsg, ok := err.(*exec.ExitError)
	if ok {
		me.rep.Exit = *waitMsg.Waitmsg
		err = nil
	}

	// No waiting: if the process exited, we kill the connection.
	if me.stdinConn != nil {
		me.stdinConn.Close()
	}

	// We could use a connection here too, but this is simpler.
	me.rep.Stdout = stdout.String()
	me.rep.Stderr = stderr.String()

	return err
}

// Sorts FileAttr such deletions come reversed before additions.

func (me *Mirror) fillReply(ufs *fs.MemUnionFs) *attr.FileSet {
	yield := ufs.Reap()
	wrRoot := strings.TrimLeft(me.writableRoot, "/")
	content := me.worker.content

	files := []*attr.FileAttr{}
	reapedHashes := map[string]string{}
	for path, v := range yield {
		f := &attr.FileAttr{
			Path: filepath.Join(wrRoot, path),
		}

		if v.Attr != nil {
			f.Attr = v.Attr
		}
		f.Link = v.Link
		if !f.Deletion() && f.IsRegular() {
			contentPath := filepath.Join(wrRoot, v.Original)
			if v.Original != "" && v.Original != contentPath {
				fa := me.rpcFs.attr.Get(contentPath)
				if fa.Hash == "" {
					log.Panicf("Contents for %q disappeared.", contentPath)
				}
				f.Hash = fa.Hash
			}
			if v.Backing != "" {
				f.Hash = reapedHashes[v.Backing]
				var err error
				if f.Hash == "" {
					f.Hash, err = content.DestructiveSavePath(v.Backing)
				}
				if err != nil {
					log.Fatalf("DestructiveSavePath fail %q: %v", v.Backing, err)
				} else {
					reapedHashes[v.Backing] = f.Hash
				}
			}
		}

		files = append(files, f)
	}
	fs := attr.FileSet{Files: files}
	fs.Sort()

	return &fs
}
<commit_msg>Note TODO.<commit_after>package termite

import (
	"bytes"
	"fmt"
	"github.com/hanwen/termite/attr"
	"github.com/hanwen/termite/fs"
	"log"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"
)

type WorkerTask struct {
	req       *WorkRequest
	rep       *WorkResponse
	stdinConn net.Conn
	mirror    *Mirror
	cmd       *exec.Cmd
	taskInfo  string
}

func (me *WorkerTask) Kill() {
	if me.cmd.Process != nil {
		pid := me.cmd.Process.Pid
		err := syscall.Kill(pid, syscall.SIGQUIT)
		log.Printf("Killed pid %d, result %v", pid, err)
	}
}

func (me *WorkerTask) String() string {
	return me.taskInfo
}

func (me *WorkerTask) Run() error {
	fuseFs, err := me.mirror.newFs(me)

	if err == ShuttingDownError {
		// We can't return an error, since that would cause
		// the master to drop us directly before the other
		// jobs finished, opening a time window where there
		// might no worker running at all.
		//
		// TODO - some gentler signaling to the master?
		select {}
	}

	if err != nil {
		return err
	}

	me.mirror.worker.stats.Enter("fuse")
	err = me.runInFuse(fuseFs)
	me.mirror.worker.stats.Exit("fuse")

	defer me.mirror.returnFs(fuseFs)

	me.mirror.worker.stats.Enter("reap")
	if me.mirror.considerReap(fuseFs, me) {
		me.rep.FileSet, me.rep.TaskIds = me.mirror.reapFuse(fuseFs)
	}
	me.mirror.worker.stats.Exit("reap")

	return err
}

func (me *WorkerTask) runInFuse(fuseFs *workerFuseFs) error {
	fuseFs.SetDebug(me.req.Debug)
	stdout := &bytes.Buffer{}
	stderr := &bytes.Buffer{}

	// See /bin/true for the background of
	// /bin/true. http://code.google.com/p/go/issues/detail?id=2373
	me.cmd = &exec.Cmd{
		Path: me.req.Binary,
		Args: me.req.Argv,
	}
	cmd := me.cmd
	if os.Geteuid() == 0 {
		attr := &syscall.SysProcAttr{}
		attr.Credential = &syscall.Credential{
			Uid: uint32(me.mirror.worker.options.User.Uid),
			Gid: uint32(me.mirror.worker.options.User.Gid),
		}
		attr.Chroot = fuseFs.mount

		cmd.SysProcAttr = attr
		cmd.Dir = me.req.Dir
	} else {
		cmd.Path = filepath.Join(fuseFs.mount, me.req.Binary)
		cmd.Dir = filepath.Join(fuseFs.mount, me.req.Dir)
	}

	cmd.Env = me.req.Env
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	if me.stdinConn != nil {
		cmd.Stdin = me.stdinConn
	}

	if err := cmd.Start(); err != nil {
		return err
	}

	printCmd := fmt.Sprintf("%v", cmd.Args)
	if me.req.Debug {
		printCmd = fmt.Sprintf("%v", cmd)
	}
	me.taskInfo = fmt.Sprintf("%v, dir %v, fuse FS %v",
		printCmd, cmd.Dir, fuseFs.id)
	err := cmd.Wait()

	waitMsg, ok := err.(*exec.ExitError)
	if ok {
		me.rep.Exit = *waitMsg.Waitmsg
		err = nil
	}

	// No waiting: if the process exited, we kill the connection.
	if me.stdinConn != nil {
		me.stdinConn.Close()
	}

	// We could use a connection here too, but this is simpler.
	me.rep.Stdout = stdout.String()
	me.rep.Stderr = stderr.String()

	return err
}

// Sorts FileAttr such deletions come reversed before additions.

// TODO - rename files out of backing store, and return the FS early
func (me *Mirror) fillReply(ufs *fs.MemUnionFs) *attr.FileSet {
	yield := ufs.Reap()
	wrRoot := strings.TrimLeft(me.writableRoot, "/")
	content := me.worker.content

	files := []*attr.FileAttr{}
	reapedHashes := map[string]string{}
	for path, v := range yield {
		f := &attr.FileAttr{
			Path: filepath.Join(wrRoot, path),
		}

		if v.Attr != nil {
			f.Attr = v.Attr
		}
		f.Link = v.Link
		if !f.Deletion() && f.IsRegular() {
			contentPath := filepath.Join(wrRoot, v.Original)
			if v.Original != "" && v.Original != contentPath {
				fa := me.rpcFs.attr.Get(contentPath)
				if fa.Hash == "" {
					log.Panicf("Contents for %q disappeared.", contentPath)
				}
				f.Hash = fa.Hash
			}
			if v.Backing != "" {
				f.Hash = reapedHashes[v.Backing]
				var err error
				if f.Hash == "" {
					f.Hash, err = content.DestructiveSavePath(v.Backing)
				}
				if err != nil {
					log.Fatalf("DestructiveSavePath fail %q: %v", v.Backing, err)
				} else {
					reapedHashes[v.Backing] = f.Hash
				}
			}
		}

		files = append(files, f)
	}
	fs := attr.FileSet{Files: files}
	fs.Sort()

	return &fs
}
<|endoftext|>


<commit_before>package xdg

// XDG user directories environment variables.
var (
	envDesktopDir     = "XDG_DESKTOP_DIR"
	envDownloadDir    = "XDG_DOWNLOAD_DIR"
	envDocumentsDir   = "XDG_DOCUMENTS_DIR"
	envMusicDir       = "XDG_MUSIC_DIR"
	envPicturesDir    = "XDG_PICTURES_DIR"
	envVideosDir      = "XDG_VIDEOS_DIR"
	envTemplatesDir   = "XDG_TEMPLATES_DIR"
	envPublicShareDir = "XDG_PUBLICSHARE_DIR"
)

// UserDirectories defines the locations of well known user directories.
type UserDirectories struct {
	// Desktop defines the location of the user's desktop directory.
	Desktop string

	// Download defines a suitable location for user downloaded files.
	Download string

	// Documents defines a suitable location for user document files.
	Documents string

	// Music defines a suitable location for user audio files.
	Music string

	// Pictures defines a suitable location for user image files.
	Pictures string

	// VideosDir defines a suitable location for user video files.
	Videos string

	// Templates defines a suitable location for user template files.
	Templates string

	// PublicShare defines a suitable location for user shared files.
	PublicShare string
}
<commit_msg>Minor internal improvement<commit_after>package xdg

// XDG user directories environment variables.
const (
	envDesktopDir     = "XDG_DESKTOP_DIR"
	envDownloadDir    = "XDG_DOWNLOAD_DIR"
	envDocumentsDir   = "XDG_DOCUMENTS_DIR"
	envMusicDir       = "XDG_MUSIC_DIR"
	envPicturesDir    = "XDG_PICTURES_DIR"
	envVideosDir      = "XDG_VIDEOS_DIR"
	envTemplatesDir   = "XDG_TEMPLATES_DIR"
	envPublicShareDir = "XDG_PUBLICSHARE_DIR"
)

// UserDirectories defines the locations of well known user directories.
type UserDirectories struct {
	// Desktop defines the location of the user's desktop directory.
	Desktop string

	// Download defines a suitable location for user downloaded files.
	Download string

	// Documents defines a suitable location for user document files.
	Documents string

	// Music defines a suitable location for user audio files.
	Music string

	// Pictures defines a suitable location for user image files.
	Pictures string

	// VideosDir defines a suitable location for user video files.
	Videos string

	// Templates defines a suitable location for user template files.
	Templates string

	// PublicShare defines a suitable location for user shared files.
	PublicShare string
}
<|endoftext|>


<commit_before>package util

import (
	"fmt"
	"os"
	"strings"

	"github.com/mundipagg/boleto-api/config"
	"github.com/mundipagg/boleto-api/log"

	dc "github.com/hugocarreira/go-decent-copy"
)

//ListCert Lista os Certificados necessários e chama o método que faz a cópia
func ListCert() error {

	list := []string{
		config.Get().CertBoletoPathCrt,
		config.Get().CertBoletoPathKey,
		config.Get().CertBoletoPathCa,
		config.Get().CertICP_PathPkey,
		config.Get().CertICP_PathChainCertificates,
	}

	var err error

	for _, v := range list {

		err = copyCert(v)

		if err != nil {
			return err
		}

	}

	return nil

}

func copyCert(certificateDestiny string) error {

	originPath := strings.Replace(certificateDestiny, "boleto_cert", "boleto_orig", 1)

	err := dc.Copy(originPath, certificateDestiny)
	if err != nil {
		logCopy("Copy Certificates Error:" + err.Error())
		return err
	}

	err = os.Chmod(certificateDestiny, 0777)
	if err != nil {
		logCopy("Copy Certificates Error:" + err.Error())

		return err
	}

	logCopy("Certificate Copy Sucessful: " + certificateDestiny)

	return nil

}

func logCopy(msg string) {
	fmt.Println(msg)
	log.Info("[{Application}: Certificates] - " + msg)
}
<commit_msg>:coffin: Remove arquivo não utilizado<commit_after><|endoftext|>


<commit_before>// +build linux darwin freebsd

package util

import (
	"fmt"
	"os/exec"
	"time"

	"github.com/Songmu/timeout"
)

// timeoutDuration is option of `Runcommand()` set timeout limit of command execution.
var timeoutDuration = 30 * time.Second

// timeoutKillAfter is option of `RunCommand()` set waiting limit to `kill -kill` after terminating the command.
var timeoutKillAfter = 10 * time.Second

// RunCommand runs command (in one string) and returns stdout, stderr strings and its exit code.
func RunCommand(command string) (string, string, int, error) {
	tio := &timeout.Timeout{
		Cmd:       exec.Command("/bin/sh", "-c", command),
		Duration:  timeoutDuration,
		KillAfter: timeoutKillAfter,
	}
	exitStatus, stdout, stderr, err := tio.Run()

	if err == nil && exitStatus.IsTimedOut() {
		err = fmt.Errorf("command timed out")
	}

	return stdout, stderr, exitStatus.GetChildExitCode(), err
}
<commit_msg>add log when the command execution failed<commit_after>// +build linux darwin freebsd

package util

import (
	"fmt"
	"os/exec"
	"time"

	"github.com/Songmu/timeout"
	"github.com/mackerelio/mackerel-agent/logging"
)

var utilLogger = logging.GetLogger("util")

// timeoutDuration is option of `Runcommand()` set timeout limit of command execution.
var timeoutDuration = 30 * time.Second

// timeoutKillAfter is option of `RunCommand()` set waiting limit to `kill -kill` after terminating the command.
var timeoutKillAfter = 10 * time.Second

// RunCommand runs command (in one string) and returns stdout, stderr strings and its exit code.
func RunCommand(command string) (string, string, int, error) {
	tio := &timeout.Timeout{
		Cmd:       exec.Command("/bin/sh", "-c", command),
		Duration:  timeoutDuration,
		KillAfter: timeoutKillAfter,
	}
	exitStatus, stdout, stderr, err := tio.Run()

	if err == nil && exitStatus.IsTimedOut() {
		err = fmt.Errorf("command timed out")
	}
	if err != nil {
		utilLogger.Errorf("RunCommand error command: %s, error: %s", command, err)
	}
	return stdout, stderr, exitStatus.GetChildExitCode(), err
}
<|endoftext|>


<commit_before>package message

import (
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"socialapi/config"
	"socialapi/models"
	"socialapi/request"
	"socialapi/workers/common/response"
	"time"

	"github.com/koding/bongo"
)

func Create(u *url.URL, h http.Header, req *models.ChannelMessage, c *models.Context) (int, http.Header, interface{}, error) {

	channelId, err := fetchInitialChannelId(u, c)
	if err != nil {
		return response.NewBadRequest(err)
	}

	// override message type
	// all of the messages coming from client-side
	// should be marked as POST
	req.TypeConstant = models.ChannelMessage_TYPE_POST

	req.InitialChannelId = channelId

	if err := checkThrottle(channelId, req.AccountId); err != nil {
		return response.NewBadRequest(err)
	}

	if err := req.Create(); err != nil {
		// todo this should be internal server error
		return response.NewBadRequest(err)
	}

	cml := models.NewChannelMessageList()
	// override channel id
	cml.ChannelId = channelId
	cml.MessageId = req.Id
	cml.ClientRequestId = req.ClientRequestId
	if err := cml.Create(); err != nil {
		// todo this should be internal server error
		return response.NewBadRequest(err)
	}

	cmc := models.NewChannelMessageContainer()
	err = cmc.Fetch(req.Id, request.GetQuery(u))

	// assign client request id back to message response because
	// client uses it for latency compansation
	cmc.Message.ClientRequestId = req.ClientRequestId
	return response.HandleResultAndError(cmc, err)
}

func fetchInitialChannelId(u *url.URL, context *models.Context) (int64, error) {
	channelId, err := request.GetURIInt64(u, "id")
	if err != nil {
		return 0, err
	}

	c, err := models.ChannelById(channelId)
	if err != nil {
		return 0, err
	}

	// when somebody posts on a topic channel, for creating it both in public and topic channels
	// initialChannelId must be set as current group's public channel id
	if c.TypeConstant != models.Channel_TYPE_TOPIC {
		return channelId, nil
	}

	publicChannel := models.NewChannel()
	if err := publicChannel.FetchPublicChannel(context.GroupName); err != nil {
		return 0, err
	}

	return publicChannel.Id, nil
}

func checkThrottle(channelId, requesterId int64) error {
	c, err := models.ChannelById(channelId)
	if err != nil {
		return err
	}

	if c.TypeConstant != models.Channel_TYPE_GROUP {
		return nil
	}

	cm := models.NewChannelMessage()

	conf := config.MustGet()

	// if oit is defaul treturn  early
	if conf.Limits.PostThrottleDuration == "" {
		return nil
	}

	// if throttle count is zero, it meands it is not set
	if conf.Limits.PostThrottleCount == 0 {
		return nil
	}

	dur, err := time.ParseDuration(conf.Limits.PostThrottleDuration)
	if err != nil {
		return err
	}

	// subtrack duration from current time
	prevTime := time.Now().UTC().Truncate(dur)

	// count sends positional parameters, no need to sanitize input
	count, err := bongo.B.Count(
		cm,
		"initial_channel_id = ? and "+
			"account_id = ? and "+
			"created_at > ?",
		channelId,
		requesterId,
		prevTime.Format(time.RFC3339Nano),
	)
	if err != nil {
		return err
	}

	if count > conf.Limits.PostThrottleCount {
		return fmt.Errorf("reached to throttle, current post count %d for user %d", count, requesterId)
	}

	return nil
}

func Delete(u *url.URL, h http.Header, _ interface{}) (int, http.Header, interface{}, error) {
	id, err := request.GetURIInt64(u, "id")
	if err != nil {
		return response.NewBadRequest(err)
	}

	cm := models.NewChannelMessage()
	cm.Id = id

	if err := cm.ById(id); err != nil {
		if err == bongo.RecordNotFound {
			return response.NewNotFound()
		}
		return response.NewBadRequest(err)
	}

	// if this is a reply no need to delete it's replies
	if cm.TypeConstant == models.ChannelMessage_TYPE_REPLY {
		mr := models.NewMessageReply()
		mr.ReplyId = id
		parent, err := mr.FetchParent()
		if err != nil {
			return response.NewBadRequest(err)
		}

		// delete the message here
		err = cm.DeleteMessageAndDependencies(false)
		// then invalidate the cache of the parent message
		bongo.B.AddToCache(parent)

	} else {
		err = cm.DeleteMessageAndDependencies(true)
	}

	if err != nil {
		return response.NewBadRequest(err)
	}

	// yes it is deleted but not removed completely from our system
	return response.NewDeleted()
}

func Update(u *url.URL, h http.Header, req *models.ChannelMessage) (int, http.Header, interface{}, error) {
	id, err := request.GetURIInt64(u, "id")
	if err != nil {
		return response.NewBadRequest(err)
	}

	body := req.Body
	payload := req.Payload
	if err := req.ById(id); err != nil {
		if err == bongo.RecordNotFound {
			return response.NewNotFound()
		}
		return response.NewBadRequest(err)
	}

	if req.Id == 0 {
		return response.NewBadRequest(err)
	}

	req.Body = body
	req.Payload = payload
	if err := req.Update(); err != nil {
		return response.NewBadRequest(err)
	}

	cmc := models.NewChannelMessageContainer()
	return response.HandleResultAndError(cmc, cmc.Fetch(id, request.GetQuery(u)))
}

func Get(u *url.URL, h http.Header, _ interface{}) (int, http.Header, interface{}, error) {
	cm, err := getMessageByUrl(u)
	if err != nil {
		return response.NewBadRequest(err)
	}

	if cm.Id == 0 {
		return response.NewNotFound()
	}

	cmc := models.NewChannelMessageContainer()
	return response.HandleResultAndError(cmc, cmc.Fetch(cm.Id, request.GetQuery(u)))
}

func getMessageByUrl(u *url.URL) (*models.ChannelMessage, error) {

	// TODO
	// fmt.Println(`
	// 	------->
	//             ADD SECURTY CHECK FOR VISIBILTY OF THE MESSAGE
	//                         FOR THE REQUESTER
	//     ------->"`,
	// )

	id, err := request.GetURIInt64(u, "id")
	if err != nil {
		return nil, err
	}

	// get url query params
	q := request.GetQuery(u)

	query := &bongo.Query{
		Selector: map[string]interface{}{
			"id": id,
		},
		Pagination: *bongo.NewPagination(1, 0),
	}

	cm := models.NewChannelMessage()
	// add exempt info
	query.AddScope(models.RemoveTrollContent(cm, q.ShowExempt))

	if err := cm.One(query); err != nil {
		return nil, err
	}

	return cm, nil
}

func GetWithRelated(u *url.URL, h http.Header, _ interface{}) (int, http.Header, interface{}, error) {
	cm, err := getMessageByUrl(u)
	if err != nil {
		return response.NewBadRequest(err)
	}

	if cm.Id == 0 {
		return response.NewNotFound()
	}

	q := request.GetQuery(u)

	cmc := models.NewChannelMessageContainer()
	if err := cmc.Fetch(cm.Id, q); err != nil {
		return response.NewBadRequest(err)
	}

	cmc.AddIsInteracted(q).AddIsFollowed(q)

	return response.HandleResultAndError(cmc, cmc.Err)
}

func GetBySlug(u *url.URL, h http.Header, _ interface{}) (int, http.Header, interface{}, error) {
	q := request.GetQuery(u)

	if q.Slug == "" {
		return response.NewBadRequest(errors.New("slug is not set"))
	}

	cm := models.NewChannelMessage()
	if err := cm.BySlug(q); err != nil {
		if err == bongo.RecordNotFound {
			return response.NewNotFound()
		}
		return response.NewBadRequest(err)
	}

	cmc := models.NewChannelMessageContainer()
	if err := cmc.Fetch(cm.Id, q); err != nil {
		return response.NewBadRequest(err)
	}

	cmc.AddIsInteracted(q).AddIsFollowed(q)

	return response.HandleResultAndError(cmc, cmc.Err)
}
<commit_msg>socialapi: cache public channel<commit_after>package message

import (
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"socialapi/config"
	"socialapi/models"
	"socialapi/request"
	"socialapi/workers/common/response"
	"time"

	"github.com/koding/bongo"
)

var publicChannel *models.Channel

func Create(u *url.URL, h http.Header, req *models.ChannelMessage, c *models.Context) (int, http.Header, interface{}, error) {

	channelId, err := fetchInitialChannelId(u, c)
	if err != nil {
		return response.NewBadRequest(err)
	}

	// override message type
	// all of the messages coming from client-side
	// should be marked as POST
	req.TypeConstant = models.ChannelMessage_TYPE_POST

	req.InitialChannelId = channelId

	if err := checkThrottle(channelId, req.AccountId); err != nil {
		return response.NewBadRequest(err)
	}

	if err := req.Create(); err != nil {
		// todo this should be internal server error
		return response.NewBadRequest(err)
	}

	cml := models.NewChannelMessageList()
	// override channel id
	cml.ChannelId = channelId
	cml.MessageId = req.Id
	cml.ClientRequestId = req.ClientRequestId
	if err := cml.Create(); err != nil {
		// todo this should be internal server error
		return response.NewBadRequest(err)
	}

	cmc := models.NewChannelMessageContainer()
	err = cmc.Fetch(req.Id, request.GetQuery(u))

	// assign client request id back to message response because
	// client uses it for latency compansation
	cmc.Message.ClientRequestId = req.ClientRequestId
	return response.HandleResultAndError(cmc, err)
}

func fetchInitialChannelId(u *url.URL, context *models.Context) (int64, error) {
	channelId, err := request.GetURIInt64(u, "id")
	if err != nil {
		return 0, err
	}

	c, err := models.ChannelById(channelId)
	if err != nil {
		return 0, err
	}

	// when somebody posts on a topic channel, for creating it both in public and topic channels
	// initialChannelId must be set as current group's public channel id
	if c.TypeConstant != models.Channel_TYPE_TOPIC {
		return channelId, nil
	}

	return fetchPublicChannelId(context.GroupName)
}

// TODO when we implement Team product, we will need a better caching mechanism
func fetchPublicChannelId(groupName string) (int64, error) {

	if groupName == "koding" && publicChannel != nil {
		return publicChannel.Id, nil
	}

	channel := models.NewChannel()
	if groupName == "koding" {
		publicChannel = channel
	}

	if err := channel.FetchPublicChannel(groupName); err != nil {
		return 0, err
	}

	return channel.Id, nil
}

func checkThrottle(channelId, requesterId int64) error {
	c, err := models.ChannelById(channelId)
	if err != nil {
		return err
	}

	if c.TypeConstant != models.Channel_TYPE_GROUP {
		return nil
	}

	cm := models.NewChannelMessage()

	conf := config.MustGet()

	// if oit is defaul treturn  early
	if conf.Limits.PostThrottleDuration == "" {
		return nil
	}

	// if throttle count is zero, it meands it is not set
	if conf.Limits.PostThrottleCount == 0 {
		return nil
	}

	dur, err := time.ParseDuration(conf.Limits.PostThrottleDuration)
	if err != nil {
		return err
	}

	// subtrack duration from current time
	prevTime := time.Now().UTC().Truncate(dur)

	// count sends positional parameters, no need to sanitize input
	count, err := bongo.B.Count(
		cm,
		"initial_channel_id = ? and "+
			"account_id = ? and "+
			"created_at > ?",
		channelId,
		requesterId,
		prevTime.Format(time.RFC3339Nano),
	)
	if err != nil {
		return err
	}

	if count > conf.Limits.PostThrottleCount {
		return fmt.Errorf("reached to throttle, current post count %d for user %d", count, requesterId)
	}

	return nil
}

func Delete(u *url.URL, h http.Header, _ interface{}) (int, http.Header, interface{}, error) {
	id, err := request.GetURIInt64(u, "id")
	if err != nil {
		return response.NewBadRequest(err)
	}

	cm := models.NewChannelMessage()
	cm.Id = id

	if err := cm.ById(id); err != nil {
		if err == bongo.RecordNotFound {
			return response.NewNotFound()
		}
		return response.NewBadRequest(err)
	}

	// if this is a reply no need to delete it's replies
	if cm.TypeConstant == models.ChannelMessage_TYPE_REPLY {
		mr := models.NewMessageReply()
		mr.ReplyId = id
		parent, err := mr.FetchParent()
		if err != nil {
			return response.NewBadRequest(err)
		}

		// delete the message here
		err = cm.DeleteMessageAndDependencies(false)
		// then invalidate the cache of the parent message
		bongo.B.AddToCache(parent)

	} else {
		err = cm.DeleteMessageAndDependencies(true)
	}

	if err != nil {
		return response.NewBadRequest(err)
	}

	// yes it is deleted but not removed completely from our system
	return response.NewDeleted()
}

func Update(u *url.URL, h http.Header, req *models.ChannelMessage) (int, http.Header, interface{}, error) {
	id, err := request.GetURIInt64(u, "id")
	if err != nil {
		return response.NewBadRequest(err)
	}

	body := req.Body
	payload := req.Payload
	if err := req.ById(id); err != nil {
		if err == bongo.RecordNotFound {
			return response.NewNotFound()
		}
		return response.NewBadRequest(err)
	}

	if req.Id == 0 {
		return response.NewBadRequest(err)
	}

	req.Body = body
	req.Payload = payload
	if err := req.Update(); err != nil {
		return response.NewBadRequest(err)
	}

	cmc := models.NewChannelMessageContainer()
	return response.HandleResultAndError(cmc, cmc.Fetch(id, request.GetQuery(u)))
}

func Get(u *url.URL, h http.Header, _ interface{}) (int, http.Header, interface{}, error) {
	cm, err := getMessageByUrl(u)
	if err != nil {
		return response.NewBadRequest(err)
	}

	if cm.Id == 0 {
		return response.NewNotFound()
	}

	cmc := models.NewChannelMessageContainer()
	return response.HandleResultAndError(cmc, cmc.Fetch(cm.Id, request.GetQuery(u)))
}

func getMessageByUrl(u *url.URL) (*models.ChannelMessage, error) {

	// TODO
	// fmt.Println(`
	// 	------->
	//             ADD SECURTY CHECK FOR VISIBILTY OF THE MESSAGE
	//                         FOR THE REQUESTER
	//     ------->"`,
	// )

	id, err := request.GetURIInt64(u, "id")
	if err != nil {
		return nil, err
	}

	// get url query params
	q := request.GetQuery(u)

	query := &bongo.Query{
		Selector: map[string]interface{}{
			"id": id,
		},
		Pagination: *bongo.NewPagination(1, 0),
	}

	cm := models.NewChannelMessage()
	// add exempt info
	query.AddScope(models.RemoveTrollContent(cm, q.ShowExempt))

	if err := cm.One(query); err != nil {
		return nil, err
	}

	return cm, nil
}

func GetWithRelated(u *url.URL, h http.Header, _ interface{}) (int, http.Header, interface{}, error) {
	cm, err := getMessageByUrl(u)
	if err != nil {
		return response.NewBadRequest(err)
	}

	if cm.Id == 0 {
		return response.NewNotFound()
	}

	q := request.GetQuery(u)

	cmc := models.NewChannelMessageContainer()
	if err := cmc.Fetch(cm.Id, q); err != nil {
		return response.NewBadRequest(err)
	}

	cmc.AddIsInteracted(q).AddIsFollowed(q)

	return response.HandleResultAndError(cmc, cmc.Err)
}

func GetBySlug(u *url.URL, h http.Header, _ interface{}) (int, http.Header, interface{}, error) {
	q := request.GetQuery(u)

	if q.Slug == "" {
		return response.NewBadRequest(errors.New("slug is not set"))
	}

	cm := models.NewChannelMessage()
	if err := cm.BySlug(q); err != nil {
		if err == bongo.RecordNotFound {
			return response.NewNotFound()
		}
		return response.NewBadRequest(err)
	}

	cmc := models.NewChannelMessageContainer()
	if err := cmc.Fetch(cm.Id, q); err != nil {
		return response.NewBadRequest(err)
	}

	cmc.AddIsInteracted(q).AddIsFollowed(q)

	return response.HandleResultAndError(cmc, cmc.Err)
}
<|endoftext|>


<commit_before>package util

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/mgutz/ansi"
	"github.com/op/go-logging"
	"gopkg.in/coryb/yaml.v2"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"text/template"
	"time"
)

var log = logging.MustGetLogger("util")

func FindParentPaths(fileName string) []string {
	cwd, _ := os.Getwd()

	paths := make([]string, 0)

	// special case if homedir is not in current path then check there anyway
	homedir := os.Getenv("HOME")
	if !strings.HasPrefix(cwd, homedir) {
		file := fmt.Sprintf("%s/%s", homedir, fileName)
		if _, err := os.Stat(file); err == nil {
			paths = append(paths, file)
		}
	}

	var dir string
	for _, part := range strings.Split(cwd, string(os.PathSeparator)) {
		if dir == "/" {
			dir = fmt.Sprintf("/%s", part)
		} else {
			dir = fmt.Sprintf("%s/%s", dir, part)
		}
		file := fmt.Sprintf("%s/%s", dir, fileName)
		if _, err := os.Stat(file); err == nil {
			paths = append(paths, file)
		}
	}
	return paths
}

func FindClosestParentPath(fileName string) (string, error) {
	paths := FindParentPaths(fileName)
	if len(paths) > 0 {
		return paths[len(paths)-1], nil
	}
	return "", errors.New(fmt.Sprintf("%s not found in parent directory hierarchy", fileName))
}

func ReadFile(file string) string {
	var bytes []byte
	var err error
	if bytes, err = ioutil.ReadFile(file); err != nil {
		log.Error("Failed to read file %s: %s", file, err)
		os.Exit(1)
	}
	return string(bytes)
}

func CopyFile(src, dst string) (err error) {
	var s, d *os.File
	if s, err = os.Open(src); err == nil {
		defer s.Close()
		if d, err = os.Create(dst); err == nil {
			if _, err = io.Copy(d, s); err != nil {
				d.Close()
				return
			}
			return d.Close()
		}
	}
	return
}

func FuzzyAge(start string) (string, error) {
	if t, err := time.Parse("2006-01-02T15:04:05.000-0700", start); err != nil {
		return "", err
	} else {
		delta := time.Now().Sub(t)
		if delta.Minutes() < 2 {
			return "a minute", nil
		} else if dm := delta.Minutes(); dm < 45 {
			return fmt.Sprintf("%d minutes", int(dm)), nil
		} else if dm := delta.Minutes(); dm < 90 {
			return "an hour", nil
		} else if dh := delta.Hours(); dh < 24 {
			return fmt.Sprintf("%d hours", int(dh)), nil
		} else if dh := delta.Hours(); dh < 48 {
			return "a day", nil
		} else {
			return fmt.Sprintf("%d days", int(delta.Hours()/24)), nil
		}
	}
	return "unknown", nil
}

func RunTemplate(templateContent string, data interface{}, out io.Writer) error {

	if out == nil {
		out = os.Stdout
	}

	funcs := map[string]interface{}{
		"toJson": func(content interface{}) (string, error) {
			if bytes, err := json.MarshalIndent(content, "", "    "); err != nil {
				return "", err
			} else {
				return string(bytes), nil
			}
		},
		"append": func(more string, content interface{}) (string, error) {
			switch value := content.(type) {
			case string:
				return string(append([]byte(content.(string)), []byte(more)...)), nil
			case []byte:
				return string(append(content.([]byte), []byte(more)...)), nil
			default:
				return "", errors.New(fmt.Sprintf("Unknown type: %s", value))
			}
		},
		"indent": func(spaces int, content string) string {
			indent := make([]rune, spaces+1, spaces+1)
			indent[0] = '\n'
			for i := 1; i < spaces+1; i += 1 {
				indent[i] = ' '
			}

			lineSeps := []rune{'\n', '\u0085', '\u2028', '\u2029'}
			for _, sep := range lineSeps {
				indent[0] = sep
				content = strings.Replace(content, string(sep), string(indent), -1)
			}
			return content

		},
		"comment": func(content string) string {
			lineSeps := []rune{'\n', '\u0085', '\u2028', '\u2029'}
			for _, sep := range lineSeps {
				content = strings.Replace(content, string(sep), string([]rune{sep, '#', ' '}), -1)
			}
			return content
		},
		"color": func(color string) string {
			return ansi.ColorCode(color)
		},
		"split": func(sep string, content string) []string {
			return strings.Split(content, sep)
		},
		"abbrev": func(max int, content string) string {
			if len(content) > max {
				var buffer bytes.Buffer
				buffer.WriteString(content[:max-3])
				buffer.WriteString("...")
				return buffer.String()
			}
			return content
		},
		"rep": func(count int, content string) string {
			var buffer bytes.Buffer
			for i := 0; i < count; i += 1 {
				buffer.WriteString(content)
			}
			return buffer.String()
		},
		"age": func(content string) (string, error) {
			return FuzzyAge(content)
		},
	}
	if tmpl, err := template.New("template").Funcs(funcs).Parse(templateContent); err != nil {
		log.Error("Failed to parse template: %s", err)
		return err
	} else {
		if err := tmpl.Execute(out, data); err != nil {
			log.Error("Failed to execute template: %s", err)
			return err
		}
	}
	return nil
}

func ResponseToJson(resp *http.Response, err error) (interface{}, error) {
	if err != nil {
		return nil, err
	}

	data := JsonDecode(resp.Body)
	if resp.StatusCode == 400 {
		if val, ok := data.(map[string]interface{})["errorMessages"]; ok {
			for _, errMsg := range val.([]interface{}) {
				log.Error("%s", errMsg)
			}
		}
	}

	return data, nil
}

func JsonDecode(io io.Reader) interface{} {
	content, err := ioutil.ReadAll(io)
	var data interface{}
	err = json.Unmarshal(content, &data)
	if err != nil {
		log.Error("JSON Parse Error: %s from %s", err, content)
	}
	return data
}

func JsonEncode(data interface{}) (string, error) {
	buffer := bytes.NewBuffer(make([]byte, 0))
	enc := json.NewEncoder(buffer)

	err := enc.Encode(data)
	if err != nil {
		log.Error("Failed to encode data %s: %s", data, err)
		return "", err
	}
	return buffer.String(), nil
}

func JsonWrite(file string, data interface{}) {
	fh, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	defer fh.Close()
	if err != nil {
		log.Error("Failed to open %s: %s", file, err)
		os.Exit(1)
	}
	enc := json.NewEncoder(fh)
	enc.Encode(data)
}

func PromptYN(prompt string, yes bool) bool {
	reader := bufio.NewReader(os.Stdin)
	if !yes {
		prompt = fmt.Sprintf("%s [y/N]: ", prompt)
	} else {
		prompt = fmt.Sprintf("%s [Y/n]: ", prompt)
	}

	fmt.Printf("%s", prompt)
	text, _ := reader.ReadString('\n')
	ans := strings.ToLower(strings.TrimRight(text, "\n"))
	if ans == "" {
		return yes
	}
	if strings.HasPrefix(ans, "y") {
		return true
	}
	return false
}

func ParseYaml(file string, opts map[string]interface{}) {
	if fh, err := ioutil.ReadFile(file); err == nil {
		log.Debug("Found Config file: %s", file)
		yaml.Unmarshal(fh, &opts)
	}
}

func YamlFixup(data interface{}) (interface{}, error) {
	switch d := data.(type) {
	case map[interface{}]interface{}:
		// need to copy this map into a string map so json can encode it
		copy := make(map[string]interface{})
		for key, val := range d {
			switch k := key.(type) {
			case string:
				if fixed, err := YamlFixup(val); err != nil {
					return nil, err
				} else if fixed != nil {
					copy[k] = fixed
				}
			default:
				err := fmt.Errorf("YAML: key %s is type '%T', require 'string'", key, k)
				log.Error("%s", err)
				return nil, err
			}
		}
		return copy, nil
	case map[string]interface{}:
		copy := make(map[string]interface{})
		for k, v := range d {
			if fixed, err := YamlFixup(v); err != nil {
				return nil, err
			} else if fixed != nil {
				copy[k] = fixed
			}
		}
		return copy, nil
	case []interface{}:
		copy := make([]interface{}, 0, len(d))
		for _, val := range d {
			if fixed, err := YamlFixup(val); err != nil {
				return nil, err
			} else if fixed != nil {
				copy = append(copy, fixed)
			}
		}
		return copy, nil
	case string:
		if d == "" || d == "\n" {
			return nil, nil
		}
		return d, nil
	default:
		return d, nil
	}
}

func Mkdir(dir string) error {
	if stat, err := os.Stat(dir); err != nil && !os.IsNotExist(err) {
		log.Error("Failed to stat %s: %s", dir, err)
		return err
	} else if err == nil && !stat.IsDir() {
		err := fmt.Errorf("%s exists and is not a directory!", dir)
		log.Error("%s", err)
		return err
	} else {
		// dir does not exist, so try to create it
		if err := os.MkdirAll(dir, 0755); err != nil {
			log.Error("Failed to mkdir -p %s: %s", dir, err)
			return err
		}
	}
	return nil
}
<commit_msg>add basic prompt routine<commit_after>package util

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/mgutz/ansi"
	"github.com/op/go-logging"
	"gopkg.in/coryb/yaml.v2"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"text/template"
	"time"
)

var log = logging.MustGetLogger("util")

func FindParentPaths(fileName string) []string {
	cwd, _ := os.Getwd()

	paths := make([]string, 0)

	// special case if homedir is not in current path then check there anyway
	homedir := os.Getenv("HOME")
	if !strings.HasPrefix(cwd, homedir) {
		file := fmt.Sprintf("%s/%s", homedir, fileName)
		if _, err := os.Stat(file); err == nil {
			paths = append(paths, file)
		}
	}

	var dir string
	for _, part := range strings.Split(cwd, string(os.PathSeparator)) {
		if dir == "/" {
			dir = fmt.Sprintf("/%s", part)
		} else {
			dir = fmt.Sprintf("%s/%s", dir, part)
		}
		file := fmt.Sprintf("%s/%s", dir, fileName)
		if _, err := os.Stat(file); err == nil {
			paths = append(paths, file)
		}
	}
	return paths
}

func FindClosestParentPath(fileName string) (string, error) {
	paths := FindParentPaths(fileName)
	if len(paths) > 0 {
		return paths[len(paths)-1], nil
	}
	return "", errors.New(fmt.Sprintf("%s not found in parent directory hierarchy", fileName))
}

func ReadFile(file string) string {
	var bytes []byte
	var err error
	if bytes, err = ioutil.ReadFile(file); err != nil {
		log.Error("Failed to read file %s: %s", file, err)
		os.Exit(1)
	}
	return string(bytes)
}

func CopyFile(src, dst string) (err error) {
	var s, d *os.File
	if s, err = os.Open(src); err == nil {
		defer s.Close()
		if d, err = os.Create(dst); err == nil {
			if _, err = io.Copy(d, s); err != nil {
				d.Close()
				return
			}
			return d.Close()
		}
	}
	return
}

func FuzzyAge(start string) (string, error) {
	if t, err := time.Parse("2006-01-02T15:04:05.000-0700", start); err != nil {
		return "", err
	} else {
		delta := time.Now().Sub(t)
		if delta.Minutes() < 2 {
			return "a minute", nil
		} else if dm := delta.Minutes(); dm < 45 {
			return fmt.Sprintf("%d minutes", int(dm)), nil
		} else if dm := delta.Minutes(); dm < 90 {
			return "an hour", nil
		} else if dh := delta.Hours(); dh < 24 {
			return fmt.Sprintf("%d hours", int(dh)), nil
		} else if dh := delta.Hours(); dh < 48 {
			return "a day", nil
		} else {
			return fmt.Sprintf("%d days", int(delta.Hours()/24)), nil
		}
	}
	return "unknown", nil
}

func RunTemplate(templateContent string, data interface{}, out io.Writer) error {

	if out == nil {
		out = os.Stdout
	}

	funcs := map[string]interface{}{
		"toJson": func(content interface{}) (string, error) {
			if bytes, err := json.MarshalIndent(content, "", "    "); err != nil {
				return "", err
			} else {
				return string(bytes), nil
			}
		},
		"append": func(more string, content interface{}) (string, error) {
			switch value := content.(type) {
			case string:
				return string(append([]byte(content.(string)), []byte(more)...)), nil
			case []byte:
				return string(append(content.([]byte), []byte(more)...)), nil
			default:
				return "", errors.New(fmt.Sprintf("Unknown type: %s", value))
			}
		},
		"indent": func(spaces int, content string) string {
			indent := make([]rune, spaces+1, spaces+1)
			indent[0] = '\n'
			for i := 1; i < spaces+1; i += 1 {
				indent[i] = ' '
			}

			lineSeps := []rune{'\n', '\u0085', '\u2028', '\u2029'}
			for _, sep := range lineSeps {
				indent[0] = sep
				content = strings.Replace(content, string(sep), string(indent), -1)
			}
			return content

		},
		"comment": func(content string) string {
			lineSeps := []rune{'\n', '\u0085', '\u2028', '\u2029'}
			for _, sep := range lineSeps {
				content = strings.Replace(content, string(sep), string([]rune{sep, '#', ' '}), -1)
			}
			return content
		},
		"color": func(color string) string {
			return ansi.ColorCode(color)
		},
		"split": func(sep string, content string) []string {
			return strings.Split(content, sep)
		},
		"abbrev": func(max int, content string) string {
			if len(content) > max {
				var buffer bytes.Buffer
				buffer.WriteString(content[:max-3])
				buffer.WriteString("...")
				return buffer.String()
			}
			return content
		},
		"rep": func(count int, content string) string {
			var buffer bytes.Buffer
			for i := 0; i < count; i += 1 {
				buffer.WriteString(content)
			}
			return buffer.String()
		},
		"age": func(content string) (string, error) {
			return FuzzyAge(content)
		},
	}
	if tmpl, err := template.New("template").Funcs(funcs).Parse(templateContent); err != nil {
		log.Error("Failed to parse template: %s", err)
		return err
	} else {
		if err := tmpl.Execute(out, data); err != nil {
			log.Error("Failed to execute template: %s", err)
			return err
		}
	}
	return nil
}

func ResponseToJson(resp *http.Response, err error) (interface{}, error) {
	if err != nil {
		return nil, err
	}

	data := JsonDecode(resp.Body)
	if resp.StatusCode == 400 {
		if val, ok := data.(map[string]interface{})["errorMessages"]; ok {
			for _, errMsg := range val.([]interface{}) {
				log.Error("%s", errMsg)
			}
		}
	}

	return data, nil
}

func JsonDecode(io io.Reader) interface{} {
	content, err := ioutil.ReadAll(io)
	var data interface{}
	err = json.Unmarshal(content, &data)
	if err != nil {
		log.Error("JSON Parse Error: %s from %s", err, content)
	}
	return data
}

func JsonEncode(data interface{}) (string, error) {
	buffer := bytes.NewBuffer(make([]byte, 0))
	enc := json.NewEncoder(buffer)

	err := enc.Encode(data)
	if err != nil {
		log.Error("Failed to encode data %s: %s", data, err)
		return "", err
	}
	return buffer.String(), nil
}

func JsonWrite(file string, data interface{}) {
	fh, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	defer fh.Close()
	if err != nil {
		log.Error("Failed to open %s: %s", file, err)
		os.Exit(1)
	}
	enc := json.NewEncoder(fh)
	enc.Encode(data)
}

func PromptYN(prompt string, yes bool) bool {
	reader := bufio.NewReader(os.Stdin)
	if !yes {
		prompt = fmt.Sprintf("%s [y/N]: ", prompt)
	} else {
		prompt = fmt.Sprintf("%s [Y/n]: ", prompt)
	}

	fmt.Printf("%s", prompt)
	text, _ := reader.ReadString('\n')
	ans := strings.ToLower(strings.TrimRight(text, "\n"))
	if ans == "" {
		return yes
	}
	if strings.HasPrefix(ans, "y") {
		return true
	}
	return false
}

func Prompt(prompt string) {
	reader := bufio.NewReader(os.Stdin)
	prompt = fmt.Sprintf("%s [ENTER]", prompt)
	fmt.Printf("%s", prompt)
	reader.ReadString('\n')
}

func ParseYaml(file string, opts map[string]interface{}) {
	if fh, err := ioutil.ReadFile(file); err == nil {
		log.Debug("Found Config file: %s", file)
		yaml.Unmarshal(fh, &opts)
	}
}

func YamlFixup(data interface{}) (interface{}, error) {
	switch d := data.(type) {
	case map[interface{}]interface{}:
		// need to copy this map into a string map so json can encode it
		copy := make(map[string]interface{})
		for key, val := range d {
			switch k := key.(type) {
			case string:
				if fixed, err := YamlFixup(val); err != nil {
					return nil, err
				} else if fixed != nil {
					copy[k] = fixed
				}
			default:
				err := fmt.Errorf("YAML: key %s is type '%T', require 'string'", key, k)
				log.Error("%s", err)
				return nil, err
			}
		}
		return copy, nil
	case map[string]interface{}:
		copy := make(map[string]interface{})
		for k, v := range d {
			if fixed, err := YamlFixup(v); err != nil {
				return nil, err
			} else if fixed != nil {
				copy[k] = fixed
			}
		}
		return copy, nil
	case []interface{}:
		copy := make([]interface{}, 0, len(d))
		for _, val := range d {
			if fixed, err := YamlFixup(val); err != nil {
				return nil, err
			} else if fixed != nil {
				copy = append(copy, fixed)
			}
		}
		return copy, nil
	case string:
		if d == "" || d == "\n" {
			return nil, nil
		}
		return d, nil
	default:
		return d, nil
	}
}

func Mkdir(dir string) error {
	if stat, err := os.Stat(dir); err != nil && !os.IsNotExist(err) {
		log.Error("Failed to stat %s: %s", dir, err)
		return err
	} else if err == nil && !stat.IsDir() {
		err := fmt.Errorf("%s exists and is not a directory!", dir)
		log.Error("%s", err)
		return err
	} else {
		// dir does not exist, so try to create it
		if err := os.MkdirAll(dir, 0755); err != nil {
			log.Error("Failed to mkdir -p %s: %s", dir, err)
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package util

import (
	"errors"
	"fmt"
	"io/ioutil"

	"golang.org/x/net/context"

	log "github.com/Sirupsen/logrus"
	docker "github.com/docker/engine-api/client"
	"github.com/docker/engine-api/types"
)

const labelPrefix string = "io.conplicity"

// CheckErr checks for error, logs and optionally exits the program
func CheckErr(err error, msg string, level string) {
	if err != nil {
		switch level {
		case "debug":
			log.Debugf(msg, err)
		case "info":
			log.Infof(msg, err)
		case "warn":
			log.Warnf(msg, err)
		case "error":
			log.Errorf(msg, err)
		case "fatal":
			log.Fatalf(msg, err)
		case "panic":
			log.Panicf(msg, err)
		default:
			log.Panicf("Wrong loglevel '%v', please report this bug", level)
		}
	}
}

// GetVolumeLabel retrieves the value of given key in the io.conplicity
// namespace of the volume labels
func GetVolumeLabel(vol *types.Volume, key string) (value string, err error) {
	log.Debugf("Getting value for label %s of volume %s", key, vol.Name)
	value, ok := vol.Labels[labelPrefix+"."+key]
	if !ok {
		errMsg := fmt.Sprintf("Key %v not found in labels for volume %v", key, vol.Name)
		err = errors.New(errMsg)
	}
	return
}

// PullImage pulls an image from the registry
func PullImage(c *docker.Client, image string) (err error) {
	if _, _, err = c.ImageInspectWithRaw(context.Background(), image, false); err != nil {
		// TODO: output pull to logs
		log.WithFields(log.Fields{
			"image": image,
		}).Info("Pulling image")
		resp, err := c.ImagePull(context.Background(), image, types.ImagePullOptions{})
		if err != nil {
			log.Errorf("ImagePull returned an error: %v", err)
			return err
		}
		defer resp.Close()
		body, err := ioutil.ReadAll(resp)
		if err != nil {
			log.Errorf("Failed to read from ImagePull response: %v", err)
			return err
		}
		log.Debugf("Pull image response body: %v", string(body))
	} else {
		log.WithFields(log.Fields{
			"image": image,
		}).Debug("Image already pulled, not pulling")
	}

	return nil
}

// RemoveContainer removes a container
func RemoveContainer(c *docker.Client, id string) {
	log.WithFields(log.Fields{
		"container": id,
	}).Infof("Removing container")
	err := c.ContainerRemove(context.Background(), id, types.ContainerRemoveOptions{
		Force:         true,
		RemoveVolumes: true,
	})
	CheckErr(err, "Failed to remove container "+id+": %v", "error")
}
<commit_msg>Debug less verbose<commit_after>package util

import (
	"errors"
	"fmt"
	"io/ioutil"

	"golang.org/x/net/context"

	log "github.com/Sirupsen/logrus"
	docker "github.com/docker/engine-api/client"
	"github.com/docker/engine-api/types"
)

const labelPrefix string = "io.conplicity"

// CheckErr checks for error, logs and optionally exits the program
func CheckErr(err error, msg string, level string) {
	if err != nil {
		switch level {
		case "debug":
			log.Debugf(msg, err)
		case "info":
			log.Infof(msg, err)
		case "warn":
			log.Warnf(msg, err)
		case "error":
			log.Errorf(msg, err)
		case "fatal":
			log.Fatalf(msg, err)
		case "panic":
			log.Panicf(msg, err)
		default:
			log.Panicf("Wrong loglevel '%v', please report this bug", level)
		}
	}
}

// GetVolumeLabel retrieves the value of given key in the io.conplicity
// namespace of the volume labels
func GetVolumeLabel(vol *types.Volume, key string) (value string, err error) {
	//log.Debugf("Getting value for label %s of volume %s", key, vol.Name)
	value, ok := vol.Labels[labelPrefix+"."+key]
	if !ok {
		errMsg := fmt.Sprintf("Key %v not found in labels for volume %v", key, vol.Name)
		err = errors.New(errMsg)
	}
	return
}

// PullImage pulls an image from the registry
func PullImage(c *docker.Client, image string) (err error) {
	if _, _, err = c.ImageInspectWithRaw(context.Background(), image, false); err != nil {
		// TODO: output pull to logs
		log.WithFields(log.Fields{
			"image": image,
		}).Info("Pulling image")
		resp, err := c.ImagePull(context.Background(), image, types.ImagePullOptions{})
		if err != nil {
			log.Errorf("ImagePull returned an error: %v", err)
			return err
		}
		defer resp.Close()
		body, err := ioutil.ReadAll(resp)
		if err != nil {
			log.Errorf("Failed to read from ImagePull response: %v", err)
			return err
		}
		log.Debugf("Pull image response body: %v", string(body))
	} else {
		log.WithFields(log.Fields{
			"image": image,
		}).Debug("Image already pulled, not pulling")
	}

	return nil
}

// RemoveContainer removes a container
func RemoveContainer(c *docker.Client, id string) {
	log.WithFields(log.Fields{
		"container": id,
	}).Infof("Removing container")
	err := c.ContainerRemove(context.Background(), id, types.ContainerRemoveOptions{
		Force:         true,
		RemoveVolumes: true,
	})
	CheckErr(err, "Failed to remove container "+id+": %v", "error")
}
<|endoftext|>


<commit_before>package brdoc

import (
	"github.com/Nhanderu/tuyo/convert"
	valid "github.com/asaskevich/govalidator"
)

func init() {
	valid.CustomTypeTagMap.Set("cpf", valid.CustomTypeValidator(func(i interface{}, o interface{}) bool {
		return validate(i, IsCPF)
	}))
	valid.CustomTypeTagMap.Set("cnpj", valid.CustomTypeValidator(func(i interface{}, o interface{}) bool {
		return validate(i, IsCNPJ)
	}))
}

func validate(doc interface{}, v func(string) bool) bool {
	switch doc.(type) {
	case string:
		return v(convert.ToString(doc))
	default:
		return false
	}
}
<commit_msg>Remove unnecessary code.<commit_after>package brdoc

import (
	"github.com/Nhanderu/tuyo/convert"
	valid "github.com/asaskevich/govalidator"
)

func init() {
	valid.CustomTypeTagMap.Set("cpf", valid.CustomTypeValidator(func(i interface{}, o interface{}) bool {
		return IsCPF(convert.ToString(i))
	}))
	valid.CustomTypeTagMap.Set("cnpj", valid.CustomTypeValidator(func(i interface{}, o interface{}) bool {
		return IsCNPJ(convert.ToString(i))
	}))
}
<|endoftext|>


<commit_before>// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package scraper

// List of predefined scraper rules (alphabetically sorted)
// domain => CSS selectors
var predefinedRules = map[string]string{
	"cbc.ca":              ".story-content",
	"developpez.com":      "div[itemprop=articleBody]",
	"francetvinfo.fr":     ".text",
	"github.com":          "article.entry-content",
	"heise.de":            "div.article-content",
	"igen.fr":             "section.corps",
	"ing.dk":              "section.body",
	"lapresse.ca":         ".amorce, .entry",
	"lemonde.fr":          "div#articleBody",
	"lepoint.fr":          ".art-text",
	"lesjoiesducode.fr":   ".blog-post-content img",
	"lesnumeriques.com":   ".text",
	"linux.com":           "div.content, div[property]",
	"medium.com":          ".section-content",
	"mac4ever.com":        "div[itemprop=articleBody]",
	"monwindows.com":      ".blog-post-body",
	"npr.org":             "#storytext",
	"oneindia.com":        ".io-article-body",
	"opensource.com":      "div[property]",
	"osnews.com":          "div.newscontent1",
	"phoronix.com":        "div.content",
	"pseudo-sciences.org": "#art_main",
	"slate.fr":            ".field-items",
	"techcrunch.com":      "div.article-entry",
	"theregister.co.uk":   "#body",
	"universfreebox.com":  "#corps_corps",
	"version2.dk":         "section.body",
	"wired.com":           "main figure, article",
	"zeit.de":             ".summary, .article-body",
	"zdnet.com":           "div.storyBody",
}
<commit_msg>Add scraper rule for darkreading.com<commit_after>// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package scraper

// List of predefined scraper rules (alphabetically sorted)
// domain => CSS selectors
var predefinedRules = map[string]string{
	"cbc.ca":              ".story-content",
	"darkreading.com":     "#article-main:not(header)",
	"developpez.com":      "div[itemprop=articleBody]",
	"francetvinfo.fr":     ".text",
	"github.com":          "article.entry-content",
	"heise.de":            "div.article-content",
	"igen.fr":             "section.corps",
	"ing.dk":              "section.body",
	"lapresse.ca":         ".amorce, .entry",
	"lemonde.fr":          "div#articleBody",
	"lepoint.fr":          ".art-text",
	"lesjoiesducode.fr":   ".blog-post-content img",
	"lesnumeriques.com":   ".text",
	"linux.com":           "div.content, div[property]",
	"medium.com":          ".section-content",
	"mac4ever.com":        "div[itemprop=articleBody]",
	"monwindows.com":      ".blog-post-body",
	"npr.org":             "#storytext",
	"oneindia.com":        ".io-article-body",
	"opensource.com":      "div[property]",
	"osnews.com":          "div.newscontent1",
	"phoronix.com":        "div.content",
	"pseudo-sciences.org": "#art_main",
	"slate.fr":            ".field-items",
	"techcrunch.com":      "div.article-entry",
	"theregister.co.uk":   "#body",
	"universfreebox.com":  "#corps_corps",
	"version2.dk":         "section.body",
	"wired.com":           "main figure, article",
	"zeit.de":             ".summary, .article-body",
	"zdnet.com":           "div.storyBody",
}
<|endoftext|>


<commit_before>package util

import (
	"bufio"
	"encoding/json"
	"fmt"
	"github.com/tdewolff/parse/buffer"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/network/nw_capture"
	"github.com/watermint/toolbox/infra/recipe/rc_exec"
	"github.com/watermint/toolbox/infra/recipe/rc_recipe"
	"github.com/watermint/toolbox/infra/util/ut_io"
	"go.uber.org/zap"
	"io"
	"os"
)

type Curl struct {
	Record string
}

func (z *Curl) Preset() {
}

func (z *Curl) Exec(c app_control.Control) error {
	l := c.Log()
	w := ut_io.NewDefaultOut(c.IsTest())
	bw := bufio.NewWriter(w)
	defer bw.Flush()
	var r io.Reader
	r = os.Stdin
	if z.Record != "" {
		r = buffer.NewReader([]byte(z.Record))
	}
	br := bufio.NewReader(r)
	for {
		line, _, err := br.ReadLine()
		switch err {
		case nil:
			rec := &nw_capture.Record{}
			if pe := json.Unmarshal(line, rec); pe != nil {
				l.Error("Unable to unmarshal", zap.Error(err), zap.String("line", string(line)))
				return err
			}
			fmt.Fprintf(bw, "curl -X POST %s\\\n", rec.Req.RequestUrl)
			for k, v := range rec.Req.RequestHeaders {
				fmt.Fprintf(bw, "     --header \"%s: %s\" \\\n", k, v)
			}
			fmt.Fprintf(bw, "     --data \"%s\"\n", rec.Req.RequestParam)
			fmt.Fprintf(bw, "\n")
			fmt.Fprintf(bw, "HTTP/2 %d\n", rec.Res.ResponseCode)
			for k, v := range rec.Res.ResponseHeaders {
				fmt.Fprintf(bw, "%s: %s\n", k, v)
			}
			fmt.Fprintf(bw, "\n")
			if rec.Res.ResponseError != "" {
				fmt.Fprintln(bw, rec.Res.ResponseError)
			} else if rec.Res.ResponseBody != "" {
				fmt.Fprintln(bw, rec.Res.ResponseBody)
			} else {
				fmt.Fprintln(bw, string(rec.Res.ResponseJson))
			}
			fmt.Fprintf(bw, "\n\n\n")

		case io.EOF:
			return nil
		default:
			return err
		}
	}
}

func (z *Curl) Test(c app_control.Control) error {
	return rc_exec.Exec(c, &Curl{}, func(r rc_recipe.Recipe) {
		m := r.(*Curl)
		m.Record = `{"time":"2020-02-27T18:34:58.677+0900","req":{"method":"POST","url":"https://api.dropboxapi.com/2/files/create_folder_v2","param":"{\"path\":\"/watermint-toolbox-test/2020-02-27T18-34-37/file-sync-up/d-e-f\",\"autorename\":false}","headers":{"Authorization":"Bearer <secret>","Content-Type":"application/json"},"content_length":92},"res":{"code":429,"headers":{"Content-Disposition":"attachment; filename='error'","Content-Security-Policy":"sandbox; frame-ancestors 'none'","Content-Type":"application/json","Date":"Thu, 27 Feb 2020 09:34:58 GMT","Retry-After":"1","Server":"nginx","X-Content-Type-Options":"nosniff","X-Dropbox-Request-Id":"0b3e28a4804b2b99a4eab9c48e07536a","X-Frame-Options":"DENY"},"json":{"error_summary":"too_many_write_operations/.","error":{"reason":{".tag":"too_many_write_operations"}}},"content_length":108},"latency":857121385}`
	})
}
<commit_msg>fix #309 : dev util curl command<commit_after>package util

import (
	"bufio"
	"encoding/json"
	"fmt"
	"github.com/tdewolff/parse/buffer"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/network/nw_capture"
	"github.com/watermint/toolbox/infra/recipe/rc_exec"
	"github.com/watermint/toolbox/infra/recipe/rc_recipe"
	"github.com/watermint/toolbox/infra/util/ut_io"
	"go.uber.org/zap"
	"io"
	"os"
	"sort"
	"strings"
)

type Curl struct {
	Record string
}

func (z *Curl) Preset() {
}

func (z *Curl) Exec(c app_control.Control) error {
	l := c.Log()
	w := ut_io.NewDefaultOut(c.IsTest())
	bw := bufio.NewWriter(w)
	defer bw.Flush()
	var r io.Reader
	r = os.Stdin
	if z.Record != "" {
		r = buffer.NewReader([]byte(z.Record))
	}
	br := bufio.NewReader(r)
	for {
		line, _, err := br.ReadLine()
		switch err {
		case nil:
			rec := &nw_capture.Record{}
			if pe := json.Unmarshal(line, rec); pe != nil {
				l.Error("Unable to unmarshal", zap.Error(err), zap.String("line", string(line)))
				return err
			}
			fmt.Fprintf(bw, "curl -D - -X POST %s \\\n", rec.Req.RequestUrl)
			reqKeys := make([]string, 0)
			for k := range rec.Req.RequestHeaders {
				reqKeys = append(reqKeys, k)
			}
			sort.Strings(reqKeys)
			for _, k := range reqKeys {
				fmt.Fprintf(bw, "     --header \"%s: %s\" \\\n", k, rec.Req.RequestHeaders[k])
			}
			fmt.Fprintf(bw, "     --data \"%s\"\n", strings.ReplaceAll(rec.Req.RequestParam, "\"", "\\\""))
			fmt.Fprintf(bw, "\n")
			fmt.Fprintf(bw, "HTTP/2 %d\n", rec.Res.ResponseCode)
			resKeys := make([]string, 0)
			for k := range rec.Res.ResponseHeaders {
				resKeys = append(resKeys, k)
			}
			sort.Strings(resKeys)
			for _, k := range resKeys {
				fmt.Fprintf(bw, "%s: %s\n", strings.ToLower(k), rec.Res.ResponseHeaders[k])
			}
			fmt.Fprintf(bw, "\n")
			if rec.Res.ResponseError != "" {
				fmt.Fprintln(bw, rec.Res.ResponseError)
			} else if rec.Res.ResponseBody != "" {
				fmt.Fprintln(bw, rec.Res.ResponseBody)
			} else {
				fmt.Fprintln(bw, string(rec.Res.ResponseJson))
			}
			fmt.Fprintf(bw, "\n\n\n")

		case io.EOF:
			return nil
		default:
			return err
		}
	}
}

func (z *Curl) Test(c app_control.Control) error {
	return rc_exec.Exec(c, &Curl{}, func(r rc_recipe.Recipe) {
		m := r.(*Curl)
		m.Record = `{"time":"2020-02-27T18:34:58.677+0900","req":{"method":"POST","url":"https://api.dropboxapi.com/2/files/create_folder_v2","param":"{\"path\":\"/watermint-toolbox-test/2020-02-27T18-34-37/file-sync-up/d-e-f\",\"autorename\":false}","headers":{"Authorization":"Bearer <secret>","Content-Type":"application/json"},"content_length":92},"res":{"code":429,"headers":{"Content-Disposition":"attachment; filename='error'","Content-Security-Policy":"sandbox; frame-ancestors 'none'","Content-Type":"application/json","Date":"Thu, 27 Feb 2020 09:34:58 GMT","Retry-After":"1","Server":"nginx","X-Content-Type-Options":"nosniff","X-Dropbox-Request-Id":"0b3e28a4804b2b99a4eab9c48e07536a","X-Frame-Options":"DENY"},"json":{"error_summary":"too_many_write_operations/.","error":{"reason":{".tag":"too_many_write_operations"}}},"content_length":108},"latency":857121385}`
	})
}
<|endoftext|>


<commit_before>// Copyright 2022 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"context"
	"testing"

	"github.com/fsouza/fake-gcs-server/fakestorage"
	"github.com/jacobsa/gcloud/gcs"
	. "github.com/jacobsa/ogletest"
)

const missingObjectName string = "test/foo"

// FakeGCSServer is not handling generation and metageneration checks for Delete flow.
// Hence, we are not writing the test for this test case.
// https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/vendor/github.com/fsouza/fake-gcs-server/fakestorage/object.go#L515

func TestBucketHandle(t *testing.T) { RunTests(t) }

type BucketHandleTest struct {
	fakeStorageServer *fakestorage.Server
	bucketHandle      *bucketHandle
}

var _ SetUpInterface = &BucketHandleTest{}
var _ TearDownInterface = &BucketHandleTest{}

func init() { RegisterTestSuite(&BucketHandleTest{}) }

func (t *BucketHandleTest) SetUp(_ *TestInfo) {
	var err error
	t.fakeStorageServer, err = CreateFakeStorageServer([]fakestorage.Object{GetTestFakeStorageObject()})
	AssertEq(nil, err)

	storageClient := &storageClient{client: t.fakeStorageServer.Client()}
	t.bucketHandle, err = storageClient.BucketHandle(TestBucketName)
	AssertEq(nil, err)
	AssertNe(nil, t.bucketHandle)
}

func (t *BucketHandleTest) TearDown() {
	t.fakeStorageServer.Stop()
}

func (t *BucketHandleTest) TestNewReaderMethodWithCompleteRead() {
	rc, err := t.bucketHandle.NewReader(context.Background(),
		&gcs.ReadObjectRequest{
			Name: TestObjectName,
			Range: &gcs.ByteRange{
				Start: uint64(0),
				Limit: uint64(len(ContentInTestObject)),
			},
		})

	AssertEq(nil, err)
	defer rc.Close()
	buf := make([]byte, len(ContentInTestObject))
	_, err = rc.Read(buf)
	AssertEq(nil, err)
	ExpectEq(string(buf[:]), ContentInTestObject)
}

func (t *BucketHandleTest) TestNewReaderMethodWithRangeRead() {
	start := uint64(2)
	limit := uint64(8)

	rc, err := t.bucketHandle.NewReader(context.Background(),
		&gcs.ReadObjectRequest{
			Name: TestObjectName,
			Range: &gcs.ByteRange{
				Start: start,
				Limit: limit,
			},
		})

	AssertEq(nil, err)
	defer rc.Close()
	buf := make([]byte, limit-start)
	_, err = rc.Read(buf)
	AssertEq(nil, err)
	ExpectEq(string(buf[:]), ContentInTestObject[start:limit])
}

func (t *BucketHandleTest) TestNewReaderMethodWithInValidObject() {
	rc, err := t.bucketHandle.NewReader(context.Background(),
		&gcs.ReadObjectRequest{
			Name: missingObjectName,
			Range: &gcs.ByteRange{
				Start: uint64(0),
				Limit: uint64(len(ContentInTestObject)),
			},
		})

	AssertNe(nil, err)
	AssertEq(nil, rc)
}

func (t *BucketHandleTest) TestNewReaderMethodWithValidGeneration() {
	rc, err := t.bucketHandle.NewReader(context.Background(),
		&gcs.ReadObjectRequest{
			Name: TestObjectName,
			Range: &gcs.ByteRange{
				Start: uint64(0),
				Limit: uint64(len(ContentInTestObject)),
			},
			Generation: TestObjectGeneration,
		})

	AssertEq(nil, err)
	defer rc.Close()
	buf := make([]byte, len(ContentInTestObject))
	_, err = rc.Read(buf)
	AssertEq(nil, err)
	ExpectEq(string(buf[:]), ContentInTestObject)
}

func (t *BucketHandleTest) TestNewReaderMethodWithInvalidGeneration() {
	rc, err := t.bucketHandle.NewReader(context.Background(),
		&gcs.ReadObjectRequest{
			Name: TestObjectName,
			Range: &gcs.ByteRange{
				Start: uint64(0),
				Limit: uint64(len(ContentInTestObject)),
			},
			Generation: 222, // other than TestObjectGeneration, doesn't exist.
		})

	AssertNe(nil, err)
	AssertEq(nil, rc)
}

func (t *BucketHandleTest) TestDeleteObjectMethodWithValidObject() {
	error := t.bucketHandle.DeleteObject(context.Background(),
		&gcs.DeleteObjectRequest{
			Name:                       TestObjectName,
			Generation:                 0,
			MetaGenerationPrecondition: nil,
		})

	AssertEq(nil, error)
}

func (t *BucketHandleTest) TestDeleteObjectMethodWithMissingObject() {
	error := t.bucketHandle.DeleteObject(context.Background(),
		&gcs.DeleteObjectRequest{
			Name:                       missingObjectName,
			Generation:                 0,
			MetaGenerationPrecondition: nil,
		})

	AssertEq("storage: object doesn't exist", error.Error())
}
<commit_msg>delete object method and unit test<commit_after>// Copyright 2022 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"context"
	"testing"

	"github.com/fsouza/fake-gcs-server/fakestorage"
	"github.com/jacobsa/gcloud/gcs"
	. "github.com/jacobsa/ogletest"
)

const missingObjectName string = "test/foo"

// FakeGCSServer is not handling generation and metageneration checks for Delete flow.
// Hence, we are not writing tests for these flows.
// https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/vendor/github.com/fsouza/fake-gcs-server/fakestorage/object.go#L515

func TestBucketHandle(t *testing.T) { RunTests(t) }

type BucketHandleTest struct {
	fakeStorageServer *fakestorage.Server
	bucketHandle      *bucketHandle
}

var _ SetUpInterface = &BucketHandleTest{}
var _ TearDownInterface = &BucketHandleTest{}

func init() { RegisterTestSuite(&BucketHandleTest{}) }

func (t *BucketHandleTest) SetUp(_ *TestInfo) {
	var err error
	t.fakeStorageServer, err = CreateFakeStorageServer([]fakestorage.Object{GetTestFakeStorageObject()})
	AssertEq(nil, err)

	storageClient := &storageClient{client: t.fakeStorageServer.Client()}
	t.bucketHandle, err = storageClient.BucketHandle(TestBucketName)
	AssertEq(nil, err)
	AssertNe(nil, t.bucketHandle)
}

func (t *BucketHandleTest) TearDown() {
	t.fakeStorageServer.Stop()
}

func (t *BucketHandleTest) TestNewReaderMethodWithCompleteRead() {
	rc, err := t.bucketHandle.NewReader(context.Background(),
		&gcs.ReadObjectRequest{
			Name: TestObjectName,
			Range: &gcs.ByteRange{
				Start: uint64(0),
				Limit: uint64(len(ContentInTestObject)),
			},
		})

	AssertEq(nil, err)
	defer rc.Close()
	buf := make([]byte, len(ContentInTestObject))
	_, err = rc.Read(buf)
	AssertEq(nil, err)
	ExpectEq(string(buf[:]), ContentInTestObject)
}

func (t *BucketHandleTest) TestNewReaderMethodWithRangeRead() {
	start := uint64(2)
	limit := uint64(8)

	rc, err := t.bucketHandle.NewReader(context.Background(),
		&gcs.ReadObjectRequest{
			Name: TestObjectName,
			Range: &gcs.ByteRange{
				Start: start,
				Limit: limit,
			},
		})

	AssertEq(nil, err)
	defer rc.Close()
	buf := make([]byte, limit-start)
	_, err = rc.Read(buf)
	AssertEq(nil, err)
	ExpectEq(string(buf[:]), ContentInTestObject[start:limit])
}

func (t *BucketHandleTest) TestNewReaderMethodWithInValidObject() {
	rc, err := t.bucketHandle.NewReader(context.Background(),
		&gcs.ReadObjectRequest{
			Name: missingObjectName,
			Range: &gcs.ByteRange{
				Start: uint64(0),
				Limit: uint64(len(ContentInTestObject)),
			},
		})

	AssertNe(nil, err)
	AssertEq(nil, rc)
}

func (t *BucketHandleTest) TestNewReaderMethodWithValidGeneration() {
	rc, err := t.bucketHandle.NewReader(context.Background(),
		&gcs.ReadObjectRequest{
			Name: TestObjectName,
			Range: &gcs.ByteRange{
				Start: uint64(0),
				Limit: uint64(len(ContentInTestObject)),
			},
			Generation: TestObjectGeneration,
		})

	AssertEq(nil, err)
	defer rc.Close()
	buf := make([]byte, len(ContentInTestObject))
	_, err = rc.Read(buf)
	AssertEq(nil, err)
	ExpectEq(string(buf[:]), ContentInTestObject)
}

func (t *BucketHandleTest) TestNewReaderMethodWithInvalidGeneration() {
	rc, err := t.bucketHandle.NewReader(context.Background(),
		&gcs.ReadObjectRequest{
			Name: TestObjectName,
			Range: &gcs.ByteRange{
				Start: uint64(0),
				Limit: uint64(len(ContentInTestObject)),
			},
			Generation: 222, // other than TestObjectGeneration, doesn't exist.
		})

	AssertNe(nil, err)
	AssertEq(nil, rc)
}

func (t *BucketHandleTest) TestDeleteObjectMethodWithValidObject() {
	error := t.bucketHandle.DeleteObject(context.Background(),
		&gcs.DeleteObjectRequest{
			Name:                       TestObjectName,
			Generation:                 0,
			MetaGenerationPrecondition: nil,
		})

	AssertEq(nil, error)
}

func (t *BucketHandleTest) TestDeleteObjectMethodWithMissingObject() {
	error := t.bucketHandle.DeleteObject(context.Background(),
		&gcs.DeleteObjectRequest{
			Name:                       missingObjectName,
			Generation:                 0,
			MetaGenerationPrecondition: nil,
		})

	AssertEq("storage: object doesn't exist", error.Error())
}
<|endoftext|>


<commit_before>package worker

import (
	"bytes"
	"testing"

	gocontext "golang.org/x/net/context"

	"github.com/mitchellh/multistep"
	"github.com/stretchr/testify/assert"
	"github.com/travis-ci/worker/backend"
	"github.com/travis-ci/worker/config"
)

func setupStepWriteWorkerInfo() (*stepWriteWorkerInfo, *bytes.Buffer, multistep.StateBag) {
	s := &stepWriteWorkerInfo{}

	bp, _ := backend.NewBackendProvider("fake",
		config.ProviderConfigFromMap(map[string]string{
			"STARTUP_DURATION": "42.17s",
		}))

	ctx := gocontext.TODO()
	instance, _ := bp.Start(ctx, nil)

	logWriter := bytes.NewBufferString("")

	state := &multistep.BasicStateBag{}
	state.Put("logWriter", logWriter)
	state.Put("instance", instance)
	state.Put("hostname", "frizzlefry.example.local")

	return s, logWriter, state
}

func TestStepWriteWorkerInfo_Run(t *testing.T) {
	s, out, state := setupStepWriteWorkerInfo()

	s.Run(state)

	assert.Contains(t, out, "travis_fold:start:worker_info\r\033[0K")
	assert.Contains(t, out, "\033[33;1mWorker information\033[0m\n")
	assert.Contains(t, out, "\nhostname: frizzlefry.example.local\n")
	assert.Contains(t, out, "\nversion: "+VersionString+" "+RevisionURLString+"\n")
	assert.Contains(t, out, "\ninstance: fake\n")
	assert.Contains(t, out, "\nstartup: 42.17s\n")
	assert.Contains(t, out, "\ntravis_fold:end:worker_info\r\033[0K")
}
<commit_msg>step_write_worker_info: implement LogWriter in test case<commit_after>package worker

import (
	"bytes"
	"testing"
	"time"

	gocontext "golang.org/x/net/context"

	"github.com/mitchellh/multistep"
	"github.com/stretchr/testify/assert"
	"github.com/travis-ci/worker/backend"
	"github.com/travis-ci/worker/config"
)

type byteBufferLogWriter struct {
	*bytes.Buffer
}

func (w *byteBufferLogWriter) Close() error {
	return nil
}

func (w *byteBufferLogWriter) WriteAndClose(p []byte) (int, error) {
	return w.Write(p)
}

func (w *byteBufferLogWriter) SetTimeout(time.Duration) {
}

func (w *byteBufferLogWriter) Timeout() <-chan time.Time {
	return make(<-chan time.Time)
}

func (w *byteBufferLogWriter) SetMaxLogLength(m int) {
}

func setupStepWriteWorkerInfo() (*stepWriteWorkerInfo, *byteBufferLogWriter, multistep.StateBag) {
	s := &stepWriteWorkerInfo{}

	bp, _ := backend.NewBackendProvider("fake",
		config.ProviderConfigFromMap(map[string]string{
			"STARTUP_DURATION": "42.17s",
		}))

	ctx := gocontext.TODO()
	instance, _ := bp.Start(ctx, nil)

	logWriter := &byteBufferLogWriter{
		bytes.NewBufferString(""),
	}

	state := &multistep.BasicStateBag{}
	state.Put("logWriter", logWriter)
	state.Put("instance", instance)
	state.Put("hostname", "frizzlefry.example.local")

	return s, logWriter, state
}

func TestStepWriteWorkerInfo_Run(t *testing.T) {
	s, logWriter, state := setupStepWriteWorkerInfo()

	s.Run(state)

	out := logWriter.String()
	assert.Contains(t, out, "travis_fold:start:worker_info\r\033[0K")
	assert.Contains(t, out, "\033[33;1mWorker information\033[0m\n")
	assert.Contains(t, out, "\nhostname: frizzlefry.example.local\n")
	assert.Contains(t, out, "\nversion: "+VersionString+" "+RevisionURLString+"\n")
	assert.Contains(t, out, "\ninstance: fake\n")
	assert.Contains(t, out, "\nstartup: 42.17s\n")
	assert.Contains(t, out, "\ntravis_fold:end:worker_info\r\033[0K")
}
<|endoftext|>