<commit_before>package logger

import (
	"fmt"
	"github.com/Sirupsen/logrus"
	"io"
	"io/ioutil"
	"strings"
)

var formatter = &textFormatter{
	DisableTimestamp: false,
	FullTimestamp:    true,
}

func init() {
	logrus.SetFormatter(formatter)
	logrus.SetLevel(logrus.DebugLevel)
}

// Discard configures to logger to discard all loogs.
func Discard() {
	logrus.SetOutput(ioutil.Discard)
}

// Logger is repsonsible for logging messages from code.
type Logger interface {
	Debug(string, ...interface{})
	Info(string, ...interface{})
	Error(string, ...interface{})
	WithFields(...interface{}) Logger
}

// SetLevel sets the level of logging
func SetLevel(l string) {
	switch strings.ToLower(l) {
	case "debug":
		logrus.SetLevel(logrus.DebugLevel)
	case "info":
		logrus.SetLevel(logrus.InfoLevel)
	case "warn":
		logrus.SetLevel(logrus.WarnLevel)
	case "error":
		logrus.SetLevel(logrus.ErrorLevel)
	default:
		logrus.SetLevel(logrus.InfoLevel)
	}
}

// DisableTimestamp prevents timestamps from being displayed in the logs
func DisableTimestamp(b bool) {
	formatter := &textFormatter{
		DisableTimestamp: b,
		FullTimestamp:    true,
	}
	logrus.SetFormatter(formatter)
}

// ForceColors forces the log output formatter to use color. Useful during testing.
func ForceColors() {
	formatter.ForceColors = true
}

// New returns a new Logger instance.
func New(ns string, args ...interface{}) Logger {
	f := fields(args...)
	f["ns"] = ns
	l := logrus.WithFields(f)
	return &logger{l}
}

type logger struct {
	log *logrus.Entry
}

// Debug logs a debug message.
//
// After the first argument, arguments are key-value pairs which are written as structured logs.
//     log.Debug("Some message here", "key1", value1, "key2", value2)
func (l *logger) Debug(msg string, args ...interface{}) {
	defer recoverLogErr()
	f := fields(args...)
	l.log.WithFields(f).Debug(msg)
}

// Info logs an info message
//
// After the first argument, arguments are key-value pairs which are written as structured logs.
//     log.Info("Some message here", "key1", value1, "key2", value2)
func (l *logger) Info(msg string, args ...interface{}) {
	defer recoverLogErr()
	f := fields(args...)
	l.log.WithFields(f).Info(msg)
}

// Error logs an error message
//
// After the first argument, arguments are key-value pairs which are written as structured logs.
//     log.Error("Some message here", "key1", value1, "key2", value2)
//
// Error has a two-argument version that can be used as a shortcut.
//     err := startServer()
//     log.Error("Couldn't start server", err)
func (l *logger) Error(msg string, args ...interface{}) {
	defer recoverLogErr()
	var f map[string]interface{}
	if len(args) == 1 {
		f = fields("error", args[0])
	} else {
		f = fields(args...)
	}
	l.log.WithFields(f).Error(msg)
}

// WithFields returns a new Logger instance with the given fields added to all log messages.
func (l *logger) WithFields(args ...interface{}) Logger {
	defer recoverLogErr()
	f := fields(args...)
	n := l.log.WithFields(f)
	return &logger{n}
}

// SetOutput sets the output for all loggers.
func SetOutput(w io.Writer) {
	logrus.SetOutput(w)
}

var rootLogger = New("funnel")

// Debug logs to the global logger at the Debug level
func Debug(msg string, args ...interface{}) {
	rootLogger.Debug(msg, args...)
}

// Info logs to the global logger at the Info level
func Info(msg string, args ...interface{}) {
	rootLogger.Info(msg, args...)
}

// Error logs to the global logger at the Error level
func Error(msg string, args ...interface{}) {
	rootLogger.Error(msg, args...)
}

// recoverLogErr is used to recover from any panics during logging.
// Panics aren't expected of course, but logging should never crash
// a program, so this failsafe tries to prevent those crashes.
func recoverLogErr() {
	if r := recover(); r != nil {
		fmt.Println("Recovered from logging panic", r)
	}
}

// PrintSimpleError prints out an error message with a red "ERROR:" prefix.
func PrintSimpleError(err error) {
	fmt.Printf("\x1b[%dm%s\x1b[0m %s\n", red, "ERROR:", err.Error())
}

func fields(args ...interface{}) map[string]interface{} {
	f := make(map[string]interface{}, len(args)/2)
	if len(args) == 1 {
		f["unknown"] = args[0]
		return f
	}
	for i := 0; i < len(args); i += 2 {
		k := args[i].(string)
		v := args[i+1]
		f[k] = v
	}
	if len(args)%2 != 0 {
		f["unknown"] = args[len(args)-1]
	}
	return f
}
<commit_msg>Fix logging bug. Don't reset formatter.<commit_after>package logger

import (
	"fmt"
	"github.com/Sirupsen/logrus"
	"io"
	"io/ioutil"
	"strings"
)

var formatter = &textFormatter{
	DisableTimestamp: false,
	FullTimestamp:    true,
}

func init() {
	logrus.SetFormatter(formatter)
	logrus.SetLevel(logrus.DebugLevel)
}

// Discard configures to logger to discard all loogs.
func Discard() {
	logrus.SetOutput(ioutil.Discard)
}

// Logger is repsonsible for logging messages from code.
type Logger interface {
	Debug(string, ...interface{})
	Info(string, ...interface{})
	Error(string, ...interface{})
	WithFields(...interface{}) Logger
}

// SetLevel sets the level of logging
func SetLevel(l string) {
	switch strings.ToLower(l) {
	case "debug":
		logrus.SetLevel(logrus.DebugLevel)
	case "info":
		logrus.SetLevel(logrus.InfoLevel)
	case "warn":
		logrus.SetLevel(logrus.WarnLevel)
	case "error":
		logrus.SetLevel(logrus.ErrorLevel)
	default:
		logrus.SetLevel(logrus.InfoLevel)
	}
}

// DisableTimestamp prevents timestamps from being displayed in the logs
func DisableTimestamp(b bool) {
	formatter.DisableTimestamp = b
}

// ForceColors forces the log output formatter to use color. Useful during testing.
func ForceColors() {
	formatter.ForceColors = true
}

// New returns a new Logger instance.
func New(ns string, args ...interface{}) Logger {
	f := fields(args...)
	f["ns"] = ns
	l := logrus.WithFields(f)
	return &logger{l}
}

type logger struct {
	log *logrus.Entry
}

// Debug logs a debug message.
//
// After the first argument, arguments are key-value pairs which are written as structured logs.
//     log.Debug("Some message here", "key1", value1, "key2", value2)
func (l *logger) Debug(msg string, args ...interface{}) {
	defer recoverLogErr()
	f := fields(args...)
	l.log.WithFields(f).Debug(msg)
}

// Info logs an info message
//
// After the first argument, arguments are key-value pairs which are written as structured logs.
//     log.Info("Some message here", "key1", value1, "key2", value2)
func (l *logger) Info(msg string, args ...interface{}) {
	defer recoverLogErr()
	f := fields(args...)
	l.log.WithFields(f).Info(msg)
}

// Error logs an error message
//
// After the first argument, arguments are key-value pairs which are written as structured logs.
//     log.Error("Some message here", "key1", value1, "key2", value2)
//
// Error has a two-argument version that can be used as a shortcut.
//     err := startServer()
//     log.Error("Couldn't start server", err)
func (l *logger) Error(msg string, args ...interface{}) {
	defer recoverLogErr()
	var f map[string]interface{}
	if len(args) == 1 {
		f = fields("error", args[0])
	} else {
		f = fields(args...)
	}
	l.log.WithFields(f).Error(msg)
}

// WithFields returns a new Logger instance with the given fields added to all log messages.
func (l *logger) WithFields(args ...interface{}) Logger {
	defer recoverLogErr()
	f := fields(args...)
	n := l.log.WithFields(f)
	return &logger{n}
}

// SetOutput sets the output for all loggers.
func SetOutput(w io.Writer) {
	logrus.SetOutput(w)
}

var rootLogger = New("funnel")

// Debug logs to the global logger at the Debug level
func Debug(msg string, args ...interface{}) {
	rootLogger.Debug(msg, args...)
}

// Info logs to the global logger at the Info level
func Info(msg string, args ...interface{}) {
	rootLogger.Info(msg, args...)
}

// Error logs to the global logger at the Error level
func Error(msg string, args ...interface{}) {
	rootLogger.Error(msg, args...)
}

// recoverLogErr is used to recover from any panics during logging.
// Panics aren't expected of course, but logging should never crash
// a program, so this failsafe tries to prevent those crashes.
func recoverLogErr() {
	if r := recover(); r != nil {
		fmt.Println("Recovered from logging panic", r)
	}
}

// PrintSimpleError prints out an error message with a red "ERROR:" prefix.
func PrintSimpleError(err error) {
	fmt.Printf("\x1b[%dm%s\x1b[0m %s\n", red, "ERROR:", err.Error())
}

func fields(args ...interface{}) map[string]interface{} {
	f := make(map[string]interface{}, len(args)/2)
	if len(args) == 1 {
		f["unknown"] = args[0]
		return f
	}
	for i := 0; i < len(args); i += 2 {
		k := args[i].(string)
		v := args[i+1]
		f[k] = v
	}
	if len(args)%2 != 0 {
		f["unknown"] = args[len(args)-1]
	}
	return f
}
<|endoftext|>


<commit_before>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package discovery

import (
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/defaults"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/model"
	"golang.org/x/net/context"

	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/prometheus/prometheus/config"
	"github.com/prometheus/prometheus/util/strutil"
)

const (
	ec2Label           = model.MetaLabelPrefix + "ec2_"
	ec2LabelAZ         = ec2Label + "availability_zone"
	ec2LabelInstanceID = ec2Label + "instance_id"
	ec2LabelPublicDNS  = ec2Label + "public_dns_name"
	ec2LabelPublicIP   = ec2Label + "public_ip"
	ec2LabelPrivateIP  = ec2Label + "private_ip"
	ec2LabelSubnetID   = ec2Label + "subnet_id"
	ec2LabelTag        = ec2Label + "tag_"
	ec2LabelVPCID      = ec2Label + "vpc_id"
	subnetSeparator    = ","
)

// EC2Discovery periodically performs EC2-SD requests. It implements
// the TargetProvider interface.
type EC2Discovery struct {
	aws      *aws.Config
	interval time.Duration
	port     int
}

// NewEC2Discovery returns a new EC2Discovery which periodically refreshes its targets.
func NewEC2Discovery(conf *config.EC2SDConfig) *EC2Discovery {
	creds := credentials.NewStaticCredentials(conf.AccessKey, conf.SecretKey, "")
	if conf.AccessKey == "" && conf.SecretKey == "" {
		creds = defaults.DefaultChainCredentials
	}
	return &EC2Discovery{
		aws: &aws.Config{
			Region:      &conf.Region,
			Credentials: creds,
		},
		interval: time.Duration(conf.RefreshInterval),
		port:     conf.Port,
	}
}

// Run implements the TargetProvider interface.
func (ed *EC2Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) {
	defer close(ch)

	ticker := time.NewTicker(ed.interval)
	defer ticker.Stop()

	// Get an initial set right away.
	tg, err := ed.refresh()
	if err != nil {
		log.Error(err)
	} else {
		ch <- []*config.TargetGroup{tg}
	}

	for {
		select {
		case <-ticker.C:
			tg, err := ed.refresh()
			if err != nil {
				log.Error(err)
			} else {
				ch <- []*config.TargetGroup{tg}
			}
		case <-ctx.Done():
			return
		}
	}
}

func (ed *EC2Discovery) refresh() (*config.TargetGroup, error) {
	ec2s := ec2.New(ed.aws)
	tg := &config.TargetGroup{
		Source: *ed.aws.Region,
	}
	if err := ec2s.DescribeInstancesPages(nil, func(p *ec2.DescribeInstancesOutput, lastPage bool) bool {
		for _, r := range p.Reservations {
			for _, inst := range r.Instances {
				if inst.PrivateIpAddress == nil {
					continue
				}
				labels := model.LabelSet{
					ec2LabelInstanceID: model.LabelValue(*inst.InstanceId),
				}
				labels[ec2LabelPrivateIP] = model.LabelValue(*inst.PrivateIpAddress)
				addr := net.JoinHostPort(*inst.PrivateIpAddress, fmt.Sprintf("%d", ed.port))
				labels[model.AddressLabel] = model.LabelValue(addr)

				if inst.PublicIpAddress != nil {
					labels[ec2LabelPublicIP] = model.LabelValue(*inst.PublicIpAddress)
					labels[ec2LabelPublicDNS] = model.LabelValue(*inst.PublicDnsName)
				}

				labels[ec2LabelAZ] = model.LabelValue(*inst.Placement.AvailabilityZone)

				if inst.VpcId != nil {
					labels[ec2LabelVPCID] = model.LabelValue(*inst.VpcId)

					subnetsMap := make(map[string]struct{})
					for _, eni := range inst.NetworkInterfaces {
						subnetsMap[*eni.SubnetId] = struct{}{}
					}
					subnets := []string{}
					for k := range subnetsMap {
						subnets = append(subnets, k)
					}
					labels[ec2LabelSubnetID] = model.LabelValue(
						subnetSeparator +
							strings.Join(subnets, subnetSeparator) +
							subnetSeparator)
				}

				for _, t := range inst.Tags {
					name := strutil.SanitizeLabelName(*t.Key)
					labels[ec2LabelTag+model.LabelName(name)] = model.LabelValue(*t.Value)
				}
				tg.Targets = append(tg.Targets, labels)
			}
		}
		return true
	}); err != nil {
		return nil, fmt.Errorf("could not describe instances: %s", err)
	}
	return tg, nil
}
<commit_msg>added http_proxy support for ec2 sd<commit_after>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package discovery

import (
	"fmt"
	"net"
	"strings"
	"time"
  "net/http"
	"net/url"
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/defaults"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/model"
	"golang.org/x/net/context"

	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/prometheus/prometheus/config"
	"github.com/prometheus/prometheus/util/strutil"
)

const (
	ec2Label           = model.MetaLabelPrefix + "ec2_"
	ec2LabelAZ         = ec2Label + "availability_zone"
	ec2LabelInstanceID = ec2Label + "instance_id"
	ec2LabelPublicDNS  = ec2Label + "public_dns_name"
	ec2LabelPublicIP   = ec2Label + "public_ip"
	ec2LabelPrivateIP  = ec2Label + "private_ip"
	ec2LabelSubnetID   = ec2Label + "subnet_id"
	ec2LabelTag        = ec2Label + "tag_"
	ec2LabelVPCID      = ec2Label + "vpc_id"
	subnetSeparator    = ","
)

// EC2Discovery periodically performs EC2-SD requests. It implements
// the TargetProvider interface.
type EC2Discovery struct {
	aws      *aws.Config
	interval time.Duration
	port     int
}

// NewEC2Discovery returns a new EC2Discovery which periodically refreshes its targets.
func NewEC2Discovery(conf *config.EC2SDConfig) *EC2Discovery {
	creds := credentials.NewStaticCredentials(conf.AccessKey, conf.SecretKey, "")
	if conf.AccessKey == "" && conf.SecretKey == "" {
		creds = defaults.DefaultChainCredentials
	}

  var awsConfig = &aws.Config{
			Region:      &conf.Region,
			Credentials: creds,
	}

  // Adds HTTP proxy to aws config
  if os.Getenv("HTTP_PROXY") != "" {
    proxyUrl, err := url.Parse(os.Getenv("HTTP_PROXY")); if err != nil {
      log.Error(err)
    }
    client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}}
    awsConfig.HTTPClient = client
  }

	return &EC2Discovery{
		aws: awsConfig,
		interval: time.Duration(conf.RefreshInterval),
		port:     conf.Port,
	}
}

// Run implements the TargetProvider interface.
func (ed *EC2Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) {
	defer close(ch)

	ticker := time.NewTicker(ed.interval)
	defer ticker.Stop()

	// Get an initial set right away.
	tg, err := ed.refresh()
	if err != nil {
		log.Error(err)
	} else {
		ch <- []*config.TargetGroup{tg}
	}

	for {
		select {
		case <-ticker.C:
			tg, err := ed.refresh()
			if err != nil {
				log.Error(err)
			} else {
				ch <- []*config.TargetGroup{tg}
			}
		case <-ctx.Done():
			return
		}
	}
}

func (ed *EC2Discovery) refresh() (*config.TargetGroup, error) {
	ec2s := ec2.New(ed.aws)
	tg := &config.TargetGroup{
		Source: *ed.aws.Region,
	}
	if err := ec2s.DescribeInstancesPages(nil, func(p *ec2.DescribeInstancesOutput, lastPage bool) bool {
		for _, r := range p.Reservations {
			for _, inst := range r.Instances {
				if inst.PrivateIpAddress == nil {
					continue
				}
				labels := model.LabelSet{
					ec2LabelInstanceID: model.LabelValue(*inst.InstanceId),
				}
				labels[ec2LabelPrivateIP] = model.LabelValue(*inst.PrivateIpAddress)
				addr := net.JoinHostPort(*inst.PrivateIpAddress, fmt.Sprintf("%d", ed.port))
				labels[model.AddressLabel] = model.LabelValue(addr)

				if inst.PublicIpAddress != nil {
					labels[ec2LabelPublicIP] = model.LabelValue(*inst.PublicIpAddress)
					labels[ec2LabelPublicDNS] = model.LabelValue(*inst.PublicDnsName)
				}

				labels[ec2LabelAZ] = model.LabelValue(*inst.Placement.AvailabilityZone)

				if inst.VpcId != nil {
					labels[ec2LabelVPCID] = model.LabelValue(*inst.VpcId)

					subnetsMap := make(map[string]struct{})
					for _, eni := range inst.NetworkInterfaces {
						subnetsMap[*eni.SubnetId] = struct{}{}
					}
					subnets := []string{}
					for k := range subnetsMap {
						subnets = append(subnets, k)
					}
					labels[ec2LabelSubnetID] = model.LabelValue(
						subnetSeparator +
							strings.Join(subnets, subnetSeparator) +
							subnetSeparator)
				}

				for _, t := range inst.Tags {
					name := strutil.SanitizeLabelName(*t.Key)
					labels[ec2LabelTag+model.LabelName(name)] = model.LabelValue(*t.Value)
				}
				tg.Targets = append(tg.Targets, labels)
			}
		}
		return true
	}); err != nil {
		return nil, fmt.Errorf("could not describe instances: %s", err)
	}
	return tg, nil
}
<|endoftext|>


<commit_before>package momos

import (
	"net/http"
	"net/http/httputil"
	"net/url"

	"github.com/hemerajs/momos/logger"
	"github.com/lox/httpcache"
	"github.com/lox/httpcache/httplog"
)

var ServerLogging = false
var Log = logger.NewStdLogger(true, true, true, true, false)

type Proxy struct {
	ReverseProxy *httputil.ReverseProxy
	ProxyURL     string
	Handler      *httpcache.Handler
	Cache        httpcache.Cache
}

// PreCacheResponseHandler is an http handler to log informations about the cache status
// https://github.com/lox/httpcache
func PreCacheResponseHandler(h http.Handler) http.HandlerFunc {
	return func(res http.ResponseWriter, req *http.Request) {
		url := req.Host + req.URL.String()
		cacheHeader := res.Header().Get("X-Cache")
		Log.Noticef("PreResponse url: %v, cache: %v", url, cacheHeader)
		h.ServeHTTP(res, req)
	}
}

func New(targetUrl string) *Proxy {
	target, tErr := url.Parse(targetUrl)

	if tErr != nil {
		Log.Errorf("Invalid url: %v", targetUrl)
		panic(tErr)
	}

	httpcache.DebugLogging = ServerLogging

	p := &Proxy{}
	p.ReverseProxy = httputil.NewSingleHostReverseProxy(target)
	p.ReverseProxy.Transport = &proxyTransport{http.DefaultTransport}

	p.Cache = httpcache.NewMemoryCache()
	p.Handler = httpcache.NewHandler(p.Cache, PreCacheResponseHandler(p.ReverseProxy))
	p.Handler.Shared = true

	respLogger := httplog.NewResponseLogger(p.Handler)
	respLogger.DumpRequests = true
	respLogger.DumpResponses = true
	respLogger.DumpErrors = true

	return p
}
<commit_msg>try fix race<commit_after>package momos

import (
	"net/http"
	"net/http/httputil"
	"net/url"

	"github.com/hemerajs/momos/logger"
	"github.com/lox/httpcache"
	"github.com/lox/httpcache/httplog"
)

var ServerLogging = false
var Log = logger.NewStdLogger(true, true, true, true, false)

type Proxy struct {
	ReverseProxy *httputil.ReverseProxy
	ProxyURL     string
	Handler      *httpcache.Handler
	Cache        httpcache.Cache
}

// PreCacheResponseHandler is an http handler to log informations about the cache status
// https://github.com/lox/httpcache
func PreCacheResponseHandler(h http.Handler) http.HandlerFunc {
	return func(res http.ResponseWriter, req *http.Request) {
		url := req.Host + req.URL.String()
		cacheHeader := res.Header().Get("X-Cache")
		Log.Noticef("PreResponse url: %v, cache: %v", url, cacheHeader)
		h.ServeHTTP(res, req)
	}
}

func New(targetUrl string) *Proxy {
	target, tErr := url.Parse(targetUrl)

	if tErr != nil {
		Log.Errorf("Invalid url: %v", targetUrl)
		panic(tErr)
	}

	httpcache.DebugLogging = ServerLogging

	p := &Proxy{}
	p.ReverseProxy = httputil.NewSingleHostReverseProxy(target)
	p.ReverseProxy.Transport = &proxyTransport{http.DefaultTransport}

	p.Cache = httpcache.NewMemoryCache()
	p.Handler = httpcache.NewHandler(p.Cache, p.ReverseProxy)
	p.Handler.Shared = true

	respLogger := httplog.NewResponseLogger(p.Handler)
	respLogger.DumpRequests = true
	respLogger.DumpResponses = true
	respLogger.DumpErrors = true

	return p
}
<|endoftext|>


<commit_before>package accounting

import (
	"bytes"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/rc"
)

var (
	// Stats is global statistics counter
	Stats = NewStats()
)

func init() {
	// Set the function pointer up in fs
	fs.CountError = Stats.Error

	rc.Add(rc.Call{
		Path:  "core/stats",
		Fn:    Stats.RemoteStats,
		Title: "Returns stats about current transfers.",
		Help: `
This returns all available stats

	rclone rc core/stats

Returns the following values:

` + "```" + `
{
	"speed": average speed in bytes/sec since start of the process,
	"bytes": total transferred bytes since the start of the process,
	"errors": number of errors,
	"checks": number of checked files,
	"transfers": number of transferred files,
	"deletes" : number of deleted files,
	"elapsedTime": time in seconds since the start of the process,
	"lastError": last occurred error,
	"transferring": an array of currently active file transfers:
		[
			{
				"bytes": total transferred bytes for this file,
				"eta": estimated time in seconds until file transfer completion
				"name": name of the file,
				"percentage": progress of the file transfer in percent,
				"speed": speed in bytes/sec,
				"speedAvg": speed in bytes/sec as an exponentially weighted moving average,
				"size": size of the file in bytes
			}
		],
	"checking": an array of names of currently active file checks
		[]
}
` + "```" + `
Values for "transferring", "checking" and "lastError" are only assigned if data is available.
The value for "eta" is null if an eta cannot be determined.
`,
	})
}

// StatsInfo accounts all transfers
type StatsInfo struct {
	mu                sync.RWMutex
	bytes             int64
	errors            int64
	lastError         error
	checks            int64
	checking          *stringSet
	checkQueue        int
	checkQueueSize    int64
	transfers         int64
	transferring      *stringSet
	transferQueue     int
	transferQueueSize int64
	renameQueue       int
	renameQueueSize   int64
	deletes           int64
	start             time.Time
	inProgress        *inProgress
}

// NewStats cretates an initialised StatsInfo
func NewStats() *StatsInfo {
	return &StatsInfo{
		checking:     newStringSet(fs.Config.Checkers),
		transferring: newStringSet(fs.Config.Transfers),
		start:        time.Now(),
		inProgress:   newInProgress(),
	}
}

// RemoteStats returns stats for rc
func (s *StatsInfo) RemoteStats(in rc.Params) (out rc.Params, err error) {
	out = make(rc.Params)
	s.mu.RLock()
	dt := time.Now().Sub(s.start)
	dtSeconds := dt.Seconds()
	speed := 0.0
	if dt > 0 {
		speed = float64(s.bytes) / dtSeconds
	}
	out["speed"] = speed
	out["bytes"] = s.bytes
	out["errors"] = s.errors
	out["checks"] = s.checks
	out["transfers"] = s.transfers
	out["deletes"] = s.deletes
	out["elapsedTime"] = dtSeconds
	s.mu.RUnlock()
	if !s.checking.empty() {
		var c []string
		s.checking.mu.RLock()
		defer s.checking.mu.RUnlock()
		for name := range s.checking.items {
			c = append(c, name)
		}
		out["checking"] = c
	}
	if !s.transferring.empty() {
		var t []interface{}
		s.transferring.mu.RLock()
		defer s.transferring.mu.RUnlock()
		for name := range s.transferring.items {
			if acc := s.inProgress.get(name); acc != nil {
				t = append(t, acc.RemoteStats())
			} else {
				t = append(t, name)
			}
		}
		out["transferring"] = t
	}
	if s.errors > 0 {
		out["lastError"] = s.lastError
	}
	return out, nil
}

// String convert the StatsInfo to a string for printing
func (s *StatsInfo) String() string {
	s.mu.RLock()

	dt := time.Now().Sub(s.start)
	dtSeconds := dt.Seconds()
	speed := 0.0
	if dt > 0 {
		speed = float64(s.bytes) / dtSeconds
	}
	dtRounded := dt - (dt % (time.Second / 10))
	buf := &bytes.Buffer{}

	if fs.Config.DataRateUnit == "bits" {
		speed = speed * 8
	}

	_, _ = fmt.Fprintf(buf, `
Transferred:   %10s (%s)
Errors:        %10d
Checks:        %10d
Transferred:   %10d
Elapsed time:  %10v
`,
		fs.SizeSuffix(s.bytes).Unit("Bytes"), fs.SizeSuffix(speed).Unit(strings.Title(fs.Config.DataRateUnit)+"/s"),
		s.errors,
		s.checks,
		s.transfers,
		dtRounded)

	// checking and transferring have their own locking so unlock
	// here to prevent deadlock on GetBytes
	s.mu.RUnlock()

	if !s.checking.empty() {
		_, _ = fmt.Fprintf(buf, "Checking:\n%s\n", s.checking)
	}
	if !s.transferring.empty() {
		_, _ = fmt.Fprintf(buf, "Transferring:\n%s\n", s.transferring)
	}
	return buf.String()
}

// Log outputs the StatsInfo to the log
func (s *StatsInfo) Log() {
	fs.LogLevelPrintf(fs.Config.StatsLogLevel, nil, "%v\n", s)
}

// Bytes updates the stats for bytes bytes
func (s *StatsInfo) Bytes(bytes int64) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.bytes += bytes
}

// GetBytes returns the number of bytes transferred so far
func (s *StatsInfo) GetBytes() int64 {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.bytes
}

// Errors updates the stats for errors
func (s *StatsInfo) Errors(errors int64) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.errors += errors
}

// GetErrors reads the number of errors
func (s *StatsInfo) GetErrors() int64 {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.errors
}

// GetLastError returns the lastError
func (s *StatsInfo) GetLastError() error {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.lastError
}

// Deletes updates the stats for deletes
func (s *StatsInfo) Deletes(deletes int64) int64 {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.deletes += deletes
	return s.deletes
}

// ResetCounters sets the counters (bytes, checks, errors, transfers) to 0
func (s *StatsInfo) ResetCounters() {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.bytes = 0
	s.errors = 0
	s.checks = 0
	s.transfers = 0
	s.deletes = 0
}

// ResetErrors sets the errors count to 0
func (s *StatsInfo) ResetErrors() {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.errors = 0
}

// Errored returns whether there have been any errors
func (s *StatsInfo) Errored() bool {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.errors != 0
}

// Error adds a single error into the stats and assigns lastError
func (s *StatsInfo) Error(err error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.errors++
	s.lastError = err
}

// Checking adds a check into the stats
func (s *StatsInfo) Checking(remote string) {
	s.checking.add(remote)
}

// DoneChecking removes a check from the stats
func (s *StatsInfo) DoneChecking(remote string) {
	s.checking.del(remote)
	s.mu.Lock()
	s.checks++
	s.mu.Unlock()
}

// GetTransfers reads the number of transfers
func (s *StatsInfo) GetTransfers() int64 {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.transfers
}

// Transferring adds a transfer into the stats
func (s *StatsInfo) Transferring(remote string) {
	s.transferring.add(remote)
}

// DoneTransferring removes a transfer from the stats
//
// if ok is true then it increments the transfers count
func (s *StatsInfo) DoneTransferring(remote string, ok bool) {
	s.transferring.del(remote)
	if ok {
		s.mu.Lock()
		s.transfers++
		s.mu.Unlock()
	}
}

// SetCheckQueue sets the number of queued checks
func (s *StatsInfo) SetCheckQueue(n int, size int64) {
	s.mu.Lock()
	s.checkQueue = n
	s.checkQueueSize = size
	s.mu.Unlock()
}

// SetTransferQueue sets the number of queued transfers
func (s *StatsInfo) SetTransferQueue(n int, size int64) {
	s.mu.Lock()
	s.transferQueue = n
	s.transferQueueSize = size
	s.mu.Unlock()
}

// SetRenameQueue sets the number of queued transfers
func (s *StatsInfo) SetRenameQueue(n int, size int64) {
	s.mu.Lock()
	s.renameQueue = n
	s.renameQueueSize = size
	s.mu.Unlock()
}
<commit_msg>fs/accounting: show the total progress of the sync in the stats #379<commit_after>package accounting

import (
	"bytes"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/rc"
)

var (
	// Stats is global statistics counter
	Stats = NewStats()
)

func init() {
	// Set the function pointer up in fs
	fs.CountError = Stats.Error

	rc.Add(rc.Call{
		Path:  "core/stats",
		Fn:    Stats.RemoteStats,
		Title: "Returns stats about current transfers.",
		Help: `
This returns all available stats

	rclone rc core/stats

Returns the following values:

` + "```" + `
{
	"speed": average speed in bytes/sec since start of the process,
	"bytes": total transferred bytes since the start of the process,
	"errors": number of errors,
	"checks": number of checked files,
	"transfers": number of transferred files,
	"deletes" : number of deleted files,
	"elapsedTime": time in seconds since the start of the process,
	"lastError": last occurred error,
	"transferring": an array of currently active file transfers:
		[
			{
				"bytes": total transferred bytes for this file,
				"eta": estimated time in seconds until file transfer completion
				"name": name of the file,
				"percentage": progress of the file transfer in percent,
				"speed": speed in bytes/sec,
				"speedAvg": speed in bytes/sec as an exponentially weighted moving average,
				"size": size of the file in bytes
			}
		],
	"checking": an array of names of currently active file checks
		[]
}
` + "```" + `
Values for "transferring", "checking" and "lastError" are only assigned if data is available.
The value for "eta" is null if an eta cannot be determined.
`,
	})
}

// StatsInfo accounts all transfers
type StatsInfo struct {
	mu                sync.RWMutex
	bytes             int64
	errors            int64
	lastError         error
	checks            int64
	checking          *stringSet
	checkQueue        int
	checkQueueSize    int64
	transfers         int64
	transferring      *stringSet
	transferQueue     int
	transferQueueSize int64
	renameQueue       int
	renameQueueSize   int64
	deletes           int64
	start             time.Time
	inProgress        *inProgress
}

// NewStats cretates an initialised StatsInfo
func NewStats() *StatsInfo {
	return &StatsInfo{
		checking:     newStringSet(fs.Config.Checkers),
		transferring: newStringSet(fs.Config.Transfers),
		start:        time.Now(),
		inProgress:   newInProgress(),
	}
}

// RemoteStats returns stats for rc
func (s *StatsInfo) RemoteStats(in rc.Params) (out rc.Params, err error) {
	out = make(rc.Params)
	s.mu.RLock()
	dt := time.Now().Sub(s.start)
	dtSeconds := dt.Seconds()
	speed := 0.0
	if dt > 0 {
		speed = float64(s.bytes) / dtSeconds
	}
	out["speed"] = speed
	out["bytes"] = s.bytes
	out["errors"] = s.errors
	out["checks"] = s.checks
	out["transfers"] = s.transfers
	out["deletes"] = s.deletes
	out["elapsedTime"] = dtSeconds
	s.mu.RUnlock()
	if !s.checking.empty() {
		var c []string
		s.checking.mu.RLock()
		defer s.checking.mu.RUnlock()
		for name := range s.checking.items {
			c = append(c, name)
		}
		out["checking"] = c
	}
	if !s.transferring.empty() {
		var t []interface{}
		s.transferring.mu.RLock()
		defer s.transferring.mu.RUnlock()
		for name := range s.transferring.items {
			if acc := s.inProgress.get(name); acc != nil {
				t = append(t, acc.RemoteStats())
			} else {
				t = append(t, name)
			}
		}
		out["transferring"] = t
	}
	if s.errors > 0 {
		out["lastError"] = s.lastError
	}
	return out, nil
}

// String convert the StatsInfo to a string for printing
func (s *StatsInfo) String() string {
	s.mu.RLock()

	dt := time.Now().Sub(s.start)
	dtSeconds := dt.Seconds()
	speed := 0.0
	if dt > 0 {
		speed = float64(s.bytes) / dtSeconds
	}
	dtRounded := dt - (dt % (time.Second / 10))
	buf := &bytes.Buffer{}

	if fs.Config.DataRateUnit == "bits" {
		speed = speed * 8
	}

	percent := func(a int64, b int64) int {
		if b <= 0 {
			return 0
		}
		return int(float64(a)*100/float64(b) + 0.5)
	}

	totalChecks, totalTransfer, totalSize := int64(s.checkQueue)+s.checks, int64(s.transferQueue)+s.transfers, s.transferQueueSize+s.bytes
	eta := time.Duration(0)
	if speed > 0 {
		eta = time.Second * time.Duration(float64(s.transferQueueSize)/float64(speed)+0.5)
	}
	etaString := "-"
	if eta > 0 {
		etaString = eta.String()
	}
	xfrchk := []string{}
	if totalTransfer > 0 && s.transferQueue > 0 {
		xfrchk = append(xfrchk, fmt.Sprintf("xfr#%d/%d", s.transfers, totalTransfer))
	}
	if totalChecks > 0 && s.checkQueue > 0 {
		xfrchk = append(xfrchk, fmt.Sprintf("chk#%d/%d", s.checks, totalChecks))
	}
	xfrchkString := ""
	if len(xfrchk) > 0 {
		xfrchkString = fmt.Sprintf(" (%s)", strings.Join(xfrchk, ", "))
	}
	// FIXME make a one line display too

	_, _ = fmt.Fprintf(buf, `
Transferred:   %10s / %s, %d%%, %s, ETA %s%s
Errors:        %10d
Checks:        %10d / %d, %d%%
Transferred:   %10d / %d, %d%%
Elapsed time:  %10v
`,
		fs.SizeSuffix(s.bytes), fs.SizeSuffix(totalSize).Unit("Bytes"), percent(s.bytes, totalSize), fs.SizeSuffix(speed).Unit(strings.Title(fs.Config.DataRateUnit)+"/s"), etaString, xfrchkString,
		s.errors,
		s.checks, totalChecks, percent(s.checks, totalChecks),
		s.transfers, totalTransfer, percent(s.transfers, totalTransfer),
		dtRounded)

	// checking and transferring have their own locking so unlock
	// here to prevent deadlock on GetBytes
	s.mu.RUnlock()

	if !s.checking.empty() {
		_, _ = fmt.Fprintf(buf, "Checking:\n%s\n", s.checking)
	}
	if !s.transferring.empty() {
		_, _ = fmt.Fprintf(buf, "Transferring:\n%s\n", s.transferring)
	}
	return buf.String()
}

// Log outputs the StatsInfo to the log
func (s *StatsInfo) Log() {
	fs.LogLevelPrintf(fs.Config.StatsLogLevel, nil, "%v\n", s)
}

// Bytes updates the stats for bytes bytes
func (s *StatsInfo) Bytes(bytes int64) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.bytes += bytes
}

// GetBytes returns the number of bytes transferred so far
func (s *StatsInfo) GetBytes() int64 {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.bytes
}

// Errors updates the stats for errors
func (s *StatsInfo) Errors(errors int64) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.errors += errors
}

// GetErrors reads the number of errors
func (s *StatsInfo) GetErrors() int64 {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.errors
}

// GetLastError returns the lastError
func (s *StatsInfo) GetLastError() error {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.lastError
}

// Deletes updates the stats for deletes
func (s *StatsInfo) Deletes(deletes int64) int64 {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.deletes += deletes
	return s.deletes
}

// ResetCounters sets the counters (bytes, checks, errors, transfers) to 0
func (s *StatsInfo) ResetCounters() {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.bytes = 0
	s.errors = 0
	s.checks = 0
	s.transfers = 0
	s.deletes = 0
}

// ResetErrors sets the errors count to 0
func (s *StatsInfo) ResetErrors() {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.errors = 0
}

// Errored returns whether there have been any errors
func (s *StatsInfo) Errored() bool {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.errors != 0
}

// Error adds a single error into the stats and assigns lastError
func (s *StatsInfo) Error(err error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.errors++
	s.lastError = err
}

// Checking adds a check into the stats
func (s *StatsInfo) Checking(remote string) {
	s.checking.add(remote)
}

// DoneChecking removes a check from the stats
func (s *StatsInfo) DoneChecking(remote string) {
	s.checking.del(remote)
	s.mu.Lock()
	s.checks++
	s.mu.Unlock()
}

// GetTransfers reads the number of transfers
func (s *StatsInfo) GetTransfers() int64 {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.transfers
}

// Transferring adds a transfer into the stats
func (s *StatsInfo) Transferring(remote string) {
	s.transferring.add(remote)
}

// DoneTransferring removes a transfer from the stats
//
// if ok is true then it increments the transfers count
func (s *StatsInfo) DoneTransferring(remote string, ok bool) {
	s.transferring.del(remote)
	if ok {
		s.mu.Lock()
		s.transfers++
		s.mu.Unlock()
	}
}

// SetCheckQueue sets the number of queued checks
func (s *StatsInfo) SetCheckQueue(n int, size int64) {
	s.mu.Lock()
	s.checkQueue = n
	s.checkQueueSize = size
	s.mu.Unlock()
}

// SetTransferQueue sets the number of queued transfers
func (s *StatsInfo) SetTransferQueue(n int, size int64) {
	s.mu.Lock()
	s.transferQueue = n
	s.transferQueueSize = size
	s.mu.Unlock()
}

// SetRenameQueue sets the number of queued transfers
func (s *StatsInfo) SetRenameQueue(n int, size int64) {
	s.mu.Lock()
	s.renameQueue = n
	s.renameQueueSize = size
	s.mu.Unlock()
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package http_test

import (
	"fmt"
	"net/http"
	"os"
	"runtime"
	"sort"
	"strings"
	"testing"
	"time"
)

func TestMain(m *testing.M) {
	v := m.Run()
	if v == 0 && goroutineLeaked() {
		os.Exit(1)
	}
	os.Exit(v)
}

func interestingGoroutines() (gs []string) {
	buf := make([]byte, 2<<20)
	buf = buf[:runtime.Stack(buf, true)]
	for _, g := range strings.Split(string(buf), "\n\n") {
		sl := strings.SplitN(g, "\n", 2)
		if len(sl) != 2 {
			continue
		}
		stack := strings.TrimSpace(sl[1])
		if stack == "" ||
			strings.Contains(stack, "created by net.startServer") ||
			strings.Contains(stack, "created by testing.RunTests") ||
			strings.Contains(stack, "closeWriteAndWait") ||
			strings.Contains(stack, "testing.Main(") ||
			// These only show up with GOTRACEBACK=2; Issue 5005 (comment 28)
			strings.Contains(stack, "runtime.goexit") ||
			strings.Contains(stack, "created by runtime.gc") ||
			strings.Contains(stack, "net/http_test.interestingGoroutines") ||
			strings.Contains(stack, "runtime.MHeap_Scavenger") {
			continue
		}
		gs = append(gs, stack)
	}
	sort.Strings(gs)
	return
}

// Verify the other tests didn't leave any goroutines running.
func goroutineLeaked() bool {
	if testing.Short() {
		// not counting goroutines for leakage in -short mode
		return false
	}
	gs := interestingGoroutines()

	n := 0
	stackCount := make(map[string]int)
	for _, g := range gs {
		stackCount[g]++
		n++
	}

	if n == 0 {
		return false
	}
	fmt.Fprintf(os.Stderr, "Too many goroutines running after net/http test(s).\n")
	for stack, count := range stackCount {
		fmt.Fprintf(os.Stderr, "%d instances of:\n%s", count, stack)
	}
	return true
}

func afterTest(t *testing.T) {
	http.DefaultTransport.(*http.Transport).CloseIdleConnections()
	if testing.Short() {
		return
	}
	var bad string
	badSubstring := map[string]string{
		").readLoop(":                                  "a Transport",
		").writeLoop(":                                 "a Transport",
		"created by net/http/httptest.(*Server).Start": "an httptest.Server",
		"timeoutHandler":                               "a TimeoutHandler",
		"net.(*netFD).connect(":                        "a timing out dial",
		").noteClientGone(":                            "a closenotifier sender",
	}
	var stacks string
	for i := 0; i < 4; i++ {
		bad = ""
		stacks = strings.Join(interestingGoroutines(), "\n\n")
		for substr, what := range badSubstring {
			if strings.Contains(stacks, substr) {
				bad = what
			}
		}
		if bad == "" {
			return
		}
		// Bad stuff found, but goroutines might just still be
		// shutting down, so give it some time.
		time.Sleep(250 * time.Millisecond)
	}
	t.Errorf("Test appears to have leaked %s:\n%s", bad, stacks)
}
<commit_msg>net/http: add missing newline in list of leaked goroutines<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package http_test

import (
	"fmt"
	"net/http"
	"os"
	"runtime"
	"sort"
	"strings"
	"testing"
	"time"
)

func TestMain(m *testing.M) {
	v := m.Run()
	if v == 0 && goroutineLeaked() {
		os.Exit(1)
	}
	os.Exit(v)
}

func interestingGoroutines() (gs []string) {
	buf := make([]byte, 2<<20)
	buf = buf[:runtime.Stack(buf, true)]
	for _, g := range strings.Split(string(buf), "\n\n") {
		sl := strings.SplitN(g, "\n", 2)
		if len(sl) != 2 {
			continue
		}
		stack := strings.TrimSpace(sl[1])
		if stack == "" ||
			strings.Contains(stack, "created by net.startServer") ||
			strings.Contains(stack, "created by testing.RunTests") ||
			strings.Contains(stack, "closeWriteAndWait") ||
			strings.Contains(stack, "testing.Main(") ||
			// These only show up with GOTRACEBACK=2; Issue 5005 (comment 28)
			strings.Contains(stack, "runtime.goexit") ||
			strings.Contains(stack, "created by runtime.gc") ||
			strings.Contains(stack, "net/http_test.interestingGoroutines") ||
			strings.Contains(stack, "runtime.MHeap_Scavenger") {
			continue
		}
		gs = append(gs, stack)
	}
	sort.Strings(gs)
	return
}

// Verify the other tests didn't leave any goroutines running.
func goroutineLeaked() bool {
	if testing.Short() {
		// not counting goroutines for leakage in -short mode
		return false
	}
	gs := interestingGoroutines()

	n := 0
	stackCount := make(map[string]int)
	for _, g := range gs {
		stackCount[g]++
		n++
	}

	if n == 0 {
		return false
	}
	fmt.Fprintf(os.Stderr, "Too many goroutines running after net/http test(s).\n")
	for stack, count := range stackCount {
		fmt.Fprintf(os.Stderr, "%d instances of:\n%s\n", count, stack)
	}
	return true
}

func afterTest(t *testing.T) {
	http.DefaultTransport.(*http.Transport).CloseIdleConnections()
	if testing.Short() {
		return
	}
	var bad string
	badSubstring := map[string]string{
		").readLoop(":                                  "a Transport",
		").writeLoop(":                                 "a Transport",
		"created by net/http/httptest.(*Server).Start": "an httptest.Server",
		"timeoutHandler":                               "a TimeoutHandler",
		"net.(*netFD).connect(":                        "a timing out dial",
		").noteClientGone(":                            "a closenotifier sender",
	}
	var stacks string
	for i := 0; i < 4; i++ {
		bad = ""
		stacks = strings.Join(interestingGoroutines(), "\n\n")
		for substr, what := range badSubstring {
			if strings.Contains(stacks, substr) {
				bad = what
			}
		}
		if bad == "" {
			return
		}
		// Bad stuff found, but goroutines might just still be
		// shutting down, so give it some time.
		time.Sleep(250 * time.Millisecond)
	}
	t.Errorf("Test appears to have leaked %s:\n%s", bad, stacks)
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// TODO(thockin): This whole pkg is pretty linux-centric.  As soon as we have
// an alternate platform, we will need to abstract further.
package mount

import (
	"os"
	"path/filepath"
	"strings"
	"syscall"
)

type FileType string

const (
	// Default mount command if mounter path is not specified
	defaultMountCommand           = "mount"
	MountsInGlobalPDPath          = "mounts"
	FileTypeDirectory    FileType = "Directory"
	FileTypeFile         FileType = "File"
	FileTypeSocket       FileType = "Socket"
	FileTypeCharDev      FileType = "CharDevice"
	FileTypeBlockDev     FileType = "BlockDevice"
)

type Interface interface {
	// Mount mounts source to target as fstype with given options.
	Mount(source string, target string, fstype string, options []string) error
	// Unmount unmounts given target.
	Unmount(target string) error
	// List returns a list of all mounted filesystems.  This can be large.
	// On some platforms, reading mounts is not guaranteed consistent (i.e.
	// it could change between chunked reads). This is guaranteed to be
	// consistent.
	List() ([]MountPoint, error)
	// IsMountPointMatch determines if the mountpoint matches the dir
	IsMountPointMatch(mp MountPoint, dir string) bool
	// IsNotMountPoint determines if a directory is a mountpoint.
	// It should return ErrNotExist when the directory does not exist.
	// IsNotMountPoint is more expensive than IsLikelyNotMountPoint.
	// IsNotMountPoint detects bind mounts in linux.
	// IsNotMountPoint enumerates all the mountpoints using List() and
	// the list of mountpoints may be large, then it uses
	// IsMountPointMatch to evaluate whether the directory is a mountpoint
	IsNotMountPoint(file string) (bool, error)
	// IsLikelyNotMountPoint uses heuristics to determine if a directory
	// is a mountpoint.
	// It should return ErrNotExist when the directory does not exist.
	// IsLikelyNotMountPoint does NOT properly detect all mountpoint types
	// most notably linux bind mounts.
	IsLikelyNotMountPoint(file string) (bool, error)
	// DeviceOpened determines if the device is in use elsewhere
	// on the system, i.e. still mounted.
	DeviceOpened(pathname string) (bool, error)
	// PathIsDevice determines if a path is a device.
	PathIsDevice(pathname string) (bool, error)
	// GetDeviceNameFromMount finds the device name by checking the mount path
	// to get the global mount path which matches its plugin directory
	GetDeviceNameFromMount(mountPath, pluginDir string) (string, error)
	// MakeRShared checks that given path is on a mount with 'rshared' mount
	// propagation. If not, it bind-mounts the path as rshared.
	MakeRShared(path string) error
	// GetFileType checks for file/directory/socket/block/character devices.
	// Will operate in the host mount namespace if kubelet is running in a container
	GetFileType(pathname string) (FileType, error)
	// MakeFile creates an empty file.
	// Will operate in the host mount namespace if kubelet is running in a container
	MakeFile(pathname string) error
	// MakeDir creates a new directory.
	// Will operate in the host mount namespace if kubelet is running in a container
	MakeDir(pathname string) error
	// SafeMakeDir makes sure that the created directory does not escape given
	// base directory mis-using symlinks. The directory is created in the same
	// mount namespace as where kubelet is running. Note that the function makes
	// sure that it creates the directory somewhere under the base, nothing
	// else. E.g. if the directory already exists, it may exists outside of the
	// base due to symlinks.
	SafeMakeDir(pathname string, base string, perm os.FileMode) error
	// ExistsPath checks whether the path exists.
	// Will operate in the host mount namespace if kubelet is running in a container
	ExistsPath(pathname string) bool
	// CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given
	// pod volume directory.
	CleanSubPaths(podDir string, volumeName string) error
	// PrepareSafeSubpath does everything that's necessary to prepare a subPath
	// that's 1) inside given volumePath and 2) immutable after this call.
	//
	// newHostPath - location of prepared subPath. It should be used instead of
	// hostName when running the container.
	// cleanupAction - action to run when the container is running or it failed to start.
	//
	// CleanupAction must be called immediately after the container with given
	// subpath starts. On the other hand, Interface.CleanSubPaths must be called
	// when the pod finishes.
	PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error)
}

type Subpath struct {
	// index of the VolumeMount for this container
	VolumeMountIndex int
	// Full path to the subpath directory on the host
	Path string
	// name of the volume that is a valid directory name.
	VolumeName string
	// Full path to the volume path
	VolumePath string
	// Path to the pod's directory, including pod UID
	PodDir string
	// Name of the container
	ContainerName string
}

// Exec executes command where mount utilities are. This can be either the host,
// container where kubelet runs or even a remote pod with mount utilities.
// Usual pkg/util/exec interface is not used because kubelet.RunInContainer does
// not provide stdin/stdout/stderr streams.
type Exec interface {
	// Run executes a command and returns its stdout + stderr combined in one
	// stream.
	Run(cmd string, args ...string) ([]byte, error)
}

// Compile-time check to ensure all Mounter implementations satisfy
// the mount interface
var _ Interface = &Mounter{}

// This represents a single line in /proc/mounts or /etc/fstab.
type MountPoint struct {
	Device string
	Path   string
	Type   string
	Opts   []string
	Freq   int
	Pass   int
}

// SafeFormatAndMount probes a device to see if it is formatted.
// Namely it checks to see if a file system is present. If so it
// mounts it otherwise the device is formatted first then mounted.
type SafeFormatAndMount struct {
	Interface
	Exec
}

// FormatAndMount formats the given disk, if needed, and mounts it.
// That is if the disk is not formatted and it is not being mounted as
// read-only it will format it first then mount it. Otherwise, if the
// disk is already formatted or it is being mounted as read-only, it
// will be mounted without formatting.
func (mounter *SafeFormatAndMount) FormatAndMount(source string, target string, fstype string, options []string) error {
	return mounter.formatAndMount(source, target, fstype, options)
}

// GetMountRefsByDev finds all references to the device provided
// by mountPath; returns a list of paths.
func GetMountRefsByDev(mounter Interface, mountPath string) ([]string, error) {
	mps, err := mounter.List()
	if err != nil {
		return nil, err
	}
	slTarget, err := filepath.EvalSymlinks(mountPath)
	if err != nil {
		slTarget = mountPath
	}

	// Finding the device mounted to mountPath
	diskDev := ""
	for i := range mps {
		if slTarget == mps[i].Path {
			diskDev = mps[i].Device
			break
		}
	}

	// Find all references to the device.
	var refs []string
	for i := range mps {
		if mps[i].Device == diskDev || mps[i].Device == slTarget {
			if mps[i].Path != slTarget {
				refs = append(refs, mps[i].Path)
			}
		}
	}
	return refs, nil
}

// GetDeviceNameFromMount: given a mnt point, find the device from /proc/mounts
// returns the device name, reference count, and error code
func GetDeviceNameFromMount(mounter Interface, mountPath string) (string, int, error) {
	mps, err := mounter.List()
	if err != nil {
		return "", 0, err
	}

	// Find the device name.
	// FIXME if multiple devices mounted on the same mount path, only the first one is returned
	device := ""
	// If mountPath is symlink, need get its target path.
	slTarget, err := filepath.EvalSymlinks(mountPath)
	if err != nil {
		slTarget = mountPath
	}
	for i := range mps {
		if mps[i].Path == slTarget {
			device = mps[i].Device
			break
		}
	}

	// Find all references to the device.
	refCount := 0
	for i := range mps {
		if mps[i].Device == device {
			refCount++
		}
	}
	return device, refCount, nil
}

func isNotDirErr(err error) bool {
	if e, ok := err.(*os.PathError); ok && e.Err == syscall.ENOTDIR {
		return true
	}
	return false
}

// IsNotMountPoint determines if a directory is a mountpoint.
// It should return ErrNotExist when the directory does not exist.
// This method uses the List() of all mountpoints
// It is more extensive than IsLikelyNotMountPoint
// and it detects bind mounts in linux
func IsNotMountPoint(mounter Interface, file string) (bool, error) {
	// IsLikelyNotMountPoint provides a quick check
	// to determine whether file IS A mountpoint
	notMnt, notMntErr := mounter.IsLikelyNotMountPoint(file)
	if notMntErr != nil && os.IsPermission(notMntErr) {
		// We were not allowed to do the simple stat() check, e.g. on NFS with
		// root_squash. Fall back to /proc/mounts check below.
		notMnt = true
		notMntErr = nil
	}
	if notMntErr != nil && isNotDirErr(notMntErr) {
		return notMnt, notMntErr
	}
	// identified as mountpoint, so return this fact
	if notMnt == false {
		return notMnt, nil
	}
	// check all mountpoints since IsLikelyNotMountPoint
	// is not reliable for some mountpoint types
	mountPoints, mountPointsErr := mounter.List()
	if mountPointsErr != nil {
		return notMnt, mountPointsErr
	}
	for _, mp := range mountPoints {
		if mounter.IsMountPointMatch(mp, file) {
			notMnt = false
			break
		}
	}
	return notMnt, nil
}

// isBind detects whether a bind mount is being requested and makes the remount options to
// use in case of bind mount, due to the fact that bind mount doesn't respect mount options.
// The list equals:
//   options - 'bind' + 'remount' (no duplicate)
func isBind(options []string) (bool, []string) {
	bindRemountOpts := []string{"remount"}
	bind := false

	if len(options) != 0 {
		for _, option := range options {
			switch option {
			case "bind":
				bind = true
				break
			case "remount":
				break
			default:
				bindRemountOpts = append(bindRemountOpts, option)
			}
		}
	}

	return bind, bindRemountOpts
}

// TODO: this is a workaround for the unmount device issue caused by gci mounter.
// In GCI cluster, if gci mounter is used for mounting, the container started by mounter
// script will cause additional mounts created in the container. Since these mounts are
// irrelevant to the original mounts, they should be not considered when checking the
// mount references. Current solution is to filter out those mount paths that contain
// the string of original mount path.
// Plan to work on better approach to solve this issue.

func HasMountRefs(mountPath string, mountRefs []string) bool {
	count := 0
	for _, ref := range mountRefs {
		if !strings.Contains(ref, mountPath) {
			count = count + 1
		}
	}
	return count > 0
}

// pathWithinBase checks if give path is within given base directory.
func pathWithinBase(fullPath, basePath string) bool {
	rel, err := filepath.Rel(basePath, fullPath)
	if err != nil {
		return false
	}
	if startsWithBackstep(rel) {
		// Needed to escape the base path
		return false
	}
	return true
}

// startsWithBackstep checks if the given path starts with a backstep segment
func startsWithBackstep(rel string) bool {
	// normalize to / and check for ../
	return rel == ".." || strings.HasPrefix(filepath.ToSlash(rel), "../")
}
<commit_msg>Remove isNotDir error check<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// TODO(thockin): This whole pkg is pretty linux-centric.  As soon as we have
// an alternate platform, we will need to abstract further.
package mount

import (
	"os"
	"path/filepath"
	"strings"
)

type FileType string

const (
	// Default mount command if mounter path is not specified
	defaultMountCommand           = "mount"
	MountsInGlobalPDPath          = "mounts"
	FileTypeDirectory    FileType = "Directory"
	FileTypeFile         FileType = "File"
	FileTypeSocket       FileType = "Socket"
	FileTypeCharDev      FileType = "CharDevice"
	FileTypeBlockDev     FileType = "BlockDevice"
)

type Interface interface {
	// Mount mounts source to target as fstype with given options.
	Mount(source string, target string, fstype string, options []string) error
	// Unmount unmounts given target.
	Unmount(target string) error
	// List returns a list of all mounted filesystems.  This can be large.
	// On some platforms, reading mounts is not guaranteed consistent (i.e.
	// it could change between chunked reads). This is guaranteed to be
	// consistent.
	List() ([]MountPoint, error)
	// IsMountPointMatch determines if the mountpoint matches the dir
	IsMountPointMatch(mp MountPoint, dir string) bool
	// IsNotMountPoint determines if a directory is a mountpoint.
	// It should return ErrNotExist when the directory does not exist.
	// IsNotMountPoint is more expensive than IsLikelyNotMountPoint.
	// IsNotMountPoint detects bind mounts in linux.
	// IsNotMountPoint enumerates all the mountpoints using List() and
	// the list of mountpoints may be large, then it uses
	// IsMountPointMatch to evaluate whether the directory is a mountpoint
	IsNotMountPoint(file string) (bool, error)
	// IsLikelyNotMountPoint uses heuristics to determine if a directory
	// is a mountpoint.
	// It should return ErrNotExist when the directory does not exist.
	// IsLikelyNotMountPoint does NOT properly detect all mountpoint types
	// most notably linux bind mounts.
	IsLikelyNotMountPoint(file string) (bool, error)
	// DeviceOpened determines if the device is in use elsewhere
	// on the system, i.e. still mounted.
	DeviceOpened(pathname string) (bool, error)
	// PathIsDevice determines if a path is a device.
	PathIsDevice(pathname string) (bool, error)
	// GetDeviceNameFromMount finds the device name by checking the mount path
	// to get the global mount path which matches its plugin directory
	GetDeviceNameFromMount(mountPath, pluginDir string) (string, error)
	// MakeRShared checks that given path is on a mount with 'rshared' mount
	// propagation. If not, it bind-mounts the path as rshared.
	MakeRShared(path string) error
	// GetFileType checks for file/directory/socket/block/character devices.
	// Will operate in the host mount namespace if kubelet is running in a container
	GetFileType(pathname string) (FileType, error)
	// MakeFile creates an empty file.
	// Will operate in the host mount namespace if kubelet is running in a container
	MakeFile(pathname string) error
	// MakeDir creates a new directory.
	// Will operate in the host mount namespace if kubelet is running in a container
	MakeDir(pathname string) error
	// SafeMakeDir makes sure that the created directory does not escape given
	// base directory mis-using symlinks. The directory is created in the same
	// mount namespace as where kubelet is running. Note that the function makes
	// sure that it creates the directory somewhere under the base, nothing
	// else. E.g. if the directory already exists, it may exists outside of the
	// base due to symlinks.
	SafeMakeDir(pathname string, base string, perm os.FileMode) error
	// ExistsPath checks whether the path exists.
	// Will operate in the host mount namespace if kubelet is running in a container
	ExistsPath(pathname string) bool
	// CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given
	// pod volume directory.
	CleanSubPaths(podDir string, volumeName string) error
	// PrepareSafeSubpath does everything that's necessary to prepare a subPath
	// that's 1) inside given volumePath and 2) immutable after this call.
	//
	// newHostPath - location of prepared subPath. It should be used instead of
	// hostName when running the container.
	// cleanupAction - action to run when the container is running or it failed to start.
	//
	// CleanupAction must be called immediately after the container with given
	// subpath starts. On the other hand, Interface.CleanSubPaths must be called
	// when the pod finishes.
	PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error)
}

type Subpath struct {
	// index of the VolumeMount for this container
	VolumeMountIndex int
	// Full path to the subpath directory on the host
	Path string
	// name of the volume that is a valid directory name.
	VolumeName string
	// Full path to the volume path
	VolumePath string
	// Path to the pod's directory, including pod UID
	PodDir string
	// Name of the container
	ContainerName string
}

// Exec executes command where mount utilities are. This can be either the host,
// container where kubelet runs or even a remote pod with mount utilities.
// Usual pkg/util/exec interface is not used because kubelet.RunInContainer does
// not provide stdin/stdout/stderr streams.
type Exec interface {
	// Run executes a command and returns its stdout + stderr combined in one
	// stream.
	Run(cmd string, args ...string) ([]byte, error)
}

// Compile-time check to ensure all Mounter implementations satisfy
// the mount interface
var _ Interface = &Mounter{}

// This represents a single line in /proc/mounts or /etc/fstab.
type MountPoint struct {
	Device string
	Path   string
	Type   string
	Opts   []string
	Freq   int
	Pass   int
}

// SafeFormatAndMount probes a device to see if it is formatted.
// Namely it checks to see if a file system is present. If so it
// mounts it otherwise the device is formatted first then mounted.
type SafeFormatAndMount struct {
	Interface
	Exec
}

// FormatAndMount formats the given disk, if needed, and mounts it.
// That is if the disk is not formatted and it is not being mounted as
// read-only it will format it first then mount it. Otherwise, if the
// disk is already formatted or it is being mounted as read-only, it
// will be mounted without formatting.
func (mounter *SafeFormatAndMount) FormatAndMount(source string, target string, fstype string, options []string) error {
	return mounter.formatAndMount(source, target, fstype, options)
}

// GetMountRefsByDev finds all references to the device provided
// by mountPath; returns a list of paths.
func GetMountRefsByDev(mounter Interface, mountPath string) ([]string, error) {
	mps, err := mounter.List()
	if err != nil {
		return nil, err
	}
	slTarget, err := filepath.EvalSymlinks(mountPath)
	if err != nil {
		slTarget = mountPath
	}

	// Finding the device mounted to mountPath
	diskDev := ""
	for i := range mps {
		if slTarget == mps[i].Path {
			diskDev = mps[i].Device
			break
		}
	}

	// Find all references to the device.
	var refs []string
	for i := range mps {
		if mps[i].Device == diskDev || mps[i].Device == slTarget {
			if mps[i].Path != slTarget {
				refs = append(refs, mps[i].Path)
			}
		}
	}
	return refs, nil
}

// GetDeviceNameFromMount: given a mnt point, find the device from /proc/mounts
// returns the device name, reference count, and error code
func GetDeviceNameFromMount(mounter Interface, mountPath string) (string, int, error) {
	mps, err := mounter.List()
	if err != nil {
		return "", 0, err
	}

	// Find the device name.
	// FIXME if multiple devices mounted on the same mount path, only the first one is returned
	device := ""
	// If mountPath is symlink, need get its target path.
	slTarget, err := filepath.EvalSymlinks(mountPath)
	if err != nil {
		slTarget = mountPath
	}
	for i := range mps {
		if mps[i].Path == slTarget {
			device = mps[i].Device
			break
		}
	}

	// Find all references to the device.
	refCount := 0
	for i := range mps {
		if mps[i].Device == device {
			refCount++
		}
	}
	return device, refCount, nil
}

// IsNotMountPoint determines if a directory is a mountpoint.
// It should return ErrNotExist when the directory does not exist.
// This method uses the List() of all mountpoints
// It is more extensive than IsLikelyNotMountPoint
// and it detects bind mounts in linux
func IsNotMountPoint(mounter Interface, file string) (bool, error) {
	// IsLikelyNotMountPoint provides a quick check
	// to determine whether file IS A mountpoint
	notMnt, notMntErr := mounter.IsLikelyNotMountPoint(file)
	if notMntErr != nil && os.IsPermission(notMntErr) {
		// We were not allowed to do the simple stat() check, e.g. on NFS with
		// root_squash. Fall back to /proc/mounts check below.
		notMnt = true
		notMntErr = nil
	}
	if notMntErr != nil {
		return notMnt, notMntErr
	}
	// identified as mountpoint, so return this fact
	if notMnt == false {
		return notMnt, nil
	}
	// check all mountpoints since IsLikelyNotMountPoint
	// is not reliable for some mountpoint types
	mountPoints, mountPointsErr := mounter.List()
	if mountPointsErr != nil {
		return notMnt, mountPointsErr
	}
	for _, mp := range mountPoints {
		if mounter.IsMountPointMatch(mp, file) {
			notMnt = false
			break
		}
	}
	return notMnt, nil
}

// isBind detects whether a bind mount is being requested and makes the remount options to
// use in case of bind mount, due to the fact that bind mount doesn't respect mount options.
// The list equals:
//   options - 'bind' + 'remount' (no duplicate)
func isBind(options []string) (bool, []string) {
	bindRemountOpts := []string{"remount"}
	bind := false

	if len(options) != 0 {
		for _, option := range options {
			switch option {
			case "bind":
				bind = true
				break
			case "remount":
				break
			default:
				bindRemountOpts = append(bindRemountOpts, option)
			}
		}
	}

	return bind, bindRemountOpts
}

// TODO: this is a workaround for the unmount device issue caused by gci mounter.
// In GCI cluster, if gci mounter is used for mounting, the container started by mounter
// script will cause additional mounts created in the container. Since these mounts are
// irrelevant to the original mounts, they should be not considered when checking the
// mount references. Current solution is to filter out those mount paths that contain
// the string of original mount path.
// Plan to work on better approach to solve this issue.

func HasMountRefs(mountPath string, mountRefs []string) bool {
	count := 0
	for _, ref := range mountRefs {
		if !strings.Contains(ref, mountPath) {
			count = count + 1
		}
	}
	return count > 0
}

// pathWithinBase checks if give path is within given base directory.
func pathWithinBase(fullPath, basePath string) bool {
	rel, err := filepath.Rel(basePath, fullPath)
	if err != nil {
		return false
	}
	if startsWithBackstep(rel) {
		// Needed to escape the base path
		return false
	}
	return true
}

// startsWithBackstep checks if the given path starts with a backstep segment
func startsWithBackstep(rel string) bool {
	// normalize to / and check for ../
	return rel == ".." || strings.HasPrefix(filepath.ToSlash(rel), "../")
}
<|endoftext|>


<commit_before>package cinemate

import (
	"encoding/xml"
	"net/url"
	"strconv"
	"time"
)

// GetMovie Информация о фильме
// Пример запроса: http://api.cinemate.cc/movie?apikey=APIKEY&id=68675&format=xml
// apikey ключ разработчика
// id     ID фильма
// format необязательный параметр формата возвращаемых сервером данных: xml (по умолчанию) или json
func (api *API) GetMovie(id int64) (movie Movie, err error) {
	time.Sleep(1 * time.Second)
	var result APIResponse
	var u url.URL
	u.Scheme = "http"
	u.Host = "api.cinemate.cc"
	u.Path = "/movie"
	q := u.Query()
	q.Set("apikey", api.apikey)
	q.Set("id", strconv.FormatInt(id, 10))
	q.Set("format", "xml")
	u.RawQuery = q.Encode()
	xmlBody, err := getXML(u.String())
	if err != nil {
		return
	}
	err = xml.Unmarshal(xmlBody, &result)
	movie = result.Movies[0]
	return
}

// GetMovieList Результаты поиска фильмов, используя заданные фильтры. Возвращается 10 первых фильмов.
// Пример запроса: http://api.cinemate.cc/movie.list?apikey=APIKEY&year=2010&format=xml
// apikey         ключ разработчика
// type           тип фильмов. Возможные значения: movie, serial, short
// state          состояние фильма. Возможные значения: soon, cinema
// mode           специальный режим отображения лучших фильмов, отсортированных по рейтингу IMDB. Возможные значения: best
// year           год выпуска фильма или сериала
// genre          жанр. В качестве значения (используется slug со страницы http://cinemate.cc/movie/genre/, например, "sport")
// country        отбор по стране (используется slug со страницы http://cinemate.cc/movie/country/, например, "kazakhstan")
// order_by       критерий сортировки: create_date (по дате добавления на сайт), release_date (по дате выхода в мире), ru_release_date (по дате выхода в России, по умолчанию)
// order          порядок сортировки параметра order_by: desc (по убыванию, по умолчанию), asc (по возрастанию)
// from, to       значения среза параметра order_by в формате даты ДД.ММ.ГГГГ. Включительно.
// page, per_page страница и количество записей в выборке. По умолчанию 0 и 10 соответственно. per_page не может быть более 25.
// format         необязательный параметр формата возвращаемых сервером данных: xml (по умолчанию) или json
func (api *API) GetMovieList(ccr CCRequest) (movies []Movie, err error) {
	time.Sleep(1 * time.Second)
	var result APIResponse
	var u url.URL
	u.Scheme = "http"
	u.Host = "api.cinemate.cc"
	u.Path = "/movie.list"
	q := u.Query()
	q.Set("apikey", api.apikey)
	if ccr.Type != "" {
		q.Set("type", ccr.Type)
	}
	if ccr.State != "" {
		q.Set("state", ccr.State)
	}
	if ccr.Mode != "" {
		q.Set("mode", ccr.Mode)
	}
	if ccr.Year != 0 {
		q.Set("year", strconv.FormatInt(ccr.Year, 10))
	}
	if ccr.Genre != "" {
		q.Set("genre", ccr.Genre)
	}
	if ccr.Country != "" {
		q.Set("country", ccr.Country)
	}
	if ccr.OrderBy != "" {
		q.Set("order_by", ccr.OrderBy)
	}
	if ccr.Order != "" {
		q.Set("order", ccr.Order)
	}
	if ccr.From != "" {
		q.Set("from", ccr.From)
	}
	if ccr.To != "" {
		q.Set("to", ccr.To)
	}
	if ccr.Page != 0 {
		q.Set("page", strconv.FormatInt(ccr.Page, 10))
	}
	if ccr.PerPage != 0 {
		q.Set("per_page", strconv.FormatInt(ccr.PerPage, 10))
	}
	q.Set("format", "xml")
	u.RawQuery = q.Encode()
	xmlBody, err := getXML(u.String())
	if err != nil {
		return
	}
	err = xml.Unmarshal(xmlBody, &result)
	movies = result.Movies
	return
}

// GetMovieSearch Поиск по заголовкам фильмов
// Пример запроса: http://api.cinemate.cc/movie.search?apikey=APIKEY&term=Пираты%20кариб&format=xml
// apikey ключ разработчика
// term   искомая строка; поддерживается уточняющий поиск по году выхода фильма (год должен быть указан в конце искомой строки, например, "Пираты кариб 2003") и коррекцию ошибок при печати
// format необязательный параметр формата возвращаемых сервером данных: xml (по умолчанию) или json
func (api *API) GetMovieSearch(term string) (movies []Movie, err error) {
	time.Sleep(1 * time.Second)
	var result APIResponse
	var u url.URL
	u.Scheme = "http"
	u.Host = "api.cinemate.cc"
	u.Path = "/movie.search"
	q := u.Query()
	q.Set("apikey", api.apikey)
	q.Set("term", term)
	q.Set("format", "xml")
	u.RawQuery = q.Encode()
	xmlBody, err := getXML(u.String())
	if err != nil {
		return
	}
	err = xml.Unmarshal(xmlBody, &result)
	movies = result.Movies
	return
}
<commit_msg>check result<commit_after>package cinemate

import (
	"encoding/xml"
	"net/url"
	"strconv"
	"time"
)

// GetMovie Информация о фильме
// Пример запроса: http://api.cinemate.cc/movie?apikey=APIKEY&id=68675&format=xml
// apikey ключ разработчика
// id     ID фильма
// format необязательный параметр формата возвращаемых сервером данных: xml (по умолчанию) или json
func (api *API) GetMovie(id int64) (movie Movie, err error) {
	time.Sleep(1 * time.Second)
	var result APIResponse
	var u url.URL
	u.Scheme = "http"
	u.Host = "api.cinemate.cc"
	u.Path = "/movie"
	q := u.Query()
	q.Set("apikey", api.apikey)
	q.Set("id", strconv.FormatInt(id, 10))
	q.Set("format", "xml")
	u.RawQuery = q.Encode()
	xmlBody, err := getXML(u.String())
	if err != nil {
		return
	}
	err = xml.Unmarshal(xmlBody, &result)
	if len(result.Movies) > 0 {
		movie = result.Movies[0]
	}
	return
}

// GetMovieList Результаты поиска фильмов, используя заданные фильтры. Возвращается 10 первых фильмов.
// Пример запроса: http://api.cinemate.cc/movie.list?apikey=APIKEY&year=2010&format=xml
// apikey         ключ разработчика
// type           тип фильмов. Возможные значения: movie, serial, short
// state          состояние фильма. Возможные значения: soon, cinema
// mode           специальный режим отображения лучших фильмов, отсортированных по рейтингу IMDB. Возможные значения: best
// year           год выпуска фильма или сериала
// genre          жанр. В качестве значения (используется slug со страницы http://cinemate.cc/movie/genre/, например, "sport")
// country        отбор по стране (используется slug со страницы http://cinemate.cc/movie/country/, например, "kazakhstan")
// order_by       критерий сортировки: create_date (по дате добавления на сайт), release_date (по дате выхода в мире), ru_release_date (по дате выхода в России, по умолчанию)
// order          порядок сортировки параметра order_by: desc (по убыванию, по умолчанию), asc (по возрастанию)
// from, to       значения среза параметра order_by в формате даты ДД.ММ.ГГГГ. Включительно.
// page, per_page страница и количество записей в выборке. По умолчанию 0 и 10 соответственно. per_page не может быть более 25.
// format         необязательный параметр формата возвращаемых сервером данных: xml (по умолчанию) или json
func (api *API) GetMovieList(ccr CCRequest) (movies []Movie, err error) {
	time.Sleep(1 * time.Second)
	var result APIResponse
	var u url.URL
	u.Scheme = "http"
	u.Host = "api.cinemate.cc"
	u.Path = "/movie.list"
	q := u.Query()
	q.Set("apikey", api.apikey)
	if ccr.Type != "" {
		q.Set("type", ccr.Type)
	}
	if ccr.State != "" {
		q.Set("state", ccr.State)
	}
	if ccr.Mode != "" {
		q.Set("mode", ccr.Mode)
	}
	if ccr.Year != 0 {
		q.Set("year", strconv.FormatInt(ccr.Year, 10))
	}
	if ccr.Genre != "" {
		q.Set("genre", ccr.Genre)
	}
	if ccr.Country != "" {
		q.Set("country", ccr.Country)
	}
	if ccr.OrderBy != "" {
		q.Set("order_by", ccr.OrderBy)
	}
	if ccr.Order != "" {
		q.Set("order", ccr.Order)
	}
	if ccr.From != "" {
		q.Set("from", ccr.From)
	}
	if ccr.To != "" {
		q.Set("to", ccr.To)
	}
	if ccr.Page != 0 {
		q.Set("page", strconv.FormatInt(ccr.Page, 10))
	}
	if ccr.PerPage != 0 {
		q.Set("per_page", strconv.FormatInt(ccr.PerPage, 10))
	}
	q.Set("format", "xml")
	u.RawQuery = q.Encode()
	xmlBody, err := getXML(u.String())
	if err != nil {
		return
	}
	err = xml.Unmarshal(xmlBody, &result)
	movies = result.Movies
	return
}

// GetMovieSearch Поиск по заголовкам фильмов
// Пример запроса: http://api.cinemate.cc/movie.search?apikey=APIKEY&term=Пираты%20кариб&format=xml
// apikey ключ разработчика
// term   искомая строка; поддерживается уточняющий поиск по году выхода фильма (год должен быть указан в конце искомой строки, например, "Пираты кариб 2003") и коррекцию ошибок при печати
// format необязательный параметр формата возвращаемых сервером данных: xml (по умолчанию) или json
func (api *API) GetMovieSearch(term string) (movies []Movie, err error) {
	time.Sleep(1 * time.Second)
	var result APIResponse
	var u url.URL
	u.Scheme = "http"
	u.Host = "api.cinemate.cc"
	u.Path = "/movie.search"
	q := u.Query()
	q.Set("apikey", api.apikey)
	q.Set("term", term)
	q.Set("format", "xml")
	u.RawQuery = q.Encode()
	xmlBody, err := getXML(u.String())
	if err != nil {
		return
	}
	err = xml.Unmarshal(xmlBody, &result)
	movies = result.Movies
	return
}
<|endoftext|>


<commit_before>package mq

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/Cepave/fe/g"
	"github.com/streadway/amqp"
	"io/ioutil"
	"log"
	"net/http"
)

type Message struct {
	Hostname string `json:"hostname"`
	Mute     bool   `json:"mute"`
}

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
		panic(fmt.Sprintf("%s: %s", msg, err))
	}
}

func Start() {
	mq := g.Config().Mq
	nodes := map[string]interface{}{}
	conn, err := amqp.Dial(mq.Queue)
	failOnError(err, "Failed to connect to RabbitMQ")
	defer conn.Close()

	ch, err := conn.Channel()
	failOnError(err, "Failed to open a channel")
	defer ch.Close()

	q, err := ch.QueueDeclare(
		"mute", // name
		false,  // durable
		false,  // delete when usused
		false,  // exclusive
		false,  // no-wait
		nil,    // arguments
	)
	failOnError(err, "Failed to declare a queue")

	msgs, err := ch.Consume(
		q.Name, // queue
		"",     // consumer
		true,   // auto-ack
		false,  // exclusive
		false,  // no-local
		false,  // no-wait
		nil,    // args
	)
	failOnError(err, "Failed to register a consumer")

	forever := make(chan bool)

	var pkt Message
	go func() {
		for d := range msgs {
			json.Unmarshal(d.Body, &pkt)
			log.Printf("Received a message: %v", pkt)
			params := map[string]string{
				"host": pkt.Hostname,
				"mute": "",
			}
			if pkt.Mute {
				params["mute"] = "1"
			} else {
				params["mute"] = "0"
			}
			payload := map[string]interface{}{
				"method": "host.update",
				"params": params,
			}
			log.Println("payload =", payload)

			s, err := json.Marshal(payload)
			if err != nil {
				log.Println("json.Marshal Error:", err.Error())
			}

			url := mq.Consumer
			log.Println("url =", url)
			reqPost, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(s)))
			if err != nil {
				log.Println("reqPost Error:", err.Error())
			}
			reqPost.Header.Set("Content-Type", "application/json")

			client := &http.Client{}
			resp, err := client.Do(reqPost)
			if err != nil {
				log.Println("resp Error:", err.Error())
			} else {
				defer resp.Body.Close()
				body, _ := ioutil.ReadAll(resp.Body)
				err = json.Unmarshal(body, &nodes)
				if err != nil {
					log.Println("Unmarshal Error:", err.Error())
				}
				log.Println("nodes =", nodes)
			}
		}
	}()
	log.Println("mq.Start ok. Waiting for messages.")
	<-forever
}
<commit_msg>[OWL-708] Fix the fe shutdown problem during initialization the connection to mq.<commit_after>package mq

import (
	"bytes"
	"encoding/json"
	"errors"
	"github.com/Cepave/fe/g"
	"github.com/streadway/amqp"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

var RetriedLimit = 10
var SleepTimePeriod = time.Duration(60)
var ExitStringPrefix = "Exit mq goroutine safely: "
var LogStringFormat = "%s: %s"

type Message struct {
	Hostname string `json:"hostname"`
	Mute     bool   `json:"mute"`
}

func setup(url string) (*amqp.Connection, *amqp.Channel, error) {
	conn, err := amqp.Dial(url)
	if err != nil {
		log.Printf(LogStringFormat, "Failed to connect to RabbitMQ", err)
		return nil, nil, err
	}

	ch, err := conn.Channel()
	if err != nil {
		log.Printf(LogStringFormat, "Failed to open a channel", err)
		defer conn.Close()
		return nil, nil, err
	}
	return conn, ch, nil
}

func Start() {
	mq := g.Config().Mq
	nodes := map[string]interface{}{}

	// Retry RetriedLimit times if there is some problem during connecting
	var ch *amqp.Channel
	var conn *amqp.Connection
	var err error = errors.New("Init error.")
	for i := 0; i < RetriedLimit; i++ {
		if conn, ch, err = setup(mq.Queue); err != nil {
			time.Sleep(time.Second * SleepTimePeriod)
		}
	}
	if err != nil {
		log.Println(ExitStringPrefix + "retried too many times.")
		return
	}
	defer conn.Close()
	defer ch.Close()

	q, err := ch.QueueDeclare(
		"mute", // name
		false,  // durable
		false,  // delete when usused
		false,  // exclusive
		false,  // no-wait
		nil,    // arguments
	)
	if err != nil {
		log.Printf(LogStringFormat, ExitStringPrefix+"failed to declare a queue", err)
		return
	}

	msgs, err := ch.Consume(
		q.Name, // queue
		"",     // consumer
		true,   // auto-ack
		false,  // exclusive
		false,  // no-local
		false,  // no-wait
		nil,    // args
	)
	if err != nil {
		log.Printf(LogStringFormat, ExitStringPrefix+"failed to register a consumer", err)
		return
	}

	forever := make(chan bool)

	var pkt Message
	go func() {
		for d := range msgs {
			json.Unmarshal(d.Body, &pkt)
			log.Printf("Received a message: %v", pkt)
			params := map[string]string{
				"host": pkt.Hostname,
				"mute": "",
			}
			if pkt.Mute {
				params["mute"] = "1"
			} else {
				params["mute"] = "0"
			}
			payload := map[string]interface{}{
				"method": "host.update",
				"params": params,
			}
			log.Println("payload =", payload)

			s, err := json.Marshal(payload)
			if err != nil {
				log.Println("json.Marshal Error:", err.Error())
			}

			url := mq.Consumer
			log.Println("url =", url)
			reqPost, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(s)))
			if err != nil {
				log.Println("reqPost Error:", err.Error())
			}
			reqPost.Header.Set("Content-Type", "application/json")

			client := &http.Client{}
			resp, err := client.Do(reqPost)
			if err != nil {
				log.Println("resp Error:", err.Error())
			} else {
				defer resp.Body.Close()
				body, _ := ioutil.ReadAll(resp.Body)
				err = json.Unmarshal(body, &nodes)
				if err != nil {
					log.Println("Unmarshal Error:", err.Error())
				}
				log.Println("nodes =", nodes)
			}
		}
	}()
	log.Println("mq.Start ok. Waiting for messages.")
	<-forever
}
<|endoftext|>


<commit_before>/**
 * Slack bot for a CTF.
 *
 * Functionality:
 * - greets users the first time they join the same channel as the bot.
 * - converts latex expressions into images.
 * - fetches weather information from Yahoo and returns it.
 * - runs arbitrary commands as 'nobody'.
 */

package main

import (
	"database/sql"
	"fmt"
	"log"
	"os"
	"strings"

	_ "github.com/go-sql-driver/mysql"
)

func main() {
	config := configRead()
	fmt.Print("[OK] Config\n")

	// // For debugging purpose
	// result := kaleidoscopeMake(config, "alok", "test_image.jpg")
	// fmt.Print(result)
	// os.Exit(1)

	// Connect to database
	db, err := sql.Open("mysql", config.MysqlConn)
	if err != nil {
		log.Panicf("Failed to connect to database: %s", err)
	}
	fmt.Print("[OK] Database\n")

	// Connect to Slack using Websocket Real Time API
	ws, bot_id := slackConnect(config.SlackApiToken)
	fmt.Print("[OK] Slack\n")

	for {
		// read each incoming message
		m, err := getMessage(ws)
		if err != nil {
			log.Printf("getMessage failed: %s", err)
			os.Exit(1)
		}

		if m.Type == "message" {
			if m.Subtype == "group_join" || m.Subtype == "channel_join" {
				go func(m Message) {
					greet(config, db, ws, m.Channel)
				}(m)
			} else if m.Subtype == "" && strings.HasPrefix(m.Text, fmt.Sprintf("<@%s>", bot_id)) {
				parts := strings.Fields(m.Text)
				if len(parts) >= 2 && parts[1] == "help" {
					go func(m Message) {
						help(config, ws, m.User, m.Channel)
					}(m)
				} else if len(parts) >= 3 && parts[1] == "math" {
					go func(m Message) {
						math(config, ws, m.User, m.Channel, strings.Join(parts[2:], " "))
					}(m)
				} else if len(parts) >= 3 && parts[1] == "weather" {
					go func(m Message) {
						weather(config, db, ws, m.User, m.Channel, strings.Join(parts[2:], " "))
					}(m)
				} else if len(parts) >= 3 && parts[1] == "debug" {
					go func(m Message) {
						debug(config, ws, m.User, m.Channel, strings.Join(parts[2:], " "))
					}(m)
				} else if len(parts) >= 3 && parts[1] == "kaleidoscope" {
					go func(m Message) {
						kaleidoscope(config, ws, m.User, m.Channel, strings.Join(parts[2:], " "))
					}(m)
				} else {
					go func(m Message) {
						m.Text = fmt.Sprintf("<@%s>: sorry, I don't understand that.", m.User)
						postMessage(ws, m)
					}(m)
				}
			}
		}
	}
}
<commit_msg>Print incoming messages<commit_after>/**
 * Slack bot for a CTF.
 *
 * Functionality:
 * - greets users the first time they join the same channel as the bot.
 * - converts latex expressions into images.
 * - fetches weather information from Yahoo and returns it.
 * - runs arbitrary commands as 'nobody'.
 */

package main

import (
	"database/sql"
	"fmt"
	"log"
	"os"
	"strings"

	_ "github.com/go-sql-driver/mysql"
)

func main() {
	config := configRead()
	fmt.Print("[OK] Config\n")

	// // For debugging purpose
	// result := kaleidoscopeMake(config, "alok", "test_image.jpg")
	// fmt.Print(result)
	// os.Exit(1)

	// Connect to database
	db, err := sql.Open("mysql", config.MysqlConn)
	if err != nil {
		log.Panicf("Failed to connect to database: %s", err)
	}
	fmt.Print("[OK] Database\n")

	// Connect to Slack using Websocket Real Time API
	ws, bot_id := slackConnect(config.SlackApiToken)
	fmt.Print("[OK] Slack\n")

	for {
		// read each incoming message
		m, err := getMessage(ws)
		if err != nil {
			log.Printf("getMessage failed: %s", err)
			os.Exit(1)
		}

		if m.Type == "message" {
			if m.Subtype == "group_join" || m.Subtype == "channel_join" {
				go func(m Message) {
					greet(config, db, ws, m.Channel)
				}(m)
			} else if m.Subtype == "" && strings.HasPrefix(m.Text, fmt.Sprintf("<@%s>", bot_id)) {
				fmt.Fprintf(os.Stderr, "got msg: '%s'", m.Text)
				parts := strings.Fields(m.Text)
				if len(parts) >= 2 && parts[1] == "help" {
					go func(m Message) {
						help(config, ws, m.User, m.Channel)
					}(m)
				} else if len(parts) >= 3 && parts[1] == "math" {
					go func(m Message) {
						math(config, ws, m.User, m.Channel, strings.Join(parts[2:], " "))
					}(m)
				} else if len(parts) >= 3 && parts[1] == "weather" {
					go func(m Message) {
						weather(config, db, ws, m.User, m.Channel, strings.Join(parts[2:], " "))
					}(m)
				} else if len(parts) >= 3 && parts[1] == "debug" {
					go func(m Message) {
						debug(config, ws, m.User, m.Channel, strings.Join(parts[2:], " "))
					}(m)
				} else if len(parts) >= 3 && parts[1] == "kaleidoscope" {
					go func(m Message) {
						kaleidoscope(config, ws, m.User, m.Channel, strings.Join(parts[2:], " "))
					}(m)
				} else {
					go func(m Message) {
						m.Text = fmt.Sprintf("<@%s>: sorry, I don't understand that.", m.User)
						postMessage(ws, m)
					}(m)
				}
			}
		}
	}
}
<|endoftext|>