<commit_before>// Package scribble is a tiny JSON database
package scribble

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"

	"github.com/jcelliott/lumber"
)

// Version is the current version of the project
const Version = "1.0.4"

type (

	// Logger is a generic logger interface
	Logger interface {
		Fatal(string, ...interface{})
		Error(string, ...interface{})
		Warn(string, ...interface{})
		Info(string, ...interface{})
		Debug(string, ...interface{})
		Trace(string, ...interface{})
	}

	// Driver is what is used to interact with the scribble database. It runs
	// transactions, and provides log output
	Driver struct {
		mutex   sync.Mutex
		mutexes map[string]sync.Mutex
		dir     string // the directory where scribble will create the database
		log     Logger // the logger scribble will log to
	}
)

// Options uses for specification of working golang-scribble
type Options struct {
	Logger // the logger scribble will use (configurable)
}

// New creates a new scribble database at the desired directory location, and
// returns a *Driver to then use for interacting with the database
func New(dir string, options *Options) (*Driver, error) {

	//
	dir = filepath.Clean(dir)

	// create default options
	opts := Options{}

	// if options are passed in, use those
	if options != nil {
		opts = *options
	}

	// if no logger is provided, create a default
	if opts.Logger == nil {
		opts.Logger = lumber.NewConsoleLogger(lumber.INFO)
	}

	//
	driver := Driver{
		dir:     dir,
		mutexes: make(map[string]sync.Mutex),
		log:     opts.Logger,
	}

	// if the database already exists, just use it
	if _, err := os.Stat(dir); err == nil {
		opts.Logger.Debug("Using '%s' (database already exists)\n", dir)
		return &driver, nil
	}

	// if the database doesn't exist create it
	opts.Logger.Debug("Creating scribble database at '%s'...\n", dir)
	return &driver, os.MkdirAll(dir, 0755)
}

// Write locks the database and attempts to write the record to the database under
// the [collection] specified with the [resource] name given
func (d *Driver) Write(collection, resource string, v interface{}) error {

	// ensure there is a place to save record
	if collection == "" {
		return fmt.Errorf("Missing collection - no place to save record!")
	}

	// ensure there is a resource (name) to save record as
	if resource == "" {
		return fmt.Errorf("Missing resource - unable to save record (no name)!")
	}

	mutex := d.getOrCreateMutex(collection)
	mutex.Lock()
	defer mutex.Unlock()

	//
	dir := filepath.Join(d.dir, collection)
	fnlPath := filepath.Join(dir, resource+".json")
	tmpPath := fnlPath + ".tmp"

	// create collection directory
	if err := os.MkdirAll(dir, 0755); err != nil {
		return err
	}

	//
	b, err := json.MarshalIndent(v, "", "\t")
	if err != nil {
		return err
	}

	// write marshaled data to the temp file
	if err := ioutil.WriteFile(tmpPath, b, 0644); err != nil {
		return err
	}

	// move final file into place
	return os.Rename(tmpPath, fnlPath)
}

// Read a record from the database
func (d *Driver) Read(collection, resource string, v interface{}) error {

	// ensure there is a place to save record
	if collection == "" {
		return fmt.Errorf("Missing collection - no place to save record!")
	}

	// ensure there is a resource (name) to save record as
	if resource == "" {
		return fmt.Errorf("Missing resource - unable to save record (no name)!")
	}

	//
	record := filepath.Join(d.dir, collection, resource)

	// check to see if file exists
	if _, err := stat(record); err != nil {
		return err
	}

	// read record from database
	b, err := ioutil.ReadFile(record + ".json")
	if err != nil {
		return err
	}

	// unmarshal data
	return json.Unmarshal(b, &v)
}

// ReadAll records from a collection; this is returned as a slice of strings because
// there is no way of knowing what type the record is.
func (d *Driver) ReadAll(collection string) ([]string, error) {

	// ensure there is a collection to read
	if collection == "" {
		return nil, fmt.Errorf("Missing collection - unable to record location!")
	}

	//
	dir := filepath.Join(d.dir, collection)

	// check to see if collection (directory) exists
	if _, err := stat(dir); err != nil {
		return nil, err
	}

	// read all the files in the transaction.Collection; an error here just means
	// the collection is either empty or doesn't exist
	files, _ := ioutil.ReadDir(dir)

	// the files read from the database
	var records []string

	// iterate over each of the files, attempting to read the file. If successful
	// append the files to the collection of read files
	for _, file := range files {
		b, err := ioutil.ReadFile(filepath.Join(dir, file.Name()))
		if err != nil {
			return nil, err
		}

		// append read file
		records = append(records, string(b))
	}

	// unmarhsal the read files as a comma delimeted byte array
	return records, nil
}

// Delete locks that database and then attempts to remove the collection/resource
// specified by [path]
func (d *Driver) Delete(collection, resource string) error {
	path := filepath.Join(collection, resource)
	//
	mutex := d.getOrCreateMutex(path)
	mutex.Lock()
	defer mutex.Unlock()

	//
	dir := filepath.Join(d.dir, path)

	switch fi, err := stat(dir); {

	// if fi is nil or error is not nil return
	case fi == nil, err != nil:
		return fmt.Errorf("Unable to find file or directory named %v\n", path)

	// remove directory and all contents
	case fi.Mode().IsDir():
		return os.RemoveAll(dir)

	// remove file
	case fi.Mode().IsRegular():
		return os.RemoveAll(dir + ".json")
	}

	return nil
}

//
func stat(path string) (fi os.FileInfo, err error) {

	// check for dir, if path isn't a directory check to see if it's a file
	if fi, err = os.Stat(path); os.IsNotExist(err) {
		fi, err = os.Stat(path + ".json")
	}

	return
}

// getOrCreateMutex creates a new collection specific mutex any time a collection
// is being modfied to avoid unsafe operations
func (d *Driver) getOrCreateMutex(collection string) sync.Mutex {

	d.mutex.Lock()
	defer d.mutex.Unlock()

	m, ok := d.mutexes[collection]

	// if the mutex doesn't exist make it
	if !ok {
		m = sync.Mutex{}
		d.mutexes[collection] = m
	}

	return m
}
<commit_msg>Fix mutex being copied & wrong mutex key<commit_after>// Package scribble is a tiny JSON database
package scribble

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"

	"github.com/jcelliott/lumber"
)

// Version is the current version of the project
const Version = "1.0.4"

type (

	// Logger is a generic logger interface
	Logger interface {
		Fatal(string, ...interface{})
		Error(string, ...interface{})
		Warn(string, ...interface{})
		Info(string, ...interface{})
		Debug(string, ...interface{})
		Trace(string, ...interface{})
	}

	// Driver is what is used to interact with the scribble database. It runs
	// transactions, and provides log output
	Driver struct {
		mutex   sync.Mutex
		mutexes map[string]*sync.Mutex
		dir     string // the directory where scribble will create the database
		log     Logger // the logger scribble will log to
	}
)

// Options uses for specification of working golang-scribble
type Options struct {
	Logger // the logger scribble will use (configurable)
}

// New creates a new scribble database at the desired directory location, and
// returns a *Driver to then use for interacting with the database
func New(dir string, options *Options) (*Driver, error) {

	//
	dir = filepath.Clean(dir)

	// create default options
	opts := Options{}

	// if options are passed in, use those
	if options != nil {
		opts = *options
	}

	// if no logger is provided, create a default
	if opts.Logger == nil {
		opts.Logger = lumber.NewConsoleLogger(lumber.INFO)
	}

	//
	driver := Driver{
		dir:     dir,
		mutexes: make(map[string]*sync.Mutex),
		log:     opts.Logger,
	}

	// if the database already exists, just use it
	if _, err := os.Stat(dir); err == nil {
		opts.Logger.Debug("Using '%s' (database already exists)\n", dir)
		return &driver, nil
	}

	// if the database doesn't exist create it
	opts.Logger.Debug("Creating scribble database at '%s'...\n", dir)
	return &driver, os.MkdirAll(dir, 0755)
}

// Write locks the database and attempts to write the record to the database under
// the [collection] specified with the [resource] name given
func (d *Driver) Write(collection, resource string, v interface{}) error {

	// ensure there is a place to save record
	if collection == "" {
		return fmt.Errorf("Missing collection - no place to save record!")
	}

	// ensure there is a resource (name) to save record as
	if resource == "" {
		return fmt.Errorf("Missing resource - unable to save record (no name)!")
	}

	mutex := d.getOrCreateMutex(collection)
	mutex.Lock()
	defer mutex.Unlock()

	//
	dir := filepath.Join(d.dir, collection)
	fnlPath := filepath.Join(dir, resource+".json")
	tmpPath := fnlPath + ".tmp"

	// create collection directory
	if err := os.MkdirAll(dir, 0755); err != nil {
		return err
	}

	//
	b, err := json.MarshalIndent(v, "", "\t")
	if err != nil {
		return err
	}

	// write marshaled data to the temp file
	if err := ioutil.WriteFile(tmpPath, b, 0644); err != nil {
		return err
	}

	// move final file into place
	return os.Rename(tmpPath, fnlPath)
}

// Read a record from the database
func (d *Driver) Read(collection, resource string, v interface{}) error {

	// ensure there is a place to save record
	if collection == "" {
		return fmt.Errorf("Missing collection - no place to save record!")
	}

	// ensure there is a resource (name) to save record as
	if resource == "" {
		return fmt.Errorf("Missing resource - unable to save record (no name)!")
	}

	//
	record := filepath.Join(d.dir, collection, resource)

	// check to see if file exists
	if _, err := stat(record); err != nil {
		return err
	}

	// read record from database
	b, err := ioutil.ReadFile(record + ".json")
	if err != nil {
		return err
	}

	// unmarshal data
	return json.Unmarshal(b, &v)
}

// ReadAll records from a collection; this is returned as a slice of strings because
// there is no way of knowing what type the record is.
func (d *Driver) ReadAll(collection string) ([]string, error) {

	// ensure there is a collection to read
	if collection == "" {
		return nil, fmt.Errorf("Missing collection - unable to record location!")
	}

	//
	dir := filepath.Join(d.dir, collection)

	// check to see if collection (directory) exists
	if _, err := stat(dir); err != nil {
		return nil, err
	}

	// read all the files in the transaction.Collection; an error here just means
	// the collection is either empty or doesn't exist
	files, _ := ioutil.ReadDir(dir)

	// the files read from the database
	var records []string

	// iterate over each of the files, attempting to read the file. If successful
	// append the files to the collection of read files
	for _, file := range files {
		b, err := ioutil.ReadFile(filepath.Join(dir, file.Name()))
		if err != nil {
			return nil, err
		}

		// append read file
		records = append(records, string(b))
	}

	// unmarhsal the read files as a comma delimeted byte array
	return records, nil
}

// Delete locks that database and then attempts to remove the collection/resource
// specified by [path]
func (d *Driver) Delete(collection, resource string) error {
	path := filepath.Join(collection, resource)
	//
	mutex := d.getOrCreateMutex(collection)
	mutex.Lock()
	defer mutex.Unlock()

	//
	dir := filepath.Join(d.dir, path)

	switch fi, err := stat(dir); {

	// if fi is nil or error is not nil return
	case fi == nil, err != nil:
		return fmt.Errorf("Unable to find file or directory named %v\n", path)

	// remove directory and all contents
	case fi.Mode().IsDir():
		return os.RemoveAll(dir)

	// remove file
	case fi.Mode().IsRegular():
		return os.RemoveAll(dir + ".json")
	}

	return nil
}

//
func stat(path string) (fi os.FileInfo, err error) {

	// check for dir, if path isn't a directory check to see if it's a file
	if fi, err = os.Stat(path); os.IsNotExist(err) {
		fi, err = os.Stat(path + ".json")
	}

	return
}

// getOrCreateMutex creates a new collection specific mutex any time a collection
// is being modfied to avoid unsafe operations
func (d *Driver) getOrCreateMutex(collection string) *sync.Mutex {

	d.mutex.Lock()
	defer d.mutex.Unlock()

	m, ok := d.mutexes[collection]

	// if the mutex doesn't exist make it
	if !ok {
		m = &sync.Mutex{}
		d.mutexes[collection] = m
	}

	return m
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
)

const version = "v0.0.1"

var (
	flHelp    = flag.Bool("help", false, "Print this message and quit")
	flVersion = flag.Bool("version", false, "Print version information and quit")
)

func init() {
	flag.BoolVar(flHelp, "h", false, "Print this message and quit")
	flag.BoolVar(flVersion, "v", false, "Print version information and quit")
}

func main() {
	flag.Parse()

	if *flHelp || flag.NArg() == 0 {
		fmt.Fprintf(os.Stderr, helpText)
		os.Exit(0)
	}

	if *flVersion {
		fmt.Fprintf(os.Stderr, "md2ghost: %s\n", version)
		os.Exit(0)
	}
}

const helpText = `md2ghost - Convert a markdown files into Ghost posts.

Usage: md2ghost [option] <file|directory>

Options:

  -o, --output  Specify an output directory for Ghost posts
  -h, --help    Print this message and quit
  -v, --version Print version information and quit

Example:

  $ md2ghost .
  $ md2ghost -o path/to/output_directory path/to/your_directory
`
<commit_msg>Change a specified flags<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/jessevdk/go-flags"
)

const version = "v0.0.1"

type opts struct {
	Help    bool   `short:"h" long:"help" description:"Print this message and quit"`
	Version bool   `short:"v" long:"version" description:"Print version information and quit"`
	OutDir  string `short:"o" long:"output" description:"Specify an output directory for Ghost posts"`
}

func init() {
}

func main() {
	opts := &opts{}
	p := flags.NewParser(opts, flags.PrintErrors)
	args, err := p.Parse()
	if err != nil {
		fmt.Fprintf(os.Stderr, helpText)
		return
	}

	if opts.Help {
		fmt.Fprintf(os.Stderr, helpText)
		return
	}

	if opts.Version {
		fmt.Fprintf(os.Stderr, "md2ghost: %s\n", version)
		return
	}
}

const helpText = `md2ghost - Convert a markdown files into Ghost posts.

Usage: md2ghost [option] <file|directory>

Options:

  -o, --output  Specify an output directory for Ghost posts
  -h, --help    Print this message and quit
  -v, --version Print version information and quit

Example:

  $ md2ghost .
  $ md2ghost -o path/to/output_directory path/to/your_directory
`
<|endoftext|>


<commit_before>package feedmailer

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"log"
	"net/smtp"
	"os"
	"sync"
	"text/template"
	"time"

	rss "github.com/jteeuwen/go-pkg-rss"
)

type profile struct {
	SmtpAddr      string
	SmtpUser      string
	SmtpPass      string
	SmtpHost      string
	SrcEmail      string
	DstEmails     []string
	SubjectPrefix string
	FetchTimeout  int
	HistFile      string
	Feeds         []string
}

type FeedMailer struct {
	ErrChan chan error
	prof    profile
	history map[string]time.Time
	mutex   sync.Mutex
}

func NewFeedMailer() *FeedMailer {
	fm := &FeedMailer{}
	fm.ErrChan = make(chan error)
	fm.history = make(map[string]time.Time)
	return fm
}

func (fm *FeedMailer) Start(file string) error {
	log.Println("Reading profile")
	f, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}
	if err := json.Unmarshal(f, &fm.prof); err != nil {
		return err
	}

	log.Println("Reading history")
	f, err = ioutil.ReadFile(fm.prof.HistFile)
	if err == nil {
		json.Unmarshal(f, &fm.history)
	} else if os.IsNotExist(err) {
		log.Printf("History file (%s) not found, it will be created",
			fm.prof.HistFile)
	} else {
		return err
	}

	log.Println("Fetching feeds")
	for _, url := range fm.prof.Feeds {
		go fm.fetch(url)
	}

	return nil
}

func (fm *FeedMailer) fetch(url string) {
	feed := rss.New(fm.prof.FetchTimeout, true, fm.chanHandler, fm.itemHandler)
	for {
		if err := feed.Fetch(url, nil); err != nil {
			fm.ErrChan <- err
		}
		<-time.After(time.Duration(feed.SecondsTillUpdate()) * time.Second)
	}
}

func (fm *FeedMailer) chanHandler(feed *rss.Feed, newChannels []*rss.Channel) {
	log.Printf("%d new channel(s) in %s\n", len(newChannels), feed.Url)
}

func (fm *FeedMailer) itemHandler(feed *rss.Feed, ch *rss.Channel, newItems []*rss.Item) {
	log.Printf("%d new item(s) in %s\n", len(newItems), feed.Url)

	var lastUpdate time.Time
	for _, item := range newItems {
		itemDate, err := item.ParsedPubDate()
		if err != nil {
			fm.ErrChan <- err
		}
		if fm.history[feed.Url].IsZero() || itemDate.After(fm.history[feed.Url]) {
			if err := fm.mail(ch, item); err != nil {
				fm.ErrChan <- err
			}
			if itemDate.After(lastUpdate) {
				lastUpdate = itemDate
			}
		}
	}

	if !lastUpdate.IsZero() {
		fm.mutex.Lock()
		fm.history[feed.Url] = lastUpdate
		if err := fm.updateHistory(); err != nil {
			fm.ErrChan <- err
		}
		fm.mutex.Unlock()
	}
}

func (fm *FeedMailer) mail(ch *rss.Channel, item *rss.Item) error {
	date, _ := item.ParsedPubDate()
	data := struct {
		SubjectPrefix, ChanTitle, ItemTitle string
		Date                                time.Time
		Links                               []*rss.Link
		Description                         string
		Content                             *rss.Content
	}{fm.prof.SubjectPrefix, ch.Title, item.Title, date,
		item.Links, item.Description, item.Content}

	t, err := template.New("mail").Parse(mailTmpl)
	if err != nil {
		return err
	}
	msg := &bytes.Buffer{}
	if err := t.Execute(msg, data); err != nil {
		return err
	}

	log.Printf("Sending e-mail: [%s] %s", ch.Title, item.Title)
	auth := smtp.PlainAuth("", fm.prof.SmtpUser, fm.prof.SmtpPass, fm.prof.SmtpHost)
	err = smtp.SendMail(fm.prof.SmtpAddr, auth, fm.prof.SrcEmail,
		fm.prof.DstEmails, msg.Bytes())
	if err != nil {
		return err
	}
	return nil
}

func (fm *FeedMailer) updateHistory() error {
	log.Println("Updating history file")
	buf, err := json.Marshal(fm.history)
	if err != nil {
		return err
	}
	if err := ioutil.WriteFile(fm.prof.HistFile, buf, 0600); err != nil {
		return err
	}
	return nil
}

const mailTmpl = `Subject: {{.SubjectPrefix}} [{{.ChanTitle}}] {{.ItemTitle}}
MIME-version: 1.0;
Content-Type: text/html; charset="UTF-8";

Title: {{.ItemTitle}}

{{if not .Date.IsZero}}Date: {{.Date.Format "2 January 2006 15:04"}}
{{end}}
{{if .Links}}
Links:

{{range .Links}}
  - {{.Href}}

{{end}}
{{end}}
{{if .Description}}Description:
{{.Description}}
{{end}}
{{if .Content}}Content:
{{.Content.Text}}{{end}}`
<commit_msg>Use channels instead of mutexes<commit_after>package feedmailer

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"log"
	"net/smtp"
	"os"
	"text/template"
	"time"

	rss "github.com/jteeuwen/go-pkg-rss"
)

type profile struct {
	SmtpAddr      string
	SmtpUser      string
	SmtpPass      string
	SmtpHost      string
	SrcEmail      string
	DstEmails     []string
	SubjectPrefix string
	FetchTimeout  int
	HistFile      string
	Feeds         []string
}

type histItem struct {
	url        string
	lastUpdate time.Time
}

type FeedMailer struct {
	ErrChan  chan error
	histChan chan histItem
	prof     profile
	history  map[string]time.Time
}

func NewFeedMailer() *FeedMailer {
	fm := &FeedMailer{}
	fm.ErrChan = make(chan error)
	fm.histChan = make(chan histItem)
	fm.history = make(map[string]time.Time)
	return fm
}

func (fm *FeedMailer) Start(file string) error {
	log.Println("Reading profile")
	f, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}
	if err := json.Unmarshal(f, &fm.prof); err != nil {
		return err
	}

	log.Println("Reading history")
	f, err = ioutil.ReadFile(fm.prof.HistFile)
	if err == nil {
		json.Unmarshal(f, &fm.history)
	} else if os.IsNotExist(err) {
		log.Printf("History file (%s) not found, it will be created",
			fm.prof.HistFile)
	} else {
		return err
	}

	go fm.updateHistory()

	log.Println("Fetching feeds")
	for _, url := range fm.prof.Feeds {
		go fm.fetch(url)
	}

	return nil
}

func (fm *FeedMailer) fetch(url string) {
	feed := rss.New(fm.prof.FetchTimeout, true, fm.chanHandler, fm.itemHandler)
	for {
		if err := feed.Fetch(url, nil); err != nil {
			fm.ErrChan <- err
		}
		<-time.After(time.Duration(feed.SecondsTillUpdate()) * time.Second)
	}
}

func (fm *FeedMailer) chanHandler(feed *rss.Feed, newChannels []*rss.Channel) {
	log.Printf("%d new channel(s) in %s\n", len(newChannels), feed.Url)
}

func (fm *FeedMailer) itemHandler(feed *rss.Feed, ch *rss.Channel, newItems []*rss.Item) {
	log.Printf("%d new item(s) in %s\n", len(newItems), feed.Url)

	var lastUpdate time.Time
	for _, item := range newItems {
		itemDate, err := item.ParsedPubDate()
		if err != nil {
			fm.ErrChan <- err
		}
		if fm.history[feed.Url].IsZero() || itemDate.After(fm.history[feed.Url]) {
			if err := fm.mail(ch, item); err != nil {
				fm.ErrChan <- err
			}
			if itemDate.After(lastUpdate) {
				lastUpdate = itemDate
			}
		}
	}

	if !lastUpdate.IsZero() {
		fm.histChan <- histItem{feed.Url, lastUpdate}
	}
}

func (fm *FeedMailer) mail(ch *rss.Channel, item *rss.Item) error {
	date, _ := item.ParsedPubDate()
	data := struct {
		SubjectPrefix, ChanTitle, ItemTitle string
		Date                                time.Time
		Links                               []*rss.Link
		Description                         string
		Content                             *rss.Content
	}{fm.prof.SubjectPrefix, ch.Title, item.Title, date,
		item.Links, item.Description, item.Content}

	t, err := template.New("mail").Parse(mailTmpl)
	if err != nil {
		return err
	}
	msg := &bytes.Buffer{}
	if err := t.Execute(msg, data); err != nil {
		return err
	}

	log.Printf("Sending e-mail: [%s] %s", ch.Title, item.Title)
	auth := smtp.PlainAuth("", fm.prof.SmtpUser, fm.prof.SmtpPass, fm.prof.SmtpHost)
	err = smtp.SendMail(fm.prof.SmtpAddr, auth, fm.prof.SrcEmail,
		fm.prof.DstEmails, msg.Bytes())
	if err != nil {
		return err
	}
	return nil
}

func (fm *FeedMailer) updateHistory() {
	for {
		hist := <-fm.histChan
		log.Println("Updating history file")
		fm.history[hist.url] = hist.lastUpdate
		buf, err := json.Marshal(fm.history)
		if err != nil {
			fm.ErrChan <- err
		}
		if err := ioutil.WriteFile(fm.prof.HistFile, buf, 0600); err != nil {
			fm.ErrChan <- err
		}
	}
}

const mailTmpl = `Subject: {{.SubjectPrefix}} [{{.ChanTitle}}] {{.ItemTitle}}
MIME-version: 1.0;
Content-Type: text/html; charset="UTF-8";

Title: {{.ItemTitle}}

{{if not .Date.IsZero}}Date: {{.Date.Format "2 January 2006 15:04"}}
{{end}}
{{if .Links}}
Links:

{{range .Links}}
  - {{.Href}}

{{end}}
{{end}}
{{if .Description}}Description:
{{.Description}}
{{end}}
{{if .Content}}Content:
{{.Content.Text}}{{end}}`
<|endoftext|>


<commit_before>package main

import "fmt"

func MaxSum(a []int) int{
	sum := 0
	b := 0
	for i :=0; i<len(a); i++{
		if a[i] > b+a[i] {
			b = a[i]
		}else{
			b = b+a[i]
		}
		if b > sum {
			sum = b
		}
	}
	fmt.Println(sum)
	return sum
}

func main() {
	temp :=[]int{ 1,34,-20, -25,12,30,-23,45,12}
	MaxSum(temp)
}
<commit_msg>fix bug<commit_after>package main

import "fmt"

func MaxSum(a []int) int{
	if len(a) == 1 {
		return a[0]
	}
	sum := a[0]
	b := a[0]
	for i :=1; i<len(a); i++{
		if a[i] > b+a[i] {
			b = a[i]
		}else{
			b = b+a[i]
		}
		if b > sum {
			sum = b
		}
	}
	fmt.Println(sum)
	return sum
}

func main() {
	temp :=[]int{ 1,34,-20, -25,12,30,-23,45,12}
	MaxSum(temp)
}
<|endoftext|>


<commit_before>package ethui

import (
	"fmt"
	"github.com/ethereum/eth-go/ethchain"
	"github.com/ethereum/eth-go/ethutil"
	"github.com/go-qml/qml"
	"strings"
)

type DebuggerWindow struct {
	win    *qml.Window
	engine *qml.Engine
	lib    *UiLib
	Db     *Debugger
}

func NewDebuggerWindow(lib *UiLib) *DebuggerWindow {
	engine := qml.NewEngine()
	component, err := engine.LoadFile(lib.AssetPath("debugger/debugger.qml"))
	if err != nil {
		fmt.Println(err)

		return nil
	}

	win := component.CreateWindow(nil)
	db := &Debugger{win, make(chan bool), make(chan bool), true}

	return &DebuggerWindow{engine: engine, win: win, lib: lib, Db: db}
}

func (self *DebuggerWindow) Show() {
	context := self.engine.Context()
	context.SetVar("dbg", self)

	go func() {
		self.win.Show()
		self.win.Wait()
	}()
}

func (self *DebuggerWindow) SetCode(code string) {
	self.win.Set("codeText", code)
}

func (self *DebuggerWindow) SetData(data string) {
	self.win.Set("dataText", data)
}
func (self *DebuggerWindow) SetAsm(data string) {
	dis := ethchain.Disassemble(ethutil.FromHex(data))
	for _, str := range dis {
		self.win.Root().Call("setAsm", str)
	}
}

func (self *DebuggerWindow) Debug(valueStr, gasStr, gasPriceStr, scriptStr, dataStr string) {
	if !self.Db.done {
		self.Db.Q <- true
	}

	data := ethutil.StringToByteFunc(dataStr, func(s string) (ret []byte) {
		slice := strings.Split(dataStr, "\n")
		for _, dataItem := range slice {
			d := ethutil.FormatData(dataItem)
			ret = append(ret, d...)
		}
		return
	})

	var err error
	script := ethutil.StringToByteFunc(scriptStr, func(s string) (ret []byte) {
		ret, err = ethutil.Compile(s)
		fmt.Printf("%x\n", ret)
		return
	})

	if err != nil {
		self.Logln(err)

		return
	}

	dis := ethchain.Disassemble(script)
	self.win.Root().Call("clearAsm")
	self.win.Root().Call("clearLog")

	for _, str := range dis {
		self.win.Root().Call("setAsm", str)
	}

	// Contract addr as test address
	keyPair := ethutil.GetKeyRing().Get(0)
	callerTx := ethchain.NewContractCreationTx(ethutil.Big(valueStr), ethutil.Big(gasStr), ethutil.Big(gasPriceStr), script)
	callerTx.Sign(keyPair.PrivateKey)

	state := self.lib.eth.BlockChain().CurrentBlock.State()
	account := self.lib.eth.StateManager().TransState().GetAccount(keyPair.Address())
	contract := ethchain.MakeContract(callerTx, state)
	callerClosure := ethchain.NewClosure(account, contract, script, state, ethutil.Big(gasStr), ethutil.Big(gasPriceStr))

	block := self.lib.eth.BlockChain().CurrentBlock
	vm := ethchain.NewVm(state, self.lib.eth.StateManager(), ethchain.RuntimeVars{
		Origin:      account.Address(),
		BlockNumber: block.BlockInfo().Number,
		PrevHash:    block.PrevHash,
		Coinbase:    block.Coinbase,
		Time:        block.Time,
		Diff:        block.Difficulty,
		Value:       ethutil.Big(valueStr),
	})

	self.Db.done = false
	go func() {
		ret, _, err := callerClosure.Call(vm, data, self.Db.halting)
		if err != nil {
			self.Logln("exited with errors:", err)
		} else {
			self.Logf("exited: %v", ret)
		}

		state.Reset()

		self.Db.done = true
	}()
}

func (self *DebuggerWindow) Logf(format string, v ...interface{}) {
	self.win.Root().Call("setLog", fmt.Sprintf(format, v...))
}

func (self *DebuggerWindow) Logln(v ...interface{}) {
	str := fmt.Sprintln(v...)
	self.Logf("%s", str[:len(str)-1])
}

func (self *DebuggerWindow) Next() {
	self.Db.Next()
}

type Debugger struct {
	win  *qml.Window
	N    chan bool
	Q    chan bool
	done bool
}

type storeVal struct {
	Key, Value string
}

func (d *Debugger) halting(pc int, op ethchain.OpCode, mem *ethchain.Memory, stack *ethchain.Stack, stateObject *ethchain.StateObject) bool {
	d.win.Root().Call("setInstruction", pc)
	d.win.Root().Call("clearMem")
	d.win.Root().Call("clearStack")
	d.win.Root().Call("clearStorage")

	addr := 0
	for i := 0; i+32 <= mem.Len(); i += 32 {
		d.win.Root().Call("setMem", memAddr{fmt.Sprintf("%03d", addr), fmt.Sprintf("% x", mem.Data()[i:i+32])})
		addr++
	}

	for _, val := range stack.Data() {
		d.win.Root().Call("setStack", val.String())
	}

	stateObject.State().EachStorage(func(key string, node *ethutil.Value) {
		d.win.Root().Call("setStorage", storeVal{fmt.Sprintf("% x", key), fmt.Sprintf("% x", node.Str())})
	})

out:
	for {
		select {
		case <-d.N:
			break out
		case <-d.Q:
			d.done = true

			return false
		}
	}

	return true
}

func (d *Debugger) Next() {
	if !d.done {
		d.N <- true
	}
}
<commit_msg>Added more debugger output<commit_after>package ethui

import (
	"fmt"
	"github.com/ethereum/eth-go/ethchain"
	"github.com/ethereum/eth-go/ethutil"
	"github.com/go-qml/qml"
	"math/big"
	"strings"
)

type DebuggerWindow struct {
	win    *qml.Window
	engine *qml.Engine
	lib    *UiLib
	Db     *Debugger
}

func NewDebuggerWindow(lib *UiLib) *DebuggerWindow {
	engine := qml.NewEngine()
	component, err := engine.LoadFile(lib.AssetPath("debugger/debugger.qml"))
	if err != nil {
		fmt.Println(err)

		return nil
	}

	win := component.CreateWindow(nil)
	db := &Debugger{win, make(chan bool), make(chan bool), true}

	return &DebuggerWindow{engine: engine, win: win, lib: lib, Db: db}
}

func (self *DebuggerWindow) Show() {
	context := self.engine.Context()
	context.SetVar("dbg", self)

	go func() {
		self.win.Show()
		self.win.Wait()
	}()
}

func (self *DebuggerWindow) SetCode(code string) {
	self.win.Set("codeText", code)
}

func (self *DebuggerWindow) SetData(data string) {
	self.win.Set("dataText", data)
}
func (self *DebuggerWindow) SetAsm(data string) {
	dis := ethchain.Disassemble(ethutil.FromHex(data))
	for _, str := range dis {
		self.win.Root().Call("setAsm", str)
	}
}

func (self *DebuggerWindow) Debug(valueStr, gasStr, gasPriceStr, scriptStr, dataStr string) {
	if !self.Db.done {
		self.Db.Q <- true
	}

	data := ethutil.StringToByteFunc(dataStr, func(s string) (ret []byte) {
		slice := strings.Split(dataStr, "\n")
		for _, dataItem := range slice {
			d := ethutil.FormatData(dataItem)
			ret = append(ret, d...)
		}
		return
	})

	var err error
	script := ethutil.StringToByteFunc(scriptStr, func(s string) (ret []byte) {
		ret, err = ethutil.Compile(s)
		fmt.Printf("%x\n", ret)
		return
	})

	if err != nil {
		self.Logln(err)

		return
	}

	dis := ethchain.Disassemble(script)
	self.win.Root().Call("clearAsm")
	self.win.Root().Call("clearLog")

	for _, str := range dis {
		self.win.Root().Call("setAsm", str)
	}

	gas := ethutil.Big(gasStr)
	gasPrice := ethutil.Big(gasPriceStr)
	// Contract addr as test address
	keyPair := ethutil.GetKeyRing().Get(0)
	callerTx := ethchain.NewContractCreationTx(ethutil.Big(valueStr), gas, gasPrice, script)
	callerTx.Sign(keyPair.PrivateKey)

	state := self.lib.eth.BlockChain().CurrentBlock.State()
	account := self.lib.eth.StateManager().TransState().GetAccount(keyPair.Address())
	contract := ethchain.MakeContract(callerTx, state)
	callerClosure := ethchain.NewClosure(account, contract, script, state, gas, gasPrice)

	block := self.lib.eth.BlockChain().CurrentBlock
	vm := ethchain.NewVm(state, self.lib.eth.StateManager(), ethchain.RuntimeVars{
		Origin:      account.Address(),
		BlockNumber: block.BlockInfo().Number,
		PrevHash:    block.PrevHash,
		Coinbase:    block.Coinbase,
		Time:        block.Time,
		Diff:        block.Difficulty,
		Value:       ethutil.Big(valueStr),
	})

	self.Db.done = false
	self.Logf("callsize %d", len(script))
	go func() {
		ret, g, err := callerClosure.Call(vm, data, self.Db.halting)
		self.Logln("gas usage", g, "total price =", new(big.Int).Mul(g, gasPrice))
		if err != nil {
			self.Logln("exited with errors:", err)
		} else {
			if len(ret) > 0 {
				self.Logf("exited: % x", ret)
			} else {
				self.Logf("exited: nil")
			}
		}

		state.Reset()

		self.Db.done = true
	}()
}

func (self *DebuggerWindow) Logf(format string, v ...interface{}) {
	self.win.Root().Call("setLog", fmt.Sprintf(format, v...))
}

func (self *DebuggerWindow) Logln(v ...interface{}) {
	str := fmt.Sprintln(v...)
	self.Logf("%s", str[:len(str)-1])
}

func (self *DebuggerWindow) Next() {
	self.Db.Next()
}

type Debugger struct {
	win  *qml.Window
	N    chan bool
	Q    chan bool
	done bool
}

type storeVal struct {
	Key, Value string
}

func (d *Debugger) halting(pc int, op ethchain.OpCode, mem *ethchain.Memory, stack *ethchain.Stack, stateObject *ethchain.StateObject) bool {
	d.win.Root().Call("setInstruction", pc)
	d.win.Root().Call("clearMem")
	d.win.Root().Call("clearStack")
	d.win.Root().Call("clearStorage")

	addr := 0
	for i := 0; i+32 <= mem.Len(); i += 32 {
		d.win.Root().Call("setMem", memAddr{fmt.Sprintf("%03d", addr), fmt.Sprintf("% x", mem.Data()[i:i+32])})
		addr++
	}

	for _, val := range stack.Data() {
		d.win.Root().Call("setStack", val.String())
	}

	stateObject.State().EachStorage(func(key string, node *ethutil.Value) {
		d.win.Root().Call("setStorage", storeVal{fmt.Sprintf("% x", key), fmt.Sprintf("% x", node.Str())})
	})

out:
	for {
		select {
		case <-d.N:
			break out
		case <-d.Q:
			d.done = true

			return false
		}
	}

	return true
}

func (d *Debugger) Next() {
	if !d.done {
		d.N <- true
	}
}
<|endoftext|>


<commit_before>package events

import (
	"bytes"
	"fmt"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/allegro/marathon-consul/apps"
	"github.com/allegro/marathon-consul/marathon"
	"github.com/allegro/marathon-consul/metrics"
	"github.com/allegro/marathon-consul/service"
)

type Event struct {
	Timestamp time.Time
	EventType string
	Body      []byte
}

type eventHandler struct {
	id              int
	serviceRegistry service.ServiceRegistry
	marathon        marathon.Marathoner
	eventQueue      <-chan Event
}

type StopEvent struct{}

const (
	StatusUpdateEventType        = "status_update_event"
	HealthStatusChangedEventType = "health_status_changed_event"
)

func NewEventHandler(id int, serviceRegistry service.ServiceRegistry, marathon marathon.Marathoner, eventQueue <-chan Event) *eventHandler {
	return &eventHandler{
		id:              id,
		serviceRegistry: serviceRegistry,
		marathon:        marathon,
		eventQueue:      eventQueue,
	}
}

func (fh *eventHandler) Start() chan<- StopEvent {
	var e Event
	process := func() {
		err := fh.handleEvent(e.EventType, e.Body)
		if err != nil {
			metrics.Mark("events.processing.error")
		} else {
			metrics.Mark("events.processing.succes")
		}
	}

	quitChan := make(chan StopEvent)
	log.WithField("Id", fh.id).Println("Starting worker")
	go func() {
		for {
			select {
			case e = <-fh.eventQueue:
				metrics.Mark(fmt.Sprintf("events.handler.%d", fh.id))

				queueLength := int64(len(fh.eventQueue))
				metrics.UpdateGauge("events.queue.len", queueLength)
				queueCapacity := int64(cap(fh.eventQueue))

				utilization := int64(0)
				if queueCapacity > 0 {
					utilization = 100 * (queueLength / queueCapacity)
				}
				metrics.UpdateGauge("events.queue.util", utilization)

				metrics.UpdateGauge("events.queue.delay_ns", time.Since(e.Timestamp).Nanoseconds())
				metrics.Time("events.processing."+e.EventType, process)
			case <-quitChan:
				log.WithField("Id", fh.id).Info("Stopping worker")
			}
		}
	}()
	return quitChan
}

func (fh *eventHandler) handleEvent(eventType string, body []byte) error {

	body = replaceTaskIDWithID(body)

	switch eventType {
	case StatusUpdateEventType:
		return fh.handleStatusEvent(body)
	case HealthStatusChangedEventType:
		return fh.handleHealthyTask(body)
	default:
		err := fmt.Errorf("Unsuported event type: %s", eventType)
		log.WithError(err).WithField("EventType", eventType).Error("This should never happen. Not handled event type")
		return err
	}
}

func (fh *eventHandler) handleHealthyTask(body []byte) error {
	taskHealthChange, err := ParseTaskHealthChange(body)
	if err != nil {
		log.WithError(err).Error("Body generated error")
		return err
	}
	delay := taskHealthChange.Timestamp.Delay()
	metrics.UpdateGauge("events.read.delay.current", int64(delay))

	appID := taskHealthChange.AppID
	taskID := taskHealthChange.TaskID()
	log.WithField("Id", taskID).Info("Got HealthStatusEvent")

	if !taskHealthChange.Alive {
		log.WithField("Id", taskID).Debug("Task is not alive. Not registering")
		return nil
	}

	app, err := fh.marathon.App(appID)
	if err != nil {
		log.WithField("Id", taskID).WithError(err).Error("There was a problem obtaining app info")
		return err
	}

	if !app.IsConsulApp() {
		err = fmt.Errorf("%s is not consul app. Missing consul label", app.ID)
		log.WithField("Id", taskID).WithError(err).Debug("Skipping app registration in Consul")
		return nil
	}

	tasks := app.Tasks

	task, found := apps.FindTaskByID(taskID, tasks)
	if !found {
		log.WithField("Id", taskID).Error("Task not found")
		return err
	}

	if task.IsHealthy() {
		err := fh.serviceRegistry.Register(&task, app)
		if err != nil {
			log.WithField("Id", task.ID).WithError(err).Error("There was a problem registering task")
			return err
		}
		return nil
	}
	log.WithField("Id", task.ID).Debug("Task is not healthy. Not registering")
	return nil
}

func (fh *eventHandler) handleStatusEvent(body []byte) error {
	task, err := apps.ParseTask(body)
	if err != nil {
		log.WithError(err).WithField("Body", body).Error("Could not parse event body")
		return err
	}
	delay := task.Timestamp.Delay()
	metrics.UpdateGauge("events.read.delay.current", int64(delay))

	log.WithFields(log.Fields{
		"Id":         task.ID,
		"TaskStatus": task.TaskStatus,
	}).Info("Got StatusEvent")

	switch task.TaskStatus {
	case "TASK_FINISHED", "TASK_FAILED", "TASK_KILLING", "TASK_KILLED", "TASK_LOST":
		return fh.deregister(task.ID)
	default:
		log.WithFields(log.Fields{
			"Id":         task.ID,
			"taskStatus": task.TaskStatus,
		}).Debug("Not handled task status")
		return nil
	}
}

func (fh *eventHandler) deregister(taskID apps.TaskID) error {
	err := fh.serviceRegistry.DeregisterByTask(taskID)
	if err != nil {
		log.WithField("Id", taskID).WithError(err).Error("There was a problem deregistering task")
	}
	return err
}

// for every other use of Tasks, Marathon uses the "id" field for the task ID.
// Here, it uses "taskId", with most of the other fields being equal. We'll
// just swap "taskId" for "id" in the body so that we can successfully parse
// incoming events.
func replaceTaskIDWithID(body []byte) []byte {
	return bytes.Replace(body, []byte("taskId"), []byte("id"), -1)
}
<commit_msg>Handle empty event type (#241)<commit_after>package events

import (
	"bytes"
	"errors"
	"fmt"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/allegro/marathon-consul/apps"
	"github.com/allegro/marathon-consul/marathon"
	"github.com/allegro/marathon-consul/metrics"
	"github.com/allegro/marathon-consul/service"
)

type Event struct {
	Timestamp time.Time
	EventType string
	Body      []byte
}

type eventHandler struct {
	id              int
	serviceRegistry service.ServiceRegistry
	marathon        marathon.Marathoner
	eventQueue      <-chan Event
}

type StopEvent struct{}

const (
	StatusUpdateEventType        = "status_update_event"
	HealthStatusChangedEventType = "health_status_changed_event"
	EmptyEventType               = ""
)

func NewEventHandler(id int, serviceRegistry service.ServiceRegistry, marathon marathon.Marathoner, eventQueue <-chan Event) *eventHandler {
	return &eventHandler{
		id:              id,
		serviceRegistry: serviceRegistry,
		marathon:        marathon,
		eventQueue:      eventQueue,
	}
}

func (fh *eventHandler) Start() chan<- StopEvent {
	var e Event
	process := func() {
		err := fh.handleEvent(e.EventType, e.Body)
		if err != nil {
			metrics.Mark("events.processing.error")
		} else {
			metrics.Mark("events.processing.succes")
		}
	}

	quitChan := make(chan StopEvent)
	log.WithField("Id", fh.id).Println("Starting worker")
	go func() {
		for {
			select {
			case e = <-fh.eventQueue:
				metrics.Mark(fmt.Sprintf("events.handler.%d", fh.id))

				queueLength := int64(len(fh.eventQueue))
				metrics.UpdateGauge("events.queue.len", queueLength)
				queueCapacity := int64(cap(fh.eventQueue))

				utilization := int64(0)
				if queueCapacity > 0 {
					utilization = 100 * (queueLength / queueCapacity)
				}
				metrics.UpdateGauge("events.queue.util", utilization)

				metrics.UpdateGauge("events.queue.delay_ns", time.Since(e.Timestamp).Nanoseconds())
				metrics.Time("events.processing."+e.EventType, process)
			case <-quitChan:
				log.WithField("Id", fh.id).Info("Stopping worker")
			}
		}
	}()
	return quitChan
}

func (fh *eventHandler) handleEvent(eventType string, body []byte) error {

	body = replaceTaskIDWithID(body)

	switch eventType {
	case StatusUpdateEventType:
		return fh.handleStatusEvent(body)
	case HealthStatusChangedEventType:
		return fh.handleHealthyTask(body)
	case EmptyEventType:
		err := errors.New("Empty event type")
		log.WithError(err).Warn("Event type is empty. " +
			"This means event was not properly serialized. " +
			"This can ocure when connection with Marathon breaks " +
			"due to network error or Marathon restarts.")
		return err
	default:
		err := fmt.Errorf("Unsuported event type: %s", eventType)
		log.WithError(err).WithField("EventType", eventType).Error("This should never happen. Not handled event type")
		return err
	}
}

func (fh *eventHandler) handleHealthyTask(body []byte) error {
	taskHealthChange, err := ParseTaskHealthChange(body)
	if err != nil {
		log.WithError(err).Error("Body generated error")
		return err
	}
	delay := taskHealthChange.Timestamp.Delay()
	metrics.UpdateGauge("events.read.delay.current", int64(delay))

	appID := taskHealthChange.AppID
	taskID := taskHealthChange.TaskID()
	log.WithField("Id", taskID).Info("Got HealthStatusEvent")

	if !taskHealthChange.Alive {
		log.WithField("Id", taskID).Debug("Task is not alive. Not registering")
		return nil
	}

	app, err := fh.marathon.App(appID)
	if err != nil {
		log.WithField("Id", taskID).WithError(err).Error("There was a problem obtaining app info")
		return err
	}

	if !app.IsConsulApp() {
		err = fmt.Errorf("%s is not consul app. Missing consul label", app.ID)
		log.WithField("Id", taskID).WithError(err).Debug("Skipping app registration in Consul")
		return nil
	}

	tasks := app.Tasks

	task, found := apps.FindTaskByID(taskID, tasks)
	if !found {
		log.WithField("Id", taskID).Error("Task not found")
		return err
	}

	if task.IsHealthy() {
		err := fh.serviceRegistry.Register(&task, app)
		if err != nil {
			log.WithField("Id", task.ID).WithError(err).Error("There was a problem registering task")
			return err
		}
		return nil
	}
	log.WithField("Id", task.ID).Debug("Task is not healthy. Not registering")
	return nil
}

func (fh *eventHandler) handleStatusEvent(body []byte) error {
	task, err := apps.ParseTask(body)
	if err != nil {
		log.WithError(err).WithField("Body", body).Error("Could not parse event body")
		return err
	}
	delay := task.Timestamp.Delay()
	metrics.UpdateGauge("events.read.delay.current", int64(delay))

	log.WithFields(log.Fields{
		"Id":         task.ID,
		"TaskStatus": task.TaskStatus,
	}).Info("Got StatusEvent")

	switch task.TaskStatus {
	case "TASK_FINISHED", "TASK_FAILED", "TASK_KILLING", "TASK_KILLED", "TASK_LOST":
		return fh.deregister(task.ID)
	default:
		log.WithFields(log.Fields{
			"Id":         task.ID,
			"taskStatus": task.TaskStatus,
		}).Debug("Not handled task status")
		return nil
	}
}

func (fh *eventHandler) deregister(taskID apps.TaskID) error {
	err := fh.serviceRegistry.DeregisterByTask(taskID)
	if err != nil {
		log.WithField("Id", taskID).WithError(err).Error("There was a problem deregistering task")
	}
	return err
}

// for every other use of Tasks, Marathon uses the "id" field for the task ID.
// Here, it uses "taskId", with most of the other fields being equal. We'll
// just swap "taskId" for "id" in the body so that we can successfully parse
// incoming events.
func replaceTaskIDWithID(body []byte) []byte {
	return bytes.Replace(body, []byte("taskId"), []byte("id"), -1)
}
<|endoftext|>


<commit_before>/*
Package measured wraps a dialer to measure the delay, throughput and errors of the connection made.
A list of reporters can be plugged in to distribute the results to different target.
*/
package measured

import (
	"net"
	"strings"
	"sync/atomic"

	"github.com/getlantern/golog"
)

// Stats encapsulates the statistics to report
type Stats struct {
	Type   string
	Tags   map[string]string
	Fields map[string]interface{}
}

// Reporter encapsulates different ways to report statistics
type Reporter interface {
	Submit(*Stats) error
}

var (
	reporters   []Reporter
	defaultTags atomic.Value
	running     uint32
	log         = golog.LoggerFor("measured")
	chStats     = make(chan *Stats)
	chStop      = make(chan interface{})
)

func init() {
	defaultTags.Store(map[string]string{})
}

// DialFunc is the type of function measured can wrap
type DialFunc func(net, addr string) (net.Conn, error)

// Reset resets the measured package
func Reset() {
	reporters = []Reporter{}
}

// AddReporter add a new way to report statistics
func AddReporter(r Reporter) {
	reporters = append(reporters, r)
}

// SetDefaults set a few default tags sending every time
func SetDefaults(defaults map[string]string) {
	defaultTags.Store(defaults)
}

// Start runs the measured loop
func Start() {
	go run()
}

// Stop stops the measured loop
func Stop() {
	if atomic.LoadUint32(&running) == 0 {
		return
	}
	log.Debug("Stopping measured loop...")
	select {
	case chStop <- nil:
	default:
		log.Error("Failed to send stop signal")
	}
}

// Dialer wraps a dial function to measure various statistics
func Dialer(d DialFunc, via string) DialFunc {
	return func(net, addr string) (net.Conn, error) {
		c, err := d(net, addr)
		if err != nil {
			reportError(via, err, "dial")
		}
		return measuredConn{c, via}, err
	}
}

func run() {
	log.Debug("Measured loop started")
	atomic.StoreUint32(&running, 1)
	for {
		select {
		case s := <-chStats:
			defaults := defaultTags.Load().(map[string]string)
			for _, r := range reporters {
				for k, v := range defaults {
					s.Tags[k] = v
				}
				if err := r.Submit(s); err != nil {
					log.Errorf("Failed to report error to influxdb: %s", err)
				} else {
					log.Tracef("Submitted error to influxdb: %v", s)
				}
			}
		case <-chStop:
			log.Debug("Measured loop stopped")
			atomic.StoreUint32(&running, 0)
			return
		}
	}
}

func reportError(addr string, err error, phase string) {
	splitted := strings.Split(err.Error(), ":")
	lastIndex := len(splitted) - 1
	if lastIndex < 0 {
		lastIndex = 0
	}
	e := strings.Trim(splitted[lastIndex], " ")
	select {
	case chStats <- &Stats{
		Type: "errors",
		Tags: map[string]string{
			"server": addr,
			"error":  e,
			"phase":  phase,
		},
		Fields: map[string]interface{}{"value": 1},
	}:
	default:
	}
}

type measuredConn struct {
	net.Conn
	addr string
}

// Read() implements the function from net.Conn
func (mc measuredConn) Read(b []byte) (n int, err error) {
	n, err = mc.Conn.Read(b)
	if err != nil {
		reportError(mc.addr, err, "read")
	}
	return
}

// Write() implements the function from net.Conn
func (mc measuredConn) Write(b []byte) (n int, err error) {
	n, err = mc.Conn.Write(b)
	if err != nil {
		reportError(mc.addr, err, "write")
	}
	return
}

// Close() implements the function from net.Conn
func (mc measuredConn) Close() (err error) {
	err = mc.Conn.Close()
	if err != nil {
		reportError(mc.addr, err, "close")
	}
	return
}
<commit_msg>add error log in reportError<commit_after>/*
Package measured wraps a dialer to measure the delay, throughput and errors of the connection made.
A list of reporters can be plugged in to distribute the results to different target.
*/
package measured

import (
	"net"
	"strings"
	"sync/atomic"

	"github.com/getlantern/golog"
)

// Stats encapsulates the statistics to report
type Stats struct {
	Type   string
	Tags   map[string]string
	Fields map[string]interface{}
}

// Reporter encapsulates different ways to report statistics
type Reporter interface {
	Submit(*Stats) error
}

var (
	reporters   []Reporter
	defaultTags atomic.Value
	running     uint32
	log         = golog.LoggerFor("measured")
	chStats     = make(chan *Stats)
	chStop      = make(chan interface{})
)

func init() {
	defaultTags.Store(map[string]string{})
}

// DialFunc is the type of function measured can wrap
type DialFunc func(net, addr string) (net.Conn, error)

// Reset resets the measured package
func Reset() {
	reporters = []Reporter{}
}

// AddReporter add a new way to report statistics
func AddReporter(r Reporter) {
	reporters = append(reporters, r)
}

// SetDefaults set a few default tags sending every time
func SetDefaults(defaults map[string]string) {
	defaultTags.Store(defaults)
}

// Start runs the measured loop
func Start() {
	go run()
}

// Stop stops the measured loop
func Stop() {
	if atomic.LoadUint32(&running) == 0 {
		return
	}
	log.Debug("Stopping measured loop...")
	select {
	case chStop <- nil:
	default:
		log.Error("Failed to send stop signal")
	}
}

// Dialer wraps a dial function to measure various statistics
func Dialer(d DialFunc, via string) DialFunc {
	return func(net, addr string) (net.Conn, error) {
		c, err := d(net, addr)
		if err != nil {
			reportError(via, err, "dial")
		}
		return measuredConn{c, via}, err
	}
}

func run() {
	log.Debug("Measured loop started")
	atomic.StoreUint32(&running, 1)
	for {
		select {
		case s := <-chStats:
			defaults := defaultTags.Load().(map[string]string)
			for _, r := range reporters {
				for k, v := range defaults {
					s.Tags[k] = v
				}
				if err := r.Submit(s); err != nil {
					log.Errorf("Failed to report error to influxdb: %s", err)
				} else {
					log.Tracef("Submitted error to influxdb: %v", s)
				}
			}
		case <-chStop:
			log.Debug("Measured loop stopped")
			atomic.StoreUint32(&running, 0)
			return
		}
	}
}

func reportError(addr string, err error, phase string) {
	splitted := strings.Split(err.Error(), ":")
	lastIndex := len(splitted) - 1
	if lastIndex < 0 {
		lastIndex = 0
	}
	e := strings.Trim(splitted[lastIndex], " ")
	select {
	case chStats <- &Stats{
		Type: "errors",
		Tags: map[string]string{
			"server": addr,
			"error":  e,
			"phase":  phase,
		},
		Fields: map[string]interface{}{"value": 1},
	}:
	default:
		log.Error("Failed to send stats to reporters")
	}
}

type measuredConn struct {
	net.Conn
	addr string
}

// Read() implements the function from net.Conn
func (mc measuredConn) Read(b []byte) (n int, err error) {
	n, err = mc.Conn.Read(b)
	if err != nil {
		reportError(mc.addr, err, "read")
	}
	return
}

// Write() implements the function from net.Conn
func (mc measuredConn) Write(b []byte) (n int, err error) {
	n, err = mc.Conn.Write(b)
	if err != nil {
		reportError(mc.addr, err, "write")
	}
	return
}

// Close() implements the function from net.Conn
func (mc measuredConn) Close() (err error) {
	err = mc.Conn.Close()
	if err != nil {
		reportError(mc.addr, err, "close")
	}
	return
}
<|endoftext|>


<commit_before>package memalpha

import (
	"bytes"
	"fmt"
	"net"
	"os/exec"
	"reflect"
	"strconv"
	"testing"
	"time"
)

func freePort() (int, error) {
	l, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		return 0, err
	}
	defer l.Close()

	return l.Addr().(*net.TCPAddr).Port, nil
}

type server struct {
	cmd    *exec.Cmd
	client *Client
}

func newServer() *server {
	return &server{}
}

func (s *server) Start() error {
	port, err := freePort()
	if err != nil {
		return err
	}

	s.cmd = exec.Command("memcached", "-p", strconv.Itoa(port))
	if err = s.cmd.Start(); err != nil {
		return err
	}

	s.client = NewClient(fmt.Sprintf("localhost:%d", port))

	// Wait a bit for the socket to appear.
	for i := 0; i < 10; i++ {
		err = s.client.ensureConnected()
		if err == nil {
			return nil
		}
		time.Sleep(time.Duration(25*i) * time.Millisecond)
	}

	return err
}

func (s *server) Shutdown() error {
	_ = s.cmd.Process.Kill()
	return s.cmd.Wait()
}

func TestLocalhost(t *testing.T) {
	memd := newServer()
	err := memd.Start()
	if err != nil {
		t.Skipf("skipping test; couldn't start memcached: %s", err)
	}
	defer memd.Shutdown()

	c := memd.client

	// Set
	err = c.Set("foo", []byte("fooval"))
	if err != nil {
		t.Fatalf("first set(foo): %v", err)
	}
	err = c.Set("foo", []byte("fooval"))
	if err != nil {
		t.Fatalf("second set(foo): %v", err)
	}

	// Get
	value, _, err := c.Get("foo")
	if err != nil {
		t.Fatalf("get(foo): %v", err)
	}
	if !bytes.Equal(value, []byte("fooval")) {
		t.Fatalf("get(foo) Value = %q, want fooval", value)
	}

	// Set large item
	largeKey := string(bytes.Repeat([]byte("A"), 250))
	largeValue := bytes.Repeat([]byte("A"), 1023*1024)
	err = c.Set(largeKey, largeValue)
	if err != nil {
		t.Fatalf("set(largeKey): %v", err)
	}

	// Get large item
	value, _, err = c.Get(largeKey)
	if err != nil {
		t.Fatalf("get(largeKey): %v", err)
	}
	if !bytes.Equal(value, largeValue) {
		peekLen := len(value)
		if peekLen > 10 {
			peekLen = 10
		}
		t.Fatalf("get(largeKey) Value = %q, want fooval", value[:peekLen])
	}

	// Gets
	err = c.Set("bar", []byte("barval"))
	if err != nil {
		t.Fatalf("set(bar): %v", err)
	}
	m, err := c.Gets([]string{"foo", "bar"})
	if err != nil {
		t.Fatalf("gets(foo, bar): %v", err)
	}
	keyToValue := make(map[string]string)
	for key, response := range m {
		keyToValue[key] = string(response.Value)
	}
	expected := map[string]string{"foo": "fooval", "bar": "barval"}
	if !reflect.DeepEqual(keyToValue, expected) {
		t.Fatalf("gets(foo, bar) Value = %+v, want %+v", m, expected)
	}

	// Add
	err = c.Add("baz", []byte("baz1"))
	if err != nil {
		t.Fatalf("first add(baz): %v", err)
	}
	err = c.Add("baz", []byte("baz2"))
	if err != ErrNotStored {
		t.Fatalf("second add(baz) Error = ErrNotStored, want %+v", err)
	}

	// Replace
	err = c.Set("foo", []byte("fooval"))
	if err != nil {
		t.Fatalf("set(foo): %v", err)
	}
	err = c.Replace("foo", []byte("fooval2"))
	if err != nil {
		t.Fatalf("replace(foo): %v", err)
	}
	value, _, err = c.Get("foo")
	if err != nil {
		t.Fatalf("get(foo): %v", err)
	}
	if !bytes.Equal(value, []byte("fooval2")) {
		t.Fatalf("replace(foo, fooval2) then, get(foo) Value = %q, want fooval2", value)
	}

	// Append
	err = c.Append("foo", []byte("suffix"))
	if err != nil {
		t.Fatalf("append(foo, suffix): %v", err)
	}
	value, _, err = c.Get("foo")
	if err != nil {
		t.Fatalf("get(foo): %v", err)
	}
	if !bytes.Equal(value, []byte("fooval2suffix")) {
		t.Fatalf("append(foo, suffix) then, get(foo) Value = %q, want fooval2suffix", value)
	}

	// Prepend
	err = c.Prepend("foo", []byte("prefix"))
	if err != nil {
		t.Fatalf("prepend(foo, prefix): %v", err)
	}
	value, _, err = c.Get("foo")
	if err != nil {
		t.Fatalf("get(foo): %v", err)
	}
	if !bytes.Equal(value, []byte("prefixfooval2suffix")) {
		t.Fatalf("prepend(foo, prefix) then, get(foo) Value = %q, want prefixfooval2suffix", value)
	}

	// CompareAndSwap
	m, err = c.Gets([]string{"foo"})
	if err != nil {
		t.Fatalf("gets(foo): %v", err)
	}
	err = c.CompareAndSwap("foo", []byte("swapped"), m["foo"].CasID)
	if err != nil {
		t.Fatalf("cas(foo, swapped, casid): %v", err)
	}
	err = c.CompareAndSwap("foo", []byte("swapped"), m["foo"].CasID)
	if err != ErrCasConflict {
		t.Fatalf("cas(foo, swapped, casid) Error = %v, want %v", err, ErrCasConflict)
	}

	// Delete
	err = c.Delete("foo", false)
	if err != nil {
		t.Fatalf("delete(foo): %v", err)
	}
	_, _, err = c.Get("foo")
	if err != ErrCacheMiss {
		t.Fatalf("get(foo) Error = %q, want ErrCacheMiss", err)
	}

	// Increment
	err = c.Set("foo", []byte("35"))
	if err != nil {
		t.Fatalf("set(foo): %v", err)
	}
	num, err := c.Increment("foo", 7, false)
	if err != nil {
		t.Fatalf("incr(foo, 7): %v", err)
	}
	if num != 42 {
		t.Fatalf("incr(foo, 7) Value = %q, want 42", num)
	}

	// Decrement
	num, err = c.Decrement("foo", 10, false)
	if err != nil {
		t.Fatalf("decr(foo, 10): %v", err)
	}
	if num != 32 {
		t.Fatalf("decr(foo, 10) Value = %q, want 32", num)
	}
	value, _, err = c.Get("foo")
	if err != nil {
		t.Fatalf("get(foo): %v", err)
	}
	if !bytes.Equal(value, []byte("32")) {
		t.Fatalf("get(foo) Value = %q, want 32", value)
	}

	// Touch
	err = c.Touch("foo", 2, false)
	if err != nil {
		t.Fatalf("touch(foo, 2): %v", err)
	}
	value, _, err = c.Get("foo")
	if err != nil {
		t.Fatalf("get(foo): %v", err)
	}
	if !bytes.Equal(value, []byte("32")) {
		t.Fatalf("get(foo) Value = %q, want 32", value)
	}
	time.Sleep(2 * time.Second)
	_, _, err = c.Get("foo")
	if err != ErrCacheMiss {
		t.Fatalf("get(foo) Error = %q, want ErrCacheMiss", err)
	}

	// Touch raise ErrNotFound
	err = c.Touch("not_exists", 10, false)
	if err != ErrNotFound {
		t.Fatalf("get(not_exists) Error = %q, want ErrNotFound", err)
	}

	// Stats
	stats, err := c.Stats()
	if err != nil {
		t.Fatalf("stats(): %v", err)
	}
	if len(stats) < 2 {
		t.Fatalf("stats(): len(Value) = %q, want len(value) > 2", stats)
	}

	// StatsArg
	stats, err = c.StatsArg("slabs")
	if err != nil {
		t.Fatalf("stats(): %v", err)
	}
	if len(stats) < 2 {
		t.Fatalf("stats(): len(Value) = %q, want len(value) > 2", stats)
	}

	// FlushAll
	err = c.Set("foo", []byte("bar"))
	if err != nil {
		t.Fatalf("set(foo): %v", err)
	}
	err = c.FlushAll(0, false)
	if err != nil {
		t.Fatalf("version(): %v", err)
	}
	_, _, err = c.Get("foo")
	if err != ErrCacheMiss {
		t.Fatalf("get(foo) Error = %q, want ErrCacheMiss", err)
	}

	// Version
	ver, err := c.Version()
	if err != nil {
		t.Fatalf("version(): %v", err)
	}
	if len(ver) == 0 {
		t.Fatalf("version() Value = %q, want len(value) > 0", ver)
	}

	// Quit
	err = c.Quit()
	if err != nil {
		t.Fatalf("quit(): %v", err)
	}
	if c.conn == nil {
		t.Fatalf("net.Conn = %q, want nil", c.conn)
	}
}
<commit_msg>Test ErrNotFound of Delete<commit_after>package memalpha

import (
	"bytes"
	"fmt"
	"net"
	"os/exec"
	"reflect"
	"strconv"
	"testing"
	"time"
)

func freePort() (int, error) {
	l, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		return 0, err
	}
	defer l.Close()

	return l.Addr().(*net.TCPAddr).Port, nil
}

type server struct {
	cmd    *exec.Cmd
	client *Client
}

func newServer() *server {
	return &server{}
}

func (s *server) Start() error {
	port, err := freePort()
	if err != nil {
		return err
	}

	s.cmd = exec.Command("memcached", "-p", strconv.Itoa(port))
	if err = s.cmd.Start(); err != nil {
		return err
	}

	s.client = NewClient(fmt.Sprintf("localhost:%d", port))

	// Wait a bit for the socket to appear.
	for i := 0; i < 10; i++ {
		err = s.client.ensureConnected()
		if err == nil {
			return nil
		}
		time.Sleep(time.Duration(25*i) * time.Millisecond)
	}

	return err
}

func (s *server) Shutdown() error {
	_ = s.cmd.Process.Kill()
	return s.cmd.Wait()
}

func TestLocalhost(t *testing.T) {
	memd := newServer()
	err := memd.Start()
	if err != nil {
		t.Skipf("skipping test; couldn't start memcached: %s", err)
	}
	defer memd.Shutdown()

	c := memd.client

	// Set
	err = c.Set("foo", []byte("fooval"))
	if err != nil {
		t.Fatalf("first set(foo): %v", err)
	}
	err = c.Set("foo", []byte("fooval"))
	if err != nil {
		t.Fatalf("second set(foo): %v", err)
	}

	// Get
	value, _, err := c.Get("foo")
	if err != nil {
		t.Fatalf("get(foo): %v", err)
	}
	if !bytes.Equal(value, []byte("fooval")) {
		t.Fatalf("get(foo) Value = %q, want fooval", value)
	}

	// Set large item
	largeKey := string(bytes.Repeat([]byte("A"), 250))
	largeValue := bytes.Repeat([]byte("A"), 1023*1024)
	err = c.Set(largeKey, largeValue)
	if err != nil {
		t.Fatalf("set(largeKey): %v", err)
	}

	// Get large item
	value, _, err = c.Get(largeKey)
	if err != nil {
		t.Fatalf("get(largeKey): %v", err)
	}
	if !bytes.Equal(value, largeValue) {
		peekLen := len(value)
		if peekLen > 10 {
			peekLen = 10
		}
		t.Fatalf("get(largeKey) Value = %q, want fooval", value[:peekLen])
	}

	// Gets
	err = c.Set("bar", []byte("barval"))
	if err != nil {
		t.Fatalf("set(bar): %v", err)
	}
	m, err := c.Gets([]string{"foo", "bar"})
	if err != nil {
		t.Fatalf("gets(foo, bar): %v", err)
	}
	keyToValue := make(map[string]string)
	for key, response := range m {
		keyToValue[key] = string(response.Value)
	}
	expected := map[string]string{"foo": "fooval", "bar": "barval"}
	if !reflect.DeepEqual(keyToValue, expected) {
		t.Fatalf("gets(foo, bar) Value = %+v, want %+v", m, expected)
	}

	// Add
	err = c.Add("baz", []byte("baz1"))
	if err != nil {
		t.Fatalf("first add(baz): %v", err)
	}
	err = c.Add("baz", []byte("baz2"))
	if err != ErrNotStored {
		t.Fatalf("second add(baz) Error = ErrNotStored, want %+v", err)
	}

	// Replace
	err = c.Set("foo", []byte("fooval"))
	if err != nil {
		t.Fatalf("set(foo): %v", err)
	}
	err = c.Replace("foo", []byte("fooval2"))
	if err != nil {
		t.Fatalf("replace(foo): %v", err)
	}
	value, _, err = c.Get("foo")
	if err != nil {
		t.Fatalf("get(foo): %v", err)
	}
	if !bytes.Equal(value, []byte("fooval2")) {
		t.Fatalf("replace(foo, fooval2) then, get(foo) Value = %q, want fooval2", value)
	}

	// Append
	err = c.Append("foo", []byte("suffix"))
	if err != nil {
		t.Fatalf("append(foo, suffix): %v", err)
	}
	value, _, err = c.Get("foo")
	if err != nil {
		t.Fatalf("get(foo): %v", err)
	}
	if !bytes.Equal(value, []byte("fooval2suffix")) {
		t.Fatalf("append(foo, suffix) then, get(foo) Value = %q, want fooval2suffix", value)
	}

	// Prepend
	err = c.Prepend("foo", []byte("prefix"))
	if err != nil {
		t.Fatalf("prepend(foo, prefix): %v", err)
	}
	value, _, err = c.Get("foo")
	if err != nil {
		t.Fatalf("get(foo): %v", err)
	}
	if !bytes.Equal(value, []byte("prefixfooval2suffix")) {
		t.Fatalf("prepend(foo, prefix) then, get(foo) Value = %q, want prefixfooval2suffix", value)
	}

	// CompareAndSwap
	m, err = c.Gets([]string{"foo"})
	if err != nil {
		t.Fatalf("gets(foo): %v", err)
	}
	err = c.CompareAndSwap("foo", []byte("swapped"), m["foo"].CasID)
	if err != nil {
		t.Fatalf("cas(foo, swapped, casid): %v", err)
	}
	err = c.CompareAndSwap("foo", []byte("swapped"), m["foo"].CasID)
	if err != ErrCasConflict {
		t.Fatalf("cas(foo, swapped, casid) Error = %v, want %v", err, ErrCasConflict)
	}

	// Delete
	err = c.Delete("foo", false)
	if err != nil {
		t.Fatalf("delete(foo): %v", err)
	}
	_, _, err = c.Get("foo")
	if err != ErrCacheMiss {
		t.Fatalf("get(foo) Error = %q, want ErrCacheMiss", err)
	}

	// Touch raise ErrNotFound
	err = c.Touch("not_exists", 10, false)
	if err != ErrNotFound {
		t.Fatalf("get(not_exists) Error = %q, want ErrNotFound", err)
	}

	// Increment
	err = c.Set("foo", []byte("35"))
	if err != nil {
		t.Fatalf("set(foo): %v", err)
	}
	num, err := c.Increment("foo", 7, false)
	if err != nil {
		t.Fatalf("incr(foo, 7): %v", err)
	}
	if num != 42 {
		t.Fatalf("incr(foo, 7) Value = %q, want 42", num)
	}

	// Decrement
	num, err = c.Decrement("foo", 10, false)
	if err != nil {
		t.Fatalf("decr(foo, 10): %v", err)
	}
	if num != 32 {
		t.Fatalf("decr(foo, 10) Value = %q, want 32", num)
	}
	value, _, err = c.Get("foo")
	if err != nil {
		t.Fatalf("get(foo): %v", err)
	}
	if !bytes.Equal(value, []byte("32")) {
		t.Fatalf("get(foo) Value = %q, want 32", value)
	}

	// Touch
	err = c.Touch("foo", 2, false)
	if err != nil {
		t.Fatalf("touch(foo, 2): %v", err)
	}
	value, _, err = c.Get("foo")
	if err != nil {
		t.Fatalf("get(foo): %v", err)
	}
	if !bytes.Equal(value, []byte("32")) {
		t.Fatalf("get(foo) Value = %q, want 32", value)
	}
	time.Sleep(2 * time.Second)
	_, _, err = c.Get("foo")
	if err != ErrCacheMiss {
		t.Fatalf("get(foo) Error = %q, want ErrCacheMiss", err)
	}

	// Touch raise ErrNotFound
	err = c.Touch("not_exists", 10, false)
	if err != ErrNotFound {
		t.Fatalf("get(not_exists) Error = %q, want ErrNotFound", err)
	}

	// Stats
	stats, err := c.Stats()
	if err != nil {
		t.Fatalf("stats(): %v", err)
	}
	if len(stats) < 2 {
		t.Fatalf("stats(): len(Value) = %q, want len(value) > 2", stats)
	}

	// StatsArg
	stats, err = c.StatsArg("slabs")
	if err != nil {
		t.Fatalf("stats(): %v", err)
	}
	if len(stats) < 2 {
		t.Fatalf("stats(): len(Value) = %q, want len(value) > 2", stats)
	}

	// FlushAll
	err = c.Set("foo", []byte("bar"))
	if err != nil {
		t.Fatalf("set(foo): %v", err)
	}
	err = c.FlushAll(0, false)
	if err != nil {
		t.Fatalf("version(): %v", err)
	}
	_, _, err = c.Get("foo")
	if err != ErrCacheMiss {
		t.Fatalf("get(foo) Error = %q, want ErrCacheMiss", err)
	}

	// Version
	ver, err := c.Version()
	if err != nil {
		t.Fatalf("version(): %v", err)
	}
	if len(ver) == 0 {
		t.Fatalf("version() Value = %q, want len(value) > 0", ver)
	}

	// Quit
	err = c.Quit()
	if err != nil {
		t.Fatalf("quit(): %v", err)
	}
	if c.conn == nil {
		t.Fatalf("net.Conn = %q, want nil", c.conn)
	}
}
<|endoftext|>


<commit_before>// +build ignore

package main

import (
	"crypto/sha256"
	//_ "expvar"
	"flag"
	redigo "github.com/garyburd/redigo/redis"
	"github.com/pierrre/imageserver"
	imageserver_cache_async "github.com/pierrre/imageserver/cache/async"
	imageserver_cache_list "github.com/pierrre/imageserver/cache/list"
	imageserver_cache_memory "github.com/pierrre/imageserver/cache/memory"
	imageserver_cache_redis "github.com/pierrre/imageserver/cache/redis"
	imageserver_http "github.com/pierrre/imageserver/http"
	imageserver_http_parser_graphicsmagick "github.com/pierrre/imageserver/http/parser/graphicsmagick"
	imageserver_http_parser_list "github.com/pierrre/imageserver/http/parser/list"
	imageserver_http_parser_source "github.com/pierrre/imageserver/http/parser/source"
	imageserver_processor_graphicsmagick "github.com/pierrre/imageserver/processor/graphicsmagick"
	imageserver_processor_limit "github.com/pierrre/imageserver/processor/limit"
	imageserver_provider_cache "github.com/pierrre/imageserver/provider/cache"
	imageserver_provider_http "github.com/pierrre/imageserver/provider/http"
	"log"
	"net"
	"net/http"
	//_ "net/http/pprof"
	"os"
	"strconv"
	"time"
)

func main() {
	var verbose bool
	var httpAddr string
	flag.BoolVar(&verbose, "verbose", false, "Verbose")
	flag.StringVar(&httpAddr, "http", ":8080", "Http")
	flag.Parse()

	hostname, err := os.Hostname()
	if err != nil {
		panic(err)
	}

	var cache imageserver.Cache
	cache = &imageserver_cache_redis.RedisCache{
		Pool: &redigo.Pool{
			Dial: func() (redigo.Conn, error) {
				return redigo.Dial("tcp", "localhost:6379")
			},
			MaxIdle: 50,
		},
		Expire: time.Duration(7 * 24 * time.Hour),
	}
	cache = &imageserver_cache_async.AsyncCache{
		Cache: cache,
		ErrFunc: func(err error, key string, image *imageserver.Image, parameters imageserver.Parameters) {
			if verbose {
				log.Println("Cache error:", err)
			}
		},
	}
	cache = imageserver_cache_list.ListCache{
		imageserver_cache_memory.New(10 * 1024 * 1024),
		cache,
	}

	provider := &imageserver_provider_cache.CacheProvider{
		Provider:     &imageserver_provider_http.HTTPProvider{},
		Cache:        cache,
		CacheKeyFunc: imageserver_provider_cache.NewSourceHashCacheKeyFunc(sha256.New),
	}

	var processor imageserver.Processor
	processor = &imageserver_processor_graphicsmagick.GraphicsMagickProcessor{
		Executable: "gm",
		Timeout:    time.Duration(10 * time.Second),
		AllowedFormats: []string{
			"jpeg",
			"png",
			"bmp",
			"gif",
		},
		DefaultQualities: map[string]string{
			"jpeg": "85",
		},
	}
	processor = imageserver_processor_limit.New(processor, 16)

	imageServer := &imageserver.Server{
		Cache:        cache,
		CacheKeyFunc: imageserver.NewParametersHashCacheKeyFunc(sha256.New),
		Provider:     provider,
		Processor:    processor,
	}

	httpImageServer := &imageserver_http.Server{
		Parser: &imageserver_http_parser_list.ListParser{
			&imageserver_http_parser_source.SourceParser{},
			&imageserver_http_parser_graphicsmagick.GraphicsMagickParser{},
		},
		ImageServer: imageServer,
		ETagFunc:    imageserver_http.NewParametersHashETagFunc(sha256.New),
		Expire:      time.Duration(7 * 24 * time.Hour),
		RequestFunc: func(request *http.Request) error {
			if verbose {
				log.Println("Request:", strconv.Quote(request.URL.String()))
			}
			return nil
		},
		HeaderFunc: func(header http.Header, request *http.Request, err error) {
			header.Set("X-Hostname", hostname)
		},
		ErrorFunc: func(err error, request *http.Request) {
			if verbose {
				log.Println("Error:", err)
			}
		},
		ResponseFunc: func(request *http.Request, statusCode int, contentSize int64, err error) {
			if verbose {
				var errString string
				if err != nil {
					errString = err.Error()
				}
				log.Println("Response:", request.RemoteAddr, request.Method, strconv.Quote(request.URL.String()), statusCode, contentSize, strconv.Quote(errString))
			}
		},
	}
	http.Handle("/", httpImageServer)

	tcpAddr, err := net.ResolveTCPAddr("tcp", httpAddr)
	if err != nil {
		panic(err)
	}

	tcpListener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		panic(err)
	}

	err = http.Serve(tcpListener, nil)
	if err != nil {
		panic(err)
	}
}
<commit_msg>improve advanced example stop<commit_after>// +build ignore

package main

import (
	"crypto/sha256"
	//_ "expvar"
	"flag"
	redigo "github.com/garyburd/redigo/redis"
	"github.com/pierrre/imageserver"
	imageserver_cache_async "github.com/pierrre/imageserver/cache/async"
	imageserver_cache_list "github.com/pierrre/imageserver/cache/list"
	imageserver_cache_memory "github.com/pierrre/imageserver/cache/memory"
	imageserver_cache_redis "github.com/pierrre/imageserver/cache/redis"
	imageserver_http "github.com/pierrre/imageserver/http"
	imageserver_http_parser_graphicsmagick "github.com/pierrre/imageserver/http/parser/graphicsmagick"
	imageserver_http_parser_list "github.com/pierrre/imageserver/http/parser/list"
	imageserver_http_parser_source "github.com/pierrre/imageserver/http/parser/source"
	imageserver_processor_graphicsmagick "github.com/pierrre/imageserver/processor/graphicsmagick"
	imageserver_processor_limit "github.com/pierrre/imageserver/processor/limit"
	imageserver_provider_cache "github.com/pierrre/imageserver/provider/cache"
	imageserver_provider_http "github.com/pierrre/imageserver/provider/http"
	"log"
	"net"
	"net/http"
	//_ "net/http/pprof"
	"os"
	"os/signal"
	"strconv"
	"time"
)

func main() {
	var verbose bool
	var httpAddr string
	flag.BoolVar(&verbose, "verbose", false, "Verbose")
	flag.StringVar(&httpAddr, "http", ":8080", "Http")
	flag.Parse()

	log.Println("Start")

	hostname, err := os.Hostname()
	if err != nil {
		panic(err)
	}

	var cache imageserver.Cache
	cache = &imageserver_cache_redis.RedisCache{
		Pool: &redigo.Pool{
			Dial: func() (redigo.Conn, error) {
				return redigo.Dial("tcp", "localhost:6379")
			},
			MaxIdle: 50,
		},
		Expire: time.Duration(7 * 24 * time.Hour),
	}
	cache = &imageserver_cache_async.AsyncCache{
		Cache: cache,
		ErrFunc: func(err error, key string, image *imageserver.Image, parameters imageserver.Parameters) {
			if verbose {
				log.Println("Cache error:", err)
			}
		},
	}
	cache = imageserver_cache_list.ListCache{
		imageserver_cache_memory.New(10 * 1024 * 1024),
		cache,
	}

	provider := &imageserver_provider_cache.CacheProvider{
		Provider:     &imageserver_provider_http.HTTPProvider{},
		Cache:        cache,
		CacheKeyFunc: imageserver_provider_cache.NewSourceHashCacheKeyFunc(sha256.New),
	}

	var processor imageserver.Processor
	processor = &imageserver_processor_graphicsmagick.GraphicsMagickProcessor{
		Executable: "gm",
		Timeout:    time.Duration(10 * time.Second),
		AllowedFormats: []string{
			"jpeg",
			"png",
			"bmp",
			"gif",
		},
		DefaultQualities: map[string]string{
			"jpeg": "85",
		},
	}
	processor = imageserver_processor_limit.New(processor, 16)

	imageServer := &imageserver.Server{
		Cache:        cache,
		CacheKeyFunc: imageserver.NewParametersHashCacheKeyFunc(sha256.New),
		Provider:     provider,
		Processor:    processor,
	}

	httpImageServer := &imageserver_http.Server{
		Parser: &imageserver_http_parser_list.ListParser{
			&imageserver_http_parser_source.SourceParser{},
			&imageserver_http_parser_graphicsmagick.GraphicsMagickParser{},
		},
		ImageServer: imageServer,
		ETagFunc:    imageserver_http.NewParametersHashETagFunc(sha256.New),
		Expire:      time.Duration(7 * 24 * time.Hour),
		RequestFunc: func(request *http.Request) error {
			if verbose {
				log.Println("Request:", strconv.Quote(request.URL.String()))
			}
			return nil
		},
		HeaderFunc: func(header http.Header, request *http.Request, err error) {
			header.Set("X-Hostname", hostname)
		},
		ErrorFunc: func(err error, request *http.Request) {
			if verbose {
				log.Println("Error:", err)
			}
		},
		ResponseFunc: func(request *http.Request, statusCode int, contentSize int64, err error) {
			if verbose {
				var errString string
				if err != nil {
					errString = err.Error()
				}
				log.Println("Response:", request.RemoteAddr, request.Method, strconv.Quote(request.URL.String()), statusCode, contentSize, strconv.Quote(errString))
			}
		},
	}
	http.Handle("/", httpImageServer)

	tcpAddr, err := net.ResolveTCPAddr("tcp", httpAddr)
	if err != nil {
		panic(err)
	}

	tcpListener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		panic(err)
	}

	interrupted := false
	interruptChan := make(chan os.Signal)
	signal.Notify(interruptChan, os.Interrupt)
	go func() {
		<-interruptChan
		interrupted = true
		log.Println("Close TCP listener")
		err := tcpListener.Close()
		if err != nil {
			panic(err)
		}
	}()

	log.Println("Start HTTP server")
	err = http.Serve(tcpListener, nil)
	if err != nil {
		if interrupted {
			waitDuration := 10 * time.Second
			log.Printf("Wait clients %s (press CTRL+C again to stop the server immediatly)", waitDuration)
			select {
			case <-time.After(waitDuration):
			case <-interruptChan:
			}
		} else {
			panic(err)
		}
	}

	log.Println("Exit")
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"log"
	"os"

	"github.com/codegangsta/cli"
	"github.com/olekukonko/tablewriter"
	"golang.org/x/net/context"

	"github.com/crackcomm/cloudflare"
)

var cmdRecordsList = cli.Command{
	Name:      "list",
	Usage:     "lists zone records",
	ArgsUsage: "<zone-id>",
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name:  "list",
			Usage: "print list instead of table",
		},
	},
	Action: func(c *cli.Context) {
		if len(c.Args()) == 0 {
			log.Fatal("Usage error: zone id is required to print its records.")
		}

		records, err := client(c).Records.List(context.Background(), c.Args().First())
		if err != nil {
			log.Fatal(err)
		}

		if c.Bool("list") {
			for _, record := range records {
				fmt.Println(record.ID)
			}
			return
		}

		table := newRecordsTable()
		for _, record := range records {
			table.add(record)
		}
		table.Render()
	},
}

type recordsTable struct {
	table *tablewriter.Table
}

func newRecordsTable() *recordsTable {
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{
		"ID",
		"Type",
		"Name",
		"Content",
		"Proxiable",
		"Proxied",
		"Locked",
		"TTL",
		"Created On",
		"Modified On",
	})
	return &recordsTable{
		table: table,
	}
}

func (table *recordsTable) Render() { table.table.Render() }

func (table *recordsTable) add(record *cloudflare.Record) {
	table.table.Append([]string{
		record.ID,
		record.Type,
		record.Name,
		record.Content,
		yesOrNo(record.Proxiable),
		yesOrNo(record.Proxied),
		yesOrNo(record.Locked),
		fmt.Sprintf("%d", record.TTL),
		record.CreatedOn.Format("2006/01/02 15:04:05"),
		record.ModifiedOn.Format("2006/01/02 15:04:05"),
	})
}
<commit_msg>cf records list --domain: closes #1<commit_after>package cmd

import (
	"fmt"
	"log"
	"os"

	"github.com/codegangsta/cli"
	"github.com/olekukonko/tablewriter"
	"golang.org/x/net/context"

	"github.com/crackcomm/cloudflare"
)

var cmdRecordsList = cli.Command{
	Name:      "list",
	Usage:     "lists zone records",
	ArgsUsage: "<zone-id>",
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name:  "list",
			Usage: "print list instead of table",
		},
		cli.StringFlag{
			Name:  "domain",
			Usage: "print list instead of table",
		},
	},
	Action: func(c *cli.Context) {
		zoneID := c.Args().First()
		if zoneID == "" && c.String("domain") == "" {
			log.Fatal("Usage error: zone id or --domain is required to print its records.")
		}

		cfclient := client(c)

		if domain := c.String("domain"); domain != "" {
			zones, err := client(c).Zones.List(context.Background())
			if err != nil {
				log.Fatal(err)
			}

			for _, zone := range zones {
				if zone.Name == domain {
					zoneID = zone.ID
					break
				}
			}

			if zoneID == "" {
				log.Fatalf("Domain %q was not found", domain)
			}
		}

		records, err := cfclient.Records.List(context.Background(), zoneID)
		if err != nil {
			log.Fatal(err)
		}

		if c.Bool("list") {
			for _, record := range records {
				fmt.Println(record.ID)
			}
			return
		}

		table := newRecordsTable()
		for _, record := range records {
			table.add(record)
		}
		table.Render()
	},
}

type recordsTable struct {
	table *tablewriter.Table
}

func newRecordsTable() *recordsTable {
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{
		"ID",
		"Type",
		"Name",
		"Content",
		"Proxiable",
		"Proxied",
		"Locked",
		"TTL",
		"Created On",
		"Modified On",
	})
	return &recordsTable{
		table: table,
	}
}

func (table *recordsTable) Render() { table.table.Render() }

func (table *recordsTable) add(record *cloudflare.Record) {
	table.table.Append([]string{
		record.ID,
		record.Type,
		record.Name,
		record.Content,
		yesOrNo(record.Proxiable),
		yesOrNo(record.Proxied),
		yesOrNo(record.Locked),
		fmt.Sprintf("%d", record.TTL),
		record.CreatedOn.Format("2006/01/02 15:04:05"),
		record.ModifiedOn.Format("2006/01/02 15:04:05"),
	})
}
<|endoftext|>


<commit_before>package filesys

import (
	"context"
	"fmt"
	"math"
	"os"
	"path"
	"sync"
	"time"

	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/util/grace"

	"github.com/seaweedfs/fuse"
	"github.com/seaweedfs/fuse/fs"

	"github.com/chrislusf/seaweedfs/weed/filesys/meta_cache"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/util/chunk_cache"
)

type Option struct {
	FilerGrpcAddress   string
	GrpcDialOption     grpc.DialOption
	FilerMountRootPath string
	Collection         string
	Replication        string
	TtlSec             int32
	ChunkSizeLimit     int64
	CacheDir           string
	CacheSizeMB        int64
	DataCenter         string
	DirListCacheLimit  int64
	EntryCacheTtl      time.Duration
	Umask              os.FileMode

	MountUid   uint32
	MountGid   uint32
	MountMode  os.FileMode
	MountCtime time.Time
	MountMtime time.Time

	OutsideContainerClusterMode bool // whether the mount runs outside SeaweedFS containers
	Cipher                      bool // whether encrypt data on volume server

}

var _ = fs.FS(&WFS{})
var _ = fs.FSStatfser(&WFS{})

type WFS struct {
	option                    *Option

	// contains all open handles, protected by handlesLock
	handlesLock sync.Mutex
	handles     map[uint64]*FileHandle

	bufPool sync.Pool

	stats statsCache

	root        fs.Node
	fsNodeCache *FsCache

	chunkCache *chunk_cache.ChunkCache
	metaCache  *meta_cache.MetaCache
}
type statsCache struct {
	filer_pb.StatisticsResponse
	lastChecked int64 // unix time in seconds
}

func NewSeaweedFileSystem(option *Option) *WFS {
	wfs := &WFS{
		option:                    option,
		handles:                   make(map[uint64]*FileHandle),
		bufPool: sync.Pool{
			New: func() interface{} {
				return make([]byte, option.ChunkSizeLimit)
			},
		},
	}
	cacheUniqueId := util.Md5([]byte(option.FilerGrpcAddress))[0:4]
	cacheDir := path.Join(option.CacheDir, cacheUniqueId)
	if option.CacheSizeMB > 0 {
		os.MkdirAll(cacheDir, 0755)
		wfs.chunkCache = chunk_cache.NewChunkCache(256, cacheDir, option.CacheSizeMB)
		grace.OnInterrupt(func() {
			wfs.chunkCache.Shutdown()
		})
	}

	wfs.metaCache = meta_cache.NewMetaCache(path.Join(cacheDir, "meta"))
	startTime := time.Now()
	go meta_cache.SubscribeMetaEvents(wfs.metaCache, wfs, wfs.option.FilerMountRootPath, startTime.UnixNano())
	grace.OnInterrupt(func() {
		wfs.metaCache.Shutdown()
	})

	wfs.root = &Dir{name: wfs.option.FilerMountRootPath, wfs: wfs}
	wfs.fsNodeCache = newFsCache(wfs.root)

	return wfs
}

func (wfs *WFS) Root() (fs.Node, error) {
	return wfs.root, nil
}

func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHandle) {

	fullpath := file.fullpath()
	glog.V(4).Infof("%s AcquireHandle uid=%d gid=%d", fullpath, uid, gid)

	wfs.handlesLock.Lock()
	defer wfs.handlesLock.Unlock()

	inodeId := file.fullpath().AsInode()
	existingHandle, found := wfs.handles[inodeId]
	if found && existingHandle != nil {
		return existingHandle
	}

	fileHandle = newFileHandle(file, uid, gid)
	wfs.handles[inodeId] = fileHandle
	fileHandle.handle = inodeId
	glog.V(4).Infof("%s new fh %d", fullpath, fileHandle.handle)

	return
}

func (wfs *WFS) ReleaseHandle(fullpath util.FullPath, handleId fuse.HandleID) {
	wfs.handlesLock.Lock()
	defer wfs.handlesLock.Unlock()

	glog.V(4).Infof("%s ReleaseHandle id %d current handles length %d", fullpath, handleId, len(wfs.handles))

	delete(wfs.handles, fullpath.AsInode())

	return
}

// Statfs is called to obtain file system metadata. Implements fuse.FSStatfser
func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.StatfsResponse) error {

	glog.V(4).Infof("reading fs stats: %+v", req)

	if wfs.stats.lastChecked < time.Now().Unix()-20 {

		err := wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

			request := &filer_pb.StatisticsRequest{
				Collection:  wfs.option.Collection,
				Replication: wfs.option.Replication,
				Ttl:         fmt.Sprintf("%ds", wfs.option.TtlSec),
			}

			glog.V(4).Infof("reading filer stats: %+v", request)
			resp, err := client.Statistics(context.Background(), request)
			if err != nil {
				glog.V(0).Infof("reading filer stats %v: %v", request, err)
				return err
			}
			glog.V(4).Infof("read filer stats: %+v", resp)

			wfs.stats.TotalSize = resp.TotalSize
			wfs.stats.UsedSize = resp.UsedSize
			wfs.stats.FileCount = resp.FileCount
			wfs.stats.lastChecked = time.Now().Unix()

			return nil
		})
		if err != nil {
			glog.V(0).Infof("filer Statistics: %v", err)
			return err
		}
	}

	totalDiskSize := wfs.stats.TotalSize
	usedDiskSize := wfs.stats.UsedSize
	actualFileCount := wfs.stats.FileCount

	// Compute the total number of available blocks
	resp.Blocks = totalDiskSize / blockSize

	// Compute the number of used blocks
	numBlocks := uint64(usedDiskSize / blockSize)

	// Report the number of free and available blocks for the block size
	resp.Bfree = resp.Blocks - numBlocks
	resp.Bavail = resp.Blocks - numBlocks
	resp.Bsize = uint32(blockSize)

	// Report the total number of possible files in the file system (and those free)
	resp.Files = math.MaxInt64
	resp.Ffree = math.MaxInt64 - actualFileCount

	// Report the maximum length of a name and the minimum fragment size
	resp.Namelen = 1024
	resp.Frsize = uint32(blockSize)

	return nil
}
<commit_msg>reset cache for each new release<commit_after>package filesys

import (
	"context"
	"fmt"
	"math"
	"os"
	"path"
	"sync"
	"time"

	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/util/grace"

	"github.com/seaweedfs/fuse"
	"github.com/seaweedfs/fuse/fs"

	"github.com/chrislusf/seaweedfs/weed/filesys/meta_cache"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/util/chunk_cache"
)

type Option struct {
	FilerGrpcAddress   string
	GrpcDialOption     grpc.DialOption
	FilerMountRootPath string
	Collection         string
	Replication        string
	TtlSec             int32
	ChunkSizeLimit     int64
	CacheDir           string
	CacheSizeMB        int64
	DataCenter         string
	DirListCacheLimit  int64
	EntryCacheTtl      time.Duration
	Umask              os.FileMode

	MountUid   uint32
	MountGid   uint32
	MountMode  os.FileMode
	MountCtime time.Time
	MountMtime time.Time

	OutsideContainerClusterMode bool // whether the mount runs outside SeaweedFS containers
	Cipher                      bool // whether encrypt data on volume server

}

var _ = fs.FS(&WFS{})
var _ = fs.FSStatfser(&WFS{})

type WFS struct {
	option *Option

	// contains all open handles, protected by handlesLock
	handlesLock sync.Mutex
	handles     map[uint64]*FileHandle

	bufPool sync.Pool

	stats statsCache

	root        fs.Node
	fsNodeCache *FsCache

	chunkCache *chunk_cache.ChunkCache
	metaCache  *meta_cache.MetaCache
}
type statsCache struct {
	filer_pb.StatisticsResponse
	lastChecked int64 // unix time in seconds
}

func NewSeaweedFileSystem(option *Option) *WFS {
	wfs := &WFS{
		option:  option,
		handles: make(map[uint64]*FileHandle),
		bufPool: sync.Pool{
			New: func() interface{} {
				return make([]byte, option.ChunkSizeLimit)
			},
		},
	}
	cacheUniqueId := util.Md5([]byte(option.FilerGrpcAddress + option.FilerMountRootPath + util.Version()))[0:4]
	cacheDir := path.Join(option.CacheDir, cacheUniqueId)
	if option.CacheSizeMB > 0 {
		os.MkdirAll(cacheDir, 0755)
		wfs.chunkCache = chunk_cache.NewChunkCache(256, cacheDir, option.CacheSizeMB)
		grace.OnInterrupt(func() {
			wfs.chunkCache.Shutdown()
		})
	}

	wfs.metaCache = meta_cache.NewMetaCache(path.Join(cacheDir, "meta"))
	startTime := time.Now()
	go meta_cache.SubscribeMetaEvents(wfs.metaCache, wfs, wfs.option.FilerMountRootPath, startTime.UnixNano())
	grace.OnInterrupt(func() {
		wfs.metaCache.Shutdown()
	})

	wfs.root = &Dir{name: wfs.option.FilerMountRootPath, wfs: wfs}
	wfs.fsNodeCache = newFsCache(wfs.root)

	return wfs
}

func (wfs *WFS) Root() (fs.Node, error) {
	return wfs.root, nil
}

func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHandle) {

	fullpath := file.fullpath()
	glog.V(4).Infof("%s AcquireHandle uid=%d gid=%d", fullpath, uid, gid)

	wfs.handlesLock.Lock()
	defer wfs.handlesLock.Unlock()

	inodeId := file.fullpath().AsInode()
	existingHandle, found := wfs.handles[inodeId]
	if found && existingHandle != nil {
		return existingHandle
	}

	fileHandle = newFileHandle(file, uid, gid)
	wfs.handles[inodeId] = fileHandle
	fileHandle.handle = inodeId
	glog.V(4).Infof("%s new fh %d", fullpath, fileHandle.handle)

	return
}

func (wfs *WFS) ReleaseHandle(fullpath util.FullPath, handleId fuse.HandleID) {
	wfs.handlesLock.Lock()
	defer wfs.handlesLock.Unlock()

	glog.V(4).Infof("%s ReleaseHandle id %d current handles length %d", fullpath, handleId, len(wfs.handles))

	delete(wfs.handles, fullpath.AsInode())

	return
}

// Statfs is called to obtain file system metadata. Implements fuse.FSStatfser
func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.StatfsResponse) error {

	glog.V(4).Infof("reading fs stats: %+v", req)

	if wfs.stats.lastChecked < time.Now().Unix()-20 {

		err := wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

			request := &filer_pb.StatisticsRequest{
				Collection:  wfs.option.Collection,
				Replication: wfs.option.Replication,
				Ttl:         fmt.Sprintf("%ds", wfs.option.TtlSec),
			}

			glog.V(4).Infof("reading filer stats: %+v", request)
			resp, err := client.Statistics(context.Background(), request)
			if err != nil {
				glog.V(0).Infof("reading filer stats %v: %v", request, err)
				return err
			}
			glog.V(4).Infof("read filer stats: %+v", resp)

			wfs.stats.TotalSize = resp.TotalSize
			wfs.stats.UsedSize = resp.UsedSize
			wfs.stats.FileCount = resp.FileCount
			wfs.stats.lastChecked = time.Now().Unix()

			return nil
		})
		if err != nil {
			glog.V(0).Infof("filer Statistics: %v", err)
			return err
		}
	}

	totalDiskSize := wfs.stats.TotalSize
	usedDiskSize := wfs.stats.UsedSize
	actualFileCount := wfs.stats.FileCount

	// Compute the total number of available blocks
	resp.Blocks = totalDiskSize / blockSize

	// Compute the number of used blocks
	numBlocks := uint64(usedDiskSize / blockSize)

	// Report the number of free and available blocks for the block size
	resp.Bfree = resp.Blocks - numBlocks
	resp.Bavail = resp.Blocks - numBlocks
	resp.Bsize = uint32(blockSize)

	// Report the total number of possible files in the file system (and those free)
	resp.Files = math.MaxInt64
	resp.Ffree = math.MaxInt64 - actualFileCount

	// Report the maximum length of a name and the minimum fragment size
	resp.Namelen = 1024
	resp.Frsize = uint32(blockSize)

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
)

type Selector interface {
	DropHeaders() bool
	ParseHeaders(headers []string) error
	Select(recode []string) ([]string, error)
}

type All struct {
}

func NewAll() *All {
	return &All{}
}

func (a *All) DropHeaders() bool {
	return false
}

func (a *All) ParseHeaders(headers []string) error {
	return nil
}

func (a *All) Select(recode []string) ([]string, error) {
	return recode, nil
}

var (
	INDEXES = regexp.MustCompile(`^(?:\d*-\d*|\d+)(?:,(?:\d*-\d*|\d+))*$`)
	INDEX   = regexp.MustCompile(`(?:\d*-\d*|\d+)`)
	RANGE   = regexp.MustCompile(`^(\d*)-(\d*)$`)
)

type Indexes struct {
	list    string
	indexes []int
}

func NewIndexes(list string) *Indexes {
	return &Indexes{
		list: list,
	}
}

func (i *Indexes) DropHeaders() bool {
	return false
}

func (i *Indexes) ParseHeaders(headers []string) error {
	if i.list == "" {
		i.indexes = make([]int, 0)
		return nil
	}
	if !INDEXES.MatchString(i.list) {
		return fmt.Errorf("%q: invalid syntax", i.list)
	}

	i.indexes = make([]int, 0)
	for _, index := range INDEX.FindAllString(i.list, -1) {
		var err error
		switch {
		case RANGE.MatchString(index):
			first, last := 1, len(headers)
			matches := RANGE.FindStringSubmatch(index)
			if matches[1] != "" {
				first, err = strconv.Atoi(matches[1])
				if err != nil {
					return err
				}
				if first == 0 {
					return fmt.Errorf("indexes are numberd from 1")
				}
			}
			if matches[2] != "" {
				last, err = strconv.Atoi(matches[2])
				if err != nil {
					return err
				}
				if last == 0 {
					return fmt.Errorf("indexes are numberd from 1")
				}
			}
			for idx := first; idx <= last && idx <= len(headers); idx++ {
				i.indexes = append(i.indexes, idx-1)
			}
		default:
			idx, err := strconv.Atoi(index)
			if err != nil {
				return err
			}
			if idx == 0 {
				return fmt.Errorf("indexes are numberd from 1")
			}
			i.indexes = append(i.indexes, idx-1)
		}
	}
	return nil
}

func (i *Indexes) Select(recode []string) ([]string, error) {
	a := make([]string, len(i.indexes))
	for j, index := range i.indexes {
		if index >= 0 && index < len(recode) {
			a[j] = recode[index]
		}
	}
	return a, nil
}

var HEADER = regexp.MustCompile(`(?:[^,\\]|\\.)*`)

type Headers struct {
	indexes []int
	headers []string
}

func NewHeaders(list string) *Headers {
	if list == "" {
		return &Headers{
			headers: []string{},
		}
	}

	headers := HEADER.FindAllString(list, -1)
	for i := 0; i < len(headers); i++ {
		headers[i] = strings.Replace(headers[i], `\,`, `,`, -1)
	}
	return &Headers{
		headers: headers,
	}
}

func (h *Headers) DropHeaders() bool {
	return true
}

func (h *Headers) ParseHeaders(headers []string) error {
	indexMap := make(map[string]int)
	for i, header := range headers {
		if _, ok := indexMap[header]; ok {
			return fmt.Errorf("%q: duplicated header", header)
		}
		indexMap[header] = i
	}

	h.indexes = make([]int, len(h.headers))
	for i, header := range h.headers {
		if index, ok := indexMap[header]; ok {
			h.indexes[i] = index
		} else {
			h.indexes[i] = -1
		}
	}
	return nil
}

func (h *Headers) Select(recode []string) ([]string, error) {
	a := make([]string, len(h.indexes))
	for i, index := range h.indexes {
		if index != -1 {
			a[i] = recode[index]
		}
	}
	return a, nil
}
<commit_msg>Refactoring Indexes.ParseIndexes around variables name<commit_after>package main

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
)

type Selector interface {
	DropHeaders() bool
	ParseHeaders(headers []string) error
	Select(recode []string) ([]string, error)
}

type All struct {
}

func NewAll() *All {
	return &All{}
}

func (a *All) DropHeaders() bool {
	return false
}

func (a *All) ParseHeaders(headers []string) error {
	return nil
}

func (a *All) Select(recode []string) ([]string, error) {
	return recode, nil
}

var (
	INDEXES = regexp.MustCompile(`^(?:\d*-\d*|\d+)(?:,(?:\d*-\d*|\d+))*$`)
	INDEX   = regexp.MustCompile(`(?:\d*-\d*|\d+)`)
	RANGE   = regexp.MustCompile(`^(\d*)-(\d*)$`)
)

type Indexes struct {
	list    string
	indexes []int
}

func NewIndexes(list string) *Indexes {
	return &Indexes{
		list: list,
	}
}

func (i *Indexes) DropHeaders() bool {
	return false
}

func (i *Indexes) ParseHeaders(headers []string) error {
	if i.list == "" {
		i.indexes = make([]int, 0)
		return nil
	}
	if !INDEXES.MatchString(i.list) {
		return fmt.Errorf("%q: invalid syntax", i.list)
	}

	i.indexes = make([]int, 0)
	for _, rawIndex := range INDEX.FindAllString(i.list, -1) {
		var err error
		switch {
		case RANGE.MatchString(rawIndex):
			first, last := 1, len(headers)
			rawRange := RANGE.FindStringSubmatch(rawIndex)
			if rawRange[1] != "" {
				first, err = strconv.Atoi(rawRange[1])
				if err != nil {
					return err
				}
				if first == 0 {
					return fmt.Errorf("indexes are numberd from 1")
				}
			}
			if rawRange[2] != "" {
				last, err = strconv.Atoi(rawRange[2])
				if err != nil {
					return err
				}
				if last == 0 {
					return fmt.Errorf("indexes are numberd from 1")
				}
			}
			for index := first; index <= last && index <= len(headers); index++ {
				i.indexes = append(i.indexes, index-1)
			}
		default:
			index, err := strconv.Atoi(rawIndex)
			if err != nil {
				return err
			}
			if index == 0 {
				return fmt.Errorf("indexes are numberd from 1")
			}
			i.indexes = append(i.indexes, index-1)
		}
	}
	return nil
}

func (i *Indexes) Select(recode []string) ([]string, error) {
	a := make([]string, len(i.indexes))
	for j, index := range i.indexes {
		if index >= 0 && index < len(recode) {
			a[j] = recode[index]
		}
	}
	return a, nil
}

var HEADER = regexp.MustCompile(`(?:[^,\\]|\\.)*`)

type Headers struct {
	indexes []int
	headers []string
}

func NewHeaders(list string) *Headers {
	if list == "" {
		return &Headers{
			headers: []string{},
		}
	}

	headers := HEADER.FindAllString(list, -1)
	for i := 0; i < len(headers); i++ {
		headers[i] = strings.Replace(headers[i], `\,`, `,`, -1)
	}
	return &Headers{
		headers: headers,
	}
}

func (h *Headers) DropHeaders() bool {
	return true
}

func (h *Headers) ParseHeaders(headers []string) error {
	indexMap := make(map[string]int)
	for i, header := range headers {
		if _, ok := indexMap[header]; ok {
			return fmt.Errorf("%q: duplicated header", header)
		}
		indexMap[header] = i
	}

	h.indexes = make([]int, len(h.headers))
	for i, header := range h.headers {
		if index, ok := indexMap[header]; ok {
			h.indexes[i] = index
		} else {
			h.indexes[i] = -1
		}
	}
	return nil
}

func (h *Headers) Select(recode []string) ([]string, error) {
	a := make([]string, len(h.indexes))
	for i, index := range h.indexes {
		if index != -1 {
			a[i] = recode[index]
		}
	}
	return a, nil
}
<|endoftext|>