<commit_before>package glubcms

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	bm "github.com/microcosm-cc/bluemonday"
	bf "github.com/russross/blackfriday"
)

const (
	PSep = string(os.PathSeparator)
)

var (
	ErrHidden = errors.New("Hidden")
)

type Entry interface {
	Active() bool
	Author() string
	Date() time.Time
	HTML() template.HTML
	IsArticle() bool
	Link() string
	Priority() int
	Title() string
	Next() Entry
	Prev() Entry
}

type entries []entry

func (e entries) Len() int {
	return len(e)
}
func (e entries) Less(i, j int) bool {
	if e[i].Priority() != e[j].Priority() {
		return e[i].Priority() > e[j].Priority()
	}
	return e[i].Date().After(e[j].Date())
}
func (e entries) Swap(i, j int) {
	e[i], e[j] = e[j], e[i]
}
func (e entries) Split() (Menu, Articles entries) {
	for _, v := range e {
		if v.IsArticle() {
			Articles = append(Articles, v)
		} else {
			Menu = append(Menu, v)
		}
	}

	return
}

type GCTime time.Time

const GCTimeLayout = "2006-01-02 15:04"

func (t *GCTime) UnmarshalJSON(b []byte) error {
	tmp, err := time.Parse(GCTimeLayout, strings.Trim(string(b), "\""))
	*t = GCTime(tmp)
	return err
}

func (t GCTime) String() string {
	return time.Time(t).Format(GCTimeLayout)
}
func (t GCTime) MarshalJSON() ([]byte, error) {
	return []byte(`"` + t.String() + `"`), nil
}

type Meta struct {
	Author   string
	Date     GCTime
	Title    string
	Priority int
	Hidden   bool `json:",omitempty"`
	Unsafe   bool `json:",omitempty"`
}

type ContentRenderer interface {
	Render() []byte
}

type entry struct {
	meta       Meta
	active     bool
	html       []byte
	isarticle  bool
	link       url.URL
	next       *entry
	prev       *entry
	fs         http.FileSystem
	md_path    string
	once       sync.Once
	renderHTML ContentRenderer
}

func (e entry) Active() bool {
	return e.active
}
func (a entry) Author() string {
	return a.meta.Author
}
func (e entry) Date() time.Time {
	return time.Time(e.meta.Date)
}
func (e *entry) HTML() template.HTML {
	e.once.Do(func() {
		e.html = e.renderHTML.Render()
	})
	return template.HTML(e.html)
}
func (e entry) IsArticle() bool {
	return e.isarticle
}
func (e entry) Link() string {
	return e.link.String()
}
func (e entry) Priority() int {
	return e.meta.Priority
}
func (e entry) Title() string {
	return e.meta.Title
}
func (e entry) Next() Entry {
	if e.next != nil {
		return e.next
	}
	return nil
}
func (e entry) Prev() Entry {
	if e.prev != nil {
		return e.prev
	}
	return nil
}

func entryFromMeta(fs http.FileSystem, path string) (entry, error) {
	ret := entry{}
	f, err := fs.Open(filepath.Join(path))
	if err != nil {
		return ret, err
	}
	defer f.Close()

	err = json.NewDecoder(f).Decode(&ret.meta)
	if err != nil {
		err = &os.PathError{
			Op:   "Parsing json in",
			Path: path,
			Err:  err,
		}
	}
	return ret, err
}

func entryFromDir(fs http.FileSystem, path, activepath string) (ret entry, err error) {
	ret, err = entryFromMeta(fs, filepath.Join(path, "meta.json"))
	if err != nil {
		log.Println(err)
		return ret, err
	}

	// skip hidden folders, unless directly asked for
	if ret.meta.Hidden && activepath != path {
		return ret, ErrHidden
	}

	ret.link = url.URL{Path: path}
	// .Dir() removes trailing slashes, this prevents
	// /a/b/ beeing seen as prefix from /a/bc/
	if strings.HasPrefix(activepath+PSep, path+PSep) {
		ret.active = true
	}

	md_path := filepath.Join(path, "article.md")
	md, err := fs.Open(md_path)
	if err != nil {
		if !os.IsNotExist(err) {
			log.Println(err)
		} else {
			err = nil
		}
		return ret, err
	}
	md.Close()

	ret.isarticle = true
	ret.renderHTML = articleRenderer{
		fs:      fs,
		md_path: md_path,
		unsafe:  ret.meta.Unsafe,
	}

	return ret, nil
}

type articleRenderer struct {
	fs      http.FileSystem
	md_path string
	unsafe  bool
}

func (a articleRenderer) Render() []byte {
	md, err := a.fs.Open(a.md_path)
	if err != nil {
		if !os.IsNotExist(err) {
			log.Println(err)
		}
		return nil
	}
	defer md.Close()

	b, err := ioutil.ReadAll(md)
	if err != nil {
		log.Println(err)
	}
	html := bf.Markdown(b, bf.HtmlRenderer(bf.HTML_USE_XHTML, "", ""), bf.EXTENSION_TABLES)
	if !a.unsafe {
		html = bm.UGCPolicy().SanitizeBytes(html)
	}
	return html
}

func entriesFromDir(fs http.FileSystem, path, activepath string) entries {
	var ret entries

	dir, err := fs.Open(path)
	if err != nil {
		log.Println(err)
		return nil
	}

	dirlist, err := dir.Readdir(0)
	if err != nil {
		log.Println(err)
	}
	dir.Close()

	for _, fi := range dirlist {
		if !fi.IsDir() {
			continue
		}
		entry, err := entryFromDir(fs, filepath.Join(path, fi.Name()), activepath)
		if err == nil {
			ret = append(ret, entry)
		}
	}

	sort.Sort(ret)
	return ret
}

type Page struct {
	Menu     []entries
	Articles entries
	Content  *entry
}

func PageFromDir(fs http.FileSystem, path string) Page {
	var p Page
	path = filepath.Clean(path)
	activepath := path

	for {
		menu, articles := entriesFromDir(fs, path, activepath).Split()
		if len(menu) > 0 {
			sort.Sort(menu)
			p.Menu = append(p.Menu, menu)
		}
		if p.Articles == nil && len(articles) > 0 {
			sort.Sort(articles)
			p.Articles = articles

			// link next/prev pointers
			for i := 1; i < len(p.Articles); i++ {
				p.Articles[i].next = &p.Articles[i-1]
				p.Articles[i-1].prev = &p.Articles[i]
			}

			// set the active article as content
			for i := 0; i < len(p.Articles); i++ {
				if p.Articles[i].link.Path == activepath {
					p.Content = &p.Articles[i]
				}
			}

			// if no exact match, default to first article
			if p.Content == nil {
				p.Content = &p.Articles[0]
				p.Content.active = true
			}
		}
		if path == "." || path == "/" {
			break
		}
		path = filepath.Dir(path)
	}

	// building from deepest path to /, need to reverse
	for l, r := 0, len(p.Menu)-1; l < r; l, r = l+1, r-1 {
		p.Menu[l], p.Menu[r] = p.Menu[r], p.Menu[l]
	}

	return p
}

// For debugging
func (p Page) Outline() string {
	buf := bytes.Buffer{}

	fmt.Fprintln(&buf, "Menu:")
	for level, ms := range p.Menu {
		for _, m := range ms {
			fmt.Fprintf(&buf, "%s%q", strings.Repeat("\t", level), m.Title())
			if m.Active() {
				fmt.Fprintf(&buf, " (active)")
			}
			fmt.Fprintln(&buf)
		}
	}

	fmt.Fprintln(&buf, "Articles:")
	for _, a := range p.Articles {
		fmt.Fprintf(&buf, "%q", a.Title())
		if a.Active() {
			fmt.Fprintf(&buf, " (active)")
		}
		fmt.Fprintln(&buf)
	}
	fmt.Fprintln(&buf, "Content:")
	if p.Content != nil {
		fmt.Fprint(&buf, p.Content.Title())
	}

	return buf.String()
}

func init() {
	log.SetFlags(log.Flags() | log.Lshortfile)
}
<commit_msg>facilitate an index page<commit_after>package glubcms

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	bm "github.com/microcosm-cc/bluemonday"
	bf "github.com/russross/blackfriday"
)

const (
	PSep = string(os.PathSeparator)
)

var (
	ErrHidden = errors.New("Hidden")
)

type Entry interface {
	Active() bool
	Author() string
	Date() time.Time
	HTML() template.HTML
	IsArticle() bool
	Link() string
	Priority() int
	Title() string
	Next() Entry
	Prev() Entry
}

type entries []entry

func (e entries) Len() int {
	return len(e)
}
func (e entries) Less(i, j int) bool {
	switch {
	case e[i].meta.IsIndex && !e[j].meta.IsIndex:
		return false
	case !e[i].meta.IsIndex && e[j].meta.IsIndex:
		return true
	case e[i].Priority() != e[j].Priority():
		return e[i].Priority() > e[j].Priority()
	}
	return e[i].Date().After(e[j].Date())
}
func (e entries) Swap(i, j int) {
	e[i], e[j] = e[j], e[i]
}
func (e entries) Split() (Menu, Articles entries) {
	for _, v := range e {
		if v.IsArticle() {
			Articles = append(Articles, v)
		} else {
			Menu = append(Menu, v)
		}
	}

	return
}

type GCTime time.Time

const GCTimeLayout = "2006-01-02 15:04"

func (t *GCTime) UnmarshalJSON(b []byte) error {
	tmp, err := time.Parse(GCTimeLayout, strings.Trim(string(b), "\""))
	*t = GCTime(tmp)
	return err
}

func (t GCTime) String() string {
	return time.Time(t).Format(GCTimeLayout)
}
func (t GCTime) MarshalJSON() ([]byte, error) {
	return []byte(`"` + t.String() + `"`), nil
}

type Meta struct {
	Author   string
	Date     GCTime
	Title    string
	Priority int
	Hidden   bool `json:",omitempty"`
	Unsafe   bool `json:",omitempty"`
	IsIndex  bool `json:",omitempty"`
}

type ContentRenderer interface {
	Render() []byte
}

type entry struct {
	meta       Meta
	active     bool
	html       []byte
	isarticle  bool
	link       url.URL
	next       *entry
	prev       *entry
	fs         http.FileSystem
	md_path    string
	once       sync.Once
	renderHTML ContentRenderer
}

func (e entry) Active() bool {
	return e.active
}
func (a entry) Author() string {
	return a.meta.Author
}
func (e entry) Date() time.Time {
	return time.Time(e.meta.Date)
}
func (e *entry) HTML() template.HTML {
	e.once.Do(func() {
		e.html = e.renderHTML.Render()
	})
	return template.HTML(e.html)
}
func (e entry) IsArticle() bool {
	return e.isarticle
}
func (e entry) Link() string {
	return e.link.String()
}
func (e entry) Priority() int {
	return e.meta.Priority
}
func (e entry) Title() string {
	return e.meta.Title
}
func (e entry) Next() Entry {
	if e.next != nil {
		return e.next
	}
	return nil
}
func (e entry) Prev() Entry {
	if e.prev != nil {
		return e.prev
	}
	return nil
}

func entryFromMeta(fs http.FileSystem, path string) (entry, error) {
	ret := entry{}
	f, err := fs.Open(filepath.Join(path))
	if err != nil {
		return ret, err
	}
	defer f.Close()

	err = json.NewDecoder(f).Decode(&ret.meta)
	if err != nil {
		err = &os.PathError{
			Op:   "Parsing json in",
			Path: path,
			Err:  err,
		}
	}
	return ret, err
}

func entryFromDir(fs http.FileSystem, path, activepath string) (ret entry, err error) {
	ret, err = entryFromMeta(fs, filepath.Join(path, "meta.json"))
	if err != nil {
		log.Println(err)
		return ret, err
	}

	// skip hidden folders, unless directly asked for
	if ret.meta.Hidden && activepath != path {
		return ret, ErrHidden
	}

	ret.link = url.URL{Path: path}
	// .Dir() removes trailing slashes, this prevents
	// /a/b/ beeing seen as prefix from /a/bc/
	if strings.HasPrefix(activepath+PSep, path+PSep) {
		ret.active = true
	}

	md_path := filepath.Join(path, "article.md")
	md, err := fs.Open(md_path)
	if err != nil {
		if !os.IsNotExist(err) {
			log.Println(err)
		} else {
			err = nil
		}
		return ret, err
	}
	md.Close()

	ret.isarticle = true
	ret.renderHTML = articleRenderer{
		fs:      fs,
		md_path: md_path,
		unsafe:  ret.meta.Unsafe,
	}

	return ret, nil
}

type articleRenderer struct {
	fs      http.FileSystem
	md_path string
	unsafe  bool
}

func (a articleRenderer) Render() []byte {
	md, err := a.fs.Open(a.md_path)
	if err != nil {
		if !os.IsNotExist(err) {
			log.Println(err)
		}
		return nil
	}
	defer md.Close()

	b, err := ioutil.ReadAll(md)
	if err != nil {
		log.Println(err)
	}
	html := bf.Markdown(b, bf.HtmlRenderer(bf.HTML_USE_XHTML, "", ""), bf.EXTENSION_TABLES)
	if !a.unsafe {
		html = bm.UGCPolicy().SanitizeBytes(html)
	}
	return html
}

func entriesFromDir(fs http.FileSystem, path, activepath string) entries {
	var ret entries

	dir, err := fs.Open(path)
	if err != nil {
		log.Println(err)
		return nil
	}

	dirlist, err := dir.Readdir(0)
	if err != nil {
		log.Println(err)
	}
	dir.Close()

	for _, fi := range dirlist {
		if !fi.IsDir() {
			continue
		}
		entry, err := entryFromDir(fs, filepath.Join(path, fi.Name()), activepath)
		if err == nil {
			ret = append(ret, entry)
		}
	}

	sort.Sort(ret)
	return ret
}

type Page struct {
	Menu     []entries
	Articles entries
	Content  *entry
	Index    *entry
}

func PageFromDir(fs http.FileSystem, path string) Page {
	var p Page
	path = filepath.Clean(path)
	activepath := path

	for {
		menu, articles := entriesFromDir(fs, path, activepath).Split()
		if len(menu) > 0 {
			sort.Sort(menu)
			p.Menu = append(p.Menu, menu)
		}
		if p.Articles == nil && len(articles) > 0 {
			sort.Sort(articles)
			p.Articles = articles

			// link next/prev pointers
			for i := 1; i < len(p.Articles); i++ {
				p.Articles[i].next = &p.Articles[i-1]
				p.Articles[i-1].prev = &p.Articles[i]
			}

			// set the active article as content
			for i := 0; i < len(p.Articles); i++ {
				if p.Articles[i].link.Path == activepath {
					p.Content = &p.Articles[i]
					break
				}
			}

			// if no exact match, default to first article
			if p.Content == nil {
				p.Content = &p.Articles[0]
				p.Content.active = true
			}

			// don't list the index page as article
			for p.Articles[len(p.Articles)-1].meta.IsIndex {
				p.Index = &p.Articles[len(p.Articles)-1]
				p.Articles = p.Articles[:len(p.Articles)-1]
			}
		}
		if path == "." || path == "/" {
			break
		}
		path = filepath.Dir(path)
	}

	// building from deepest path to /, need to reverse
	for l, r := 0, len(p.Menu)-1; l < r; l, r = l+1, r-1 {
		p.Menu[l], p.Menu[r] = p.Menu[r], p.Menu[l]
	}

	return p
}

// For debugging
func (p Page) Outline() string {
	buf := bytes.Buffer{}

	fmt.Fprintln(&buf, "Menu:")
	for level, ms := range p.Menu {
		for _, m := range ms {
			fmt.Fprintf(&buf, "%s%q", strings.Repeat("\t", level), m.Title())
			if m.Active() {
				fmt.Fprintf(&buf, " (active)")
			}
			fmt.Fprintln(&buf)
		}
	}

	fmt.Fprintln(&buf, "Articles:")
	for _, a := range p.Articles {
		fmt.Fprintf(&buf, "%q", a.Title())
		if a.Active() {
			fmt.Fprintf(&buf, " (active)")
		}
		fmt.Fprintln(&buf)
	}
	fmt.Fprintln(&buf, "Content:")
	if p.Content != nil {
		fmt.Fprint(&buf, p.Content.Title())
	}

	return buf.String()
}

func init() {
	log.SetFlags(log.Flags() | log.Lshortfile)
}
<|endoftext|>


<commit_before>package queues
/*
 *  Filename:    priority.go
 *  Package:     queues
 *  Author:      Bryan Matsuo <bmatsuo@soe.ucsc.edu>
 *  Created:     Wed Jul  6 22:18:57 PDT 2011
 *  Description:
 */
import (
    "container/heap"
)

type PrioritizedTask interface {
    Task
    Key() float64
    SetKey(float64)
}
type PTask struct {
    F func(int64)
    P float64
}
func (pt PTask) Func() func(int64) {
    return pt.F
}
func (pt PTask) Key() float64 {
    return pt.P
}
func (pt *PTask) SetKey(k float64) {
    pt.P = k
}

type pQueue struct {
    elements []RegisteredTask
}
func newPQueue() *pQueue {
    var h = new(pQueue)
    h.elements = make([]RegisteredTask, 0, 5)
    return h
}
func (h *pQueue) GetPTask(i int) PrioritizedTask {
    if n := len(h.elements) ; i < 0 || i >= n {
        panic("badindex")
    }
    return h.elements[i].Task().(PrioritizedTask)
}
func (h *pQueue) Len() int {
    return len(h.elements)
}
func (h *pQueue) Less(i, j int) bool {
    return h.GetPTask(i).Key() < h.GetPTask(j).Key()
}
func (h *pQueue) Swap(i, j int) {
    if n := len(h.elements) ; i < 0 || i >=n || j < 0 || j >= n {
        panic("badindex")
    }
    var tmp = h.elements[i]
    h.elements[i] = h.elements[j]
    h.elements[j] = tmp
}
func (h *pQueue) Push(x interface{}) {
    switch x.(type) {
    case PrioritizedTask:
        h.elements = append(h.elements, x.(RegisteredTask))
    default:
        panic("badtype")
    }
}
func (h *pQueue) Pop() interface{} {
    if len(h.elements) <= 0 {
        panic("empty")
    }
    var head = h.elements[0]
    h.elements = h.elements[1:]
    return head
}
func (h *pQueue) FindId(id int64) (int, RegisteredTask) {
    for i, elm := range h.elements {
        if elm.Id() == id {
            return i, elm
        }
    }
    return -1, nil
}

type PriorityQueue struct {
    h  *pQueue
}

func NewPriorityQueue() *PriorityQueue {
    var pq = new(PriorityQueue)
    pq.h = newPQueue()
    // No need to call heap.Init(pq.h) on an empty heap.
    return pq
}

func (pq *PriorityQueue) Len() int {
    return pq.h.Len()
}
func (pq *PriorityQueue) Dequeue() RegisteredTask {
    if pq.Len() <= 0 {
        panic("empty")
    }
    return heap.Pop(pq.h).(RegisteredTask)
}
func (pq *PriorityQueue) Enqueue(task RegisteredTask) {
    switch task.Task().(type) {
    case PrioritizedTask:
        heap.Push(pq.h, task)
    default:
        panic("nokey")
    }
}
func (pq *PriorityQueue) SetKey(id int64, k float64) {
    var i, task = pq.h.FindId(id)
    if i < 0 {
        return
    }
    heap.Remove(pq.h, i)
    task.Task().(PrioritizedTask).SetKey(k)
    heap.Push(pq.h, task)
}
<commit_msg>Add some debug info to panic in (*PriorityQueue)Enqueue()<commit_after>package queues
/*
 *  Filename:    priority.go
 *  Package:     queues
 *  Author:      Bryan Matsuo <bmatsuo@soe.ucsc.edu>
 *  Created:     Wed Jul  6 22:18:57 PDT 2011
 *  Description:
 */
import (
    "container/heap"
)

type PrioritizedTask interface {
    Task
    Key() float64
    SetKey(float64)
}
type PTask struct {
    F func(int64)
    P float64
}
func (pt PTask) Func() func(int64) {
    return pt.F
}
func (pt PTask) Key() float64 {
    return pt.P
}
func (pt *PTask) SetKey(k float64) {
    pt.P = k
}

type pQueue struct {
    elements []RegisteredTask
}
func newPQueue() *pQueue {
    var h = new(pQueue)
    h.elements = make([]RegisteredTask, 0, 5)
    return h
}
func (h *pQueue) GetPTask(i int) PrioritizedTask {
    if n := len(h.elements) ; i < 0 || i >= n {
        panic("badindex")
    }
    return h.elements[i].Task().(PrioritizedTask)
}
func (h *pQueue) Len() int {
    return len(h.elements)
}
func (h *pQueue) Less(i, j int) bool {
    return h.GetPTask(i).Key() < h.GetPTask(j).Key()
}
func (h *pQueue) Swap(i, j int) {
    if n := len(h.elements) ; i < 0 || i >=n || j < 0 || j >= n {
        panic("badindex")
    }
    var tmp = h.elements[i]
    h.elements[i] = h.elements[j]
    h.elements[j] = tmp
}
func (h *pQueue) Push(x interface{}) {
    switch x.(type) {
    case PrioritizedTask:
        h.elements = append(h.elements, x.(RegisteredTask))
    default:
        panic("badtype")
    }
}
func (h *pQueue) Pop() interface{} {
    if len(h.elements) <= 0 {
        panic("empty")
    }
    var head = h.elements[0]
    h.elements = h.elements[1:]
    return head
}
func (h *pQueue) FindId(id int64) (int, RegisteredTask) {
    for i, elm := range h.elements {
        if elm.Id() == id {
            return i, elm
        }
    }
    return -1, nil
}

type PriorityQueue struct {
    h  *pQueue
}

func NewPriorityQueue() *PriorityQueue {
    var pq = new(PriorityQueue)
    pq.h = newPQueue()
    // No need to call heap.Init(pq.h) on an empty heap.
    return pq
}

func (pq *PriorityQueue) Len() int {
    return pq.h.Len()
}
func (pq *PriorityQueue) Dequeue() RegisteredTask {
    if pq.Len() <= 0 {
        panic("empty")
    }
    return heap.Pop(pq.h).(RegisteredTask)
}
func (pq *PriorityQueue) Enqueue(task RegisteredTask) {
    switch task.Task().(type) {
    case PrioritizedTask:
        heap.Push(pq.h, task)
    default:
        panic(fmt.Sprintf("nokey %v", task))
    }
}
func (pq *PriorityQueue) SetKey(id int64, k float64) {
    var i, task = pq.h.FindId(id)
    if i < 0 {
        return
    }
    heap.Remove(pq.h, i)
    task.Task().(PrioritizedTask).SetKey(k)
    heap.Push(pq.h, task)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"regexp"
	"syscall"
	"text/template"

	"github.com/codegangsta/cli"
	godaemon "github.com/yookoala/go-daemon"
)

func handleShutdown(l net.Listener, pidfile string) {
	c := make(chan os.Signal, 1)
	signal.Notify(c,
		os.Interrupt,
		syscall.SIGTERM,
		syscall.SIGQUIT,
		syscall.SIGKILL)

	for {
		select {
		case <-c:
			l.Close()
			if pidfile != "" {
				os.Remove(pidfile)
			}
			os.Exit(0)
		}
	}
}

type commandFn func(stdout, stderr io.Writer) error

func handleConnection(conn net.Conn, stdout, stderr io.Writer, fn commandFn) {
	log.Printf("server: handleConnection")
	defer conn.Close()

	for {
		bufbytes := make([]byte, 1024)
		nr, err := conn.Read(bufbytes)

		// handle error
		if err == io.EOF {
			log.Printf("server: client connect closed")
			return
		} else if err != nil {
			log.Printf("server read error: %#v", err.Error())
			return
		}

		data := bufbytes[0:nr]
		fmt.Fprintf(conn, "echo: ")
		conn.Write(data)
		log.Printf("server got: %s", data)

		rw := io.MultiWriter(conn, stdout)
		ew := io.MultiWriter(conn, stderr)

		if err := fn(rw, ew); err == io.EOF {
			log.Printf("server: connection terminated")
			return
		} else if err != nil {
			log.Printf("callback error: %s", err.Error())
			return
		}
	}
}

// address returns networkk and address that fits
// the use of either net.Dial or net.Listen
func address(listen string) (network, address string) {
	reIP := regexp.MustCompile("^(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})\\:(\\d{2,5}$)")
	rePort := regexp.MustCompile("^(\\d+)$")
	switch {
	case reIP.MatchString(listen):
		network = "tcp"
		address = listen
	case rePort.MatchString(listen):
		network = "tcp"
		address = ":" + listen
	default:
		network = "unix"
		address = listen
	}
	return
}

func actionServer(c *cli.Context) {

	var stdout io.Writer = os.Stdout
	var stderr io.Writer = os.Stderr

	if output := c.String("output"); output != "" {
		var f *os.File
		var err error
		if f, err = os.Create(output); err != nil {
			log.Fatalf("error opening output logfile %#v: %s",
				output, err.Error())
			return
		}
		stdout = f
		stderr = f
		log.SetOutput(f)
	} else if c.Bool("daemon") {
		var f *os.File
		var err error
		if f, err = os.Create(os.DevNull); err != nil {
			log.Fatalf("error opening output logfile %#v: %s",
				output, err.Error())
			return
		}
		stdout = f
		stderr = f
		log.SetOutput(f)
	}

	// daemonized server
	if c.Bool("daemon") {
		context := new(godaemon.Context)
		if child, _ := context.Reborn(); child != nil {
			return
		}
		defer context.Release()
		actionServerMain(c, stdout, stderr)
		return
	}

	// normal server output
	actionServerMain(c, stdout, stderr)
}

func actionServerMain(c *cli.Context, stdout, stderr io.Writer) {

	l, err := net.Listen(address(c.String("listen")))
	if err != nil {
		log.Fatalf("error: %s", err)
	}

	pidfile := c.String("pidfile")
	if pidfile != "" {
		// get current pid and write to file
		pid := fmt.Sprintf("%d", os.Getpid())
		ioutil.WriteFile(pidfile, []byte(pid), 0600)
	}

	// cleanly disconnect the socket
	go handleShutdown(l, pidfile)

	// define git source to update from
	src := gitSource{mustGitRootPath(c.String("gitrepo")),
		c.String("remote"), c.String("branch")}

	for {
		conn, err := l.Accept()
		if err != nil {
			panic(err)
		}

		go handleConnection(conn, stdout, stderr, gitActionsFor(src))
	}
}

func actionOnce(c *cli.Context) {

	var stdout io.Writer = os.Stdout
	var stderr io.Writer = os.Stderr
	if output := c.String("output"); output != "" {
		var f *os.File
		var err error
		if f, err = os.Create(output); err != nil {
			log.Fatalf("error opening output logfile %#v: %s",
				output, err.Error())
			return
		}
		stdout = f
		stderr = f
		log.SetOutput(f)
	}

	// define git source to update from
	src := gitSource{mustGitRootPath(c.String("gitrepo")),
		c.String("remote"), c.String("branch")}

	if err := gitActionsFor(src)(stdout, stderr); err != io.EOF {
		log.Fatalf("error: %s", err.Error())
	}
}

func actionClient(c *cli.Context) {
	conn, err := net.Dial(address(c.String("conn")))
	if err != nil {
		log.Fatalf("connection error: %s", err.Error())
		return
	}

	conn.Write([]byte("hello\n"))

	bufbytes := make([]byte, 1024)
	for {
		nr, err := conn.Read(bufbytes)

		// handle error
		if err == io.EOF {
			log.Printf("client: server connect closed")
			return
		} else if err != nil {
			log.Printf("client read error: %#v", err.Error())
			return
		}

		data := bufbytes[0:nr]
		fmt.Printf("%s", data)
	}
}

func createHookScript(filename, command string) (err error) {

	// template for git hook script
	tpl := template.Must(template.New("gitsocket").Parse(`#!/bin/sh
#
# An example hook script to prepare a packed repository for use over
# dumb transports.
#
# To enable this hook, rename this file to "post-checkout".
{{ .Command }}
`))

	f, err := os.Create(filename)
	if err != nil {
		log.Fatalf("error: %s", err)
		return
	}
	err = tpl.Execute(f, map[string]interface{}{
		"Command": command,
	})
	f.Close()
	if err != nil {
		return
	}

	err = os.Chmod(filename, 0777)
	return
}

func actionSetup(c *cli.Context) {

	// define git source to update from
	rootPath := mustGitRootPath(c.String("gitrepo"))
	filename := path.Join(rootPath, ".git/hooks/post-checkout")

	if command := c.String("command"); command != "" {
		// if file not exists, create the file
		if _, err := os.Stat(filename); os.IsNotExist(err) {
			createHookScript(filename, command)
			return
		} else if c.Bool("force") {
			createHookScript(filename, command)
			return
		}
		fmt.Println("post-checkout script already exists. If you want to " +
			"overwrite, please use the -f flag")
		os.Exit(1)
		return
	}

	// if file not exists, create the file
	if _, err := os.Stat(filename); os.IsNotExist(err) {
		createHookScript(filename, "exec echo \"checkout completed.\"\n")
	}

	cmd := exec.Command("vi", filename)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stdout
	if err := cmd.Run(); err != nil {
		log.Fatalf("error: %s", err)
	}
}
<commit_msg>server: daemon waits for socket ready before exit<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"regexp"
	"syscall"
	"text/template"
	"time"

	"github.com/codegangsta/cli"
	godaemon "github.com/yookoala/go-daemon"
)

func handleShutdown(l net.Listener, pidfile string) {
	c := make(chan os.Signal, 1)
	signal.Notify(c,
		os.Interrupt,
		syscall.SIGTERM,
		syscall.SIGQUIT,
		syscall.SIGKILL)

	for {
		select {
		case <-c:
			l.Close()
			if pidfile != "" {
				os.Remove(pidfile)
			}
			os.Exit(0)
		}
	}
}

type commandFn func(stdout, stderr io.Writer) error

func handleConnection(conn net.Conn, stdout, stderr io.Writer, fn commandFn) {
	log.Printf("server: handleConnection")
	defer conn.Close()

	for {
		bufbytes := make([]byte, 1024)
		nr, err := conn.Read(bufbytes)

		// handle error
		if err == io.EOF {
			log.Printf("server: client connect closed")
			return
		} else if err != nil {
			log.Printf("server read error: %#v", err.Error())
			return
		}

		data := bufbytes[0:nr]
		fmt.Fprintf(conn, "echo: ")
		conn.Write(data)
		log.Printf("server got: %s", data)

		rw := io.MultiWriter(conn, stdout)
		ew := io.MultiWriter(conn, stderr)

		if err := fn(rw, ew); err == io.EOF {
			log.Printf("server: connection terminated")
			return
		} else if err != nil {
			log.Printf("callback error: %s", err.Error())
			return
		}
	}
}

// address returns networkk and address that fits
// the use of either net.Dial or net.Listen
func address(listen string) (network, address string) {
	reIP := regexp.MustCompile("^(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})\\:(\\d{2,5}$)")
	rePort := regexp.MustCompile("^(\\d+)$")
	switch {
	case reIP.MatchString(listen):
		network = "tcp"
		address = listen
	case rePort.MatchString(listen):
		network = "tcp"
		address = ":" + listen
	default:
		network = "unix"
		address = listen
	}
	return
}

func actionServer(c *cli.Context) {

	var stdout io.Writer = os.Stdout
	var stderr io.Writer = os.Stderr

	if output := c.String("output"); output != "" {
		var f *os.File
		var err error
		if f, err = os.Create(output); err != nil {
			log.Fatalf("error opening output logfile %#v: %s",
				output, err.Error())
			return
		}
		stdout = f
		stderr = f
		log.SetOutput(f)
	} else if c.Bool("daemon") {
		var f *os.File
		var err error
		if f, err = os.Create(os.DevNull); err != nil {
			log.Fatalf("error opening output logfile %#v: %s",
				output, err.Error())
			return
		}
		stdout = f
		stderr = f
		log.SetOutput(f)
	}

	// daemonized server
	if c.Bool("daemon") {
		context := new(godaemon.Context)
		if child, _ := context.Reborn(); child != nil {

			// set timeout time
			timeout := time.After(time.Second * 30)

			// test if the socket is ready
			ready := make(chan int)
			go func() {
				for {
					conn, err := net.Dial(address(c.String("listen")))
					if err == nil {
						conn.Close() // close the test connection
						break
					}
				}
				ready <- 0
			}()

			// wait until timeout or socket ready by child
			select {
			case <-timeout:
				log.Fatalf("timeout: socket not ready in %d seconds", 30)
			case <-ready:
				return
			}
		}
		defer context.Release()
		actionServerMain(c, stdout, stderr)
		return
	}

	// normal server output
	actionServerMain(c, stdout, stderr)
}

func actionServerMain(c *cli.Context, stdout, stderr io.Writer) {

	l, err := net.Listen(address(c.String("listen")))
	if err != nil {
		log.Fatalf("error: %s", err)
	}

	pidfile := c.String("pidfile")
	if pidfile != "" {
		// get current pid and write to file
		pid := fmt.Sprintf("%d", os.Getpid())
		ioutil.WriteFile(pidfile, []byte(pid), 0600)
	}

	// cleanly disconnect the socket
	go handleShutdown(l, pidfile)

	// define git source to update from
	src := gitSource{mustGitRootPath(c.String("gitrepo")),
		c.String("remote"), c.String("branch")}

	for {
		conn, err := l.Accept()
		if err != nil {
			panic(err)
		}

		go handleConnection(conn, stdout, stderr, gitActionsFor(src))
	}
}

func actionOnce(c *cli.Context) {

	var stdout io.Writer = os.Stdout
	var stderr io.Writer = os.Stderr
	if output := c.String("output"); output != "" {
		var f *os.File
		var err error
		if f, err = os.Create(output); err != nil {
			log.Fatalf("error opening output logfile %#v: %s",
				output, err.Error())
			return
		}
		stdout = f
		stderr = f
		log.SetOutput(f)
	}

	// define git source to update from
	src := gitSource{mustGitRootPath(c.String("gitrepo")),
		c.String("remote"), c.String("branch")}

	if err := gitActionsFor(src)(stdout, stderr); err != io.EOF {
		log.Fatalf("error: %s", err.Error())
	}
}

func actionClient(c *cli.Context) {
	conn, err := net.Dial(address(c.String("conn")))
	if err != nil {
		log.Fatalf("connection error: %s", err.Error())
		return
	}

	conn.Write([]byte("hello\n"))

	bufbytes := make([]byte, 1024)
	for {
		nr, err := conn.Read(bufbytes)

		// handle error
		if err == io.EOF {
			log.Printf("client: server connect closed")
			return
		} else if err != nil {
			log.Printf("client read error: %#v", err.Error())
			return
		}

		data := bufbytes[0:nr]
		fmt.Printf("%s", data)
	}
}

func createHookScript(filename, command string) (err error) {

	// template for git hook script
	tpl := template.Must(template.New("gitsocket").Parse(`#!/bin/sh
#
# An example hook script to prepare a packed repository for use over
# dumb transports.
#
# To enable this hook, rename this file to "post-checkout".
{{ .Command }}
`))

	f, err := os.Create(filename)
	if err != nil {
		log.Fatalf("error: %s", err)
		return
	}
	err = tpl.Execute(f, map[string]interface{}{
		"Command": command,
	})
	f.Close()
	if err != nil {
		return
	}

	err = os.Chmod(filename, 0777)
	return
}

func actionSetup(c *cli.Context) {

	// define git source to update from
	rootPath := mustGitRootPath(c.String("gitrepo"))
	filename := path.Join(rootPath, ".git/hooks/post-checkout")

	if command := c.String("command"); command != "" {
		// if file not exists, create the file
		if _, err := os.Stat(filename); os.IsNotExist(err) {
			createHookScript(filename, command)
			return
		} else if c.Bool("force") {
			createHookScript(filename, command)
			return
		}
		fmt.Println("post-checkout script already exists. If you want to " +
			"overwrite, please use the -f flag")
		os.Exit(1)
		return
	}

	// if file not exists, create the file
	if _, err := os.Stat(filename); os.IsNotExist(err) {
		createHookScript(filename, "exec echo \"checkout completed.\"\n")
	}

	cmd := exec.Command("vi", filename)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stdout
	if err := cmd.Run(); err != nil {
		log.Fatalf("error: %s", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"os"
	"strings"
	"text/tabwriter"

	"github.com/bgentry/heroku-go"
)

var cmdAddons = &Command{
	Run:      runAddons,
	Usage:    "addons [<service>:<plan>...]",
	NeedsApp: true,
	Category: "add-on",
	Short:    "list addons",
	Long: `
Lists addons.

Examples:

    $ hk addons
    heroku-postgresql-blue  heroku-postgresql:crane  Nov 19 12:40
    pgbackups               pgbackups:plus           Sep 30 15:43

    $ hk addons pgbackups
    pgbackups  pgbackups:plus  Sep 30 15:43
`,
}

func runAddons(cmd *Command, names []string) {
	w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
	defer w.Flush()

	appname := mustApp()
	addons, err := client.AddonList(appname, nil)
	if err != nil {
		printError(err.Error())
	}
	for i, s := range names {
		names[i] = strings.ToLower(s)
	}
	for _, a := range addons {
		if len(names) == 0 || addonMatch(a, names) {
			listAddon(w, a)
		}
	}
}

func addonMatch(a heroku.Addon, names []string) bool {
	for _, name := range names {
		if name == strings.ToLower(a.Name) {
			return true
		}
		if name == strings.ToLower(a.Plan.Name) {
			return true
		}
		if name == strings.ToLower(a.Id) {
			return true
		}
	}
	return false
}

func listAddon(w io.Writer, a heroku.Addon) {
	name := a.Name
	if name == "" {
		name = "[unnamed]"
	}
	listRec(w,
		name,
		a.Plan.Name,
		prettyTime{a.CreatedAt},
	)
}

var cmdAddonAdd = &Command{
	Run:      runAddonAdd,
	Usage:    "addon-add <service>[:<plan>] [<config>=<value>...]",
	NeedsApp: true,
	Category: "add-on",
	Short:    "add an addon",
	Long: `
Adds an addon to an app.

Examples:

    $ hk addon-add heroku-postgresql
    Added heroku-postgresql:hobby-dev to myapp.

    $ hk addon-add heroku-postgresql:standard-tengu
    Added heroku-postgresql:standard-tengu to myapp.
`,
}

func runAddonAdd(cmd *Command, args []string) {
	appname := mustApp()
	if len(args) == 0 {
		cmd.printUsage()
		os.Exit(2)
	}
	plan := args[0]
	var opts heroku.AddonCreateOpts
	if len(args) > 1 {
		config, err := parseAddonAddConfig(args[1:])
		if err != nil {
			log.Println(err)
			os.Exit(2)
		}
		opts = heroku.AddonCreateOpts{Config: config}
	}
	addon, err := client.AddonCreate(appname, plan, &opts)
	must(err)
	log.Printf("Added %s to %s.", addon.Plan.Name, appname)
}

func parseAddonAddConfig(config []string) (*map[string]string, error) {
	conf := make(map[string]string, len(config))
	for _, kv := range config {
		iEq := strings.IndexRune(kv, '=')
		if iEq < 1 || len(kv) < iEq+2 {
			return nil, fmt.Errorf("Invalid option '%s', must be of form 'key=value'", kv)
		}
		val := kv[iEq+1:]
		if val[0] == '\'' {
			val = strings.Trim(val, "'")
		} else if val[0] == '"' {
			val = strings.Trim(val, "\"")
		}
		conf[kv[:iEq]] = val
	}
	return &conf, nil
}

var cmdAddonRemove = &Command{
	Run:      runAddonRemove,
	Usage:    "addon-remove <name>",
	NeedsApp: true,
	Category: "add-on",
	Short:    "remove an addon",
	Long: `
Removes an addon from an app.

Examples:

    $ hk addon-remove heroku-postgresql-blue
    Removed heroku-postgresql-blue from myapp.

    $ hk addon-remove redistogo
    Removed redistogo from myapp.
`,
}

func runAddonRemove(cmd *Command, args []string) {
	appname := mustApp()
	if len(args) != 1 {
		cmd.printUsage()
		os.Exit(2)
	}
	name := args[0]
	if strings.IndexRune(name, ':') != -1 {
		// specified an addon with plan name, unsupported in v3
		log.Println("Please specify an addon name, not a plan name.")
		cmd.printUsage()
		os.Exit(2)
	}
	checkAddonError(client.AddonDelete(appname, name))
	log.Printf("Removed %s from %s.", name, appname)
}

var cmdAddonOpen = &Command{
	Run:      runAddonOpen,
	Usage:    "addon-open <name>",
	NeedsApp: true,
	Category: "add-on",
	Short:    "open an addon" + extra,
	Long: `
Open the addon's management page in your default web browser.

Examples:

    $ hk addon-open heroku-postgresql-blue

    $ hk addon-open redistogo
`,
}

func runAddonOpen(cmd *Command, args []string) {
	appname := mustApp()
	if len(args) != 1 {
		cmd.printUsage()
		os.Exit(2)
	}
	name := args[0]
	// look up addon to make sure it exists and to get plan name
	a, err := client.AddonInfo(appname, name)
	checkAddonError(err)
	must(openURL("https://addons-sso.heroku.com/apps/" + appname + "/addons/" + a.Plan.Name))
}

func checkAddonError(err error) {
	if err != nil {
		if hkerr, ok := err.(heroku.Error); ok && hkerr.Id == "not_found" {
			printError(err.Error() + " Choose an addon name from `hk addons`.")
		} else {
			printError(err.Error())
		}
		os.Exit(2)
	}
}
<commit_msg>tell the user the name of their addon when they've added it<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"os"
	"strings"
	"text/tabwriter"

	"github.com/bgentry/heroku-go"
)

var cmdAddons = &Command{
	Run:      runAddons,
	Usage:    "addons [<service>:<plan>...]",
	NeedsApp: true,
	Category: "add-on",
	Short:    "list addons",
	Long: `
Lists addons.

Examples:

    $ hk addons
    heroku-postgresql-blue  heroku-postgresql:crane  Nov 19 12:40
    pgbackups               pgbackups:plus           Sep 30 15:43

    $ hk addons pgbackups
    pgbackups  pgbackups:plus  Sep 30 15:43
`,
}

func runAddons(cmd *Command, names []string) {
	w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
	defer w.Flush()

	appname := mustApp()
	addons, err := client.AddonList(appname, nil)
	if err != nil {
		printError(err.Error())
	}
	for i, s := range names {
		names[i] = strings.ToLower(s)
	}
	for _, a := range addons {
		if len(names) == 0 || addonMatch(a, names) {
			listAddon(w, a)
		}
	}
}

func addonMatch(a heroku.Addon, names []string) bool {
	for _, name := range names {
		if name == strings.ToLower(a.Name) {
			return true
		}
		if name == strings.ToLower(a.Plan.Name) {
			return true
		}
		if name == strings.ToLower(a.Id) {
			return true
		}
	}
	return false
}

func listAddon(w io.Writer, a heroku.Addon) {
	name := a.Name
	if name == "" {
		name = "[unnamed]"
	}
	listRec(w,
		name,
		a.Plan.Name,
		prettyTime{a.CreatedAt},
	)
}

var cmdAddonAdd = &Command{
	Run:      runAddonAdd,
	Usage:    "addon-add <service>[:<plan>] [<config>=<value>...]",
	NeedsApp: true,
	Category: "add-on",
	Short:    "add an addon",
	Long: `
Adds an addon to an app.

Examples:

    $ hk addon-add heroku-postgresql
    Added heroku-postgresql:hobby-dev to myapp as heroku-postgresql-yellow.

    $ hk addon-add heroku-postgresql:standard-tengu
    Added heroku-postgresql:standard-tengu to myapp as heroku-postgresql-orange.
`,
}

func runAddonAdd(cmd *Command, args []string) {
	appname := mustApp()
	if len(args) == 0 {
		cmd.printUsage()
		os.Exit(2)
	}
	plan := args[0]
	var opts heroku.AddonCreateOpts
	if len(args) > 1 {
		config, err := parseAddonAddConfig(args[1:])
		if err != nil {
			log.Println(err)
			os.Exit(2)
		}
		opts = heroku.AddonCreateOpts{Config: config}
	}
	addon, err := client.AddonCreate(appname, plan, &opts)
	must(err)
	log.Printf("Added %s to %s as %s.", addon.Plan.Name, appname, addon.Name)
}

func parseAddonAddConfig(config []string) (*map[string]string, error) {
	conf := make(map[string]string, len(config))
	for _, kv := range config {
		iEq := strings.IndexRune(kv, '=')
		if iEq < 1 || len(kv) < iEq+2 {
			return nil, fmt.Errorf("Invalid option '%s', must be of form 'key=value'", kv)
		}
		val := kv[iEq+1:]
		if val[0] == '\'' {
			val = strings.Trim(val, "'")
		} else if val[0] == '"' {
			val = strings.Trim(val, "\"")
		}
		conf[kv[:iEq]] = val
	}
	return &conf, nil
}

var cmdAddonRemove = &Command{
	Run:      runAddonRemove,
	Usage:    "addon-remove <name>",
	NeedsApp: true,
	Category: "add-on",
	Short:    "remove an addon",
	Long: `
Removes an addon from an app.

Examples:

    $ hk addon-remove heroku-postgresql-blue
    Removed heroku-postgresql-blue from myapp.

    $ hk addon-remove redistogo
    Removed redistogo from myapp.
`,
}

func runAddonRemove(cmd *Command, args []string) {
	appname := mustApp()
	if len(args) != 1 {
		cmd.printUsage()
		os.Exit(2)
	}
	name := args[0]
	if strings.IndexRune(name, ':') != -1 {
		// specified an addon with plan name, unsupported in v3
		log.Println("Please specify an addon name, not a plan name.")
		cmd.printUsage()
		os.Exit(2)
	}
	checkAddonError(client.AddonDelete(appname, name))
	log.Printf("Removed %s from %s.", name, appname)
}

var cmdAddonOpen = &Command{
	Run:      runAddonOpen,
	Usage:    "addon-open <name>",
	NeedsApp: true,
	Category: "add-on",
	Short:    "open an addon" + extra,
	Long: `
Open the addon's management page in your default web browser.

Examples:

    $ hk addon-open heroku-postgresql-blue

    $ hk addon-open redistogo
`,
}

func runAddonOpen(cmd *Command, args []string) {
	appname := mustApp()
	if len(args) != 1 {
		cmd.printUsage()
		os.Exit(2)
	}
	name := args[0]
	// look up addon to make sure it exists and to get plan name
	a, err := client.AddonInfo(appname, name)
	checkAddonError(err)
	must(openURL("https://addons-sso.heroku.com/apps/" + appname + "/addons/" + a.Plan.Name))
}

func checkAddonError(err error) {
	if err != nil {
		if hkerr, ok := err.(heroku.Error); ok && hkerr.Id == "not_found" {
			printError(err.Error() + " Choose an addon name from `hk addons`.")
		} else {
			printError(err.Error())
		}
		os.Exit(2)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"github.com/fatih/color"
	mkr "github.com/mackerelio/mackerel-client-go"
	"github.com/mackerelio/mkr/logger"
)

var commandAlerts = cli.Command{
	Name:  "alerts",
	Usage: "Retrieve/Close alerts",
	Description: `
    Retrieve/Close alerts. Without subcommand, show all alerts.
    Request APIs under "/api/v0/alerts". See http://help-ja.mackerel.io/entry/spec/api/v0 .
`,
	Action: doAlertsRetrieve,
	Subcommands: []cli.Command{
		{
			Name:        "list",
			Usage:       "list alerts",
			Description: "Show alerts with human-readable format.",
			Action:      doAlertsList,
			Flags: []cli.Flag{
				cli.StringSliceFlag{
					Name:  "service, s",
					Value: &cli.StringSlice{},
					Usage: "Filter alerts by service. Multiple choice allow.",
				},
				cli.StringSliceFlag{
					Name:  "host-status, S",
					Value: &cli.StringSlice{},
					Usage: "Filter alerts by status of each host. Multiple choice allow.",
				},
				cli.BoolTFlag{Name: "color, c", Usage: "Colorize output. default: true"},
			},
		},
		{
			Name:        "close",
			Usage:       "close alerts",
			Description: "Close alerts. Multiple alert IDs can be specified.",
			Action:      doAlertsClose,
			Flags: []cli.Flag{
				cli.StringFlag{Name: "reason, r", Value: "", Usage: "Reason of closing alert."},
				cli.BoolFlag{Name: "verbose, v", Usage: "Verbose output mode"},
			},
		},
	},
}

type alertSet struct {
	Alert   *mkr.Alert
	Host    *mkr.Host
	Monitor *mkr.Monitor
}

func joinMonitorsAndHosts(client *mkr.Client, alerts []*mkr.Alert) []*alertSet {
	hostsJSON, err := client.FindHosts(&mkr.FindHostsParam{
		Statuses: []string{"working", "standby", "poweroff", "maintenance"},
	})
	logger.DieIf(err)

	hosts := map[string]*mkr.Host{}
	for _, host := range hostsJSON {
		hosts[host.ID] = host
	}

	monitorsJSON, err := client.FindMonitors()
	logger.DieIf(err)

	monitors := map[string]*mkr.Monitor{}
	for _, monitor := range monitorsJSON {
		monitors[monitor.ID] = monitor
	}

	alertSets := []*alertSet{}
	for _, alert := range alerts {
		alertSets = append(
			alertSets,
			&alertSet{Alert: alert, Host: hosts[alert.HostID], Monitor: monitors[alert.MonitorID]},
		)
	}
	return alertSets
}

func formatJoinedAlert(alertSet *alertSet, colorize bool) string {
	const layout = "2006-01-02 15:04:05"

	host := alertSet.Host
	monitor := alertSet.Monitor
	alert := alertSet.Alert

	hostMsg := ""
	if host != nil {
		statusMsg := host.Status
		if host.IsRetired == true {
			statusMsg = "retired"
		}
		if colorize {
			switch statusMsg {
			case "working":
				statusMsg = color.BlueString("working")
			case "standby":
				statusMsg = color.GreenString("standby")
			case "poweroff":
				statusMsg = "poweroff"
			case "maintenance":
				statusMsg = color.YellowString("maintenance")
			}
		}
		hostMsg = fmt.Sprintf(" %s %s", host.Name, statusMsg)
		roleMsgs := []string{}
		for service, roles := range host.Roles {
			roleMsgs = append(roleMsgs, fmt.Sprintf("%s:%s", service, strings.Join(roles, ",")))
		}
		hostMsg += " [" + strings.Join(roleMsgs, ", ") + "]"
	}

	monitorMsg := ""
	if monitor != nil {
		switch monitor.Type {
		case "connectivity":
			monitorMsg = fmt.Sprintf("%s", monitor.Type)
		case "host":
			switch alert.Status {
			case "CRITICAL":
				monitorMsg = fmt.Sprintf("%s %.2f %s %.2f", monitor.Metric, alert.Value, monitor.Operator, monitor.Critical)
			case "WARNING":
				monitorMsg = fmt.Sprintf("%s %.2f %s %.2f", monitor.Metric, alert.Value, monitor.Operator, monitor.Warning)
			default:
				monitorMsg = fmt.Sprintf("%s %.2f %s %.2f", monitor.Metric, alert.Value, monitor.Operator, monitor.Critical)
			}
		case "service":
			switch alert.Status {
			case "CRITICAL":
				monitorMsg = fmt.Sprintf("%s %s %.2f %s %.2f", monitor.Service, monitor.Metric, alert.Value, monitor.Operator, monitor.Critical)
			case "WARNING":
				monitorMsg = fmt.Sprintf("%s %s %.2f %s %.2f", monitor.Service, monitor.Metric, alert.Value, monitor.Operator, monitor.Warning)
			default:
				monitorMsg = fmt.Sprintf("%s %s %.2f %s %.2f", monitor.Service, monitor.Metric, alert.Value, monitor.Operator, monitor.Critical)
			}
		case "external":
			statusRegexp, _ := regexp.Compile("^[2345][0-9][0-9]$")
			switch alert.Status {
			case "CRITICAL":
				if statusRegexp.MatchString(alert.Message) {
					monitorMsg = fmt.Sprintf("%s %s %.2f > %.2f msec, status:%s", monitor.Name, monitor.URL, alert.Value, monitor.ResponseTimeCritical, alert.Message)
				} else {
					monitorMsg = fmt.Sprintf("%s %s %.2f msec, %s", monitor.Name, monitor.URL, alert.Value, alert.Message)
				}
			case "WARNING":
				if statusRegexp.MatchString(alert.Message) {
					monitorMsg = fmt.Sprintf("%s %.2f > %.2f msec, status:%s", monitor.Name, alert.Value, monitor.ResponseTimeWarning, alert.Message)
				} else {
					monitorMsg = fmt.Sprintf("%s %.2f msec, %s", monitor.Name, alert.Value, alert.Message)
				}
			default:
				monitorMsg = fmt.Sprintf("%s %.2f > %.2f msec, status:%s", monitor.Name, alert.Value, monitor.ResponseTimeCritical, alert.Message)
			}
		case "check":
			monitorMsg = fmt.Sprintf("%s", monitor.Type)
		default:
			monitorMsg = fmt.Sprintf("%s", monitor.Type)
		}
	}
	statusMsg := alert.Status
	if colorize {
		switch alert.Status {
		case "CRITICAL":
			statusMsg = color.RedString("CRITICAL")
		case "WARNING":
			statusMsg = color.YellowString("WARNING ")
		}
	}
	return fmt.Sprintf("%s %s %s %s%s", alert.ID, time.Unix(alert.OpenedAt, 0).Format(layout), statusMsg, monitorMsg, hostMsg)
}

func doAlertsRetrieve(c *cli.Context) {
	conffile := c.GlobalString("conf")
	client := newMackerel(conffile)

	alerts, err := client.FindAlerts()
	logger.DieIf(err)
	PrettyPrintJSON(alerts)
	return
}

func doAlertsList(c *cli.Context) {
	conffile := c.GlobalString("conf")
	filterServices := c.StringSlice("service")
	filterStatuses := c.StringSlice("status")
	client := newMackerel(conffile)

	alerts, err := client.FindAlerts()
	logger.DieIf(err)
	joinedAlerts := joinMonitorsAndHosts(client, alerts)

	for _, joinAlert := range joinedAlerts {
		if len(filterServices) > 0 {
			found := false
			for _, filterService := range filterServices {
				if joinAlert.Host != nil {
					if _, ok := joinAlert.Host.Roles[filterService]; ok {
						found = true
					}
				} else if joinAlert.Monitor.Service == filterService {
					found = true
				}
			}
			if !found {
				continue
			}
		}
		if len(filterStatuses) > 0 {
			found := false
			for _, filterStatus := range filterStatuses {
				if joinAlert.Host != nil && joinAlert.Host.Status == filterStatus {
					found = true
				}
			}
			if found == false {
				continue
			}
		}
		fmt.Println(formatJoinedAlert(joinAlert, c.BoolT("color")))
	}
}

func doAlertsClose(c *cli.Context) {
	conffile := c.GlobalString("conf")
	isVerbose := c.Bool("verbose")
	argAlertIDs := c.Args()
	reason := c.String("reason")

	if len(argAlertIDs) < 1 {
		cli.ShowCommandHelp(c, "alerts")
		os.Exit(1)
	}

	client := newMackerel(conffile)
	for _, alertID := range argAlertIDs {
		alert, err := client.CloseAlert(alertID, reason)
		logger.DieIf(err)

		logger.Log("Alert closed", alertID)
		if isVerbose == true {
			PrettyPrintJSON(alert)
		}
	}
}
<commit_msg>simplify if condition<commit_after>package main

import (
	"fmt"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"github.com/fatih/color"
	mkr "github.com/mackerelio/mackerel-client-go"
	"github.com/mackerelio/mkr/logger"
)

var commandAlerts = cli.Command{
	Name:  "alerts",
	Usage: "Retrieve/Close alerts",
	Description: `
    Retrieve/Close alerts. Without subcommand, show all alerts.
    Request APIs under "/api/v0/alerts". See http://help-ja.mackerel.io/entry/spec/api/v0 .
`,
	Action: doAlertsRetrieve,
	Subcommands: []cli.Command{
		{
			Name:        "list",
			Usage:       "list alerts",
			Description: "Show alerts with human-readable format.",
			Action:      doAlertsList,
			Flags: []cli.Flag{
				cli.StringSliceFlag{
					Name:  "service, s",
					Value: &cli.StringSlice{},
					Usage: "Filter alerts by service. Multiple choice allow.",
				},
				cli.StringSliceFlag{
					Name:  "host-status, S",
					Value: &cli.StringSlice{},
					Usage: "Filter alerts by status of each host. Multiple choice allow.",
				},
				cli.BoolTFlag{Name: "color, c", Usage: "Colorize output. default: true"},
			},
		},
		{
			Name:        "close",
			Usage:       "close alerts",
			Description: "Close alerts. Multiple alert IDs can be specified.",
			Action:      doAlertsClose,
			Flags: []cli.Flag{
				cli.StringFlag{Name: "reason, r", Value: "", Usage: "Reason of closing alert."},
				cli.BoolFlag{Name: "verbose, v", Usage: "Verbose output mode"},
			},
		},
	},
}

type alertSet struct {
	Alert   *mkr.Alert
	Host    *mkr.Host
	Monitor *mkr.Monitor
}

func joinMonitorsAndHosts(client *mkr.Client, alerts []*mkr.Alert) []*alertSet {
	hostsJSON, err := client.FindHosts(&mkr.FindHostsParam{
		Statuses: []string{"working", "standby", "poweroff", "maintenance"},
	})
	logger.DieIf(err)

	hosts := map[string]*mkr.Host{}
	for _, host := range hostsJSON {
		hosts[host.ID] = host
	}

	monitorsJSON, err := client.FindMonitors()
	logger.DieIf(err)

	monitors := map[string]*mkr.Monitor{}
	for _, monitor := range monitorsJSON {
		monitors[monitor.ID] = monitor
	}

	alertSets := []*alertSet{}
	for _, alert := range alerts {
		alertSets = append(
			alertSets,
			&alertSet{Alert: alert, Host: hosts[alert.HostID], Monitor: monitors[alert.MonitorID]},
		)
	}
	return alertSets
}

func formatJoinedAlert(alertSet *alertSet, colorize bool) string {
	const layout = "2006-01-02 15:04:05"

	host := alertSet.Host
	monitor := alertSet.Monitor
	alert := alertSet.Alert

	hostMsg := ""
	if host != nil {
		statusMsg := host.Status
		if host.IsRetired == true {
			statusMsg = "retired"
		}
		if colorize {
			switch statusMsg {
			case "working":
				statusMsg = color.BlueString("working")
			case "standby":
				statusMsg = color.GreenString("standby")
			case "poweroff":
				statusMsg = "poweroff"
			case "maintenance":
				statusMsg = color.YellowString("maintenance")
			}
		}
		hostMsg = fmt.Sprintf(" %s %s", host.Name, statusMsg)
		roleMsgs := []string{}
		for service, roles := range host.Roles {
			roleMsgs = append(roleMsgs, fmt.Sprintf("%s:%s", service, strings.Join(roles, ",")))
		}
		hostMsg += " [" + strings.Join(roleMsgs, ", ") + "]"
	}

	monitorMsg := ""
	if monitor != nil {
		switch monitor.Type {
		case "connectivity":
			monitorMsg = fmt.Sprintf("%s", monitor.Type)
		case "host":
			switch alert.Status {
			case "CRITICAL":
				monitorMsg = fmt.Sprintf("%s %.2f %s %.2f", monitor.Metric, alert.Value, monitor.Operator, monitor.Critical)
			case "WARNING":
				monitorMsg = fmt.Sprintf("%s %.2f %s %.2f", monitor.Metric, alert.Value, monitor.Operator, monitor.Warning)
			default:
				monitorMsg = fmt.Sprintf("%s %.2f %s %.2f", monitor.Metric, alert.Value, monitor.Operator, monitor.Critical)
			}
		case "service":
			switch alert.Status {
			case "CRITICAL":
				monitorMsg = fmt.Sprintf("%s %s %.2f %s %.2f", monitor.Service, monitor.Metric, alert.Value, monitor.Operator, monitor.Critical)
			case "WARNING":
				monitorMsg = fmt.Sprintf("%s %s %.2f %s %.2f", monitor.Service, monitor.Metric, alert.Value, monitor.Operator, monitor.Warning)
			default:
				monitorMsg = fmt.Sprintf("%s %s %.2f %s %.2f", monitor.Service, monitor.Metric, alert.Value, monitor.Operator, monitor.Critical)
			}
		case "external":
			statusRegexp, _ := regexp.Compile("^[2345][0-9][0-9]$")
			switch alert.Status {
			case "CRITICAL":
				if statusRegexp.MatchString(alert.Message) {
					monitorMsg = fmt.Sprintf("%s %s %.2f > %.2f msec, status:%s", monitor.Name, monitor.URL, alert.Value, monitor.ResponseTimeCritical, alert.Message)
				} else {
					monitorMsg = fmt.Sprintf("%s %s %.2f msec, %s", monitor.Name, monitor.URL, alert.Value, alert.Message)
				}
			case "WARNING":
				if statusRegexp.MatchString(alert.Message) {
					monitorMsg = fmt.Sprintf("%s %.2f > %.2f msec, status:%s", monitor.Name, alert.Value, monitor.ResponseTimeWarning, alert.Message)
				} else {
					monitorMsg = fmt.Sprintf("%s %.2f msec, %s", monitor.Name, alert.Value, alert.Message)
				}
			default:
				monitorMsg = fmt.Sprintf("%s %.2f > %.2f msec, status:%s", monitor.Name, alert.Value, monitor.ResponseTimeCritical, alert.Message)
			}
		case "check":
			monitorMsg = fmt.Sprintf("%s", monitor.Type)
		default:
			monitorMsg = fmt.Sprintf("%s", monitor.Type)
		}
	}
	statusMsg := alert.Status
	if colorize {
		switch alert.Status {
		case "CRITICAL":
			statusMsg = color.RedString("CRITICAL")
		case "WARNING":
			statusMsg = color.YellowString("WARNING ")
		}
	}
	return fmt.Sprintf("%s %s %s %s%s", alert.ID, time.Unix(alert.OpenedAt, 0).Format(layout), statusMsg, monitorMsg, hostMsg)
}

func doAlertsRetrieve(c *cli.Context) {
	conffile := c.GlobalString("conf")
	client := newMackerel(conffile)

	alerts, err := client.FindAlerts()
	logger.DieIf(err)
	PrettyPrintJSON(alerts)
	return
}

func doAlertsList(c *cli.Context) {
	conffile := c.GlobalString("conf")
	filterServices := c.StringSlice("service")
	filterStatuses := c.StringSlice("status")
	client := newMackerel(conffile)

	alerts, err := client.FindAlerts()
	logger.DieIf(err)
	joinedAlerts := joinMonitorsAndHosts(client, alerts)

	for _, joinAlert := range joinedAlerts {
		if len(filterServices) > 0 {
			found := false
			for _, filterService := range filterServices {
				if joinAlert.Host != nil {
					if _, ok := joinAlert.Host.Roles[filterService]; ok {
						found = true
					}
				} else if joinAlert.Monitor.Service == filterService {
					found = true
				}
			}
			if !found {
				continue
			}
		}
		if len(filterStatuses) > 0 {
			found := false
			for _, filterStatus := range filterStatuses {
				if joinAlert.Host != nil && joinAlert.Host.Status == filterStatus {
					found = true
				}
			}
			if !found {
				continue
			}
		}
		fmt.Println(formatJoinedAlert(joinAlert, c.BoolT("color")))
	}
}

func doAlertsClose(c *cli.Context) {
	conffile := c.GlobalString("conf")
	isVerbose := c.Bool("verbose")
	argAlertIDs := c.Args()
	reason := c.String("reason")

	if len(argAlertIDs) < 1 {
		cli.ShowCommandHelp(c, "alerts")
		os.Exit(1)
	}

	client := newMackerel(conffile)
	for _, alertID := range argAlertIDs {
		alert, err := client.CloseAlert(alertID, reason)
		logger.DieIf(err)

		logger.Log("Alert closed", alertID)
		if isVerbose == true {
			PrettyPrintJSON(alert)
		}
	}
}
<|endoftext|>


<commit_before>package spotify

import (
	"fmt"
	"net/url"
	"strconv"
	"strings"
)

// SimpleArtist contains basic info about an artist.
type SimpleArtist struct {
	Name string `json:"name"`
	ID   ID     `json:"id"`
	// The Spotify URI for the artist.
	URI URI `json:"uri"`
	// A link to the Web API enpoint providing full details of the artist.
	Endpoint     string            `json:"href"`
	ExternalURLs map[string]string `json:"external_urls"`
}

// FullArtist provides extra artist data in addition to what is provided by SimpleArtist.
type FullArtist struct {
	SimpleArtist
	// The popularity of the artist, expressed as an integer between 0 and 100.
	// The artist's popularity is calculated from the popularity of the artist's tracks.
	Popularity int `json:"popularity"`
	// A list of genres the artist is associated with.  For example, "Prog Rock"
	// or "Post-Grunge".  If not yet classified, the slice is empty.
	Genres    []string  `json:"genres"`
	Followers Followers `json:"followers"`
	// Images of the artist in various sizes, widest first.
	Images []Image `json:"images"`
}

// GetArtist gets Spotify catalog information for a single artist, given its Spotify ID.
func (c *Client) GetArtist(id ID) (*FullArtist, error) {
	spotifyURL := fmt.Sprintf("%sartists/%s", c.baseURL, id)

	var a FullArtist
	err := c.get(spotifyURL, &a)
	if err != nil {
		return nil, err
	}

	return &a, nil
}

// GetArtists gets spotify catalog information for several artists based on their
// Spotify IDs.  It supports up to 50 artists in a single call.  Artists are
// returned in the order requested.  If an artist is not found, that position
// in the result will be nil.  Duplicate IDs will result in duplicate artists
// in the result.
func (c *Client) GetArtists(ids ...ID) ([]*FullArtist, error) {
	spotifyURL := fmt.Sprintf("%sartists?ids=%s", c.baseURL, strings.Join(toStringSlice(ids), ","))

	var a struct {
		Artists []*FullArtist
	}

	err := c.get(spotifyURL, &a)
	if err != nil {
		return nil, err
	}

	return a.Artists, nil
}

// GetArtistsTopTracks gets Spotify catalog information about an artist's top
// tracks in a particular country.  It returns a maximum of 10 tracks.  The
// country is specified as an ISO 3166-1 alpha-2 country code.
func (c *Client) GetArtistsTopTracks(artistID ID, country string) ([]FullTrack, error) {
	spotifyURL := fmt.Sprintf("%sartists/%s/top-tracks?country=%s", c.baseURL, artistID, country)

	var t struct {
		Tracks []FullTrack `json:"tracks"`
	}

	err := c.get(spotifyURL, &t)
	if err != nil {
		return nil, err
	}

	return t.Tracks, nil
}

// GetRelatedArtists gets Spotify catalog information about artists similar to a
// given artist.  Similarity is based on analysis of the Spotify community's
// listening history.  This function returns up to 20 artists that are considered
// related to the specified artist.
func (c *Client) GetRelatedArtists(id ID) ([]FullArtist, error) {
	spotifyURL := fmt.Sprintf("%sartists/%s/related-artists", c.baseURL, id)

	var a struct {
		Artists []FullArtist `json:"artists"`
	}

	err := c.get(spotifyURL, &a)
	if err != nil {
		return nil, err
	}

	return a.Artists, nil
}

// GetArtistAlbums gets Spotify catalog information about an artist's albums.
// It is equivalent to GetArtistAlbumsOpt(artistID, nil).
func (c *Client) GetArtistAlbums(artistID ID) (*SimpleAlbumPage, error) {
	return c.GetArtistAlbumsOpt(artistID, nil, nil)
}

// GetArtistAlbumsOpt is just like GetArtistAlbums, but it accepts optional
// parameters used to filter and sort the result.
//
// The AlbumType argument can be used to find a particular types of album.
// If the market (Options) is not specified, Spotify will likely return a lot
// of duplicates (one for each market in which the album is available)
func (c *Client) GetArtistAlbumsOpt(artistID ID, options *Options, ts ...*AlbumType) (*SimpleAlbumPage, error) {
	spotifyURL := fmt.Sprintf("%sartists/%s/albums", c.baseURL, artistID)
	// add optional query string if options were specified
	values := url.Values{}
	if ts != nil {
		types := make([]string, len(ts))
		for i := range ts {
			types[i] = ts[i].encode()
		}
		values.Set("album_type", strings.Join(types, ","))
	}
	if options != nil {
		if options.Country != nil {
			values.Set("market", *options.Country)
		}
		if options.Limit != nil {
			values.Set("limit", strconv.Itoa(*options.Limit))
		}
		if options.Offset != nil {
			values.Set("offset", strconv.Itoa(*options.Offset))
		}
	}
	if query := values.Encode(); query != "" {
		spotifyURL += "?" + query
	}

	var p SimpleAlbumPage

	err := c.get(spotifyURL, &p)
	if err != nil {
		return nil, err
	}

	return &p, nil
}
<commit_msg>bug: Use the correct param for album type<commit_after>package spotify

import (
	"fmt"
	"net/url"
	"strconv"
	"strings"
)

// SimpleArtist contains basic info about an artist.
type SimpleArtist struct {
	Name string `json:"name"`
	ID   ID     `json:"id"`
	// The Spotify URI for the artist.
	URI URI `json:"uri"`
	// A link to the Web API enpoint providing full details of the artist.
	Endpoint     string            `json:"href"`
	ExternalURLs map[string]string `json:"external_urls"`
}

// FullArtist provides extra artist data in addition to what is provided by SimpleArtist.
type FullArtist struct {
	SimpleArtist
	// The popularity of the artist, expressed as an integer between 0 and 100.
	// The artist's popularity is calculated from the popularity of the artist's tracks.
	Popularity int `json:"popularity"`
	// A list of genres the artist is associated with.  For example, "Prog Rock"
	// or "Post-Grunge".  If not yet classified, the slice is empty.
	Genres    []string  `json:"genres"`
	Followers Followers `json:"followers"`
	// Images of the artist in various sizes, widest first.
	Images []Image `json:"images"`
}

// GetArtist gets Spotify catalog information for a single artist, given its Spotify ID.
func (c *Client) GetArtist(id ID) (*FullArtist, error) {
	spotifyURL := fmt.Sprintf("%sartists/%s", c.baseURL, id)

	var a FullArtist
	err := c.get(spotifyURL, &a)
	if err != nil {
		return nil, err
	}

	return &a, nil
}

// GetArtists gets spotify catalog information for several artists based on their
// Spotify IDs.  It supports up to 50 artists in a single call.  Artists are
// returned in the order requested.  If an artist is not found, that position
// in the result will be nil.  Duplicate IDs will result in duplicate artists
// in the result.
func (c *Client) GetArtists(ids ...ID) ([]*FullArtist, error) {
	spotifyURL := fmt.Sprintf("%sartists?ids=%s", c.baseURL, strings.Join(toStringSlice(ids), ","))

	var a struct {
		Artists []*FullArtist
	}

	err := c.get(spotifyURL, &a)
	if err != nil {
		return nil, err
	}

	return a.Artists, nil
}

// GetArtistsTopTracks gets Spotify catalog information about an artist's top
// tracks in a particular country.  It returns a maximum of 10 tracks.  The
// country is specified as an ISO 3166-1 alpha-2 country code.
func (c *Client) GetArtistsTopTracks(artistID ID, country string) ([]FullTrack, error) {
	spotifyURL := fmt.Sprintf("%sartists/%s/top-tracks?country=%s", c.baseURL, artistID, country)

	var t struct {
		Tracks []FullTrack `json:"tracks"`
	}

	err := c.get(spotifyURL, &t)
	if err != nil {
		return nil, err
	}

	return t.Tracks, nil
}

// GetRelatedArtists gets Spotify catalog information about artists similar to a
// given artist.  Similarity is based on analysis of the Spotify community's
// listening history.  This function returns up to 20 artists that are considered
// related to the specified artist.
func (c *Client) GetRelatedArtists(id ID) ([]FullArtist, error) {
	spotifyURL := fmt.Sprintf("%sartists/%s/related-artists", c.baseURL, id)

	var a struct {
		Artists []FullArtist `json:"artists"`
	}

	err := c.get(spotifyURL, &a)
	if err != nil {
		return nil, err
	}

	return a.Artists, nil
}

// GetArtistAlbums gets Spotify catalog information about an artist's albums.
// It is equivalent to GetArtistAlbumsOpt(artistID, nil).
func (c *Client) GetArtistAlbums(artistID ID) (*SimpleAlbumPage, error) {
	return c.GetArtistAlbumsOpt(artistID, nil, nil)
}

// GetArtistAlbumsOpt is just like GetArtistAlbums, but it accepts optional
// parameters used to filter and sort the result.
//
// The AlbumType argument can be used to find a particular types of album.
// If the market (Options) is not specified, Spotify will likely return a lot
// of duplicates (one for each market in which the album is available)
func (c *Client) GetArtistAlbumsOpt(artistID ID, options *Options, ts ...*AlbumType) (*SimpleAlbumPage, error) {
	spotifyURL := fmt.Sprintf("%sartists/%s/albums", c.baseURL, artistID)
	// add optional query string if options were specified
	values := url.Values{}
	if ts != nil {
		types := make([]string, len(ts))
		for i := range ts {
			types[i] = ts[i].encode()
		}
		values.Set("include_groups", strings.Join(types, ","))
	}
	if options != nil {
		if options.Country != nil {
			values.Set("market", *options.Country)
		}
		if options.Limit != nil {
			values.Set("limit", strconv.Itoa(*options.Limit))
		}
		if options.Offset != nil {
			values.Set("offset", strconv.Itoa(*options.Offset))
		}
	}
	if query := values.Encode(); query != "" {
		spotifyURL += "?" + query
	}

	var p SimpleAlbumPage

	err := c.get(spotifyURL, &p)
	if err != nil {
		return nil, err
	}

	return &p, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"io"
	"regexp"
	"strings"
	"flag"
	"time"
	"compress/gzip"
)

const TIME_LAYOUT string = "[2006-01-02 15:04:05 MST]"
var job_regexp       *regexp.Regexp = regexp.MustCompile("^P[0-9]+(DJ|PW)[0-9]*")
var timestamp_regexp *regexp.Regexp = regexp.MustCompile("^(\\[[0-9-]+ [0-9:]+ UTC\\])")
var request_regexp   *regexp.Regexp = regexp.MustCompile("\\] (P[0-9]+[A-Za-z]+[0-9]+) ")

func main() {
	hide_jobs_flag := flag.Bool("hide_jobs", false, "Hide background jobs")
	hide_sql_flag  := flag.Bool("hide_sql", false, "Hide SQL statements")
	hide_ntlm_flag := flag.Bool("hide_ntlm", false, "Hide NTLM lines")
	full_flag      := flag.Bool("full", false, "Show the full request/job for each found line")
	neat_flag      := flag.Bool("neat", false, "Hide clutter - equivalent to -hide_jobs -hide_sql -hide_ntlm")
	after_str      := flag.String("after", "", "Show logs after this time (YYYY-MM-DD HH:II::SS")
	find_str       := flag.String("find", "", "Find lines matching this regexp")

	flag.Parse()
	args := flag.Args()

	// Time layouts must use the
	// reference time `Mon Jan 2 15:04:05 MST 2006` to show the
	// pattern with which to format/parse a given time/string

	time_after, e    := time.Parse(TIME_LAYOUT, fmt.Sprintf("[%s UTC]", *after_str))
	parse_time       := false

	if e != nil {
		if len(*after_str) > 0 {
			msg(fmt.Sprintf("Invalid time format \"%s\" - Must be YYYY-MM-DD HH::II::SS", *after_str))
			usage()
			os.Exit(2)
		}
	} else {
		parse_time = true
	}

	if len(args) < 1 {
		usage()
		os.Exit(2)
	}

	if *neat_flag {
		*hide_jobs_flag = true
		*hide_sql_flag = true
		*hide_ntlm_flag = true
	}

	msg(fmt.Sprintf("Show full requests/jobs: %t", *full_flag))
	msg(fmt.Sprintf("Show background job lines: %t", !*hide_jobs_flag))
	msg(fmt.Sprintf("Show SQL lines: %t", !*hide_sql_flag))
	msg(fmt.Sprintf("Show NTLM lines: %t", !*hide_ntlm_flag))
	msg(fmt.Sprintf("Show lines after: %s", *after_str))

	filename := args[0]
	msg(fmt.Sprintf("Opening file: %s", filename))

	file := openFile(filename)
	defer file.Close()

	is_gzip      := isGzip(filename)
	file_size    := float64(fileSize(file))
	show_percent := !is_gzip
	var read_size int64 = 0

	var reader io.Reader = file

	sql_regexp := regexp.MustCompile("(SQL \\()|(EXEC sp_executesql N)|( CACHE \\()")
	ntlm_regexp := regexp.MustCompile(" \\(NTLM\\) ")

	var unique_map map[string]bool;

	line_strexp := *find_str

	if line_regexp, err := regexp.Compile(line_strexp); *full_flag && len(line_strexp) > 0 && err == nil {
		if is_gzip {
			// for some reason if you create a reader but don't use it,
			// an error is given when the output reader is created below
			parse_gz_reader := getGzipReader(file)
			defer parse_gz_reader.Close()

			reader = parse_gz_reader
		}

		line_count  := 0
		line_after  := !parse_time // if not parsing time, then all lines are valid
		request_ids := make([]string, 0)

		scanner := bufio.NewScanner(reader);

		for scanner.Scan() {
			line := scanner.Text();
			if line_regexp.MatchString(line) {

				if !line_after {
					if timestamp := extractTimestamp(line); len(timestamp) > 1 {
						if isAfterTime(timestamp, &time_after) {
							line_after = true
						}
					}
				}

				if line_after {
					if request_id := extractRequestId(line); len(request_id) > 1 {
						if !*hide_jobs_flag || !isJob(request_id) {
							request_ids = append(request_ids, request_id)
						}
					}
				}
			}

			read_size += int64(len(line))

			if line_count++; line_count % 20000 == 0 {
				if show_percent {
					showPercent(line_count, float64(read_size) / file_size, line_after, len(request_ids))
				} else {
					showBytes(line_count, float64(read_size), line_after, len(request_ids))
				}
			}
		}

		file_size = float64(read_size) // set the filesize to the total known size
		msg("") // empty line

		if err := scanner.Err(); err != nil {
			log.Fatal(err)
		}

		msg(fmt.Sprintf("Found %d lines matching \"%s\"", len(request_ids), line_strexp))
		unique_map = generateRequestIdMap(&request_ids)

		if len(unique_map) < 1 {
			msg(fmt.Sprintf("Found 0 request identifiers for \"%s\"", line_strexp))
			os.Exit(2)
		}

		rewindFile(file)
	} else {
		msg("Not printing -full requests, skipping request collection phase")
	}

	if is_gzip {
		output_gz_reader := getGzipReader(file)
		defer output_gz_reader.Close()

		reader = output_gz_reader
	}

	show_percent = read_size > int64(0)
	read_size = 0

	line_count := 0
	line_after := !parse_time // if not parsing time, then all lines are valid
	has_requests := len(unique_map) > 0

	line_regexp, err := regexp.Compile(line_strexp);
	has_matcher      := len(line_strexp) > 0 && err == nil

	output_scanner := bufio.NewScanner(reader);

	for output_scanner.Scan() {
		line := output_scanner.Text();

		output := false

		if !line_after {
			read_size += int64(len(line))

			if line_count++; line_count % 5000 == 0 {
				if show_percent {
					fmt.Fprint(os.Stderr, fmt.Sprintf("Reading: %.2f%%\r", (float64(read_size) / file_size) * 100))
				} else {
					fmt.Fprint(os.Stderr, fmt.Sprintf("Reading: %d lines, %0.3f GB\r", line_count, float64(read_size)/1024/1024/1024))
				}
			}

			if timestamp := extractTimestamp(line); len(timestamp) > 1 {
				if isAfterTime(timestamp, &time_after) {
					msg("\n") // empty line
					line_after = true
				}
			}
		}

		if line_after {
			request_id := extractRequestId(line)

			if has_requests {
				if len(request_id) > 0 && unique_map[request_id] {
					if *hide_jobs_flag && isJob(request_id) {
						output = false
					} else {
						output = true
					}
				}
			} else if has_matcher {
				output = line_regexp.MatchString(line)
			} else {
				output = true
			}
		}

		if output {
			if *hide_sql_flag && sql_regexp.MatchString(line) {
				output = false
			} else if *hide_ntlm_flag && ntlm_regexp.MatchString(line) {
				output = false
			}
		}

		if output {
			fmt.Println(line)
		}
	}

	if err := output_scanner.Err(); err != nil {
		log.Fatal(err)
	}
}

func usage() {
	msg("This tool can be used to extract the logs for specific requests from an AppVolumes Manager log")
	msg("")
	msg("Example:avmlog -after=\"2015-10-19 09:00:00\" -find \"apvuser2599\" -full -neat ~/Documents/scale.log.gz")
	msg("")
	flag.PrintDefaults()
	msg("")
}

func isAfterTime(timestamp string, time_after *time.Time) bool {
	if line_time, e := time.Parse(TIME_LAYOUT, timestamp); e != nil {
		msg(fmt.Sprintf("Got error %s", e))
		return false
	} else if line_time.Before(*time_after) {
		return false
	}

	return true
}

func isJob(request_id string) bool {
	return job_regexp.MatchString(request_id)
}

func extractTimestamp(line string) string {
	if timestamp_match := timestamp_regexp.FindStringSubmatch(line); len(timestamp_match) > 1 {
		return timestamp_match[1]
	} else {
		return ""
	}
}

func extractRequestId(line string) string {
	if request_match := request_regexp.FindStringSubmatch(line); len(request_match) > 1 {
		return request_match[1]
	} else {
		return ""
	}
}

func generateRequestIdMap(request_ids *[]string) map[string]bool {
	unique_map := make(map[string]bool, len(*request_ids))

	for _, x := range *request_ids {
		unique_map[x] = true
	}

	for k, _ := range unique_map {
		msg(fmt.Sprintf("Request ID: %s", k))
	}

	return unique_map
}

func openFile(filename string) *os.File {
	file, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	return file
}

func fileSize(file *os.File) int64 {
	if fi, err := file.Stat(); err != nil {
		msg("Unable to determine file size")

		return 1;
	} else {
		msg(fmt.Sprintf("The file is %d bytes long", fi.Size()))

		return fi.Size();
	}
}

func isGzip(filename string) bool {
	return strings.HasSuffix(filename, ".gz")
}

func getGzipReader(file *os.File) *gzip.Reader {
	gz_reader, err := gzip.NewReader(file)
	if err != nil {
	log.Fatal(err)
	}

	return gz_reader
}

func rewindFile(file *os.File) {
	file.Seek(0, 0)  // go back to the top (rewind)
}

func msg(output string) {
	fmt.Fprintln(os.Stderr, output)
}

func showPercent(line_count int, position float64, after bool, matches int) {
	fmt.Fprintf(
		os.Stderr,
		"Reading: %d lines, %.2f%% (after: %v, matches: %d)\r",
		line_count,
		position * 100,
		after,
		matches)
}

func showBytes(line_count int, position float64, after bool, matches int) {
	fmt.Fprintf(
		os.Stderr,
		"Reading: %d lines, %0.3f GB (after: %v, matches: %d)\r",
		line_count,
		position / 1024 / 1024 / 1024,
		after,
		matches)
}<commit_msg>Make sql and ntlm regexp's global variables<commit_after>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"io"
	"regexp"
	"strings"
	"flag"
	"time"
	"compress/gzip"
)

// Time layouts must use the reference time `Mon Jan 2 15:04:05 MST 2006` to
// convey the pattern with which to format/parse a given time/string
const TIME_LAYOUT string = "[2006-01-02 15:04:05 MST]"

var job_regexp       *regexp.Regexp = regexp.MustCompile("^P[0-9]+(DJ|PW)[0-9]*")
var timestamp_regexp *regexp.Regexp = regexp.MustCompile("^(\\[[0-9-]+ [0-9:]+ UTC\\])")
var request_regexp   *regexp.Regexp = regexp.MustCompile("\\] (P[0-9]+[A-Za-z]+[0-9]+) ")
var sql_regexp       *regexp.Regexp = regexp.MustCompile("(SQL \\()|(EXEC sp_executesql N)|( CACHE \\()")
var ntlm_regexp      *regexp.Regexp = regexp.MustCompile(" \\(NTLM\\) ")

func main() {
	hide_jobs_flag := flag.Bool("hide_jobs", false, "Hide background jobs")
	hide_sql_flag  := flag.Bool("hide_sql", false, "Hide SQL statements")
	hide_ntlm_flag := flag.Bool("hide_ntlm", false, "Hide NTLM lines")
	full_flag      := flag.Bool("full", false, "Show the full request/job for each found line")
	neat_flag      := flag.Bool("neat", false, "Hide clutter - equivalent to -hide_jobs -hide_sql -hide_ntlm")
	after_str      := flag.String("after", "", "Show logs after this time (YYYY-MM-DD HH:II::SS")
	find_str       := flag.String("find", "", "Find lines matching this regexp")

	flag.Parse()
	args := flag.Args()

	time_after, e    := time.Parse(TIME_LAYOUT, fmt.Sprintf("[%s UTC]", *after_str))
	parse_time       := false

	if e != nil {
		if len(*after_str) > 0 {
			msg(fmt.Sprintf("Invalid time format \"%s\" - Must be YYYY-MM-DD HH::II::SS", *after_str))
			usage()
			os.Exit(2)
		}
	} else {
		parse_time = true
	}

	if len(args) < 1 {
		usage()
		os.Exit(2)
	}

	if *neat_flag {
		*hide_jobs_flag = true
		*hide_sql_flag = true
		*hide_ntlm_flag = true
	}

	msg(fmt.Sprintf("Show full requests/jobs: %t", *full_flag))
	msg(fmt.Sprintf("Show background job lines: %t", !*hide_jobs_flag))
	msg(fmt.Sprintf("Show SQL lines: %t", !*hide_sql_flag))
	msg(fmt.Sprintf("Show NTLM lines: %t", !*hide_ntlm_flag))
	msg(fmt.Sprintf("Show lines after: %s", *after_str))

	filename := args[0]
	msg(fmt.Sprintf("Opening file: %s", filename))

	file := openFile(filename)
	defer file.Close()

	is_gzip      := isGzip(filename)
	file_size    := float64(fileSize(file))
	show_percent := !is_gzip
	var read_size int64 = 0

	var reader io.Reader = file
	var unique_map map[string]bool;

	line_strexp := *find_str

	if line_regexp, err := regexp.Compile(line_strexp); *full_flag && len(line_strexp) > 0 && err == nil {
		if is_gzip {
			// for some reason if you create a reader but don't use it,
			// an error is given when the output reader is created below
			parse_gz_reader := getGzipReader(file)
			defer parse_gz_reader.Close()

			reader = parse_gz_reader
		}

		line_count  := 0
		line_after  := !parse_time // if not parsing time, then all lines are valid
		request_ids := make([]string, 0)

		scanner := bufio.NewScanner(reader);

		for scanner.Scan() {
			line := scanner.Text();
			if line_regexp.MatchString(line) {

				if !line_after {
					if timestamp := extractTimestamp(line); len(timestamp) > 1 {
						if isAfterTime(timestamp, &time_after) {
							line_after = true
						}
					}
				}

				if line_after {
					if request_id := extractRequestId(line); len(request_id) > 1 {
						if !*hide_jobs_flag || !isJob(request_id) {
							request_ids = append(request_ids, request_id)
						}
					}
				}
			}

			read_size += int64(len(line))

			if line_count++; line_count % 20000 == 0 {
				if show_percent {
					showPercent(line_count, float64(read_size) / file_size, line_after, len(request_ids))
				} else {
					showBytes(line_count, float64(read_size), line_after, len(request_ids))
				}
			}
		}

		file_size = float64(read_size) // set the filesize to the total known size
		msg("") // empty line

		if err := scanner.Err(); err != nil {
			log.Fatal(err)
		}

		msg(fmt.Sprintf("Found %d lines matching \"%s\"", len(request_ids), line_strexp))
		unique_map = generateRequestIdMap(&request_ids)

		if len(unique_map) < 1 {
			msg(fmt.Sprintf("Found 0 request identifiers for \"%s\"", line_strexp))
			os.Exit(2)
		}

		rewindFile(file)
	} else {
		msg("Not printing -full requests, skipping request collection phase")
	}

	if is_gzip {
		output_gz_reader := getGzipReader(file)
		defer output_gz_reader.Close()

		reader = output_gz_reader
	}

	show_percent = read_size > int64(0)
	read_size = 0

	line_count := 0
	line_after := !parse_time // if not parsing time, then all lines are valid
	has_requests := len(unique_map) > 0

	line_regexp, err := regexp.Compile(line_strexp);
	has_matcher      := len(line_strexp) > 0 && err == nil

	output_scanner := bufio.NewScanner(reader);

	for output_scanner.Scan() {
		line := output_scanner.Text();

		output := false

		if !line_after {
			read_size += int64(len(line))

			if line_count++; line_count % 5000 == 0 {
				if show_percent {
					fmt.Fprint(os.Stderr, fmt.Sprintf("Reading: %.2f%%\r", (float64(read_size) / file_size) * 100))
				} else {
					fmt.Fprint(os.Stderr, fmt.Sprintf("Reading: %d lines, %0.3f GB\r", line_count, float64(read_size)/1024/1024/1024))
				}
			}

			if timestamp := extractTimestamp(line); len(timestamp) > 1 {
				if isAfterTime(timestamp, &time_after) {
					msg("\n") // empty line
					line_after = true
				}
			}
		}

		if line_after {
			request_id := extractRequestId(line)

			if has_requests {
				if len(request_id) > 0 && unique_map[request_id] {
					if *hide_jobs_flag && isJob(request_id) {
						output = false
					} else {
						output = true
					}
				}
			} else if has_matcher {
				output = line_regexp.MatchString(line)
			} else {
				output = true
			}
		}

		if output {
			if *hide_sql_flag && sql_regexp.MatchString(line) {
				output = false
			} else if *hide_ntlm_flag && ntlm_regexp.MatchString(line) {
				output = false
			}
		}

		if output {
			fmt.Println(line)
		}
	}

	if err := output_scanner.Err(); err != nil {
		log.Fatal(err)
	}
}

func usage() {
	msg("This tool can be used to extract the logs for specific requests from an AppVolumes Manager log")
	msg("")
	msg("Example:avmlog -after=\"2015-10-19 09:00:00\" -find \"apvuser2599\" -full -neat ~/Documents/scale.log.gz")
	msg("")
	flag.PrintDefaults()
	msg("")
}

func isAfterTime(timestamp string, time_after *time.Time) bool {
	if line_time, e := time.Parse(TIME_LAYOUT, timestamp); e != nil {
		msg(fmt.Sprintf("Got error %s", e))
		return false
	} else if line_time.Before(*time_after) {
		return false
	}

	return true
}

func isJob(request_id string) bool {
	return job_regexp.MatchString(request_id)
}

func extractTimestamp(line string) string {
	if timestamp_match := timestamp_regexp.FindStringSubmatch(line); len(timestamp_match) > 1 {
		return timestamp_match[1]
	} else {
		return ""
	}
}

func extractRequestId(line string) string {
	if request_match := request_regexp.FindStringSubmatch(line); len(request_match) > 1 {
		return request_match[1]
	} else {
		return ""
	}
}

func generateRequestIdMap(request_ids *[]string) map[string]bool {
	unique_map := make(map[string]bool, len(*request_ids))

	for _, x := range *request_ids {
		unique_map[x] = true
	}

	for k, _ := range unique_map {
		msg(fmt.Sprintf("Request ID: %s", k))
	}

	return unique_map
}

func openFile(filename string) *os.File {
	file, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	return file
}

func fileSize(file *os.File) int64 {
	if fi, err := file.Stat(); err != nil {
		msg("Unable to determine file size")

		return 1;
	} else {
		msg(fmt.Sprintf("The file is %d bytes long", fi.Size()))

		return fi.Size();
	}
}

func isGzip(filename string) bool {
	return strings.HasSuffix(filename, ".gz")
}

func getGzipReader(file *os.File) *gzip.Reader {
	gz_reader, err := gzip.NewReader(file)
	if err != nil {
	log.Fatal(err)
	}

	return gz_reader
}

func rewindFile(file *os.File) {
	file.Seek(0, 0)  // go back to the top (rewind)
}

func msg(output string) {
	fmt.Fprintln(os.Stderr, output)
}

func showPercent(line_count int, position float64, after bool, matches int) {
	fmt.Fprintf(
		os.Stderr,
		"Reading: %d lines, %.2f%% (after: %v, matches: %d)\r",
		line_count,
		position * 100,
		after,
		matches)
}

func showBytes(line_count int, position float64, after bool, matches int) {
	fmt.Fprintf(
		os.Stderr,
		"Reading: %d lines, %0.3f GB (after: %v, matches: %d)\r",
		line_count,
		position / 1024 / 1024 / 1024,
		after,
		matches)
}<|endoftext|>


<commit_before>package main

import (
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sync"

	"github.com/gosuri/uiprogress"
	"github.com/gosuri/uiprogress/util/strutil"

	"gopkg.in/kothar/go-backblaze.v0"
)

// TODO support subdirectories
// TODO support destination path
// TODO support version id downloads
type Get struct {
	Threads int `short:"j" long:"threads" default:"5" description:"Maximum simultaneous downloads to process"`
}

func init() {
	parser.AddCommand("get", "Download a file",
		"Downloads one or more files to the current directory. Specify the bucket with -b, and the filenames to download as extra arguments.",
		&Get{})
}

func (o *Get) Execute(args []string) error {
	client, err := Client()
	if err != nil {
		return err
	}

	bucket, err := client.Bucket(opts.Bucket)
	if err != nil {
		return err
	}
	if bucket == nil {
		return errors.New("Bucket not found: " + opts.Bucket)
	}

	uiprogress.Start()
	fmt.Printf("Making a pool for %d downloads\n", o.Threads)
	pool := make(chan bool, o.Threads)
	group := sync.WaitGroup{}
	var downloadError error

	for _, file := range args {
		// TODO handle wildcards

		fileInfo, reader, err := bucket.DownloadFileByName(file)
		if err != nil {
			downloadError = err
			break
		}

		// Get a ticket to process a download
		pool <- true

		if downloadError != nil {
			break
		}

		group.Add(1)
		go func() {
			defer func() {
				// Allow next entry into pool
				group.Done()
				<-pool
			}()

			err := download(fileInfo, reader, file)
			if err != nil {
				fmt.Println(err)
				downloadError = err
			}
		}()
	}

	group.Wait()

	return downloadError
}

type progressWriter struct {
	bar *uiprogress.Bar
	w   io.Writer
}

func (p *progressWriter) Write(b []byte) (int, error) {
	written, err := p.w.Write(b)
	p.bar.Set(p.bar.Current() + written)
	return written, err
}

func download(fileInfo *backblaze.File, reader io.ReadCloser, path string) error {
	defer reader.Close()

	// Display download progress
	bar := uiprogress.AddBar(int(fileInfo.ContentLength))
	bar.AppendFunc(func(b *uiprogress.Bar) string {
		speed := (float32(b.Current()) / 1024) / float32(b.TimeElapsed().Seconds())
		return fmt.Sprintf("%7.2f KB/s", speed)
	})
	bar.AppendCompleted()
	bar.PrependFunc(func(b *uiprogress.Bar) string { return fmt.Sprintf("%10d", b.Total) })
	bar.PrependFunc(func(b *uiprogress.Bar) string { return strutil.Resize(fileInfo.Name, 50) })
	bar.Width = 20

	err := os.MkdirAll(filepath.Dir(path), 0777)
	if err != nil {
		return err
	}

	writer, err := os.Create(path)
	if err != nil {
		return err
	}
	defer writer.Close()

	sha := sha1.New()
	tee := io.MultiWriter(sha, &progressWriter{bar, writer})

	_, err = io.Copy(tee, reader)
	if err != nil {
		return err
	}

	// Check SHA
	sha1Hash := hex.EncodeToString(sha.Sum(nil))
	if sha1Hash != fileInfo.ContentSha1 {
		return errors.New("Downloaded data does not match SHA1 hash")
	}

	return nil
}
<commit_msg>Fix parallel download + Remove progress unless verbose requested<commit_after>package main

import (
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"

	"github.com/gosuri/uiprogress"
	"github.com/gosuri/uiprogress/util/strutil"

	"gopkg.in/kothar/go-backblaze.v0"
)

// TODO support subdirectories
// TODO support destination path
// TODO support version id downloads
type Get struct {
	Threads int `short:"j" long:"threads" default:"5" description:"Maximum simultaneous downloads to process"`
}

func init() {
	parser.AddCommand("get", "Download a file",
		"Downloads one or more files to the current directory. Specify the bucket with -b, and the filenames to download as extra arguments.",
		&Get{})
}

func (o *Get) Execute(args []string) error {
	client, err := Client()
	if err != nil {
		return err
	}

	bucket, err := client.Bucket(opts.Bucket)
	if err != nil {
		return err
	}
	if bucket == nil {
		return errors.New("Bucket not found: " + opts.Bucket)
	}

	uiprogress.Start()
	pool := make(chan bool, o.Threads)
	group := sync.WaitGroup{}
	var downloadError error

	for _, file := range args {
		// TODO handle wildcards

		fileInfo, reader, err := bucket.DownloadFileByName(file)
		if err != nil {
			downloadError = err
			break
		}

		// Get a ticket to process a download
		pool <- true

		if downloadError != nil {
			break
		}

		// Start next parallel download
		group.Add(1)
		go func(fileInfo *backblaze.File, reader io.ReadCloser, path string) {
			err := download(fileInfo, reader, path)
			if err != nil {
				fmt.Println(err)
				downloadError = err
			}

			// Allow next entry into pool
			group.Done()
			<-pool
		}(fileInfo, reader, file)
	}

	group.Wait()

	return downloadError
}

type progressWriter struct {
	bar *uiprogress.Bar
}

func (p *progressWriter) Write(b []byte) (int, error) {
	written := len(b)
	p.bar.Set(p.bar.Current() + written)
	return written, nil
}

func download(fileInfo *backblaze.File, reader io.ReadCloser, path string) error {
	defer reader.Close()

	// Display download progress
	var w io.Writer = ioutil.Discard
	if opts.Verbose {
		bar := uiprogress.AddBar(int(fileInfo.ContentLength))
		bar.AppendFunc(func(b *uiprogress.Bar) string {
			speed := (float32(b.Current()) / 1024) / float32(b.TimeElapsed().Seconds())
			return fmt.Sprintf("%7.2f KB/s", speed)
		})
		bar.AppendCompleted()
		bar.PrependFunc(func(b *uiprogress.Bar) string { return fmt.Sprintf("%10d", b.Total) })
		bar.PrependFunc(func(b *uiprogress.Bar) string { return strutil.Resize(fileInfo.Name, 50) })
		bar.Width = 20

		w = &progressWriter{bar}
	}

	err := os.MkdirAll(filepath.Dir(path), 0777)
	if err != nil {
		return err
	}

	writer, err := os.Create(path)
	if err != nil {
		return err
	}
	defer writer.Close()

	sha := sha1.New()
	tee := io.MultiWriter(sha, writer, w)

	_, err = io.Copy(tee, reader)
	if err != nil {
		return err
	}

	// Check SHA
	sha1Hash := hex.EncodeToString(sha.Sum(nil))
	if sha1Hash != fileInfo.ContentSha1 {
		return errors.New("Downloaded data does not match SHA1 hash")
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	concurrent = flag.Int("j", runtime.NumCPU(), "number of concurrent")
	timeout    = flag.Int64("timeout", 10, "timeout second for HTTP request")
)

var (
	errRedirectNotFollowed = errors.New("redirection not followed")
)

var customTransport http.RoundTripper = &http.Transport{
	Proxy: http.ProxyFromEnvironment,
	Dial: (&net.Dialer{
		Timeout:   30 * time.Second,
		KeepAlive: 30 * time.Second,
	}).Dial,
	TLSHandshakeTimeout: 10 * time.Second,
	MaxIdleConnsPerHost: 3 * http.DefaultMaxIdleConnsPerHost,
}

var httpClient = &http.Client{
	Transport: customTransport,
	CheckRedirect: func(req *http.Request, via []*http.Request) error {
		return errRedirectNotFollowed
	},
	Timeout: time.Duration(*timeout * int64(time.Second)),
}

type responseSet struct {
	res    *http.Response
	urlStr string
}

func main() {
	flag.Parse()

	var stock []*responseSet
	var m sync.Mutex
	printq := make(chan string, *concurrent)
	stockq := make(chan bool, *concurrent)
	waitingq := make(chan bool, *concurrent)
	quitq := make(chan bool)
	eof := false

	go func() {
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			urlStr := strings.TrimRight(scanner.Text(), "\n")
			printq <- urlStr

			go func() {
				waitingq <- true
				res, _ := fetch(urlStr)
				<-waitingq
				m.Lock()
				stock = append(stock, &responseSet{res: res, urlStr: urlStr})
				m.Unlock()
				stockq <- true
			}()
		}
		if err := scanner.Err(); err != nil {
			fmt.Fprintln(os.Stderr, "reading standard input:", err)
		}
		eof = true
	}()

	go func() {
		for {
			urlStr := <-printq
		L:
			for {
				for i, set := range stock {
					if set.urlStr == urlStr {
						fmt.Fprintf(os.Stdout, "%s\t%s\n", edit(set.res), set.urlStr)
						m.Lock()
						stock = append(stock[:i], stock[i+1:]...)
						m.Unlock()
						if eof && len(waitingq) == 0 && len(stock) == 0 {
							quitq <- true
						}
						break L
					}
				}
				<-stockq
			}
		}
	}()

	<-quitq
}

func fetch(urlStr string) (*http.Response, error) {
	res, err := httpClient.Head(urlStr)
	if err != nil {
		ue, ok := err.(*url.Error)
		if ok {
			if ue.Err == errRedirectNotFollowed {
				// ignore
			} else {
				return nil, err
			}
		} else {
			return nil, err
		}
	}
	return res, nil
}

func edit(res *http.Response) string {
	if res == nil {
		return "???"
	}
	// TODO Implement other style
	return strconv.Itoa(res.StatusCode)
}
<commit_msg>Unlock time of makeing object for responseSet<commit_after>package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	concurrent = flag.Int("j", runtime.NumCPU(), "number of concurrent")
	timeout    = flag.Int64("timeout", 10, "timeout second for HTTP request")
)

var (
	errRedirectNotFollowed = errors.New("redirection not followed")
)

var customTransport http.RoundTripper = &http.Transport{
	Proxy: http.ProxyFromEnvironment,
	Dial: (&net.Dialer{
		Timeout:   30 * time.Second,
		KeepAlive: 30 * time.Second,
	}).Dial,
	TLSHandshakeTimeout: 10 * time.Second,
	MaxIdleConnsPerHost: 3 * http.DefaultMaxIdleConnsPerHost,
}

var httpClient = &http.Client{
	Transport: customTransport,
	CheckRedirect: func(req *http.Request, via []*http.Request) error {
		return errRedirectNotFollowed
	},
	Timeout: time.Duration(*timeout * int64(time.Second)),
}

type responseSet struct {
	res    *http.Response
	urlStr string
}

func main() {
	flag.Parse()

	var stock []*responseSet
	var m sync.Mutex
	printq := make(chan string, *concurrent)
	stockq := make(chan bool, *concurrent)
	waitingq := make(chan bool, *concurrent)
	quitq := make(chan bool)
	eof := false

	go func() {
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			urlStr := strings.TrimRight(scanner.Text(), "\n")
			printq <- urlStr

			go func() {
				waitingq <- true
				res, _ := fetch(urlStr)
				<-waitingq
				set := &responseSet{
					res:    res,
					urlStr: urlStr,
				}
				m.Lock()
				stock = append(stock, set)
				m.Unlock()
				stockq <- true
			}()
		}
		if err := scanner.Err(); err != nil {
			fmt.Fprintln(os.Stderr, "reading standard input:", err)
		}
		eof = true
	}()

	go func() {
		for {
			urlStr := <-printq
		L:
			for {
				for i, set := range stock {
					if set.urlStr == urlStr {
						fmt.Fprintf(os.Stdout, "%s\t%s\n", edit(set.res), set.urlStr)
						m.Lock()
						stock = append(stock[:i], stock[i+1:]...)
						m.Unlock()
						if eof && len(waitingq) == 0 && len(stock) == 0 {
							quitq <- true
						}
						break L
					}
				}
				<-stockq
			}
		}
	}()

	<-quitq
}

func fetch(urlStr string) (*http.Response, error) {
	res, err := httpClient.Head(urlStr)
	if err != nil {
		ue, ok := err.(*url.Error)
		if ok {
			if ue.Err == errRedirectNotFollowed {
				// ignore
			} else {
				return nil, err
			}
		} else {
			return nil, err
		}
	}
	return res, nil
}

func edit(res *http.Response) string {
	if res == nil {
		return "???"
	}
	// TODO Implement other style
	return strconv.Itoa(res.StatusCode)
}
<|endoftext|>


<commit_before>package listener

import (
	"github.com/jeffpierce/cassabon/config"
	"github.com/jeffpierce/cassabon/pearson"
)

type indexedLine struct {
	peerIndex int
	statLine  string
}

// peerList contains an ordered list of Cassabon peers.
type peerList struct {
	target   chan indexedLine // Channel for forwarding a stat line to a Cassabon peer
	hostPort string           // Host:port on which the local server is listening
	peers    []string         // Host:port information for all Cassabon peers (inclusive)
}

// isInitialized indicates whether the structure has ever been updated.
func (pl *peerList) isInitialized() bool {
	return pl.hostPort != ""
}

// start records the current peer list and starts the forwarder goroutine.
func (pl *peerList) start(hostPort string, peers []string) {

	// Record the current set of peers.
	pl.hostPort = hostPort
	pl.peers = make([]string, len(peers))
	for i, v := range peers {
		pl.peers[i] = v
	}

	// Start the forwarder goroutine.
	config.G.OnReload2WG.Add(1)
	go pl.run()
}

// isEqual indicates whether the given new configuration is equal to the current.
func (pl *peerList) isEqual(hostPort string, peers []string) bool {
	if pl.hostPort != hostPort {
		return false
	}
	if len(pl.peers) != len(peers) {
		return false
	}
	for i, v := range pl.peers {
		if peers[i] != v {
			return false
		}
	}
	return true
}

// ownerOf determines which host owns a particular stats path.
func (pl *peerList) ownerOf(statPath string) (int, bool) {
	peerIndex := int(pearson.Hash8(statPath)) % len(pl.peers)
	if pl.hostPort == pl.peers[peerIndex] {
		config.G.Log.System.LogInfo("Mine! %-30s %d %s", statPath, peerIndex, pl.peers[peerIndex])
		return peerIndex, true
	} else {
		//config.G.Log.System.LogInfo("      %-30s %d %s", statPath, peerIndex, pl.peers[peerIndex])
		return peerIndex, false
	}
}

// run listens for stat lines on a channel and sends them to the appropriate Cassabon peer.
func (pl *peerList) run() {

	// Create the channel on which stats to forward are received.
	pl.target = make(chan indexedLine, 1)
	defer close(pl.target)

	for {
		select {
		case <-config.G.OnReload2:
			config.G.Log.System.LogDebug("peerList::run received QUIT message")
			config.G.OnReload2WG.Done()
			return
		case il := <-pl.target:
			if pl.hostPort != pl.peers[il.peerIndex] {
				config.G.Log.System.LogInfo("Forwarding to %d %s: \"%s\"",
					il.peerIndex, pl.peers[il.peerIndex], il.statLine)
			}
		}
	}
}
<commit_msg>Undo move of channel make, as it causes a race<commit_after>package listener

import (
	"github.com/jeffpierce/cassabon/config"
	"github.com/jeffpierce/cassabon/pearson"
)

type indexedLine struct {
	peerIndex int
	statLine  string
}

// peerList contains an ordered list of Cassabon peers.
type peerList struct {
	target   chan indexedLine // Channel for forwarding a stat line to a Cassabon peer
	hostPort string           // Host:port on which the local server is listening
	peers    []string         // Host:port information for all Cassabon peers (inclusive)
}

// isInitialized indicates whether the structure has ever been updated.
func (pl *peerList) isInitialized() bool {
	return pl.hostPort != ""
}

// start records the current peer list and starts the forwarder goroutine.
func (pl *peerList) start(hostPort string, peers []string) {

	// Create the channel on which stats to forward are received.
	pl.target = make(chan indexedLine, 1)

	// Record the current set of peers.
	pl.hostPort = hostPort
	pl.peers = make([]string, len(peers))
	for i, v := range peers {
		pl.peers[i] = v
	}

	// Start the forwarder goroutine.
	config.G.OnReload2WG.Add(1)
	go pl.run()
}

// isEqual indicates whether the given new configuration is equal to the current.
func (pl *peerList) isEqual(hostPort string, peers []string) bool {
	if pl.hostPort != hostPort {
		return false
	}
	if len(pl.peers) != len(peers) {
		return false
	}
	for i, v := range pl.peers {
		if peers[i] != v {
			return false
		}
	}
	return true
}

// ownerOf determines which host owns a particular stats path.
func (pl *peerList) ownerOf(statPath string) (int, bool) {
	peerIndex := int(pearson.Hash8(statPath)) % len(pl.peers)
	if pl.hostPort == pl.peers[peerIndex] {
		config.G.Log.System.LogInfo("Mine! %-30s %d %s", statPath, peerIndex, pl.peers[peerIndex])
		return peerIndex, true
	} else {
		//config.G.Log.System.LogInfo("      %-30s %d %s", statPath, peerIndex, pl.peers[peerIndex])
		return peerIndex, false
	}
}

// run listens for stat lines on a channel and sends them to the appropriate Cassabon peer.
func (pl *peerList) run() {

	defer close(pl.target)

	for {
		select {
		case <-config.G.OnReload2:
			config.G.Log.System.LogDebug("peerList::run received QUIT message")
			config.G.OnReload2WG.Done()
			return
		case il := <-pl.target:
			if pl.hostPort != pl.peers[il.peerIndex] {
				config.G.Log.System.LogInfo("Forwarding to %d %s: \"%s\"",
					il.peerIndex, pl.peers[il.peerIndex], il.statLine)
			}
		}
	}
}
<|endoftext|>