<commit_before>/*
The memberlist package is used to provide a lightweight gossip based
mechanism for node membership and failure detection. It is loosely
based on the SWIM paper (Scalable Weakly-consistent Infection-style
process group Membership protocol). There are a few notable differences,
including the uses of additional gossip (instead of purely piggybacking on
failure detection) and the addition of a state push/pull mechanism.

An independent gossip mechanism is used because it allows for changes to be propogated
more quickly, and also enables us to gossip at a different interval that we perform
failure checks. The gossip rate is tunable, and can be disabled.

A Push/Pull mechanism is also included because it allows new nodes to
get an almost complete member list upon joining. It also is used as
a periodic anti-entropy mechanism to ensure very high convergence rates.
The frequency of this can be adjusted to change the overhead, or disabled
entirely.

*/
package memberlist

import (
	"fmt"
	"log"
	"net"
	"os"
	"sync"
	"time"
)

type Delegate interface {
	// NodeMeta is used to retrieve meta-data about the current node
	// when broadcasting an alive message. It's length is limited to
	// the given byte size.
	NodeMeta(limit int) []byte

	// NotifyMsg is called when a user-data message is received.
	// This should not block
	NotifyMsg([]byte)

	// GetBroadcasts is called when user data messages can be broadcast.
	// It can return a list of buffers to send. Each buffer should assume an
	// overhead as provided with a limit on the total byte size allowed.
	GetBroadcasts(overhead, limit int) [][]byte

	// LocalState is used for a TCP Push/Pull. This is sent to
	// the remote side as well as membership information
	LocalState() []byte

	// MergeRemoteState is invoked after a TCP Push/Pull. This is the
	// state received from the remote side.
	MergeRemoteState([]byte)
}

type Config struct {
	Name             string        // Node name (FQDN)
	BindAddr         string        // Binding address
	UDPPort          int           // UDP port to listen on
	TCPPort          int           // TCP port to listen on
	TCPTimeout       time.Duration // TCP timeout
	IndirectChecks   int           // Number of indirect checks to use
	RetransmitMult   int           // Retransmits = RetransmitMult * log(N+1)
	SuspicionMult    int           // Suspicion time = SuspcicionMult * log(N+1) * Interval
	PushPullInterval time.Duration // How often we do a Push/Pull update
	RTT              time.Duration // 99% precentile of round-trip-time
	ProbeInterval    time.Duration // Failure probing interval length

	GossipNodes    int           // Number of nodes to gossip to per GossipInterval
	GossipInterval time.Duration // Gossip interval for non-piggyback messages (only if GossipNodes > 0)

	JoinCh       chan<- *Node
	LeaveCh      chan<- *Node
	UserDelegate Delegate // Delegate for user data
}

type Memberlist struct {
	config   *Config
	shutdown bool
	leave    bool

	udpListener *net.UDPConn
	tcpListener *net.TCPListener

	sequenceNum uint32 // Local sequence number
	incarnation uint32 // Local incarnation number

	nodeLock sync.RWMutex
	nodes    []*NodeState          // Known nodes
	nodeMap  map[string]*NodeState // Maps Addr.String() -> NodeState

	tickerLock sync.Mutex
	tickers    []*time.Ticker
	stopTick   chan struct{}
	probeIndex int

	ackLock     sync.Mutex
	ackHandlers map[uint32]*ackHandler

	broadcasts *TransmitLimitedQueue
}

func DefaultConfig() *Config {
	hostname, _ := os.Hostname()
	return &Config{
		hostname,
		"0.0.0.0",
		7946,
		7946,
		10 * time.Second,      // Timeout after 10 seconds
		3,                     // Use 3 nodes for the indirect ping
		4,                     // Retransmit a message 4 * log(N+1) nodes
		5,                     // Suspect a node for 5 * log(N+1) * Interval
		30 * time.Second,      // Low frequency
		20 * time.Millisecond, // Reasonable RTT time for LAN
		1 * time.Second,       // Failure check every second

		3, // Gossip to 3 nodes
		200 * time.Millisecond, // Gossip more rapidly

		nil,
		nil,
		nil,
	}
}

// newMemberlist creates the network listeners.
// Does not schedule exeuction of background maintenence.
func newMemberlist(conf *Config) (*Memberlist, error) {
	tcpAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.TCPPort)
	tcpLn, err := net.Listen("tcp", tcpAddr)
	if err != nil {
		return nil, fmt.Errorf("Failed to start TCP listener. Err: %s", err)
	}

	udpAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.UDPPort)
	udpLn, err := net.ListenPacket("udp", udpAddr)
	if err != nil {
		tcpLn.Close()
		return nil, fmt.Errorf("Failed to start UDP listener. Err: %s", err)
	}

	// Set the UDP receive window size
	setUDPRecvBuf(udpLn.(*net.UDPConn))

	m := &Memberlist{config: conf,
		udpListener: udpLn.(*net.UDPConn),
		tcpListener: tcpLn.(*net.TCPListener),
		nodeMap:     make(map[string]*NodeState),
		stopTick:    make(chan struct{}, 32),
		ackHandlers: make(map[uint32]*ackHandler),
		broadcasts:  &TransmitLimitedQueue{RetransmitMult: conf.RetransmitMult},
	}
	m.broadcasts.NumNodes = func() int { return len(m.nodes) }
	go m.tcpListen()
	go m.udpListen()
	return m, nil
}

// Create will start memberlist and create a new gossip pool, but
// will not connect to an existing node. This should only be used
// for the first node in the cluster.
func Create(conf *Config) (*Memberlist, error) {
	m, err := newMemberlist(conf)
	if err != nil {
		return nil, err
	}
	if err := m.setAlive(); err != nil {
		m.Shutdown()
		return nil, err
	}
	m.schedule()
	return m, nil
}

// Join will start memberlist and perform an initial push/pull with
// all the given hosts. If none of the existing hosts could be contacted,
// the join will fail.
func Join(conf *Config, existing []string) (*Memberlist, error) {
	m, err := newMemberlist(conf)
	if err != nil {
		return nil, err
	}
	if err := m.setAlive(); err != nil {
		m.Shutdown()
		return nil, err
	}

	// Attempt to join any of them
	success := false
	for _, exist := range existing {
		addr, err := net.ResolveIPAddr("ip", exist)
		if err != nil {
			log.Printf("[ERR] Failed to resolve %s: %s", exist, err)
			continue
		}

		if err := m.pushPullNode(addr.IP); err != nil {
			log.Printf("[ERR] Failed to contact %s: %s", exist, err)
			continue
		}

		// Mark success, but keep exchanging with other hosts
		// to get more complete state data
		success = true
	}

	// Only continue on success
	if !success {
		m.Shutdown()
		return nil, fmt.Errorf("Failed to contact existing hosts")
	}

	// Schedule background work
	m.schedule()
	return m, nil
}

// setAlive is used to mark this node as being alive
func (m *Memberlist) setAlive() error {
	// Pick a private IP address
	var ipAddr []byte
	if m.config.BindAddr == "0.0.0.0" {
		// Get the interfaces
		addresses, err := net.InterfaceAddrs()
		if err != nil {
			return fmt.Errorf("Failed to get interface addresses! Err: %vn", err)
		}

		// Find private IPv4 address
		for _, addr := range addresses {
			ip, ok := addr.(*net.IPNet)
			if !ok {
				continue
			}
			if ip.IP.To4() == nil {
				continue
			}
			if !isPrivateIP(ip.IP.String()) {
				continue
			}
			ipAddr = ip.IP
			break
		}

		// Failed to find private IP, use loopback
		if ipAddr == nil {
			ipAddr = []byte{127, 0, 0, 1}
		}

	} else {
		addr := m.tcpListener.Addr().(*net.TCPAddr)
		ipAddr = addr.IP
	}

	// Get the node meta data
	var meta []byte
	if m.config.UserDelegate != nil {
		meta = m.config.UserDelegate.NodeMeta(metaMaxSize)
		if len(meta) > metaMaxSize {
			meta = meta[:metaMaxSize]
		}
	}

	a := alive{
		Incarnation: m.nextIncarnation(),
		Node:        m.config.Name,
		Addr:        ipAddr,
		Meta:        meta,
	}
	m.aliveNode(&a)
	return nil
}

// Members is used to return a list of all known live nodes
func (m *Memberlist) Members() []*Node {
	nodes := make([]*Node, 0, len(m.nodes))
	m.nodeLock.RLock()
	defer m.nodeLock.RUnlock()
	for _, n := range m.nodes {
		if n.State != StateDead {
			nodes = append(nodes, &n.Node)
		}
	}
	return nodes
}

// NumMembers provides an efficient way to determine
// the number of alive members
func (m *Memberlist) NumMembers() (alive int) {
	m.nodeLock.RLock()
	defer m.nodeLock.RUnlock()
	for _, n := range m.nodes {
		if n.State != StateDead {
			alive++
		}
	}
	return
}

// Leave will broadcast a leave message but will not shutdown
// the memberlist background maintenence. This should be followed
// by a Shutdown(). Note that this just enqueues the message,
// some time should be allowed for it to propogate.
func (m *Memberlist) Leave() error {
	m.leave = true
	d := dead{Incarnation: m.incarnation, Node: m.config.Name}
	m.deadNode(&d)
	return nil
}

// Shutdown will stop the memberlist background maintenence
// but will not broadcast a leave message prior. If no prior
// leave was issued, other nodes will detect this as a failure.
func (m *Memberlist) Shutdown() error {
	m.shutdown = true
	m.deschedule()
	m.udpListener.Close()
	m.tcpListener.Close()
	return nil
}
<commit_msg>Do not silently trim the meta data<commit_after>/*
The memberlist package is used to provide a lightweight gossip based
mechanism for node membership and failure detection. It is loosely
based on the SWIM paper (Scalable Weakly-consistent Infection-style
process group Membership protocol). There are a few notable differences,
including the uses of additional gossip (instead of purely piggybacking on
failure detection) and the addition of a state push/pull mechanism.

An independent gossip mechanism is used because it allows for changes to be propogated
more quickly, and also enables us to gossip at a different interval that we perform
failure checks. The gossip rate is tunable, and can be disabled.

A Push/Pull mechanism is also included because it allows new nodes to
get an almost complete member list upon joining. It also is used as
a periodic anti-entropy mechanism to ensure very high convergence rates.
The frequency of this can be adjusted to change the overhead, or disabled
entirely.

*/
package memberlist

import (
	"fmt"
	"log"
	"net"
	"os"
	"sync"
	"time"
)

type Delegate interface {
	// NodeMeta is used to retrieve meta-data about the current node
	// when broadcasting an alive message. It's length is limited to
	// the given byte size.
	NodeMeta(limit int) []byte

	// NotifyMsg is called when a user-data message is received.
	// This should not block
	NotifyMsg([]byte)

	// GetBroadcasts is called when user data messages can be broadcast.
	// It can return a list of buffers to send. Each buffer should assume an
	// overhead as provided with a limit on the total byte size allowed.
	GetBroadcasts(overhead, limit int) [][]byte

	// LocalState is used for a TCP Push/Pull. This is sent to
	// the remote side as well as membership information
	LocalState() []byte

	// MergeRemoteState is invoked after a TCP Push/Pull. This is the
	// state received from the remote side.
	MergeRemoteState([]byte)
}

type Config struct {
	Name             string        // Node name (FQDN)
	BindAddr         string        // Binding address
	UDPPort          int           // UDP port to listen on
	TCPPort          int           // TCP port to listen on
	TCPTimeout       time.Duration // TCP timeout
	IndirectChecks   int           // Number of indirect checks to use
	RetransmitMult   int           // Retransmits = RetransmitMult * log(N+1)
	SuspicionMult    int           // Suspicion time = SuspcicionMult * log(N+1) * Interval
	PushPullInterval time.Duration // How often we do a Push/Pull update
	RTT              time.Duration // 99% precentile of round-trip-time
	ProbeInterval    time.Duration // Failure probing interval length

	GossipNodes    int           // Number of nodes to gossip to per GossipInterval
	GossipInterval time.Duration // Gossip interval for non-piggyback messages (only if GossipNodes > 0)

	JoinCh       chan<- *Node
	LeaveCh      chan<- *Node
	UserDelegate Delegate // Delegate for user data
}

type Memberlist struct {
	config   *Config
	shutdown bool
	leave    bool

	udpListener *net.UDPConn
	tcpListener *net.TCPListener

	sequenceNum uint32 // Local sequence number
	incarnation uint32 // Local incarnation number

	nodeLock sync.RWMutex
	nodes    []*NodeState          // Known nodes
	nodeMap  map[string]*NodeState // Maps Addr.String() -> NodeState

	tickerLock sync.Mutex
	tickers    []*time.Ticker
	stopTick   chan struct{}
	probeIndex int

	ackLock     sync.Mutex
	ackHandlers map[uint32]*ackHandler

	broadcasts *TransmitLimitedQueue
}

func DefaultConfig() *Config {
	hostname, _ := os.Hostname()
	return &Config{
		hostname,
		"0.0.0.0",
		7946,
		7946,
		10 * time.Second,      // Timeout after 10 seconds
		3,                     // Use 3 nodes for the indirect ping
		4,                     // Retransmit a message 4 * log(N+1) nodes
		5,                     // Suspect a node for 5 * log(N+1) * Interval
		30 * time.Second,      // Low frequency
		20 * time.Millisecond, // Reasonable RTT time for LAN
		1 * time.Second,       // Failure check every second

		3, // Gossip to 3 nodes
		200 * time.Millisecond, // Gossip more rapidly

		nil,
		nil,
		nil,
	}
}

// newMemberlist creates the network listeners.
// Does not schedule exeuction of background maintenence.
func newMemberlist(conf *Config) (*Memberlist, error) {
	tcpAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.TCPPort)
	tcpLn, err := net.Listen("tcp", tcpAddr)
	if err != nil {
		return nil, fmt.Errorf("Failed to start TCP listener. Err: %s", err)
	}

	udpAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.UDPPort)
	udpLn, err := net.ListenPacket("udp", udpAddr)
	if err != nil {
		tcpLn.Close()
		return nil, fmt.Errorf("Failed to start UDP listener. Err: %s", err)
	}

	// Set the UDP receive window size
	setUDPRecvBuf(udpLn.(*net.UDPConn))

	m := &Memberlist{config: conf,
		udpListener: udpLn.(*net.UDPConn),
		tcpListener: tcpLn.(*net.TCPListener),
		nodeMap:     make(map[string]*NodeState),
		stopTick:    make(chan struct{}, 32),
		ackHandlers: make(map[uint32]*ackHandler),
		broadcasts:  &TransmitLimitedQueue{RetransmitMult: conf.RetransmitMult},
	}
	m.broadcasts.NumNodes = func() int { return len(m.nodes) }
	go m.tcpListen()
	go m.udpListen()
	return m, nil
}

// Create will start memberlist and create a new gossip pool, but
// will not connect to an existing node. This should only be used
// for the first node in the cluster.
func Create(conf *Config) (*Memberlist, error) {
	m, err := newMemberlist(conf)
	if err != nil {
		return nil, err
	}
	if err := m.setAlive(); err != nil {
		m.Shutdown()
		return nil, err
	}
	m.schedule()
	return m, nil
}

// Join will start memberlist and perform an initial push/pull with
// all the given hosts. If none of the existing hosts could be contacted,
// the join will fail.
func Join(conf *Config, existing []string) (*Memberlist, error) {
	m, err := newMemberlist(conf)
	if err != nil {
		return nil, err
	}
	if err := m.setAlive(); err != nil {
		m.Shutdown()
		return nil, err
	}

	// Attempt to join any of them
	success := false
	for _, exist := range existing {
		addr, err := net.ResolveIPAddr("ip", exist)
		if err != nil {
			log.Printf("[ERR] Failed to resolve %s: %s", exist, err)
			continue
		}

		if err := m.pushPullNode(addr.IP); err != nil {
			log.Printf("[ERR] Failed to contact %s: %s", exist, err)
			continue
		}

		// Mark success, but keep exchanging with other hosts
		// to get more complete state data
		success = true
	}

	// Only continue on success
	if !success {
		m.Shutdown()
		return nil, fmt.Errorf("Failed to contact existing hosts")
	}

	// Schedule background work
	m.schedule()
	return m, nil
}

// setAlive is used to mark this node as being alive
func (m *Memberlist) setAlive() error {
	// Pick a private IP address
	var ipAddr []byte
	if m.config.BindAddr == "0.0.0.0" {
		// Get the interfaces
		addresses, err := net.InterfaceAddrs()
		if err != nil {
			return fmt.Errorf("Failed to get interface addresses! Err: %vn", err)
		}

		// Find private IPv4 address
		for _, addr := range addresses {
			ip, ok := addr.(*net.IPNet)
			if !ok {
				continue
			}
			if ip.IP.To4() == nil {
				continue
			}
			if !isPrivateIP(ip.IP.String()) {
				continue
			}
			ipAddr = ip.IP
			break
		}

		// Failed to find private IP, use loopback
		if ipAddr == nil {
			ipAddr = []byte{127, 0, 0, 1}
		}

	} else {
		addr := m.tcpListener.Addr().(*net.TCPAddr)
		ipAddr = addr.IP
	}

	// Get the node meta data
	var meta []byte
	if m.config.UserDelegate != nil {
		meta = m.config.UserDelegate.NodeMeta(metaMaxSize)
		if len(meta) > metaMaxSize {
			panic("Node meta data provided is longer than the limit")
		}
	}

	a := alive{
		Incarnation: m.nextIncarnation(),
		Node:        m.config.Name,
		Addr:        ipAddr,
		Meta:        meta,
	}
	m.aliveNode(&a)
	return nil
}

// Members is used to return a list of all known live nodes
func (m *Memberlist) Members() []*Node {
	nodes := make([]*Node, 0, len(m.nodes))
	m.nodeLock.RLock()
	defer m.nodeLock.RUnlock()
	for _, n := range m.nodes {
		if n.State != StateDead {
			nodes = append(nodes, &n.Node)
		}
	}
	return nodes
}

// NumMembers provides an efficient way to determine
// the number of alive members
func (m *Memberlist) NumMembers() (alive int) {
	m.nodeLock.RLock()
	defer m.nodeLock.RUnlock()
	for _, n := range m.nodes {
		if n.State != StateDead {
			alive++
		}
	}
	return
}

// Leave will broadcast a leave message but will not shutdown
// the memberlist background maintenence. This should be followed
// by a Shutdown(). Note that this just enqueues the message,
// some time should be allowed for it to propogate.
func (m *Memberlist) Leave() error {
	m.leave = true
	d := dead{Incarnation: m.incarnation, Node: m.config.Name}
	m.deadNode(&d)
	return nil
}

// Shutdown will stop the memberlist background maintenence
// but will not broadcast a leave message prior. If no prior
// leave was issued, other nodes will detect this as a failure.
func (m *Memberlist) Shutdown() error {
	m.shutdown = true
	m.deschedule()
	m.udpListener.Close()
	m.tcpListener.Close()
	return nil
}
<|endoftext|>


<commit_before>package db

import (
	"testing"

	."github.com/smartystreets/goconvey/convey"

)

func TestRecord(t *testing.T) {

	Convey("Should be able to set the Id of LegderRecord", t, func() {
		record := new(LedgerRecord)
        record.Id = 5
        So(record.Id, ShouldEqual, 5)

		Convey("PagingToken() returns an id ", func() {
            So(record.PagingToken(), ShouldEqual, "5")
		})
	})
}<commit_msg>formatting<commit_after>package db

import (
	"testing"

	."github.com/smartystreets/goconvey/convey"

)

func TestRecord(t *testing.T) {

    Convey("Should be able to set the Id of LegderRecord", t, func() {
        record := new(LedgerRecord)
        record.Id = 5
        So(record.Id, ShouldEqual, 5)
        Convey("PagingToken() returns an id ", func() {
            So(record.PagingToken(), ShouldEqual, "5")
		})
	})
}<|endoftext|>


<commit_before>package deliveries

import (
	"encoding/json"
	"fmt"
	"time"
)

const (
	// DeliveriesGetDestinationsEndpoint returns all destinations for a team. Requires team id.
	DeliveriesGetDestinationsEndpoint = "/v1/teams/getDeliveryDestinations"
	// DeliveriesDeleteDestinationEndpoint markes a delivery destination as deleted. It requires a delivery destination id.
	DeliveriesDeleteDestinationEndpoint = "/v1/teams/deleteDeliveryDestination"
)

// Destination is a representation of a single location that a team can deliver results to.
type Destination struct {
	ID        string     `json:"id"`
	TeamID    string     `json:"team_id"`
	Location  string     `json:"location"`
	Region    string     `json:"region"`
	Name      string     `json:"name"`
	DestType  string     `json:"type"`
	DeletedAt *time.Time `json:"deleted_at,omitempty"`
}

// String returns a JSON formatted string of the delivery object
func (p Destination) String() string {
	b, err := json.Marshal(p)
	if err != nil {
		return fmt.Sprintf("failed to format delivery: %v", err.Error())
	}
	return string(b)
}
<commit_msg>create destination struct<commit_after>package deliveries

import (
	"encoding/json"
	"fmt"
	"time"
)

const (
	// DeliveriesGetDestinationsEndpoint returns all destinations for a team. Requires team id.
	DeliveriesGetDestinationsEndpoint = "/v1/teams/getDeliveryDestinations"
	// DeliveriesDeleteDestinationEndpoint markes a delivery destination as deleted. It requires a delivery destination id.
	DeliveriesDeleteDestinationEndpoint = "/v1/teams/deleteDeliveryDestination"
)

// Destination is a representation of a single location that a team can deliver results to.
type Destination struct {
	ID        string     `json:"id"`
	TeamID    string     `json:"team_id"`
	Location  string     `json:"location"`
	Region    string     `json:"region"`
	Name      string     `json:"name"`
	DestType  string     `json:"type"`
	DeletedAt *time.Time `json:"deleted_at,omitempty"`
}

// CreateDestinationInput is an input representation of a single location that a team can deliver results to.
type CreateDestinationInput struct {
	Destination
	AccessKey string `json:"access_key"`
	SecretKey string `json:"secret_key"`
}

// String returns a JSON formatted string of the delivery object
func (p Destination) String() string {
	b, err := json.Marshal(p)
	if err != nil {
		return fmt.Sprintf("failed to format delivery: %v", err.Error())
	}
	return string(b)
}
<|endoftext|>


<commit_before>package lib

import (
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

var (
	postTemplate *template.Template
	config       Config
	ConfigFile   string
	PublicDir    string
	PostsDir     string
	TemplatesDir string
	RssURL       string

	specFiles = map[string]struct{}{
		"favicon.ico":          struct{}{},
		"robots.txt":           struct{}{},
		"humans.txt":           struct{}{},
		"apple-touch-icon.png": struct{}{},
	}
)

func init() {
	pwd, err := os.Getwd()
	if err != nil {
		log.Fatal("FATAL", err)
	}
	PublicDir = filepath.Join(pwd, "public")
	PostsDir = filepath.Join(pwd, "post")
	TemplatesDir = filepath.Join(pwd, "template")
	ConfigFile = filepath.Join(pwd, "config.json")
	config = GetConfig(ConfigFile)
}

func storeRssURL() {
	base, err := url.Parse(config.BaseURL)
	if err != nil {
		fmt.Errorf("Error parsing the baseurl: %s", err)
	}
	rss, err := base.Parse("/rss")
	if err != nil {
		fmt.Errorf("Error parsing the rss url: %s", err)
	}

	RssURL = rss.String()
}

type posts []*LongPost

func (p posts) Len() int           { return len(p) }
func (p posts) Less(i, j int) bool { return p[i].PublishDate.Before(p[j].PublishDate) }
func (p posts) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

func filter(file []os.FileInfo) []os.FileInfo {
	for i := 0; i < len(file); {
		if file[i].IsDir() || filepath.Ext(file[i].Name()) != ".md" {
			file[i], file = file[len(file)-1], file[:len(file)-1]
		} else {
			i++
		}
	}

	return file
}

func clearPublishDir() error {
	files, err := ioutil.ReadDir(PublicDir)
	if err != nil {
		return fmt.Errorf("Error while getting public directory files: %s", err)
	}

	for _, file := range files {
		if !file.IsDir() && !strings.HasPrefix(file.Name(), ".") {
			if _, ok := specFiles[file.Name()]; !ok {
				err = os.Remove(filepath.Join(PublicDir, file.Name()))
				if err != nil {
					return fmt.Errorf("Error deleting file %s: %s", file.Name(), err)
				}
			}
		}
	}

	return nil
}

func getPosts(files []os.FileInfo) (allPosts []*LongPost, recentPosts []*LongPost) {
	allPosts = make([]*LongPost, 0, len(files))
	for _, file := range files {
		longPost, err := newLongPost(file)
		if err != nil {
			log.Printf("Post ignored: %s; Error: %s\n", file.Name(), err)
		} else {
			allPosts = append(allPosts, longPost)
		}
	}

	sort.Sort(sort.Reverse(posts(allPosts)))
	recent := config.RecentPostsCount
	if length := len(allPosts); length < recent {
		recent = length
	}
	recentPosts = allPosts[:recent]
	return
}

func loadTemplates() {
	postTemplate = template.Must(template.ParseFiles("template/post.html", "template/base.html"))
}

func GenerateSite() error {
	storeRssURL()
	loadTemplates()
	files, err := ioutil.ReadDir(PostsDir)
	if err != nil {
		return err
	}

	files = filter(files)

	allPosts, recentPosts := getPosts(files)

	if err := clearPublishDir(); err != nil {
		return err
	}

	for i, p := range allPosts {
		pt := newPostTempalte(p, i, recentPosts, allPosts)
		if err := generateFile(pt, i == 0); err != nil {
			return err
		}
	}

	pt := newPostTempalte(nil, 0, recentPosts, allPosts)
	return generateRss(pt)
}

func generateRss(pt *PostTempalte) error {
	rss := NewRss(config.SiteName, config.Slogan, config.BaseURL)
	base, err := url.Parse(config.BaseURL)
	if err != nil {
		return fmt.Errorf("Error parsing base URL: %s", err)
	}

	for _, p := range pt.Recent {
		u, err := base.Parse(p.Slug)
		if err != nil {
			return fmt.Errorf("Error parsing post URL: %s", err)
		}
		rss.Channels[0].AppendItem(NewRssItem(p.Title, p.Description, u.String(), p.Author, "", p.PublishDate))
	}

	return rss.WriteToFile(filepath.Join(PublicDir, "rss"))
}

func generateFile(pt *PostTempalte, index bool) error {
	var w io.Writer

	fileWriter, err := os.Create(filepath.Join(PublicDir, pt.Post.Slug))
	if err != nil {
		return fmt.Errorf("Error creating static file %s: %s", pt.Post.Slug, err)
	}
	defer fileWriter.Close()

	w = fileWriter
	if index {
		indexWriter, err := os.Create(filepath.Join(PublicDir, "index.html"))
		if err != nil {
			return fmt.Errorf("Error creating static file index.html: %s", err)
		}
		defer indexWriter.Close()
		w = io.MultiWriter(fileWriter, indexWriter)
	}

	return postTemplate.ExecuteTemplate(w, "base", pt)
}
<commit_msg>fix bugs<commit_after>package lib

import (
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

var (
	postTemplate *template.Template
	config       Config
	ConfigFile   string
	PublicDir    string
	PostsDir     string
	TemplatesDir string
	RssURL       string

	specFiles = map[string]struct{}{
		"favicon.ico":          struct{}{},
		"robots.txt":           struct{}{},
		"humans.txt":           struct{}{},
		"apple-touch-icon.png": struct{}{},
	}
)

func init() {
	pwd, err := os.Getwd()
	if err != nil {
		log.Fatal("FATAL", err)
	}
	PublicDir = filepath.Join(pwd, "public")
	PostsDir = filepath.Join(pwd, "post")
	TemplatesDir = filepath.Join(pwd, "template")
	ConfigFile = filepath.Join(pwd, "config.json")
	config = GetConfig(ConfigFile)
}

func storeRssURL() {
	base, err := url.Parse(config.BaseURL)
	if err != nil {
		fmt.Errorf("Error parsing the baseurl: %s", err)
	}
	rss, err := base.Parse("/rss")
	if err != nil {
		fmt.Errorf("Error parsing the rss url: %s", err)
	}

	RssURL = rss.String()
}

type posts []*LongPost

func (p posts) Len() int           { return len(p) }
func (p posts) Less(i, j int) bool { return p[i].PublishDate.Before(p[j].PublishDate) }
func (p posts) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

func filter(file []os.FileInfo) []os.FileInfo {
	for i := 0; i < len(file); {
		if file[i].IsDir() || filepath.Ext(file[i].Name()) != ".md" {
			file[i], file = file[len(file)-1], file[:len(file)-1]
		} else {
			i++
		}
	}

	return file
}

func clearPublishDir() error {
	files, err := ioutil.ReadDir(PublicDir)
	if err != nil {
		return fmt.Errorf("Error while getting public directory files: %s", err)
	}

	for _, file := range files {
		if !file.IsDir() && !strings.HasPrefix(file.Name(), ".") {
			if _, ok := specFiles[file.Name()]; !ok {
				err = os.Remove(filepath.Join(PublicDir, file.Name()))
				if err != nil {
					return fmt.Errorf("Error deleting file %s: %s", file.Name(), err)
				}
			}
		}
	}

	return nil
}

func getPosts(files []os.FileInfo) (allPosts []*LongPost, recentPosts []*LongPost) {
	allPosts = make([]*LongPost, 0, len(files))
	for _, file := range files {
		longPost, err := newLongPost(file)
		if err != nil {
			log.Printf("Post ignored: %s; Error: %s\n", file.Name(), err)
		} else {
			allPosts = append(allPosts, longPost)
		}
	}

	sort.Sort(sort.Reverse(posts(allPosts)))
	recent := config.RecentPostsCount
	if length := len(allPosts); length < recent {
		recent = length
	}
	recentPosts = allPosts[:recent]
	return
}

func loadTemplates() {
	postTemplate = template.Must(template.ParseFiles("template/post.html", "template/base.html"))
}

func GenerateSite() error {
	storeRssURL()
	loadTemplates()
	files, err := ioutil.ReadDir(PostsDir)
	if err != nil {
		return err
	}

	files = filter(files)

	allPosts, recentPosts := getPosts(files)

	if err := clearPublishDir(); err != nil {
		return err
	}

	for i, p := range allPosts {
		pt := newPostTempalte(p, i, recentPosts, allPosts)
		if err := generateFile(pt, i == 0); err != nil {
			return err
		}
	}

	pt := newPostTempalte(nil, 0, recentPosts, allPosts)
	return generateRss(pt)
}

func generateRss(pt *PostTempalte) error {
	rss := NewRss(config.SiteName, config.Slogan, config.BaseURL)
	base, err := url.Parse(config.BaseURL)
	if err != nil {
		return fmt.Errorf("Error parsing base URL: %s", err)
	}

	for _, p := range pt.Recent {
		u, err := base.Parse(p.Slug)
		if err != nil {
			return fmt.Errorf("Error parsing post URL: %s", err)
		}
		rss.Channels[0].AppendItem(NewRssItem(p.Title, p.Description, u.String(), p.Author, "", p.PublishDate))
	}

	return rss.WriteToFile(filepath.Join(PublicDir, "rss.xml"))
}

func generateFile(pt *PostTempalte, index bool) error {
	var w io.Writer

	fileWriter, err := os.Create(filepath.Join(PublicDir, pt.Post.Slug))
	if err != nil {
		return fmt.Errorf("Error creating static file %s: %s", pt.Post.Slug, err)
	}
	defer fileWriter.Close()

	w = fileWriter
	if index {
		indexWriter, err := os.Create(filepath.Join(PublicDir, "index.html"))
		if err != nil {
			return fmt.Errorf("Error creating static file index.html: %s", err)
		}
		defer indexWriter.Close()
		w = io.MultiWriter(fileWriter, indexWriter)
	}

	return postTemplate.ExecuteTemplate(w, "base", pt)
}
<|endoftext|>


<commit_before>package Dictionary

import "strings"
import "sort"

type KeyValuePair struct {
	Key       string
	Lower_key string
	Value     string
}

type Dictionary struct {
	m map[string]*KeyValuePair
}

func NewDictionary() *Dictionary {
	this := new(Dictionary)
	this.m = make(map[string]*KeyValuePair)
	return this
}

func (this *Dictionary) Get(key string) string {
	lower_key := strings.ToLower(key)

	node, ok := this.m[lower_key]
	if ok {
		return node.Value
	} else {
		return ""
	}
}

func (this *Dictionary) Remove(key string) {
	delete(this.m,strings.ToLower(key))
}

func (this *Dictionary) Set(key string, value string) {
	if value == "" {
		this.Remove(key)
		return
	}
	tmp := new(KeyValuePair)
	tmp.Key = key
	tmp.Lower_key = strings.ToLower(key)
	tmp.Value = value
	this.m[tmp.Lower_key] = tmp
}

func (this *Dictionary) Iter() chan *KeyValuePair {
	ch := make(chan *KeyValuePair, 0)
	go func() {
		for _, pair := range this.m {
			ch <- pair
		}
		close(ch)
		return
	}()
	return ch
}

type KeyValueList struct {
	list []KeyValuePair
}

func (this *KeyValueList) Len() int {
	return len(this.list)
}

func (this *KeyValueList) Less(i, j int) bool {
	return this.list[i].Key < this.list[j].Key
}

func (this *KeyValueList) Swap(i, j int) {
	this.list[i], this.list[j] = this.list[j], this.list[i]
}

func (this *Dictionary) ToArray() []KeyValuePair {
	list := []KeyValuePair{}
	for _, pair := range this.m {
		list = append(list, *pair)
	}
	return list
}

func (this *Dictionary) SortIter() chan KeyValuePair {
	array := new(KeyValueList)
	array.list = this.ToArray()
	sort.Sort(array)
	ch := make(chan KeyValuePair)
	go func() {
		for i := 0; i < len(array.list); i++ {
			ch <- array.list[i]
		}
		close(ch)
	}()
	return ch
}
<commit_msg>go fmt<commit_after>package Dictionary

import "strings"
import "sort"

type KeyValuePair struct {
	Key       string
	Lower_key string
	Value     string
}

type Dictionary struct {
	m map[string]*KeyValuePair
}

func NewDictionary() *Dictionary {
	this := new(Dictionary)
	this.m = make(map[string]*KeyValuePair)
	return this
}

func (this *Dictionary) Get(key string) string {
	lower_key := strings.ToLower(key)

	node, ok := this.m[lower_key]
	if ok {
		return node.Value
	} else {
		return ""
	}
}

func (this *Dictionary) Remove(key string) {
	delete(this.m, strings.ToLower(key))
}

func (this *Dictionary) Set(key string, value string) {
	if value == "" {
		this.Remove(key)
		return
	}
	tmp := new(KeyValuePair)
	tmp.Key = key
	tmp.Lower_key = strings.ToLower(key)
	tmp.Value = value
	this.m[tmp.Lower_key] = tmp
}

func (this *Dictionary) Iter() chan *KeyValuePair {
	ch := make(chan *KeyValuePair, 0)
	go func() {
		for _, pair := range this.m {
			ch <- pair
		}
		close(ch)
		return
	}()
	return ch
}

type KeyValueList struct {
	list []KeyValuePair
}

func (this *KeyValueList) Len() int {
	return len(this.list)
}

func (this *KeyValueList) Less(i, j int) bool {
	return this.list[i].Key < this.list[j].Key
}

func (this *KeyValueList) Swap(i, j int) {
	this.list[i], this.list[j] = this.list[j], this.list[i]
}

func (this *Dictionary) ToArray() []KeyValuePair {
	list := []KeyValuePair{}
	for _, pair := range this.m {
		list = append(list, *pair)
	}
	return list
}

func (this *Dictionary) SortIter() chan KeyValuePair {
	array := new(KeyValueList)
	array.list = this.ToArray()
	sort.Sort(array)
	ch := make(chan KeyValuePair)
	go func() {
		for i := 0; i < len(array.list); i++ {
			ch <- array.list[i]
		}
		close(ch)
	}()
	return ch
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/google/go-github/github"
)

// MirroredPR contains the upstream and downstream PR ids
type MirroredPR struct {
	DownstreamID int
	UpstreamID   int
}

// PRMirror contains various different variables
type PRMirror struct {
	GitHubClient  *github.Client
	Context       *context.Context
	Configuration *Config
	Database      *Database
	GitLock       *SpinLock
}

// GitHubEventMonitor passes in an instance of the PRMirror struct to all HTTP calls to the webhook listener
type GitHubEventMonitor struct {
	Mirrorer PRMirror
}

// HandleEvent handles github events and acts like an event handler
func (p PRMirror) HandleEvent(event *github.Event) {
	seenEvent, _ := p.Database.SeenEvent(event.GetID())
	if seenEvent {
		return
	}

	eventType := event.GetType()
	if eventType != "PullRequestEvent" {
		return
	}

	prEvent := github.PullRequestEvent{}
	err := json.Unmarshal(event.GetRawPayload(), &prEvent)
	if err != nil {
		panic(err)
	}

	p.HandlePREvent(&prEvent)
	p.Database.AddEvent(event.GetID())
}

func (p PRMirror) HandlePREvent(prEvent *github.PullRequestEvent) {
	//repoName := prEvent.Repo.GetName()
	//repoOwner := prEvent.Repo.Owner.GetName()
	prEventURL := prEvent.PullRequest.GetURL()

	//if p.Configuration.UseWebhook repoName != p.Configuration.DownstreamRepo {
	//	log.Warningf("Ignoring PR Event: %s because %s != %s\n", prEventURL, repoName, p.Configuration.UpstreamRepo)
	//	return
	//} //else if repoOwner != p.Configuration.DownstreamOwner {
	//log.Warningf("Ignoring PR Event: %s because %s != %s\n", prEventURL, repoOwner, p.Configuration.UpstreamOwner)
	//return
	//}

	log.Debugf("Handling PR Event: %s\n", prEventURL)

	prAction := prEvent.GetAction()
	if prAction == "closed" && prEvent.PullRequest.GetMerged() == true {
		prID, err := p.MirrorPR(prEvent.PullRequest)
		if err != nil {
			log.Errorf("Error while creating a new PR: %s\n", err.Error())
		} else {
			p.AddLabels(prID, []string{"Upstream PR Merged"})
			p.Database.StoreMirror(prID, prEvent.PullRequest.GetNumber())
		}
	}
}

// RunEventScraper runs the GitHub repo event API scraper
func (p PRMirror) RunEventScraper() {
	for {
		events, pollInterval, err := p.GetRepoEvents()
		if err == nil {
			for _, event := range events {
				p.HandleEvent(event)
			}
		}

		log.Debugf("Sleeping for %d as specified by GitHub\n", pollInterval)
		time.Sleep(time.Duration(pollInterval) * time.Second)
	}
}

// ServeHTTP handles HTTP requests to the webhook endpoint
func (s GitHubEventMonitor) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	payload, err := github.ValidatePayload(r, []byte(s.Mirrorer.Configuration.WebhookSecret))
	if err != nil {
		log.Errorf("Error validating the payload\n")
		return
	}
	event, err := github.ParseWebHook(github.WebHookType(r), payload)
	if err != nil {
		log.Errorf("Error parsing the payload\n")
	}

	switch event := event.(type) {
	case *github.PullRequestEvent:
		s.Mirrorer.HandlePREvent(event)
	}
}

// RunWebhookListener acts a webhook listener which GitHub will call with events
func (p PRMirror) RunWebhookListener() {
	server := GitHubEventMonitor{Mirrorer: p}
	err := http.ListenAndServe(fmt.Sprintf(":%d", p.Configuration.WebhookPort), server)
	log.Fatal(err)
}

// MirrorPR will mirror a PR from an upstream to the downstream
func (p PRMirror) MirrorPR(pr *github.PullRequest) (int, error) {
	p.GitLock.Lock()
	defer p.GitLock.Unlock()

	downstreamID, err := p.Database.GetDownstreamID(pr.GetNumber())
	if downstreamID != 0 {
		log.Warningf("Refusing to mirror already existing PR: %s - %s\n", pr.GetTitle(), pr.GetNumber())
		return 0, errors.New("prmirror: tried to mirror a PR which has already been mirrored")
	}

	log.Infof("Mirroring PR [%d]: %s from %s\n", pr.GetNumber(), pr.GetTitle(), pr.User.GetLogin())

	cmd := exec.Command(fmt.Sprintf("%s%s", p.Configuration.RepoPath, p.Configuration.ToolPath), strconv.Itoa(pr.GetNumber()), pr.GetTitle())
	cmd.Dir = p.Configuration.RepoPath
	cmdoutput, err := cmd.CombinedOutput()
	if err != nil {
		log.Criticalf("Error while mirroring %d: %s\n", pr.GetNumber(), err)
		return 0, err
	}

	logpath := fmt.Sprintf("./logs/upstream-merge-%d.log", pr.GetNumber())
	ioutil.WriteFile(logpath, cmdoutput, 0600)
	log.Debugf("Wrote log to %s\n", logpath)

	base := "master"
	head := fmt.Sprintf("upstream-merge-%d", pr.GetNumber())
	maintainerCanModify := true // We are the owner of the PR so we can specify this as true
	title := fmt.Sprintf("[MIRROR] %s", pr.GetTitle())
	body := fmt.Sprintf("Original PR: %s\n--------------------\n%s", pr.GetHTMLURL(), strings.Replace(pr.GetBody(), "@", "@ ", -1))

	newPR := github.NewPullRequest{}
	newPR.Title = &title
	newPR.Body = &body
	newPR.Base = &base
	newPR.Head = &head
	newPR.MaintainerCanModify = &maintainerCanModify

	pr, _, err = p.GitHubClient.PullRequests.Create(*p.Context, p.Configuration.DownstreamOwner, p.Configuration.DownstreamRepo, &newPR)
	if err != nil {
		return 0, err
	}

	if strings.Contains(string(cmdoutput), "Rejected hunk") {
		p.AddLabels(pr.GetNumber(), []string{"Auto Merge Rejections"})
	}

	return pr.GetNumber(), nil
}
<commit_msg>Fixes non-master branches being mirrored<commit_after>package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/google/go-github/github"
)

// MirroredPR contains the upstream and downstream PR ids
type MirroredPR struct {
	DownstreamID int
	UpstreamID   int
}

// PRMirror contains various different variables
type PRMirror struct {
	GitHubClient  *github.Client
	Context       *context.Context
	Configuration *Config
	Database      *Database
	GitLock       *SpinLock
}

// GitHubEventMonitor passes in an instance of the PRMirror struct to all HTTP calls to the webhook listener
type GitHubEventMonitor struct {
	Mirrorer PRMirror
}

// HandleEvent handles github events and acts like an event handler
func (p PRMirror) HandleEvent(event *github.Event) {
	seenEvent, _ := p.Database.SeenEvent(event.GetID())
	if seenEvent {
		return
	}

	eventType := event.GetType()
	if eventType != "PullRequestEvent" {
		return
	}

	prEvent := github.PullRequestEvent{}
	err := json.Unmarshal(event.GetRawPayload(), &prEvent)
	if err != nil {
		panic(err)
	}

	p.HandlePREvent(&prEvent)
	p.Database.AddEvent(event.GetID())
}

func (p PRMirror) HandlePREvent(prEvent *github.PullRequestEvent) {
	//repoName := prEvent.Repo.GetName()
	//repoOwner := prEvent.Repo.Owner.GetName()
	prEventURL := prEvent.PullRequest.GetURL()

	//if p.Configuration.UseWebhook repoName != p.Configuration.DownstreamRepo {
	//	log.Warningf("Ignoring PR Event: %s because %s != %s\n", prEventURL, repoName, p.Configuration.UpstreamRepo)
	//	return
	//} //else if repoOwner != p.Configuration.DownstreamOwner {
	//log.Warningf("Ignoring PR Event: %s because %s != %s\n", prEventURL, repoOwner, p.Configuration.UpstreamOwner)
	//return
	//}

	log.Debugf("Handling PR Event: %s\n", prEventURL)

	prAction := prEvent.GetAction()
	if prAction == "closed" && prEvent.PullRequest.GetMerged() == true && prEvent.PullRequest.Base.GetRef() == "master" {
		prID, err := p.MirrorPR(prEvent.PullRequest)
		if err != nil {
			log.Errorf("Error while creating a new PR: %s\n", err.Error())
		} else {
			p.AddLabels(prID, []string{"Upstream PR Merged"})
			p.Database.StoreMirror(prID, prEvent.PullRequest.GetNumber())
		}
	}
}

// RunEventScraper runs the GitHub repo event API scraper
func (p PRMirror) RunEventScraper() {
	for {
		events, pollInterval, err := p.GetRepoEvents()
		if err == nil {
			for _, event := range events {
				p.HandleEvent(event)
			}
		}

		log.Debugf("Sleeping for %d as specified by GitHub\n", pollInterval)
		time.Sleep(time.Duration(pollInterval) * time.Second)
	}
}

// ServeHTTP handles HTTP requests to the webhook endpoint
func (s GitHubEventMonitor) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	payload, err := github.ValidatePayload(r, []byte(s.Mirrorer.Configuration.WebhookSecret))
	if err != nil {
		log.Errorf("Error validating the payload\n")
		return
	}
	event, err := github.ParseWebHook(github.WebHookType(r), payload)
	if err != nil {
		log.Errorf("Error parsing the payload\n")
	}

	switch event := event.(type) {
	case *github.PullRequestEvent:
		s.Mirrorer.HandlePREvent(event)
	}
}

// RunWebhookListener acts a webhook listener which GitHub will call with events
func (p PRMirror) RunWebhookListener() {
	server := GitHubEventMonitor{Mirrorer: p}
	err := http.ListenAndServe(fmt.Sprintf(":%d", p.Configuration.WebhookPort), server)
	log.Fatal(err)
}

// MirrorPR will mirror a PR from an upstream to the downstream
func (p PRMirror) MirrorPR(pr *github.PullRequest) (int, error) {
	p.GitLock.Lock()
	defer p.GitLock.Unlock()

	downstreamID, err := p.Database.GetDownstreamID(pr.GetNumber())
	if downstreamID != 0 {
		log.Warningf("Refusing to mirror already existing PR: %s - %s\n", pr.GetTitle(), pr.GetNumber())
		return 0, errors.New("prmirror: tried to mirror a PR which has already been mirrored")
	}

	log.Infof("Mirroring PR [%d]: %s from %s\n", pr.GetNumber(), pr.GetTitle(), pr.User.GetLogin())

	cmd := exec.Command(fmt.Sprintf("%s%s", p.Configuration.RepoPath, p.Configuration.ToolPath), strconv.Itoa(pr.GetNumber()), pr.GetTitle())
	cmd.Dir = p.Configuration.RepoPath
	cmdoutput, err := cmd.CombinedOutput()
	if err != nil {
		log.Criticalf("Error while mirroring %d: %s\n", pr.GetNumber(), err)
		return 0, err
	}

	logpath := fmt.Sprintf("./logs/upstream-merge-%d.log", pr.GetNumber())
	ioutil.WriteFile(logpath, cmdoutput, 0600)
	log.Debugf("Wrote log to %s\n", logpath)

	base := "master"
	head := fmt.Sprintf("upstream-merge-%d", pr.GetNumber())
	maintainerCanModify := true // We are the owner of the PR so we can specify this as true
	title := fmt.Sprintf("[MIRROR] %s", pr.GetTitle())
	body := fmt.Sprintf("Original PR: %s\n--------------------\n%s", pr.GetHTMLURL(), strings.Replace(pr.GetBody(), "@", "@ ", -1))

	newPR := github.NewPullRequest{}
	newPR.Title = &title
	newPR.Body = &body
	newPR.Base = &base
	newPR.Head = &head
	newPR.MaintainerCanModify = &maintainerCanModify

	pr, _, err = p.GitHubClient.PullRequests.Create(*p.Context, p.Configuration.DownstreamOwner, p.Configuration.DownstreamRepo, &newPR)
	if err != nil {
		return 0, err
	}

	if strings.Contains(string(cmdoutput), "Rejected hunk") {
		p.AddLabels(pr.GetNumber(), []string{"Auto Merge Rejections"})
	}

	return pr.GetNumber(), nil
}
<|endoftext|>


<commit_before>package ioutils

import (
	"github.com/bancek/pb"
	"io"
)

type ReaderAtSeeker interface {
	io.ReaderAt
	io.ReadSeeker
}

type ProgressReader struct {
	ReaderAtSeeker
	bar *pb.ProgressBar
}

func NewProgressReader(r ReaderAtSeeker) (pr *ProgressReader, err error) {
	len, err := r.Seek(0, 2)

	if err != nil {
		return
	}

	if _, err = r.Seek(0, 0); err != nil {
		return
	}

	bar := pb.New(0)
	bar.Total = len

	bar.Units = pb.U_BYTES
	bar.Start()

	pr = &ProgressReader{
		ReaderAtSeeker: r,
		bar:            bar,
	}

	return
}

func (pr *ProgressReader) Read(p []byte) (len int, err error) {
	defer pr.bar.Read(p)
	return pr.ReaderAtSeeker.Read(p)
}

func (pr *ProgressReader) ReadAt(p []byte, off int64) (n int, err error) {
	if off == 0 {
		pr.bar.Set(0)
	}
	defer pr.bar.Read(p)
	return pr.ReaderAtSeeker.ReadAt(p, off)
}
<commit_msg>move to forked dependency<commit_after>package ioutils

import (
	"github.com/koofr/pb"
	"io"
)

type ReaderAtSeeker interface {
	io.ReaderAt
	io.ReadSeeker
}

type ProgressReader struct {
	ReaderAtSeeker
	bar *pb.ProgressBar
}

func NewProgressReader(r ReaderAtSeeker) (pr *ProgressReader, err error) {
	len, err := r.Seek(0, 2)

	if err != nil {
		return
	}

	if _, err = r.Seek(0, 0); err != nil {
		return
	}

	bar := pb.New(0)
	bar.Total = len

	bar.Units = pb.U_BYTES
	bar.Start()

	pr = &ProgressReader{
		ReaderAtSeeker: r,
		bar:            bar,
	}

	return
}

func (pr *ProgressReader) Read(p []byte) (len int, err error) {
	defer pr.bar.Read(p)
	return pr.ReaderAtSeeker.Read(p)
}

func (pr *ProgressReader) ReadAt(p []byte, off int64) (n int, err error) {
	if off == 0 {
		pr.bar.Set(0)
	}
	defer pr.bar.Read(p)
	return pr.ReaderAtSeeker.ReadAt(p, off)
}
<|endoftext|>


<commit_before>package restic

import (
	"fmt"
	"sync"
	"time"
)

type Progress struct {
	OnUpdate ProgressFunc
	OnDone   ProgressFunc
	fnM      sync.Mutex

	cur    Stat
	curM   sync.Mutex
	start  time.Time
	c      *time.Ticker
	cancel chan struct{}
	o      sync.Once
	d      time.Duration

	running bool
}

type Stat struct {
	Files uint64
	Dirs  uint64
	Bytes uint64
}

type ProgressFunc func(s Stat, runtime time.Duration, ticker bool)

// NewProgress returns a new progress reporter. After Start() has been called,
// the function OnUpdate is called when new data arrives or at least every d
// interval. The function OnDone is called when Done() is called. Both
// functions are called synchronously and can use shared state.
func NewProgress(d time.Duration) *Progress {
	return &Progress{d: d}
}

// Start runs resets and runs the progress reporter.
func (p *Progress) Start() {
	if p == nil {
		return
	}

	if p.running {
		panic("truing to reset a running Progress")
	}

	p.o = sync.Once{}
	p.cancel = make(chan struct{})
	p.running = true
	p.Reset()
	p.start = time.Now()
	p.c = time.NewTicker(p.d)

	go p.reporter()
}

// Report adds the statistics from s to the current state and tries to report
// the accumulated statistics via the feedback channel.
func (p *Progress) Report(s Stat) {
	if p == nil {
		return
	}

	if !p.running {
		panic("reporting in a non-running Progress")
	}

	p.curM.Lock()
	p.cur.Add(s)
	cur := p.cur
	p.curM.Unlock()

	// update progress
	if p.OnUpdate != nil {
		p.fnM.Lock()
		p.OnUpdate(cur, time.Since(p.start), false)
		p.fnM.Unlock()
	}
}

func (p *Progress) reporter() {
	if p == nil {
		return
	}

	for {
		select {
		case <-p.c.C:
			p.curM.Lock()
			cur := p.cur
			p.curM.Unlock()

			if p.OnUpdate != nil {
				p.fnM.Lock()
				p.OnUpdate(cur, time.Since(p.start), true)
				p.fnM.Unlock()
			}
		case <-p.cancel:
			p.c.Stop()
			return
		}
	}
}

// Reset resets all statistic counters to zero.
func (p *Progress) Reset() {
	if p == nil {
		return
	}

	if !p.running {
		panic("resetting a non-running Progress")
	}

	p.curM.Lock()
	p.cur = Stat{}
	p.curM.Unlock()
}

// Done closes the progress report.
func (p *Progress) Done() {
	if p == nil {
		return
	}

	if !p.running {
		panic("Done() called on non-running Progress")
	}

	if p.running {
		p.running = false
		p.o.Do(func() {
			close(p.cancel)
		})

		cur := p.cur

		if p.OnDone != nil {
			p.fnM.Lock()
			p.OnDone(cur, time.Since(p.start), false)
			p.fnM.Unlock()
		}
	}
}

// Current returns the current stat value.
func (p *Progress) Current() Stat {
	p.curM.Lock()
	s := p.cur
	p.curM.Unlock()

	return s
}

// Add accumulates other into s.
func (s *Stat) Add(other Stat) {
	s.Bytes += other.Bytes
	s.Dirs += other.Dirs
	s.Files += other.Files
}

func (s Stat) String() string {
	b := float64(s.Bytes)
	var str string

	switch {
	case s.Bytes > 1<<40:
		str = fmt.Sprintf("%.3f TiB", b/(1<<40))
	case s.Bytes > 1<<30:
		str = fmt.Sprintf("%.3f GiB", b/(1<<30))
	case s.Bytes > 1<<20:
		str = fmt.Sprintf("%.3f MiB", b/(1<<20))
	case s.Bytes > 1<<10:
		str = fmt.Sprintf("%.3f KiB", b/(1<<10))
	default:
		str = fmt.Sprintf("%dB", s.Bytes)
	}

	return fmt.Sprintf("Stat(%d files, %d dirs, %v)",
		s.Files, s.Dirs, str)
}
<commit_msg>Progress: Add convenience functions<commit_after>package restic

import (
	"fmt"
	"sync"
	"time"
)

type Progress struct {
	OnUpdate ProgressFunc
	OnDone   ProgressFunc
	fnM      sync.Mutex

	cur    Stat
	curM   sync.Mutex
	start  time.Time
	c      *time.Ticker
	cancel chan struct{}
	o      sync.Once
	d      time.Duration

	running bool
}

type Stat struct {
	Files uint64
	Dirs  uint64
	Bytes uint64
}

type ProgressFunc func(s Stat, runtime time.Duration, ticker bool)

// NewProgress returns a new progress reporter. After Start() has been called,
// the function OnUpdate is called when new data arrives or at least every d
// interval. The function OnDone is called when Done() is called. Both
// functions are called synchronously and can use shared state.
func NewProgress(d time.Duration) *Progress {
	return &Progress{d: d}
}

// Start runs resets and runs the progress reporter.
func (p *Progress) Start() {
	if p == nil {
		return
	}

	if p.running {
		panic("truing to reset a running Progress")
	}

	p.o = sync.Once{}
	p.cancel = make(chan struct{})
	p.running = true
	p.Reset()
	p.start = time.Now()
	p.c = time.NewTicker(p.d)

	go p.reporter()
}

// Report adds the statistics from s to the current state and tries to report
// the accumulated statistics via the feedback channel.
func (p *Progress) Report(s Stat) {
	if p == nil {
		return
	}

	if !p.running {
		panic("reporting in a non-running Progress")
	}

	p.curM.Lock()
	p.cur.Add(s)
	cur := p.cur
	p.curM.Unlock()

	// update progress
	if p.OnUpdate != nil {
		p.fnM.Lock()
		p.OnUpdate(cur, time.Since(p.start), false)
		p.fnM.Unlock()
	}
}

// Report a file with the given size.
func (p *Progress) ReportFile(size uint64) {
	p.Report(Stat{Files: 1, Bytes: size})
}

// Report a directory.
func (p *Progress) ReportDir() {
	p.Report(Stat{Dirs: 1})
}

func (p *Progress) reporter() {
	if p == nil {
		return
	}

	for {
		select {
		case <-p.c.C:
			p.curM.Lock()
			cur := p.cur
			p.curM.Unlock()

			if p.OnUpdate != nil {
				p.fnM.Lock()
				p.OnUpdate(cur, time.Since(p.start), true)
				p.fnM.Unlock()
			}
		case <-p.cancel:
			p.c.Stop()
			return
		}
	}
}

// Reset resets all statistic counters to zero.
func (p *Progress) Reset() {
	if p == nil {
		return
	}

	if !p.running {
		panic("resetting a non-running Progress")
	}

	p.curM.Lock()
	p.cur = Stat{}
	p.curM.Unlock()
}

// Done closes the progress report.
func (p *Progress) Done() {
	if p == nil {
		return
	}

	if !p.running {
		panic("Done() called on non-running Progress")
	}

	if p.running {
		p.running = false
		p.o.Do(func() {
			close(p.cancel)
		})

		cur := p.cur

		if p.OnDone != nil {
			p.fnM.Lock()
			p.OnDone(cur, time.Since(p.start), false)
			p.fnM.Unlock()
		}
	}
}

// Current returns the current stat value.
func (p *Progress) Current() Stat {
	p.curM.Lock()
	s := p.cur
	p.curM.Unlock()

	return s
}

// Add accumulates other into s.
func (s *Stat) Add(other Stat) {
	s.Bytes += other.Bytes
	s.Dirs += other.Dirs
	s.Files += other.Files
}

func (s Stat) String() string {
	b := float64(s.Bytes)
	var str string

	switch {
	case s.Bytes > 1<<40:
		str = fmt.Sprintf("%.3f TiB", b/(1<<40))
	case s.Bytes > 1<<30:
		str = fmt.Sprintf("%.3f GiB", b/(1<<30))
	case s.Bytes > 1<<20:
		str = fmt.Sprintf("%.3f MiB", b/(1<<20))
	case s.Bytes > 1<<10:
		str = fmt.Sprintf("%.3f KiB", b/(1<<10))
	default:
		str = fmt.Sprintf("%dB", s.Bytes)
	}

	return fmt.Sprintf("Stat(%d files, %d dirs, %v)",
		s.Files, s.Dirs, str)
}
<|endoftext|>


<commit_before>package mpb

import (
	"context"
	"errors"
	"io"
	"os"
	"sort"
	"sync"
	"time"

	"github.com/vbauerster/mpb/cwriter"
)

// ErrCallAfterStop thrown by panic, if Progress methods like AddBar() are called
// after Stop() has been called
var ErrCallAfterStop = errors.New("method call on stopped Progress instance")

type opType uint

const (
	opBarAdd opType = iota
	opBarRemove
)

type SortType uint

const (
	SortNone SortType = iota
	SortTop
	SortBottom
)

// default RefreshRate
const rr = 100

// Progress represents the container that renders Progress bars
type Progress struct {
	// Context for canceling bars rendering
	ctx context.Context
	// WaitGroup for internal rendering sync
	wg *sync.WaitGroup

	out   io.Writer
	width int
	sort  SortType

	operationCh    chan *operation
	rrChangeReqCh  chan time.Duration
	outChangeReqCh chan io.Writer
	barCountReqCh  chan chan int
	done           chan struct{}
}

type operation struct {
	kind   opType
	bar    *Bar
	result chan bool
}

// New creates new Progress instance, which will orchestrate bars rendering
// process. It acceepts context.Context, for cancellation.
// If you don't plan to cancel, it is safe to feed with nil
func New(ctx context.Context) *Progress {
	if ctx == nil {
		ctx = context.Background()
	}
	p := &Progress{
		width:          70,
		operationCh:    make(chan *operation),
		rrChangeReqCh:  make(chan time.Duration),
		outChangeReqCh: make(chan io.Writer),
		barCountReqCh:  make(chan chan int),
		done:           make(chan struct{}),
		wg:             new(sync.WaitGroup),
		ctx:            ctx,
	}
	go p.server(cwriter.New(os.Stdout), time.NewTicker(rr*time.Millisecond))
	return p
}

// SetWidth sets the width for all underlying bars
func (p *Progress) SetWidth(n int) *Progress {
	if n <= 0 {
		return p
	}
	p.width = n
	return p
}

// SetOut sets underlying writer of progress. Default is os.Stdout
// pancis, if called on stopped Progress instance, i.e after Stop()
func (p *Progress) SetOut(w io.Writer) *Progress {
	if p.isDone() {
		panic(ErrCallAfterStop)
	}
	if w == nil {
		return p
	}
	p.outChangeReqCh <- w
	return p
}

// RefreshRate overrides default (30ms) refreshRate value
// pancis, if called on stopped Progress instance, i.e after Stop()
func (p *Progress) RefreshRate(d time.Duration) *Progress {
	if p.isDone() {
		panic(ErrCallAfterStop)
	}
	p.rrChangeReqCh <- d
	return p
}

// func (p *Progress) WithContext(ctx context.Context) *Progress {
// 	if p.BarCount() > 0 {
// 		panic("cannot apply ctx after AddBar has been called")
// 	}
// 	if ctx == nil {
// 		panic("nil context")
// 	}
// 	p.ctx = ctx
// 	return p
// }

// WithSort sorts the bars, while redering
func (p *Progress) WithSort(sort SortType) *Progress {
	p.sort = sort
	return p
}

// AddBar creates a new progress bar and adds to the container
// pancis, if called on stopped Progress instance, i.e after Stop()
func (p *Progress) AddBar(total int64) *Bar {
	if p.isDone() {
		panic(ErrCallAfterStop)
	}
	result := make(chan bool)
	bar := newBar(p.ctx, p.wg, total, p.width)
	p.operationCh <- &operation{opBarAdd, bar, result}
	if <-result {
		p.wg.Add(1)
	}
	return bar
}

// RemoveBar removes bar at any time
// pancis, if called on stopped Progress instance, i.e after Stop()
func (p *Progress) RemoveBar(b *Bar) bool {
	if p.isDone() {
		panic(ErrCallAfterStop)
	}
	result := make(chan bool)
	p.operationCh <- &operation{opBarRemove, b, result}
	return <-result
}

// BarCount returns bars count in the container.
// Pancis if called on stopped Progress instance, i.e after Stop()
func (p *Progress) BarCount() int {
	if p.isDone() {
		panic(ErrCallAfterStop)
	}
	respCh := make(chan int)
	p.barCountReqCh <- respCh
	return <-respCh
}

// Stop waits for bars to finish rendering and stops the rendering goroutine
func (p *Progress) Stop() {
	p.wg.Wait()
	if !p.isDone() {
		close(p.done)
		close(p.operationCh)
	}
}

// server monitors underlying channels and renders any progress bars
func (p *Progress) server(cw *cwriter.Writer, t *time.Ticker) {
	const numDigesters = 4
	bars := make([]*Bar, 0, 4)
	for {
		select {
		case w := <-p.outChangeReqCh:
			cw.Flush()
			cw = cwriter.New(w)
		case op, ok := <-p.operationCh:
			if !ok {
				t.Stop()
				return
			}
			switch op.kind {
			case opBarAdd:
				bars = append(bars, op.bar)
				op.result <- true
			case opBarRemove:
				var ok bool
				for i, b := range bars {
					if b == op.bar {
						bars = append(bars[:i], bars[i+1:]...)
						ok = true
						b.remove()
						break
					}
				}
				op.result <- ok
			}
		case respCh := <-p.barCountReqCh:
			respCh <- len(bars)
		case <-t.C:
			switch p.sort {
			case SortTop:
				sort.Sort(sort.Reverse(SortableBarSlice(bars)))
			case SortBottom:
				sort.Sort(SortableBarSlice(bars))
			}

			width, _ := cwriter.TerminalWidth()
			ibars := iBarsGen(p.ctx.Done(), bars, width)
			c := make(chan *indexedBarBuffer)
			var wg sync.WaitGroup
			wg.Add(numDigesters)
			for i := 0; i < numDigesters; i++ {
				go func() {
					drawer(p.ctx.Done(), ibars, c)
					wg.Done()
				}()
			}
			go func() {
				wg.Wait()
				close(c)
			}()

			m := make(map[int][]byte, len(bars))
			for r := range c {
				m[r.index] = r.buff
			}
			for i := 0; i < len(bars); i++ {
				m[i] = append(m[i], '\n')
				cw.Write(m[i])
			}
			cw.Flush()
			go flushed(p.ctx.Done(), bars)

		case d := <-p.rrChangeReqCh:
			t.Stop()
			t = time.NewTicker(d)
		case <-p.ctx.Done():
			t.Stop()
			close(p.done)
			return
		}
	}
}

type indexedBarBuffer struct {
	index int
	buff  []byte
}

type indexedBar struct {
	index int
	width int
	bar   *Bar
}

func drawer(done <-chan struct{}, ibars <-chan *indexedBar, c chan<- *indexedBarBuffer) {
	for b := range ibars {
		select {
		case c <- &indexedBarBuffer{b.index, b.bar.bytes(b.width)}:
		case <-done:
			return
		}
	}
}

func iBarsGen(done <-chan struct{}, bars []*Bar, width int) <-chan *indexedBar {
	ibars := make(chan *indexedBar)
	go func() {
		defer close(ibars)
		for i, b := range bars {
			select {
			case ibars <- &indexedBar{i, width, b}:
			case <-done:
				return
			}
		}
	}()
	return ibars
}

func flushed(done <-chan struct{}, bars []*Bar) {
	for _, b := range bars {
		select {
		case <-done:
			return
		default:
			b.flushDone()
		}
	}
}

func (p *Progress) isDone() bool {
	select {
	case <-p.done:
		return true
	default:
		return false
	}
}
<commit_msg>refactoring<commit_after>package mpb

import (
	"context"
	"errors"
	"io"
	"os"
	"sort"
	"sync"
	"time"

	"github.com/vbauerster/mpb/cwriter"
)

// ErrCallAfterStop thrown by panic, if Progress methods like AddBar() are called
// after Stop() has been called
var ErrCallAfterStop = errors.New("method call on stopped Progress instance")

type (
	// SortType defines sort direction of bar
	SortType uint
	opType   uint

	operation struct {
		kind   opType
		bar    *Bar
		result chan bool
	}

	indexedBarBuffer struct {
		index int
		buff  []byte
	}

	indexedBar struct {
		index int
		width int
		bar   *Bar
	}
)

const (
	opBarAdd opType = iota
	opBarRemove
)

const (
	SortNone SortType = iota
	SortTop
	SortBottom
)

// default RefreshRate
const rr = 100

// Progress represents the container that renders Progress bars
type Progress struct {
	// Context for canceling bars rendering
	ctx context.Context
	// WaitGroup for internal rendering sync
	wg *sync.WaitGroup

	out   io.Writer
	width int
	sort  SortType

	operationCh    chan *operation
	rrChangeReqCh  chan time.Duration
	outChangeReqCh chan io.Writer
	barCountReqCh  chan chan int
	done           chan struct{}
}

// New creates new Progress instance, which will orchestrate bars rendering
// process. It acceepts context.Context, for cancellation.
// If you don't plan to cancel, it is safe to feed with nil
func New(ctx context.Context) *Progress {
	if ctx == nil {
		ctx = context.Background()
	}
	p := &Progress{
		width:          70,
		operationCh:    make(chan *operation),
		rrChangeReqCh:  make(chan time.Duration),
		outChangeReqCh: make(chan io.Writer),
		barCountReqCh:  make(chan chan int),
		done:           make(chan struct{}),
		wg:             new(sync.WaitGroup),
		ctx:            ctx,
	}
	go p.server(cwriter.New(os.Stdout), time.NewTicker(rr*time.Millisecond))
	return p
}

// SetWidth sets the width for all underlying bars
func (p *Progress) SetWidth(n int) *Progress {
	if n <= 0 {
		return p
	}
	p.width = n
	return p
}

// SetOut sets underlying writer of progress. Default is os.Stdout
// pancis, if called on stopped Progress instance, i.e after Stop()
func (p *Progress) SetOut(w io.Writer) *Progress {
	if p.isDone() {
		panic(ErrCallAfterStop)
	}
	if w == nil {
		return p
	}
	p.outChangeReqCh <- w
	return p
}

// RefreshRate overrides default (30ms) refreshRate value
// pancis, if called on stopped Progress instance, i.e after Stop()
func (p *Progress) RefreshRate(d time.Duration) *Progress {
	if p.isDone() {
		panic(ErrCallAfterStop)
	}
	p.rrChangeReqCh <- d
	return p
}

// WithSort sorts the bars, while redering
func (p *Progress) WithSort(sort SortType) *Progress {
	p.sort = sort
	return p
}

// AddBar creates a new progress bar and adds to the container
// pancis, if called on stopped Progress instance, i.e after Stop()
func (p *Progress) AddBar(total int64) *Bar {
	if p.isDone() {
		panic(ErrCallAfterStop)
	}
	result := make(chan bool)
	bar := newBar(p.ctx, p.wg, total, p.width)
	p.operationCh <- &operation{opBarAdd, bar, result}
	if <-result {
		p.wg.Add(1)
	}
	return bar
}

// RemoveBar removes bar at any time
// pancis, if called on stopped Progress instance, i.e after Stop()
func (p *Progress) RemoveBar(b *Bar) bool {
	if p.isDone() {
		panic(ErrCallAfterStop)
	}
	result := make(chan bool)
	p.operationCh <- &operation{opBarRemove, b, result}
	return <-result
}

// BarCount returns bars count in the container.
// Pancis if called on stopped Progress instance, i.e after Stop()
func (p *Progress) BarCount() int {
	if p.isDone() {
		panic(ErrCallAfterStop)
	}
	respCh := make(chan int)
	p.barCountReqCh <- respCh
	return <-respCh
}

// Stop waits for bars to finish rendering and stops the rendering goroutine
func (p *Progress) Stop() {
	p.wg.Wait()
	if !p.isDone() {
		close(p.done)
		close(p.operationCh)
	}
}

func (p *Progress) isDone() bool {
	select {
	case <-p.done:
		return true
	default:
		return false
	}
}

// server monitors underlying channels and renders any progress bars
func (p *Progress) server(cw *cwriter.Writer, t *time.Ticker) {
	const numDrawers = 4
	bars := make([]*Bar, 0, 4)
	for {
		select {
		case w := <-p.outChangeReqCh:
			cw.Flush()
			cw = cwriter.New(w)
		case op, ok := <-p.operationCh:
			if !ok {
				t.Stop()
				return
			}
			switch op.kind {
			case opBarAdd:
				bars = append(bars, op.bar)
				op.result <- true
			case opBarRemove:
				var ok bool
				for i, b := range bars {
					if b == op.bar {
						bars = append(bars[:i], bars[i+1:]...)
						ok = true
						b.remove()
						break
					}
				}
				op.result <- ok
			}
		case respCh := <-p.barCountReqCh:
			respCh <- len(bars)
		case <-t.C:
			switch p.sort {
			case SortTop:
				sort.Sort(sort.Reverse(SortableBarSlice(bars)))
			case SortBottom:
				sort.Sort(SortableBarSlice(bars))
			}

			width, _ := cwriter.TerminalWidth()
			ibars := iBarsGen(bars, width)
			c := make(chan indexedBarBuffer)
			var wg sync.WaitGroup
			wg.Add(numDrawers)
			for i := 0; i < numDrawers; i++ {
				go func() {
					drawer(ibars, c)
					wg.Done()
				}()
			}
			go func() {
				wg.Wait()
				close(c)
			}()

			m := make(map[int][]byte, len(bars))
			for r := range c {
				m[r.index] = r.buff
			}
			for i := 0; i < len(bars); i++ {
				m[i] = append(m[i], '\n')
				cw.Write(m[i])
			}

			cw.Flush()

			go func() {
				for _, b := range bars {
					b.flushDone()
				}
			}()
		case d := <-p.rrChangeReqCh:
			t.Stop()
			t = time.NewTicker(d)
		case <-p.ctx.Done():
			t.Stop()
			close(p.done)
			return
		}
	}
}

func drawer(ibars <-chan indexedBar, c chan<- indexedBarBuffer) {
	for b := range ibars {
		c <- indexedBarBuffer{b.index, b.bar.bytes(b.width)}
	}
}

func iBarsGen(bars []*Bar, width int) <-chan indexedBar {
	ibars := make(chan indexedBar)
	go func() {
		defer close(ibars)
		for i, b := range bars {
			ibars <- indexedBar{i, width, b}
		}
	}()
	return ibars
}
<|endoftext|>


<commit_before>package main

/*
free quote downloader

Usage:
  quote -h | -help
  quote -v | -version
  quote [-y <years>|(-b <beginDate> [-e <endDate>])] [flags] [-i <symFile>|etf|nyse|amex|nasdaq|<symbol>...]

Options:
  -h -help                 Show help
  -v -version              Show version
  -infile <symbolFile>  List of symbols to download
  -years <years>        Number of years to download [default: 5]
  -begin <beginDate>    yyyy-mm-dd
  -end <endDate>        yyyy-mm-dd
  -period <period>      1m|5m|15m|30m|1h|d|w|m [default: d]
  -source <source>      yahoo|google [default: yahoo]
  -outfile <outFile>    Output filename
  -format <outFormat>   (csv|json) [default: json]
  -all <allInOne>      All in one file (true|false) [default: true]`
*/
// TODO:
// version flag
// yahoo adjust prices flag, pacing flag
// stdout/stdin? piping
// log file

import (
	"flag"
	"fmt"
	"github.com/markcheno/go-quote"
	"io/ioutil"
	"os"
	"strings"
	"time"
)

const version = "0.1"
const dateFormat = "2006-01-02"

var yearsFlag int
var beginFlag string
var endFlag string
var periodFlag string
var sourceFlag string
var inFlag string
var outFlag string
var formatFlag string
var allFlag bool

func init() {
	const (
		yearsUsage  = "Number of years to download"
		beginUsage  = "Begin date (yyyy[-mm[-dd]])"
		endUsage    = "End date (yyyy[-mm[-dd]])"
		periodUsage = "1m|5m|15m|30m|1h|d|w|m"
		sourceUsage = "yahoo|google"
		inUsage     = "Input filename"
		outUsage    = "Output filename"
		formatUsage = "csv|json"
		allUsage    = "all output in one file"
	)
	//flag.IntVar(&yearsFlag, "y", 5, yearsUsage)
	flag.IntVar(&yearsFlag, "years", 5, yearsUsage)

	//flag.StringVar(&beginFlag, "b", "", beginUsage)
	flag.StringVar(&beginFlag, "begin", "", beginUsage)

	//flag.StringVar(&endFlag, "e", "", endUsage)
	flag.StringVar(&endFlag, "end", "", endUsage)

	//flag.StringVar(&periodFlag, "p", "d", periodUsage)
	flag.StringVar(&periodFlag, "period", "d", periodUsage)

	//flag.StringVar(&sourceFlag, "s", "yahoo", sourceUsage)
	flag.StringVar(&sourceFlag, "source", "yahoo", sourceUsage)

	//flag.StringVar(&inFlag, "i", "", inUsage)
	flag.StringVar(&inFlag, "infile", "", inUsage)

	//flag.StringVar(&outFlag, "o", "", outUsage)
	flag.StringVar(&outFlag, "outfile", "", outUsage)

	//flag.StringVar(&formatFlag, "f", "csv", formatUsage)
	flag.StringVar(&formatFlag, "format", "csv", formatUsage)

	//flag.BoolVar(&allFlag, "a", true, allUsage)
	flag.BoolVar(&allFlag, "all", true, allUsage)

	flag.Parse()
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func main() {

	// determine symbols
	var symbols []string
	if inFlag != "" {
		raw, err := ioutil.ReadFile(inFlag)
		check(err)
		symbols = strings.Split(string(raw), "\n")
	} else {
		symbols = flag.Args()
	}
	if len(symbols) == 0 {
		panic(fmt.Errorf("no symbols"))
	}
	//fmt.Println(symbols)

	// determine outfile
	//fmt.Println("outFlag=" + outFlag)

	// validate source
	if sourceFlag != "yahoo" && sourceFlag != "google" {
		panic(fmt.Errorf("invalid source"))
	}

	// determine period
	var period quote.Period
	switch periodFlag {
	case "1m":
		period = quote.Min1
	case "5m":
		period = quote.Min5
	case "15m":
		period = quote.Min15
	case "30m":
		period = quote.Min30
	case "60m":
		period = quote.Min60
	case "d":
		period = quote.Daily
	case "w":
		period = quote.Weekly
	case "m":
		period = quote.Monthly
	case "y":
		period = quote.Yearly
	}
	//fmt.Println("period=" + period)

	// handle exchanges
	switch symbols[0] {
	case "etf":
		quote.NewEtfFile(outFlag)
		os.Exit(0)
	case "nyse":
		quote.NewExchangeFile("nyse", outFlag)
		os.Exit(0)
	case "nasdaq":
		quote.NewExchangeFile("nasdaq", outFlag)
		os.Exit(0)
	case "amex":
		quote.NewExchangeFile("amex", outFlag)
		os.Exit(0)
	}

	// determine begin/end times
	var from, to time.Time

	if beginFlag != "" {

		from = quote.ParseDTString(beginFlag)

		if endFlag != "" {
			to = quote.ParseDTString(endFlag)
		} else {
			to = time.Now()
		}
	} else {
		to = time.Now()
		from = to.Add(-time.Duration(int(time.Hour) * 24 * 365 * yearsFlag))
	}
	//fmt.Printf("from=%s, to=%s", from, to)

	if allFlag {
		quotes := quote.Quotes{}
		if sourceFlag == "yahoo" {
			quotes, _ = quote.NewQuotesFromYahooSyms(symbols, from.Format(dateFormat), to.Format(dateFormat), period, true)
		} else if sourceFlag == "google" {
			quotes, _ = quote.NewQuotesFromGoogleSyms(symbols, from.Format(dateFormat), to.Format(dateFormat), period)
		}
		if formatFlag == "csv" {
			quotes.WriteCSV(outFlag)
		} else if formatFlag == "json" {
			quotes.WriteJSON(outFlag, false)
		}
		os.Exit(0) // done
	} else {
		// output individual symbol files
		for _, sym := range symbols {
			var q quote.Quote
			if sourceFlag == "yahoo" {
				q, _ = quote.NewQuoteFromYahoo(sym, from.Format(dateFormat), to.Format(dateFormat), period, true)
			} else if sourceFlag == "google" {
				q, _ = quote.NewQuoteFromGoogle(sym, from.Format(dateFormat), to.Format(dateFormat), period)
			}
			if formatFlag == "csv" {
				q.WriteCSV(outFlag)
			} else if formatFlag == "json" {
				q.WriteJSON(outFlag, false)
			}
		}
	}
}
<commit_msg>minor fix<commit_after>package main

/*
free quote downloader

Usage:
  quote -h | -help
  quote -v | -version
  quote [-y <years>|(-b <beginDate> [-e <endDate>])] [flags] [-i <symFile>|etf|nyse|amex|nasdaq|<symbol>...]

Options:
  -h -help              Show help
  -v -version           Show version
  -infile <symbolFile>  List of symbols to download
  -years <years>        Number of years to download [default: 5]
  -begin <beginDate>    yyyy-mm-dd
  -end <endDate>        yyyy-mm-dd
  -period <period>      1m|5m|15m|30m|1h|d|w|m [default: d]
  -source <source>      yahoo|google [default: yahoo]
  -outfile <outFile>    Output filename
  -format <outFormat>   (csv|json) [default: json]
  -all <allInOne>       All in one file (true|false) [default: true]`
*/
// TODO:
// version flag
// yahoo adjust prices flag, pacing flag
// stdout/stdin? piping
// log file

import (
	"flag"
	"fmt"
	"github.com/markcheno/go-quote"
	"io/ioutil"
	"os"
	"strings"
	"time"
)

const version = "0.1"
const dateFormat = "2006-01-02"

var yearsFlag int
var beginFlag string
var endFlag string
var periodFlag string
var sourceFlag string
var inFlag string
var outFlag string
var formatFlag string
var allFlag bool

func init() {
	const (
		yearsUsage  = "Number of years to download"
		beginUsage  = "Begin date (yyyy[-mm[-dd]])"
		endUsage    = "End date (yyyy[-mm[-dd]])"
		periodUsage = "1m|5m|15m|30m|1h|d|w|m"
		sourceUsage = "yahoo|google"
		inUsage     = "Input filename"
		outUsage    = "Output filename"
		formatUsage = "csv|json"
		allUsage    = "all output in one file"
	)
	//flag.IntVar(&yearsFlag, "y", 5, yearsUsage)
	flag.IntVar(&yearsFlag, "years", 5, yearsUsage)

	//flag.StringVar(&beginFlag, "b", "", beginUsage)
	flag.StringVar(&beginFlag, "begin", "", beginUsage)

	//flag.StringVar(&endFlag, "e", "", endUsage)
	flag.StringVar(&endFlag, "end", "", endUsage)

	//flag.StringVar(&periodFlag, "p", "d", periodUsage)
	flag.StringVar(&periodFlag, "period", "d", periodUsage)

	//flag.StringVar(&sourceFlag, "s", "yahoo", sourceUsage)
	flag.StringVar(&sourceFlag, "source", "yahoo", sourceUsage)

	//flag.StringVar(&inFlag, "i", "", inUsage)
	flag.StringVar(&inFlag, "infile", "", inUsage)

	//flag.StringVar(&outFlag, "o", "", outUsage)
	flag.StringVar(&outFlag, "outfile", "", outUsage)

	//flag.StringVar(&formatFlag, "f", "csv", formatUsage)
	flag.StringVar(&formatFlag, "format", "csv", formatUsage)

	//flag.BoolVar(&allFlag, "a", true, allUsage)
	flag.BoolVar(&allFlag, "all", true, allUsage)

	flag.Parse()
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func main() {

	// determine symbols
	var symbols []string
	if inFlag != "" {
		raw, err := ioutil.ReadFile(inFlag)
		check(err)
		symbols = strings.Split(string(raw), "\n")
	} else {
		symbols = flag.Args()
	}
	if len(symbols) == 0 {
		panic(fmt.Errorf("no symbols"))
	}
	//fmt.Println(symbols)

	// determine outfile
	//fmt.Println("outFlag=" + outFlag)

	// validate source
	if sourceFlag != "yahoo" && sourceFlag != "google" {
		panic(fmt.Errorf("invalid source"))
	}

	// determine period
	var period quote.Period
	switch periodFlag {
	case "1m":
		period = quote.Min1
	case "5m":
		period = quote.Min5
	case "15m":
		period = quote.Min15
	case "30m":
		period = quote.Min30
	case "60m":
		period = quote.Min60
	case "d":
		period = quote.Daily
	case "w":
		period = quote.Weekly
	case "m":
		period = quote.Monthly
	case "y":
		period = quote.Yearly
	}
	//fmt.Println("period=" + period)

	// handle exchanges
	switch symbols[0] {
	case "etf":
		quote.NewEtfFile(outFlag)
		os.Exit(0)
	case "nyse":
		quote.NewExchangeFile("nyse", outFlag)
		os.Exit(0)
	case "nasdaq":
		quote.NewExchangeFile("nasdaq", outFlag)
		os.Exit(0)
	case "amex":
		quote.NewExchangeFile("amex", outFlag)
		os.Exit(0)
	}

	// determine begin/end times
	var from, to time.Time

	if beginFlag != "" {

		from = quote.ParseDTString(beginFlag)

		if endFlag != "" {
			to = quote.ParseDTString(endFlag)
		} else {
			to = time.Now()
		}
	} else {
		to = time.Now()
		from = to.Add(-time.Duration(int(time.Hour) * 24 * 365 * yearsFlag))
	}
	//fmt.Printf("from=%s, to=%s", from, to)

	if allFlag {
		quotes := quote.Quotes{}
		if sourceFlag == "yahoo" {
			quotes, _ = quote.NewQuotesFromYahooSyms(symbols, from.Format(dateFormat), to.Format(dateFormat), period, true)
		} else if sourceFlag == "google" {
			quotes, _ = quote.NewQuotesFromGoogleSyms(symbols, from.Format(dateFormat), to.Format(dateFormat), period)
		}
		if formatFlag == "csv" {
			quotes.WriteCSV(outFlag)
		} else if formatFlag == "json" {
			quotes.WriteJSON(outFlag, false)
		}
		os.Exit(0) // done
	} else {
		// output individual symbol files
		for _, sym := range symbols {
			var q quote.Quote
			if sourceFlag == "yahoo" {
				q, _ = quote.NewQuoteFromYahoo(sym, from.Format(dateFormat), to.Format(dateFormat), period, true)
			} else if sourceFlag == "google" {
				q, _ = quote.NewQuoteFromGoogle(sym, from.Format(dateFormat), to.Format(dateFormat), period)
			}
			if formatFlag == "csv" {
				q.WriteCSV(outFlag)
			} else if formatFlag == "json" {
				q.WriteJSON(outFlag, false)
			}
		}
	}
}
<|endoftext|>


<commit_before>package model

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"text/template"
	"time"

	"github.com/minodisk/qiitactl/api"

	"gopkg.in/yaml.v2"
)

const (
	postTemplate = `<!--
{{.Meta.Format}}
-->
# {{.Title}}
{{.Body}}`
	DirMine = "mine"
)

var (
	rPostDecoder = regexp.MustCompile(`^(?ms:\n*<!--(.*)-->\n*# +(.*?)\n+(.*))$`)
	tmpl         = func() (t *template.Template) {
		t = template.New("postfile")
		template.Must(t.Parse(postTemplate))
		return
	}()
	rInvalidFilename = regexp.MustCompile(`[^a-zA-Z0-9\-]+`)
	rHyphens         = regexp.MustCompile(`\-{2,}`)
)

type Post struct {
	Meta
	User         User   `json:"user"`
	Title        string `json:"title"`         // 投稿のタイトル
	Body         string `json:"body"`          // Markdown形式の本文
	RenderedBody string `json:"rendered_body"` // HTML形式の本文
	Team         *Team  `json:"-"`             // チーム
}

type CreationOptions struct {
	Tweet bool `json:"tweet"`
	Gist  bool `json:"gist"`
}

type CreationPost struct {
	Post
	CreationOptions
}

func NewPost(title string, createdAt *Time, team *Team) (post Post) {
	if createdAt == nil {
		createdAt = &Time{Time: time.Now()}
	}
	post.CreatedAt = *createdAt
	post.UpdatedAt = *createdAt
	post.Title = title
	post.Team = team
	return
}

func NewPostWithFile(path string) (post Post, err error) {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return
	}
	err = post.Decode(b)
	if err != nil {
		return
	}
	return
}

func (post *Post) Create(client api.Client, opts CreationOptions) (err error) {
	subDomain := ""
	if post.Team != nil {
		subDomain = post.Team.ID
	}

	cPost := CreationPost{
		Post:            *post,
		CreationOptions: opts,
	}
	body, _, err := client.Post(subDomain, "/items", cPost)
	if err != nil {
		return
	}

	err = json.Unmarshal(body, post)
	if err != nil {
		return
	}
	return
}

func FetchPost(client api.Client, team *Team, id string) (post Post, err error) {
	subDomain := ""
	if team != nil {
		subDomain = team.ID
	}
	body, _, err := client.Get(subDomain, fmt.Sprintf("/items/%s", id), nil)
	if err != nil {
		return
	}
	err = json.Unmarshal(body, &post)
	if err != nil {
		return
	}
	post.Team = team
	return
}

func (post *Post) Update(client api.Client) (err error) {
	if post.ID == "" {
		err = EmptyIDError{}
		return
	}

	subDomain := ""
	if post.Team != nil {
		subDomain = post.Team.ID
	}
	body, _, err := client.Patch(subDomain, fmt.Sprintf("/items/%s", post.ID), post)
	if err != nil {
		return
	}
	err = json.Unmarshal(body, post)
	if err != nil {
		return
	}
	return
}

func (post *Post) Delete(client api.Client) (err error) {
	if post.ID == "" {
		err = EmptyIDError{}
		return
	}

	subDomain := ""
	if post.Team != nil {
		subDomain = post.Team.ID
	}
	body, _, err := client.Delete(subDomain, fmt.Sprintf("/items/%s", post.ID), post)
	if err != nil {
		return
	}
	err = json.Unmarshal(body, post)
	if err != nil {
		return
	}
	return
}

func (post Post) Save() (err error) {
	var path string
	if post.ID != "" {
		path, err = post.findPath()
		if err != nil {
			return err
		}
	}
	if path == "" {
		path = post.createPath()
	}

	dir := filepath.Dir(path)
	err = os.MkdirAll(dir, 0755)
	if err != nil {
		return
	}

	fmt.Printf("Make file: %s\n", path)
	f, err := os.Create(path)
	defer f.Close()
	if err != nil {
		return
	}
	err = post.Encode(f)
	if err != nil {
		return
	}
	return
}

func (post Post) findPath() (path string, err error) {
	dir, err := os.Getwd()
	if err != nil {
		return
	}
	found := false
	filepath.Walk(dir, func(p string, info os.FileInfo, e error) (err error) {
		if e != nil {
			err = e
			return
		}
		if found {
			return filepath.SkipDir
		}
		if info.IsDir() {
			return
		}
		if filepath.Ext(p) != ".md" {
			return
		}

		postInLocal, err := NewPostWithFile(p)
		if err != nil {
			err = nil
			return
		}
		if postInLocal.ID == post.ID {
			path = p
			found = true
			return filepath.SkipDir
		}
		return
	})
	return
}

func (post Post) createPath() (path string) {
	var dirname string
	if post.Team == nil {
		dirname = DirMine
	} else {
		dirname = post.Team.ID
	}
	dirname = filepath.Join(dirname, post.CreatedAt.Format("2006/01"))

	filename := fmt.Sprintf("%s-%s", post.CreatedAt.Format("02"), post.Title)
	filename = rInvalidFilename.ReplaceAllString(filename, "-")
	filename = strings.ToLower(filename)
	filename = rHyphens.ReplaceAllString(filename, "-")
	filename = strings.TrimRight(filename, "-")

	for {
		path = filepath.Join(dirname, fmt.Sprintf("%s.md", filename))
		_, err := os.Stat(path)
		// no error means: a file exists at the path
		// error occurs means: no file exists at the path
		if err != nil { //TODO test me
			break
		}
		filename += "-"
	}
	return
}

func (post Post) Encode(w io.Writer) (err error) {
	err = tmpl.Execute(w, post)
	return
}

func (post *Post) Decode(b []byte) (err error) {
	matched := rPostDecoder.FindSubmatch(b)
	if len(matched) != 4 {
		err = fmt.Errorf("wrong format")
		return
	}

	err = yaml.Unmarshal((bytes.TrimSpace(matched[1])), &post.Meta)
	if err != nil {
		return
	}
	post.Title = string(bytes.TrimSpace(matched[2]))
	post.Body = string(bytes.TrimSpace(matched[3]))
	return
}

type EmptyIDError struct{}

func (err EmptyIDError) Error() (msg string) {
	msg = "empty ID"
	return
}
<commit_msg>Stop walking after finding path<commit_after>package model

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"text/template"
	"time"

	"github.com/minodisk/qiitactl/api"

	"gopkg.in/yaml.v2"
)

const (
	postTemplate = `<!--
{{.Meta.Format}}
-->
# {{.Title}}
{{.Body}}`
	DirMine = "mine"
)

var (
	rPostDecoder = regexp.MustCompile(`^(?ms:\n*<!--(.*)-->\n*# +(.*?)\n+(.*))$`)
	tmpl         = func() (t *template.Template) {
		t = template.New("postfile")
		template.Must(t.Parse(postTemplate))
		return
	}()
	rInvalidFilename = regexp.MustCompile(`[^a-zA-Z0-9\-]+`)
	rHyphens         = regexp.MustCompile(`\-{2,}`)
)

type Post struct {
	Meta
	User         User   `json:"user"`
	Title        string `json:"title"`         // 投稿のタイトル
	Body         string `json:"body"`          // Markdown形式の本文
	RenderedBody string `json:"rendered_body"` // HTML形式の本文
	Team         *Team  `json:"-"`             // チーム
}

type CreationOptions struct {
	Tweet bool `json:"tweet"`
	Gist  bool `json:"gist"`
}

type CreationPost struct {
	Post
	CreationOptions
}

func NewPost(title string, createdAt *Time, team *Team) (post Post) {
	if createdAt == nil {
		createdAt = &Time{Time: time.Now()}
	}
	post.CreatedAt = *createdAt
	post.UpdatedAt = *createdAt
	post.Title = title
	post.Team = team
	return
}

func NewPostWithFile(path string) (post Post, err error) {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return
	}
	err = post.Decode(b)
	if err != nil {
		return
	}
	return
}

func (post *Post) Create(client api.Client, opts CreationOptions) (err error) {
	subDomain := ""
	if post.Team != nil {
		subDomain = post.Team.ID
	}

	cPost := CreationPost{
		Post:            *post,
		CreationOptions: opts,
	}
	body, _, err := client.Post(subDomain, "/items", cPost)
	if err != nil {
		return
	}

	err = json.Unmarshal(body, post)
	if err != nil {
		return
	}
	return
}

func FetchPost(client api.Client, team *Team, id string) (post Post, err error) {
	subDomain := ""
	if team != nil {
		subDomain = team.ID
	}
	body, _, err := client.Get(subDomain, fmt.Sprintf("/items/%s", id), nil)
	if err != nil {
		return
	}
	err = json.Unmarshal(body, &post)
	if err != nil {
		return
	}
	post.Team = team
	return
}

func (post *Post) Update(client api.Client) (err error) {
	if post.ID == "" {
		err = EmptyIDError{}
		return
	}

	subDomain := ""
	if post.Team != nil {
		subDomain = post.Team.ID
	}
	body, _, err := client.Patch(subDomain, fmt.Sprintf("/items/%s", post.ID), post)
	if err != nil {
		return
	}
	err = json.Unmarshal(body, post)
	if err != nil {
		return
	}
	return
}

func (post *Post) Delete(client api.Client) (err error) {
	if post.ID == "" {
		err = EmptyIDError{}
		return
	}

	subDomain := ""
	if post.Team != nil {
		subDomain = post.Team.ID
	}
	body, _, err := client.Delete(subDomain, fmt.Sprintf("/items/%s", post.ID), post)
	if err != nil {
		return
	}
	err = json.Unmarshal(body, post)
	if err != nil {
		return
	}
	return
}

func (post Post) Save() (err error) {
	var path string
	if post.ID != "" {
		path, err = post.findPath()
		if err != nil {
			return err
		}
	}
	if path == "" {
		path = post.createPath()
	}

	dir := filepath.Dir(path)
	err = os.MkdirAll(dir, 0755)
	if err != nil {
		return
	}

	fmt.Printf("Make file: %s\n", path)
	f, err := os.Create(path)
	defer f.Close()
	if err != nil {
		return
	}
	err = post.Encode(f)
	if err != nil {
		return
	}
	return
}

func (post Post) findPath() (path string, err error) {
	dir, err := os.Getwd()
	if err != nil {
		return
	}

	found := errors.New("found")
	err = filepath.Walk(dir, func(p string, info os.FileInfo, e error) (err error) {
		if e != nil {
			err = e
			return
		}
		if info.IsDir() {
			return
		}
		if filepath.Ext(p) != ".md" {
			return
		}

		postInLocal, err := NewPostWithFile(p)
		if err != nil {
			err = nil
			return
		}
		if postInLocal.ID == post.ID {
			path = p
			return found
		}
		return
	})
	if err == found {
		err = nil
	}
	return
}

func (post Post) createPath() (path string) {
	var dirname string
	if post.Team == nil {
		dirname = DirMine
	} else {
		dirname = post.Team.ID
	}
	dirname = filepath.Join(dirname, post.CreatedAt.Format("2006/01"))

	filename := fmt.Sprintf("%s-%s", post.CreatedAt.Format("02"), post.Title)
	filename = rInvalidFilename.ReplaceAllString(filename, "-")
	filename = strings.ToLower(filename)
	filename = rHyphens.ReplaceAllString(filename, "-")
	filename = strings.TrimRight(filename, "-")

	for {
		path = filepath.Join(dirname, fmt.Sprintf("%s.md", filename))
		_, err := os.Stat(path)
		// no error means: a file exists at the path
		// error occurs means: no file exists at the path
		if err != nil { //TODO test me
			break
		}
		filename += "-"
	}
	return
}

func (post Post) Encode(w io.Writer) (err error) {
	err = tmpl.Execute(w, post)
	return
}

func (post *Post) Decode(b []byte) (err error) {
	matched := rPostDecoder.FindSubmatch(b)
	if len(matched) != 4 {
		err = fmt.Errorf("wrong format")
		return
	}

	err = yaml.Unmarshal((bytes.TrimSpace(matched[1])), &post.Meta)
	if err != nil {
		return
	}
	post.Title = string(bytes.TrimSpace(matched[2]))
	post.Body = string(bytes.TrimSpace(matched[3]))
	return
}

type EmptyIDError struct{}

func (err EmptyIDError) Error() (msg string) {
	msg = "empty ID"
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"golang.org/x/sys/unix"
)

var (
	fstab   = flag.String("fstab", "/etc/fstab", "Specify path to fstab(5).")
	fstype  = flag.String("type", "", "Specify filesystem type.")
	options = flag.String("options", "", "Specify mount options.")
)

type fsEntry struct {
	devpath string
	mntpt   string
	fstype  string
	options []string
}

func main() {
	log.SetPrefix("mount: ")
	log.SetFlags(0)

	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: mount [options] device [mountpoint]")
		fmt.Fprintln(os.Stderr, "       mount")
		flag.PrintDefaults()
	}
	flag.Parse()

	if flag.NArg() == 0 {
		if err := printMounts(); err != nil {
			log.Fatal(err)
		}
		os.Exit(0)
	}

	if flag.NArg() > 2 {
		flag.Usage()
		os.Exit(1)
	}

	entry, fstabErr := readEntry(flag.Arg(0))
	if fstabErr != nil && !os.IsNotExist(fstabErr) {
		log.Fatal(fstabErr)
	}

	if entry.mntpt == "" && flag.Arg(1) == "" {
		log.Print(fstabErr)
		log.Fatal("mountpoint not specified")
	} else if flag.Arg(1) != "" {
		entry.mntpt = flag.Arg(1)
	}

	if *fstype != "" {
		entry.fstype = *fstype
	}
	if entry.fstype == "" {
		log.Print(fstabErr)
		log.Fatal("filesystem type not specified")
	}
	if *options != "" {
		entry.options = strings.Split(*options, ",")
	}

	var flags uintptr
	for _, s := range entry.options {
		switch s {
		case "ro":
			flags |= unix.MS_RDONLY
		case "rw":
			// rw is default
		case "remount":
			flags |= unix.MS_REMOUNT
		default:
			log.Printf("unrecognized option %q", s)
		}
	}
	if err := unix.Mount(entry.devpath, entry.mntpt, entry.fstype, flags, ""); err != nil {
		log.Fatal(err)
	}
}

func printMounts() error {
	f, err := os.Open("/proc/mounts")
	if err != nil {
		return err
	}
	defer f.Close()
	if _, err := io.Copy(os.Stdout, f); err != nil {
		return err
	}
	return nil
}

func readEntry(devpath string) (*fsEntry, error) {
	// default is rw if no entry
	e := fsEntry{
		devpath: devpath,
		options: []string{"rw"},
	}

	f, err := os.Open(*fstab)
	if err != nil {
		return &e, err
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "#") {
			continue
		}
		fields := strings.Fields(line)
		if len(fields) < 4 {
			continue
		}
		if fields[0] != devpath {
			continue
		}
		e.mntpt = fields[1]
		e.fstype = fields[2]
		e.options = strings.Split(fields[3], ",")
		break
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return &e, nil
}
<commit_msg>mount: don't try to print fstab i/o error message<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"golang.org/x/sys/unix"
)

var (
	fstab   = flag.String("fstab", "/etc/fstab", "Specify path to fstab(5).")
	fstype  = flag.String("type", "", "Specify filesystem type.")
	options = flag.String("options", "", "Specify mount options.")
)

type fsEntry struct {
	devpath string
	mntpt   string
	fstype  string
	options []string
}

func main() {
	log.SetPrefix("mount: ")
	log.SetFlags(0)

	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: mount [options] device [mountpoint]")
		fmt.Fprintln(os.Stderr, "       mount")
		flag.PrintDefaults()
	}
	flag.Parse()

	if flag.NArg() == 0 {
		if err := printMounts(); err != nil {
			log.Fatal(err)
		}
		os.Exit(0)
	}

	if flag.NArg() > 2 {
		flag.Usage()
		os.Exit(1)
	}

	entry, err := readEntry(flag.Arg(0))
	if err != nil && !os.IsNotExist(err) {
		log.Fatal(err)
	}

	if entry.mntpt == "" && flag.Arg(1) == "" {
		log.Fatal("mountpoint not specified")
	} else if flag.Arg(1) != "" {
		entry.mntpt = flag.Arg(1)
	}

	if *fstype != "" {
		entry.fstype = *fstype
	}
	if entry.fstype == "" {
		log.Fatal("filesystem type not specified")
	}
	if *options != "" {
		entry.options = strings.Split(*options, ",")
	}

	var flags uintptr
	for _, s := range entry.options {
		switch s {
		case "ro":
			flags |= unix.MS_RDONLY
		case "rw":
			// rw is default
		case "remount":
			flags |= unix.MS_REMOUNT
		default:
			log.Printf("unrecognized option %q", s)
		}
	}
	if err := unix.Mount(entry.devpath, entry.mntpt, entry.fstype, flags, ""); err != nil {
		log.Fatal(err)
	}
}

func printMounts() error {
	f, err := os.Open("/proc/mounts")
	if err != nil {
		return err
	}
	defer f.Close()
	if _, err := io.Copy(os.Stdout, f); err != nil {
		return err
	}
	return nil
}

func readEntry(devpath string) (*fsEntry, error) {
	// default is rw if no entry
	e := fsEntry{
		devpath: devpath,
		options: []string{"rw"},
	}

	f, err := os.Open(*fstab)
	if err != nil {
		return &e, err
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "#") {
			continue
		}
		fields := strings.Fields(line)
		if len(fields) < 4 {
			continue
		}
		if fields[0] != devpath {
			continue
		}
		e.mntpt = fields[1]
		e.fstype = fields[2]
		e.options = strings.Split(fields[3], ",")
		break
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return &e, nil
}
<|endoftext|>