<commit_before>package renter

// Downloads can be written directly to a file, can be written to an http
// stream, or can be written to an in-memory buffer. The core download loop only
// has the concept of writing using WriteAt, so to support writing to a stream
// or to an in-memory buffer, we need to wrap the function with something that
// will transform the WriteAt call into an in-order stream or otherwise write it
// to the right place.

import (
	"errors"
	"io"
	"sync"
)

// downloadDestination is a wrapper for the different types of writing that we
// can do when reovering and writing the logical data of a file. The wrapper
// needs to convert the various write-at calls into writes that make sense to
// the underlying file, buffer, or stream.
//
// For example, if the underlying object is a file, the WriteAt call is just a
// passthrough function. But if the underlying object is a stream, WriteAt may
// block while it waits for previous data to be written.
type downloadDestination interface {
	Close() error
	WriteAt(data []byte, offset int64) (int, error)
}

// downloadDestinationBuffer writes logical chunk data to an in-memory buffer.
// This buffer is primarily used when performing repairs on uploads.
type downloadDestinationBuffer []byte

// Close implements Close for the downloadDestination interface.
func (dw downloadDestinationBuffer) Close() error {
	return nil
}

// WriteAt writes the provided data to the downloadDestinationBuffer.
func (dw downloadDestinationBuffer) WriteAt(data []byte, offset int64) (int, error) {
	if len(data)+int(offset) > len(dw) || offset < 0 {
		return 0, errors.New("write at specified offset exceeds buffer size")
	}
	i := copy(dw[offset:], data)
	return i, nil
}

// downloadDestinationWriter is a downloadDestination that writes to an
// underlying data stream. The data stream is expecting sequential data while
// the download chunks will be written in an aribtrary order using calls to
// WriteAt. We need to block the calls to WriteAt until all prior data has been
// written.
//
// TODO: There is no timeout protection here. If there's some misalignment of
// data, we'll never know it'll just hang forever.
type downloadDestinationWriter struct {
	failed         bool
	mu             sync.Mutex // Protects the underlying data structures.
	progress       int64      // How much data has been written yet.
	io.WriteCloser            // The underlying writer.

	// A list of write calls and their corresponding locks. When one write call
	// completes, it'll search through the list of write calls for the next one.
	// The next write call can be unblocked by unlocking the corresponding mutex
	// in the next array.
	blockingWriteCalls   []int64 // A list of write calls that are waiting for their turn
	blockingWriteSignals []*sync.Mutex
}

// errFailedStreamWrite gets returned if a prior error occurred when writing to
// the stream.
var errFailedStreamWrite = errors.New("downloadDestinationWriter has a broken stream due to a prior failed write")

// newDownloadDestinationWriter takes a writer and converts it into a
func newDownloadDestinationWriter(w io.WriteCloser) downloadDestination {
	return &downloadDestinationWriter{WriteCloser: w}
}

// nextWrite will iterate over all of the blocking write calls and unblock the
// one that is next in line, if the next-in-line call is available.
func (ddw *downloadDestinationWriter) nextWrite() {
	for i, offset := range ddw.blockingWriteCalls {
		if offset == ddw.progress {
			ddw.blockingWriteSignals[i].Unlock()
			ddw.blockingWriteCalls = append(ddw.blockingWriteCalls[0:i], ddw.blockingWriteCalls[i+1:]...)
			ddw.blockingWriteSignals = append(ddw.blockingWriteSignals[0:i], ddw.blockingWriteSignals[i+1:]...)
			return
		}
		if offset < ddw.progress {
			// Sanity check - there should not be a call to WriteAt that occurs
			// earlier than the current progress. If there is, the
			// downloadDestinationWriter is being used incorrectly in an
			// unrecoverable way.
			panic("incorrect write order for downloadDestinationWriter")
		}
	}
}

// WriteAt will block until the stream has progressed to or past 'offset', and
// then it will write its own data.
func (ddw *downloadDestinationWriter) WriteAt(data []byte, offset int64) (int, error) {
	write := func() (int, error) {
		// If the stream writer has already failed, return an error.
		if ddw.failed {
			return 0, errFailedStreamWrite
		}

		// Write the data to the stream.
		n, err := ddw.Write(data)
		if err != nil {
			// If there is an error, marked the stream write as failed and then
			// unlock/unblock all of the waiting WriteAt calls.
			ddw.failed = true
			ddw.Close()
			for i := range ddw.blockingWriteSignals {
				ddw.blockingWriteSignals[i].Unlock()
			}
			return n, err
		}

		// Update the progress and unblock the next write.
		ddw.progress += int64(n)
		ddw.nextWrite()
		return n, nil
	}

	ddw.mu.Lock()
	// Check if the stream has already failed. If so, return immediately with
	// the failed stream error.
	if ddw.failed {
		ddw.mu.Unlock()
		return 0, errFailedStreamWrite
	}

	// Check if we are writing to the correct offset for the stream. If so, call
	// write() and return.
	if offset == ddw.progress {
		// This write is the next write in line.
		n, err := write()
		ddw.mu.Unlock()
		return n, err
	}

	// Block until we are the correct offset for the stream. The blocking is
	// coordinated by a new mutex which gets added to an array. When the earlier
	// data is written, the mutex will be unlocked, allowing us to write.
	myMu := new(sync.Mutex)
	myMu.Lock()
	ddw.blockingWriteCalls = append(ddw.blockingWriteCalls, offset)
	ddw.blockingWriteSignals = append(ddw.blockingWriteSignals, myMu)
	ddw.mu.Unlock()
	myMu.Lock()
	ddw.mu.Lock()
	n, err := write()
	ddw.mu.Unlock()
	return n, err
}

// httpWriteCloser wraps an hhtpWriter with a closer function so that it can be
// passed to the newDownloadDestinationWriter function.
type httpWriteCloser struct {
	io.Writer
}

// Close is a blank function that allows an httpWriter to become an
// io.WriteCloser.
func (httpWriteCloser) Close() error { return nil }

func newDownloadDestinationHTTPWriter(w io.Writer) downloadDestination {
	var hwc httpWriteCloser
	hwc.Writer = w
	return newDownloadDestinationWriter(hwc)
}
<commit_msg>clean up downloadDestinationWriter concurrency<commit_after>package renter

// Downloads can be written directly to a file, can be written to an http
// stream, or can be written to an in-memory buffer. The core download loop only
// has the concept of writing using WriteAt, so to support writing to a stream
// or to an in-memory buffer, we need to wrap the function with something that
// will transform the WriteAt call into an in-order stream or otherwise write it
// to the right place.

import (
	"errors"
	"io"
	"sync"
)

// downloadDestination is a wrapper for the different types of writing that we
// can do when reovering and writing the logical data of a file. The wrapper
// needs to convert the various write-at calls into writes that make sense to
// the underlying file, buffer, or stream.
//
// For example, if the underlying object is a file, the WriteAt call is just a
// passthrough function. But if the underlying object is a stream, WriteAt may
// block while it waits for previous data to be written.
type downloadDestination interface {
	Close() error
	WriteAt(data []byte, offset int64) (int, error)
}

// downloadDestinationBuffer writes logical chunk data to an in-memory buffer.
// This buffer is primarily used when performing repairs on uploads.
type downloadDestinationBuffer []byte

// Close implements Close for the downloadDestination interface.
func (dw downloadDestinationBuffer) Close() error {
	return nil
}

// WriteAt writes the provided data to the downloadDestinationBuffer.
func (dw downloadDestinationBuffer) WriteAt(data []byte, offset int64) (int, error) {
	if len(data)+int(offset) > len(dw) || offset < 0 {
		return 0, errors.New("write at specified offset exceeds buffer size")
	}
	i := copy(dw[offset:], data)
	return i, nil
}

// downloadDestinationWriter is a downloadDestination that writes to an
// underlying data stream. The data stream is expecting sequential data while
// the download chunks will be written in an aribtrary order using calls to
// WriteAt. We need to block the calls to WriteAt until all prior data has been
// written.
//
// NOTE: If the caller accedentally leaves a gap between calls to WriteAt, for
// example writes bytes 0-100 and then writes bytes 110-200, and accidentally
// never writes bytes 100-110, the downloadDestinationWriter will block forever
// waiting for those gap bytes to be written.
//
// NOTE: Calling WriteAt has linear time performance in the number of concurrent
// calls to WriteAt.
type downloadDestinationWriter struct {
	closed         bool
	mu             sync.Mutex // Protects the underlying data structures.
	progress       int64      // How much data has been written yet.
	io.WriteCloser            // The underlying writer.

	// A list of write calls and their corresponding locks. When one write call
	// completes, it'll search through the list of write calls for the next one.
	// The next write call can be unblocked by unlocking the corresponding mutex
	// in the next array.
	blockingWriteCalls   []int64 // A list of write calls that are waiting for their turn
	blockingWriteSignals []*sync.Mutex
}

var (
	// errClosedStream gets returned if the stream was closed but we are trying
	// to write.
	errClosedStream = errors.New("unable to write because stream has been closed")

	// errOffsetAlreadyWritten gets returned if a call to WriteAt tries to write
	// to a place in the stream which has already had data written to it.
	errOffsetAlreadyWritten = errors.New("cannot write to that offset in stream, data already written")
)

// newDownloadDestinationWriter takes a writer and converts it into a
func newDownloadDestinationWriter(w io.WriteCloser) downloadDestination {
	return &downloadDestinationWriter{WriteCloser: w}
}

// unblockNextWrites will iterate over all of the blocking write calls and
// unblock any whose offsets have been reached by the current progress of the
// stream.
//
// NOTE: unblockNextWrites has linear time performance in the number of currently
// blocking calls.
func (ddw *downloadDestinationWriter) unblockNextWrites() {
	for i, offset := range ddw.blockingWriteCalls {
		if offset <= ddw.progress {
			ddw.blockingWriteSignals[i].Unlock()
			ddw.blockingWriteCalls = append(ddw.blockingWriteCalls[0:i], ddw.blockingWriteCalls[i+1:]...)
			ddw.blockingWriteSignals = append(ddw.blockingWriteSignals[0:i], ddw.blockingWriteSignals[i+1:]...)
		}
	}
}

// Close will unblock any hanging calls to WriteAt, and then call Close on the
// underlying WriteCloser.
func (ddw *downloadDestinationWriter) Close() error {
	ddw.mu.Lock()
	if ddw.closed {
		ddw.mu.Unlock()
		return errClosedStream
	}
	ddw.closed = true
	for i := range ddw.blockingWriteSignals {
		ddw.blockingWriteSignals[i].Unlock()
	}
	ddw.mu.Unlock()
	return ddw.WriteCloser.Close()
}

// WriteAt will block until the stream has progressed to 'offset', and then it
// will write its own data. An error will be returned if the stream has already
// progressed beyond 'offset'.
func (ddw *downloadDestinationWriter) WriteAt(data []byte, offset int64) (int, error) {
	write := func() (int, error) {
		// Error if the stream has been closed.
		if ddw.closed {
			return 0, errClosedStream
		}
		// Error if the stream has progressed beyond 'offset'.
		if offset < ddw.progress {
			ddw.mu.Unlock()
			return 0, errOffsetAlreadyWritten
		}

		// Write the data to the stream, and the update the progress and unblock
		// the next write.
		n, err := ddw.Write(data)
		ddw.progress += int64(n)
		ddw.unblockNextWrites()
		return n, err
	}

	ddw.mu.Lock()
	// Attempt to write if the stream progress is at or beyond the offset. The
	// write call will perform error handling.
	if offset <= ddw.progress {
		n, err := write()
		ddw.mu.Unlock()
		return n, err
	}

	// The stream has not yet progressed to 'offset'. We will block until the
	// stream has made progress. We perform the block by creating a
	// thread-specific mutex 'myMu' and adding it to the object's list of
	// blocking threads. When other threads successfully call WriteAt, they will
	// reference this list and unblock any which have enough progress. The
	// result is a somewhat strange construction where we lock myMu twice in a
	// row, but between those two calls to lock, we put myMu in a place where
	// another thread can unlock myMu.
	//
	// myMu will be unblocked when another thread calls 'unblockNextWrites'.
	myMu := new(sync.Mutex)
	myMu.Lock()
	ddw.blockingWriteCalls = append(ddw.blockingWriteCalls, offset)
	ddw.blockingWriteSignals = append(ddw.blockingWriteSignals, myMu)
	ddw.mu.Unlock()
	myMu.Lock()
	ddw.mu.Lock()
	n, err := write()
	ddw.mu.Unlock()
	return n, err
}

// httpWriteCloser wraps an hhtpWriter with a closer function so that it can be
// passed to the newDownloadDestinationWriter function.
type httpWriteCloser struct {
	io.Writer
}

// Close is a blank function that allows an httpWriter to become an
// io.WriteCloser.
func (httpWriteCloser) Close() error { return nil }

// newDownloadDestinationHTTPWriter wraps an io.Writer (typically an HTTPWriter)
// with a do-nothing Close function so that it satisfies the WriteCloser
// interface.
//
// TODO: Reconsider the name of this funciton.
func newDownloadDestinationHTTPWriter(w io.Writer) downloadDestination {
	var hwc httpWriteCloser
	hwc.Writer = w
	return newDownloadDestinationWriter(hwc)
}
<|endoftext|>


<commit_before>package handler

import (
	"context"
	"sort"

	"github.com/micro/go-micro/errors"
	"github.com/micro/go-micro/network"
	pbNet "github.com/micro/go-micro/network/proto"
	pbRtr "github.com/micro/go-micro/router/proto"
)

// Network implements network handler
type Network struct {
	Network network.Network
}

// ListRoutes returns a list of routing table routes
func (n *Network) ListRoutes(ctx context.Context, req *pbRtr.Request, resp *pbRtr.ListResponse) error {
	routes, err := n.Network.Options().Router.Table().List()
	if err != nil {
		return errors.InternalServerError("go.micro.network", "failed to list routes: %s", err)
	}

	var respRoutes []*pbRtr.Route
	for _, route := range routes {
		respRoute := &pbRtr.Route{
			Service: route.Service,
			Address: route.Address,
			Gateway: route.Gateway,
			Network: route.Network,
			Router:  route.Router,
			Link:    route.Link,
			Metric:  int64(route.Metric),
		}
		respRoutes = append(respRoutes, respRoute)
	}

	resp.Routes = respRoutes

	return nil
}

// ListNodes returns a list of all accessible nodes in the network
func (n *Network) ListNodes(ctx context.Context, req *pbNet.ListRequest, resp *pbNet.ListResponse) error {
	nodes := n.Network.Nodes()

	var respNodes []*pbNet.Node
	for _, node := range nodes {
		respNode := &pbNet.Node{
			Id:      node.Id(),
			Address: node.Address(),
		}
		respNodes = append(respNodes, respNode)
	}

	resp.Nodes = respNodes

	return nil
}

// Neighbourhood returns a list of immediate neighbours
func (n *Network) Neighbourhood(ctx context.Context, req *pbNet.NeighbourhoodRequest, resp *pbNet.NeighbourhoodResponse) error {
	// extract the id of the node to query
	id := req.Id
	// if no id is passed, we assume local node
	if id == "" {
		id = n.Network.Id()
	}

	// get all the nodes in the network
	nodes := n.Network.Nodes()

	var neighbours []*pbNet.Neighbour
	// find a node with a given id
	i := sort.Search(len(nodes), func(i int) bool { return nodes[i].Id() == id })
	// collect all the nodes in the neighbourhood of the found node
	if i < len(nodes) && nodes[i].Id() == id {
		for _, neighbour := range nodes[i].Neighbourhood() {
			var nodeNeighbours []*pbNet.Node
			for _, nodeNeighbour := range neighbour.Neighbourhood() {
				nn := &pbNet.Node{
					Id:      nodeNeighbour.Id(),
					Address: nodeNeighbour.Address(),
				}
				nodeNeighbours = append(nodeNeighbours, nn)
			}
			// node is present at node[i]
			neighbour := &pbNet.Neighbour{
				Node: &pbNet.Node{
					Id:      neighbour.Id(),
					Address: neighbour.Address(),
				},
				Neighbours: nodeNeighbours,
			}
			neighbours = append(neighbours, neighbour)
		}
	}

	resp.Neighbours = neighbours

	return nil
}
<commit_msg>Sort the returned slice of nodes before searching<commit_after>package handler

import (
	"context"
	"sort"

	"github.com/micro/go-micro/errors"
	"github.com/micro/go-micro/network"
	pbNet "github.com/micro/go-micro/network/proto"
	pbRtr "github.com/micro/go-micro/router/proto"
)

// Network implements network handler
type Network struct {
	Network network.Network
}

// ListRoutes returns a list of routing table routes
func (n *Network) ListRoutes(ctx context.Context, req *pbRtr.Request, resp *pbRtr.ListResponse) error {
	routes, err := n.Network.Options().Router.Table().List()
	if err != nil {
		return errors.InternalServerError("go.micro.network", "failed to list routes: %s", err)
	}

	var respRoutes []*pbRtr.Route
	for _, route := range routes {
		respRoute := &pbRtr.Route{
			Service: route.Service,
			Address: route.Address,
			Gateway: route.Gateway,
			Network: route.Network,
			Router:  route.Router,
			Link:    route.Link,
			Metric:  int64(route.Metric),
		}
		respRoutes = append(respRoutes, respRoute)
	}

	resp.Routes = respRoutes

	return nil
}

// ListNodes returns a list of all accessible nodes in the network
func (n *Network) ListNodes(ctx context.Context, req *pbNet.ListRequest, resp *pbNet.ListResponse) error {
	nodes := n.Network.Nodes()

	var respNodes []*pbNet.Node
	for _, node := range nodes {
		respNode := &pbNet.Node{
			Id:      node.Id(),
			Address: node.Address(),
		}
		respNodes = append(respNodes, respNode)
	}

	resp.Nodes = respNodes

	return nil
}

// Neighbourhood returns a list of immediate neighbours
func (n *Network) Neighbourhood(ctx context.Context, req *pbNet.NeighbourhoodRequest, resp *pbNet.NeighbourhoodResponse) error {
	// extract the id of the node to query
	id := req.Id
	// if no id is passed, we assume local node
	if id == "" {
		id = n.Network.Id()
	}

	// get all the nodes in the network
	nodes := n.Network.Nodes()

	// sort the slice of nodes
	sort.Slice(nodes, func(i, j int) bool { return nodes[i].Id() <= nodes[j].Id() })
	// find a node with a given id
	i := sort.Search(len(nodes), func(j int) bool { return nodes[j].Id() >= id })

	var neighbours []*pbNet.Neighbour
	// collect all the nodes in the neighbourhood of the found node
	if i < len(nodes) && nodes[i].Id() == id {
		for _, neighbour := range nodes[i].Neighbourhood() {
			var nodeNeighbours []*pbNet.Node
			for _, nodeNeighbour := range neighbour.Neighbourhood() {
				// don't return yourself in response
				if nodeNeighbour.Id() == n.Network.Id() {
					continue
				}
				nn := &pbNet.Node{
					Id:      nodeNeighbour.Id(),
					Address: nodeNeighbour.Address(),
				}
				nodeNeighbours = append(nodeNeighbours, nn)
			}
			// node is present at node[i]
			neighbour := &pbNet.Neighbour{
				Node: &pbNet.Node{
					Id:      neighbour.Id(),
					Address: neighbour.Address(),
				},
				Neighbours: nodeNeighbours,
			}
			neighbours = append(neighbours, neighbour)
		}
	}

	resp.Neighbours = neighbours

	return nil
}
<|endoftext|>


<commit_before>// Package trigram is a dumb trigram index
package trigram

// T is a trigram
type T uint32

func (t T) String() string {
	b := [3]byte{byte(t >> 16), byte(t >> 8), byte(t)}
	return string(b[:])
}

// DocID is a document ID
type DocID uint32

// Index is a trigram index
type Index map[T][]DocID

// Extract returns a list of trigrams in s
func Extract(s string, trigrams []T) []T {

	for i := 0; i <= len(s)-3; i++ {
		t := T(uint32(s[i])<<16 | uint32(s[i+1])<<8 | uint32(s[i+2]))
		trigrams = appendIfUnique(trigrams, t)
	}

	return trigrams
}

func appendIfUnique(t []T, n T) []T {
	for _, v := range t {
		if v == n {
			return t
		}
	}

	return append(t, n)
}

// NewIndex returns an index for the strings in docs
func NewIndex(docs []string) Index {

	idx := make(Index)

	var trigrams []T

	for id, d := range docs {
		ts := Extract(d, trigrams)
		docid := DocID(id)
		for _, t := range ts {
			idx[t] = append(idx[t], docid)
		}
		trigrams = trigrams[:0]
	}

	return idx
}

// Add adds a new string to the search index
func (idx Index) Add(s string) {

	id := DocID(len(idx))

	ts := Extract(s, nil)
	for _, t := range ts {
		idx[t] = append(idx[t], id)
	}
}

// Query returns a list of document IDs that match the trigrams in the query s
func (idx Index) Query(s string) []DocID {
	ts := Extract(s, nil)
	return idx.QueryTrigrams(ts)
}

// QueryTrigrams returns a list of document IDs that match the trigram set ts
func (idx Index) QueryTrigrams(ts []T) []DocID {

	midx := 0
	mtri := ts[midx]

	for i, t := range ts {
		if len(idx[t]) < len(idx[mtri]) {
			midx = i
			mtri = t
		}
	}

	ts[0], ts[midx] = ts[midx], ts[0]

	return idx.Filter(idx[mtri], ts[1:]...)
}

// Filter removes documents that don't contain the specified trigrams
func (idx Index) Filter(docs []DocID, ts ...T) []DocID {
	for _, t := range ts {
		docs = intersect(docs, idx[t])
	}

	return docs
}

func intersect(a, b []DocID) []DocID {

	// TODO(dgryski): reduce allocations by reusing A

	var aidx, bidx int

	var result []DocID

scan:
	for aidx < len(a) && bidx < len(b) {
		if a[aidx] == b[bidx] {
			result = append(result, a[aidx])
			aidx++
			bidx++
			if aidx >= len(a) || bidx >= len(b) {
				break scan
			}
		}

		for a[aidx] < b[bidx] {
			aidx++
			if aidx >= len(a) {
				break scan
			}
		}

		for bidx < len(b) && a[aidx] > b[bidx] {
			bidx++
			if bidx >= len(b) {
				break scan
			}
		}
	}

	return result
}
<commit_msg>QueryTrigrams should return all known documents on empty trigram list<commit_after>// Package trigram is a dumb trigram index
package trigram

// T is a trigram
type T uint32

func (t T) String() string {
	b := [3]byte{byte(t >> 16), byte(t >> 8), byte(t)}
	return string(b[:])
}

// DocID is a document ID
type DocID uint32

// Index is a trigram index
type Index map[T][]DocID

// a special (and invalid) trigram that holds all the document IDs
const tAllDocIDs T = 0xFFFFFFFF

// Extract returns a list of trigrams in s
func Extract(s string, trigrams []T) []T {

	for i := 0; i <= len(s)-3; i++ {
		t := T(uint32(s[i])<<16 | uint32(s[i+1])<<8 | uint32(s[i+2]))
		trigrams = appendIfUnique(trigrams, t)
	}

	return trigrams
}

func appendIfUnique(t []T, n T) []T {
	for _, v := range t {
		if v == n {
			return t
		}
	}

	return append(t, n)
}

// NewIndex returns an index for the strings in docs
func NewIndex(docs []string) Index {

	idx := make(Index)

	var allDocIDs []DocID

	var trigrams []T

	for id, d := range docs {
		ts := Extract(d, trigrams)
		docid := DocID(id)
		allDocIDs = append(allDocIDs, docid)
		for _, t := range ts {
			idx[t] = append(idx[t], docid)
		}
		trigrams = trigrams[:0]
	}

	idx[tAllDocIDs] = allDocIDs

	return idx
}

// Add adds a new string to the search index
func (idx Index) Add(s string) {

	id := DocID(len(idx))

	ts := Extract(s, nil)
	for _, t := range ts {
		idx[t] = append(idx[t], id)
	}

	idx[tAllDocIDs] = append(idx[tAllDocIDs], id)
}

// Query returns a list of document IDs that match the trigrams in the query s
func (idx Index) Query(s string) []DocID {
	ts := Extract(s, nil)
	return idx.QueryTrigrams(ts)
}

// QueryTrigrams returns a list of document IDs that match the trigram set ts
func (idx Index) QueryTrigrams(ts []T) []DocID {

	if len(ts) == 0 {
		return idx[tAllDocIDs]
	}

	midx := 0
	mtri := ts[midx]

	for i, t := range ts {
		if len(idx[t]) < len(idx[mtri]) {
			midx = i
			mtri = t
		}
	}

	ts[0], ts[midx] = ts[midx], ts[0]

	return idx.Filter(idx[mtri], ts[1:]...)
}

// Filter removes documents that don't contain the specified trigrams
func (idx Index) Filter(docs []DocID, ts ...T) []DocID {
	for _, t := range ts {
		docs = intersect(docs, idx[t])
	}

	return docs
}

func intersect(a, b []DocID) []DocID {

	// TODO(dgryski): reduce allocations by reusing A

	var aidx, bidx int

	var result []DocID

scan:
	for aidx < len(a) && bidx < len(b) {
		if a[aidx] == b[bidx] {
			result = append(result, a[aidx])
			aidx++
			bidx++
			if aidx >= len(a) || bidx >= len(b) {
				break scan
			}
		}

		for a[aidx] < b[bidx] {
			aidx++
			if aidx >= len(a) {
				break scan
			}
		}

		for bidx < len(b) && a[aidx] > b[bidx] {
			bidx++
			if bidx >= len(b) {
				break scan
			}
		}
	}

	return result
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
)

func http_request(url string) []byte {
	response, error := http.Get(url)
	if error != nil {
		fmt.Println("Could not connect to server and retrieve data, exiting.\n%s",
			error)
		os.Exit(1)
	}

	defer response.Body.Close()
	contents, error := ioutil.ReadAll(response.Body)
	if error != nil {
		fmt.Println("Failed to read HTTP response from server, exiting.\n%s",
			error)
		os.Exit(1)
	}

	return contents

}

func main() {
	url := "https://twitter.com"
        contents := http_request(url)
	fmt.Println("%s", string(contents))
}
<commit_msg>WIP: read stuffz on socket and echo back.<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
)

func http_request(url string) []byte {
	response, error := http.Get(url)
	check_error(error)

	defer response.Body.Close()
	contents, error := ioutil.ReadAll(response.Body)
	check_error(error)

	return contents
}

func main() {
	// 29418 is the default Gerrit Port, here we should send git
	// events when receving the "stream-events" command.

	//url := "https://twitter.com"
        //contents := http_request(url)
	//fmt.Println("%s", string(contents))

	service := ":9000"
	tcpAddr, error := net.ResolveTCPAddr("ipv4", service)
	check_error(error)

	listener, error := net.ListenTCP("tcp", tcpAddr)
	check_error(error)

	for {
		conn, error := listener.Accept()
		if error != nil {
			continue
		}

		go handle_connection(conn)
	}
}

func handle_connection(conn net.Conn) {
	// close connection on exit
	defer conn.Close()
	buffer := make([]byte, 1024)
	_, error := conn.Read(buffer)
	if error != nil {
		// do something good to clean up?
	} else {
		switch {
		case string(buffer) == "gerrit":
			conn.Write([]byte("gerrit command"))
		default:
			conn.Write(buffer[0:])
		}
	}
}

func check_error(error error) {
	if error != nil {
		fmt.Fprintf(os.Stderr, "Fatal error %s", error.Error())
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/dghubble/go-twitter/twitter"
	"github.com/dghubble/oauth1"
	"strings"
	"time"
)

var (
	twitterBot *TwitterBot
)

// TwitterBot ...
type TwitterBot struct {
	ID      string
	ImgPath string
	Client  *twitter.Client
	Follows map[string]string
}

// NewTwitterBot ...
func NewTwitterBot(cfg *TwitterConfig) *TwitterBot {
	config := oauth1.NewConfig(cfg.ConsumerKey, cfg.ConsumerSecret)
	token := oauth1.NewToken(cfg.AccessToken, cfg.AccessSecret)
	httpClient := config.Client(oauth1.NoContext, token)
	client := twitter.NewClient(httpClient)
	bot := &TwitterBot{
		ID:      cfg.IDSelf,
		ImgPath: cfg.ImgPath,
		Client:  client,
		Follows: map[string]string{
			"KanColle_STAFF": "294025417",
			"komatan":        "96604067",
			"maesanpicture":  "2381595966",
			"Strangestone":   "93332575",
			"kazuharukina":   "28787294",
		},
	}
	return bot
}

func hasHashTags(s string, tags []twitter.HashtagEntity) bool {
	for _, tag := range tags {
		if s == tag.Text {
			return true
		}
	}
	return false
}

func getMedias(tweet *twitter.Tweet) []twitter.MediaEntity {
	ee := tweet.ExtendedEntities
	if ee != nil {
		return ee.Media
	}
	return tweet.Entities.Media
}

func sendPics(medias []twitter.MediaEntity) {
	for _, media := range medias {
		switch media.Type {
		case "photo":
			go qqBot.SendPics(qqBot.SendGroupMsg, media.MediaURLHttps)
		}
	}
}

func logAllTrack(msg interface{}) {
	logger.Debug(msg)
}

func (t *TwitterBot) trackTweet(tweet *twitter.Tweet) {
	if tweet.RetweetedStatus != nil {
		// logger.Debugf("ignore retweet (%s):{%s}", tweet.User.Name, tweet.Text)
		return
	}
	flattenedText := strings.Replace(tweet.Text, "\n", " ", -1)
	medias := getMedias(tweet)
	switch tweet.User.IDStr {
	case t.Follows["KanColle_STAFF"]:
		msg := tweet.Text
		if tweet.Truncated {
			msg = tweet.FullText
		}
		logger.Infof("(%s):{%s}", tweet.User.Name, flattenedText)
		qqBot.SendGroupMsg(tweet.User.Name + "\n" + tweet.CreatedAt + "\n" + msg)
		sendPics(medias)

	case t.Follows["komatan"]:
		sendPics(medias)

	case t.Follows["maesanpicture"]:
		if !hasHashTags("毎日五月雨", tweet.Entities.Hashtags) || (len(medias) == 0) {
			logger.Debugf("(%s):{%s}", tweet.User.Name, flattenedText)
			return
		}
		logger.Infof("(%s):{%s}", tweet.User.Name, flattenedText)
		qqBot.SendGroupMsg(tweet.Text)
		sendPics(medias)

	case t.Follows["Strangestone"]:
		if !strings.HasPrefix(tweet.Text, "月曜日のたわわ") || (len(medias) == 0) {
			logger.Debugf("(%s):{%s}", tweet.User.Name, flattenedText)
			return
		}
		logger.Infof("(%s):{%s}", tweet.User.Name, flattenedText)
		qqBot.SendGroupMsg(tweet.Text)
		sendPics(medias)

	case t.Follows["kazuharukina"]:
		if !hasHashTags("和遥キナ毎日JK企画", tweet.Entities.Hashtags) || (len(medias) == 0) {
			logger.Debugf("(%s):{%s}", tweet.User.Name, strings.Replace(tweet.Text, "\n", " ", -1))
			return
		}
		logger.Infof("(%s):{%s}", tweet.User.Name, flattenedText)
		sendPics(medias)

	default:
		logger.Debugf("(%s):{%s}", tweet.User.Name, strings.Replace(tweet.Text, "\n", " ", -1))
	}
}

func (t *TwitterBot) selfProceedPics(medias []twitter.MediaEntity, action int) {
	for _, media := range medias {
		switch media.Type {
		case "photo":
			switch action {
			case 1:
				downloadFile(media.MediaURLHttps, t.ImgPath)
				go qqBot.SendPics(qqBot.SendSelfMsg, media.MediaURLHttps)
			case -1:
				removeFile(media.MediaURLHttps, t.ImgPath)
			}
		}
	}
}

func (t *TwitterBot) selfEvent(event *twitter.Event) {
	if event.Source.IDStr != t.ID {
		logger.Debugf(
			"favorited: (%s):{%s}",
			event.Source.Name,
			strings.Replace(event.TargetObject.Text, "\n", " ", -1))
		return
	}
	switch event.Event {
	case "favorite":
		medias := getMedias(event.TargetObject)
		logger.Infof(
			"favorite: (%s):{%s} %d medias",
			event.TargetObject.User.Name,
			strings.Replace(event.TargetObject.Text, "\n", " ", -1),
			len(medias))
		go t.selfProceedPics(medias, 1)
	case "unfavorite":
		medias := getMedias(event.TargetObject)
		logger.Debugf(
			"unfavorite: (%s):{%s} %d medias",
			event.TargetObject.User.Name,
			strings.Replace(event.TargetObject.Text, "\n", " ", -1),
			len(medias))
		go t.selfProceedPics(medias, -1)
	default:
		logger.Debug(event.Event)
	}
}

func (t *TwitterBot) selfTweet(tweet *twitter.Tweet) {
	if qqBot.Config.NameGroup != "" {
		if hasHashTags(qqBot.Config.NameGroup, tweet.Entities.Hashtags) {
			if tweet.QuotedStatus != nil {
				logger.Infof("(%s):{%s}", qqBot.Config.NameGroup, strings.Replace(tweet.QuotedStatus.Text, "\n", " ", -1))
				sendPics(getMedias(tweet.QuotedStatus))
			} else {
				logger.Infof("(%s):{%s}", qqBot.Config.NameGroup, strings.Replace(tweet.Text, "\n", " ", -1))
				sendPics(getMedias(tweet))
			}
		}
	}
}

// Track ...
func (t *TwitterBot) Track() {
	follows := []string{}
	for _, value := range t.Follows {
		follows = append(follows, value)
	}
	for i := 1; ; i++ {
		demux := twitter.NewSwitchDemux()
		demux.Tweet = t.trackTweet
		filterParams := &twitter.StreamFilterParams{
			Follow: follows,
		}
		stream, err := t.Client.Streams.Filter(filterParams)
		if err != nil {
			logger.Error(err)
			time.Sleep(time.Duration(i) * time.Second)
		}
		demux.HandleChan(stream.Messages)
	}
}

// Self ...
func (t *TwitterBot) Self() {
	for i := 1; ; i++ {
		demux := twitter.NewSwitchDemux()
		demux.Event = t.selfEvent
		demux.Tweet = t.selfTweet
		userParams := &twitter.StreamUserParams{
			With: t.ID,
		}
		stream, err := t.Client.Streams.User(userParams)
		if err != nil {
			logger.Error(err)
			time.Sleep(time.Duration(i) * time.Second)
		}
		demux.HandleChan(stream.Messages)
	}
}
<commit_msg>fix flatten text<commit_after>package main

import (
	"github.com/dghubble/go-twitter/twitter"
	"github.com/dghubble/oauth1"
	"strings"
	"time"
)

var (
	twitterBot *TwitterBot
)

// TwitterBot ...
type TwitterBot struct {
	ID      string
	ImgPath string
	Client  *twitter.Client
	Follows map[string]string
}

// NewTwitterBot ...
func NewTwitterBot(cfg *TwitterConfig) *TwitterBot {
	config := oauth1.NewConfig(cfg.ConsumerKey, cfg.ConsumerSecret)
	token := oauth1.NewToken(cfg.AccessToken, cfg.AccessSecret)
	httpClient := config.Client(oauth1.NoContext, token)
	client := twitter.NewClient(httpClient)
	bot := &TwitterBot{
		ID:      cfg.IDSelf,
		ImgPath: cfg.ImgPath,
		Client:  client,
		Follows: map[string]string{
			"KanColle_STAFF": "294025417",
			"komatan":        "96604067",
			"maesanpicture":  "2381595966",
			"Strangestone":   "93332575",
			"kazuharukina":   "28787294",
		},
	}
	return bot
}

func hasHashTags(s string, tags []twitter.HashtagEntity) bool {
	for _, tag := range tags {
		if s == tag.Text {
			return true
		}
	}
	return false
}

func getMedias(tweet *twitter.Tweet) []twitter.MediaEntity {
	ee := tweet.ExtendedEntities
	if ee != nil {
		return ee.Media
	}
	return tweet.Entities.Media
}

func sendPics(medias []twitter.MediaEntity) {
	for _, media := range medias {
		switch media.Type {
		case "photo":
			go qqBot.SendPics(qqBot.SendGroupMsg, media.MediaURLHttps)
		}
	}
}

func logAllTrack(msg interface{}) {
	logger.Debug(msg)
}

func (t *TwitterBot) trackTweet(tweet *twitter.Tweet) {
	if tweet.RetweetedStatus != nil {
		// logger.Debugf("ignore retweet (%s):{%s}", tweet.User.Name, tweet.Text)
		return
	}
	flattenedText := strings.Replace(tweet.Text, "\n", `\n`, -1)
	medias := getMedias(tweet)
	switch tweet.User.IDStr {
	case t.Follows["KanColle_STAFF"]:
		msg := tweet.Text
		if tweet.Truncated {
			msg = tweet.FullText
		}
		logger.Infof("(%s):{%s}", tweet.User.Name, flattenedText)
		qqBot.SendGroupMsg(tweet.User.Name + "\n" + tweet.CreatedAt + "\n" + msg)
		sendPics(medias)

	case t.Follows["komatan"]:
		sendPics(medias)

	case t.Follows["maesanpicture"]:
		if !hasHashTags("毎日五月雨", tweet.Entities.Hashtags) || (len(medias) == 0) {
			logger.Debugf("(%s):{%s}", tweet.User.Name, flattenedText)
			return
		}
		logger.Infof("(%s):{%s}", tweet.User.Name, flattenedText)
		qqBot.SendGroupMsg(tweet.Text)
		sendPics(medias)

	case t.Follows["Strangestone"]:
		if !strings.HasPrefix(tweet.Text, "月曜日のたわわ") || (len(medias) == 0) {
			logger.Debugf("(%s):{%s}", tweet.User.Name, flattenedText)
			return
		}
		logger.Infof("(%s):{%s}", tweet.User.Name, flattenedText)
		qqBot.SendGroupMsg(tweet.Text)
		sendPics(medias)

	case t.Follows["kazuharukina"]:
		if !hasHashTags("和遥キナ毎日JK企画", tweet.Entities.Hashtags) || (len(medias) == 0) {
			logger.Debugf("(%s):{%s}", tweet.User.Name, flattenedText)
			return
		}
		logger.Infof("(%s):{%s}", tweet.User.Name, flattenedText)
		sendPics(medias)

	default:
		logger.Debugf("(%s):{%s}", tweet.User.Name, flattenedText)
	}
}

func (t *TwitterBot) selfProceedPics(medias []twitter.MediaEntity, action int) {
	for _, media := range medias {
		switch media.Type {
		case "photo":
			switch action {
			case 1:
				downloadFile(media.MediaURLHttps, t.ImgPath)
				go qqBot.SendPics(qqBot.SendSelfMsg, media.MediaURLHttps)
			case -1:
				removeFile(media.MediaURLHttps, t.ImgPath)
			}
		}
	}
}

func (t *TwitterBot) selfEvent(event *twitter.Event) {
	flattenedText := strings.Replace(event.TargetObject.Text, "\n", `\n`, -1)
	if event.Source.IDStr != t.ID {
		logger.Debugf("favorited: (%s):{%s}", event.Source.Name, flattenedText)
		return
	}
	switch event.Event {
	case "favorite":
		medias := getMedias(event.TargetObject)
		logger.Infof("favorite: (%s):{%s} %d medias", event.TargetObject.User.Name, flattenedText, len(medias))
		go t.selfProceedPics(medias, 1)
	case "unfavorite":
		medias := getMedias(event.TargetObject)
		logger.Debugf("unfavorite: (%s):{%s} %d medias", event.TargetObject.User.Name, flattenedText, len(medias))
		go t.selfProceedPics(medias, -1)
	default:
		logger.Debug(event.Event)
	}
}

func (t *TwitterBot) selfTweet(tweet *twitter.Tweet) {
	if qqBot.Config.NameGroup != "" {
		if hasHashTags(qqBot.Config.NameGroup, tweet.Entities.Hashtags) {
			if tweet.QuotedStatus != nil {
				logger.Infof("(%s):{%s}", qqBot.Config.NameGroup, strings.Replace(tweet.QuotedStatus.Text, "\n", `\n`, -1))
				sendPics(getMedias(tweet.QuotedStatus))
			} else {
				logger.Infof("(%s):{%s}", qqBot.Config.NameGroup, strings.Replace(tweet.Text, "\n", `\n`, -1))
				sendPics(getMedias(tweet))
			}
		}
	}
}

// Track ...
func (t *TwitterBot) Track() {
	follows := []string{}
	for _, value := range t.Follows {
		follows = append(follows, value)
	}
	for i := 1; ; i++ {
		demux := twitter.NewSwitchDemux()
		demux.Tweet = t.trackTweet
		filterParams := &twitter.StreamFilterParams{
			Follow: follows,
		}
		stream, err := t.Client.Streams.Filter(filterParams)
		if err != nil {
			logger.Error(err)
			time.Sleep(time.Duration(i) * time.Second)
		}
		demux.HandleChan(stream.Messages)
	}
}

// Self ...
func (t *TwitterBot) Self() {
	for i := 1; ; i++ {
		demux := twitter.NewSwitchDemux()
		demux.Event = t.selfEvent
		demux.Tweet = t.selfTweet
		userParams := &twitter.StreamUserParams{
			With: t.ID,
		}
		stream, err := t.Client.Streams.User(userParams)
		if err != nil {
			logger.Error(err)
			time.Sleep(time.Duration(i) * time.Second)
		}
		demux.HandleChan(stream.Messages)
	}
}
<|endoftext|>


<commit_before>package shufn

import (
	"math/rand"
)

// Iter defines the common interface to thread-safe and -unsafe variants
// of the iterator.
type Iter interface {
	Next() (i uint64, ok bool)

	Mult() uint64
	Mod() uint64
	Start() uint64

	Min() uint64
	Max() uint64
}

// New creates a non-thread-safe iterator over the numeric range.
func New(mult, mod, min, max, start uint64) *iter {
	if start == 0 {
		start = rand.Uint64() % (max - min)
	}
	start = start % (max - min)
	if start == 0 {
		start = 1
	}

	return &iter{
		mult:  mult,
		mod:   mod,
		start: start,
		min:   min,
		max:   max,
	}
}

type iter struct {
	mult  uint64
	mod   uint64
	start uint64
	i     uint64
	max   uint64
	min   uint64
}

var _ Iter = (*iter)(nil)

func (i *iter) Mod() uint64   { return i.mod }
func (i *iter) Mult() uint64  { return i.mult }
func (i *iter) Start() uint64 { return i.start }
func (i *iter) Max() uint64   { return i.max }
func (i *iter) Min() uint64   { return i.min }

func (i *iter) Next() (v uint64, more bool) {
	v, more = i.next()
	for more && v > i.max {
		v, more = i.next()
	}
	return
}

func (i *iter) next() (v uint64, more bool) {
	if i.i == 0 {
		i.i = i.start
		more = true
	} else {
		i.i = (i.i * i.mult) % i.mod
		more = i.i != i.start
	}

	v = i.i + i.min - 1
	return
}
<commit_msg>Change interface<commit_after>package shufn

import (
	"math/rand"
)

// New creates a non-thread-safe iterator over the numeric range.
func New(mult, mod, min, max, start uint64) *Iter {
	if start == 0 {
		start = rand.Uint64()
	}
	start = start % (max - min)
	if start == 0 {
		start = 1
	}

	return &Iter{
		Mult:  mult,
		Mod:   mod,
		Start: start,
		Min:   min,
		Max:   max,
	}
}

type Iter struct {
	Mult  uint64
	Mod   uint64
	Start uint64
	i     uint64
	I     uint64
	Max   uint64
	Min   uint64
}

// Next returns whether there are more numbers in the sequence; and indicates
// the next is available on i.I. (Not thread safe!)
func (i *Iter) Next() (more bool) {
	i.I, more = i.NextI()
	return
}

func (i *Iter) NextI() (I uint64, more bool) {
	I, more = i.next()
	for more && I > i.Max {
		I, more = i.next()
	}
	return
}

func (i *Iter) next() (I uint64, more bool) {
	if i.i == 0 {
		i.i = i.Start
		more = true
	} else {
		i.i = (i.i * i.Mult) % i.Mod
		more = i.i != i.Start
	}

	I = i.i + i.Min - 1
	return
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package ldb_test

import (
	"github.com/conformal/btcdb"
	"github.com/conformal/btcwire"
	"os"
	"testing"
)

// we need to test for empty databas and make certain it returns proper value

func TestEmptyDB(t *testing.T) {

	dbname := "tstdbempty"
	_ = os.RemoveAll(dbname)
	db, err := btcdb.CreateDB("leveldb", dbname)
	if err != nil {
		t.Errorf("Failed to open test database %v", err)
		return
	}
	defer os.RemoveAll(dbname)

	// This is a reopen test
	db.Close()

	db, err = btcdb.OpenDB("leveldb", dbname)
	if err != nil {
		t.Errorf("Failed to open test database %v", err)
		return
	}
	defer db.Close()

	sha, height, err := db.NewestSha()
	if !sha.IsEqual(&btcwire.ShaHash{}) {
		t.Errorf("sha not nil")
	}
	if height != -1 {
		t.Errorf("height not -1 %v", height)
	}
}
<commit_msg>Update error message for zero hash change.<commit_after>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package ldb_test

import (
	"github.com/conformal/btcdb"
	"github.com/conformal/btcwire"
	"os"
	"testing"
)

// we need to test for empty databas and make certain it returns proper value

func TestEmptyDB(t *testing.T) {

	dbname := "tstdbempty"
	_ = os.RemoveAll(dbname)
	db, err := btcdb.CreateDB("leveldb", dbname)
	if err != nil {
		t.Errorf("Failed to open test database %v", err)
		return
	}
	defer os.RemoveAll(dbname)

	// This is a reopen test
	db.Close()

	db, err = btcdb.OpenDB("leveldb", dbname)
	if err != nil {
		t.Errorf("Failed to open test database %v", err)
		return
	}
	defer db.Close()

	sha, height, err := db.NewestSha()
	if !sha.IsEqual(&btcwire.ShaHash{}) {
		t.Errorf("sha not zero hash")
	}
	if height != -1 {
		t.Errorf("height not -1 %v", height)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Neugram Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"reflect"
	"strings"
	"time"

	"neugram.io/ng/eval"
	"neugram.io/ng/eval/environ"
	"neugram.io/ng/eval/shell"
	"neugram.io/ng/parser"
	"neugram.io/ng/tipe"

	"github.com/kr/pretty"
	"github.com/peterh/liner"
)

var (
	origMode liner.ModeApplier

	lineNg        *liner.State // ng-mode line reader
	historyNgFile = ""
	historyNg     = make(chan string, 1)
	historyShFile = ""
	historySh     = make(chan string, 1)

	prg *eval.Program
)

func exit(code int) {
	lineNg.Close()
	os.Exit(code)
}

func exitf(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "ng: "+format, args...)
	exit(1)
}

func mode() liner.ModeApplier {
	m, err := liner.TerminalMode()
	if err != nil {
		exitf("terminal mode: %v", err)
	}
	return m
}

func main() {
	shell.Init()

	origMode = mode()
	lineNg = liner.NewLiner()
	loop()
}

func setWindowSize(env map[interface{}]interface{}) {
	// TODO windowsize
	// TODO
	// TODO
	// TODO
	/*
		rows, cols, err := job.WindowSize(os.Stderr.Fd())
		if err != nil {
			fmt.Printf("ng: could not get window size: %v\n", err)
		} else {
			// TODO: these are meant to be shell variables, not
			// environment variables. But then, how do programs
			// like `ls` read them?
			env["LINES"] = strconv.Itoa(rows)
			env["COLUMNS"] = strconv.Itoa(cols)
		}
	*/
}

func ps1(env *environ.Environ) string {
	v := env.Get("PS1")
	if v == "" {
		return "ng$ "
	}
	if strings.IndexByte(v, '\\') == -1 {
		return v
	}
	var buf []byte
	for {
		i := strings.IndexByte(v, '\\')
		if i == -1 || i == len(v)-1 {
			break
		}
		buf = append(buf, v[:i]...)
		b := v[i+1]
		v = v[i+2:]
		switch b {
		case 'h', 'H':
			out, err := exec.Command("hostname").CombinedOutput()
			if err != nil {
				fmt.Fprintf(os.Stderr, "ng: %v\n", err)
				continue
			}
			if b == 'h' {
				if i := bytes.IndexByte(out, '.'); i >= 0 {
					out = out[:i]
				}
			}
			if len(out) > 0 && out[len(out)-1] == '\n' {
				out = out[:len(out)-1]
			}
			buf = append(buf, out...)
		case 'n':
			buf = append(buf, '\n')
		case 'w', 'W':
			cwd := env.Get("PWD")
			if home := env.Get("HOME"); home != "" {
				cwd = strings.Replace(cwd, home, "~", 1)
			}
			if b == 'W' {
				cwd = filepath.Base(cwd)
			}
			buf = append(buf, cwd...)
		}
		// TODO: '!', '#', '$', 'nnn', 's', 'j', and more.
	}
	buf = append(buf, v...)
	return string(buf)
}

func loop() {
	// TODO: support starting via a shebang: #!/bin/ng.
	// When doing so, path = os.Args[0]
	path, err := os.Getwd()
	if err != nil {
		panic(err)
	}
	path = filepath.Join(path, "ng-interactive")

	p := parser.New()
	prg = eval.New(path)
	shell.Env = prg.Environ()
	shell.Alias = prg.Alias()

	// TODO this env setup could be done in neugram code
	env := prg.Environ()
	for _, s := range os.Environ() {
		i := strings.Index(s, "=")
		env.Set(s[:i], s[i+1:])
	}
	wd, err := os.Getwd()
	if err == nil {
		env.Set("PWD", wd)
	}
	//setWindowSize(env)

	lineNg.SetTabCompletionStyle(liner.TabPrints)
	lineNg.SetWordCompleter(completer)

	if f, err := os.Open(historyShFile); err == nil {
		lineNg.SetMode("sh")
		lineNg.ReadHistory(f)
		f.Close()
	}
	go historyWriter(historyShFile, historySh)

	if f, err := os.Open(historyNgFile); err == nil {
		lineNg.SetMode("ng")
		lineNg.ReadHistory(f)
		f.Close()
	}
	go historyWriter(historyNgFile, historyNg)

	sigint := make(chan os.Signal, 1)
	state := parser.StateStmt
	if os.Args[0] == "ngsh" || os.Args[0] == "-ngsh" {
		initFile := filepath.Join(os.Getenv("HOME"), ".ngshinit")
		if f, err := os.Open(initFile); err == nil {
			scanner := bufio.NewScanner(f)
			for scanner.Scan() {
				res := p.ParseLine(scanner.Bytes())
				handleResult(res, sigint)
				state = res.State
			}
			if err := scanner.Err(); err != nil {
				exitf(".ngshinit: %v", err)
			}
			f.Close()
		}
		switch state {
		case parser.StateStmtPartial, parser.StateCmdPartial:
			exitf(".ngshinit: ends in a partial statement")
		case parser.StateStmt:
			res := p.ParseLine([]byte("$$"))
			handleResult(res, sigint)
			state = res.State
		}
	}

	signal.Notify(sigint, os.Interrupt)
	lineNg.SetCtrlCAborts(true)

	for {
		var (
			mode    string
			prompt  string
			history chan string
		)
		switch state {
		case parser.StateUnknown:
			mode, prompt, history = "ng", "??> ", historyNg
		case parser.StateStmt:
			mode, prompt, history = "ng", "ng> ", historyNg
		case parser.StateStmtPartial:
			mode, prompt, history = "ng", "..> ", historyNg
		case parser.StateCmd:
			mode, prompt, history = "sh", ps1(env), historySh
		case parser.StateCmdPartial:
			mode, prompt, history = "sh", "..$ ", historySh
		default:
			exitf("unkown parser state: %v", state)
		}
		lineNg.SetMode(mode)
		data, err := lineNg.Prompt(prompt)
		if err == liner.ErrPromptAborted {
			switch state {
			case parser.StateStmtPartial:
				fmt.Printf("TODO interrupt partial statement\n")
			case parser.StateCmdPartial:
				fmt.Printf("TODO interrupt partial command\n")
			}
		} else if err != nil {
			if err == io.EOF {
				exit(0)
			}
			exitf("error reading input: %v", err)
		}
		if data == "" {
			continue
		}
		lineNg.AppendHistory(mode, data)
		history <- data
		select { // drain sigint
		case <-sigint:
		default:
		}
		res := p.ParseLine([]byte(data))
		handleResult(res, sigint)
		state = res.State
	}
}

func handleResult(res parser.Result, sigint <-chan os.Signal) {
	for _, s := range res.Stmts {
		v, err := prg.Eval(s, sigint)
		if err != nil {
			fmt.Printf("ng: %v\n", err)
			continue
		}
		if len(v) > 1 {
			fmt.Print("(")
		}
		for i, val := range v {
			if i > 0 {
				fmt.Print(", ")
			}
			if val == (reflect.Value{}) {
				fmt.Print("<nil>")
			} else {
				pretty.Print(val.Interface())
			}
		}
		if len(v) > 1 {
			fmt.Println(")")
		} else if len(v) == 1 {
			fmt.Println("")
		}
	}
	for _, err := range res.Errs {
		fmt.Println(err.Error())
	}
	//editMode := mode()
	//origMode.ApplyMode()
	for _, cmd := range res.Cmds {
		j := &shell.Job{
			Cmd:    cmd,
			Params: prg,
			Stdin:  os.Stdin,
			Stdout: os.Stdout,
			Stderr: os.Stderr,
		}
		if err := j.Start(); err != nil {
			fmt.Println(err)
			continue
		}
		done, err := j.Wait()
		if err != nil {
			fmt.Println(err)
			continue
		}
		if !done {
			break // TODO not right, instead we should just have one cmd, not Cmds here.
		}
	}
	//editMode.ApplyMode()
}

func printValue(t tipe.Type, v interface{}) {
	// This is, effectively, a primitive type-aware printf implementation
	// that understands the neugram evaluator data layout. A far better
	// version of this would be an "ngfmt" package, that implemented the
	// printing command in neugram, using a "ngreflect" package. But it
	// will be a while until I build a reflect package, so this will have
	// to do.
	//
	// Still: avoid putting too much machinary in this. At some point soon
	// it's not worth the effort.
	/*switch t := tipe.Underlying(t).(type) {
	case *tipe.Struct:
	fmt.Print("{")
	for i, name := range t.FieldNames {
		fmt.Printf("%s: ", name)
		printValue(t.Fields[i], v.(*eval.StructVal).Fields[i].Value)
		if i < len(t.FieldNames)-1 {
			fmt.Print(", ")
		}
	}
	fmt.Print("}")
	default:
	}*/
	fmt.Print(v)
}

func init() {
	if home := os.Getenv("HOME"); home != "" {
		historyNgFile = filepath.Join(home, ".ng_history")
		historyShFile = filepath.Join(home, ".ngsh_history")
	}
}

func historyWriter(dst string, src <-chan string) {
	var batch []string
	ticker := time.Tick(250 * time.Millisecond)
	for {
		select {
		case line := <-src:
			batch = append(batch, line)
		case <-ticker:
			if len(batch) > 0 && dst != "" {
				// TODO: FcntlFlock
				f, err := os.OpenFile(dst, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0664)
				if err == nil {
					for _, line := range batch {
						fmt.Fprintf(f, "%s\n", line)
					}
					f.Close()
				}
			}
			batch = nil
		}
	}
}
<commit_msg>ng: add -e flag and usage message<commit_after>// Copyright 2016 The Neugram Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"reflect"
	"strings"
	"time"

	"neugram.io/ng/eval"
	"neugram.io/ng/eval/environ"
	"neugram.io/ng/eval/shell"
	"neugram.io/ng/parser"
	"neugram.io/ng/tipe"

	"github.com/kr/pretty"
	"github.com/peterh/liner"
)

var (
	origMode liner.ModeApplier

	lineNg        *liner.State // ng-mode line reader
	historyNgFile = ""
	historyNg     = make(chan string, 1)
	historyShFile = ""
	historySh     = make(chan string, 1)
	sigint        = make(chan os.Signal, 1)

	p   *parser.Parser
	prg *eval.Program
)

func exit(code int) {
	lineNg.Close()
	os.Exit(code)
}

func exitf(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "ng: "+format, args...)
	exit(1)
}

func mode() liner.ModeApplier {
	m, err := liner.TerminalMode()
	if err != nil {
		exitf("terminal mode: %v", err)
	}
	return m
}

const usageLine = "ng [programfile | -e cmd] [arguments]"

func usage() {
	fmt.Fprintf(os.Stderr, `ng - neugram scripting language and shell

Usage:
	%s

Options:
`, usageLine)
	flag.PrintDefaults()
}

func main() {
	shell.Init()

	help := flag.Bool("h", false, "display help message and exit")
	e := flag.String("e", "", "program passed as a string")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: %s\n", usageLine)
		os.Exit(1)
	}
	flag.Parse()

	if *help {
		usage()
		os.Exit(0)
	}
	if *e != "" {
		initProgram(filepath.Join(cwd, "ng-arg"))
		res := p.ParseLine([]byte(*e))
		handleResult(res)
		return
	}

	origMode = mode()
	lineNg = liner.NewLiner()
	loop()
}

func setWindowSize(env map[interface{}]interface{}) {
	// TODO windowsize
	// TODO
	// TODO
	// TODO
	/*
		rows, cols, err := job.WindowSize(os.Stderr.Fd())
		if err != nil {
			fmt.Printf("ng: could not get window size: %v\n", err)
		} else {
			// TODO: these are meant to be shell variables, not
			// environment variables. But then, how do programs
			// like `ls` read them?
			env["LINES"] = strconv.Itoa(rows)
			env["COLUMNS"] = strconv.Itoa(cols)
		}
	*/
}

func ps1(env *environ.Environ) string {
	v := env.Get("PS1")
	if v == "" {
		return "ng$ "
	}
	if strings.IndexByte(v, '\\') == -1 {
		return v
	}
	var buf []byte
	for {
		i := strings.IndexByte(v, '\\')
		if i == -1 || i == len(v)-1 {
			break
		}
		buf = append(buf, v[:i]...)
		b := v[i+1]
		v = v[i+2:]
		switch b {
		case 'h', 'H':
			out, err := exec.Command("hostname").CombinedOutput()
			if err != nil {
				fmt.Fprintf(os.Stderr, "ng: %v\n", err)
				continue
			}
			if b == 'h' {
				if i := bytes.IndexByte(out, '.'); i >= 0 {
					out = out[:i]
				}
			}
			if len(out) > 0 && out[len(out)-1] == '\n' {
				out = out[:len(out)-1]
			}
			buf = append(buf, out...)
		case 'n':
			buf = append(buf, '\n')
		case 'w', 'W':
			cwd := env.Get("PWD")
			if home := env.Get("HOME"); home != "" {
				cwd = strings.Replace(cwd, home, "~", 1)
			}
			if b == 'W' {
				cwd = filepath.Base(cwd)
			}
			buf = append(buf, cwd...)
		}
		// TODO: '!', '#', '$', 'nnn', 's', 'j', and more.
	}
	buf = append(buf, v...)
	return string(buf)
}

var cwd string

func init() {
	var err error
	cwd, err = os.Getwd()
	if err != nil {
		panic(err)
	}
}

func initProgram(path string) {
	p = parser.New()
	prg = eval.New(path)
	shell.Env = prg.Environ()
	shell.Alias = prg.Alias()

	// TODO this env setup could be done in neugram code
	env := prg.Environ()
	for _, s := range os.Environ() {
		i := strings.Index(s, "=")
		env.Set(s[:i], s[i+1:])
	}
	wd, err := os.Getwd()
	if err == nil {
		env.Set("PWD", wd)
	}
	//setWindowSize(env)

	signal.Notify(sigint, os.Interrupt)
}

func loop() {
	// TODO: support starting via a shebang: #!/bin/ng.
	// When doing so, path = os.Args[0]
	path := filepath.Join(cwd, "ng-interactive")
	initProgram(path)

	state := parser.StateStmt
	if os.Args[0] == "ngsh" || os.Args[0] == "-ngsh" {
		initFile := filepath.Join(os.Getenv("HOME"), ".ngshinit")
		if f, err := os.Open(initFile); err == nil {
			scanner := bufio.NewScanner(f)
			for scanner.Scan() {
				res := p.ParseLine(scanner.Bytes())
				handleResult(res)
				state = res.State
			}
			if err := scanner.Err(); err != nil {
				exitf(".ngshinit: %v", err)
			}
			f.Close()
		}
		switch state {
		case parser.StateStmtPartial, parser.StateCmdPartial:
			exitf(".ngshinit: ends in a partial statement")
		case parser.StateStmt:
			res := p.ParseLine([]byte("$$"))
			handleResult(res)
			state = res.State
		}
	}

	lineNg.SetTabCompletionStyle(liner.TabPrints)
	lineNg.SetWordCompleter(completer)
	lineNg.SetCtrlCAborts(true)

	if f, err := os.Open(historyShFile); err == nil {
		lineNg.SetMode("sh")
		lineNg.ReadHistory(f)
		f.Close()
	}
	go historyWriter(historyShFile, historySh)

	if f, err := os.Open(historyNgFile); err == nil {
		lineNg.SetMode("ng")
		lineNg.ReadHistory(f)
		f.Close()
	}
	go historyWriter(historyNgFile, historyNg)

	for {
		var (
			mode    string
			prompt  string
			history chan string
		)
		switch state {
		case parser.StateUnknown:
			mode, prompt, history = "ng", "??> ", historyNg
		case parser.StateStmt:
			mode, prompt, history = "ng", "ng> ", historyNg
		case parser.StateStmtPartial:
			mode, prompt, history = "ng", "..> ", historyNg
		case parser.StateCmd:
			mode, prompt, history = "sh", ps1(prg.Environ()), historySh
		case parser.StateCmdPartial:
			mode, prompt, history = "sh", "..$ ", historySh
		default:
			exitf("unkown parser state: %v", state)
		}
		lineNg.SetMode(mode)
		data, err := lineNg.Prompt(prompt)
		if err == liner.ErrPromptAborted {
			switch state {
			case parser.StateStmtPartial:
				fmt.Printf("TODO interrupt partial statement\n")
			case parser.StateCmdPartial:
				fmt.Printf("TODO interrupt partial command\n")
			}
		} else if err != nil {
			if err == io.EOF {
				exit(0)
			}
			exitf("error reading input: %v", err)
		}
		if data == "" {
			continue
		}
		lineNg.AppendHistory(mode, data)
		history <- data
		select { // drain sigint
		case <-sigint:
		default:
		}
		res := p.ParseLine([]byte(data))
		handleResult(res)
		state = res.State
	}
}

func handleResult(res parser.Result) {
	for _, s := range res.Stmts {
		v, err := prg.Eval(s, sigint)
		if err != nil {
			fmt.Printf("ng: %v\n", err)
			continue
		}
		if len(v) > 1 {
			fmt.Print("(")
		}
		for i, val := range v {
			if i > 0 {
				fmt.Print(", ")
			}
			if val == (reflect.Value{}) {
				fmt.Print("<nil>")
			} else {
				pretty.Print(val.Interface())
			}
		}
		if len(v) > 1 {
			fmt.Println(")")
		} else if len(v) == 1 {
			fmt.Println("")
		}
	}
	for _, err := range res.Errs {
		fmt.Println(err.Error())
	}
	//editMode := mode()
	//origMode.ApplyMode()
	for _, cmd := range res.Cmds {
		j := &shell.Job{
			Cmd:    cmd,
			Params: prg,
			Stdin:  os.Stdin,
			Stdout: os.Stdout,
			Stderr: os.Stderr,
		}
		if err := j.Start(); err != nil {
			fmt.Println(err)
			continue
		}
		done, err := j.Wait()
		if err != nil {
			fmt.Println(err)
			continue
		}
		if !done {
			break // TODO not right, instead we should just have one cmd, not Cmds here.
		}
	}
	//editMode.ApplyMode()
}

func printValue(t tipe.Type, v interface{}) {
	// This is, effectively, a primitive type-aware printf implementation
	// that understands the neugram evaluator data layout. A far better
	// version of this would be an "ngfmt" package, that implemented the
	// printing command in neugram, using a "ngreflect" package. But it
	// will be a while until I build a reflect package, so this will have
	// to do.
	//
	// Still: avoid putting too much machinary in this. At some point soon
	// it's not worth the effort.
	/*switch t := tipe.Underlying(t).(type) {
	case *tipe.Struct:
	fmt.Print("{")
	for i, name := range t.FieldNames {
		fmt.Printf("%s: ", name)
		printValue(t.Fields[i], v.(*eval.StructVal).Fields[i].Value)
		if i < len(t.FieldNames)-1 {
			fmt.Print(", ")
		}
	}
	fmt.Print("}")
	default:
	}*/
	fmt.Print(v)
}

func init() {
	if home := os.Getenv("HOME"); home != "" {
		historyNgFile = filepath.Join(home, ".ng_history")
		historyShFile = filepath.Join(home, ".ngsh_history")
	}
}

func historyWriter(dst string, src <-chan string) {
	var batch []string
	ticker := time.Tick(250 * time.Millisecond)
	for {
		select {
		case line := <-src:
			batch = append(batch, line)
		case <-ticker:
			if len(batch) > 0 && dst != "" {
				// TODO: FcntlFlock
				f, err := os.OpenFile(dst, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0664)
				if err == nil {
					for _, line := range batch {
						fmt.Fprintf(f, "%s\n", line)
					}
					f.Close()
				}
			}
			batch = nil
		}
	}
}
<|endoftext|>


<commit_before>package common

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"os/signal"
	"strings"
)

var (
	GoPath = os.Getenv("GOPATH")
)

func TrapSignal(cb func()) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	signal.Notify(c, os.Kill)
	go func() {
		for sig := range c {
			fmt.Printf("captured %v, exiting...\n", sig)
			if cb != nil {
				cb()
			}
			os.Exit(1)
		}
	}()
	select {}
}

func Exit(s string) {
	fmt.Printf(s + "\n")
	os.Exit(1)
}

func EnsureDir(dir string, mode os.FileMode) error {
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		err := os.MkdirAll(dir, mode)
		if err != nil {
			return fmt.Errorf("Could not create directory %v. %v", dir, err)
		}
	}
	return nil
}

func FileExists(filePath string) bool {
	_, err := os.Stat(filePath)
	return !os.IsNotExist(err)
}

func ReadFile(filePath string) ([]byte, error) {
	return ioutil.ReadFile(filePath)
}

func MustReadFile(filePath string) []byte {
	fileBytes, err := ioutil.ReadFile(filePath)
	if err != nil {
		Exit(Fmt("MustReadFile failed: %v", err))
		return nil
	}
	return fileBytes
}

func WriteFile(filePath string, contents []byte, mode os.FileMode) error {
	err := ioutil.WriteFile(filePath, contents, mode)
	if err != nil {
		return err
	}
	// fmt.Printf("File written to %v.\n", filePath)
	return nil
}

func MustWriteFile(filePath string, contents []byte, mode os.FileMode) {
	err := WriteFile(filePath, contents, mode)
	if err != nil {
		Exit(Fmt("MustWriteFile failed: %v", err))
	}
}

// Writes to newBytes to filePath.
// Guaranteed not to lose *both* oldBytes and newBytes,
// (assuming that the OS is perfect)
func WriteFileAtomic(filePath string, newBytes []byte, mode os.FileMode) error {
	// If a file already exists there, copy to filePath+".bak" (overwrite anything)
	if _, err := os.Stat(filePath); !os.IsNotExist(err) {
		fileBytes, err := ioutil.ReadFile(filePath)
		if err != nil {
			return fmt.Errorf("Could not read file %v. %v", filePath, err)
		}
		err = ioutil.WriteFile(filePath+".bak", fileBytes, mode)
		if err != nil {
			return fmt.Errorf("Could not write file %v. %v", filePath+".bak", err)
		}
	}
	// Write newBytes to filePath.new
	err := ioutil.WriteFile(filePath+".new", newBytes, mode)
	if err != nil {
		return fmt.Errorf("Could not write file %v. %v", filePath+".new", err)
	}
	// Move filePath.new to filePath
	err = os.Rename(filePath+".new", filePath)
	return err
}

//--------------------------------------------------------------------------------

func Tempfile(prefix string) (*os.File, string) {
	file, err := ioutil.TempFile("", prefix)
	if err != nil {
		PanicCrisis(err)
	}
	return file, file.Name()
}

//--------------------------------------------------------------------------------

func Prompt(prompt string, defaultValue string) (string, error) {
	fmt.Print(prompt)
	reader := bufio.NewReader(os.Stdin)
	line, err := reader.ReadString('\n')
	if err != nil {
		return defaultValue, err
	} else {
		line = strings.TrimSpace(line)
		if line == "" {
			return defaultValue, nil
		}
		return line, nil
	}
}
<commit_msg>Add Tempdir<commit_after>package common

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"os/signal"
	"strings"
)

var (
	GoPath = os.Getenv("GOPATH")
)

func TrapSignal(cb func()) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	signal.Notify(c, os.Kill)
	go func() {
		for sig := range c {
			fmt.Printf("captured %v, exiting...\n", sig)
			if cb != nil {
				cb()
			}
			os.Exit(1)
		}
	}()
	select {}
}

func Exit(s string) {
	fmt.Printf(s + "\n")
	os.Exit(1)
}

func EnsureDir(dir string, mode os.FileMode) error {
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		err := os.MkdirAll(dir, mode)
		if err != nil {
			return fmt.Errorf("Could not create directory %v. %v", dir, err)
		}
	}
	return nil
}

func FileExists(filePath string) bool {
	_, err := os.Stat(filePath)
	return !os.IsNotExist(err)
}

func ReadFile(filePath string) ([]byte, error) {
	return ioutil.ReadFile(filePath)
}

func MustReadFile(filePath string) []byte {
	fileBytes, err := ioutil.ReadFile(filePath)
	if err != nil {
		Exit(Fmt("MustReadFile failed: %v", err))
		return nil
	}
	return fileBytes
}

func WriteFile(filePath string, contents []byte, mode os.FileMode) error {
	err := ioutil.WriteFile(filePath, contents, mode)
	if err != nil {
		return err
	}
	// fmt.Printf("File written to %v.\n", filePath)
	return nil
}

func MustWriteFile(filePath string, contents []byte, mode os.FileMode) {
	err := WriteFile(filePath, contents, mode)
	if err != nil {
		Exit(Fmt("MustWriteFile failed: %v", err))
	}
}

// Writes to newBytes to filePath.
// Guaranteed not to lose *both* oldBytes and newBytes,
// (assuming that the OS is perfect)
func WriteFileAtomic(filePath string, newBytes []byte, mode os.FileMode) error {
	// If a file already exists there, copy to filePath+".bak" (overwrite anything)
	if _, err := os.Stat(filePath); !os.IsNotExist(err) {
		fileBytes, err := ioutil.ReadFile(filePath)
		if err != nil {
			return fmt.Errorf("Could not read file %v. %v", filePath, err)
		}
		err = ioutil.WriteFile(filePath+".bak", fileBytes, mode)
		if err != nil {
			return fmt.Errorf("Could not write file %v. %v", filePath+".bak", err)
		}
	}
	// Write newBytes to filePath.new
	err := ioutil.WriteFile(filePath+".new", newBytes, mode)
	if err != nil {
		return fmt.Errorf("Could not write file %v. %v", filePath+".new", err)
	}
	// Move filePath.new to filePath
	err = os.Rename(filePath+".new", filePath)
	return err
}

//--------------------------------------------------------------------------------

func Tempfile(prefix string) (*os.File, string) {
	file, err := ioutil.TempFile("", prefix)
	if err != nil {
		PanicCrisis(err)
	}
	return file, file.Name()
}

func Tempdir(prefix string) (*os.File, string) {
	tempDir := os.TempDir() + "/" + prefix + RandStr(12)
	err := EnsureDir(tempDir, 0700)
	if err != nil {
		panic(Fmt("Error creating temp dir: %v", err))
	}
	dir, err := os.Open(tempDir)
	if err != nil {
		panic(Fmt("Error opening temp dir: %v", err))
	}
	return dir, tempDir
}

//--------------------------------------------------------------------------------

func Prompt(prompt string, defaultValue string) (string, error) {
	fmt.Print(prompt)
	reader := bufio.NewReader(os.Stdin)
	line, err := reader.ReadString('\n')
	if err != nil {
		return defaultValue, err
	} else {
		line = strings.TrimSpace(line)
		if line == "" {
			return defaultValue, nil
		}
		return line, nil
	}
}
<|endoftext|>


<commit_before>package z

import (
	"archive/tar"
	"archive/zip"
	"bufio"
	"bytes"
	"compress/gzip"
	"crypto/md5"
	"crypto/sha1"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

// 获取本地MAC地址，只限Linux系统
func GetMac() string {
	var mac string
	var stdout, stderr bytes.Buffer
	cmd := exec.Command("/sbin/ifconfig", "-a")
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	cmd.Run()
	sOut := stdout.String()
	sErr := stderr.String()
	if len(sErr) == 0 {
		rx, _ := regexp.Compile("[0-9A-F]{2}:[0-9A-F]{2}:[0-9A-F]{2}:[0-9A-F]{2}:[0-9A-F]{2}:[0-9A-F]{2}")
		macStr := rx.FindString(strings.ToUpper(sOut))
		str := strings.ToUpper(macStr)
		mac = strings.Replace(str, ":", "", -1)
	} else {
		log.Panic(sErr)
	}
	return Trim(mac)
}

func GetIntMac(v string) string {
	var mac string
	var stdout, stderr bytes.Buffer
	cmd := exec.Command("/sbin/ifconfig", v)
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	cmd.Run()
	sOut := stdout.String()
	sErr := stderr.String()
	if len(sErr) == 0 {
		rx, _ := regexp.Compile("[0-9A-F]{2}:[0-9A-F]{2}:[0-9A-F]{2}:[0-9A-F]{2}:[0-9A-F]{2}:[0-9A-F]{2}")
		macStr := rx.FindString(strings.ToUpper(sOut))
		str := strings.ToUpper(macStr)
		mac = strings.Replace(str, ":", "", -1)
	} else {
		log.Panic(sErr)
	}
	return Trim(mac), nil
}

// 计算一个文件的 MD5 指纹, 文件路径为磁盘绝对路径
func MD5(ph string) string {
	return Finger(md5.New(), ph)
}

// 将磁盘某个文件按照某种算法计算成加密指纹
func Finger(h hash.Hash, ph string) string {
	// 打开文件
	f, err := os.Open(ph)
	if err != nil {
		return ""
	}
	defer f.Close()
	// 读取
	io.Copy(h, bufio.NewReader(f))
	// 返回计算结果
	return fmt.Sprintf("%x", h.Sum(nil))
}

// 对字符串进行SHA1哈希
func StrSHA1(data string) string {
	t := sha1.New()
	io.WriteString(t, data)
	return fmt.Sprintf("%x", t.Sum(nil))
}

// 通过唯一时间的字符串，返回唯一的SHA1哈希
func RandomSHA1() string {
	return StrSHA1(UnixNano())
}

// 生成一个 UUID 字符串（小写，去掉减号），需要系统支持 "uuidgen" 命令
// 返回的字符串格式如 "1694108edc6348b08364e604dee1bf35"
func UU() string {
	return strings.Replace(UU16(), "-", "", -1)
}

// 生成一个 UUID 字符串（小写），需要系统支持 "uuidgen" 命令
// 返回的字符串格式如 "1694108e-dc63-48b0-8364-e604dee1bf35"
func UU16() string {
	bs, err := exec.Command("uuidgen").Output()
	if nil != err {
		log.Fatal("fail to found command 'uuidgen' in $PATH")
	}
	return strings.ToLower(TrimBytes(bs))
}

// 解压Tar文件
func Untar(file, path string) error {
	// 打开文件
	f, err := os.Open(file)
	if err != nil {
		return err
	}
	defer f.Close()
	// 读取GZIP
	gr, err := gzip.NewReader(f)
	if err != nil {
		return err
	}
	defer gr.Close()
	// 读取TAR
	tr := tar.NewReader(gr)
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}
		if hdr.FileInfo().IsDir() {
			os.MkdirAll(path+string(os.PathSeparator)+hdr.Name, hdr.FileInfo().Mode())
		} else {
			fw, err := os.OpenFile(path+string(os.PathSeparator)+hdr.Name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, hdr.FileInfo().Mode())
			if err != nil {
				return err
			}
			defer fw.Close()
			_, err = io.Copy(fw, tr)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// 运行命令脚本，只限Linux系统
func LinuxCmd(sh string) error {
	var stderr bytes.Buffer
	var stdout bytes.Buffer
	cmd := exec.Command("/bin/sh", sh)
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("[%s] [%s]", sh, err)
	}
	sOut := stdout.String()
	if len(sOut) != 0 {
		log.Println(sOut)
	}
	sErr := stderr.String()
	if len(sErr) != 0 {
		return fmt.Errorf(sh, sErr)
	}
	return nil
}

// 运行系统命令，只限Linux系统
func LinuxBash(sh string) error {
	var stderr bytes.Buffer
	var stdout bytes.Buffer
	cmd := exec.Command(sh)
	cmd.Stderr = &stderr
	cmd.Stdout = &stdout
	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("[%s] [%s]", sh, err)
	}
	sOut := stdout.String()
	if len(sOut) != 0 {
		log.Println(sOut)
	}
	sErr := stderr.String()
	if len(sErr) != 0 {
		return fmt.Errorf(sErr)
	}
	return nil
}

// 创建压缩文件
func CreateZip(path, ph string) error {
	// 创建写入缓冲区
	buf := new(bytes.Buffer)
	// 创建压缩缓冲区
	w := zip.NewWriter(buf)
	// 文件列表
	files := make([]string, 0)
	// 读取文件列表
	err := filepath.Walk(path, func(aph string, f os.FileInfo, err error) error {
		// 文件不存在
		if f == nil {
			return nil
		}
		// 跳过文件夹
		if f.IsDir() {
			return nil
		}
		files = append(files, Range(aph, len(path), len(aph)))
		return nil
	})
	// 判断是否出错
	if err != nil {
		return err
	}
	// 将文件读取
	for _, file := range files {
		f, err := w.Create(file)
		if err != nil {
			return err
		}
		r, err := os.Open(path + "/" + file)
		if err != nil {
			return err
		}
		data, err := ioutil.ReadAll(r)
		if err != nil {
			return err
		}
		_, err = f.Write(data)
		if err != nil {
			return err
		}
		r.Close()
	}
	// 关闭缓冲区
	err = w.Close()
	if err != nil {
		return err
	}
	// 写入
	FileWF(ph, func(f *os.File) {
		f.Write(buf.Bytes())
	})
	// 返回
	return nil
}

// 字符串
func Range(str string, start, end int) string {
	var data string
	for i, s := range str {
		if i >= start && i < end {
			data += string(s)
		}
	}
	return data
}
<commit_msg>update os.go<commit_after>package z

import (
	"archive/tar"
	"archive/zip"
	"bufio"
	"bytes"
	"compress/gzip"
	"crypto/md5"
	"crypto/sha1"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

// 获取本地MAC地址，只限Linux系统
func GetMac() string {
	var mac string
	var stdout, stderr bytes.Buffer
	cmd := exec.Command("/sbin/ifconfig", "-a")
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	cmd.Run()
	sOut := stdout.String()
	sErr := stderr.String()
	if len(sErr) == 0 {
		rx, _ := regexp.Compile("[0-9A-F]{2}:[0-9A-F]{2}:[0-9A-F]{2}:[0-9A-F]{2}:[0-9A-F]{2}:[0-9A-F]{2}")
		macStr := rx.FindString(strings.ToUpper(sOut))
		str := strings.ToUpper(macStr)
		mac = strings.Replace(str, ":", "", -1)
	} else {
		log.Panic(sErr)
	}
	return Trim(mac)
}

func GetIntMac(v string) string {
	var mac string
	var stdout, stderr bytes.Buffer
	cmd := exec.Command("/sbin/ifconfig", v)
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	cmd.Run()
	sOut := stdout.String()
	sErr := stderr.String()
	if len(sErr) == 0 {
		rx, _ := regexp.Compile("[0-9A-F]{2}:[0-9A-F]{2}:[0-9A-F]{2}:[0-9A-F]{2}:[0-9A-F]{2}:[0-9A-F]{2}")
		macStr := rx.FindString(strings.ToUpper(sOut))
		str := strings.ToUpper(macStr)
		mac = strings.Replace(str, ":", "", -1)
	} else {
		log.Panic(sErr)
	}
	return Trim(mac)
}

// 计算一个文件的 MD5 指纹, 文件路径为磁盘绝对路径
func MD5(ph string) string {
	return Finger(md5.New(), ph)
}

// 将磁盘某个文件按照某种算法计算成加密指纹
func Finger(h hash.Hash, ph string) string {
	// 打开文件
	f, err := os.Open(ph)
	if err != nil {
		return ""
	}
	defer f.Close()
	// 读取
	io.Copy(h, bufio.NewReader(f))
	// 返回计算结果
	return fmt.Sprintf("%x", h.Sum(nil))
}

// 对字符串进行SHA1哈希
func StrSHA1(data string) string {
	t := sha1.New()
	io.WriteString(t, data)
	return fmt.Sprintf("%x", t.Sum(nil))
}

// 通过唯一时间的字符串，返回唯一的SHA1哈希
func RandomSHA1() string {
	return StrSHA1(UnixNano())
}

// 生成一个 UUID 字符串（小写，去掉减号），需要系统支持 "uuidgen" 命令
// 返回的字符串格式如 "1694108edc6348b08364e604dee1bf35"
func UU() string {
	return strings.Replace(UU16(), "-", "", -1)
}

// 生成一个 UUID 字符串（小写），需要系统支持 "uuidgen" 命令
// 返回的字符串格式如 "1694108e-dc63-48b0-8364-e604dee1bf35"
func UU16() string {
	bs, err := exec.Command("uuidgen").Output()
	if nil != err {
		log.Fatal("fail to found command 'uuidgen' in $PATH")
	}
	return strings.ToLower(TrimBytes(bs))
}

// 解压Tar文件
func Untar(file, path string) error {
	// 打开文件
	f, err := os.Open(file)
	if err != nil {
		return err
	}
	defer f.Close()
	// 读取GZIP
	gr, err := gzip.NewReader(f)
	if err != nil {
		return err
	}
	defer gr.Close()
	// 读取TAR
	tr := tar.NewReader(gr)
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}
		if hdr.FileInfo().IsDir() {
			os.MkdirAll(path+string(os.PathSeparator)+hdr.Name, hdr.FileInfo().Mode())
		} else {
			fw, err := os.OpenFile(path+string(os.PathSeparator)+hdr.Name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, hdr.FileInfo().Mode())
			if err != nil {
				return err
			}
			defer fw.Close()
			_, err = io.Copy(fw, tr)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// 运行命令脚本，只限Linux系统
func LinuxCmd(sh string) error {
	var stderr bytes.Buffer
	var stdout bytes.Buffer
	cmd := exec.Command("/bin/sh", sh)
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("[%s] [%s]", sh, err)
	}
	sOut := stdout.String()
	if len(sOut) != 0 {
		log.Println(sOut)
	}
	sErr := stderr.String()
	if len(sErr) != 0 {
		return fmt.Errorf(sh, sErr)
	}
	return nil
}

// 运行系统命令，只限Linux系统
func LinuxBash(sh string) error {
	var stderr bytes.Buffer
	var stdout bytes.Buffer
	cmd := exec.Command(sh)
	cmd.Stderr = &stderr
	cmd.Stdout = &stdout
	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("[%s] [%s]", sh, err)
	}
	sOut := stdout.String()
	if len(sOut) != 0 {
		log.Println(sOut)
	}
	sErr := stderr.String()
	if len(sErr) != 0 {
		return fmt.Errorf(sErr)
	}
	return nil
}

// 创建压缩文件
func CreateZip(path, ph string) error {
	// 创建写入缓冲区
	buf := new(bytes.Buffer)
	// 创建压缩缓冲区
	w := zip.NewWriter(buf)
	// 文件列表
	files := make([]string, 0)
	// 读取文件列表
	err := filepath.Walk(path, func(aph string, f os.FileInfo, err error) error {
		// 文件不存在
		if f == nil {
			return nil
		}
		// 跳过文件夹
		if f.IsDir() {
			return nil
		}
		files = append(files, Range(aph, len(path), len(aph)))
		return nil
	})
	// 判断是否出错
	if err != nil {
		return err
	}
	// 将文件读取
	for _, file := range files {
		f, err := w.Create(file)
		if err != nil {
			return err
		}
		r, err := os.Open(path + "/" + file)
		if err != nil {
			return err
		}
		data, err := ioutil.ReadAll(r)
		if err != nil {
			return err
		}
		_, err = f.Write(data)
		if err != nil {
			return err
		}
		r.Close()
	}
	// 关闭缓冲区
	err = w.Close()
	if err != nil {
		return err
	}
	// 写入
	FileWF(ph, func(f *os.File) {
		f.Write(buf.Bytes())
	})
	// 返回
	return nil
}

// 字符串
func Range(str string, start, end int) string {
	var data string
	for i, s := range str {
		if i >= start && i < end {
			data += string(s)
		}
	}
	return data
}
<|endoftext|>


<commit_before>package storages

import (
	"errors"
	"fmt"
	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
	"mime"
	"path"
	"time"
)

var ACLs = map[string]s3.ACL{
	"private":                   s3.Private,
	"public-read":               s3.PublicRead,
	"public-read-write":         s3.PublicReadWrite,
	"authenticated-read":        s3.AuthenticatedRead,
	"bucket-owner-read":         s3.BucketOwnerRead,
	"bucket-owner-full-control": s3.BucketOwnerFull,
}

const LastModifiedFormat = "%a, %d %b %Y %H:%M:%S %Z"

func NewS3Storage(accessKeyId string, secretAccessKey string, bucketName string, location string, region aws.Region, acl s3.ACL) (Storage, error) {
	return &S3Storage{
		AccessKeyId:     accessKeyId,
		SecretAccessKey: secretAccessKey,
		BucketName:      bucketName,
		Location:        location,
		Region:          region,
		ACL:             acl,
	}, nil
}

func (s *S3Storage) Params(params map[string]string) error {
	ACL, ok := ACLs[params["acl"]]

	if !ok {
		return errors.New(fmt.Sprintf("The ACL %s does not exist", params["acl"]))
	}

	Region, ok := aws.Regions[params["region"]]

	if !ok {
		return errors.New(fmt.Sprintf("The Region %s does not exist", params["region"]))
	}

	s.AccessKeyId = params["access_key_id"]
	s.SecretAccessKey = params["secret_access_key"]
	s.BucketName = params["bucket_name"]
	s.Location = params["location"]
	s.Region = Region
	s.ACL = ACL

	return nil
}

type S3Storage struct {
	AccessKeyId     string
	SecretAccessKey string
	BucketName      string
	Location        string
	Region          aws.Region
	ACL             s3.ACL
}

// Auth returns a Auth instance
func (s *S3Storage) Auth() (auth aws.Auth, err error) {
	return aws.GetAuth(s.AccessKeyId, s.SecretAccessKey)
}

// Client returns a S3 instance
func (s *S3Storage) Client() (*s3.S3, error) {
	auth, err := s.Auth()

	if err != nil {
		return nil, err
	}

	return s3.New(auth, s.Region), nil
}

// Bucket returns a bucket instance
func (s *S3Storage) Bucket() (*s3.Bucket, error) {
	client, err := s.Client()

	if err != nil {
		return nil, err
	}

	return client.Bucket(s.BucketName), nil
}

// Open returns the file content in a dedicated bucket
func (s *S3Storage) Open(filepath string) ([]byte, error) {
	bucket, err := s.Bucket()

	if err != nil {
		return nil, err
	}

	return bucket.Get(s.Path(filepath))
}

// Delete the file from the bucket
func (s *S3Storage) Delete(filepath string) error {
	bucket, err := s.Bucket()

	if err != nil {
		return err
	}

	return bucket.Del(s.Path(filepath))
}

func (s *S3Storage) GetKey(filepath string) (*s3.Key, error) {
	bucket, err := s.Bucket()

	if err != nil {
		return nil, err
	}

	key, err := bucket.GetKey(s.Path(filepath))

	if err != nil {
		return nil, err
	}

	return key, nil
}

// Exists checks if the given file is in the bucket
func (s *S3Storage) Exists(filepath string) bool {
	key, err := s.GetKey(filepath)

	if err != nil {
		return false
	}

	return true
}

// Save saves a file at the given path in the bucket
func (s *S3Storage) SaveWithContentType(filepath string, content []byte, contentType string) error {
	bucket, err := s.Bucket()

	if err != nil {
		return err
	}

	err = bucket.Put(s.Path(filepath), content, contentType, s.ACL)

	return err
}

func (s *S3Storage) Save(filepath string, content []byte) error {
	return s.SaveWithContentType(filepath, content, mime.TypeByExtension(filepath))
}

// Path joins the given file to the storage output directory
func (s *S3Storage) Path(filepath string) string {
	return path.Join(s.Location, filepath)
}

func (s *S3Storage) Size(filepath string) int64 {
	key, err := s.GetKey(filepath)

	if err != nil {
		return 0
	}

	return key.Size
}

func (s *S3Storage) ModifiedTime(filepath string) (time.Time, error) {
	key, err := s.GetKey(filepath)

	if err != nil {
		return time.Time{}, err
	}

	return time.Parse(LastModifiedFormat, key.LastModified)
}
<commit_msg>Remove warning on S3Storage.Exists<commit_after>package storages

import (
	"errors"
	"fmt"
	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
	"mime"
	"path"
	"time"
)

var ACLs = map[string]s3.ACL{
	"private":                   s3.Private,
	"public-read":               s3.PublicRead,
	"public-read-write":         s3.PublicReadWrite,
	"authenticated-read":        s3.AuthenticatedRead,
	"bucket-owner-read":         s3.BucketOwnerRead,
	"bucket-owner-full-control": s3.BucketOwnerFull,
}

const LastModifiedFormat = "%a, %d %b %Y %H:%M:%S %Z"

func NewS3Storage(accessKeyId string, secretAccessKey string, bucketName string, location string, region aws.Region, acl s3.ACL) (Storage, error) {
	return &S3Storage{
		AccessKeyId:     accessKeyId,
		SecretAccessKey: secretAccessKey,
		BucketName:      bucketName,
		Location:        location,
		Region:          region,
		ACL:             acl,
	}, nil
}

func (s *S3Storage) Params(params map[string]string) error {
	ACL, ok := ACLs[params["acl"]]

	if !ok {
		return errors.New(fmt.Sprintf("The ACL %s does not exist", params["acl"]))
	}

	Region, ok := aws.Regions[params["region"]]

	if !ok {
		return errors.New(fmt.Sprintf("The Region %s does not exist", params["region"]))
	}

	s.AccessKeyId = params["access_key_id"]
	s.SecretAccessKey = params["secret_access_key"]
	s.BucketName = params["bucket_name"]
	s.Location = params["location"]
	s.Region = Region
	s.ACL = ACL

	return nil
}

type S3Storage struct {
	AccessKeyId     string
	SecretAccessKey string
	BucketName      string
	Location        string
	Region          aws.Region
	ACL             s3.ACL
}

// Auth returns a Auth instance
func (s *S3Storage) Auth() (auth aws.Auth, err error) {
	return aws.GetAuth(s.AccessKeyId, s.SecretAccessKey)
}

// Client returns a S3 instance
func (s *S3Storage) Client() (*s3.S3, error) {
	auth, err := s.Auth()

	if err != nil {
		return nil, err
	}

	return s3.New(auth, s.Region), nil
}

// Bucket returns a bucket instance
func (s *S3Storage) Bucket() (*s3.Bucket, error) {
	client, err := s.Client()

	if err != nil {
		return nil, err
	}

	return client.Bucket(s.BucketName), nil
}

// Open returns the file content in a dedicated bucket
func (s *S3Storage) Open(filepath string) ([]byte, error) {
	bucket, err := s.Bucket()

	if err != nil {
		return nil, err
	}

	return bucket.Get(s.Path(filepath))
}

// Delete the file from the bucket
func (s *S3Storage) Delete(filepath string) error {
	bucket, err := s.Bucket()

	if err != nil {
		return err
	}

	return bucket.Del(s.Path(filepath))
}

func (s *S3Storage) GetKey(filepath string) (*s3.Key, error) {
	bucket, err := s.Bucket()

	if err != nil {
		return nil, err
	}

	key, err := bucket.GetKey(s.Path(filepath))

	if err != nil {
		return nil, err
	}

	return key, nil
}

// Exists checks if the given file is in the bucket
func (s *S3Storage) Exists(filepath string) bool {
	_, err := s.GetKey(filepath)

	if err != nil {
		return false
	}

	return true
}

// Save saves a file at the given path in the bucket
func (s *S3Storage) SaveWithContentType(filepath string, content []byte, contentType string) error {
	bucket, err := s.Bucket()

	if err != nil {
		return err
	}

	err = bucket.Put(s.Path(filepath), content, contentType, s.ACL)

	return err
}

func (s *S3Storage) Save(filepath string, content []byte) error {
	return s.SaveWithContentType(filepath, content, mime.TypeByExtension(filepath))
}

// Path joins the given file to the storage output directory
func (s *S3Storage) Path(filepath string) string {
	return path.Join(s.Location, filepath)
}

func (s *S3Storage) Size(filepath string) int64 {
	key, err := s.GetKey(filepath)

	if err != nil {
		return 0
	}

	return key.Size
}

func (s *S3Storage) ModifiedTime(filepath string) (time.Time, error) {
	key, err := s.GetKey(filepath)

	if err != nil {
		return time.Time{}, err
	}

	return time.Parse(LastModifiedFormat, key.LastModified)
}
<|endoftext|>