<commit_before>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tchannel_test

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"testing"
	"time"

	. "github.com/uber/tchannel-go"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"github.com/uber/tchannel-go/testutils"
	"github.com/uber/tchannel-go/testutils/goroutines"
	"golang.org/x/net/context"
)

const streamRequestError = byte(255)

func makeRepeatedBytes(n byte) []byte {
	data := make([]byte, int(n))
	for i := byte(0); i < n; i++ {
		data[i] = n
	}
	return data
}

func writeFlushBytes(w ArgWriter, bs []byte) error {
	if _, err := w.Write(bs); err != nil {
		return err
	}
	return w.Flush()
}

type streamHelper struct {
	t *testing.T
}

// startCall starts a call to echoStream and returns the arg3 reader and writer.
func (h streamHelper) startCall(ctx context.Context, ch *Channel, hostPort, serviceName string) (ArgWriter, ArgReader) {
	call, err := ch.BeginCall(ctx, hostPort, serviceName, "echoStream", nil)
	require.NoError(h.t, err, "BeginCall to echoStream failed")

	// Write empty headers
	require.NoError(h.t, NewArgWriter(call.Arg2Writer()).Write(nil), "Write empty headers failed")

	// Flush arg3 to force the call to start without any arg3.
	writer, err := call.Arg3Writer()
	require.NoError(h.t, err, "Arg3Writer failed")
	require.NoError(h.t, writer.Flush(), "Arg3Writer flush failed")

	// Read empty Headers
	response := call.Response()
	var arg2 []byte
	require.NoError(h.t, NewArgReader(response.Arg2Reader()).Read(&arg2), "Read headers failed")
	require.False(h.t, response.ApplicationError(), "echoStream failed due to application error")

	reader, err := response.Arg3Reader()
	require.NoError(h.t, err, "Arg3Reader failed")

	return writer, reader
}

// streamPartialHandler returns a streaming handler that has the following contract:
// read a byte, write N bytes where N = the byte that was read.
// The results are be written as soon as the byte is read.
func streamPartialHandler(t *testing.T, reportErrors bool) HandlerFunc {
	return func(ctx context.Context, call *InboundCall) {
		response := call.Response()
		onError := func(err error) {
			if reportErrors {
				t.Errorf("Handler error: %v", err)
			}
			response.SendSystemError(fmt.Errorf("failed to read arg2"))
		}

		var arg2 []byte
		if err := NewArgReader(call.Arg2Reader()).Read(&arg2); err != nil {
			onError(fmt.Errorf("failed to read arg2"))
			return
		}

		if err := NewArgWriter(response.Arg2Writer()).Write(nil); err != nil {
			onError(fmt.Errorf(""))
			return
		}

		argReader, err := call.Arg3Reader()
		if err != nil {
			onError(fmt.Errorf("failed to read arg3"))
			return
		}

		argWriter, err := response.Arg3Writer()
		if err != nil {
			onError(fmt.Errorf("arg3 writer failed"))
			return
		}

		// Flush arg3 which will force a frame with just arg2 to be sent.
		// The test reads arg2 before arg3 has been sent.
		if err := argWriter.Flush(); err != nil {
			onError(fmt.Errorf("arg3 flush failed"))
			return
		}

		arg3 := make([]byte, 1)
		for {
			n, err := argReader.Read(arg3)
			if err == io.EOF {
				break
			}
			if n == 0 && err == nil {
				err = fmt.Errorf("read 0 bytes")
			}
			if err != nil {
				onError(fmt.Errorf("arg3 Read failed: %v", err))
				return
			}

			// Magic number to cause a failure
			if arg3[0] == streamRequestError {
				// Make sure that the reader is closed.
				if err := argReader.Close(); err != nil {
					onError(fmt.Errorf("request error failed to close argReader: %v", err))
					return
				}

				response.SendSystemError(errors.New("intentional failure"))
				return
			}

			// Write the number of bytes as specified by arg3[0]
			if _, err := argWriter.Write(makeRepeatedBytes(arg3[0])); err != nil {
				onError(fmt.Errorf("argWriter Write failed: %v", err))
				return
			}
			if err := argWriter.Flush(); err != nil {
				onError(fmt.Errorf("argWriter flush failed: %v", err))
				return
			}
		}

		if err := argReader.Close(); err != nil {
			onError(fmt.Errorf("argReader Close failed: %v", err))
			return
		}

		if err := argWriter.Close(); err != nil {
			onError(fmt.Errorf("arg3writer Close failed: %v", err))
			return
		}
	}
}

func testStreamArg(t *testing.T, f func(argWriter ArgWriter, argReader ArgReader)) {
	defer testutils.SetTimeout(t, 2*time.Second)()
	ctx, cancel := NewContext(time.Second)
	defer cancel()

	helper := streamHelper{t}
	WithVerifiedServer(t, nil, func(ch *Channel, hostPort string) {
		ch.Register(streamPartialHandler(t, true /* report errors */), "echoStream")

		argWriter, argReader := helper.startCall(ctx, ch, hostPort, ch.ServiceName())
		verifyBytes := func(n byte) {
			require.NoError(t, writeFlushBytes(argWriter, []byte{n}), "arg3 write failed")

			arg3 := make([]byte, int(n))
			_, err := io.ReadFull(argReader, arg3)
			require.NoError(t, err, "arg3 read failed")

			assert.Equal(t, makeRepeatedBytes(n), arg3, "arg3 result mismatch")
		}

		verifyBytes(0)
		verifyBytes(5)
		verifyBytes(100)
		verifyBytes(1)

		f(argWriter, argReader)
	})
}

func TestStreamPartialArg(t *testing.T) {
	testStreamArg(t, func(argWriter ArgWriter, argReader ArgReader) {
		require.NoError(t, argWriter.Close(), "arg3 close failed")

		// Once closed, we expect the reader to return EOF
		n, err := io.Copy(ioutil.Discard, argReader)
		assert.Equal(t, int64(0), n, "arg2 reader expected to EOF after arg3 writer is closed")
		assert.NoError(t, err, "Copy should not fail")
		assert.NoError(t, argReader.Close(), "close arg reader failed")
	})
}

func TestStreamSendError(t *testing.T) {
	testStreamArg(t, func(argWriter ArgWriter, argReader ArgReader) {
		// Send the magic number to request an error.
		_, err := argWriter.Write([]byte{streamRequestError})
		require.NoError(t, err, "arg3 write failed")
		require.NoError(t, argWriter.Close(), "arg3 close failed")

		// Now we expect an error on our next read.
		_, err = ioutil.ReadAll(argReader)
		assert.Error(t, err, "ReadAll should fail")
		assert.True(t, strings.Contains(err.Error(), "intentional failure"), "err %v unexpected", err)
	})
}

func TestStreamCancelled(t *testing.T) {
	server := testutils.NewServer(t, nil)
	server.Register(streamPartialHandler(t, false /* report errors */), "echoStream")

	ctx, cancel := NewContext(testutils.Timeout(50 * time.Millisecond))
	defer cancel()

	helper := streamHelper{t}
	WithVerifiedServer(t, nil, func(ch *Channel, _ string) {
		callCtx, callCancel := context.WithCancel(ctx)
		cancelContext := make(chan struct{})

		arg3Writer, arg3Reader := helper.startCall(callCtx, ch, server.PeerInfo().HostPort, server.ServiceName())
		go func() {
			for i := 0; i < 10; i++ {
				assert.NoError(t, writeFlushBytes(arg3Writer, []byte{1}), "Write failed")
			}

			// Our reads and writes should fail now.
			<-cancelContext
			callCancel()

			_, err := arg3Writer.Write([]byte{1})
			// The write will succeed since it's buffered.
			assert.NoError(t, err, "Write after fail should be buffered")
			assert.Error(t, arg3Writer.Flush(), "writer.Flush should fail after cancel")
			assert.Error(t, arg3Writer.Close(), "writer.Close should fail after cancel")
		}()

		for i := 0; i < 10; i++ {
			arg3 := make([]byte, 1)
			n, err := arg3Reader.Read(arg3)
			assert.Equal(t, 1, n, "Read did not correct number of bytes")
			assert.NoError(t, err, "Read failed")
		}

		close(cancelContext)

		n, err := io.Copy(ioutil.Discard, arg3Reader)
		assert.EqualValues(t, 0, n, "Read should not read any bytes after cancel")
		assert.Error(t, err, "Read should fail after cancel")
		assert.Error(t, arg3Reader.Close(), "reader.Close should fail after cancel")
	})

	// TODO(prashant): Once calls are cancelled when the connection is closed, this
	// can be removed, since the calls should fail.

	<-ctx.Done()

	server.Close()
	waitForChannelClose(t, server)
	goroutines.VerifyNoLeaks(t, nil)
}
<commit_msg>Add test to ensure req stream fails after resp stream ends<commit_after>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tchannel_test

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"testing"
	"time"

	. "github.com/uber/tchannel-go"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"github.com/uber/tchannel-go/testutils"
	"github.com/uber/tchannel-go/testutils/goroutines"
	"golang.org/x/net/context"
)

const (
	streamRequestError = byte(255)
	streamRequestClose = byte(254)
)

func makeRepeatedBytes(n byte) []byte {
	data := make([]byte, int(n))
	for i := byte(0); i < n; i++ {
		data[i] = n
	}
	return data
}

func writeFlushBytes(w ArgWriter, bs []byte) error {
	if _, err := w.Write(bs); err != nil {
		return err
	}
	return w.Flush()
}

type streamHelper struct {
	t *testing.T
}

// startCall starts a call to echoStream and returns the arg3 reader and writer.
func (h streamHelper) startCall(ctx context.Context, ch *Channel, hostPort, serviceName string) (ArgWriter, ArgReader) {
	call, err := ch.BeginCall(ctx, hostPort, serviceName, "echoStream", nil)
	require.NoError(h.t, err, "BeginCall to echoStream failed")

	// Write empty headers
	require.NoError(h.t, NewArgWriter(call.Arg2Writer()).Write(nil), "Write empty headers failed")

	// Flush arg3 to force the call to start without any arg3.
	writer, err := call.Arg3Writer()
	require.NoError(h.t, err, "Arg3Writer failed")
	require.NoError(h.t, writer.Flush(), "Arg3Writer flush failed")

	// Read empty Headers
	response := call.Response()
	var arg2 []byte
	require.NoError(h.t, NewArgReader(response.Arg2Reader()).Read(&arg2), "Read headers failed")
	require.False(h.t, response.ApplicationError(), "echoStream failed due to application error")

	reader, err := response.Arg3Reader()
	require.NoError(h.t, err, "Arg3Reader failed")

	return writer, reader
}

// streamPartialHandler returns a streaming handler that has the following contract:
// read a byte, write N bytes where N = the byte that was read.
// The results are be written as soon as the byte is read.
func streamPartialHandler(t *testing.T, reportErrors bool) HandlerFunc {
	return func(ctx context.Context, call *InboundCall) {
		response := call.Response()
		onError := func(err error) {
			if reportErrors {
				t.Errorf("Handler error: %v", err)
			}
			response.SendSystemError(fmt.Errorf("failed to read arg2"))
		}

		var arg2 []byte
		if err := NewArgReader(call.Arg2Reader()).Read(&arg2); err != nil {
			onError(fmt.Errorf("failed to read arg2"))
			return
		}

		if err := NewArgWriter(response.Arg2Writer()).Write(nil); err != nil {
			onError(fmt.Errorf(""))
			return
		}

		argReader, err := call.Arg3Reader()
		if err != nil {
			onError(fmt.Errorf("failed to read arg3"))
			return
		}

		argWriter, err := response.Arg3Writer()
		if err != nil {
			onError(fmt.Errorf("arg3 writer failed"))
			return
		}

		// Flush arg3 which will force a frame with just arg2 to be sent.
		// The test reads arg2 before arg3 has been sent.
		if err := argWriter.Flush(); err != nil {
			onError(fmt.Errorf("arg3 flush failed"))
			return
		}

		arg3 := make([]byte, 1)
		for {
			n, err := argReader.Read(arg3)
			if err == io.EOF {
				break
			}
			if n == 0 && err == nil {
				err = fmt.Errorf("read 0 bytes")
			}
			if err != nil {
				onError(fmt.Errorf("arg3 Read failed: %v", err))
				return
			}

			// Magic number to cause a failure
			if arg3[0] == streamRequestError {
				// Make sure that the reader is closed.
				if err := argReader.Close(); err != nil {
					onError(fmt.Errorf("request error failed to close argReader: %v", err))
					return
				}

				response.SendSystemError(errors.New("intentional failure"))
				return
			}
			if arg3[0] == streamRequestClose {
				if err := argWriter.Close(); err != nil {
					onError(err)
				}
				return
			}

			// Write the number of bytes as specified by arg3[0]
			if _, err := argWriter.Write(makeRepeatedBytes(arg3[0])); err != nil {
				onError(fmt.Errorf("argWriter Write failed: %v", err))
				return
			}
			if err := argWriter.Flush(); err != nil {
				onError(fmt.Errorf("argWriter flush failed: %v", err))
				return
			}
		}

		if err := argReader.Close(); err != nil {
			onError(fmt.Errorf("argReader Close failed: %v", err))
			return
		}

		if err := argWriter.Close(); err != nil {
			onError(fmt.Errorf("arg3writer Close failed: %v", err))
			return
		}
	}
}

func testStreamArg(t *testing.T, f func(argWriter ArgWriter, argReader ArgReader)) {
	defer testutils.SetTimeout(t, 2*time.Second)()
	ctx, cancel := NewContext(time.Second)
	defer cancel()

	helper := streamHelper{t}
	WithVerifiedServer(t, nil, func(ch *Channel, hostPort string) {
		ch.Register(streamPartialHandler(t, true /* report errors */), "echoStream")

		argWriter, argReader := helper.startCall(ctx, ch, hostPort, ch.ServiceName())
		verifyBytes := func(n byte) {
			require.NoError(t, writeFlushBytes(argWriter, []byte{n}), "arg3 write failed")

			arg3 := make([]byte, int(n))
			_, err := io.ReadFull(argReader, arg3)
			require.NoError(t, err, "arg3 read failed")

			assert.Equal(t, makeRepeatedBytes(n), arg3, "arg3 result mismatch")
		}

		verifyBytes(0)
		verifyBytes(5)
		verifyBytes(100)
		verifyBytes(1)

		f(argWriter, argReader)
	})
}

func TestStreamPartialArg(t *testing.T) {
	testStreamArg(t, func(argWriter ArgWriter, argReader ArgReader) {
		require.NoError(t, argWriter.Close(), "arg3 close failed")

		// Once closed, we expect the reader to return EOF
		n, err := io.Copy(ioutil.Discard, argReader)
		assert.Equal(t, int64(0), n, "arg2 reader expected to EOF after arg3 writer is closed")
		assert.NoError(t, err, "Copy should not fail")
		assert.NoError(t, argReader.Close(), "close arg reader failed")
	})
}

func TestStreamSendError(t *testing.T) {
	testStreamArg(t, func(argWriter ArgWriter, argReader ArgReader) {
		// Send the magic number to request an error.
		_, err := argWriter.Write([]byte{streamRequestError})
		require.NoError(t, err, "arg3 write failed")
		require.NoError(t, argWriter.Close(), "arg3 close failed")

		// Now we expect an error on our next read.
		_, err = ioutil.ReadAll(argReader)
		assert.Error(t, err, "ReadAll should fail")
		assert.True(t, strings.Contains(err.Error(), "intentional failure"), "err %v unexpected", err)
	})
}

func TestStreamCancelled(t *testing.T) {
	server := testutils.NewServer(t, nil)
	server.Register(streamPartialHandler(t, false /* report errors */), "echoStream")

	ctx, cancel := NewContext(testutils.Timeout(50 * time.Millisecond))
	defer cancel()

	helper := streamHelper{t}
	WithVerifiedServer(t, nil, func(ch *Channel, _ string) {
		callCtx, callCancel := context.WithCancel(ctx)
		cancelContext := make(chan struct{})

		arg3Writer, arg3Reader := helper.startCall(callCtx, ch, server.PeerInfo().HostPort, server.ServiceName())
		go func() {
			for i := 0; i < 10; i++ {
				assert.NoError(t, writeFlushBytes(arg3Writer, []byte{1}), "Write failed")
			}

			// Our reads and writes should fail now.
			<-cancelContext
			callCancel()

			_, err := arg3Writer.Write([]byte{1})
			// The write will succeed since it's buffered.
			assert.NoError(t, err, "Write after fail should be buffered")
			assert.Error(t, arg3Writer.Flush(), "writer.Flush should fail after cancel")
			assert.Error(t, arg3Writer.Close(), "writer.Close should fail after cancel")
		}()

		for i := 0; i < 10; i++ {
			arg3 := make([]byte, 1)
			n, err := arg3Reader.Read(arg3)
			assert.Equal(t, 1, n, "Read did not correct number of bytes")
			assert.NoError(t, err, "Read failed")
		}

		close(cancelContext)

		n, err := io.Copy(ioutil.Discard, arg3Reader)
		assert.EqualValues(t, 0, n, "Read should not read any bytes after cancel")
		assert.Error(t, err, "Read should fail after cancel")
		assert.Error(t, arg3Reader.Close(), "reader.Close should fail after cancel")
	})

	// TODO(prashant): Once calls are cancelled when the connection is closed, this
	// can be removed, since the calls should fail.

	<-ctx.Done()

	server.Close()
	waitForChannelClose(t, server)
	goroutines.VerifyNoLeaks(t, nil)
}

func TestResponseClosedBeforeRequest(t *testing.T) {
	server := testutils.NewServer(t, nil)
	server.Register(streamPartialHandler(t, false /* report errors */), "echoStream")

	ctx, cancel := NewContext(testutils.Timeout(50 * time.Millisecond))
	defer cancel()

	helper := streamHelper{t}
	WithVerifiedServer(t, nil, func(ch *Channel, _ string) {
		responseClosed := make(chan struct{})
		writerDone := make(chan struct{})

		arg3Writer, arg3Reader := helper.startCall(ctx, ch, server.PeerInfo().HostPort, server.ServiceName())
		go func() {
			defer close(writerDone)

			for i := 0; i < 10; i++ {
				assert.NoError(t, writeFlushBytes(arg3Writer, []byte{1}), "Write failed")
			}
			assert.NoError(t, writeFlushBytes(arg3Writer, []byte{streamRequestClose}), "Write failed")

			// Wait until our reader gets the EOF.
			<-responseClosed

			// Now our writes should fail, since the stream is shutdown
			err := writeFlushBytes(arg3Writer, []byte{1})
			if assert.Error(t, err, "Req write should fail since response stream ended") {
				assert.Contains(t, err.Error(), "mex has been shutdown")
			}
		}()

		for i := 0; i < 10; i++ {
			arg3 := make([]byte, 1)
			n, err := arg3Reader.Read(arg3)
			assert.Equal(t, 1, n, "Read did not correct number of bytes")
			assert.NoError(t, err, "Read failed")
		}

		eofBuf := make([]byte, 1)
		_, err := arg3Reader.Read(eofBuf)
		assert.Equal(t, io.EOF, err, "Response should EOF after request close")
		assert.NoError(t, arg3Reader.Close(), "Close should succeed")
		close(responseClosed)
		<-writerDone
	})

	server.Close()
	waitForChannelClose(t, server)
	goroutines.VerifyNoLeaks(t, nil)
}
<|endoftext|>


<commit_before>package streams

import (
	"fmt"
	"log"
	"os"
	"time"

	"github.com/MediaMath/keryxlib/pg/wal"
)

//WalStream is an abstraction around WAL entries.
type WalStream struct {
	dataDir             string
	publish             chan<- *wal.Entry
	stop                chan interface{}
	cursor              *wal.Cursor
	lastOffsetPublished uint64
}

// NewWalStream creates a new WalStream pointed at the provided dataDir
func NewWalStream(dataDir string) (*WalStream, error) {
	s := &WalStream{dataDir, nil, make(chan interface{}), nil, 0}

	return s, nil
}

// Start begins streaming of events in a go routine and returns a channel of WAL entries
func (streamer *WalStream) Start() (<-chan *wal.Entry, error) {
	out := make(chan *wal.Entry)

	if streamer.publish == nil {
		streamer.publish = out

		err := streamer.startAtCheckpoint()
		if err != nil {
			return nil, err
		}

		tick := time.Tick(50 * time.Millisecond)

		go func() {
			for !streamer.publishUntilErrorOrStopped() {
				<-tick
			}
			close(out)
			streamer.publish = nil
		}()
	} else {
		return nil, fmt.Errorf("already publishing")
	}

	return out, nil
}

// Stop will end publishing to the channel returned by start.
func (streamer *WalStream) Stop() {
	streamer.stop <- true
}

func (streamer *WalStream) isStopped() (stopped bool) {
	select {
	case <-streamer.stop:
		stopped = true
	default:
		stopped = false
	}

	return
}

func (streamer *WalStream) startAtCheckpoint() error {
	cursor, err := wal.NewCursorAtCheckpoint(streamer.dataDir)
	if err == nil {
		streamer.cursor = cursor
	}

	return err
}

func (streamer *WalStream) publishUntilErrorOrStopped() (stopped bool) {
	stopped = false

	var ent *wal.Entry
	var previousCursor, currentCursor wal.Cursor
	var err error

	currentCursor = *streamer.cursor
	keepReading := true
	for keepReading {

		previousCursor = currentCursor

		ent, currentCursor, err = currentCursor.ReadEntry()

		if err == nil && ent != nil {
			if ent.ReadFrom.Offset() > streamer.lastOffsetPublished {
				streamer.publish <- ent
				*streamer.cursor = currentCursor

				streamer.lastOffsetPublished = ent.ReadFrom.Offset()
			}
		} else {
			keepReading = false
		}

		stopped = streamer.isStopped()
		keepReading = keepReading && !stopped

		if previousCursor.String() == currentCursor.String() {
			keepReading = false
		}
	}

	if err != nil {
		//Known oddity of keryx attempting to look for a file that hasnt been created yet
		//dont bother logging that.
		if !os.IsNotExist(err) {
			log.Printf("error while reading wal: %v", err)
		}
		streamer.startAtCheckpoint()
	}

	return
}
<commit_msg>update comment to more acurately reflect reality<commit_after>package streams

import (
	"fmt"
	"log"
	"os"
	"time"

	"github.com/MediaMath/keryxlib/pg/wal"
)

//WalStream is an abstraction around WAL entries.
type WalStream struct {
	dataDir             string
	publish             chan<- *wal.Entry
	stop                chan interface{}
	cursor              *wal.Cursor
	lastOffsetPublished uint64
}

// NewWalStream creates a new WalStream pointed at the provided dataDir
func NewWalStream(dataDir string) (*WalStream, error) {
	s := &WalStream{dataDir, nil, make(chan interface{}), nil, 0}

	return s, nil
}

// Start begins streaming of events in a go routine and returns a channel of WAL entries
func (streamer *WalStream) Start() (<-chan *wal.Entry, error) {
	out := make(chan *wal.Entry)

	if streamer.publish == nil {
		streamer.publish = out

		err := streamer.startAtCheckpoint()
		if err != nil {
			return nil, err
		}

		tick := time.Tick(50 * time.Millisecond)

		go func() {
			for !streamer.publishUntilErrorOrStopped() {
				<-tick
			}
			close(out)
			streamer.publish = nil
		}()
	} else {
		return nil, fmt.Errorf("already publishing")
	}

	return out, nil
}

// Stop will end publishing to the channel returned by start.
func (streamer *WalStream) Stop() {
	streamer.stop <- true
}

func (streamer *WalStream) isStopped() (stopped bool) {
	select {
	case <-streamer.stop:
		stopped = true
	default:
		stopped = false
	}

	return
}

func (streamer *WalStream) startAtCheckpoint() error {
	cursor, err := wal.NewCursorAtCheckpoint(streamer.dataDir)
	if err == nil {
		streamer.cursor = cursor
	}

	return err
}

func (streamer *WalStream) publishUntilErrorOrStopped() (stopped bool) {
	stopped = false

	var ent *wal.Entry
	var previousCursor, currentCursor wal.Cursor
	var err error

	currentCursor = *streamer.cursor
	keepReading := true
	for keepReading {

		previousCursor = currentCursor

		ent, currentCursor, err = currentCursor.ReadEntry()

		if err == nil && ent != nil {
			if ent.ReadFrom.Offset() > streamer.lastOffsetPublished {
				streamer.publish <- ent
				*streamer.cursor = currentCursor

				streamer.lastOffsetPublished = ent.ReadFrom.Offset()
			}
		} else {
			keepReading = false
		}

		stopped = streamer.isStopped()
		keepReading = keepReading && !stopped

		if previousCursor.String() == currentCursor.String() {
			keepReading = false
		}
	}

	if err != nil {
		//the file can not exist for 2 reasons
		//1 - can happen a lot, if keryx is staying ahead of the wal log
		//2 - hopefully not often, if keryx is falling too far behind and the wal is being removed
		//we are gambling on number 1 being the cause of this error and therefore are not logging it as it is very
		//noisy if we do
		if !os.IsNotExist(err) {
			log.Printf("error while reading wal: %v", err)
		}
		streamer.startAtCheckpoint()
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"crypto/md5"
	"fmt"
	"os"
	"regexp"
	"strings"
)

const version = "0.1.0+Go"

type ansiColorer func(color int) string

type options struct {
	foreground ansiColorer
	background ansiColorer
	pattern    string
}

type colorMode int

const (
	mode16Color colorMode = iota
	mode256color
)

func main() {
	interactLinewise(colorLineMaker(parseArgs(os.Args[1:])))
}

func parseArgs(args []string) options {
	colorForeground, colorBackground := true, false
	regexps := make([]string, 0)
	colorDepth := mode256color

	acceptingFlags := true
	for _, arg := range args {
		if !acceptingFlags {
			regexps = append(regexps, arg)
			continue
		}

		if arg == "-h" || arg == "--help" {
			printUsage()
			os.Exit(0)
		} else if arg == "-v" || arg == "--version" {
			fmt.Printf("Synesthesia version %v\n", version)
			os.Exit(0)
		} else if arg == "-f" || arg == "--foreground" {
			colorForeground = true
		} else if arg == "--no-foreground" {
			colorForeground = false
		} else if arg == "-b" || arg == "--background" {
			colorBackground = true
		} else if arg == "--no-background" {
			colorBackground = false
		} else if arg == "--16" {
			colorDepth = mode16Color
		} else if arg == "--256" {
			colorDepth = mode256color
		} else if arg == "--" {
			acceptingFlags = false
		} else if strings.HasPrefix(arg, "-") {
			fmt.Printf("Unrecognized flag: %s\n", arg)
			printUsage()
			os.Exit(1)
		} else {
			regexps = append(regexps, arg)
		}
	}

	foregroundColorer, backgroundColorer := colorNoop, colorNoop
	if colorDepth == mode16Color {
		if colorForeground {
			foregroundColorer = colorAnsi16Foreground
		}
		if colorBackground {
			backgroundColorer = colorAnsi16Background
		}
	} else if colorDepth == mode256color {
		if colorForeground {
			foregroundColorer = colorAnsi256Foreground
		}
		if colorBackground {
			backgroundColorer = colorAnsi256Background
		}
	}

	validateRegexps(regexps)
	for i, pattern := range regexps {
		regexps[i] = fmt.Sprintf("(?:%v)", pattern)
	}

	return options{foregroundColorer, backgroundColorer, strings.Join(regexps, "|")}
}

func printUsage() {
	fmt.Fprintf(os.Stderr, `Usage: %v ([flags]|[patterns])... [-- [patterns]...]
Color standard input based on the values matched by the provided regular
expressions.
Before --, any parameters starting with a dash will be interpreted as flags.
After it, all parameters are interpreted as patterns.
A pattern is any valid Go regular expression.
Multiples of flags controlling the same option are allowed; the last flag
specified will take precedence over the ones that come before it.
Flags:
-b --background     Turn on or off background colorization. (default off)
   --no-background
-f --foreground     Turn on or off foreground colorization. (default on)
   --no-foreground
-h --help           Print usage information.
`, os.Args[0])
}

func validateRegexps(regexps []string) {
	valid := true
	for _, pattern := range regexps {
		if _, err := regexp.Compile(pattern); err != nil {
			valid = false
			fmt.Fprintf(os.Stderr, "Invalid regex: %v\n", pattern)
		}
	}
	if !valid {
		os.Exit(1)
	}
}

func interactLinewise(transform func([]byte) []byte) {
	input := bufio.NewReader(os.Stdin)
	line, err := input.ReadBytes('\n')
	for err == nil {
		fmt.Print(string(transform(line)))
		line, err = input.ReadBytes('\n')
	}
	fmt.Print(string(transform(line)))
}

func colorLineMaker(options options) func([]byte) []byte {
	pattern := regexp.MustCompile(options.pattern)
	return func(line []byte) []byte {
		return colorPatternInString(pattern, line, options.foreground, options.background)
	}
}

func colorPatternInString(pattern *regexp.Regexp, buf []byte, foreground, background ansiColorer) []byte {
	return pattern.ReplaceAllFunc(buf, colorMatchMaker(foreground, background))
}

func colorMatchMaker(foreground, background ansiColorer) func([]byte) []byte {
	return func(match []byte) []byte {
		return color(match, foreground, background)
	}
}

func color(buf []byte, foreground, background ansiColorer) []byte {
	sum := md5.Sum(buf)
	color := int(sum[md5.Size-3])<<16 | int(sum[md5.Size-2])<<8 | int(sum[md5.Size-1])
	return []byte(fmt.Sprintf("%v%v%v\033[m", foreground(color), background(0xFFFFFF-color), string(buf)))
}

func colorNoop(color int) string {
	return ""
}

func colorAnsi16Foreground(color int) string {
	color = color % 16
	index := color%8 + 30
	if color < 8 {
		return fmt.Sprintf("\033[%vm", index)
	}
	return fmt.Sprintf("\033[%v;1m", index)
}

func colorAnsi16Background(color int) string {
	color = color % 16
	index := color%8 + 40
	if color < 8 {
		return fmt.Sprintf("\033[%vm", index)
	}
	return fmt.Sprintf("\033[%v;1m", index)
}

func colorAnsi256Foreground(color int) string {
	r, g, b := color>>(16+3), color>>(8+3)&0x1F, color>>3&0x1F
	return fmt.Sprintf("\033[38;5;%vm", rgbTo216Color(r, g, b)+16)
}

func colorAnsi256Background(color int) string {
	r, g, b := color>>(16+3), color>>(8+3)&0x1F, color>>3&0x1F
	return fmt.Sprintf("\033[48;5;%vm", rgbTo216Color(r, g, b)+16)
}

func rgbFrom216Color(color int) (int, int, int) {
	return color / 36, color / 6 % 6, color % 6
}

func rgbTo216Color(r, g, b int) int {
	return r*36 + g*6 + b
}
<commit_msg>Add color escape support for Windows<commit_after>package main

import (
	"bufio"
	"crypto/md5"
	"fmt"
	"os"
	"regexp"
	"strings"

	"github.com/shiena/ansicolor"
)

const version = "0.1.0+Go"

type ansiColorer func(color int) string

type options struct {
	foreground ansiColorer
	background ansiColorer
	pattern    string
}

type colorMode int

const (
	mode16Color colorMode = iota
	mode256color
)

func main() {
	interactLinewise(colorLineMaker(parseArgs(os.Args[1:])))
}

func parseArgs(args []string) options {
	colorForeground, colorBackground := true, false
	regexps := make([]string, 0)
	colorDepth := mode256color

	acceptingFlags := true
	for _, arg := range args {
		if !acceptingFlags {
			regexps = append(regexps, arg)
			continue
		}

		if arg == "-h" || arg == "--help" {
			printUsage()
			os.Exit(0)
		} else if arg == "-v" || arg == "--version" {
			fmt.Printf("Synesthesia version %v\n", version)
			os.Exit(0)
		} else if arg == "-f" || arg == "--foreground" {
			colorForeground = true
		} else if arg == "--no-foreground" {
			colorForeground = false
		} else if arg == "-b" || arg == "--background" {
			colorBackground = true
		} else if arg == "--no-background" {
			colorBackground = false
		} else if arg == "--16" {
			colorDepth = mode16Color
		} else if arg == "--256" {
			colorDepth = mode256color
		} else if arg == "--" {
			acceptingFlags = false
		} else if strings.HasPrefix(arg, "-") {
			fmt.Printf("Unrecognized flag: %s\n", arg)
			printUsage()
			os.Exit(1)
		} else {
			regexps = append(regexps, arg)
		}
	}

	foregroundColorer, backgroundColorer := colorNoop, colorNoop
	if colorDepth == mode16Color {
		if colorForeground {
			foregroundColorer = colorAnsi16Foreground
		}
		if colorBackground {
			backgroundColorer = colorAnsi16Background
		}
	} else if colorDepth == mode256color {
		if colorForeground {
			foregroundColorer = colorAnsi256Foreground
		}
		if colorBackground {
			backgroundColorer = colorAnsi256Background
		}
	}

	validateRegexps(regexps)
	for i, pattern := range regexps {
		regexps[i] = fmt.Sprintf("(?:%v)", pattern)
	}

	return options{foregroundColorer, backgroundColorer, strings.Join(regexps, "|")}
}

func printUsage() {
	fmt.Fprintf(os.Stderr, `Usage: %v ([flags]|[patterns])... [-- [patterns]...]
Color standard input based on the values matched by the provided regular
expressions.
Before --, any parameters starting with a dash will be interpreted as flags.
After it, all parameters are interpreted as patterns.
A pattern is any valid Go regular expression.
Multiples of flags controlling the same option are allowed; the last flag
specified will take precedence over the ones that come before it.
Flags:
-b --background     Turn on or off background colorization. (default off)
   --no-background
-f --foreground     Turn on or off foreground colorization. (default on)
   --no-foreground
-h --help           Print usage information.
`, os.Args[0])
}

func validateRegexps(regexps []string) {
	valid := true
	for _, pattern := range regexps {
		if _, err := regexp.Compile(pattern); err != nil {
			valid = false
			fmt.Fprintf(os.Stderr, "Invalid regex: %v\n", pattern)
		}
	}
	if !valid {
		os.Exit(1)
	}
}

func interactLinewise(transform func([]byte) []byte) {
	input := bufio.NewReader(os.Stdin)
	output := ansicolor.NewAnsiColorWriter(os.Stdout)
	line, err := input.ReadBytes('\n')
	for err == nil {
		fmt.Fprint(output, string(transform(line)))
		line, err = input.ReadBytes('\n')
	}
	fmt.Fprint(output, string(transform(line)))
}

func colorLineMaker(options options) func([]byte) []byte {
	pattern := regexp.MustCompile(options.pattern)
	return func(line []byte) []byte {
		return colorPatternInString(pattern, line, options.foreground, options.background)
	}
}

func colorPatternInString(pattern *regexp.Regexp, buf []byte, foreground, background ansiColorer) []byte {
	return pattern.ReplaceAllFunc(buf, colorMatchMaker(foreground, background))
}

func colorMatchMaker(foreground, background ansiColorer) func([]byte) []byte {
	return func(match []byte) []byte {
		return color(match, foreground, background)
	}
}

func color(buf []byte, foreground, background ansiColorer) []byte {
	sum := md5.Sum(buf)
	color := int(sum[md5.Size-3])<<16 | int(sum[md5.Size-2])<<8 | int(sum[md5.Size-1])
	return []byte(fmt.Sprintf("%v%v%v\033[m", foreground(color), background(0xFFFFFF-color), string(buf)))
}

func colorNoop(color int) string {
	return ""
}

func colorAnsi16Foreground(color int) string {
	color = color % 16
	index := color%8 + 30
	if color < 8 {
		return fmt.Sprintf("\033[%vm", index)
	}
	return fmt.Sprintf("\033[%v;1m", index)
}

func colorAnsi16Background(color int) string {
	color = color % 16
	index := color%8 + 40
	if color < 8 {
		return fmt.Sprintf("\033[%vm", index)
	}
	return fmt.Sprintf("\033[%v;1m", index)
}

func colorAnsi256Foreground(color int) string {
	r, g, b := color>>(16+3), color>>(8+3)&0x1F, color>>3&0x1F
	return fmt.Sprintf("\033[38;5;%vm", rgbTo216Color(r, g, b)+16)
}

func colorAnsi256Background(color int) string {
	r, g, b := color>>(16+3), color>>(8+3)&0x1F, color>>3&0x1F
	return fmt.Sprintf("\033[48;5;%vm", rgbTo216Color(r, g, b)+16)
}

func rgbFrom216Color(color int) (int, int, int) {
	return color / 36, color / 6 % 6, color % 6
}

func rgbTo216Color(r, g, b int) int {
	return r*36 + g*6 + b
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Package tailer provides a class that is responsible for tailing log files
// and extracting new log lines to be passed into the virtual machines.
package tailer

// For regular files, mtail gets notified on modifications (i.e. appends) to
// log files that are being watched, in order to read the new lines. Log files
// can also be rotated, so mtail is also notified of creates in the log file
// directory.

import (
	"errors"
	"expvar"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sync"
	"syscall"
	"time"
	"unicode/utf8"

	"github.com/golang/glog"

	"github.com/google/mtail/watcher"

	"github.com/spf13/afero"
)

var (
	logCount     = expvar.NewInt("log_count")
	logErrors    = expvar.NewMap("log_errors_total")
	logRotations = expvar.NewMap("log_rotations_total")
)

// Tailer receives notification of changes from a Watcher and extracts new log
// lines from files. It also handles new log file creation events and log
// rotations.
type Tailer struct {
	w watcher.Watcher

	watched      map[string]struct{}   // Names of logs being watched.
	watchedLock  sync.RWMutex          // protects `watched'
	lines        chan<- string         // Logfile lines being emitted.
	files        map[string]afero.File // File handles for each pathname.
	filesLock    sync.Mutex            // protects `files'
	partials     map[string]string     // Accumulator for the currently read line for each pathname.
	partialsLock sync.Mutex            // protects 'partials'

	fs afero.Fs // mockable filesystem interface
}

// Options configures a Tailer
type Options struct {
	Lines chan<- string
	W     watcher.Watcher // Not required, will use watcher.LogWatcher if it is zero.
	FS    afero.Fs        // Not required, will use afero.OsFs if it is zero.
}

// New returns a new Tailer, configured with the supplied Options
func New(o Options) (*Tailer, error) {
	if o.Lines == nil {
		return nil, errors.New("tailer needs lines")
	}
	fs := o.FS
	if fs == nil {
		fs = &afero.OsFs{}
	}
	w := o.W
	if w == nil {
		var err error
		w, err = watcher.NewLogWatcher()
		if err != nil {
			return nil, fmt.Errorf("Couldn't create a watcher for tailer: %s", err)
		}
	}
	t := &Tailer{
		w:        w,
		watched:  make(map[string]struct{}),
		lines:    o.Lines,
		files:    make(map[string]afero.File),
		partials: make(map[string]string),
		fs:       fs,
	}
	go t.run()
	return t, nil
}

// addWatched adds a path to the list of watched items.
func (t *Tailer) addWatched(path string) {
	t.watchedLock.Lock()
	defer t.watchedLock.Unlock()
	t.watched[path] = struct{}{}
}

// isWatching indicates if the path is being watched. It includes both
// filenames and directories.
func (t *Tailer) isWatching(path string) bool {
	t.watchedLock.RLock()
	defer t.watchedLock.RUnlock()
	_, ok := t.watched[path]
	return ok
}

// Tail registers a file path to be tailed.
func (t *Tailer) Tail(pathname string) {
	fullpath, err := filepath.Abs(pathname)
	if err != nil {
		glog.Infof("Failed to find absolute path for %q: %s\n", pathname, err)
		return
	}
	if !t.isWatching(fullpath) {
		t.addWatched(fullpath)
		logCount.Add(1)
		t.openLogPath(fullpath, false)
	}
}

// TailFile registers a file handle to be tailed.  There is no filesystem to
// watch, so no watches are registered, and no file paths are opened.
func (t *Tailer) TailFile(f afero.File) error {
	logCount.Add(1)
	return t.startNewFile(f, false)
}

// handleLogUpdate reads all available bytes from an already opened file
// identified by pathname, and sends them to be processed on the lines channel.
func (t *Tailer) handleLogUpdate(pathname string) {
	t.filesLock.Lock()
	fd, ok := t.files[pathname]
	t.filesLock.Unlock()
	if !ok {
		glog.Warningf("No file descriptor found for %q, but is being watched", pathname)
		return
	}
	var err error
	t.partialsLock.Lock()
	t.partials[pathname], err = t.read(fd, t.partials[pathname])
	t.partialsLock.Unlock()
	if err != nil && err != io.EOF {
		glog.Info(err)
	}
}

// read reads blocks of 4096 bytes from the File, sending lines to the
// channel as it encounters newlines.  If EOF is encountered, the partial line
// is returned to be concatenated with on the next call.
func (t *Tailer) read(f afero.File, partialIn string) (partialOut string, err error) {
	partial := partialIn
	b := make([]byte, 0, 4096)
	for {
		n, err := f.Read(b[:cap(b)])
		b = b[:n]
		if err != nil {
			return partial, err
		}

		for i, width := 0, 0; i < len(b) && i < n; i += width {
			var rune rune
			rune, width = utf8.DecodeRune(b[i:])
			switch {
			case rune != '\n':
				partial += string(rune)
			default:
				// send off line for processing, blocks if not ready
				t.lines <- partial
				// reset accumulator
				partial = ""
			}
		}
	}
}

// inode returns the inode number of a file, or 0 if the file has no underlying Sys implementation.
func inode(f os.FileInfo) uint64 {
	s := f.Sys()
	if s == nil {
		return 0
	}
	switch s := s.(type) {
	case *syscall.Stat_t:
		return uint64(s.Ino)
	default:
		return 0
	}
}

// handleLogCreate handles both new and rotated log files.
func (t *Tailer) handleLogCreate(pathname string) {
	t.filesLock.Lock()
	fd, ok := t.files[pathname]
	t.filesLock.Unlock()
	if ok {
		s1, err := fd.Stat()
		if err != nil {
			glog.Infof("Stat failed on %q: %s", t.files[pathname].Name(), err)
			return
		}
		s2, err := t.fs.Stat(pathname)
		if err != nil {
			glog.Infof("Stat failed on %q: %s", pathname, err)
			return
		}
		if inode(s1) != inode(s2) {
			glog.V(1).Infof("New inode detected for %s, treating as rotation.", pathname)
			logRotations.Add(pathname, 1)
			// flush the old log, pathname is still an index into t.files with the old inode.
			t.handleLogUpdate(pathname)
			fd.Close()
			go func() {
				// Run in goroutine as Remove may block waiting on event processing.
				err := t.w.Remove(pathname)
				if err != nil {
					glog.Infof("Failed removing watches on %s: %s", pathname, err)
				}
				// openLogPath readds the file to the watcher, so must be strictly after the Remove succeeds.
				t.openLogPath(pathname, true)
			}()
		} else {
			glog.V(1).Infof("Path %s already being watched, and inode not changed.",
				pathname)
		}
	} else {
		// Freshly opened log file, never seen before.
		t.openLogPath(pathname, true)
	}
}

// openLogPath opens a log file named by pathname.
func (t *Tailer) openLogPath(pathname string, seenBefore bool) {
	d := path.Dir(pathname)
	if !t.isWatching(d) {
		err := t.w.Add(d)
		if err != nil {
			glog.Infof("Failed to create new watch on directory %q: %s", d, err)
		}
		t.addWatched(d)
	}

	retries := 3
	retryDelay := 1 * time.Millisecond
	var f afero.File
	var err error
	for retries > 0 {
		f, err = t.fs.Open(pathname)
		if err == nil {
			break
		}
		// Doesn't exist yet. We're watching the directory, so we'll pick it up
		// again on create; return successfully.
		if os.IsNotExist(err) {
			return
		}
		glog.Infof("Failed to open %q for reading: %s", pathname, err)
		logErrors.Add(pathname, 1)
		// seenBefore indicates also that we're rotating a file that previously worked, so retry.
		if seenBefore {
			retries = retries - 1
			time.Sleep(retryDelay)
			retryDelay = retryDelay + retryDelay
		} else {
			return
		}
	}
	err = t.startNewFile(f, seenBefore)
	if err != nil {
		glog.Error(err)
	}
}

// startNewFile optionally seeks to the start or end of the file, then starts
// the consumption of log lines. Rotated logs should read from the start, but
// logs opened for the first time read from the end.
func (t *Tailer) startNewFile(f afero.File, seekStart bool) error {
	fi, err := f.Stat()
	if err != nil {
		// Stat failed, log error and return.
		logErrors.Add(f.Name(), 1)
		return fmt.Errorf("Failed to stat %q: %s", f.Name(), err)
	}
	switch m := fi.Mode(); {
	case m&os.ModeType == 0:
		if seekStart {
			f.Seek(0, os.SEEK_SET)
		} else {
			f.Seek(0, os.SEEK_END)
		}
		err = t.w.Add(f.Name())
		if err != nil {
			return fmt.Errorf("Adding a change watch failed on %q: %s", f.Name(), err)
		}
		// In case the new log has been written to already, attempt to read the
		// first lines.
		t.partialsLock.Lock()
		t.partials[f.Name()], err = t.read(f, "")
		t.partialsLock.Unlock()
		if err != nil {
			if err == io.EOF {
				// Don't worry about EOF on first read, that's expected.
				break
			}
			return err
		}
	case m&os.ModeType == os.ModeNamedPipe:
		go t.readForever(f)
	default:
		return fmt.Errorf("Can't open files with mode %v: %s", m&os.ModeType, f.Name())
	}
	t.filesLock.Lock()
	t.files[f.Name()] = f
	t.filesLock.Unlock()
	glog.Infof("Tailing %s", f.Name())

	return nil
}

// start is the main event loop for the Tailer.
// It receives notification of log file changes from the watcher channel, and
// handles them.
func (t *Tailer) run() {
	for e := range t.w.Events() {
		switch e := e.(type) {
		case watcher.UpdateEvent:
			if t.isWatching(e.Pathname) {
				t.handleLogUpdate(e.Pathname)
			}
		case watcher.CreateEvent:
			if t.isWatching(e.Pathname) {
				t.handleLogCreate(e.Pathname)
			}
		case watcher.DeleteEvent:
		default:
			glog.Infof("Unexpected event %#v", e)
		}
	}
	glog.Infof("Shutting down tailer.")
	close(t.lines)
}

// readForever handles non-logfile inputs by reading from the File until it is closed.
func (t *Tailer) readForever(f afero.File) {
	var err error
	partial := ""
	for {
		partial, err = t.read(f, partial)
		// We want to exit at EOF, because the FD has been closed.
		if err != nil {
			glog.Infof("%s: %s", err, f.Name())
			return
		}
		// TODO(jaq): nonblocking read, handle eagain, and do a little sleep if so, so the read can be interrupted
	}
}

// Close signals termination to the watcher.
func (t *Tailer) Close() {
	t.w.Close()
}
<commit_msg>Return the error from tailer close, if any.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Package tailer provides a class that is responsible for tailing log files
// and extracting new log lines to be passed into the virtual machines.
package tailer

// For regular files, mtail gets notified on modifications (i.e. appends) to
// log files that are being watched, in order to read the new lines. Log files
// can also be rotated, so mtail is also notified of creates in the log file
// directory.

import (
	"errors"
	"expvar"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sync"
	"syscall"
	"time"
	"unicode/utf8"

	"github.com/golang/glog"

	"github.com/google/mtail/watcher"

	"github.com/spf13/afero"
)

var (
	logCount     = expvar.NewInt("log_count")
	logErrors    = expvar.NewMap("log_errors_total")
	logRotations = expvar.NewMap("log_rotations_total")
)

// Tailer receives notification of changes from a Watcher and extracts new log
// lines from files. It also handles new log file creation events and log
// rotations.
type Tailer struct {
	w watcher.Watcher

	watched      map[string]struct{}   // Names of logs being watched.
	watchedLock  sync.RWMutex          // protects `watched'
	lines        chan<- string         // Logfile lines being emitted.
	files        map[string]afero.File // File handles for each pathname.
	filesLock    sync.Mutex            // protects `files'
	partials     map[string]string     // Accumulator for the currently read line for each pathname.
	partialsLock sync.Mutex            // protects 'partials'

	fs afero.Fs // mockable filesystem interface
}

// Options configures a Tailer
type Options struct {
	Lines chan<- string
	W     watcher.Watcher // Not required, will use watcher.LogWatcher if it is zero.
	FS    afero.Fs        // Not required, will use afero.OsFs if it is zero.
}

// New returns a new Tailer, configured with the supplied Options
func New(o Options) (*Tailer, error) {
	if o.Lines == nil {
		return nil, errors.New("tailer needs lines")
	}
	fs := o.FS
	if fs == nil {
		fs = &afero.OsFs{}
	}
	w := o.W
	if w == nil {
		var err error
		w, err = watcher.NewLogWatcher()
		if err != nil {
			return nil, fmt.Errorf("Couldn't create a watcher for tailer: %s", err)
		}
	}
	t := &Tailer{
		w:        w,
		watched:  make(map[string]struct{}),
		lines:    o.Lines,
		files:    make(map[string]afero.File),
		partials: make(map[string]string),
		fs:       fs,
	}
	go t.run()
	return t, nil
}

// addWatched adds a path to the list of watched items.
func (t *Tailer) addWatched(path string) {
	t.watchedLock.Lock()
	defer t.watchedLock.Unlock()
	t.watched[path] = struct{}{}
}

// isWatching indicates if the path is being watched. It includes both
// filenames and directories.
func (t *Tailer) isWatching(path string) bool {
	t.watchedLock.RLock()
	defer t.watchedLock.RUnlock()
	_, ok := t.watched[path]
	return ok
}

// Tail registers a file path to be tailed.
func (t *Tailer) Tail(pathname string) {
	fullpath, err := filepath.Abs(pathname)
	if err != nil {
		glog.Infof("Failed to find absolute path for %q: %s\n", pathname, err)
		return
	}
	if !t.isWatching(fullpath) {
		t.addWatched(fullpath)
		logCount.Add(1)
		t.openLogPath(fullpath, false)
	}
}

// TailFile registers a file handle to be tailed.  There is no filesystem to
// watch, so no watches are registered, and no file paths are opened.
func (t *Tailer) TailFile(f afero.File) error {
	logCount.Add(1)
	return t.startNewFile(f, false)
}

// handleLogUpdate reads all available bytes from an already opened file
// identified by pathname, and sends them to be processed on the lines channel.
func (t *Tailer) handleLogUpdate(pathname string) {
	t.filesLock.Lock()
	fd, ok := t.files[pathname]
	t.filesLock.Unlock()
	if !ok {
		glog.Warningf("No file descriptor found for %q, but is being watched", pathname)
		return
	}
	var err error
	t.partialsLock.Lock()
	t.partials[pathname], err = t.read(fd, t.partials[pathname])
	t.partialsLock.Unlock()
	if err != nil && err != io.EOF {
		glog.Info(err)
	}
}

// read reads blocks of 4096 bytes from the File, sending lines to the
// channel as it encounters newlines.  If EOF is encountered, the partial line
// is returned to be concatenated with on the next call.
func (t *Tailer) read(f afero.File, partialIn string) (partialOut string, err error) {
	partial := partialIn
	b := make([]byte, 0, 4096)
	for {
		n, err := f.Read(b[:cap(b)])
		b = b[:n]
		if err != nil {
			return partial, err
		}

		for i, width := 0, 0; i < len(b) && i < n; i += width {
			var rune rune
			rune, width = utf8.DecodeRune(b[i:])
			switch {
			case rune != '\n':
				partial += string(rune)
			default:
				// send off line for processing, blocks if not ready
				t.lines <- partial
				// reset accumulator
				partial = ""
			}
		}
	}
}

// inode returns the inode number of a file, or 0 if the file has no underlying Sys implementation.
func inode(f os.FileInfo) uint64 {
	s := f.Sys()
	if s == nil {
		return 0
	}
	switch s := s.(type) {
	case *syscall.Stat_t:
		return uint64(s.Ino)
	default:
		return 0
	}
}

// handleLogCreate handles both new and rotated log files.
func (t *Tailer) handleLogCreate(pathname string) {
	t.filesLock.Lock()
	fd, ok := t.files[pathname]
	t.filesLock.Unlock()
	if ok {
		s1, err := fd.Stat()
		if err != nil {
			glog.Infof("Stat failed on %q: %s", t.files[pathname].Name(), err)
			return
		}
		s2, err := t.fs.Stat(pathname)
		if err != nil {
			glog.Infof("Stat failed on %q: %s", pathname, err)
			return
		}
		if inode(s1) != inode(s2) {
			glog.V(1).Infof("New inode detected for %s, treating as rotation.", pathname)
			logRotations.Add(pathname, 1)
			// flush the old log, pathname is still an index into t.files with the old inode.
			t.handleLogUpdate(pathname)
			fd.Close()
			go func() {
				// Run in goroutine as Remove may block waiting on event processing.
				err := t.w.Remove(pathname)
				if err != nil {
					glog.Infof("Failed removing watches on %s: %s", pathname, err)
				}
				// openLogPath readds the file to the watcher, so must be strictly after the Remove succeeds.
				t.openLogPath(pathname, true)
			}()
		} else {
			glog.V(1).Infof("Path %s already being watched, and inode not changed.",
				pathname)
		}
	} else {
		// Freshly opened log file, never seen before.
		t.openLogPath(pathname, true)
	}
}

// openLogPath opens a log file named by pathname.
func (t *Tailer) openLogPath(pathname string, seenBefore bool) {
	d := path.Dir(pathname)
	if !t.isWatching(d) {
		err := t.w.Add(d)
		if err != nil {
			glog.Infof("Failed to create new watch on directory %q: %s", d, err)
		}
		t.addWatched(d)
	}

	retries := 3
	retryDelay := 1 * time.Millisecond
	var f afero.File
	var err error
	for retries > 0 {
		f, err = t.fs.Open(pathname)
		if err == nil {
			break
		}
		// Doesn't exist yet. We're watching the directory, so we'll pick it up
		// again on create; return successfully.
		if os.IsNotExist(err) {
			return
		}
		glog.Infof("Failed to open %q for reading: %s", pathname, err)
		logErrors.Add(pathname, 1)
		// seenBefore indicates also that we're rotating a file that previously worked, so retry.
		if seenBefore {
			retries = retries - 1
			time.Sleep(retryDelay)
			retryDelay = retryDelay + retryDelay
		} else {
			return
		}
	}
	err = t.startNewFile(f, seenBefore)
	if err != nil {
		glog.Error(err)
	}
}

// startNewFile optionally seeks to the start or end of the file, then starts
// the consumption of log lines. Rotated logs should read from the start, but
// logs opened for the first time read from the end.
func (t *Tailer) startNewFile(f afero.File, seekStart bool) error {
	fi, err := f.Stat()
	if err != nil {
		// Stat failed, log error and return.
		logErrors.Add(f.Name(), 1)
		return fmt.Errorf("Failed to stat %q: %s", f.Name(), err)
	}
	switch m := fi.Mode(); {
	case m&os.ModeType == 0:
		if seekStart {
			f.Seek(0, os.SEEK_SET)
		} else {
			f.Seek(0, os.SEEK_END)
		}
		err = t.w.Add(f.Name())
		if err != nil {
			return fmt.Errorf("Adding a change watch failed on %q: %s", f.Name(), err)
		}
		// In case the new log has been written to already, attempt to read the
		// first lines.
		t.partialsLock.Lock()
		t.partials[f.Name()], err = t.read(f, "")
		t.partialsLock.Unlock()
		if err != nil {
			if err == io.EOF {
				// Don't worry about EOF on first read, that's expected.
				break
			}
			return err
		}
	case m&os.ModeType == os.ModeNamedPipe:
		go t.readForever(f)
	default:
		return fmt.Errorf("Can't open files with mode %v: %s", m&os.ModeType, f.Name())
	}
	t.filesLock.Lock()
	t.files[f.Name()] = f
	t.filesLock.Unlock()
	glog.Infof("Tailing %s", f.Name())

	return nil
}

// start is the main event loop for the Tailer.
// It receives notification of log file changes from the watcher channel, and
// handles them.
func (t *Tailer) run() {
	for e := range t.w.Events() {
		switch e := e.(type) {
		case watcher.UpdateEvent:
			if t.isWatching(e.Pathname) {
				t.handleLogUpdate(e.Pathname)
			}
		case watcher.CreateEvent:
			if t.isWatching(e.Pathname) {
				t.handleLogCreate(e.Pathname)
			}
		case watcher.DeleteEvent:
		default:
			glog.Infof("Unexpected event %#v", e)
		}
	}
	glog.Infof("Shutting down tailer.")
	close(t.lines)
}

// readForever handles non-logfile inputs by reading from the File until it is closed.
func (t *Tailer) readForever(f afero.File) {
	var err error
	partial := ""
	for {
		partial, err = t.read(f, partial)
		// We want to exit at EOF, because the FD has been closed.
		if err != nil {
			glog.Infof("%s: %s", err, f.Name())
			return
		}
		// TODO(jaq): nonblocking read, handle eagain, and do a little sleep if so, so the read can be interrupted
	}
}

// Close signals termination to the watcher.
func (t *Tailer) Close() error {
	return t.w.Close()
}
<|endoftext|>


<commit_before>// errchk $G $D/$F.go

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

const (
	A int = 1;
	B byte;	// ERROR "type without expr"
)
<commit_msg>Recognize gccgo error message.<commit_after>// errchk $G $D/$F.go

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

const (
	A int = 1;
	B byte;	// ERROR "type without expr|expected .=."
)
<|endoftext|>


<commit_before>package testio

import (
	"bytes"
	"os"
	"testing"
)

func TestBrokenWriter(t *testing.T) {
	buf := NewBrokenWriter(2)
	data := []byte{1, 2}

	n, err := buf.Write(data)
	if err != nil {
		t.Fatalf("%v", err)
	} else if n != 2 {
		t.Fatalf("expected write size of 2, have %d", n)
	}

	_, err = buf.Write(data)
	if err == nil {
		t.Fatal("expected a write failure")
	}

	buf.Reset()
	_, err = buf.Write(data)
	if err == nil {
		t.Fatalf("expected a write failure after reset")
	}

	buf.Extend(2)
	_, err = buf.Write(data)
	if err != nil {
		t.Fatalf("%v", err)
	}
}

func TestBufCloser(t *testing.T) {
	var data = []byte{1, 2}
	var read = make([]byte, 2)

	buf := NewBufCloser(data)
	_, err := buf.Read(read)
	if err != nil {
		t.Fatalf("%v", err)
	}

	_, err = buf.Write(data)
	if err != nil {
		t.Fatalf("%v", err)
	}

	buf.Close()
	buf.Reset()

	s := "hi"
	buf = NewBufCloserString(s)

	read = buf.Bytes()
	if string(read) != s {
		t.Fatalf("expected %s, have %s", s, read)
	}
}

func TestLoggingBuffer(t *testing.T) {
	src := &bytes.Buffer{}
	data := []byte("AB")
	lb := NewLoggingBuffer(src)
	_, err := lb.Write(data)
	if err != nil {
		t.Fatalf("%v", err)
	}

	src.Reset()
	lb.SetName("TEST")
	out := &bytes.Buffer{}
	lb.LogTo(out)

	_, err = lb.Write(data)
	if err != nil {
		t.Fatalf("%v", err)
	}

	expected := "[TEST] [WRITE] 4142\n"
	if string(out.Bytes()) != expected {
		t.Fatalf("expected '%s', have '%s'", expected, string(out.Bytes()))
	}

	out.Reset()
	src = bytes.NewBuffer(data)
	read := make([]byte, 2)

	_, err = lb.Read(read)
	if err != nil {
		t.Fatalf("%v", err)
	}

	expected = "[TEST] [READ] 4142\n"
	if string(out.Bytes()) != expected {
		t.Fatalf("expected '%s', have '%s'", expected, string(out.Bytes()))
	}

	out.Reset()
	lb.SetName("")
	lb.LogTo(os.Stderr)
	lb.Write([]byte("AB"))

	lb.LogTo(out)
	_, err = lb.Read(read)
	if err != nil {
		t.Fatalf("%v", err)
	}

	expected = "[READ] 4142\n"
	if string(out.Bytes()) != expected {
		t.Fatalf("expected '%s', have '%s'", expected, string(out.Bytes()))
	}

	src.Reset()
	_, err = lb.Read(read)
	if err == nil {
		t.Fatal("expected a read failure")
	}
}

func TestBrokenReadWriter(t *testing.T) {
	brw := NewBrokenReadWriter(0, 0)
	lb := NewLoggingBuffer(brw)

	var p = make([]byte, 2)
	var data = []byte("HI")
	_, err := lb.Write(data)
	if err == nil {
		t.Fatal("expected a write failure")
	}

	_, err = lb.Read(p)
	if err == nil {
		t.Fatal("expected a read failure")
	}

	brw.Extend(1, 0)
	_, err = lb.Write(data)
	if err == nil {
		t.Fatal("expected a write failure")
	}

	brw.Extend(2, 0)
	_, err = lb.Write(data)
	if err != nil {
		t.Fatalf("%v", err)
	}

	brw.Extend(4, 1)
	_, err = lb.Write(data)
	if err != nil {
		t.Fatalf("%v", err)
	}

	_, err = lb.Read(p)
	if err == nil {
		t.Fatal("expected a read failure")
	}

	brw.Reset()
	brw.Extend(10, 2)
	_, err = lb.Write(data)
	if err != nil {
		t.Fatalf("%v", err)
	}

	p = make([]byte, 1)
	_, err = lb.Read(p)
	if err != nil {
		t.Fatalf("%v", err)
	}
}

func TestBufferConn(t *testing.T) {
	bc := NewBufferConn()

	client := []byte("AB")
	peer := []byte("XY")

	_, err := bc.WritePeer(peer)
	if err != nil {
		t.Fatalf("%v", err)
	}

	var p = make([]byte, 2)
	_, err = bc.Write(client)
	if err != nil {
		t.Fatalf("%v", err)
	}

	_, err = bc.Read(p)
	if err != nil {
		t.Fatalf("%v", err)
	}

	if !bytes.Equal(p, peer) {
		t.Fatalf("client should have read %x, but read %x",
			peer, p)
	}

	_, err = bc.ReadClient(p)
	if err != nil {
		t.Fatalf("%v", err)
	}

	if !bytes.Equal(client, p) {
		t.Fatalf("client should have sent %x, but sent %x",
			client, p)
	}

}
<commit_msg>Back to 100% test coverage.<commit_after>package testio

import (
	"bytes"
	"os"
	"testing"
)

func TestBrokenWriter(t *testing.T) {
	buf := NewBrokenWriter(2)
	data := []byte{1, 2}

	n, err := buf.Write(data)
	if err != nil {
		t.Fatalf("%v", err)
	} else if n != 2 {
		t.Fatalf("expected write size of 2, have %d", n)
	}

	_, err = buf.Write(data)
	if err == nil {
		t.Fatal("expected a write failure")
	}

	buf.Reset()
	_, err = buf.Write(data)
	if err == nil {
		t.Fatalf("expected a write failure after reset")
	}

	buf.Extend(2)
	_, err = buf.Write(data)
	if err != nil {
		t.Fatalf("%v", err)
	}
}

func TestBufCloser(t *testing.T) {
	var data = []byte{1, 2}
	var read = make([]byte, 2)

	buf := NewBufCloser(data)
	_, err := buf.Read(read)
	if err != nil {
		t.Fatalf("%v", err)
	}

	_, err = buf.Write(data)
	if err != nil {
		t.Fatalf("%v", err)
	}

	buf.Close()
	buf.Reset()

	s := "hi"
	buf = NewBufCloserString(s)

	read = buf.Bytes()
	if string(read) != s {
		t.Fatalf("expected %s, have %s", s, read)
	}
}

func TestLoggingBuffer(t *testing.T) {
	src := &bytes.Buffer{}
	data := []byte("AB")
	lb := NewLoggingBuffer(src)
	_, err := lb.Write(data)
	if err != nil {
		t.Fatalf("%v", err)
	}

	src.Reset()
	lb.SetName("TEST")
	out := &bytes.Buffer{}
	lb.LogTo(out)

	_, err = lb.Write(data)
	if err != nil {
		t.Fatalf("%v", err)
	}

	expected := "[TEST] [WRITE] 4142\n"
	if string(out.Bytes()) != expected {
		t.Fatalf("expected '%s', have '%s'", expected, string(out.Bytes()))
	}

	out.Reset()
	src = bytes.NewBuffer(data)
	read := make([]byte, 2)

	_, err = lb.Read(read)
	if err != nil {
		t.Fatalf("%v", err)
	}

	expected = "[TEST] [READ] 4142\n"
	if string(out.Bytes()) != expected {
		t.Fatalf("expected '%s', have '%s'", expected, string(out.Bytes()))
	}

	out.Reset()
	lb.SetName("")
	lb.LogTo(os.Stderr)
	lb.Write([]byte("AB"))

	lb.LogTo(out)
	_, err = lb.Read(read)
	if err != nil {
		t.Fatalf("%v", err)
	}

	expected = "[READ] 4142\n"
	if string(out.Bytes()) != expected {
		t.Fatalf("expected '%s', have '%s'", expected, string(out.Bytes()))
	}

	src.Reset()
	_, err = lb.Read(read)
	if err == nil {
		t.Fatal("expected a read failure")
	}
}

func TestBrokenReadWriter(t *testing.T) {
	brw := NewBrokenReadWriter(0, 0)
	lb := NewLoggingBuffer(brw)

	var p = make([]byte, 2)
	var data = []byte("HI")
	_, err := lb.Write(data)
	if err == nil {
		t.Fatal("expected a write failure")
	}

	_, err = lb.Read(p)
	if err == nil {
		t.Fatal("expected a read failure")
	}

	brw.Extend(1, 0)
	_, err = lb.Write(data)
	if err == nil {
		t.Fatal("expected a write failure")
	}

	brw.Extend(2, 0)
	_, err = lb.Write(data)
	if err != nil {
		t.Fatalf("%v", err)
	}

	brw.Extend(4, 1)
	_, err = lb.Write(data)
	if err != nil {
		t.Fatalf("%v", err)
	}

	_, err = lb.Read(p)
	if err == nil {
		t.Fatal("expected a read failure")
	}

	brw.Reset()
	brw.Extend(10, 2)
	_, err = lb.Write(data)
	if err != nil {
		t.Fatalf("%v", err)
	}

	p = make([]byte, 1)
	_, err = lb.Read(p)
	if err != nil {
		t.Fatalf("%v", err)
	}
}

func TestBufferConn(t *testing.T) {
	bc := NewBufferConn()

	client := []byte("AB")
	peer := []byte("XY")

	_, err := bc.WritePeer(peer)
	if err != nil {
		t.Fatalf("%v", err)
	}

	var p = make([]byte, 2)
	_, err = bc.Write(client)
	if err != nil {
		t.Fatalf("%v", err)
	}

	_, err = bc.Read(p)
	if err != nil {
		t.Fatalf("%v", err)
	}

	if !bytes.Equal(p, peer) {
		t.Fatalf("client should have read %x, but read %x",
			peer, p)
	}

	_, err = bc.ReadClient(p)
	if err != nil {
		t.Fatalf("%v", err)
	}

	if !bytes.Equal(client, p) {
		t.Fatalf("client should have sent %x, but sent %x",
			client, p)
	}

	err = bc.Close()
	if err != nil {
		t.Fatalf("Close should always return nil, but it returned %v", err)
	}
}
<|endoftext|>


<commit_before>package timer

import (
	"errors"
	"fmt"
	"log"
	"time"

	"github.com/mfine30/prodda/client"
)

const (
	MinimumFrequency = time.Duration(1 * time.Minute)
)

type Alarm struct {
	running    bool
	NextTime   time.Time
	cancelChan chan struct{}
	task       Task
	frequency  time.Duration
}

type Task interface {
	Run() error
}

type TravisTask struct {
	client  *client.Travis
	token   string
	buildID uint
}

func NewTravisTask(token string, buildID uint) *TravisTask {
	return &TravisTask{
		client:  client.NewTravisClient("https://api.travis-ci.org"),
		token:   token,
		buildID: buildID,
	}
}

func (t TravisTask) Run() error {
	fmt.Printf("Travis task running\n")

	resp, err := t.client.TriggerBuild(t.token, t.buildID)
	if err != nil {
		return err
	}
	log.Printf("response: %+v\n", resp)
	return nil
}

// NewAlarm creates an alarm
// An error will be thrown if time is not in the future
// An error will be thrown if the task is nil
// An error will be thrown if the frequency is between 0 and MinimumFrequency (exclusive)
func NewAlarm(t time.Time, task Task, frequency time.Duration) (*Alarm, error) {
	currentTime := time.Now()
	if t.Before(currentTime) {
		return nil, errors.New("Time must not be in the past")
	}

	if task == nil {
		return nil, errors.New("Task must not be nil.")
	}

	err := validateFrequency(frequency)
	if err != nil {
		return nil, err
	}

	return &Alarm{
		NextTime:   t,
		cancelChan: make(chan struct{}),
		task:       task,
		frequency:  frequency,
	}, nil
}

func validateFrequency(frequency time.Duration) error {
	if frequency != 0 && frequency < MinimumFrequency {
		return fmt.Errorf("Frequency must be 0 or greater than %v", MinimumFrequency)
	}
	return nil
}

// Update will change the time the alarm will finish at.
// If the alarm is currently running it will be canceled.
// An error will be thrown if time is not in the future
// An error will be thrown if the frequency is between 0 and MinimumFrequency (exclusive)
func (a *Alarm) Update(t time.Time, frequency time.Duration) error {
	currentTime := time.Now()
	if t.Before(currentTime) {
		return errors.New("Time must not be in the past")
	}

	err := validateFrequency(frequency)
	if err != nil {
		return err
	}

	if a.running {
		a.Cancel()
	}

	a.NextTime = t
	a.cancelChan = make(chan struct{})

	return nil
}

// Cancel will cancel the alarm if it is running
// It will return an error if the alarm is not running.
func (a *Alarm) Cancel() error {
	if !a.running {
		return errors.New("Alarm not running")
	}

	close(a.cancelChan)
	return nil
}

// Start will block until either the timer goes off or the Alarm is canceled
func (a *Alarm) Start() <-chan error {
	resultChan := make(chan error)

	go func() {
		for {
			a.running = true
			durationUntilNext := a.NextTime.Sub(time.Now())
			select {
			case <-time.After(durationUntilNext):
				fmt.Printf("Alarm time has gone off\n")
				err := a.task.Run()
				if err != nil {
					fmt.Printf("Error running task: %v\n", err)
					resultChan <- err
					a.running = false
					close(resultChan)
					return
				}
				if a.frequency == 0 {
					a.running = false
					close(resultChan)
					return
				}
				a.NextTime = time.Now().Add(a.frequency)
			case <-a.cancelChan:
				fmt.Printf("Alarm canceled\n")
				a.running = false
				close(resultChan)
				return
			}
		}
	}()
	return resultChan
}
<commit_msg>Add mutex in timer to prevent race condition.<commit_after>package timer

import (
	"errors"
	"fmt"
	"log"
	"sync"
	"time"

	"github.com/mfine30/prodda/client"
)

const (
	MinimumFrequency = time.Duration(1 * time.Minute)
)

type Alarm struct {
	mutex      *sync.Mutex
	running    bool
	NextTime   time.Time
	cancelChan chan struct{}
	task       Task
	frequency  time.Duration
}

type Task interface {
	Run() error
}

type TravisTask struct {
	client  *client.Travis
	token   string
	buildID uint
}

func NewTravisTask(token string, buildID uint) *TravisTask {
	return &TravisTask{
		client:  client.NewTravisClient("https://api.travis-ci.org"),
		token:   token,
		buildID: buildID,
	}
}

func (t TravisTask) Run() error {
	fmt.Printf("Travis task running\n")

	resp, err := t.client.TriggerBuild(t.token, t.buildID)
	if err != nil {
		return err
	}
	log.Printf("response: %+v\n", resp)
	return nil
}

// NewAlarm creates an alarm
// An error will be thrown if time is not in the future
// An error will be thrown if the task is nil
// An error will be thrown if the frequency is between 0 and MinimumFrequency (exclusive)
func NewAlarm(t time.Time, task Task, frequency time.Duration) (*Alarm, error) {
	currentTime := time.Now()
	if t.Before(currentTime) {
		return nil, errors.New("Time must not be in the past")
	}

	if task == nil {
		return nil, errors.New("Task must not be nil.")
	}

	err := validateFrequency(frequency)
	if err != nil {
		return nil, err
	}

	return &Alarm{
		NextTime:   t,
		cancelChan: make(chan struct{}),
		task:       task,
		frequency:  frequency,
		mutex:      &sync.Mutex{},
	}, nil
}

func validateFrequency(frequency time.Duration) error {
	if frequency != 0 && frequency < MinimumFrequency {
		return fmt.Errorf("Frequency must be 0 or greater than %v", MinimumFrequency)
	}
	return nil
}

// Update will change the time the alarm will finish at.
// If the alarm is currently running it will be canceled.
// An error will be thrown if time is not in the future
// An error will be thrown if the frequency is between 0 and MinimumFrequency (exclusive)
func (a *Alarm) Update(t time.Time, frequency time.Duration) error {
	a.mutex.Lock()
	defer a.mutex.Unlock()

	currentTime := time.Now()
	if t.Before(currentTime) {
		return errors.New("Time must not be in the past")
	}

	err := validateFrequency(frequency)
	if err != nil {
		return err
	}

	if a.running {
		a.Cancel()
	}

	a.NextTime = t
	a.cancelChan = make(chan struct{})

	return nil
}

// Cancel will cancel the alarm if it is running
// It will return an error if the alarm is not running.
func (a *Alarm) Cancel() error {
	a.mutex.Lock()
	defer a.mutex.Unlock()

	if !a.running {
		return errors.New("Alarm not running")
	}

	close(a.cancelChan)
	return nil
}

// Start will block until either the timer goes off or the Alarm is canceled
func (a *Alarm) Start() <-chan error {
	resultChan := make(chan error)

	go func() {
		for {
			a.mutex.Lock()
			a.running = true
			a.mutex.Unlock()

			durationUntilNext := a.NextTime.Sub(time.Now())
			select {
			case <-time.After(durationUntilNext):
				fmt.Printf("Alarm time has gone off\n")
				err := a.task.Run()
				if err != nil {
					fmt.Printf("Error running task: %v\n", err)

					a.mutex.Lock()
					a.running = false
					a.mutex.Unlock()

					resultChan <- err
					close(resultChan)
					return
				}
				if a.frequency == 0 {
					a.mutex.Lock()
					a.running = false
					a.mutex.Unlock()

					close(resultChan)
					return
				}
				a.NextTime = time.Now().Add(a.frequency)
			case <-a.cancelChan:
				fmt.Printf("Alarm canceled\n")

				a.mutex.Lock()
				a.running = false
				a.mutex.Unlock()

				close(resultChan)
				return
			}
		}
	}()
	return resultChan
}
<|endoftext|>


<commit_before>// Package tlsdefaults provides sensible defaults for using TLS.
package tlsdefaults

import (
	"crypto/tls"
)

var (
	// The ECDHE cipher suites are preferred for performance and forward
	// secrecy.  See https://community.qualys.com/blogs/securitylabs/2013/06/25/ssl-labs-deploying-forward-secrecy.
	preferredCipherSuites = []uint16{
		tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
		tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
		tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,
		tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA,
		tls.TLS_RSA_WITH_AES_128_CBC_SHA,
		tls.TLS_RSA_WITH_AES_256_CBC_SHA,
		tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
		tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
	}
)

// Server provides a tls.Config with sensible defaults for server use.
func Server() *tls.Config {
	return &tls.Config{
		MinVersion:               tls.VersionTLS10, // mitigate against POODLE
		PreferServerCipherSuites: true,
		CipherSuites:             preferredCipherSuites, // maximize likelihood of perfect forward secrecy
	}
}
<commit_msg>Using more of Go's defaults<commit_after>// Package tlsdefaults provides sensible defaults for using TLS.
package tlsdefaults

import (
	"crypto/tls"
)

// Server provides a tls.Config with sensible defaults for server use. At this
// point, it mostly trusts the defaults from Go (assumes Go version 1.5 or
// or newer).
func Server() *tls.Config {
	return &tls.Config{
		PreferServerCipherSuites: true,
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package token defines constants representing the lexical tokens of the Go+
// programming language and basic operations on tokens (printing, predicates).
//
package token

import (
	"go/token"
	"strconv"
)

// Token is the set of lexical tokens of the Go programming language.
type Token int

// The list of tokens.
const (
	// Special tokens
	ILLEGAL Token = iota
	EOF
	COMMENT

	literal_beg
	// Identifiers and basic type literals
	// (these tokens stand for classes of literals)
	IDENT  // main
	INT    // 12345
	FLOAT  // 123.45
	IMAG   // 123.45i
	CHAR   // 'a'
	STRING // "abc"
	literal_end

	operator_beg
	// Operators and delimiters
	ADD // +
	SUB // -
	MUL // *
	QUO // /
	REM // %

	AND     // &
	OR      // |
	XOR     // ^
	SHL     // <<
	SHR     // >>
	AND_NOT // &^

	ADD_ASSIGN // +=
	SUB_ASSIGN // -=
	MUL_ASSIGN // *=
	QUO_ASSIGN // /=
	REM_ASSIGN // %=

	AND_ASSIGN     // &=
	OR_ASSIGN      // |=
	XOR_ASSIGN     // ^=
	SHL_ASSIGN     // <<=
	SHR_ASSIGN     // >>=
	AND_NOT_ASSIGN // &^=

	LAND  // &&
	LOR   // ||
	ARROW // <-
	INC   // ++
	DEC   // --

	EQL    // ==
	LSS    // <
	GTR    // >
	ASSIGN // =
	NOT    // !

	NEQ      // !=
	LEQ      // <=
	GEQ      // >=
	DEFINE   // :=
	ELLIPSIS // ...

	LPAREN // (
	LBRACK // [
	LBRACE // {
	COMMA  // ,
	PERIOD // .

	RPAREN    // )
	RBRACK    // ]
	RBRACE    // }
	SEMICOLON // ;
	COLON     // :
	operator_end

	keyword_beg
	// Keywords
	BREAK
	CASE
	CHAN
	CONST
	CONTINUE

	DEFAULT
	DEFER
	ELSE
	FALLTHROUGH
	FOR

	FUNC
	GO
	GOTO
	IF
	IMPORT

	INTERFACE
	MAP
	PACKAGE
	RANGE
	RETURN

	SELECT
	STRUCT
	SWITCH
	TYPE
	VAR
	keyword_end

	RAT      = literal_end  // 123.5r
	QUESTION = operator_end // ~
)

var tokens = [...]string{
	ILLEGAL: "ILLEGAL",

	EOF:     "EOF",
	COMMENT: "COMMENT",

	IDENT:  "IDENT",
	INT:    "INT",
	FLOAT:  "FLOAT",
	IMAG:   "IMAG",
	CHAR:   "CHAR",
	STRING: "STRING",

	ADD: "+",
	SUB: "-",
	MUL: "*",
	QUO: "/",
	REM: "%",

	AND:     "&",
	OR:      "|",
	XOR:     "^",
	SHL:     "<<",
	SHR:     ">>",
	AND_NOT: "&^",

	ADD_ASSIGN: "+=",
	SUB_ASSIGN: "-=",
	MUL_ASSIGN: "*=",
	QUO_ASSIGN: "/=",
	REM_ASSIGN: "%=",

	AND_ASSIGN:     "&=",
	OR_ASSIGN:      "|=",
	XOR_ASSIGN:     "^=",
	SHL_ASSIGN:     "<<=",
	SHR_ASSIGN:     ">>=",
	AND_NOT_ASSIGN: "&^=",

	LAND:  "&&",
	LOR:   "||",
	ARROW: "<-",
	INC:   "++",
	DEC:   "--",

	EQL:    "==",
	LSS:    "<",
	GTR:    ">",
	ASSIGN: "=",
	NOT:    "!",

	NEQ:      "!=",
	LEQ:      "<=",
	GEQ:      ">=",
	DEFINE:   ":=",
	ELLIPSIS: "...",

	LPAREN: "(",
	LBRACK: "[",
	LBRACE: "{",
	COMMA:  ",",
	PERIOD: ".",

	RPAREN:    ")",
	RBRACK:    "]",
	RBRACE:    "}",
	SEMICOLON: ";",
	COLON:     ":",
	QUESTION:  "?",

	BREAK:    "break",
	CASE:     "case",
	CHAN:     "chan",
	CONST:    "const",
	CONTINUE: "continue",

	DEFAULT:     "default",
	DEFER:       "defer",
	ELSE:        "else",
	FALLTHROUGH: "fallthrough",
	FOR:         "for",

	FUNC:   "func",
	GO:     "go",
	GOTO:   "goto",
	IF:     "if",
	IMPORT: "import",

	INTERFACE: "interface",
	MAP:       "map",
	PACKAGE:   "package",
	RANGE:     "range",
	RETURN:    "return",

	SELECT: "select",
	STRUCT: "struct",
	SWITCH: "switch",
	TYPE:   "type",
	VAR:    "var",
}

// String returns the string corresponding to the token tok.
// For operators, delimiters, and keywords the string is the actual
// token character sequence (e.g., for the token ADD, the string is
// "+"). For all other tokens the string corresponds to the token
// constant name (e.g. for the token IDENT, the string is "IDENT").
//
func (tok Token) String() string {
	s := ""
	if 0 <= tok && tok < Token(len(tokens)) {
		s = tokens[tok]
	}
	if s == "" {
		s = "token(" + strconv.Itoa(int(tok)) + ")"
	}
	return s
}

// A set of constants for precedence-based expression parsing.
// Non-operators have lowest precedence, followed by operators
// starting with precedence 1 up to unary operators. The highest
// precedence serves as "catch-all" precedence for selector,
// indexing, and other operator and delimiter tokens.
//
const (
	LowestPrec  = 0 // non-operators
	UnaryPrec   = 6
	HighestPrec = 7
)

// Precedence returns the operator precedence of the binary
// operator op. If op is not a binary operator, the result
// is LowestPrecedence.
//
func (op Token) Precedence() int {
	switch op {
	case LOR:
		return 1
	case LAND:
		return 2
	case EQL, NEQ, LSS, LEQ, GTR, GEQ:
		return 3
	case ADD, SUB, OR, XOR:
		return 4
	case MUL, QUO, REM, SHL, SHR, AND, AND_NOT:
		return 5
	}
	return LowestPrec
}

var keywords map[string]Token

func init() {
	keywords = make(map[string]Token)
	for i := keyword_beg + 1; i < keyword_end; i++ {
		keywords[tokens[i]] = i
	}
}

// Lookup maps an identifier to its keyword token or IDENT (if not a keyword).
//
func Lookup(ident string) Token {
	if tok, is_keyword := keywords[ident]; is_keyword {
		return tok
	}
	return IDENT
}

// Predicates

// IsLiteral returns true for tokens corresponding to identifiers
// and basic type literals; it returns false otherwise.
//
func (tok Token) IsLiteral() bool { return literal_beg < tok && tok <= literal_end }

// IsOperator returns true for tokens corresponding to operators and
// delimiters; it returns false otherwise.
//
func (tok Token) IsOperator() bool { return operator_beg < tok && tok <= operator_end }

// IsKeyword returns true for tokens corresponding to keywords;
// it returns false otherwise.
//
func (tok Token) IsKeyword() bool { return keyword_beg < tok && tok < keyword_end }

// IsExported reports whether name starts with an upper-case letter.
//
func IsExported(name string) bool {
	return token.IsExported(name)
}

// IsKeyword reports whether name is a Go keyword, such as "func" or "return".
//
func IsKeyword(name string) bool {
	return token.IsKeyword(name)
}

// IsIdentifier reports whether name is a Go identifier, that is, a non-empty
// string made up of letters, digits, and underscores, where the first character
// is not a digit. Keywords are not identifiers.
//
func IsIdentifier(name string) bool {
	return token.IsIdentifier(name)
}
<commit_msg>Minor reformat of token.go<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package token defines constants representing the lexical tokens of the Go+
// programming language and basic operations on tokens (printing, predicates).
//
package token

import (
	"go/token"
	"strconv"
)

// Token is the set of lexical tokens of the Go programming language.
type Token int

// The list of tokens.
const (
	// Special tokens
	ILLEGAL Token = iota
	EOF
	COMMENT

	literal_beg
	// Identifiers and basic type literals
	// (these tokens stand for classes of literals)
	IDENT  // main
	INT    // 12345
	FLOAT  // 123.45
	IMAG   // 123.45i
	CHAR   // 'a'
	STRING // "abc"
	literal_end

	operator_beg
	// Operators and delimiters
	ADD // +
	SUB // -
	MUL // *
	QUO // /
	REM // %

	AND     // &
	OR      // |
	XOR     // ^
	SHL     // <<
	SHR     // >>
	AND_NOT // &^

	ADD_ASSIGN // +=
	SUB_ASSIGN // -=
	MUL_ASSIGN // *=
	QUO_ASSIGN // /=
	REM_ASSIGN // %=

	AND_ASSIGN     // &=
	OR_ASSIGN      // |=
	XOR_ASSIGN     // ^=
	SHL_ASSIGN     // <<=
	SHR_ASSIGN     // >>=
	AND_NOT_ASSIGN // &^=

	LAND  // &&
	LOR   // ||
	ARROW // <-
	INC   // ++
	DEC   // --

	EQL    // ==
	LSS    // <
	GTR    // >
	ASSIGN // =
	NOT    // !

	NEQ      // !=
	LEQ      // <=
	GEQ      // >=
	DEFINE   // :=
	ELLIPSIS // ...

	LPAREN // (
	LBRACK // [
	LBRACE // {
	COMMA  // ,
	PERIOD // .

	RPAREN    // )
	RBRACK    // ]
	RBRACE    // }
	SEMICOLON // ;
	COLON     // :
	operator_end

	keyword_beg
	// Keywords
	BREAK
	CASE
	CHAN
	CONST
	CONTINUE

	DEFAULT
	DEFER
	ELSE
	FALLTHROUGH
	FOR

	FUNC
	GO
	GOTO
	IF
	IMPORT

	INTERFACE
	MAP
	PACKAGE
	RANGE
	RETURN

	SELECT
	STRUCT
	SWITCH
	TYPE
	VAR
	keyword_end

	RAT      = literal_end  // 123.5r
	QUESTION = operator_end // ~
)

var tokens = [...]string{
	ILLEGAL: "ILLEGAL",

	EOF:     "EOF",
	COMMENT: "COMMENT",

	IDENT:  "IDENT",
	INT:    "INT",
	FLOAT:  "FLOAT",
	IMAG:   "IMAG",
	CHAR:   "CHAR",
	STRING: "STRING",

	ADD: "+",
	SUB: "-",
	MUL: "*",
	QUO: "/",
	REM: "%",

	AND:     "&",
	OR:      "|",
	XOR:     "^",
	SHL:     "<<",
	SHR:     ">>",
	AND_NOT: "&^",

	ADD_ASSIGN: "+=",
	SUB_ASSIGN: "-=",
	MUL_ASSIGN: "*=",
	QUO_ASSIGN: "/=",
	REM_ASSIGN: "%=",

	AND_ASSIGN:     "&=",
	OR_ASSIGN:      "|=",
	XOR_ASSIGN:     "^=",
	SHL_ASSIGN:     "<<=",
	SHR_ASSIGN:     ">>=",
	AND_NOT_ASSIGN: "&^=",

	LAND:  "&&",
	LOR:   "||",
	ARROW: "<-",
	INC:   "++",
	DEC:   "--",

	EQL:    "==",
	LSS:    "<",
	GTR:    ">",
	ASSIGN: "=",
	NOT:    "!",

	NEQ:      "!=",
	LEQ:      "<=",
	GEQ:      ">=",
	DEFINE:   ":=",
	ELLIPSIS: "...",

	LPAREN: "(",
	LBRACK: "[",
	LBRACE: "{",
	COMMA:  ",",
	PERIOD: ".",

	RPAREN:    ")",
	RBRACK:    "]",
	RBRACE:    "}",
	SEMICOLON: ";",
	COLON:     ":",
	QUESTION:  "?",

	BREAK:    "break",
	CASE:     "case",
	CHAN:     "chan",
	CONST:    "const",
	CONTINUE: "continue",

	DEFAULT:     "default",
	DEFER:       "defer",
	ELSE:        "else",
	FALLTHROUGH: "fallthrough",
	FOR:         "for",

	FUNC:   "func",
	GO:     "go",
	GOTO:   "goto",
	IF:     "if",
	IMPORT: "import",

	INTERFACE: "interface",
	MAP:       "map",
	PACKAGE:   "package",
	RANGE:     "range",
	RETURN:    "return",

	SELECT: "select",
	STRUCT: "struct",
	SWITCH: "switch",
	TYPE:   "type",
	VAR:    "var",
}

// String returns the string corresponding to the token tok.
// For operators, delimiters, and keywords the string is the actual
// token character sequence (e.g., for the token ADD, the string is
// "+"). For all other tokens the string corresponds to the token
// constant name (e.g. for the token IDENT, the string is "IDENT").
//
func (tok Token) String() string {
	s := ""
	if 0 <= tok && tok < Token(len(tokens)) {
		s = tokens[tok]
	}
	if s == "" {
		s = "token(" + strconv.Itoa(int(tok)) + ")"
	}
	return s
}

// A set of constants for precedence-based expression parsing.
// Non-operators have lowest precedence, followed by operators
// starting with precedence 1 up to unary operators. The highest
// precedence serves as "catch-all" precedence for selector,
// indexing, and other operator and delimiter tokens.
//
const (
	LowestPrec  = 0 // non-operators
	UnaryPrec   = 6
	HighestPrec = 7
)

// Precedence returns the operator precedence of the binary
// operator op. If op is not a binary operator, the result
// is LowestPrecedence.
//
func (op Token) Precedence() int {
	switch op {
	case LOR:
		return 1
	case LAND:
		return 2
	case EQL, NEQ, LSS, LEQ, GTR, GEQ:
		return 3
	case ADD, SUB, OR, XOR:
		return 4
	case MUL, QUO, REM, SHL, SHR, AND, AND_NOT:
		return 5
	}
	return LowestPrec
}

var keywords map[string]Token

func init() {
	keywords = make(map[string]Token)
	for i := keyword_beg + 1; i < keyword_end; i++ {
		keywords[tokens[i]] = i
	}
}

// Lookup maps an identifier to its keyword token or IDENT (if not a keyword).
//
func Lookup(ident string) Token {
	if tok, is_keyword := keywords[ident]; is_keyword {
		return tok
	}
	return IDENT
}

// Predicates

// IsLiteral returns true for tokens corresponding to identifiers
// and basic type literals; it returns false otherwise.
//
func (tok Token) IsLiteral() bool {
	return literal_beg < tok && tok <= literal_end
}

// IsOperator returns true for tokens corresponding to operators and
// delimiters; it returns false otherwise.
//
func (tok Token) IsOperator() bool {
	return operator_beg < tok && tok <= operator_end
}

// IsKeyword returns true for tokens corresponding to keywords;
// it returns false otherwise.
//
func (tok Token) IsKeyword() bool {
	return keyword_beg < tok && tok < keyword_end
}

// IsExported reports whether name starts with an upper-case letter.
//
func IsExported(name string) bool {
	return token.IsExported(name)
}

// IsKeyword reports whether name is a Go keyword, such as "func" or "return".
//
func IsKeyword(name string) bool {
	return token.IsKeyword(name)
}

// IsIdentifier reports whether name is a Go identifier, that is, a non-empty
// string made up of letters, digits, and underscores, where the first character
// is not a digit. Keywords are not identifiers.
//
func IsIdentifier(name string) bool {
	return token.IsIdentifier(name)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/gorilla/feeds"
)

type Video struct {
	MediaName string
	MediaID   string
	VideoID   string
}

func main() {
	fmt.Println("Parsing trailers")

	trailers, _ := FindTrailers()

	fmt.Println("Generating feed")

	feed := &feeds.RssFeed{
		Title:          "Trailers RSS Feed",
		Link:           "http://mediafeeds.malone.me.uk",
		Description:    "An RSS feed of IMDB trailers.",
		ManagingEditor: "CjMalone@mail.com (Cj Malone)",
		Category:       "Trailers",
	}

	feed.Items = make([]*feeds.RssItem, len(trailers))

	for i := 0; i < len(trailers); i++ {
		fileInfo, err := os.Stat("/var/www/mediafeeds/Trailers/" + trailers[i].VideoID + ".mp4")
		if err != nil {
			cmd := exec.Command("youtube-dl", "-o", "/var/www/mediafeeds/Trailers/"+trailers[i].VideoID+".mp4", "http://www.imdb.com/video/imdb/"+trailers[i].VideoID+"/imdb/embed")

			var out bytes.Buffer
			cmd.Stdout = &out

			fmt.Printf("Downloading %s\n", trailers[i].MediaName)
			err := cmd.Run()
			if err != nil {
				fmt.Println(err)
				fmt.Println(out.String())
			}
		}

		fileInfo, err = os.Stat("/var/www/mediafeeds/Trailers/" + trailers[i].VideoID + ".mp4")
		fileLength := "0"
		if err != nil {
			fmt.Println(err)
		} else {
			fileLength = strconv.FormatInt(fileInfo.Size(), 10)
		}

		feed.Items[i] = &feeds.RssItem{
			Title:       trailers[i].MediaName,
			Link:        "http://www.imdb.com/title/" + trailers[i].MediaID,
			Description: "A trailer for \"" + trailers[i].MediaName + "\" (" + trailers[i].MediaID + ")",
			Author:      "CjMalone@mail.com (Cj Malone)",
			Category:    "Trailer",
			Enclosure: &feeds.RssEnclosure{
				Url:    "http://mediafeeds.malone.me.uk/Trailers/" + trailers[i].VideoID + ".mp4",
				Length: fileLength,
				Type:   "video/mp4",
			},
			Guid: trailers[i].VideoID,
			PubDate: time.Now().Format(time.RFC1123Z),
		}
	}
	rss, err := feeds.ToXML(feed)
	if err != nil {
		fmt.Println(err)
	}

	fmt.Println("Feed created")

	rssFile, err := os.OpenFile("/var/www/mediafeeds/trailers.xml", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(0644))
	defer rssFile.Close()

	if err != nil {
		fmt.Println(err)
	}

	rssFile.WriteString(rss)
}

func FindTrailers() ([]Video, error) {
	doc, err := goquery.NewDocument("http://www.imdb.com/trailers")
	if err != nil {
		return nil, err
	}

	recentTab := doc.Find("#recAddTab")
	lengthStr, exists := recentTab.Find(".gridlist-item").Last().Attr("data-index")
	if !exists {
		return nil, nil //TODO
	}
	length, err := strconv.Atoi(lengthStr)
	var trailers = make([]Video, length)
	current := 0

	recentTab.Find(".gridlist-item").Each(func(i int, s *goquery.Selection) {
		rawTitle := s.Find(".trailer-caption").Text()
		mediaLink, mediaLinkExists := s.Find(".trailer-caption").ChildrenFiltered("a").Attr("href")
		videoLink, videoLinkExists := s.Find(".video-link").Attr("href")
		if !mediaLinkExists || !videoLinkExists {
			return //TODO
		}

		title := rawTitle[2 : len(rawTitle)-5]
		var trailer Video
		trailer.MediaName = title
		trailer.MediaID = mediaLink[7:16] //TODO
		splitLink := strings.Split(videoLink, "/")
		trailer.VideoID = splitLink[3]

		trailers[current] = trailer

		current += 1
	})
	return trailers, nil
}
<commit_msg>Changed feed description and title<commit_after>package main

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/gorilla/feeds"
)

type Video struct {
	MediaName string
	MediaID   string
	VideoID   string
}

func main() {
	fmt.Println("Parsing trailers")

	trailers, _ := FindTrailers()

	fmt.Println("Generating feed")

	feed := &feeds.RssFeed{
		Title:          "Trailers",
		Link:           "http://mediafeeds.malone.me.uk",
		Description:    "Movie and TV show trailers that are published on IMDb",
		ManagingEditor: "CjMalone@mail.com (Cj Malone)",
		Category:       "Trailers",
	}

	feed.Items = make([]*feeds.RssItem, len(trailers))

	for i := 0; i < len(trailers); i++ {
		fileInfo, err := os.Stat("/var/www/mediafeeds/Trailers/" + trailers[i].VideoID + ".mp4")
		if err != nil {
			cmd := exec.Command("youtube-dl", "-o", "/var/www/mediafeeds/Trailers/"+trailers[i].VideoID+".mp4", "http://www.imdb.com/video/imdb/"+trailers[i].VideoID+"/imdb/embed")

			var out bytes.Buffer
			cmd.Stdout = &out

			fmt.Printf("Downloading %s\n", trailers[i].MediaName)
			err := cmd.Run()
			if err != nil {
				fmt.Println(err)
				fmt.Println(out.String())
			}
		}

		fileInfo, err = os.Stat("/var/www/mediafeeds/Trailers/" + trailers[i].VideoID + ".mp4")
		fileLength := "0"
		if err != nil {
			fmt.Println(err)
		} else {
			fileLength = strconv.FormatInt(fileInfo.Size(), 10)
		}

		feed.Items[i] = &feeds.RssItem{
			Title:       trailers[i].MediaName,
			Link:        "http://www.imdb.com/title/" + trailers[i].MediaID,
			Description: "A trailer for \"" + trailers[i].MediaName + "\" (" + trailers[i].MediaID + ")",
			Author:      "CjMalone@mail.com (Cj Malone)",
			Category:    "Trailer",
			Enclosure: &feeds.RssEnclosure{
				Url:    "http://mediafeeds.malone.me.uk/Trailers/" + trailers[i].VideoID + ".mp4",
				Length: fileLength,
				Type:   "video/mp4",
			},
			Guid: trailers[i].VideoID,
			PubDate: time.Now().Format(time.RFC1123Z),
		}
	}
	rss, err := feeds.ToXML(feed)
	if err != nil {
		fmt.Println(err)
	}

	fmt.Println("Feed created")

	rssFile, err := os.OpenFile("/var/www/mediafeeds/trailers.xml", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(0644))
	defer rssFile.Close()

	if err != nil {
		fmt.Println(err)
	}

	rssFile.WriteString(rss)
}

func FindTrailers() ([]Video, error) {
	doc, err := goquery.NewDocument("http://www.imdb.com/trailers")
	if err != nil {
		return nil, err
	}

	recentTab := doc.Find("#recAddTab")
	lengthStr, exists := recentTab.Find(".gridlist-item").Last().Attr("data-index")
	if !exists {
		return nil, nil //TODO
	}
	length, err := strconv.Atoi(lengthStr)
	var trailers = make([]Video, length)
	current := 0

	recentTab.Find(".gridlist-item").Each(func(i int, s *goquery.Selection) {
		rawTitle := s.Find(".trailer-caption").Text()
		mediaLink, mediaLinkExists := s.Find(".trailer-caption").ChildrenFiltered("a").Attr("href")
		videoLink, videoLinkExists := s.Find(".video-link").Attr("href")
		if !mediaLinkExists || !videoLinkExists {
			return //TODO
		}

		title := rawTitle[2 : len(rawTitle)-5]
		var trailer Video
		trailer.MediaName = title
		trailer.MediaID = mediaLink[7:16] //TODO
		splitLink := strings.Split(videoLink, "/")
		trailer.VideoID = splitLink[3]

		trailers[current] = trailer

		current += 1
	})
	return trailers, nil
}
<|endoftext|>