<commit_before>package demoinfocs

import (
	"errors"
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/gogo/protobuf/proto"

	"github.com/markus-wa/demoinfocs-golang/common"
	"github.com/markus-wa/demoinfocs-golang/events"
	"github.com/markus-wa/demoinfocs-golang/msg"
)

const maxOsPath = 260

const (
	playerWeaponPrefix    = "m_hMyWeapons."
	playerWeaponPrePrefix = "bcc_nonlocaldata."
	gameRulesPrefix       = "cs_gamerules_data"
)

// Parsing errors
var (
	// ErrCancelled signals that parsing was cancelled via Parser.Cancel()
	ErrCancelled = errors.New("parsing was cancelled before it finished (ErrCancelled)")

	// ErrUnexpectedEndOfDemo signals that the demo is incomplete / corrupt -
	// these demos may still be useful, check how far the parser got.
	ErrUnexpectedEndOfDemo = errors.New("demo stream ended unexpectedly (ErrUnexpectedEndOfDemo)")

	// ErrInvalidFileType signals that the input isn't a valid CS:GO demo.
	ErrInvalidFileType = errors.New("invalid File-Type; expecting HL2DEMO in the first 8 bytes (ErrInvalidFileType)")
)

// ParseHeader attempts to parse the header of the demo and returns it.
// If not done manually this will be called by Parser.ParseNextFrame() or Parser.ParseToEnd().
//
// Returns ErrInvalidFileType if the filestamp (first 8 bytes) doesn't match HL2DEMO.
func (p *Parser) ParseHeader() (common.DemoHeader, error) {
	var h common.DemoHeader
	h.Filestamp = p.bitReader.ReadCString(8)
	h.Protocol = p.bitReader.ReadSignedInt(32)
	h.NetworkProtocol = p.bitReader.ReadSignedInt(32)
	h.ServerName = p.bitReader.ReadCString(maxOsPath)
	h.ClientName = p.bitReader.ReadCString(maxOsPath)
	h.MapName = p.bitReader.ReadCString(maxOsPath)
	h.GameDirectory = p.bitReader.ReadCString(maxOsPath)
	h.PlaybackTime = time.Duration(p.bitReader.ReadFloat() * float32(time.Second))
	h.PlaybackTicks = p.bitReader.ReadSignedInt(32)
	h.PlaybackFrames = p.bitReader.ReadSignedInt(32)
	h.SignonLength = p.bitReader.ReadSignedInt(32)

	if h.Filestamp != "HL2DEMO" {
		return h, ErrInvalidFileType
	}

	// Initialize queue if the buffer size wasn't specified, the amount of ticks
	// seems to be a good indicator of how many events we'll get
	if p.msgQueue == nil {
		p.initMsgQueue(h.PlaybackTicks)
	}

	p.header = &h

	return h, nil
}

// ParseToEnd attempts to parse the demo until the end.
// Aborts and returns ErrCancelled if Cancel() is called before the end.
//
// See also: ParseNextFrame() for other possible errors.
func (p *Parser) ParseToEnd() (err error) {
	defer func() {
		// Make sure all the messages of the demo are handled
		p.msgDispatcher.SyncAllQueues()

		// Close msgQueue
		if p.msgQueue != nil {
			close(p.msgQueue)
		}

		if err == nil {
			err = recoverFromUnexpectedEOF(recover())
		}
	}()

	if p.header == nil {
		_, err = p.ParseHeader()
		if err != nil {
			return
		}
	}

	for {
		select {
		case <-p.cancelChan:
			return ErrCancelled

		default:
			if !p.parseFrame() {
				return p.error()
			}
		}

		if err = p.error(); err != nil {
			return
		}
	}
}

func recoverFromUnexpectedEOF(r interface{}) error {
	if r != nil {
		if r == io.ErrUnexpectedEOF || r == io.EOF {
			return ErrUnexpectedEndOfDemo
		}
		panic(r)
	}
	return nil
}

// Cancel aborts ParseToEnd().
// All information that was already read up to this point may still be used (and new events may still be sent out).
func (p *Parser) Cancel() {
	p.cancelChan <- struct{}{}
}

/*
ParseNextFrame attempts to parse the next frame / demo-tick (not ingame tick).

Returns true unless the demo command 'stop' or an error was encountered.

May return ErrUnexpectedEndOfDemo for incomplete / corrupt demos.
May panic if the demo is corrupt in some way.

See also: ParseToEnd() for parsing the complete demo in one go (faster).
*/
func (p *Parser) ParseNextFrame() (moreFrames bool, err error) {
	defer func() {
		// Make sure all the messages of the frame are handled
		p.msgDispatcher.SyncAllQueues()

		// Close msgQueue (only if we are done)
		if p.msgQueue != nil && !moreFrames {
			close(p.msgQueue)
		}

		if err == nil {
			err = recoverFromUnexpectedEOF(recover())
		}
	}()

	if p.header == nil {
		_, err = p.ParseHeader()
		if err != nil {
			return
		}
	}

	moreFrames = p.parseFrame()

	return moreFrames, p.error()
}

// Demo commands as documented at https://developer.valvesoftware.com/wiki/DEM_Format
type demoCommand byte

const (
	dcSignon         demoCommand = 1
	dcPacket         demoCommand = 2
	dcSynctick       demoCommand = 3
	dcConsoleCommand demoCommand = 4
	dcUserCommand    demoCommand = 5
	dcDataTables     demoCommand = 6
	dcStop           demoCommand = 7
	dcCustomData     demoCommand = 8
	dcStringTables   demoCommand = 9
)

func (p *Parser) parseFrame() bool {
	cmd := demoCommand(p.bitReader.ReadSingleByte())

	// Send ingame tick number update
	p.msgQueue <- ingameTickNumber(p.bitReader.ReadSignedInt(32))

	// Skip 'player slot'
	p.bitReader.Skip(8)

	debugDemoCommand(cmd)

	switch cmd {
	case dcSynctick:
		// Ignore

	case dcStop:
		return false

	case dcConsoleCommand:
		// Skip
		p.bitReader.Skip(p.bitReader.ReadSignedInt(32) << 3)

	case dcDataTables:
		p.msgDispatcher.SyncAllQueues()

		p.bitReader.BeginChunk(p.bitReader.ReadSignedInt(32) << 3)
		p.stParser.ParsePacket(p.bitReader)
		p.bitReader.EndChunk()

		debugAllServerClasses(p.ServerClasses())

		p.mapEquipment()
		p.bindEntities()

		p.eventDispatcher.Dispatch(events.DataTablesParsed{})

	case dcStringTables:
		p.msgDispatcher.SyncAllQueues()

		p.parseStringTables()

	case dcUserCommand:
		// Skip
		p.bitReader.Skip(32)
		p.bitReader.Skip(p.bitReader.ReadSignedInt(32) << 3)

	case dcSignon:
		fallthrough
	case dcPacket:
		p.parsePacket()

	case dcCustomData:
		// Might as well panic since we'll be way off if we dont skip the whole thing
		panic("Found CustomData but not handled")

	default:
		panic(fmt.Sprintf("I haven't programmed that pathway yet (command %v unknown)", cmd))
	}

	// Queue up some post processing
	p.msgQueue <- frameParsedToken

	return true
}

var byteSlicePool = sync.Pool{
	New: func() interface{} {
		s := make([]byte, 0, 256)
		return &s
	},
}

func (p *Parser) parsePacket() {
	// Booooring
	// 152 bytes CommandInfo, 4 bytes SeqNrIn, 4 bytes SeqNrOut
	// See at the bottom of the file what the CommandInfo would contain if you are interested.
	p.bitReader.Skip((152 + 4 + 4) << 3)

	// Here we go
	p.bitReader.BeginChunk(p.bitReader.ReadSignedInt(32) << 3)
	for !p.bitReader.ChunkFinished() {
		cmd := int(p.bitReader.ReadVarInt32())
		size := int(p.bitReader.ReadVarInt32())

		p.bitReader.BeginChunk(size << 3)
		var m proto.Message
		switch cmd {
		case int(msg.SVC_Messages_svc_PacketEntities):
			// We could pool CSVCMsg_PacketEntities as they take up A LOT of the allocations
			// but unless we're on a system that's doing a lot of concurrent parsing there isn't really a point
			// as handling packets is a lot slower than creating them and we can't pool until they are handled.
			m = new(msg.CSVCMsg_PacketEntities)

		case int(msg.SVC_Messages_svc_GameEventList):
			m = new(msg.CSVCMsg_GameEventList)

		case int(msg.SVC_Messages_svc_GameEvent):
			m = new(msg.CSVCMsg_GameEvent)

		case int(msg.SVC_Messages_svc_CreateStringTable):
			m = new(msg.CSVCMsg_CreateStringTable)

		case int(msg.SVC_Messages_svc_UpdateStringTable):
			m = new(msg.CSVCMsg_UpdateStringTable)

		case int(msg.SVC_Messages_svc_UserMessage):
			m = new(msg.CSVCMsg_UserMessage)

		default:
			var name string
			if cmd < 8 || cmd >= 100 {
				name = msg.NET_Messages_name[int32(cmd)]
			} else {
				name = msg.SVC_Messages_name[int32(cmd)]
			}

			debugUnhandledMessage(cmd, name)

			if name != "" {
				// Handle additional net-messages as defined by the user
				creator := p.additionalNetMessageCreators[cmd]
				if creator != nil {
					m = creator()
					break
				}
			} else {
				// Send a warning if the command is unknown
				// This might mean our proto files are out of date
				p.eventDispatcher.Dispatch(events.ParserWarn{Message: fmt.Sprintf("Unknown message command %q", cmd)})
			}

			// On to the next one
			p.bitReader.EndChunk()
			continue
		}

		b := byteSlicePool.Get().(*[]byte)
		p.bitReader.ReadBytesInto(b, size)

		if proto.Unmarshal(*b, m) != nil {
			// TODO: Don't crash here, happens with demos that work in gotv
			panic(fmt.Sprintf("Failed to unmarshal cmd %d", cmd))
		}
		p.msgQueue <- m

		// Reset length to 0 and pool
		*b = (*b)[:0]
		byteSlicePool.Put(b)

		p.bitReader.EndChunk()
	}
	p.bitReader.EndChunk()
}

type frameParsedTokenType struct{}

var frameParsedToken = new(frameParsedTokenType)

func (p *Parser) handleFrameParsed(*frameParsedTokenType) {
	// PlayerFlashed events need to be dispatched at the end of the tick
	// because Player.FlashDuration is updated after the game-events are parsed.
	for _, e := range p.delayedEvents {
		p.eventDispatcher.Dispatch(e)
	}
	p.delayedEvents = p.delayedEvents[:0]

	p.currentFrame++
	p.eventDispatcher.Dispatch(events.TickDone{})
	p.eventDispatcher.Dispatch(events.FrameDone{})
}

/*
Format of 'CommandInfos' - I honestly have no clue what they are good for.
This data is skipped in Parser.parsePacket().
If you find a use for this please let me know!

Here is all i know:

CommandInfo [152 bytes]
- [2]Split

Split [76 bytes]
- flags [4 bytes]
- viewOrigin [12 bytes]
- viewAngles [12 bytes]
- localViewAngles [12 bytes]
- viewOrigin2 [12 bytes]
- viewAngles2 [12 bytes]
- localViewAngles2 [12 bytes]

Origin [12 bytes]
- X [4 bytes]
- Y [4 bytes]
- Z [4 bytes]

Angle [12 bytes]
- X [4 bytes]
- Y [4 bytes]
- Z [4 bytes]

They are parsed in the following order:
split1.flags
split1.viewOrigin.x
split1.viewOrigin.y
split1.viewOrigin.z
split1.viewAngles.x
split1.viewAngles.y
split1.viewAngles.z
split1.localViewAngles.x
split1.localViewAngles.y
split1.localViewAngles.z
split1.viewOrigin2...
split1.viewAngles2...
split1.localViewAngles2...
split2.flags
...

Or just check this file's history for an example on how to parse them
*/
<commit_msg>parsing: refactored net-message creation<commit_after>package demoinfocs

import (
	"errors"
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/gogo/protobuf/proto"

	"github.com/markus-wa/demoinfocs-golang/common"
	"github.com/markus-wa/demoinfocs-golang/events"
	"github.com/markus-wa/demoinfocs-golang/msg"
)

const maxOsPath = 260

const (
	playerWeaponPrefix    = "m_hMyWeapons."
	playerWeaponPrePrefix = "bcc_nonlocaldata."
	gameRulesPrefix       = "cs_gamerules_data"
)

// Parsing errors
var (
	// ErrCancelled signals that parsing was cancelled via Parser.Cancel()
	ErrCancelled = errors.New("parsing was cancelled before it finished (ErrCancelled)")

	// ErrUnexpectedEndOfDemo signals that the demo is incomplete / corrupt -
	// these demos may still be useful, check how far the parser got.
	ErrUnexpectedEndOfDemo = errors.New("demo stream ended unexpectedly (ErrUnexpectedEndOfDemo)")

	// ErrInvalidFileType signals that the input isn't a valid CS:GO demo.
	ErrInvalidFileType = errors.New("invalid File-Type; expecting HL2DEMO in the first 8 bytes (ErrInvalidFileType)")
)

// ParseHeader attempts to parse the header of the demo and returns it.
// If not done manually this will be called by Parser.ParseNextFrame() or Parser.ParseToEnd().
//
// Returns ErrInvalidFileType if the filestamp (first 8 bytes) doesn't match HL2DEMO.
func (p *Parser) ParseHeader() (common.DemoHeader, error) {
	var h common.DemoHeader
	h.Filestamp = p.bitReader.ReadCString(8)
	h.Protocol = p.bitReader.ReadSignedInt(32)
	h.NetworkProtocol = p.bitReader.ReadSignedInt(32)
	h.ServerName = p.bitReader.ReadCString(maxOsPath)
	h.ClientName = p.bitReader.ReadCString(maxOsPath)
	h.MapName = p.bitReader.ReadCString(maxOsPath)
	h.GameDirectory = p.bitReader.ReadCString(maxOsPath)
	h.PlaybackTime = time.Duration(p.bitReader.ReadFloat() * float32(time.Second))
	h.PlaybackTicks = p.bitReader.ReadSignedInt(32)
	h.PlaybackFrames = p.bitReader.ReadSignedInt(32)
	h.SignonLength = p.bitReader.ReadSignedInt(32)

	if h.Filestamp != "HL2DEMO" {
		return h, ErrInvalidFileType
	}

	// Initialize queue if the buffer size wasn't specified, the amount of ticks
	// seems to be a good indicator of how many events we'll get
	if p.msgQueue == nil {
		p.initMsgQueue(h.PlaybackTicks)
	}

	p.header = &h

	return h, nil
}

// ParseToEnd attempts to parse the demo until the end.
// Aborts and returns ErrCancelled if Cancel() is called before the end.
//
// See also: ParseNextFrame() for other possible errors.
func (p *Parser) ParseToEnd() (err error) {
	defer func() {
		// Make sure all the messages of the demo are handled
		p.msgDispatcher.SyncAllQueues()

		// Close msgQueue
		if p.msgQueue != nil {
			close(p.msgQueue)
		}

		if err == nil {
			err = recoverFromUnexpectedEOF(recover())
		}
	}()

	if p.header == nil {
		_, err = p.ParseHeader()
		if err != nil {
			return
		}
	}

	for {
		select {
		case <-p.cancelChan:
			return ErrCancelled

		default:
			if !p.parseFrame() {
				return p.error()
			}
		}

		if err = p.error(); err != nil {
			return
		}
	}
}

func recoverFromUnexpectedEOF(r interface{}) error {
	if r != nil {
		if r == io.ErrUnexpectedEOF || r == io.EOF {
			return ErrUnexpectedEndOfDemo
		}
		panic(r)
	}
	return nil
}

// Cancel aborts ParseToEnd().
// All information that was already read up to this point may still be used (and new events may still be sent out).
func (p *Parser) Cancel() {
	p.cancelChan <- struct{}{}
}

/*
ParseNextFrame attempts to parse the next frame / demo-tick (not ingame tick).

Returns true unless the demo command 'stop' or an error was encountered.

May return ErrUnexpectedEndOfDemo for incomplete / corrupt demos.
May panic if the demo is corrupt in some way.

See also: ParseToEnd() for parsing the complete demo in one go (faster).
*/
func (p *Parser) ParseNextFrame() (moreFrames bool, err error) {
	defer func() {
		// Make sure all the messages of the frame are handled
		p.msgDispatcher.SyncAllQueues()

		// Close msgQueue (only if we are done)
		if p.msgQueue != nil && !moreFrames {
			close(p.msgQueue)
		}

		if err == nil {
			err = recoverFromUnexpectedEOF(recover())
		}
	}()

	if p.header == nil {
		_, err = p.ParseHeader()
		if err != nil {
			return
		}
	}

	moreFrames = p.parseFrame()

	return moreFrames, p.error()
}

// Demo commands as documented at https://developer.valvesoftware.com/wiki/DEM_Format
type demoCommand byte

const (
	dcSignon         demoCommand = 1
	dcPacket         demoCommand = 2
	dcSynctick       demoCommand = 3
	dcConsoleCommand demoCommand = 4
	dcUserCommand    demoCommand = 5
	dcDataTables     demoCommand = 6
	dcStop           demoCommand = 7
	dcCustomData     demoCommand = 8
	dcStringTables   demoCommand = 9
)

func (p *Parser) parseFrame() bool {
	cmd := demoCommand(p.bitReader.ReadSingleByte())

	// Send ingame tick number update
	p.msgQueue <- ingameTickNumber(p.bitReader.ReadSignedInt(32))

	// Skip 'player slot'
	p.bitReader.Skip(8)

	debugDemoCommand(cmd)

	switch cmd {
	case dcSynctick:
		// Ignore

	case dcStop:
		return false

	case dcConsoleCommand:
		// Skip
		p.bitReader.Skip(p.bitReader.ReadSignedInt(32) << 3)

	case dcDataTables:
		p.msgDispatcher.SyncAllQueues()

		p.bitReader.BeginChunk(p.bitReader.ReadSignedInt(32) << 3)
		p.stParser.ParsePacket(p.bitReader)
		p.bitReader.EndChunk()

		debugAllServerClasses(p.ServerClasses())

		p.mapEquipment()
		p.bindEntities()

		p.eventDispatcher.Dispatch(events.DataTablesParsed{})

	case dcStringTables:
		p.msgDispatcher.SyncAllQueues()

		p.parseStringTables()

	case dcUserCommand:
		// Skip
		p.bitReader.Skip(32)
		p.bitReader.Skip(p.bitReader.ReadSignedInt(32) << 3)

	case dcSignon:
		fallthrough
	case dcPacket:
		p.parsePacket()

	case dcCustomData:
		// Might as well panic since we'll be way off if we dont skip the whole thing
		panic("Found CustomData but not handled")

	default:
		panic(fmt.Sprintf("I haven't programmed that pathway yet (command %v unknown)", cmd))
	}

	// Queue up some post processing
	p.msgQueue <- frameParsedToken

	return true
}

var byteSlicePool = sync.Pool{
	New: func() interface{} {
		s := make([]byte, 0, 256)
		return &s
	},
}

var defaultNetMessageCreators = map[int]NetMessageCreator{
	// We could pool CSVCMsg_PacketEntities as they take up A LOT of the allocations
	// but unless we're on a system that's doing a lot of concurrent parsing there isn't really a point
	// as handling packets is a lot slower than creating them and we can't pool until they are handled.
	int(msg.SVC_Messages_svc_PacketEntities):    func() proto.Message { return new(msg.CSVCMsg_PacketEntities) },
	int(msg.SVC_Messages_svc_GameEventList):     func() proto.Message { return new(msg.CSVCMsg_GameEventList) },
	int(msg.SVC_Messages_svc_GameEvent):         func() proto.Message { return new(msg.CSVCMsg_GameEvent) },
	int(msg.SVC_Messages_svc_CreateStringTable): func() proto.Message { return new(msg.CSVCMsg_CreateStringTable) },
	int(msg.SVC_Messages_svc_UpdateStringTable): func() proto.Message { return new(msg.CSVCMsg_UpdateStringTable) },
	int(msg.SVC_Messages_svc_UserMessage):       func() proto.Message { return new(msg.CSVCMsg_UserMessage) },
}

func (p *Parser) parsePacket() {
	// Booooring
	// 152 bytes CommandInfo, 4 bytes SeqNrIn, 4 bytes SeqNrOut
	// See at the bottom of the file what the CommandInfo would contain if you are interested.
	p.bitReader.Skip((152 + 4 + 4) << 3)

	// Here we go
	p.bitReader.BeginChunk(p.bitReader.ReadSignedInt(32) << 3)

	for !p.bitReader.ChunkFinished() {
		cmd := int(p.bitReader.ReadVarInt32())
		size := int(p.bitReader.ReadVarInt32())

		p.bitReader.BeginChunk(size << 3)

		msgCreator := defaultNetMessageCreators[cmd]

		if msgCreator == nil {
			var msgName string
			if cmd < 8 || cmd >= 100 {
				msgName = msg.NET_Messages_name[int32(cmd)]
			} else {
				msgName = msg.SVC_Messages_name[int32(cmd)]
			}

			if msgName == "" {
				// Send a warning if the command is unknown
				// This might mean our proto files are out of date
				p.eventDispatcher.Dispatch(events.ParserWarn{Message: fmt.Sprintf("Unknown message command %q", cmd)})
			}

			// Handle additional net-messages as defined by the user
			msgCreator = p.additionalNetMessageCreators[cmd]
			if msgCreator == nil {
				debugUnhandledMessage(cmd, msgName)

				// On to the next one
				p.bitReader.EndChunk()
				continue
			}
		}

		b := byteSlicePool.Get().(*[]byte)
		p.bitReader.ReadBytesInto(b, size)

		m := msgCreator()
		if proto.Unmarshal(*b, m) != nil {
			// TODO: Don't crash here, happens with demos that work in gotv
			panic(fmt.Sprintf("Failed to unmarshal cmd %d", cmd))
		}
		p.msgQueue <- m

		// Reset length to 0 and pool
		*b = (*b)[:0]
		byteSlicePool.Put(b)

		p.bitReader.EndChunk()
	}

	p.bitReader.EndChunk()
}

type frameParsedTokenType struct{}

var frameParsedToken = new(frameParsedTokenType)

func (p *Parser) handleFrameParsed(*frameParsedTokenType) {
	// PlayerFlashed events need to be dispatched at the end of the tick
	// because Player.FlashDuration is updated after the game-events are parsed.
	for _, e := range p.delayedEvents {
		p.eventDispatcher.Dispatch(e)
	}
	p.delayedEvents = p.delayedEvents[:0]

	p.currentFrame++
	p.eventDispatcher.Dispatch(events.TickDone{})
	p.eventDispatcher.Dispatch(events.FrameDone{})
}

/*
Format of 'CommandInfos' - I honestly have no clue what they are good for.
This data is skipped in Parser.parsePacket().
If you find a use for this please let me know!

Here is all i know:

CommandInfo [152 bytes]
- [2]Split

Split [76 bytes]
- flags [4 bytes]
- viewOrigin [12 bytes]
- viewAngles [12 bytes]
- localViewAngles [12 bytes]
- viewOrigin2 [12 bytes]
- viewAngles2 [12 bytes]
- localViewAngles2 [12 bytes]

Origin [12 bytes]
- X [4 bytes]
- Y [4 bytes]
- Z [4 bytes]

Angle [12 bytes]
- X [4 bytes]
- Y [4 bytes]
- Z [4 bytes]

They are parsed in the following order:
split1.flags
split1.viewOrigin.x
split1.viewOrigin.y
split1.viewOrigin.z
split1.viewAngles.x
split1.viewAngles.y
split1.viewAngles.z
split1.localViewAngles.x
split1.localViewAngles.y
split1.localViewAngles.z
split1.viewOrigin2...
split1.viewAngles2...
split1.localViewAngles2...
split2.flags
...

Or just check this file's history for an example on how to parse them
*/
<|endoftext|>


<commit_before>package remote

import (
	"github.com/mitchellh/mapstructure"
	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/gruntwork-io/terragrunt/options"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/aws"
	"fmt"
	"github.com/gruntwork-io/terragrunt/util"
	"github.com/gruntwork-io/terragrunt/shell"
	"github.com/gruntwork-io/terragrunt/aws_helper"
	"time"
)

// A representation of the configuration options available for S3 remote state
type RemoteStateConfigS3 struct {
	Encrypt string
	Bucket  string
	Key     string
	Region  string
}

const MAX_RETRIES_WAITING_FOR_S3_BUCKET = 12
const SLEEP_BETWEEN_RETRIES_WAITING_FOR_S3_BUCKET = 5 * time.Second

// Initialize the remote state S3 bucket specified in the given config. This function will validate the config
// parameters, create the S3 bucket if it doesn't already exist, and check that versioning is enabled.
func InitializeRemoteStateS3(config map[string]string, terragruntOptions *options.TerragruntOptions) error {
	s3Config, err := parseS3Config(config)
	if err != nil {
		return err
	}

	if err := validateS3Config(s3Config, terragruntOptions); err != nil {
		return err
	}

	s3Client, err := CreateS3Client(s3Config.Region)
	if err != nil {
		return err
	}

	if err := createS3BucketIfNecessary(s3Client, s3Config, terragruntOptions); err != nil {
		return err
	}

	if err := checkIfVersioningEnabled(s3Client, s3Config, terragruntOptions); err != nil {
		return err
	}

	return nil
}

// Parse the given map into an S3 config
func parseS3Config(config map[string]string) (*RemoteStateConfigS3, error) {
	var s3Config RemoteStateConfigS3
	if err := mapstructure.Decode(config, &s3Config); err != nil {
		return nil, errors.WithStackTrace(err)
	}

	return &s3Config, nil
}

// Validate all the parameters of the given S3 remote state configuration
func validateS3Config(config *RemoteStateConfigS3, terragruntOptions *options.TerragruntOptions) error {
	if config.Region == "" {
		return errors.WithStackTrace(MissingRequiredS3RemoteStateConfig("region"))
	}

	if config.Bucket == "" {
		return errors.WithStackTrace(MissingRequiredS3RemoteStateConfig("bucket"))
	}

	if config.Key == "" {
		return errors.WithStackTrace(MissingRequiredS3RemoteStateConfig("key"))
	}

	if config.Encrypt != "true" {
		util.Logger.Printf("WARNING: encryption is not enabled on the S3 remote state bucket %s. Terraform state files may contain secrets, so we STRONGLY recommend enablying encryption!", config.Bucket)
	}

	return nil
}

// If the bucket specified in the given config doesn't already exist, prompt the user to create it, and if the user
// confirms, create the bucket and enable versioning for it.
func createS3BucketIfNecessary(s3Client *s3.S3, config *RemoteStateConfigS3, terragruntOptions *options.TerragruntOptions) error {
	if !DoesS3BucketExist(s3Client, config) {
		prompt := fmt.Sprintf("Remote state S3 bucket %s does not exist or you don't have permissions to access it. Would you Terragrunt to create it?", config.Bucket)
		shouldCreateBucket, err := shell.PromptUserForYesNo(prompt, terragruntOptions)
		if err != nil {
			return err
		}

		if shouldCreateBucket {
			return CreateS3BucketWithVersioning(s3Client, config)
		}
	}

	return nil
}

// Check if versioning is enabled for the S3 bucket specified in the given config and warn the user if it is not
func checkIfVersioningEnabled(s3Client *s3.S3, config *RemoteStateConfigS3, terragruntOptions *options.TerragruntOptions) error {
	out, err := s3Client.GetBucketVersioning(&s3.GetBucketVersioningInput{Bucket: aws.String(config.Bucket)})
	if err != nil {
		return errors.WithStackTrace(err)
	}

	if *out.Status != s3.BucketVersioningStatusEnabled {
		util.Logger.Printf("WARNING: Versioning is not enabled for the remote state S3 bucket %s. We recommend enabling versioning so that you can roll back to previous versions of your Terraform state in case of error.", config.Bucket)
	}

	return nil
}

// Create the given S3 bucket and enable versioning for it
func CreateS3BucketWithVersioning(s3Client *s3.S3, config *RemoteStateConfigS3) error {
	if err := CreateS3Bucket(s3Client, config); err != nil {
		return err
	}

	if err := WaitUntilS3BucketExists(s3Client, config); err != nil {
		return err
	}

	if err := EnableVersioningForS3Bucket(s3Client, config); err != nil {
		return err
	}

	return nil
}

// AWS is eventually consistent, so after creating an S3 bucket, this method can be used to wait until the information
// about that S3 bucket has propagated everywhere
func WaitUntilS3BucketExists(s3Client *s3.S3, config *RemoteStateConfigS3) error {
	for retries := 0; retries < MAX_RETRIES_WAITING_FOR_S3_BUCKET; retries++ {
		if DoesS3BucketExist(s3Client, config) {
			util.Logger.Printf("S3 bucket %s created.", config.Bucket)
			return nil
		} else if retries < MAX_RETRIES_WAITING_FOR_S3_BUCKET - 1 {
			util.Logger.Printf("S3 bucket %s has not been created yet. Sleeping for %s and will check again.", config.Bucket, SLEEP_BETWEEN_RETRIES_WAITING_FOR_S3_BUCKET)
			time.Sleep(SLEEP_BETWEEN_RETRIES_WAITING_FOR_S3_BUCKET)
		}
	}

	return errors.WithStackTrace(MaxRetriesWaitingForS3BucketExceeded(config.Bucket))
}

// Create the S3 bucket specified in the given config
func CreateS3Bucket(s3Client *s3.S3, config *RemoteStateConfigS3) error {
	util.Logger.Printf("Creating S3 bucket %s", config.Bucket)
	_, err := s3Client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(config.Bucket)})
	return errors.WithStackTrace(err)
}

// Enable versioning for the S3 bucket specified in the given config
func EnableVersioningForS3Bucket(s3Client *s3.S3, config *RemoteStateConfigS3) error {
	util.Logger.Printf("Enabling versioning on S3 bucket %s", config.Bucket)
	input := s3.PutBucketVersioningInput{
		Bucket: aws.String(config.Bucket),
		VersioningConfiguration: &s3.VersioningConfiguration{Status: aws.String(s3.BucketVersioningStatusEnabled)},
	}
	_, err := s3Client.PutBucketVersioning(&input)
	return errors.WithStackTrace(err)
}

// Returns true if the S3 bucket specified in the given config exists and the current user has the ability to access
// it.
func DoesS3BucketExist(s3Client *s3.S3, config *RemoteStateConfigS3) bool {
	_, err := s3Client.HeadBucket(&s3.HeadBucketInput{Bucket: aws.String(config.Bucket)})
	return err == nil
}

// Create an authenticated client for DynamoDB
func CreateS3Client(awsRegion string) (*s3.S3, error) {
	config, err := aws_helper.CreateAwsConfig(awsRegion)
	if err != nil {
		return nil, err
	}

	return s3.New(session.New(), config), nil
}

// Custom error types

type MissingRequiredS3RemoteStateConfig string
func (configName MissingRequiredS3RemoteStateConfig) Error() string {
	return fmt.Sprintf("Missing required S3 remote state configuration %s", string(configName))
}

type MaxRetriesWaitingForS3BucketExceeded string
func (err MaxRetriesWaitingForS3BucketExceeded) Error() string {
	return fmt.Sprintf("Exceeded max retries (%d) waiting for bucket S3 bucket %s", MAX_RETRIES_WAITING_FOR_S3_BUCKET, string(err))
}<commit_msg>Typo fix.<commit_after>package remote

import (
	"github.com/mitchellh/mapstructure"
	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/gruntwork-io/terragrunt/options"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/aws"
	"fmt"
	"github.com/gruntwork-io/terragrunt/util"
	"github.com/gruntwork-io/terragrunt/shell"
	"github.com/gruntwork-io/terragrunt/aws_helper"
	"time"
)

// A representation of the configuration options available for S3 remote state
type RemoteStateConfigS3 struct {
	Encrypt string
	Bucket  string
	Key     string
	Region  string
}

const MAX_RETRIES_WAITING_FOR_S3_BUCKET = 12
const SLEEP_BETWEEN_RETRIES_WAITING_FOR_S3_BUCKET = 5 * time.Second

// Initialize the remote state S3 bucket specified in the given config. This function will validate the config
// parameters, create the S3 bucket if it doesn't already exist, and check that versioning is enabled.
func InitializeRemoteStateS3(config map[string]string, terragruntOptions *options.TerragruntOptions) error {
	s3Config, err := parseS3Config(config)
	if err != nil {
		return err
	}

	if err := validateS3Config(s3Config, terragruntOptions); err != nil {
		return err
	}

	s3Client, err := CreateS3Client(s3Config.Region)
	if err != nil {
		return err
	}

	if err := createS3BucketIfNecessary(s3Client, s3Config, terragruntOptions); err != nil {
		return err
	}

	if err := checkIfVersioningEnabled(s3Client, s3Config, terragruntOptions); err != nil {
		return err
	}

	return nil
}

// Parse the given map into an S3 config
func parseS3Config(config map[string]string) (*RemoteStateConfigS3, error) {
	var s3Config RemoteStateConfigS3
	if err := mapstructure.Decode(config, &s3Config); err != nil {
		return nil, errors.WithStackTrace(err)
	}

	return &s3Config, nil
}

// Validate all the parameters of the given S3 remote state configuration
func validateS3Config(config *RemoteStateConfigS3, terragruntOptions *options.TerragruntOptions) error {
	if config.Region == "" {
		return errors.WithStackTrace(MissingRequiredS3RemoteStateConfig("region"))
	}

	if config.Bucket == "" {
		return errors.WithStackTrace(MissingRequiredS3RemoteStateConfig("bucket"))
	}

	if config.Key == "" {
		return errors.WithStackTrace(MissingRequiredS3RemoteStateConfig("key"))
	}

	if config.Encrypt != "true" {
		util.Logger.Printf("WARNING: encryption is not enabled on the S3 remote state bucket %s. Terraform state files may contain secrets, so we STRONGLY recommend enablying encryption!", config.Bucket)
	}

	return nil
}

// If the bucket specified in the given config doesn't already exist, prompt the user to create it, and if the user
// confirms, create the bucket and enable versioning for it.
func createS3BucketIfNecessary(s3Client *s3.S3, config *RemoteStateConfigS3, terragruntOptions *options.TerragruntOptions) error {
	if !DoesS3BucketExist(s3Client, config) {
		prompt := fmt.Sprintf("Remote state S3 bucket %s does not exist or you don't have permissions to access it. Would you like Terragrunt to create it?", config.Bucket)
		shouldCreateBucket, err := shell.PromptUserForYesNo(prompt, terragruntOptions)
		if err != nil {
			return err
		}

		if shouldCreateBucket {
			return CreateS3BucketWithVersioning(s3Client, config)
		}
	}

	return nil
}

// Check if versioning is enabled for the S3 bucket specified in the given config and warn the user if it is not
func checkIfVersioningEnabled(s3Client *s3.S3, config *RemoteStateConfigS3, terragruntOptions *options.TerragruntOptions) error {
	out, err := s3Client.GetBucketVersioning(&s3.GetBucketVersioningInput{Bucket: aws.String(config.Bucket)})
	if err != nil {
		return errors.WithStackTrace(err)
	}

	if *out.Status != s3.BucketVersioningStatusEnabled {
		util.Logger.Printf("WARNING: Versioning is not enabled for the remote state S3 bucket %s. We recommend enabling versioning so that you can roll back to previous versions of your Terraform state in case of error.", config.Bucket)
	}

	return nil
}

// Create the given S3 bucket and enable versioning for it
func CreateS3BucketWithVersioning(s3Client *s3.S3, config *RemoteStateConfigS3) error {
	if err := CreateS3Bucket(s3Client, config); err != nil {
		return err
	}

	if err := WaitUntilS3BucketExists(s3Client, config); err != nil {
		return err
	}

	if err := EnableVersioningForS3Bucket(s3Client, config); err != nil {
		return err
	}

	return nil
}

// AWS is eventually consistent, so after creating an S3 bucket, this method can be used to wait until the information
// about that S3 bucket has propagated everywhere
func WaitUntilS3BucketExists(s3Client *s3.S3, config *RemoteStateConfigS3) error {
	for retries := 0; retries < MAX_RETRIES_WAITING_FOR_S3_BUCKET; retries++ {
		if DoesS3BucketExist(s3Client, config) {
			util.Logger.Printf("S3 bucket %s created.", config.Bucket)
			return nil
		} else if retries < MAX_RETRIES_WAITING_FOR_S3_BUCKET - 1 {
			util.Logger.Printf("S3 bucket %s has not been created yet. Sleeping for %s and will check again.", config.Bucket, SLEEP_BETWEEN_RETRIES_WAITING_FOR_S3_BUCKET)
			time.Sleep(SLEEP_BETWEEN_RETRIES_WAITING_FOR_S3_BUCKET)
		}
	}

	return errors.WithStackTrace(MaxRetriesWaitingForS3BucketExceeded(config.Bucket))
}

// Create the S3 bucket specified in the given config
func CreateS3Bucket(s3Client *s3.S3, config *RemoteStateConfigS3) error {
	util.Logger.Printf("Creating S3 bucket %s", config.Bucket)
	_, err := s3Client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(config.Bucket)})
	return errors.WithStackTrace(err)
}

// Enable versioning for the S3 bucket specified in the given config
func EnableVersioningForS3Bucket(s3Client *s3.S3, config *RemoteStateConfigS3) error {
	util.Logger.Printf("Enabling versioning on S3 bucket %s", config.Bucket)
	input := s3.PutBucketVersioningInput{
		Bucket: aws.String(config.Bucket),
		VersioningConfiguration: &s3.VersioningConfiguration{Status: aws.String(s3.BucketVersioningStatusEnabled)},
	}
	_, err := s3Client.PutBucketVersioning(&input)
	return errors.WithStackTrace(err)
}

// Returns true if the S3 bucket specified in the given config exists and the current user has the ability to access
// it.
func DoesS3BucketExist(s3Client *s3.S3, config *RemoteStateConfigS3) bool {
	_, err := s3Client.HeadBucket(&s3.HeadBucketInput{Bucket: aws.String(config.Bucket)})
	return err == nil
}

// Create an authenticated client for DynamoDB
func CreateS3Client(awsRegion string) (*s3.S3, error) {
	config, err := aws_helper.CreateAwsConfig(awsRegion)
	if err != nil {
		return nil, err
	}

	return s3.New(session.New(), config), nil
}

// Custom error types

type MissingRequiredS3RemoteStateConfig string
func (configName MissingRequiredS3RemoteStateConfig) Error() string {
	return fmt.Sprintf("Missing required S3 remote state configuration %s", string(configName))
}

type MaxRetriesWaitingForS3BucketExceeded string
func (err MaxRetriesWaitingForS3BucketExceeded) Error() string {
	return fmt.Sprintf("Exceeded max retries (%d) waiting for bucket S3 bucket %s", MAX_RETRIES_WAITING_FOR_S3_BUCKET, string(err))
}
<|endoftext|>


<commit_before>package replication

import (
	"bytes"
	"compress/gzip"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"
	"time"

	osm "github.com/paulmach/go.osm"
	"github.com/paulmach/go.osm/osmxml"
)

// ChangesetSeqNum indicates the sequence of the changeset replication found here:
// http://planet.osm.org/replication/changesets/
type ChangesetSeqNum uint64

// CurrentChangesetState returns the current state of the changeset replication.
// Delegates to the DefaultDatasource and uses its http.Client to make the request.
func CurrentChangesetState(ctx context.Context) (ChangesetSeqNum, State, error) {
	return DefaultDatasource.CurrentChangesetState(ctx)
}

// CurrentChangesetState returns the current state of the changeset replication.
func (ds *Datasource) CurrentChangesetState(ctx context.Context) (ChangesetSeqNum, State, error) {
	url := ds.baseURL() + "/replication/changesets/state.yaml"
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return 0, State{}, err
	}

	resp, err := ds.client().Do(req.WithContext(ctx))
	if err != nil {
		return 0, State{}, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return 0, State{}, &UnexpectedStatusCodeError{
			Code: resp.StatusCode,
			URL:  url,
		}
	}

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return 0, State{}, err
	}

	s, err := decodeChangesetState(data)
	return ChangesetSeqNum(s.SeqNum), s, err
}

func decodeChangesetState(data []byte) (State, error) {
	// example
	// ---
	// last_run: 2016-07-02 22:46:01.422137422 Z
	// sequence: 1912325

	lines := bytes.Split(data, []byte("\n"))

	parts := bytes.Split(lines[1], []byte(":"))
	timeString := string(bytes.TrimSpace(bytes.Join(parts[1:], []byte(":"))))
	t, err := time.Parse(
		"2006-01-02 15:04:05.999999999 Z",
		timeString)
	if err != nil {
		return State{}, err
	}

	parts = bytes.Split(lines[2], []byte(":"))
	n, err := strconv.ParseUint(string(bytes.TrimSpace(parts[1])), 10, 64)
	if err != nil {
		return State{}, err
	}

	return State{
		SeqNum:    n,
		Timestamp: t,
	}, nil
}

// Changesets returns the complete list of changesets in for the given replication sequence.
// Delegates to the DefaultDatasource and uses its http.Client to make the request.
func Changesets(ctx context.Context, n ChangesetSeqNum) (osm.Changesets, error) {
	return DefaultDatasource.Changesets(ctx, n)
}

// Changesets returns the complete list of changesets in for the given replication sequence.
func (ds *Datasource) Changesets(ctx context.Context, n ChangesetSeqNum) (osm.Changesets, error) {
	r, err := ds.changesetReader(ctx, n)
	if err != nil {
		return nil, err
	}
	defer r.Close()

	gzReader, err := gzip.NewReader(r)
	if err != nil {
		return nil, err
	}
	defer gzReader.Close()

	var changesets []*osm.Changeset
	scanner := osmxml.New(ctx, gzReader)
	for scanner.Scan() {
		e := scanner.Element()
		changesets = append(changesets, e.(*osm.Changeset))
	}

	return changesets, scanner.Err()
}

// changesetReader will return a ReadCloser with the data from the changeset.
// It will be gzip compressed, so the caller must decompress.
// It is the caller's responsibility to call Close on the Reader when done.
func (ds *Datasource) changesetReader(ctx context.Context, n ChangesetSeqNum) (io.ReadCloser, error) {
	url := ds.changesetURL(n)
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	resp, err := ds.client().Do(req.WithContext(ctx))
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != 200 {
		resp.Body.Close()
		return nil, &UnexpectedStatusCodeError{
			Code: resp.StatusCode,
			URL:  url,
		}
	}

	return resp.Body, nil
}

func (ds *Datasource) changesetURL(n ChangesetSeqNum) string {
	return fmt.Sprintf("%s/replication/changesets/%03d/%03d/%03d.osm.gz",
		ds.baseURL(),
		n/1000000,
		(n%1000000)/1000,
		(n % 1000))
}
<commit_msg>replication: the time format for changeset state has change :(<commit_after>package replication

import (
	"bytes"
	"compress/gzip"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"
	"time"

	osm "github.com/paulmach/go.osm"
	"github.com/paulmach/go.osm/osmxml"
)

// ChangesetSeqNum indicates the sequence of the changeset replication found here:
// http://planet.osm.org/replication/changesets/
type ChangesetSeqNum uint64

// CurrentChangesetState returns the current state of the changeset replication.
// Delegates to the DefaultDatasource and uses its http.Client to make the request.
func CurrentChangesetState(ctx context.Context) (ChangesetSeqNum, State, error) {
	return DefaultDatasource.CurrentChangesetState(ctx)
}

// CurrentChangesetState returns the current state of the changeset replication.
func (ds *Datasource) CurrentChangesetState(ctx context.Context) (ChangesetSeqNum, State, error) {
	url := ds.baseURL() + "/replication/changesets/state.yaml"
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return 0, State{}, err
	}

	resp, err := ds.client().Do(req.WithContext(ctx))
	if err != nil {
		return 0, State{}, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return 0, State{}, &UnexpectedStatusCodeError{
			Code: resp.StatusCode,
			URL:  url,
		}
	}

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return 0, State{}, err
	}

	s, err := decodeChangesetState(data)
	return ChangesetSeqNum(s.SeqNum), s, err
}

var timeFormats = []string{
	"2006-01-02 15:04:05.999999999 Z",
	"2006-01-02 15:04:05.999999999 +00:00",
}

func decodeChangesetState(data []byte) (State, error) {
	// example
	// ---
	// last_run: 2016-07-02 22:46:01.422137422 +00:00  (or Z)
	// sequence: 1912325

	lines := bytes.Split(data, []byte("\n"))

	parts := bytes.Split(lines[1], []byte(":"))
	timeString := string(bytes.TrimSpace(bytes.Join(parts[1:], []byte(":"))))

	var (
		t   time.Time
		err error
	)
	for _, format := range timeFormats {
		t, err = time.Parse(format, timeString)
		if err == nil {
			break
		}
	}

	if err != nil {
		return State{}, err
	}

	parts = bytes.Split(lines[2], []byte(":"))
	n, err := strconv.ParseUint(string(bytes.TrimSpace(parts[1])), 10, 64)
	if err != nil {
		return State{}, err
	}

	return State{
		SeqNum:    n,
		Timestamp: t,
	}, nil
}

// Changesets returns the complete list of changesets in for the given replication sequence.
// Delegates to the DefaultDatasource and uses its http.Client to make the request.
func Changesets(ctx context.Context, n ChangesetSeqNum) (osm.Changesets, error) {
	return DefaultDatasource.Changesets(ctx, n)
}

// Changesets returns the complete list of changesets in for the given replication sequence.
func (ds *Datasource) Changesets(ctx context.Context, n ChangesetSeqNum) (osm.Changesets, error) {
	r, err := ds.changesetReader(ctx, n)
	if err != nil {
		return nil, err
	}
	defer r.Close()

	gzReader, err := gzip.NewReader(r)
	if err != nil {
		return nil, err
	}
	defer gzReader.Close()

	var changesets []*osm.Changeset
	scanner := osmxml.New(ctx, gzReader)
	for scanner.Scan() {
		e := scanner.Element()
		changesets = append(changesets, e.(*osm.Changeset))
	}

	return changesets, scanner.Err()
}

// changesetReader will return a ReadCloser with the data from the changeset.
// It will be gzip compressed, so the caller must decompress.
// It is the caller's responsibility to call Close on the Reader when done.
func (ds *Datasource) changesetReader(ctx context.Context, n ChangesetSeqNum) (io.ReadCloser, error) {
	url := ds.changesetURL(n)
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	resp, err := ds.client().Do(req.WithContext(ctx))
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != 200 {
		resp.Body.Close()
		return nil, &UnexpectedStatusCodeError{
			Code: resp.StatusCode,
			URL:  url,
		}
	}

	return resp.Body, nil
}

func (ds *Datasource) changesetURL(n ChangesetSeqNum) string {
	return fmt.Sprintf("%s/replication/changesets/%03d/%03d/%03d.osm.gz",
		ds.baseURL(),
		n/1000000,
		(n%1000000)/1000,
		(n % 1000))
}
<|endoftext|>


<commit_before>package osutil

import (
	"io/ioutil"
	"os"
)

// checks whether a given filepath exists, file or dir

func Exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return true, err
}

func EmptyAll(path string) error {
	aEntries, err := ioutil.ReadDir(path)
	if err != nil {
		return err
	}
	for _, f := range aEntries {
		if f.Name() == "." || f.Name() == ".." {
			continue
		}
		err = os.Remove(path + "/" + f.Name())
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>add FileModAge()<commit_after>package osutil

import (
	"io/ioutil"
	"os"
	"time"
)

// checks whether a given filepath exists, file or dir

func Exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return true, err
}

func EmptyAll(path string) error {
	aEntries, err := ioutil.ReadDir(path)
	if err != nil {
		return err
	}
	for _, f := range aEntries {
		if f.Name() == "." || f.Name() == ".." {
			continue
		}
		err = os.Remove(path + "/" + f.Name())
		if err != nil {
			return err
		}
	}
	return nil
}

func FileModAge(filepath string) (time.Duration, error) {
	stat, err := os.Stat(filepath)
	if err != nil {
		dur0, _ := time.ParseDuration("0s")
		return dur0, err
	}
	mod := stat.ModTime()
	dt := time.Now()
	dur := dt.Sub(mod)
	return dur, nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math

// Atan returns the arc tangent of y/x, using
// the signs of the two to determine the quadrant
// of the return value.
func Atan2(x, y float64) float64 {
	// Determine the quadrant and call atan.
	if x+y == x {
		if x >= 0 {
			return Pi / 2
		}
		return -Pi / 2;
	}
	q := Atan(x / y);
	if y < 0 {
		if q <= 0 {
			return q + Pi
		}
		return q - Pi;
	}
	return q;
}
<commit_msg>math: fix argument names in Atan2 (error introduced converting from arg1, arg2)<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math

// Atan2 returns the arc tangent of y/x, using
// the signs of the two to determine the quadrant
// of the return value.
func Atan2(y, x float64) float64 {
	// Determine the quadrant and call atan.
	if y+x == y {
		if y >= 0 {
			return Pi / 2
		}
		return -Pi / 2;
	}
	q := Atan(y / x);
	if x < 0 {
		if q <= 0 {
			return q + Pi
		}
		return q - Pi;
	}
	return q;
}
<|endoftext|>


<commit_before>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package pat

import (
	"net/http"
	"net/url"
	"path"
	"strings"

	"code.google.com/p/gorilla/mux"
)

// New returns a new router.
func New() *Router {
	return &Router{}
}

// Router is a request router that implements a pat-like API.
//
// pat docs: http://gopkgdoc.appspot.com/pkg/github.com/bmizerany/pat
type Router struct {
	mux.Router
}

// Add registers a pattern with a handler for the given request method.
func (r *Router) Add(meth, pat string, h http.Handler) *mux.Route {
	return r.NewRoute().PathPrefix(pat).Handler(h).Methods(meth)
}

// Delete registers a pattern with a handler for DELETE requests.
func (r *Router) Delete(pat string, h http.HandlerFunc) *mux.Route {
	return r.Add("DELETE", pat, h)
}

// Get registers a pattern with a handler for GET requests.
func (r *Router) Get(pat string, h http.HandlerFunc) *mux.Route {
	return r.Add("GET", pat, h)
}

// Post registers a pattern with a handler for POST requests.
func (r *Router) Post(pat string, h http.HandlerFunc) *mux.Route {
	return r.Add("POST", pat, h)
}

// Put registers a pattern with a handler for PUT requests.
func (r *Router) Put(pat string, h http.HandlerFunc) *mux.Route {
	return r.Add("PUT", pat, h)
}

// ServeHTTP dispatches the handler registered in the matched route.
func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// Clean path to canonical form and redirect.
	if p := cleanPath(req.URL.Path); p != req.URL.Path {
		w.Header().Set("Location", p)
		w.WriteHeader(http.StatusMovedPermanently)
		return
	}
	var match mux.RouteMatch
	var handler http.Handler
	if matched := r.Match(req, &match); matched {
		handler = match.Handler
		registerVars(req, match.Vars)
	}
	if handler == nil {
		if r.NotFoundHandler == nil {
			r.NotFoundHandler = http.NotFoundHandler()
		}
		handler = r.NotFoundHandler
	}
	handler.ServeHTTP(w, req)
}

// registerVars adds the matched route variables to the URL query.
func registerVars(r *http.Request, vars map[string]string) {
	parts, i := make([]string, len(vars)), 0
	for key, value := range vars {
		parts[i] = url.QueryEscape(":"+key) + "=" + url.QueryEscape(value)
		i++
	}
	r.URL.RawQuery = strings.Join(parts, "&") + "&" + r.URL.RawQuery
}

// cleanPath returns the canonical path for p, eliminating . and .. elements.
// Borrowed from the net/http package.
func cleanPath(p string) string {
	if p == "" {
		return "/"
	}
	if p[0] != '/' {
		p = "/" + p
	}
	np := path.Clean(p)
	// path.Clean removes trailing slash except for root;
	// put the trailing slash back if necessary.
	if p[len(p)-1] == '/' && np != "/" {
		np += "/"
	}
	return np
}
<commit_msg>pat: minor tweak in registerVars().<commit_after>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package pat

import (
	"net/http"
	"net/url"
	"path"
	"strings"

	"code.google.com/p/gorilla/mux"
)

// New returns a new router.
func New() *Router {
	return &Router{}
}

// Router is a request router that implements a pat-like API.
//
// pat docs: http://gopkgdoc.appspot.com/pkg/github.com/bmizerany/pat
type Router struct {
	mux.Router
}

// Add registers a pattern with a handler for the given request method.
func (r *Router) Add(meth, pat string, h http.Handler) *mux.Route {
	return r.NewRoute().PathPrefix(pat).Handler(h).Methods(meth)
}

// Delete registers a pattern with a handler for DELETE requests.
func (r *Router) Delete(pat string, h http.HandlerFunc) *mux.Route {
	return r.Add("DELETE", pat, h)
}

// Get registers a pattern with a handler for GET requests.
func (r *Router) Get(pat string, h http.HandlerFunc) *mux.Route {
	return r.Add("GET", pat, h)
}

// Post registers a pattern with a handler for POST requests.
func (r *Router) Post(pat string, h http.HandlerFunc) *mux.Route {
	return r.Add("POST", pat, h)
}

// Put registers a pattern with a handler for PUT requests.
func (r *Router) Put(pat string, h http.HandlerFunc) *mux.Route {
	return r.Add("PUT", pat, h)
}

// ServeHTTP dispatches the handler registered in the matched route.
func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// Clean path to canonical form and redirect.
	if p := cleanPath(req.URL.Path); p != req.URL.Path {
		w.Header().Set("Location", p)
		w.WriteHeader(http.StatusMovedPermanently)
		return
	}
	var match mux.RouteMatch
	var handler http.Handler
	if matched := r.Match(req, &match); matched {
		handler = match.Handler
		registerVars(req, match.Vars)
	}
	if handler == nil {
		if r.NotFoundHandler == nil {
			r.NotFoundHandler = http.NotFoundHandler()
		}
		handler = r.NotFoundHandler
	}
	handler.ServeHTTP(w, req)
}

// registerVars adds the matched route variables to the URL query.
func registerVars(r *http.Request, vars map[string]string) {
	parts, i := make([]string, len(vars)), 0
	for key, value := range vars {
		parts[i] = url.QueryEscape(":"+key) + "=" + url.QueryEscape(value)
		i++
	}
	q := strings.Join(parts, "&")
	if r.URL.RawQuery == "" {
		r.URL.RawQuery = q
	} else {
		r.URL.RawQuery += "&" + q
	}
}

// cleanPath returns the canonical path for p, eliminating . and .. elements.
// Borrowed from the net/http package.
func cleanPath(p string) string {
	if p == "" {
		return "/"
	}
	if p[0] != '/' {
		p = "/" + p
	}
	np := path.Clean(p)
	// path.Clean removes trailing slash except for root;
	// put the trailing slash back if necessary.
	if p[len(p)-1] == '/' && np != "/" {
		np += "/"
	}
	return np
}
<|endoftext|>


<commit_before>package qshell

import (
	"bufio"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/qiniu/api.v6/auth/digest"
	"github.com/qiniu/api.v6/conf"
	fio "github.com/qiniu/api.v6/io"
	rio "github.com/qiniu/api.v6/resumable/io"
	"github.com/qiniu/api.v6/rs"
	"github.com/qiniu/log"
	"github.com/qiniu/rpc"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/opt"
	"io/ioutil"
	"os"
	"os/user"
	"runtime"
	"strconv"
	"strings"
	"sync"
)

/*
Config file like:

{
	"up_host"		:	"http://upload.qiniu.com",
	"src_dir" 		:	"/Users/jemy/Photos",
	"access_key" 	:	"<Your AccessKey>",
	"secret_key"	:	"<Your SecretKey>",
	"bucket"		:	"test-bucket",
	"ignore_dir"	:	false,
	"key_prefix"	:	"2014/12/01/",
	"overwrite"		:	false,
	"check_exists"	:	true
}

or without up_host and key_prefix and ignore_dir and check_exists

{
	"src_dir" 		:	"/Users/jemy/Photos",
	"access_key" 	:	"<Your AccessKey>",
	"secret_key"	:	"<Your SecretKey>",
	"bucket"		:	"test-bucket",
}
*/

const (
	PUT_THRESHOLD           int64 = 100 * 1 << 20
	MIN_UPLOAD_THREAD_COUNT int64 = 1
	MAX_UPLOAD_THREAD_COUNT int64 = 100
)

type UploadConfig struct {
	SrcDir      string `json:"src_dir"`
	AccessKey   string `json:"access_key"`
	SecretKey   string `json:"secret_key"`
	Bucket      string `json:"bucket"`
	UpHost      string `json:"up_host,omitempty"`
	KeyPrefix   string `json:"key_prefix,omitempty"`
	IgnoreDir   bool   `json:"ignore_dir,omitempty"`
	Overwrite   bool   `json:"overwrite,omitempty"`
	CheckExists bool   `json:"check_exists,omitempty"`
}

var upSettings = rio.Settings{
	ChunkSize: 4 * 1024 * 1024,
	TryTimes:  5,
}

func QiniuUpload(threadCount int, uploadConfigFile string) {
	fp, err := os.Open(uploadConfigFile)
	if err != nil {
		log.Error(fmt.Sprintf("Open upload config file `%s' error due to `%s'", uploadConfigFile, err))
		return
	}
	defer fp.Close()
	configData, err := ioutil.ReadAll(fp)
	if err != nil {
		log.Error(fmt.Sprintf("Read upload config file `%s' error due to `%s'", uploadConfigFile, err))
		return
	}
	var uploadConfig UploadConfig
	err = json.Unmarshal(configData, &uploadConfig)
	if err != nil {
		log.Error(fmt.Sprintf("Parse upload config file `%s' errror due to `%s'", uploadConfigFile, err))
		return
	}
	if _, err := os.Stat(uploadConfig.SrcDir); err != nil {
		log.Error("Upload config error for parameter `SrcDir`,", err)
		return
	}
	dirCache := DirCache{}
	currentUser, err := user.Current()
	if err != nil {
		log.Error("Failed to get current user", err)
		return
	}
	pathSep := string(os.PathSeparator)
	//create job id
	md5Hasher := md5.New()
	md5Hasher.Write([]byte(uploadConfig.SrcDir + ":" + uploadConfig.Bucket))
	jobId := fmt.Sprintf("%x", md5Hasher.Sum(nil))

	//local storage path
	storePath := fmt.Sprintf("%s%s.qshell%squpload%s%s", currentUser.HomeDir, pathSep, pathSep, pathSep, jobId)
	err = os.MkdirAll(storePath, 0775)
	if err != nil {
		log.Error(fmt.Sprintf("Failed to mkdir `%s' due to `%s'", storePath, err))
		return
	}

	//cache file
	cacheFileName := fmt.Sprintf("%s%s%s.cache", storePath, pathSep, jobId)
	//leveldb folder
	leveldbFileName := fmt.Sprintf("%s%s%s.ldb", storePath, pathSep, jobId)

	totalFileCount := dirCache.Cache(uploadConfig.SrcDir, cacheFileName)
	ldb, err := leveldb.OpenFile(leveldbFileName, nil)
	if err != nil {
		log.Error(fmt.Sprintf("Open leveldb `%s' failed due to `%s'", leveldbFileName, err))
		return
	}
	defer ldb.Close()
	//sync
	ufp, err := os.Open(cacheFileName)
	if err != nil {
		log.Error(fmt.Sprintf("Open cache file `%s' failed due to `%s'", cacheFileName, err))
		return
	}
	defer ufp.Close()
	bScanner := bufio.NewScanner(ufp)
	bScanner.Split(bufio.ScanLines)
	currentFileCount := 0
	ldbWOpt := opt.WriteOptions{
		Sync: true,
	}

	upWorkGroup := sync.WaitGroup{}
	upCounter := 0
	threadThreshold := threadCount + 1

	//use host if not empty
	if uploadConfig.UpHost != "" {
		conf.UP_HOST = uploadConfig.UpHost
	}
	//set settings
	rio.SetSettings(&upSettings)
	mac := digest.Mac{uploadConfig.AccessKey, []byte(uploadConfig.SecretKey)}
	//check thread count
	for bScanner.Scan() {
		line := strings.TrimSpace(bScanner.Text())
		items := strings.Split(line, "\t")
		if len(items) > 1 {
			cacheFname := items[0]
			cacheFlmd, _ := strconv.Atoi(items[2])
			uploadFileKey := cacheFname
			if uploadConfig.IgnoreDir {
				if i := strings.LastIndex(uploadFileKey, pathSep); i != -1 {
					uploadFileKey = uploadFileKey[i+1:]
				}
			}
			if uploadConfig.KeyPrefix != "" {
				uploadFileKey = strings.Join([]string{uploadConfig.KeyPrefix, uploadFileKey}, "")
			}
			//convert \ to / under windows
			if runtime.GOOS == "windows" {
				uploadFileKey = strings.Replace(uploadFileKey, "\\", "/", -1)
			}
			cacheFilePath := strings.Join([]string{uploadConfig.SrcDir, cacheFname}, pathSep)
			fstat, err := os.Stat(cacheFilePath)
			if err != nil {
				log.Error(fmt.Sprintf("Error stat local file `%s' due to `%s'", cacheFilePath, err))
				return
			}
			fsize := fstat.Size()

			//check leveldb
			currentFileCount += 1
			ldbKey := fmt.Sprintf("%s => %s", cacheFilePath, uploadFileKey)
			log.Debug(fmt.Sprintf("Checking %s ...", ldbKey))
			//check last modified
			ldbFlmd, err := ldb.Get([]byte(ldbKey), nil)
			flmd, _ := strconv.Atoi(string(ldbFlmd))
			//not exist, return ErrNotFound
			if err == nil && cacheFlmd == flmd {
				continue
			}

			fmt.Print("\033[2K\r")
			fmt.Printf("Uploading %s (%d/%d, %.1f%%) ...", ldbKey, currentFileCount, totalFileCount,
				float32(currentFileCount)*100/float32(totalFileCount))
			os.Stdout.Sync()
			rsClient := rs.New(&mac)
			//worker
			upCounter += 1
			if upCounter%threadThreshold == 0 {
				upWorkGroup.Wait()
			}
			upWorkGroup.Add(1)
			go func() {
				defer upWorkGroup.Done()
				//check exists
				if uploadConfig.CheckExists {
					rsEntry, checkErr := rsClient.Stat(nil, uploadConfig.Bucket, uploadFileKey)
					if checkErr == nil {
						//compare hash
						localEtag, cErr := GetEtag(cacheFilePath)
						if cErr != nil {
							log.Error("Calc local file hash failed,", cErr)
							return
						}
						if rsEntry.Hash == localEtag {
							log.Info("File already exists in bucket, ignore this upload")
							return
						}
					} else {
						if _, ok := checkErr.(*rpc.ErrorInfo); !ok {
							//not logic error, should be network error
							return
						}
					}
				}
				//upload
				policy := rs.PutPolicy{}
				policy.Scope = uploadConfig.Bucket
				if uploadConfig.Overwrite {
					policy.Scope = uploadConfig.Bucket + ":" + uploadFileKey
					policy.InsertOnly = 0
				}
				policy.Expires = 24 * 3600
				uptoken := policy.Token(&mac)
				if fsize > PUT_THRESHOLD {
					putRet := rio.PutRet{}
					err := rio.PutFile(nil, &putRet, uptoken, uploadFileKey, cacheFilePath, nil)
					if err != nil {
						log.Error(fmt.Sprintf("Put file `%s' => `%s' failed due to `%s'", cacheFilePath, uploadFileKey, err))
					} else {
						perr := ldb.Put([]byte(ldbKey), []byte("Y"), &ldbWOpt)
						if perr != nil {
							log.Error(fmt.Sprintf("Put key `%s' into leveldb error due to `%s'", ldbKey, perr))
						}
					}
				} else {
					putRet := fio.PutRet{}
					err := fio.PutFile(nil, &putRet, uptoken, uploadFileKey, cacheFilePath, nil)
					if err != nil {
						log.Error(fmt.Sprintf("Put file `%s' => `%s' failed due to `%s'", cacheFilePath, uploadFileKey, err))
					} else {
						perr := ldb.Put([]byte(ldbKey), []byte(strconv.Itoa(cacheFlmd)), &ldbWOpt)
						if perr != nil {
							log.Error(fmt.Sprintf("Put key `%s' into leveldb error due to `%s'", ldbKey, perr))
						}
					}
				}
			}()
		} else {
			log.Error(fmt.Sprintf("Error cache line `%s'", line))
		}
	}
	upWorkGroup.Wait()
	fmt.Println()
	fmt.Println("Upload done!")
}
<commit_msg>update upload threshold to 10MB<commit_after>package qshell

import (
	"bufio"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/qiniu/api.v6/auth/digest"
	"github.com/qiniu/api.v6/conf"
	fio "github.com/qiniu/api.v6/io"
	rio "github.com/qiniu/api.v6/resumable/io"
	"github.com/qiniu/api.v6/rs"
	"github.com/qiniu/log"
	"github.com/qiniu/rpc"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/opt"
	"io/ioutil"
	"os"
	"os/user"
	"runtime"
	"strconv"
	"strings"
	"sync"
)

/*
Config file like:

{
	"up_host"		:	"http://upload.qiniu.com",
	"src_dir" 		:	"/Users/jemy/Photos",
	"access_key" 	:	"<Your AccessKey>",
	"secret_key"	:	"<Your SecretKey>",
	"bucket"		:	"test-bucket",
	"ignore_dir"	:	false,
	"key_prefix"	:	"2014/12/01/",
	"overwrite"		:	false,
	"check_exists"	:	true
}

or without up_host and key_prefix and ignore_dir and check_exists

{
	"src_dir" 		:	"/Users/jemy/Photos",
	"access_key" 	:	"<Your AccessKey>",
	"secret_key"	:	"<Your SecretKey>",
	"bucket"		:	"test-bucket",
}
*/

const (
	PUT_THRESHOLD           int64 = 10 * 1 << 20
	MIN_UPLOAD_THREAD_COUNT int64 = 1
	MAX_UPLOAD_THREAD_COUNT int64 = 100
)

type UploadConfig struct {
	SrcDir      string `json:"src_dir"`
	AccessKey   string `json:"access_key"`
	SecretKey   string `json:"secret_key"`
	Bucket      string `json:"bucket"`
	UpHost      string `json:"up_host,omitempty"`
	KeyPrefix   string `json:"key_prefix,omitempty"`
	IgnoreDir   bool   `json:"ignore_dir,omitempty"`
	Overwrite   bool   `json:"overwrite,omitempty"`
	CheckExists bool   `json:"check_exists,omitempty"`
}

var upSettings = rio.Settings{
	ChunkSize: 4 * 1024 * 1024,
	TryTimes:  5,
}

func QiniuUpload(threadCount int, uploadConfigFile string) {
	fp, err := os.Open(uploadConfigFile)
	if err != nil {
		log.Error(fmt.Sprintf("Open upload config file `%s' error due to `%s'", uploadConfigFile, err))
		return
	}
	defer fp.Close()
	configData, err := ioutil.ReadAll(fp)
	if err != nil {
		log.Error(fmt.Sprintf("Read upload config file `%s' error due to `%s'", uploadConfigFile, err))
		return
	}
	var uploadConfig UploadConfig
	err = json.Unmarshal(configData, &uploadConfig)
	if err != nil {
		log.Error(fmt.Sprintf("Parse upload config file `%s' errror due to `%s'", uploadConfigFile, err))
		return
	}
	if _, err := os.Stat(uploadConfig.SrcDir); err != nil {
		log.Error("Upload config error for parameter `SrcDir`,", err)
		return
	}
	dirCache := DirCache{}
	currentUser, err := user.Current()
	if err != nil {
		log.Error("Failed to get current user", err)
		return
	}
	pathSep := string(os.PathSeparator)
	//create job id
	md5Hasher := md5.New()
	md5Hasher.Write([]byte(uploadConfig.SrcDir + ":" + uploadConfig.Bucket))
	jobId := fmt.Sprintf("%x", md5Hasher.Sum(nil))

	//local storage path
	storePath := fmt.Sprintf("%s%s.qshell%squpload%s%s", currentUser.HomeDir, pathSep, pathSep, pathSep, jobId)
	err = os.MkdirAll(storePath, 0775)
	if err != nil {
		log.Error(fmt.Sprintf("Failed to mkdir `%s' due to `%s'", storePath, err))
		return
	}

	//cache file
	cacheFileName := fmt.Sprintf("%s%s%s.cache", storePath, pathSep, jobId)
	//leveldb folder
	leveldbFileName := fmt.Sprintf("%s%s%s.ldb", storePath, pathSep, jobId)

	totalFileCount := dirCache.Cache(uploadConfig.SrcDir, cacheFileName)
	ldb, err := leveldb.OpenFile(leveldbFileName, nil)
	if err != nil {
		log.Error(fmt.Sprintf("Open leveldb `%s' failed due to `%s'", leveldbFileName, err))
		return
	}
	defer ldb.Close()
	//sync
	ufp, err := os.Open(cacheFileName)
	if err != nil {
		log.Error(fmt.Sprintf("Open cache file `%s' failed due to `%s'", cacheFileName, err))
		return
	}
	defer ufp.Close()
	bScanner := bufio.NewScanner(ufp)
	bScanner.Split(bufio.ScanLines)
	currentFileCount := 0
	ldbWOpt := opt.WriteOptions{
		Sync: true,
	}

	upWorkGroup := sync.WaitGroup{}
	upCounter := 0
	threadThreshold := threadCount + 1

	//use host if not empty
	if uploadConfig.UpHost != "" {
		conf.UP_HOST = uploadConfig.UpHost
	}
	//set settings
	rio.SetSettings(&upSettings)
	mac := digest.Mac{uploadConfig.AccessKey, []byte(uploadConfig.SecretKey)}
	//check thread count
	for bScanner.Scan() {
		line := strings.TrimSpace(bScanner.Text())
		items := strings.Split(line, "\t")
		if len(items) > 1 {
			cacheFname := items[0]
			cacheFlmd, _ := strconv.Atoi(items[2])
			uploadFileKey := cacheFname
			if uploadConfig.IgnoreDir {
				if i := strings.LastIndex(uploadFileKey, pathSep); i != -1 {
					uploadFileKey = uploadFileKey[i+1:]
				}
			}
			if uploadConfig.KeyPrefix != "" {
				uploadFileKey = strings.Join([]string{uploadConfig.KeyPrefix, uploadFileKey}, "")
			}
			//convert \ to / under windows
			if runtime.GOOS == "windows" {
				uploadFileKey = strings.Replace(uploadFileKey, "\\", "/", -1)
			}
			cacheFilePath := strings.Join([]string{uploadConfig.SrcDir, cacheFname}, pathSep)
			fstat, err := os.Stat(cacheFilePath)
			if err != nil {
				log.Error(fmt.Sprintf("Error stat local file `%s' due to `%s'", cacheFilePath, err))
				return
			}
			fsize := fstat.Size()

			//check leveldb
			currentFileCount += 1
			ldbKey := fmt.Sprintf("%s => %s", cacheFilePath, uploadFileKey)
			log.Debug(fmt.Sprintf("Checking %s ...", ldbKey))
			//check last modified
			ldbFlmd, err := ldb.Get([]byte(ldbKey), nil)
			flmd, _ := strconv.Atoi(string(ldbFlmd))
			//not exist, return ErrNotFound
			if err == nil && cacheFlmd == flmd {
				continue
			}

			fmt.Print("\033[2K\r")
			fmt.Printf("Uploading %s (%d/%d, %.1f%%) ...", ldbKey, currentFileCount, totalFileCount,
				float32(currentFileCount)*100/float32(totalFileCount))
			os.Stdout.Sync()
			rsClient := rs.New(&mac)
			//worker
			upCounter += 1
			if upCounter%threadThreshold == 0 {
				upWorkGroup.Wait()
			}
			upWorkGroup.Add(1)
			go func() {
				defer upWorkGroup.Done()
				//check exists
				if uploadConfig.CheckExists {
					rsEntry, checkErr := rsClient.Stat(nil, uploadConfig.Bucket, uploadFileKey)
					if checkErr == nil {
						//compare hash
						localEtag, cErr := GetEtag(cacheFilePath)
						if cErr != nil {
							log.Error("Calc local file hash failed,", cErr)
							return
						}
						if rsEntry.Hash == localEtag {
							log.Info("File already exists in bucket, ignore this upload")
							return
						}
					} else {
						if _, ok := checkErr.(*rpc.ErrorInfo); !ok {
							//not logic error, should be network error
							return
						}
					}
				}
				//upload
				policy := rs.PutPolicy{}
				policy.Scope = uploadConfig.Bucket
				if uploadConfig.Overwrite {
					policy.Scope = uploadConfig.Bucket + ":" + uploadFileKey
					policy.InsertOnly = 0
				}
				policy.Expires = 24 * 3600
				uptoken := policy.Token(&mac)
				if fsize > PUT_THRESHOLD {
					putRet := rio.PutRet{}
					err := rio.PutFile(nil, &putRet, uptoken, uploadFileKey, cacheFilePath, nil)
					if err != nil {
						log.Error(fmt.Sprintf("Put file `%s' => `%s' failed due to `%s'", cacheFilePath, uploadFileKey, err))
					} else {
						perr := ldb.Put([]byte(ldbKey), []byte("Y"), &ldbWOpt)
						if perr != nil {
							log.Error(fmt.Sprintf("Put key `%s' into leveldb error due to `%s'", ldbKey, perr))
						}
					}
				} else {
					putRet := fio.PutRet{}
					err := fio.PutFile(nil, &putRet, uptoken, uploadFileKey, cacheFilePath, nil)
					if err != nil {
						log.Error(fmt.Sprintf("Put file `%s' => `%s' failed due to `%s'", cacheFilePath, uploadFileKey, err))
					} else {
						perr := ldb.Put([]byte(ldbKey), []byte(strconv.Itoa(cacheFlmd)), &ldbWOpt)
						if perr != nil {
							log.Error(fmt.Sprintf("Put key `%s' into leveldb error due to `%s'", ldbKey, perr))
						}
					}
				}
			}()
		} else {
			log.Error(fmt.Sprintf("Error cache line `%s'", line))
		}
	}
	upWorkGroup.Wait()
	fmt.Println()
	fmt.Println("Upload done!")
}
<|endoftext|>


<commit_before>// Copyright 2016 IBM Corporation
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package commands

import (
	"fmt"

	"github.com/amalgam8/amalgam8/cli/api"
	"github.com/amalgam8/amalgam8/cli/common"
	"github.com/amalgam8/amalgam8/cli/terminal"
	"github.com/amalgam8/amalgam8/cli/utils"
	"github.com/urfave/cli"
)

// RuleUpdateCommand is used for the rule-update command.
type RuleUpdateCommand struct {
	ctx        *cli.Context
	controller api.ControllerClient
	term       terminal.UI
}

// NewRuleUpdateCommand constructs a new Rule Update.
func NewRuleUpdateCommand(term terminal.UI) (cmd *RuleUpdateCommand) {
	return &RuleUpdateCommand{
		term: term,
	}
}

// GetMetadata returns the metadata.
func (cmd *RuleUpdateCommand) GetMetadata() cli.Command {
	T := utils.Language(common.DefaultLanguage)
	return cli.Command{
		Name:        "rule-update",
		Description: T("rule_update_description"),
		Usage:       T("rule_update_usage"),
		// TODO: Complete UsageText
		UsageText: T("rule_update_usage"),
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "file, f",
				Usage: T("rule_update_file_usage"),
			},
			cli.BoolFlag{
				Name:  "redirection, r",
				Usage: T("rule_update_input_redirection_usage"),
			},
		},
		Before:       cmd.Before,
		OnUsageError: cmd.OnUsageError,
		Action:       cmd.Action,
	}
}

// Before runs before the Action
// https://godoc.org/github.com/urfave/cli#BeforeFunc
func (cmd *RuleUpdateCommand) Before(ctx *cli.Context) error {
	// Update the context
	cmd.ctx = ctx
	return nil
}

// OnUsageError is executed if an usage error occurs.
func (cmd *RuleUpdateCommand) OnUsageError(ctx *cli.Context, err error, isSubcommand bool) error {
	return cli.ShowCommandHelp(ctx, cmd.GetMetadata().FullName())
}

// Action runs when no subcommands are specified
// https://godoc.org/github.com/urfave/cli#ActionFunc
func (cmd *RuleUpdateCommand) Action(ctx *cli.Context) error {
	T := utils.Language(common.DefaultLanguage)
	controller, err := api.NewControllerClient(ctx)
	if err != nil {
		// Exit if the controller returned an error
		return nil
	}
	// Update the controller
	cmd.controller = controller

	if ctx.IsSet("file") {
		reader, format, err := utils.ReadInputFile(ctx.String("file"))
		if err != nil {
			return err
		}

		// Convert YAML to JSON
		if format == utils.YAML {
			reader, err = utils.YAMLToJSON(reader, &api.RuleList{})
			if err != nil {
				return err
			}
		}
		// Add errors in client
		_, err = cmd.controller.UpdateRules(reader)
		if err != nil {
			return err
		}

		fmt.Fprintln(cmd.ctx.App.Writer, T("request_completed"))
		return nil
	}

	return cmd.DefaultAction(ctx)
}

// DefaultAction captures the rules provided in the terminal.
func (cmd *RuleUpdateCommand) DefaultAction(ctx *cli.Context) error {
	T := utils.Language(common.DefaultLanguage)
	reader, format, err := utils.ScannerLines(cmd.ctx.App.Writer, "Enter DSL Rules", ctx.Bool("redirection"))

	fmt.Fprintf(cmd.ctx.App.Writer, "\n\n")

	if err != nil {
		fmt.Fprintf(cmd.ctx.App.Writer, "%s\n\n", err.Error())
		return nil
	}

	// Convert YAML to JSON
	if format == utils.YAML {
		reader, err = utils.YAMLToJSON(reader, &api.RuleList{})
		if err != nil {
			return err
		}
	}

	// TODO: return user-friendly errors
	_, err = cmd.controller.UpdateRules(reader)
	if err != nil {
		return err
	}

	fmt.Fprintln(cmd.ctx.App.Writer, T("request_completed"))
	return nil
}
<commit_msg>Updated rule-update command<commit_after>// Copyright 2016 IBM Corporation
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package commands

import (
	"fmt"

	"github.com/amalgam8/amalgam8/cli/common"
	"github.com/amalgam8/amalgam8/cli/terminal"
	"github.com/amalgam8/amalgam8/cli/utils"
	ctrl "github.com/amalgam8/amalgam8/controller/client"
	"github.com/amalgam8/amalgam8/pkg/api"
	"github.com/urfave/cli"
)

// RuleUpdateCommand is used for the rule-update command.
type RuleUpdateCommand struct {
	ctx        *cli.Context
	controller *ctrl.Client
	term       terminal.UI
}

// NewRuleUpdateCommand constructs a new Rule Update.
func NewRuleUpdateCommand(term terminal.UI) (cmd *RuleUpdateCommand) {
	return &RuleUpdateCommand{
		term: term,
	}
}

// GetMetadata returns the metadata.
func (cmd *RuleUpdateCommand) GetMetadata() cli.Command {
	T := utils.Language(common.DefaultLanguage)
	return cli.Command{
		Name:        "rule-update",
		Description: T("rule_update_description"),
		Usage:       T("rule_update_usage"),
		// TODO: Complete UsageText
		UsageText: T("rule_update_usage"),
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "file, f",
				Usage: T("rule_update_file_usage"),
			},
			cli.BoolFlag{
				Name:  "redirection, r",
				Usage: T("rule_update_input_redirection_usage"),
			},
		},
		Before:       cmd.Before,
		OnUsageError: cmd.OnUsageError,
		Action:       cmd.Action,
	}
}

// Before runs before the Action
// https://godoc.org/github.com/urfave/cli#BeforeFunc
func (cmd *RuleUpdateCommand) Before(ctx *cli.Context) error {
	// Update the context
	cmd.ctx = ctx
	return nil
}

// OnUsageError is executed if an usage error occurs.
func (cmd *RuleUpdateCommand) OnUsageError(ctx *cli.Context, err error, isSubcommand bool) error {
	return cli.ShowCommandHelp(ctx, cmd.GetMetadata().FullName())
}

// Action runs when no subcommands are specified
// https://godoc.org/github.com/urfave/cli#ActionFunc
func (cmd *RuleUpdateCommand) Action(ctx *cli.Context) error {
	T := utils.Language(common.DefaultLanguage)
	controller, err := Controller(ctx)
	if err != nil {
		// Exit if the controller returned an error
		return nil
	}
	// Update the controller
	cmd.controller = controller

	if ctx.IsSet("file") {
		reader, format, err := utils.ReadInputFile(ctx.String("file"))
		if err != nil {
			return err
		}

		rules := &api.RulesSet{}

		// Read rules
		err = utils.UnmarshallReader(reader, format, rules)
		if err != nil {
			return err
		}

		// Add errors in client
		_, err = cmd.controller.UpdateRules(rules)
		if err != nil {
			return err
		}

		fmt.Fprintln(cmd.ctx.App.Writer, T("request_completed"))
		return nil
	}

	return cmd.DefaultAction(ctx)
}

// DefaultAction captures the rules provided in the terminal.
func (cmd *RuleUpdateCommand) DefaultAction(ctx *cli.Context) error {
	T := utils.Language(common.DefaultLanguage)
	reader, format, err := utils.ScannerLines(cmd.ctx.App.Writer, "Enter DSL Rules", ctx.Bool("redirection"))

	fmt.Fprintf(cmd.ctx.App.Writer, "\n\n")

	if err != nil {
		fmt.Fprintf(cmd.ctx.App.Writer, "%s\n\n", err.Error())
		return nil
	}

	rules := &api.RulesSet{}

	// Read Rules
	err = utils.UnmarshallReader(reader, format, rules)
	if err != nil {
		return err
	}

	// TODO: return user-friendly errors
	_, err = cmd.controller.UpdateRules(rules)
	if err != nil {
		return err
	}

	fmt.Fprintln(cmd.ctx.App.Writer, T("request_completed"))
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/northbright/keygen"
	"os"
)

const (
	defaultKeySize    int         = 512
	defaultOutputFile string      = "./key.dat"
	defaultPerm       os.FileMode = 0666
)

var (
	keySize    int    // Key Size
	outputFile string // Output Key File
)

func main() {

	flag.StringVar(&outputFile, "o", defaultOutputFile, "Output key file. Ex: -a='./mykey.dat'. Default: ./key.dat")
	flag.IntVar(&keySize, "b", defaultKeySize, "Key Size(bits). Ex: -b='256'. Default: 512 bits")

	flag.Parse()

	if err := keygen.GenSymmetricKeyFile(keySize, outputFile, defaultPerm); err != nil {
		fmt.Printf("Generate symmetric key error: %v\n", err)
	} else {
		fmt.Printf("Generate symmetric key successfully.\n")
	}
}
<commit_msg>Import non-standard packages after standard ones.<commit_after>package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/northbright/keygen"
)

const (
	defaultKeySize    int         = 512
	defaultOutputFile string      = "./key.dat"
	defaultPerm       os.FileMode = 0666
)

var (
	keySize    int    // Key Size
	outputFile string // Output Key File
)

func main() {

	flag.StringVar(&outputFile, "o", defaultOutputFile, "Output key file. Ex: -a='./mykey.dat'. Default: ./key.dat")
	flag.IntVar(&keySize, "b", defaultKeySize, "Key Size(bits). Ex: -b='256'. Default: 512 bits")

	flag.Parse()

	if err := keygen.GenSymmetricKeyFile(keySize, outputFile, defaultPerm); err != nil {
		fmt.Printf("Generate symmetric key error: %v\n", err)
	} else {
		fmt.Printf("Generate symmetric key successfully.\n")
	}
}
<|endoftext|>


<commit_before>package rangeexpr

import (
	"fmt"

	"rangeops"
	"rangestore"
)

type Type uint8

const (
	typeData Type = iota
	// set operations
	typeUnion
	typeIntersection
	typeDifference
	// cluster lookup
	typeClusterLookup
	typeKeyLookup // 5
	// reverse lookup
	typeKeyReverseLookup
	typeKeyReverseLookupAttr
	typeKeyReverseLookupHint
)

type ByteCode struct {
	T     Type
	Value string
}

type Expression struct {
	Code []ByteCode
	Top  int
}

func (e *Expression) Init(expression string) {
	e.Code = make([]ByteCode, len(expression))
}

func (e *Expression) addOperator(operator Type) {
	code, top := e.Code, e.Top
	e.Top++
	code[top].T = operator
}

func (e *Expression) addValue(value string) {
	code, top := e.Code, e.Top
	e.Top++
	code[top].Value = value
}

// Accepts the interface for connection to store.
// Returns a pointer to array of strings and error
func (e *Expression) Evaluate(s interface{}) (*[]string, []error) {
	// simplest case, no ByteCode because expr was an empty string
	if len(e.Code) == 0 {
		return nil, nil
	}

	// typecast the store (s) to the generic store
	var store rangestore.Store
	store = s.(rangestore.Store)

	// Create an array of errors
	var errs = make([]error, 0)

	// create a stack to hold pointers to string, and a index
	// to track the stack. stack pointers will undergo inplace
	// modifications
	fmt.Println(e.Code, e.Top)
	stack, top := make([]*[]string, len(e.Code)), 0 // array of point to array of string
	// LOGIC:
	// =====
	// Rule 1: As soon as we hit a cluster set operation, take the
	//         top 2 elements of the stack the replace the top most
	//         with the set operation result. [Binary Operators]
	// Rule 2: If it is not a cluster set operation, try to Peek Forward
	//         Sensibly to make more sense of the input stream, if no
	//         sense is made, push on to the stack. [Unary Operators]
	//         Denition of Peek Forward Sensibly is as follows:
	//         a. If it is a ClusterLookup, try to see whether it
	//            is a lookup with key involved (ie, KeyLookup)
	//         b. If it is a KeyReverseLookup, try to check whether
	//            we have an attr (ReverseLookupAttr) and or a hint
	//            (KeyReverseLookupHint)
	// Rule 3: stack[0] is always the result
	var ptr = 0 // ptr to iterate over e.Code
	// we will have to manually iterate the e.Code,
	// this is because we might have to jump ahead due to
	// our lookahead cases
	for ptr < e.Top {
		code := e.Code[ptr]
		switch code.T {
		// Data Insertion
		// if it not an operation, but just data
		case typeData:
			// convert the data as an array string
			stack[top] = &[]string{code.Value}
			top++

		// Cluster Lookup
		// --------------
		// if type == ClusterLookup, peek 1 ahead to check whether
		// it is a KeyLookup, if yes continue and let KeyLookup take care
		// of expanding, else expand in place
		// if type == KeyLookup, take the next elem from the stack and
		// do an inplace expand.
		// eg,
		// Case 1:
		//   %d1 => [d1, ClusterLookup]
		//   stack => [ nil, [d1,], ] <= push d1
		//   stack => [ nil, lookup([d1,]), ] <= ClusterLookup (inplace)
		// Case 2:
		//   d2 => [d2]
		//   stack => [nil, [d2,] ] <= push d2
		// Case 3:
		//   %d1:D2 => [d1, ClusterLookup, KeyLookup, D2]
		//   stack => [nil, [d1,] ] <= push d1
		//   stack => [nil, [d1,] ] <= (at ClusterLookup) peeks and sees KeyLookup so passes
		//   stack => [nil, lookup([d1,], d2) ] <= (at KeyLookup) it takes the next value and lookups with key

		case typeClusterLookup:
			// peek first, if true continue
			if ptr < e.Top-1 && e.Code[ptr+1].T == typeKeyLookup {
				ptr++
				continue
			}
			result, err := store.ClusterLookup((*stack[top-1])[0])
			// store the addr of the result
			stack[top-1] = result
			// append the errors
			if err != nil {
				errs = append(errs, err)
			}

		case typeKeyLookup:
			var key = e.Code[ptr+1].Value // this will be the key
			result, err := store.KeyLookup((*stack[top-1])[0], key)
			// store the addr of the result
			stack[top-1] = result
			// append the errors
			if err != nil {
				errs = append(errs, err)
			}

		// Reverse Lookup
		// --------------
		// if type == ReverseKeyLookup, peek 1 ahead to check whether it is a
		// ReverseKeyLookupAttr, if yes continue and let ReverseKeyLookupAttr
		// take care of expanding, else expand in place
		// if type == ReverseKeyLookupAttr, peek 1 ahead to check whether it is a
		// ReverseLookupHint, if yes continue and let ReverseLookupHint take care
		// of expanding
		// if type == ReverseKeyLookupHint, take the values from the stack and
		// do an inplace expand
		// eg,
		// Case 1:
		//   *d1 => [d1, ReverseKeyLookup]
		//   stack => [ nil, [d1,], ] <= push d1
		//   stack => [ nil, reverse([d1,]), ] <= ReverseKeyLookup (inplace)
		// Case 2:
		//   *d1;A => [d1, ReverseKeyLookup, ReverseKeyLookupAttr, A]
		//   stack => [nil, reverse([d1,], A) ] <=
		// Case 3:
		//   %d1;A:d2 => [d1, ReverseKeyLookup, ReverseKeyLookupAttr, A, ReverseKeyLookupHint, d2]
		//   stack => [nil, [d1,] ] <= push d1
		//   stack => [nil, [d1,] ] <= (at ReverseKeyLookup) peeks and sees ReverseKeyLookupAttr so passes
		//   stack => [nil, [d1,], A] ] <= (at ReverseKeyLookupAttr) peeks and sees ReverseKeyLookupHint so passes
		//   stack => [nil, reverse([d1,], A, d2) ] <= (at ReverseKeyLookupHint) takes the 2 vales from stack + next value

		case typeKeyReverseLookup:
			// peek first, if true continue
			if ptr < e.Top-1 && e.Code[ptr+1].T == typeKeyReverseLookupAttr {
				ptr++
				continue
			}
			result, err := store.KeyReverseLookup((*stack[top-1])[0])
			// store the addr of the result
			stack[top-1] = result
			// append the errors
			if err != nil {
				errs = append(errs, err)
			}

		case typeKeyReverseLookupAttr:
			// peek first, if true continue
			if ptr < e.Top-1 && e.Code[ptr+1].T == typeKeyReverseLookupHint {
				ptr++
				continue
			}
			var attr = e.Code[ptr+1].Value // this will be the attr
			result, err := store.KeyReverseLookupAttr((*stack[top-1])[0], attr)
			// store the addr of the result
			stack[top-1] = result
			// append the errors
			if err != nil {
				errs = append(errs, err)
			}

		case typeKeyReverseLookupHint:
			var hint = e.Code[ptr+1].Value // this will be the hint
			result, err := store.KeyReverseLookupHint((*stack[top-1])[0], (*stack[top-2])[0], hint)
			// store the addr of the result
			stack[top-2] = result
			// reset top to nil, that value is no more useful to us
			stack[top-1] = nil
			// append the errors
			if err != nil {
				errs = append(errs, err)
			}

		// Range Set Operations
		// --------------------
		// All Set Operations are binary, pop off the stack the last two elements,
		// do the operation and push it back.
		case typeUnion:
			var result = make([]string, 0)
			rangeops.Union(stack[top-2], stack[top-1], &result)
			// store the addr of the result
			stack[top-2] = &result
			// reset top to nil, that value is no more useful to us
			stack[top-1] = nil

		case typeIntersection:
			var result = make([]string, 0)
			rangeops.Intersection(stack[top-2], stack[top-1], &result)
			// store the addr of the result
			stack[top-2] = &result
			// reset top to nil, that value is no more useful to us
			stack[top-1] = nil

		case typeDifference:
			var result = make([]string, 0)
			rangeops.Difference(stack[top-2], stack[top-1], &result)
			// store the addr of the result
			stack[top-2] = &result
			// reset top to nil, that value is no more useful to us
			stack[top-1] = nil

		} // switch

		ptr++
	}

	return stack[0], errs
}
<commit_msg>fixing the AST processing for binary operators<commit_after>package rangeexpr

import (
	"rangeops"
	"rangestore"
)

type Type uint8

const (
	typeData Type = iota
	// set operations
	typeUnion
	typeIntersection
	typeDifference
	// cluster lookup
	typeClusterLookup
	typeKeyLookup // 5
	// reverse lookup
	typeKeyReverseLookup
	typeKeyReverseLookupAttr
	typeKeyReverseLookupHint
)

type ByteCode struct {
	T     Type
	Value string
}

type Expression struct {
	Code []ByteCode
	Top  int
}

func (e *Expression) Init(expression string) {
	e.Code = make([]ByteCode, len(expression))
}

func (e *Expression) addOperator(operator Type) {
	code, top := e.Code, e.Top
	e.Top++
	code[top].T = operator
}

func (e *Expression) addValue(value string) {
	code, top := e.Code, e.Top
	e.Top++
	code[top].Value = value
}

// Accepts the interface for connection to store.
// Returns a pointer to array of strings and error
func (e *Expression) Evaluate(s interface{}) (*[]string, []error) {
	// simplest case, no ByteCode because expr was an empty string
	if len(e.Code) == 0 {
		return nil, nil
	}

	// typecast the store (s) to the generic store
	var store rangestore.Store
	store = s.(rangestore.Store)

	// Create an array of errors
	var errs = make([]error, 0)

	// create a stack to hold pointers to string, and a index
	// to track the stack. stack pointers will undergo inplace
	// modifications
	stack, top := make([]*[]string, len(e.Code)), 0 // array of point to array of string
	// Rules:
	// =====
	// Rule 0: Follow the my basic rules, we will get our AST processed in
	//         1-pass :-).
	// Rule 1: As soon as we hit a cluster set operation, take the
	//         top 2 elements of the stack the replace the top most
	//         with the set operation result. [Binary Operators]
	// Rule 2: If it is not a cluster set operation, try to Peek Forward
	//         Sensibly to make more sense of the input stream, if no
	//         sense is made, push on to the stack. [Unary Operators]
	//         Denition of Peek Forward Sensibly is as follows:
	//         a. If it is a ClusterLookup, try to see whether it
	//            is a lookup with key involved (ie, KeyLookup)
	//         b. If it is a KeyReverseLookup, try to check whether
	//            we have an attr (ReverseLookupAttr) and or a hint
	//            (KeyReverseLookupHint)
	// Rule 3: stack[0] is always the result
	var ptr = 0 // ptr to iterate over e.Code
	// we will have to manually iterate the e.Code,
	// this is because we might have to jump ahead due to
	// our lookahead cases
	for ptr < e.Top {
		code := e.Code[ptr]
		switch code.T {
		// Data Insertion
		// if it not an operation, but just data
		case typeData:
			// convert the data as an array string
			stack[top] = &[]string{code.Value}
			top++

		// Cluster Lookup
		// --------------
		// if type == ClusterLookup, peek 1 ahead to check whether
		// it is a KeyLookup, if yes continue and let KeyLookup take care
		// of expanding, else expand in place
		// if type == KeyLookup, take the next elem from the stack and
		// do an inplace expand.
		// eg,
		// Case 1:
		//   %d1 => [d1, ClusterLookup]
		//   stack => [ nil, [d1,], ] <= push d1
		//   stack => [ nil, lookup([d1,]), ] <= ClusterLookup (inplace)
		// Case 2:
		//   d2 => [d2]
		//   stack => [nil, [d2,] ] <= push d2
		// Case 3:
		//   %d1:D2 => [d1, ClusterLookup, KeyLookup, D2]
		//   stack => [nil, [d1,] ] <= push d1
		//   stack => [nil, [d1,] ] <= (at ClusterLookup) peeks and sees KeyLookup so passes
		//   stack => [nil, lookup([d1,], d2) ] <= (at KeyLookup) it takes the next value and lookups with key

		case typeClusterLookup:
			// peek first, if true continue
			if ptr < e.Top-1 && e.Code[ptr+1].T == typeKeyLookup {
				ptr++
				continue
			}
			result, err := store.ClusterLookup(stack[top-1])

			// store the addr of the result
			stack[top-1] = result
			// append the errors
			if err != nil {
				errs = append(errs, err)
			}

		case typeKeyLookup:
			var key = e.Code[ptr+1].Value // this will be the key
			result, err := store.KeyLookup(stack[top-1], key)
			// store the addr of the result
			stack[top-1] = result
			// append the errors
			if err != nil {
				errs = append(errs, err)
			}

		// Reverse Lookup
		// --------------
		// if type == ReverseKeyLookup, peek 1 ahead to check whether it is a
		// ReverseKeyLookupAttr, if yes continue and let ReverseKeyLookupAttr
		// take care of expanding, else expand in place
		// if type == ReverseKeyLookupAttr, peek 1 ahead to check whether it is a
		// ReverseLookupHint, if yes continue and let ReverseLookupHint take care
		// of expanding
		// if type == ReverseKeyLookupHint, take the values from the stack and
		// do an inplace expand
		// eg,
		// Case 1:
		//   *d1 => [d1, ReverseKeyLookup]
		//   stack => [ nil, [d1,], ] <= push d1
		//   stack => [ nil, reverse([d1,]), ] <= ReverseKeyLookup (inplace)
		// Case 2:
		//   *d1;A => [d1, ReverseKeyLookup, ReverseKeyLookupAttr, A]
		//   stack => [nil, reverse([d1,], A) ] <=
		// Case 3:
		//   %d1;A:d2 => [d1, ReverseKeyLookup, ReverseKeyLookupAttr, A, ReverseKeyLookupHint, d2]
		//   stack => [nil, [d1,] ] <= push d1
		//   stack => [nil, [d1,] ] <= (at ReverseKeyLookup) peeks and sees ReverseKeyLookupAttr so passes
		//   stack => [nil, [d1,], A] ] <= (at ReverseKeyLookupAttr) peeks and sees ReverseKeyLookupHint so passes
		//   stack => [nil, reverse([d1,], A, d2) ] <= (at ReverseKeyLookupHint) takes the 2 vales from stack + next value

		case typeKeyReverseLookup:
			// peek first, if true continue
			if ptr < e.Top-1 && e.Code[ptr+1].T == typeKeyReverseLookupAttr {
				ptr++
				continue
			}
			result, err := store.KeyReverseLookup((*stack[top-1])[0])
			// store the addr of the result
			stack[top-1] = result
			// append the errors
			if err != nil {
				errs = append(errs, err)
			}

		case typeKeyReverseLookupAttr:
			// peek first, if true continue
			if ptr < e.Top-1 && e.Code[ptr+1].T == typeKeyReverseLookupHint {
				ptr++
				continue
			}
			var attr = e.Code[ptr+1].Value // this will be the attr
			result, err := store.KeyReverseLookupAttr((*stack[top-1])[0], attr)
			// store the addr of the result
			stack[top-1] = result
			// append the errors
			if err != nil {
				errs = append(errs, err)
			}

		case typeKeyReverseLookupHint:
			var hint = e.Code[ptr+1].Value // this will be the hint
			result, err := store.KeyReverseLookupHint((*stack[top-1])[0], (*stack[top-2])[0], hint)
			// store the addr of the result
			stack[top-2] = result
			// reset top to nil, that value is no more useful to us
			stack[top-1] = nil
			// append the errors
			if err != nil {
				errs = append(errs, err)
			}

		// Range Set Operations
		// --------------------
		// All Set Operations are binary, pop off the stack the last two elements,
		// do the operation and push it back.
		case typeUnion:
			var result = make([]string, 0)
			rangeops.Union(stack[top-2], stack[top-1], &result)
			// store the addr of the result
			stack[top-2] = &result
			// reset top to nil, that value is no more useful to us
			stack[top-1] = nil
			top-- // merged two values to 1

		case typeIntersection:
			var result = make([]string, 0)
			rangeops.Intersection(stack[top-2], stack[top-1], &result)
			// store the addr of the result
			stack[top-2] = &result
			top-- // merged two values to 1

		case typeDifference:
			var result = make([]string, 0)
			rangeops.Difference(stack[top-2], stack[top-1], &result)
			// store the addr of the result
			stack[top-2] = &result
			top-- // merged two values to 1

		} // switch

		ptr++
	}

	return stack[0], errs
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"sync"
	"time"

	"github.com/andrew-d/go-termutil"
	"github.com/codegangsta/cli"
	"github.com/streadway/amqp"
)

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
		panic(fmt.Sprintf("%s: %s", msg, err))
	}
}

func main() {
	app := cli.NewApp()
	app.Name = "pipecat"
	app.Usage = "Connect unix pipes and message queues"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "amqpUri",
			Value:  "amqp://guest:guest@localhost:5672/",
			Usage:  "AMQP URI",
			EnvVar: "AMQP_URI",
		},
		cli.StringFlag{
			Name:   "fack",
			Value:  os.DevNull,
			Usage:  "Writes the acknowledged lines to file handle",
			EnvVar: "FACK",
		},
	}

	app.Action = func(c *cli.Context) {
		queueName := c.Args().First()
		if queueName == "" {
			fmt.Println("Please provide name of the queue")
			os.Exit(1)
		}

		conn, err := amqp.Dial(c.String("amqpUri"))
		failOnError(err, "Failed to connect to AMPQ broker")
		defer conn.Close()

		channel, err := conn.Channel()
		failOnError(err, "Failed to open a channel")
		defer channel.Close()

		q, err := channel.QueueDeclare(
			queueName, // name
			true,      // durable
			false,     // delete when unused
			false,     // exclusive
			false,     // no-wait
			nil,       // arguments
		)
		failOnError(err, "Failed to declare a queue")

		var mutex sync.Mutex
		unackedMessages := make([]amqp.Delivery, 100)

		msgs, err := channel.Consume(
			q.Name, // queue
			"",     // consumer
			false,  // auto-ack
			false,  // exclusive
			false,  // no-local
			false,  // no-wait
			nil,    // args
		)
		failOnError(err, "Failed to register a consumer")

		stdack, err := os.OpenFile(c.String("fack"), os.O_APPEND, 0660)
		failOnError(err, "Could not open ack file")
		defer stdack.Close()

		if termutil.Isatty(os.Stdin.Fd()) {
			// only read
			forever := make(chan bool)

			go func() {

				scanner := bufio.NewScanner(os.Stdin)
				for scanner.Scan() {
					ackedLine := scanner.Text()

					mutex.Lock()
					for _, msg := range unackedMessages {
						unackedLine := fmt.Sprintf("%s", msg.Body)
						if unackedLine == ackedLine {
							fmt.Fprintln(stdack, ackedLine)
							msg.Ack(false)
						}
					}
					mutex.Unlock()
				}
				if err := scanner.Err(); err != nil {
					fmt.Fprintln(os.Stderr, "Reading standard input:", err)
				}

			}()

			go func() {
				for msg := range msgs {
					mutex.Lock()
					unackedMessages = append(unackedMessages, msg)
					mutex.Unlock()

					time.Sleep(100 * time.Millisecond)
					line := fmt.Sprintf("%s", msg.Body)
					fmt.Println(line)
				}
			}()
			<-forever
		} else {

			scanner := bufio.NewScanner(os.Stdin)
			for scanner.Scan() {
				line := scanner.Text()
				err = channel.Publish(
					"",     // exchange
					q.Name, // routing key
					false,  // mandatory
					false,  // immediate
					amqp.Publishing{
						ContentType: "text/plain",
						Body:        []byte(line),
					})

				failOnError(err, "Failed to publish a message")
				fmt.Println(line)
				fmt.Fprintln(stdack, line)
			}
			if err := scanner.Err(); err != nil {
				fmt.Fprintln(os.Stderr, "Reading standard input:", err)
			}

		}

		/*} else {
			scanner := bufio.NewScanner(os.Stdin)
			for scanner.Scan() {
				line := scanner.Text()
				err = channel.Publish(
					"",     // exchange
					q.Name, // routing key
					false,  // mandatory
					false,  // immediate
					amqp.Publishing{
						ContentType: "text/plain",
						Body:        []byte(line),
					})
				fmt.Println(line)
				failOnError(err, "Failed to publish a message")
			}
			if err := scanner.Err(); err != nil {
				fmt.Fprintln(os.Stderr, "Reading standard input:", err)
			}
		}*/
	}

	app.Run(os.Args)
}
<commit_msg>Pipecat works<commit_after>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"sync"

	"github.com/codegangsta/cli"
	"github.com/streadway/amqp"
)

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
		panic(fmt.Sprintf("%s: %s", msg, err))
	}
}

func main() {
	app := cli.NewApp()
	app.Name = "pipecat"
	app.Usage = "Connect unix pipes and message queues"

	globalFlags := []cli.Flag{
		cli.StringFlag{
			Name:   "amqpUri",
			Value:  "amqp://guest:guest@localhost:5672/",
			Usage:  "AMQP URI",
			EnvVar: "AMQP_URI",
		},
		cli.StringFlag{
			Name:   "fack",
			Value:  os.DevNull,
			Usage:  "ACK file",
			EnvVar: "FACK",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:    "publish",
			Aliases: []string{"p"},
			Usage:   "Publish messages to queue",
			Flags:   globalFlags,
			Action: func(c *cli.Context) {
				queueName := c.Args().First()
				if queueName == "" {
					fmt.Println("Please provide name of the queue")
					os.Exit(1)
				}

				conn, err := amqp.Dial(c.String("amqpUri"))
				failOnError(err, "Failed to connect to AMPQ broker")
				defer conn.Close()

				channel, err := conn.Channel()
				failOnError(err, "Failed to open a channel")
				defer channel.Close()

				q, err := channel.QueueDeclare(
					queueName, // name
					true,      // durable
					false,     // delete when unused
					false,     // exclusive
					false,     // no-wait
					nil,       // arguments
				)
				failOnError(err, "Failed to declare a queue")

				stdack, err := os.OpenFile(c.String("fack"), os.O_APPEND, 0660)
				failOnError(err, "Could not open ack file")
				defer stdack.Close()

				scanner := bufio.NewScanner(os.Stdin)
				for scanner.Scan() {
					line := scanner.Text()
					err = channel.Publish(
						"",     // exchange
						q.Name, // routing key
						false,  // mandatory
						false,  // immediate
						amqp.Publishing{
							ContentType: "text/plain",
							Body:        []byte(line),
						})

					failOnError(err, "Failed to publish a message")
					fmt.Println(line)
					fmt.Fprintln(stdack, line)
				}
				if err := scanner.Err(); err != nil {
					fmt.Fprintln(os.Stderr, "Reading standard input:", err)
				}

			},
		},
		{
			Name:    "consume",
			Flags:   globalFlags,
			Aliases: []string{"c"},
			Usage:   "Consume messages from queue",
			Action: func(c *cli.Context) {
				queueName := c.Args().First()
				if queueName == "" {
					fmt.Println("Please provide name of the queue")
					os.Exit(1)
				}

				conn, err := amqp.Dial(c.String("amqpUri"))
				failOnError(err, "Failed to connect to AMPQ broker")
				defer conn.Close()

				channel, err := conn.Channel()
				failOnError(err, "Failed to open a channel")
				defer channel.Close()

				var mutex sync.Mutex
				unackedMessages := make([]amqp.Delivery, 100)

				msgs, err := channel.Consume(
					queueName, // queue
					"",        // consumer
					false,     // auto-ack
					false,     // exclusive
					false,     // no-local
					false,     // no-wait
					nil,       // args
				)
				failOnError(err, "Failed to register a consumer")

				stdack, err := os.OpenFile(c.String("fack"), os.O_APPEND, 0660)
				failOnError(err, "Could not open ack file")
				defer stdack.Close()
				forever := make(chan bool)

				go func() {

					scanner := bufio.NewScanner(os.Stdin)
					for scanner.Scan() {
						ackedLine := scanner.Text()

						mutex.Lock()
						for _, msg := range unackedMessages {
							unackedLine := fmt.Sprintf("%s", msg.Body)
							if unackedLine == ackedLine {
								fmt.Fprintln(stdack, ackedLine)
								msg.Ack(false)
							}
						}
						mutex.Unlock()
					}
					if err := scanner.Err(); err != nil {
						fmt.Fprintln(os.Stderr, "Reading standard input:", err)
					}

				}()

				go func() {
					for msg := range msgs {
						mutex.Lock()
						unackedMessages = append(unackedMessages, msg)
						mutex.Unlock()

						line := fmt.Sprintf("%s", msg.Body)
						fmt.Println(line)
					}
				}()
				<-forever

			},
		},
	}

	app.Run(os.Args)
}
<|endoftext|>