<commit_before>package main

import (
	"fmt"
	"log"
	"math/rand"
	"os"
	"time"

	"github.com/go-audio/music/theory"

	"github.com/go-audio/midi"
)

var (
	ppq uint32 = 480
)

func main() {
	f, err := os.Create("output.mid")
	if err != nil {
		log.Fatal(err)
	}

	rand.Seed(int64(time.Now().Nanosecond()))

	enc := midi.NewEncoder(f, midi.SingleTrack, uint16(ppq))
	tr := enc.NewTrack()
	tr.AddAfterDelta(0, midi.CopyrightEvent("Generated by Go-Audio"))

	defer func() {
		tr.AddAfterDelta(0, midi.EndOfTrack())
		if err := enc.Write(); err != nil {
			log.Fatal(err)
		}
		f.Close()
	}()

	// generate a chord progression
	// scale
	scale := theory.MajorScale
	// if t := time.Now().Nanosecond() % 2; t == 0 {
	// 	scale = theory.MelodicMinorScale
	// }

	// random root
	rootInt := rand.Intn(12)

	// Play the scale notes
	root := midi.Notes[rootInt]
	scaleName := fmt.Sprintf("%s %s scale", root, scale)
	fmt.Println(scaleName)
	keys, notes := theory.ScaleNotes(root, scale)
	fmt.Println("Notes in scale:", notes)
	tr.SetName(fmt.Sprintf("%s %v", scaleName, notes))
	octaveBump := 12 * 4
	// move to the 3rd octave
	for i := 0; i < len(keys); i++ {
		keys[i] += octaveBump
	}

	var progression []int
	if scale == theory.MajorScale {
		n := rand.Intn(len(theory.MajorProgressions))
		progression = theory.MajorProgressions[n]
	} else {
		n := rand.Intn(len(theory.MinorProgressions))
		progression = theory.MinorProgressions[n]
	}

	playScale(tr, root, scale)
	fmt.Println()
	playProgression(tr, root, scale, progression)
	fmt.Println()
	playScaleChords(tr, root, scale)
}

func playScaleChords(tr *midi.Track, root string, scale theory.ScaleName) {
	fmt.Println("Chords in scale")
	var timeBuffer uint32
	keys, _ := theory.ScaleNotes(root, scale)
	octaveBump := 12 * 2

	start := keys[0] + octaveBump
	// move to the 3rd octave
	for i := 0; i < len(keys); i++ {
		keys[i] += octaveBump
		// keep going up
		if keys[i] < start {
			keys[i] += 12
		}
	}

	var firstChord *theory.Chord

	for chordTypeIDX := 0; chordTypeIDX < len(theory.RichScaleChords[scale][0]); chordTypeIDX++ {
		fmt.Println("Chord Type", chordTypeIDX)
		scaleChords := theory.RichScaleChords[scale]
		// play all chords in the scale
		for i, roman := range theory.RomanNumerals[scale] {
			if i > len(scaleChords) || chordTypeIDX > len(scaleChords[i]) {
				fmt.Println(roman, "not found")
				continue
			}
			chordName := fmt.Sprintf("%s%s\n",
				midi.Notes[keys[i]%12],
				scaleChords[i][chordTypeIDX])

			fmt.Printf("%s\t%s", roman, chordName)
			c := theory.NewChordFromAbbrev(chordName)
			if i == 0 {
				firstChord = c
			}

			for i, k := range c.Keys {
				tr.AddAfterDelta(timeBuffer, midi.NoteOn(1, k+octaveBump, 99))
				if i == 0 {
					timeBuffer = 0
				}
			}
			for i, k := range c.Keys {
				if i == 0 {
					timeBuffer = ppq * 2
				}
				tr.AddAfterDelta(timeBuffer, midi.NoteOff(1, k+octaveBump))
				if i == 0 {
					timeBuffer = 0
				}
			}
		}

		// back to the first chord
		for i, k := range firstChord.Keys {
			if i == 0 {
				timeBuffer = ppq * 2
			}
			tr.AddAfterDelta(0, midi.NoteOn(1, k+24, 99))
		}
		for i, k := range firstChord.Keys {
			if i == 0 {
				timeBuffer = ppq * 2
			}
			tr.AddAfterDelta(timeBuffer, midi.NoteOff(1, k+24))
			if i == 0 {
				timeBuffer = 0
			}
		}
	}
}

func playScale(tr *midi.Track, root string, scale theory.ScaleName) {
	keys, _ := theory.ScaleNotes(root, scale)
	octaveBump := 12 * 4

	start := keys[0] + octaveBump
	// move to the 3rd octave
	for i := 0; i < len(keys); i++ {
		keys[i] += octaveBump
		// keep going up
		if keys[i] < start {
			keys[i] += 12
		}
	}

	for _, k := range keys {
		tr.AddAfterDelta(0, midi.NoteOn(1, k, 99))
		tr.AddAfterDelta(ppq, midi.NoteOff(1, k))
	}

	// back to the first note
	tr.AddAfterDelta(0, midi.NoteOn(1, keys[0]+12, 99))
	tr.AddAfterDelta(ppq, midi.NoteOff(1, keys[0]+12))
}

func playProgression(tr *midi.Track, root string, scale theory.ScaleName, progression []int) {
	var timeBuffer uint32
	octaveBump := 12 * 2
	keys, _ := theory.ScaleNotes(root, scale)
	var c *theory.Chord

	// play the same chord twice
	repeatedChords := func(rate uint32) func() {
		return func() {
			repeats := int((ppq / rate) * 2)
			timeBuffer = 0
			for n := 0; n < repeats; n++ {
				// note on
				for i, k := range c.Keys {
					tr.AddAfterDelta(timeBuffer, midi.NoteOn(1, k+octaveBump, 99))
					if i == 0 {
						timeBuffer = 0
					}
				}
				// note off
				for i, k := range c.Keys {
					if i == 0 {
						timeBuffer = rate
					}
					tr.AddAfterDelta(timeBuffer, midi.NoteOff(1, k+octaveBump))
					if i == 0 {
						timeBuffer = 0
					}
				}
			}
		}
	}

	// play the same chord twice once at a lower octave then up
	downUp := func(rate uint32) func() {
		return func() {
			repeats := int((ppq / rate) * 4)
			timeBuffer = 0
			for n := 0; n < repeats; n++ {
				bump := octaveBump + ((n % 2) * 12)
				// note on
				for i, k := range c.Keys {
					if i == 0 {
						k -= 12
					}
					tr.AddAfterDelta(timeBuffer, midi.NoteOn(1, k+bump, 99))
					if i == 0 {
						timeBuffer = 0
						if n%2 == 0 {
							break
						}
					}
				}
				// note off
				for i, k := range c.Keys {
					if i == 0 {
						timeBuffer = rate
						k -= 12
					}
					tr.AddAfterDelta(timeBuffer, midi.NoteOff(1, k+bump))
					if i == 0 {
						timeBuffer = 0
						if n%2 == 0 {
							break
						}
					}
				}
			}
		}
	}

	arps := []func(){
		repeatedChords(ppq * 2),
		repeatedChords(ppq),
		downUp(ppq),
		downUp(ppq / 2)}

	fmt.Println("Chord progression:")
	for i, fn := range arps {
		for _, k := range progression {
			// testing using triad vs 7th
			var chordType int
			if k%2 == 0 {
				chordType = 1
			}
			chordName := fmt.Sprintf("%s%s\n",
				midi.Notes[keys[k]%12],
				theory.RichScaleChords[scale][k][chordType])

			if i == 0 {
				fmt.Printf("%s\t%s", theory.RomanNumerals[scale][k], chordName)
			}
			c = theory.NewChordFromAbbrev(chordName)
			if c == nil {
				fmt.Println("Couldn't find chord", chordName)
				continue
			}

			fn()
		}
	}
}
<commit_msg>more options for the generation<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"strings"
	"time"

	"github.com/go-audio/music/theory"

	"github.com/go-audio/midi"
)

var (
	ppq          uint32 = 480
	flagFromFreq        = flag.Float64("freq", 0, "Use this frequency as the root of the generated data")
	flagFromNote        = flag.String("root", "", "Root note to use")
	flagOctave          = flag.Int("octave", 3, "Default octave to start from")
	flagIsMinor         = flag.Bool("minor", false, "should we use a minor scale")
)

func main() {
	flag.Parse()
	f, err := os.Create("output.mid")
	if err != nil {
		log.Fatal(err)
	}

	rand.Seed(int64(time.Now().Nanosecond()))

	enc := midi.NewEncoder(f, midi.SingleTrack, uint16(ppq))
	tr := enc.NewTrack()
	tr.AddAfterDelta(0, midi.CopyrightEvent("Generated by Go-Audio"))

	defer func() {
		tr.AddAfterDelta(0, midi.EndOfTrack())
		if err := enc.Write(); err != nil {
			log.Fatal(err)
		}
		f.Close()
	}()

	// generate a chord progression
	// scale
	var scale theory.ScaleName
	if *flagIsMinor {
		scale = theory.MelodicMinorScale
	} else {
		scale = theory.MajorScale
	}

	var rootInt int
	if *flagFromNote != "" {
		rootInt = midi.NotesToInt[strings.ToUpper(*flagFromNote)]
	} else if *flagFromFreq != 0 {
		rootInt = midi.FreqToNote(*flagFromFreq)
	} else {
		// random root
		rootInt = rand.Intn(12)
	}

	// Play the scale notes
	root := midi.Notes[rootInt]
	scaleName := fmt.Sprintf("%s %s scale", root, scale)
	fmt.Println(scaleName)
	keys, notes := theory.ScaleNotes(root, scale)
	fmt.Println("Notes in scale:", notes)
	tr.SetName(fmt.Sprintf("%s %v", scaleName, notes))
	octaveBump := 12 * (*flagOctave + 1)
	// move to the 3rd octave
	for i := 0; i < len(keys); i++ {
		keys[i] += octaveBump
	}

	var progression []int
	if scale == theory.MajorScale {
		n := rand.Intn(len(theory.MajorProgressions))
		progression = theory.MajorProgressions[n]
	} else {
		n := rand.Intn(len(theory.MinorProgressions))
		progression = theory.MinorProgressions[n]
	}

	// playScale(tr, root, scale)
	// fmt.Println()
	playProgression(tr, root, scale, progression)
	fmt.Println()
	playScaleChords(tr, root, scale)
}

func playScaleChords(tr *midi.Track, root string, scale theory.ScaleName) {
	fmt.Println("Chords in scale")
	var timeBuffer uint32
	keys, _ := theory.ScaleNotes(root, scale)
	octaveBump := 12 * 2

	start := keys[0] + octaveBump
	// move to the 3rd octave
	for i := 0; i < len(keys); i++ {
		keys[i] += octaveBump
		// keep going up
		if keys[i] < start {
			keys[i] += 12
		}
	}

	var firstChord *theory.Chord

	for chordTypeIDX := 0; chordTypeIDX < len(theory.RichScaleChords[scale][0]); chordTypeIDX++ {
		fmt.Println("Chord Type", chordTypeIDX)
		scaleChords := theory.RichScaleChords[scale]
		// play all chords in the scale
		for i, roman := range theory.RomanNumerals[scale] {
			if i > len(scaleChords) || chordTypeIDX > len(scaleChords[i]) {
				fmt.Println(roman, "not found")
				continue
			}
			chordName := fmt.Sprintf("%s%s\n",
				midi.Notes[keys[i]%12],
				scaleChords[i][chordTypeIDX])

			fmt.Printf("%s\t%s", roman, chordName)
			c := theory.NewChordFromAbbrev(chordName)
			if i == 0 {
				firstChord = c
			}
			if c == nil {
				fmt.Println("failed to find chord named", chordName)
				continue
			}

			for i, k := range c.Keys {
				tr.AddAfterDelta(timeBuffer, midi.NoteOn(1, k+octaveBump, 99))
				if i == 0 {
					timeBuffer = 0
				}
			}
			for i, k := range c.Keys {
				if i == 0 {
					timeBuffer = ppq * 2
				}
				tr.AddAfterDelta(timeBuffer, midi.NoteOff(1, k+octaveBump))
				if i == 0 {
					timeBuffer = 0
				}
			}
		}

		// back to the first chord
		for i, k := range firstChord.Keys {
			if i == 0 {
				timeBuffer = ppq * 2
			}
			tr.AddAfterDelta(0, midi.NoteOn(1, k+24, 99))
		}
		for i, k := range firstChord.Keys {
			if i == 0 {
				timeBuffer = ppq * 2
			}
			tr.AddAfterDelta(timeBuffer, midi.NoteOff(1, k+24))
			if i == 0 {
				timeBuffer = 0
			}
		}
	}
}

func playScale(tr *midi.Track, root string, scale theory.ScaleName) {
	keys, _ := theory.ScaleNotes(root, scale)
	octaveBump := 12 * 4

	start := keys[0] + octaveBump
	// move to the 3rd octave
	for i := 0; i < len(keys); i++ {
		keys[i] += octaveBump
		// keep going up
		if keys[i] < start {
			keys[i] += 12
		}
	}

	for _, k := range keys {
		tr.AddAfterDelta(0, midi.NoteOn(1, k, 99))
		tr.AddAfterDelta(ppq, midi.NoteOff(1, k))
	}

	// back to the first note
	tr.AddAfterDelta(0, midi.NoteOn(1, keys[0]+12, 99))
	tr.AddAfterDelta(ppq, midi.NoteOff(1, keys[0]+12))
}

func playProgression(tr *midi.Track, root string, scale theory.ScaleName, progression []int) {
	var timeBuffer uint32
	octaveBump := 12 * 2
	keys, _ := theory.ScaleNotes(root, scale)
	var c *theory.Chord

	// play the same chord twice
	repeatedChords := func(rate uint32) func() {
		return func() {
			repeats := int((ppq / rate) * 2)
			timeBuffer = 0
			for n := 0; n < repeats; n++ {
				// note on
				for i, k := range c.Keys {
					tr.AddAfterDelta(timeBuffer, midi.NoteOn(1, k+octaveBump, 99))
					if i == 0 {
						timeBuffer = 0
					}
				}
				// note off
				for i, k := range c.Keys {
					if i == 0 {
						timeBuffer = rate
					}
					tr.AddAfterDelta(timeBuffer, midi.NoteOff(1, k+octaveBump))
					if i == 0 {
						timeBuffer = 0
					}
				}
			}
		}
	}

	// play the same chord twice once at a lower octave then up
	downUp := func(rate uint32) func() {
		return func() {
			repeats := int((ppq / rate) * 4)
			timeBuffer = 0
			for n := 0; n < repeats; n++ {
				bump := octaveBump + ((n % 2) * 12)
				// note on
				for i, k := range c.Keys {
					if i == 0 {
						k -= 12
					}
					tr.AddAfterDelta(timeBuffer, midi.NoteOn(1, k+bump, 99))
					if i == 0 {
						timeBuffer = 0
						if n%2 == 0 {
							break
						}
					}
				}
				// note off
				for i, k := range c.Keys {
					if i == 0 {
						timeBuffer = rate
						k -= 12
					}
					tr.AddAfterDelta(timeBuffer, midi.NoteOff(1, k+bump))
					if i == 0 {
						timeBuffer = 0
						if n%2 == 0 {
							break
						}
					}
				}
			}
		}
	}

	arps := []func(){
		repeatedChords(ppq * 2),
		repeatedChords(ppq),
		downUp(ppq),
		downUp(ppq / 2)}

	fmt.Println("Chord progression:")
	for i, fn := range arps {
		for _, k := range progression {
			// testing using triad vs 7th
			var chordType int
			if k%2 == 0 {
				chordType = 1
			}
			chordName := fmt.Sprintf("%s%s\n",
				midi.Notes[keys[k]%12],
				theory.RichScaleChords[scale][k][chordType])

			if i == 0 {
				fmt.Printf("%s\t%s", theory.RomanNumerals[scale][k], chordName)
			}
			c = theory.NewChordFromAbbrev(chordName)
			if c == nil {
				fmt.Println("Couldn't find chord", chordName)
				continue
			}

			fn()
		}
	}
}
<|endoftext|>


<commit_before>package poloniex

import (
	"fmt"
	"time"

	"github.com/beatgammit/turnpike"
	"github.com/gogo/protobuf/types"
	"github.com/golang/glog"

	"github.com/crypto-bank/go-exchanges/common"
	"github.com/crypto-bank/proto/currency"
	"github.com/crypto-bank/proto/exchange"
	"github.com/crypto-bank/proto/order"
	"github.com/crypto-bank/proto/orderbook"
)

// Stream - Poloniex stream.
type Stream struct {
	client *turnpike.Client
	events chan *orderbook.Event
	errors chan error

	// hbchan - heartbeat channel
	hbchan chan struct{}

	// subs - active subscriptions
	subs []*currency.Pair

	// timestamp - time of last event
	timestamp time.Time
}

const (
	// WebsocketAddress - Poloniex Websocket address.
	WebsocketAddress = "wss://api.poloniex.com"

	// WebsocketRealm - Poloniex Websocket realm name.
	WebsocketRealm = "realm1"
)

// NewStream - Creates a new connected poloniex stream.
func NewStream() (stream *Stream, err error) {
	client, err := connectWS()
	if err != nil {
		return
	}
	stream = &Stream{
		client: client,
		events: make(chan *orderbook.Event, 1000),
		errors: make(chan error, 10),
		hbchan: make(chan struct{}, 10),
	}
	go stream.heartbeatPoll()
	return
}

// connect - Connects to poloniex websocket server
func connectWS() (client *turnpike.Client, err error) {
	// Create WS client and connect
	client, err = turnpike.NewWebsocketClient(turnpike.JSON, WebsocketAddress, nil)
	if err != nil {
		return
	}
	// Join to poloniex realm
	_, err = client.JoinRealm(WebsocketRealm, nil)
	if err != nil {
		return nil, err
	}
	return
}

// Subscribe - Sends subscription to the server.
// Stores subscription pair in local memory,
// in case of disconnection it will re-subscribe.
func (stream *Stream) Subscribe(pair *currency.Pair) error {
	// Add pair to local subscriptions map
	// if disconnect happens we might have to reuse it
	stream.subs = append(stream.subs, pair)
	// Send subscription to the server
	return stream.subscribe(pair)
}

// subscribe - Sends subscription to the server.
func (stream *Stream) subscribe(pair *currency.Pair) error {
	return stream.client.Subscribe(pair.Concat("_"), stream.eventHandler(pair))
}

// Events - Channel of orderbook.
func (stream *Stream) Events() <-chan *orderbook.Event {
	return stream.events
}

// Errors - Channel of errors.
func (stream *Stream) Errors() <-chan error {
	return stream.errors
}

// Close - Closes a stream.
func (stream *Stream) Close() (err error) {
	// We have to first close the stream
	err = stream.client.Close()
	if err != nil {
		return
	}

	// It should be safe to close channels now
	close(stream.errors)
	close(stream.events)
	close(stream.hbchan)
	return
}

// eventHandler - Creates stream event handler for currency pair.
func (stream *Stream) eventHandler(pair *currency.Pair) turnpike.EventHandler {
	return func(args []interface{}, kwargs map[string]interface{}) {
		// Send a heartbeat to a goroutine,
		// which polls for it and re-connects in case of heartbeat stop.
		stream.hbchan <- struct{}{}

		// If more than one argument is present
		// it means we received an event.
		// We have to store timestamp of last event,
		// so in case no events in latest 60 seconds will be sent
		// heartbeats will increase to 8 seconds (sent by Poloniex)
		// and we won't reconnect.
		if len(args) >= 1 {
			stream.timestamp = time.Now()
		}

		// Parse and emit all events
		for _, v := range args {
			// Cast to underlying value type
			value := v.(map[string]interface{})

			// Message type
			typ := value["type"].(string)

			// Message data
			data := value["data"].(map[string]interface{})

			// Handle message
			res, err := parseEvent(typ, pair, data)
			if err != nil {
				stream.errors <- fmt.Errorf("unhandled poloniex message: %v", err)
			} else {
				stream.events <- res
			}
		}
	}
}

// heartbeatPoll - Polls for heartbeats and re-connects in case of stop.
// Should be executed in separate goroutine.
func (stream *Stream) heartbeatPoll() {
	for {
		// Calculate heartbeat interval,
		// adding one second in case of connection problems.
		timeout := stream.heartbeatInterval() + time.Second

		// Poll on either a heartbeat or a timeout channel.
		select {
		case _, ok := <-stream.hbchan:
			// If ok is false, it means heartbeat channel was closed
			// and this goroutine should be garbage collected
			if !ok {
				return
			}
		case <-time.After(timeout):
			// Timeout passed for a heartbeat to arrive.
			// Warn about reconnecting and start the process.
			glog.Warningf("No heartbeat received for %s, reconnecting.", timeout)
			// Start reconnecting to websocket
			stream.reconnect()
		}
	}
}

func (stream *Stream) reconnect() {
	// Channel on which event will be sent if we are done connecting
	done := make(chan struct{}, 2)
	result := make(chan *turnpike.Client, 1)

	// Defer closing of the channels
	defer func() {
		close(done)
		close(result)
	}()

	// Start re-connecting in goroutine
	go func() {
		for {
			glog.V(1).Info("Re-connecting to poloniex")

			// Connect to websocket
			client, err := connectWS()
			if err != nil {
				glog.Warningf("Re-connect error: %v", err)
				stream.errors <- err
			} else {
				glog.V(1).Info("Re-connected successfuly")
				result <- client
				return
			}

			select {
			case <-time.After(time.Millisecond * 250):
				continue
			case <-done:
				// If we receive on `done` channel
				// it means we have received heartbeat before
				// if we've got a client we have to close it
				if client != nil {
					glog.V(1).Info("Closing re-connected client")
					client.Close()
				}
			}
		}
	}()

	select {
	case hb, ok := <-stream.hbchan:
		glog.V(1).Info("Heartbeat restored before re-connect")
		// Send event telling we are done connecting
		done <- struct{}{}
		// If ok is false, it means heartbeat channel was closed
		// and this goroutine should be garbage collected
		if !ok {
			return
		}
		// Re-send heartbeat on the channel
		stream.hbchan <- hb
		// Try to read from result channel
		if client, ok := <-result; ok {
			glog.V(1).Info("Closing re-connected client")
			// Close client, it won't be used anymore
			client.Close()
		}
		return
	case client := <-result:
		glog.V(1).Info("Replacing old client")
		// First we have to close old client
		if err := stream.client.Close(); err != nil {
			glog.Warningf("Client close error: %v", err)
			stream.errors <- err
		}

		// now we can replace it with new client
		stream.client = client
	}

	// Re-subscribe after re-connection
	glog.V(1).Infof("Re-subscribing to %d channels", len(stream.subs))
	stream.resubscribe()
}

func (stream *Stream) heartbeatInterval() time.Duration {
	if time.Since(stream.timestamp) > time.Minute {
		return time.Second * 8
	}
	return time.Second

}

// resubscribe - Re-subscribes client after re-connection.
func (stream *Stream) resubscribe() (err error) {
	for _, pair := range stream.subs {
		err = stream.subscribe(pair)
		if err != nil {
			return
		}
	}
	return
}

var handlers = map[string]func(*currency.Pair, map[string]interface{}) (*orderbook.Event, error){
	"orderBookRemove": parseRemove,
	"orderBookModify": parseModify,
	"newTrade":        parseTrade,
}

func parseEvent(typ string, pair *currency.Pair, data map[string]interface{}) (_ *orderbook.Event, err error) {
	// Get handler by message type
	handler, ok := handlers[typ]
	if !ok {
		return nil, fmt.Errorf("unknown type: %q", typ)
	}

	// Handle message
	return handler(pair, data)
}

func parseRemove(pair *currency.Pair, data map[string]interface{}) (_ *orderbook.Event, err error) {
	res := &order.Order{
		Exchange: exchange.Poloniex,
		Volume:   currency.NewVolume(pair.Second, 0.0),
	}
	res.Type, err = order.TypeFromString(data["type"].(string))
	if err != nil {
		return
	}
	res.Rate, err = common.ParseIVolume(pair.First, data["rate"])
	if err != nil {
		return
	}
	return orderbook.NewEvent(res), nil
}

func parseModify(pair *currency.Pair, data map[string]interface{}) (_ *orderbook.Event, err error) {
	res := &order.Order{
		Exchange: exchange.Poloniex,
	}
	res.Type, err = order.TypeFromString(data["type"].(string))
	if err != nil {
		return
	}
	res.Rate, err = common.ParseIVolume(pair.First, data["rate"])
	if err != nil {
		return
	}
	res.Volume, err = common.ParseIVolume(pair.Second, data["amount"])
	if err != nil {
		return
	}
	return orderbook.NewEvent(res), nil
}

func parseTrade(pair *currency.Pair, data map[string]interface{}) (_ *orderbook.Event, err error) {
	res := new(order.Trade)
	res.ID, err = common.ParseIInt64(data["tradeID"])
	if err != nil {
		return
	}
	t, err := time.Parse("2006-01-02 15:04:05", data["date"].(string))
	if err != nil {
		return
	}
	res.Time, err = types.TimestampProto(t)
	if err != nil {
		return
	}
	res.Order = &order.Order{
		Exchange: exchange.Poloniex,
	}
	res.Order.Type, err = order.TypeFromString(data["type"].(string))
	if err != nil {
		return
	}
	res.Order.Rate, err = common.ParseIVolume(pair.First, data["rate"])
	if err != nil {
		return
	}
	res.Order.Volume, err = common.ParseIVolume(pair.Second, data["amount"])
	if err != nil {
		return
	}
	return orderbook.NewEvent(res), nil
}
<commit_msg>feat(*): events id's<commit_after>package poloniex

import (
	"fmt"
	"time"

	"github.com/beatgammit/turnpike"
	"github.com/golang/glog"

	"github.com/crypto-bank/go-exchanges/common"
	"github.com/crypto-bank/proto/currency"
	"github.com/crypto-bank/proto/exchange"
	"github.com/crypto-bank/proto/order"
	"github.com/crypto-bank/proto/orderbook"
)

// Stream - Poloniex stream.
type Stream struct {
	client *turnpike.Client
	events chan *orderbook.Event
	errors chan error

	// hbchan - heartbeat channel
	hbchan chan struct{}

	// subs - active subscriptions
	subs []*currency.Pair

	// timestamp - time of last event
	timestamp time.Time
}

const (
	// WebsocketAddress - Poloniex Websocket address.
	WebsocketAddress = "wss://api.poloniex.com"

	// WebsocketRealm - Poloniex Websocket realm name.
	WebsocketRealm = "realm1"
)

// NewStream - Creates a new connected poloniex stream.
func NewStream() (stream *Stream, err error) {
	client, err := connectWS()
	if err != nil {
		return
	}
	stream = &Stream{
		client: client,
		events: make(chan *orderbook.Event, 1000),
		errors: make(chan error, 1000),
		hbchan: make(chan struct{}, 1000),
	}
	go stream.heartbeatPoll()
	return
}

// connect - Connects to poloniex websocket server
func connectWS() (client *turnpike.Client, err error) {
	// Create WS client and connect
	client, err = turnpike.NewWebsocketClient(turnpike.JSON, WebsocketAddress, nil)
	if err != nil {
		return
	}
	// Join to poloniex realm
	_, err = client.JoinRealm(WebsocketRealm, nil)
	if err != nil {
		return nil, err
	}
	return
}

// Subscribe - Sends subscription to the server.
// Stores subscription pair in local memory,
// in case of disconnection it will re-subscribe.
func (stream *Stream) Subscribe(pair *currency.Pair) error {
	// Add pair to local subscriptions map
	// if disconnect happens we might have to reuse it
	stream.subs = append(stream.subs, pair)
	// Send subscription to the server
	return stream.subscribe(pair)
}

// subscribe - Sends subscription to the server.
func (stream *Stream) subscribe(pair *currency.Pair) error {
	return stream.client.Subscribe(pair.Concat("_"), stream.eventHandler(pair))
}

// Events - Channel of orderbook.
func (stream *Stream) Events() <-chan *orderbook.Event {
	return stream.events
}

// Errors - Channel of errors.
func (stream *Stream) Errors() <-chan error {
	return stream.errors
}

// Close - Closes a stream.
func (stream *Stream) Close() (err error) {
	// We have to first close the stream
	err = stream.client.Close()
	if err != nil {
		return
	}

	// It should be safe to close channels now
	close(stream.errors)
	close(stream.events)
	close(stream.hbchan)
	return
}

// eventHandler - Creates stream event handler for currency pair.
func (stream *Stream) eventHandler(pair *currency.Pair) turnpike.EventHandler {
	return func(args []interface{}, kwargs map[string]interface{}) {
		// Send a heartbeat to a goroutine,
		// which polls for it and re-connects in case of heartbeat stop.
		stream.hbchan <- struct{}{}

		// If more than one argument is present
		// it means we received an event.
		// We have to store timestamp of last event,
		// so in case no events in latest 60 seconds will be sent
		// heartbeats will increase to 8 seconds (sent by Poloniex)
		// and we won't reconnect.
		if len(args) >= 1 {
			stream.timestamp = time.Now()
		}

		// If no sequence ID we are not having the right thing
		fseq, ok := kwargs["seq"].(float64)
		if !ok {
			glog.Warningf("No sequence number in kwargs")
			return
		}

		// Convert `float64` sequence ID to `int64`
		seq := int64(fseq)

		// Parse and emit all events
		for _, v := range args {
			// Cast to underlying value type
			value := v.(map[string]interface{})

			// Message type
			typ := value["type"].(string)

			// Message data
			data := value["data"].(map[string]interface{})

			// Handle message
			res, err := parseEvent(typ, seq, pair, data)
			if err != nil {
				stream.errors <- fmt.Errorf("unhandled poloniex message: %v", err)
			} else {
				stream.events <- res
			}
		}
	}
}

// heartbeatPoll - Polls for heartbeats and re-connects in case of stop.
// Should be executed in separate goroutine.
func (stream *Stream) heartbeatPoll() {
	for {
		// Calculate heartbeat interval,
		// adding two seconds in case of connection problems.
		timeout := stream.heartbeatInterval() + (2 * time.Second)

		// Poll on either a heartbeat or a timeout channel.
		select {
		case _, ok := <-stream.hbchan:
			// If ok is false, it means heartbeat channel was closed
			// and this goroutine should be garbage collected
			if !ok {
				return
			}
		case <-time.After(timeout):
			// Timeout passed for a heartbeat to arrive.
			// Warn about reconnecting and start the process.
			glog.Warningf("No heartbeat received for %s, reconnecting.", timeout)
			// Start reconnecting to websocket
			stream.reconnect()
		}
	}
}

func (stream *Stream) reconnect() {
	// Channel on which event will be sent if we are done connecting
	done := make(chan struct{}, 2)
	result := make(chan *turnpike.Client, 1)

	// Defer closing of the channels
	defer func() {
		close(done)
		close(result)
	}()

	// Start re-connecting in goroutine
	go func() {
		for {
			glog.V(1).Info("Re-connecting to poloniex")

			// Connect to websocket
			client, err := connectWS()
			if err != nil {
				glog.Warningf("Re-connect error: %v", err)
				stream.errors <- err
			} else {
				glog.V(1).Info("Re-connected successfuly")
				result <- client
				return
			}

			select {
			case <-time.After(time.Millisecond * 250):
				continue
			case <-done:
				// If we receive on `done` channel
				// it means we have received heartbeat before
				// if we've got a client we have to close it
				if client != nil {
					glog.V(1).Info("Closing re-connected client")
					client.Close()
				}
			}
		}
	}()

	select {
	case hb, ok := <-stream.hbchan:
		glog.V(1).Infof("Heartbeat restored before re-connect (state: %v)", ok)
		// Send event telling we are done connecting
		done <- struct{}{}
		// If ok is false, it means heartbeat channel was closed
		// and this goroutine should be garbage collected
		if !ok {
			glog.V(1).Info("Closing re-connected client")
			return
		}
		// Re-send heartbeat on the channel
		stream.hbchan <- hb
		// Try to read from result channel
		if client, ok := <-result; ok {
			glog.V(1).Info("Closing re-connected client")
			// Close client, it won't be used anymore
			client.Close()
		} else {
			glog.V(1).Info("Re-connected client was not recovered")
		}
		return
	case client := <-result:
		glog.V(1).Info("Replacing old client")
		// First we have to close old client
		if err := stream.client.Close(); err != nil {
			glog.Warningf("Client close error: %v", err)
			stream.errors <- err
		}

		// now we can replace it with new client
		stream.client = client
	}

	// Re-subscribe after re-connection
	glog.V(1).Infof("Re-subscribing to %d channels", len(stream.subs))
	stream.resubscribe()
}

func (stream *Stream) heartbeatInterval() time.Duration {
	if time.Since(stream.timestamp) > time.Minute {
		return time.Second * 8
	}
	return time.Second

}

// resubscribe - Re-subscribes client after re-connection.
func (stream *Stream) resubscribe() (err error) {
	for _, pair := range stream.subs {
		err = stream.subscribe(pair)
		if err != nil {
			return
		}
	}
	return
}

var handlers = map[string]func(int64, *currency.Pair, map[string]interface{}) (*orderbook.Event, error){
	"orderBookRemove": parseRemove,
	"orderBookModify": parseModify,
	"newTrade":        parseTrade,
}

func parseEvent(typ string, seq int64, pair *currency.Pair, data map[string]interface{}) (_ *orderbook.Event, err error) {
	// Get handler by message type
	handler, ok := handlers[typ]
	if !ok {
		return nil, fmt.Errorf("unknown type: %q", typ)
	}

	// Handle message
	return handler(seq, pair, data)
}

func parseRemove(seq int64, pair *currency.Pair, data map[string]interface{}) (_ *orderbook.Event, err error) {
	res := &order.Order{
		Exchange: exchange.Poloniex,
		Volume:   currency.NewVolume(pair.Second, 0.0),
	}
	res.Type, err = order.TypeFromString(data["type"].(string))
	if err != nil {
		return
	}
	res.Rate, err = common.ParseIVolume(pair.First, data["rate"])
	if err != nil {
		return
	}
	return orderbook.NewEvent(seq, res), nil
}

func parseModify(seq int64, pair *currency.Pair, data map[string]interface{}) (_ *orderbook.Event, err error) {
	res := &order.Order{
		Exchange: exchange.Poloniex,
	}
	res.Type, err = order.TypeFromString(data["type"].(string))
	if err != nil {
		return
	}
	res.Rate, err = common.ParseIVolume(pair.First, data["rate"])
	if err != nil {
		return
	}
	res.Volume, err = common.ParseIVolume(pair.Second, data["amount"])
	if err != nil {
		return
	}
	return orderbook.NewEvent(seq, res), nil
}

func parseTrade(seq int64, pair *currency.Pair, data map[string]interface{}) (_ *orderbook.Event, err error) {
	res := new(order.Trade)
	res.ID, err = common.ParseIInt64(data["tradeID"])
	if err != nil {
		return
	}
	res.Time, err = common.ParseTime("2006-01-02 15:04:05", data["date"].(string))
	if err != nil {
		return
	}
	res.Order = &order.Order{
		Exchange: exchange.Poloniex,
	}
	res.Order.Type, err = order.TypeFromString(data["type"].(string))
	if err != nil {
		return
	}
	res.Order.Rate, err = common.ParseIVolume(pair.First, data["rate"])
	if err != nil {
		return
	}
	res.Order.Volume, err = common.ParseIVolume(pair.Second, data["amount"])
	if err != nil {
		return
	}
	return orderbook.NewEvent(seq, res), nil
}
<|endoftext|>


<commit_before>package api

import (
	"emp/db"
	"emp/objects"
	"fmt"
	"quibit"
	"time"
)

func Start(config *ApiConfig) {
	var err error
	var frame quibit.Frame

	defer quit(config)

	// Start Database Services
	err = db.Initialize(config.Log, config.DbFile)
	defer db.Cleanup()
	if err != nil {
		config.Log <- fmt.Sprintf("Error initializing database: %s", err)
		config.Log <- "Quit"
		return
	}
	config.LocalVersion.Timestamp = time.Now().Round(time.Second)

	locVersion := objects.MakeFrame(objects.VERSION, objects.REQUEST, &config.LocalVersion)
	for str, _ := range config.NodeList.Nodes {
		locVersion.Peer = str
		config.SendQueue <- *locVersion
	}

	for {
		select {
		case frame = <-config.RecvQueue:
			if frame.Header.Command != objects.GETOBJ {
				config.Log <- fmt.Sprintf("Received %s frame...", CmdString(frame.Header.Command))
			} else {
				config.Log <- fmt.Sprintf("Received %s frame for %s...", CmdString(frame.Header.Command), frame.Payload)
			}
			switch frame.Header.Command {
			case objects.VERSION:
				version := new(objects.Version)
				err = version.FromBytes(frame.Payload)
				if err != nil {
					config.Log <- fmt.Sprintf("Error parsing version: %s", err)
				} else {
					go fVERSION(config, frame, version)
				}
			case objects.PEER:
				nodeList := new(objects.NodeList)
				err = nodeList.FromBytes(frame.Payload)
				if err != nil {
					config.Log <- fmt.Sprintf("Error parsing peer list: %s", err)
				} else {
					go fPEER(config, frame, nodeList)
				}
			case objects.OBJ:
				obj := new(objects.Obj)
				err = obj.FromBytes(frame.Payload)
				if err != nil {
					config.Log <- fmt.Sprintf("Error parsing obj list: %s", err)
				} else {
					go fOBJ(config, frame, obj)
				}
			case objects.GETOBJ:
				getObj := new(objects.Hash)
				if len(frame.Payload) == 0 {
					break
				}
				err = getObj.FromBytes(frame.Payload)
				if err != nil {
					config.Log <- fmt.Sprintf("Error parsing getobj hash: %s", err)
				} else {
					go fGETOBJ(config, frame, getObj)
				}
			case objects.PUBKEY_REQUEST:
				pubReq := new(objects.Hash)
				err = pubReq.FromBytes(frame.Payload)
				if err != nil {
					config.Log <- fmt.Sprintf("Error parsing pubkey request hash: %s", err)
				} else {
					go fPUBKEY_REQUEST(config, frame, pubReq)
				}
			case objects.PUBKEY:
				pub := new(objects.EncryptedPubkey)
				err = pub.FromBytes(frame.Payload)
				if err != nil {
					config.Log <- fmt.Sprintf("Error parsing pubkey: %s", err)
				} else {
					go fPUBKEY(config, frame, pub)
				}
			case objects.MSG:
				msg := new(objects.Message)
				err = msg.FromBytes(frame.Payload)
				if err != nil {
					config.Log <- fmt.Sprintf("Error parsing message: %s", err)
				} else {
					go fMSG(config, frame, msg)
				}
				fmt.Println("Finished select!")
			case objects.PURGE:
				purge := new(objects.Purge)
				err = purge.FromBytes(frame.Payload)
				if err != nil {
					config.Log <- fmt.Sprintf("Error parsing purge: %s", err)
				} else {
					go fPURGE(config, frame, purge)
				}
			default:
				config.Log <- fmt.Sprintf("Received invalid frame for command: %d", frame.Header.Command)
			}
		case <-config.Quit:
			fmt.Println()
			return
		}
	}

	// Should NEVER get here!
	panic("Must've been a cosmic ray!")
}

func quit(config *ApiConfig) {
	config.Log <- "Quit"
}
<commit_msg>Removed debugging<commit_after>package api

import (
	"emp/db"
	"emp/objects"
	"fmt"
	"quibit"
	"time"
)

func Start(config *ApiConfig) {
	var err error
	var frame quibit.Frame

	defer quit(config)

	// Start Database Services
	err = db.Initialize(config.Log, config.DbFile)
	defer db.Cleanup()
	if err != nil {
		config.Log <- fmt.Sprintf("Error initializing database: %s", err)
		config.Log <- "Quit"
		return
	}
	config.LocalVersion.Timestamp = time.Now().Round(time.Second)

	locVersion := objects.MakeFrame(objects.VERSION, objects.REQUEST, &config.LocalVersion)
	for str, _ := range config.NodeList.Nodes {
		locVersion.Peer = str
		config.SendQueue <- *locVersion
	}

	for {
		select {
		case frame = <-config.RecvQueue:
			config.Log <- fmt.Sprintf("Received %s frame...", CmdString(frame.Header.Command))
			switch frame.Header.Command {
			case objects.VERSION:
				version := new(objects.Version)
				err = version.FromBytes(frame.Payload)
				if err != nil {
					config.Log <- fmt.Sprintf("Error parsing version: %s", err)
				} else {
					go fVERSION(config, frame, version)
				}
			case objects.PEER:
				nodeList := new(objects.NodeList)
				err = nodeList.FromBytes(frame.Payload)
				if err != nil {
					config.Log <- fmt.Sprintf("Error parsing peer list: %s", err)
				} else {
					go fPEER(config, frame, nodeList)
				}
			case objects.OBJ:
				obj := new(objects.Obj)
				err = obj.FromBytes(frame.Payload)
				if err != nil {
					config.Log <- fmt.Sprintf("Error parsing obj list: %s", err)
				} else {
					go fOBJ(config, frame, obj)
				}
			case objects.GETOBJ:
				getObj := new(objects.Hash)
				if len(frame.Payload) == 0 {
					break
				}
				err = getObj.FromBytes(frame.Payload)
				if err != nil {
					config.Log <- fmt.Sprintf("Error parsing getobj hash: %s", err)
				} else {
					go fGETOBJ(config, frame, getObj)
				}
			case objects.PUBKEY_REQUEST:
				pubReq := new(objects.Hash)
				err = pubReq.FromBytes(frame.Payload)
				if err != nil {
					config.Log <- fmt.Sprintf("Error parsing pubkey request hash: %s", err)
				} else {
					go fPUBKEY_REQUEST(config, frame, pubReq)
				}
			case objects.PUBKEY:
				pub := new(objects.EncryptedPubkey)
				err = pub.FromBytes(frame.Payload)
				if err != nil {
					config.Log <- fmt.Sprintf("Error parsing pubkey: %s", err)
				} else {
					go fPUBKEY(config, frame, pub)
				}
			case objects.MSG:
				msg := new(objects.Message)
				err = msg.FromBytes(frame.Payload)
				if err != nil {
					config.Log <- fmt.Sprintf("Error parsing message: %s", err)
				} else {
					go fMSG(config, frame, msg)
				}
				fmt.Println("Finished select!")
			case objects.PURGE:
				purge := new(objects.Purge)
				err = purge.FromBytes(frame.Payload)
				if err != nil {
					config.Log <- fmt.Sprintf("Error parsing purge: %s", err)
				} else {
					go fPURGE(config, frame, purge)
				}
			default:
				config.Log <- fmt.Sprintf("Received invalid frame for command: %d", frame.Header.Command)
			}
		case <-config.Quit:
			fmt.Println()
			return
		}
	}

	// Should NEVER get here!
	panic("Must've been a cosmic ray!")
}

func quit(config *ApiConfig) {
	config.Log <- "Quit"
}
<|endoftext|>


<commit_before>package fmttab

import (
	"bytes"
	"fmt"
	"io"
	"strconv"
	"strings"
	"unicode/utf8"

	"math"
)

//A Border of table
type Border int

//A BorderKind type of element on the border of the table
type BorderKind int

//A Align text alignment in column of the table
type Align bool

const (
	//WidthAuto auto sizing of width column
	WidthAuto = 0
	//BorderNone table without borders
	BorderNone = Border(0)
	//BorderThin table with a thin border
	BorderThin = Border(1)
	//BorderDouble table with a double border
	BorderDouble = Border(2)
	//AlignLeft align text along the left edge
	AlignLeft = Align(false)
	//AlignRight align text along the right edge
	AlignRight = Align(true)
)

//The concrete type of the object on the border of the table
const (
	BKLeftTop BorderKind = iota
	BKRighttop
	BKRightBottom
	BKLeftBottom
	BKLeftToRight
	BKRightToLeft
	BKTopToBottom
	BKBottomToTop
	BKBottomCross
	BKHorizontal
	BKVertical
	BKHorizontalBorder
	BKVerticalBorder
)

const (
	trimend = ".."
)

//Borders predefined border types
var Borders = map[Border]map[BorderKind]string{
	BorderNone: map[BorderKind]string{
		BKVertical: " ",
	},
	BorderThin: map[BorderKind]string{
		BKLeftTop:          "\u250c",
		BKRighttop:         "\u2510",
		BKRightBottom:      "\u2518",
		BKLeftBottom:       "\u2514",
		BKLeftToRight:      "\u251c",
		BKRightToLeft:      "\u2524",
		BKTopToBottom:      "\u252c",
		BKBottomToTop:      "\u2534",
		BKBottomCross:      "\u253c",
		BKHorizontal:       "\u2500",
		BKVertical:         "\u2502",
		BKHorizontalBorder: "\u2500",
		BKVerticalBorder:   "\u2502",
	},
	BorderDouble: map[BorderKind]string{
		BKLeftTop:          "\u2554",
		BKRighttop:         "\u2557",
		BKRightBottom:      "\u255d",
		BKLeftBottom:       "\u255a",
		BKLeftToRight:      "\u255f",
		BKRightToLeft:      "\u2562",
		BKTopToBottom:      "\u2564",
		BKBottomToTop:      "\u2567",
		BKBottomCross:      "\u253c",
		BKHorizontal:       "\u2500",
		BKVertical:         "\u2502",
		BKHorizontalBorder: "\u2550",
		BKVerticalBorder:   "\u2551",
	},
}

//A DataGetter functional type for table data
type DataGetter func() (bool, map[string]interface{})

//A Column type of table columns
type Column struct {
	maxLen int
	Name   string
	Width  int
	Aling  Align
}

//A Table is the repository for the columns, the data that are used for printing the table
type Table struct {
	dataget  DataGetter
	border   Border
	caption  string
	autoSize int
	Columns  []*Column
	Data     []map[string]interface{}
}

// A trimEnds supplements the text with special characters by limiting the length of the text column width
func trimEnds(val, end string, max int) string {
	l := utf8.RuneCountInString(val)
	if l <= max {
		return val
	}
	lend := utf8.RuneCountInString(end)
	if lend >= max {
		return end[:max]
	}
	return string([]rune(val)[:(max-lend)]) + end
}

//GetMaskFormat returns a pattern string for formatting text in table column alignment
func (t *Table) GetMaskFormat(c *Column) string {
	if c.Aling == AlignLeft {
		return "%-" + strconv.Itoa(t.getWidth(c)) + "v"
	}
	return "%" + strconv.Itoa(t.getWidth(c)) + "v"
}

//must be calculated before call
func (t *Table) getWidth(c *Column) int {
	if c.Width == WidthAuto || t.autoSize > 0 {
		return c.maxLen
	}
	return c.Width
}

//AddColumn adds a column to the table
func (t *Table) AddColumn(name string, width int, aling Align) *Table {
	//TODO: check dublicate
	t.Columns = append(t.Columns, &Column{
		Name:  name,
		Width: width,
		Aling: aling,
	})
	return t
}

//AppendData adds the data to the table
func (t *Table) AppendData(rec map[string]interface{}) *Table {
	t.Data = append(t.Data, rec)
	return t
}

//ClearData removes data from a table
func (t *Table) ClearData() *Table {
	t.Data = nil
	return t
}

//AutoSize fit columns
func (t *Table) AutoSize(enabled bool, destWidth int) {
	if enabled {
		t.autoSize = destWidth
	} else {
		t.autoSize = 0
	}
}

//CountData the amount of data in the table
func (t *Table) CountData() int {
	return len(t.Data)
}

func (t *Table) writeHeader(w io.Writer) (int, error) {
	var cntwrite int
	dataout := ""
	if t.caption != "" {
		dataout += t.caption + "\n"
	}
	dataout += Borders[t.border][BKLeftTop]
	cntCols := len(t.Columns)
	for num, c := range t.Columns {
		dataout += strings.Repeat(Borders[t.border][BKHorizontalBorder], t.getWidth(c))
		var delim string
		if num < cntCols-1 {
			delim = Borders[t.border][BKTopToBottom]
		} else {
			delim = Borders[t.border][BKRighttop] + "\n"
		}
		dataout += delim
	}
	dataout += Borders[t.border][BKVerticalBorder]
	for num, c := range t.Columns {
		caption := fmt.Sprintf(t.GetMaskFormat(c), c.Name)
		dataout += trimEnds(caption, trimend, t.getWidth(c))
		var delim string
		if num < cntCols-1 {
			delim = Borders[t.border][BKVertical]
		} else {
			delim = Borders[t.border][BKVerticalBorder]
		}
		dataout += delim
	}
	dataout += "\n" + Borders[t.border][BKLeftToRight]
	dataout += t.writeBorderTopButtomData(BKHorizontal, BKBottomCross, BKRightToLeft)
	if n, err := w.Write([]byte(dataout)); err == nil {
		cntwrite += n
	} else {
		return -1, err
	}
	return cntwrite, nil
}

func (t *Table) writeBorderTopButtomData(hr, vbwnCol, vright BorderKind) (data string) {
	cntCols := len(t.Columns)
	empty := true
	for num, c := range t.Columns {
		s := strings.Repeat(Borders[t.border][hr], t.getWidth(c))
		if len(s) > 0 {
			empty = false
		}
		data += s
		var delim string
		if num < cntCols-1 {
			delim = Borders[t.border][vbwnCol]
		} else {
			delim = Borders[t.border][vright]
			if !empty {
				delim += "\n"
			}
		}
		if len(delim) > 0 {
			empty = false
		}
		data += delim
	}
	return data
}

func (t *Table) writeBottomBorder(w io.Writer) (int, error) {
	var cntwrite int
	data := Borders[t.border][BKLeftBottom] + t.writeBorderTopButtomData(BKHorizontalBorder, BKBottomToTop, BKRightBottom)
	if n, err := w.Write([]byte(data)); err == nil {
		cntwrite = n
	} else {
		return -1, err
	}
	return cntwrite, nil
}

func (t *Table) writeRecord(data map[string]interface{}, w io.Writer) (int, error) {
	var cntwrite int
	cntCols := len(t.Columns)
	if n, err := w.Write([]byte(Borders[t.border][BKVerticalBorder])); err == nil {
		cntwrite += n
	} else {
		return -1, err
	}
	for num, c := range t.Columns {
		val, mok := data[c.Name]
		if !mok || val == nil {
			val = ""
		}
		caption := fmt.Sprintf(t.GetMaskFormat(c), val)
		if n, err := w.Write([]byte(trimEnds(caption, trimend, t.getWidth(c)))); err == nil {
			cntwrite += n
		} else {
			return -1, err
		}
		var delim string
		if num < cntCols-1 {
			delim = Borders[t.border][BKVertical]
		} else {
			delim = Borders[t.border][BKVerticalBorder]
		}
		if n, err := w.Write([]byte(delim)); err == nil {
			cntwrite += n
		} else {
			return -1, err
		}
	}
	if n, err := w.Write([]byte("\n")); err == nil {
		cntwrite += n
	} else {
		return -1, err
	}
	return cntwrite, nil
}

func (t *Table) writeData(w io.Writer) (int, error) {
	var cntwrite int
	if t.dataget != nil {
		for {
			ok, data := t.dataget()
			if !ok {
				break
			}
			if n, err := t.writeRecord(data, w); err == nil {
				cntwrite += n
			} else {
				return -1, err
			}
		}
	} else if t.CountData() != 0 {
		for _, data := range t.Data {
			if n, err := t.writeRecord(data, w); err == nil {
				cntwrite += n
			} else {
				return -1, err
			}
		}
	}
	return cntwrite, nil
}

//New creates a Table object. DataGetter can be nil
func New(caption string, border Border, datagetter DataGetter) *Table {
	return &Table{
		caption: caption,
		border:  border,
		dataget: datagetter,
	}
}

// String returns the contents of the table with borders
// as a string.  If error, it returns "".
func (t *Table) String() string {
	var buf bytes.Buffer
	if _, err := t.WriteTo(&buf); err != nil {
		return ""
	}
	return buf.String()
}

func (t *Table) autoWidth() error {
	//each column
	var wa []*Column
	for i := range t.Columns {
		if t.Columns[i].Width == WidthAuto || t.autoSize > 0 {
			t.Columns[i].maxLen = len(t.Columns[i].Name)
			wa = append(wa, t.Columns[i])
		}
	}
	if len(wa) == 0 {
		return nil
	}
	for _, data := range t.Data {
		for i := range wa {
			curval := fmt.Sprintf("%v", data[wa[i].Name])
			curlen := len(curval)
			if curlen > wa[i].maxLen {
				wa[i].maxLen = curlen
			}
		}
	}
	//autosize table
	if t.autoSize > 0 {
		termwidth := t.autoSize - utf8.RuneCountInString(Borders[t.border][BKVertical]) * len(t.Columns) - utf8.RuneCountInString(Borders[t.border][BKVerticalBorder])*2
		nowwidths := make([]int, len(t.Columns))
		allcolswidth := 0
		for i := range t.Columns {
			if t.Columns[i].maxLen > t.Columns[i].Width || t.Columns[i].Width == WidthAuto {
				nowwidths[i] = t.Columns[i].maxLen
			} else {
				nowwidths[i] = t.Columns[i].Width
			}
			allcolswidth += nowwidths[i]
		}
		//todo: allcolswidth - borders
		twAll := 0
		for i := range t.Columns {
			t.Columns[i].maxLen = int(math.Trunc(float64(termwidth) * (float64(nowwidths[i]) / float64(allcolswidth))))
			twAll += t.Columns[i].maxLen
		}
		i := 0
		//distrib mod
		for {
			if twAll >= termwidth || twAll <= 0 {
				break
			}
			if i+1 >= len(t.Columns) {
				i = 0
			}
			t.Columns[i].maxLen = t.Columns[i].maxLen + 1

			twAll = twAll + 1
			i = i + 1
		}
	}

	return nil
}

// WriteTo writes data to w until the buffer is drained or an error occurs.
// The return value n is the number of bytes written; it always fits into an
// int, but it is int64 to match the io.WriterTo interface. Any error
// encountered during the write is also returned.
func (t *Table) WriteTo(w io.Writer) (int64, error) {
	if len(t.Columns) == 0 {
		return 0, nil
	}
	if err := t.autoWidth(); err != nil {
		return 0, err
	}
	var cntwrite int64
	if n, err := t.writeHeader(w); err == nil {
		cntwrite += int64(n)
	} else {
		return -1, err
	}
	if n, err := t.writeData(w); err == nil {
		cntwrite += int64(n)
	} else {
		return -1, err
	}
	if n, err := t.writeBottomBorder(w); err == nil {
		cntwrite += int64(n)
	} else {
		return -1, err
	}
	return cntwrite, nil
}
<commit_msg>func SetBorder<commit_after>package fmttab

import (
	"bytes"
	"fmt"
	"io"
	"strconv"
	"strings"
	"unicode/utf8"

	"math"
)

//A Border of table
type Border int

//A BorderKind type of element on the border of the table
type BorderKind int

//A Align text alignment in column of the table
type Align bool

const (
	//WidthAuto auto sizing of width column
	WidthAuto = 0
	//BorderNone table without borders
	BorderNone = Border(0)
	//BorderThin table with a thin border
	BorderThin = Border(1)
	//BorderDouble table with a double border
	BorderDouble = Border(2)
	//AlignLeft align text along the left edge
	AlignLeft = Align(false)
	//AlignRight align text along the right edge
	AlignRight = Align(true)
)

//The concrete type of the object on the border of the table
const (
	BKLeftTop BorderKind = iota
	BKRighttop
	BKRightBottom
	BKLeftBottom
	BKLeftToRight
	BKRightToLeft
	BKTopToBottom
	BKBottomToTop
	BKBottomCross
	BKHorizontal
	BKVertical
	BKHorizontalBorder
	BKVerticalBorder
)

const (
	trimend = ".."
)

//Borders predefined border types
var Borders = map[Border]map[BorderKind]string{
	BorderNone: map[BorderKind]string{
		BKVertical: " ",
	},
	BorderThin: map[BorderKind]string{
		BKLeftTop:          "\u250c",
		BKRighttop:         "\u2510",
		BKRightBottom:      "\u2518",
		BKLeftBottom:       "\u2514",
		BKLeftToRight:      "\u251c",
		BKRightToLeft:      "\u2524",
		BKTopToBottom:      "\u252c",
		BKBottomToTop:      "\u2534",
		BKBottomCross:      "\u253c",
		BKHorizontal:       "\u2500",
		BKVertical:         "\u2502",
		BKHorizontalBorder: "\u2500",
		BKVerticalBorder:   "\u2502",
	},
	BorderDouble: map[BorderKind]string{
		BKLeftTop:          "\u2554",
		BKRighttop:         "\u2557",
		BKRightBottom:      "\u255d",
		BKLeftBottom:       "\u255a",
		BKLeftToRight:      "\u255f",
		BKRightToLeft:      "\u2562",
		BKTopToBottom:      "\u2564",
		BKBottomToTop:      "\u2567",
		BKBottomCross:      "\u253c",
		BKHorizontal:       "\u2500",
		BKVertical:         "\u2502",
		BKHorizontalBorder: "\u2550",
		BKVerticalBorder:   "\u2551",
	},
}

//A DataGetter functional type for table data
type DataGetter func() (bool, map[string]interface{})

//A Column type of table columns
type Column struct {
	maxLen int
	Name   string
	Width  int
	Aling  Align
}

//A Table is the repository for the columns, the data that are used for printing the table
type Table struct {
	dataget  DataGetter
	border   Border
	caption  string
	autoSize int
	Columns  []*Column
	Data     []map[string]interface{}
}

// A trimEnds supplements the text with special characters by limiting the length of the text column width
func trimEnds(val, end string, max int) string {
	l := utf8.RuneCountInString(val)
	if l <= max {
		return val
	}
	lend := utf8.RuneCountInString(end)
	if lend >= max {
		return end[:max]
	}
	return string([]rune(val)[:(max-lend)]) + end
}

//GetMaskFormat returns a pattern string for formatting text in table column alignment
func (t *Table) GetMaskFormat(c *Column) string {
	if c.Aling == AlignLeft {
		return "%-" + strconv.Itoa(t.getWidth(c)) + "v"
	}
	return "%" + strconv.Itoa(t.getWidth(c)) + "v"
}

//must be calculated before call
func (t *Table) getWidth(c *Column) int {
	if c.Width == WidthAuto || t.autoSize > 0 {
		return c.maxLen
	}
	return c.Width
}

//AddColumn adds a column to the table
func (t *Table) AddColumn(name string, width int, aling Align) *Table {
	//TODO: check dublicate
	t.Columns = append(t.Columns, &Column{
		Name:  name,
		Width: width,
		Aling: aling,
	})
	return t
}

//AppendData adds the data to the table
func (t *Table) AppendData(rec map[string]interface{}) *Table {
	t.Data = append(t.Data, rec)
	return t
}

//ClearData removes data from a table
func (t *Table) ClearData() *Table {
	t.Data = nil
	return t
}

//AutoSize fit columns
func (t *Table) AutoSize(enabled bool, destWidth int) {
	if enabled {
		t.autoSize = destWidth
	} else {
		t.autoSize = 0
	}
}

//CountData the amount of data in the table
func (t *Table) CountData() int {
	return len(t.Data)
}

//SetBorder - set  type of border table
func (t *Table) SetBorder(b Border) {
	t.border = b
}

func (t *Table) writeHeader(w io.Writer) (int, error) {
	var cntwrite int
	dataout := ""
	if t.caption != "" {
		dataout += t.caption + "\n"
	}
	dataout += Borders[t.border][BKLeftTop]
	cntCols := len(t.Columns)
	for num, c := range t.Columns {
		dataout += strings.Repeat(Borders[t.border][BKHorizontalBorder], t.getWidth(c))
		var delim string
		if num < cntCols-1 {
			delim = Borders[t.border][BKTopToBottom]
		} else {
			delim = Borders[t.border][BKRighttop] + "\n"
		}
		dataout += delim
	}
	dataout += Borders[t.border][BKVerticalBorder]
	for num, c := range t.Columns {
		caption := fmt.Sprintf(t.GetMaskFormat(c), c.Name)
		dataout += trimEnds(caption, trimend, t.getWidth(c))
		var delim string
		if num < cntCols-1 {
			delim = Borders[t.border][BKVertical]
		} else {
			delim = Borders[t.border][BKVerticalBorder]
		}
		dataout += delim
	}
	dataout += "\n" + Borders[t.border][BKLeftToRight]
	dataout += t.writeBorderTopButtomData(BKHorizontal, BKBottomCross, BKRightToLeft)
	if n, err := w.Write([]byte(dataout)); err == nil {
		cntwrite += n
	} else {
		return -1, err
	}
	return cntwrite, nil
}

func (t *Table) writeBorderTopButtomData(hr, vbwnCol, vright BorderKind) (data string) {
	cntCols := len(t.Columns)
	empty := true
	for num, c := range t.Columns {
		s := strings.Repeat(Borders[t.border][hr], t.getWidth(c))
		if len(s) > 0 {
			empty = false
		}
		data += s
		var delim string
		if num < cntCols-1 {
			delim = Borders[t.border][vbwnCol]
		} else {
			delim = Borders[t.border][vright]
			if !empty {
				delim += "\n"
			}
		}
		if len(delim) > 0 {
			empty = false
		}
		data += delim
	}
	return data
}

func (t *Table) writeBottomBorder(w io.Writer) (int, error) {
	var cntwrite int
	data := Borders[t.border][BKLeftBottom] + t.writeBorderTopButtomData(BKHorizontalBorder, BKBottomToTop, BKRightBottom)
	if n, err := w.Write([]byte(data)); err == nil {
		cntwrite = n
	} else {
		return -1, err
	}
	return cntwrite, nil
}

func (t *Table) writeRecord(data map[string]interface{}, w io.Writer) (int, error) {
	var cntwrite int
	cntCols := len(t.Columns)
	if n, err := w.Write([]byte(Borders[t.border][BKVerticalBorder])); err == nil {
		cntwrite += n
	} else {
		return -1, err
	}
	for num, c := range t.Columns {
		val, mok := data[c.Name]
		if !mok || val == nil {
			val = ""
		}
		caption := fmt.Sprintf(t.GetMaskFormat(c), val)
		if n, err := w.Write([]byte(trimEnds(caption, trimend, t.getWidth(c)))); err == nil {
			cntwrite += n
		} else {
			return -1, err
		}
		var delim string
		if num < cntCols-1 {
			delim = Borders[t.border][BKVertical]
		} else {
			delim = Borders[t.border][BKVerticalBorder]
		}
		if n, err := w.Write([]byte(delim)); err == nil {
			cntwrite += n
		} else {
			return -1, err
		}
	}
	if n, err := w.Write([]byte("\n")); err == nil {
		cntwrite += n
	} else {
		return -1, err
	}
	return cntwrite, nil
}

func (t *Table) writeData(w io.Writer) (int, error) {
	var cntwrite int
	if t.dataget != nil {
		for {
			ok, data := t.dataget()
			if !ok {
				break
			}
			if n, err := t.writeRecord(data, w); err == nil {
				cntwrite += n
			} else {
				return -1, err
			}
		}
	} else if t.CountData() != 0 {
		for _, data := range t.Data {
			if n, err := t.writeRecord(data, w); err == nil {
				cntwrite += n
			} else {
				return -1, err
			}
		}
	}
	return cntwrite, nil
}

//New creates a Table object. DataGetter can be nil
func New(caption string, border Border, datagetter DataGetter) *Table {
	return &Table{
		caption: caption,
		border:  border,
		dataget: datagetter,
	}
}

// String returns the contents of the table with borders
// as a string.  If error, it returns "".
func (t *Table) String() string {
	var buf bytes.Buffer
	if _, err := t.WriteTo(&buf); err != nil {
		return ""
	}
	return buf.String()
}

func (t *Table) autoWidth() error {
	//each column
	var wa []*Column
	for i := range t.Columns {
		if t.Columns[i].Width == WidthAuto || t.autoSize > 0 {
			t.Columns[i].maxLen = len(t.Columns[i].Name)
			wa = append(wa, t.Columns[i])
		}
	}
	if len(wa) == 0 {
		return nil
	}
	for _, data := range t.Data {
		for i := range wa {
			curval := fmt.Sprintf("%v", data[wa[i].Name])
			curlen := len(curval)
			if curlen > wa[i].maxLen {
				wa[i].maxLen = curlen
			}
		}
	}
	//autosize table
	if t.autoSize > 0 {
		termwidth := t.autoSize - utf8.RuneCountInString(Borders[t.border][BKVertical])*len(t.Columns) - utf8.RuneCountInString(Borders[t.border][BKVerticalBorder])*2
		nowwidths := make([]int, len(t.Columns))
		allcolswidth := 0
		for i := range t.Columns {
			if t.Columns[i].maxLen > t.Columns[i].Width || t.Columns[i].Width == WidthAuto {
				nowwidths[i] = t.Columns[i].maxLen
			} else {
				nowwidths[i] = t.Columns[i].Width
			}
			allcolswidth += nowwidths[i]
		}
		//todo: allcolswidth - borders
		twAll := 0
		for i := range t.Columns {
			t.Columns[i].maxLen = int(math.Trunc(float64(termwidth) * (float64(nowwidths[i]) / float64(allcolswidth))))
			twAll += t.Columns[i].maxLen
		}
		i := 0
		//distrib mod
		for {
			if twAll >= termwidth || twAll <= 0 {
				break
			}
			if i+1 >= len(t.Columns) {
				i = 0
			}
			t.Columns[i].maxLen = t.Columns[i].maxLen + 1

			twAll = twAll + 1
			i = i + 1
		}
	}

	return nil
}

// WriteTo writes data to w until the buffer is drained or an error occurs.
// The return value n is the number of bytes written; it always fits into an
// int, but it is int64 to match the io.WriterTo interface. Any error
// encountered during the write is also returned.
func (t *Table) WriteTo(w io.Writer) (int64, error) {
	if len(t.Columns) == 0 {
		return 0, nil
	}
	if err := t.autoWidth(); err != nil {
		return 0, err
	}
	var cntwrite int64
	if n, err := t.writeHeader(w); err == nil {
		cntwrite += int64(n)
	} else {
		return -1, err
	}
	if n, err := t.writeData(w); err == nil {
		cntwrite += int64(n)
	} else {
		return -1, err
	}
	if n, err := t.writeBottomBorder(w); err == nil {
		cntwrite += int64(n)
	} else {
		return -1, err
	}
	return cntwrite, nil
}
<|endoftext|>


<commit_before>package otaru

import (
	"fmt"
	"log"

	"github.com/nyaxt/otaru/blobstore"
	"github.com/nyaxt/otaru/btncrypt"
	fl "github.com/nyaxt/otaru/flags"
	"github.com/nyaxt/otaru/inodedb"
	. "github.com/nyaxt/otaru/util" // FIXME
)

const (
	ChunkSplitSize = 256 * 1024 * 1024 // 256MB
)

const (
	NewChunk      = true
	ExistingChunk = false
)

type ChunksArrayIO interface {
	Read() ([]inodedb.FileChunk, error)
	Write(cs []inodedb.FileChunk) error
}

type ChunkedFileIO struct {
	bs blobstore.RandomAccessBlobStore
	c  btncrypt.Cipher

	caio ChunksArrayIO

	newChunkIO func(blobstore.BlobHandle, btncrypt.Cipher) blobstore.BlobHandle
}

func NewChunkedFileIO(bs blobstore.RandomAccessBlobStore, c btncrypt.Cipher, caio ChunksArrayIO) *ChunkedFileIO {
	return &ChunkedFileIO{
		bs: bs,
		c:  c,

		caio: caio,

		newChunkIO: func(bh blobstore.BlobHandle, c btncrypt.Cipher) blobstore.BlobHandle { return NewChunkIO(bh, c) },
	}
}

func (cfio *ChunkedFileIO) OverrideNewChunkIOForTesting(newChunkIO func(blobstore.BlobHandle, btncrypt.Cipher) blobstore.BlobHandle) {
	cfio.newChunkIO = newChunkIO
}

func (cfio *ChunkedFileIO) newFileChunk(newo int64) (inodedb.FileChunk, error) {
	bpath, err := blobstore.GenerateNewBlobPath(cfio.bs)
	if err != nil {
		return inodedb.FileChunk{}, err
	}
	fc := inodedb.FileChunk{Offset: newo, Length: 0, BlobPath: bpath}
	fmt.Printf("new chunk %v\n", fc)
	return fc, nil
}

func (cfio *ChunkedFileIO) PWrite(offset int64, p []byte) error {
	remo := offset
	remp := p
	if len(remp) == 0 {
		return nil
	}

	cs, err := cfio.caio.Read()
	if err != nil {
		return fmt.Errorf("Failed to read cs array: %v", err)
	}
	csUpdated := false

	writeToChunk := func(c *inodedb.FileChunk, isNewChunk bool, maxChunkLen int64) error {
		if !fl.IsReadWriteAllowed(cfio.bs.Flags()) {
			return EPERM
		}

		flags := fl.O_RDWR
		if isNewChunk {
			flags |= fl.O_CREATE | fl.O_EXCL
		}
		bh, err := cfio.bs.Open(c.BlobPath, flags)
		if err != nil {
			return fmt.Errorf("Failed to open path \"%s\" for writing (isNewChunk: %t): %v", c.BlobPath, isNewChunk, err)
		}
		cio := cfio.newChunkIO(bh, cfio.c)

		coff := remo - c.Offset
		n := IntMin(len(remp), int(maxChunkLen-coff))
		if n < 0 {
			return nil
		}
		if err := cio.PWrite(coff, remp[:n]); err != nil {
			return err
		}
		if err := cio.Close(); err != nil {
			return err
		}
		oldLength := c.Length
		c.Length = int64(cio.Size())
		if oldLength != c.Length {
			csUpdated = true
		}

		remo += int64(n)
		remp = remp[n:]
		return nil
	}

	for i := 0; i < len(cs); i++ {
		c := &cs[i]
		if c.Left() > remo {
			// Insert a new chunk @ i

			// try best to align offset at ChunkSplitSize
			newo := remo / ChunkSplitSize * ChunkSplitSize
			maxlen := int64(ChunkSplitSize)
			if i > 0 {
				prev := cs[i-1]
				pright := prev.Right()
				if newo < pright {
					maxlen -= pright - newo
					newo = pright
				}
			}
			if i < len(cs)-1 {
				next := cs[i+1]
				if newo+maxlen > next.Left() {
					maxlen = next.Left() - newo
				}
			}

			newc, err := cfio.newFileChunk(newo)
			if err != nil {
				return err
			}
			cs = append(cs, inodedb.FileChunk{})
			copy(cs[i+1:], cs[i:])
			cs[i] = newc
			csUpdated = true

			if err := writeToChunk(&newc, NewChunk, maxlen); err != nil {
				return err
			}
			if len(remp) == 0 {
				break
			}

			continue
		}

		// Write to the chunk
		maxlen := int64(ChunkSplitSize)
		if i < len(cs)-1 {
			next := cs[i+1]
			if c.Left()+maxlen > next.Left() {
				maxlen = next.Left() - c.Left()
			}
		}
		if err := writeToChunk(c, ExistingChunk, maxlen); err != nil {
			return err
		}
		if len(remp) == 0 {
			break
		}
	}

	for len(remp) > 0 {
		// Append a new chunk at the end
		newo := remo / ChunkSplitSize * ChunkSplitSize
		maxlen := int64(ChunkSplitSize)

		if len(cs) > 0 {
			last := cs[len(cs)-1]
			lastRight := last.Right()
			if newo < lastRight {
				maxlen -= lastRight - newo
				newo = lastRight
			}
		}

		newc, err := cfio.newFileChunk(newo)
		if err != nil {
			return err
		}
		if err := writeToChunk(&newc, NewChunk, maxlen); err != nil {
			return err
		}

		cs = append(cs, newc)
		csUpdated = true
	}

	if csUpdated {
		if err := cfio.caio.Write(cs); err != nil {
			return fmt.Errorf("Failed to write updated cs array: %v", err)
		}
	}
	return nil
}

func (cfio *ChunkedFileIO) PRead(offset int64, p []byte) error {
	remo := offset
	remp := p

	if offset < 0 {
		return fmt.Errorf("negative offset %d given", offset)
	}

	cs, err := cfio.caio.Read()
	if err != nil {
		return fmt.Errorf("Failed to read cs array: %v", err)
	}
	// fmt.Printf("cs: %v\n", cs)
	for i := 0; i < len(cs) && len(remp) > 0; i++ {
		c := cs[i]
		if c.Left() > remo+int64(len(remp)) {
			break
		}
		if c.Right() <= remo {
			continue
		}

		coff := remo - c.Left()
		if coff < 0 {
			// Fill gap with zero
			n := Int64Min(int64(len(remp)), -coff)
			for j := int64(0); j < n; j++ {
				remp[j] = 0
			}
			remo += n
			coff = 0
			if len(remp) == 0 {
				return nil
			}
		}

		if !fl.IsReadAllowed(cfio.bs.Flags()) {
			return EPERM
		}

		bh, err := cfio.bs.Open(c.BlobPath, fl.O_RDONLY)
		if err != nil {
			return fmt.Errorf("Failed to open path \"%s\" for reading: %v", c.BlobPath, err)
		}
		cio := cfio.newChunkIO(bh, cfio.c)

		n := Int64Min(int64(len(p)), c.Length-coff)
		if err := cio.PRead(coff, remp[:n]); err != nil {
			return err
		}
		if err := cio.Close(); err != nil {
			return err
		}

		remo += n
		remp = remp[n:]

		if len(remp) == 0 {
			return nil
		}
	}

	log.Printf("cs: %+v", cs)
	return fmt.Errorf("Attempt to read over file size by %d", len(remp))
}

func (cfio *ChunkedFileIO) Size() int64 {
	cs, err := cfio.caio.Read()
	if err != nil {
		log.Printf("Failed to read cs array: %v", err)
		return 0
	}
	if len(cs) == 0 {
		return 0
	}
	return cs[len(cs)-1].Right()
}

func (cfio *ChunkedFileIO) Close() error {
	return nil
}

func (cfio *ChunkedFileIO) Truncate(size int64) error {
	if !fl.IsReadWriteAllowed(cfio.bs.Flags()) {
		return EPERM
	}

	cs, err := cfio.caio.Read()
	if err != nil {
		return fmt.Errorf("Failed to read cs array: %v", err)
	}

	for i := len(cs) - 1; i >= 0; i-- {
		c := &cs[i]

		if c.Left() >= size {
			// drop the chunk
			continue
		}

		if c.Right() > size {
			// trim the chunk
			chunksize := size - c.Left()

			bh, err := cfio.bs.Open(c.BlobPath, fl.O_RDWR)
			if err != nil {
				return err
			}
			cio := cfio.newChunkIO(bh, cfio.c)
			if err := cio.Truncate(chunksize); err != nil {
				return err
			}
			if err := cio.Close(); err != nil {
				return err
			}
			c.Length = int64(cio.Size())
		}

		cs = cs[:i+1]
		if err := cfio.caio.Write(cs); err != nil {
			return fmt.Errorf("Failed to write updated cs array: %v", err)
		}
		return nil
	}
	if err := cfio.caio.Write([]inodedb.FileChunk{}); err != nil {
		return fmt.Errorf("Failed to write updated cs array (empty): %v", err)
	}
	return nil
}
<commit_msg>chunkedfileio: close handles<commit_after>package otaru

import (
	"fmt"
	"log"

	"github.com/nyaxt/otaru/blobstore"
	"github.com/nyaxt/otaru/btncrypt"
	fl "github.com/nyaxt/otaru/flags"
	"github.com/nyaxt/otaru/inodedb"
	. "github.com/nyaxt/otaru/util" // FIXME
)

const (
	ChunkSplitSize = 256 * 1024 * 1024 // 256MB
)

const (
	NewChunk      = true
	ExistingChunk = false
)

type ChunksArrayIO interface {
	Read() ([]inodedb.FileChunk, error)
	Write(cs []inodedb.FileChunk) error
}

type ChunkedFileIO struct {
	bs blobstore.RandomAccessBlobStore
	c  btncrypt.Cipher

	caio ChunksArrayIO

	newChunkIO func(blobstore.BlobHandle, btncrypt.Cipher) blobstore.BlobHandle
}

func NewChunkedFileIO(bs blobstore.RandomAccessBlobStore, c btncrypt.Cipher, caio ChunksArrayIO) *ChunkedFileIO {
	return &ChunkedFileIO{
		bs: bs,
		c:  c,

		caio: caio,

		newChunkIO: func(bh blobstore.BlobHandle, c btncrypt.Cipher) blobstore.BlobHandle { return NewChunkIO(bh, c) },
	}
}

func (cfio *ChunkedFileIO) OverrideNewChunkIOForTesting(newChunkIO func(blobstore.BlobHandle, btncrypt.Cipher) blobstore.BlobHandle) {
	cfio.newChunkIO = newChunkIO
}

func (cfio *ChunkedFileIO) newFileChunk(newo int64) (inodedb.FileChunk, error) {
	bpath, err := blobstore.GenerateNewBlobPath(cfio.bs)
	if err != nil {
		return inodedb.FileChunk{}, err
	}
	fc := inodedb.FileChunk{Offset: newo, Length: 0, BlobPath: bpath}
	fmt.Printf("new chunk %v\n", fc)
	return fc, nil
}

func (cfio *ChunkedFileIO) PWrite(offset int64, p []byte) error {
	remo := offset
	remp := p
	if len(remp) == 0 {
		return nil
	}

	cs, err := cfio.caio.Read()
	if err != nil {
		return fmt.Errorf("Failed to read cs array: %v", err)
	}
	csUpdated := false

	writeToChunk := func(c *inodedb.FileChunk, isNewChunk bool, maxChunkLen int64) error {
		if !fl.IsReadWriteAllowed(cfio.bs.Flags()) {
			return EPERM
		}

		flags := fl.O_RDWR
		if isNewChunk {
			flags |= fl.O_CREATE | fl.O_EXCL
		}
		bh, err := cfio.bs.Open(c.BlobPath, flags)
		if err != nil {
			return fmt.Errorf("Failed to open path \"%s\" for writing (isNewChunk: %t): %v", c.BlobPath, isNewChunk, err)
		}
		defer func() {
			if err := bh.Close(); err != nil {
				log.Printf("blobhandle Close failed: %v", err)
			}
		}()

		cio := cfio.newChunkIO(bh, cfio.c)
		defer func() {
			if err := cio.Close(); err != nil {
				log.Printf("cio Close failed: %v", err)
			}
		}()

		coff := remo - c.Offset
		n := IntMin(len(remp), int(maxChunkLen-coff))
		if n < 0 {
			return nil
		}
		if err := cio.PWrite(coff, remp[:n]); err != nil {
			return err
		}
		oldLength := c.Length
		c.Length = int64(cio.Size())
		if oldLength != c.Length {
			csUpdated = true
		}

		remo += int64(n)
		remp = remp[n:]
		return nil
	}

	for i := 0; i < len(cs); i++ {
		c := &cs[i]
		if c.Left() > remo {
			// Insert a new chunk @ i

			// try best to align offset at ChunkSplitSize
			newo := remo / ChunkSplitSize * ChunkSplitSize
			maxlen := int64(ChunkSplitSize)
			if i > 0 {
				prev := cs[i-1]
				pright := prev.Right()
				if newo < pright {
					maxlen -= pright - newo
					newo = pright
				}
			}
			if i < len(cs)-1 {
				next := cs[i+1]
				if newo+maxlen > next.Left() {
					maxlen = next.Left() - newo
				}
			}

			newc, err := cfio.newFileChunk(newo)
			if err != nil {
				return err
			}
			cs = append(cs, inodedb.FileChunk{})
			copy(cs[i+1:], cs[i:])
			cs[i] = newc
			csUpdated = true

			if err := writeToChunk(&newc, NewChunk, maxlen); err != nil {
				return err
			}
			if len(remp) == 0 {
				break
			}

			continue
		}

		// Write to the chunk
		maxlen := int64(ChunkSplitSize)
		if i < len(cs)-1 {
			next := cs[i+1]
			if c.Left()+maxlen > next.Left() {
				maxlen = next.Left() - c.Left()
			}
		}
		if err := writeToChunk(c, ExistingChunk, maxlen); err != nil {
			return err
		}
		if len(remp) == 0 {
			break
		}
	}

	for len(remp) > 0 {
		// Append a new chunk at the end
		newo := remo / ChunkSplitSize * ChunkSplitSize
		maxlen := int64(ChunkSplitSize)

		if len(cs) > 0 {
			last := cs[len(cs)-1]
			lastRight := last.Right()
			if newo < lastRight {
				maxlen -= lastRight - newo
				newo = lastRight
			}
		}

		newc, err := cfio.newFileChunk(newo)
		if err != nil {
			return err
		}
		if err := writeToChunk(&newc, NewChunk, maxlen); err != nil {
			return err
		}

		cs = append(cs, newc)
		csUpdated = true
	}

	if csUpdated {
		if err := cfio.caio.Write(cs); err != nil {
			return fmt.Errorf("Failed to write updated cs array: %v", err)
		}
	}
	return nil
}

func (cfio *ChunkedFileIO) PRead(offset int64, p []byte) error {
	remo := offset
	remp := p

	if offset < 0 {
		return fmt.Errorf("negative offset %d given", offset)
	}

	cs, err := cfio.caio.Read()
	if err != nil {
		return fmt.Errorf("Failed to read cs array: %v", err)
	}
	// fmt.Printf("cs: %v\n", cs)
	for i := 0; i < len(cs) && len(remp) > 0; i++ {
		c := cs[i]
		if c.Left() > remo+int64(len(remp)) {
			break
		}
		if c.Right() <= remo {
			continue
		}

		coff := remo - c.Left()
		if coff < 0 {
			// Fill gap with zero
			n := Int64Min(int64(len(remp)), -coff)
			for j := int64(0); j < n; j++ {
				remp[j] = 0
			}
			remo += n
			coff = 0
			if len(remp) == 0 {
				return nil
			}
		}

		if !fl.IsReadAllowed(cfio.bs.Flags()) {
			return EPERM
		}

		bh, err := cfio.bs.Open(c.BlobPath, fl.O_RDONLY)
		if err != nil {
			return fmt.Errorf("Failed to open path \"%s\" for reading: %v", c.BlobPath, err)
		}
		defer func() {
			if err := bh.Close(); err != nil {
				log.Printf("blobhandle Close failed: %v", err)
			}
		}()

		cio := cfio.newChunkIO(bh, cfio.c)
		defer func() {
			if err := cio.Close(); err != nil {
				log.Printf("cio Close failed: %v", err)
			}
		}()

		n := Int64Min(int64(len(p)), c.Length-coff)
		if err := cio.PRead(coff, remp[:n]); err != nil {
			return err
		}

		remo += n
		remp = remp[n:]

		if len(remp) == 0 {
			return nil
		}
	}

	log.Printf("cs: %+v", cs)
	return fmt.Errorf("Attempt to read over file size by %d", len(remp))
}

func (cfio *ChunkedFileIO) Size() int64 {
	cs, err := cfio.caio.Read()
	if err != nil {
		log.Printf("Failed to read cs array: %v", err)
		return 0
	}
	if len(cs) == 0 {
		return 0
	}
	return cs[len(cs)-1].Right()
}

func (cfio *ChunkedFileIO) Close() error {
	return nil
}

func (cfio *ChunkedFileIO) Truncate(size int64) error {
	if !fl.IsReadWriteAllowed(cfio.bs.Flags()) {
		return EPERM
	}

	cs, err := cfio.caio.Read()
	if err != nil {
		return fmt.Errorf("Failed to read cs array: %v", err)
	}

	for i := len(cs) - 1; i >= 0; i-- {
		c := &cs[i]

		if c.Left() >= size {
			// drop the chunk
			continue
		}

		if c.Right() > size {
			// trim the chunk
			chunksize := size - c.Left()

			bh, err := cfio.bs.Open(c.BlobPath, fl.O_RDWR)
			if err != nil {
				return err
			}
			cio := cfio.newChunkIO(bh, cfio.c)
			if err := cio.Truncate(chunksize); err != nil {
				return err
			}
			if err := cio.Close(); err != nil {
				return err
			}
			c.Length = int64(cio.Size())
		}

		cs = cs[:i+1]
		if err := cfio.caio.Write(cs); err != nil {
			return fmt.Errorf("Failed to write updated cs array: %v", err)
		}
		return nil
	}
	if err := cfio.caio.Write([]inodedb.FileChunk{}); err != nil {
		return fmt.Errorf("Failed to write updated cs array (empty): %v", err)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package build gathers information about Go packages.
//
// Go Path
//
// The Go path is a list of directory trees containing Go source code.
// It is consulted to resolve imports that cannot be found in the standard
// Go tree.  The default path is the value of the GOPATH environment
// variable, interpreted as a path list appropriate to the operating system
// (on Unix, the variable is a colon-separated string;
// on Windows, a semicolon-separated string;
// on Plan 9, a list).
//
// Each directory listed in the Go path must have a prescribed structure:
//
// The src/ directory holds source code.  The path below 'src' determines
// the import path or executable name.
//
// The pkg/ directory holds installed package objects.
// As in the Go tree, each target operating system and
// architecture pair has its own subdirectory of pkg
// (pkg/GOOS_GOARCH).
//
// If DIR is a directory listed in the Go path, a package with
// source in DIR/src/foo/bar can be imported as "foo/bar" and
// has its compiled form installed to "DIR/pkg/GOOS_GOARCH/foo/bar.a"
// (or, for gccgo, "DIR/pkg/gccgo/foo/libbar.a").
//
// The bin/ directory holds compiled commands.
// Each command is named for its source directory, but only
// using the final element, not the entire path.  That is, the
// command with source in DIR/src/foo/quux is installed into
// DIR/bin/quux, not DIR/bin/foo/quux.  The foo/ is stripped
// so that you can add DIR/bin to your PATH to get at the
// installed commands.
//
// Here's an example directory layout:
//
//	GOPATH=/home/user/gocode
//
//	/home/user/gocode/
//	    src/
//	        foo/
//	            bar/               (go code in package bar)
//	                x.go
//	            quux/              (go code in package main)
//	                y.go
//	    bin/
//	        quux                   (installed command)
//	    pkg/
//	        linux_amd64/
//	            foo/
//	                bar.a          (installed package object)
//
// Build Constraints
//
// A build constraint, also known as a build tag, is a line comment that begins
//
//	// +build
//
// that lists the conditions under which a file should be included in the package.
// Constraints may appear in any kind of source file (not just Go), but
// they must appear near the top of the file, preceded
// only by blank lines and other line comments. These rules mean that in Go
// files a build constraint must appear before the package clause.
//
// To distinguish build constraints from package documentation, a series of
// build constraints must be followed by a blank line.
//
// A build constraint is evaluated as the OR of space-separated options;
// each option evaluates as the AND of its comma-separated terms;
// and each term is an alphanumeric word or, preceded by !, its negation.
// That is, the build constraint:
//
//	// +build linux,386 darwin,!cgo
//
// corresponds to the boolean formula:
//
//	(linux AND 386) OR (darwin AND (NOT cgo))
//
// A file may have multiple build constraints. The overall constraint is the AND
// of the individual constraints. That is, the build constraints:
//
//	// +build linux darwin
//	// +build 386
//
// corresponds to the boolean formula:
//
//	(linux OR darwin) AND 386
//
// During a particular build, the following words are satisfied:
//
//	- the target operating system, as spelled by runtime.GOOS
//	- the target architecture, as spelled by runtime.GOARCH
//	- the compiler being used, either "gc" or "gccgo"
//	- "cgo", if ctxt.CgoEnabled is true
//	- "go1.1", from Go version 1.1 onward
//	- "go1.2", from Go version 1.2 onward
//	- any additional words listed in ctxt.BuildTags
//
// If a file's name, after stripping the extension and a possible _test suffix,
// matches any of the following patterns:
//	*_GOOS
// 	*_GOARCH
// 	*_GOOS_GOARCH
// (example: source_windows_amd64.go) or the literals:
//	GOOS
// 	GOARCH
// (example: windows.go) where GOOS and GOARCH represent any known operating
// system and architecture values respectively, then the file is considered to
// have an implicit build constraint requiring those terms.
//
// To keep a file from being considered for the build:
//
//	// +build ignore
//
// (any other unsatisfied word will work as well, but ``ignore'' is conventional.)
//
// To build a file only when using cgo, and only on Linux and OS X:
//
//	// +build linux,cgo darwin,cgo
//
// Such a file is usually paired with another file implementing the
// default functionality for other systems, which in this case would
// carry the constraint:
//
//	// +build !linux,!darwin !cgo
//
// Naming a file dns_windows.go will cause it to be included only when
// building the package for Windows; similarly, math_386.s will be included
// only when building the package for 32-bit x86.
//
package build
<commit_msg>go/build: update doc.go for go1.3 build tag.<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package build gathers information about Go packages.
//
// Go Path
//
// The Go path is a list of directory trees containing Go source code.
// It is consulted to resolve imports that cannot be found in the standard
// Go tree.  The default path is the value of the GOPATH environment
// variable, interpreted as a path list appropriate to the operating system
// (on Unix, the variable is a colon-separated string;
// on Windows, a semicolon-separated string;
// on Plan 9, a list).
//
// Each directory listed in the Go path must have a prescribed structure:
//
// The src/ directory holds source code.  The path below 'src' determines
// the import path or executable name.
//
// The pkg/ directory holds installed package objects.
// As in the Go tree, each target operating system and
// architecture pair has its own subdirectory of pkg
// (pkg/GOOS_GOARCH).
//
// If DIR is a directory listed in the Go path, a package with
// source in DIR/src/foo/bar can be imported as "foo/bar" and
// has its compiled form installed to "DIR/pkg/GOOS_GOARCH/foo/bar.a"
// (or, for gccgo, "DIR/pkg/gccgo/foo/libbar.a").
//
// The bin/ directory holds compiled commands.
// Each command is named for its source directory, but only
// using the final element, not the entire path.  That is, the
// command with source in DIR/src/foo/quux is installed into
// DIR/bin/quux, not DIR/bin/foo/quux.  The foo/ is stripped
// so that you can add DIR/bin to your PATH to get at the
// installed commands.
//
// Here's an example directory layout:
//
//	GOPATH=/home/user/gocode
//
//	/home/user/gocode/
//	    src/
//	        foo/
//	            bar/               (go code in package bar)
//	                x.go
//	            quux/              (go code in package main)
//	                y.go
//	    bin/
//	        quux                   (installed command)
//	    pkg/
//	        linux_amd64/
//	            foo/
//	                bar.a          (installed package object)
//
// Build Constraints
//
// A build constraint, also known as a build tag, is a line comment that begins
//
//	// +build
//
// that lists the conditions under which a file should be included in the package.
// Constraints may appear in any kind of source file (not just Go), but
// they must appear near the top of the file, preceded
// only by blank lines and other line comments. These rules mean that in Go
// files a build constraint must appear before the package clause.
//
// To distinguish build constraints from package documentation, a series of
// build constraints must be followed by a blank line.
//
// A build constraint is evaluated as the OR of space-separated options;
// each option evaluates as the AND of its comma-separated terms;
// and each term is an alphanumeric word or, preceded by !, its negation.
// That is, the build constraint:
//
//	// +build linux,386 darwin,!cgo
//
// corresponds to the boolean formula:
//
//	(linux AND 386) OR (darwin AND (NOT cgo))
//
// A file may have multiple build constraints. The overall constraint is the AND
// of the individual constraints. That is, the build constraints:
//
//	// +build linux darwin
//	// +build 386
//
// corresponds to the boolean formula:
//
//	(linux OR darwin) AND 386
//
// During a particular build, the following words are satisfied:
//
//	- the target operating system, as spelled by runtime.GOOS
//	- the target architecture, as spelled by runtime.GOARCH
//	- the compiler being used, either "gc" or "gccgo"
//	- "cgo", if ctxt.CgoEnabled is true
//	- "go1.1", from Go version 1.1 onward
//	- "go1.2", from Go version 1.2 onward
//	- "go1.3", from Go version 1.3 onward
//	- any additional words listed in ctxt.BuildTags
//
// If a file's name, after stripping the extension and a possible _test suffix,
// matches any of the following patterns:
//	*_GOOS
// 	*_GOARCH
// 	*_GOOS_GOARCH
// (example: source_windows_amd64.go) or the literals:
//	GOOS
// 	GOARCH
// (example: windows.go) where GOOS and GOARCH represent any known operating
// system and architecture values respectively, then the file is considered to
// have an implicit build constraint requiring those terms.
//
// To keep a file from being considered for the build:
//
//	// +build ignore
//
// (any other unsatisfied word will work as well, but ``ignore'' is conventional.)
//
// To build a file only when using cgo, and only on Linux and OS X:
//
//	// +build linux,cgo darwin,cgo
//
// Such a file is usually paired with another file implementing the
// default functionality for other systems, which in this case would
// carry the constraint:
//
//	// +build !linux,!darwin !cgo
//
// Naming a file dns_windows.go will cause it to be included only when
// building the package for Windows; similarly, math_386.s will be included
// only when building the package for 32-bit x86.
//
package build
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"regexp";
	"testing";
)

type DialErrorTest struct {
	Net	string;
	Laddr	string;
	Raddr	string;
	Pattern	string;
}

var dialErrorTests = []DialErrorTest{
	DialErrorTest{
		"datakit", "", "mh/astro/r70",
		"dial datakit mh/astro/r70: unknown network datakit",
	},
	DialErrorTest{
		"tcp", "", "127.0.0.1:☺",
		"dial tcp 127.0.0.1:☺: unknown port tcp/☺",
	},
	DialErrorTest{
		"tcp", "", "no-such-name.google.com.:80",
		"dial tcp no-such-name.google.com.:80: lookup no-such-name.google.com.( on .*)?: no (.*)",
	},
	DialErrorTest{
		"tcp", "", "no-such-name.no-such-top-level-domain.:80",
		"dial tcp no-such-name.no-such-top-level-domain.:80: lookup no-such-name.no-such-top-level-domain.( on .*)?: no (.*)",
	},
	DialErrorTest{
		"tcp", "", "no-such-name:80",
		`dial tcp no-such-name:80: lookup no-such-name\.(.*\.)?( on .*)?: no (.*)`,
	},
	DialErrorTest{
		"tcp", "", "mh/astro/r70:http",
		"dial tcp mh/astro/r70:http: lookup mh/astro/r70: invalid domain name",
	},
	DialErrorTest{
		"unix", "", "/etc/file-not-found",
		"dial unix /etc/file-not-found: no such file or directory",
	},
	DialErrorTest{
		"unix", "", "/etc/",
		"dial unix /etc/: (permission denied|socket operation on non-socket)",
	},
}

func TestDialError(t *testing.T) {
	for i, tt := range dialErrorTests {
		c, e := Dial(tt.Net, tt.Laddr, tt.Raddr);
		if c != nil {
			c.Close()
		}
		if e == nil {
			t.Errorf("#%d: nil error, want match for %#q", i, tt.Pattern);
			continue;
		}
		s := e.String();
		match, _ := regexp.MatchString(tt.Pattern, s);
		if !match {
			t.Errorf("#%d: %q, want match for %#q", i, s, tt.Pattern)
		}
	}
}
<commit_msg>net: disable dns error test<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"flag";
	"regexp";
	"testing";
)

var runErrorTest = flag.Bool("run_error_test", false, "let TestDialError check for dns errors")

type DialErrorTest struct {
	Net	string;
	Laddr	string;
	Raddr	string;
	Pattern	string;
}

var dialErrorTests = []DialErrorTest{
	DialErrorTest{
		"datakit", "", "mh/astro/r70",
		"dial datakit mh/astro/r70: unknown network datakit",
	},
	DialErrorTest{
		"tcp", "", "127.0.0.1:☺",
		"dial tcp 127.0.0.1:☺: unknown port tcp/☺",
	},
	DialErrorTest{
		"tcp", "", "no-such-name.google.com.:80",
		"dial tcp no-such-name.google.com.:80: lookup no-such-name.google.com.( on .*)?: no (.*)",
	},
	DialErrorTest{
		"tcp", "", "no-such-name.no-such-top-level-domain.:80",
		"dial tcp no-such-name.no-such-top-level-domain.:80: lookup no-such-name.no-such-top-level-domain.( on .*)?: no (.*)",
	},
	DialErrorTest{
		"tcp", "", "no-such-name:80",
		`dial tcp no-such-name:80: lookup no-such-name\.(.*\.)?( on .*)?: no (.*)`,
	},
	DialErrorTest{
		"tcp", "", "mh/astro/r70:http",
		"dial tcp mh/astro/r70:http: lookup mh/astro/r70: invalid domain name",
	},
	DialErrorTest{
		"unix", "", "/etc/file-not-found",
		"dial unix /etc/file-not-found: no such file or directory",
	},
	DialErrorTest{
		"unix", "", "/etc/",
		"dial unix /etc/: (permission denied|socket operation on non-socket)",
	},
}

func TestDialError(t *testing.T) {
	if !*runErrorTest {
		t.Logf("test disabled; use --run_error_test to enable");
		return;
	}
	for i, tt := range dialErrorTests {
		c, e := Dial(tt.Net, tt.Laddr, tt.Raddr);
		if c != nil {
			c.Close()
		}
		if e == nil {
			t.Errorf("#%d: nil error, want match for %#q", i, tt.Pattern);
			continue;
		}
		s := e.String();
		match, _ := regexp.MatchString(tt.Pattern, s);
		if !match {
			t.Errorf("#%d: %q, want match for %#q", i, s, tt.Pattern)
		}
	}
}
<|endoftext|>