<commit_before>package kafka

import (
	"errors"
	"fmt"
	"log"
	"math"
	"sync"
	"time"
)

const megabyte = 1048576

var (
	// Returned by consumer Fetch when retry limit was set and exceeded during
	// single function call
	ErrNoData = errors.New("no data")
)

type clusterMetadata struct {
	Nodes     map[int32]string // node ID to address
	Endpoints map[string]int32 // topic:partition to leader node ID
}

type BrokerConfig struct {
	ClientID string
}

type Broker struct {
	config BrokerConfig

	mu       sync.Mutex
	metadata clusterMetadata
	conns    map[int32]*connection
}

func Dial(nodeAddresses []string, config BrokerConfig) (*Broker, error) {
	broker := &Broker{
		config: config,
		conns:  make(map[int32]*connection),
		metadata: clusterMetadata{
			Nodes:     make(map[int32]string),
			Endpoints: make(map[string]int32),
		},
	}

	for _, addr := range nodeAddresses {
		conn, err := newConnection(addr)
		if err != nil {
			log.Printf("could not connect to %s: %s", addr, err)
			continue
		}
		resp, err := conn.Metadata(&MetadataReq{
			ClientID: broker.config.ClientID,
			Topics:   nil,
		})
		if err != nil {
			log.Printf("could not fetch metadata from %s: %s", addr, err)
			continue
		}
		for _, node := range resp.Brokers {
			broker.metadata.Nodes[node.NodeID] = fmt.Sprintf("%s:%d", node.Host, node.Port)
		}
		for _, topic := range resp.Topics {
			for _, part := range topic.Partitions {
				broker.metadata.Endpoints[fmt.Sprintf("%s:%d", topic.Name, part.ID)] = part.Leader
			}
		}
		return broker, nil
	}
	return nil, errors.New("could not connect")
}

func (b *Broker) leaderConnection(topic string, partition int32) (conn *connection, err error) {
	b.mu.Lock()
	defer b.mu.Unlock()

	endpoint := fmt.Sprintf("%s:%d", topic, partition)
	nodeID, ok := b.metadata.Endpoints[endpoint]
	if !ok {
		// TODO(husio) refresh metadata and check again
		return nil, ErrUnknownTopicOrPartition
	}
	conn, ok = b.conns[nodeID]
	if !ok {
		addr, ok := b.metadata.Nodes[nodeID]
		if !ok {
			return nil, ErrBrokerNotAvailable
		}
		conn, err = newConnection(addr)
		if err != nil {
			return nil, err
		}
		b.conns[nodeID] = conn
	}
	return conn, nil
}

type ProducerConfig struct {
	Timeout      time.Duration
	RequiredAcks int16
}

// NewProducerConfig return default producer configuration
func NewProducerConfig() ProducerConfig {
	return ProducerConfig{
		Timeout:      time.Second,
		RequiredAcks: RequiredAcksAll,
	}
}

type Producer struct {
	// TODO(husio) configuration
	config ProducerConfig
	broker *Broker
}

func (b *Broker) Producer(config ProducerConfig) *Producer {
	return &Producer{
		config: config,
		broker: b,
	}
}

func (p *Producer) Config() ProducerConfig {
	return p.config
}

func (p *Producer) Produce(topic string, partition int32, messages ...*Message) (offset int64, err error) {
	conn, err := p.broker.leaderConnection(topic, partition)
	if err != nil {
		return 0, err
	}
	req := ProduceReq{
		ClientID:     p.broker.config.ClientID,
		RequiredAcks: p.config.RequiredAcks,
		Timeout:      p.config.Timeout,
		Topics: []ProduceReqTopic{
			ProduceReqTopic{
				Name: topic,
				Partitions: []ProduceReqPartition{
					ProduceReqPartition{
						ID:       partition,
						Messages: messages,
					},
				},
			},
		},
	}
	resp, err := conn.Produce(&req)
	if err != nil {
		// TODO(husio) handle some of the errors
		return 0, err
	}

	// we expect single partition response
	found := false
	for _, t := range resp.Topics {
		if t.Name != topic {
			log.Printf("unexpected topic information received: %s", t.Name)
			continue
		}
		for _, part := range t.Partitions {
			if part.ID != partition {
				log.Printf("unexpected partition information received: %s:%d", t.Name, part.ID)
				continue
			}
			found = true
			offset = part.Offset
			err = part.Err
		}
	}

	if !found {
		return 0, errors.New("incomplete produce response")
	}
	return offset, err
}

type ConsumerConfig struct {
	Topic     string
	Partition int32
	// FetchTimeout controlls fetch request timeout. This operation is blocking
	// the whole connection, so it should always be set to small value. By
	// default it's set to 0.
	FetchTimeout time.Duration
	// FetchRetryLimit limits fetching messages given amount of times before
	// returning ErrNoData error. By default set to -1, which turns this limit
	// off.
	FetchRetryLimit int
	// MinFetchSize is minimum size of messages to fetch in bytes. By default
	// set to 1 to fetch any message available.
	MinFetchSize int32
	MaxFetchSize int32
}

// NewConsumerConfig return default consumer configuration
func NewConsumerConfig(topic string, partition int32) ConsumerConfig {
	return ConsumerConfig{
		Topic:           topic,
		Partition:       partition,
		FetchTimeout:    0,
		FetchRetryLimit: -1,
		MinFetchSize:    1,
		MaxFetchSize:    megabyte * 2,
	}
}

type Consumer struct {
	broker *Broker
	conn   *connection
	config ConsumerConfig
	offset int64
	msgbuf []*Message
}

func (b *Broker) Consumer(config ConsumerConfig) (consumer *Consumer, err error) {
	conn, err := b.leaderConnection(config.Topic, config.Partition)
	if err != nil {
		return nil, err
	}
	consumer = &Consumer{
		broker: b,
		conn:   conn,
		config: config,
		msgbuf: make([]*Message, 0),
		offset: 0,
	}
	return consumer, nil
}

func (c *Consumer) Config() ConsumerConfig {
	return c.config
}

func (c *Consumer) Fetch() (*Message, error) {
	var retry int

	for len(c.msgbuf) == 0 {
		req := FetchReq{
			ClientID:    c.broker.config.ClientID,
			MaxWaitTime: c.config.FetchTimeout,
			MinBytes:    c.config.MinFetchSize,
			Topics: []FetchReqTopic{
				FetchReqTopic{
					Name: c.config.Topic,
					Partitions: []FetchReqPartition{
						FetchReqPartition{
							ID:          c.config.Partition,
							FetchOffset: c.offset + 1,
							MaxBytes:    c.config.MaxFetchSize,
						},
					},
				},
			},
		}
		resp, err := c.conn.Fetch(&req)
		if err != nil {
			// TODO(husio) handle some of the errors
			return nil, err
		}

		found := false
		for _, topic := range resp.Topics {
			if topic.Name != c.config.Topic {
				log.Printf("unexpected topic information received: %s (expecting %s)", topic.Name)
				continue
			}
			for _, part := range topic.Partitions {
				if part.ID != c.config.Partition {
					log.Printf("unexpected partition information received: %s:%d", topic.Name, part.ID)
					continue
				}

				found = true
				if len(part.Messages) == 0 {
					time.Sleep(time.Duration(math.Log(float64(retry+2))*250) * time.Millisecond)
					retry += 1
					if c.config.FetchRetryLimit != -1 && retry > c.config.FetchRetryLimit {
						return nil, ErrNoData
					}
				} else {
					last := part.Messages[len(part.Messages)-1]
					c.offset = last.Offset
					c.msgbuf = part.Messages
				}
			}
		}
		if !found {
			return nil, errors.New("incomplete fetch response")
		}
	}

	msg := c.msgbuf[0]
	c.msgbuf = c.msgbuf[1:]
	return msg, nil
}
<commit_msg>broker documentation<commit_after>package kafka

import (
	"errors"
	"fmt"
	"log"
	"math"
	"sync"
	"time"
)

const megabyte = 1048576

var (
	// Returned by consumer Fetch when retry limit was set and exceeded during
	// single function call
	ErrNoData = errors.New("no data")
)

type clusterMetadata struct {
	Nodes     map[int32]string // node ID to address
	Endpoints map[string]int32 // topic:partition to leader node ID
}

type BrokerConfig struct {
	ClientID string
}

// Broker is abstrac connection to kafka cluster, managing connections to all
// kafka nodes.
type Broker struct {
	config BrokerConfig

	mu       sync.Mutex
	metadata clusterMetadata
	conns    map[int32]*connection
}

// Dial connects to any node from given list of kafka addresses and after
// successful metadata fetch, returns broker.
// Returned broker is not initially connected to any kafka node.
func Dial(nodeAddresses []string, config BrokerConfig) (*Broker, error) {
	broker := &Broker{
		config: config,
		conns:  make(map[int32]*connection),
		metadata: clusterMetadata{
			Nodes:     make(map[int32]string),
			Endpoints: make(map[string]int32),
		},
	}

	for _, addr := range nodeAddresses {
		conn, err := newConnection(addr)
		if err != nil {
			log.Printf("could not connect to %s: %s", addr, err)
			continue
		}
		defer conn.Close()
		resp, err := conn.Metadata(&MetadataReq{
			ClientID: broker.config.ClientID,
			Topics:   nil,
		})
		if err != nil {
			log.Printf("could not fetch metadata from %s: %s", addr, err)
			continue
		}
		for _, node := range resp.Brokers {
			broker.metadata.Nodes[node.NodeID] = fmt.Sprintf("%s:%d", node.Host, node.Port)
		}
		for _, topic := range resp.Topics {
			for _, part := range topic.Partitions {
				broker.metadata.Endpoints[fmt.Sprintf("%s:%d", topic.Name, part.ID)] = part.Leader
			}
		}
		return broker, nil
	}
	return nil, errors.New("could not connect")
}

// leaderConnection returns connection to leader for given partition. If
// connection does not exist, broker will try to connect first and add store
// connection for any further use.
func (b *Broker) leaderConnection(topic string, partition int32) (conn *connection, err error) {
	b.mu.Lock()
	defer b.mu.Unlock()

	endpoint := fmt.Sprintf("%s:%d", topic, partition)
	nodeID, ok := b.metadata.Endpoints[endpoint]
	if !ok {
		// TODO(husio) refresh metadata and check again
		return nil, ErrUnknownTopicOrPartition
	}
	conn, ok = b.conns[nodeID]
	if !ok {
		addr, ok := b.metadata.Nodes[nodeID]
		if !ok {
			return nil, ErrBrokerNotAvailable
		}
		conn, err = newConnection(addr)
		if err != nil {
			return nil, err
		}
		b.conns[nodeID] = conn
	}
	return conn, nil
}

type ProducerConfig struct {
	Timeout      time.Duration
	RequiredAcks int16
}

// NewProducerConfig return default producer configuration
func NewProducerConfig() ProducerConfig {
	return ProducerConfig{
		Timeout:      time.Second,
		RequiredAcks: RequiredAcksAll,
	}
}

// Producer is link to broker with extra configuration.
type Producer struct {
	config ProducerConfig
	broker *Broker
}

func (b *Broker) Producer(config ProducerConfig) *Producer {
	return &Producer{
		config: config,
		broker: b,
	}
}

func (p *Producer) Config() ProducerConfig {
	return p.config
}

// Produce writes messages to given destination. Write within single Produce
// call are atomic, meaning either all or none of them are written to kafka.
func (p *Producer) Produce(topic string, partition int32, messages ...*Message) (offset int64, err error) {
	conn, err := p.broker.leaderConnection(topic, partition)
	if err != nil {
		return 0, err
	}
	req := ProduceReq{
		ClientID:     p.broker.config.ClientID,
		RequiredAcks: p.config.RequiredAcks,
		Timeout:      p.config.Timeout,
		Topics: []ProduceReqTopic{
			ProduceReqTopic{
				Name: topic,
				Partitions: []ProduceReqPartition{
					ProduceReqPartition{
						ID:       partition,
						Messages: messages,
					},
				},
			},
		},
	}
	resp, err := conn.Produce(&req)
	if err != nil {
		// TODO(husio) handle some of the errors
		return 0, err
	}

	// we expect single partition response
	found := false
	for _, t := range resp.Topics {
		if t.Name != topic {
			log.Printf("unexpected topic information received: %s", t.Name)
			continue
		}
		for _, part := range t.Partitions {
			if part.ID != partition {
				log.Printf("unexpected partition information received: %s:%d", t.Name, part.ID)
				continue
			}
			found = true
			offset = part.Offset
			err = part.Err
		}
	}

	if !found {
		return 0, errors.New("incomplete produce response")
	}
	return offset, err
}

type ConsumerConfig struct {
	Topic     string
	Partition int32
	// FetchTimeout controlls fetch request timeout. This operation is blocking
	// the whole connection, so it should always be set to small value. By
	// default it's set to 0.
	FetchTimeout time.Duration
	// FetchRetryLimit limits fetching messages given amount of times before
	// returning ErrNoData error. By default set to -1, which turns this limit
	// off.
	FetchRetryLimit int
	// MinFetchSize is minimum size of messages to fetch in bytes. By default
	// set to 1 to fetch any message available.
	MinFetchSize int32
	MaxFetchSize int32
}

// NewConsumerConfig return default consumer configuration
func NewConsumerConfig(topic string, partition int32) ConsumerConfig {
	return ConsumerConfig{
		Topic:           topic,
		Partition:       partition,
		FetchTimeout:    0,
		FetchRetryLimit: -1,
		MinFetchSize:    1,
		MaxFetchSize:    megabyte * 2,
	}
}

// Consumer is representing single partition reading buffer.
type Consumer struct {
	broker *Broker
	conn   *connection
	config ConsumerConfig
	offset int64
	msgbuf []*Message
}

// Consumer creates cursor capable of reading messages from given source.
func (b *Broker) Consumer(config ConsumerConfig) (consumer *Consumer, err error) {
	conn, err := b.leaderConnection(config.Topic, config.Partition)
	if err != nil {
		return nil, err
	}
	consumer = &Consumer{
		broker: b,
		conn:   conn,
		config: config,
		msgbuf: make([]*Message, 0),
		offset: 0,
	}
	return consumer, nil
}

func (c *Consumer) Config() ConsumerConfig {
	return c.config
}

// Fetch is returning single message from consumed partition.
func (c *Consumer) Fetch() (*Message, error) {
	var retry int

	for len(c.msgbuf) == 0 {
		req := FetchReq{
			ClientID:    c.broker.config.ClientID,
			MaxWaitTime: c.config.FetchTimeout,
			MinBytes:    c.config.MinFetchSize,
			Topics: []FetchReqTopic{
				FetchReqTopic{
					Name: c.config.Topic,
					Partitions: []FetchReqPartition{
						FetchReqPartition{
							ID:          c.config.Partition,
							FetchOffset: c.offset + 1,
							MaxBytes:    c.config.MaxFetchSize,
						},
					},
				},
			},
		}
		resp, err := c.conn.Fetch(&req)
		if err != nil {
			// TODO(husio) handle some of the errors
			return nil, err
		}

		found := false
		for _, topic := range resp.Topics {
			if topic.Name != c.config.Topic {
				log.Printf("unexpected topic information received: %s (expecting %s)", topic.Name)
				continue
			}
			for _, part := range topic.Partitions {
				if part.ID != c.config.Partition {
					log.Printf("unexpected partition information received: %s:%d", topic.Name, part.ID)
					continue
				}

				found = true
				if len(part.Messages) == 0 {
					time.Sleep(time.Duration(math.Log(float64(retry+2))*250) * time.Millisecond)
					retry += 1
					if c.config.FetchRetryLimit != -1 && retry > c.config.FetchRetryLimit {
						return nil, ErrNoData
					}
				} else {
					last := part.Messages[len(part.Messages)-1]
					c.offset = last.Offset
					c.msgbuf = part.Messages
				}
			}
		}
		if !found {
			return nil, errors.New("incomplete fetch response")
		}
	}

	msg := c.msgbuf[0]
	c.msgbuf = c.msgbuf[1:]
	return msg, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"strconv"

	"time"

	. "github.com/claudetech/loggo/default"
	"github.com/orcaman/concurrent-map"
)

var instances cmap.ConcurrentMap
var chunkPath string
var chunkSize int64

func init() {
	instances = cmap.New()
}

// Buffer is a buffered stream
type Buffer struct {
	numberOfInstances int
	client            *http.Client
	object            *APIObject
	tempDir           string
	chunkSize         int64
	preload           bool
	chunkDir          string
}

// GetBufferInstance gets a singleton instance of buffer
func GetBufferInstance(client *http.Client, object *APIObject) (*Buffer, error) {
	if !instances.Has(object.ObjectID) {
		i, err := newBuffer(client, object)
		if nil != err {
			return nil, err
		}

		instances.Set(object.ObjectID, i)
	}

	instance, ok := instances.Get(object.ObjectID)
	// if buffer allocation failed due to race conditions it will try to fetch a new one
	if !ok {
		i, err := GetBufferInstance(client, object)
		if nil != err {
			return nil, err
		}
		instance = i
	}
	instance.(*Buffer).numberOfInstances++
	return instance.(*Buffer), nil
}

// SetChunkPath sets the global chunk path
func SetChunkPath(path string) {
	chunkPath = path
}

// SetChunkSize sets the global chunk size
func SetChunkSize(size int64) {
	chunkSize = size
}

// NewBuffer creates a new buffer instance
func newBuffer(client *http.Client, object *APIObject) (*Buffer, error) {
	Log.Infof("Starting playback of %v", object.Name)
	Log.Debugf("Creating buffer for object %v", object.ObjectID)

	tempDir := filepath.Join(chunkPath, object.ObjectID)
	if err := os.MkdirAll(tempDir, 0777); nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not create temp path for object %v", object.ObjectID)
	}

	if 0 == chunkSize {
		Log.Debugf("ChunkSize was 0, setting to default (5 MB)")
		chunkSize = 5 * 1024 * 1024
	}

	buffer := Buffer{
		numberOfInstances: 0,
		client:            client,
		object:            object,
		tempDir:           tempDir,
		chunkSize:         chunkSize,
		preload:           true,
	}

	return &buffer, nil
}

// Close all handles
func (b *Buffer) Close() error {
	b.numberOfInstances--
	if 0 == b.numberOfInstances {
		Log.Infof("Stopping playback of %v", b.object.Name)
		Log.Debugf("Stop buffering for object %v", b.object.ObjectID)

		b.preload = false
		instances.Remove(b.object.ObjectID)
	}
	return nil
}

// ReadBytes on a specific location
func (b *Buffer) ReadBytes(start, size int64, isPreload bool) ([]byte, error) {
	fOffset := start % b.chunkSize
	offset := start - fOffset
	offsetEnd := offset + b.chunkSize

	Log.Debugf("Getting object %v bytes %v - %v (is preload: %v)", b.object.ObjectID, offset, offsetEnd, isPreload)

	filename := filepath.Join(b.tempDir, strconv.Itoa(int(offset)))
	if f, err := os.Open(filename); nil == err {
		defer f.Close()
		buf := make([]byte, size)
		if _, err := f.ReadAt(buf, fOffset); nil == err {
			Log.Debugf("Found object %v bytes %v - %v in cache", b.object.ObjectID, offset, offsetEnd)

			// update the last modified time for files that are often in use
			if err := os.Chtimes(filename, time.Now(), time.Now()); nil != err {
				Log.Warningf("Could not update last modified time for %v", filename)
			}

			return buf[:size], nil
		}
	}

	Log.Debugf("Requesting object %v bytes %v - %v from API", b.object.ObjectID, offset, offsetEnd)
	req, err := http.NewRequest("GET", b.object.DownloadURL, nil)
	if nil != err {
		return nil, err
	}

	req.Header.Add("Range", fmt.Sprintf("bytes=%v-%v", offset, offsetEnd))

	Log.Tracef("Sending HTTP Request %v", req)

	res, err := b.client.Do(req)
	if nil != err {
		return nil, err
	}

	if res.StatusCode != 206 {
		return nil, fmt.Errorf("Wrong status code %v", res)
	}

	bytes, err := ioutil.ReadAll(res.Body)
	if nil != err {
		return nil, err
	}

	f, err := os.Create(filename)
	if nil != err {
		return nil, err
	}
	defer f.Close()

	_, err = f.Write(bytes)
	if nil != err {
		return nil, err
	}

	if !isPreload && b.preload && uint64(offsetEnd) < b.object.Size {
		go func() {
			b.ReadBytes(offsetEnd+1, size, true)
		}()
	}

	return bytes[fOffset:int64(math.Min(float64(fOffset+size), float64(len(bytes))))], nil
}
<commit_msg><4k file offset bugfix<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"strconv"

	"time"

	. "github.com/claudetech/loggo/default"
	"github.com/orcaman/concurrent-map"
)

var instances cmap.ConcurrentMap
var chunkPath string
var chunkSize int64

func init() {
	instances = cmap.New()
}

// Buffer is a buffered stream
type Buffer struct {
	numberOfInstances int
	client            *http.Client
	object            *APIObject
	tempDir           string
	chunkSize         int64
	preload           bool
	chunkDir          string
}

// GetBufferInstance gets a singleton instance of buffer
func GetBufferInstance(client *http.Client, object *APIObject) (*Buffer, error) {
	if !instances.Has(object.ObjectID) {
		i, err := newBuffer(client, object)
		if nil != err {
			return nil, err
		}

		instances.Set(object.ObjectID, i)
	}

	instance, ok := instances.Get(object.ObjectID)
	// if buffer allocation failed due to race conditions it will try to fetch a new one
	if !ok {
		i, err := GetBufferInstance(client, object)
		if nil != err {
			return nil, err
		}
		instance = i
	}
	instance.(*Buffer).numberOfInstances++
	return instance.(*Buffer), nil
}

// SetChunkPath sets the global chunk path
func SetChunkPath(path string) {
	chunkPath = path
}

// SetChunkSize sets the global chunk size
func SetChunkSize(size int64) {
	chunkSize = size
}

// NewBuffer creates a new buffer instance
func newBuffer(client *http.Client, object *APIObject) (*Buffer, error) {
	Log.Infof("Starting playback of %v", object.Name)
	Log.Debugf("Creating buffer for object %v", object.ObjectID)

	tempDir := filepath.Join(chunkPath, object.ObjectID)
	if err := os.MkdirAll(tempDir, 0777); nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not create temp path for object %v", object.ObjectID)
	}

	if 0 == chunkSize {
		Log.Debugf("ChunkSize was 0, setting to default (5 MB)")
		chunkSize = 5 * 1024 * 1024
	}

	buffer := Buffer{
		numberOfInstances: 0,
		client:            client,
		object:            object,
		tempDir:           tempDir,
		chunkSize:         chunkSize,
		preload:           true,
	}

	return &buffer, nil
}

// Close all handles
func (b *Buffer) Close() error {
	b.numberOfInstances--
	if 0 == b.numberOfInstances {
		Log.Infof("Stopping playback of %v", b.object.Name)
		Log.Debugf("Stop buffering for object %v", b.object.ObjectID)

		b.preload = false
		instances.Remove(b.object.ObjectID)
	}
	return nil
}

// ReadBytes on a specific location
func (b *Buffer) ReadBytes(start, size int64, isPreload bool) ([]byte, error) {
	fOffset := start % b.chunkSize
	offset := start - fOffset
	offsetEnd := offset + b.chunkSize

	Log.Debugf("Getting object %v bytes %v - %v (is preload: %v)", b.object.ObjectID, offset, offsetEnd, isPreload)

	filename := filepath.Join(b.tempDir, strconv.Itoa(int(offset)))
	if f, err := os.Open(filename); nil == err {
		defer f.Close()
		buf := make([]byte, size)
		if n, err := f.ReadAt(buf, fOffset); nil == err && n > 0 {
			Log.Debugf("Found object %v bytes %v - %v in cache", b.object.ObjectID, offset, offsetEnd)

			// update the last modified time for files that are often in use
			if err := os.Chtimes(filename, time.Now(), time.Now()); nil != err {
				Log.Warningf("Could not update last modified time for %v", filename)
			}

			return buf[:size], nil
		}
	}

	Log.Debugf("Requesting object %v bytes %v - %v from API", b.object.ObjectID, offset, offsetEnd)
	req, err := http.NewRequest("GET", b.object.DownloadURL, nil)
	if nil != err {
		return nil, err
	}

	req.Header.Add("Range", fmt.Sprintf("bytes=%v-%v", offset, offsetEnd))

	Log.Tracef("Sending HTTP Request %v", req)

	res, err := b.client.Do(req)
	if nil != err {
		return nil, err
	}

	if res.StatusCode != 206 {
		return nil, fmt.Errorf("Wrong status code %v", res)
	}

	bytes, err := ioutil.ReadAll(res.Body)
	if nil != err {
		return nil, err
	}

	f, err := os.Create(filename)
	if nil != err {
		return nil, err
	}
	defer f.Close()

	_, err = f.Write(bytes)
	if nil != err {
		return nil, err
	}

	if !isPreload && b.preload && uint64(offsetEnd) < b.object.Size {
		go func() {
			b.ReadBytes(offsetEnd+1, size, true)
		}()
	}

	return bytes[fOffset:int64(math.Min(float64(fOffset+size), float64(len(bytes))))], nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"strconv"

	"time"

	"strings"

	. "github.com/claudetech/loggo/default"
	"github.com/mxk/go-flowrate/flowrate"
	"github.com/orcaman/concurrent-map"
)

var instances cmap.ConcurrentMap
var chunkPath string
var chunkSize int64
var chunkDirMaxSize int64
var speedLimit int64

func init() {
	instances = cmap.New()
}

// Buffer is a buffered stream
type Buffer struct {
	numberOfInstances int
	client            *http.Client
	object            *APIObject
	tempDir           string
	preload           bool
	chunks            cmap.ConcurrentMap
}

// GetBufferInstance gets a singleton instance of buffer
func GetBufferInstance(client *http.Client, object *APIObject) (*Buffer, error) {
	if !instances.Has(object.ObjectID) {
		i, err := newBuffer(client, object)
		if nil != err {
			return nil, err
		}

		instances.Set(object.ObjectID, i)
	}

	instance, ok := instances.Get(object.ObjectID)
	// if buffer allocation failed due to race conditions it will try to fetch a new one
	if !ok {
		i, err := GetBufferInstance(client, object)
		if nil != err {
			return nil, err
		}
		instance = i
	}
	instance.(*Buffer).numberOfInstances++
	return instance.(*Buffer), nil
}

// SetChunkPath sets the global chunk path
func SetChunkPath(path string) {
	chunkPath = path
}

// SetChunkSize sets the global chunk size
func SetChunkSize(size int64) {
	chunkSize = size
}

// SetChunkDirMaxSize sets the maximum size of the chunk directory
func SetChunkDirMaxSize(size int64) {
	chunkDirMaxSize = size
}

// SetDownloadSpeedLimit sets the download speed limit per chunk
func SetDownloadSpeedLimit(downloadSpeedLimit int64) {
	speedLimit = downloadSpeedLimit
}

// NewBuffer creates a new buffer instance
func newBuffer(client *http.Client, object *APIObject) (*Buffer, error) {
	Log.Infof("Starting playback of %v", object.Name)
	Log.Debugf("Creating buffer for object %v", object.ObjectID)

	tempDir := filepath.Join(chunkPath, object.ObjectID)
	if err := os.MkdirAll(tempDir, 0777); nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not create temp path for object %v", object.ObjectID)
	}

	if 0 == chunkSize {
		Log.Debugf("ChunkSize was 0, setting to default (5 MB)")
		chunkSize = 5 * 1024 * 1024
	}

	buffer := Buffer{
		numberOfInstances: 0,
		client:            client,
		object:            object,
		tempDir:           tempDir,
		preload:           true,
		chunks:            cmap.New(),
	}

	return &buffer, nil
}

// Close all handles
func (b *Buffer) Close() error {
	b.numberOfInstances--
	if 0 == b.numberOfInstances {
		Log.Infof("Stopping playback of %v", b.object.Name)
		Log.Debugf("Stop buffering for object %v", b.object.ObjectID)

		b.preload = false
		instances.Remove(b.object.ObjectID)
	}
	return nil
}

// ReadBytes on a specific location
func (b *Buffer) ReadBytes(start, size int64, preload bool, delay int32) ([]byte, error) {
	fOffset := start % chunkSize
	offset := start - fOffset
	offsetEnd := offset + chunkSize

	Log.Tracef("Getting object %v - chunk %v - offset %v for %v bytes",
		b.object.ObjectID, strconv.Itoa(int(offset)), fOffset, size)

	if !preload && b.preload && uint64(offsetEnd) < b.object.Size {
		defer func() {
			go func() {
				preloadStart := strconv.Itoa(int(offsetEnd))
				if !b.chunks.Has(preloadStart) {
					b.chunks.Set(preloadStart, true)
					b.ReadBytes(offsetEnd, size, true, 0)
				}
			}()
		}()
	}

	filename := filepath.Join(b.tempDir, strconv.Itoa(int(offset)))
	if f, err := os.Open(filename); nil == err {
		defer f.Close()

		buf := make([]byte, size)
		if n, err := f.ReadAt(buf, fOffset); n > 0 && (nil == err || io.EOF == err) {
			Log.Tracef("Found file %s bytes %v - %v in cache", filename, offset, offsetEnd)

			// update the last modified time for files that are often in use
			if err := os.Chtimes(filename, time.Now(), time.Now()); nil != err {
				Log.Warningf("Could not update last modified time for %v", filename)
			}

			return buf[:size], nil
		}

		Log.Debugf("%v", err)
		Log.Debugf("Could not read file %s at %v", filename, fOffset)
	}

	if chunkDirMaxSize > 0 {
		go func() {
			if err := cleanChunkDir(chunkPath); nil != err {
				Log.Debugf("%v", err)
				Log.Warningf("Could not delete oldest chunk")
			}
		}()
	}

	// sleep if request is throttled
	if delay > 0 {
		time.Sleep(time.Duration(delay) * time.Second)
	}

	Log.Debugf("Requesting object %v bytes %v - %v from API", b.object.ObjectID, offset, offsetEnd)
	req, err := http.NewRequest("GET", b.object.DownloadURL, nil)
	if nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not create request object %v from API", b.object.ObjectID)
	}

	req.Header.Add("Range", fmt.Sprintf("bytes=%v-%v", offset, offsetEnd))

	Log.Tracef("Sending HTTP Request %v", req)

	res, err := b.client.Do(req)
	if nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not request object %v from API", b.object.ObjectID)
	}
	defer res.Body.Close()

	reader := res.Body
	if speedLimit > 0 {
		reader = flowrate.NewReader(res.Body, speedLimit)
	}

	if res.StatusCode != 206 {
		if res.StatusCode != 403 {
			return nil, fmt.Errorf("Wrong status code %v", res)
		}

		// throttle requests
		if delay > 8 {
			return nil, fmt.Errorf("Maximum throttle interval has been reached")
		}
		bytes, err := ioutil.ReadAll(reader)
		if nil != err {
			Log.Debugf("%v", err)
			return nil, fmt.Errorf("Could not read body of 403 error")
		}
		body := string(bytes)
		if strings.Contains(body, "dailyLimitExceeded") ||
			strings.Contains(body, "userRateLimitExceeded") ||
			strings.Contains(body, "rateLimitExceeded") ||
			strings.Contains(body, "backendError") {
			if 0 == delay {
				delay = 1
			} else {
				delay = delay * 2
			}
			return b.ReadBytes(start, size, true, delay)
		}
	}

	bytes, err := ioutil.ReadAll(reader)
	if nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not read objects %v API response", b.object.ObjectID)
	}

	if _, err := os.Stat(b.tempDir); os.IsNotExist(err) {
		if err := os.MkdirAll(b.tempDir, 0777); nil != err {
			Log.Debugf("%v", err)
			return nil, fmt.Errorf("Could not create chunk temp path for chunk %v", filename)
		}
	}

	if err := ioutil.WriteFile(filename, bytes, 0777); nil != err {
		Log.Debugf("%v", err)
		Log.Warningf("Could not write chunk temp file %v", filename)
	}

	sOffset := int64(math.Min(float64(fOffset), float64(len(bytes))))
	eOffset := int64(math.Min(float64(fOffset+size), float64(len(bytes))))
	return bytes[sOffset:eOffset], nil
}

// cleanChunkDir checks if the chunk folder is grown to big and clears the oldest file if necessary
func cleanChunkDir(chunkPath string) error {
	chunkDirSize, err := dirSize(chunkPath)
	if nil != err {
		return err
	}

	if chunkDirSize+chunkSize*2 > chunkDirMaxSize {
		if err := deleteOldestFile(chunkPath); nil != err {
			return err
		}
	}

	return nil
}

// deleteOldestFile deletes the oldest file in the directory
func deleteOldestFile(path string) error {
	var fpath string
	lastMod := time.Now()

	err := filepath.Walk(path, func(file string, info os.FileInfo, err error) error {
		if err != nil {
			Log.Errorf("Error during walk through cache directory: %+v", err)
			return filepath.SkipDir
		}
		if info == nil {
			Log.Errorf("File info for %s was nil", file)
			return filepath.SkipDir
		}
		if !info.IsDir() {
			modTime := info.ModTime()
			if modTime.Before(lastMod) {
				lastMod = modTime
				fpath = file
			}
		}
		return err
	})

	Log.Debugf("Deleting oldest chunk file %v", fpath)
	os.Remove(fpath)

	return err
}

// dirSize gets the total directory size
func dirSize(path string) (int64, error) {
	var size int64
	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
		if nil == err && nil != info && !info.IsDir() {
			size += info.Size()
		}
		return err
	})
	return size, err
}
<commit_msg>checked unexpected EOF<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"strconv"

	"time"

	"strings"

	. "github.com/claudetech/loggo/default"
	"github.com/mxk/go-flowrate/flowrate"
	"github.com/orcaman/concurrent-map"
)

var instances cmap.ConcurrentMap
var chunkPath string
var chunkSize int64
var chunkDirMaxSize int64
var speedLimit int64

func init() {
	instances = cmap.New()
}

// Buffer is a buffered stream
type Buffer struct {
	numberOfInstances int
	client            *http.Client
	object            *APIObject
	tempDir           string
	preload           bool
	chunks            cmap.ConcurrentMap
}

// GetBufferInstance gets a singleton instance of buffer
func GetBufferInstance(client *http.Client, object *APIObject) (*Buffer, error) {
	if !instances.Has(object.ObjectID) {
		i, err := newBuffer(client, object)
		if nil != err {
			return nil, err
		}

		instances.Set(object.ObjectID, i)
	}

	instance, ok := instances.Get(object.ObjectID)
	// if buffer allocation failed due to race conditions it will try to fetch a new one
	if !ok {
		i, err := GetBufferInstance(client, object)
		if nil != err {
			return nil, err
		}
		instance = i
	}
	instance.(*Buffer).numberOfInstances++
	return instance.(*Buffer), nil
}

// SetChunkPath sets the global chunk path
func SetChunkPath(path string) {
	chunkPath = path
}

// SetChunkSize sets the global chunk size
func SetChunkSize(size int64) {
	chunkSize = size
}

// SetChunkDirMaxSize sets the maximum size of the chunk directory
func SetChunkDirMaxSize(size int64) {
	chunkDirMaxSize = size
}

// SetDownloadSpeedLimit sets the download speed limit per chunk
func SetDownloadSpeedLimit(downloadSpeedLimit int64) {
	speedLimit = downloadSpeedLimit
}

// NewBuffer creates a new buffer instance
func newBuffer(client *http.Client, object *APIObject) (*Buffer, error) {
	Log.Infof("Starting playback of %v", object.Name)
	Log.Debugf("Creating buffer for object %v", object.ObjectID)

	tempDir := filepath.Join(chunkPath, object.ObjectID)
	if err := os.MkdirAll(tempDir, 0777); nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not create temp path for object %v", object.ObjectID)
	}

	if 0 == chunkSize {
		Log.Debugf("ChunkSize was 0, setting to default (5 MB)")
		chunkSize = 5 * 1024 * 1024
	}

	buffer := Buffer{
		numberOfInstances: 0,
		client:            client,
		object:            object,
		tempDir:           tempDir,
		preload:           true,
		chunks:            cmap.New(),
	}

	return &buffer, nil
}

// Close all handles
func (b *Buffer) Close() error {
	b.numberOfInstances--
	if 0 == b.numberOfInstances {
		Log.Infof("Stopping playback of %v", b.object.Name)
		Log.Debugf("Stop buffering for object %v", b.object.ObjectID)

		b.preload = false
		instances.Remove(b.object.ObjectID)
	}
	return nil
}

// ReadBytes on a specific location
func (b *Buffer) ReadBytes(start, size int64, preload bool, delay int32) ([]byte, error) {
	fOffset := start % chunkSize
	offset := start - fOffset
	offsetEnd := offset + chunkSize

	Log.Tracef("Getting object %v - chunk %v - offset %v for %v bytes",
		b.object.ObjectID, strconv.Itoa(int(offset)), fOffset, size)

	if !preload && b.preload && uint64(offsetEnd) < b.object.Size {
		defer func() {
			go func() {
				preloadStart := strconv.Itoa(int(offsetEnd))
				if !b.chunks.Has(preloadStart) {
					b.chunks.Set(preloadStart, true)
					b.ReadBytes(offsetEnd, size, true, 0)
				}
			}()
		}()
	}

	filename := filepath.Join(b.tempDir, strconv.Itoa(int(offset)))
	if f, err := os.Open(filename); nil == err {
		defer f.Close()

		buf := make([]byte, size)
		if n, err := f.ReadAt(buf, fOffset); n > 0 && (nil == err || io.EOF == err || io.ErrUnexpectedEOF == err) {
			Log.Tracef("Found file %s bytes %v - %v in cache", filename, offset, offsetEnd)

			// update the last modified time for files that are often in use
			if err := os.Chtimes(filename, time.Now(), time.Now()); nil != err {
				Log.Warningf("Could not update last modified time for %v", filename)
			}

			return buf[:size], nil
		}

		Log.Debugf("%v", err)
		Log.Debugf("Could not read file %s at %v", filename, fOffset)
	}

	if chunkDirMaxSize > 0 {
		go func() {
			if err := cleanChunkDir(chunkPath); nil != err {
				Log.Debugf("%v", err)
				Log.Warningf("Could not delete oldest chunk")
			}
		}()
	}

	// sleep if request is throttled
	if delay > 0 {
		time.Sleep(time.Duration(delay) * time.Second)
	}

	Log.Debugf("Requesting object %v bytes %v - %v from API", b.object.ObjectID, offset, offsetEnd)
	req, err := http.NewRequest("GET", b.object.DownloadURL, nil)
	if nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not create request object %v from API", b.object.ObjectID)
	}

	req.Header.Add("Range", fmt.Sprintf("bytes=%v-%v", offset, offsetEnd))

	Log.Tracef("Sending HTTP Request %v", req)

	res, err := b.client.Do(req)
	if nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not request object %v from API", b.object.ObjectID)
	}
	defer res.Body.Close()

	reader := res.Body
	if speedLimit > 0 {
		reader = flowrate.NewReader(res.Body, speedLimit)
	}

	if res.StatusCode != 206 {
		if res.StatusCode != 403 {
			return nil, fmt.Errorf("Wrong status code %v", res)
		}

		// throttle requests
		if delay > 8 {
			return nil, fmt.Errorf("Maximum throttle interval has been reached")
		}
		bytes, err := ioutil.ReadAll(reader)
		if nil != err {
			Log.Debugf("%v", err)
			return nil, fmt.Errorf("Could not read body of 403 error")
		}
		body := string(bytes)
		if strings.Contains(body, "dailyLimitExceeded") ||
			strings.Contains(body, "userRateLimitExceeded") ||
			strings.Contains(body, "rateLimitExceeded") ||
			strings.Contains(body, "backendError") {
			if 0 == delay {
				delay = 1
			} else {
				delay = delay * 2
			}
			return b.ReadBytes(start, size, true, delay)
		}
	}

	bytes, err := ioutil.ReadAll(reader)
	if nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not read objects %v API response", b.object.ObjectID)
	}

	if _, err := os.Stat(b.tempDir); os.IsNotExist(err) {
		if err := os.MkdirAll(b.tempDir, 0777); nil != err {
			Log.Debugf("%v", err)
			return nil, fmt.Errorf("Could not create chunk temp path for chunk %v", filename)
		}
	}

	if err := ioutil.WriteFile(filename, bytes, 0777); nil != err {
		Log.Debugf("%v", err)
		Log.Warningf("Could not write chunk temp file %v", filename)
	}

	sOffset := int64(math.Min(float64(fOffset), float64(len(bytes))))
	eOffset := int64(math.Min(float64(fOffset+size), float64(len(bytes))))
	return bytes[sOffset:eOffset], nil
}

// cleanChunkDir checks if the chunk folder is grown to big and clears the oldest file if necessary
func cleanChunkDir(chunkPath string) error {
	chunkDirSize, err := dirSize(chunkPath)
	if nil != err {
		return err
	}

	if chunkDirSize+chunkSize*2 > chunkDirMaxSize {
		if err := deleteOldestFile(chunkPath); nil != err {
			return err
		}
	}

	return nil
}

// deleteOldestFile deletes the oldest file in the directory
func deleteOldestFile(path string) error {
	var fpath string
	lastMod := time.Now()

	err := filepath.Walk(path, func(file string, info os.FileInfo, err error) error {
		if err != nil {
			Log.Errorf("Error during walk through cache directory: %+v", err)
			return filepath.SkipDir
		}
		if info == nil {
			Log.Errorf("File info for %s was nil", file)
			return filepath.SkipDir
		}
		if !info.IsDir() {
			modTime := info.ModTime()
			if modTime.Before(lastMod) {
				lastMod = modTime
				fpath = file
			}
		}
		return err
	})

	Log.Debugf("Deleting oldest chunk file %v", fpath)
	os.Remove(fpath)

	return err
}

// dirSize gets the total directory size
func dirSize(path string) (int64, error) {
	var size int64
	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
		if nil == err && nil != info && !info.IsDir() {
			size += info.Size()
		}
		return err
	})
	return size, err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"strconv"

	"time"

	. "github.com/claudetech/loggo/default"
	"github.com/orcaman/concurrent-map"
)

var instances cmap.ConcurrentMap
var chunkPath string
var chunkSize int64
var chunkDirMaxSize int64

func init() {
	instances = cmap.New()
}

// Buffer is a buffered stream
type Buffer struct {
	numberOfInstances int
	client            *http.Client
	object            *APIObject
	tempDir           string
	preload           bool
	chunkDir          string
}

// GetBufferInstance gets a singleton instance of buffer
func GetBufferInstance(client *http.Client, object *APIObject) (*Buffer, error) {
	if !instances.Has(object.ObjectID) {
		i, err := newBuffer(client, object)
		if nil != err {
			return nil, err
		}

		instances.Set(object.ObjectID, i)
	}

	instance, ok := instances.Get(object.ObjectID)
	// if buffer allocation failed due to race conditions it will try to fetch a new one
	if !ok {
		i, err := GetBufferInstance(client, object)
		if nil != err {
			return nil, err
		}
		instance = i
	}
	instance.(*Buffer).numberOfInstances++
	return instance.(*Buffer), nil
}

// SetChunkPath sets the global chunk path
func SetChunkPath(path string) {
	chunkPath = path
}

// SetChunkSize sets the global chunk size
func SetChunkSize(size int64) {
	chunkSize = size
}

// SetChunkDirMaxSize sets the maximum size of the chunk directory
func SetChunkDirMaxSize(size int64) {
	chunkDirMaxSize = size
}

// NewBuffer creates a new buffer instance
func newBuffer(client *http.Client, object *APIObject) (*Buffer, error) {
	Log.Infof("Starting playback of %v", object.Name)
	Log.Debugf("Creating buffer for object %v", object.ObjectID)

	tempDir := filepath.Join(chunkPath, object.ObjectID)
	if err := os.MkdirAll(tempDir, 0777); nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not create temp path for object %v", object.ObjectID)
	}

	if 0 == chunkSize {
		Log.Debugf("ChunkSize was 0, setting to default (5 MB)")
		chunkSize = 5 * 1024 * 1024
	}

	buffer := Buffer{
		numberOfInstances: 0,
		client:            client,
		object:            object,
		tempDir:           tempDir,
		preload:           true,
	}

	return &buffer, nil
}

// Close all handles
func (b *Buffer) Close() error {
	b.numberOfInstances--
	if 0 == b.numberOfInstances {
		Log.Infof("Stopping playback of %v", b.object.Name)
		Log.Debugf("Stop buffering for object %v", b.object.ObjectID)

		b.preload = false
		instances.Remove(b.object.ObjectID)
	}
	return nil
}

// ReadBytes on a specific location
func (b *Buffer) ReadBytes(start, size int64, isPreload bool) ([]byte, error) {
	fOffset := start % chunkSize
	offset := start - fOffset
	offsetEnd := offset + chunkSize

	Log.Debugf("Getting object %v bytes %v - %v (is preload: %v)", b.object.ObjectID, offset, offsetEnd, isPreload)

	filename := filepath.Join(b.tempDir, strconv.Itoa(int(offset)))
	if f, err := os.Open(filename); nil == err {
		defer f.Close()
		buf := make([]byte, size)
		if n, err := f.ReadAt(buf, fOffset); nil == err && n > 0 {
			Log.Debugf("Found object %v bytes %v - %v in cache", b.object.ObjectID, offset, offsetEnd)

			// update the last modified time for files that are often in use
			if err := os.Chtimes(filename, time.Now(), time.Now()); nil != err {
				Log.Warningf("Could not update last modified time for %v", filename)
			}

			return buf[:size], nil
		}
	}

	if chunkDirMaxSize > 0 {
		if err := cleanChunkDir(chunkPath); nil != err {
			Log.Debugf("%v", err)
			return nil, fmt.Errorf("Could not delete oldest chunk")
		}
	}

	Log.Debugf("Requesting object %v bytes %v - %v from API", b.object.ObjectID, offset, offsetEnd)
	req, err := http.NewRequest("GET", b.object.DownloadURL, nil)
	if nil != err {
		return nil, err
	}

	req.Header.Add("Range", fmt.Sprintf("bytes=%v-%v", offset, offsetEnd))

	Log.Tracef("Sending HTTP Request %v", req)

	res, err := b.client.Do(req)
	if nil != err {
		return nil, err
	}

	if res.StatusCode != 206 {
		return nil, fmt.Errorf("Wrong status code %v", res)
	}

	bytes, err := ioutil.ReadAll(res.Body)
	if nil != err {
		return nil, err
	}

	f, err := os.Create(filename)
	if nil != err {
		return nil, err
	}
	defer f.Close()

	_, err = f.Write(bytes)
	if nil != err {
		return nil, err
	}

	if !isPreload && b.preload && uint64(offsetEnd) < b.object.Size {
		go func() {
			b.ReadBytes(offsetEnd+1, size, true)
		}()
	}

	return bytes[fOffset:int64(math.Min(float64(fOffset+size), float64(len(bytes))))], nil
}

// cleanChunkDir checks if the chunk folder is grown to big and clears the oldest file if necessary
func cleanChunkDir(chunkPath string) error {
	chunkDirSize, err := dirSize(chunkPath)
	if nil != err {
		return err
	}

	if chunkDirSize+chunkSize > chunkDirMaxSize {
		if err := deleteOldestFile(chunkPath); nil != err {
			return err
		}
	}

	return nil
}

// deleteOldestFile deletes the oldest file in the directory
func deleteOldestFile(path string) error {
	var fpath string
	lastMod := time.Now()

	err := filepath.Walk(path, func(file string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			modTime := info.ModTime()
			if modTime.Before(lastMod) {
				lastMod = modTime
				fpath = file
			}
		}
		return err
	})

	os.Remove(fpath)

	return err
}

// dirSize gets the total directory size
func dirSize(path string) (int64, error) {
	var size int64
	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			size += info.Size()
		}
		return err
	})
	return size, err
}
<commit_msg>Deal with last block from cache<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"strconv"

	"time"

	. "github.com/claudetech/loggo/default"
	"github.com/orcaman/concurrent-map"
)

var instances cmap.ConcurrentMap
var chunkPath string
var chunkSize int64
var chunkDirMaxSize int64

func init() {
	instances = cmap.New()
}

// Buffer is a buffered stream
type Buffer struct {
	numberOfInstances int
	client            *http.Client
	object            *APIObject
	tempDir           string
	preload           bool
	chunkDir          string
}

// GetBufferInstance gets a singleton instance of buffer
func GetBufferInstance(client *http.Client, object *APIObject) (*Buffer, error) {
	if !instances.Has(object.ObjectID) {
		i, err := newBuffer(client, object)
		if nil != err {
			return nil, err
		}

		instances.Set(object.ObjectID, i)
	}

	instance, ok := instances.Get(object.ObjectID)
	// if buffer allocation failed due to race conditions it will try to fetch a new one
	if !ok {
		i, err := GetBufferInstance(client, object)
		if nil != err {
			return nil, err
		}
		instance = i
	}
	instance.(*Buffer).numberOfInstances++
	return instance.(*Buffer), nil
}

// SetChunkPath sets the global chunk path
func SetChunkPath(path string) {
	chunkPath = path
}

// SetChunkSize sets the global chunk size
func SetChunkSize(size int64) {
	chunkSize = size
}

// SetChunkDirMaxSize sets the maximum size of the chunk directory
func SetChunkDirMaxSize(size int64) {
	chunkDirMaxSize = size
}

// NewBuffer creates a new buffer instance
func newBuffer(client *http.Client, object *APIObject) (*Buffer, error) {
	Log.Infof("Starting playback of %v", object.Name)
	Log.Debugf("Creating buffer for object %v", object.ObjectID)

	tempDir := filepath.Join(chunkPath, object.ObjectID)
	if err := os.MkdirAll(tempDir, 0777); nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not create temp path for object %v", object.ObjectID)
	}

	if 0 == chunkSize {
		Log.Debugf("ChunkSize was 0, setting to default (5 MB)")
		chunkSize = 5 * 1024 * 1024
	}

	buffer := Buffer{
		numberOfInstances: 0,
		client:            client,
		object:            object,
		tempDir:           tempDir,
		preload:           true,
	}

	return &buffer, nil
}

// Close all handles
func (b *Buffer) Close() error {
	b.numberOfInstances--
	if 0 == b.numberOfInstances {
		Log.Infof("Stopping playback of %v", b.object.Name)
		Log.Debugf("Stop buffering for object %v", b.object.ObjectID)

		b.preload = false
		instances.Remove(b.object.ObjectID)
	}
	return nil
}

// ReadBytes on a specific location
func (b *Buffer) ReadBytes(start, size int64, isPreload bool) ([]byte, error) {
	fOffset := start % chunkSize
	offset := start - fOffset
	offsetEnd := offset + chunkSize

	Log.Debugf("Getting object %v - chunk %v - offset %v for %v bytes (is preload: %v)", b.object.ObjectID, offset, fOffset, size, isPreload)

	filename := filepath.Join(b.tempDir, strconv.Itoa(int(offset)))
	if f, err := os.Open(filename); nil == err {
		defer f.Close()
		buf := make([]byte, size)
		if n, err := f.ReadAt(buf, fOffset); n > 0 {
			Log.Debugf("Found object %v bytes %v - %v in cache", b.object.ObjectID, offset, offsetEnd)

			// update the last modified time for files that are often in use
			if err := os.Chtimes(filename, time.Now(), time.Now()); nil != err {
				Log.Warningf("Could not update last modified time for %v", filename)
			}

			return buf[:size], nil
		} else {
			Log.Debugf("Could not read file %s at %v - err : %v", filename, fOffset, err)
		}
	}

	if chunkDirMaxSize > 0 {
		if err := cleanChunkDir(chunkPath); nil != err {
			Log.Debugf("%v", err)
			return nil, fmt.Errorf("Could not delete oldest chunk")
		}
	}

	Log.Debugf("Requesting object %v bytes %v - %v from API", b.object.ObjectID, offset, offsetEnd)
	req, err := http.NewRequest("GET", b.object.DownloadURL, nil)
	if nil != err {
		return nil, err
	}

	req.Header.Add("Range", fmt.Sprintf("bytes=%v-%v", offset, offsetEnd))

	Log.Tracef("Sending HTTP Request %v", req)

	res, err := b.client.Do(req)
	if nil != err {
		return nil, err
	}

	if res.StatusCode != 206 {
		return nil, fmt.Errorf("Wrong status code %v", res)
	}

	bytes, err := ioutil.ReadAll(res.Body)
	if nil != err {
		return nil, err
	}

	f, err := os.Create(filename)
	if nil != err {
		return nil, err
	}
	defer f.Close()

	_, err = f.Write(bytes)
	if nil != err {
		return nil, err
	}

	if !isPreload && b.preload && uint64(offsetEnd) < b.object.Size {
		go func() {
			b.ReadBytes(offsetEnd+1, size, true)
		}()
	}

	return bytes[fOffset:int64(math.Min(float64(fOffset+size), float64(len(bytes))))], nil
}

// cleanChunkDir checks if the chunk folder is grown to big and clears the oldest file if necessary
func cleanChunkDir(chunkPath string) error {
	chunkDirSize, err := dirSize(chunkPath)
	if nil != err {
		return err
	}

	if chunkDirSize+chunkSize > chunkDirMaxSize {
		if err := deleteOldestFile(chunkPath); nil != err {
			return err
		}
	}

	return nil
}

// deleteOldestFile deletes the oldest file in the directory
func deleteOldestFile(path string) error {
	var fpath string
	lastMod := time.Now()

	err := filepath.Walk(path, func(file string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			modTime := info.ModTime()
			if modTime.Before(lastMod) {
				lastMod = modTime
				fpath = file
			}
		}
		return err
	})

	os.Remove(fpath)

	return err
}

// dirSize gets the total directory size
func dirSize(path string) (int64, error) {
	var size int64
	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			size += info.Size()
		}
		return err
	})
	return size, err
}
<|endoftext|>


<commit_before>//
// markup.go
// memeposting markup parser
//
package srnd

import (
  "html"
  "regexp"
  "strings"
)

// copypasted from https://stackoverflow.com/questions/161738/what-is-the-best-regular-expression-to-check-if-a-string-is-a-valid-url
// var re_external_link = regexp.MustCompile(`((?:(?:https?|ftp):\/\/)(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?)`);
var re_external_link = regexp.MustCompile(`((https?|ftp):\/\/[\w\d\.]*\/[\w\d\.\/]*)`);

func formatline(line string) (markup string) {
  line = strings.Trim(line, "\t\r\n ")
  if len(line) > 0 {
    if strings.HasPrefix(line, ">") {
      // le ebin meme arrows
      markup += "
"
      markup += html.EscapeString(line)
      markup += "
"
    } else if strings.HasPrefix(line, "==") && strings.HasSuffix(line, "==") {
      // redtext
      markup += "
"
      markup += html.EscapeString(line[2:len(line)-2])
      markup += "
"
    } else {
      // regular line
      markup += "
"
      // linkify it
      line = html.EscapeString(line)
      markup += re_external_link.ReplaceAllString(line, `<a href="$1">$1</a>`)
      markup += "
"
    }
  }
  return
}

// format lines inside a code tag
func formatcodeline(line string) (markup string) {
  markup += "
"
  markup += html.EscapeString(line)
  markup += "
"
  return
}

func memeposting(src string) (markup string) {
  found_tag := false
  tag_content := ""
  tag := ""
  // for each line...
  for _, line := range strings.Split(src, "\n") {
    // beginning of code tag ?
    if strings.Count(line, "[code]") > 0 {
      // yes there's a code tag
      found_tag = true
      tag = "code"
    } else if strings.Count(line, "[spoiler]") > 0 {
      // spoiler tag
      found_tag = true
      tag = "spoiler"
    } else if strings.Count(line, "[psy]") > 0 {
      // psy tag
      found_tag = true
      tag = "psy"
    }
    if found_tag {
      // collect content of tag
      tag_content += line + "\n"
      // end of our tag ?
      if strings.Count(line, "[/"+tag+"]") == 1 {
        // yah
        found_tag = false
        var tag_open, tag_close string
        if tag == "code" {
          tag_open = "
"
          tag_close = "
"
        } else if tag == "spoiler" {
          tag_open = ""
          tag_close = ""
        } else if tag == "psy" {
          tag_open = "
"
          tag_close = "
"
        }
        markup += tag_open
        // remove open tag, only once so we can have a code tag verbatum inside
        tag_content = strings.Replace(tag_content, "["+tag+"]", "", 1)
        // remove all close tags, should only have 1
        tag_content = strings.Replace(tag_content, "[/"+tag+"]", "", -1)
        // make into lines
        for _, tag_line := range strings.Split(tag_content, "\n") {
          if tag == "code" {
            markup += formatcodeline(tag_line)
          } else {
            markup += formatline(tag_line)
          }
        }
        // close pre tag
        markup += tag_close
        // reset content buffer
        tag_content = ""
      }
      // next line
      continue
    }
    // format line regularlly
    markup += formatline(line)
  }
  // flush the rest of an incomplete code tag
  for _, line := range strings.Split(tag_content, "\n") {
    markup += formatline(line)
  }
  return
}
<commit_msg>ammend url regex again<commit_after>//
// markup.go
// memeposting markup parser
//
package srnd

import (
  "html"
  "regexp"
  "strings"
)

// copypasted from https://stackoverflow.com/questions/161738/what-is-the-best-regular-expression-to-check-if-a-string-is-a-valid-url
// var re_external_link = regexp.MustCompile(`((?:(?:https?|ftp):\/\/)(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?)`);
var re_external_link = regexp.MustCompile(`((https?|ftp):\/\/[\w\d\.\-_]*\/[\w\d\.\/\-_]*)`);

func formatline(line string) (markup string) {
  line = strings.Trim(line, "\t\r\n ")
  if len(line) > 0 {
    if strings.HasPrefix(line, ">") {
      // le ebin meme arrows
      markup += "
"
      markup += html.EscapeString(line)
      markup += "
"
    } else if strings.HasPrefix(line, "==") && strings.HasSuffix(line, "==") {
      // redtext
      markup += "
"
      markup += html.EscapeString(line[2:len(line)-2])
      markup += "
"
    } else {
      // regular line
      markup += "
"
      // linkify it
      line = html.EscapeString(line)
      markup += re_external_link.ReplaceAllString(line, `<a href="$1">$1</a>`)
      markup += "
"
    }
  }
  return
}

// format lines inside a code tag
func formatcodeline(line string) (markup string) {
  markup += "
"
  markup += html.EscapeString(line)
  markup += "
"
  return
}

func memeposting(src string) (markup string) {
  found_tag := false
  tag_content := ""
  tag := ""
  // for each line...
  for _, line := range strings.Split(src, "\n") {
    // beginning of code tag ?
    if strings.Count(line, "[code]") > 0 {
      // yes there's a code tag
      found_tag = true
      tag = "code"
    } else if strings.Count(line, "[spoiler]") > 0 {
      // spoiler tag
      found_tag = true
      tag = "spoiler"
    } else if strings.Count(line, "[psy]") > 0 {
      // psy tag
      found_tag = true
      tag = "psy"
    }
    if found_tag {
      // collect content of tag
      tag_content += line + "\n"
      // end of our tag ?
      if strings.Count(line, "[/"+tag+"]") == 1 {
        // yah
        found_tag = false
        var tag_open, tag_close string
        if tag == "code" {
          tag_open = "
"
          tag_close = "
"
        } else if tag == "spoiler" {
          tag_open = ""
          tag_close = ""
        } else if tag == "psy" {
          tag_open = "
"
          tag_close = "
"
        }
        markup += tag_open
        // remove open tag, only once so we can have a code tag verbatum inside
        tag_content = strings.Replace(tag_content, "["+tag+"]", "", 1)
        // remove all close tags, should only have 1
        tag_content = strings.Replace(tag_content, "[/"+tag+"]", "", -1)
        // make into lines
        for _, tag_line := range strings.Split(tag_content, "\n") {
          if tag == "code" {
            markup += formatcodeline(tag_line)
          } else {
            markup += formatline(tag_line)
          }
        }
        // close pre tag
        markup += tag_close
        // reset content buffer
        tag_content = ""
      }
      // next line
      continue
    }
    // format line regularlly
    markup += formatline(line)
  }
  // flush the rest of an incomplete code tag
  for _, line := range strings.Split(tag_content, "\n") {
    markup += formatline(line)
  }
  return
}
<|endoftext|>


<commit_before>// written by Daniel Oaks <daniel@danieloaks.net>
// released under the ISC license

package ircbnc

import (
	"bufio"
	"database/sql"
	"fmt"
	"log"
	"net"
	"strconv"
	"strings"

	"github.com/goshuirc/irc-go/client"
	"github.com/goshuirc/irc-go/eventmgr"
	"github.com/goshuirc/irc-go/ircfmt"
	"github.com/goshuirc/irc-go/ircmsg"
)

// ServerConnectionAddress represents an address a ServerConnection can join.
type ServerConnectionAddress struct {
	Address string
	Port    int
	UseTLS  bool
}

// ServerConnection represents a connection to an IRC server.
type ServerConnection struct {
	Name      string
	User      User
	Connected bool

	Nickname   string
	FbNickname string
	Username   string
	Realname   string
	Channels   map[string]string

	receiveLines  chan *string
	ReceiveEvents chan Message

	storingConnectMessages bool
	connectMessages        []ircmsg.IrcMessage
	currentServer          *gircclient.ServerConnection
	Listeners              []Listener

	Password  string
	Addresses []ServerConnectionAddress
}

// LoadServerConnection loads the given server connection from our database.
func LoadServerConnection(name string, user User, db *sql.DB) (*ServerConnection, error) {
	var sc ServerConnection
	sc.storingConnectMessages = true
	sc.receiveLines = make(chan *string)
	sc.ReceiveEvents = make(chan Message)
	sc.Name = name
	sc.User = user

	row := db.QueryRow(`SELECT nickname, fallback_nickname, username, realname, password FROM server_connections WHERE user_id = ? AND name = ?`,
		user.ID, name)
	err := row.Scan(&sc.Nickname, &sc.FbNickname, &sc.Username, &sc.Realname, &sc.Password)
	if err != nil {
		return nil, fmt.Errorf("Could not create new ServerConnection (loading sc details from db): %s", err.Error())
	}

	// set default values
	if sc.Nickname == "" {
		sc.Nickname = user.DefaultNick
	}
	if sc.FbNickname == "" {
		sc.FbNickname = user.DefaultFbNick
	}
	if sc.Username == "" {
		sc.Username = user.DefaultUser
	}
	if sc.Realname == "" {
		sc.Realname = user.DefaultReal
	}

	// load channels
	sc.Channels = make(map[string]string)
	rows, err := db.Query(`SELECT name, key FROM server_connection_channels WHERE user_id = ? AND sc_name = ?`,
		user.ID, name)
	if err != nil {
		return nil, fmt.Errorf("Could not create new ServerConnection (loading address details from db): %s", err.Error())
	}
	for rows.Next() {
		var name, key string
		rows.Scan(&name, &key)

		sc.Channels[name] = key
	}

	// load addresses
	rows, err = db.Query(`SELECT address, port, use_tls FROM server_connection_addresses WHERE user_id = ? AND sc_name = ?`,
		user.ID, name)
	if err != nil {
		return nil, fmt.Errorf("Could not create new ServerConnection (loading address details from db): %s", err.Error())
	}
	for rows.Next() {
		var address, portString string
		var useTLS bool

		rows.Scan(&address, &portString, &useTLS)

		port, err := strconv.Atoi(portString)
		if err != nil {
			return nil, fmt.Errorf("Could not create new ServerConnection (port did not load correctly): %s", err.Error())
		} else if port < 1 || port > 65535 {
			return nil, fmt.Errorf("Could not create new ServerConnection (port %d is not valid)", port)
		}

		var newAddress ServerConnectionAddress
		newAddress.Address = address
		newAddress.Port = port
		newAddress.UseTLS = useTLS
		sc.Addresses = append(sc.Addresses, newAddress)
	}

	return &sc, nil
}

var storedConnectLines = map[string]bool{
	"001": true,
	"002": true,
	"003": true,
	"004": true,
	"005": true,
	"250": true,
	"251": true,
	"252": true,
	"254": true,
	"255": true,
	"265": true,
	"266": true,
	"372": true,
	"375": true,
	"376": true,
	"422": true,
}

// disconnectHandler extracts and stores .
func (sc *ServerConnection) disconnectHandler(event string, info eventmgr.InfoMap) {
	sc.currentServer = nil

	for _, listener := range sc.Listeners {
		listener.Send(nil, listener.Bouncer.StatusSource, "PRIVMSG", "Disconnected from server")
	}
}

// connectLinesHandler extracts and stores the connection lines.
func (sc *ServerConnection) connectLinesHandler(event string, info eventmgr.InfoMap) {
	if !sc.storingConnectMessages {
		return
	}

	line := info["data"].(string)
	message, err := ircmsg.ParseLine(line)
	if err != nil {
		return
	}

	_, storeMessage := storedConnectLines[message.Command]
	if storeMessage {
		// fmt.Println("IN:", message)
		sc.connectMessages = append(sc.connectMessages, message)
	}

	if message.Command == "376" || message.Command == "422" {
		sc.storingConnectMessages = false
	}
}

// DumpRegistration dumps the registration messages of this server to the given Listener.
func (sc *ServerConnection) DumpRegistration(listener *Listener) {
	// if server is not currently connected, just dump a nil connect
	if sc.currentServer == nil {
		listener.SendNilConnect()
		return
	}

	// change nick if user has a different one set
	//TODO(dan): If nick if diff. we may want to dump a NICK message, but maybe not.
	// If clients get nick from 001, it'll be fine.
	listener.ClientNick = sc.currentServer.Nick

	// dump reg
	for _, message := range sc.connectMessages {
		message.Params[0] = listener.ClientNick
		listener.Send(&message.Tags, message.Prefix, message.Command, message.Params...)
	}

}

// rawHandler prints raw messages to and from the server.
//TODO(dan): This is only VERY INITIAL, for use while we are debugging.
func rawHandler(event string, info eventmgr.InfoMap) {
	server := info["server"].(*gircclient.ServerConnection)
	direction := info["direction"].(string)
	line := info["data"].(string)

	var arrow string
	if direction == "in" {
		arrow = "<- "
	} else {
		arrow = " ->"
	}

	fmt.Println(server.Name, arrow, ircfmt.Escape(strings.Trim(line, "\r\n")))
}

func (sc *ServerConnection) lineReceiveLoop(server *gircclient.ServerConnection) {
	// wait for the connection to become available
	server.WaitForConnection()

	reader := bufio.NewReader(server.RawConnection)

	for {
		line, err := reader.ReadString('\n')
		if err != nil {
			sc.receiveLines <- nil
			break
		}

		sc.receiveLines <- &line
	}

	server.Disconnect()
}

// ReceiveLoop runs a loop of receiving and dispatching new messages.
func (sc *ServerConnection) ReceiveLoop(server *gircclient.ServerConnection) {
	var msg Message
	var line *string
	for {
		select {
		case line = <-sc.receiveLines:
			if line == nil {
				break
			}
			server.ProcessIncomingLine(*line)
		case msg = <-sc.ReceiveEvents:
			if msg.Type == AddListenerMT {
				listener := msg.Info[ListenerIK].(*Listener)
				sc.Listeners = append(sc.Listeners, *listener)
				listener.ServerConnection = sc
			} else {
				log.Fatal("Got an event I cannot parse")
				fmt.Println(msg)
			}
		}
	}
}

// AddListener adds the given listener to this ServerConnection.
func (sc *ServerConnection) AddListener(listener *Listener) {
	message := NewMessage(AddListenerMT, NoMV)
	message.Info[ListenerIK] = listener
	sc.ReceiveEvents <- message
}

// Start opens and starts connecting to the server.
func (sc *ServerConnection) Start(reactor gircclient.Reactor) {
	name := fmt.Sprintf("%s %s", sc.User.ID, sc.Name)
	server := reactor.CreateServer(name)
	sc.currentServer = server

	server.InitialNick = sc.Nickname
	server.InitialUser = sc.Username
	server.InitialRealName = sc.Realname
	server.ConnectionPass = sc.Password
	server.FallbackNicks = append(server.FallbackNicks, sc.FbNickname)

	server.RegisterEvent("in", "raw", sc.connectLinesHandler, 0)
	server.RegisterEvent("out", "server disconnected", sc.disconnectHandler, 0)
	server.RegisterEvent("in", "raw", rawHandler, 0)
	server.RegisterEvent("out", "raw", rawHandler, 0)

	var err error
	for _, address := range sc.Addresses {
		fullAddress := net.JoinHostPort(address.Address, strconv.Itoa(address.Port))

		err = server.Connect(fullAddress, address.UseTLS, nil)
		if err == nil {
			break
		}
	}

	if err != nil {
		fmt.Println("ERROR: Could not connect to", name, err.Error())
		return
	}

	go sc.lineReceiveLoop(server)
	go sc.ReceiveLoop(server)
}
<commit_msg>Update deps<commit_after>// written by Daniel Oaks <daniel@danieloaks.net>
// released under the ISC license

package ircbnc

import (
	"bufio"
	"database/sql"
	"fmt"
	"log"
	"net"
	"strconv"
	"strings"

	"github.com/goshuirc/eventmgr"
	"github.com/goshuirc/irc-go/client"
	"github.com/goshuirc/irc-go/ircfmt"
	"github.com/goshuirc/irc-go/ircmsg"
)

// ServerConnectionAddress represents an address a ServerConnection can join.
type ServerConnectionAddress struct {
	Address string
	Port    int
	UseTLS  bool
}

// ServerConnection represents a connection to an IRC server.
type ServerConnection struct {
	Name      string
	User      User
	Connected bool

	Nickname   string
	FbNickname string
	Username   string
	Realname   string
	Channels   map[string]string

	receiveLines  chan *string
	ReceiveEvents chan Message

	storingConnectMessages bool
	connectMessages        []ircmsg.IrcMessage
	currentServer          *gircclient.ServerConnection
	Listeners              []Listener

	Password  string
	Addresses []ServerConnectionAddress
}

// LoadServerConnection loads the given server connection from our database.
func LoadServerConnection(name string, user User, db *sql.DB) (*ServerConnection, error) {
	var sc ServerConnection
	sc.storingConnectMessages = true
	sc.receiveLines = make(chan *string)
	sc.ReceiveEvents = make(chan Message)
	sc.Name = name
	sc.User = user

	row := db.QueryRow(`SELECT nickname, fallback_nickname, username, realname, password FROM server_connections WHERE user_id = ? AND name = ?`,
		user.ID, name)
	err := row.Scan(&sc.Nickname, &sc.FbNickname, &sc.Username, &sc.Realname, &sc.Password)
	if err != nil {
		return nil, fmt.Errorf("Could not create new ServerConnection (loading sc details from db): %s", err.Error())
	}

	// set default values
	if sc.Nickname == "" {
		sc.Nickname = user.DefaultNick
	}
	if sc.FbNickname == "" {
		sc.FbNickname = user.DefaultFbNick
	}
	if sc.Username == "" {
		sc.Username = user.DefaultUser
	}
	if sc.Realname == "" {
		sc.Realname = user.DefaultReal
	}

	// load channels
	sc.Channels = make(map[string]string)
	rows, err := db.Query(`SELECT name, key FROM server_connection_channels WHERE user_id = ? AND sc_name = ?`,
		user.ID, name)
	if err != nil {
		return nil, fmt.Errorf("Could not create new ServerConnection (loading address details from db): %s", err.Error())
	}
	for rows.Next() {
		var name, key string
		rows.Scan(&name, &key)

		sc.Channels[name] = key
	}

	// load addresses
	rows, err = db.Query(`SELECT address, port, use_tls FROM server_connection_addresses WHERE user_id = ? AND sc_name = ?`,
		user.ID, name)
	if err != nil {
		return nil, fmt.Errorf("Could not create new ServerConnection (loading address details from db): %s", err.Error())
	}
	for rows.Next() {
		var address, portString string
		var useTLS bool

		rows.Scan(&address, &portString, &useTLS)

		port, err := strconv.Atoi(portString)
		if err != nil {
			return nil, fmt.Errorf("Could not create new ServerConnection (port did not load correctly): %s", err.Error())
		} else if port < 1 || port > 65535 {
			return nil, fmt.Errorf("Could not create new ServerConnection (port %d is not valid)", port)
		}

		var newAddress ServerConnectionAddress
		newAddress.Address = address
		newAddress.Port = port
		newAddress.UseTLS = useTLS
		sc.Addresses = append(sc.Addresses, newAddress)
	}

	return &sc, nil
}

var storedConnectLines = map[string]bool{
	"001": true,
	"002": true,
	"003": true,
	"004": true,
	"005": true,
	"250": true,
	"251": true,
	"252": true,
	"254": true,
	"255": true,
	"265": true,
	"266": true,
	"372": true,
	"375": true,
	"376": true,
	"422": true,
}

// disconnectHandler extracts and stores .
func (sc *ServerConnection) disconnectHandler(event string, info eventmgr.InfoMap) {
	sc.currentServer = nil

	for _, listener := range sc.Listeners {
		listener.Send(nil, listener.Bouncer.StatusSource, "PRIVMSG", "Disconnected from server")
	}
}

// connectLinesHandler extracts and stores the connection lines.
func (sc *ServerConnection) connectLinesHandler(event string, info eventmgr.InfoMap) {
	if !sc.storingConnectMessages {
		return
	}

	line := info["data"].(string)
	message, err := ircmsg.ParseLine(line)
	if err != nil {
		return
	}

	_, storeMessage := storedConnectLines[message.Command]
	if storeMessage {
		// fmt.Println("IN:", message)
		sc.connectMessages = append(sc.connectMessages, message)
	}

	if message.Command == "376" || message.Command == "422" {
		sc.storingConnectMessages = false
	}
}

// DumpRegistration dumps the registration messages of this server to the given Listener.
func (sc *ServerConnection) DumpRegistration(listener *Listener) {
	// if server is not currently connected, just dump a nil connect
	if sc.currentServer == nil {
		listener.SendNilConnect()
		return
	}

	// change nick if user has a different one set
	//TODO(dan): If nick if diff. we may want to dump a NICK message, but maybe not.
	// If clients get nick from 001, it'll be fine.
	listener.ClientNick = sc.currentServer.Nick

	// dump reg
	for _, message := range sc.connectMessages {
		message.Params[0] = listener.ClientNick
		listener.Send(&message.Tags, message.Prefix, message.Command, message.Params...)
	}

}

// rawHandler prints raw messages to and from the server.
//TODO(dan): This is only VERY INITIAL, for use while we are debugging.
func rawHandler(event string, info eventmgr.InfoMap) {
	server := info["server"].(*gircclient.ServerConnection)
	direction := info["direction"].(string)
	line := info["data"].(string)

	var arrow string
	if direction == "in" {
		arrow = "<- "
	} else {
		arrow = " ->"
	}

	fmt.Println(server.Name, arrow, ircfmt.Escape(strings.Trim(line, "\r\n")))
}

func (sc *ServerConnection) lineReceiveLoop(server *gircclient.ServerConnection) {
	// wait for the connection to become available
	server.WaitForConnection()

	reader := bufio.NewReader(server.RawConnection)

	for {
		line, err := reader.ReadString('\n')
		if err != nil {
			sc.receiveLines <- nil
			break
		}

		sc.receiveLines <- &line
	}

	server.Disconnect()
}

// ReceiveLoop runs a loop of receiving and dispatching new messages.
func (sc *ServerConnection) ReceiveLoop(server *gircclient.ServerConnection) {
	var msg Message
	var line *string
	for {
		select {
		case line = <-sc.receiveLines:
			if line == nil {
				break
			}
			server.ProcessIncomingLine(*line)
		case msg = <-sc.ReceiveEvents:
			if msg.Type == AddListenerMT {
				listener := msg.Info[ListenerIK].(*Listener)
				sc.Listeners = append(sc.Listeners, *listener)
				listener.ServerConnection = sc
			} else {
				log.Fatal("Got an event I cannot parse")
				fmt.Println(msg)
			}
		}
	}
}

// AddListener adds the given listener to this ServerConnection.
func (sc *ServerConnection) AddListener(listener *Listener) {
	message := NewMessage(AddListenerMT, NoMV)
	message.Info[ListenerIK] = listener
	sc.ReceiveEvents <- message
}

// Start opens and starts connecting to the server.
func (sc *ServerConnection) Start(reactor gircclient.Reactor) {
	name := fmt.Sprintf("%s %s", sc.User.ID, sc.Name)
	server := reactor.CreateServer(name)
	sc.currentServer = server

	server.InitialNick = sc.Nickname
	server.InitialUser = sc.Username
	server.InitialRealName = sc.Realname
	server.ConnectionPass = sc.Password
	server.FallbackNicks = append(server.FallbackNicks, sc.FbNickname)

	server.RegisterEvent("in", "raw", sc.connectLinesHandler, 0)
	server.RegisterEvent("out", "server disconnected", sc.disconnectHandler, 0)
	server.RegisterEvent("in", "raw", rawHandler, 0)
	server.RegisterEvent("out", "raw", rawHandler, 0)

	var err error
	for _, address := range sc.Addresses {
		fullAddress := net.JoinHostPort(address.Address, strconv.Itoa(address.Port))

		err = server.Connect(fullAddress, address.UseTLS, nil)
		if err == nil {
			break
		}
	}

	if err != nil {
		fmt.Println("ERROR: Could not connect to", name, err.Error())
		return
	}

	go sc.lineReceiveLoop(server)
	go sc.ReceiveLoop(server)
}
<|endoftext|>


<commit_before>package trace

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"log"
	"os"
)

type Image [][]Color

const maxColor = 65535

func colorComponentToBytes(c uint16) []byte {
	buf := new(bytes.Buffer)
	err := binary.Write(buf, binary.BigEndian, c)
	if err != nil {
		log.Fatal(err)
	}
	return buf.Bytes()
}

func makeImage(w, h uint32) Image {
	image := make([][]Color, h)
	for i := range image {
		image[i] = make([]Color, w)
		for j := range image[i] {
			image[i][j] = Color{0.0, 0.0, 0.0}
		}
	}
	return image
}

// TODO: gamma correct
func normalizeImage(image Image) {
	// find max
	max := 0.0
	for i := range image {
		for j := range image[i] {
			if image[i][j].R > max {
				max = image[i][j].R
			}
			if image[i][j].G > max {
				max = image[i][j].G
			}
			if image[i][j].B > max {
				max = image[i][j].B
			}
		}
	}

	// normalize
	for i := range image {
		for j := range image[i] {
			image[i][j].Scale(1.0 / max)
		}
	}
}

func WriteImageToPPM(image Image, name string) {
	f, err := os.Create(name + ".ppm")
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	log.Printf("Writing image to %s.ppm\n", name)
	normalizeImage(image)

	_, err = f.WriteString(fmt.Sprintf("P6 %d %d %d\n", len(image[0]), len(image), maxColor))
	if err != nil {
		log.Fatal(err)
	}
	for y := range image {
		for x := range image[y] {
			n, err := f.Write(colorComponentToBytes(uint16(image[y][x].R * maxColor)))
			if err != nil {
				log.Fatal(err)
			}
			if n != 2 {
				log.Fatal(fmt.Sprintf("r != 2 bytes: %.3f", image[y][x].R))
			}
			n, err = f.Write(colorComponentToBytes(uint16(image[y][x].G * maxColor)))
			if err != nil {
				log.Fatal(err)
			}
			if n != 2 {
				log.Fatal(fmt.Sprintf("g != 2 bytes: %.3f", image[y][x].G))
			}
			n, err = f.Write(colorComponentToBytes(uint16(image[y][x].B * maxColor)))
			if err != nil {
				log.Fatal(err)
			}
			if n != 2 {
				log.Fatal(fmt.Sprintf("b != 2 bytes: %.3f", image[y][x].B))
			}
		}
	}

}

<commit_msg>Optimise file writing by writing a row at a time<commit_after>package trace

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"log"
	"os"
)

type Image [][]Color

const maxColor = 65535

// Convert a row of uint16s to a byte slice
func colorToBytes(r [][3]uint16) []byte {
	buf := new(bytes.Buffer)
	err := binary.Write(buf, binary.BigEndian, r)
	if err != nil {
		log.Fatal(err)
	}
	return buf.Bytes()
}

// Create an image w x h pixels
func makeImage(w, h uint32) Image {
	image := make([][]Color, h)
	for i := range image {
		image[i] = make([]Color, w)
		for j := range image[i] {
			image[i][j] = Color{0.0, 0.0, 0.0}
		}
	}
	return image
}

// TODO: gamma correct
// Ensure the brightest component of any colour is 1.0
func normalizeImage(image Image) {
	// find max
	max := 0.0
	for i := range image {
		for j := range image[i] {
			if image[i][j].R > max {
				max = image[i][j].R
			}
			if image[i][j].G > max {
				max = image[i][j].G
			}
			if image[i][j].B > max {
				max = image[i][j].B
			}
		}
	}

	// normalize
	for i := range image {
		for j := range image[i] {
			image[i][j].Scale(1.0 / max)
		}
	}
}

// Write the given image to a PPM format file of the given name
func WriteImageToPPM(image Image, name string) {
	f, err := os.Create(name + ".ppm")
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	log.Printf("Normalizing image\n")
	normalizeImage(image)

	log.Printf("Converting image\n")
	outImage := make([][][3]uint16, len(image))
	for i := range outImage {
		outImage[i] = make([][3]uint16, len(image[i]))
		for j := range outImage[i] {
			outImage[i][j][0] = uint16(image[i][j].R * maxColor)
			outImage[i][j][1] = uint16(image[i][j].G * maxColor)
			outImage[i][j][2] = uint16(image[i][j].B * maxColor)
		}
	}

	log.Printf("Writing image to %s.ppm\n", name)
	_, err = f.WriteString(fmt.Sprintf("P6 %d %d %d\n", len(outImage[0]), len(outImage), maxColor))
	if err != nil {
		log.Fatal(err)
	}
	for y := range outImage {
		// Write a row of the image to the file
		_, err := f.Write(colorToBytes(outImage[y]))
		if err != nil {
			log.Fatal(err)
		}
	}
	log.Println("done")
}

<|endoftext|>


<commit_before>package beauties

import (
	"crypto/sha512"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
)

// Storage is a Storage interface
type Storage interface {
	String() string
	Get(token, filename string) (reader File, contentType string, contentLength int64, err error)
	Head(token, filename string) (contentType string, contentLength int64, err error)
	Put(token, filename string, reader io.Reader, contentLength int64) error
	Delete(token, filename string) (err error)
	IsNotExist(err error) bool
}

// File is a File interface combining io.Reader, io.Seeker and io.Closer
type File interface {
	io.Reader
	io.Seeker
	io.Closer
}

// LocalStorage is an implementation of a Storage interface using
// local directory to store files
type LocalStorage struct {
	Storage
	Name    string
	basedir string
}

// NewLocalStorage returns a LocalStorage instance
func NewLocalStorage(basedir string) (storage *LocalStorage, err error) {
	storage = &LocalStorage{basedir: basedir, Name: fmt.Sprintf("LocalStorage %s", basedir)}
	err = os.MkdirAll(basedir, 0750)
	return
}

func (s *LocalStorage) hash(filename string) (hash string) {
	fn := []byte(filename)
	hasher := sha512.New()
	hasher.Write(fn)
	hash = fmt.Sprintf("%x", hasher.Sum(nil))
	return
}

func (s *LocalStorage) getPath(token, filename string) (path string) {
	path = filepath.Join(s.basedir, s.hash(token+filename))
	return
}

// String returns a string representation of LocalStorage
func (s *LocalStorage) String() string {
	return s.Name
}

// Head returns content type and content length to use in e.g. HTTP
// HEAD method
func (s *LocalStorage) Head(token string, filename string) (contentType string, contentLength int64, err error) {
	path := s.getPath(token, filename)

	var fi os.FileInfo
	if fi, err = os.Lstat(path); err != nil {
		return
	}

	contentLength = int64(fi.Size())
	contentType = s.getContentType(path)

	return
}

// Get retrieves file from a storage
func (s *LocalStorage) Get(token string, filename string) (reader File, contentType string, contentLength int64, err error) {
	path := s.getPath(token, filename)

	// content type , content length
	if reader, err = os.Open(path); err != nil {
		return
	}

	var fi os.FileInfo
	if fi, err = os.Lstat(path); err != nil {
		return
	}

	contentLength = int64(fi.Size())
	contentType = s.getContentType(path)

	return
}

// Delete deletes file from a storage
func (s *LocalStorage) Delete(token, filename string) (err error) {
	path := s.getPath(token, filename)
	err = os.RemoveAll(path)
	return
}

// IsNotExist checks whether error meaning is file doesn't exists
func (s *LocalStorage) IsNotExist(err error) bool {
	return os.IsNotExist(err)
}

// Put puts file in a storage
func (s *LocalStorage) Put(token string, filename string, reader io.Reader, contentLength int64) error {
	var f io.WriteCloser
	var err error

	path := s.getPath(token, filename)

	if f, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600); err != nil && !os.IsExist(err) {
		fmt.Printf("%s", err)
		return err
	}

	defer f.Close()

	if _, err = io.Copy(f, reader); err != nil {
		return err
	}

	return nil
}

func (s *LocalStorage) getContentType(path string) (ct string) {
	var reader File
	var err error

	reader, err = os.Open(path)
	if reader, err = os.Open(path); err != nil {
		return
	}

	buffer := make([]byte, 512)
	if _, err = reader.Read(buffer); err != nil {
		return
	}

	reader.Close()

	ct = http.DetectContentType(buffer)

	if ct == "application/octet-stream" {
		ct = ""
	}

	return

}
<commit_msg>I don't know what these changes are supposed to accomplish but somebody told me to make them.<commit_after>package beauties

import (
	"crypto/sha512"
	"fmt"
	"io"
	"mime"
	"net/http"
	"os"
	"path/filepath"
)

// Storage is a Storage interface
type Storage interface {
	String() string
	Get(token, filename string) (reader File, contentType string, contentLength int64, err error)
	Head(token, filename string) (contentType string, contentLength int64, err error)
	Put(token, filename string, reader io.Reader, contentLength int64) error
	Delete(token, filename string) (err error)
	IsNotExist(err error) bool
}

// File is a File interface combining io.Reader, io.Seeker and io.Closer
type File interface {
	io.Reader
	io.Seeker
	io.Closer
}

// LocalStorage is an implementation of a Storage interface using
// local directory to store files
type LocalStorage struct {
	Storage
	Name    string
	basedir string
}

// NewLocalStorage returns a LocalStorage instance
func NewLocalStorage(basedir string) (storage *LocalStorage, err error) {
	storage = &LocalStorage{basedir: basedir, Name: fmt.Sprintf("LocalStorage %s", basedir)}
	err = os.MkdirAll(basedir, 0750)
	return
}

func (s *LocalStorage) hash(filename string) (hash string) {
	fn := []byte(filename)
	hasher := sha512.New()
	hasher.Write(fn)
	hash = fmt.Sprintf("%x", hasher.Sum(nil))
	return
}

func (s *LocalStorage) getPath(token, filename string) (path string) {
	path = filepath.Join(s.basedir, s.hash(token+filename))
	return
}

// String returns a string representation of LocalStorage
func (s *LocalStorage) String() string {
	return s.Name
}

// Head returns content type and content length to use in e.g. HTTP
// HEAD method
func (s *LocalStorage) Head(token string, filename string) (contentType string, contentLength int64, err error) {
	path := s.getPath(token, filename)

	var fi os.FileInfo
	if fi, err = os.Lstat(path); err != nil {
		return
	}

	contentLength = int64(fi.Size())
	contentType = s.getContentType(path)

	return
}

// Get retrieves file from a storage
func (s *LocalStorage) Get(token string, filename string) (reader File, contentType string, contentLength int64, err error) {
	path := s.getPath(token, filename)

	// content type , content length
	if reader, err = os.Open(path); err != nil {
		return
	}

	var fi os.FileInfo
	if fi, err = os.Lstat(path); err != nil {
		return
	}

	contentLength = int64(fi.Size())
	contentType = s.getContentType(path)

	return
}

// Delete deletes file from a storage
func (s *LocalStorage) Delete(token, filename string) (err error) {
	path := s.getPath(token, filename)
	err = os.RemoveAll(path)
	return
}

// IsNotExist checks whether error meaning is file doesn't exists
func (s *LocalStorage) IsNotExist(err error) bool {
	return os.IsNotExist(err)
}

// Put puts file in a storage
func (s *LocalStorage) Put(token string, filename string, reader io.Reader, contentLength int64) error {
	var f io.WriteCloser
	var err error

	path := s.getPath(token, filename)

	if f, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600); err != nil && !os.IsExist(err) {
		fmt.Printf("%s", err)
		return err
	}

	defer f.Close()

	if _, err = io.Copy(f, reader); err != nil {
		return err
	}

	return nil
}

func (s *LocalStorage) getContentType(path string) (ct string) {
	ct = mime.TypeByExtension(filepath.Ext(path))
	if ct != "" {
		return
	}

	var reader File
	var err error

	reader, err = os.Open(path)
	if reader, err = os.Open(path); err != nil {
		return
	}

	buffer := make([]byte, 512)
	if _, err = reader.Read(buffer); err != nil {
		return
	}

	reader.Close()

	ct = http.DetectContentType(buffer)

	if ct == "application/octet-stream" {
		ct = ""
	}

	return

}
<|endoftext|>


<commit_before>package signals

import (
	"encoding/gob"
	"errors"
	"io"
	"net/http"
	"net/url"
	"encoding/base64"
	"os"
	"regexp"
	"strconv"
	"strings"
	"fmt"
)

func init() {
	gob.Register(&Wave{})
}

const bufferSize = 16

// a PCM-Signal, read as required, from a URL.
// if queried for a property value from an x that is more than 32 samples lower than a previous query, will return zero.
type Wave struct {
	Offset
	URL    string
	reader io.Reader
}

func (s *Wave) property(p x) y {
	if s.reader == nil {
		wav, err := NewWave(s.URL)
		failOn(err)
		s.Offset = wav.Offset
		s.reader = wav.reader
	}
	for p > s.MaxX() {
		// append available data onto the PCM slice.
		// also possibly shift off some data, shortening the PCM slice, retaining at least two buffer lengths.
		// partial samples are read but not accessed by property.
		switch st := s.Offset.LimitedSignal.(type) {
		case PCM8bit:
			sd := PCM8bit{st.PCM}
			sd.Data = append(sd.Data, make([]byte, bufferSize)...)
			n, err := s.reader.Read(sd.Data[len(sd.Data)-bufferSize:])
			failOn(err)
			sd.Data = sd.Data[:len(sd.Data)-bufferSize+n]
			if len(sd.Data) > bufferSize*3 {
				sd.Data = sd.Data[bufferSize:]
				s.Offset = Offset{sd, s.Offset.Offset + bufferSize*st.samplePeriod}
			} else {
				s.Offset = Offset{sd, s.Offset.Offset}
			}
		case PCM16bit:
			sd := PCM16bit{st.PCM}
			sd.Data = append(sd.Data, make([]byte, bufferSize*2)...)
			n, err := s.reader.Read(sd.Data[len(sd.Data)-bufferSize*2:])
			failOn(err)
			sd.Data = sd.Data[:len(sd.Data)-bufferSize*2+n]
			if len(sd.Data) > bufferSize*2*3 {
				sd.Data = sd.Data[bufferSize*2:]
				s.Offset = Offset{sd, s.Offset.Offset + bufferSize*st.samplePeriod}
			} else {
				s.Offset = Offset{sd, s.Offset.Offset}
			}
		case PCM24bit:
			sd := PCM24bit{st.PCM}
			sd.Data = append(sd.Data, make([]byte, bufferSize*3)...)
			n, err := s.reader.Read(sd.Data[len(sd.Data)-bufferSize*3:])
			failOn(err)
			sd.Data = sd.Data[:len(sd.Data)-bufferSize*3+n]
			if len(sd.Data) > bufferSize*3*3 {
				sd.Data = sd.Data[bufferSize*3:]
				s.Offset = Offset{sd, s.Offset.Offset + bufferSize*st.samplePeriod}
			} else {
				s.Offset = Offset{sd, s.Offset.Offset}
			}
		case PCM32bit:
			sd := PCM32bit{st.PCM}
			sd.Data = append(sd.Data, make([]byte, bufferSize*4)...)
			n, err := s.reader.Read(sd.Data[len(sd.Data)-bufferSize*4:])
			failOn(err)
			sd.Data = sd.Data[:len(sd.Data)-bufferSize*4+n]
			if len(sd.Data) > bufferSize*4*3 {
				sd.Data = sd.Data[bufferSize*4:]
				s.Offset = Offset{sd, s.Offset.Offset + bufferSize*st.samplePeriod}
			} else {
				s.Offset = Offset{sd, s.Offset.Offset}
			}
		case PCM48bit:
			sd := PCM48bit{st.PCM}
			sd.Data = append(sd.Data, make([]byte, bufferSize*6)...)
			n, err := s.reader.Read(sd.Data[len(sd.Data)-bufferSize*6:])
			failOn(err)
			sd.Data = sd.Data[:len(sd.Data)-bufferSize*6+n]
			if len(sd.Data) > bufferSize*6*3 {
				sd.Data = sd.Data[bufferSize*6:]
				s.Offset = Offset{sd, s.Offset.Offset + bufferSize*st.samplePeriod}
			} else {
				s.Offset = Offset{sd, s.Offset.Offset}
			}
		case PCM64bit:
			sd := PCM64bit{st.PCM}
			sd.Data = append(sd.Data, make([]byte, bufferSize*8)...)
			n, err := s.reader.Read(sd.Data[len(sd.Data)-bufferSize*8:])
			failOn(err)
			sd.Data = sd.Data[:len(sd.Data)-bufferSize*8+n]
			if len(sd.Data) > bufferSize*8*3 {
				sd.Data = sd.Data[bufferSize*8:]
				s.Offset = Offset{sd, s.Offset.Offset + bufferSize*st.samplePeriod}
			} else {
				s.Offset = Offset{sd, s.Offset.Offset}
			}
		}
	}
	return s.Offset.property(p)
}

//func updateShifted(s Shifted, r io.Reader, b *[]byte, blockSize int) (err error){
//	b=append(b,make([]byte,bufferSize*blockSize)...)
//	n, err := r.Read(b[len(b)-bufferSize*blockSize:])
//	failOn(err)
//	b=b[:len(b)-bufferSize*blockSize+n]
//	if len(b)>bufferSize*blockSize*3{
//		b=b[bufferSize*blockSize:]
//		s.Offset+=bufferSize*s.samplePeriod
//	}
//}

func NewWave(URL string) (*Wave, error) {
	r, channels, bytes, rate, err := PCMReader(URL)
	if err != nil {
		return nil, err
	}
	if channels != 1 {
		return nil, errors.New(URL + ":Needs to be mono.")
	}
	b := make([]byte, bufferSize*bytes)
	n, err := r.Read(b)
	failOn(err)
	b = b[:n]
	switch bytes {
	case 1:
		return &Wave{Offset{NewPCM8bit(rate, b), 0}, URL, r}, nil
	case 2:
		return &Wave{Offset{NewPCM16bit(rate, b), 0}, URL, r}, nil
	case 3:
		return &Wave{Offset{NewPCM24bit(rate, b), 0}, URL, r}, nil
	case 4:
		return &Wave{Offset{NewPCM32bit(rate, b), 0}, URL, r}, nil
	case 6:
		return &Wave{Offset{NewPCM48bit(rate, b), 0}, URL, r}, nil
	case 8:
		return &Wave{Offset{NewPCM64bit(rate, b), 0}, URL, r}, nil
	}
	return nil, ErrWaveParse{"Source bit rate not supported."}
}

var contentTypeParse = regexp.MustCompile(`^audio/l(\d+);rate=(\d+)$`)

// returns a reader to a resource, along with its Channel count, Precision (bytes) and Samples per second.
func PCMReader(resourceLocation string) (io.Reader, uint16, uint16, uint32, error) {
	//	resp, err := http.Get(resourceLocation)
	url, err := url.Parse(resourceLocation)
	if err != nil {
		return nil, 0, 0, 0, err
	}
	switch url.Scheme {
	case "file":
		file, err := os.Open(url.Path)
		if err != nil {
			return nil, 0, 0, 0, err
		}
		_, format, err := readWaveHeader(file)
		if err != nil {
			return nil, 0, 0, 0, err
		}
		return file, format.Channels, format.SampleBytes, format.SampleRate, nil
	case "data":
		mimeAndRest := strings.SplitN(url.Opaque, ";", 2)
		encodingAndData := strings.SplitN(mimeAndRest[1], ",", 2)
		r := strings.NewReader(encodingAndData[1])
		dr:= base64.NewDecoder(base64.StdEncoding, r)
		_, format, err := readWaveHeader(dr)
		if err != nil {
			return nil, 0, 0, 0, err
		}
		fmt.Println(format)
		return dr, format.Channels, format.SampleBytes, format.SampleRate, nil
	default: // whatever supported and placed in Body, currently basically "http" or "https"
		resp, err := http.DefaultClient.Do(&http.Request{Method: "GET", URL: url})

		if err != nil {
			return nil, 0, 0, 0, err
		}
		if resp.Header["Content-Type"][0] == "sound/wav" || resp.Header["Content-Type"][0] == "audio/x-wav" {
			_, format, err := readWaveHeader(resp.Body)
			if err != nil {
				return nil, 0, 0, 0, err
			}
			return resp.Body, format.Channels, format.SampleBytes, format.SampleRate, nil
		}
		pcmFormat := contentTypeParse.FindStringSubmatch(resp.Header["Content-Type"][0])
		if pcmFormat != nil {
			bits, err := strconv.ParseUint(pcmFormat[1], 10, 19)
			if err != nil {
				return nil, 0, 0, 0, err
			}
			rate, err := strconv.ParseUint(pcmFormat[2], 10, 32)
			if err != nil {
				return nil, 0, 0, 0, err
			}
			return resp.Body, 1, uint16(bits / 8), uint32(rate), nil
		}
		return nil, 0, 0, 0, errors.New("Source in unrecognized format.")
	}
	return nil, 0, 0, 0, errors.New("Source has unrecognized Scheme." + url.Scheme)
}

func failOn(e error) {
	if e != nil {
		panic(e)
	}
}

<commit_msg>base64 and wave/audio in data url support<commit_after>package signals

import (
	"encoding/gob"
	"errors"
	"io"
	"net/http"
	"net/url"
	"encoding/base64"
	"os"
	"regexp"
	"strconv"
	"strings"
)

func init() {
	gob.Register(&Wave{})
}

const bufferSize = 16

// a PCM-Signal, read as required, from a URL.
// if queried for a property value from an x that is more than 32 samples lower than a previous query, will return zero.
type Wave struct {
	Offset
	URL    string
	reader io.Reader
}

func (s *Wave) property(p x) y {
	if s.reader == nil {
		wav, err := NewWave(s.URL)
		failOn(err)
		s.Offset = wav.Offset
		s.reader = wav.reader
	}
	for p > s.MaxX() {
		// append available data onto the PCM slice.
		// also possibly shift off some data, shortening the PCM slice, retaining at least two buffer lengths.
		// partial samples are read but not accessed by property.
		switch st := s.Offset.LimitedSignal.(type) {
		case PCM8bit:
			sd := PCM8bit{st.PCM}
			sd.Data = append(sd.Data, make([]byte, bufferSize)...)
			n, err := s.reader.Read(sd.Data[len(sd.Data)-bufferSize:])
			failOn(err)
			sd.Data = sd.Data[:len(sd.Data)-bufferSize+n]
			if len(sd.Data) > bufferSize*3 {
				sd.Data = sd.Data[bufferSize:]
				s.Offset = Offset{sd, s.Offset.Offset + bufferSize*st.samplePeriod}
			} else {
				s.Offset = Offset{sd, s.Offset.Offset}
			}
		case PCM16bit:
			sd := PCM16bit{st.PCM}
			sd.Data = append(sd.Data, make([]byte, bufferSize*2)...)
			n, err := s.reader.Read(sd.Data[len(sd.Data)-bufferSize*2:])
			failOn(err)
			sd.Data = sd.Data[:len(sd.Data)-bufferSize*2+n]
			if len(sd.Data) > bufferSize*2*3 {
				sd.Data = sd.Data[bufferSize*2:]
				s.Offset = Offset{sd, s.Offset.Offset + bufferSize*st.samplePeriod}
			} else {
				s.Offset = Offset{sd, s.Offset.Offset}
			}
		case PCM24bit:
			sd := PCM24bit{st.PCM}
			sd.Data = append(sd.Data, make([]byte, bufferSize*3)...)
			n, err := s.reader.Read(sd.Data[len(sd.Data)-bufferSize*3:])
			failOn(err)
			sd.Data = sd.Data[:len(sd.Data)-bufferSize*3+n]
			if len(sd.Data) > bufferSize*3*3 {
				sd.Data = sd.Data[bufferSize*3:]
				s.Offset = Offset{sd, s.Offset.Offset + bufferSize*st.samplePeriod}
			} else {
				s.Offset = Offset{sd, s.Offset.Offset}
			}
		case PCM32bit:
			sd := PCM32bit{st.PCM}
			sd.Data = append(sd.Data, make([]byte, bufferSize*4)...)
			n, err := s.reader.Read(sd.Data[len(sd.Data)-bufferSize*4:])
			failOn(err)
			sd.Data = sd.Data[:len(sd.Data)-bufferSize*4+n]
			if len(sd.Data) > bufferSize*4*3 {
				sd.Data = sd.Data[bufferSize*4:]
				s.Offset = Offset{sd, s.Offset.Offset + bufferSize*st.samplePeriod}
			} else {
				s.Offset = Offset{sd, s.Offset.Offset}
			}
		case PCM48bit:
			sd := PCM48bit{st.PCM}
			sd.Data = append(sd.Data, make([]byte, bufferSize*6)...)
			n, err := s.reader.Read(sd.Data[len(sd.Data)-bufferSize*6:])
			failOn(err)
			sd.Data = sd.Data[:len(sd.Data)-bufferSize*6+n]
			if len(sd.Data) > bufferSize*6*3 {
				sd.Data = sd.Data[bufferSize*6:]
				s.Offset = Offset{sd, s.Offset.Offset + bufferSize*st.samplePeriod}
			} else {
				s.Offset = Offset{sd, s.Offset.Offset}
			}
		case PCM64bit:
			sd := PCM64bit{st.PCM}
			sd.Data = append(sd.Data, make([]byte, bufferSize*8)...)
			n, err := s.reader.Read(sd.Data[len(sd.Data)-bufferSize*8:])
			failOn(err)
			sd.Data = sd.Data[:len(sd.Data)-bufferSize*8+n]
			if len(sd.Data) > bufferSize*8*3 {
				sd.Data = sd.Data[bufferSize*8:]
				s.Offset = Offset{sd, s.Offset.Offset + bufferSize*st.samplePeriod}
			} else {
				s.Offset = Offset{sd, s.Offset.Offset}
			}
		}
	}
	return s.Offset.property(p)
}

//func updateShifted(s Shifted, r io.Reader, b *[]byte, blockSize int) (err error){
//	b=append(b,make([]byte,bufferSize*blockSize)...)
//	n, err := r.Read(b[len(b)-bufferSize*blockSize:])
//	failOn(err)
//	b=b[:len(b)-bufferSize*blockSize+n]
//	if len(b)>bufferSize*blockSize*3{
//		b=b[bufferSize*blockSize:]
//		s.Offset+=bufferSize*s.samplePeriod
//	}
//}

func NewWave(URL string) (*Wave, error) {
	r, channels, bytes, rate, err := PCMReader(URL)
	if err != nil {
		return nil, err
	}
	if channels != 1 {
		return nil, errors.New(URL + ":Needs to be mono.")
	}
	b := make([]byte, bufferSize*bytes)
	n, err := r.Read(b)
	failOn(err)
	b = b[:n]
	switch bytes {
	case 1:
		return &Wave{Offset{NewPCM8bit(rate, b), 0}, URL, r}, nil
	case 2:
		return &Wave{Offset{NewPCM16bit(rate, b), 0}, URL, r}, nil
	case 3:
		return &Wave{Offset{NewPCM24bit(rate, b), 0}, URL, r}, nil
	case 4:
		return &Wave{Offset{NewPCM32bit(rate, b), 0}, URL, r}, nil
	case 6:
		return &Wave{Offset{NewPCM48bit(rate, b), 0}, URL, r}, nil
	case 8:
		return &Wave{Offset{NewPCM64bit(rate, b), 0}, URL, r}, nil
	}
	return nil, ErrWaveParse{"Source bit rate not supported."}
}

var contentTypeParse = regexp.MustCompile(`^audio/l(\d+);rate=(\d+)$`)

// returns a reader to a resource, along with its Channel count, Precision (bytes) and Samples per second.
func PCMReader(resourceLocation string) (io.Reader, uint16, uint16, uint32, error) {
	//	resp, err := http.Get(resourceLocation)
	url, err := url.Parse(resourceLocation)
	if err != nil {
		return nil, 0, 0, 0, err
	}
	switch url.Scheme {
	case "file":
		file, err := os.Open(url.Path)
		if err != nil {
			return nil, 0, 0, 0, err
		}
		_, format, err := readWaveHeader(file)
		if err != nil {
			return nil, 0, 0, 0, err
		}
		return file, format.Channels, format.SampleBytes, format.SampleRate, nil
	case "data":
		mimeAndRest := strings.SplitN(url.Opaque, ";", 2)
		encodingAndData := strings.SplitN(mimeAndRest[1], ",", 2)
		var r io.Reader
		if encodingAndData[0]=="base64" {
			r= base64.NewDecoder(base64.StdEncoding, strings.NewReader(encodingAndData[1]))
		}else{
			r= strings.NewReader(encodingAndData[1])
		}
		if mimeAndRest[0] == "sound/wav" || mimeAndRest[0] == "audio/x-wav" {
			_, format, err := readWaveHeader(r)
			if err != nil {
				return nil, 0, 0, 0, err
			}
			return r, format.Channels, format.SampleBytes, format.SampleRate, nil
		}
		pcmFormat := contentTypeParse.FindStringSubmatch(mimeAndRest[0])
		if pcmFormat != nil {
			bits, err := strconv.ParseUint(pcmFormat[1], 10, 19)
			if err != nil {
				return nil, 0, 0, 0, err
			}
			rate, err := strconv.ParseUint(pcmFormat[2], 10, 32)
			if err != nil {
				return nil, 0, 0, 0, err
			}
			return r, 1, uint16(bits / 8), uint32(rate), nil
		}
	default: // whatever supported and placed in Body, currently basically "http" or "https"
		resp, err := http.DefaultClient.Do(&http.Request{Method: "GET", URL: url})

		if err != nil {
			return nil, 0, 0, 0, err
		}
		if resp.Header["Content-Type"][0] == "sound/wav" || resp.Header["Content-Type"][0] == "audio/x-wav" {
			_, format, err := readWaveHeader(resp.Body)
			if err != nil {
				return nil, 0, 0, 0, err
			}
			return resp.Body, format.Channels, format.SampleBytes, format.SampleRate, nil
		}
		pcmFormat := contentTypeParse.FindStringSubmatch(resp.Header["Content-Type"][0])
		if pcmFormat != nil {
			bits, err := strconv.ParseUint(pcmFormat[1], 10, 19)
			if err != nil {
				return nil, 0, 0, 0, err
			}
			rate, err := strconv.ParseUint(pcmFormat[2], 10, 32)
			if err != nil {
				return nil, 0, 0, 0, err
			}
			return resp.Body, 1, uint16(bits / 8), uint32(rate), nil
		}
		return nil, 0, 0, 0, errors.New("Source in unrecognized format.")
	}
	return nil, 0, 0, 0, errors.New("Source has unrecognized Scheme." + url.Scheme)
}

func failOn(e error) {
	if e != nil {
		panic(e)
	}
}

<|endoftext|>