<commit_before>package connection

import (
	"encoding/hex"
	"log"
	"net"

	"github.com/piot/hasty-protocol/channel"
	"github.com/piot/hasty-protocol/chunk"
	"github.com/piot/hasty-protocol/commands"
	"github.com/piot/hasty-protocol/packet"
	"github.com/piot/hasty-protocol/packetserializers"
	"github.com/piot/hasty-protocol/serializer"
	"github.com/piot/hasty-protocol/timestamp"
	"github.com/piot/hasty-server/authorization"
	"github.com/piot/hasty-server/master"
	"github.com/piot/hasty-server/storage"
	"github.com/piot/hasty-server/subscribers"
)

type StreamInfo struct {
	lastOffsetSent uint64
}

type ConnectionHandler struct {
	conn          *net.Conn
	storage       *filestorage.StreamStorage
	subscribers   *subscribers.Subscribers
	streamInfos   map[uint32]*StreamInfo
	connectionID  packet.ConnectionID
	masterHandler *master.MasterCommandHandler
	chunkStreams  map[uint32]*chunk.Stream
}

// NewConnectionHandler : todo
func NewConnectionHandler(connection *net.Conn, masterHandler *master.MasterCommandHandler, storage *filestorage.StreamStorage, subs *subscribers.Subscribers, connectionID packet.ConnectionID) *ConnectionHandler {
	return &ConnectionHandler{connectionID: connectionID, masterHandler: masterHandler, conn: connection, storage: storage, subscribers: subs, streamInfos: map[uint32]*StreamInfo{}, chunkStreams: map[uint32]*chunk.Stream{}}
}

// HandleConnect : todo
func (in *ConnectionHandler) HandleConnect(cmd commands.Connect) error {
	log.Printf("%s %s", in.connectionID, cmd)

	// _ := commands.NewConnectResult(cmd.Realm(), cmd.ProtocolVersion())
	octetsToSend := packetserializers.ConnectResultToOctets()
	in.sendPacket(octetsToSend)
	return nil
}

func (in *ConnectionHandler) sendPong(echoedTime timestamp.Time) {
	log.Printf("%s sendPong %s", in.connectionID, echoedTime)
	now := timestamp.Now()
	octetsToSend := packetserializers.PongToOctets(now, echoedTime)
	in.sendPacket(octetsToSend)
}

func (in *ConnectionHandler) sendLoginResult(worked bool) {
	log.Printf("%s sendLoginResult %t", in.connectionID, worked)
	octetsToSend := packetserializers.LoginResultToOctets()
	in.sendPacket(octetsToSend)
}

// HandlePing : todo
func (in *ConnectionHandler) HandlePing(cmd commands.Ping) {
	log.Printf("%s %s", in.connectionID, cmd)
	in.sendPong(cmd.SentTime())
}

// HandlePong : todo
func (in *ConnectionHandler) HandlePong(cmd commands.Pong) {
	now := timestamp.Now()
	latency := now.Raw() - cmd.EchoedTime().Raw()
	log.Printf("%s Latency: %d ms", in.connectionID, latency)
}

// HandlePublishStream : todo
func (in *ConnectionHandler) HandlePublishStream(cmd commands.PublishStream) error {
	log.Printf("%s %s", in.connectionID, cmd)
	return nil
}

func (in *ConnectionHandler) StreamChanged(channelID channel.ID) {
	info := in.streamInfos[channelID.Raw()]
	file, fileErr := in.storage.ReadStream(channelID)
	if fileErr != nil {
		return
	}
	file.Seek(info.lastOffsetSent)
	data := make([]byte, 32*1024)
	octetsRead, readErr := file.Read(data)
	if readErr != nil {
		return
	}
	file.Close()
	in.sendStreamData(channelID, uint32(info.lastOffsetSent), data[:octetsRead])
	info.lastOffsetSent += uint64(octetsRead)
}

func (in *ConnectionHandler) sendStreamData(channelID channel.ID, lastOffsetSent uint32, data []byte) {
	log.Printf("%s sendStreamData %s offset:%d", in.connectionID, channelID, lastOffsetSent)
	payload := packetserializers.StreamDataToOctets(channelID, lastOffsetSent, data)
	in.sendPacket(payload)
}

func (in *ConnectionHandler) fetchOrCreateStreamInfo(channelID channel.ID) *StreamInfo {
	infos := in.streamInfos[channelID.Raw()]
	if infos == nil {
		infos = &StreamInfo{}
		in.streamInfos[channelID.Raw()] = infos
	}
	return infos
}

// HandleSubscribeStream : todo
func (in *ConnectionHandler) HandleSubscribeStream(cmd commands.SubscribeStream) {
	log.Printf("%s %s", in.connectionID, cmd)

	for _, v := range cmd.Infos() {
		readFile, err := in.storage.ReadStream(v.Channel())

		if err != nil {
			return
		}

		buf := make([]byte, 64*1024)
		octetsRead, readErr := readFile.Read(buf)
		if readErr != nil {
			return
		}
		data := buf[:octetsRead]

		octetsToSend := packetserializers.StreamDataToOctets(v.Channel(), 0, data)
		in.sendPacket(octetsToSend)
		infos := in.fetchOrCreateStreamInfo(v.Channel())
		infos.lastOffsetSent = uint64(octetsRead)
		in.subscribers.AddStreamSubscriber(v.Channel(), in)
	}
}

// HandleUnsubscribeStream : todo
func (in *ConnectionHandler) HandleUnsubscribeStream(cmd commands.UnsubscribeStream) {
	log.Printf("%s %s", in.connectionID, cmd)
}

// HandleCreateStream : todo
func (in *ConnectionHandler) HandleCreateStream(cmd commands.CreateStream) (channel.ID, error) {
	log.Printf("%s %s", in.connectionID, cmd)
	return channel.ID{}, nil
}

func (in *ConnectionHandler) fetchOrAssoicateChunkStream(channelID channel.ID) *chunk.Stream {
	stream := in.chunkStreams[channelID.Raw()]
	if stream == nil {
		stream = chunk.NewChunkStream(in.connectionID, channelID)
		in.chunkStreams[channelID.Raw()] = stream
	}
	return stream
}

func (in *ConnectionHandler) publishMasterStream(channel channel.ID, payload []byte) {
	fakeClient := authorization.AdminClient{}
	cmd := commands.NewPublishStream(channel, payload)
	in.masterHandler.HandlePublishStream(fakeClient, cmd)
}

// HandleStreamData : todo
func (in *ConnectionHandler) HandleStreamData(cmd commands.StreamData) {
	log.Printf("%s %s", in.connectionID, cmd)
	chunkStream := in.fetchOrAssoicateChunkStream(cmd.Channel())
	chunkStream.Feed(cmd.Data())
	foundChunk, fetchErr := chunkStream.FetchChunk()
	if fetchErr != nil {
		_, isNotDoneError := fetchErr.(*chunk.NotDoneError)
		if isNotDoneError {
		} else {
			log.Printf("Fetcherror:%s", fetchErr)
		}
	} else {
		in.publishMasterStream(cmd.Channel(), foundChunk.Payload())
	}
}

// HandleLogin : todo
func (in *ConnectionHandler) HandleLogin(cmd commands.Login) error {
	log.Printf("%s", cmd)
	in.sendLoginResult(true)
	return nil
}

func (in *ConnectionHandler) sendPacket(octets []byte) {
	payloadLength := uint16(len(octets))
	hexPayload := hex.Dump(octets)
	lengthBuf, lengthErr := serializer.SmallLengthToOctets(payloadLength)
	if lengthErr != nil {
		log.Printf("We couldn't write length")
		return
	}
	log.Printf("%s Sending packet (size %d) %s", in.connectionID, payloadLength, hexPayload)
	(*in.conn).Write(lengthBuf)
	(*in.conn).Write(octets)
}

// HandleTransportDisconnect : todo
func (in *ConnectionHandler) HandleTransportDisconnect() {
	log.Printf("%s Transport disconnect", in.connectionID)
}
<commit_msg>Now seeks to the request stream offset in subscribe stream.<commit_after>package connection

import (
	"encoding/hex"
	"log"
	"net"

	"github.com/piot/hasty-protocol/channel"
	"github.com/piot/hasty-protocol/chunk"
	"github.com/piot/hasty-protocol/commands"
	"github.com/piot/hasty-protocol/packet"
	"github.com/piot/hasty-protocol/packetserializers"
	"github.com/piot/hasty-protocol/serializer"
	"github.com/piot/hasty-protocol/timestamp"
	"github.com/piot/hasty-server/authorization"
	"github.com/piot/hasty-server/master"
	"github.com/piot/hasty-server/storage"
	"github.com/piot/hasty-server/subscribers"
)

type StreamInfo struct {
	lastOffsetSent uint64
}

type ConnectionHandler struct {
	conn          *net.Conn
	storage       *filestorage.StreamStorage
	subscribers   *subscribers.Subscribers
	streamInfos   map[uint32]*StreamInfo
	connectionID  packet.ConnectionID
	masterHandler *master.MasterCommandHandler
	chunkStreams  map[uint32]*chunk.Stream
}

// NewConnectionHandler : todo
func NewConnectionHandler(connection *net.Conn, masterHandler *master.MasterCommandHandler, storage *filestorage.StreamStorage, subs *subscribers.Subscribers, connectionID packet.ConnectionID) *ConnectionHandler {
	return &ConnectionHandler{connectionID: connectionID, masterHandler: masterHandler, conn: connection, storage: storage, subscribers: subs, streamInfos: map[uint32]*StreamInfo{}, chunkStreams: map[uint32]*chunk.Stream{}}
}

// HandleConnect : todo
func (in *ConnectionHandler) HandleConnect(cmd commands.Connect) error {
	log.Printf("%s %s", in.connectionID, cmd)

	// _ := commands.NewConnectResult(cmd.Realm(), cmd.ProtocolVersion())
	octetsToSend := packetserializers.ConnectResultToOctets()
	in.sendPacket(octetsToSend)
	return nil
}

func (in *ConnectionHandler) sendPong(echoedTime timestamp.Time) {
	log.Printf("%s sendPong %s", in.connectionID, echoedTime)
	now := timestamp.Now()
	octetsToSend := packetserializers.PongToOctets(now, echoedTime)
	in.sendPacket(octetsToSend)
}

func (in *ConnectionHandler) sendLoginResult(worked bool) {
	log.Printf("%s sendLoginResult %t", in.connectionID, worked)
	octetsToSend := packetserializers.LoginResultToOctets()
	in.sendPacket(octetsToSend)
}

// HandlePing : todo
func (in *ConnectionHandler) HandlePing(cmd commands.Ping) {
	log.Printf("%s %s", in.connectionID, cmd)
	in.sendPong(cmd.SentTime())
}

// HandlePong : todo
func (in *ConnectionHandler) HandlePong(cmd commands.Pong) {
	now := timestamp.Now()
	latency := now.Raw() - cmd.EchoedTime().Raw()
	log.Printf("%s Latency: %d ms", in.connectionID, latency)
}

// HandlePublishStream : todo
func (in *ConnectionHandler) HandlePublishStream(cmd commands.PublishStream) error {
	log.Printf("%s %s", in.connectionID, cmd)
	return nil
}

func (in *ConnectionHandler) StreamChanged(channelID channel.ID) {
	info := in.streamInfos[channelID.Raw()]
	file, fileErr := in.storage.ReadStream(channelID)
	if fileErr != nil {
		return
	}
	file.Seek(info.lastOffsetSent)
	data := make([]byte, 32*1024)
	octetsRead, readErr := file.Read(data)
	if readErr != nil {
		return
	}
	file.Close()
	in.sendStreamData(channelID, uint32(info.lastOffsetSent), data[:octetsRead])
	info.lastOffsetSent += uint64(octetsRead)
}

func (in *ConnectionHandler) sendStreamData(channelID channel.ID, lastOffsetSent uint32, data []byte) {
	log.Printf("%s sendStreamData %s offset:%d", in.connectionID, channelID, lastOffsetSent)
	payload := packetserializers.StreamDataToOctets(channelID, lastOffsetSent, data)
	in.sendPacket(payload)
}

func (in *ConnectionHandler) fetchOrCreateStreamInfo(channelID channel.ID) *StreamInfo {
	infos := in.streamInfos[channelID.Raw()]
	if infos == nil {
		infos = &StreamInfo{}
		in.streamInfos[channelID.Raw()] = infos
	}
	return infos
}

// HandleSubscribeStream : todo
func (in *ConnectionHandler) HandleSubscribeStream(cmd commands.SubscribeStream) {
	log.Printf("%s %s", in.connectionID, cmd)

	for _, v := range cmd.Infos() {
		readFile, err := in.storage.ReadStream(v.Channel())

		if err != nil {
			return
		}
		offset := v.Offset()

		const maxSizeBuffer = 1 * 1024 * 1024
		buf := make([]byte, maxSizeBuffer)
		readFile.Seek(uint64(offset))
		octetsRead, readErr := readFile.Read(buf)
		if readErr != nil {
			return
		}
		data := buf[:octetsRead]
		lastOffset := int(offset) + octetsRead
		octetsToSend := packetserializers.StreamDataToOctets(v.Channel(), offset, data)
		in.sendPacket(octetsToSend)
		infos := in.fetchOrCreateStreamInfo(v.Channel())
		infos.lastOffsetSent = uint64(lastOffset)
		in.subscribers.AddStreamSubscriber(v.Channel(), in)
	}
}

// HandleUnsubscribeStream : todo
func (in *ConnectionHandler) HandleUnsubscribeStream(cmd commands.UnsubscribeStream) {
	log.Printf("%s %s", in.connectionID, cmd)
}

// HandleCreateStream : todo
func (in *ConnectionHandler) HandleCreateStream(cmd commands.CreateStream) (channel.ID, error) {
	log.Printf("%s %s", in.connectionID, cmd)
	return channel.ID{}, nil
}

func (in *ConnectionHandler) fetchOrAssoicateChunkStream(channelID channel.ID) *chunk.Stream {
	stream := in.chunkStreams[channelID.Raw()]
	if stream == nil {
		stream = chunk.NewChunkStream(in.connectionID, channelID)
		in.chunkStreams[channelID.Raw()] = stream
	}
	return stream
}

func (in *ConnectionHandler) publishMasterStream(channel channel.ID, payload []byte) {
	fakeClient := authorization.AdminClient{}
	cmd := commands.NewPublishStream(channel, payload)
	in.masterHandler.HandlePublishStream(fakeClient, cmd)
}

// HandleStreamData : todo
func (in *ConnectionHandler) HandleStreamData(cmd commands.StreamData) {
	log.Printf("%s %s", in.connectionID, cmd)
	chunkStream := in.fetchOrAssoicateChunkStream(cmd.Channel())
	chunkStream.Feed(cmd.Data())
	foundChunk, fetchErr := chunkStream.FetchChunk()
	if fetchErr != nil {
		_, isNotDoneError := fetchErr.(*chunk.NotDoneError)
		if isNotDoneError {
		} else {
			log.Printf("Fetcherror:%s", fetchErr)
		}
	} else {
		in.publishMasterStream(cmd.Channel(), foundChunk.Payload())
	}
}

// HandleLogin : todo
func (in *ConnectionHandler) HandleLogin(cmd commands.Login) error {
	log.Printf("%s", cmd)
	in.sendLoginResult(true)
	return nil
}

func (in *ConnectionHandler) sendPacket(octets []byte) {
	payloadLength := uint16(len(octets))
	hexPayload := hex.Dump(octets)
	lengthBuf, lengthErr := serializer.SmallLengthToOctets(payloadLength)
	if lengthErr != nil {
		log.Printf("We couldn't write length")
		return
	}
	log.Printf("%s Sending packet (size %d) %s", in.connectionID, payloadLength, hexPayload)
	(*in.conn).Write(lengthBuf)
	(*in.conn).Write(octets)
}

// HandleTransportDisconnect : todo
func (in *ConnectionHandler) HandleTransportDisconnect() {
	log.Printf("%s Transport disconnect", in.connectionID)
}
<|endoftext|>


<commit_before>/*
Package firego is a REST client for Firebase (https://firebase.com).
*/
package firego

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	_url "net/url"
	"strings"
	"sync"
	"time"
)

// TimeoutDuration is the length of time any request will have to establish
// a connection and receive headers from Firebase before returning
// an ErrTimeout error.
var TimeoutDuration = 30 * time.Second

var defaultRedirectLimit = 30

// ErrTimeout is an error type is that is returned if a request
// exceeds the TimeoutDuration configured.
type ErrTimeout struct {
	error
}

// query parameter constants
const (
	authParam         = "auth"
	shallowParam      = "shallow"
	formatParam       = "format"
	formatVal         = "export"
	orderByParam      = "orderBy"
	limitToFirstParam = "limitToFirst"
	limitToLastParam  = "limitToLast"
	startAtParam      = "startAt"
	endAtParam        = "endAt"
	equalToParam      = "equalTo"
)

const defaultHeartbeat = 2 * time.Minute

// Firebase represents a location in the cloud.
type Firebase struct {
	url           string
	params        _url.Values
	client        *http.Client
	clientTimeout time.Duration

	eventMtx   sync.Mutex
	eventFuncs map[string]chan struct{}

	watchMtx       sync.Mutex
	watching       bool
	watchHeartbeat time.Duration
	stopWatching   chan struct{}
}

// New creates a new Firebase reference,
// if client is nil, http.DefaultClient is used.
func New(url string, client *http.Client) *Firebase {
	fb := &Firebase{
		url:            sanitizeURL(url),
		params:         _url.Values{},
		clientTimeout:  TimeoutDuration,
		stopWatching:   make(chan struct{}),
		watchHeartbeat: defaultHeartbeat,
		eventFuncs:     map[string]chan struct{}{},
	}
	if client == nil {
		var tr *http.Transport
		tr = &http.Transport{
			DisableKeepAlives: true, // https://code.google.com/p/go/issues/detail?id=3514
			Dial: func(network, address string) (net.Conn, error) {
				start := time.Now()
				c, err := net.DialTimeout(network, address, fb.clientTimeout)
				tr.ResponseHeaderTimeout = fb.clientTimeout - time.Since(start)
				return c, err
			},
		}

		client = &http.Client{
			Transport:     tr,
			CheckRedirect: redirectPreserveHeaders,
		}
	}

	fb.client = client
	return fb
}

// Auth sets the custom Firebase token used to authenticate to Firebase.
func (fb *Firebase) Auth(token string) {
	fb.params.Set(authParam, token)
}

// Unauth removes the current token being used to authenticate to Firebase.
func (fb *Firebase) Unauth() {
	fb.params.Del(authParam)
}

// Ref returns a copy of an existing Firebase reference with a new path.
func (fb *Firebase) Ref(path string) (*Firebase, error) {
	newFB := fb.copy()
	parsedURL, err := _url.Parse(fb.url)
	if err != nil {
		return newFB, err
	}
	newFB.url = parsedURL.Scheme + "://" + parsedURL.Host + "/" + strings.Trim(path, "/")
	return newFB, nil
}

// SetURL changes the url for a firebase reference.
func (fb *Firebase) SetURL(url string) {
	fb.url = sanitizeURL(url)
}

// Push creates a reference to an auto-generated child location.
func (fb *Firebase) Push(v interface{}) (*Firebase, error) {
	bytes, err := json.Marshal(v)
	if err != nil {
		return nil, err
	}
	bytes, err = fb.doRequest("POST", bytes)
	if err != nil {
		return nil, err
	}
	var m map[string]string
	if err := json.Unmarshal(bytes, &m); err != nil {
		return nil, err
	}
	newRef := fb.copy()
	newRef.url = fb.url + "/" + m["name"]
	return newRef, err
}

// Remove the Firebase reference from the cloud.
func (fb *Firebase) Remove() error {
	_, err := fb.doRequest("DELETE", nil)
	if err != nil {
		return err
	}
	return nil
}

// Set the value of the Firebase reference.
func (fb *Firebase) Set(v interface{}) error {
	bytes, err := json.Marshal(v)
	if err != nil {
		return err
	}
	_, err = fb.doRequest("PUT", bytes)
	return err
}

// Update the specific child with the given value.
func (fb *Firebase) Update(v interface{}) error {
	bytes, err := json.Marshal(v)
	if err != nil {
		return err
	}
	_, err = fb.doRequest("PATCH", bytes)
	return err
}

// Value gets the value of the Firebase reference.
func (fb *Firebase) Value(v interface{}) error {
	bytes, err := fb.doRequest("GET", nil)
	if err != nil {
		return err
	}
	return json.Unmarshal(bytes, v)
}

// String returns the string representation of the
// Firebase reference.
func (fb *Firebase) String() string {
	path := fb.url + "/.json"

	if len(fb.params) > 0 {
		path += "?" + fb.params.Encode()
	}
	return path
}

// Child creates a new Firebase reference for the requested
// child with the same configuration as the parent.
func (fb *Firebase) Child(child string) *Firebase {
	c := fb.copy()
	c.url = c.url + "/" + child
	return c
}

func (fb *Firebase) copy() *Firebase {
	c := &Firebase{
		url:            fb.url,
		params:         _url.Values{},
		client:         fb.client,
		clientTimeout:  fb.clientTimeout,
		stopWatching:   make(chan struct{}),
		watchHeartbeat: defaultHeartbeat,
		eventFuncs:     map[string]chan struct{}{},
	}

	// making sure to manually copy the map items into a new
	// map to avoid modifying the map reference.
	for k, v := range fb.params {
		c.params[k] = v
	}
	return c
}

func sanitizeURL(url string) string {
	if !strings.HasPrefix(url, "https://") && !strings.HasPrefix(url, "http://") {
		url = "https://" + url
	}

	if strings.HasSuffix(url, "/") {
		url = url[:len(url)-1]
	}

	return url
}

// Preserve headers on redirect.
//
// Reference https://github.com/golang/go/issues/4800
func redirectPreserveHeaders(req *http.Request, via []*http.Request) error {
	if len(via) == 0 {
		// No redirects
		return nil
	}

	if len(via) > defaultRedirectLimit {
		return fmt.Errorf("%d consecutive requests(redirects)", len(via))
	}

	// mutate the subsequent redirect requests with the first Header
	for key, val := range via[0].Header {
		req.Header[key] = val
	}
	return nil
}

func (fb *Firebase) doRequest(method string, body []byte) ([]byte, error) {
	req, err := http.NewRequest(method, fb.String(), bytes.NewReader(body))
	if err != nil {
		return nil, err
	}

	resp, err := fb.client.Do(req)
	switch err := err.(type) {
	default:
		return nil, err
	case nil:
		// carry on

	case *_url.Error:
		// `http.Client.Do` will return a `url.Error` that wraps a `net.Error`
		// when exceeding it's `Transport`'s `ResponseHeadersTimeout`
		e1, ok := err.Err.(net.Error)
		if ok && e1.Timeout() {
			return nil, ErrTimeout{err}
		}

		return nil, err

	case net.Error:
		// `http.Client.Do` will return a `net.Error` directly when Dial times
		// out, or when the Client's RoundTripper otherwise returns an err
		if err.Timeout() {
			return nil, ErrTimeout{err}
		}

		return nil, err
	}

	defer resp.Body.Close()
	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode/200 != 1 {
		return nil, errors.New(string(respBody))
	}
	return respBody, nil
}
<commit_msg>URL()<commit_after>/*
Package firego is a REST client for Firebase (https://firebase.com).
*/
package firego

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	_url "net/url"
	"strings"
	"sync"
	"time"
)

// TimeoutDuration is the length of time any request will have to establish
// a connection and receive headers from Firebase before returning
// an ErrTimeout error.
var TimeoutDuration = 30 * time.Second

var defaultRedirectLimit = 30

// ErrTimeout is an error type is that is returned if a request
// exceeds the TimeoutDuration configured.
type ErrTimeout struct {
	error
}

// query parameter constants
const (
	authParam         = "auth"
	shallowParam      = "shallow"
	formatParam       = "format"
	formatVal         = "export"
	orderByParam      = "orderBy"
	limitToFirstParam = "limitToFirst"
	limitToLastParam  = "limitToLast"
	startAtParam      = "startAt"
	endAtParam        = "endAt"
	equalToParam      = "equalTo"
)

const defaultHeartbeat = 2 * time.Minute

// Firebase represents a location in the cloud.
type Firebase struct {
	url           string
	params        _url.Values
	client        *http.Client
	clientTimeout time.Duration

	eventMtx   sync.Mutex
	eventFuncs map[string]chan struct{}

	watchMtx       sync.Mutex
	watching       bool
	watchHeartbeat time.Duration
	stopWatching   chan struct{}
}

// New creates a new Firebase reference,
// if client is nil, http.DefaultClient is used.
func New(url string, client *http.Client) *Firebase {
	fb := &Firebase{
		url:            sanitizeURL(url),
		params:         _url.Values{},
		clientTimeout:  TimeoutDuration,
		stopWatching:   make(chan struct{}),
		watchHeartbeat: defaultHeartbeat,
		eventFuncs:     map[string]chan struct{}{},
	}
	if client == nil {
		var tr *http.Transport
		tr = &http.Transport{
			DisableKeepAlives: true, // https://code.google.com/p/go/issues/detail?id=3514
			Dial: func(network, address string) (net.Conn, error) {
				start := time.Now()
				c, err := net.DialTimeout(network, address, fb.clientTimeout)
				tr.ResponseHeaderTimeout = fb.clientTimeout - time.Since(start)
				return c, err
			},
		}

		client = &http.Client{
			Transport:     tr,
			CheckRedirect: redirectPreserveHeaders,
		}
	}

	fb.client = client
	return fb
}

// Auth sets the custom Firebase token used to authenticate to Firebase.
func (fb *Firebase) Auth(token string) {
	fb.params.Set(authParam, token)
}

// Unauth removes the current token being used to authenticate to Firebase.
func (fb *Firebase) Unauth() {
	fb.params.Del(authParam)
}

// Ref returns a copy of an existing Firebase reference with a new path.
func (fb *Firebase) Ref(path string) (*Firebase, error) {
	newFB := fb.copy()
	parsedURL, err := _url.Parse(fb.url)
	if err != nil {
		return newFB, err
	}
	newFB.url = parsedURL.Scheme + "://" + parsedURL.Host + "/" + strings.Trim(path, "/")
	return newFB, nil
}

// SetURL changes the url for a firebase reference.
func (fb *Firebase) SetURL(url string) {
	fb.url = sanitizeURL(url)
}

// URL returns firebase reference URL
func (fb *Firebase) URL() string {
	return fb.url
}

// Push creates a reference to an auto-generated child location.
func (fb *Firebase) Push(v interface{}) (*Firebase, error) {
	bytes, err := json.Marshal(v)
	if err != nil {
		return nil, err
	}
	bytes, err = fb.doRequest("POST", bytes)
	if err != nil {
		return nil, err
	}
	var m map[string]string
	if err := json.Unmarshal(bytes, &m); err != nil {
		return nil, err
	}
	newRef := fb.copy()
	newRef.url = fb.url + "/" + m["name"]
	return newRef, err
}

// Remove the Firebase reference from the cloud.
func (fb *Firebase) Remove() error {
	_, err := fb.doRequest("DELETE", nil)
	if err != nil {
		return err
	}
	return nil
}

// Set the value of the Firebase reference.
func (fb *Firebase) Set(v interface{}) error {
	bytes, err := json.Marshal(v)
	if err != nil {
		return err
	}
	_, err = fb.doRequest("PUT", bytes)
	return err
}

// Update the specific child with the given value.
func (fb *Firebase) Update(v interface{}) error {
	bytes, err := json.Marshal(v)
	if err != nil {
		return err
	}
	_, err = fb.doRequest("PATCH", bytes)
	return err
}

// Value gets the value of the Firebase reference.
func (fb *Firebase) Value(v interface{}) error {
	bytes, err := fb.doRequest("GET", nil)
	if err != nil {
		return err
	}
	return json.Unmarshal(bytes, v)
}

// String returns the string representation of the
// Firebase reference.
func (fb *Firebase) String() string {
	path := fb.url + "/.json"

	if len(fb.params) > 0 {
		path += "?" + fb.params.Encode()
	}
	return path
}

// Child creates a new Firebase reference for the requested
// child with the same configuration as the parent.
func (fb *Firebase) Child(child string) *Firebase {
	c := fb.copy()
	c.url = c.url + "/" + child
	return c
}

func (fb *Firebase) copy() *Firebase {
	c := &Firebase{
		url:            fb.url,
		params:         _url.Values{},
		client:         fb.client,
		clientTimeout:  fb.clientTimeout,
		stopWatching:   make(chan struct{}),
		watchHeartbeat: defaultHeartbeat,
		eventFuncs:     map[string]chan struct{}{},
	}

	// making sure to manually copy the map items into a new
	// map to avoid modifying the map reference.
	for k, v := range fb.params {
		c.params[k] = v
	}
	return c
}

func sanitizeURL(url string) string {
	if !strings.HasPrefix(url, "https://") && !strings.HasPrefix(url, "http://") {
		url = "https://" + url
	}

	if strings.HasSuffix(url, "/") {
		url = url[:len(url)-1]
	}

	return url
}

// Preserve headers on redirect.
//
// Reference https://github.com/golang/go/issues/4800
func redirectPreserveHeaders(req *http.Request, via []*http.Request) error {
	if len(via) == 0 {
		// No redirects
		return nil
	}

	if len(via) > defaultRedirectLimit {
		return fmt.Errorf("%d consecutive requests(redirects)", len(via))
	}

	// mutate the subsequent redirect requests with the first Header
	for key, val := range via[0].Header {
		req.Header[key] = val
	}
	return nil
}

func (fb *Firebase) doRequest(method string, body []byte) ([]byte, error) {
	req, err := http.NewRequest(method, fb.String(), bytes.NewReader(body))
	if err != nil {
		return nil, err
	}

	resp, err := fb.client.Do(req)
	switch err := err.(type) {
	default:
		return nil, err
	case nil:
		// carry on

	case *_url.Error:
		// `http.Client.Do` will return a `url.Error` that wraps a `net.Error`
		// when exceeding it's `Transport`'s `ResponseHeadersTimeout`
		e1, ok := err.Err.(net.Error)
		if ok && e1.Timeout() {
			return nil, ErrTimeout{err}
		}

		return nil, err

	case net.Error:
		// `http.Client.Do` will return a `net.Error` directly when Dial times
		// out, or when the Client's RoundTripper otherwise returns an err
		if err.Timeout() {
			return nil, ErrTimeout{err}
		}

		return nil, err
	}

	defer resp.Body.Close()
	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode/200 != 1 {
		return nil, errors.New(string(respBody))
	}
	return respBody, nil
}
<|endoftext|>


<commit_before>package multi

import (
	"os"
	"path"
	"sync"

	"github.com/gotgo/fw/logging"
)

type ImageProcessor struct {

	// Uploader - required
	Uploader Uploader

	// LocalPath - defaults to os.TempDir()/downloads
	LocalPath string

	Log logging.Logger

	MaxHeight int
	MaxWidth  int

	downloader *TaskRun
	phasher    *TaskRun
	resizer    *TaskRun
	uploader   *TaskRun

	outstanding sync.WaitGroup
	complete    chan *ImageProcessorOutput
}

func (ip *ImageProcessor) setup() {
	uploader := ip.Uploader
	if uploader == nil {
		panic("uploader not set")
	}

	if ip.Log == nil {
		ip.Log = &logging.NoOpLogger{}
	}

	var tempFolder = ip.LocalPath
	if tempFolder == "" {
		tempFolder = path.Join(os.TempDir(), "downloads")
	}

	os.MkdirAll(tempFolder, 0774)

	ip.complete = make(chan *ImageProcessorOutput, 100)

	ip.downloader = &TaskRun{
		Action:       &FileDownloadTask{Folder: tempFolder},
		Concurrency:  10,
		MaxQueuedIn:  10 * 5,
		MaxQueuedOut: 10 * 10,
	}

	ip.phasher = &TaskRun{
		Action:       &PHashTask{},
		Concurrency:  2,
		MaxQueuedIn:  2,
		MaxQueuedOut: 100,
	}

	ip.resizer = &TaskRun{
		Action:       &ResizeImageTask{MaxHeight: ip.MaxHeight, MaxWidth: ip.MaxWidth},
		Concurrency:  6,
		MaxQueuedIn:  12,
		MaxQueuedOut: 100,
	}

	ip.uploader = &TaskRun{
		Action:       &FileUploadTask{Uploader: uploader},
		Concurrency:  8,
		MaxQueuedIn:  8 * 10,
		MaxQueuedOut: 100,
	}
}

func (p *ImageProcessor) Startup() {
	p.setup()
	p.doScavenge()
}

func (p *ImageProcessor) Shutdown() {
	p.downloader.Shutdown()
	p.outstanding.Wait()
}

func (p *ImageProcessor) Injest(url, filename string, ctx *DataContext) {
	in := &FileDownloadInput{
		Url:      url,
		Filename: filename,
	}

	p.downloader.Add(in, ctx)
}

func (p *ImageProcessor) doScavenge() {
	p.downloader.Startup()
	p.phasher.Startup()
	p.resizer.Startup()
	p.uploader.Startup()

	p.outstanding.Add(1)
	// phash
	go p.phash()
	p.outstanding.Add(1)
	// resize
	go p.resize()
	p.outstanding.Add(1)
	// upload
	go p.upload()
	p.outstanding.Add(1)
	// collector
	go p.wrapUp()
}

func (p *ImageProcessor) handleError(message string, result *TaskRunOutput) {
	p.Log.Error(message, result.Error())
	p.complete <- &ImageProcessorOutput{Error: result.Error()}
}

func (p *ImageProcessor) phash() {
	for dl := range p.downloader.Completed() {
		if dl.Error() != nil {
			p.handleError("download failed", dl)
		} else {
			result := dl.Context.Get(p.downloader.Name()).(*TaskRunResult)
			dlo := result.Output.(*FileDownloadOutput)
			p.phasher.Add(dlo.Path, dl.Context)
		}
	}
	p.phasher.Shutdown()
	p.outstanding.Done()
}

func (p *ImageProcessor) resize() {
	for ph := range p.phasher.Completed() {
		if ph.Error() != nil {
			p.handleError("Phash failed", ph)
		} else {
			result := ph.Context.Get(p.downloader.Name()).(*TaskRunResult)
			dlo := result.Output.(*FileDownloadOutput)
			p.resizer.Add(dlo.Path, ph.Context)
		}
	}
	p.resizer.Shutdown()
	p.outstanding.Done()
}

func (p *ImageProcessor) upload() {
	for rz := range p.resizer.Completed() {
		if rz.Error() != nil {
			p.handleError("resize failed", rz)
		} else {
			rzOut := rz.Output().(*ImageResizeOutput)
			p.uploader.Add(rzOut.FilePath, rz.Context)
		}
	}
	p.uploader.Shutdown()
	p.outstanding.Done()
}

func (p *ImageProcessor) wrapUp() {
	for result := range p.uploader.Completed() {
		if result.Error() != nil {
			p.handleError("upload failed", result)
		} else {
			dl := result.Previous(p.downloader.Name()).Output.(*FileDownloadOutput)
			dlin := result.Previous(p.downloader.Name()).Output.(*FileDownloadInput)
			rz := result.Previous(p.resizer.Name()).Output.(*ImageResizeOutput)
			phash := result.Previous(p.phasher.Name()).Output.(uint64)
			ul := result.Previous(p.uploader.Name()).Output.(*FileUploadOutput)

			r := &ImageProcessorOutput{
				DownloadSize:        dl.Size,
				DownloadContentType: dl.ContentType,
				DownloadUrl:         dlin.Url,
				PHash:               phash,
				FileSize:            rz.FileSize,
				Height:              rz.Height,
				Width:               rz.Width,
				ContentType:         rz.ContentType,
				DestinationUrl:      ul.Url,
				Error:               result.Error(),
			}

			p.complete <- r
		}
	}
	close(p.complete)
	p.outstanding.Done()
}

func (p *ImageProcessor) Completed() <-chan *ImageProcessorOutput {
	return p.complete
}

type ImageProcessorOutput struct {
	DownloadSize        int64
	DownloadContentType string
	DownloadUrl         string
	PHash               uint64

	FileSize    int64
	Height      int
	Width       int
	ContentType string

	DestinationUrl string
	Error          error
}
<commit_msg>bug fix<commit_after>package multi

import (
	"os"
	"path"
	"sync"

	"github.com/gotgo/fw/logging"
)

type ImageProcessor struct {

	// Uploader - required
	Uploader Uploader

	// LocalPath - defaults to os.TempDir()/downloads
	LocalPath string

	Log logging.Logger

	MaxHeight int
	MaxWidth  int

	downloader *TaskRun
	phasher    *TaskRun
	resizer    *TaskRun
	uploader   *TaskRun

	outstanding sync.WaitGroup
	complete    chan *ImageProcessorOutput
}

func (ip *ImageProcessor) setup() {
	uploader := ip.Uploader
	if uploader == nil {
		panic("uploader not set")
	}

	if ip.Log == nil {
		ip.Log = &logging.NoOpLogger{}
	}

	var tempFolder = ip.LocalPath
	if tempFolder == "" {
		tempFolder = path.Join(os.TempDir(), "downloads")
	}

	os.MkdirAll(tempFolder, 0774)

	ip.complete = make(chan *ImageProcessorOutput, 100)

	ip.downloader = &TaskRun{
		Action:       &FileDownloadTask{Folder: tempFolder},
		Concurrency:  10,
		MaxQueuedIn:  10 * 5,
		MaxQueuedOut: 10 * 10,
	}

	ip.phasher = &TaskRun{
		Action:       &PHashTask{},
		Concurrency:  2,
		MaxQueuedIn:  2,
		MaxQueuedOut: 100,
	}

	ip.resizer = &TaskRun{
		Action:       &ResizeImageTask{MaxHeight: ip.MaxHeight, MaxWidth: ip.MaxWidth},
		Concurrency:  6,
		MaxQueuedIn:  12,
		MaxQueuedOut: 100,
	}

	ip.uploader = &TaskRun{
		Action:       &FileUploadTask{Uploader: uploader},
		Concurrency:  8,
		MaxQueuedIn:  8 * 10,
		MaxQueuedOut: 100,
	}
}

func (p *ImageProcessor) Startup() {
	p.setup()
	p.doScavenge()
}

func (p *ImageProcessor) Shutdown() {
	p.downloader.Shutdown()
	p.outstanding.Wait()
}

func (p *ImageProcessor) Injest(url, filename string, ctx *DataContext) {
	in := &FileDownloadInput{
		Url:      url,
		Filename: filename,
	}

	p.downloader.Add(in, ctx)
}

func (p *ImageProcessor) doScavenge() {
	p.downloader.Startup()
	p.phasher.Startup()
	p.resizer.Startup()
	p.uploader.Startup()

	p.outstanding.Add(1)
	// phash
	go p.phash()
	p.outstanding.Add(1)
	// resize
	go p.resize()
	p.outstanding.Add(1)
	// upload
	go p.upload()
	p.outstanding.Add(1)
	// collector
	go p.wrapUp()
}

func (p *ImageProcessor) handleError(message string, result *TaskRunOutput) {
	p.Log.Error(message, result.Error())
	p.complete <- &ImageProcessorOutput{
		Error:   result.Error(),
		Context: result.Context,
	}
}

func (p *ImageProcessor) phash() {
	for dl := range p.downloader.Completed() {
		if dl.Error() != nil {
			p.handleError("download failed", dl)
		} else {
			result := dl.Context.Get(p.downloader.Name()).(*TaskRunResult)
			dlo := result.Output.(*FileDownloadOutput)
			p.phasher.Add(dlo.Path, dl.Context)
		}
	}
	p.phasher.Shutdown()
	p.outstanding.Done()
}

func (p *ImageProcessor) resize() {
	for ph := range p.phasher.Completed() {
		if ph.Error() != nil {
			p.handleError("Phash failed", ph)
		} else {
			result := ph.Context.Get(p.downloader.Name()).(*TaskRunResult)
			dlo := result.Output.(*FileDownloadOutput)
			p.resizer.Add(dlo.Path, ph.Context)
		}
	}
	p.resizer.Shutdown()
	p.outstanding.Done()
}

func (p *ImageProcessor) upload() {
	for rz := range p.resizer.Completed() {
		if rz.Error() != nil {
			p.handleError("resize failed", rz)
		} else {
			rzOut := rz.Output().(*ImageResizeOutput)
			p.uploader.Add(rzOut.FilePath, rz.Context)
		}
	}
	p.uploader.Shutdown()
	p.outstanding.Done()
}

func (p *ImageProcessor) wrapUp() {
	for result := range p.uploader.Completed() {
		if result.Error() != nil {
			p.handleError("upload failed", result)
		} else {
			dl := result.Previous(p.downloader.Name()).Output.(*FileDownloadOutput)
			dlin := result.Previous(p.downloader.Name()).Output.(*FileDownloadInput)
			rz := result.Previous(p.resizer.Name()).Output.(*ImageResizeOutput)
			phash := result.Previous(p.phasher.Name()).Output.(uint64)
			ul := result.Previous(p.uploader.Name()).Output.(*FileUploadOutput)

			r := &ImageProcessorOutput{
				DownloadSize:        dl.Size,
				DownloadContentType: dl.ContentType,
				DownloadUrl:         dlin.Url,
				PHash:               phash,
				FileSize:            rz.FileSize,
				Height:              rz.Height,
				Width:               rz.Width,
				ContentType:         rz.ContentType,
				DestinationUrl:      ul.Url,
				Error:               result.Error(),
				Context:             result.Context,
			}

			p.complete <- r
		}
	}
	close(p.complete)
	p.outstanding.Done()
}

func (p *ImageProcessor) Completed() <-chan *ImageProcessorOutput {
	return p.complete
}

type ImageProcessorOutput struct {
	DownloadSize        int64
	DownloadContentType string
	DownloadUrl         string
	PHash               uint64

	FileSize    int64
	Height      int
	Width       int
	ContentType string

	DestinationUrl string
	Error          error
	Context        *DataContext
}
<|endoftext|>


<commit_before>package colonycore

import (
	"github.com/eaciit/orm/v1"
)

type Application struct {
	orm.ModelBase
	ID       string `json:"_id",bson:"_id"`
	AppsName string `json:"AppsName", bson:"AppsName"`
	Enable   bool   `json:"Enable", bson:"Enable"`
	AppPath  string `json:"AppPath", bson:"AppPath"`
}

func (a *Application) TableName() string {
	return "applications"
}

func (a *Application) RecordID() interface{} {
	return a.ID
}
<commit_msg>Revert "update application struct"<commit_after>package colonycore

import (
	"github.com/eaciit/orm/v1"
)

type Application struct {
	orm.ModelBase `json:"-"`
	ID            string `json:"_id"`
	Enable        bool
}

func (a *Application) TableName() string {
	return "applications"
}

func (a *Application) RecordID() interface{} {
	return a.ID
}
<|endoftext|>


<commit_before>/*
The MIT License (MIT)

Copyright (c) 2014 Yasen Visulchev

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

/*
Written by Yasen Visulchev - Sofia University FMI/Golang
	20141130
*/
package gw2api

import (
	"fmt"
	"testing"
)

// Tests

/* GW2API UNEXPORTED FUNCTIONS */

//OK
func TestFetchJSON(t *testing.T) {
	//t.Skip()
	id := "fetchJSON"
	_, err := fetchJSON("v1", "world_names", "")
	if err != nil {
		t.Errorf("Failed %s for world_names! Got:\n%s", id, err.Error())
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

/* DYNAMIC EVENTS */

//EXPERIMENTAL
func TestEvents(t *testing.T) {
	t.Skip()
	id := "Events"
	es, err := Events(0, 0, "")
	if err != nil {
		t.Errorf("Failed to get %s! Got:\n%s", id, err.Error())
	}
	if es == nil {
		t.Errorf("%s is nil! API broken?: %s", id, es)
	}
	fmt.Printf("\t-%s\t\t\tOK\n", id)
}

//OK
func TestEventNames(t *testing.T) {
	//t.Skip()
	id := "EventNames"
	_, err := EventNames("es")
	if err != nil {
		t.Errorf("Failed to get %s! Got:\n%s", id, err.Error())
	}

	_, err = EventNames("")
	if err != nil {
		t.Errorf("Failed to get %s with empty appendix! Got:\n%s", id, err.Error)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

//OK
func TestMapNames(t *testing.T) {
	//t.Skip()
	id := "MapNames"
	_, err := MapNames("es")
	if err != nil {
		t.Errorf("Failed to get %s! Got:\n%s", id, err.Error())
	}

	_, err = MapNames("")
	if err != nil {
		t.Errorf("Failed to get %s with empty appendix! Got:\n%s", id, err.Error)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

//OK
func TestWorldNames(t *testing.T) {
	//t.Skip()
	id := "WorldNames"
	_, err := WorldNames("es")
	if err != nil {
		t.Errorf("Failed to get %s! Got:\n%s", id, err.Error())
	}

	_, err = WorldNames("")
	if err != nil {
		t.Errorf("Failed to get %s with empty appendix! Got:\n%s", id, err.Error)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

//OK
func TestEventDetails(t *testing.T) {
	//t.Skip()
	id := "EventDetails"
	_, err := EventDetails("", "es")
	if err != nil {
		t.Errorf("Failed to get %s in Spanish! Got:\n%s", id, err.Error())
	}

	_, err = EventDetails("EED8A79F-B374-4AE6-BA6F-B7B98D9D7142", "")
	if err != nil {
		t.Errorf("Failed to get %s for EED8A79F-B374-4AE6-BA6F-B7B98D9D7142", id)
	}

	_, err = EventDetails("EED8A79F-B374-4AE6-BA6F-B7B98D9D7142", "es")
	if err != nil {
		t.Errorf("Failed to get %s for EED8A79F-B374-4AE6-BA6F-B7B98D9D7142 in Spanish!", id)
	}

	_, err = EventDetails("", "")
	if err != nil {
		t.Errorf("Failed to get %s with empty appendix! Got:\n%s", id, err.Error)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

/* GUILDS */

//OK
func TestGuildDetails(t *testing.T) {
	//t.Skip()
	id := "GuildDetails"
	_, err := GuildDetails("", "")
	if err == nil {
		t.Errorf("No error getting %s with empty required parameters\n", id)
	}

	_, err = GuildDetails("", "When%20Time%20Flies")
	if err != nil {
		t.Errorf("Error getting %s for When Time Flies:\n\t%s\n", id, err)
	}
	_, err = GuildDetails("B986E58F-17A6-4D68-8C50-21E3C47FE103", "")
	if err != nil {
		t.Errorf("Error getting %s for B986E58F-17A6-4D68-8C50-21E3C47FE103:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

/* ITEMS */

//OK
func TestItems(t *testing.T) {
	//t.Skip()
	id := "Items"
	_, err := Items("")
	if err != nil {
		t.Errorf("Error getting %s with empty lang parameter:\n\t%s\n", id, err)
	}

	_, err = Items("es")
	if err != nil {
		t.Errorf("Error getting %s in Spanish:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\t\tOK\n", id)
}

//OK
func TestItemDetails(t *testing.T) {
	//t.Skip()
	id := "ItemDetails"
	_, err := ItemDetails("", "arf arf arf")
	if err == nil {
		t.Errorf("No error getting %s with empty required parameters\n", id)
	}

	_, err = ItemDetails("54770", "es")
	if err != nil {
		t.Errorf("Error getting %s for 54770 in Spanish:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

//OK
func TestRecipes(t *testing.T) {
	//t.Skip()
	id := "Recipes"
	_, err := Recipes("")
	if err != nil {
		t.Errorf("Error getting %s with empty lang parameter:\n\t%s\n", id, err)
	}

	_, err = Recipes("es")
	if err != nil {
		t.Errorf("Error getting %s in Spanish:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

//OK
func TestRecipeDetails(t *testing.T) {
	//t.Skip()
	id := "RecipeDetails"
	_, err := RecipeDetails("", "arf arf arf")
	if err == nil {
		t.Errorf("No error getting %s with empty required parameters\n", id)
	}

	_, err = RecipeDetails("3147", "es")
	if err != nil {
		t.Errorf("Error getting %s for 3147 in Spanish:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

//OK
func TestSkins(t *testing.T) {
	//t.Skip()
	id := "Skins"
	_, err := Skins()
	if err != nil {
		t.Errorf("Error getting %s:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\t\tOK\n", id)
}

//OK
func TestSkinDetails(t *testing.T) {
	//t.Skip()
	id := "SkinDetails"
	_, err := SkinDetails("", "arf arf arf")
	if err == nil {
		t.Errorf("No error getting %s with empty required parameters\n", id)
	}

	_, err = SkinDetails("900", "es")
	if err != nil {
		t.Errorf("Error getting %s for 900 in Spanish!:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

/* MAP INFORMATION */

//OK
func TestContinents(t *testing.T) {
	//t.Skip()
	id := "Continents"
	_, err := Continents("")
	if err != nil {
		t.Errorf("Error getting %s with empty lang parameter:\n\t%s\n", id, err)
	}

	_, err = Continents("es")
	if err != nil {
		t.Errorf("Error getting %s in Spanish:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

//OK
func TestMaps(t *testing.T) {
	//t.Skip()
	id := "Maps"
	_, err := Maps("", "")
	if err != nil {
		t.Errorf("Error getting %s with empty parameters:\n\t%s\n", id, err)
	}

	_, err = Maps("15", "es")
	if err != nil {
		t.Errorf("Error getting %s 15 in Spanish:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\t\tOK\n", id)
}

//OK
func TestMapFloor(t *testing.T) {
	//t.Skip()
	id := "MapFloor"
	_, err := MapFloor("", "", "arf arf arf")
	if err == nil {
		t.Errorf("No error getting %s with empty parameters!", id)
	}

	_, err = MapFloor("1", "2", "es")
	if err != nil {
		t.Errorf("Error getting %s, continent 1, floor 2 in Spanish:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

/* WORLD VS. WORLD */

//OK
func TestWVWMatches(t *testing.T) {
	//t.Skip()
	id := "WVWMatches"
	_, err := WVWMatches()
	if err != nil {
		t.Errorf("Error getting %s:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

//OK
func TestWVWMatchDetails(t *testing.T) {
	//t.Skip()
	id := "WVWMatchDetails"
	_, err := WVWMatchDetails("")
	if err == nil {
		t.Errorf("No error getting %s with empty required parameters\n", id)
	}

	_, err = WVWMatchDetails("1-4")
	if err != nil {
		t.Errorf("Error getting %s for 1-4:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\tOK\n", id)
}

//OK
func TestWVWObjectiveNames(t *testing.T) {
	//t.Skip()
	id := "WVWObjectiveNames"
	_, err := WVWObjectiveNames("es")
	if err != nil {
		t.Errorf("Failed to get %s! Got:\n%s", id, err.Error())
	}

	_, err = WVWObjectiveNames("")
	if err != nil {
		t.Errorf("Failed to get %s with empty appendix! Got:\n%s", id, err.Error)
	}
	fmt.Printf("\t-%s\tOK\n", id)
}

/* MISCELLANEOUS */

//OK
func TestBuild(t *testing.T) {
	//t.Skip()
	id := "Build"
	_, err := Build()
	if err != nil {
		t.Errorf("Error getting %s:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\t\tOK\n", id)
}

//OK
func TestColors(t *testing.T) {
	//t.Skip()
	id := "Colors"
	_, err := Colors("")
	if err != nil {
		t.Errorf("Error getting %s with empty lang parameter:\n\t%s\n", id, err)
	}

	_, err = Colors("es")
	if err != nil {
		t.Errorf("Error getting %s in Spanish:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\t\tOK\n", id)
}

//OK
func TestFiles(t *testing.T) {
	//t.Skip()
	id := "Files"
	_, err := Files()
	if err != nil {
		t.Errorf("Error getting %s:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\t\tOK\n", id)
}
<commit_msg>Update gw2api_test.go<commit_after>/*
The MIT License (MIT)

Copyright (c) 2014 Yasen Visulchev

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

package gw2api

import (
	"fmt"
	"testing"
)

// Tests

/* GW2API UNEXPORTED FUNCTIONS */

//OK
func TestFetchJSON(t *testing.T) {
	//t.Skip()
	id := "fetchJSON"
	_, err := fetchJSON("v1", "world_names", "")
	if err != nil {
		t.Errorf("Failed %s for world_names! Got:\n%s", id, err.Error())
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

/* DYNAMIC EVENTS */

//EXPERIMENTAL
func TestEvents(t *testing.T) {
	t.Skip()
	id := "Events"
	es, err := Events(0, 0, "")
	if err != nil {
		t.Errorf("Failed to get %s! Got:\n%s", id, err.Error())
	}
	if es == nil {
		t.Errorf("%s is nil! API broken?: %s", id, es)
	}
	fmt.Printf("\t-%s\t\t\tOK\n", id)
}

//OK
func TestEventNames(t *testing.T) {
	//t.Skip()
	id := "EventNames"
	_, err := EventNames("es")
	if err != nil {
		t.Errorf("Failed to get %s! Got:\n%s", id, err.Error())
	}

	_, err = EventNames("")
	if err != nil {
		t.Errorf("Failed to get %s with empty appendix! Got:\n%s", id, err.Error)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

//OK
func TestMapNames(t *testing.T) {
	//t.Skip()
	id := "MapNames"
	_, err := MapNames("es")
	if err != nil {
		t.Errorf("Failed to get %s! Got:\n%s", id, err.Error())
	}

	_, err = MapNames("")
	if err != nil {
		t.Errorf("Failed to get %s with empty appendix! Got:\n%s", id, err.Error)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

//OK
func TestWorldNames(t *testing.T) {
	//t.Skip()
	id := "WorldNames"
	_, err := WorldNames("es")
	if err != nil {
		t.Errorf("Failed to get %s! Got:\n%s", id, err.Error())
	}

	_, err = WorldNames("")
	if err != nil {
		t.Errorf("Failed to get %s with empty appendix! Got:\n%s", id, err.Error)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

//OK
func TestEventDetails(t *testing.T) {
	//t.Skip()
	id := "EventDetails"
	_, err := EventDetails("", "es")
	if err != nil {
		t.Errorf("Failed to get %s in Spanish! Got:\n%s", id, err.Error())
	}

	_, err = EventDetails("EED8A79F-B374-4AE6-BA6F-B7B98D9D7142", "")
	if err != nil {
		t.Errorf("Failed to get %s for EED8A79F-B374-4AE6-BA6F-B7B98D9D7142", id)
	}

	_, err = EventDetails("EED8A79F-B374-4AE6-BA6F-B7B98D9D7142", "es")
	if err != nil {
		t.Errorf("Failed to get %s for EED8A79F-B374-4AE6-BA6F-B7B98D9D7142 in Spanish!", id)
	}

	_, err = EventDetails("", "")
	if err != nil {
		t.Errorf("Failed to get %s with empty appendix! Got:\n%s", id, err.Error)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

/* GUILDS */

//OK
func TestGuildDetails(t *testing.T) {
	//t.Skip()
	id := "GuildDetails"
	_, err := GuildDetails("", "")
	if err == nil {
		t.Errorf("No error getting %s with empty required parameters\n", id)
	}

	_, err = GuildDetails("", "When%20Time%20Flies")
	if err != nil {
		t.Errorf("Error getting %s for When Time Flies:\n\t%s\n", id, err)
	}
	_, err = GuildDetails("B986E58F-17A6-4D68-8C50-21E3C47FE103", "")
	if err != nil {
		t.Errorf("Error getting %s for B986E58F-17A6-4D68-8C50-21E3C47FE103:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

/* ITEMS */

//OK
func TestItems(t *testing.T) {
	//t.Skip()
	id := "Items"
	_, err := Items("")
	if err != nil {
		t.Errorf("Error getting %s with empty lang parameter:\n\t%s\n", id, err)
	}

	_, err = Items("es")
	if err != nil {
		t.Errorf("Error getting %s in Spanish:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\t\tOK\n", id)
}

//OK
func TestItemDetails(t *testing.T) {
	//t.Skip()
	id := "ItemDetails"
	_, err := ItemDetails("", "arf arf arf")
	if err == nil {
		t.Errorf("No error getting %s with empty required parameters\n", id)
	}

	_, err = ItemDetails("54770", "es")
	if err != nil {
		t.Errorf("Error getting %s for 54770 in Spanish:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

//OK
func TestRecipes(t *testing.T) {
	//t.Skip()
	id := "Recipes"
	_, err := Recipes("")
	if err != nil {
		t.Errorf("Error getting %s with empty lang parameter:\n\t%s\n", id, err)
	}

	_, err = Recipes("es")
	if err != nil {
		t.Errorf("Error getting %s in Spanish:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

//OK
func TestRecipeDetails(t *testing.T) {
	//t.Skip()
	id := "RecipeDetails"
	_, err := RecipeDetails("", "arf arf arf")
	if err == nil {
		t.Errorf("No error getting %s with empty required parameters\n", id)
	}

	_, err = RecipeDetails("3147", "es")
	if err != nil {
		t.Errorf("Error getting %s for 3147 in Spanish:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

//OK
func TestSkins(t *testing.T) {
	//t.Skip()
	id := "Skins"
	_, err := Skins()
	if err != nil {
		t.Errorf("Error getting %s:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\t\tOK\n", id)
}

//OK
func TestSkinDetails(t *testing.T) {
	//t.Skip()
	id := "SkinDetails"
	_, err := SkinDetails("", "arf arf arf")
	if err == nil {
		t.Errorf("No error getting %s with empty required parameters\n", id)
	}

	_, err = SkinDetails("900", "es")
	if err != nil {
		t.Errorf("Error getting %s for 900 in Spanish!:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

/* MAP INFORMATION */

//OK
func TestContinents(t *testing.T) {
	//t.Skip()
	id := "Continents"
	_, err := Continents("")
	if err != nil {
		t.Errorf("Error getting %s with empty lang parameter:\n\t%s\n", id, err)
	}

	_, err = Continents("es")
	if err != nil {
		t.Errorf("Error getting %s in Spanish:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

//OK
func TestMaps(t *testing.T) {
	//t.Skip()
	id := "Maps"
	_, err := Maps("", "")
	if err != nil {
		t.Errorf("Error getting %s with empty parameters:\n\t%s\n", id, err)
	}

	_, err = Maps("15", "es")
	if err != nil {
		t.Errorf("Error getting %s 15 in Spanish:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\t\tOK\n", id)
}

//OK
func TestMapFloor(t *testing.T) {
	//t.Skip()
	id := "MapFloor"
	_, err := MapFloor("", "", "arf arf arf")
	if err == nil {
		t.Errorf("No error getting %s with empty parameters!", id)
	}

	_, err = MapFloor("1", "2", "es")
	if err != nil {
		t.Errorf("Error getting %s, continent 1, floor 2 in Spanish:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

/* WORLD VS. WORLD */

//OK
func TestWVWMatches(t *testing.T) {
	//t.Skip()
	id := "WVWMatches"
	_, err := WVWMatches()
	if err != nil {
		t.Errorf("Error getting %s:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\tOK\n", id)
}

//OK
func TestWVWMatchDetails(t *testing.T) {
	//t.Skip()
	id := "WVWMatchDetails"
	_, err := WVWMatchDetails("")
	if err == nil {
		t.Errorf("No error getting %s with empty required parameters\n", id)
	}

	_, err = WVWMatchDetails("1-4")
	if err != nil {
		t.Errorf("Error getting %s for 1-4:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\tOK\n", id)
}

//OK
func TestWVWObjectiveNames(t *testing.T) {
	//t.Skip()
	id := "WVWObjectiveNames"
	_, err := WVWObjectiveNames("es")
	if err != nil {
		t.Errorf("Failed to get %s! Got:\n%s", id, err.Error())
	}

	_, err = WVWObjectiveNames("")
	if err != nil {
		t.Errorf("Failed to get %s with empty appendix! Got:\n%s", id, err.Error)
	}
	fmt.Printf("\t-%s\tOK\n", id)
}

/* MISCELLANEOUS */

//OK
func TestBuild(t *testing.T) {
	//t.Skip()
	id := "Build"
	_, err := Build()
	if err != nil {
		t.Errorf("Error getting %s:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\t\tOK\n", id)
}

//OK
func TestColors(t *testing.T) {
	//t.Skip()
	id := "Colors"
	_, err := Colors("")
	if err != nil {
		t.Errorf("Error getting %s with empty lang parameter:\n\t%s\n", id, err)
	}

	_, err = Colors("es")
	if err != nil {
		t.Errorf("Error getting %s in Spanish:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\t\tOK\n", id)
}

//OK
func TestFiles(t *testing.T) {
	//t.Skip()
	id := "Files"
	_, err := Files()
	if err != nil {
		t.Errorf("Error getting %s:\n\t%s\n", id, err)
	}
	fmt.Printf("\t-%s\t\t\tOK\n", id)
}
<|endoftext|>


<commit_before>// Copyright 2018 The Terraformer Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package terraform_output

import (
	"context"
	"log"
	"strings"

	"cloud.google.com/go/storage"
)

type BucketState struct {
	Name string
}

func (b BucketState) BucketGetTfData(path string) interface{} {
	bucketStateData := map[string]interface{}{
		"terraform": map[string]interface{}{
			"backend": map[string]interface{}{
				"gcs": map[string]interface{}{
					"bucket": b.Name,
					"prefix": b.BucketPrefix(path),
				},
			},
		},
	}
	return bucketStateData
}

func (b BucketState) BucketPrefix(path string) string {
	return path
}

func (b BucketState) BucketUpload(path string, file []byte) error {
	ctx := context.Background()
	client, err := storage.NewClient(ctx)
	if err != nil {
		log.Fatalf("Failed to create client: %v", err)
	}
	name := strings.Replace(b.Name, "gs://", "", -1)
	wc := client.Bucket(name).Object(b.BucketPrefix(path) + "/default.tfstate").NewWriter(ctx)
	if _, err = wc.Write(file); err != nil {
		return err
	}
	if err := wc.Close(); err != nil {
		return err
	}
	return nil
}
<commit_msg>fix bucket.tf<commit_after>// Copyright 2018 The Terraformer Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package terraform_output

import (
	"context"
	"log"
	"strings"

	"cloud.google.com/go/storage"
)

type BucketState struct {
	Name string
}

func (b BucketState) BucketGetTfData(path string) interface{} {
	bucketStateData := map[string]interface{}{
		"terraform": map[string]interface{}{
			"backend": []map[string]interface{}{
				{
					"gcs": map[string]interface{}{
						"bucket": b.Name,
						"prefix": b.BucketPrefix(path),
					},
				},
			},
		},
	}
	return bucketStateData
}

func (b BucketState) BucketPrefix(path string) string {
	return path
}

func (b BucketState) BucketUpload(path string, file []byte) error {
	ctx := context.Background()
	client, err := storage.NewClient(ctx)
	if err != nil {
		log.Fatalf("Failed to create client: %v", err)
	}
	name := strings.Replace(b.Name, "gs://", "", -1)
	wc := client.Bucket(name).Object(b.BucketPrefix(path) + "/default.tfstate").NewWriter(ctx)
	if _, err = wc.Write(file); err != nil {
		return err
	}
	if err := wc.Close(); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package validator_test

import(
  "testing"
  "github.com/codeadict/ecuadorianid/validator"
)

func TestValid(t *testing.T) {
	tests := []string{"1719956383"}

	for _, tt := range tests {
		if p, err := Validate(tt); err != nil {
			t.Errorf("Fail at '%v', out '%+v', err: %+v", tt, p, err)
		}
	}
}

func TestInvalid(t *testing.T) {
	tests := []string{"1719956382"}

	for _, tt := range tests {
		if p, err := Validate(tt); err == nil {
			t.Errorf("Fail at '%v', out '%+v', should have produced error", tt, p)
		}
	}
}
<commit_msg>Added better testing<commit_after>package validator

import(
  "testing"
  "github.com/codeadict/ecuadorianid"
)

func TestValid(t *testing.T) {
	tests := []string{"1719956383"}

	for _, tt := range tests {
		if p, err := validator.Validate(tt); err != nil {
			t.Errorf("Fail at '%v', out '%+v', err: %+v", tt, p, err)
		}
	}
}

func TestInvalid(t *testing.T) {
	tests := []string{"1719956382"}

	for _, tt := range tests {
		if p, err := validator.Validate(tt); err == nil {
			t.Errorf("Fail at '%v', out '%+v', should have produced error", tt, p)
		}
	}
}
<|endoftext|>


<commit_before>// +build windows

package disk

import (
	"syscall"
	"unsafe"
)

// Space returns total and free bytes available in a directory, e.g. `C:\`.
// It returns free space available to the user (including quota limitations),
// so it can be lower than the free space of the disk.
func Space(path string) (total, free uint64, err error) {
	kernel32, err := syscall.LoadLibrary("Kernel32.dll")
	if err != nil {
		return 0, 0, err
	}
	defer syscall.FreeLibrary(kernel32)
	GetDiskFreeSpaceEx, err := syscall.GetProcAddress(syscall.Handle(kernel32), "GetDiskFreeSpaceExW")
	if err != nil {
		return 0, 0, err
	}
	lpFreeBytesAvailable := int64(0)
	lpTotalNumberOfBytes := int64(0)
	lpTotalNumberOfFreeBytes := int64(0)
	r1, _, e1 := syscall.Syscall6(uintptr(GetDiskFreeSpaceEx), 4,
		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
		uintptr(unsafe.Pointer(&lpFreeBytesAvailable)),
		uintptr(unsafe.Pointer(&lpTotalNumberOfBytes)),
		uintptr(unsafe.Pointer(&lpTotalNumberOfFreeBytes)), 0, 0)
	if r1 == 0 {
		err := syscall.EINVAL
		if e1 != 0 {
			err = error(e1)
		}
		return 0, 0, err
	}
	total := uint64(lpTotalNumberOfBytes)
	free := uint64(lpFreeBytesAvailable)
	return total, free, nil
}
<commit_msg>lib/disk: windows fixes<commit_after>// +build windows

package disk

import (
	"syscall"
	"unsafe"

	"golang.org/x/sys/windows"
)

// Space returns total and free bytes available in a directory, e.g. `C:\`.
// It returns free space available to the user (including quota limitations),
// so it can be lower than the free space of the disk.
func Space(path string) (total, free uint64, err error) {
	kernel32, err := windows.LoadLibrary("Kernel32.dll")
	if err != nil {
		return 0, 0, err
	}
	defer windows.FreeLibrary(kernel32)
	GetDiskFreeSpaceEx, err := windows.GetProcAddress(windows.Handle(kernel32), "GetDiskFreeSpaceExW")
	if err != nil {
		return 0, 0, err
	}
	lpFreeBytesAvailable := int64(0)
	lpTotalNumberOfBytes := int64(0)
	lpTotalNumberOfFreeBytes := int64(0)
	r1, _, e1 := syscall.Syscall6(GetDiskFreeSpaceEx, 4,
		uintptr(unsafe.Pointer(windows.StringToUTF16Ptr(path))),
		uintptr(unsafe.Pointer(&lpFreeBytesAvailable)),
		uintptr(unsafe.Pointer(&lpTotalNumberOfBytes)),
		uintptr(unsafe.Pointer(&lpTotalNumberOfFreeBytes)), 0, 0)
	if r1 == 0 {
		err := syscall.EINVAL
		if e1 != 0 {
			err = syscall.Errno(e1)
		}
		return 0, 0, err
	}
	return uint64(lpTotalNumberOfBytes), uint64(lpFreeBytesAvailable), nil
}
<|endoftext|>


<commit_before>package aws

import (
	"encoding/json"
	"log"
	"os/exec"
	"strings"

	"github.com/cockroachdb/roachprod/vm"
	"github.com/pkg/errors"
)

// Both M5 and I3 machines expose their EBS or local SSD volumes as NVMe block devices, but
// the actual device numbers vary a bit between the two types.
// This user-data script will create a filesystem, mount the data volume, and chmod 777.
// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nvme-ebs-volumes.html
const awsStartupScript = `#!/usr/bin/env bash
set -x
disknum=0
for d in $(ls /dev/nvme?n1); do
  if ! mount | grep ${d}; then
    let "disknum++"
    echo "Disk ${d} not mounted, creating..."
    mountpoint="/mnt/data${disknum}"
    mkdir -p "${mountpoint}"
    mkfs.ext4 ${d}
    mount -o discard,defaults ${d} ${mountpoint}
    chmod 777 ${mountpoint}
    echo "${d} ${mountpoint} ext4 discard,defaults 1 1" | tee -a /etc/fstab
  else
    echo "Disk ${disknum}: ${d} already mounted, skipping..."
  fi
done
if [ "${disknum}" -eq "0" ]; then
  echo "No disks mounted, creating /mnt/data1"
  mkdir -p /mnt/data1
  chmod 777 /mnt/data1
fi

sudo apt-get update
sudo apt-get install -qy chrony
echo -e "\nserver 169.254.169.123 prefer iburst" | sudo tee -a /etc/chrony/chrony.conf
sudo /etc/init.d/chrony restart

sudo touch /mnt/data1/.roachprod-initialized
`

// runCommand is used to invoke an AWS command for which no output is expected.
func runCommand(args []string) error {
	cmd := exec.Command("aws", args...)

	_, err := cmd.Output()
	if err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			log.Println(string(exitErr.Stderr))
		}
		return errors.Wrapf(err, "failed to run: aws %s", strings.Join(args, " "))
	}
	return nil
}

// runJSONCommand invokes an aws command and parses the json output.
func runJSONCommand(args []string, parsed interface{}) error {
	// force json output in case the user has overridden the default behavior
	args = append(args[:len(args):len(args)], "--output", "json")
	cmd := exec.Command("aws", args...)

	rawJSON, err := cmd.Output()
	if err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			log.Println(string(exitErr.Stderr))
		}
		return errors.Wrapf(err, "failed to run: aws %s", strings.Join(args, " "))
	}

	if err := json.Unmarshal(rawJSON, &parsed); err != nil {
		return errors.Wrapf(err, "failed to parse json %s", rawJSON)
	}

	return nil
}

// splitMap splits a list of `key:value` pairs into a map.
func splitMap(data []string) (map[string]string, error) {
	ret := make(map[string]string, len(data))
	for _, part := range data {
		parts := strings.Split(part, ":")
		if len(parts) != 2 {
			return nil, errors.Errorf("Could not split Region:AMI: %s", part)
		}
		ret[parts[0]] = parts[1]
	}
	return ret, nil
}

// regionMap collates VM instances by their region.
func regionMap(vms vm.List) (map[string]vm.List, error) {
	// Fan out the work by region
	byRegion := make(map[string]vm.List)
	for _, m := range vms {
		region, err := zoneToRegion(m.Zone)
		if err != nil {
			return nil, err
		}
		byRegion[region] = append(byRegion[region], m)
	}
	return byRegion, nil
}

// zoneToRegion converts an availability zone like us-east-2a to the zone name us-east-2
func zoneToRegion(zone string) (string, error) {
	return zone[0 : len(zone)-1], nil
}
<commit_msg>Wait for chrony to synchronize<commit_after>package aws

import (
	"encoding/json"
	"log"
	"os/exec"
	"strings"

	"github.com/cockroachdb/roachprod/vm"
	"github.com/pkg/errors"
)

// Both M5 and I3 machines expose their EBS or local SSD volumes as NVMe block devices, but
// the actual device numbers vary a bit between the two types.
// This user-data script will create a filesystem, mount the data volume, and chmod 777.
// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nvme-ebs-volumes.html
const awsStartupScript = `#!/usr/bin/env bash
set -x
disknum=0
for d in $(ls /dev/nvme?n1); do
  if ! mount | grep ${d}; then
    let "disknum++"
    echo "Disk ${d} not mounted, creating..."
    mountpoint="/mnt/data${disknum}"
    mkdir -p "${mountpoint}"
    mkfs.ext4 ${d}
    mount -o discard,defaults ${d} ${mountpoint}
    chmod 777 ${mountpoint}
    echo "${d} ${mountpoint} ext4 discard,defaults 1 1" | tee -a /etc/fstab
  else
    echo "Disk ${disknum}: ${d} already mounted, skipping..."
  fi
done
if [ "${disknum}" -eq "0" ]; then
  echo "No disks mounted, creating /mnt/data1"
  mkdir -p /mnt/data1
  chmod 777 /mnt/data1
fi

sudo apt-get update
sudo apt-get install -qy chrony
echo -e "\nserver 169.254.169.123 prefer iburst" | sudo tee -a /etc/chrony/chrony.conf
echo -e "\nmakestep 0.1 3" | sudo tee -a /etc/chrony/chrony.conf
sudo /etc/init.d/chrony restart
sudo chronyc -a waitsync 30 0.01 | sudo tee -a /root/chrony.log

sudo touch /mnt/data1/.roachprod-initialized
`

// runCommand is used to invoke an AWS command for which no output is expected.
func runCommand(args []string) error {
	cmd := exec.Command("aws", args...)

	_, err := cmd.Output()
	if err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			log.Println(string(exitErr.Stderr))
		}
		return errors.Wrapf(err, "failed to run: aws %s", strings.Join(args, " "))
	}
	return nil
}

// runJSONCommand invokes an aws command and parses the json output.
func runJSONCommand(args []string, parsed interface{}) error {
	// force json output in case the user has overridden the default behavior
	args = append(args[:len(args):len(args)], "--output", "json")
	cmd := exec.Command("aws", args...)

	rawJSON, err := cmd.Output()
	if err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			log.Println(string(exitErr.Stderr))
		}
		return errors.Wrapf(err, "failed to run: aws %s", strings.Join(args, " "))
	}

	if err := json.Unmarshal(rawJSON, &parsed); err != nil {
		return errors.Wrapf(err, "failed to parse json %s", rawJSON)
	}

	return nil
}

// splitMap splits a list of `key:value` pairs into a map.
func splitMap(data []string) (map[string]string, error) {
	ret := make(map[string]string, len(data))
	for _, part := range data {
		parts := strings.Split(part, ":")
		if len(parts) != 2 {
			return nil, errors.Errorf("Could not split Region:AMI: %s", part)
		}
		ret[parts[0]] = parts[1]
	}
	return ret, nil
}

// regionMap collates VM instances by their region.
func regionMap(vms vm.List) (map[string]vm.List, error) {
	// Fan out the work by region
	byRegion := make(map[string]vm.List)
	for _, m := range vms {
		region, err := zoneToRegion(m.Zone)
		if err != nil {
			return nil, err
		}
		byRegion[region] = append(byRegion[region], m)
	}
	return byRegion, nil
}

// zoneToRegion converts an availability zone like us-east-2a to the zone name us-east-2
func zoneToRegion(zone string) (string, error) {
	return zone[0 : len(zone)-1], nil
}
<|endoftext|>


<commit_before>package martini

import (
	"log"
	"net/http"
	"path"
	"strings"
)

// StaticOptions is a struct for specifying configuration options for the martini.Static middleware.
type StaticOptions struct {
	// Prefix is the optional prefix used to serve the static directory content
	Prefix string
	// SkipLogging will disable [Static] log messages when a static file is served.
	SkipLogging bool
	// IndexFile defines which file to serve as index if it exists.
	IndexFile string
	// Expires defines which user-defined function to use for producing a HTTP Expires Header
	// https://developers.google.com/speed/docs/insights/LeverageBrowserCaching
	Expires func() string
}

func prepareStaticOptions(options []StaticOptions) StaticOptions {
	var opt StaticOptions
	if len(options) > 0 {
		opt = options[0]
	}

	// Defaults
	if len(opt.IndexFile) == 0 {
		opt.IndexFile = "index.html"
	}
	// Normalize the prefix if provided
	if opt.Prefix != "" {
		// Ensure we have a leading '/'
		if opt.Prefix[0] != '/' {
			opt.Prefix = "/" + opt.Prefix
		}
		// Remove any trailing '/'
		opt.Prefix = strings.TrimRight(opt.Prefix, "/")
	}
	return opt
}

// Static returns a middleware handler that serves static files in the given directory.
func Static(directory string, staticOpt ...StaticOptions) Handler {
	if !path.IsAbs(directory) {
		directory = path.Join(Root, directory)
	}
	dir := http.Dir(directory)
	opt := prepareStaticOptions(staticOpt)

	return func(res http.ResponseWriter, req *http.Request, log *log.Logger) {
		if req.Method != "GET" && req.Method != "HEAD" {
			return
		}
		file := req.URL.Path
		// if we have a prefix, filter requests by stripping the prefix
		if opt.Prefix != "" {
			if !strings.HasPrefix(file, opt.Prefix) {
				return
			}
			file = file[len(opt.Prefix):]
			if file != "" && file[0] != '/' {
				return
			}
		}
		f, err := dir.Open(file)
		if err != nil {
			// discard the error?
			return
		}
		defer f.Close()

		fi, err := f.Stat()
		if err != nil {
			return
		}

		// try to serve index file
		if fi.IsDir() {
			// redirect if missing trailing slash
			if !strings.HasSuffix(req.URL.Path, "/") {
				http.Redirect(res, req, req.URL.Path+"/", http.StatusFound)
				return
			}

			file = path.Join(file, opt.IndexFile)
			f, err = dir.Open(file)
			if err != nil {
				return
			}
			defer f.Close()

			fi, err = f.Stat()
			if err != nil || fi.IsDir() {
				return
			}
		}

		if !opt.SkipLogging {
			log.Println("[Static] Serving " + file)
		}

		// Add an Expires header to the static content
		if opt.Expires != nil {
			res.Header().Set("Expires", opt.Expires())
		}

		http.ServeContent(res, req, file, fi.ModTime(), f)
	}
}
<commit_msg>fix #282<commit_after>package martini

import (
	"log"
	"net/http"
	"path/filepath"
	"strings"
)

// StaticOptions is a struct for specifying configuration options for the martini.Static middleware.
type StaticOptions struct {
	// Prefix is the optional prefix used to serve the static directory content
	Prefix string
	// SkipLogging will disable [Static] log messages when a static file is served.
	SkipLogging bool
	// IndexFile defines which file to serve as index if it exists.
	IndexFile string
	// Expires defines which user-defined function to use for producing a HTTP Expires Header
	// https://developers.google.com/speed/docs/insights/LeverageBrowserCaching
	Expires func() string
}

func prepareStaticOptions(options []StaticOptions) StaticOptions {
	var opt StaticOptions
	if len(options) > 0 {
		opt = options[0]
	}

	// Defaults
	if len(opt.IndexFile) == 0 {
		opt.IndexFile = "index.html"
	}
	// Normalize the prefix if provided
	if opt.Prefix != "" {
		// Ensure we have a leading '/'
		if opt.Prefix[0] != '/' {
			opt.Prefix = "/" + opt.Prefix
		}
		// Remove any trailing '/'
		opt.Prefix = strings.TrimRight(opt.Prefix, "/")
	}
	return opt
}

// Static returns a middleware handler that serves static files in the given directory.
func Static(directory string, staticOpt ...StaticOptions) Handler {
	if !filepath.IsAbs(directory) {
		directory = filepath.Join(Root, directory)
	}
	dir := http.Dir(directory)
	opt := prepareStaticOptions(staticOpt)

	return func(res http.ResponseWriter, req *http.Request, log *log.Logger) {
		if req.Method != "GET" && req.Method != "HEAD" {
			return
		}
		file := req.URL.Path
		// if we have a prefix, filter requests by stripping the prefix
		if opt.Prefix != "" {
			if !strings.HasPrefix(file, opt.Prefix) {
				return
			}
			file = file[len(opt.Prefix):]
			if file != "" && file[0] != '/' {
				return
			}
		}
		f, err := dir.Open(file)
		if err != nil {
			// discard the error?
			return
		}
		defer f.Close()

		fi, err := f.Stat()
		if err != nil {
			return
		}

		// try to serve index file
		if fi.IsDir() {
			// redirect if missing trailing slash
			if !strings.HasSuffix(req.URL.Path, "/") {
				http.Redirect(res, req, req.URL.Path+"/", http.StatusFound)
				return
			}

			file = filepath.Join(file, opt.IndexFile)
			f, err = dir.Open(file)
			if err != nil {
				return
			}
			defer f.Close()

			fi, err = f.Stat()
			if err != nil || fi.IsDir() {
				return
			}
		}

		if !opt.SkipLogging {
			log.Println("[Static] Serving " + file)
		}

		// Add an Expires header to the static content
		if opt.Expires != nil {
			res.Header().Set("Expires", opt.Expires())
		}

		http.ServeContent(res, req, file, fi.ModTime(), f)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strings"
)

var cmdStream = &Command{
	Run:       runStream,
	UsageLine: "stream [flags] workspaceId",
	Short:     "hooks into server-sent events for changes to the workspace",
	Long: `
This automatically fetches changes to the workspace.

Use these flags to download the wanted files.
-vhdl 	For generated VHDL

-csim 	For generated C Simulation
`,
}

var (
	streamVHDL bool
	streamCSim bool
)

func init() {
	cmdStream.Flag.BoolVar(&streamVHDL, "vhdl", false, "download generated vhdl")
	cmdStream.Flag.BoolVar(&streamCSim, "csim", false, "download generated C Simulation code")
}

func runStream(cmd *Command, args []string) {
	if len(args) == 0 {
		cmd.Usage()
	}

	wp, err := OpenWorkspace(args[0])
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %s\n", err)
		setExitStatus(1)
		return
	}
	fmt.Println("WP Open:", wp)

	done := make(chan bool, 1)
	err = wp.OpenEventStream(done)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %s\n", err)
		setExitStatus(1)
		return
	}

	if streamVHDL {
		fmt.Println("Downloading generated VHDL...")
		go func() {
			for ev := range wp.Events {
				subj := ev.GetSubject()
				if subj == "P:COMPILER:VHDL" {
					err := ev.DownloadFiles()
					if err != nil {
						fmt.Fprintf(os.Stderr, "Could not load all files. %s", err)
						break
					}
					fmt.Println("[*] Download finished..")
				}
			}
		}()
	}

	if streamCSim {
		fmt.Println("Downloading generated C Simulation code...")
		go func() {
			for ev := range wp.Events {
				subj := ev.GetSubject()
				if subj == "P:COMPILER:C" {
					err := ev.DownloadFiles()
					if err != nil {
						fmt.Fprintf(os.Stderr, "Could not load all files. %s", err)
						break
					}
					fmt.Println("[*] Download finished..")
				}
			}
		}()
	}

	fmt.Println("Displaying events...")
	go func() {
		for ev := range wp.Events {
			subj := ev.GetSubject()
			switch {

			case strings.HasPrefix(subj, "P:WORKSPACE:"):
				fmt.Println("Worskpace Changed:", subj)
				for _, file := range ev.GetFiles() {
					fmt.Printf("[*] %s\n", file.RelPath)
				}

			case subj == "P:COMPILER:VHDL":
				fmt.Println("New VHDL Code")

			case subj == "P:COMPILER:C":
				fmt.Println("New C-Sim Code")
			}
		}
	}()

	<-done
}
<commit_msg>simplified stream command - should not miss events anymore<commit_after>package main

import (
	"fmt"
	"os"
	"strings"
)

var cmdStream = &Command{
	Run:       runStream,
	UsageLine: "stream [flags] workspaceId",
	Short:     "hooks into server-sent events for changes to the workspace",
	Long: `
This automatically fetches changes to the workspace.

Use these flags to download the wanted files.
-vhdl 	For generated VHDL

-csim 	For generated C Simulation
`,
}

var (
	streamVHDL bool
	streamCSim bool
)

func init() {
	cmdStream.Flag.BoolVar(&streamVHDL, "vhdl", false, "download generated vhdl")
	cmdStream.Flag.BoolVar(&streamCSim, "csim", false, "download generated C Simulation code")
}

func runStream(cmd *Command, args []string) {
	if len(args) == 0 {
		cmd.Usage()
	}

	wp, err := OpenWorkspace(args[0])
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %s\n", err)
		setExitStatus(1)
		return
	}
	fmt.Println("WP Open:", wp)

	done := make(chan bool, 1)
	err = wp.OpenEventStream(done)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %s\n", err)
		setExitStatus(1)
		return
	}

	fmt.Println("Displaying events...")
	go func() {
		for ev := range wp.Events {
			subj := ev.GetSubject()
			switch {

			case strings.HasPrefix(subj, "P:WORKSPACE:"):
				fmt.Println("Worskpace Changed:", subj)
				for _, file := range ev.GetFiles() {
					fmt.Printf("[*] %s\n", file.RelPath)
				}

			case subj == "P:COMPILER:VHDL" && streamVHDL:
				fmt.Println("New VHDL Code")
				err := ev.DownloadFiles()
				if err != nil {
					fmt.Fprintf(os.Stderr, "Could not load all files. %s", err)
					break
				}
				fmt.Println("[*] Download finished..")

			case subj == "P:COMPILER:C" && streamCSim:
				fmt.Println("New C-Sim Code")
				err := ev.DownloadFiles()
				if err != nil {
					fmt.Fprintf(os.Stderr, "Could not load all files. %s", err)
					break
				}
				fmt.Println("[*] Download finished..")
			}
		}
	}()

	<-done
}
<|endoftext|>


<commit_before>package yamux

import (
	"bytes"
	"io"
	"sync"
	"sync/atomic"
	"time"
)

type streamState int

const (
	streamInit streamState = iota
	streamSYNSent
	streamSYNReceived
	streamEstablished
	streamLocalClose
	streamRemoteClose
	streamClosed
	streamReset
)

// Stream is used to represent a logical stream
// within a session.
type Stream struct {
	recvWindow uint32
	sendWindow uint32

	id      uint32
	session *Session

	state     streamState
	stateLock sync.Mutex

	recvBuf  bytes.Buffer
	recvLock sync.Mutex

	controlHdr     header
	controlErr     chan error
	controlHdrLock sync.Mutex

	sendHdr  header
	sendErr  chan error
	sendLock sync.Mutex

	recvNotifyCh chan struct{}
	sendNotifyCh chan struct{}

	readDeadline  time.Time
	writeDeadline time.Time
}

// newStream is used to construct a new stream within
// a given session for an ID
func newStream(session *Session, id uint32, state streamState) *Stream {
	s := &Stream{
		id:           id,
		session:      session,
		state:        state,
		controlHdr:   header(make([]byte, headerSize)),
		controlErr:   make(chan error, 1),
		sendHdr:      header(make([]byte, headerSize)),
		sendErr:      make(chan error, 1),
		recvWindow:   initialStreamWindow,
		sendWindow:   initialStreamWindow,
		recvNotifyCh: make(chan struct{}, 1),
		sendNotifyCh: make(chan struct{}, 1),
	}
	return s
}

// Session returns the associated stream session
func (s *Stream) Session() *Session {
	return s.session
}

// StreamID returns the ID of this stream
func (s *Stream) StreamID() uint32 {
	return s.id
}

// Read is used to read from the stream
func (s *Stream) Read(b []byte) (n int, err error) {
	defer asyncNotify(s.recvNotifyCh)
START:
	s.stateLock.Lock()
	switch s.state {
	case streamLocalClose:
		fallthrough
	case streamRemoteClose:
		fallthrough
	case streamClosed:
		if s.recvBuf.Len() == 0 {
			s.stateLock.Unlock()
			return 0, io.EOF
		}
	case streamReset:
		s.stateLock.Unlock()
		return 0, ErrConnectionReset
	}
	s.stateLock.Unlock()

	// If there is no data available, block
	s.recvLock.Lock()
	if s.recvBuf.Len() == 0 {
		s.recvLock.Unlock()
		goto WAIT
	}

	// Read any bytes
	n, _ = s.recvBuf.Read(b)
	s.recvLock.Unlock()

	// Send a window update potentially
	err = s.sendWindowUpdate()
	return n, err

WAIT:
	var timeout <-chan time.Time
	if !s.readDeadline.IsZero() {
		delay := s.readDeadline.Sub(time.Now())
		timeout = time.After(delay)
	}
	select {
	case <-s.recvNotifyCh:
		goto START
	case <-timeout:
		return 0, ErrTimeout
	}
}

// Write is used to write to the stream
func (s *Stream) Write(b []byte) (n int, err error) {
	s.sendLock.Lock()
	defer s.sendLock.Unlock()
	total := 0
	for total < len(b) {
		n, err := s.write(b[total:])
		total += n
		if err != nil {
			return total, err
		}
	}
	return total, nil
}

// write is used to write to the stream, may return on
// a short write.
func (s *Stream) write(b []byte) (n int, err error) {
	var flags uint16
	var max uint32
	var body io.Reader
START:
	s.stateLock.Lock()
	switch s.state {
	case streamLocalClose:
		fallthrough
	case streamClosed:
		s.stateLock.Unlock()
		return 0, ErrStreamClosed
	case streamReset:
		s.stateLock.Unlock()
		return 0, ErrConnectionReset
	}
	s.stateLock.Unlock()

	// If there is no data available, block
	window := atomic.LoadUint32(&s.sendWindow)
	if window == 0 {
		goto WAIT
	}

	// Determine the flags if any
	flags = s.sendFlags()

	// Send up to our send window
	max = min(window, uint32(len(b)))
	body = bytes.NewReader(b[:max])

	// Send the header
	s.sendHdr.encode(typeData, flags, s.id, max)
	if err := s.session.waitForSendErr(s.sendHdr, body, s.sendErr); err != nil {
		return 0, err
	}

	// Reduce our send window
	atomic.AddUint32(&s.sendWindow, ^uint32(max-1))

	// Unlock
	return int(max), err

WAIT:
	var timeout <-chan time.Time
	if !s.writeDeadline.IsZero() {
		delay := s.writeDeadline.Sub(time.Now())
		timeout = time.After(delay)
	}
	select {
	case <-s.sendNotifyCh:
		goto START
	case <-timeout:
		return 0, ErrTimeout
	}
	return 0, nil
}

// sendFlags determines any flags that are appropriate
// based on the current stream state
func (s *Stream) sendFlags() uint16 {
	s.stateLock.Lock()
	defer s.stateLock.Unlock()
	var flags uint16
	switch s.state {
	case streamInit:
		flags |= flagSYN
		s.state = streamSYNSent
	case streamSYNReceived:
		flags |= flagACK
		s.state = streamEstablished
	}
	return flags
}

// sendWindowUpdate potentially sends a window update enabling
// further writes to take place. Must be invoked with the lock.
func (s *Stream) sendWindowUpdate() error {
	s.controlHdrLock.Lock()
	defer s.controlHdrLock.Unlock()

	// Determine the delta update
	max := s.session.config.MaxStreamWindowSize
	delta := max - atomic.LoadUint32(&s.recvWindow)

	// Determine the flags if any
	flags := s.sendFlags()

	// Check if we can omit the update
	if delta < (max/2) && flags == 0 {
		return nil
	}

	// Update our window
	atomic.AddUint32(&s.recvWindow, delta)

	// Send the header
	s.controlHdr.encode(typeWindowUpdate, flags, s.id, delta)
	if err := s.session.waitForSendErr(s.controlHdr, nil, s.controlErr); err != nil {
		return err
	}
	return nil
}

// sendClose is used to send a FIN
func (s *Stream) sendClose() error {
	s.controlHdrLock.Lock()
	defer s.controlHdrLock.Unlock()

	flags := s.sendFlags()
	flags |= flagFIN
	s.controlHdr.encode(typeWindowUpdate, flags, s.id, 0)
	if err := s.session.waitForSendErr(s.controlHdr, nil, s.controlErr); err != nil {
		return err
	}
	return nil
}

// Close is used to close the stream
func (s *Stream) Close() error {
	s.stateLock.Lock()
	switch s.state {
	// Opened means we need to signal a close
	case streamSYNSent:
		fallthrough
	case streamSYNReceived:
		fallthrough
	case streamEstablished:
		s.state = streamLocalClose
		goto SEND_CLOSE

	case streamLocalClose:
	case streamRemoteClose:
		s.state = streamClosed
		s.session.closeStream(s.id)
		goto SEND_CLOSE

	case streamClosed:
	case streamReset:
	default:
		panic("unhandled state")
	}
	s.stateLock.Unlock()
	return nil
SEND_CLOSE:
	s.stateLock.Unlock()
	s.sendClose()
	s.notifyWaiting()
	return nil
}

// forceClose is used for when the session is exiting
func (s *Stream) forceClose() {
	s.stateLock.Lock()
	s.state = streamClosed
	s.stateLock.Unlock()
	s.notifyWaiting()
}

// processFlags is used to update the state of the stream
// based on set flags, if any. Lock must be held
func (s *Stream) processFlags(flags uint16) error {
	s.stateLock.Lock()
	defer s.stateLock.Unlock()
	if flags&flagACK == flagACK {
		if s.state == streamSYNSent {
			s.state = streamEstablished
		}

	} else if flags&flagFIN == flagFIN {
		switch s.state {
		case streamSYNSent:
			fallthrough
		case streamSYNReceived:
			fallthrough
		case streamEstablished:
			s.state = streamRemoteClose
			s.notifyWaiting()
		case streamLocalClose:
			s.state = streamClosed
			s.session.closeStream(s.id)
			s.notifyWaiting()
		default:
			s.session.logger.Printf("[ERR] yamux: unexpected FIN flag in state %d", s.state)
			return ErrUnexpectedFlag
		}
	} else if flags&flagRST == flagRST {
		s.state = streamReset
		s.session.closeStream(s.id)
		s.notifyWaiting()
	}
	return nil
}

// notifyWaiting notifies all the waiting channels
func (s *Stream) notifyWaiting() {
	asyncNotify(s.recvNotifyCh)
	asyncNotify(s.sendNotifyCh)
}

// incrSendWindow updates the size of our send window
func (s *Stream) incrSendWindow(hdr header, flags uint16) error {
	if err := s.processFlags(flags); err != nil {
		return err
	}

	// Increase window, unblock a sender
	atomic.AddUint32(&s.sendWindow, hdr.Length())
	asyncNotify(s.sendNotifyCh)
	return nil
}

// readData is used to handle a data frame
func (s *Stream) readData(hdr header, flags uint16, conn io.Reader) error {
	if err := s.processFlags(flags); err != nil {
		return err
	}

	// Check that our recv window is not exceeded
	length := hdr.Length()
	if length == 0 {
		return nil
	}
	if remain := atomic.LoadUint32(&s.recvWindow); length > remain {
		s.session.logger.Printf("[ERR] yamux: receive window exceeded (stream: %d, remain: %d, recv: %d)", s.id, remain, length)
		return ErrRecvWindowExceeded
	}

	// Wrap in a limited reader
	conn = &io.LimitedReader{R: conn, N: int64(length)}

	// Copy into buffer
	s.recvLock.Lock()
	if _, err := io.Copy(&s.recvBuf, conn); err != nil {
		s.session.logger.Printf("[ERR] yamux: Failed to read stream data: %v", err)
		s.recvLock.Unlock()
		return err
	}

	// Decrement the receive window
	atomic.AddUint32(&s.recvWindow, ^uint32(length-1))
	s.recvLock.Unlock()

	// Unblock any readers
	asyncNotify(s.recvNotifyCh)
	return nil
}

// SetDeadline sets the read and write deadlines
func (s *Stream) SetDeadline(t time.Time) error {
	if err := s.SetReadDeadline(t); err != nil {
		return err
	}
	if err := s.SetWriteDeadline(t); err != nil {
		return err
	}
	return nil
}

// SetReadDeadline sets the deadline for future Read calls.
func (s *Stream) SetReadDeadline(t time.Time) error {
	s.readDeadline = t
	return nil
}

// SetWriteDeadline sets the deadline for future Write calls
func (s *Stream) SetWriteDeadline(t time.Time) error {
	s.writeDeadline = t
	return nil
}
<commit_msg>Prevent deadlock with closeStream race<commit_after>package yamux

import (
	"bytes"
	"io"
	"sync"
	"sync/atomic"
	"time"
)

type streamState int

const (
	streamInit streamState = iota
	streamSYNSent
	streamSYNReceived
	streamEstablished
	streamLocalClose
	streamRemoteClose
	streamClosed
	streamReset
)

// Stream is used to represent a logical stream
// within a session.
type Stream struct {
	recvWindow uint32
	sendWindow uint32

	id      uint32
	session *Session

	state     streamState
	stateLock sync.Mutex

	recvBuf  bytes.Buffer
	recvLock sync.Mutex

	controlHdr     header
	controlErr     chan error
	controlHdrLock sync.Mutex

	sendHdr  header
	sendErr  chan error
	sendLock sync.Mutex

	recvNotifyCh chan struct{}
	sendNotifyCh chan struct{}

	readDeadline  time.Time
	writeDeadline time.Time
}

// newStream is used to construct a new stream within
// a given session for an ID
func newStream(session *Session, id uint32, state streamState) *Stream {
	s := &Stream{
		id:           id,
		session:      session,
		state:        state,
		controlHdr:   header(make([]byte, headerSize)),
		controlErr:   make(chan error, 1),
		sendHdr:      header(make([]byte, headerSize)),
		sendErr:      make(chan error, 1),
		recvWindow:   initialStreamWindow,
		sendWindow:   initialStreamWindow,
		recvNotifyCh: make(chan struct{}, 1),
		sendNotifyCh: make(chan struct{}, 1),
	}
	return s
}

// Session returns the associated stream session
func (s *Stream) Session() *Session {
	return s.session
}

// StreamID returns the ID of this stream
func (s *Stream) StreamID() uint32 {
	return s.id
}

// Read is used to read from the stream
func (s *Stream) Read(b []byte) (n int, err error) {
	defer asyncNotify(s.recvNotifyCh)
START:
	s.stateLock.Lock()
	switch s.state {
	case streamLocalClose:
		fallthrough
	case streamRemoteClose:
		fallthrough
	case streamClosed:
		if s.recvBuf.Len() == 0 {
			s.stateLock.Unlock()
			return 0, io.EOF
		}
	case streamReset:
		s.stateLock.Unlock()
		return 0, ErrConnectionReset
	}
	s.stateLock.Unlock()

	// If there is no data available, block
	s.recvLock.Lock()
	if s.recvBuf.Len() == 0 {
		s.recvLock.Unlock()
		goto WAIT
	}

	// Read any bytes
	n, _ = s.recvBuf.Read(b)
	s.recvLock.Unlock()

	// Send a window update potentially
	err = s.sendWindowUpdate()
	return n, err

WAIT:
	var timeout <-chan time.Time
	if !s.readDeadline.IsZero() {
		delay := s.readDeadline.Sub(time.Now())
		timeout = time.After(delay)
	}
	select {
	case <-s.recvNotifyCh:
		goto START
	case <-timeout:
		return 0, ErrTimeout
	}
}

// Write is used to write to the stream
func (s *Stream) Write(b []byte) (n int, err error) {
	s.sendLock.Lock()
	defer s.sendLock.Unlock()
	total := 0
	for total < len(b) {
		n, err := s.write(b[total:])
		total += n
		if err != nil {
			return total, err
		}
	}
	return total, nil
}

// write is used to write to the stream, may return on
// a short write.
func (s *Stream) write(b []byte) (n int, err error) {
	var flags uint16
	var max uint32
	var body io.Reader
START:
	s.stateLock.Lock()
	switch s.state {
	case streamLocalClose:
		fallthrough
	case streamClosed:
		s.stateLock.Unlock()
		return 0, ErrStreamClosed
	case streamReset:
		s.stateLock.Unlock()
		return 0, ErrConnectionReset
	}
	s.stateLock.Unlock()

	// If there is no data available, block
	window := atomic.LoadUint32(&s.sendWindow)
	if window == 0 {
		goto WAIT
	}

	// Determine the flags if any
	flags = s.sendFlags()

	// Send up to our send window
	max = min(window, uint32(len(b)))
	body = bytes.NewReader(b[:max])

	// Send the header
	s.sendHdr.encode(typeData, flags, s.id, max)
	if err := s.session.waitForSendErr(s.sendHdr, body, s.sendErr); err != nil {
		return 0, err
	}

	// Reduce our send window
	atomic.AddUint32(&s.sendWindow, ^uint32(max-1))

	// Unlock
	return int(max), err

WAIT:
	var timeout <-chan time.Time
	if !s.writeDeadline.IsZero() {
		delay := s.writeDeadline.Sub(time.Now())
		timeout = time.After(delay)
	}
	select {
	case <-s.sendNotifyCh:
		goto START
	case <-timeout:
		return 0, ErrTimeout
	}
	return 0, nil
}

// sendFlags determines any flags that are appropriate
// based on the current stream state
func (s *Stream) sendFlags() uint16 {
	s.stateLock.Lock()
	defer s.stateLock.Unlock()
	var flags uint16
	switch s.state {
	case streamInit:
		flags |= flagSYN
		s.state = streamSYNSent
	case streamSYNReceived:
		flags |= flagACK
		s.state = streamEstablished
	}
	return flags
}

// sendWindowUpdate potentially sends a window update enabling
// further writes to take place. Must be invoked with the lock.
func (s *Stream) sendWindowUpdate() error {
	s.controlHdrLock.Lock()
	defer s.controlHdrLock.Unlock()

	// Determine the delta update
	max := s.session.config.MaxStreamWindowSize
	delta := max - atomic.LoadUint32(&s.recvWindow)

	// Determine the flags if any
	flags := s.sendFlags()

	// Check if we can omit the update
	if delta < (max/2) && flags == 0 {
		return nil
	}

	// Update our window
	atomic.AddUint32(&s.recvWindow, delta)

	// Send the header
	s.controlHdr.encode(typeWindowUpdate, flags, s.id, delta)
	if err := s.session.waitForSendErr(s.controlHdr, nil, s.controlErr); err != nil {
		return err
	}
	return nil
}

// sendClose is used to send a FIN
func (s *Stream) sendClose() error {
	s.controlHdrLock.Lock()
	defer s.controlHdrLock.Unlock()

	flags := s.sendFlags()
	flags |= flagFIN
	s.controlHdr.encode(typeWindowUpdate, flags, s.id, 0)
	if err := s.session.waitForSendErr(s.controlHdr, nil, s.controlErr); err != nil {
		return err
	}
	return nil
}

// Close is used to close the stream
func (s *Stream) Close() error {
	closeStream := false
	s.stateLock.Lock()
	switch s.state {
	// Opened means we need to signal a close
	case streamSYNSent:
		fallthrough
	case streamSYNReceived:
		fallthrough
	case streamEstablished:
		s.state = streamLocalClose
		goto SEND_CLOSE

	case streamLocalClose:
	case streamRemoteClose:
		s.state = streamClosed
		closeStream = true
		goto SEND_CLOSE

	case streamClosed:
	case streamReset:
	default:
		panic("unhandled state")
	}
	s.stateLock.Unlock()
	return nil
SEND_CLOSE:
	s.stateLock.Unlock()
	s.sendClose()
	s.notifyWaiting()
	if closeStream {
		s.session.closeStream(s.id)
	}
	return nil
}

// forceClose is used for when the session is exiting
func (s *Stream) forceClose() {
	s.stateLock.Lock()
	s.state = streamClosed
	s.stateLock.Unlock()
	s.notifyWaiting()
}

// processFlags is used to update the state of the stream
// based on set flags, if any. Lock must be held
func (s *Stream) processFlags(flags uint16) error {
	// Close the stream without holding the state lock
	closeStream := false
	defer func() {
		if closeStream {
			s.session.closeStream(s.id)
		}
	}()

	s.stateLock.Lock()
	defer s.stateLock.Unlock()
	if flags&flagACK == flagACK {
		if s.state == streamSYNSent {
			s.state = streamEstablished
		}

	} else if flags&flagFIN == flagFIN {
		switch s.state {
		case streamSYNSent:
			fallthrough
		case streamSYNReceived:
			fallthrough
		case streamEstablished:
			s.state = streamRemoteClose
			s.notifyWaiting()
		case streamLocalClose:
			s.state = streamClosed
			closeStream = true
			s.notifyWaiting()
		default:
			s.session.logger.Printf("[ERR] yamux: unexpected FIN flag in state %d", s.state)
			return ErrUnexpectedFlag
		}
	} else if flags&flagRST == flagRST {
		s.state = streamReset
		closeStream = true
		s.notifyWaiting()
	}
	return nil
}

// notifyWaiting notifies all the waiting channels
func (s *Stream) notifyWaiting() {
	asyncNotify(s.recvNotifyCh)
	asyncNotify(s.sendNotifyCh)
}

// incrSendWindow updates the size of our send window
func (s *Stream) incrSendWindow(hdr header, flags uint16) error {
	if err := s.processFlags(flags); err != nil {
		return err
	}

	// Increase window, unblock a sender
	atomic.AddUint32(&s.sendWindow, hdr.Length())
	asyncNotify(s.sendNotifyCh)
	return nil
}

// readData is used to handle a data frame
func (s *Stream) readData(hdr header, flags uint16, conn io.Reader) error {
	if err := s.processFlags(flags); err != nil {
		return err
	}

	// Check that our recv window is not exceeded
	length := hdr.Length()
	if length == 0 {
		return nil
	}
	if remain := atomic.LoadUint32(&s.recvWindow); length > remain {
		s.session.logger.Printf("[ERR] yamux: receive window exceeded (stream: %d, remain: %d, recv: %d)", s.id, remain, length)
		return ErrRecvWindowExceeded
	}

	// Wrap in a limited reader
	conn = &io.LimitedReader{R: conn, N: int64(length)}

	// Copy into buffer
	s.recvLock.Lock()
	if _, err := io.Copy(&s.recvBuf, conn); err != nil {
		s.session.logger.Printf("[ERR] yamux: Failed to read stream data: %v", err)
		s.recvLock.Unlock()
		return err
	}

	// Decrement the receive window
	atomic.AddUint32(&s.recvWindow, ^uint32(length-1))
	s.recvLock.Unlock()

	// Unblock any readers
	asyncNotify(s.recvNotifyCh)
	return nil
}

// SetDeadline sets the read and write deadlines
func (s *Stream) SetDeadline(t time.Time) error {
	if err := s.SetReadDeadline(t); err != nil {
		return err
	}
	if err := s.SetWriteDeadline(t); err != nil {
		return err
	}
	return nil
}

// SetReadDeadline sets the deadline for future Read calls.
func (s *Stream) SetReadDeadline(t time.Time) error {
	s.readDeadline = t
	return nil
}

// SetWriteDeadline sets the deadline for future Write calls
func (s *Stream) SetWriteDeadline(t time.Time) error {
	s.writeDeadline = t
	return nil
}
<|endoftext|>