<commit_before>/*
Copyright 2015 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

*/

package teleport

import (
	"fmt"
	"strings"

	"github.com/gravitational/trace"
)

// Role identifies the role of an SSH connection. Unlike "user roles"
// introduced as part of RBAC in Teleport 1.4+ these are built-in roles used
// for different Teleport components when connecting to each other.
type Role string
type Roles []Role

const (
	// RoleAuth is for teleport auth server (authority, authentication and authorization)
	RoleAuth Role = "Auth"
	// RoleWeb is for web access users
	RoleWeb Role = "Web"
	// RoleNode is a role for SSH node in the cluster
	RoleNode Role = "Node"
	// RoleProxy is a role for SSH proxy in the cluster
	RoleProxy Role = "Proxy"
	// RoleAdmin is admin role
	RoleAdmin Role = "Admin"
	// RoleProvisionToken is a role for nodes authenticated using provisioning tokens
	RoleProvisionToken Role = "ProvisionToken"
	// RoleTrustedCluster is a role needed for tokens used to add trusted clusters.
	RoleTrustedCluster Role = "trusted_cluster"
	// RoleSignup is for first time signing up users
	RoleSignup Role = "Signup"
	// RoleNop is used for actions that already using external authz mechanisms
	// e.g. tokens or passwords
	RoleNop Role = "Nop"
)

// this constant exists for backwards compatibility reasons, needed to upgrade to 2.3
const LegacyClusterTokenType Role = "Trustedcluster"

// ParseRoles takes a comma-separated list of roles and returns a slice
// of roles, or an error if parsing failed
func ParseRoles(str string) (roles Roles, err error) {
	for _, s := range strings.Split(str, ",") {
		r := Role(strings.Title(strings.ToLower(strings.TrimSpace(s))))
		if err = r.Check(); err != nil {
			return nil, trace.Wrap(err)
		}
		roles = append(roles, r)
	}
	return roles, nil
}

// Includes returns 'true' if a given list of roles includes a given role
func (roles Roles) Include(role Role) bool {
	for _, r := range roles {
		if r == role {
			return true
		}
	}
	return false
}

// Equals compares two sets of roles
func (roles Roles) Equals(other Roles) bool {
	if len(roles) != len(other) {
		return false
	}
	for _, r := range roles {
		if !other.Include(r) {
			return false
		}
	}
	return true
}

// Check returns an error if the role set is incorrect (contains unknown roles)
func (roles Roles) Check() (err error) {
	for _, role := range roles {
		if err = role.Check(); err != nil {
			return trace.Wrap(err)
		}
	}
	return nil
}

func (roles Roles) String() string {
	s := make([]string, 0)
	for _, r := range roles {
		s = append(s, string(r))
	}
	return strings.Join(s, ",")
}

// Set sets the value of the role from string, used to integrate with CLI tools
func (r *Role) Set(v string) error {
	val := Role(strings.Title(v))
	if err := val.Check(); err != nil {
		return trace.Wrap(err)
	}
	*r = val
	return nil
}

// String returns debug-friendly representation of this role
func (r *Role) String() string {
	return fmt.Sprintf("%v", strings.ToUpper(string(*r)))
}

// Check checks if this a a valid role value, returns nil
// if it's ok, false otherwise
func (r *Role) Check() error {
	switch *r {
	case RoleAuth, RoleWeb, RoleNode,
		RoleAdmin, RoleProvisionToken,
		RoleTrustedCluster, LegacyClusterTokenType,
		RoleSignup, RoleProxy, RoleNop:
		return nil
	}
	return trace.BadParameter("role %v is not registered", *r)
}

// ContextUser is a user set in the context of the request
const ContextUser = "teleport-user"

// LocalUsername is a local username
type LocalUser struct {
	// Username is local username
	Username string
}

// BuiltinRole is monitoring
type BuiltinRole struct {
	// Role is the builtin role this username is associated with
	Role Role
}

// RemoteUser defines encoded remote user
type RemoteUser struct {
	// Username is a name of the remote user
	Username string `json:"username"`
	// ClusterName is a name of the remote cluster
	// of the user
	ClusterName string `json:"cluster_name"`
	// RemoteRoles is optional list of remote roles
	RemoteRoles []string `json:"remote_roles"`
}
<commit_msg>Fix name of role, has to start with uppercase.<commit_after>/*
Copyright 2015 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

*/

package teleport

import (
	"fmt"
	"strings"

	"github.com/gravitational/trace"
)

// Role identifies the role of an SSH connection. Unlike "user roles"
// introduced as part of RBAC in Teleport 1.4+ these are built-in roles used
// for different Teleport components when connecting to each other.
type Role string
type Roles []Role

const (
	// RoleAuth is for teleport auth server (authority, authentication and authorization)
	RoleAuth Role = "Auth"
	// RoleWeb is for web access users
	RoleWeb Role = "Web"
	// RoleNode is a role for SSH node in the cluster
	RoleNode Role = "Node"
	// RoleProxy is a role for SSH proxy in the cluster
	RoleProxy Role = "Proxy"
	// RoleAdmin is admin role
	RoleAdmin Role = "Admin"
	// RoleProvisionToken is a role for nodes authenticated using provisioning tokens
	RoleProvisionToken Role = "ProvisionToken"
	// RoleTrustedCluster is a role needed for tokens used to add trusted clusters.
	RoleTrustedCluster Role = "Trusted_cluster"
	// RoleSignup is for first time signing up users
	RoleSignup Role = "Signup"
	// RoleNop is used for actions that already using external authz mechanisms
	// e.g. tokens or passwords
	RoleNop Role = "Nop"
)

// this constant exists for backwards compatibility reasons, needed to upgrade to 2.3
const LegacyClusterTokenType Role = "Trustedcluster"

// ParseRoles takes a comma-separated list of roles and returns a slice
// of roles, or an error if parsing failed
func ParseRoles(str string) (roles Roles, err error) {
	for _, s := range strings.Split(str, ",") {
		r := Role(strings.Title(strings.ToLower(strings.TrimSpace(s))))
		if err = r.Check(); err != nil {
			return nil, trace.Wrap(err)
		}
		roles = append(roles, r)
	}
	return roles, nil
}

// Includes returns 'true' if a given list of roles includes a given role
func (roles Roles) Include(role Role) bool {
	for _, r := range roles {
		if r == role {
			return true
		}
	}
	return false
}

// Equals compares two sets of roles
func (roles Roles) Equals(other Roles) bool {
	if len(roles) != len(other) {
		return false
	}
	for _, r := range roles {
		if !other.Include(r) {
			return false
		}
	}
	return true
}

// Check returns an error if the role set is incorrect (contains unknown roles)
func (roles Roles) Check() (err error) {
	for _, role := range roles {
		if err = role.Check(); err != nil {
			return trace.Wrap(err)
		}
	}
	return nil
}

func (roles Roles) String() string {
	s := make([]string, 0)
	for _, r := range roles {
		s = append(s, string(r))
	}
	return strings.Join(s, ",")
}

// Set sets the value of the role from string, used to integrate with CLI tools
func (r *Role) Set(v string) error {
	val := Role(strings.Title(v))
	if err := val.Check(); err != nil {
		return trace.Wrap(err)
	}
	*r = val
	return nil
}

// String returns debug-friendly representation of this role
func (r *Role) String() string {
	return fmt.Sprintf("%v", strings.ToUpper(string(*r)))
}

// Check checks if this a a valid role value, returns nil
// if it's ok, false otherwise
func (r *Role) Check() error {
	switch *r {
	case RoleAuth, RoleWeb, RoleNode,
		RoleAdmin, RoleProvisionToken,
		RoleTrustedCluster, LegacyClusterTokenType,
		RoleSignup, RoleProxy, RoleNop:
		return nil
	}
	return trace.BadParameter("role %v is not registered", *r)
}

// ContextUser is a user set in the context of the request
const ContextUser = "teleport-user"

// LocalUsername is a local username
type LocalUser struct {
	// Username is local username
	Username string
}

// BuiltinRole is monitoring
type BuiltinRole struct {
	// Role is the builtin role this username is associated with
	Role Role
}

// RemoteUser defines encoded remote user
type RemoteUser struct {
	// Username is a name of the remote user
	Username string `json:"username"`
	// ClusterName is a name of the remote cluster
	// of the user
	ClusterName string `json:"cluster_name"`
	// RemoteRoles is optional list of remote roles
	RemoteRoles []string `json:"remote_roles"`
}
<|endoftext|>


<commit_before>package api_test

import (
	"encoding/json"
	"fmt"
	"github.com/nerdzeu/nerdz-api/nerdz"
	"net/http"
	"testing"
)

var (
	userID          = "1"
	numPosts        = 10
	mainURL         = fmt.Sprintf("http://localhost:%d", nerdz.Configuration.Port)
	userInfoURL     = fmt.Sprintf("%s/users/%s", mainURL, userID)
	userFriendsURL  = fmt.Sprintf("%s/users/%s/friends", mainURL, userID)
	allUserPostsURL = fmt.Sprintf("%s/users/%s/posts", mainURL, userID)
	nUserPostsURL   = fmt.Sprintf("%s/users/%s/posts?n=%d", mainURL, userID, numPosts)
)

func TestUserInfo(t *testing.T) {
	t.Log("Trying to retrieve User(1)'s information")

	res, err := http.DefaultClient.Get(userInfoURL)

	if err != nil {
		t.Errorf("Error in GET request: %+v", err)
		t.FailNow()
	} else if res.StatusCode != http.StatusOK {
		t.Errorf("Error in GET request: status code=%s", res.Status)
		t.FailNow()
	}

	dec := json.NewDecoder(res.Body)

	mapData := map[string]interface{}{}

	if err := dec.Decode(&mapData); err != nil {
		t.Errorf("Unable to decode received data: %+v", err)
		t.FailNow()
	}

}

func TestUserFriends(t *testing.T) {

	t.Log("Trying to retrieve User(1)'s friends")

	res, err := http.DefaultClient.Get(userFriendsURL)

	if err != nil {
		t.Errorf("Error in GET request: %+v", err)
		t.FailNow()
	} else if res.StatusCode != http.StatusOK {
		t.Errorf("Error in GET request: status code=%s", res.Status)
		t.FailNow()
	}

	dec := json.NewDecoder(res.Body)

	friendsData := map[string]interface{}{}

	if err := dec.Decode(&friendsData); err != nil {
		t.Errorf("Unable to decode received data: %+v", err)
		t.FailNow()
	}

	// User 1 has friends
	if lenData := len(friendsData["data"].(map[string]interface{})); lenData != 3 {
		t.Errorf("Incorrect retrived friends. User(1) has 3 friends, got %d", lenData)
	}

}

func TestAllUserPosts(t *testing.T) {
	t.Log("Trying to retrieve all User(1)'s posts")

	res, err := http.DefaultClient.Get(allUserPostsURL)

	if err != nil {
		t.Errorf("Error in GET request: %+v", err)
		t.FailNow()
	} else if res.StatusCode != http.StatusOK {
		t.Errorf("Error in GET request: status code=%s", res.Status)
		t.FailNow()
	}

	dec := json.NewDecoder(res.Body)

	mapData := map[string]interface{}{}

	if err := dec.Decode(&mapData); err != nil {
		t.Errorf("Unable to decode received data: %+v", err)
		t.FailNow()
	}
}

func TestTenUserPosts(t *testing.T) {
	t.Logf("Trying to retrieve <%d> User(1)'s posts", numPosts)

	res, err := http.DefaultClient.Get(userInfoURL)

	if err != nil {
		t.Errorf("Error in GET request: %+v", err)
		t.FailNow()
	} else if res.StatusCode != http.StatusOK {
		t.Errorf("Error in GET request: status code=%s", res.Status)
		t.FailNow()
	}

	dec := json.NewDecoder(res.Body)

	mapData := map[string]interface{}{}

	if err := dec.Decode(&mapData); err != nil {
		t.Errorf("Unable to decode received data: %+v", err)
		t.FailNow()
	}

	if lenData := len(mapData["data"].(map[string]interface{})); lenData > numPosts {
		t.Errorf("Unable to retrieve correctly posts: lenData=%d > numPosts=%d", lenData, numPosts)
		t.FailNow()
	}
}
<commit_msg>Created entry point for an API handler; Simple tests for API handler<commit_after>package api_test

import (
	"encoding/json"
	"fmt"
	"github.com/nerdzeu/nerdz-api/nerdz"
	"net/http"
	"testing"
)

var (
	userID          = "1"
	numPosts        = 10
	mainURL         = fmt.Sprintf("http://localhost:%d", nerdz.Configuration.Port)
	userInfoURL     = fmt.Sprintf("%s/users/%s", mainURL, userID)
	userFriendsURL  = fmt.Sprintf("%s/users/%s/friends", mainURL, userID)
	allUserPostsURL = fmt.Sprintf("%s/users/%s/posts", mainURL, userID)
	nUserPostsURL   = fmt.Sprintf("%s/users/%s/posts?n=%d", mainURL, userID, numPosts)
)

func TestUserInfo(t *testing.T) {
	t.Log("Trying to retrieve User(1)'s information")

	res, err := http.DefaultClient.Get(userInfoURL)

	if err != nil {
		t.Errorf("Error in GET request: %+v", err)
		t.FailNow()
	} else if res.StatusCode != http.StatusOK {
		t.Errorf("Error in GET request: status code=%s", res.Status)
		t.FailNow()
	}

	dec := json.NewDecoder(res.Body)

	mapData := map[string]interface{}{}

	if err := dec.Decode(&mapData); err != nil {
		t.Errorf("Unable to decode received data: %+v", err)
		t.FailNow()
	}

}

func TestUserFriends(t *testing.T) {

	t.Log("Trying to retrieve User(1)'s friends")

	res, err := http.DefaultClient.Get(userFriendsURL)

	if err != nil {
		t.Errorf("Error in GET request: %+v", err)
		t.FailNow()
	} else if res.StatusCode != http.StatusOK {
		t.Errorf("Error in GET request: status code=%s", res.Status)
		t.FailNow()
	}

	dec := json.NewDecoder(res.Body)

	friendsData := map[string]interface{}{}

	if err := dec.Decode(&friendsData); err != nil {
		t.Errorf("Unable to decode received data: %+v", err)
		t.FailNow()
	}

	// User 1 has friends
	if lenData := len(friendsData["data"].(map[string]interface{})); lenData != 3 {
		t.Errorf("Incorrect retrived friends. User(1) has 3 friends, got %d", lenData)
	}

}

func TestAllUserPosts(t *testing.T) {
	t.Log("Trying to retrieve all User(1)'s posts")

	res, err := http.DefaultClient.Get(allUserPostsURL)

	if err != nil {
		t.Errorf("Error in GET request: %+v", err)
		t.FailNow()
	} else if res.StatusCode != http.StatusOK {
		t.Errorf("Error in GET request: status code=%s", res.Status)
		t.FailNow()
	}

	dec := json.NewDecoder(res.Body)

	mapData := map[string]interface{}{}

	if err := dec.Decode(&mapData); err != nil {
		t.Errorf("Unable to decode received data: %+v", err)
		t.FailNow()
	}
}

func TestTenUserPosts(t *testing.T) {
	t.Logf("Trying to retrieve <%d> User(1)'s posts", numPosts)

	res, err := http.DefaultClient.Get(userInfoURL)

	if err != nil {
		t.Errorf("Error in GET request: %+v", err)
		t.FailNow()
	} else if res.StatusCode != http.StatusOK {
		t.Errorf("Error in GET request: status code=%s", res.Status)
		t.FailNow()
	}

	dec := json.NewDecoder(res.Body)

	mapData := map[string]interface{}{}

	if err := dec.Decode(&mapData); err != nil {
		t.Errorf("Unable to decode received data: %+v", err)
		t.FailNow()
	}

	if lenData := len(mapData["data"].(map[string]interface{})); lenData == numPosts {
		t.Errorf("Unable to retrieve correctly posts: lenData=%d > numPosts=%d", lenData, numPosts)
		t.FailNow()
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 The Walk Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package walk

import (
	"errors"
	"fmt"
	"reflect"
)

var (
	errValidationFailed = errors.New("validation failed")
)

type ErrorPresenter interface {
	PresentError(err error, widget Widget)
}

type DataBinder struct {
	dataSource                interface{}
	boundWidgets              []Widget
	properties                []*Property
	property2Widget           map[*Property]Widget
	property2ChangedHandle    map[*Property]int
	widget2Property2Error     map[Widget]map[*Property]error
	errorPresenter            ErrorPresenter
	canSubmitChangedPublisher EventPublisher
}

func NewDataBinder() *DataBinder {
	return new(DataBinder)
}

func (db *DataBinder) DataSource() interface{} {
	return db.dataSource
}

func (db *DataBinder) SetDataSource(dataSource interface{}) {
	db.dataSource = dataSource
}

func (db *DataBinder) BoundWidgets() []Widget {
	return db.boundWidgets
}

func (db *DataBinder) SetBoundWidgets(boundWidgets []Widget) {
	for prop, handle := range db.property2ChangedHandle {
		prop.Changed().Detach(handle)
	}

	db.boundWidgets = boundWidgets

	db.property2Widget = make(map[*Property]Widget)
	db.property2ChangedHandle = make(map[*Property]int)
	db.widget2Property2Error = make(map[Widget]map[*Property]error)

	for _, widget := range boundWidgets {
		widget := widget

		for _, prop := range widget.BaseWidget().name2Property {
			prop := prop
			if _, ok := prop.Source().(string); !ok {
				continue
			}

			db.properties = append(db.properties, prop)
			db.property2Widget[prop] = widget

			db.property2ChangedHandle[prop] = prop.Changed().Attach(func() {
				db.validateProperty(prop, widget)
			})
		}
	}
}

func (db *DataBinder) validateProperty(prop *Property, widget Widget) {
	validator := prop.Validator()
	if validator == nil {
		return
	}

	var changed bool
	prop2Err := db.widget2Property2Error[widget]

	err := validator.Validate(prop.Get())
	if err != nil {
		changed = len(db.widget2Property2Error) == 0

		if prop2Err == nil {
			prop2Err = make(map[*Property]error)
			db.widget2Property2Error[widget] = prop2Err
		}
		prop2Err[prop] = err
	} else {
		if prop2Err == nil {
			return
		}

		delete(prop2Err, prop)

		if len(prop2Err) == 0 {
			delete(db.widget2Property2Error, widget)

			changed = len(db.widget2Property2Error) == 0
		}
	}

	if db.errorPresenter != nil {
		db.errorPresenter.PresentError(err, widget)
	}

	if changed {
		db.canSubmitChangedPublisher.Publish()
	}
}

func (db *DataBinder) ErrorPresenter() ErrorPresenter {
	return db.errorPresenter
}

func (db *DataBinder) SetErrorPresenter(ep ErrorPresenter) {
	db.errorPresenter = ep
}

func (db *DataBinder) CanSubmit() bool {
	return len(db.widget2Property2Error) == 0
}

func (db *DataBinder) CanSubmitChanged() *Event {
	return db.canSubmitChangedPublisher.Event()
}

func (db *DataBinder) Reset() error {
	return db.forEach(func(prop *Property, field reflect.Value) error {
		if f64, ok := prop.Get().(float64); ok {
			switch v := field.Interface().(type) {
			case float32:
				f64 = float64(v)

			case float64:
				f64 = v

			case int:
				f64 = float64(v)

			case int8:
				f64 = float64(v)

			case int16:
				f64 = float64(v)

			case int32:
				f64 = float64(v)

			case int64:
				f64 = float64(v)

			case uint:
				f64 = float64(v)

			case uint8:
				f64 = float64(v)

			case uint16:
				f64 = float64(v)

			case uint32:
				f64 = float64(v)

			case uint64:
				f64 = float64(v)

			case uintptr:
				f64 = float64(v)

			default:
				return newError(fmt.Sprintf("Field '%s': Can't convert %s to float64.", prop.Source().(string), field.Type().Name()))
			}

			if err := prop.Set(f64); err != nil {
				return err
			}
		} else {
			if err := prop.Set(field.Interface()); err != nil {
				return err
			}
		}

		db.validateProperty(prop, db.property2Widget[prop])
		return nil
	})
}

func (db *DataBinder) Submit() error {
	if !db.CanSubmit() {
		return errValidationFailed
	}

	return db.forEach(func(prop *Property, field reflect.Value) error {
		value := prop.Get()
		if value == nil {
			// This happens e.g. if CurrentIndex() of a ComboBox returns -1.
			// FIXME: Should we handle this differently?
			return nil
		}

		if f64, ok := value.(float64); ok {
			switch field.Kind() {
			case reflect.Float32, reflect.Float64:
				field.SetFloat(f64)

			case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
				field.SetInt(int64(f64))

			case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
				field.SetUint(uint64(f64))

			default:
				return newError(fmt.Sprintf("Field '%s': Can't convert float64 to %s.", prop.Source().(string), field.Type().Name()))
			}

			return nil
		}

		field.Set(reflect.ValueOf(value))

		return nil
	})
}

func (db *DataBinder) forEach(f func(prop *Property, field reflect.Value) error) error {
	p := reflect.ValueOf(db.dataSource)
	if p.Type().Kind() != reflect.Ptr {
		return newError("DataSource must be a pointer to a struct.")
	}

	if p.IsNil() {
		return nil
	}

	s := reflect.Indirect(p)
	if s.Type().Kind() != reflect.Struct {
		return newError("DataSource must be a pointer to a struct.")
	}

	for _, prop := range db.properties {
		if field := s.FieldByName(prop.Source().(string)); field.IsValid() {
			if err := f(prop, field); err != nil {
				return err
			}
		} else {
			return newError(fmt.Sprintf("Struct '%s' has no field '%s'.", s.Type().Name(), prop.Source().(string)))
		}
	}

	return nil
}

func validateBindingMemberSyntax(member string) error {
	// FIXME
	return nil
}
<commit_msg>DataBinder: Add support for data source composition (only pointers to structs for now)<commit_after>// Copyright 2012 The Walk Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package walk

import (
	"errors"
	"fmt"
	"reflect"
	"strings"
)

var (
	errValidationFailed = errors.New("validation failed")
)

type ErrorPresenter interface {
	PresentError(err error, widget Widget)
}

type DataBinder struct {
	dataSource                interface{}
	boundWidgets              []Widget
	properties                []*Property
	property2Widget           map[*Property]Widget
	property2ChangedHandle    map[*Property]int
	widget2Property2Error     map[Widget]map[*Property]error
	errorPresenter            ErrorPresenter
	canSubmitChangedPublisher EventPublisher
}

func NewDataBinder() *DataBinder {
	return new(DataBinder)
}

func (db *DataBinder) DataSource() interface{} {
	return db.dataSource
}

func (db *DataBinder) SetDataSource(dataSource interface{}) {
	db.dataSource = dataSource
}

func (db *DataBinder) BoundWidgets() []Widget {
	return db.boundWidgets
}

func (db *DataBinder) SetBoundWidgets(boundWidgets []Widget) {
	for prop, handle := range db.property2ChangedHandle {
		prop.Changed().Detach(handle)
	}

	db.boundWidgets = boundWidgets

	db.property2Widget = make(map[*Property]Widget)
	db.property2ChangedHandle = make(map[*Property]int)
	db.widget2Property2Error = make(map[Widget]map[*Property]error)

	for _, widget := range boundWidgets {
		widget := widget

		for _, prop := range widget.BaseWidget().name2Property {
			prop := prop
			if _, ok := prop.Source().(string); !ok {
				continue
			}

			db.properties = append(db.properties, prop)
			db.property2Widget[prop] = widget

			db.property2ChangedHandle[prop] = prop.Changed().Attach(func() {
				db.validateProperty(prop, widget)
			})
		}
	}
}

func (db *DataBinder) validateProperty(prop *Property, widget Widget) {
	validator := prop.Validator()
	if validator == nil {
		return
	}

	var changed bool
	prop2Err := db.widget2Property2Error[widget]

	err := validator.Validate(prop.Get())
	if err != nil {
		changed = len(db.widget2Property2Error) == 0

		if prop2Err == nil {
			prop2Err = make(map[*Property]error)
			db.widget2Property2Error[widget] = prop2Err
		}
		prop2Err[prop] = err
	} else {
		if prop2Err == nil {
			return
		}

		delete(prop2Err, prop)

		if len(prop2Err) == 0 {
			delete(db.widget2Property2Error, widget)

			changed = len(db.widget2Property2Error) == 0
		}
	}

	if db.errorPresenter != nil {
		db.errorPresenter.PresentError(err, widget)
	}

	if changed {
		db.canSubmitChangedPublisher.Publish()
	}
}

func (db *DataBinder) ErrorPresenter() ErrorPresenter {
	return db.errorPresenter
}

func (db *DataBinder) SetErrorPresenter(ep ErrorPresenter) {
	db.errorPresenter = ep
}

func (db *DataBinder) CanSubmit() bool {
	return len(db.widget2Property2Error) == 0
}

func (db *DataBinder) CanSubmitChanged() *Event {
	return db.canSubmitChangedPublisher.Event()
}

func (db *DataBinder) Reset() error {
	return db.forEach(func(prop *Property, field reflect.Value) error {
		if f64, ok := prop.Get().(float64); ok {
			switch v := field.Interface().(type) {
			case float32:
				f64 = float64(v)

			case float64:
				f64 = v

			case int:
				f64 = float64(v)

			case int8:
				f64 = float64(v)

			case int16:
				f64 = float64(v)

			case int32:
				f64 = float64(v)

			case int64:
				f64 = float64(v)

			case uint:
				f64 = float64(v)

			case uint8:
				f64 = float64(v)

			case uint16:
				f64 = float64(v)

			case uint32:
				f64 = float64(v)

			case uint64:
				f64 = float64(v)

			case uintptr:
				f64 = float64(v)

			default:
				return newError(fmt.Sprintf("Field '%s': Can't convert %s to float64.", prop.Source().(string), field.Type().Name()))
			}

			if err := prop.Set(f64); err != nil {
				return err
			}
		} else {
			if err := prop.Set(field.Interface()); err != nil {
				return err
			}
		}

		db.validateProperty(prop, db.property2Widget[prop])
		return nil
	})
}

func (db *DataBinder) Submit() error {
	if !db.CanSubmit() {
		return errValidationFailed
	}

	return db.forEach(func(prop *Property, field reflect.Value) error {
		value := prop.Get()
		if value == nil {
			// This happens e.g. if CurrentIndex() of a ComboBox returns -1.
			// FIXME: Should we handle this differently?
			return nil
		}

		if f64, ok := value.(float64); ok {
			switch field.Kind() {
			case reflect.Float32, reflect.Float64:
				field.SetFloat(f64)

			case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
				field.SetInt(int64(f64))

			case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
				field.SetUint(uint64(f64))

			default:
				return newError(fmt.Sprintf("Field '%s': Can't convert float64 to %s.", prop.Source().(string), field.Type().Name()))
			}

			return nil
		}

		field.Set(reflect.ValueOf(value))

		return nil
	})
}

func (db *DataBinder) forEach(f func(prop *Property, field reflect.Value) error) error {
	p := reflect.ValueOf(db.dataSource)
	if p.Type().Kind() != reflect.Ptr {
		return newError("DataSource must be a pointer to a struct.")
	}

	if p.IsNil() {
		return nil
	}

	s := reflect.Indirect(p)
	if s.Type().Kind() != reflect.Struct {
		return newError("DataSource must be a pointer to a struct.")
	}

	for _, prop := range db.properties {
		path := prop.Source().(string)
		names := strings.Split(path, ".")

		p := p
		s := s

		for i, name := range names {
			field := s.FieldByName(name)
			if !field.IsValid() {
				return newError(fmt.Sprintf("Struct '%s' has no field '%s'.",
					s.Type().Name(), name))
			}

			if i == len(names)-1 {
				if err := f(prop, field); err != nil {
					return err
				}
			} else if p.Type().Kind() == reflect.Ptr {
				p = field
			} else {
				return newError("Field must be a pointer to a struct.")
			}

			if p.IsNil() {
				return newError("Pointer must not be nil.")
			}

			s = reflect.Indirect(p)
			if s.Type().Kind() != reflect.Struct {
				return newError("Pointer must point to a struct.")
			}
		}
	}

	return nil
}

func validateBindingMemberSyntax(member string) error {
	// FIXME
	return nil
}
<|endoftext|>


<commit_before>package cldb

// Level DB cached wrapper to improve write performance using batching.

import (
	"firempq/common"
	"firempq/conf"
	"firempq/log"
	"sync"
	"time"

	. "firempq/api"

	"github.com/jmhodges/levigo"
)

// Default LevelDB read options.
var defaultReadOptions = levigo.NewReadOptions()

// Default LevelDB write options.
var defaultWriteOptions = levigo.NewWriteOptions()

// CLevelDBStorage A high level cached structure on top of LevelDB.
// It caches item storing them into database
// as multiple large batches later.
type CLevelDBStorage struct {
	db             *levigo.DB        // Pointer the the instance of level db.
	dbName         string            // LevelDB database name.
	itemCache      map[string]string // Active cache for item metadata.
	tmpItemCache   map[string]string // Active cache during flush operation.
	cacheLock      sync.Mutex        // Used for caches access.
	flushLock      sync.Mutex        // Used to prevent double flush.
	saveLock       sync.Mutex        // Used to prevent double flush.
	closed         bool
	flushSync      *sync.WaitGroup // Use to wait until flush happens.
	forceFlushChan chan bool
}

// NewLevelDBStorage is a constructor of DataStorage.
func NewLevelDBStorage(dbName string) (*CLevelDBStorage, error) {
	ds := CLevelDBStorage{
		dbName:         dbName,
		itemCache:      make(map[string]string),
		tmpItemCache:   nil,
		closed:         false,
		forceFlushChan: make(chan bool, 1),
		flushSync:      &sync.WaitGroup{},
	}

	// LevelDB write options.
	opts := levigo.NewOptions()
	opts.SetCreateIfMissing(true)
	opts.SetWriteBufferSize(10 * 1024 * 1024)
	opts.SetCompression(levigo.SnappyCompression)

	db, err := levigo.Open(dbName, opts)
	if err != nil {
		return nil, err
	}
	ds.db = db
	go ds.periodicCacheFlush()
	return &ds, nil
}

func (ds *CLevelDBStorage) GetStats() map[string]interface{} {
	return map[string]interface{}{
		"DBName":       ds.dbName,
		"CacheSize":    len(ds.itemCache),
		"TmpCacheSize": len(ds.tmpItemCache),
		"Close":        ds.closed,
	}
}

func (ds *CLevelDBStorage) periodicCacheFlush() {
	ds.flushSync.Add(1)
	for !ds.closed {
		select {
		case <-ds.forceFlushChan:
			break
		case <-time.After(conf.CFG.DbFlushInterval * time.Millisecond):
			break
		}
		ds.flushLock.Lock()
		oldFlushSync := ds.flushSync
		ds.flushSync = &sync.WaitGroup{}
		ds.flushSync.Add(1)
		if !ds.closed {
			ds.FlushCache()
		}
		oldFlushSync.Done()
		ds.flushLock.Unlock()
	}
	ds.flushSync.Done()
}

// WaitFlush waits until all data is flushed on disk.
func (ds *CLevelDBStorage) WaitFlush() {
	ds.flushLock.Lock()
	s := ds.flushSync
	ds.flushLock.Unlock()
	s.Wait()
}

// CachedStoreItem stores data into the cache.
func (ds *CLevelDBStorage) CachedStore(data ...string) {
	if len(data)%2 != 0 {
		panic("Number of arguments must be even!")
	}
	ds.cacheLock.Lock()
	for i := 0; i < len(data); i += 2 {
		ds.itemCache[data[i]] = data[i+1]
	}
	ds.cacheLock.Unlock()
}

// DeleteDataWithPrefix deletes all service data such as service metadata, items and payloads.
func (ds *CLevelDBStorage) DeleteDataWithPrefix(prefix string) int {
	ds.flushLock.Lock()
	defer ds.flushLock.Unlock()
	ds.FlushCache()

	limitCounter := 0
	total := 0
	iter := ds.IterData(prefix)
	wb := levigo.NewWriteBatch()

	for iter.Valid() {
		total++
		if limitCounter < 1000 {
			wb.Delete(iter.GetKey())
			limitCounter++
		} else {
			limitCounter = 0
			ds.db.Write(defaultWriteOptions, wb)
			wb.Close()
			wb = levigo.NewWriteBatch()
		}
		iter.Next()
	}
	wb.Close()

	ds.db.Write(defaultWriteOptions, wb)
	return total
}

// FlushCache flushes all cache into database.
func (ds *CLevelDBStorage) FlushCache() {
	ds.saveLock.Lock()
	ds.cacheLock.Lock()
	ds.tmpItemCache = ds.itemCache
	ds.itemCache = make(map[string]string)
	ds.cacheLock.Unlock()

	wb := levigo.NewWriteBatch()
	counter := 0
	for k, v := range ds.tmpItemCache {
		if counter >= 10000 {
			ds.db.Write(defaultWriteOptions, wb)
			wb.Clear()
		}
		key := common.UnsafeStringToBytes(k)
		if v == "" {
			wb.Delete(key)
		} else {
			wb.Put(key, common.UnsafeStringToBytes(v))
		}
		counter++
	}
	ds.db.Write(defaultWriteOptions, wb)
	wb.Close()
	ds.saveLock.Unlock()
}

// IterData returns an iterator over all data with prefix.
func (ds *CLevelDBStorage) IterData(prefix string) ItemIterator {
	iter := ds.db.NewIterator(defaultReadOptions)
	return makeItemIterator(iter, common.UnsafeStringToBytes(prefix))
}

// StoreData data directly into the database stores service metadata into database.
func (ds *CLevelDBStorage) StoreData(data ...string) error {
	if len(data)%2 != 0 {
		panic("Number of arguments must be even!")
	}
	wb := levigo.NewWriteBatch()
	defer wb.Close()
	for i := 0; i < len(data); i += 2 {
		wb.Put(common.UnsafeStringToBytes(data[i]),
			common.UnsafeStringToBytes(data[i+1]))
	}
	return ds.db.Write(defaultWriteOptions, wb)
}

func (ds *CLevelDBStorage) DeleteData(id ...string) {
	wb := levigo.NewWriteBatch()
	defer wb.Close()
	for _, i := range id {
		wb.Delete(common.UnsafeStringToBytes(i))
	}
	ds.db.Write(defaultWriteOptions, wb)
}

// GetData looks data looks for and item going through each layer of cache finally looking into database.
func (ds *CLevelDBStorage) GetData(id string) string {
	ds.cacheLock.Lock()
	data, ok := ds.itemCache[id]
	if ok {
		ds.cacheLock.Unlock()
		return data
	}
	data, ok = ds.tmpItemCache[id]
	if ok {
		ds.cacheLock.Unlock()
		return data
	}
	ds.cacheLock.Unlock()
	value, _ := ds.db.Get(defaultReadOptions, common.UnsafeStringToBytes(id))
	return common.UnsafeBytesToString(value)
}

// CachedDeleteData deletes item metadata and payload, affects cache only until flushed.
func (ds *CLevelDBStorage) CachedDeleteData(id ...string) {
	ds.cacheLock.Lock()
	for _, i := range id {
		ds.itemCache[i] = ""
	}
	ds.cacheLock.Unlock()
}

func (ds *CLevelDBStorage) IsClosed() bool {
	ds.flushLock.Lock()
	defer ds.flushLock.Unlock()
	return ds.closed
}

// Close flushes data on disk and closes database.
func (ds *CLevelDBStorage) Close() {
	ds.flushLock.Lock()
	defer ds.flushLock.Unlock()
	if !ds.closed {
		log.Info("Flushing database cache")
		ds.FlushCache()
		ds.closed = true
		log.Info("Closing the database")
		ds.db.Close()
		log.Info("The database has been closed.")
	} else {
		log.Error("Attempt to close database more than once!")
	}
}
<commit_msg>Fixed problem with closed writebatch.<commit_after>package cldb

// Level DB cached wrapper to improve write performance using batching.

import (
	"firempq/common"
	"firempq/conf"
	"firempq/log"
	"sync"
	"time"

	. "firempq/api"

	"github.com/jmhodges/levigo"
)

// Default LevelDB read options.
var defaultReadOptions = levigo.NewReadOptions()

// Default LevelDB write options.
var defaultWriteOptions = levigo.NewWriteOptions()

// CLevelDBStorage A high level cached structure on top of LevelDB.
// It caches item storing them into database
// as multiple large batches later.
type CLevelDBStorage struct {
	db             *levigo.DB        // Pointer the the instance of level db.
	dbName         string            // LevelDB database name.
	itemCache      map[string]string // Active cache for item metadata.
	tmpItemCache   map[string]string // Active cache during flush operation.
	cacheLock      sync.Mutex        // Used for caches access.
	flushLock      sync.Mutex        // Used to prevent double flush.
	saveLock       sync.Mutex        // Used to prevent double flush.
	closed         bool
	flushSync      *sync.WaitGroup // Use to wait until flush happens.
	forceFlushChan chan bool
}

// NewLevelDBStorage is a constructor of DataStorage.
func NewLevelDBStorage(dbName string) (*CLevelDBStorage, error) {
	ds := CLevelDBStorage{
		dbName:         dbName,
		itemCache:      make(map[string]string),
		tmpItemCache:   nil,
		closed:         false,
		forceFlushChan: make(chan bool, 1),
		flushSync:      &sync.WaitGroup{},
	}

	// LevelDB write options.
	opts := levigo.NewOptions()
	opts.SetCreateIfMissing(true)
	opts.SetWriteBufferSize(10 * 1024 * 1024)
	opts.SetCompression(levigo.SnappyCompression)

	db, err := levigo.Open(dbName, opts)
	if err != nil {
		return nil, err
	}
	ds.db = db
	go ds.periodicCacheFlush()
	return &ds, nil
}

func (ds *CLevelDBStorage) GetStats() map[string]interface{} {
	return map[string]interface{}{
		"DBName":       ds.dbName,
		"CacheSize":    len(ds.itemCache),
		"TmpCacheSize": len(ds.tmpItemCache),
		"Close":        ds.closed,
	}
}

func (ds *CLevelDBStorage) periodicCacheFlush() {
	ds.flushSync.Add(1)
	for !ds.closed {
		select {
		case <-ds.forceFlushChan:
			break
		case <-time.After(conf.CFG.DbFlushInterval * time.Millisecond):
			break
		}
		ds.flushLock.Lock()
		oldFlushSync := ds.flushSync
		ds.flushSync = &sync.WaitGroup{}
		ds.flushSync.Add(1)
		if !ds.closed {
			ds.FlushCache()
		}
		oldFlushSync.Done()
		ds.flushLock.Unlock()
	}
	ds.flushSync.Done()
}

// WaitFlush waits until all data is flushed on disk.
func (ds *CLevelDBStorage) WaitFlush() {
	ds.flushLock.Lock()
	s := ds.flushSync
	ds.flushLock.Unlock()
	s.Wait()
}

// CachedStoreItem stores data into the cache.
func (ds *CLevelDBStorage) CachedStore(data ...string) {
	if len(data)%2 != 0 {
		panic("Number of arguments must be even!")
	}
	ds.cacheLock.Lock()
	for i := 0; i < len(data); i += 2 {
		ds.itemCache[data[i]] = data[i+1]
	}
	ds.cacheLock.Unlock()
}

// DeleteDataWithPrefix deletes all service data such as service metadata, items and payloads.
func (ds *CLevelDBStorage) DeleteDataWithPrefix(prefix string) int {
	ds.flushLock.Lock()
	ds.FlushCache()

	limitCounter := 0
	total := 0
	iter := ds.IterData(prefix)
	wb := levigo.NewWriteBatch()

	for iter.Valid() {
		if limitCounter < 1000 {
			key := iter.GetKey()
			iter.Next()
			wb.Delete(key)
			limitCounter++
			total++
		} else {
			limitCounter = 0
			ds.db.Write(defaultWriteOptions, wb)
			wb.Clear()
			wb = levigo.NewWriteBatch()
		}

	}
	if limitCounter > 0 {
		ds.db.Write(defaultWriteOptions, wb)
	}
	wb.Close()
	ds.flushLock.Unlock()
	return total
}

// FlushCache flushes all cache into database.
func (ds *CLevelDBStorage) FlushCache() {
	ds.saveLock.Lock()
	ds.cacheLock.Lock()
	ds.tmpItemCache = ds.itemCache
	ds.itemCache = make(map[string]string)
	ds.cacheLock.Unlock()

	wb := levigo.NewWriteBatch()
	counter := 0
	for k, v := range ds.tmpItemCache {
		if counter >= 10000 {
			ds.db.Write(defaultWriteOptions, wb)
			wb.Clear()
		}
		key := common.UnsafeStringToBytes(k)
		if v == "" {
			wb.Delete(key)
		} else {
			wb.Put(key, common.UnsafeStringToBytes(v))
		}
		counter++
	}
	ds.db.Write(defaultWriteOptions, wb)
	wb.Close()
	ds.saveLock.Unlock()
}

// IterData returns an iterator over all data with prefix.
func (ds *CLevelDBStorage) IterData(prefix string) ItemIterator {
	iter := ds.db.NewIterator(defaultReadOptions)
	return makeItemIterator(iter, []byte(prefix))
}

// StoreData data directly into the database stores service metadata into database.
func (ds *CLevelDBStorage) StoreData(data ...string) error {
	if len(data)%2 != 0 {
		panic("Number of arguments must be even!")
	}
	wb := levigo.NewWriteBatch()
	for i := 0; i < len(data); i += 2 {
		wb.Put(common.UnsafeStringToBytes(data[i]),
			common.UnsafeStringToBytes(data[i+1]))
	}
	res := ds.db.Write(defaultWriteOptions, wb)
	wb.Close()
	return res
}

func (ds *CLevelDBStorage) DeleteData(id ...string) {
	wb := levigo.NewWriteBatch()
	for _, i := range id {
		wb.Delete(common.UnsafeStringToBytes(i))
	}
	ds.db.Write(defaultWriteOptions, wb)
	wb.Close()
}

// GetData looks data looks for and item going through each layer of cache finally looking into database.
func (ds *CLevelDBStorage) GetData(id string) string {
	ds.cacheLock.Lock()
	data, ok := ds.itemCache[id]
	if ok {
		ds.cacheLock.Unlock()
		return data
	}
	data, ok = ds.tmpItemCache[id]
	if ok {
		ds.cacheLock.Unlock()
		return data
	}
	ds.cacheLock.Unlock()
	value, _ := ds.db.Get(defaultReadOptions, common.UnsafeStringToBytes(id))
	return common.UnsafeBytesToString(value)
}

// CachedDeleteData deletes item metadata and payload, affects cache only until flushed.
func (ds *CLevelDBStorage) CachedDeleteData(id ...string) {
	ds.cacheLock.Lock()
	for _, i := range id {
		ds.itemCache[i] = ""
	}
	ds.cacheLock.Unlock()
}

func (ds *CLevelDBStorage) IsClosed() bool {
	ds.flushLock.Lock()
	cl := ds.closed
	ds.flushLock.Unlock()
	return cl
}

// Close flushes data on disk and closes database.
func (ds *CLevelDBStorage) Close() {
	ds.flushLock.Lock()
	if !ds.closed {
		log.Info("Flushing database cache")
		ds.FlushCache()
		ds.closed = true
		log.Info("Closing the database")
		ds.db.Close()
		log.Info("The database has been closed.")
	} else {
		log.Error("Attempt to close database more than once!")
	}
	ds.flushLock.Unlock()
}
<|endoftext|>


<commit_before>package engi

import (
	"log"
	"os"
	"time"
	"unsafe"

	"azul3d.org/audio.v1"
	"azul3d.org/native/al.v1"
	"golang.org/x/net/context"

	_ "azul3d.org/audio/wav.v1"
	_ "github.com/guregu/audio-flac"
)

var (
	audioDevice  *al.Device
	audioSources []uint32
	audioContext context.Context
	audioCancel  func()
	muted        bool
	gain         float32
)

const (
	audioSourcesCount = 31
	buffersPerStream  = 3
)

type Sound interface {
	Play()
	Loop()
	Stop()
	Playing() bool
}

func ToggleMute() {
	muted = !muted
	g := gain
	if muted {
		g = 0
	}

	for _, src := range audioSources {
		audioDevice.Sourcef(src, al.GAIN, g)
	}
}

func Muted() bool {
	return muted
}

func SetGain(g float32) {
	if g != gain {
		gain = g

		if muted {
			return
		}

		for _, src := range audioSources {
			audioDevice.Sourcef(src, al.GAIN, g)
		}
	}
}

type Music struct {
	source  uint32
	buffers []uint32
	looping bool
	context context.Context
	cancel  func()

	decoder audio.Decoder
	file    *os.File
}

func (s *Music) Play() {
	s.looping = false
	s.play()
}

func (s *Music) Loop() {
	s.looping = true
	s.play()
}

func (s *Music) play() {
	s.context, s.cancel = context.WithCancel(audioContext)
	s.source = nextAvailableSource()
	s.reset()
	audioDevice.Sourcei(s.source, al.LOOPING, al.FALSE)
	// fill all buffers first
	for _, buf := range s.buffers {
		s.fill(buf)
	}

	audioDevice.SourceQueueBuffers(s.source, s.buffers)
	go s.run()

	audioDevice.SourcePlay(s.source)
}

func (s *Music) reset() {
	// flac pkg can't seek yet so ghetto seek to 0
	s.file.Seek(0, os.SEEK_SET)
	var err error
	s.decoder, _, err = audio.NewDecoder(s.file)
	if err != nil {
		panic(err)
	}
}

func (s *Music) run() {
	for {
		select {
		case <-s.context.Done():
			return
		default:
			processed := s.unqueue()
			if len(processed) == 0 {
				continue
			}

			for _, buf := range processed {
				err := s.fill(buf)
				switch {
				case err == audio.EOS:
					if s.looping {
						// time.Sleep(500 * time.Millisecond)
						// start over
						s.reset()
					}
				case err != nil:
					panic(err)
				}
				audioDevice.SourceQueueBuffers(s.source, []uint32{buf})
				if err == audio.EOS && !s.looping {
					return
				}
			}
		}
	}
}

func (s *Music) unqueue() []uint32 {
	var processed int32

	audioDevice.GetSourcei(s.source, al.BUFFERS_PROCESSED, &processed)
	if processed == 0 {
		return nil
	}
	available := make([]uint32, processed)
	audioDevice.SourceUnqueueBuffers(s.source, available)
	return available
}

func (s *Music) fill(buffer uint32) error {
	config := s.decoder.Config()
	bufSize := config.SampleRate
	samples := make(audio.PCM16Samples, bufSize)

	read, err := s.decoder.Read(samples)
	if err != nil && err != audio.EOS {
		return err
	}

	if read > 0 {
		data := []audio.PCM16(samples[:read])
		if config.Channels == 1 {
			audioDevice.BufferData(buffer, al.FORMAT_MONO16, unsafe.Pointer(&data[0]), int32(int(unsafe.Sizeof(data[0]))*read), int32(config.SampleRate))
		} else {
			audioDevice.BufferData(buffer, al.FORMAT_STEREO16, unsafe.Pointer(&data[0]), int32(int(unsafe.Sizeof(data[0]))*read), int32(config.SampleRate))
		}
	}

	return err
}

func (s *Music) Delete() {
	if s.cancel != nil {
		s.cancel()
	}
	s.file.Close()
	audioDevice.DeleteBuffers(buffersPerStream, &s.buffers[0])
}

func (s *Music) Playing() bool {
	if s.source == 0 {
		return false
	}
	var state int32
	audioDevice.GetSourcei(s.source, al.SOURCE_STATE, &state)
	return state == al.PLAYING
}

func (s *Music) Stop() {
	if s.cancel != nil {
		s.cancel()
	}
	audioDevice.SourceStop(s.source)
	s.unqueue()
	s.source = 0
}

func loadMusic(r Resource) (*Music, error) {
	// func readSoundFile(filename string) (samples []audio.PCM16, config audio.Config, duration time.Duration, err error) {
	file, err := os.Open(r.url)
	if err != nil {
		return nil, err
	}

	s := Music{
		buffers: make([]uint32, buffersPerStream),
		file:    file,
	}

	audioDevice.GenBuffers(buffersPerStream, &s.buffers[0])

	return &s, nil
}

type SFX struct {
	source     uint32
	buffer     uint32
	duration   time.Duration // seconds
	sampleRate int
	looping    bool
}

func (s *SFX) bind() {
	s.source = nextAvailableSource()
	audioDevice.Sourcei(s.source, al.BUFFER, int32(s.buffer))
	if s.looping {
		audioDevice.Sourcei(s.source, al.LOOPING, al.TRUE)
	} else {
		audioDevice.Sourcei(s.source, al.LOOPING, al.FALSE)
	}
}

func (s *SFX) Play() {
	s.looping = false
	s.bind()
	audioDevice.SourcePlay(s.source)
}

func (s *SFX) Loop() {
	s.looping = true
	s.bind()
	audioDevice.SourcePlay(s.source)
}

func (s *SFX) Stop() {
	audioDevice.SourceStop(s.source)
	s.unqueue()
}

func (s *SFX) unqueue() []uint32 {
	var processed int32
	audioDevice.GetSourcei(s.source, al.BUFFERS_PROCESSED, &processed)
	if processed == 0 {
		return nil
	}
	available := make([]uint32, processed)
	audioDevice.SourceUnqueueBuffers(s.source, available)
	return available
}

func (s *SFX) Delete() {
	audioDevice.DeleteBuffers(1, &s.buffer)
}

func (s *SFX) Playing() bool {
	if s.source == 0 {
		return false
	}

	var state int32
	audioDevice.GetSourcei(s.source, al.SOURCE_STATE, &state)
	return state == al.PLAYING
}

func (s *SFX) Duration() time.Duration {
	return s.duration
}

func setupAudio() {
	var err error
	audioDevice, err = al.OpenDevice("", nil)
	fatalErr(err)

	audioSources = make([]uint32, audioSourcesCount)
	audioDevice.GenSources(audioSourcesCount, &audioSources[0])

	audioContext, audioCancel = context.WithCancel(context.Background())
}

func nextAvailableSource() uint32 {
	// find unused source
	for _, source := range audioSources {
		var state int32
		audioDevice.GetSourcei(source, al.SOURCE_STATE, &state)
		if state != al.PLAYING {
			audioDevice.Sourcei(source, al.BUFFER, 0)
			return source
		}
	}

	// no free sounds. find non-looping one and cut it short
	for _, source := range audioSources {
		var looping int32
		audioDevice.GetSourcei(source, al.LOOPING, &looping)
		if looping != al.TRUE {
			audioDevice.SourceStop(source)
			return source
		}
	}

	// give up, take the last one
	source := audioSources[len(audioSources)-1]
	audioDevice.SourceStop(source)
	return source
}

func loadSFX(r Resource) (*SFX, error) {
	samples, config, duration, err := readSFXFile(r.url)
	if err != nil {
		return nil, err
	}

	s := SFX{
		duration:   duration,
		sampleRate: config.SampleRate,
	}
	audioDevice.GenBuffers(1, &s.buffer)
	if config.Channels == 1 {
		audioDevice.BufferData(s.buffer, al.FORMAT_MONO16, unsafe.Pointer(&samples[0]), int32(int(unsafe.Sizeof(samples[0]))*len(samples)), int32(config.SampleRate))
	} else {
		audioDevice.BufferData(s.buffer, al.FORMAT_STEREO16, unsafe.Pointer(&samples[0]), int32(int(unsafe.Sizeof(samples[0]))*len(samples)), int32(config.SampleRate))
	}
	return &s, err
}

func readSFXFile(filename string) (samples []audio.PCM16, config audio.Config, duration time.Duration, err error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, audio.Config{}, 0, err
	}
	defer file.Close()
	fi, err := file.Stat()
	if err != nil {
		return nil, audio.Config{}, 0, err
	}

	decoder, _, err := audio.NewDecoder(file)
	if err != nil {
		return nil, audio.Config{}, 0, err
	}

	config = decoder.Config()

	// Convert everything to 16-bit samples
	bufSize := int(fi.Size())
	samples = make(audio.PCM16Samples, 0, bufSize)

	// TODO: surely there is a better way to do this
	var read int
	buf := make(audio.PCM16Samples, 1024*1024)
	err = nil
	for err != audio.EOS {
		var r int
		r, err = decoder.Read(buf)
		if err != nil && err != audio.EOS {
			return nil, audio.Config{}, 0, err
		}
		read += r
		samples = append(samples, buf[:r]...)
	}

	secs := 1 / float64(config.SampleRate) * float64(read)
	duration = time.Duration(float64(time.Second) * secs)
	return []audio.PCM16(samples)[:read], config, duration, nil
}

func cleanupAudio() {
	audioCancel()
	audioDevice.DeleteSources(int32(len(audioSources)), &audioSources[0])
	for _, s := range Files.sfx {
		s.Delete()
	}
	for _, s := range Files.music {
		s.Delete()
	}
	if audioDevice != nil {
		audioDevice.Close()
	}
}

func init() {
	al.SetErrorHandler(func(err error) {
		log.Println("[audio]", err)
	})
}

var _ Sound = (*Music)(nil)
var _ Sound = (*SFX)(nil)
<commit_msg>use my native-al<commit_after>package engi

import (
	"log"
	"os"
	"time"
	"unsafe"

	"azul3d.org/audio.v1"
	"github.com/guregu/native-al"
	"golang.org/x/net/context"

	_ "azul3d.org/audio/wav.v1"
	_ "github.com/guregu/audio-flac"
)

var (
	audioDevice  *al.Device
	audioSources []uint32
	audioContext context.Context
	audioCancel  func()
	muted        bool
	gain         float32
)

const (
	audioSourcesCount = 31
	buffersPerStream  = 3
)

type Sound interface {
	Play()
	Loop()
	Stop()
	Playing() bool
}

func ToggleMute() {
	muted = !muted
	g := gain
	if muted {
		g = 0
	}

	for _, src := range audioSources {
		audioDevice.Sourcef(src, al.GAIN, g)
	}
}

func Muted() bool {
	return muted
}

func SetGain(g float32) {
	if g != gain {
		gain = g

		if muted {
			return
		}

		for _, src := range audioSources {
			audioDevice.Sourcef(src, al.GAIN, g)
		}
	}
}

type Music struct {
	source  uint32
	buffers []uint32
	looping bool
	context context.Context
	cancel  func()

	decoder audio.Decoder
	file    *os.File
}

func (s *Music) Play() {
	s.looping = false
	s.play()
}

func (s *Music) Loop() {
	s.looping = true
	s.play()
}

func (s *Music) play() {
	s.context, s.cancel = context.WithCancel(audioContext)
	s.source = nextAvailableSource()
	s.reset()
	audioDevice.Sourcei(s.source, al.LOOPING, al.FALSE)
	// fill all buffers first
	for _, buf := range s.buffers {
		s.fill(buf)
	}

	audioDevice.SourceQueueBuffers(s.source, s.buffers)
	go s.run()

	audioDevice.SourcePlay(s.source)
}

func (s *Music) reset() {
	// flac pkg can't seek yet so ghetto seek to 0
	s.file.Seek(0, os.SEEK_SET)
	var err error
	s.decoder, _, err = audio.NewDecoder(s.file)
	if err != nil {
		panic(err)
	}
}

func (s *Music) run() {
	for {
		select {
		case <-s.context.Done():
			return
		default:
			processed := s.unqueue()
			if len(processed) == 0 {
				continue
			}

			for _, buf := range processed {
				err := s.fill(buf)
				switch {
				case err == audio.EOS:
					if s.looping {
						// time.Sleep(500 * time.Millisecond)
						// start over
						s.reset()
					}
				case err != nil:
					panic(err)
				}
				audioDevice.SourceQueueBuffers(s.source, []uint32{buf})
				if err == audio.EOS && !s.looping {
					return
				}
			}
		}
	}
}

func (s *Music) unqueue() []uint32 {
	var processed int32

	audioDevice.GetSourcei(s.source, al.BUFFERS_PROCESSED, &processed)
	if processed == 0 {
		return nil
	}
	available := make([]uint32, processed)
	audioDevice.SourceUnqueueBuffers(s.source, available)
	return available
}

func (s *Music) fill(buffer uint32) error {
	config := s.decoder.Config()
	bufSize := config.SampleRate
	samples := make(audio.PCM16Samples, bufSize)

	read, err := s.decoder.Read(samples)
	if err != nil && err != audio.EOS {
		return err
	}

	if read > 0 {
		data := []audio.PCM16(samples[:read])
		if config.Channels == 1 {
			audioDevice.BufferData(buffer, al.FORMAT_MONO16, unsafe.Pointer(&data[0]), int32(int(unsafe.Sizeof(data[0]))*read), int32(config.SampleRate))
		} else {
			audioDevice.BufferData(buffer, al.FORMAT_STEREO16, unsafe.Pointer(&data[0]), int32(int(unsafe.Sizeof(data[0]))*read), int32(config.SampleRate))
		}
	}

	return err
}

func (s *Music) Delete() {
	if s.cancel != nil {
		s.cancel()
	}
	s.file.Close()
	audioDevice.DeleteBuffers(buffersPerStream, &s.buffers[0])
}

func (s *Music) Playing() bool {
	if s.source == 0 {
		return false
	}
	var state int32
	audioDevice.GetSourcei(s.source, al.SOURCE_STATE, &state)
	return state == al.PLAYING
}

func (s *Music) Stop() {
	if s.cancel != nil {
		s.cancel()
	}
	audioDevice.SourceStop(s.source)
	s.unqueue()
	s.source = 0
}

func loadMusic(r Resource) (*Music, error) {
	// func readSoundFile(filename string) (samples []audio.PCM16, config audio.Config, duration time.Duration, err error) {
	file, err := os.Open(r.url)
	if err != nil {
		return nil, err
	}

	s := Music{
		buffers: make([]uint32, buffersPerStream),
		file:    file,
	}

	audioDevice.GenBuffers(buffersPerStream, &s.buffers[0])

	return &s, nil
}

type SFX struct {
	source     uint32
	buffer     uint32
	duration   time.Duration // seconds
	sampleRate int
	looping    bool
}

func (s *SFX) bind() {
	s.source = nextAvailableSource()
	audioDevice.Sourcei(s.source, al.BUFFER, int32(s.buffer))
	if s.looping {
		audioDevice.Sourcei(s.source, al.LOOPING, al.TRUE)
	} else {
		audioDevice.Sourcei(s.source, al.LOOPING, al.FALSE)
	}
}

func (s *SFX) Play() {
	s.looping = false
	s.bind()
	audioDevice.SourcePlay(s.source)
}

func (s *SFX) Loop() {
	s.looping = true
	s.bind()
	audioDevice.SourcePlay(s.source)
}

func (s *SFX) Stop() {
	audioDevice.SourceStop(s.source)
	s.unqueue()
}

func (s *SFX) unqueue() []uint32 {
	var processed int32
	audioDevice.GetSourcei(s.source, al.BUFFERS_PROCESSED, &processed)
	if processed == 0 {
		return nil
	}
	available := make([]uint32, processed)
	audioDevice.SourceUnqueueBuffers(s.source, available)
	return available
}

func (s *SFX) Delete() {
	audioDevice.DeleteBuffers(1, &s.buffer)
}

func (s *SFX) Playing() bool {
	if s.source == 0 {
		return false
	}

	var state int32
	audioDevice.GetSourcei(s.source, al.SOURCE_STATE, &state)
	return state == al.PLAYING
}

func (s *SFX) Duration() time.Duration {
	return s.duration
}

func setupAudio() {
	var err error
	audioDevice, err = al.OpenDevice("", nil)
	fatalErr(err)

	audioSources = make([]uint32, audioSourcesCount)
	audioDevice.GenSources(audioSourcesCount, &audioSources[0])

	audioContext, audioCancel = context.WithCancel(context.Background())
}

func nextAvailableSource() uint32 {
	// find unused source
	for _, source := range audioSources {
		var state int32
		audioDevice.GetSourcei(source, al.SOURCE_STATE, &state)
		if state != al.PLAYING {
			audioDevice.Sourcei(source, al.BUFFER, 0)
			return source
		}
	}

	// no free sounds. find non-looping one and cut it short
	for _, source := range audioSources {
		var looping int32
		audioDevice.GetSourcei(source, al.LOOPING, &looping)
		if looping != al.TRUE {
			audioDevice.SourceStop(source)
			return source
		}
	}

	// give up, take the last one
	source := audioSources[len(audioSources)-1]
	audioDevice.SourceStop(source)
	return source
}

func loadSFX(r Resource) (*SFX, error) {
	samples, config, duration, err := readSFXFile(r.url)
	if err != nil {
		return nil, err
	}

	s := SFX{
		duration:   duration,
		sampleRate: config.SampleRate,
	}
	audioDevice.GenBuffers(1, &s.buffer)
	if config.Channels == 1 {
		audioDevice.BufferData(s.buffer, al.FORMAT_MONO16, unsafe.Pointer(&samples[0]), int32(int(unsafe.Sizeof(samples[0]))*len(samples)), int32(config.SampleRate))
	} else {
		audioDevice.BufferData(s.buffer, al.FORMAT_STEREO16, unsafe.Pointer(&samples[0]), int32(int(unsafe.Sizeof(samples[0]))*len(samples)), int32(config.SampleRate))
	}
	return &s, err
}

func readSFXFile(filename string) (samples []audio.PCM16, config audio.Config, duration time.Duration, err error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, audio.Config{}, 0, err
	}
	defer file.Close()
	fi, err := file.Stat()
	if err != nil {
		return nil, audio.Config{}, 0, err
	}

	decoder, _, err := audio.NewDecoder(file)
	if err != nil {
		return nil, audio.Config{}, 0, err
	}

	config = decoder.Config()

	// Convert everything to 16-bit samples
	bufSize := int(fi.Size())
	samples = make(audio.PCM16Samples, 0, bufSize)

	// TODO: surely there is a better way to do this
	var read int
	buf := make(audio.PCM16Samples, 1024*1024)
	err = nil
	for err != audio.EOS {
		var r int
		r, err = decoder.Read(buf)
		if err != nil && err != audio.EOS {
			return nil, audio.Config{}, 0, err
		}
		read += r
		samples = append(samples, buf[:r]...)
	}

	secs := 1 / float64(config.SampleRate) * float64(read)
	duration = time.Duration(float64(time.Second) * secs)
	return []audio.PCM16(samples)[:read], config, duration, nil
}

func cleanupAudio() {
	audioCancel()
	audioDevice.DeleteSources(int32(len(audioSources)), &audioSources[0])
	for _, s := range Files.sfx {
		s.Delete()
	}
	for _, s := range Files.music {
		s.Delete()
	}
	if audioDevice != nil {
		audioDevice.Close()
	}
}

func init() {
	al.SetErrorHandler(func(err error) {
		log.Println("[audio]", err)
	})
}

var _ Sound = (*Music)(nil)
var _ Sound = (*SFX)(nil)
<|endoftext|>


<commit_before>package cf

import (
	"encoding/json"
	"fmt"
	"net/http"
)

type spaceMetadata struct {
	Guid string `json:"guid"`
}

type spaceEntity struct {
	Name         string       `json:"name"`
	Organization Organization `json:"organization"`
}

type Space struct {
	spaceMetadata `json:"metadata"`
	spaceEntity   `json:"entity"`
}

func (target *Target) SpacesGet(orgGUID string) (spaces []Space, err error) {
	url := fmt.Sprintf("%s/v2/organizations/%s/spaces", target.TargetUrl, orgGUID)
	req, _ := http.NewRequest("GET", url, nil)
	resp, err := target.sendRequest(req)
	if err != nil {
		return nil, err
	}
	decoder := json.NewDecoder(resp.Body)

	var res struct {
		Spaces []Space `json:"resources"`
	}
	err = decoder.Decode(&res)
	spaces = res.Spaces
	return
}
<commit_msg>get all spaces, not just org. specific<commit_after>package cf

import (
	"encoding/json"
	"fmt"
	"net/http"
)

type spaceMetadata struct {
	Guid string `json:"guid"`
}

type spaceEntity struct {
	Name         string       `json:"name"`
	Organization Organization `json:"organization"`
}

type Space struct {
	spaceMetadata `json:"metadata"`
	spaceEntity   `json:"entity"`
}

func (target *Target) SpacesGet(orgGUID string) (spaces []Space, err error) {
	url := fmt.Sprintf("%s/v2/spaces?inline-relations-depth=1", target.TargetUrl)
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return
	}
	resp, err := target.sendRequest(req)
	if err != nil {
		return
	}
	decoder := json.NewDecoder(resp.Body)

	var res struct {
		Spaces []Space `json:"resources"`
	}
	err = decoder.Decode(&res)
	spaces = res.Spaces
	return
}
<|endoftext|>


<commit_before>package fastq

import (
	"bytes"
	"fmt"
	"gongs/scan"
	"gongs/xopen"
	"io"
	"strings"
)

type Fastq struct {
	Name string
	Seq  []byte
	Qual []byte
}

func (fq Fastq) String() string {
	return fmt.Sprintf("@%v\n%v\n+\n%v", fq.Name, string(fq.Seq), string(fq.Qual))
}

func (fq Fastq) Id() string {
	if n := strings.IndexByte(fq.Name, ' '); n >= 0 {
		return fq.Name[:n]
	}

	// for old solexa data format
	if n := strings.IndexByte(fq.Name, '#'); n >= 0 {
		return fq.Name[:n]
	}

	return fq.Name
}

type FastqFile struct {
	Name  string
	file  io.ReadCloser
	s     *scan.Scanner
	name  []byte
	seq   []byte
	qual  []byte
	err   error
	stage int
}

func Open(filename string) (*FastqFile, error) {
	file, err := xopen.Xopen(filename)
	if err != nil {
		return nil, err
	}

	return &FastqFile{
		Name: filename,
		s:    scan.New(file),
		file: file,
	}, nil
}

func (ff *FastqFile) Close() error {
	return ff.file.Close()
}

func (ff *FastqFile) Err() error {
	if ff.err == nil || ff.err == io.EOF {
		if err := ff.s.Err(); err != nil {
			return err
		}
		return nil
	}
	return ff.err
}

func (ff *FastqFile) setErr(err error) {
	if ff.err == nil {
		ff.err = err
	}
}

func (ff *FastqFile) Next() bool {
	if ff.err != nil {
		return false
	}

	var line []byte
	for ff.s.Scan() {
		line = bytes.TrimSpace(ff.s.Bytes())
		if len(line) == 0 { // ingore empty line
			continue
		}
		switch ff.stage {
		case 0: // get fastq name
			if len(line) > 0 && line[0] != '@' {
				ff.setErr(fmt.Errorf("file: %v Wrong Fastq Record Name %s at line: %d", ff.Name, string(line), ff.s.Lid()))
				return false
			}
			ff.stage++
			ff.name = line[1:]
			ff.seq = ff.seq[:0]   // clear seq
			ff.qual = ff.qual[:0] // clear qual
		case 1: // get fastq seq
			if len(line) > 0 && line[0] == '+' {
				ff.stage += 2
				break
			}
			ff.seq = append(ff.seq, line...)
		case 2: // get + line
		case 3: // get fastq qual
			ff.qual = append(ff.qual, line...)
			if len(ff.qual) == len(ff.seq) {
				ff.stage = 0
				return true
			} else if len(ff.qual) > len(ff.seq) {
				ff.setErr(fmt.Errorf("file: %v Fastq Record (%s) qual length (%d) != seq length (%d) at line: %d",
					ff.Name, string(ff.name), len(ff.qual), len(ff.seq), ff.s.Lid()))
				return false
			}
		}
	}
	if len(ff.qual) < len(ff.seq) {
		ff.setErr(fmt.Errorf("file: %v Fastq Record (%s) qual length (%d) != seq length (%d) at line: %d",
			ff.Name, string(ff.name), len(ff.qual), len(ff.seq), ff.s.Lid()))
	}
	ff.setErr(io.EOF)
	return false
}

func (ff *FastqFile) Value() *Fastq {
	return &Fastq{Name: string(ff.name), Seq: ff.seq, Qual: ff.qual}
}

func (ff *FastqFile) Iter() <-chan *Fastq {
	ch := make(chan *Fastq)
	go func(ch chan *Fastq) {
		for ff.Next() {
			ch <- ff.Value()
		}
		close(ch)
	}(ch)
	return ch
}

func Opens(filenames ...string) ([]*FastqFile, error) {
	fqfiles := make([]*FastqFile, len(filenames))
	for i, filename := range filenames {
		fqfile, err := Open(filename)
		if err != nil {
			return nil, err
		}
		fqfiles[i] = fqfile
	}
	return fqfiles, nil
}
<commit_msg>add Load and LoadMix function<commit_after>package fastq

import (
	"bytes"
	"fmt"
	"gongs/scan"
	"gongs/xopen"
	"io"
	"os"
	"strings"
	"sync"
)

type Fastq struct {
	Name string
	Seq  []byte
	Qual []byte
}

func (fq Fastq) String() string {
	return fmt.Sprintf("@%v\n%v\n+\n%v", fq.Name, string(fq.Seq), string(fq.Qual))
}

func (fq Fastq) Id() string {
	if n := strings.IndexByte(fq.Name, ' '); n >= 0 {
		return fq.Name[:n]
	}

	// for old solexa data format
	if n := strings.IndexByte(fq.Name, '#'); n >= 0 {
		return fq.Name[:n]
	}

	return fq.Name
}

type FastqFile struct {
	Name  string
	file  io.ReadCloser
	s     *scan.Scanner
	name  []byte
	seq   []byte
	qual  []byte
	err   error
	stage int
}

func Open(filename string) (*FastqFile, error) {
	file, err := xopen.Xopen(filename)
	if err != nil {
		return nil, err
	}

	return &FastqFile{
		Name: filename,
		s:    scan.New(file),
		file: file,
	}, nil
}

func (ff *FastqFile) Close() error {
	return ff.file.Close()
}

func (ff *FastqFile) Err() error {
	if ff.err == nil || ff.err == io.EOF {
		if err := ff.s.Err(); err != nil {
			return err
		}
		return nil
	}
	return ff.err
}

func (ff *FastqFile) setErr(err error) {
	if ff.err == nil {
		ff.err = err
	}
}

func (ff *FastqFile) Next() bool {
	if ff.err != nil {
		return false
	}

	var line []byte
	for ff.s.Scan() {
		line = bytes.TrimSpace(ff.s.Bytes())
		if len(line) == 0 { // ingore empty line
			continue
		}
		switch ff.stage {
		case 0: // get fastq name
			if len(line) > 0 && line[0] != '@' {
				ff.setErr(fmt.Errorf("file: %v Wrong Fastq Record Name %s at line: %d", ff.Name, string(line), ff.s.Lid()))
				return false
			}
			ff.stage++
			ff.name = line[1:]
			ff.seq = ff.seq[:0]   // clear seq
			ff.qual = ff.qual[:0] // clear qual
		case 1: // get fastq seq
			if len(line) > 0 && line[0] == '+' {
				ff.stage += 2
				break
			}
			ff.seq = append(ff.seq, line...)
		case 2: // get + line
		case 3: // get fastq qual
			ff.qual = append(ff.qual, line...)
			if len(ff.qual) == len(ff.seq) {
				ff.stage = 0
				return true
			} else if len(ff.qual) > len(ff.seq) {
				ff.setErr(fmt.Errorf("file: %v Fastq Record (%s) qual length (%d) != seq length (%d) at line: %d",
					ff.Name, string(ff.name), len(ff.qual), len(ff.seq), ff.s.Lid()))
				return false
			}
		}
	}
	if len(ff.qual) < len(ff.seq) {
		ff.setErr(fmt.Errorf("file: %v Fastq Record (%s) qual length (%d) != seq length (%d) at line: %d",
			ff.Name, string(ff.name), len(ff.qual), len(ff.seq), ff.s.Lid()))
	}
	ff.setErr(io.EOF)
	return false
}

func (ff *FastqFile) Value() *Fastq {
	return &Fastq{Name: string(ff.name), Seq: ff.seq, Qual: ff.qual}
}

func (ff *FastqFile) Iter() <-chan *Fastq {
	ch := make(chan *Fastq)
	go func(ch chan *Fastq) {
		for ff.Next() {
			ch <- ff.Value()
		}
		close(ch)
	}(ch)
	return ch
}

func Opens(filenames ...string) ([]*FastqFile, error) {
	fqfiles := make([]*FastqFile, len(filenames))
	for i, filename := range filenames {
		fqfile, err := Open(filename)
		if err != nil {
			return nil, err
		}
		fqfiles[i] = fqfile
	}
	return fqfiles, nil
}

func Load(filenames ...string) (<-chan *Fastq, <-chan error) {
	fqChan := make(chan *Fastq, 2*len(fqfiles))
	errChan := make(chan error, 1)

	fqfiles, err := Opens(filenames...)
	if err != nil {
		errChan <- err
		return nil, errChan
	}

	go func(fqChan chan *Fastq, errChan chan error, fqfiles []*FastqFile) {
		for _, fqfile := range fqfiles {
			defer fqfile.Close()
			for fqfile.Next() {
				fqChan <- fqfile.Value()
			}
			if err := fqfile.Err(); err != nil {
				errChan <- err
			}
		}
		close(fqChan)
	}(wg, fqChan, errChan, fqfiles)
	return fqChan, errChan
}

func LoadMix(filenames ...string) (<-chan *Fastq, <-chan error) {
	fqChan := make(chan *Fastq, 2*len(fqfiles))
	errChan := make(chan error, 1)

	fqfiles, err := Opens(filenames...)
	if err != nil {
		errChan <- err
		return nil, errChan
	}

	wg := &sync.WaitGroup{}
	go func(wg *sync.WaitGroup, fqChan chan *Fastq, errChan chan error, fqfiles []*FastqFile) {
		for _, fqfile := range fqfiles {
			wg.Add(1)
			go func(wg *sync.WaitGroup, fqChan chan *Fastq, errChan chan error, fqfile *FastqFile) {
				defer fqfile.Close()
				for fqfile.Next() {
					fmt.Fprintln(os.Stderr, fqfile.Value())
					fqChan <- fqfile.Value()
				}
				if err := fqfile.Err(); err != nil {
					errChan <- err
				}
				wg.Done()
			}(wg, fqChan, errChan, fqfile)
		}
		wg.Wait()
		close(fqChan)
	}(wg, fqChan, errChan, fqfiles)
	return fqChan, errChan
}
<|endoftext|>


<commit_before>package stealer

import "errors"

type Stealer struct {
	Datas map[string][]string
}

// Steal some variable data from path phpfile
func Steal(path string) (error, *Stealer) {
	var stealer Stealer
	var err error
	err, stealer.Datas = ReadFile(path)
	return err, &stealer
}

// save all the variables and its values to new path
// TODO : write test case where the path is not exist and need to create the path first
// lets say path/to/specific/file.go need to create the folder first
func (s *Stealer) Save(path, packageName string) error {
	var err error
	if s.Datas == nil {
		err = errors.New("Stealer Datas are empty")
		return err
	}
	err = WriteFile(path, packageName, s.Datas)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>fix variable name<commit_after>package stealer

import "errors"

type Stealer struct {
	Datas map[string][]string
}

// Steal some variable data from path phpfile
func Steal(path string) (error, *Stealer) {
	var stealer Stealer
	var err error
	err, stealer.Datas = ReadFile(path)
	return err, &stealer
}

// save all the variables and its values to new path
// TODO : write test case where the path is not exist and need to create the path first
// lets say path/to/specific/file.go need to create the folder first
func (s *Stealer) Save(savePath, packageName string) error {
	var err error
	if s.Datas == nil {
		err = errors.New("Stealer Datas are empty")
		return err
	}
	err = WriteFile(savePath, packageName, s.Datas)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
)

const BUFSIZE = 8192

// TODO make chunking user-configurable, default to 32MB
// chunk limit should be a multiple of BUFSIZE for max efficiency
const CHUNKLIMIT = BUFSIZE * 4086

// Information about a LOB
type LOBInfo struct {
	// SHA of the LOB
	SHA string
	// Total size of the LOB (all chunks)
	Size int64
	// Number of chunks that make up the whole LOB (integrity check)
	NumChunks int
}

// Gets the root folder of this git repository (the one containing .git)
func GetRepoRoot() (path string, isSeparateGitDir bool) {
	// We could call 'git rev-parse --git-dir' but this requires shelling out = slow, especially on Windows
	// We should try to avoid that whenever we can
	// So let's just find it ourselves; first containing folder with a .git folder/file
	curDir, err := os.Getwd()
	if err != nil {
		LogErrorf("Getwd failed: %v\n", err)
		return "", false
	}
	for {
		exists, isDir := FileOrDirExists(filepath.Join(curDir, ".git"))
		if exists {
			return curDir, !isDir
		}
		curDir = filepath.Dir(curDir)
		if curDir == string(filepath.Separator) || curDir == "." {
			// Not a repo
			LogError("Couldn't find repo root, not a git folder")
			return "", false
		}
	}
}

// Gets the git data dir of git repository (the .git dir, or where .git file points)
func GetGitDir() string {
	root, isSeparate := GetRepoRoot()
	git := filepath.Join(root, ".git")
	if isSeparate {
		// Git repo folder is separate, read location from file
		filebytes, err := ioutil.ReadFile(git)
		if err != nil {
			LogErrorf("Can't read .git file %v: %v\n", git, err)
			return ""
		}
		filestr := string(filebytes)
		match := regexp.MustCompile("gitdir:[\\s]+([^\\r\\n]+)").FindStringSubmatch(filestr)
		if match == nil {
			LogErrorf("Unexpected contents of .git file %v: %v\n", git, filestr)
			return ""
		}
		return match[1]
	} else {
		// Regular git dir
		return git
	}

}

// Gets the root directory for LOB files & creates if necessary
func GetLOBRoot() string {
	ret := filepath.Join(GetGitDir(), "git-lob")
	err := os.MkdirAll(ret, 0777)
	if err != nil {
		LogErrorf("Unable to create LOB root folder at %v: %v", ret, err)
		panic(err)
	}
	return ret
}

// Gets the containing folder for a given LOB SHA & creates if necessary
// LOBs are 'splayed' based on first 2 chars of SHA
func GetLOBDir(sha string) string {
	if len(sha) != 40 {
		LogErrorf("Invalid SHA format: %v\n", sha)
		return ""
	}
	return filepath.Join(GetLOBRoot(), sha[:2])
}

func getLOBMetaFilename(sha string) string {
	fld := GetLOBDir(sha)
	return filepath.Join(fld, sha+"_meta")
}
func getLOBChunkFilename(sha string, chunkIdx int) string {
	fld := GetLOBDir(sha)
	return filepath.Join(fld, fmt.Sprintf("%v_%d", sha, chunkIdx))
}

// Retrieve information about an existing stored LOB
func GetLOBInfo(sha string) (*LOBInfo, error) {
	meta := getLOBMetaFilename(sha)
	infobytes, err := ioutil.ReadFile(meta)

	if err != nil {
		// Maybe just that it's not been downloaded yet
		// Let caller decide
		return nil, err
	}
	// Read JSON metadata
	info := &LOBInfo{}
	err = json.Unmarshal(infobytes, info)
	if err != nil {
		// Fatal, corruption
		LogErrorf("Unable to interpret meta file %v: %v\n", meta, err)
		return nil, err
	}

	return info, nil

}

// Retrieve LOB from storage
func RetrieveLOB(sha string, out io.Writer) (info *LOBInfo, err error) {
	info, err = GetLOBInfo(sha)

	if err != nil {
		if os.IsNotExist(err) {
			// We don't have this file yet
			// Potentially auto-download
			// TODO
		} else {
			// A problem
			LogErrorf("Unable to retrieve LOB with SHA %v: %v\n", sha, err)
			return nil, err
		}
	}

	for i := 0; i < info.NumChunks; i++ {
		// Check each chunk file exists, and is correct size
		// if not, maybe download (again)
		// TODO
	}

	return

}

// Read from a stream and calculate SHA, while also writing content to chunked content
// leader is a slice of bytes that has already been read (probe for SHA)
func StoreLOB(in io.Reader, leader []byte) (*LOBInfo, error) {

	sha := sha1.New()
	// Write chunks to temporary files, then move based on SHA filename once calculated
	chunkFilenames := make([]string, 0, 5)

	var outf *os.File
	var err error
	writeLeader := true
	buf := make([]byte, BUFSIZE)
	var fatalError error
	currentChunkSize := 0
	var totalSize int64 = 0

	for {
		// New chunk file?
		if outf == nil {
			outf, err = ioutil.TempFile("", "tempchunk")
			if err != nil {
				LogErrorf("Unable to create chunk %d: %v\n", len(chunkFilenames), err)
				fatalError = err
				break
			}
			LogDebugf("Creating temporary chunk file #%d: %v\n", len(chunkFilenames), outf.Name())
			chunkFilenames = append(chunkFilenames, outf.Name())
			currentChunkSize = 0
		}
		if writeLeader {
			LogDebugf("Writing leader of size %d to %v\n", len(leader), outf.Name())
			sha.Write(leader)
			c, err := outf.Write(leader)
			if err != nil {
				LogErrorf("I/O error writing leader: %v wrote %d bytes of %d\n", err, c, len(leader))
				fatalError = err
				break
			}
			currentChunkSize += c
			totalSize += int64(c)
			writeLeader = false
		}
		// Read from incoming
		var bytesToRead = BUFSIZE
		if BUFSIZE+currentChunkSize > CHUNKLIMIT {
			// Read less than BUFSIZE so we stick to CHUNKLIMIT
			bytesToRead = CHUNKLIMIT - currentChunkSize
		}
		c, err := in.Read(buf[:bytesToRead])
		// Write any data to SHA & output
		if c > 0 {
			currentChunkSize += c
			totalSize += int64(c)
			sha.Write(buf[:c])
			cw, err := outf.Write(buf)
			if err != nil || cw != c {
				LogErrorf("I/O error writing chunk %d: %v\n", len(chunkFilenames), err)
				fatalError = err
				break
			}
		}
		if err != nil {
			if err == io.EOF {
				// End of input
				outf.Close()
				break
			} else {
				LogErrorf("I/O error reading chunk %d: %v", len(chunkFilenames), err)
				outf.Close()
				fatalError = err
				break
			}
		}
		// Deal with chunk limit
		// NB right now assumes BUFSIZE is an exact divisor of CHUNKSIZE
		if currentChunkSize >= CHUNKLIMIT {
			// Close this output, next iteration will create the next file
			outf.Close()
			outf = nil
		}

	}

	if fatalError != nil {
		// Clean up temporaries
		for _, f := range chunkFilenames {
			os.Remove(f)
		}
		return nil, fatalError
	}

	shaStr := fmt.Sprintf("%x", string(sha.Sum(nil)))

	// We *may* now move the data to LOB dir
	// We won't if it already exists & is the correct size
	// Construct LOBInfo & write to final location
	info := &LOBInfo{SHA: shaStr, Size: totalSize, NumChunks: len(chunkFilenames)}
	infoBytes, err := json.Marshal(info)
	if err != nil {
		LogErrorf("Unable to convert LOB info to JSON: %v\n", err)
		return nil, err
	}
	infoFilename := getLOBMetaFilename(shaStr)
	if !FileExistsAndIsOfSize(infoFilename, int64(len(infoBytes))) {
		// Since all the details are derived from the SHA the only variant is chunking or incomplete writes so
		// we don't need to worry about needing to update the content (it must be correct)
		LogDebugf("Writing LOB metadata file: %v\n", infoFilename)
		ioutil.WriteFile(infoFilename, infoBytes, 0777)
	} else {
		LogDebugf("LOB metadata file already exists & is valid: %v\n", infoFilename)
	}
	// Check each chunk file
	for i, f := range chunkFilenames {
		sz := CHUNKLIMIT
		if i+1 == len(chunkFilenames) {
			// Last chunk, get size
			sz = currentChunkSize
		}
		destFile := getLOBChunkFilename(shaStr, i)
		if !FileExistsAndIsOfSize(destFile, int64(sz)) {
			LogDebugf("Saving final LOB metadata file: %v\n", destFile)
			// delete any existing (incorrectly sized) file since will probably not be allowed to rename over it
			// ignore any errors
			os.Remove(destFile)
			os.Rename(f, destFile)
		} else {
			LogDebugf("LOB chunk file already exists & is valid: %v\n", destFile)
		}
	}

	return info, nil

}
<commit_msg>Make sure we create subdir of git-lob storage area<commit_after>package main

import (
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
)

const BUFSIZE = 8192

// TODO make chunking user-configurable, default to 32MB
// chunk limit should be a multiple of BUFSIZE for max efficiency
const CHUNKLIMIT = BUFSIZE * 4086

// Information about a LOB
type LOBInfo struct {
	// SHA of the LOB
	SHA string
	// Total size of the LOB (all chunks)
	Size int64
	// Number of chunks that make up the whole LOB (integrity check)
	NumChunks int
}

// Gets the root folder of this git repository (the one containing .git)
func GetRepoRoot() (path string, isSeparateGitDir bool) {
	// We could call 'git rev-parse --git-dir' but this requires shelling out = slow, especially on Windows
	// We should try to avoid that whenever we can
	// So let's just find it ourselves; first containing folder with a .git folder/file
	curDir, err := os.Getwd()
	if err != nil {
		LogErrorf("Getwd failed: %v\n", err)
		return "", false
	}
	for {
		exists, isDir := FileOrDirExists(filepath.Join(curDir, ".git"))
		if exists {
			return curDir, !isDir
		}
		curDir = filepath.Dir(curDir)
		if curDir == string(filepath.Separator) || curDir == "." {
			// Not a repo
			LogError("Couldn't find repo root, not a git folder")
			return "", false
		}
	}
}

// Gets the git data dir of git repository (the .git dir, or where .git file points)
func GetGitDir() string {
	root, isSeparate := GetRepoRoot()
	git := filepath.Join(root, ".git")
	if isSeparate {
		// Git repo folder is separate, read location from file
		filebytes, err := ioutil.ReadFile(git)
		if err != nil {
			LogErrorf("Can't read .git file %v: %v\n", git, err)
			return ""
		}
		filestr := string(filebytes)
		match := regexp.MustCompile("gitdir:[\\s]+([^\\r\\n]+)").FindStringSubmatch(filestr)
		if match == nil {
			LogErrorf("Unexpected contents of .git file %v: %v\n", git, filestr)
			return ""
		}
		return match[1]
	} else {
		// Regular git dir
		return git
	}

}

// Gets the root directory for LOB files & creates if necessary
func GetLOBRoot() string {
	ret := filepath.Join(GetGitDir(), "git-lob")
	err := os.MkdirAll(ret, 0777)
	if err != nil {
		LogErrorf("Unable to create LOB root folder at %v: %v", ret, err)
		panic(err)
	}
	return ret
}

// Gets the containing folder for a given LOB SHA & creates if necessary
// LOBs are 'splayed' based on first 2 chars of SHA
func GetLOBDir(sha string) string {
	if len(sha) != 40 {
		LogErrorf("Invalid SHA format: %v\n", sha)
		return ""
	}
	ret := filepath.Join(GetLOBRoot(), sha[:2])
	err := os.MkdirAll(ret, 0777)
	if err != nil {
		LogErrorf("Unable to create LOB 2nd-levle folder at %v: %v", ret, err)
		panic(err)
	}
	return ret
}

func getLOBMetaFilename(sha string) string {
	fld := GetLOBDir(sha)
	return filepath.Join(fld, sha+"_meta")
}
func getLOBChunkFilename(sha string, chunkIdx int) string {
	fld := GetLOBDir(sha)
	return filepath.Join(fld, fmt.Sprintf("%v_%d", sha, chunkIdx))
}

// Retrieve information about an existing stored LOB
func GetLOBInfo(sha string) (*LOBInfo, error) {
	meta := getLOBMetaFilename(sha)
	infobytes, err := ioutil.ReadFile(meta)

	if err != nil {
		// Maybe just that it's not been downloaded yet
		// Let caller decide
		return nil, err
	}
	// Read JSON metadata
	info := &LOBInfo{}
	err = json.Unmarshal(infobytes, info)
	if err != nil {
		// Fatal, corruption
		LogErrorf("Unable to interpret meta file %v: %v\n", meta, err)
		return nil, err
	}

	return info, nil

}

// Retrieve LOB from storage
func RetrieveLOB(sha string, out io.Writer) (info *LOBInfo, err error) {
	info, err = GetLOBInfo(sha)

	if err != nil {
		if os.IsNotExist(err) {
			// We don't have this file yet
			// Potentially auto-download
			// TODO
		} else {
			// A problem
			LogErrorf("Unable to retrieve LOB with SHA %v: %v\n", sha, err)
			return nil, err
		}
	}

	for i := 0; i < info.NumChunks; i++ {
		// Check each chunk file exists, and is correct size
		// if not, maybe download (again)
		// TODO
	}

	return

}

// Read from a stream and calculate SHA, while also writing content to chunked content
// leader is a slice of bytes that has already been read (probe for SHA)
func StoreLOB(in io.Reader, leader []byte) (*LOBInfo, error) {

	sha := sha1.New()
	// Write chunks to temporary files, then move based on SHA filename once calculated
	chunkFilenames := make([]string, 0, 5)

	var outf *os.File
	var err error
	writeLeader := true
	buf := make([]byte, BUFSIZE)
	var fatalError error
	currentChunkSize := 0
	var totalSize int64 = 0

	for {
		// New chunk file?
		if outf == nil {
			outf, err = ioutil.TempFile("", "tempchunk")
			if err != nil {
				LogErrorf("Unable to create chunk %d: %v\n", len(chunkFilenames), err)
				fatalError = err
				break
			}
			LogDebugf("Creating temporary chunk file #%d: %v\n", len(chunkFilenames), outf.Name())
			chunkFilenames = append(chunkFilenames, outf.Name())
			currentChunkSize = 0
		}
		if writeLeader {
			LogDebugf("Writing leader of size %d to %v\n", len(leader), outf.Name())
			sha.Write(leader)
			c, err := outf.Write(leader)
			if err != nil {
				LogErrorf("I/O error writing leader: %v wrote %d bytes of %d\n", err, c, len(leader))
				fatalError = err
				break
			}
			currentChunkSize += c
			totalSize += int64(c)
			writeLeader = false
		}
		// Read from incoming
		var bytesToRead = BUFSIZE
		if BUFSIZE+currentChunkSize > CHUNKLIMIT {
			// Read less than BUFSIZE so we stick to CHUNKLIMIT
			bytesToRead = CHUNKLIMIT - currentChunkSize
		}
		c, err := in.Read(buf[:bytesToRead])
		// Write any data to SHA & output
		if c > 0 {
			currentChunkSize += c
			totalSize += int64(c)
			sha.Write(buf[:c])
			cw, err := outf.Write(buf)
			if err != nil || cw != c {
				LogErrorf("I/O error writing chunk %d: %v\n", len(chunkFilenames), err)
				fatalError = err
				break
			}
		}
		if err != nil {
			if err == io.EOF {
				// End of input
				outf.Close()
				break
			} else {
				LogErrorf("I/O error reading chunk %d: %v", len(chunkFilenames), err)
				outf.Close()
				fatalError = err
				break
			}
		}
		// Deal with chunk limit
		// NB right now assumes BUFSIZE is an exact divisor of CHUNKSIZE
		if currentChunkSize >= CHUNKLIMIT {
			// Close this output, next iteration will create the next file
			outf.Close()
			outf = nil
		}

	}

	if fatalError != nil {
		// Clean up temporaries
		for _, f := range chunkFilenames {
			os.Remove(f)
		}
		return nil, fatalError
	}

	shaStr := fmt.Sprintf("%x", string(sha.Sum(nil)))

	// We *may* now move the data to LOB dir
	// We won't if it already exists & is the correct size
	// Construct LOBInfo & write to final location
	info := &LOBInfo{SHA: shaStr, Size: totalSize, NumChunks: len(chunkFilenames)}
	infoBytes, err := json.Marshal(info)
	if err != nil {
		LogErrorf("Unable to convert LOB info to JSON: %v\n", err)
		return nil, err
	}
	infoFilename := getLOBMetaFilename(shaStr)
	if !FileExistsAndIsOfSize(infoFilename, int64(len(infoBytes))) {
		// Since all the details are derived from the SHA the only variant is chunking or incomplete writes so
		// we don't need to worry about needing to update the content (it must be correct)
		LogDebugf("Writing LOB metadata file: %v\n", infoFilename)
		ioutil.WriteFile(infoFilename, infoBytes, 0777)
	} else {
		LogDebugf("LOB metadata file already exists & is valid: %v\n", infoFilename)
	}
	// Check each chunk file
	for i, f := range chunkFilenames {
		sz := CHUNKLIMIT
		if i+1 == len(chunkFilenames) {
			// Last chunk, get size
			sz = currentChunkSize
		}
		destFile := getLOBChunkFilename(shaStr, i)
		if !FileExistsAndIsOfSize(destFile, int64(sz)) {
			LogDebugf("Saving final LOB metadata file: %v\n", destFile)
			// delete any existing (incorrectly sized) file since will probably not be allowed to rename over it
			// ignore any errors
			os.Remove(destFile)
			os.Rename(f, destFile)
		} else {
			LogDebugf("LOB chunk file already exists & is valid: %v\n", destFile)
		}
	}

	return info, nil

}
<|endoftext|>


<commit_before>package anaconda

import (
	"time"
)

type Tweet struct {
	Contributors         []int64     `json:"contributors"`
	Coordinates          interface{} `json:"coordinates"`
	CreatedAt            string      `json:"created_at"`
	Entities             Entities    `json:"entities"`
	FavoriteCount        int         `json:"favorite_count"`
	Favorited            bool        `json:"favorited"`
	Geo                  interface{} `json:"geo"`
	Id                   int64       `json:"id"`
	IdStr                string      `json:"id_str"`
	InReplyToScreenName  string      `json:"in_reply_to_screen_name"`
	InReplyToStatusID    int64       `json:"in_reply_to_status_id"`
	InReplyToStatusIdStr string      `json:"in_reply_to_status_id_str"`
	InReplyToUserID      int64       `json:"in_reply_to_user_id"`
	InReplyToUserIdStr   string      `json:"in_reply_to_user_id_str"`
	Place                Place       `json:"place"`
	PossiblySensitive    bool        `json:"possibly_sensitive"`
	RetweetCount         int         `json:"retweet_count"`
	Retweeted            bool        `json:"retweeted"`
	Source               string      `json:"source"`
	Text                 string      `json:"text"`
	Truncated            bool        `json:"truncated"`
	User                 User        `json:"user"`
}

// CreatedAtTime is a convenience wrapper that returns the Created_at time, parsed as a time.Time struct
func (t Tweet) CreatedAtTime() (time.Time, error) {
	return time.Parse(time.RubyDate, t.CreatedAt)
}
<commit_msg>Parse `tweet.retweeted_status` field (close #6)<commit_after>package anaconda

import (
	"time"
)

type Tweet struct {
	Contributors         []int64     `json:"contributors"`
	Coordinates          interface{} `json:"coordinates"`
	CreatedAt            string      `json:"created_at"`
	Entities             Entities    `json:"entities"`
	FavoriteCount        int         `json:"favorite_count"`
	Favorited            bool        `json:"favorited"`
	Geo                  interface{} `json:"geo"`
	Id                   int64       `json:"id"`
	IdStr                string      `json:"id_str"`
	InReplyToScreenName  string      `json:"in_reply_to_screen_name"`
	InReplyToStatusID    int64       `json:"in_reply_to_status_id"`
	InReplyToStatusIdStr string      `json:"in_reply_to_status_id_str"`
	InReplyToUserID      int64       `json:"in_reply_to_user_id"`
	InReplyToUserIdStr   string      `json:"in_reply_to_user_id_str"`
	Place                Place       `json:"place"`
	PossiblySensitive    bool        `json:"possibly_sensitive"`
	RetweetCount         int         `json:"retweet_count"`
	Retweeted            bool        `json:"retweeted"`
	RetweetedStatus      *Tweet      `json:"retweeted_status"`
	Source               string      `json:"source"`
	Text                 string      `json:"text"`
	Truncated            bool        `json:"truncated"`
	User                 User        `json:"user"`
}

// CreatedAtTime is a convenience wrapper that returns the Created_at time, parsed as a time.Time struct
func (t Tweet) CreatedAtTime() (time.Time, error) {
	return time.Parse(time.RubyDate, t.CreatedAt)
}
<|endoftext|>