<commit_before>/*
Incoming!!

Roadmap:
- document. make sure to mention that adblock (at least on chrome) causes high
  CPU usage. recommend to disable adblock for the page that uses incoming.
--> 0.1 finished

- (optional) file verification after upload: checksum in browser and backend, then
  assert that checksum is the same. Most likely error scenario: user updated file on
  disk while upload was running.
- go through ways for web app to retrieve file
  - available on filesystem? just give it the path to the file then (good enough if
    incoming!! serves only one web app). web app must move file away (or copy it), then
    tell incoming!! that it is finished. This is what we have now.
  - web app could download the file (very bad idea with most web apps, as it takes time,
    but this should be easy to implement)
  - if stored in cloud storage (ceph?): object id will work. coolest solution, as the file
    is stored in the right place right away

open questions:
- web app frontend must know name of incoming!! server. how will it get to know that?
    - for now: web app backend knows. html includes URL to js file.
- Incoming!! js code must know name of incoming!! server. how will it know?
    - for now, there is a function set_server_hostname in the incoming lib that must
	  be called by the web app frontend. Can we simplify this?
*/
package main

import (
	"fmt"
	"log"
	"net/http"
	"net/url"
	"path"
	"path/filepath"
	"strconv"
	"time"

	"bitbucket.org/kardianos/osext"
	"github.com/gorilla/mux"

	"source.uit.no/star-apt/incoming/upload"
)

type appVarsT struct {
	uploaders upload.UploaderPool
	config    *appConfigT
}

var appVars *appVarsT

/* NewUploadHandler receives an http request from a webapp wanting to do
an upload, and makes an Uploader for it. It responds with the uploader's id
(string).
*/
func NewUploadHandler(w http.ResponseWriter, r *http.Request) {
	log.Printf("got new upload request")

	// read upload parameters from request

	// upload to file or... (nothing else supported yet)
	destType := r.FormValue("destType") // 'file' or nothing. Default: file
	if destType == "" {
		destType = "file"
	}

	// which URL to POST to when file is here
	signalFinishURL, err := url.ParseRequestURI(r.FormValue("signalFinishURL"))
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "signalFinishURL invalid: %s", err.Error())
		return
	}

	// should we remove the file when it's all over or not?
	removeFileWhenFinishedStr := r.FormValue("removeFileWhenFinished")
	if removeFileWhenFinishedStr == "" { // true or false. Default: true
		removeFileWhenFinishedStr = "true"
	}
	removeFileWhenFinished, err := strconv.ParseBool(removeFileWhenFinishedStr)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "removeFileWhenFinished invalid: %s", err.Error())
		return
	}

	// secret cookie to POST to finish URL later
	backendSecret := r.FormValue("backendSecret") // optional

	// make (and pool) new uploader
	storageDirAbsolute, _ := filepath.Abs(appVars.config.StorageDir)
	uploader := upload.NewUploadToLocalFile(appVars.uploaders,
		storageDirAbsolute, signalFinishURL,
		removeFileWhenFinished, backendSecret,
		time.Duration(appVars.config.UploadMaxIdleDurationS)*time.Second)

	// answer request with id of new uploader
	fmt.Fprint(w, uploader.GetId())
	return
}

func ServeJSFileHandler(w http.ResponseWriter, r *http.Request) {
	programDir, _ := osext.ExecutableFolder()
	filePath := path.Join(programDir, "incoming_jslib.js")
	http.ServeFile(w, r, filePath)
}

func FinishUploadHandler(w http.ResponseWriter, r *http.Request) {
	// fetch uploader for given id
	id := r.FormValue("id")
	if id == "" {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprint(w, "id not given")
		return
	}
	uploader, ok := appVars.uploaders.Get(id)
	if !ok {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprint(w, "id unknown")
		return
	}

	// tell uploader that handover is done
	err := uploader.HandoverDone()

	// return error message or "ok"
	if err != nil {
		fmt.Fprint(w, err.Error())
	} else {
		fmt.Fprint(w, "ok")
	}
}

func CancelUploadHandler(w http.ResponseWriter, r *http.Request) {
	// fetch uploader for given id
	id := r.FormValue("id")
	if id == "" {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprint(w, "id not given")
		return
	}
	uploader, ok := appVars.uploaders.Get(id)
	if !ok {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprint(w, "id unknown")
		return
	}

	// let uploader cancel (async because this method should return quickly)
	go func() {
		uploader.Cancel(false, "Cancelled by request",
			time.Duration(appVars.config.HandoverTimeoutS)*time.Second)
		uploader.CleanUp()
	}()
	fmt.Fprint(w, "ok")
	return
}

func main() {
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds | log.Lshortfile)

	// --- init application-wide things (config, data structures)
	appVars = new(appVarsT)
	var err error

	// load config
	appVars.config, err = LoadConfig()
	if err != nil {
		log.Printf("Couldn't load config!")
		log.Fatal(err)
		return
	}

	// init upload module
	err = upload.InitModule(appVars.config.StorageDir)
	if err != nil {
		log.Fatal(err)
		return
	}

	// init uploader pool
	appVars.uploaders = upload.NewLockedUploaderPool()

	// --- set up http server
	routes := mux.NewRouter()
	routes.HandleFunc("/incoming/backend/new_upload", NewUploadHandler).
		Methods("POST")
	routes.HandleFunc("/incoming/backend/cancel_upload", CancelUploadHandler).
		Methods("POST")
	routes.HandleFunc("/incoming/backend/finish_upload", FinishUploadHandler).
		Methods("POST")
	routes.HandleFunc("/incoming/frontend/upload_ws", websocketHandler).
		Methods("GET")
	routes.HandleFunc("/incoming/frontend/incoming.js", ServeJSFileHandler).
		Methods("GET")

	// --- run server forever
	serverHost := fmt.Sprintf("%s:%d", appVars.config.IncomingIP,
		appVars.config.IncomingPort)
	log.Printf("Will start server on %s", serverHost)
	log.Fatal(http.ListenAndServe(serverHost, routes))
}
<commit_msg>"cancel upload" and "finish upload" HTTP functions now check backendSecret<commit_after>/*
Incoming!!

Roadmap:
- document. make sure to mention that adblock (at least on chrome) causes high
  CPU usage. recommend to disable adblock for the page that uses incoming.
--> 0.1 finished

- (optional) file verification after upload: checksum in browser and backend, then
  assert that checksum is the same. Most likely error scenario: user updated file on
  disk while upload was running.
- go through ways for web app to retrieve file
  - available on filesystem? just give it the path to the file then (good enough if
    incoming!! serves only one web app). web app must move file away (or copy it), then
    tell incoming!! that it is finished. This is what we have now.
  - web app could download the file (very bad idea with most web apps, as it takes time,
    but this should be easy to implement)
  - if stored in cloud storage (ceph?): object id will work. coolest solution, as the file
    is stored in the right place right away

open questions:
- web app frontend must know name of incoming!! server. how will it get to know that?
    - for now: web app backend knows. html includes URL to js file.
- Incoming!! js code must know name of incoming!! server. how will it know?
    - for now, there is a function set_server_hostname in the incoming lib that must
	  be called by the web app frontend. Can we simplify this?
*/
package main

import (
	"fmt"
	"log"
	"net/http"
	"net/url"
	"path"
	"path/filepath"
	"strconv"
	"time"

	"bitbucket.org/kardianos/osext"
	"github.com/gorilla/mux"

	"source.uit.no/star-apt/incoming/upload"
)

type appVarsT struct {
	uploaders upload.UploaderPool
	config    *appConfigT
}

var appVars *appVarsT

/* NewUploadHandler receives an http request from a webapp wanting to do
an upload, and makes an Uploader for it. It responds with the uploader's id
(string).
*/
func NewUploadHandler(w http.ResponseWriter, r *http.Request) {
	log.Printf("got new upload request")

	// read upload parameters from request

	// upload to file or... (nothing else supported yet)
	destType := r.FormValue("destType") // 'file' or nothing. Default: file
	if destType == "" {
		destType = "file"
	}

	// which URL to POST to when file is here
	signalFinishURL, err := url.ParseRequestURI(r.FormValue("signalFinishURL"))
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "signalFinishURL invalid: %s", err.Error())
		return
	}

	// should we remove the file when it's all over or not?
	removeFileWhenFinishedStr := r.FormValue("removeFileWhenFinished")
	if removeFileWhenFinishedStr == "" { // true or false. Default: true
		removeFileWhenFinishedStr = "true"
	}
	removeFileWhenFinished, err := strconv.ParseBool(removeFileWhenFinishedStr)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "removeFileWhenFinished invalid: %s", err.Error())
		return
	}

	// secret cookie to POST to finish URL later
	backendSecret := r.FormValue("backendSecret") // optional, "" if not given

	// make (and pool) new uploader
	storageDirAbsolute, _ := filepath.Abs(appVars.config.StorageDir)
	uploader := upload.NewUploadToLocalFile(appVars.uploaders,
		storageDirAbsolute, signalFinishURL,
		removeFileWhenFinished, backendSecret,
		time.Duration(appVars.config.UploadMaxIdleDurationS)*time.Second)

	// answer request with id of new uploader
	fmt.Fprint(w, uploader.GetId())
	return
}

func ServeJSFileHandler(w http.ResponseWriter, r *http.Request) {
	programDir, _ := osext.ExecutableFolder()
	filePath := path.Join(programDir, "incoming_jslib.js")
	http.ServeFile(w, r, filePath)
}

func FinishUploadHandler(w http.ResponseWriter, r *http.Request) {
	// fetch uploader for given id
	id := r.FormValue("id")
	if id == "" {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprint(w, "id not given")
		return
	}
	uploader, ok := appVars.uploaders.Get(id)
	if !ok {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprint(w, "id unknown")
		return
	}

	// assert that 'backend secret string' matches (if it's not given, it's an
	// empty string, which might be just fine)
	if uploader.GetBackendSecret() != r.FormValue("backendSecret") {
		w.WriteHeader(http.StatusForbidden)
		fmt.Fprint(w, "backendSecret not given or wrong")
		return
	}

	// tell uploader that handover is done
	err := uploader.HandoverDone()

	// return error message or "ok"
	if err != nil {
		fmt.Fprint(w, err.Error())
	} else {
		fmt.Fprint(w, "ok")
	}
}

func CancelUploadHandler(w http.ResponseWriter, r *http.Request) {
	// fetch uploader for given id
	id := r.FormValue("id")
	if id == "" {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprint(w, "id not given")
		return
	}
	uploader, ok := appVars.uploaders.Get(id)
	if !ok {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprint(w, "id unknown")
		return
	}

	// assert that 'backend secret string' matches (if it's not given, it's an
	// empty string, which might be just fine)
	if uploader.GetBackendSecret() != r.FormValue("backendSecret") {
		w.WriteHeader(http.StatusForbidden)
		fmt.Fprint(w, "backendSecret not given or wrong")
		return
	}

	// let uploader cancel (async because this method should return quickly)
	go func() {
		uploader.Cancel(false, "Cancelled by request",
			time.Duration(appVars.config.HandoverTimeoutS)*time.Second)
		uploader.CleanUp()
	}()

	fmt.Fprint(w, "ok")
	return
}

func main() {
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds | log.Lshortfile)

	// --- init application-wide things (config, data structures)
	appVars = new(appVarsT)
	var err error

	// load config
	appVars.config, err = LoadConfig()
	if err != nil {
		log.Printf("Couldn't load config!")
		log.Fatal(err)
		return
	}

	// init upload module
	err = upload.InitModule(appVars.config.StorageDir)
	if err != nil {
		log.Fatal(err)
		return
	}

	// init uploader pool
	appVars.uploaders = upload.NewLockedUploaderPool()

	// --- set up http server
	routes := mux.NewRouter()
	routes.HandleFunc("/incoming/backend/new_upload", NewUploadHandler).
		Methods("POST")
	routes.HandleFunc("/incoming/backend/cancel_upload", CancelUploadHandler).
		Methods("POST")
	routes.HandleFunc("/incoming/backend/finish_upload", FinishUploadHandler).
		Methods("POST")
	routes.HandleFunc("/incoming/frontend/upload_ws", websocketHandler).
		Methods("GET")
	routes.HandleFunc("/incoming/frontend/incoming.js", ServeJSFileHandler).
		Methods("GET")

	// --- run server forever
	serverHost := fmt.Sprintf("%s:%d", appVars.config.IncomingIP,
		appVars.config.IncomingPort)
	log.Printf("Will start server on %s", serverHost)
	log.Fatal(http.ListenAndServe(serverHost, routes))
}
<|endoftext|>


<commit_before>/*
Package simpleforce is a dead simple wrapper around the Force.com REST API.

It allows you to query for Force.com objects by using idiomatic Go constructs, or you can short
circuit the query engine and qrite your own SOQL. In either case, data is returned to you via
structs of your own creation, allowing you full control over what data is returned.
*/
package simpleforce

import (
	"bytes"
	"github.com/bitly/go-simplejson"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"reflect"
	"time"
)

const (
	DateFormat     = "2006-01-02"
	DateTimeFormat = time.RFC3339Nano
)

type Force struct {
	session string
	url     string
}

// Returns a new Force object with the given login credentials. This object is the main
// point of entry for all your Force.com needs.
func New(session, url string) Force {
	return Force{
		session,
		url,
	}
}

func NewWithCredentials(loginUrl, consumerKey, consumerSecret, username, password string) (Force, error) {
	resp, err := http.PostForm(loginUrl+"/services/oauth2/token", url.Values{
		"grant_type":    {"password"},
		"client_id":     {consumerKey},
		"client_secret": {consumerSecret},
		"username":      {username},
		"password":      {password},
	})
	if err != nil {
		return Force{}, err
	}
	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return Force{}, err
	}
	respJson, err := simplejson.NewJson(respBytes)
	if err != nil {
		return Force{}, err
	}
	session := respJson.Get("access_token").MustString()
	url := respJson.Get("instance_url").MustString() + "/services/data/v27.0"
	return New(session, url), err
}

func (f Force) authorizeRequest(method, urlStr string, body io.Reader) (*http.Request, error) {
	r, err := http.NewRequest(method, urlStr, body)
	if err != nil {
		return nil, err
	}
	r.Header.Add("Authorization", "Bearer "+f.session)
	return r, nil
}

// Run a raw SOQL query string. This will fill the given destination slice with the results of your query.
func (f Force) Query(query string, dest interface{}) error {
	vals := url.Values{}
	vals.Set("q", query)
	url := f.url + "/query?" + vals.Encode()
	req, err := f.authorizeRequest("GET", url, bytes.NewBufferString(""))
	if err != nil {
		return err
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	respJson, err := simplejson.NewJson(respBytes)
	if err != nil {
		return err
	}
	err = unmarshal(respJson, dest)
	return err
}

func unmarshal(source *simplejson.Json, dest interface{}) error {
	sliceValPtr := reflect.ValueOf(dest)
	sliceVal := sliceValPtr.Elem()
	elemType := reflect.TypeOf(dest).Elem().Elem()
	for i := 0; i < source.Get("totalSize").MustInt(); i++ {
		v := source.Get("records").GetIndex(i)
		val, err := unmarshalIndividualObject(v, elemType)
		if err != nil {
			return err
		}
		sliceVal.Set(reflect.Append(sliceVal, val))
	}
	return nil
}

func unmarshalIndividualObject(source *simplejson.Json, valType reflect.Type) (reflect.Value, error) {
	valPtr := reflect.New(valType)
	val := reflect.Indirect(valPtr)
	for f := 0; f < valType.NumField(); f++ {
		field := val.Field(f)
		switch field.Kind() {
		case reflect.Bool:
			boolVal := source.Get(valType.Field(f).Name).MustBool()
			field.SetBool(boolVal)
		case reflect.Int:
			intVal := source.Get(valType.Field(f).Name).MustInt64()
			field.SetInt(intVal)
		case reflect.Int64:
			intVal := source.Get(valType.Field(f).Name).MustInt64()
			field.SetInt(intVal)
		case reflect.Float32:
			floatVal := source.Get(valType.Field(f).Name).MustFloat64()
			field.SetFloat(floatVal)
		case reflect.Float64:
			floatVal := source.Get(valType.Field(f).Name).MustFloat64()
			field.SetFloat(floatVal)
		case reflect.String:
			strVal := source.Get(valType.Field(f).Name).MustString()
			field.SetString(strVal)
		case reflect.Struct:
			strVal := source.Get(valType.Field(f).Name).MustString()
			if valType.Field(f).Type.Name() == "Time" {
				if t, err := time.Parse(DateTimeFormat, strVal); err == nil {
					// it's a datetime string, probably!
					field.Set(reflect.ValueOf(t))
				} else if t, err = time.Parse(DateFormat, strVal); err == nil {
					// nope, it's a date string!
					field.Set(reflect.ValueOf(t))
				} else {
					return val, err
				}
			}
		case reflect.Ptr:
			objJson := source.Get(valType.Field(f).Name)
			if objJson != nil {
				objType := valType.Field(f).Type.Elem()
				objVal, err := unmarshalIndividualObject(objJson, objType)
				if err != nil {
					return val, err
				}
				field.Set(objVal.Addr())
			}
		case reflect.Slice:
			objJson := source.Get(valType.Field(f).Name).Get("records")
			length := source.Get(valType.Field(f).Name).Get("totalSize").MustInt()
			if objJson != nil {
				elemType := field.Type().Elem()
				objSlicePtr := reflect.New(field.Type())
				objSlice := reflect.Indirect(objSlicePtr)
				for i := 0; i < length; i++ {
					o := objJson.GetIndex(i)
					obj, err := unmarshalIndividualObject(o, elemType)
					if err != nil {
						return val, err
					}
					objSlice.Set(reflect.Append(objSlice, obj))
				}
				field.Set(objSlice)
			}
		}
	}
	return val, nil
}
<commit_msg>added NewFromEnvironment convenience func.<commit_after>/*
Package simpleforce is a dead simple wrapper around the Force.com REST API.

It allows you to query for Force.com objects by using idiomatic Go constructs, or you can short
circuit the query engine and qrite your own SOQL. In either case, data is returned to you via
structs of your own creation, allowing you full control over what data is returned.
*/
package simpleforce

import (
	"bytes"
	"github.com/bitly/go-simplejson"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"reflect"
	"time"
)

const (
	DateFormat     = "2006-01-02"
	DateTimeFormat = time.RFC3339Nano
)

type Force struct {
	session string
	url     string
}

// Returns a new Force object with the given login credentials. This object is the main
// point of entry for all your Force.com needs.
func New(session, url string) Force {
	return Force{
		session,
		url,
	}
}

func NewWithCredentials(loginUrl, consumerKey, consumerSecret, username, password string) (Force, error) {
	resp, err := http.PostForm(loginUrl+"/services/oauth2/token", url.Values{
		"grant_type":    {"password"},
		"client_id":     {consumerKey},
		"client_secret": {consumerSecret},
		"username":      {username},
		"password":      {password},
	})
	if err != nil {
		return Force{}, err
	}
	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return Force{}, err
	}
	respJson, err := simplejson.NewJson(respBytes)
	if err != nil {
		return Force{}, err
	}
	session := respJson.Get("access_token").MustString()
	url := respJson.Get("instance_url").MustString() + "/services/data/v27.0"
	return New(session, url), err
}

func NewFromEnvironment() (Force, error) {
	return NewWithCredentials(os.Getenv("SF_LOGIN_URL"), os.Getenv("SF_CLIENT_ID"), os.Getenv("SF_CLIENT_SECRET"), os.Getenv("SF_USERNAME"), os.Getenv("SF_PASSWORD")+os.Getenv("SF_TOKEN"))
}

func (f Force) authorizeRequest(method, urlStr string, body io.Reader) (*http.Request, error) {
	r, err := http.NewRequest(method, urlStr, body)
	if err != nil {
		return nil, err
	}
	r.Header.Add("Authorization", "Bearer "+f.session)
	return r, nil
}

// Run a raw SOQL query string. This will fill the given destination slice with the results of your query.
func (f Force) Query(query string, dest interface{}) error {
	vals := url.Values{}
	vals.Set("q", query)
	url := f.url + "/query?" + vals.Encode()
	req, err := f.authorizeRequest("GET", url, bytes.NewBufferString(""))
	if err != nil {
		return err
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	respJson, err := simplejson.NewJson(respBytes)
	if err != nil {
		return err
	}
	err = unmarshal(respJson, dest)
	return err
}

func unmarshal(source *simplejson.Json, dest interface{}) error {
	sliceValPtr := reflect.ValueOf(dest)
	sliceVal := sliceValPtr.Elem()
	elemType := reflect.TypeOf(dest).Elem().Elem()
	for i := 0; i < source.Get("totalSize").MustInt(); i++ {
		v := source.Get("records").GetIndex(i)
		val, err := unmarshalIndividualObject(v, elemType)
		if err != nil {
			return err
		}
		sliceVal.Set(reflect.Append(sliceVal, val))
	}
	return nil
}

func unmarshalIndividualObject(source *simplejson.Json, valType reflect.Type) (reflect.Value, error) {
	valPtr := reflect.New(valType)
	val := reflect.Indirect(valPtr)
	for f := 0; f < valType.NumField(); f++ {
		field := val.Field(f)
		switch field.Kind() {
		case reflect.Bool:
			boolVal := source.Get(valType.Field(f).Name).MustBool()
			field.SetBool(boolVal)
		case reflect.Int:
			intVal := source.Get(valType.Field(f).Name).MustInt64()
			field.SetInt(intVal)
		case reflect.Int64:
			intVal := source.Get(valType.Field(f).Name).MustInt64()
			field.SetInt(intVal)
		case reflect.Float32:
			floatVal := source.Get(valType.Field(f).Name).MustFloat64()
			field.SetFloat(floatVal)
		case reflect.Float64:
			floatVal := source.Get(valType.Field(f).Name).MustFloat64()
			field.SetFloat(floatVal)
		case reflect.String:
			strVal := source.Get(valType.Field(f).Name).MustString()
			field.SetString(strVal)
		case reflect.Struct:
			strVal := source.Get(valType.Field(f).Name).MustString()
			if valType.Field(f).Type.Name() == "Time" {
				if t, err := time.Parse(DateTimeFormat, strVal); err == nil {
					// it's a datetime string, probably!
					field.Set(reflect.ValueOf(t))
				} else if t, err = time.Parse(DateFormat, strVal); err == nil {
					// nope, it's a date string!
					field.Set(reflect.ValueOf(t))
				} else {
					return val, err
				}
			}
		case reflect.Ptr:
			objJson := source.Get(valType.Field(f).Name)
			if objJson != nil {
				objType := valType.Field(f).Type.Elem()
				objVal, err := unmarshalIndividualObject(objJson, objType)
				if err != nil {
					return val, err
				}
				field.Set(objVal.Addr())
			}
		case reflect.Slice:
			objJson := source.Get(valType.Field(f).Name).Get("records")
			length := source.Get(valType.Field(f).Name).Get("totalSize").MustInt()
			if objJson != nil {
				elemType := field.Type().Elem()
				objSlicePtr := reflect.New(field.Type())
				objSlice := reflect.Indirect(objSlicePtr)
				for i := 0; i < length; i++ {
					o := objJson.GetIndex(i)
					obj, err := unmarshalIndividualObject(o, elemType)
					if err != nil {
						return val, err
					}
					objSlice.Set(reflect.Append(objSlice, obj))
				}
				field.Set(objSlice)
			}
		}
	}
	return val, nil
}
<|endoftext|>


<commit_before>package middleware

import(
	"github.com/justinas/alice"
	"github.com/ethereal-go/ethereal/root/app"
	"net/http"
)

/**
/ Add middleware in App under certain condition..
*/
type AddMiddleware interface {
	Add(*[]alice.Constructor, *app.Application)
}

type Middleware struct {
	// all middleware
	AllMiddleware []AddMiddleware
	// middleware only included in application
	IncludeMiddleware []alice.Constructor

}

func (m *Middleware) AddMiddleware(middleware ...AddMiddleware) {
	m.AllMiddleware = append(m.AllMiddleware, middleware...)
}

// Method loading middleware for application
func (m Middleware) LoadApplication(application *app.Application) []alice.Constructor {
	for _, middleware := range m.AllMiddleware {
		middleware.Add(&m.IncludeMiddleware, application)
	}
	return m.IncludeMiddleware
}

func (m Middleware) GetHandler(h http.HandlerFunc) http.Handler {
	return alice.New(m.IncludeMiddleware...).Then(h)
}
// ---- waiting for your implementation ------

// middleware set Accept-Language
func middlewareLocal(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// TODO Pipline choose
		// TODO set locale from request
		//app.Locale = parserLocale(r.Header["Accept-Language"])
		next.ServeHTTP(w, r)
	})
}<commit_msg>fix middleware<commit_after>package middleware

import(
	"github.com/justinas/alice"
	"github.com/ethereal-go/ethereal/root/app"
	"net/http"
	"log"
)

/**
/ Add middleware in App under certain condition..
*/
type AddMiddleware interface {
	Add(*[]alice.Constructor, *app.Application)
}

type Middleware struct {
	// all middleware
	AllMiddleware []AddMiddleware
	// middleware only included in application
	IncludeMiddleware []alice.Constructor

}

func (m *Middleware) AddMiddleware(middleware ...AddMiddleware) {
	m.AllMiddleware = append(m.AllMiddleware, middleware...)
}

// Method loading middleware for application
func (m *Middleware) LoadApplication(application *app.Application) []alice.Constructor {
	for _, middleware := range m.AllMiddleware {
		middleware.Add(&m.IncludeMiddleware, application)
	}
	return m.IncludeMiddleware
}

func (m Middleware) GetHandler(h http.HandlerFunc) http.Handler {
	log.Println(m.IncludeMiddleware, "middlewares")
	return alice.New(m.IncludeMiddleware...).Then(h)
}
// ---- waiting for your implementation ------

// middleware set Accept-Language
func middlewareLocal(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// TODO Pipline choose
		// TODO set locale from request
		//app.Locale = parserLocale(r.Header["Accept-Language"])
		next.ServeHTTP(w, r)
	})
}<|endoftext|>


<commit_before>package clickhouse

import (
	"database/sql/driver"
	"net"
	"sync/atomic"
	"time"
)

var tick int32

func dial(network string, hosts []string, noDelay bool, r, w time.Duration, logf func(string, ...interface{})) (*connect, error) {
	var (
		err error
		abs = func(v int) int {
			if v < 0 {
				return -1
			}
			return v
		}
		conn  net.Conn
		index = abs(int(atomic.AddInt32(&tick, 1)))
	)
	for i := 0; i <= len(hosts); i++ {
		if conn, err = net.DialTimeout(network, hosts[(index+1)%len(hosts)], 2*time.Second); err == nil {
			logf("[connect] num=%d -> %s", tick, conn.RemoteAddr())
			if tcp, ok := conn.(*net.TCPConn); ok {
				tcp.SetNoDelay(noDelay) // Disable or enable the Nagle Algorithm for this tcp socket
			}
			return &connect{
				Conn:         conn,
				logf:         logf,
				readTimeout:  r,
				writeTimeout: w,
			}, nil
		}
	}
	return nil, err
}

type connect struct {
	net.Conn
	logf         func(string, ...interface{})
	readTimeout  time.Duration
	writeTimeout time.Duration
}

func (conn *connect) Read(b []byte) (int, error) {
	if conn.readTimeout != 0 {
		conn.SetReadDeadline(time.Now().Add(conn.readTimeout))
	}
	n, err := conn.Conn.Read(b)
	if err != nil {
		conn.logf("[connect] read error: %v", err)
		return n, driver.ErrBadConn
	}
	return n, nil
}

func (conn *connect) Write(b []byte) (int, error) {
	if conn.writeTimeout != 0 {
		conn.SetWriteDeadline(time.Now().Add(conn.writeTimeout))
	}
	n, err := conn.Conn.Write(b)
	if err != nil {
		conn.logf("[connect] write error: %v", err)
		return n, driver.ErrBadConn
	}
	return n, nil
}
<commit_msg>#34 use buffer for read from network<commit_after>package clickhouse

import (
	"bufio"
	"database/sql/driver"
	"net"
	"sync/atomic"
	"time"
)

var tick int32

func dial(network string, hosts []string, noDelay bool, r, w time.Duration, logf func(string, ...interface{})) (*connect, error) {
	var (
		err error
		abs = func(v int) int {
			if v < 0 {
				return -1
			}
			return v
		}
		conn  net.Conn
		index = abs(int(atomic.AddInt32(&tick, 1)))
	)
	for i := 0; i <= len(hosts); i++ {
		if conn, err = net.DialTimeout(network, hosts[(index+1)%len(hosts)], 2*time.Second); err == nil {
			logf("[connect] num=%d -> %s", tick, conn.RemoteAddr())
			if tcp, ok := conn.(*net.TCPConn); ok {
				tcp.SetNoDelay(noDelay) // Disable or enable the Nagle Algorithm for this tcp socket
			}
			return &connect{
				Conn:         conn,
				buf:          bufio.NewReader(conn),
				logf:         logf,
				readTimeout:  r,
				writeTimeout: w,
			}, nil
		}
	}
	return nil, err
}

type connect struct {
	net.Conn
	buf          *bufio.Reader
	logf         func(string, ...interface{})
	readTimeout  time.Duration
	writeTimeout time.Duration
}

func (conn *connect) Read(b []byte) (int, error) {
	if conn.readTimeout != 0 {
		conn.SetReadDeadline(time.Now().Add(conn.readTimeout))
	}
	n, err := conn.buf.Read(b)
	if err != nil {
		conn.logf("[connect] read error: %v", err)
		return n, driver.ErrBadConn
	}
	return n, nil
}

func (conn *connect) Write(b []byte) (int, error) {
	if conn.writeTimeout != 0 {
		conn.SetWriteDeadline(time.Now().Add(conn.writeTimeout))
	}
	n, err := conn.Conn.Write(b)
	if err != nil {
		conn.logf("[connect] write error: %v", err)
		return n, driver.ErrBadConn
	}
	return n, nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"reflect"
	"strings"
	"time"
)

// GetTypesRecursive links in namesmap a flag with there flildstruct Type
// You can whether provide objValue on a structure or a pointer to structure as first argument
// Flags are genereted from field name or from structags
func getTypesRecursive(objValue reflect.Value, namesmap map[string]reflect.Type, key string) error {
	name := key
	switch objValue.Kind() {
	case reflect.Struct:
		name += objValue.Type().Name()
		for i := 0; i < objValue.NumField(); i++ {
			if tag := objValue.Type().Field(i).Tag.Get("description"); len(tag) > 0 {
				fieldName := objValue.Type().Field(i).Name
				if tag := objValue.Type().Field(i).Tag.Get("long"); len(tag) > 0 {
					fieldName = tag
				}
				if tag := objValue.Type().Field(i).Tag.Get("short"); len(tag) > 0 {
					if _, ok := namesmap[strings.ToLower(tag)]; ok {
						return errors.New("Tag already exists: " + tag)
					}
					namesmap[strings.ToLower(tag)] = objValue.Field(i).Type()
				}
				if len(key) == 0 {
					name = fieldName
				} else {
					name = key + "." + fieldName
				}
				if _, ok := namesmap[strings.ToLower(name)]; ok {
					return errors.New("Tag already exists: " + name)
				}
				namesmap[strings.ToLower(name)] = objValue.Field(i).Type()
				if err := getTypesRecursive(objValue.Field(i), namesmap, name); err != nil {
					return err
				}
			}
		}
	case reflect.Array, reflect.Map, reflect.Slice, reflect.Ptr:
		typ := objValue.Type().Elem()
		inst := reflect.New(typ).Elem()
		if err := getTypesRecursive(inst, namesmap, name); err != nil {
			return err
		}
	default:
		return nil
	}
	return nil
}

//ParseArgs : parses args into a map[tag]value, using map[type]parser
//args must be formated as like as flag documentation. See https://golang.org/pkg/flag
func parseArgs(args []string, tagsmap map[string]reflect.Type, parsers map[reflect.Type]flag.Value) (map[string]flag.Value, error) {
	newParsers := map[string]flag.Value{}
	flagSet := flag.NewFlagSet("flaeg.ParseArgs", flag.ExitOnError)
	valmap := make(map[string]flag.Value)
	for tag, rType := range tagsmap {
		if parser, ok := parsers[rType]; ok {
			newparser := reflect.New(reflect.TypeOf(parser).Elem()).Interface().(flag.Value)
			flagSet.Var(newparser, tag, "help")
			newParsers[tag] = newparser
		}
	}
	if err := flagSet.Parse(args); err != nil {
		return nil, err
	}
	for tag, newParser := range newParsers {
		valmap[tag] = newParser
	}
	return valmap, nil
}

//FillStructRecursive initialize a value of any taged Struct given by reference
func fillStructRecursive(objValue reflect.Value, valmap map[string]flag.Value, key string) error {
	name := key
	// fmt.Printf("objValue begin : %+v\n", objValue)
	switch objValue.Kind() {
	case reflect.Struct:
		name += objValue.Type().Name()
		// inst := reflect.New(objValue.Type()).Elem()
		// for i := 0; i < inst.NumField(); i++ {
		for i := 0; i < objValue.Type().NumField(); i++ {
			if tag := objValue.Type().Field(i).Tag.Get("description"); len(tag) > 0 {
				fieldName := objValue.Type().Field(i).Name
				if tag := objValue.Type().Field(i).Tag.Get("long"); len(tag) > 0 {
					fieldName = tag
				}
				if tag := objValue.Type().Field(i).Tag.Get("short"); len(tag) > 0 {
					if err := setFields(objValue.Field(i), valmap, strings.ToLower(tag)); err != nil {
						return err
					}

				}
				if len(key) == 0 {
					name = fieldName
				} else {
					name = key + "." + fieldName
				}
				// fmt.Printf("tag : %s\n", name)
				if err := setFields(objValue.Field(i), valmap, strings.ToLower(name)); err != nil {
					return err
				}
				if err := fillStructRecursive(objValue.Field(i), valmap, name); err != nil {
					return err
				}
			}
		}
	case reflect.Ptr:
		if objValue.IsNil() {
			inst := reflect.New(objValue.Type().Elem())
			if err := fillStructRecursive(inst.Elem(), valmap, name); err != nil {
				return err
			}
			objValue.Set(inst)
		} else {
			if err := fillStructRecursive(objValue.Elem(), valmap, name); err != nil {
				return err
			}
		}
	default:
		return nil
	}
	// fmt.Printf("objValue end : %+v\n", objValue)
	return nil
}

// SetFields sets value to fieldValue using tag as key in valmap
func setFields(fieldValue reflect.Value, valmap map[string]flag.Value, tag string) error {
	if reflect.DeepEqual(fieldValue.Interface(), reflect.New(fieldValue.Type()).Elem().Interface()) {
		if fieldValue.CanSet() {
			if val, ok := valmap[tag]; ok {
				// fmt.Printf("tag %s : set %s in a %s\n", tag, val, fieldValue.Kind())
				fieldValue.Set(reflect.ValueOf(val).Elem().Convert(fieldValue.Type()))
			}
		} else {
			return errors.New(fieldValue.Type().String() + " is not settable.")
		}

	}
	return nil
}

//loadParsers loads default parsers and custom parsers given as parameter. Return a map [reflect.Type]parsers
func loadParsers(customParsers map[reflect.Type]flag.Value) (map[reflect.Type]flag.Value, error) {
	parsers := map[reflect.Type]flag.Value{}
	var stringParser stringValue
	var boolParser boolValue
	var intParser intValue
	var timeParser timeValue
	parsers[reflect.TypeOf("")] = &stringParser
	parsers[reflect.TypeOf(true)] = &boolParser
	parsers[reflect.TypeOf(1)] = &intParser
	parsers[reflect.TypeOf(time.Now())] = &timeParser
	for rType, parser := range customParsers {
		parsers[rType] = parser
	}
	return parsers, nil
}

//Load initializes config : struct fields given by reference, with args : arguments.
//Some custom parsers may be given.
func Load(config interface{}, args []string, customParsers map[reflect.Type]flag.Value) error {
	parsers, err := loadParsers(customParsers)
	if err != nil {
		return err
	}
	tagsmap := make(map[string]reflect.Type)
	if err := getTypesRecursive(reflect.ValueOf(config), tagsmap, ""); err != nil {
		return err
	}
	valmap, err := parseArgs(args, tagsmap, parsers)
	if err != nil {
		return err
	}
	if err := fillStructRecursive(reflect.ValueOf(config), valmap, ""); err != nil {
		return err
	}

	return nil
}
<commit_msg>parseArgs returns only changed value<commit_after>package main

import (
	"errors"
	"flag"
	"reflect"
	"strings"
	"time"
)

// GetTypesRecursive links in namesmap a flag with there flildstruct Type
// You can whether provide objValue on a structure or a pointer to structure as first argument
// Flags are genereted from field name or from structags
func getTypesRecursive(objValue reflect.Value, namesmap map[string]reflect.Type, key string) error {
	name := key
	switch objValue.Kind() {
	case reflect.Struct:
		name += objValue.Type().Name()
		for i := 0; i < objValue.NumField(); i++ {
			if tag := objValue.Type().Field(i).Tag.Get("description"); len(tag) > 0 {
				fieldName := objValue.Type().Field(i).Name
				if tag := objValue.Type().Field(i).Tag.Get("long"); len(tag) > 0 {
					fieldName = tag
				}
				if tag := objValue.Type().Field(i).Tag.Get("short"); len(tag) > 0 {
					if _, ok := namesmap[strings.ToLower(tag)]; ok {
						return errors.New("Tag already exists: " + tag)
					}
					namesmap[strings.ToLower(tag)] = objValue.Field(i).Type()
				}
				if len(key) == 0 {
					name = fieldName
				} else {
					name = key + "." + fieldName
				}
				if _, ok := namesmap[strings.ToLower(name)]; ok {
					return errors.New("Tag already exists: " + name)
				}
				namesmap[strings.ToLower(name)] = objValue.Field(i).Type()
				if err := getTypesRecursive(objValue.Field(i), namesmap, name); err != nil {
					return err
				}
			}
		}
	case reflect.Array, reflect.Map, reflect.Slice, reflect.Ptr:
		typ := objValue.Type().Elem()
		inst := reflect.New(typ).Elem()
		if err := getTypesRecursive(inst, namesmap, name); err != nil {
			return err
		}
	default:
		return nil
	}
	return nil
}

//ParseArgs : parses args into a map[tag]value, using map[type]parser
//args must be formated as like as flag documentation. See https://golang.org/pkg/flag
func parseArgs(args []string, tagsmap map[string]reflect.Type, parsers map[reflect.Type]flag.Value) (map[string]flag.Value, error) {
	newParsers := map[string]flag.Value{}
	defaultValParsers := map[string]string{}
	flagSet := flag.NewFlagSet("flaeg.ParseArgs", flag.ExitOnError)
	valmap := make(map[string]flag.Value)
	for tag, rType := range tagsmap {

		if parser, ok := parsers[rType]; ok {
			newparser := reflect.New(reflect.TypeOf(parser).Elem()).Interface().(flag.Value)
			flagSet.Var(newparser, tag, "help")
			newParsers[tag] = newparser
			defaultValParsers[tag] = newparser.String()
		}
	}

	if err := flagSet.Parse(args); err != nil {
		return nil, err
	}
	for tag, newParser := range newParsers {

		if newParser.String() != defaultValParsers[tag] {
			valmap[tag] = newParser
			// fmt.Printf("tag : %s, value : %s default : %s\n", tag, newParser.String(), defaultValParsers[tag])
		}
	}
	return valmap, nil
}

//FillStructRecursive initialize a value of any taged Struct given by reference
func fillStructRecursive(objValue reflect.Value, valmap map[string]flag.Value, key string) error {
	name := key
	// fmt.Printf("objValue begin : %+v\n", objValue)
	switch objValue.Kind() {
	case reflect.Struct:
		name += objValue.Type().Name()
		// inst := reflect.New(objValue.Type()).Elem()
		// for i := 0; i < inst.NumField(); i++ {
		for i := 0; i < objValue.Type().NumField(); i++ {
			if tag := objValue.Type().Field(i).Tag.Get("description"); len(tag) > 0 {
				fieldName := objValue.Type().Field(i).Name
				if tag := objValue.Type().Field(i).Tag.Get("long"); len(tag) > 0 {
					fieldName = tag
				}
				if tag := objValue.Type().Field(i).Tag.Get("short"); len(tag) > 0 {
					if err := setFields(objValue.Field(i), valmap, strings.ToLower(tag)); err != nil {
						return err
					}

				}
				if len(key) == 0 {
					name = fieldName
				} else {
					name = key + "." + fieldName
				}
				// fmt.Printf("tag : %s\n", name)
				if err := setFields(objValue.Field(i), valmap, strings.ToLower(name)); err != nil {
					return err
				}
				if err := fillStructRecursive(objValue.Field(i), valmap, name); err != nil {
					return err
				}
			}
		}
	case reflect.Ptr:
		if objValue.IsNil() {
			inst := reflect.New(objValue.Type().Elem())
			if err := fillStructRecursive(inst.Elem(), valmap, name); err != nil {
				return err
			}
			objValue.Set(inst)
		} else {
			if err := fillStructRecursive(objValue.Elem(), valmap, name); err != nil {
				return err
			}
		}
	default:
		return nil
	}
	// fmt.Printf("objValue end : %+v\n", objValue)
	return nil
}

// SetFields sets value to fieldValue using tag as key in valmap
func setFields(fieldValue reflect.Value, valmap map[string]flag.Value, tag string) error {
	if reflect.DeepEqual(fieldValue.Interface(), reflect.New(fieldValue.Type()).Elem().Interface()) {
		if fieldValue.CanSet() {
			if val, ok := valmap[tag]; ok {
				// fmt.Printf("tag %s : set %s in a %s\n", tag, val, fieldValue.Kind())
				fieldValue.Set(reflect.ValueOf(val).Elem().Convert(fieldValue.Type()))
			}
		} else {
			return errors.New(fieldValue.Type().String() + " is not settable.")
		}

	}
	return nil
}

//loadParsers loads default parsers and custom parsers given as parameter. Return a map [reflect.Type]parsers
func loadParsers(customParsers map[reflect.Type]flag.Value) (map[reflect.Type]flag.Value, error) {
	parsers := map[reflect.Type]flag.Value{}
	var stringParser stringValue
	var boolParser boolValue
	var intParser intValue
	var timeParser timeValue
	parsers[reflect.TypeOf("")] = &stringParser
	parsers[reflect.TypeOf(true)] = &boolParser
	parsers[reflect.TypeOf(1)] = &intParser
	parsers[reflect.TypeOf(time.Now())] = &timeParser
	for rType, parser := range customParsers {
		parsers[rType] = parser
	}
	return parsers, nil
}

//Load initializes config : struct fields given by reference, with args : arguments.
//Some custom parsers may be given.
func Load(config interface{}, args []string, customParsers map[reflect.Type]flag.Value) error {
	parsers, err := loadParsers(customParsers)
	if err != nil {
		return err
	}
	tagsmap := make(map[string]reflect.Type)
	if err := getTypesRecursive(reflect.ValueOf(config), tagsmap, ""); err != nil {
		return err
	}
	valmap, err := parseArgs(args, tagsmap, parsers)
	if err != nil {
		return err
	}
	if err := fillStructRecursive(reflect.ValueOf(config), valmap, ""); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package hbasekv

import (
	"fmt"
	"math/rand"
	"strings"
	"sync"
	"time"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/pingcap/go-hbase"
	"github.com/pingcap/go-themis"
	"github.com/pingcap/go-themis/oracle"
	"github.com/pingcap/go-themis/oracle/oracles"
	"github.com/pingcap/tidb/kv"
)

const (
	// hbaseColFamily is the hbase column family name.
	hbaseColFamily = "f"
	// hbaseQualifier is the hbase column name.
	hbaseQualifier = "q"
	// hbaseFmlAndQual is a shortcut.
	hbaseFmlAndQual = hbaseColFamily + ":" + hbaseQualifier
	// fix length conn pool
	hbaseConnPoolSize = 10
)

var (
	hbaseColFamilyBytes = []byte(hbaseColFamily)
	hbaseQualifierBytes = []byte(hbaseQualifier)
)

var (
	_ kv.Storage = (*hbaseStore)(nil)
)

var (
	// ErrInvalidDSN is returned when store dsn is invalid.
	ErrInvalidDSN = errors.New("invalid dsn")
)

type storeCache struct {
	mu    sync.Mutex
	cache map[string]*hbaseStore
}

var mc storeCache

func init() {
	mc.cache = make(map[string]*hbaseStore)
	rand.Seed(time.Now().UnixNano())
}

type hbaseStore struct {
	mu        sync.Mutex
	dsn       string
	storeName string
	oracle    oracle.Oracle
	conns     []hbase.HBaseClient
}

func (s *hbaseStore) getHBaseClient() hbase.HBaseClient {
	// return hbase connection randomly
	return s.conns[rand.Intn(hbaseConnPoolSize)]
}

func (s *hbaseStore) Begin() (kv.Transaction, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	hbaseCli := s.getHBaseClient()
	t, err := themis.NewTxn(hbaseCli, s.oracle)
	if err != nil {
		return nil, errors.Trace(err)
	}
	txn := newHbaseTxn(t, s.storeName)
	return txn, nil
}

func (s *hbaseStore) GetSnapshot(ver kv.Version) (kv.Snapshot, error) {
	hbaseCli := s.getHBaseClient()
	t, err := themis.NewTxn(hbaseCli, s.oracle)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return newHbaseSnapshot(t, s.storeName), nil
}

func (s *hbaseStore) Close() error {
	mc.mu.Lock()
	defer mc.mu.Unlock()

	delete(mc.cache, s.dsn)

	var err error
	for _, conn := range s.conns {
		err = conn.Close()
		if err != nil {
			log.Error(err)
		}
	}
	// return last error
	return err
}

func (s *hbaseStore) UUID() string {
	return fmt.Sprintf("hbase.%s.%s", s.storeName, s.dsn)
}

func (s *hbaseStore) CurrentVersion() (kv.Version, error) {
	hbaseCli := s.getHBaseClient()
	t, err := themis.NewTxn(hbaseCli, s.oracle)
	if err != nil {
		return kv.Version{Ver: 0}, errors.Trace(err)
	}
	defer t.Release()

	return kv.Version{Ver: t.GetStartTS()}, nil
}

// Driver implements engine Driver.
type Driver struct {
}

// Open opens or creates an HBase storage with given dsn, format should be 'zk1,zk2,zk3|tsoaddr:port/tblName'.
// If tsoAddr is not provided, it will use a local oracle instead.
func (d Driver) Open(dsn string) (kv.Storage, error) {
	mc.mu.Lock()
	defer mc.mu.Unlock()

	if store, ok := mc.cache[dsn]; ok {
		// TODO: check the cache store has the same engine with this Driver.
		return store, nil
	}

	zks, oracleAddr, tableName, err := parseDSN(dsn)
	if err != nil {
		return nil, errors.Trace(err)
	}

	// create buffered HBase connections, HBaseClient is goroutine-safe, so
	// it's OK to redistribute to transactions.
	conns := make([]hbase.HBaseClient, 0, hbaseConnPoolSize)
	for i := 0; i < hbaseConnPoolSize; i++ {
		var c hbase.HBaseClient
		c, err = hbase.NewClient(zks, "/hbase")
		if err != nil {
			return nil, errors.Trace(err)
		}
		conns = append(conns, c)
	}

	c := conns[0]
	var b bool
	b, err = c.TableExists(tableName)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if !b {
		// Create new hbase table for store.
		t := hbase.NewTableDesciptor(hbase.NewTableNameWithDefaultNS(tableName))
		cf := hbase.NewColumnFamilyDescriptor(hbaseColFamily)
		cf.AddStrAddr("THEMIS_ENABLE", "true")
		t.AddColumnDesc(cf)
		//TODO: specify split?
		if err := c.CreateTable(t, nil); err != nil {
			return nil, errors.Trace(err)
		}
	}

	var ora oracle.Oracle
	if len(oracleAddr) == 0 {
		ora = oracles.NewLocalOracle()
	} else {
		ora = oracles.NewRemoteOracle(oracleAddr)
	}

	s := &hbaseStore{
		dsn:       dsn,
		storeName: tableName,
		oracle:    ora,
		conns:     conns,
	}
	mc.cache[dsn] = s
	return s, nil
}

func parseDSN(dsn string) (zks []string, oracleAddr, tableName string, err error) {
	pos := strings.LastIndex(dsn, "/")
	if pos == -1 {
		err = errors.Trace(ErrInvalidDSN)
		return
	}
	tableName = dsn[pos+1:]
	addrs := dsn[:pos]

	pos = strings.LastIndex(addrs, "|")
	if pos != -1 {
		oracleAddr = addrs[pos+1:]
		addrs = addrs[:pos]
	}
	zks = strings.Split(addrs, ",")
	return
}
<commit_msg>store/hbase: update for go-hbase.<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package hbasekv

import (
	"fmt"
	"math/rand"
	"strings"
	"sync"
	"time"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/pingcap/go-hbase"
	"github.com/pingcap/go-themis"
	"github.com/pingcap/go-themis/oracle"
	"github.com/pingcap/go-themis/oracle/oracles"
	"github.com/pingcap/tidb/kv"
)

const (
	// hbaseColFamily is the hbase column family name.
	hbaseColFamily = "f"
	// hbaseQualifier is the hbase column name.
	hbaseQualifier = "q"
	// hbaseFmlAndQual is a shortcut.
	hbaseFmlAndQual = hbaseColFamily + ":" + hbaseQualifier
	// fix length conn pool
	hbaseConnPoolSize = 10
)

var (
	hbaseColFamilyBytes = []byte(hbaseColFamily)
	hbaseQualifierBytes = []byte(hbaseQualifier)
)

var (
	_ kv.Storage = (*hbaseStore)(nil)
)

var (
	// ErrInvalidDSN is returned when store dsn is invalid.
	ErrInvalidDSN = errors.New("invalid dsn")
)

type storeCache struct {
	mu    sync.Mutex
	cache map[string]*hbaseStore
}

var mc storeCache

func init() {
	mc.cache = make(map[string]*hbaseStore)
	rand.Seed(time.Now().UnixNano())
}

type hbaseStore struct {
	mu        sync.Mutex
	dsn       string
	storeName string
	oracle    oracle.Oracle
	conns     []hbase.HBaseClient
}

func (s *hbaseStore) getHBaseClient() hbase.HBaseClient {
	// return hbase connection randomly
	return s.conns[rand.Intn(hbaseConnPoolSize)]
}

func (s *hbaseStore) Begin() (kv.Transaction, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	hbaseCli := s.getHBaseClient()
	t, err := themis.NewTxn(hbaseCli, s.oracle)
	if err != nil {
		return nil, errors.Trace(err)
	}
	txn := newHbaseTxn(t, s.storeName)
	return txn, nil
}

func (s *hbaseStore) GetSnapshot(ver kv.Version) (kv.Snapshot, error) {
	hbaseCli := s.getHBaseClient()
	t, err := themis.NewTxn(hbaseCli, s.oracle)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return newHbaseSnapshot(t, s.storeName), nil
}

func (s *hbaseStore) Close() error {
	mc.mu.Lock()
	defer mc.mu.Unlock()

	delete(mc.cache, s.dsn)

	var err error
	for _, conn := range s.conns {
		err = conn.Close()
		if err != nil {
			log.Error(err)
		}
	}
	// return last error
	return err
}

func (s *hbaseStore) UUID() string {
	return fmt.Sprintf("hbase.%s.%s", s.storeName, s.dsn)
}

func (s *hbaseStore) CurrentVersion() (kv.Version, error) {
	hbaseCli := s.getHBaseClient()
	t, err := themis.NewTxn(hbaseCli, s.oracle)
	if err != nil {
		return kv.Version{Ver: 0}, errors.Trace(err)
	}
	defer t.Release()

	return kv.Version{Ver: t.GetStartTS()}, nil
}

// Driver implements engine Driver.
type Driver struct {
}

// Open opens or creates an HBase storage with given dsn, format should be 'zk1,zk2,zk3|tsoaddr:port/tblName'.
// If tsoAddr is not provided, it will use a local oracle instead.
func (d Driver) Open(dsn string) (kv.Storage, error) {
	mc.mu.Lock()
	defer mc.mu.Unlock()

	if store, ok := mc.cache[dsn]; ok {
		// TODO: check the cache store has the same engine with this Driver.
		return store, nil
	}

	zks, oracleAddr, tableName, err := parseDSN(dsn)
	if err != nil {
		return nil, errors.Trace(err)
	}

	// create buffered HBase connections, HBaseClient is goroutine-safe, so
	// it's OK to redistribute to transactions.
	conns := make([]hbase.HBaseClient, 0, hbaseConnPoolSize)
	for i := 0; i < hbaseConnPoolSize; i++ {
		var c hbase.HBaseClient
		c, err = hbase.NewClient(zks, "/hbase")
		if err != nil {
			return nil, errors.Trace(err)
		}
		conns = append(conns, c)
	}

	c := conns[0]
	var b bool
	b, err = c.TableExists(tableName)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if !b {
		// Create new hbase table for store.
		t := hbase.NewTableDesciptor(hbase.NewTableNameWithDefaultNS(tableName))
		cf := hbase.NewColumnFamilyDescriptor(hbaseColFamily)
		cf.AddAttr("THEMIS_ENABLE", "true")
		t.AddColumnDesc(cf)
		//TODO: specify split?
		if err := c.CreateTable(t, nil); err != nil {
			return nil, errors.Trace(err)
		}
	}

	var ora oracle.Oracle
	if len(oracleAddr) == 0 {
		ora = oracles.NewLocalOracle()
	} else {
		ora = oracles.NewRemoteOracle(oracleAddr)
	}

	s := &hbaseStore{
		dsn:       dsn,
		storeName: tableName,
		oracle:    ora,
		conns:     conns,
	}
	mc.cache[dsn] = s
	return s, nil
}

func parseDSN(dsn string) (zks []string, oracleAddr, tableName string, err error) {
	pos := strings.LastIndex(dsn, "/")
	if pos == -1 {
		err = errors.Trace(ErrInvalidDSN)
		return
	}
	tableName = dsn[pos+1:]
	addrs := dsn[:pos]

	pos = strings.LastIndex(addrs, "|")
	if pos != -1 {
		oracleAddr = addrs[pos+1:]
		addrs = addrs[:pos]
	}
	zks = strings.Split(addrs, ",")
	return
}
<|endoftext|>


<commit_before>package remote

import (
	"log"
	"sync"
	"time"

	"github.com/yinqiwen/gsnova/common/event"
)

type ConnEventQueue struct {
	event.EventQueue
	id         ConnId
	activeTime time.Time
}

var queueTable map[ConnId]*ConnEventQueue = make(map[ConnId]*ConnEventQueue)
var queueMutex sync.Mutex

var freeQueueTable = make(map[*ConnEventQueue]bool)
var freeQueueMutex sync.Mutex

func GetEventQueueSize() int {
	queueMutex.Lock()
	defer queueMutex.Unlock()
	return len(queueTable)
}

func removeExpiredConnEventQueue(id ConnId) {
	queueMutex.Lock()
	defer queueMutex.Unlock()
	delete(queueTable, id)
}

func getEventQueue(cid ConnId, createIfMissing bool) *ConnEventQueue {
	queueMutex.Lock()
	defer queueMutex.Unlock()
	q := queueTable[cid]
	if nil == q {
		if createIfMissing {
			q = new(ConnEventQueue)
			q.EventQueue = *(event.NewEventQueue())
			q.activeTime = time.Now()
			q.id = cid
			queueTable[cid] = q
			return q
		} else {
			return nil
		}
	}
	return q
}

func GetEventQueue(cid ConnId, createIfMissing bool) *ConnEventQueue {
	q := getEventQueue(cid, createIfMissing)
	if nil != q {
		freeQueueMutex.Lock()
		delete(freeQueueTable, q)
		freeQueueMutex.Unlock()
	}
	return q
}

func ReleaseEventQueue(q *ConnEventQueue) {
	if nil != q {
		freeQueueMutex.Lock()
		freeQueueTable[q] = true
		freeQueueMutex.Unlock()
	}
}

func init() {
	go func() {
		ticker := time.NewTicker(1 * time.Minute)
		for {
			select {
			case <-ticker.C:
				freeQueueMutex.Lock()
				for q, _ := range freeQueueTable {
					if q.activeTime.Add(30 * time.Second).Before(time.Now()) {
						removeExpiredConnEventQueue(q.id)
						delete(freeQueueTable, q)
						log.Printf("Remove old conn event queue by id:%v", q.id)
					}
				}
				freeQueueMutex.Unlock()
			}
		}
	}()
}
<commit_msg>update<commit_after>package remote

import (
	"log"
	"sync"
	"time"

	"github.com/yinqiwen/gsnova/common/event"
)

type ConnEventQueue struct {
	event.EventQueue
	id         ConnId
	activeTime time.Time
}

var queueTable map[ConnId]*ConnEventQueue = make(map[ConnId]*ConnEventQueue)
var queueMutex sync.Mutex

var freeQueueTable = make(map[*ConnEventQueue]bool)
var freeQueueMutex sync.Mutex

func GetEventQueueSize() int {
	queueMutex.Lock()
	defer queueMutex.Unlock()
	return len(queueTable)
}

func removeExpiredConnEventQueue(id ConnId) {
	queueMutex.Lock()
	defer queueMutex.Unlock()
	delete(queueTable, id)
}

func getEventQueue(cid ConnId, createIfMissing bool) *ConnEventQueue {
	queueMutex.Lock()
	defer queueMutex.Unlock()
	q := queueTable[cid]
	if nil == q {
		if createIfMissing {
			q = new(ConnEventQueue)
			q.EventQueue = *(event.NewEventQueue())
			q.activeTime = time.Now()
			q.id = cid
			queueTable[cid] = q
			return q
		} else {
			return nil
		}
	}
	return q
}

func GetEventQueue(cid ConnId, createIfMissing bool) *ConnEventQueue {
	q := getEventQueue(cid, createIfMissing)
	if nil != q {
		freeQueueMutex.Lock()
		delete(freeQueueTable, q)
		freeQueueMutex.Unlock()
	}
	return q
}

func ReleaseEventQueue(q *ConnEventQueue) {
	if nil != q {
		q.activeTime = time.Now()
		freeQueueMutex.Lock()
		freeQueueTable[q] = true
		freeQueueMutex.Unlock()
	}
}

func init() {
	go func() {
		ticker := time.NewTicker(1 * time.Minute)
		for {
			select {
			case <-ticker.C:
				freeQueueMutex.Lock()
				for q, _ := range freeQueueTable {
					if q.activeTime.Add(30 * time.Second).Before(time.Now()) {
						removeExpiredConnEventQueue(q.id)
						delete(freeQueueTable, q)
						log.Printf("Remove old conn event queue by id:%v", q.id)
					}
				}
				freeQueueMutex.Unlock()
			}
		}
	}()
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/stretchr/testify/assert"
	"testing"
)

var cr = CassandraRepository{}

func TestWhenIFindWithID1_ShoudReturnSucessful(t *testing.T) {
	actualValue, _ := cr.find(1)
	assert.Equal(t, sampleAlert, actualValue, "Must return correct Alert")
}

/*
func TestWhenIUpsertAlert_ShouldReturnOK(t *testing.T) {
	insertedAlert, err := cr.upsert(sampleAlert)
	assert.Equal(t, sampleAlert, insertedAlert, "Insert alert to map should be successful")
	assert.Nil(t, err, "Error must be nil")
}*/
<commit_msg>#13 @kavasoglu Add separation between unit and integration tests<commit_after>// +build !unit

package main

import (
	"github.com/stretchr/testify/assert"
	"testing"
)

var cr = CassandraRepository{}

func TestWhenIFindWithID1_ShoudReturnSucessful(t *testing.T) {
	actualValue, _ := cr.find(1)
	assert.Equal(t, sampleAlert, actualValue, "Must return correct Alert")
}

/*
func TestWhenIUpsertAlert_ShouldReturnOK(t *testing.T) {
	insertedAlert, err := cr.upsert(sampleAlert)
	assert.Equal(t, sampleAlert, insertedAlert, "Insert alert to map should be successful")
	assert.Nil(t, err, "Error must be nil")
}*/
<|endoftext|>


<commit_before>// Package fs provides file system handleFuncs that can be used with kite
// library
package fs

import (
	"errors"
	"log"
	"os"
	"path"
	"sync"

	"github.com/howeyc/fsnotify"
	"github.com/koding/kite"
	"github.com/koding/kite/dnode"
)

var (
	// watcher variables
	once               sync.Once
	newPaths, oldPaths = make(chan string), make(chan string)
	watchCallbacks     = make(map[string]func(*fsnotify.FileEvent), 100) // Limit of watching folders
)

func ReadDirectory(r *kite.Request) (interface{}, error) {
	var params struct {
		Path     string
		OnChange dnode.Function
	}

	if r.Args == nil {
		return nil, errors.New("arguments are not passed")
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		log.Println("params", params)
		return nil, errors.New("{ path: [string], onChange: [function]}")
	}

	response := make(map[string]interface{})

	if params.OnChange.IsValid() {
		onceBody := func() { startWatcher() }
		go once.Do(onceBody)

		// notify new paths to the watcher
		newPaths <- params.Path

		var eventType string
		var fileEntry *FileEntry

		changer := func(ev *fsnotify.FileEvent) {
			if ev.IsCreate() {
				eventType = "added"
				fileEntry, _ = getInfo(ev.Name)
			} else if ev.IsDelete() {
				eventType = "removed"
				fileEntry = NewFileEntry(path.Base(ev.Name), ev.Name)
			}

			event := map[string]interface{}{
				"event": eventType,
				"file":  fileEntry,
			}

			params.OnChange.Call(event)
			return
		}

		watchCallbacks[params.Path] = changer

		// TODO: handle them together
		r.Client.OnDisconnect(func() {
			delete(watchCallbacks, params.Path)
			oldPaths <- params.Path
		})

		// this callback is called whenever we receive a 'stopWatching' from the client
		response["stopWatching"] = dnode.Callback(func(r *dnode.Partial) {
			delete(watchCallbacks, params.Path)
			oldPaths <- params.Path
		})
	}

	files, err := readDirectory(params.Path)
	if err != nil {
		return nil, err
	}

	response["files"] = files
	return response, nil
}

func startWatcher() {
	var err error
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	go func() {
		for {
			select {
			case p := <-newPaths:
				err := watcher.Watch(p)
				if err != nil {
					log.Println("watch path adding", err)
				}
			case p := <-oldPaths:
				err := watcher.RemoveWatch(p)
				if err != nil {
					log.Println("watch remove adding", err)
				}
			}
		}
	}()

	for event := range watcher.Event {
		f, ok := watchCallbacks[path.Dir(event.Name)]
		if !ok {
			continue
		}

		f(event)
	}
}

func Glob(r *kite.Request) (interface{}, error) {
	var params struct {
		Pattern string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Pattern == "" {
		return nil, errors.New("{ pattern: [string] }")
	}

	return glob(params.Pattern)
}

func ReadFile(r *kite.Request) (interface{}, error) {
	var params struct {
		Path string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string] }")
	}

	return readFile(params.Path)
}

type writeFileParams struct {
	Path           string
	Content        []byte
	DoNotOverwrite bool
	Append         bool
}

func WriteFile(r *kite.Request) (interface{}, error) {
	var params writeFileParams
	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string] }")
	}

	return writeFile(params.Path, params.Content, params.DoNotOverwrite, params.Append)
}

func UniquePath(r *kite.Request) (interface{}, error) {
	var params struct {
		Path string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string] }")
	}

	return uniquePath(params.Path)
}

func GetInfo(r *kite.Request) (interface{}, error) {
	var params struct {
		Path string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string] }")
	}

	return getInfo(params.Path)
}

type setPermissionsParams struct {
	Path      string
	Mode      os.FileMode
	Recursive bool
}

func SetPermissions(r *kite.Request) (interface{}, error) {
	var params setPermissionsParams

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string], mode: [integer], recursive: [bool] }")
	}

	err := setPermissions(params.Path, params.Mode, params.Recursive)
	if err != nil {
		return nil, err
	}

	return true, nil
}

func Remove(r *kite.Request) (interface{}, error) {
	var params struct {
		Path      string
		Recursive bool
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string], recursive: [bool] }")
	}

	if err := remove(params.Path, params.Recursive); err != nil {
		return nil, err
	}

	return true, nil
}

func Rename(r *kite.Request) (interface{}, error) {
	var params struct {
		OldPath string
		NewPath string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.OldPath == "" || params.NewPath == "" {
		return nil, errors.New("{ oldPath: [string], newPath: [string] }")
	}

	err := rename(params.OldPath, params.NewPath)
	if err != nil {
		return nil, err
	}

	return true, nil
}

func CreateDirectory(r *kite.Request) (interface{}, error) {
	var params struct {
		Path      string
		Recursive bool
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string], recursive: [bool] }")
	}

	err := createDirectory(params.Path, params.Recursive)
	if err != nil {
		return nil, err
	}

	return true, nil
}

func Move(r *kite.Request) (interface{}, error) {
	var params struct {
		OldPath string
		NewPath string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.OldPath == "" || params.NewPath == "" {
		return nil, errors.New("{ oldPath: [string], newPath: [string] }")
	}

	err := rename(params.OldPath, params.NewPath)
	if err != nil {
		return nil, err
	}

	return true, nil
}

func Copy(r *kite.Request) (interface{}, error) {
	var params struct {
		SrcPath string
		DstPath string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.SrcPath == "" || params.DstPath == "" {
		return nil, errors.New("{ srcPath: [string], dstPath: [string] }")
	}

	err := cp(params.SrcPath, params.DstPath)
	if err != nil {
		return nil, err
	}

	return true, nil
}
<commit_msg>kite-handler/fs: fix race condition on watcher callbacks<commit_after>// Package fs provides file system handleFuncs that can be used with kite
// library
package fs

import (
	"errors"
	"log"
	"os"
	"path"
	"sync"

	"github.com/howeyc/fsnotify"
	"github.com/koding/kite"
	"github.com/koding/kite/dnode"
)

var (
	// watcher variables
	once               sync.Once
	newPaths, oldPaths = make(chan string), make(chan string)

	// Limit of watching folders
	watchCallbacks = make(map[string]func(*fsnotify.FileEvent), 100)
	mu             sync.Mutex // protects watchCallbacks
)

func ReadDirectory(r *kite.Request) (interface{}, error) {
	var params struct {
		Path     string
		OnChange dnode.Function
	}

	if r.Args == nil {
		return nil, errors.New("arguments are not passed")
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		log.Println("params", params)
		return nil, errors.New("{ path: [string], onChange: [function]}")
	}

	response := make(map[string]interface{})

	if params.OnChange.IsValid() {
		onceBody := func() { startWatcher() }
		go once.Do(onceBody)

		// notify new paths to the watcher
		newPaths <- params.Path

		var eventType string
		var fileEntry *FileEntry

		changer := func(ev *fsnotify.FileEvent) {
			if ev.IsCreate() {
				eventType = "added"
				fileEntry, _ = getInfo(ev.Name)
			} else if ev.IsDelete() {
				eventType = "removed"
				fileEntry = NewFileEntry(path.Base(ev.Name), ev.Name)
			}

			event := map[string]interface{}{
				"event": eventType,
				"file":  fileEntry,
			}

			params.OnChange.Call(event)
			return
		}

		mu.Lock()
		watchCallbacks[params.Path] = changer
		mu.Unlock()

		removePath := func() {
			mu.Lock()
			delete(watchCallbacks, params.Path)
			mu.Unlock()

			oldPaths <- params.Path
		}

		// remove the path when the remote client disconnects
		r.Client.OnDisconnect(removePath)

		// this callback is called whenever we receive a 'stopWatching' from the client
		response["stopWatching"] = dnode.Callback(func(r *dnode.Partial) {
			removePath()
		})
	}

	files, err := readDirectory(params.Path)
	if err != nil {
		return nil, err
	}

	response["files"] = files
	return response, nil
}

func startWatcher() {
	var err error
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	go func() {
		for {
			select {
			case p := <-newPaths:
				err := watcher.Watch(p)
				if err != nil {
					log.Println("watch path adding", err)
				}
			case p := <-oldPaths:
				err := watcher.RemoveWatch(p)
				if err != nil {
					log.Println("watch remove adding", err)
				}
			}
		}
	}()

	for event := range watcher.Event {
		mu.Lock()
		f, ok := watchCallbacks[path.Dir(event.Name)]
		mu.Unlock()

		if !ok {
			continue
		}

		f(event)
	}
}

func Glob(r *kite.Request) (interface{}, error) {
	var params struct {
		Pattern string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Pattern == "" {
		return nil, errors.New("{ pattern: [string] }")
	}

	return glob(params.Pattern)
}

func ReadFile(r *kite.Request) (interface{}, error) {
	var params struct {
		Path string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string] }")
	}

	return readFile(params.Path)
}

type writeFileParams struct {
	Path           string
	Content        []byte
	DoNotOverwrite bool
	Append         bool
}

func WriteFile(r *kite.Request) (interface{}, error) {
	var params writeFileParams
	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string] }")
	}

	return writeFile(params.Path, params.Content, params.DoNotOverwrite, params.Append)
}

func UniquePath(r *kite.Request) (interface{}, error) {
	var params struct {
		Path string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string] }")
	}

	return uniquePath(params.Path)
}

func GetInfo(r *kite.Request) (interface{}, error) {
	var params struct {
		Path string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string] }")
	}

	return getInfo(params.Path)
}

type setPermissionsParams struct {
	Path      string
	Mode      os.FileMode
	Recursive bool
}

func SetPermissions(r *kite.Request) (interface{}, error) {
	var params setPermissionsParams

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string], mode: [integer], recursive: [bool] }")
	}

	err := setPermissions(params.Path, params.Mode, params.Recursive)
	if err != nil {
		return nil, err
	}

	return true, nil
}

func Remove(r *kite.Request) (interface{}, error) {
	var params struct {
		Path      string
		Recursive bool
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string], recursive: [bool] }")
	}

	if err := remove(params.Path, params.Recursive); err != nil {
		return nil, err
	}

	return true, nil
}

func Rename(r *kite.Request) (interface{}, error) {
	var params struct {
		OldPath string
		NewPath string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.OldPath == "" || params.NewPath == "" {
		return nil, errors.New("{ oldPath: [string], newPath: [string] }")
	}

	err := rename(params.OldPath, params.NewPath)
	if err != nil {
		return nil, err
	}

	return true, nil
}

func CreateDirectory(r *kite.Request) (interface{}, error) {
	var params struct {
		Path      string
		Recursive bool
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string], recursive: [bool] }")
	}

	err := createDirectory(params.Path, params.Recursive)
	if err != nil {
		return nil, err
	}

	return true, nil
}

func Move(r *kite.Request) (interface{}, error) {
	var params struct {
		OldPath string
		NewPath string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.OldPath == "" || params.NewPath == "" {
		return nil, errors.New("{ oldPath: [string], newPath: [string] }")
	}

	err := rename(params.OldPath, params.NewPath)
	if err != nil {
		return nil, err
	}

	return true, nil
}

func Copy(r *kite.Request) (interface{}, error) {
	var params struct {
		SrcPath string
		DstPath string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.SrcPath == "" || params.DstPath == "" {
		return nil, errors.New("{ srcPath: [string], dstPath: [string] }")
	}

	err := cp(params.SrcPath, params.DstPath)
	if err != nil {
		return nil, err
	}

	return true, nil
}
<|endoftext|>


<commit_before>package render

import (
	"net"
	"strings"

	"github.com/weaveworks/scope/probe/endpoint"
	"github.com/weaveworks/scope/report"
)

// Constants are used in the tests.
const (
	TheInternetID      = "theinternet"
	IncomingInternetID = "in-" + TheInternetID
	OutgoingInternetID = "out-" + TheInternetID
)

// MakePseudoNodeID joins the parts of an id into the id of a pseudonode
func MakePseudoNodeID(parts ...string) string {
	return strings.Join(append([]string{"pseudo"}, parts...), ":")
}

// MakeGroupNodeTopology joins the parts of a group topology into the topology of a group node
func MakeGroupNodeTopology(originalTopology, key string) string {
	return strings.Join([]string{"group", originalTopology, key}, ":")
}

// NewDerivedNode makes a node based on node, but with a new ID
func NewDerivedNode(id string, node report.Node) report.Node {
	return report.MakeNode(id).WithChildren(node.Children.Add(node))
}

// NewDerivedPseudoNode makes a new pseudo node with the node as a child
func NewDerivedPseudoNode(id string, node report.Node) report.Node {
	output := NewDerivedNode(id, node).WithTopology(Pseudo)
	return output
}

// NewDerivedExternalNode figures out if a node should be considered external and creates the corresponding pseudo node
func NewDerivedExternalNode(n report.Node, addr string, local report.Networks) (report.Node, bool) {
	// First, check if it's a known service and emit a a specific node if it
	// is. This needs to be done before checking IPs since known services can
	// live in the same network, see https://github.com/weaveworks/scope/issues/2163
	if hostname, found := DNSFirstMatch(n, isKnownService); found {
		return NewDerivedPseudoNode(ServiceNodeIDPrefix+hostname, n), true
	}

	// If the dstNodeAddr is not in a network local to this report, we emit an
	// internet pseudoNode
	if ip := net.ParseIP(addr); ip != nil && !local.Contains(ip) {
		// emit one internet node for incoming, one for outgoing
		if len(n.Adjacency) > 0 {
			return NewDerivedPseudoNode(IncomingInternetID, n), true
		}
		return NewDerivedPseudoNode(OutgoingInternetID, n), true
	}

	// The node is not external
	return report.Node{}, false
}

// DNSFirstMatch returns the first DNS name where match() returns
// true, from a prioritized list of snooped and reverse-resolved DNS
// names associated with node n.
func DNSFirstMatch(n report.Node, match func(name string) bool) (string, bool) {
	// we rely on Sets being sorted, to make selection for display more
	// deterministic
	// prioritize snooped names
	snoopedNames, _ := n.Sets.Lookup(endpoint.SnoopedDNSNames)
	for _, hostname := range snoopedNames {
		if match(hostname) {
			return hostname, true
		}
	}
	reverseNames, _ := n.Sets.Lookup(endpoint.ReverseDNSNames)
	for _, hostname := range reverseNames {
		if match(hostname) {
			return hostname, true
		}
	}
	return "", false
}
<commit_msg>Refactor: extract externalNodeID function<commit_after>package render

import (
	"net"
	"strings"

	"github.com/weaveworks/scope/probe/endpoint"
	"github.com/weaveworks/scope/report"
)

// Constants are used in the tests.
const (
	TheInternetID      = "theinternet"
	IncomingInternetID = "in-" + TheInternetID
	OutgoingInternetID = "out-" + TheInternetID
)

// MakePseudoNodeID joins the parts of an id into the id of a pseudonode
func MakePseudoNodeID(parts ...string) string {
	return strings.Join(append([]string{"pseudo"}, parts...), ":")
}

// MakeGroupNodeTopology joins the parts of a group topology into the topology of a group node
func MakeGroupNodeTopology(originalTopology, key string) string {
	return strings.Join([]string{"group", originalTopology, key}, ":")
}

// NewDerivedNode makes a node based on node, but with a new ID
func NewDerivedNode(id string, node report.Node) report.Node {
	return report.MakeNode(id).WithChildren(node.Children.Add(node))
}

// NewDerivedPseudoNode makes a new pseudo node with the node as a child
func NewDerivedPseudoNode(id string, node report.Node) report.Node {
	output := NewDerivedNode(id, node).WithTopology(Pseudo)
	return output
}

// NewDerivedExternalNode figures out if a node should be considered external and creates the corresponding pseudo node
func NewDerivedExternalNode(n report.Node, addr string, local report.Networks) (report.Node, bool) {
	id, ok := externalNodeID(n, addr, local)
	if !ok {
		return report.Node{}, false
	}
	return NewDerivedPseudoNode(id, n), true
}

// figure out if a node should be considered external and returns an ID which can be used to create a pseudo node
func externalNodeID(n report.Node, addr string, local report.Networks) (string, bool) {
	// First, check if it's a known service and emit a a specific node if it
	// is. This needs to be done before checking IPs since known services can
	// live in the same network, see https://github.com/weaveworks/scope/issues/2163
	if hostname, found := DNSFirstMatch(n, isKnownService); found {
		return ServiceNodeIDPrefix + hostname, true
	}

	// If the dstNodeAddr is not in a network local to this report, we emit an
	// internet pseudoNode
	if ip := net.ParseIP(addr); ip != nil && !local.Contains(ip) {
		// emit one internet node for incoming, one for outgoing
		if len(n.Adjacency) > 0 {
			return IncomingInternetID, true
		}
		return OutgoingInternetID, true
	}

	// The node is not external
	return "", false
}

// DNSFirstMatch returns the first DNS name where match() returns
// true, from a prioritized list of snooped and reverse-resolved DNS
// names associated with node n.
func DNSFirstMatch(n report.Node, match func(name string) bool) (string, bool) {
	// we rely on Sets being sorted, to make selection for display more
	// deterministic
	// prioritize snooped names
	snoopedNames, _ := n.Sets.Lookup(endpoint.SnoopedDNSNames)
	for _, hostname := range snoopedNames {
		if match(hostname) {
			return hostname, true
		}
	}
	reverseNames, _ := n.Sets.Lookup(endpoint.ReverseDNSNames)
	for _, hostname := range reverseNames {
		if match(hostname) {
			return hostname, true
		}
	}
	return "", false
}
<|endoftext|>


<commit_before>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build js

package oto

import (
	"errors"
	"strings"

	"github.com/gopherjs/gopherjs/js"
)

type player struct {
	sampleRate       int
	channelNum       int
	bytesPerSample   int
	nextPosInSamples int64
	tmp              []uint8
	bufferSize       int
	context          *js.Object
}

func isIOS() bool {
	ua := js.Global.Get("navigator").Get("userAgent").String()
	if !strings.Contains(ua, "iPhone") {
		return false
	}
	return true
}

func isAndroidChrome() bool {
	ua := js.Global.Get("navigator").Get("userAgent").String()
	if !strings.Contains(ua, "Android") {
		return false
	}
	if !strings.Contains(ua, "Chrome") {
		return false
	}
	return true
}

func newPlayer(sampleRate, channelNum, bytesPerSample, bufferSize int) (*player, error) {
	class := js.Global.Get("AudioContext")
	if class == js.Undefined {
		class = js.Global.Get("webkitAudioContext")
	}
	if class == js.Undefined {
		return nil, errors.New("oto: audio couldn't be initialized")
	}
	p := &player{
		sampleRate:     sampleRate,
		channelNum:     channelNum,
		bytesPerSample: bytesPerSample,
		context:        class.New(),
		bufferSize:     bufferSize,
	}
	// iOS and Android Chrome requires touch event to use AudioContext.
	if isIOS() || isAndroidChrome() {
		js.Global.Get("document").Call("addEventListener", "touchend", func() {
			// Resuming is necessary as of Chrome 55+ in some cases like different
			// domain page in an iframe.
			p.context.Call("resume")
			p.context.Call("createBufferSource").Call("start", 0)
			p.nextPosInSamples = int64(p.context.Get("currentTime").Float() * float64(p.sampleRate))
		})
	}
	p.nextPosInSamples = int64(p.context.Get("currentTime").Float() * float64(p.sampleRate))
	return p, nil
}

func toLR(data []uint8) ([]int16, []int16) {
	l := make([]int16, len(data)/4)
	r := make([]int16, len(data)/4)
	for i := 0; i < len(data)/4; i++ {
		l[i] = int16(data[4*i]) | int16(data[4*i+1])<<8
		r[i] = int16(data[4*i+2]) | int16(data[4*i+3])<<8
	}
	return l, r
}

func (p *player) SetUnderrunCallback(f func()) {
	//TODO
}

func (p *player) Write(data []uint8) (int, error) {
	n := min(len(data), p.bufferSize-len(p.tmp))
	p.tmp = append(p.tmp, data[:n]...)

	c := int64(p.context.Get("currentTime").Float() * float64(p.sampleRate))

	if p.nextPosInSamples < c {
		p.nextPosInSamples = c
	}

	sizeInSamples := p.bufferSize / p.bytesPerSample / p.channelNum

	// It's too early to enqueue a buffer.
	// Highly likely, there are two playing buffers now.
	if c+int64(sizeInSamples) < p.nextPosInSamples {
		return n, nil
	}

	if len(p.tmp) < p.bufferSize {
		return n, nil
	}

	buf := p.context.Call("createBuffer", p.channelNum, sizeInSamples, p.sampleRate)
	l := buf.Call("getChannelData", 0)
	r := buf.Call("getChannelData", 1)
	il, ir := toLR(p.tmp)
	const max = 1 << 15
	for i := 0; i < len(il); i++ {
		l.SetIndex(i, float64(il[i])/max)
		r.SetIndex(i, float64(ir[i])/max)
	}

	s := p.context.Call("createBufferSource")
	s.Set("buffer", buf)
	s.Call("connect", p.context.Get("destination"))
	s.Call("start", float64(p.nextPosInSamples)/float64(p.sampleRate))
	p.nextPosInSamples += int64(len(il))

	p.tmp = nil
	return n, nil
}

func (p *player) Close() error {
	return nil
}
<commit_msg>js: Bug fix: tapping made audio faster<commit_after>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build js

package oto

import (
	"errors"
	"strings"

	"github.com/gopherjs/gopherjs/js"
)

type player struct {
	sampleRate       int
	channelNum       int
	bytesPerSample   int
	nextPosInSamples int64
	tmp              []uint8
	bufferSize       int
	context          *js.Object
}

func isIOS() bool {
	ua := js.Global.Get("navigator").Get("userAgent").String()
	if !strings.Contains(ua, "iPhone") {
		return false
	}
	return true
}

func isAndroidChrome() bool {
	ua := js.Global.Get("navigator").Get("userAgent").String()
	if !strings.Contains(ua, "Android") {
		return false
	}
	if !strings.Contains(ua, "Chrome") {
		return false
	}
	return true
}

func newPlayer(sampleRate, channelNum, bytesPerSample, bufferSize int) (*player, error) {
	class := js.Global.Get("AudioContext")
	if class == js.Undefined {
		class = js.Global.Get("webkitAudioContext")
	}
	if class == js.Undefined {
		return nil, errors.New("oto: audio couldn't be initialized")
	}
	p := &player{
		sampleRate:     sampleRate,
		channelNum:     channelNum,
		bytesPerSample: bytesPerSample,
		context:        class.New(),
		bufferSize:     bufferSize,
	}
	// iOS and Android Chrome requires touch event to use AudioContext.
	if isIOS() || isAndroidChrome() {
		var f *js.Object
		f = js.MakeFunc(func(this *js.Object, arguments []*js.Object) interface{} {
			// Resuming is necessary as of Chrome 55+ in some cases like different
			// domain page in an iframe.
			p.context.Call("resume")
			p.context.Call("createBufferSource").Call("start", 0)
			p.nextPosInSamples = int64(p.context.Get("currentTime").Float() * float64(p.sampleRate))
			js.Global.Get("document").Call("removeEventListener", "touchend", f)
			return nil
		})
		js.Global.Get("document").Call("addEventListener", "touchend", f)
	}
	p.nextPosInSamples = int64(p.context.Get("currentTime").Float() * float64(p.sampleRate))
	return p, nil
}

func toLR(data []uint8) ([]int16, []int16) {
	l := make([]int16, len(data)/4)
	r := make([]int16, len(data)/4)
	for i := 0; i < len(data)/4; i++ {
		l[i] = int16(data[4*i]) | int16(data[4*i+1])<<8
		r[i] = int16(data[4*i+2]) | int16(data[4*i+3])<<8
	}
	return l, r
}

func (p *player) SetUnderrunCallback(f func()) {
	//TODO
}

func (p *player) Write(data []uint8) (int, error) {
	n := min(len(data), p.bufferSize-len(p.tmp))
	p.tmp = append(p.tmp, data[:n]...)

	c := int64(p.context.Get("currentTime").Float() * float64(p.sampleRate))

	if p.nextPosInSamples < c {
		p.nextPosInSamples = c
	}

	sizeInSamples := p.bufferSize / p.bytesPerSample / p.channelNum

	// It's too early to enqueue a buffer.
	// Highly likely, there are two playing buffers now.
	if c+int64(sizeInSamples) < p.nextPosInSamples {
		return n, nil
	}

	if len(p.tmp) < p.bufferSize {
		return n, nil
	}

	buf := p.context.Call("createBuffer", p.channelNum, sizeInSamples, p.sampleRate)
	l := buf.Call("getChannelData", 0)
	r := buf.Call("getChannelData", 1)
	il, ir := toLR(p.tmp)
	const max = 1 << 15
	for i := 0; i < len(il); i++ {
		l.SetIndex(i, float64(il[i])/max)
		r.SetIndex(i, float64(ir[i])/max)
	}

	s := p.context.Call("createBufferSource")
	s.Set("buffer", buf)
	s.Call("connect", p.context.Get("destination"))
	s.Call("start", float64(p.nextPosInSamples)/float64(p.sampleRate))
	p.nextPosInSamples += int64(len(il))

	p.tmp = nil
	return n, nil
}

func (p *player) Close() error {
	return nil
}
<|endoftext|>


<commit_before>package cli

import (
	"errors"
	"flag"
	"strconv"
	"strings"
	"time"
)

// Context is a type that is passed through to
// each Handler action in a cli application. Context
// can be used to retrieve context-specific Args and
// parsed command-line options.
type Context struct {
	App     *App
	Command Command

	flagSet       *flag.FlagSet
	parentContext *Context
}

// NewContext creates a new context. For use in when invoking an App or Command action.
func NewContext(app *App, set *flag.FlagSet, parentCtx *Context) *Context {
	return &Context{App: app, flagSet: set, parentContext: parentCtx}
}

// Int looks up the value of a local int flag, returns 0 if no int flag exists
func (c *Context) Int(name string) int {
	if fs := lookupFlagSet(name, c); fs != nil {
		return lookupInt(name, fs)
	}
	return 0
}

// Duration looks up the value of a local time.Duration flag, returns 0 if no
// time.Duration flag exists
func (c *Context) Duration(name string) time.Duration {
	if fs := lookupFlagSet(name, c); fs != nil {
		return lookupDuration(name, fs)
	}
	return 0
}

// Float64 looks up the value of a local float64 flag, returns 0 if no float64
// flag exists
func (c *Context) Float64(name string) float64 {
	if fs := lookupFlagSet(name, c); fs != nil {
		return lookupFloat64(name, fs)
	}
	return 0
}

// Bool looks up the value of a local bool flag, returns false if no bool flag exists
func (c *Context) Bool(name string) bool {
	if fs := lookupFlagSet(name, c); fs != nil {
		return lookupBool(name, fs)
	}
	return false
}

// BoolT looks up the value of a local boolT flag, returns false if no bool flag exists
func (c *Context) BoolT(name string) bool {
	if fs := lookupFlagSet(name, c); fs != nil {
		return lookupBoolT(name, fs)
	}
	return true
}

// String looks up the value of a local string flag, returns "" if no string flag exists
func (c *Context) String(name string) string {
	if fs := lookupFlagSet(name, c); fs != nil {
		return lookupString(name, fs)
	}
	return ""
}

// StringSlice looks up the value of a local string slice flag, returns nil if no
// string slice flag exists
func (c *Context) StringSlice(name string) []string {
	if fs := lookupFlagSet(name, c); fs != nil {
		return lookupStringSlice(name, fs)
	}
	return nil
}

// IntSlice looks up the value of a local int slice flag, returns nil if no int
// slice flag exists
func (c *Context) IntSlice(name string) []int {
	if fs := lookupFlagSet(name, c); fs != nil {
		return lookupIntSlice(name, fs)
	}
	return nil
}

// Generic looks up the value of a local generic flag, returns nil if no generic
// flag exists
func (c *Context) Generic(name string) interface{} {
	if fs := lookupFlagSet(name, c); fs != nil {
		return lookupGeneric(name, fs)
	}
	return nil
}

// NumFlags returns the number of flags set
func (c *Context) NumFlags() int {
	return c.flagSet.NFlag()
}

// Set sets a context flag to a value.
func (c *Context) Set(name, value string) error {
	return c.flagSet.Set(name, value)
}

// IsSet determines if the flag was actually set
func (c *Context) IsSet(name string) bool {
	if fs := lookupFlagSet(name, c); fs != nil {
		isSet := false
		fs.Visit(func(f *flag.Flag) {
			if f.Name == name {
				isSet = true
			}
		})
		return isSet
	}
	return false
}

// LocalFlagNames returns a slice of flag names used in this context.
func (c *Context) LocalFlagNames() []string {
	names := []string{}
	c.flagSet.Visit(makeFlagNameVisitor(&names))
	return names
}

// FlagNames returns a slice of flag names used by the this context and all of
// its parent contexts.
func (c *Context) FlagNames() []string {
	names := []string{}

	for _, ctx := range c.Lineage() {
		ctx.flagSet.Visit(makeFlagNameVisitor(&names))
	}

	return names
}

// Lineage returns *this* context and all of its ancestor contexts in order from
// child to parent
func (c *Context) Lineage() []*Context {
	lineage := []*Context{}

	for cur := c; cur != nil; cur = cur.parentContext {
		lineage = append(lineage, cur)
	}

	return lineage
}

// Args contains apps console arguments
type Args []string

// Args returns the command line arguments associated with the context.
func (c *Context) Args() Args {
	args := Args(c.flagSet.Args())
	return args
}

// NArg returns the number of the command line arguments.
func (c *Context) NArg() int {
	return len(c.Args())
}

// Get returns the nth argument, or else a blank string
func (a Args) Get(n int) string {
	if len(a) > n {
		return a[n]
	}
	return ""
}

// First returns the first argument, or else a blank string
func (a Args) First() string {
	return a.Get(0)
}

// Tail returns the rest of the arguments (not the first one)
// or else an empty string slice
func (a Args) Tail() []string {
	if len(a) >= 2 {
		return []string(a)[1:]
	}
	return []string{}
}

// Present checks if there are any arguments present
func (a Args) Present() bool {
	return len(a) != 0
}

// Swap swaps arguments at the given indexes
func (a Args) Swap(from, to int) error {
	if from >= len(a) || to >= len(a) {
		return errors.New("index out of range")
	}
	a[from], a[to] = a[to], a[from]
	return nil
}

func lookupFlagSet(name string, ctx *Context) *flag.FlagSet {
	for _, c := range ctx.Lineage() {
		if f := c.flagSet.Lookup(name); f != nil {
			return c.flagSet
		}
	}

	return nil
}

func lookupInt(name string, set *flag.FlagSet) int {
	f := set.Lookup(name)
	if f != nil {
		val, err := strconv.Atoi(f.Value.String())
		if err != nil {
			return 0
		}
		return val
	}

	return 0
}

func lookupDuration(name string, set *flag.FlagSet) time.Duration {
	f := set.Lookup(name)
	if f != nil {
		val, err := time.ParseDuration(f.Value.String())
		if err == nil {
			return val
		}
	}

	return 0
}

func lookupFloat64(name string, set *flag.FlagSet) float64 {
	f := set.Lookup(name)
	if f != nil {
		val, err := strconv.ParseFloat(f.Value.String(), 64)
		if err != nil {
			return 0
		}
		return val
	}

	return 0
}

func lookupString(name string, set *flag.FlagSet) string {
	f := set.Lookup(name)
	if f != nil {
		return f.Value.String()
	}

	return ""
}

func lookupStringSlice(name string, set *flag.FlagSet) []string {
	f := set.Lookup(name)
	if f != nil {
		return (f.Value.(*StringSlice)).Value()

	}

	return nil
}

func lookupIntSlice(name string, set *flag.FlagSet) []int {
	f := set.Lookup(name)
	if f != nil {
		return (f.Value.(*IntSlice)).Value()

	}

	return nil
}

func lookupGeneric(name string, set *flag.FlagSet) interface{} {
	f := set.Lookup(name)
	if f != nil {
		return f.Value
	}
	return nil
}

func lookupBool(name string, set *flag.FlagSet) bool {
	f := set.Lookup(name)
	if f != nil {
		val, err := strconv.ParseBool(f.Value.String())
		if err != nil {
			return false
		}
		return val
	}

	return false
}

func lookupBoolT(name string, set *flag.FlagSet) bool {
	f := set.Lookup(name)
	if f != nil {
		val, err := strconv.ParseBool(f.Value.String())
		if err != nil {
			return true
		}
		return val
	}

	return true
}

func copyFlag(name string, ff *flag.Flag, set *flag.FlagSet) {
	switch ff.Value.(type) {
	case Serializeder:
		set.Set(name, ff.Value.(Serializeder).Serialized())
	default:
		set.Set(name, ff.Value.String())
	}
}

func normalizeFlags(flags []Flag, set *flag.FlagSet) error {
	visited := make(map[string]bool)
	set.Visit(func(f *flag.Flag) {
		visited[f.Name] = true
	})
	for _, f := range flags {
		parts := strings.Split(f.GetName(), ",")
		if len(parts) == 1 {
			continue
		}
		var ff *flag.Flag
		for _, name := range parts {
			name = strings.Trim(name, " ")
			if visited[name] {
				if ff != nil {
					return errors.New("Cannot use two forms of the same flag: " + name + " " + ff.Name)
				}
				ff = set.Lookup(name)
			}
		}
		if ff == nil {
			continue
		}
		for _, name := range parts {
			name = strings.Trim(name, " ")
			if !visited[name] {
				copyFlag(name, ff, set)
			}
		}
	}
	return nil
}

func makeFlagNameVisitor(names *[]string) func(*flag.Flag) {
	return func(f *flag.Flag) {
		name := strings.Split(f.Name, ",")[0]
		if name != "help" && name != "version" {
			(*names) = append(*names, name)
		}
	}
}
<commit_msg>Tidy up the flag name visitor func in Context<commit_after>package cli

import (
	"errors"
	"flag"
	"strconv"
	"strings"
	"time"
)

// Context is a type that is passed through to
// each Handler action in a cli application. Context
// can be used to retrieve context-specific Args and
// parsed command-line options.
type Context struct {
	App     *App
	Command Command

	flagSet       *flag.FlagSet
	parentContext *Context
}

// NewContext creates a new context. For use in when invoking an App or Command action.
func NewContext(app *App, set *flag.FlagSet, parentCtx *Context) *Context {
	return &Context{App: app, flagSet: set, parentContext: parentCtx}
}

// Int looks up the value of a local int flag, returns 0 if no int flag exists
func (c *Context) Int(name string) int {
	if fs := lookupFlagSet(name, c); fs != nil {
		return lookupInt(name, fs)
	}
	return 0
}

// Duration looks up the value of a local time.Duration flag, returns 0 if no
// time.Duration flag exists
func (c *Context) Duration(name string) time.Duration {
	if fs := lookupFlagSet(name, c); fs != nil {
		return lookupDuration(name, fs)
	}
	return 0
}

// Float64 looks up the value of a local float64 flag, returns 0 if no float64
// flag exists
func (c *Context) Float64(name string) float64 {
	if fs := lookupFlagSet(name, c); fs != nil {
		return lookupFloat64(name, fs)
	}
	return 0
}

// Bool looks up the value of a local bool flag, returns false if no bool flag exists
func (c *Context) Bool(name string) bool {
	if fs := lookupFlagSet(name, c); fs != nil {
		return lookupBool(name, fs)
	}
	return false
}

// BoolT looks up the value of a local boolT flag, returns false if no bool flag exists
func (c *Context) BoolT(name string) bool {
	if fs := lookupFlagSet(name, c); fs != nil {
		return lookupBoolT(name, fs)
	}
	return true
}

// String looks up the value of a local string flag, returns "" if no string flag exists
func (c *Context) String(name string) string {
	if fs := lookupFlagSet(name, c); fs != nil {
		return lookupString(name, fs)
	}
	return ""
}

// StringSlice looks up the value of a local string slice flag, returns nil if no
// string slice flag exists
func (c *Context) StringSlice(name string) []string {
	if fs := lookupFlagSet(name, c); fs != nil {
		return lookupStringSlice(name, fs)
	}
	return nil
}

// IntSlice looks up the value of a local int slice flag, returns nil if no int
// slice flag exists
func (c *Context) IntSlice(name string) []int {
	if fs := lookupFlagSet(name, c); fs != nil {
		return lookupIntSlice(name, fs)
	}
	return nil
}

// Generic looks up the value of a local generic flag, returns nil if no generic
// flag exists
func (c *Context) Generic(name string) interface{} {
	if fs := lookupFlagSet(name, c); fs != nil {
		return lookupGeneric(name, fs)
	}
	return nil
}

// NumFlags returns the number of flags set
func (c *Context) NumFlags() int {
	return c.flagSet.NFlag()
}

// Set sets a context flag to a value.
func (c *Context) Set(name, value string) error {
	return c.flagSet.Set(name, value)
}

// IsSet determines if the flag was actually set
func (c *Context) IsSet(name string) bool {
	if fs := lookupFlagSet(name, c); fs != nil {
		isSet := false
		fs.Visit(func(f *flag.Flag) {
			if f.Name == name {
				isSet = true
			}
		})
		return isSet
	}
	return false
}

// LocalFlagNames returns a slice of flag names used in this context.
func (c *Context) LocalFlagNames() []string {
	names := []string{}
	c.flagSet.Visit(makeFlagNameVisitor(&names))
	return names
}

// FlagNames returns a slice of flag names used by the this context and all of
// its parent contexts.
func (c *Context) FlagNames() []string {
	names := []string{}
	for _, ctx := range c.Lineage() {
		ctx.flagSet.Visit(makeFlagNameVisitor(&names))
	}
	return names
}

// Lineage returns *this* context and all of its ancestor contexts in order from
// child to parent
func (c *Context) Lineage() []*Context {
	lineage := []*Context{}

	for cur := c; cur != nil; cur = cur.parentContext {
		lineage = append(lineage, cur)
	}

	return lineage
}

// Args contains apps console arguments
type Args []string

// Args returns the command line arguments associated with the context.
func (c *Context) Args() Args {
	args := Args(c.flagSet.Args())
	return args
}

// NArg returns the number of the command line arguments.
func (c *Context) NArg() int {
	return len(c.Args())
}

// Get returns the nth argument, or else a blank string
func (a Args) Get(n int) string {
	if len(a) > n {
		return a[n]
	}
	return ""
}

// First returns the first argument, or else a blank string
func (a Args) First() string {
	return a.Get(0)
}

// Tail returns the rest of the arguments (not the first one)
// or else an empty string slice
func (a Args) Tail() []string {
	if len(a) >= 2 {
		return []string(a)[1:]
	}
	return []string{}
}

// Present checks if there are any arguments present
func (a Args) Present() bool {
	return len(a) != 0
}

// Swap swaps arguments at the given indexes
func (a Args) Swap(from, to int) error {
	if from >= len(a) || to >= len(a) {
		return errors.New("index out of range")
	}
	a[from], a[to] = a[to], a[from]
	return nil
}

func lookupFlagSet(name string, ctx *Context) *flag.FlagSet {
	for _, c := range ctx.Lineage() {
		if f := c.flagSet.Lookup(name); f != nil {
			return c.flagSet
		}
	}

	return nil
}

func lookupInt(name string, set *flag.FlagSet) int {
	f := set.Lookup(name)
	if f != nil {
		val, err := strconv.Atoi(f.Value.String())
		if err != nil {
			return 0
		}
		return val
	}

	return 0
}

func lookupDuration(name string, set *flag.FlagSet) time.Duration {
	f := set.Lookup(name)
	if f != nil {
		val, err := time.ParseDuration(f.Value.String())
		if err == nil {
			return val
		}
	}

	return 0
}

func lookupFloat64(name string, set *flag.FlagSet) float64 {
	f := set.Lookup(name)
	if f != nil {
		val, err := strconv.ParseFloat(f.Value.String(), 64)
		if err != nil {
			return 0
		}
		return val
	}

	return 0
}

func lookupString(name string, set *flag.FlagSet) string {
	f := set.Lookup(name)
	if f != nil {
		return f.Value.String()
	}

	return ""
}

func lookupStringSlice(name string, set *flag.FlagSet) []string {
	f := set.Lookup(name)
	if f != nil {
		return (f.Value.(*StringSlice)).Value()

	}

	return nil
}

func lookupIntSlice(name string, set *flag.FlagSet) []int {
	f := set.Lookup(name)
	if f != nil {
		return (f.Value.(*IntSlice)).Value()

	}

	return nil
}

func lookupGeneric(name string, set *flag.FlagSet) interface{} {
	f := set.Lookup(name)
	if f != nil {
		return f.Value
	}
	return nil
}

func lookupBool(name string, set *flag.FlagSet) bool {
	f := set.Lookup(name)
	if f != nil {
		val, err := strconv.ParseBool(f.Value.String())
		if err != nil {
			return false
		}
		return val
	}

	return false
}

func lookupBoolT(name string, set *flag.FlagSet) bool {
	f := set.Lookup(name)
	if f != nil {
		val, err := strconv.ParseBool(f.Value.String())
		if err != nil {
			return true
		}
		return val
	}

	return true
}

func copyFlag(name string, ff *flag.Flag, set *flag.FlagSet) {
	switch ff.Value.(type) {
	case Serializeder:
		set.Set(name, ff.Value.(Serializeder).Serialized())
	default:
		set.Set(name, ff.Value.String())
	}
}

func normalizeFlags(flags []Flag, set *flag.FlagSet) error {
	visited := make(map[string]bool)
	set.Visit(func(f *flag.Flag) {
		visited[f.Name] = true
	})
	for _, f := range flags {
		parts := strings.Split(f.GetName(), ",")
		if len(parts) == 1 {
			continue
		}
		var ff *flag.Flag
		for _, name := range parts {
			name = strings.Trim(name, " ")
			if visited[name] {
				if ff != nil {
					return errors.New("Cannot use two forms of the same flag: " + name + " " + ff.Name)
				}
				ff = set.Lookup(name)
			}
		}
		if ff == nil {
			continue
		}
		for _, name := range parts {
			name = strings.Trim(name, " ")
			if !visited[name] {
				copyFlag(name, ff, set)
			}
		}
	}
	return nil
}

func makeFlagNameVisitor(names *[]string) func(*flag.Flag) {
	return func(f *flag.Flag) {
		nameParts := strings.Split(f.Name, ",")
		name := strings.TrimSpace(nameParts[0])

		for _, part := range nameParts {
			part = strings.TrimSpace(part)
			if len(part) > len(name) {
				name = part
			}
		}

		if name != "" {
			(*names) = append(*names, name)
		}
	}
}
<|endoftext|>