<commit_before>package colly

import (
	"crypto/sha1"
	"encoding/gob"
	"encoding/hex"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"path"
	"regexp"
	"sync"
	"time"

	"github.com/gobwas/glob"
)

type httpBackend struct {
	LimitRules []*LimitRule
	Client     *http.Client
	lock       *sync.RWMutex
}

// LimitRule provides connection restrictions for domains.
// Both DomainRegexp and DomainGlob can be used to specify
// the included domains patterns, but at least one is required.
// There can be two kind of limitations:
//  - Parallelism: Set limit for the number of concurrent requests to matching domains
//  - Delay: Wait specified amount of time between requests (parallelism is 1 in this case)
type LimitRule struct {
	// DomainRegexp is a regular expression to match against domains
	DomainRegexp string
	// DomainRegexp is a glob pattern to match against domains
	DomainGlob string
	// Delay is the duration to wait before creating a new request to the matching domains
	Delay time.Duration
	// RandomDelay is the extra randomized duration to wait added to Delay before creating a new request
	RandomDelay time.Duration
	// Parallelism is the number of the maximum allowed concurrent requests of the matching domains
	Parallelism    int
	waitChan       chan bool
	compiledRegexp *regexp.Regexp
	compiledGlob   glob.Glob
}

// Init initializes the private members of LimitRule
func (r *LimitRule) Init() error {
	waitChanSize := 1
	if r.Parallelism > 1 {
		waitChanSize = r.Parallelism
	}
	r.waitChan = make(chan bool, waitChanSize)
	hasPattern := false
	if r.DomainRegexp != "" {
		c, err := regexp.Compile(r.DomainRegexp)
		if err != nil {
			return err
		}
		r.compiledRegexp = c
		hasPattern = true
	}
	if r.DomainGlob != "" {
		c, err := glob.Compile(r.DomainGlob)
		if err != nil {
			return err
		}
		r.compiledGlob = c
		hasPattern = true
	}
	if !hasPattern {
		return ErrNoPattern
	}
	return nil
}

func (h *httpBackend) Init(jar http.CookieJar) {
	rand.Seed(time.Now().UnixNano())
	h.LimitRules = make([]*LimitRule, 0, 8)
	h.Client = &http.Client{
		Jar:     jar,
		Timeout: 10 * time.Second,
	}
	h.lock = &sync.RWMutex{}
}

// Match checks that the domain parameter triggers the rule
func (r *LimitRule) Match(domain string) bool {
	match := false
	if r.compiledRegexp != nil && r.compiledRegexp.MatchString(domain) {
		match = true
	}
	if r.compiledGlob != nil && r.compiledGlob.Match(domain) {
		match = true
	}
	return match
}

func (h *httpBackend) GetMatchingRule(domain string) *LimitRule {
	h.lock.RLock()
	defer h.lock.RUnlock()
	for _, r := range h.LimitRules {
		if r.Match(domain) {
			return r
		}
	}
	return nil
}

func (h *httpBackend) Cache(request *http.Request, bodySize int, cacheDir string) (*Response, error) {
	if cacheDir == "" || request.Method != "GET" {
		return h.Do(request, bodySize)
	}
	sum := sha1.Sum([]byte(request.URL.String()))
	hash := hex.EncodeToString(sum[:])
	dir := path.Join(cacheDir, hash[:2])
	filename := path.Join(dir, hash)
	if file, err := os.Open(filename); err == nil {
		resp := new(Response)
		err := gob.NewDecoder(file).Decode(resp)
		file.Close()
		if resp.StatusCode < 500 {
			return resp, err
		}
	}
	resp, err := h.Do(request, bodySize)
	if err != nil || resp.StatusCode >= 500 {
		return resp, err
	}
	if _, err := os.Stat(dir); err != nil {
		if err := os.MkdirAll(dir, 0750); err != nil {
			return resp, err
		}
	}
	file, err := os.Create(filename + "~")
	if err != nil {
		return resp, err
	}
	if err := gob.NewEncoder(file).Encode(resp); err != nil {
		file.Close()
		return resp, err
	}
	file.Close()
	return resp, os.Rename(filename+"~", filename)
}

func (h *httpBackend) Do(request *http.Request, bodySize int) (*Response, error) {
	r := h.GetMatchingRule(request.URL.Host)
	if r != nil {
		r.waitChan <- true
		defer func(r *LimitRule) {
			randomDelay := time.Duration(0)
			if r.RandomDelay != 0 {
				randomDelay = time.Duration(rand.Intn(int(r.RandomDelay)))
			}
			time.Sleep(r.Delay + randomDelay)
			<-r.waitChan
		}(r)
	}

	res, err := h.Client.Do(request)
	if err != nil {
		return nil, err
	}
	*request = *res.Request

	var bodyReader io.Reader = res.Body
	if bodySize > 0 {
		bodyReader = io.LimitReader(bodyReader, int64(bodySize))
	}
	body, err := ioutil.ReadAll(bodyReader)
	defer res.Body.Close()
	if err != nil {
		return nil, err
	}
	return &Response{
		StatusCode: res.StatusCode,
		Body:       body,
		Headers:    &res.Header,
	}, nil
}

func (h *httpBackend) Limit(rule *LimitRule) error {
	h.lock.Lock()
	h.LimitRules = append(h.LimitRules, rule)
	h.lock.Unlock()
	return rule.Init()
}

func (h *httpBackend) Limits(rules []*LimitRule) error {
	for _, r := range rules {
		if err := h.Limit(r); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>[mod] optimize limit rule initialization<commit_after>package colly

import (
	"crypto/sha1"
	"encoding/gob"
	"encoding/hex"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"path"
	"regexp"
	"sync"
	"time"

	"github.com/gobwas/glob"
)

type httpBackend struct {
	LimitRules []*LimitRule
	Client     *http.Client
	lock       *sync.RWMutex
}

// LimitRule provides connection restrictions for domains.
// Both DomainRegexp and DomainGlob can be used to specify
// the included domains patterns, but at least one is required.
// There can be two kind of limitations:
//  - Parallelism: Set limit for the number of concurrent requests to matching domains
//  - Delay: Wait specified amount of time between requests (parallelism is 1 in this case)
type LimitRule struct {
	// DomainRegexp is a regular expression to match against domains
	DomainRegexp string
	// DomainRegexp is a glob pattern to match against domains
	DomainGlob string
	// Delay is the duration to wait before creating a new request to the matching domains
	Delay time.Duration
	// RandomDelay is the extra randomized duration to wait added to Delay before creating a new request
	RandomDelay time.Duration
	// Parallelism is the number of the maximum allowed concurrent requests of the matching domains
	Parallelism    int
	waitChan       chan bool
	compiledRegexp *regexp.Regexp
	compiledGlob   glob.Glob
}

// Init initializes the private members of LimitRule
func (r *LimitRule) Init() error {
	waitChanSize := 1
	if r.Parallelism > 1 {
		waitChanSize = r.Parallelism
	}
	r.waitChan = make(chan bool, waitChanSize)
	hasPattern := false
	if r.DomainRegexp != "" {
		c, err := regexp.Compile(r.DomainRegexp)
		if err != nil {
			return err
		}
		r.compiledRegexp = c
		hasPattern = true
	}
	if r.DomainGlob != "" {
		c, err := glob.Compile(r.DomainGlob)
		if err != nil {
			return err
		}
		r.compiledGlob = c
		hasPattern = true
	}
	if !hasPattern {
		return ErrNoPattern
	}
	return nil
}

func (h *httpBackend) Init(jar http.CookieJar) {
	rand.Seed(time.Now().UnixNano())
	h.Client = &http.Client{
		Jar:     jar,
		Timeout: 10 * time.Second,
	}
	h.lock = &sync.RWMutex{}
}

// Match checks that the domain parameter triggers the rule
func (r *LimitRule) Match(domain string) bool {
	match := false
	if r.compiledRegexp != nil && r.compiledRegexp.MatchString(domain) {
		match = true
	}
	if r.compiledGlob != nil && r.compiledGlob.Match(domain) {
		match = true
	}
	return match
}

func (h *httpBackend) GetMatchingRule(domain string) *LimitRule {
	if h.LimitRules == nil {
		return nil
	}
	h.lock.RLock()
	defer h.lock.RUnlock()
	for _, r := range h.LimitRules {
		if r.Match(domain) {
			return r
		}
	}
	return nil
}

func (h *httpBackend) Cache(request *http.Request, bodySize int, cacheDir string) (*Response, error) {
	if cacheDir == "" || request.Method != "GET" {
		return h.Do(request, bodySize)
	}
	sum := sha1.Sum([]byte(request.URL.String()))
	hash := hex.EncodeToString(sum[:])
	dir := path.Join(cacheDir, hash[:2])
	filename := path.Join(dir, hash)
	if file, err := os.Open(filename); err == nil {
		resp := new(Response)
		err := gob.NewDecoder(file).Decode(resp)
		file.Close()
		if resp.StatusCode < 500 {
			return resp, err
		}
	}
	resp, err := h.Do(request, bodySize)
	if err != nil || resp.StatusCode >= 500 {
		return resp, err
	}
	if _, err := os.Stat(dir); err != nil {
		if err := os.MkdirAll(dir, 0750); err != nil {
			return resp, err
		}
	}
	file, err := os.Create(filename + "~")
	if err != nil {
		return resp, err
	}
	if err := gob.NewEncoder(file).Encode(resp); err != nil {
		file.Close()
		return resp, err
	}
	file.Close()
	return resp, os.Rename(filename+"~", filename)
}

func (h *httpBackend) Do(request *http.Request, bodySize int) (*Response, error) {
	r := h.GetMatchingRule(request.URL.Host)
	if r != nil {
		r.waitChan <- true
		defer func(r *LimitRule) {
			randomDelay := time.Duration(0)
			if r.RandomDelay != 0 {
				randomDelay = time.Duration(rand.Intn(int(r.RandomDelay)))
			}
			time.Sleep(r.Delay + randomDelay)
			<-r.waitChan
		}(r)
	}

	res, err := h.Client.Do(request)
	if err != nil {
		return nil, err
	}
	*request = *res.Request

	var bodyReader io.Reader = res.Body
	if bodySize > 0 {
		bodyReader = io.LimitReader(bodyReader, int64(bodySize))
	}
	body, err := ioutil.ReadAll(bodyReader)
	defer res.Body.Close()
	if err != nil {
		return nil, err
	}
	return &Response{
		StatusCode: res.StatusCode,
		Body:       body,
		Headers:    &res.Header,
	}, nil
}

func (h *httpBackend) Limit(rule *LimitRule) error {
	h.lock.Lock()
	if h.LimitRules == nil {
		h.LimitRules = make([]*LimitRule, 0, 8)
	}
	h.LimitRules = append(h.LimitRules, rule)
	h.lock.Unlock()
	return rule.Init()
}

func (h *httpBackend) Limits(rules []*LimitRule) error {
	for _, r := range rules {
		if err := h.Limit(r); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2014 Michael Wendland
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 *	Authors:
 * 		Michael Wendland <michael@michiwend.com>
 */

package gomusicbrainz

// TODO use testdata from https://github.com/metabrainz/mmd-schema/tree/master/test-data/valid

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/aryann/difflib"
	"github.com/davecgh/go-spew/spew"
)

var (
	mux    *http.ServeMux
	server *httptest.Server
	client *WS2Client
)

// Init multiplexer and httptest server
func setupHTTPTesting() {
	mux = http.NewServeMux()
	server = httptest.NewServer(mux)

	client, _ = NewWS2Client(
		server.URL,
		"Application Name",
		"Version",
		"Contact",
	)

	// NOTE this fixes testing since the test server does not listen on /ws/2
	client.WS2RootURL.Path = ""
}

// handleFunc passes response to the http client.
/*
func handleFunc(url string, response *string, t *testing.T) {
	mux.HandleFunc(url, func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, *response)
	})
}*/

// serveTestFile responses to the http client with content of a test file
// located in ./testdata
func serveTestFile(url string, testfile string, t *testing.T) {

	//TODO check request URL if it matches one of the following patterns
	//lookup:   /<ENTITY>/<MBID>?inc=<INC>
	//browse:   /<ENTITY>?<ENTITY>=<MBID>&limit=<LIMIT>&offset=<OFFSET>&inc=<INC>
	//search:   /<ENTITY>?query=<QUERY>&limit=<LIMIT>&offset=<OFFSET>

	t.Log("Listening on", url)

	mux.HandleFunc(url, func(w http.ResponseWriter, r *http.Request) {
		if testing.Verbose() {
			fmt.Println("GET request:", r.URL.String())
		}

		http.ServeFile(w, r, "./testdata/"+testfile)
	})
}

// pretty prints a diff
func requestDiff(want, returned interface{}) string {
	spew.Config.SortKeys = true
	spew.Config.ContinueOnMethod = true
	recs := difflib.Diff(
		// FIXME splits "strings with whitespaces" into sperate fields
		strings.Fields(spew.Sprintf("%#v", returned)),
		strings.Fields(spew.Sprintf("%#v", want)))

	out := "\n"
	for _, rec := range recs {
		if rec.Delta == difflib.RightOnly {
			out += fmt.Sprintf("%-10s%s\n", "want:", rec.Payload)
		} else if rec.Delta == difflib.LeftOnly {
			out += fmt.Sprintf("%-10s%s\n", "returned:", rec.Payload)
		}
	}
	return out

	/*
		out := "\n"

		for _, diff := range pretty.Diff(want, returned) {
			out += fmt.Sprintln("difference in", diff)
		}
		return out
	*/

}
<commit_msg>cleanup testing<commit_after>/*
 * Copyright (c) 2014 Michael Wendland
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 *	Authors:
 * 		Michael Wendland <michael@michiwend.com>
 */

package gomusicbrainz

// TODO use testdata from https://github.com/metabrainz/mmd-schema/tree/master/test-data/valid

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"path"
	"strings"
	"testing"

	"github.com/aryann/difflib"
	"github.com/davecgh/go-spew/spew"
)

var (
	mux    *http.ServeMux
	server *httptest.Server
	client *WS2Client
)

// Init multiplexer and httptest server
func setupHTTPTesting() {
	mux = http.NewServeMux()
	server = httptest.NewServer(mux)

	client, _ = NewWS2Client(
		server.URL,
		"Application Name",
		"Version",
		"Contact",
	)

	// NOTE this fixes testing since the test server does not listen on /ws/2
	client.WS2RootURL.Path = ""
}

// serveTestFile responses to the http client with content of a test file
// located in ./testdata
func serveTestFile(endpoint string, testfile string, t *testing.T) {

	//TODO check request URL if it matches one of the following patterns
	//lookup:   /<ENTITY>/<MBID>?inc=<INC>
	//browse:   /<ENTITY>?<ENTITY>=<MBID>&limit=<LIMIT>&offset=<OFFSET>&inc=<INC>
	//search:   /<ENTITY>?query=<QUERY>&limit=<LIMIT>&offset=<OFFSET>

	t.Log("Handling endpoint", endpoint)
	t.Log("Serving test file", testfile)

	mux.HandleFunc(endpoint, func(w http.ResponseWriter, r *http.Request) {

		t.Log("GET request was:", r.URL.String())

		http.ServeFile(w, r, path.Join("./testdata", testfile))
	})
}

// pretty prints a diff
func requestDiff(want, returned interface{}) string {
	spew.Config.SortKeys = true
	spew.Config.ContinueOnMethod = true
	recs := difflib.Diff(
		// FIXME splits "strings with whitespaces" into sperate fields
		strings.Fields(spew.Sprintf("%#v", returned)),
		strings.Fields(spew.Sprintf("%#v", want)))

	out := "\n"
	for _, rec := range recs {
		if rec.Delta == difflib.RightOnly {
			out += fmt.Sprintf("%-10s%s\n", "want:", rec.Payload)
		} else if rec.Delta == difflib.LeftOnly {
			out += fmt.Sprintf("%-10s%s\n", "returned:", rec.Payload)
		}
	}
	return out

	/*
		out := "\n"

		for _, diff := range pretty.Diff(want, returned) {
			out += fmt.Sprintln("difference in", diff)
		}
		return out
	*/

}
<|endoftext|>


<commit_before>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package gossip

import (
	"bytes"
	"testing"
	"time"

	"github.com/pkg/errors"
	"golang.org/x/net/context"
	"google.golang.org/grpc"

	"github.com/cockroachdb/cockroach/base"
	"github.com/cockroachdb/cockroach/gossip/resolver"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/rpc"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/leaktest"
	"github.com/cockroachdb/cockroach/util/stop"
)

// TestGossipInfoStore verifies operation of gossip instance infostore.
func TestGossipInfoStore(t *testing.T) {
	defer leaktest.AfterTest(t)()
	stopper := stop.NewStopper()
	defer stopper.Stop()
	rpcContext := rpc.NewContext(nil, nil, stopper)
	g := New(rpcContext, nil, stopper)
	// Have to call g.SetNodeID before call g.AddInfo
	g.SetNodeID(roachpb.NodeID(1))
	slice := []byte("b")
	if err := g.AddInfo("s", slice, time.Hour); err != nil {
		t.Fatal(err)
	}
	if val, err := g.GetInfo("s"); !bytes.Equal(val, slice) || err != nil {
		t.Errorf("error fetching string: %v", err)
	}
	if _, err := g.GetInfo("s2"); err == nil {
		t.Errorf("expected error fetching nonexistent key \"s2\"")
	}
}

func TestGossipGetNextBootstrapAddress(t *testing.T) {
	defer leaktest.AfterTest(t)()

	resolverSpecs := []string{
		"127.0.0.1:9000",
		"127.0.0.1:9001",
		"localhost:9004",
	}

	resolvers := []resolver.Resolver{}
	for _, rs := range resolverSpecs {
		resolver, err := resolver.NewResolver(&base.Context{Insecure: true}, rs)
		if err == nil {
			resolvers = append(resolvers, resolver)
		}
	}
	if len(resolvers) != 3 {
		t.Errorf("expected 3 resolvers; got %d", len(resolvers))
	}
	g := New(nil, resolvers, nil)

	// Using specified resolvers, fetch bootstrap addresses 3 times
	// and verify the results match expected addresses.
	expAddresses := []string{
		"127.0.0.1:9000",
		"127.0.0.1:9001",
		"localhost:9004",
	}
	for i := 0; i < len(expAddresses); i++ {
		if addr := g.getNextBootstrapAddress(); addr == nil {
			t.Errorf("%d: unexpected nil addr when expecting %s", i, expAddresses[i])
		} else if addrStr := addr.String(); addrStr != expAddresses[i] {
			t.Errorf("%d: expected addr %s; got %s", i, expAddresses[i], addrStr)
		}
	}
}

// TestGossipNoForwardSelf verifies that when a Gossip instance is full, it
// redirects clients elsewhere (in particular not to itself).
//
// NB: Stress testing this test really stresses the OS networking stack
// more than anything else. For example, on Linux it may quickly deplete
// the ephemeral port range (due to the TIME_WAIT state).
// On a box which only runs tests, this can be circumvented by running
//
//	sudo bash -c "echo 1 > /proc/sys/net/ipv4/tcp_tw_recycle"
//
// See https://vincent.bernat.im/en/blog/2014-tcp-time-wait-state-linux.html
// for details.
//
// On OSX, things similarly fall apart. See #7524 and #5218 for some discussion
// of this.
func TestGossipNoForwardSelf(t *testing.T) {
	defer leaktest.AfterTest(t)()

	stopper := stop.NewStopper()
	defer stopper.Stop()
	local := startGossip(1, stopper, t)

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	// Start one loopback client plus enough additional clients to fill the
	// incoming clients.
	peers := []*Gossip{local}
	for i := 0; i < local.server.incoming.maxSize; i++ {
		peers = append(peers, startGossip(roachpb.NodeID(i+2), stopper, t))
	}

	for _, peer := range peers {
		c := newClient(&local.is.NodeAddr)

		util.SucceedsSoon(t, func() error {
			conn, err := peer.rpcContext.GRPCDial(c.addr.String(), grpc.WithBlock())
			if err != nil {
				return err
			}

			stream, err := NewGossipClient(conn).Gossip(ctx)
			if err != nil {
				return err
			}

			if err := c.requestGossip(peer, peer.is.NodeAddr, stream); err != nil {
				return err
			}

			// Wait until the server responds, so we know we're connected.
			_, err = stream.Recv()
			return err
		})
	}

	numClients := len(peers) * 2
	disconnectedCh := make(chan *client)

	// Start a few overflow peers and assert that they don't get forwarded to us
	// again.
	for i := 0; i < numClients; i++ {
		peer := startGossip(roachpb.NodeID(i+local.server.incoming.maxSize+2), stopper, t)

		for {
			c := newClient(&local.is.NodeAddr)
			c.start(peer, disconnectedCh, peer.rpcContext, stopper)

			disconnectedClient := <-disconnectedCh
			if disconnectedClient != c {
				t.Fatalf("expected %p to be disconnected, got %p", c, disconnectedClient)
			} else if c.forwardAddr == nil {
				// Under high load, clients sometimes fail to connect for reasons
				// unrelated to the test, so we need to permit some.
				t.Logf("node #%d: got nil forwarding address", peer.is.NodeID)
				continue
			} else if *c.forwardAddr == local.is.NodeAddr {
				t.Errorf("node #%d: got local's forwarding address", peer.is.NodeID)
			}
			break
		}
	}
}

// TestGossipCullNetwork verifies that a client will be culled from
// the network periodically (at cullInterval duration intervals).
func TestGossipCullNetwork(t *testing.T) {
	defer leaktest.AfterTest(t)()

	stopper := stop.NewStopper()
	defer stopper.Stop()
	local := startGossip(1, stopper, t)
	local.SetCullInterval(5 * time.Millisecond)

	local.mu.Lock()
	for i := 0; i < minPeers; i++ {
		peer := startGossip(roachpb.NodeID(i+2), stopper, t)
		local.startClient(&peer.is.NodeAddr)
	}
	local.mu.Unlock()

	util.SucceedsSoon(t, func() error {
		if peers := len(local.Outgoing()); peers != minPeers {
			return errors.Errorf("%d of %d peers connected", peers, minPeers)
		}
		return nil
	})

	local.manage()

	util.SucceedsSoon(t, func() error {
		// Verify that a client is closed within the cull interval.
		if peers := len(local.Outgoing()); peers != minPeers-1 {
			return errors.Errorf("%d of %d peers connected", peers, minPeers-1)
		}
		return nil
	})
}
<commit_msg>gossip: longer timeout for TestGossipCullNetwork<commit_after>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package gossip

import (
	"bytes"
	"testing"
	"time"

	"github.com/pkg/errors"
	"golang.org/x/net/context"
	"google.golang.org/grpc"

	"github.com/cockroachdb/cockroach/base"
	"github.com/cockroachdb/cockroach/gossip/resolver"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/rpc"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/leaktest"
	"github.com/cockroachdb/cockroach/util/stop"
)

// TestGossipInfoStore verifies operation of gossip instance infostore.
func TestGossipInfoStore(t *testing.T) {
	defer leaktest.AfterTest(t)()
	stopper := stop.NewStopper()
	defer stopper.Stop()
	rpcContext := rpc.NewContext(nil, nil, stopper)
	g := New(rpcContext, nil, stopper)
	// Have to call g.SetNodeID before call g.AddInfo
	g.SetNodeID(roachpb.NodeID(1))
	slice := []byte("b")
	if err := g.AddInfo("s", slice, time.Hour); err != nil {
		t.Fatal(err)
	}
	if val, err := g.GetInfo("s"); !bytes.Equal(val, slice) || err != nil {
		t.Errorf("error fetching string: %v", err)
	}
	if _, err := g.GetInfo("s2"); err == nil {
		t.Errorf("expected error fetching nonexistent key \"s2\"")
	}
}

func TestGossipGetNextBootstrapAddress(t *testing.T) {
	defer leaktest.AfterTest(t)()

	resolverSpecs := []string{
		"127.0.0.1:9000",
		"127.0.0.1:9001",
		"localhost:9004",
	}

	resolvers := []resolver.Resolver{}
	for _, rs := range resolverSpecs {
		resolver, err := resolver.NewResolver(&base.Context{Insecure: true}, rs)
		if err == nil {
			resolvers = append(resolvers, resolver)
		}
	}
	if len(resolvers) != 3 {
		t.Errorf("expected 3 resolvers; got %d", len(resolvers))
	}
	g := New(nil, resolvers, nil)

	// Using specified resolvers, fetch bootstrap addresses 3 times
	// and verify the results match expected addresses.
	expAddresses := []string{
		"127.0.0.1:9000",
		"127.0.0.1:9001",
		"localhost:9004",
	}
	for i := 0; i < len(expAddresses); i++ {
		if addr := g.getNextBootstrapAddress(); addr == nil {
			t.Errorf("%d: unexpected nil addr when expecting %s", i, expAddresses[i])
		} else if addrStr := addr.String(); addrStr != expAddresses[i] {
			t.Errorf("%d: expected addr %s; got %s", i, expAddresses[i], addrStr)
		}
	}
}

// TestGossipNoForwardSelf verifies that when a Gossip instance is full, it
// redirects clients elsewhere (in particular not to itself).
//
// NB: Stress testing this test really stresses the OS networking stack
// more than anything else. For example, on Linux it may quickly deplete
// the ephemeral port range (due to the TIME_WAIT state).
// On a box which only runs tests, this can be circumvented by running
//
//	sudo bash -c "echo 1 > /proc/sys/net/ipv4/tcp_tw_recycle"
//
// See https://vincent.bernat.im/en/blog/2014-tcp-time-wait-state-linux.html
// for details.
//
// On OSX, things similarly fall apart. See #7524 and #5218 for some discussion
// of this.
func TestGossipNoForwardSelf(t *testing.T) {
	defer leaktest.AfterTest(t)()

	stopper := stop.NewStopper()
	defer stopper.Stop()
	local := startGossip(1, stopper, t)

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	// Start one loopback client plus enough additional clients to fill the
	// incoming clients.
	peers := []*Gossip{local}
	for i := 0; i < local.server.incoming.maxSize; i++ {
		peers = append(peers, startGossip(roachpb.NodeID(i+2), stopper, t))
	}

	for _, peer := range peers {
		c := newClient(&local.is.NodeAddr)

		util.SucceedsSoon(t, func() error {
			conn, err := peer.rpcContext.GRPCDial(c.addr.String(), grpc.WithBlock())
			if err != nil {
				return err
			}

			stream, err := NewGossipClient(conn).Gossip(ctx)
			if err != nil {
				return err
			}

			if err := c.requestGossip(peer, peer.is.NodeAddr, stream); err != nil {
				return err
			}

			// Wait until the server responds, so we know we're connected.
			_, err = stream.Recv()
			return err
		})
	}

	numClients := len(peers) * 2
	disconnectedCh := make(chan *client)

	// Start a few overflow peers and assert that they don't get forwarded to us
	// again.
	for i := 0; i < numClients; i++ {
		peer := startGossip(roachpb.NodeID(i+local.server.incoming.maxSize+2), stopper, t)

		for {
			c := newClient(&local.is.NodeAddr)
			c.start(peer, disconnectedCh, peer.rpcContext, stopper)

			disconnectedClient := <-disconnectedCh
			if disconnectedClient != c {
				t.Fatalf("expected %p to be disconnected, got %p", c, disconnectedClient)
			} else if c.forwardAddr == nil {
				// Under high load, clients sometimes fail to connect for reasons
				// unrelated to the test, so we need to permit some.
				t.Logf("node #%d: got nil forwarding address", peer.is.NodeID)
				continue
			} else if *c.forwardAddr == local.is.NodeAddr {
				t.Errorf("node #%d: got local's forwarding address", peer.is.NodeID)
			}
			break
		}
	}
}

// TestGossipCullNetwork verifies that a client will be culled from
// the network periodically (at cullInterval duration intervals).
func TestGossipCullNetwork(t *testing.T) {
	defer leaktest.AfterTest(t)()

	stopper := stop.NewStopper()
	defer stopper.Stop()
	local := startGossip(1, stopper, t)
	local.SetCullInterval(5 * time.Millisecond)

	local.mu.Lock()
	for i := 0; i < minPeers; i++ {
		peer := startGossip(roachpb.NodeID(i+2), stopper, t)
		local.startClient(&peer.is.NodeAddr)
	}
	local.mu.Unlock()

	const slowGossipDuration = time.Minute

	if err := util.RetryForDuration(slowGossipDuration, func() error {
		if peers := len(local.Outgoing()); peers != minPeers {
			return errors.Errorf("%d of %d peers connected", peers, minPeers)
		}
		return nil
	}); err != nil {
		t.Fatalf("condition failed to evaluate within %s: %s", slowGossipDuration, err)
	}

	local.manage()

	if err := util.RetryForDuration(slowGossipDuration, func() error {
		// Verify that a client is closed within the cull interval.
		if peers := len(local.Outgoing()); peers != minPeers-1 {
			return errors.Errorf("%d of %d peers connected", peers, minPeers-1)
		}
		return nil
	}); err != nil {
		t.Fatalf("condition failed to evaluate within %s: %s", slowGossipDuration, err)
	}
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"time"

	"github.com/dnaeon/gru/catalog"
	"github.com/dnaeon/gru/task"

	"github.com/codegangsta/cli"
	"github.com/gosuri/uiprogress"
	"github.com/gosuri/uitable"
)

// NewRunCommand creates a new sub-command for submitting
// tasks to minions
func NewRunCommand() cli.Command {
	cmd := cli.Command{
		Name:   "run",
		Usage:  "send task to minion(s)",
		Action: execRunCommand,
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "with-classifier",
				Value: "",
				Usage: "match minions with given classifier pattern",
			},
		},
	}

	return cmd
}

// Executes the "run" command
func execRunCommand(c *cli.Context) {
	if len(c.Args()) < 1 {
		displayError(errNoModuleName, 64)
	}

	// Create the task that we send to our minions
	main := c.Args()[0]
	katalog, err := catalog.Load(main, c.GlobalString("modulepath"))
	if err != nil {
		displayError(err, 1)
	}

	t := task.New(katalog)

	client := newEtcdMinionClientFromFlags(c)

	cFlag := c.String("with-classifier")
	minions, err := parseClassifierPattern(client, cFlag)

	if err != nil {
		displayError(err, 1)
	}

	numMinions := len(minions)
	if numMinions == 0 {
		displayError(errNoMinionFound, 1)
	}

	fmt.Printf("Found %d minion(s) for task processing\n\n", numMinions)

	// Progress bar to display while submitting task
	progress := uiprogress.New()
	bar := progress.AddBar(numMinions)
	bar.AppendCompleted()
	bar.PrependElapsed()
	progress.Start()

	// Number of minions to which submitting the task has failed
	failed := 0

	// Submit task to minions
	fmt.Println("Submitting task to minion(s) ...")
	for _, m := range minions {
		err = client.MinionSubmitTask(m, t)
		if err != nil {
			fmt.Printf("Failed to submit task to %s: %s\n", m, err)
			failed++
		}
		bar.Incr()
	}

	// Stop progress bar and sleep for a bit to make sure the
	// progress bar gets updated if we were too fast for it
	progress.Stop()
	time.Sleep(time.Millisecond * 100)

	// Display task report
	fmt.Println()
	table := uitable.New()
	table.MaxColWidth = 80
	table.Wrap = true
	table.AddRow("TASK", "SUBMITTED", "FAILED", "TOTAL")
	table.AddRow(t.TaskID, numMinions-failed, failed, numMinions)
	fmt.Println(table)
}
<commit_msg>gructl run: be able to specify environment to use<commit_after>package command

import (
	"fmt"
	"time"

	"github.com/dnaeon/gru/catalog"
	"github.com/dnaeon/gru/task"

	"github.com/codegangsta/cli"
	"github.com/gosuri/uiprogress"
	"github.com/gosuri/uitable"
)

// NewRunCommand creates a new sub-command for submitting
// tasks to minions
func NewRunCommand() cli.Command {
	cmd := cli.Command{
		Name:   "run",
		Usage:  "send task to minion(s)",
		Action: execRunCommand,
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "environment",
				Value: "production",
				Usage: "specify environment to use",
			},
			cli.StringFlag{
				Name:  "with-classifier",
				Value: "",
				Usage: "match minions with given classifier pattern",
			},
		},
	}

	return cmd
}

// Executes the "run" command
func execRunCommand(c *cli.Context) {
	if len(c.Args()) < 1 {
		displayError(errNoModuleName, 64)
	}

	// Create the task that we send to our minions
	main := c.Args()[0]
	katalog, err := catalog.Load(main, c.GlobalString("modulepath"))
	if err != nil {
		displayError(err, 1)
	}

	t := task.New(katalog)
	t.Environment = c.String("environment")

	client := newEtcdMinionClientFromFlags(c)

	cFlag := c.String("with-classifier")
	minions, err := parseClassifierPattern(client, cFlag)

	if err != nil {
		displayError(err, 1)
	}

	numMinions := len(minions)
	if numMinions == 0 {
		displayError(errNoMinionFound, 1)
	}

	fmt.Printf("Found %d minion(s) for task processing\n\n", numMinions)

	// Progress bar to display while submitting task
	progress := uiprogress.New()
	bar := progress.AddBar(numMinions)
	bar.AppendCompleted()
	bar.PrependElapsed()
	progress.Start()

	// Number of minions to which submitting the task has failed
	failed := 0

	// Submit task to minions
	fmt.Println("Submitting task to minion(s) ...")
	for _, m := range minions {
		err = client.MinionSubmitTask(m, t)
		if err != nil {
			fmt.Printf("Failed to submit task to %s: %s\n", m, err)
			failed++
		}
		bar.Incr()
	}

	// Stop progress bar and sleep for a bit to make sure the
	// progress bar gets updated if we were too fast for it
	progress.Stop()
	time.Sleep(time.Millisecond * 100)

	// Display task report
	fmt.Println()
	table := uitable.New()
	table.MaxColWidth = 80
	table.Wrap = true
	table.AddRow("TASK", "SUBMITTED", "FAILED", "TOTAL")
	table.AddRow(t.TaskID, numMinions-failed, failed, numMinions)
	fmt.Println(table)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"math"

	"github.com/davecgh/go-spew/spew"
	"github.com/ninjasphere/go-hue"
	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/channels"
	"github.com/ninjasphere/go-ninja/devices"
	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-ninja/model"
)

var defaultTransitionTime = 500 // 500ms
var defaultBrightness = 1.0     // Full brightness
var defaultColor = channels.ColorWhite

var info = ninja.LoadModuleInfo("./package.json")

type HueDriver struct {
	log       *logger.Logger
	config    *HueDriverConfig
	conn      *ninja.Connection
	bridge    *hue.Bridge
	user      *hue.User
	sendEvent func(event string, payload interface{}) error
}

func NewHueDriver() {
	d := &HueDriver{
		log: logger.GetLogger(info.Name),
	}

	conn, err := ninja.Connect(info.ID)
	d.conn = conn
	if err != nil {
		d.log.Fatalf("Failed to connect to MQTT: %s", err)
	}

	err = conn.ExportDriver(d)
	if err != nil {
		d.log.Fatalf("Failed to export driver: %s", err)
	}
}

type HueDriverConfig struct {
}

func (d *HueDriver) Start(config *HueDriverConfig) error {
	d.config = config

	d.bridge = getBridge()
	d.user = getUser(d, d.bridge)

	allLights, err := d.user.GetLights()
	if err != nil {
		d.log.HandleError(err, "Couldn't get lights")
		return err
	}

	for _, l := range allLights {
		_, err := d.newLight(&l)
		if err != nil {
			d.log.HandleError(err, "Error creating light instance")
		}

	}
	return nil
}

func (d *HueDriver) GetModuleInfo() *model.Module {
	return info
}

func (d *HueDriver) SetEventHandler(sendEvent func(event string, payload interface{}) error) {
	d.sendEvent = sendEvent
}

func (d *HueDriver) newLight(bulb *hue.Light) (*HueLightContext, error) { //TODO cut this down!

	name := bulb.Name

	d.log.Infof("Making light on Bridge: %s ID: %s Label: %s", d.bridge.UniqueId, bulb.Id, name)

	d.log.Infof("connection %s", d.conn)

	attrs, _ := d.user.GetLightAttributes(bulb.Id)

	light, err := devices.CreateLightDevice(d, &model.Device{
		NaturalID:     bulb.Id,
		NaturalIDType: "hue",
		Name:          &name,
		Signatures: &map[string]string{
			"ninja:manufacturer":          "Phillips",
			"ninja:productName":           "Hue",
			"ninja:productType":           "Light",
			"ninja:thingType":             "light",
			"manufacturer:productModelId": attrs.ModelId,
		},
	}, d.conn)

	if err != nil {
		d.log.FatalError(err, "Could not create light device")
	}
	//func NewLight(l *hue.Light, bus *ninja.DriverBus, bridge *hue.Bridge, user *hue.User) (*HueLightContext, error) {

	if err := light.EnableOnOffChannel(); err != nil {
		d.log.FatalError(err, "Could not enable hue on-off channel")
	}

	if err := light.EnableBrightnessChannel(); err != nil {
		d.log.FatalError(err, "Could not enable hue brightness channel")
	}

	if err := light.EnableColorChannel("temperature", "hue"); err != nil {
		d.log.FatalError(err, "Could not enable hue color channel")
	}

	if err := light.EnableTransitionChannel(); err != nil {
		d.log.FatalError(err, "Could not enable hue transition channel")
	}

	hl := &HueLightContext{
		ID:     bulb.Id,
		Name:   bulb.Name,
		Bridge: d.bridge,
		User:   d.user,
		Light:  light,
		log:    logger.GetLogger(fmt.Sprintf("huelight:%s", bulb.Id)),
	}

	light.ApplyIdentify = hl.ApplyIdentify

	if err := light.EnableIdentifyChannel(); err != nil {
		d.log.FatalError(err, "Could not enable identify channel")
	}

	light.ApplyLightState = hl.ApplyLightState

	return hl, hl.updateState()
}

type HueLightContext struct {
	ID             string
	Name           string
	Bridge         *hue.Bridge
	User           *hue.User
	Light          *devices.LightDevice
	lastState      *devices.LightDeviceState // Used to fill an incomplete request, using the previous state
	desiredState   devices.LightDeviceState  // Used when the globe is updated while it is off
	lastTransition *int
	log            *logger.Logger
}

func (hl *HueLightContext) ApplyIdentify() error {
	state := &hue.LightState{
		Alert: "lselect", // Toggles 10x
	}

	return hl.User.SetLightState(hl.ID, state)
}

func (hl *HueLightContext) ApplyLightState(state *devices.LightDeviceState) error {

	hl.log.Infof("Applying light state: %s", dump(*state))

	if state.OnOff == nil {

		if state.Color == nil && state.Brightness == nil {
			// We've been given nothing. Nothing to do!

			if state.Transition != nil {
				// We at least got a transition time... keep that for later
				hl.desiredState.Transition = state.Transition
			}
			return nil
		}

		// We have been color or brightness, but not an on-off state, assume they want it on.
		on := true
		state.OnOff = &on
	}

	if state.Transition == nil {
		if hl.desiredState.Transition != nil {
			// Use the transition previously asked for
			state.Transition = hl.desiredState.Transition
		} else if hl.lastState != nil && hl.lastState.Transition != nil {
			// Use the transition we last used
			state.Transition = hl.lastState.Transition
		} else {
			// Use the default
			state.Transition = &defaultTransitionTime
		}
	}

	hl.lastTransition = state.Transition

	// Hue doesn't like you setting anything if you're off
	if !*state.OnOff {

		// cache the desired state so we can send this later
		// note this is to get around the fact we can't send color to the hue bulb if it is OFF
		// but we need to keep these changes for when the user turns it on!
		if state.Brightness != nil {
			hl.desiredState.Brightness = state.Brightness
		}
		if state.Transition != nil {
			hl.desiredState.Transition = state.Transition
		}
		if state.Color != nil {
			hl.desiredState.Color = state.Color
		}

		// just set the on-off state and transition and pass it to the globe
		on := false
		return hl.setLightState(&hue.LightState{
			On:             &on,
			TransitionTime: getTransitionTime(state),
		})
	}

	if state.Brightness == nil {
		// We have no brightness, but hue requires one to be sent.

		if hl.desiredState.Brightness != nil {
			// Use the brightness previously asked for
			state.Brightness = hl.desiredState.Brightness
		} else if hl.lastState != nil && hl.lastState.Brightness != nil {
			// Use the previously seen brightness
			state.Brightness = hl.lastState.Brightness
		} else {
			// Use the default
			state.Brightness = &defaultBrightness
		}
	}

	if state.Color == nil {
		// We have no color, but hue requires one to be sent.

		if hl.desiredState.Color != nil {
			// Use the color previously asked for
			state.Color = hl.desiredState.Color
		} else if hl.lastState != nil && hl.lastState.Color != nil {
			// Use the color we last saw
			state.Color = hl.lastState.Color
		} else {
			// Set it to the default color
			state.Color = &defaultColor
		}
	}

	if state.OnOff == nil || state.Color == nil || state.Brightness == nil || state.Transition == nil {
		// This shouldn't happen... but just in case...
		return fmt.Errorf("Values missing from light state... %s", dump(state))
	}

	hl.log.Debugf("Applying state (after defaults + desired) : %s", dump(state))

	outgoingState := &hue.LightState{
		On:             &*state.OnOff, // Always true at this point
		Brightness:     getBrightness(state),
		TransitionTime: getTransitionTime(state),
	}

	switch state.Color.Mode {
	case "hue":

		if state.Color.Hue == nil {
			return fmt.Errorf("Missing color hue")
		}

		if state.Color.Saturation == nil {
			return fmt.Errorf("Missing color saturation")
		}

		outgoingState.Hue = getHue(state)
		outgoingState.Saturation = getSaturation(state)

	case "xy":

		if state.Color.X == nil || state.Color.Y == nil {
			return fmt.Errorf("Missing X or Y from xy color state: %s", dump(state.Color))
		}

		outgoingState.XY = []float64{*state.Color.X, *state.Color.Y}

	case "temperature":

		if state.Color.Temperature == nil {
			return fmt.Errorf("Missing color temperature: %+v", *state.Color)
		}

		outgoingState.ColorTemp = getColorTemp(state)

		hl.log.Debugf("color temp: %d ", *outgoingState.ColorTemp)
	default:
		return fmt.Errorf("Unknown color mode %s", state.Color.Mode)
	}

	return hl.setLightState(outgoingState)
}

func (hl *HueLightContext) setLightState(lightState *hue.LightState) error {

	hl.log.Debugf("Sending light state to hue bulb: %s", dump(lightState))

	if err := hl.User.SetLightState(hl.ID, lightState); err != nil {
		return err
	}

	return hl.updateState()
}

func (hl *HueLightContext) updateState() error {

	la, err := hl.User.GetLightAttributes(hl.ID)
	if err != nil {
		return err
	}

	state := hl.toNinjaLightState(la.State)

	// If the light is on, we can destroy our desired state.. as it has either been used,
	// or the user has set it to something else using another app.
	if *state.OnOff {
		hl.desiredState = devices.LightDeviceState{}
	}

	// if we have desired state... use that.
	if hl.desiredState.Color != nil {
		state.Color = hl.desiredState.Color
	}
	if hl.desiredState.Brightness != nil {
		state.Brightness = hl.desiredState.Brightness
	}
	if hl.desiredState.Transition != nil {
		state.Transition = hl.desiredState.Transition
	}

	hl.log.Debugf("Updating light state: %+v", state)
	hl.lastState = state

	hl.Light.SetLightState(state)

	return nil
}

// ES: Yes, it isn't great. Whatever. Go fix something else.
func dump(obj interface{}) string {
	x, err := json.Marshal(obj)
	if err != nil {
		return spew.Sprintf("%+v", obj)
	}
	return string(x)
}

func (hl *HueLightContext) toNinjaLightState(huestate *hue.LightState) *devices.LightDeviceState {

	hl.log.Debugf("Converting hue state to ninja %s", dump(huestate))

	onOff := *huestate.On

	transition := defaultTransitionTime

	if hl.lastTransition != nil {
		transition = *hl.lastTransition
	}

	brightness := float64(*huestate.Brightness) / float64(math.MaxUint8)

	hl.log.Infof("Converted brightness from %v to %v", *huestate.Brightness, brightness)

	var lds *devices.LightDeviceState

	switch huestate.ColorMode {
	case "ct":
		// see: http://en.wikipedia.org/wiki/Mired
		// t = 1000000 / m
		temp := float64(1000000 / int(*huestate.ColorTemp))

		lds = &devices.LightDeviceState{
			Color: &channels.ColorState{
				Mode:        "temperature",
				Temperature: &temp,
			},
			Brightness: &brightness,
			OnOff:      &onOff,
			Transition: &transition,
		}
	case "xy":

		lds = &devices.LightDeviceState{
			Color: &channels.ColorState{
				Mode: "temperature",
				X:    &huestate.XY[0],
				Y:    &huestate.XY[1],
			},
			Brightness: &brightness,
			OnOff:      &onOff,
			Transition: &transition,
		}
	case "hs":
		hue := float64(*huestate.Hue) / float64(math.MaxUint16)
		saturation := float64(*huestate.Saturation) / float64(math.MaxUint16)

		lds = &devices.LightDeviceState{
			Color: &channels.ColorState{
				Mode:       "hue",
				Hue:        &hue,
				Saturation: &saturation,
			},
			Brightness: &brightness,
			OnOff:      &onOff,
			Transition: &transition,
		}
	default:
		hl.log.FatalError(errors.New("Invalid color mode"), "Failed to load hue state")
	}

	hl.log.Infof("Converted to ninja state %s", dump(lds))

	return lds
}

func getTransitionTime(state *devices.LightDeviceState) *uint16 {

	var transTime uint16
	if *state.Transition > 0 && *state.Transition < math.MaxUint16 {
		transTime = uint16(*state.Transition / 100) //HUE API uses 1/10th of a second
	} else {
		transTime = 0
	}
	return &transTime
}

func getHue(state *devices.LightDeviceState) *uint16 {
	hue := uint16(*state.Color.Hue * math.MaxUint16)
	return &hue
}

func getSaturation(state *devices.LightDeviceState) *uint8 {
	saturation := uint8(*state.Color.Saturation * math.MaxUint16)
	return &saturation
}

func getBrightness(state *devices.LightDeviceState) *uint8 {
	brightness := uint8(*state.Brightness * math.MaxUint8)
	return &brightness
}

func getColorTemp(state *devices.LightDeviceState) *uint16 {
	// see: http://en.wikipedia.org/wiki/Mired
	// m = 1000000 / t
	temp := uint16(1000000 / int(*state.Color.Temperature))
	return &temp
}
<commit_msg>Add default color<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"math"

	"github.com/davecgh/go-spew/spew"
	"github.com/ninjasphere/go-hue"
	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/channels"
	"github.com/ninjasphere/go-ninja/devices"
	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-ninja/model"
)

var defaultTransitionTime = 500 // 500ms
var defaultBrightness = 1.0     // Full brightness
var xxx float64
var defaultColor = channels.ColorState{
	Mode:       "Hue",
	Hue:        &xxx,
	Saturation: &xxx,
}

var info = ninja.LoadModuleInfo("./package.json")

type HueDriver struct {
	log       *logger.Logger
	config    *HueDriverConfig
	conn      *ninja.Connection
	bridge    *hue.Bridge
	user      *hue.User
	sendEvent func(event string, payload interface{}) error
}

func NewHueDriver() {
	d := &HueDriver{
		log: logger.GetLogger(info.Name),
	}

	conn, err := ninja.Connect(info.ID)
	d.conn = conn
	if err != nil {
		d.log.Fatalf("Failed to connect to MQTT: %s", err)
	}

	err = conn.ExportDriver(d)
	if err != nil {
		d.log.Fatalf("Failed to export driver: %s", err)
	}
}

type HueDriverConfig struct {
}

func (d *HueDriver) Start(config *HueDriverConfig) error {
	d.config = config

	d.bridge = getBridge()
	d.user = getUser(d, d.bridge)

	allLights, err := d.user.GetLights()
	if err != nil {
		d.log.HandleError(err, "Couldn't get lights")
		return err
	}

	for _, l := range allLights {
		_, err := d.newLight(&l)
		if err != nil {
			d.log.HandleError(err, "Error creating light instance")
		}

	}
	return nil
}

func (d *HueDriver) GetModuleInfo() *model.Module {
	return info
}

func (d *HueDriver) SetEventHandler(sendEvent func(event string, payload interface{}) error) {
	d.sendEvent = sendEvent
}

func (d *HueDriver) newLight(bulb *hue.Light) (*HueLightContext, error) { //TODO cut this down!

	name := bulb.Name

	d.log.Infof("Making light on Bridge: %s ID: %s Label: %s", d.bridge.UniqueId, bulb.Id, name)

	d.log.Infof("connection %s", d.conn)

	attrs, _ := d.user.GetLightAttributes(bulb.Id)

	light, err := devices.CreateLightDevice(d, &model.Device{
		NaturalID:     bulb.Id,
		NaturalIDType: "hue",
		Name:          &name,
		Signatures: &map[string]string{
			"ninja:manufacturer":          "Phillips",
			"ninja:productName":           "Hue",
			"ninja:productType":           "Light",
			"ninja:thingType":             "light",
			"manufacturer:productModelId": attrs.ModelId,
		},
	}, d.conn)

	if err != nil {
		d.log.FatalError(err, "Could not create light device")
	}
	//func NewLight(l *hue.Light, bus *ninja.DriverBus, bridge *hue.Bridge, user *hue.User) (*HueLightContext, error) {

	if err := light.EnableOnOffChannel(); err != nil {
		d.log.FatalError(err, "Could not enable hue on-off channel")
	}

	if err := light.EnableBrightnessChannel(); err != nil {
		d.log.FatalError(err, "Could not enable hue brightness channel")
	}

	if err := light.EnableColorChannel("temperature", "hue"); err != nil {
		d.log.FatalError(err, "Could not enable hue color channel")
	}

	if err := light.EnableTransitionChannel(); err != nil {
		d.log.FatalError(err, "Could not enable hue transition channel")
	}

	hl := &HueLightContext{
		ID:     bulb.Id,
		Name:   bulb.Name,
		Bridge: d.bridge,
		User:   d.user,
		Light:  light,
		log:    logger.GetLogger(fmt.Sprintf("huelight:%s", bulb.Id)),
	}

	light.ApplyIdentify = hl.ApplyIdentify

	if err := light.EnableIdentifyChannel(); err != nil {
		d.log.FatalError(err, "Could not enable identify channel")
	}

	light.ApplyLightState = hl.ApplyLightState

	return hl, hl.updateState()
}

type HueLightContext struct {
	ID             string
	Name           string
	Bridge         *hue.Bridge
	User           *hue.User
	Light          *devices.LightDevice
	lastState      *devices.LightDeviceState // Used to fill an incomplete request, using the previous state
	desiredState   devices.LightDeviceState  // Used when the globe is updated while it is off
	lastTransition *int
	log            *logger.Logger
}

func (hl *HueLightContext) ApplyIdentify() error {
	state := &hue.LightState{
		Alert: "lselect", // Toggles 10x
	}

	return hl.User.SetLightState(hl.ID, state)
}

func (hl *HueLightContext) ApplyLightState(state *devices.LightDeviceState) error {

	hl.log.Infof("Applying light state: %s", dump(*state))

	if state.OnOff == nil {

		if state.Color == nil && state.Brightness == nil {
			// We've been given nothing. Nothing to do!

			if state.Transition != nil {
				// We at least got a transition time... keep that for later
				hl.desiredState.Transition = state.Transition
			}
			return nil
		}

		// We have been color or brightness, but not an on-off state, assume they want it on.
		on := true
		state.OnOff = &on
	}

	if state.Transition == nil {
		if hl.desiredState.Transition != nil {
			// Use the transition previously asked for
			state.Transition = hl.desiredState.Transition
		} else if hl.lastState != nil && hl.lastState.Transition != nil {
			// Use the transition we last used
			state.Transition = hl.lastState.Transition
		} else {
			// Use the default
			state.Transition = &defaultTransitionTime
		}
	}

	hl.lastTransition = state.Transition

	// Hue doesn't like you setting anything if you're off
	if !*state.OnOff {

		// cache the desired state so we can send this later
		// note this is to get around the fact we can't send color to the hue bulb if it is OFF
		// but we need to keep these changes for when the user turns it on!
		if state.Brightness != nil {
			hl.desiredState.Brightness = state.Brightness
		}
		if state.Transition != nil {
			hl.desiredState.Transition = state.Transition
		}
		if state.Color != nil {
			hl.desiredState.Color = state.Color
		}

		// just set the on-off state and transition and pass it to the globe
		on := false
		return hl.setLightState(&hue.LightState{
			On:             &on,
			TransitionTime: getTransitionTime(state),
		})
	}

	if state.Brightness == nil {
		// We have no brightness, but hue requires one to be sent.

		if hl.desiredState.Brightness != nil {
			// Use the brightness previously asked for
			state.Brightness = hl.desiredState.Brightness
		} else if hl.lastState != nil && hl.lastState.Brightness != nil {
			// Use the previously seen brightness
			state.Brightness = hl.lastState.Brightness
		} else {
			// Use the default
			state.Brightness = &defaultBrightness
		}
	}

	if state.Color == nil {
		// We have no color, but hue requires one to be sent.

		if hl.desiredState.Color != nil {
			// Use the color previously asked for
			state.Color = hl.desiredState.Color
		} else if hl.lastState != nil && hl.lastState.Color != nil {
			// Use the color we last saw
			state.Color = hl.lastState.Color
		} else {
			// Set it to the default color
			state.Color = &defaultColor
		}
	}

	if state.OnOff == nil || state.Color == nil || state.Brightness == nil || state.Transition == nil {
		// This shouldn't happen... but just in case...
		return fmt.Errorf("Values missing from light state... %s", dump(state))
	}

	hl.log.Debugf("Applying state (after defaults + desired) : %s", dump(state))

	outgoingState := &hue.LightState{
		On:             &*state.OnOff, // Always true at this point
		Brightness:     getBrightness(state),
		TransitionTime: getTransitionTime(state),
	}

	switch state.Color.Mode {
	case "hue":

		if state.Color.Hue == nil {
			return fmt.Errorf("Missing color hue")
		}

		if state.Color.Saturation == nil {
			return fmt.Errorf("Missing color saturation")
		}

		outgoingState.Hue = getHue(state)
		outgoingState.Saturation = getSaturation(state)

	case "xy":

		if state.Color.X == nil || state.Color.Y == nil {
			return fmt.Errorf("Missing X or Y from xy color state: %s", dump(state.Color))
		}

		outgoingState.XY = []float64{*state.Color.X, *state.Color.Y}

	case "temperature":

		if state.Color.Temperature == nil {
			return fmt.Errorf("Missing color temperature: %+v", *state.Color)
		}

		outgoingState.ColorTemp = getColorTemp(state)

		hl.log.Debugf("color temp: %d ", *outgoingState.ColorTemp)
	default:
		return fmt.Errorf("Unknown color mode %s", state.Color.Mode)
	}

	return hl.setLightState(outgoingState)
}

func (hl *HueLightContext) setLightState(lightState *hue.LightState) error {

	hl.log.Debugf("Sending light state to hue bulb: %s", dump(lightState))

	if err := hl.User.SetLightState(hl.ID, lightState); err != nil {
		return err
	}

	return hl.updateState()
}

func (hl *HueLightContext) updateState() error {

	la, err := hl.User.GetLightAttributes(hl.ID)
	if err != nil {
		return err
	}

	state := hl.toNinjaLightState(la.State)

	// If the light is on, we can destroy our desired state.. as it has either been used,
	// or the user has set it to something else using another app.
	if *state.OnOff {
		hl.desiredState = devices.LightDeviceState{}
	}

	// if we have desired state... use that.
	if hl.desiredState.Color != nil {
		state.Color = hl.desiredState.Color
	}
	if hl.desiredState.Brightness != nil {
		state.Brightness = hl.desiredState.Brightness
	}
	if hl.desiredState.Transition != nil {
		state.Transition = hl.desiredState.Transition
	}

	hl.log.Debugf("Updating light state: %+v", state)
	hl.lastState = state

	hl.Light.SetLightState(state)

	return nil
}

// ES: Yes, it isn't great. Whatever. Go fix something else.
func dump(obj interface{}) string {
	x, err := json.Marshal(obj)
	if err != nil {
		return spew.Sprintf("%+v", obj)
	}
	return string(x)
}

func (hl *HueLightContext) toNinjaLightState(huestate *hue.LightState) *devices.LightDeviceState {

	hl.log.Debugf("Converting hue state to ninja %s", dump(huestate))

	onOff := *huestate.On

	transition := defaultTransitionTime

	if hl.lastTransition != nil {
		transition = *hl.lastTransition
	}

	brightness := float64(*huestate.Brightness) / float64(math.MaxUint8)

	hl.log.Infof("Converted brightness from %v to %v", *huestate.Brightness, brightness)

	var lds *devices.LightDeviceState

	switch huestate.ColorMode {
	case "ct":
		// see: http://en.wikipedia.org/wiki/Mired
		// t = 1000000 / m
		temp := float64(1000000 / int(*huestate.ColorTemp))

		lds = &devices.LightDeviceState{
			Color: &channels.ColorState{
				Mode:        "temperature",
				Temperature: &temp,
			},
			Brightness: &brightness,
			OnOff:      &onOff,
			Transition: &transition,
		}
	case "xy":

		lds = &devices.LightDeviceState{
			Color: &channels.ColorState{
				Mode: "temperature",
				X:    &huestate.XY[0],
				Y:    &huestate.XY[1],
			},
			Brightness: &brightness,
			OnOff:      &onOff,
			Transition: &transition,
		}
	case "hs":
		hue := float64(*huestate.Hue) / float64(math.MaxUint16)
		saturation := float64(*huestate.Saturation) / float64(math.MaxUint16)

		lds = &devices.LightDeviceState{
			Color: &channels.ColorState{
				Mode:       "hue",
				Hue:        &hue,
				Saturation: &saturation,
			},
			Brightness: &brightness,
			OnOff:      &onOff,
			Transition: &transition,
		}
	default:
		hl.log.FatalError(errors.New("Invalid color mode"), "Failed to load hue state")
	}

	hl.log.Infof("Converted to ninja state %s", dump(lds))

	return lds
}

func getTransitionTime(state *devices.LightDeviceState) *uint16 {

	var transTime uint16
	if *state.Transition > 0 && *state.Transition < math.MaxUint16 {
		transTime = uint16(*state.Transition / 100) //HUE API uses 1/10th of a second
	} else {
		transTime = 0
	}
	return &transTime
}

func getHue(state *devices.LightDeviceState) *uint16 {
	hue := uint16(*state.Color.Hue * math.MaxUint16)
	return &hue
}

func getSaturation(state *devices.LightDeviceState) *uint8 {
	saturation := uint8(*state.Color.Saturation * math.MaxUint16)
	return &saturation
}

func getBrightness(state *devices.LightDeviceState) *uint8 {
	brightness := uint8(*state.Brightness * math.MaxUint8)
	return &brightness
}

func getColorTemp(state *devices.LightDeviceState) *uint16 {
	// see: http://en.wikipedia.org/wiki/Mired
	// m = 1000000 / t
	temp := uint16(1000000 / int(*state.Color.Temperature))
	return &temp
}
<|endoftext|>


<commit_before>// TODO : Documentation

package view

// TODO Implementation of API

func (devView DeviceView) IsDescriptionPresent(description string, index int, timeout int) (bool, error) {
	// TODO : implement this method
	return false, nil
}

func (devView DeviceView) IsMatchingDescriptionPresnt(description string, index int, timeout int) (bool, error) {
	// TODO : implement this method
	return false, nil
}

func (devView DeviceView) ClickDescription(description string, index int, timeout int) (bool, error) {
	// TODO : implement this method
	return false, nil
}

func (devView DeviceView) ClickMatchingDescription(description string, index int, timeout int) (bool, error) {
	// TODO : Implement this method
	return false, nil
}

func (devView DeviceView) GetDescriptionForText(text string, index int, timeout int) (bool, error) {
	// TODO : Implement this method
	return false, nil
}

func (devView DeviceView) GetDescriptionForType(typename string, index int, timeout int) (bool, error) {
	// TODO : Implement this method
	return false, nil
}

func (devView DeviceView) GetDescriptionForresource(resource string, index int, timeout int) (bool, error) {
	// TODO : Implement this method
	return false, nil
}
<commit_msg>Added description based view operations<commit_after>package view

import (
	"errors"
	"fmt"
	"github.com/kunaldawn/goandroid/input"
	"github.com/kunaldawn/goandroid/logging"
	"time"
)

func (devView DeviceView) IsDescriptionPresent(description string, index int, timeout int) error {
	logging.Log("IsDescriptionPresent : description [%s] : index [%d] : timeout [%d]", description, index, timeout)
	start := time.Now()
	for {
		current := time.Now()
		delta := current.Sub(start)
		if delta.Seconds() >= float64(timeout) {
			break
		}
		vws, err := devView.GetViewes()
		if err != nil {
			return err
		}
		_, found := vws.GetByDescription(description, index)
		if found {
			return nil
		}
	}
	return errors.New(fmt.Sprintf("Timeout occured after [%d] seconds while searching for description [%s]", timeout, description))
}

func (devView DeviceView) IsMatchingDescriptionPresnt(description string, index int, timeout int) error {
	logging.Log("IsMatchingDescriptionPresnt : description [%s] : index [%d] : timeout [%d]", description, index, timeout)
	start := time.Now()
	for {
		current := time.Now()
		delta := current.Sub(start)
		if delta.Seconds() >= float64(timeout) {
			break
		}
		vws, err := devView.GetViewes()
		if err != nil {
			return err
		}
		_, found := vws.GetByMatchingDescription(description, index)
		if found {
			return nil
		}
	}
	return errors.New(fmt.Sprintf("Timeout occured after %d seconds while searching for matching description [%s]", timeout, description))
}

func (devView DeviceView) ClickDescription(description string, index int, timeout int) error {
	logging.Log("ClickDescription : description [%s] : index [%d] : timeout [%d]", description, index, timeout)
	start := time.Now()
	for {
		current := time.Now()
		delta := current.Sub(start)
		if delta.Seconds() >= float64(timeout) {
			break
		}
		vws, err := devView.GetViewes()
		if err != nil {
			return err
		}
		vw, found := vws.GetByDescription(description, index)
		if found {
			im := input.NewInputManager(devView.dev)
			return im.TouchScreen.Tap(vw.Center.X, vw.Center.Y)
		}
	}
	return errors.New(fmt.Sprintf("Timeout occured after %d seconds while searching for description [%s]", timeout, description))
}

func (devView DeviceView) ClickMatchingDescription(description string, index int, timeout int) error {
	logging.Log("ClickMatchingDescription : description [%s] : index [%d] : timeout [%d]", description, index, timeout)
	start := time.Now()
	for {
		current := time.Now()
		delta := current.Sub(start)
		if delta.Seconds() >= float64(timeout) {
			break
		}
		vws, err := devView.GetViewes()
		if err != nil {
			return err
		}
		vw, found := vws.GetByMatchingDescription(description, index)
		if found {
			im := input.NewInputManager(devView.dev)
			return im.TouchScreen.Tap(vw.Center.X, vw.Center.Y)
		}
	}
	return errors.New(fmt.Sprintf("Timeout occured after %d seconds while searching for matching description [%s]", timeout, description))
}

func (devView DeviceView) GetDescriptionForText(text string, index int, timeout int) (string, error) {
	logging.Log("GetTypeForText : text [%s] : index [%d] : timeout [%d]", text, index, timeout)
	start := time.Now()
	for {
		current := time.Now()
		delta := current.Sub(start)
		if delta.Seconds() >= float64(timeout) {
			break
		}
		vws, err := devView.GetViewes()
		if err != nil {
			return "", err
		}
		vw, found := vws.GetByText(text, index)
		if found {
			return vw.Description, nil
		}
	}
	return "", errors.New(fmt.Sprintf("Timeout occured after %d seconds while searching for text [%s]", timeout, text))
}

func (devView DeviceView) GetDescriptionForMatchingText(text string, index int, timeout int) (string, error) {
	logging.Log("GetTypeForMatchingText : text [%s] : index [%d] : timeout [%d]", text, index, timeout)
	start := time.Now()
	for {
		current := time.Now()
		delta := current.Sub(start)
		if delta.Seconds() >= float64(timeout) {
			break
		}
		vws, err := devView.GetViewes()
		if err != nil {
			return "", err
		}
		vw, found := vws.GetByMatchingText(text, index)
		if found {
			return vw.Description, nil
		}
	}
	return "", errors.New(fmt.Sprintf("Timeout occured after %d seconds while searching for matcnhing text [%s]", timeout, text))
}

func (devView DeviceView) GetDescriptionForResource(resource string, index int, timeout int) (string, error) {
	logging.Log("GetTypeForResource : resource [%s] : index [%d] : timeout [%d]", resource, index, timeout)
	start := time.Now()
	for {
		current := time.Now()
		delta := current.Sub(start)
		if delta.Seconds() >= float64(timeout) {
			break
		}
		vws, err := devView.GetViewes()
		if err != nil {
			return "", err
		}
		vw, found := vws.GetByResource(resource, index)
		if found {
			return vw.Description, nil
		}
	}
	return "", errors.New(fmt.Sprintf("Timeout occured after %d seconds while searching for resource [%s]", timeout, resource))
}

func (devView DeviceView) GetDescriptionForMatchingResource(resource string, index int, timeout int) (string, error) {
	logging.Log("GettypeForMatchingResource : resource [%s] : index [%d] : timeout [%d]", resource, index, timeout)
	start := time.Now()
	for {
		current := time.Now()
		delta := current.Sub(start)
		if delta.Seconds() >= float64(timeout) {
			break
		}
		vws, err := devView.GetViewes()
		if err != nil {
			return "", err
		}
		vw, found := vws.GetByMatchingResource(resource, index)
		if found {
			return vw.Description, nil
		}
	}
	return "", errors.New(fmt.Sprintf("Timeout occured after %d seconds while searching for matching resource [%s]", timeout, resource))
}

func (devView DeviceView) GetDescriptionFortype(typename string, index int, timeout int) (string, error) {
	logging.Log("GetTypeForDescription : type [%s] : index [%d] : timeout [%d]", typename, index, timeout)
	start := time.Now()
	for {
		current := time.Now()
		delta := current.Sub(start)
		if delta.Seconds() >= float64(timeout) {
			break
		}
		vws, err := devView.GetViewes()
		if err != nil {
			return "", err
		}
		vw, found := vws.GetByType(typename, index)
		if found {
			return vw.Description, nil
		}
	}
	return "", errors.New(fmt.Sprintf("Timeout occured after %d seconds while searching for type [%s]", timeout, typename))
}

func (devView DeviceView) GetDescriptionForMatchingType(typename string, index int, timeout int) (string, error) {
	logging.Log("GetTypeForMatchingDescription : type [%s] : index [%d] : timeout [%d]", typename, index, timeout)
	start := time.Now()
	for {
		current := time.Now()
		delta := current.Sub(start)
		if delta.Seconds() >= float64(timeout) {
			break
		}
		vws, err := devView.GetViewes()
		if err != nil {
			return "", err
		}
		vw, found := vws.GetByMatchingType(typename, index)
		if found {
			return vw.Class, nil
		}
	}
	return "", errors.New(fmt.Sprintf("Timeout occured after %d seconds while searching for matching type [%s]", timeout, typename))
}
<|endoftext|>


<commit_before>// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

package ghttp

import (
	"fmt"
	"reflect"
	"strings"

	"github.com/gogf/gf/os/gfile"
	"github.com/gogf/gf/text/gregex"
	"github.com/gogf/gf/text/gstr"
)

// BindObject registers object to server routes with given pattern.
//
// The optional parameter <method> is used to specify the method to be registered, which
// supports multiple method names, multiple methods are separated by char ',', case sensitive.
//
// Note that the route method should be defined as ghttp.HandlerFunc.
func (s *Server) BindObject(pattern string, object interface{}, method ...string) {
	bindMethod := ""
	if len(method) > 0 {
		bindMethod = method[0]
	}
	s.doBindObject(pattern, object, bindMethod, nil, "")
}

// BindObjectMethod registers specified method of object to server routes with given pattern.
//
// The optional parameter <method> is used to specify the method to be registered, which
// does not supports multiple method names but only one, case sensitive.
//
// Note that the route method should be defined as ghttp.HandlerFunc.
func (s *Server) BindObjectMethod(pattern string, object interface{}, method string) {
	s.doBindObjectMethod(pattern, object, method, nil, "")
}

// BindObjectRest registers object in REST API style to server with specified pattern.
// Note that the route method should be defined as ghttp.HandlerFunc.
func (s *Server) BindObjectRest(pattern string, object interface{}) {
	s.doBindObjectRest(pattern, object, nil, "")
}

func (s *Server) doBindObject(
	pattern string, object interface{}, method string,
	middleware []HandlerFunc, source string,
) {
	// Convert input method to map for convenience and high performance searching purpose.
	var methodMap map[string]bool
	if len(method) > 0 {
		methodMap = make(map[string]bool)
		for _, v := range strings.Split(method, ",") {
			methodMap[strings.TrimSpace(v)] = true
		}
	}
	// 当pattern中的method为all时，去掉该method，以便于后续方法判断
	domain, method, path, err := s.parsePattern(pattern)
	if err != nil {
		s.Logger().Fatal(err)
		return
	}
	if strings.EqualFold(method, defaultMethod) {
		pattern = s.serveHandlerKey("", path, domain)
	}
	m := make(map[string]*handlerItem)
	v := reflect.ValueOf(object)
	t := v.Type()
	initFunc := (func(*Request))(nil)
	shutFunc := (func(*Request))(nil)
	structName := t.Elem().Name()
	if v.MethodByName("Init").IsValid() {
		initFunc = v.MethodByName("Init").Interface().(func(*Request))
	}
	if v.MethodByName("Shut").IsValid() {
		shutFunc = v.MethodByName("Shut").Interface().(func(*Request))
	}
	pkgPath := t.Elem().PkgPath()
	pkgName := gfile.Basename(pkgPath)
	for i := 0; i < v.NumMethod(); i++ {
		methodName := t.Method(i).Name
		if methodMap != nil && !methodMap[methodName] {
			continue
		}
		if methodName == "Init" || methodName == "Shut" {
			continue
		}
		objName := gstr.Replace(t.String(), fmt.Sprintf(`%s.`, pkgName), "")
		if objName[0] == '*' {
			objName = fmt.Sprintf(`(%s)`, objName)
		}
		itemFunc, ok := v.Method(i).Interface().(func(*Request))
		if !ok {
			if len(methodMap) > 0 {
				s.Logger().Errorf(
					`invalid route method: %s.%s.%s defined as "%s", but "func(*ghttp.Request)" is required for object registry`,
					pkgPath, objName, methodName, v.Method(i).Type().String(),
				)
			} else {
				s.Logger().Debugf(
					`ignore route method: %s.%s.%s defined as "%s", no match "func(*ghttp.Request)" for object registry`,
					pkgPath, objName, methodName, v.Method(i).Type().String(),
				)
			}
			continue
		}
		key := s.mergeBuildInNameToPattern(pattern, structName, methodName, true)
		m[key] = &handlerItem{
			itemName:   fmt.Sprintf(`%s.%s.%s`, pkgPath, objName, methodName),
			itemType:   handlerTypeObject,
			itemFunc:   itemFunc,
			initFunc:   initFunc,
			shutFunc:   shutFunc,
			middleware: middleware,
			source:     source,
		}
		// If there's "Index" method, then an additional route is automatically added
		// to match the main URI, for example:
		// If pattern is "/user", then "/user" and "/user/index" are both automatically
		// registered.
		//
		// Note that if there's built-in variables in pattern, this route will not be added
		// automatically.
		if strings.EqualFold(methodName, "Index") && !gregex.IsMatchString(`\{\.\w+\}`, pattern) {
			p := gstr.PosRI(key, "/index")
			k := key[0:p] + key[p+6:]
			if len(k) == 0 || k[0] == '@' {
				k = "/" + k
			}
			m[k] = &handlerItem{
				itemName:   fmt.Sprintf(`%s.%s.%s`, pkgPath, objName, methodName),
				itemType:   handlerTypeObject,
				itemFunc:   itemFunc,
				initFunc:   initFunc,
				shutFunc:   shutFunc,
				middleware: middleware,
				source:     source,
			}
		}
	}
	s.bindHandlerByMap(m)
}

func (s *Server) doBindObjectMethod(
	pattern string, object interface{}, method string,
	middleware []HandlerFunc, source string,
) {
	m := make(map[string]*handlerItem)
	v := reflect.ValueOf(object)
	t := v.Type()
	structName := t.Elem().Name()
	methodName := strings.TrimSpace(method)
	methodValue := v.MethodByName(methodName)
	if !methodValue.IsValid() {
		s.Logger().Fatal("invalid method name: " + methodName)
		return
	}
	initFunc := (func(*Request))(nil)
	shutFunc := (func(*Request))(nil)
	if v.MethodByName("Init").IsValid() {
		initFunc = v.MethodByName("Init").Interface().(func(*Request))
	}
	if v.MethodByName("Shut").IsValid() {
		shutFunc = v.MethodByName("Shut").Interface().(func(*Request))
	}
	pkgPath := t.Elem().PkgPath()
	pkgName := gfile.Basename(pkgPath)
	objName := gstr.Replace(t.String(), fmt.Sprintf(`%s.`, pkgName), "")
	if objName[0] == '*' {
		objName = fmt.Sprintf(`(%s)`, objName)
	}
	itemFunc, ok := methodValue.Interface().(func(*Request))
	if !ok {
		s.Logger().Errorf(
			`invalid route method: %s.%s.%s defined as "%s", but "func(*ghttp.Request)" is required for object registry`,
			pkgPath, objName, methodName, methodValue.Type().String(),
		)
		return
	}
	key := s.mergeBuildInNameToPattern(pattern, structName, methodName, false)
	m[key] = &handlerItem{
		itemName:   fmt.Sprintf(`%s.%s.%s`, pkgPath, objName, methodName),
		itemType:   handlerTypeObject,
		itemFunc:   itemFunc,
		initFunc:   initFunc,
		shutFunc:   shutFunc,
		middleware: middleware,
		source:     source,
	}

	s.bindHandlerByMap(m)
}

func (s *Server) doBindObjectRest(
	pattern string, object interface{},
	middleware []HandlerFunc, source string,
) {
	m := make(map[string]*handlerItem)
	v := reflect.ValueOf(object)
	t := v.Type()
	initFunc := (func(*Request))(nil)
	shutFunc := (func(*Request))(nil)
	structName := t.Elem().Name()
	if v.MethodByName("Init").IsValid() {
		initFunc = v.MethodByName("Init").Interface().(func(*Request))
	}
	if v.MethodByName("Shut").IsValid() {
		shutFunc = v.MethodByName("Shut").Interface().(func(*Request))
	}
	pkgPath := t.Elem().PkgPath()
	for i := 0; i < v.NumMethod(); i++ {
		methodName := t.Method(i).Name
		if _, ok := methodsMap[strings.ToUpper(methodName)]; !ok {
			continue
		}
		pkgName := gfile.Basename(pkgPath)
		objName := gstr.Replace(t.String(), fmt.Sprintf(`%s.`, pkgName), "")
		if objName[0] == '*' {
			objName = fmt.Sprintf(`(%s)`, objName)
		}
		itemFunc, ok := v.Method(i).Interface().(func(*Request))
		if !ok {
			s.Logger().Errorf(
				`invalid route method: %s.%s.%s defined as "%s", but "func(*ghttp.Request)" is required for object registry`,
				pkgPath, objName, methodName, v.Method(i).Type().String(),
			)
			continue
		}
		key := s.mergeBuildInNameToPattern(methodName+":"+pattern, structName, methodName, false)
		m[key] = &handlerItem{
			itemName:   fmt.Sprintf(`%s.%s.%s`, pkgPath, objName, methodName),
			itemType:   handlerTypeObject,
			itemFunc:   itemFunc,
			initFunc:   initFunc,
			shutFunc:   shutFunc,
			middleware: middleware,
			source:     source,
		}
	}
	s.bindHandlerByMap(m)
}
<commit_msg>add struct object support in group router registering for package ghttp<commit_after>// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

package ghttp

import (
	"fmt"
	"reflect"
	"strings"

	"github.com/gogf/gf/os/gfile"
	"github.com/gogf/gf/text/gregex"
	"github.com/gogf/gf/text/gstr"
)

// BindObject registers object to server routes with given pattern.
//
// The optional parameter <method> is used to specify the method to be registered, which
// supports multiple method names, multiple methods are separated by char ',', case sensitive.
//
// Note that the route method should be defined as ghttp.HandlerFunc.
func (s *Server) BindObject(pattern string, object interface{}, method ...string) {
	bindMethod := ""
	if len(method) > 0 {
		bindMethod = method[0]
	}
	s.doBindObject(pattern, object, bindMethod, nil, "")
}

// BindObjectMethod registers specified method of object to server routes with given pattern.
//
// The optional parameter <method> is used to specify the method to be registered, which
// does not supports multiple method names but only one, case sensitive.
//
// Note that the route method should be defined as ghttp.HandlerFunc.
func (s *Server) BindObjectMethod(pattern string, object interface{}, method string) {
	s.doBindObjectMethod(pattern, object, method, nil, "")
}

// BindObjectRest registers object in REST API style to server with specified pattern.
// Note that the route method should be defined as ghttp.HandlerFunc.
func (s *Server) BindObjectRest(pattern string, object interface{}) {
	s.doBindObjectRest(pattern, object, nil, "")
}

func (s *Server) doBindObject(
	pattern string, object interface{}, method string,
	middleware []HandlerFunc, source string,
) {
	// Convert input method to map for convenience and high performance searching purpose.
	var methodMap map[string]bool
	if len(method) > 0 {
		methodMap = make(map[string]bool)
		for _, v := range strings.Split(method, ",") {
			methodMap[strings.TrimSpace(v)] = true
		}
	}
	// If the `method` in `pattern` is `defaultMethod`,
	// it removes for convenience for next statement control.
	domain, method, path, err := s.parsePattern(pattern)
	if err != nil {
		s.Logger().Fatal(err)
		return
	}
	if strings.EqualFold(method, defaultMethod) {
		pattern = s.serveHandlerKey("", path, domain)
	}
	var (
		m        = make(map[string]*handlerItem)
		v        = reflect.ValueOf(object)
		t        = v.Type()
		initFunc func(*Request)
		shutFunc func(*Request)
	)
	// If given `object` is not pointer, it then creates a temporary one,
	// of which the value is `v`.
	if v.Kind() == reflect.Struct {
		newValue := reflect.New(t)
		newValue.Elem().Set(v)
		v = newValue
		t = v.Type()
	}
	structName := t.Elem().Name()
	if v.MethodByName("Init").IsValid() {
		initFunc = v.MethodByName("Init").Interface().(func(*Request))
	}
	if v.MethodByName("Shut").IsValid() {
		shutFunc = v.MethodByName("Shut").Interface().(func(*Request))
	}
	pkgPath := t.Elem().PkgPath()
	pkgName := gfile.Basename(pkgPath)
	for i := 0; i < v.NumMethod(); i++ {
		methodName := t.Method(i).Name
		if methodMap != nil && !methodMap[methodName] {
			continue
		}
		if methodName == "Init" || methodName == "Shut" {
			continue
		}
		objName := gstr.Replace(t.String(), fmt.Sprintf(`%s.`, pkgName), "")
		if objName[0] == '*' {
			objName = fmt.Sprintf(`(%s)`, objName)
		}
		itemFunc, ok := v.Method(i).Interface().(func(*Request))
		if !ok {
			if len(methodMap) > 0 {
				s.Logger().Errorf(
					`invalid route method: %s.%s.%s defined as "%s", but "func(*ghttp.Request)" is required for object registry`,
					pkgPath, objName, methodName, v.Method(i).Type().String(),
				)
			} else {
				s.Logger().Debugf(
					`ignore route method: %s.%s.%s defined as "%s", no match "func(*ghttp.Request)" for object registry`,
					pkgPath, objName, methodName, v.Method(i).Type().String(),
				)
			}
			continue
		}
		key := s.mergeBuildInNameToPattern(pattern, structName, methodName, true)
		m[key] = &handlerItem{
			itemName:   fmt.Sprintf(`%s.%s.%s`, pkgPath, objName, methodName),
			itemType:   handlerTypeObject,
			itemFunc:   itemFunc,
			initFunc:   initFunc,
			shutFunc:   shutFunc,
			middleware: middleware,
			source:     source,
		}
		// If there's "Index" method, then an additional route is automatically added
		// to match the main URI, for example:
		// If pattern is "/user", then "/user" and "/user/index" are both automatically
		// registered.
		//
		// Note that if there's built-in variables in pattern, this route will not be added
		// automatically.
		if strings.EqualFold(methodName, "Index") && !gregex.IsMatchString(`\{\.\w+\}`, pattern) {
			p := gstr.PosRI(key, "/index")
			k := key[0:p] + key[p+6:]
			if len(k) == 0 || k[0] == '@' {
				k = "/" + k
			}
			m[k] = &handlerItem{
				itemName:   fmt.Sprintf(`%s.%s.%s`, pkgPath, objName, methodName),
				itemType:   handlerTypeObject,
				itemFunc:   itemFunc,
				initFunc:   initFunc,
				shutFunc:   shutFunc,
				middleware: middleware,
				source:     source,
			}
		}
	}
	s.bindHandlerByMap(m)
}

func (s *Server) doBindObjectMethod(
	pattern string, object interface{}, method string,
	middleware []HandlerFunc, source string,
) {
	var (
		m        = make(map[string]*handlerItem)
		v        = reflect.ValueOf(object)
		t        = v.Type()
		initFunc func(*Request)
		shutFunc func(*Request)
	)
	// If given `object` is not pointer, it then creates a temporary one,
	// of which the value is `v`.
	if v.Kind() == reflect.Struct {
		newValue := reflect.New(t)
		newValue.Elem().Set(v)
		v = newValue
		t = v.Type()
	}
	structName := t.Elem().Name()
	methodName := strings.TrimSpace(method)
	methodValue := v.MethodByName(methodName)
	if !methodValue.IsValid() {
		s.Logger().Fatal("invalid method name: " + methodName)
		return
	}
	if v.MethodByName("Init").IsValid() {
		initFunc = v.MethodByName("Init").Interface().(func(*Request))
	}
	if v.MethodByName("Shut").IsValid() {
		shutFunc = v.MethodByName("Shut").Interface().(func(*Request))
	}
	pkgPath := t.Elem().PkgPath()
	pkgName := gfile.Basename(pkgPath)
	objName := gstr.Replace(t.String(), fmt.Sprintf(`%s.`, pkgName), "")
	if objName[0] == '*' {
		objName = fmt.Sprintf(`(%s)`, objName)
	}
	itemFunc, ok := methodValue.Interface().(func(*Request))
	if !ok {
		s.Logger().Errorf(
			`invalid route method: %s.%s.%s defined as "%s", but "func(*ghttp.Request)" is required for object registry`,
			pkgPath, objName, methodName, methodValue.Type().String(),
		)
		return
	}
	key := s.mergeBuildInNameToPattern(pattern, structName, methodName, false)
	m[key] = &handlerItem{
		itemName:   fmt.Sprintf(`%s.%s.%s`, pkgPath, objName, methodName),
		itemType:   handlerTypeObject,
		itemFunc:   itemFunc,
		initFunc:   initFunc,
		shutFunc:   shutFunc,
		middleware: middleware,
		source:     source,
	}

	s.bindHandlerByMap(m)
}

func (s *Server) doBindObjectRest(
	pattern string, object interface{},
	middleware []HandlerFunc, source string,
) {
	var (
		m        = make(map[string]*handlerItem)
		v        = reflect.ValueOf(object)
		t        = v.Type()
		initFunc func(*Request)
		shutFunc func(*Request)
	)
	// If given `object` is not pointer, it then creates a temporary one,
	// of which the value is `v`.
	if v.Kind() == reflect.Struct {
		newValue := reflect.New(t)
		newValue.Elem().Set(v)
		v = newValue
		t = v.Type()
	}
	structName := t.Elem().Name()
	if v.MethodByName("Init").IsValid() {
		initFunc = v.MethodByName("Init").Interface().(func(*Request))
	}
	if v.MethodByName("Shut").IsValid() {
		shutFunc = v.MethodByName("Shut").Interface().(func(*Request))
	}
	pkgPath := t.Elem().PkgPath()
	for i := 0; i < v.NumMethod(); i++ {
		methodName := t.Method(i).Name
		if _, ok := methodsMap[strings.ToUpper(methodName)]; !ok {
			continue
		}
		pkgName := gfile.Basename(pkgPath)
		objName := gstr.Replace(t.String(), fmt.Sprintf(`%s.`, pkgName), "")
		if objName[0] == '*' {
			objName = fmt.Sprintf(`(%s)`, objName)
		}
		itemFunc, ok := v.Method(i).Interface().(func(*Request))
		if !ok {
			s.Logger().Errorf(
				`invalid route method: %s.%s.%s defined as "%s", but "func(*ghttp.Request)" is required for object registry`,
				pkgPath, objName, methodName, v.Method(i).Type().String(),
			)
			continue
		}
		key := s.mergeBuildInNameToPattern(methodName+":"+pattern, structName, methodName, false)
		m[key] = &handlerItem{
			itemName:   fmt.Sprintf(`%s.%s.%s`, pkgPath, objName, methodName),
			itemType:   handlerTypeObject,
			itemFunc:   itemFunc,
			initFunc:   initFunc,
			shutFunc:   shutFunc,
			middleware: middleware,
			source:     source,
		}
	}
	s.bindHandlerByMap(m)
}
<|endoftext|>


<commit_before>package test

import "testing"
import "strconv"
import "go-cache"
import "go-cache/arc"
import "math/rand"
import "time"

func TestGet(t *testing.T) {
	cacheSize := 100
	countCleaned := 0
	countAdded := 0
	countAccess := 2000

	arc := arc.NewArcCache(cacheSize)

	arc.SetCleanFunc(func (obj cache.CacheObject) error {
		countCleaned += 1
		return nil
	})
	arc.SetFetchFunc(func (key string) (cache.CacheObject, error) {
		countAdded += 1
		return key, nil
	})

	rand.Seed(time.Now().Unix())

	for i := 0; i < countAccess; i ++ {
		j := rand.Intn(cacheSize*2)
		val, err := arc.Get("key"+strconv.Itoa(j))
		if err != cache.CacheMiss && err != nil {
			t.Errorf("unexpected err:", err.Error())
		}
		if val == nil || val.(string) != "key"+strconv.Itoa(j) {
			t.Errorf("key does not match the value")
		}
	}

	countMiss := 0
	for i := 0; i < countAccess; i ++ {
		j := rand.Intn(cacheSize*2)
		val, err := arc.Get("key"+strconv.Itoa(j))
		if err == cache.CacheMiss {
			countMiss += 1
		}
		if val == nil || val.(string) != "key"+strconv.Itoa(j) {
			t.Errorf("key does not match the value")
		}
	}
	println("cache hit rate:", (100*(countAccess-countMiss))/countAccess)

	arc.CheckCache()

	if countCleaned + cacheSize != countAdded {
		t.Errorf("numbers of data items dont match: %d != %d + %d\n", countAdded, countCleaned, cacheSize)
	}

	for key, obj := range(arc.GetAllObjects()) {
		if key != obj.(string) {
			t.Errorf("key does not match the cached value")
		}
	}
}<commit_msg>add test for lru<commit_after>package test

import "testing"
import "strconv"
import "go-cache"
import "go-cache/arc"
import "go-cache/lru"
import "math/rand"
import "time"

func TestGetARC(t *testing.T) {
	cacheSize := 100
	countCleaned := 0
	countAdded := 0
	countAccess := 2000

	c := arc.NewArcCache(cacheSize)

	c.SetCleanFunc(func (obj cache.CacheObject) error {
		countCleaned += 1
		//println("replacing", obj.(string))
		return nil
	})
	c.SetFetchFunc(func (key string) (cache.CacheObject, error) {
		countAdded += 1
		return key, nil
	})

	rand.Seed(time.Now().Unix())

	for i := 0; i < countAccess; i ++ {
		j := rand.Intn(cacheSize*2)
		val, err := c.Get("key"+strconv.Itoa(j))
		if err != cache.CacheMiss && err != nil {
			t.Errorf("unexpected err:", err.Error())
		}
		if val == nil || val.(string) != "key"+strconv.Itoa(j) {
			t.Errorf("key does not match the value")
		}
	}

	countMiss := 0
	countAccess = 10
	c.SetCleanFunc(func (obj cache.CacheObject) error {
		countCleaned += 1
		println("replacing", obj.(string))
		return nil
	})
	for i := 0; i < countAccess; i ++ {
		j := rand.Intn(cacheSize*2)
		val, err := c.Get("key"+strconv.Itoa(j))
		if err == cache.CacheMiss {
			countMiss += 1
		}
		if val == nil || val.(string) != "key"+strconv.Itoa(j) {
			t.Errorf("key does not match the value")
		}
	}
	println("cache hit rate:", (100*(countAccess-countMiss))/countAccess, countMiss)

	c.CheckCache()

	if countCleaned + cacheSize != countAdded {
		t.Errorf("numbers of data items dont match: %d != %d + %d\n", countAdded, countCleaned, cacheSize)
	}

	for key, obj := range(c.GetAllObjects()) {
		if key != obj.(string) {
			t.Errorf("key does not match the cached value")
		}
	}
}

func TestGetLRU(t *testing.T) {
	cacheSize := 100
	countCleaned := 0
	countAdded := 0
	countAccess := 2000

	c := lru.NewLRUCache(cacheSize)

	c.SetCleanFunc(func (obj cache.CacheObject) error {
		countCleaned += 1
		//println("replacing", obj.(string))
		return nil
	})
	c.SetFetchFunc(func (key string) (cache.CacheObject, error) {
		countAdded += 1
		return key, nil
	})

	rand.Seed(time.Now().Unix())

	for i := 0; i < countAccess; i ++ {
		j := rand.Intn(cacheSize*2)
		val, err := c.Get("key"+strconv.Itoa(j))
		if err != cache.CacheMiss && err != nil {
			t.Errorf("unexpected err:", err.Error())
		}
		if val == nil || val.(string) != "key"+strconv.Itoa(j) {
			t.Errorf("key does not match the value")
		}
	}

	countMiss := 0
	countAccess = 10
	c.SetCleanFunc(func (obj cache.CacheObject) error {
		countCleaned += 1
		println("replacing", obj.(string))
		return nil
	})
	for i := 0; i < countAccess; i ++ {
		j := rand.Intn(cacheSize*2)
		val, err := c.Get("key"+strconv.Itoa(j))
		if err == cache.CacheMiss {
			countMiss += 1
		}
		if val == nil || val.(string) != "key"+strconv.Itoa(j) {
			t.Errorf("key does not match the value")
		}
	}
	println("cache hit rate:", (100*(countAccess-countMiss))/countAccess, countMiss)

	c.CheckCache()

	if countCleaned + cacheSize != countAdded {
		t.Errorf("numbers of data items dont match: %d != %d + %d\n", countAdded, countCleaned, cacheSize)
	}

	for key, obj := range(c.GetAllObjects()) {
		if key != obj.(string) {
			t.Errorf("key does not match the cached value")
		}
	}
}<|endoftext|>


<commit_before>package parse_test

import (
	. "github.com/yanda15/hdc/hive"
	"log"
	"testing"
	"time"
)

var h *Hive

type Sample7 struct {
	Code        string `tag_name:"code"`
	Description string `tag_name:"description"`
	Total_emp   int    `tag_name:"total_emp"`
	Salary      int    `tag_name:"salary"`
}

type SampleParse struct {
	Code        string    `tag_name:"code"`
	Description string    `tag_name:"description"`
	Total_emp   int       `tag_name:"total_emp"`
	Salary      int       `tag_name:"salary"`
	Date        time.Time `tag_name:"date"`
}

func TestParseOutput(t *testing.T) {
	h = HiveConfig("192.168.0.223:10000", "default", "developer", "b1gD@T@", "")
	h.Header = []string{"Code", "Description", "Total_emp", "Salary", "Date"}

	h.OutputType = "csv"
	h.DateFormat = "yyyy-MM-dd"
	res := []string{"'00-0000','All Occupations CSV','134354250','40690','2014-05-01'"}
	tmp := []SampleParse{}
	h.ParseOutput(res, &tmp)
	log.Println(tmp)

	h.OutputType = "csv"
	h.DateFormat = "YYYY-MM-dd"
	res = []string{"00-0000,All Occupations CSV2,134354250,40690,2014-05-01"}
	tmp = []SampleParse{}
	h.ParseOutput(res, &tmp)
	log.Println(tmp)

	h.OutputType = "tsv"
	h.DateFormat = "YYYY-MMM-dd"
	res = []string{"'00-0000'\t'All Occupations TSV'\t'134354250'\t'40690'\t'2014-Dec-05'"}
	tmp = []SampleParse{}
	h.ParseOutput(res, &tmp)
	log.Println(tmp)

	//try to parse json with different line
	h.OutputType = "json"
	resj := []string{"{ \"code\" : \"00-0000\" , \"description\" : \"All Occupations JSON\" "}
	tmpj := []SampleParse{}
	h.ParseOutput(resj, &tmpj)
	log.Println(tmpj)

	resj = []string{", \"total_emp\" : 134354, \"salary\" : 40690,\"Date\" : \"2012-04-23T18:25:43Z\" },{ \"code\" : \"00-2222\""}
	tmpj = []SampleParse{}
	h.ParseOutput(resj, &tmpj)
	log.Println(tmpj)

	resj = []string{",\"description\" : \"All Occupations INTERFACE\" , \"total_emp\" : 222, \"salary\" : 2222,\"Date\" : \"2012-05-23T18:25:43Z\" },{ \"code\" : \"00-2222\",\"description\" : \"All Occupations SISANYA\" , \"total_emp\" : 222, \"salary\" : 2222,\"Date\" : \"2012-05-23T18:25:43Z\" }", "{ \"code\" : \"00-2222\",\"description\" : \"All Occupations Belakang DW\" , \"total_emp\" : 222, \"salary\" : 2222,\"Date\" : \"2012-05-23T18:25:43Z\" }"}
	var tmpx interface{}
	h.ParseOutput(resj, &tmpx)
	log.Println(tmpx)

}

// func TestConstructHeader(t *testing.T) {
// 	h = HiveConfig("192.168.0.223:10000", "default", "developer", "b1gD@T@", "")
// 	h.ConstructHeader("'sample_07.code'\t'sample_07.description'\t'sample_07.total_emp'\t'sample_07.salary'", "\t")
// 	log.Println(h.Header)
// }

// func TestUpperGoroutine(t *testing.T) {
// 	jobs := make(chan int, 1000)
// 	results := make(chan int, 1000)

// 	for w := 1; w <= 100; w++ {
// 		go worker(w, jobs, results)
// 	}

// 	for j := 1; j <= 150; j++ {
// 		jobs <- j
// 	}

// 	close(jobs)

// 	for a := 1; a <= 150; a++ {
// 		<-results
// 	}

// }

// func worker(id int, jobs <-chan int, results chan<- int) {
// 	for j := range jobs {
// 		log.Println("worker", id, "processing job", j)
// 		time.Sleep(time.Second)
// 		results <- j * 2
// 	}
// }
<commit_msg>handle multi json in one line<commit_after>package parse_test

import (
	. "github.com/yanda15/hdc/hive"
	"log"
	"testing"
	"time"
)

var h *Hive

type Sample7 struct {
	Code        string `tag_name:"code"`
	Description string `tag_name:"description"`
	Total_emp   int    `tag_name:"total_emp"`
	Salary      int    `tag_name:"salary"`
}

type SampleParse struct {
	Code        string    `tag_name:"code"`
	Description string    `tag_name:"description"`
	Total_emp   int       `tag_name:"total_emp"`
	Salary      int       `tag_name:"salary"`
	Date        time.Time `tag_name:"date"`
}

func TestParseOutput(t *testing.T) {
	h = HiveConfig("192.168.0.223:10000", "default", "developer", "b1gD@T@", "")
	h.Header = []string{"Code", "Description", "Total_emp", "Salary", "Date"}

	h.OutputType = "csv"
	h.DateFormat = "yyyy-MM-dd"
	res := []string{"'00-0000','All Occupations CSV','134354250','40690','2014-05-01'"}
	tmp := []SampleParse{}
	h.ParseOutput(res, &tmp)
	log.Println(tmp)

	h.OutputType = "csv"
	h.DateFormat = "YYYY-MM-dd"
	res = []string{"00-0000,All Occupations CSV2,134354250,40690,2014-05-01"}
	tmp = []SampleParse{}
	h.ParseOutput(res, &tmp)
	log.Println(tmp)

	h.OutputType = "tsv"
	h.DateFormat = "YYYY-MMM-dd"
	res = []string{"'00-0000'\t'All Occupations TSV'\t'134354250'\t'40690'\t'2014-Dec-05'"}
	tmp = []SampleParse{}
	h.ParseOutput(res, &tmp)
	log.Println(tmp)

	//try to parse json with different line
	h.OutputType = "json"
	resj := []string{"{ \"code\" : \"00-0000\" , \"description\" : \"All Occupations JSON\" "}
	tmpj := []SampleParse{}
	h.ParseOutput(resj, &tmpj)
	log.Println(tmpj)

	resj = []string{", \"total_emp\" : 134354, \"salary\" : 40690,\"Date\" : \"2012-04-23T18:25:43Z\" },{ \"code\" : \"00-2222\""}
	tmpj = []SampleParse{}
	h.ParseOutput(resj, &tmpj)
	log.Println(tmpj)

	resj = []string{",\"description\" : \"All Occupations INTERFACE\" , \"total_emp\" : 222, \"salary\" : 2222,\"Date\" : \"2012-05-23T18:25:43Z\" },{ \"code\" : \"00-2222\",\"description\" : \"All Occupations NEXT\" , \"total_emp\" : 222, \"salary\" : 2222,\"Date\" : \"2012-05-23T18:25:43Z\" }", "{ \"code\" : \"00-2222\",\"description\" : \"All Occupations Last\" , \"total_emp\" : 222, \"salary\" : 2222,\"Date\" : \"2012-05-23T18:25:43Z\" }"}
	var tmpx interface{}
	h.ParseOutput(resj, &tmpx)
	log.Println(tmpx)

}

// func TestConstructHeader(t *testing.T) {
// 	h = HiveConfig("192.168.0.223:10000", "default", "developer", "b1gD@T@", "")
// 	h.ConstructHeader("'sample_07.code'\t'sample_07.description'\t'sample_07.total_emp'\t'sample_07.salary'", "\t")
// 	log.Println(h.Header)
// }

// func TestUpperGoroutine(t *testing.T) {
// 	jobs := make(chan int, 1000)
// 	results := make(chan int, 1000)

// 	for w := 1; w <= 100; w++ {
// 		go worker(w, jobs, results)
// 	}

// 	for j := 1; j <= 150; j++ {
// 		jobs <- j
// 	}

// 	close(jobs)

// 	for a := 1; a <= 150; a++ {
// 		<-results
// 	}

// }

// func worker(id int, jobs <-chan int, results chan<- int) {
// 	for j := range jobs {
// 		log.Println("worker", id, "processing job", j)
// 		time.Sleep(time.Second)
// 		results <- j * 2
// 	}
// }
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"log"
	"net"
	"net/url"
	"os/exec"
	"strings"
	"time"

	"gopkg.in/yaml.v2"
)

type DockerCompose struct {
	Host string
}

func NewDockerCompose(dockerHost string) Platform {
	var host string
	if dockerHost != "" {
		var err error
		dockerHostURL, err := url.Parse(dockerHost)
		if err != nil {
			log.Panic(err)
		}
		host, _, err = net.SplitHostPort(dockerHostURL.Host)
		if err != nil {
			log.Panic(err)
		}
	} else {
		host = ""
	}

	return DockerCompose{host}
}

func (p DockerCompose) Build(s Service) error {
	_, err := exec.Command("docker-compose", "-f", p.ConfigPath(s), "build").CombinedOutput()
	return err
}

func (p DockerCompose) Stop(s Service) error {
	_, err := exec.Command("docker-compose", "-f", p.ConfigPath(s), "stop").CombinedOutput()
	return err
}

func (p DockerCompose) Start(s Service) error {
	err := exec.Command("docker-compose", "-f", p.ConfigPath(s), "up").Start()
	if err != nil {
		return err
	}

	time.Sleep(5 * time.Second)

	// loop through docker-compose.yml and look for exposed ip, save it
	ymlBytes, err := ioutil.ReadFile(p.ConfigPath(s))
	if err != nil {
		log.Panic(err)
	}
	m := make(map[string]interface{})
	err = yaml.Unmarshal(ymlBytes, &m)
	if err != nil {
		log.Panic(err)
	}

	for serviceName, conf := range m {
		if ports, ok := conf.(map[interface{}]interface{})["ports"]; ok {
			// TODO handle multiple exposed ports
			portValue := ports.([]interface{})[0].(string)
			var innerPort string

			// TODO handle 127.0.0.1:8001:8001 style config
			if len(strings.Split(portValue, ":")) == 2 {
				innerPort = strings.Split(portValue, ":")[1]
			} else {
				innerPort = portValue
			}
			urlStr, err := p.getExposedURLString(s, serviceName, innerPort)
			if err != nil {
				log.Panic(err)
			}
			u, err := url.Parse("http://" + urlStr)
			if err != nil {
				log.Panic(err)
			}
			if p.Host != "" {
				_, originPort, err := net.SplitHostPort(u.Host)
				if err != nil {
					return err
				}
				u.Host = p.Host + ":" + originPort
			}
			s.ExposedURLs[serviceName] = u
		}
	}

	return nil

}

func (p DockerCompose) getExposedURLString(s Service, serviceName string, port string) (string, error) {
	output, err := exec.Command("docker-compose", "-f", p.ConfigPath(s), "port", serviceName, port).CombinedOutput()
	if err != nil {
		return "", err
	}
	return strings.Trim(string(output), "\n"), nil
}

func (p DockerCompose) ConfigPath(s Service) string {
	return s.Path + "/docker-compose.yml"
}
<commit_msg>show build output if error occured<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/url"
	"os/exec"
	"strings"
	"time"

	"gopkg.in/yaml.v2"
)

type DockerCompose struct {
	Host string
}

func NewDockerCompose(dockerHost string) Platform {
	var host string
	if dockerHost != "" {
		var err error
		dockerHostURL, err := url.Parse(dockerHost)
		if err != nil {
			log.Panic(err)
		}
		host, _, err = net.SplitHostPort(dockerHostURL.Host)
		if err != nil {
			log.Panic(err)
		}
	} else {
		host = ""
	}

	return DockerCompose{host}
}

func (p DockerCompose) Build(s Service) error {
	o, err := exec.Command("docker-compose", "-f", p.ConfigPath(s), "build").CombinedOutput()
	if err != nil {
		fmt.Println(string(o))
	}
	return err
}

func (p DockerCompose) Stop(s Service) error {
	_, err := exec.Command("docker-compose", "-f", p.ConfigPath(s), "stop").CombinedOutput()
	return err
}

func (p DockerCompose) Start(s Service) error {
	err := exec.Command("docker-compose", "-f", p.ConfigPath(s), "up").Start()
	if err != nil {
		return err
	}

	time.Sleep(5 * time.Second)

	// loop through docker-compose.yml and look for exposed ip, save it
	ymlBytes, err := ioutil.ReadFile(p.ConfigPath(s))
	if err != nil {
		log.Panic(err)
	}
	m := make(map[string]interface{})
	err = yaml.Unmarshal(ymlBytes, &m)
	if err != nil {
		log.Panic(err)
	}

	for serviceName, conf := range m {
		if ports, ok := conf.(map[interface{}]interface{})["ports"]; ok {
			// TODO handle multiple exposed ports
			portValue := ports.([]interface{})[0].(string)
			var innerPort string

			// TODO handle 127.0.0.1:8001:8001 style config
			if len(strings.Split(portValue, ":")) == 2 {
				innerPort = strings.Split(portValue, ":")[1]
			} else {
				innerPort = portValue
			}
			urlStr, err := p.getExposedURLString(s, serviceName, innerPort)
			if err != nil {
				log.Panic(err)
			}
			u, err := url.Parse("http://" + urlStr)
			if err != nil {
				log.Panic(err)
			}
			if p.Host != "" {
				_, originPort, err := net.SplitHostPort(u.Host)
				if err != nil {
					return err
				}
				u.Host = p.Host + ":" + originPort
			}
			s.ExposedURLs[serviceName] = u
		}
	}

	return nil

}

func (p DockerCompose) getExposedURLString(s Service, serviceName string, port string) (string, error) {
	output, err := exec.Command("docker-compose", "-f", p.ConfigPath(s), "port", serviceName, port).CombinedOutput()
	if err != nil {
		return "", err
	}
	return strings.Trim(string(output), "\n"), nil
}

func (p DockerCompose) ConfigPath(s Service) string {
	return s.Path + "/docker-compose.yml"
}
<|endoftext|>