<commit_before>package service

import (
	"encoding/json"
	"fmt"
	"github.com/jinzhu/gorm"
	memcache "github.com/patrickmn/go-cache"
	"github.com/zwirec/TGChatScanner/TGBotApi"
	"github.com/zwirec/TGChatScanner/clarifaiApi"
	"github.com/zwirec/TGChatScanner/modelManager"
	"github.com/zwirec/TGChatScanner/requestHandler"
	"github.com/zwirec/TGChatScanner/requestHandler/appContext"
	"github.com/zwirec/TGChatScanner/requestHandler/deforkers"
	file "github.com/zwirec/TGChatScanner/requestHandler/filetypes"
	"github.com/zwirec/TGChatScanner/requestHandler/forkers"
	"github.com/zwirec/TGChatScanner/requestHandler/recognizers"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"os/user"
	"sync"
	"syscall"
	"time"
)

var (
	home      = os.Getenv("HOME")
	configURL = os.Getenv("TGCHATSCANNER_REMOTE_CONFIG")
)

const (
	DefaultWorkersNumber   = 5
	DefaultCacheExpiraiton = 5 * time.Minute
	DefaultCacheClean      = 10 * time.Minute
)

func init() {
	if home == "" {
		u, err := user.Current()
		if err != nil {
			log.Fatal(err)
		}
		home := u.HomeDir
		fmt.Fprint(ioutil.Discard, home)
	}
	if configURL == "" {

		configURL = home + "/.config/tgchatscanner/config.json"
	}
}

type Config map[string]map[string]interface{}

type Service struct {
	sock         net.Listener
	mux          *http.ServeMux
	srv          *http.Server
	reqHandler   *requestHandler.RequestHandler
	config       Config
	errLogger    *log.Logger
	accessLogger *log.Logger
	notifier     chan os.Signal
	poolsWG      sync.WaitGroup
	endpointWg   sync.WaitGroup
	poolsDone    chan struct{}
}

func NewService() *Service {
	accL, errL := createLoggers()
	return &Service{
		reqHandler:   requestHandler.NewRequestHandler(),
		mux:          http.NewServeMux(),
		notifier:     make(chan os.Signal),
		poolsDone:    make(chan struct{}),
		errLogger:    errL,
		accessLogger: accL,
	}
}

func (s *Service) Run() error {

	if err := s.parseConfig(configURL); err != nil {
		s.errLogger.Println(err)
		return err
	}

	s.signalProcessing()

	db, err := s.initModels()

	if err != nil {
		s.errLogger.Println(err)
		return err
	}

	hostname := s.getHostname()

	clAPI := s.initClarifaiAPI()

	botAPI := s.initBotApi()

	workersNumber := s.getWorkersNumber()

	cache := memcache.New(DefaultCacheExpiraiton, DefaultCacheClean)

	imgPath, err := s.createImgPath()
	if err != nil {
		s.errLogger.Println(err)
		return err
	}

	downloadRequests := s.initPools(workersNumber)

	context := appContext.AppContext{
		DB:               db,
		DownloadRequests: downloadRequests,
		BotAPI:           botAPI,
		CfAPI:            clAPI,
		Cache:            cache,
		ErrLogger:        s.errLogger,
		AccessLogger:     s.accessLogger,
		ImagesPath:       imgPath,
		Hostname:         hostname,
	}

	appContext.SetAppContext(&context)
	s.reqHandler.RegisterHandlers()

	s.srv = &http.Server{Handler: s.reqHandler}

	s.endpointWg.Add(1)

	go func() {
		defer s.endpointWg.Done()
		s.endpoint()
	}()
	s.endpointWg.Wait()
	return nil
}

func (s *Service) endpoint() (err error) {
	s.sock, err = net.Listen("unix", s.config["server"]["socket"].(string))
	if err != nil {
		s.errLogger.Println(err)
		os.Remove(s.config["server"]["socket"].(string))
		s.sock, _ = net.Listen("unix", s.config["server"]["socket"].(string))
	}
	if err := os.Chmod(s.config["server"]["socket"].(string), 0777); err != nil {
		s.errLogger.Println(err)
		s.notifier <- syscall.SIGINT
	}

	s.errLogger.Println("Socket opened")
	s.errLogger.Println("Server started")
	log.Println("Server started")
	if err := s.srv.Serve(s.sock); err != nil {
		s.errLogger.Println(err)
	}
	return nil
}

func (s *Service) parseConfig(URL string) error {
	var configRaw []byte

	_, err := url.Parse(URL)

	if err == nil {
		res, err := http.Get(URL)
		if err != nil {
			s.errLogger.Println(err)
			return err
		}

		configRaw, err = ioutil.ReadAll(res.Body)
		res.Body.Close()
		if err != nil {
			s.errLogger.Println(err)
			return err
		}

	} else {
		var err error
		configRaw, err = ioutil.ReadFile(URL)
		if err != nil {
			s.errLogger.Println(err)
			return err
		}
	}

	if err := json.Unmarshal(configRaw, &s.config); err != nil {
		s.errLogger.Println(err)
		return err
	}

	return nil
}

func (s *Service) signalProcessing() {
	signal.Notify(s.notifier, syscall.SIGINT)
	go s.handler(s.notifier)
}

func (s *Service) handler(c chan os.Signal) {
	for {
		<-c
		s.errLogger.Println("Gracefully stopping...")
		log.Println("Gracefully stopping...")
		close(appContext.DownloadRequests)
		close(s.poolsDone)
		s.poolsWG.Wait()
		if err := s.srv.Shutdown(nil); err != nil {
			s.errLogger.Println(err)
			return
		}
	}
}

func (s *Service) initPools(workersNumber int) chan *file.FileBasic {
	dr := make(chan *file.FileBasic, workersNumber*2)

	fp := &requestHandler.FilePreparationsPool{In: dr, Done: s.poolsDone, WorkersNumber: workersNumber}
	fpOut := fp.Run(workersNumber*2, &s.poolsWG)

	forker := &forkers.ForkersPool{
		In:             fpOut,
		Done:           s.poolsDone,
		WorkersNumber:  workersNumber,
		ForkToFileInfo: requestHandler.CastToFileInfo,
		ForkToFileLink: requestHandler.CastToFileLink,
	}

	fdIn, prIn := forker.Run(workersNumber, workersNumber, &s.poolsWG)

	fd := &requestHandler.FileDownloadersPool{In: fdIn, Done: s.poolsDone, WorkersNumber: workersNumber}
	fdOut := fd.Run(workersNumber, &s.poolsWG)

	pr := &recognizers.PhotoRecognizersPool{In: prIn, Done: s.poolsDone, WorkersNumber: workersNumber}
	prOut := pr.Run(workersNumber, &s.poolsWG)

	deforker := &deforkers.DeforkersPool{
		In1:              fdOut,
		In2:              prOut,
		WorkersNumber:    workersNumber,
		DeforkDownloaded: requestHandler.CastFromDownloadedFile,
		DeforkRecognized: requestHandler.CastFromRecognizedPhoto,
	}

	dbsIn := deforker.Run(workersNumber*2, &s.poolsWG)

	dbs := &requestHandler.DbStoragesPool{In: dbsIn, WorkersNumber: workersNumber}
	dbs.Run(&s.poolsWG)
	return dr
}

func (s *Service) initClarifaiAPI() *clarifaiAPI.ClarifaiAPI {
	key := s.config["clarifai"]["api_key"].(string)
	url := s.config["clarifai"]["url"].(string)
	clAPI := clarifaiAPI.NewClarifaiAPI(key, url)
	return clAPI
}

func (s *Service) initBotApi() *TGBotAPI.BotAPI {
	key := s.config["tg_bot_api"]["token"].(string)
	return TGBotAPI.NewBotAPI(key)
}

func (s *Service) getWorkersNumber() int {

	wn, ok := s.config["server"]["workers"].(int)

	if !ok {
		wn = DefaultWorkersNumber
	}
	return wn
}

func createLoggers() (accLog *log.Logger, errLog *log.Logger) {
	errorlog, err := os.OpenFile("error.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModePerm)
	if err != nil {
		errorlog = os.Stderr
	}
	accesslog, err := os.OpenFile("access.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModePerm)
	if err != nil {
		accesslog = os.Stdout
	}
	errLog = log.New(errorlog, "", log.LstdFlags|log.Llongfile)
	accLog = log.New(accesslog, "", log.LstdFlags)
	return accLog, errLog
}

func (s *Service) initModels() (*gorm.DB, error) {
	db, err := modelManager.ConnectToDB(s.config["db"])
	if err != nil {
		return nil, err
	}

	if err := modelManager.InitDB(db); err != nil {
		return nil, err
	}
	return db, err
}

func (s *Service) createImgPath() (string, error) {
	imgPath, ok := s.config["chatscanner"]["images_path"].(string)

	if !ok {
		wd, _ := os.Getwd()
		imgPath = wd + "/uploads/"
	}

	if err := os.MkdirAll(imgPath, os.ModePerm); err != nil {
		return "", err
	}
	return imgPath, nil
}

func (s *Service) getHostname() string {
	hostname, _ := s.config["chatscanner"]["host"].(string)

	_, err := url.Parse(hostname)
	if err != nil {
		hostname, err = os.Hostname()
		if err != nil {
			s.errLogger.Println(err)
			hostname = "localhost"
		}
	}
	return hostname
}
<commit_msg>Fix homedir<commit_after>package service

import (
	"encoding/json"
	"github.com/jinzhu/gorm"
	memcache "github.com/patrickmn/go-cache"
	"github.com/zwirec/TGChatScanner/TGBotApi"
	"github.com/zwirec/TGChatScanner/clarifaiApi"
	"github.com/zwirec/TGChatScanner/modelManager"
	"github.com/zwirec/TGChatScanner/requestHandler"
	"github.com/zwirec/TGChatScanner/requestHandler/appContext"
	"github.com/zwirec/TGChatScanner/requestHandler/deforkers"
	file "github.com/zwirec/TGChatScanner/requestHandler/filetypes"
	"github.com/zwirec/TGChatScanner/requestHandler/forkers"
	"github.com/zwirec/TGChatScanner/requestHandler/recognizers"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"os/user"
	"sync"
	"syscall"
	"time"
)

var configURL = os.Getenv("TGCHATSCANNER_REMOTE_CONFIG")

const (
	DefaultWorkersNumber   = 5
	DefaultCacheExpiraiton = 5 * time.Minute
	DefaultCacheClean      = 10 * time.Minute
)

func init() {
	u, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}

	if configURL == "" {
		configURL = u.HomeDir + "/.config/tgchatscanner/config.json"
	}
}

type Config map[string]map[string]interface{}

type Service struct {
	sock         net.Listener
	mux          *http.ServeMux
	srv          *http.Server
	reqHandler   *requestHandler.RequestHandler
	config       Config
	errLogger    *log.Logger
	accessLogger *log.Logger
	notifier     chan os.Signal
	poolsWG      sync.WaitGroup
	endpointWg   sync.WaitGroup
	poolsDone    chan struct{}
}

func NewService() *Service {
	accL, errL := createLoggers()
	return &Service{
		reqHandler:   requestHandler.NewRequestHandler(),
		mux:          http.NewServeMux(),
		notifier:     make(chan os.Signal),
		poolsDone:    make(chan struct{}),
		errLogger:    errL,
		accessLogger: accL,
	}
}

func (s *Service) Run() error {

	if err := s.parseConfig(configURL); err != nil {
		s.errLogger.Println(err)
		return err
	}

	s.signalProcessing()

	db, err := s.initModels()

	if err != nil {
		s.errLogger.Println(err)
		return err
	}

	hostname := s.getHostname()

	clAPI := s.initClarifaiAPI()

	botAPI := s.initBotApi()

	workersNumber := s.getWorkersNumber()

	cache := memcache.New(DefaultCacheExpiraiton, DefaultCacheClean)

	imgPath, err := s.createImgPath()
	if err != nil {
		s.errLogger.Println(err)
		return err
	}

	downloadRequests := s.initPools(workersNumber)

	context := appContext.AppContext{
		DB:               db,
		DownloadRequests: downloadRequests,
		BotAPI:           botAPI,
		CfAPI:            clAPI,
		Cache:            cache,
		ErrLogger:        s.errLogger,
		AccessLogger:     s.accessLogger,
		ImagesPath:       imgPath,
		Hostname:         hostname,
	}

	appContext.SetAppContext(&context)
	s.reqHandler.RegisterHandlers()

	s.srv = &http.Server{Handler: s.reqHandler}

	s.endpointWg.Add(1)

	go func() {
		defer s.endpointWg.Done()
		s.endpoint()
	}()
	s.endpointWg.Wait()
	return nil
}

func (s *Service) endpoint() (err error) {
	s.sock, err = net.Listen("unix", s.config["server"]["socket"].(string))
	if err != nil {
		s.errLogger.Println(err)
		os.Remove(s.config["server"]["socket"].(string))
		s.sock, _ = net.Listen("unix", s.config["server"]["socket"].(string))
	}
	if err := os.Chmod(s.config["server"]["socket"].(string), 0777); err != nil {
		s.errLogger.Println(err)
		s.notifier <- syscall.SIGINT
	}

	s.errLogger.Println("Socket opened")
	s.errLogger.Println("Server started")
	log.Println("Server started")
	if err := s.srv.Serve(s.sock); err != nil {
		s.errLogger.Println(err)
	}
	return nil
}

func (s *Service) parseConfig(URL string) error {
	var configRaw []byte

	_, err := url.Parse(URL)

	if err == nil {
		res, err := http.Get(URL)
		if err != nil {
			s.errLogger.Println(err)
			return err
		}

		configRaw, err = ioutil.ReadAll(res.Body)
		res.Body.Close()
		if err != nil {
			s.errLogger.Println(err)
			return err
		}

	} else {
		var err error
		configRaw, err = ioutil.ReadFile(URL)
		if err != nil {
			s.errLogger.Println(err)
			return err
		}
	}

	if err := json.Unmarshal(configRaw, &s.config); err != nil {
		s.errLogger.Println(err)
		return err
	}

	return nil
}

func (s *Service) signalProcessing() {
	signal.Notify(s.notifier, syscall.SIGINT)
	go s.handler(s.notifier)
}

func (s *Service) handler(c chan os.Signal) {
	for {
		<-c
		s.errLogger.Println("Gracefully stopping...")
		log.Println("Gracefully stopping...")
		close(appContext.DownloadRequests)
		close(s.poolsDone)
		s.poolsWG.Wait()
		if err := s.srv.Shutdown(nil); err != nil {
			s.errLogger.Println(err)
			return
		}
	}
}

func (s *Service) initPools(workersNumber int) chan *file.FileBasic {
	dr := make(chan *file.FileBasic, workersNumber*2)

	fp := &requestHandler.FilePreparationsPool{In: dr, Done: s.poolsDone, WorkersNumber: workersNumber}
	fpOut := fp.Run(workersNumber*2, &s.poolsWG)

	forker := &forkers.ForkersPool{
		In:             fpOut,
		Done:           s.poolsDone,
		WorkersNumber:  workersNumber,
		ForkToFileInfo: requestHandler.CastToFileInfo,
		ForkToFileLink: requestHandler.CastToFileLink,
	}

	fdIn, prIn := forker.Run(workersNumber, workersNumber, &s.poolsWG)

	fd := &requestHandler.FileDownloadersPool{In: fdIn, Done: s.poolsDone, WorkersNumber: workersNumber}
	fdOut := fd.Run(workersNumber, &s.poolsWG)

	pr := &recognizers.PhotoRecognizersPool{In: prIn, Done: s.poolsDone, WorkersNumber: workersNumber}
	prOut := pr.Run(workersNumber, &s.poolsWG)

	deforker := &deforkers.DeforkersPool{
		In1:              fdOut,
		In2:              prOut,
		WorkersNumber:    workersNumber,
		DeforkDownloaded: requestHandler.CastFromDownloadedFile,
		DeforkRecognized: requestHandler.CastFromRecognizedPhoto,
	}

	dbsIn := deforker.Run(workersNumber*2, &s.poolsWG)

	dbs := &requestHandler.DbStoragesPool{In: dbsIn, WorkersNumber: workersNumber}
	dbs.Run(&s.poolsWG)
	return dr
}

func (s *Service) initClarifaiAPI() *clarifaiAPI.ClarifaiAPI {
	key := s.config["clarifai"]["api_key"].(string)
	url := s.config["clarifai"]["url"].(string)
	clAPI := clarifaiAPI.NewClarifaiAPI(key, url)
	return clAPI
}

func (s *Service) initBotApi() *TGBotAPI.BotAPI {
	key := s.config["tg_bot_api"]["token"].(string)
	return TGBotAPI.NewBotAPI(key)
}

func (s *Service) getWorkersNumber() int {

	wn, ok := s.config["server"]["workers"].(int)

	if !ok {
		wn = DefaultWorkersNumber
	}
	return wn
}

func createLoggers() (accLog *log.Logger, errLog *log.Logger) {
	errorlog, err := os.OpenFile("error.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModePerm)
	if err != nil {
		errorlog = os.Stderr
	}
	accesslog, err := os.OpenFile("access.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModePerm)
	if err != nil {
		accesslog = os.Stdout
	}
	errLog = log.New(errorlog, "", log.LstdFlags|log.Llongfile)
	accLog = log.New(accesslog, "", log.LstdFlags)
	return accLog, errLog
}

func (s *Service) initModels() (*gorm.DB, error) {
	db, err := modelManager.ConnectToDB(s.config["db"])
	if err != nil {
		return nil, err
	}

	if err := modelManager.InitDB(db); err != nil {
		return nil, err
	}
	return db, err
}

func (s *Service) createImgPath() (string, error) {
	imgPath, ok := s.config["chatscanner"]["images_path"].(string)

	if !ok {
		wd, _ := os.Getwd()
		imgPath = wd + "/uploads/"
	}

	if err := os.MkdirAll(imgPath, os.ModePerm); err != nil {
		return "", err
	}
	return imgPath, nil
}

func (s *Service) getHostname() string {
	hostname, _ := s.config["chatscanner"]["host"].(string)

	_, err := url.Parse(hostname)
	if err != nil {
		hostname, err = os.Hostname()
		if err != nil {
			s.errLogger.Println(err)
			hostname = "localhost"
		}
	}
	return hostname
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"strconv"

	"github.com/hyperledger/fabric/core/chaincode/shim"
	pb "github.com/hyperledger/fabric/protos/peer"
)

// SimpleChaincode example simple Chaincode implementation
type SimpleChaincode struct {
}

func (t *SimpleChaincode) Init(stub shim.ChaincodeStubInterface) pb.Response {
	fmt.Println("ex02 Init")
	_, args := stub.GetFunctionAndParameters()
	var A, B string    // Entities
	var Aval, Bval int // Asset holdings
	var err error

	if len(args) != 4 {
		return shim.Error("Incorrect number of arguments. Expecting 4")
	}

	// Initialize the chaincode
	A = args[0]
	Aval, err = strconv.Atoi(args[1])
	if err != nil {
		return shim.Error("Expecting integer value for asset holding")
	}
	B = args[2]
	Bval, err = strconv.Atoi(args[3])
	if err != nil {
		return shim.Error("Expecting integer value for asset holding")
	}
	fmt.Printf("Aval = %d, Bval = %d\n", Aval, Bval)

	// Write the state to the ledger
	err = stub.PutState(A, []byte(strconv.Itoa(Aval)))
	if err != nil {
		return shim.Error(err.Error())
	}

	err = stub.PutState(B, []byte(strconv.Itoa(Bval)))
	if err != nil {
		return shim.Error(err.Error())
	}

	return shim.Success(nil)
}

func (t *SimpleChaincode) Invoke(stub shim.ChaincodeStubInterface) pb.Response {
	fmt.Println("ex02 Invoke")
	function, args := stub.GetFunctionAndParameters()
	if function == "invoke" {
		// Make payment of X units from A to B
		return t.invoke(stub, args)
	} else if function == "delete" {
		// Deletes an entity from its state
		return t.delete(stub, args)
	} else if function == "query" {
		// the old "Query" is now implemtned in invoke
		return t.query(stub, args)
	}

	return shim.Error("Invalid invoke function name. Expecting \"invoke\" \"delete\" \"query\"")
}

// Transaction makes payment of X units from A to B
func (t *SimpleChaincode) invoke(stub shim.ChaincodeStubInterface, args []string) pb.Response {
	var A, B string    // Entities
	var Aval, Bval int // Asset holdings
	var X int          // Transaction value
	var err error

	if len(args) != 3 {
		return shim.Error("Incorrect number of arguments. Expecting 3")
	}

	A = args[0]
	B = args[1]

	// Get the state from the ledger
	// TODO: will be nice to have a GetAllState call to ledger
	Avalbytes, err := stub.GetState(A)
	if err != nil {
		return shim.Error("Failed to get state")
	}
	if Avalbytes == nil {
		return shim.Error("Entity not found")
	}
	Aval, _ = strconv.Atoi(string(Avalbytes))

	Bvalbytes, err := stub.GetState(B)
	if err != nil {
		return shim.Error("Failed to get state")
	}
	if Bvalbytes == nil {
		return shim.Error("Entity not found")
	}
	Bval, _ = strconv.Atoi(string(Bvalbytes))

	// Perform the execution
	X, err = strconv.Atoi(args[2])
	if err != nil {
		return shim.Error("Invalid transaction amount, expecting a integer value")
	}
	Aval = Aval - X
	Bval = Bval + X
	fmt.Printf("Aval = %d, Bval = %d\n", Aval, Bval)

	// Write the state back to the ledger
	err = stub.PutState(A, []byte(strconv.Itoa(Aval)))
	if err != nil {
		return shim.Error(err.Error())
	}

	err = stub.PutState(B, []byte(strconv.Itoa(Bval)))
	if err != nil {
		return shim.Error(err.Error())
	}

	return shim.Success(nil)
}

// Deletes an entity from state
func (t *SimpleChaincode) delete(stub shim.ChaincodeStubInterface, args []string) pb.Response {
	if len(args) != 1 {
		return shim.Error("Incorrect number of arguments. Expecting 1")
	}

	A := args[0]

	// Delete the key from the state in ledger
	err := stub.DelState(A)
	if err != nil {
		return shim.Error("Failed to delete state")
	}

	return shim.Success(nil)
}

// query callback representing the query of a chaincode
func (t *SimpleChaincode) query(stub shim.ChaincodeStubInterface, args []string) pb.Response {
	var A string // Entities
	var err error

	if len(args) != 1 {
		return shim.Error("Incorrect number of arguments. Expecting name of the person to query")
	}

	A = args[0]

	// Get the state from the ledger
	Avalbytes, err := stub.GetState(A)
	if err != nil {
		jsonResp := "{\"Error\":\"Failed to get state for " + A + "\"}"
		return shim.Error(jsonResp)
	}

	if Avalbytes == nil {
		jsonResp := "{\"Error\":\"Nil amount for " + A + "\"}"
		return shim.Error(jsonResp)
	}

	jsonResp := "{\"Name\":\"" + A + "\",\"Amount\":\"" + string(Avalbytes) + "\"}"
	fmt.Printf("Query Response:%s\n", jsonResp)
	return shim.Success(Avalbytes)
}

func main() {
	err := shim.Start(new(SimpleChaincode))
	if err != nil {
		fmt.Printf("Error starting Simple chaincode: %s", err)
	}
}
<commit_msg>[FAB-9732] Remove dead cc example from e2e_cli dir<commit_after><|endoftext|>


<commit_before>// Copyright (c) 2016 Blue Medora, Inc. All rights reserved.
// This file is subject to the terms and conditions defined in the included file 'LICENSE.txt'.

package webserver

import (
    "fmt"

    "github.com/cloudfoundry/sonde-go/events"
    "github.com/cloudfoundry/gosteno"
)

const (
    metronAgentOrigin			= "MetronAgent"
    syslogDrainBinderOrigin		= "syslog_drain_binder"
    tpsWatcherOrigin			= "tps_watcher"
    tpsListenerOrigin			= "tps_listener"
    stagerOrigin				= "stager"
    sshProxyOrigin				= "ssh-proxy"
    senderOrigin				= "sender"
    routeEmitterOrigin			= "route_emitters"
    repOrigin					= "rep"
    receptorOrigin				= "receptor"
    nsyncListenerOrigin			= "nsync_listener"
    nsyncBulkerOrigin			= "nsync_bulker"
    gardenLinuxOrigin			= "garden-linux"
    fileServerOrigin			= "file_server"
    fetcherOrigin				= "fetcher"
    convergerOrigin				= "converger"
    ccUploaderOrigin			= "cc_uploader"
    bbsOrigin					= "bbs"
	auctioneerOrigin			= "auctioneer"
	etcdOrigin					= "etcd"
	dopplerServerOrigin			= "DopplerServer"
	cloudControllerOrigin		= "cc"
	trafficControllerOrigin		= "LoggregatorTrafficController"
	goRouterOrigin				= "gorouter"
)

//Resource represents cloud controller data
type Resource struct {
    Deployment      string
    Job             string
    Index           string
    IP              string
    ValueMetrics    map[string]float64
    CounterMetrics  map[string]float64
}

func createEnvelopeKey(envelope *events.Envelope) string {
	return fmt.Sprintf("%s | %s | %s | %s", envelope.GetDeployment(), envelope.GetJob(), envelope.GetIndex(), envelope.GetIp())
}

func addMetric(envelope *events.Envelope, valueMetricMap map[string]float64, counterMetricMap map[string]float64, logger *gosteno.Logger) {
    if envelope.GetEventType() == events.Envelope_ValueMetric {
        valueMetric := envelope.GetValueMetric()

		valueMetricMap[valueMetric.GetName()] = valueMetric.GetValue()
		logger.Debugf("Addeding Counter Event Name %s, Value %d", valueMetric.GetName(), valueMetric.GetValue())
    } else if envelope.GetEventType() == events.Envelope_CounterEvent {
        counterEvent := envelope.GetCounterEvent()

		counterMetricMap[counterEvent.GetName()] = float64(counterEvent.GetTotal())
		logger.Debugf("Addeding Counter Event Name %s, Value %d", counterEvent.GetName(), counterEvent.GetTotal())
    } else {
		logger.Errorf("Unkown event type %s", envelope.GetEventType())
	}
}

func getValues(resourceMap map[string]Resource) []Resource {
    resources := make([]Resource, 0, len(resourceMap))

    for _, resource := range resourceMap {
        resources = append(resources, resource)
    }

    return resources
}<commit_msg>corrected misspelling on router-emitter origin<commit_after>// Copyright (c) 2016 Blue Medora, Inc. All rights reserved.
// This file is subject to the terms and conditions defined in the included file 'LICENSE.txt'.

package webserver

import (
    "fmt"

    "github.com/cloudfoundry/sonde-go/events"
    "github.com/cloudfoundry/gosteno"
)

const (
    metronAgentOrigin			= "MetronAgent"
    syslogDrainBinderOrigin		= "syslog_drain_binder"
    tpsWatcherOrigin			= "tps_watcher"
    tpsListenerOrigin			= "tps_listener"
    stagerOrigin				= "stager"
    sshProxyOrigin				= "ssh-proxy"
    senderOrigin				= "sender"
    routeEmitterOrigin			= "route_emitter"
    repOrigin					= "rep"
    receptorOrigin				= "receptor"
    nsyncListenerOrigin			= "nsync_listener"
    nsyncBulkerOrigin			= "nsync_bulker"
    gardenLinuxOrigin			= "garden-linux"
    fileServerOrigin			= "file_server"
    fetcherOrigin				= "fetcher"
    convergerOrigin				= "converger"
    ccUploaderOrigin			= "cc_uploader"
    bbsOrigin					= "bbs"
	auctioneerOrigin			= "auctioneer"
	etcdOrigin					= "etcd"
	dopplerServerOrigin			= "DopplerServer"
	cloudControllerOrigin		= "cc"
	trafficControllerOrigin		= "LoggregatorTrafficController"
	goRouterOrigin				= "gorouter"
)

//Resource represents cloud controller data
type Resource struct {
    Deployment      string
    Job             string
    Index           string
    IP              string
    ValueMetrics    map[string]float64
    CounterMetrics  map[string]float64
}

func createEnvelopeKey(envelope *events.Envelope) string {
	return fmt.Sprintf("%s | %s | %s | %s", envelope.GetDeployment(), envelope.GetJob(), envelope.GetIndex(), envelope.GetIp())
}

func addMetric(envelope *events.Envelope, valueMetricMap map[string]float64, counterMetricMap map[string]float64, logger *gosteno.Logger) {
    if envelope.GetEventType() == events.Envelope_ValueMetric {
        valueMetric := envelope.GetValueMetric()

		valueMetricMap[valueMetric.GetName()] = valueMetric.GetValue()
		logger.Debugf("Addeding Counter Event Name %s, Value %d", valueMetric.GetName(), valueMetric.GetValue())
    } else if envelope.GetEventType() == events.Envelope_CounterEvent {
        counterEvent := envelope.GetCounterEvent()

		counterMetricMap[counterEvent.GetName()] = float64(counterEvent.GetTotal())
		logger.Debugf("Addeding Counter Event Name %s, Value %d", counterEvent.GetName(), counterEvent.GetTotal())
    } else {
		logger.Errorf("Unkown event type %s", envelope.GetEventType())
	}
}

func getValues(resourceMap map[string]Resource) []Resource {
    resources := make([]Resource, 0, len(resourceMap))

    for _, resource := range resourceMap {
        resources = append(resources, resource)
    }

    return resources
}<|endoftext|>


<commit_before>package state

import (
	"log"
	"math/rand"
	"time"

	"config"
	"logger"
	"node"
)

const (
	FOLLOWER  = iota
	CANDIDATE = iota
	LEADER    = iota
)

var HeartbeatChan chan string
var statusInput, statusOutput chan int
var leaderInput, leaderOutput chan *node.Node

const (
	CONNECTION_TIMEOUT        = 100   // 100ms
	HEARTBEAT_INTERVAL        = 1000  // 1s
	MIN_WAIT_BEFORE_CANDIDACY = 5000  // 1000ms
	MAX_WAIT_BEFORE_CANDIDACY = 10000 // 5000ms
)

var random *rand.Rand
var VoteChan chan string

func initStatus(i chan int, o chan int) {
	status := FOLLOWER
	for {
		select {
		case status = <-i:
		case o <- status:
		}
	}
}

func initLeader(i chan *node.Node, o chan *node.Node) {
	var leader *node.Node
	leader = nil
	for {
		select {
		case leader = <-i:
		case o <- leader:
		}
	}
}

func GetMyState() int {
	return <-statusOutput
}

func AmILeader() bool {
	return GetMyState() == LEADER
}

func GetLeader() *node.Node {
	return <-leaderOutput
}

func getCandidacyTimeout() int {
	return random.Intn(MAX_WAIT_BEFORE_CANDIDACY-MIN_WAIT_BEFORE_CANDIDACY) +
		MIN_WAIT_BEFORE_CANDIDACY
}

func sendHeartBeats() {
	//TODO make the following nonblocking with a timeout
	requestSender := func(node *node.Node) {
		node.SendRequest(config.HeartbeatPath + config.UniqueId)
	}
	for {
		if GetMyState() != LEADER {
			return
		}
		node.ForAll(requestSender)
		time.Sleep(HEARTBEAT_INTERVAL * time.Millisecond)
	}
}

func transitionToLeader() {
	if GetMyState() != CANDIDATE {
		panic("should be follower")
	}
	log.Print("I am the leader now.")
	statusInput <- LEADER
	go sendHeartBeats()
}

func captureVotes() {
	nVotes := 0
	voters := make(map[string]bool)
	for {
		sender := <-VoteChan
		if sender == "" {
			log.Print("Someone asked us not to be the leader. Stepping down.")
			transitionToFollower()
			return
		} else {
			if _, ok := voters[sender]; ok {
				log.Print("vote from ", sender, " is already processed")
				continue
			}
			if node.FindNode(sender) == nil {
				log.Print("Received vote from unknown sender: ", sender)
				continue
			}
			nVotes++
			if nVotes > config.NProcesses/2 {
				transitionToLeader()
				return
			}
		}
	}
}

func VoteIfEligible(sender string, term int) {
	highestTerm := logger.GetHighestTerm()
	if term <= highestTerm {
		log.Print("Ignoring vote request for term: ", term,
			" since we are at ", highestTerm)
	} else {
		node.VoteFor(sender)
		logger.SetHighestTerm(term)
	}
}

func transitionToCandidate() {
	if GetMyState() == CANDIDATE {
		// restart the vote requests with a new term.
		VoteChan <- ""
	} else if GetMyState() == LEADER {
		log.Fatal("A leader should not be getting votes.")
	}
	log.Print("I am a candidate now.")
	statusInput <- CANDIDATE
	logger.IncrementNextTerm()
	go captureVotes()
	node.SendVoteRequests()
	VoteChan <- config.UniqueId
}

func transitionToFollower() {
	log.Print("I am a follower now.")
	statusInput <- FOLLOWER
}

func selectLeader() {
	heartbeat := true
	for {
		select {
		case sender := <-HeartbeatChan:
			if config.UniqueId != sender {
				log.Print("Got heartbeat from myself")
			} else {
				log.Print("Got heartbeat from " + sender)
				VoteChan <- ""
				node := node.FindNode(sender)
				if node == nil {
					log.Fatal("sender is not part of config: ", sender)
				}
				leaderInput <- node
			}
			heartbeat = true
		case <-time.After(time.Duration(getCandidacyTimeout()) * time.Millisecond):
			if !heartbeat {
				if GetMyState() == LEADER {
					log.Print("No heartbeat from myself")
				} else {
					log.Print("Timer Expired, claim the throne")
					transitionToCandidate()
				}
			}
			heartbeat = false
		}
	}
}

func Init() {
	statusInput, statusOutput = make(chan int), make(chan int)
	leaderInput, leaderOutput = make(chan *node.Node), make(chan *node.Node)
	go initStatus(statusInput, statusOutput)
	go initLeader(leaderInput, leaderOutput)
	HeartbeatChan = make(chan string)
	transitionToFollower()
	random = rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
	VoteChan = make(chan string)
	go selectLeader()
}
<commit_msg>Fix the condition of checking sender and self.<commit_after>package state

import (
	"log"
	"math/rand"
	"time"

	"config"
	"logger"
	"node"
)

const (
	FOLLOWER  = iota
	CANDIDATE = iota
	LEADER    = iota
)

var HeartbeatChan chan string
var statusInput, statusOutput chan int
var leaderInput, leaderOutput chan *node.Node

const (
	CONNECTION_TIMEOUT        = 100   // 100ms
	HEARTBEAT_INTERVAL        = 1000  // 1s
	MIN_WAIT_BEFORE_CANDIDACY = 5000  // 1000ms
	MAX_WAIT_BEFORE_CANDIDACY = 10000 // 5000ms
)

var random *rand.Rand
var VoteChan chan string

func initStatus(i chan int, o chan int) {
	status := FOLLOWER
	for {
		select {
		case status = <-i:
		case o <- status:
		}
	}
}

func initLeader(i chan *node.Node, o chan *node.Node) {
	var leader *node.Node
	leader = nil
	for {
		select {
		case leader = <-i:
		case o <- leader:
		}
	}
}

func GetMyState() int {
	return <-statusOutput
}

func AmILeader() bool {
	return GetMyState() == LEADER
}

func GetLeader() *node.Node {
	return <-leaderOutput
}

func getCandidacyTimeout() int {
	return random.Intn(MAX_WAIT_BEFORE_CANDIDACY-MIN_WAIT_BEFORE_CANDIDACY) +
		MIN_WAIT_BEFORE_CANDIDACY
}

func sendHeartBeats() {
	//TODO make the following nonblocking with a timeout
	requestSender := func(node *node.Node) {
		node.SendRequest(config.HeartbeatPath + config.UniqueId)
	}
	for {
		if GetMyState() != LEADER {
			return
		}
		node.ForAll(requestSender)
		time.Sleep(HEARTBEAT_INTERVAL * time.Millisecond)
	}
}

func transitionToLeader() {
	if GetMyState() != CANDIDATE {
		panic("should be follower")
	}
	log.Print("I am the leader now.")
	statusInput <- LEADER
	go sendHeartBeats()
}

func captureVotes() {
	nVotes := 0
	voters := make(map[string]bool)
	for {
		sender := <-VoteChan
		if sender == "" {
			log.Print("Someone asked us not to be the leader. Stepping down.")
			transitionToFollower()
			return
		} else {
			if _, ok := voters[sender]; ok {
				log.Print("vote from ", sender, " is already processed")
				continue
			}
			if node.FindNode(sender) == nil {
				log.Print("Received vote from unknown sender: ", sender)
				continue
			}
			nVotes++
			if nVotes > config.NProcesses/2 {
				transitionToLeader()
				return
			}
		}
	}
}

func VoteIfEligible(sender string, term int) {
	highestTerm := logger.GetHighestTerm()
	if term <= highestTerm {
		log.Print("Ignoring vote request for term: ", term,
			" since we are at ", highestTerm)
	} else {
		node.VoteFor(sender)
		logger.SetHighestTerm(term)
	}
}

func transitionToCandidate() {
	if GetMyState() == CANDIDATE {
		// restart the vote requests with a new term.
		VoteChan <- ""
	} else if GetMyState() == LEADER {
		log.Fatal("A leader should not be getting votes.")
	}
	log.Print("I am a candidate now.")
	statusInput <- CANDIDATE
	logger.IncrementNextTerm()
	go captureVotes()
	node.SendVoteRequests()
	VoteChan <- config.UniqueId
}

func transitionToFollower() {
	log.Print("I am a follower now.")
	statusInput <- FOLLOWER
}

func selectLeader() {
	heartbeat := true
	for {
		select {
		case sender := <-HeartbeatChan:
			if config.UniqueId == sender {
				log.Print("Got heartbeat from myself")
			} else {
				log.Print("I am: " + config.UniqueId + ", got heartbeat from " + sender)
				VoteChan <- ""
				node := node.FindNode(sender)
				if node == nil {
					log.Fatal("sender is not part of config: ", sender)
				}
				leaderInput <- node
			}
			heartbeat = true
		case <-time.After(time.Duration(getCandidacyTimeout()) * time.Millisecond):
			if !heartbeat {
				if GetMyState() == LEADER {
					log.Print("No heartbeat from myself")
				} else {
					log.Print("Timer Expired, claim the throne")
					transitionToCandidate()
				}
			}
			heartbeat = false
		}
	}
}

func Init() {
	statusInput, statusOutput = make(chan int), make(chan int)
	leaderInput, leaderOutput = make(chan *node.Node), make(chan *node.Node)
	go initStatus(statusInput, statusOutput)
	go initLeader(leaderInput, leaderOutput)
	HeartbeatChan = make(chan string)
	transitionToFollower()
	random = rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
	VoteChan = make(chan string)
	go selectLeader()
}
<|endoftext|>


<commit_before>package ungocheck

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

type MultiError []error

func (me MultiError) Err() error {
	if len(me) == 0 {
		return nil
	}
	var s []string
	for _, err := range me {
		s = append(s, err.Error())
	}
	return errors.New(strings.Join(s, "\n"))
}

func (me MultiError) Error() (s string) {
	if err := me.Err(); err != nil {
		s = err.Error()
	}
	return
}

func (me *MultiError) Consume(v ...interface{}) *MultiError {
	for _, v := range v {
		if err, ok := v.(error); ok && err != nil {
			*me = append(*me, err)
		} else if err, ok := v.(MultiError); ok && err != nil && len(err) != 0 {
			for _, err := range err {
				*me = append(*me, err)
			}
		}
	}
	return me
}

type Ungocheck struct {
	FS Filesystem
}

func New() *Ungocheck {
	return &Ungocheck{}
}

func (u Ungocheck) Packages(args []string) (pkgs []string) {
	// TODO(rjeczalik): filter out space-separated values for -test.* flags
	for _, arg := range args {
		if !strings.HasPrefix(arg, "-") {
			pkgs = append(pkgs, arg)
		}
	}
	if len(pkgs) == 0 {
		pkgs = append(pkgs, ".")
	}
	return
}

func istestfile(name string) bool {
	return strings.HasSuffix(name, "_test.go") && !strings.HasSuffix(name, "_ungocheck_test.go")
}

var vcs = map[string]struct{}{
	".bzr": {},
	".git": {},
	".hg":  {},
	".svn": {},
}

func isnotvcs(name string) bool {
	_, ok := vcs[name]
	return !ok
}

func (u Ungocheck) Files(pkgs []string) (files []string, err error) {
	var (
		wd    string
		f     File
		fi    []os.FileInfo
		fs    = u.fs()
		dirs  = make(map[string]struct{})
		paths = strings.Split(os.Getenv("GOPATH"), string(os.PathListSeparator))
	)
	if wd, err = os.Getwd(); err != nil {
		return
	}
	for _, pkg := range pkgs {
		switch pkg {
		case ".":
			dirs[wd] = struct{}{}
		case "./...":
			glob, dir := []string{wd}, ""
			for len(glob) > 0 {
				dir, glob = glob[len(glob)-1], glob[:len(glob)-1]
				if f, err = fs.Open(dir); err != nil {
					return
				}
				if fi, err = f.Readdir(0); err != nil {
					f.Close()
					return
				}
				f.Close()
				for _, fi := range fi {
					if fi.IsDir() && isnotvcs(fi.Name()) {
						glob = append(glob, filepath.Join(dir, fi.Name()))
					}
				}
				dirs[dir] = struct{}{}
			}
		default:
			for _, path := range paths {
				dir := filepath.Join(path, "src", pkg)
				if _, err = fs.Stat(dir); err == nil {
					dirs[dir] = struct{}{}
					break
				}
			}
		}
	}
	for dir := range dirs {
		if f, err = fs.Open(dir); err != nil {
			files = nil
			return
		}
		if fi, err = f.Readdir(0); err != nil {
			f.Close()
			files = nil
			return
		}
		f.Close()
		for _, fi := range fi {
			if !fi.IsDir() && istestfile(fi.Name()) {
				files = append(files, filepath.Join(dir, fi.Name()))
			}
		}
	}
	return
}

func (u Ungocheck) Rewrite(files []string) (tests []string, err error) {
	fs := u.fs()
	defer func() {
		if err != nil {
			for _, test := range tests {
				fs.Remove(test)
			}
			tests = nil
		}
	}()
	var n int
	for _, file := range files {
		test := filepath.Base(file)
		test = test[:len(test)-len("_test.go")] + "_ungocheck_test.go"
		test = filepath.Join(filepath.Dir(file), test)
		if n, err = u.rewriteSingle(file, test); err != nil {
			return
		}
		if n == 0 {
			fs.Remove(test)
		} else {
			tests = append(tests, test)
		}
	}
	return
}

var re = regexp.MustCompile(`func \(.*\) Test([\w\d_]+)\(([\w\d_]+) \*C\) {`)

const f = "func Test%s(%s *testing.T) { // ungocheck\n	%s.Parallel() // ungocheck\n"

func (u Ungocheck) rewriteSingle(file, test string) (n int, err error) {
	var (
		fs  = u.fs()
		src File
		dst File
	)
	if src, err = fs.Open(file); err != nil {
		return
	}
	defer src.Close()
	if dst, err = fs.OpenFile(test, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644); err != nil {
		return
	}
	defer dst.Close()
	// TODO(rjeczalik): reimplement with go/ast
	r, w, p := bufio.NewReader(src), bytes.NewBuffer([]byte("// +build ungocheck\n\n")), []byte{}
	for {
		if p, err = r.ReadBytes('\n'); len(p) == 0 {
			if err == io.EOF {
				_, err = io.Copy(dst, bytes.NewReader(w.Bytes()))
				return
			}
			if err != nil {
				return
			}
		}
		if bytes.HasPrefix(p, []byte("package ")) {
			w.Write(p)
			w.WriteString("\nimport \"testing\"\n")
			continue
		}
		if bytes.Contains(p, []byte("// +build")) && bytes.Contains(p, []byte("!ungocheck")) {
			continue
		}
		m := re.FindAllSubmatch(p, 1)
		if m != nil && len(m) == 1 && len(m[0]) == 3 {
			n += 1
			p = []byte(fmt.Sprintf(f, m[0][1], m[0][2], m[0][2]))
		} else if bytes.Contains(p, []byte("c *C")) {
			p = bytes.Replace(p, []byte("c *C"), []byte("c *testing.T"), 1)
		}

		w.Write(p)
	}
	return
}

func (u Ungocheck) Run(args []string) (err error) {
	var (
		tests []string
		me    MultiError
	)
	defer func() {
		err = me.Consume(err).Consume(u.runTest(args, tests)).Err()
	}()
	s := u.Packages(args[1:])
	if s, err = u.Files(s); err != nil {
		return
	}
	tests, err = u.Rewrite(s)
	return
}
func (u Ungocheck) fs() (fs Filesystem) {
	fs = DefaultFilesystem
	if u.FS != nil {
		fs = u.FS
	}
	return
}

func (u Ungocheck) runTest(args, tests []string) error {
	var (
		me  MultiError
		arg = append([]string{"test", "-tags=ungocheck"}, args[1:]...)
	)
	fs := u.fs()
	out, err := exec.Command("go", arg...).CombinedOutput()
	me.Consume(err).Consume(io.Copy(os.Stdout, bytes.NewReader(out)))
	for _, test := range tests {
		me.Consume(fs.Remove(test))
	}
	return me
}
<commit_msg>Support relative package globbing<commit_after>package ungocheck

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

type MultiError []error

func (me MultiError) Err() error {
	if len(me) == 0 {
		return nil
	}
	var s []string
	for _, err := range me {
		s = append(s, err.Error())
	}
	return errors.New(strings.Join(s, "\n"))
}

func (me MultiError) Error() (s string) {
	if err := me.Err(); err != nil {
		s = err.Error()
	}
	return
}

func (me *MultiError) Consume(v ...interface{}) *MultiError {
	for _, v := range v {
		if err, ok := v.(error); ok && err != nil {
			*me = append(*me, err)
		} else if err, ok := v.(MultiError); ok && err != nil && len(err) != 0 {
			for _, err := range err {
				*me = append(*me, err)
			}
		}
	}
	return me
}

type Ungocheck struct {
	FS Filesystem
}

func New() *Ungocheck {
	return &Ungocheck{}
}

func (u Ungocheck) Packages(args []string) (pkgs []string) {
	// TODO(rjeczalik): filter out space-separated values for -test.* flags
	for _, arg := range args {
		if !strings.HasPrefix(arg, "-") {
			pkgs = append(pkgs, arg)
		}
	}
	if len(pkgs) == 0 {
		pkgs = append(pkgs, ".")
	}
	return
}

func istestfile(name string) bool {
	return strings.HasSuffix(name, "_test.go") && !strings.HasSuffix(name, "_ungocheck_test.go")
}

var vcs = map[string]struct{}{
	".bzr": {},
	".git": {},
	".hg":  {},
	".svn": {},
}

func isnotvcs(name string) bool {
	_, ok := vcs[name]
	return !ok
}

func lookup(fs Filesystem, paths []string, pkg string) (dir string, err error) {
	for _, path := range paths {
		dir = filepath.Join(path, "src", pkg)
		if _, err = fs.Stat(dir); err == nil {
			break
		}
	}
	return
}

func (u Ungocheck) Files(pkgs []string) (files []string, err error) {
	var (
		wd, dir string
		f       File
		fi      []os.FileInfo
		fs      = u.fs()
		dirs    = make(map[string]struct{})
		paths   = strings.Split(os.Getenv("GOPATH"), string(os.PathListSeparator))
	)
	if wd, err = os.Getwd(); err != nil {
		return
	}
	for _, pkg := range pkgs {
		switch {
		case pkg == ".":
			dirs[wd] = struct{}{}
		case strings.HasSuffix(pkg, "/..."):
			var glob []string
			pkg = pkg[:len(pkg)-4]
			if pkg == "." {
				glob = append(glob, wd)
			} else {
				if dir, err = lookup(fs, paths, pkg); err != nil {
					return
				}
				glob = append(glob, dir)
			}
			for len(glob) > 0 {
				dir, glob = glob[len(glob)-1], glob[:len(glob)-1]
				if f, err = fs.Open(dir); err != nil {
					return
				}
				if fi, err = f.Readdir(0); err != nil {
					f.Close()
					return
				}
				f.Close()
				for _, fi := range fi {
					if fi.IsDir() && isnotvcs(fi.Name()) {
						glob = append(glob, filepath.Join(dir, fi.Name()))
					}
				}
				dirs[dir] = struct{}{}
			}
		default:
			if dir, err = lookup(fs, paths, pkg); err != nil {
				return
			}
			dirs[dir] = struct{}{}
		}
	}
	for dir := range dirs {
		if f, err = fs.Open(dir); err != nil {
			files = nil
			return
		}
		if fi, err = f.Readdir(0); err != nil {
			f.Close()
			files = nil
			return
		}
		f.Close()
		for _, fi := range fi {
			if !fi.IsDir() && istestfile(fi.Name()) {
				files = append(files, filepath.Join(dir, fi.Name()))
			}
		}
	}
	return
}

func (u Ungocheck) Rewrite(files []string) (tests []string, err error) {
	fs := u.fs()
	defer func() {
		if err != nil {
			for _, test := range tests {
				fs.Remove(test)
			}
			tests = nil
		}
	}()
	var n int
	for _, file := range files {
		test := filepath.Base(file)
		test = test[:len(test)-len("_test.go")] + "_ungocheck_test.go"
		test = filepath.Join(filepath.Dir(file), test)
		if n, err = u.rewriteSingle(file, test); err != nil {
			return
		}
		if n == 0 {
			fs.Remove(test)
		} else {
			tests = append(tests, test)
		}
	}
	return
}

var re = regexp.MustCompile(`func \(.*\) Test([\w\d_]+)\(([\w\d_]+) \*C\) {`)

const f = "func Test%s(%s *testing.T) { // ungocheck\n	%s.Parallel() // ungocheck\n"

func (u Ungocheck) rewriteSingle(file, test string) (n int, err error) {
	var (
		fs  = u.fs()
		src File
		dst File
	)
	if src, err = fs.Open(file); err != nil {
		return
	}
	defer src.Close()
	if dst, err = fs.OpenFile(test, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644); err != nil {
		return
	}
	defer dst.Close()
	// TODO(rjeczalik): reimplement with go/ast
	r, w, p := bufio.NewReader(src), bytes.NewBuffer([]byte("// +build ungocheck\n\n")), []byte{}
	for {
		if p, err = r.ReadBytes('\n'); len(p) == 0 {
			if err == io.EOF {
				_, err = io.Copy(dst, bytes.NewReader(w.Bytes()))
				return
			}
			if err != nil {
				return
			}
		}
		if bytes.HasPrefix(p, []byte("package ")) {
			w.Write(p)
			w.WriteString("\nimport \"testing\"\n")
			continue
		}
		if bytes.Contains(p, []byte("// +build")) && bytes.Contains(p, []byte("!ungocheck")) {
			continue
		}
		m := re.FindAllSubmatch(p, 1)
		if m != nil && len(m) == 1 && len(m[0]) == 3 {
			n += 1
			p = []byte(fmt.Sprintf(f, m[0][1], m[0][2], m[0][2]))
		} else if bytes.Contains(p, []byte("c *C")) {
			p = bytes.Replace(p, []byte("c *C"), []byte("c *testing.T"), 1)
		}

		w.Write(p)
	}
	return
}

func (u Ungocheck) Run(args []string) (err error) {
	var (
		tests []string
		me    MultiError
	)
	defer func() {
		err = me.Consume(err).Consume(u.runTest(args, tests)).Err()
	}()
	s := u.Packages(args[1:])
	if s, err = u.Files(s); err != nil {
		return
	}
	tests, err = u.Rewrite(s)
	return
}
func (u Ungocheck) fs() (fs Filesystem) {
	fs = DefaultFilesystem
	if u.FS != nil {
		fs = u.FS
	}
	return
}

func (u Ungocheck) runTest(args, tests []string) error {
	var (
		me  MultiError
		arg = append([]string{"test", "-tags=ungocheck"}, args[1:]...)
	)
	fs := u.fs()
	out, err := exec.Command("go", arg...).CombinedOutput()
	me.Consume(err).Consume(io.Copy(os.Stdout, bytes.NewReader(out)))
	for _, test := range tests {
		me.Consume(fs.Remove(test))
	}
	return me
}
<|endoftext|>


<commit_before>package tools

import (
	"bytes"
	"encoding/json"
	"github.com/boltdb/bolt"
	"github.com/golang/protobuf/proto"
	"github.com/scascketta/capmetricsd/daemon/agency"
	"github.com/scascketta/capmetricsd/daemon/agency/capmetro"
	"log"
	"os"
	"strconv"
	"time"
)

type VehicleLocationCollection struct {
	Count     int                      `json:"count"`
	StartDate string                   `json:"start_date"`
	EndDate   string                   `json:"end_date"`
	Data      []agency.VehicleLocation `json:"data"`
}

func GetData(dbPath, dest string, min int64, max int64) error {
	minStr, maxStr := strconv.FormatInt(min, 10), strconv.FormatInt(max, 10)
	minIso, maxIso := time.Unix(min, 0).Format(Iso8601Format), time.Unix(max, 0).Format(Iso8601Format)
	log.Printf("Get data between %s and %s\n", minIso, maxIso)

	log.Println("dbPath: ", dbPath)
	db, err := bolt.Open(dbPath, 0600, nil)
	if err != nil {
		return err
	}

	locations := []agency.VehicleLocation{}

	err = db.View(func(tx *bolt.Tx) error {
		topBucket := tx.Bucket([]byte(capmetro.BucketName))

		err = topBucket.ForEach(func(tripID, _ []byte) error {
			tripBucket := topBucket.Bucket(tripID)
			c := tripBucket.Cursor()

			for k, v := c.Seek([]byte(minStr)); k != nil && bytes.Compare(k, []byte(maxStr)) <= 0; k, v = c.Next() {
				pbData := v

				var loc agency.VehicleLocation
				if err = proto.Unmarshal(pbData, &loc); err != nil {
					return err
				}

				locations = append(locations, loc)
			}

			return nil
		})

		return err
	})

	if err != nil {
		return err
	}

	log.Printf("Writing %d vehicle locations to %s.\n", len(locations), dest)

	coll := VehicleLocationCollection{
		Count:     len(locations),
		StartDate: minIso,
		EndDate:   maxIso,
		Data:      locations,
	}

	f, err := os.Create(dest)
	if err != nil {
		log.Println(err)
		return err
	}
	defer f.Close()

	enc := json.NewEncoder(f)
	if err = enc.Encode(coll); err != nil {
		dlog.Println(err)
		return err
	}

	return nil
}
<commit_msg>Refactor slice script<commit_after>package tools

import (
	"bytes"
	"encoding/json"
	"github.com/boltdb/bolt"
	"github.com/golang/protobuf/proto"
	"github.com/scascketta/capmetricsd/daemon/agency"
	"github.com/scascketta/capmetricsd/daemon/agency/capmetro"
	"log"
	"os"
	"strconv"
	"time"
)

type VehicleLocationCollection struct {
	Count     int                      `json:"count"`
	StartDate string                   `json:"start_date"`
	EndDate   string                   `json:"end_date"`
	Data      []agency.VehicleLocation `json:"data"`
}

func readBoltData(db *bolt.DB, min, max string) (*[]agency.VehicleLocation, error) {
	locations := []agency.VehicleLocation{}

	err := db.View(func(tx *bolt.Tx) error {
		topBucket := tx.Bucket([]byte(capmetro.BucketName))

		err := topBucket.ForEach(func(tripID, _ []byte) error {
			tripBucket := topBucket.Bucket(tripID)
			c := tripBucket.Cursor()

			for k, v := c.Seek([]byte(min)); k != nil && bytes.Compare(k, []byte(max)) <= 0; k, v = c.Next() {
				var loc agency.VehicleLocation
				if err := proto.Unmarshal(v, &loc); err != nil {
					return err
				}

				locations = append(locations, loc)
			}

			return nil
		})

		return err
	})

	return &locations, err
}

func writeData(dest, min, max string, locations *[]agency.VehicleLocation) error {
	log.Printf("Writing %d vehicle locations to %s.\n", len(*locations), dest)

	coll := VehicleLocationCollection{
		Count:     len(*locations),
		StartDate: min,
		EndDate:   max,
		Data:      *locations,
	}

	f, err := os.Create(dest)
	if err != nil {
		return err
	}
	defer f.Close()

	enc := json.NewEncoder(f)
	if err = enc.Encode(coll); err != nil {
		log.Println(err)
		return err
	}

	return nil
}

func GetData(dbPath, dest string, min int64, max int64) error {
	minStr, maxStr := strconv.FormatInt(min, 10), strconv.FormatInt(max, 10)
	minIso, maxIso := time.Unix(min, 0).Format(Iso8601Format), time.Unix(max, 0).Format(Iso8601Format)
	log.Printf("Get data between %s and %s\n", minIso, maxIso)

	log.Println("dbPath: ", dbPath)
	db, err := bolt.Open(dbPath, 0600, nil)
	if err != nil {
		return err
	}

	locations, err := readBoltData(db, minStr, maxStr)
	if err != nil {
		return err
	}

	err = writeData(dest, minIso, maxIso, locations)
	return err
}
<|endoftext|>


<commit_before>// Binary tor2web is a simple tor to web gateway.
//
// It requires a TOR transparent proxy and local
// DNS resolver configured for .onion addresses.
//
// Send traffic to it, e.g. hiddenservice.onion.example.net
// and it will respond with hiddenservice.onion.
//
// Use cgo to rely on system DNS resolver.
// GODEBUG=netdns=cgo go run tor2web.go
package main

import (
	"flag"
	"log"
	"net/http"
	"net/http/httputil"
	"strings"
)

var flagListen = flag.String("listen", ":9050", "Address to listen on.")

const onionTLD = ".onion"

func main() {
	flag.Parse()
	http.Handle("/", &httputil.ReverseProxy{
		Director: func(req *http.Request) {
			req.URL.Scheme = "http"
			host, port := req.Host, ""
			if p := strings.Index(host, ":"); p > 0 {
				host, port = host[:p], host[p:]
			}
			p := strings.Index(host, onionTLD)
			if p == -1 {
				panic("only for " + onionTLD)
			}
			req.URL.Host = host[:p+len(onionTLD)]
			if port != "" {
				req.URL.Host += port
			}
			if _, ok := req.Header["User-Agent"]; !ok {
				// explicitly disable User-Agent so it's not set to default value
				req.Header.Set("User-Agent", "")
			}
		},
	})
	log.Printf("Listening on %v", *flagListen)
	log.Fatal(http.ListenAndServe(*flagListen, nil))
}
<commit_msg>tor: tor2web: use tor via socks directly<commit_after>// Binary tor2web is a simple tor to web gateway.
package main

import (
	"flag"
	"log"
	"net/http"
	"net/http/httputil"
	"os"
	"strings"
)

var (
	flagListen = flag.String("listen", ":9080", "Address to listen on.")
	flagServer = flag.String("server", "", "TOR socks server to proxy connections to.")
)

const onionTLD = ".onion"

func main() {
	flag.Parse()
	if *flagServer == "" {
		flag.Usage()
		os.Exit(1)
	}
	os.Setenv("HTTP_PROXY", "socks5://"+*flagServer)
	http.Handle("/", &httputil.ReverseProxy{
		Director: func(req *http.Request) {
			req.URL.Scheme = "http"
			host, port := req.Host, ""
			if p := strings.Index(host, ":"); p > 0 {
				host, port = host[:p], host[p:]
			}
			p := strings.Index(host, onionTLD)
			if p == -1 {
				panic("only for " + onionTLD)
			}
			req.URL.Host = host[:p+len(onionTLD)]
			if port != "" {
				req.URL.Host += port
			}
			if _, ok := req.Header["User-Agent"]; !ok {
				// explicitly disable User-Agent so it's not set to default value
				req.Header.Set("User-Agent", "")
			}
		},
	})
	log.Printf("Listening on %v", *flagListen)
	log.Fatal(http.ListenAndServe(*flagListen, nil))
}
<|endoftext|>


<commit_before>package redis

import "time"

// UniversalOptions information is required by UniversalClient to establish
// connections.
type UniversalOptions struct {
	// Either a single address or a seed list of host:port addresses
	// of cluster/sentinel nodes.
	Addrs []string

	// The sentinel master name.
	// Only failover clients.
	MasterName string

	// Database to be selected after connecting to the server.
	// Only single-node and failover clients.
	DB int

	// Only cluster clients.

	// Enables read only queries on slave nodes.
	ReadOnly bool

	MaxRedirects   int
	RouteByLatency bool

	// Common options

	MaxRetries         int
	Password           string
	DialTimeout        time.Duration
	ReadTimeout        time.Duration
	WriteTimeout       time.Duration
	PoolSize           int
	PoolTimeout        time.Duration
	IdleTimeout        time.Duration
	IdleCheckFrequency time.Duration
}

func (o *UniversalOptions) cluster() *ClusterOptions {
	if len(o.Addrs) == 0 {
		o.Addrs = []string{"127.0.0.1:6379"}
	}

	return &ClusterOptions{
		Addrs:          o.Addrs,
		MaxRedirects:   o.MaxRedirects,
		RouteByLatency: o.RouteByLatency,
		ReadOnly:       o.ReadOnly,

		MaxRetries:         o.MaxRetries,
		Password:           o.Password,
		DialTimeout:        o.DialTimeout,
		ReadTimeout:        o.ReadTimeout,
		WriteTimeout:       o.WriteTimeout,
		PoolSize:           o.PoolSize,
		PoolTimeout:        o.PoolTimeout,
		IdleTimeout:        o.IdleTimeout,
		IdleCheckFrequency: o.IdleCheckFrequency,
	}
}

func (o *UniversalOptions) failover() *FailoverOptions {
	if len(o.Addrs) == 0 {
		o.Addrs = []string{"127.0.0.1:26379"}
	}

	return &FailoverOptions{
		SentinelAddrs: o.Addrs,
		MasterName:    o.MasterName,
		DB:            o.DB,

		MaxRetries:         o.MaxRetries,
		Password:           o.Password,
		DialTimeout:        o.DialTimeout,
		ReadTimeout:        o.ReadTimeout,
		WriteTimeout:       o.WriteTimeout,
		PoolSize:           o.PoolSize,
		PoolTimeout:        o.PoolTimeout,
		IdleTimeout:        o.IdleTimeout,
		IdleCheckFrequency: o.IdleCheckFrequency,
	}
}

func (o *UniversalOptions) simple() *Options {
	addr := "127.0.0.1:6379"
	if len(o.Addrs) > 0 {
		addr = o.Addrs[0]
	}

	return &Options{
		Addr: addr,
		DB:   o.DB,

		MaxRetries:         o.MaxRetries,
		Password:           o.Password,
		DialTimeout:        o.DialTimeout,
		ReadTimeout:        o.ReadTimeout,
		WriteTimeout:       o.WriteTimeout,
		PoolSize:           o.PoolSize,
		PoolTimeout:        o.PoolTimeout,
		IdleTimeout:        o.IdleTimeout,
		IdleCheckFrequency: o.IdleCheckFrequency,
	}
}

// --------------------------------------------------------------------

// UniversalClient is an abstract client which - based on the provided options -
// can connect to either clusters, or sentinel-backed failover instances or simple
// single-instance servers. This can be useful for testing cluster-specific
// applications locally.
type UniversalClient interface {
	Cmdable
	Process(cmd Cmder) error
	Subscribe(channels ...string) *PubSub
	PSubscribe(channels ...string) *PubSub
	Close() error
}

var _ UniversalClient = (*Client)(nil)
var _ UniversalClient = (*ClusterClient)(nil)

// NewUniversalClient returns a new multi client. The type of client returned depends
// on the following three conditions:
//
// 1. if a MasterName is passed a sentinel-backed FailoverClient will be returned
// 2. if the number of Addrs is two or more, a ClusterClient will be returned
// 3. otherwise, a single-node redis Client will be returned.
func NewUniversalClient(opts *UniversalOptions) UniversalClient {
	if opts.MasterName != "" {
		return NewFailoverClient(opts.failover())
	} else if len(opts.Addrs) > 1 {
		return NewClusterClient(opts.cluster())
	}
	return NewClient(opts.simple())
}
<commit_msg>add WrapProcess to UniversalClient interface (#728)<commit_after>package redis

import "time"

// UniversalOptions information is required by UniversalClient to establish
// connections.
type UniversalOptions struct {
	// Either a single address or a seed list of host:port addresses
	// of cluster/sentinel nodes.
	Addrs []string

	// The sentinel master name.
	// Only failover clients.
	MasterName string

	// Database to be selected after connecting to the server.
	// Only single-node and failover clients.
	DB int

	// Only cluster clients.

	// Enables read only queries on slave nodes.
	ReadOnly bool

	MaxRedirects   int
	RouteByLatency bool

	// Common options

	MaxRetries         int
	Password           string
	DialTimeout        time.Duration
	ReadTimeout        time.Duration
	WriteTimeout       time.Duration
	PoolSize           int
	PoolTimeout        time.Duration
	IdleTimeout        time.Duration
	IdleCheckFrequency time.Duration
}

func (o *UniversalOptions) cluster() *ClusterOptions {
	if len(o.Addrs) == 0 {
		o.Addrs = []string{"127.0.0.1:6379"}
	}

	return &ClusterOptions{
		Addrs:          o.Addrs,
		MaxRedirects:   o.MaxRedirects,
		RouteByLatency: o.RouteByLatency,
		ReadOnly:       o.ReadOnly,

		MaxRetries:         o.MaxRetries,
		Password:           o.Password,
		DialTimeout:        o.DialTimeout,
		ReadTimeout:        o.ReadTimeout,
		WriteTimeout:       o.WriteTimeout,
		PoolSize:           o.PoolSize,
		PoolTimeout:        o.PoolTimeout,
		IdleTimeout:        o.IdleTimeout,
		IdleCheckFrequency: o.IdleCheckFrequency,
	}
}

func (o *UniversalOptions) failover() *FailoverOptions {
	if len(o.Addrs) == 0 {
		o.Addrs = []string{"127.0.0.1:26379"}
	}

	return &FailoverOptions{
		SentinelAddrs: o.Addrs,
		MasterName:    o.MasterName,
		DB:            o.DB,

		MaxRetries:         o.MaxRetries,
		Password:           o.Password,
		DialTimeout:        o.DialTimeout,
		ReadTimeout:        o.ReadTimeout,
		WriteTimeout:       o.WriteTimeout,
		PoolSize:           o.PoolSize,
		PoolTimeout:        o.PoolTimeout,
		IdleTimeout:        o.IdleTimeout,
		IdleCheckFrequency: o.IdleCheckFrequency,
	}
}

func (o *UniversalOptions) simple() *Options {
	addr := "127.0.0.1:6379"
	if len(o.Addrs) > 0 {
		addr = o.Addrs[0]
	}

	return &Options{
		Addr: addr,
		DB:   o.DB,

		MaxRetries:         o.MaxRetries,
		Password:           o.Password,
		DialTimeout:        o.DialTimeout,
		ReadTimeout:        o.ReadTimeout,
		WriteTimeout:       o.WriteTimeout,
		PoolSize:           o.PoolSize,
		PoolTimeout:        o.PoolTimeout,
		IdleTimeout:        o.IdleTimeout,
		IdleCheckFrequency: o.IdleCheckFrequency,
	}
}

// --------------------------------------------------------------------

// UniversalClient is an abstract client which - based on the provided options -
// can connect to either clusters, or sentinel-backed failover instances or simple
// single-instance servers. This can be useful for testing cluster-specific
// applications locally.
type UniversalClient interface {
	Cmdable
	Process(cmd Cmder) error
	WrapProcess(fn func(oldProcess func(cmd Cmder) error) func(cmd Cmder) error)
	Subscribe(channels ...string) *PubSub
	PSubscribe(channels ...string) *PubSub
	Close() error
}

var _ UniversalClient = (*Client)(nil)
var _ UniversalClient = (*ClusterClient)(nil)

// NewUniversalClient returns a new multi client. The type of client returned depends
// on the following three conditions:
//
// 1. if a MasterName is passed a sentinel-backed FailoverClient will be returned
// 2. if the number of Addrs is two or more, a ClusterClient will be returned
// 3. otherwise, a single-node redis Client will be returned.
func NewUniversalClient(opts *UniversalOptions) UniversalClient {
	if opts.MasterName != "" {
		return NewFailoverClient(opts.failover())
	} else if len(opts.Addrs) > 1 {
		return NewClusterClient(opts.cluster())
	}
	return NewClient(opts.simple())
}
<|endoftext|>


<commit_before>package wincred

import (
	"testing"

	"github.com/docker/docker-credential-helpers/credentials"
)

func TestWinCredHelper(t *testing.T) {
	creds := &credentials.Credentials{
		ServerURL: "https://foobar.docker.io:2376/v1",
		Username:  "foobar",
		Secret:    "foobarbaz",
	}

	helper := Wincred{}
	if err := helper.Add(creds); err != nil {
		t.Fatal(err)
	}

	username, secret, err := helper.Get(creds.ServerURL)
	if err != nil {
		t.Fatal(err)
	}

	if username != "foobar" {
		t.Fatalf("expected %s, got %s\n", "foobar", username)
	}

	if secret != "foobarbaz" {
		t.Fatalf("expected %s, got %s\n", "foobarbaz", secret)
	}

	paths, accts, err := helper.List()
	if err != nil || len(paths) == 0 || len(accts) == 0 {
		t.Fatal(err)
	}

	helper.Add(creds1)
	defer helper.Delete(creds1.ServerURL)
	newpaths, newaccts, err := helper.List()
	if len(newpaths)-len(paths) != 1 || len(newaccts)-len(accts) != 1 {
		if(err == nil) {
			t.Fatalf("Error: len(newpaths): %d, len(paths): %d\n len(newaccts): %d, len(accts): %d\n Error= %s", len(newpaths), len(paths), len(newaccts), len(accts), "")
		}
		t.Fatalf("Error: len(newpaths): %d, len(paths): %d\n len(newaccts): %d, len(accts): %d\n Error= %s", len(newpaths), len(paths), len(newaccts), len(accts), err.Error())
	}

	if err := helper.Delete(creds.ServerURL); err != nil {
		t.Fatal(err)
	}
}

func TestMissingCredentials(t *testing.T) {
	helper := Wincred{}
	_, _, err := helper.Get("https://adsfasdf.wrewerwer.com/asdfsdddd")
	if !credentials.IsErrCredentialsNotFound(err) {
		t.Fatalf("expected ErrCredentialsNotFound, got %v", err)
	}
}
<commit_msg>fixed issue with the windows test<commit_after>package wincred

import (
	"testing"

	"github.com/docker/docker-credential-helpers/credentials"
)

func TestWinCredHelper(t *testing.T) {
	creds := &credentials.Credentials{
		ServerURL: "https://foobar.docker.io:2376/v1",
		Username:  "foobar",
		Secret:    "foobarbaz",
	}
	creds1 := &credentials.Credentials{
		ServerURL: "https://foobar.docker.io:2376/v2",
		Username:  "foobarbaz",
		Secret:    "foobar",
	}

	helper := Wincred{}
	if err := helper.Add(creds); err != nil {
		t.Fatal(err)
	}

	username, secret, err := helper.Get(creds.ServerURL)
	if err != nil {
		t.Fatal(err)
	}

	if username != "foobar" {
		t.Fatalf("expected %s, got %s\n", "foobar", username)
	}

	if secret != "foobarbaz" {
		t.Fatalf("expected %s, got %s\n", "foobarbaz", secret)
	}

	paths, accts, err := helper.List()
	if err != nil || len(paths) == 0 || len(accts) == 0 {
		t.Fatal(err)
	}

	helper.Add(creds1)
	defer helper.Delete(creds1.ServerURL)
	newpaths, newaccts, err := helper.List()
	if len(newpaths)-len(paths) != 1 || len(newaccts)-len(accts) != 1 {
		if(err == nil) {
			t.Fatalf("Error: len(newpaths): %d, len(paths): %d\n len(newaccts): %d, len(accts): %d\n Error= %s", len(newpaths), len(paths), len(newaccts), len(accts), "")
		}
		t.Fatalf("Error: len(newpaths): %d, len(paths): %d\n len(newaccts): %d, len(accts): %d\n Error= %s", len(newpaths), len(paths), len(newaccts), len(accts), err.Error())
	}

	if err := helper.Delete(creds.ServerURL); err != nil {
		t.Fatal(err)
	}
}

func TestMissingCredentials(t *testing.T) {
	helper := Wincred{}
	_, _, err := helper.Get("https://adsfasdf.wrewerwer.com/asdfsdddd")
	if !credentials.IsErrCredentialsNotFound(err) {
		t.Fatalf("expected ErrCredentialsNotFound, got %v", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strconv"
	"syscall"
	"unsafe"
)

type Termios struct {
	Iflag  uint32
	Oflag  uint32
	Cflag  uint32
	Lflag  uint32
	Cc     [20]byte
	Ispeed uint32
	Ospeed uint32
}

var origTermios *Termios

func TcSetAttr(fd uintptr, termios *Termios) error {
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(syscall.TCSETS+1), uintptr(unsafe.Pointer(termios))); err != 0 {
		return err
	}
	return nil
}

func TcGetAttr(fd uintptr) (*Termios, error) {
	var termios = &Termios{}
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, syscall.TCGETS, uintptr(unsafe.Pointer(termios))); err != 0 {
		return nil, err
	}
	return termios, nil
}

func enableRawMode() {
	origTermios, _ = TcGetAttr(os.Stdin.Fd())
	var raw Termios
	raw = *origTermios
	raw.Iflag &^= syscall.IXON | syscall.ICRNL
	raw.Oflag &^= syscall.OPOST
	raw.Lflag &^= syscall.ECHO | syscall.ICANON | syscall.ISIG
	if e := TcSetAttr(os.Stdin.Fd(), &raw); e != nil {
		fmt.Fprintf(os.Stderr, "Problem enabling raw mode: %s\n", e)
	}
}

func disableRawMode() {
	if e := TcSetAttr(os.Stdin.Fd(), origTermios); e != nil {
		fmt.Fprintf(os.Stderr, "Problem disabling raw mode: %s\n", e)
	}
}

func main() {
	enableRawMode()
	defer disableRawMode()
	buffer := make([]byte, 1)
	for cc, err := os.Stdin.Read(buffer); buffer[0] != 'q' && err == nil && cc == 1; cc, err = os.Stdin.Read(buffer) {
		var r rune
		r = rune(buffer[0])
		if (strconv.IsPrint(r)) {
			fmt.Printf("%d  %c\r\n", buffer[0], r)
		} else {
			fmt.Printf("%d\r\n", buffer[0])
		}
	}
}
<commit_msg>Step 18 - add error handling, out of order<commit_after>package main

import (
	"fmt"
	"os"
	"strconv"
	"syscall"
	"unsafe"
)

type Termios struct {
	Iflag  uint32
	Oflag  uint32
	Cflag  uint32
	Lflag  uint32
	Cc     [20]byte
	Ispeed uint32
	Ospeed uint32
}

var origTermios *Termios

func TcSetAttr(fd uintptr, termios *Termios) error {
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(syscall.TCSETS+1), uintptr(unsafe.Pointer(termios))); err != 0 {
		return err
	}
	return nil
}

func TcGetAttr(fd uintptr) *Termios {
	var termios = &Termios{}
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, syscall.TCGETS, uintptr(unsafe.Pointer(termios))); err != 0 {
		fmt.Fprintf(os.Stderr, "Problem getting termial attributes\n")
		os.Exit(2)
	}
	return termios
}

func enableRawMode() {
	origTermios = TcGetAttr(os.Stdin.Fd())
	var raw Termios
	raw = *origTermios
	raw.Iflag &^= syscall.IXON | syscall.ICRNL
	raw.Oflag &^= syscall.OPOST
	raw.Lflag &^= syscall.ECHO | syscall.ICANON | syscall.ISIG
	if e := TcSetAttr(os.Stdin.Fd(), &raw); e != nil {
		fmt.Fprintf(os.Stderr, "Problem enabling raw mode: %s\n", e)
		os.Exit(2)
	}
}

func disableRawMode() {
	if e := TcSetAttr(os.Stdin.Fd(), origTermios); e != nil {
		fmt.Fprintf(os.Stderr, "Problem disabling raw mode: %s\n", e)
		os.Exit(1)
	}
}

func main() {
	enableRawMode()
	defer disableRawMode()
	buffer := make([]byte, 1)
	for cc, err := os.Stdin.Read(buffer); buffer[0] != 'q' && err == nil && cc == 1; cc, err = os.Stdin.Read(buffer) {
		var r rune
		r = rune(buffer[0])
		if (strconv.IsPrint(r)) {
			fmt.Printf("%d  %c\r\n", buffer[0], r)
		} else {
			fmt.Printf("%d\r\n", buffer[0])
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Alex Browne.  All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

// File transaction.go contains code related to the
// transactions abstraction.

package zoom

import (
	"fmt"

	"github.com/garyburd/redigo/redis"
)

// Transaction is an abstraction layer around a Redis transaction.
// Transactions consist of a set of actions which are either Redis
// commands or lua scripts. Transactions feature delayed execution,
// so nothing touches the database until you call Exec.
type Transaction struct {
	conn     redis.Conn
	actions  []*Action
	err      error
	watching []string
}

// Action is a single step in a transaction and must be either a command
// or a script with optional arguments and a reply handler.
type Action struct {
	kind    actionKind
	name    string
	script  *redis.Script
	args    redis.Args
	handler ReplyHandler
}

// actionKind is either a command or a script
type actionKind int

const (
	commandAction actionKind = iota
	scriptAction
)

// NewTransaction instantiates and returns a new transaction.
func (p *Pool) NewTransaction() *Transaction {
	t := &Transaction{
		conn: p.NewConn(),
	}
	return t
}

// SetError sets the err property of the transaction iff it was not already
// set. This will cause exec to fail immediately.
func (t *Transaction) setError(err error) {
	if t.err == nil {
		t.err = err
	}
}

// Watch issues a Redis WATCH command using the key for the given model. If the
// model changes before the transaction is executed, Exec will return a
// WatchError and the commands in the transaction will not be executed. Unlike
// most other transaction methods, Watch does not use delayed execution. Because
// of how the WATCH command works, Watch must send a command to Redis
// immediately. You must call Watch or WatchKey before any other transaction
// methods.
func (t *Transaction) Watch(model Model) error {
	if len(t.actions) != 0 {
		return fmt.Errorf("Cannot call Watch after other commands have been added to the transaction")
	}
	col, err := getCollectionForModel(model)
	if err != nil {
		return err
	}
	key := col.ModelKey(model.ModelId())
	return t.WatchKey(key)
}

// WatchKey issues a Redis WATCH command using the given key. If the key changes
// before the transaction is executed, Exec will return a WatchError and the
// commands in the transaction will not be executed. Unlike most other
// transaction methods, WatchKey does not use delayed execution. Because of how
// the WATCH command works, WatchKey must send a command to Redis immediately.
// You must call Watch or WatchKey before any other transaction methods.
func (t *Transaction) WatchKey(key string) error {
	if len(t.actions) != 0 {
		return fmt.Errorf("Cannot call WatchKey after other commands have been added to the transaction")
	}
	if _, err := t.conn.Do("WATCH", key); err != nil {
		return err
	}
	t.watching = append(t.watching, key)
	return nil
}

// Command adds a command action to the transaction with the given args.
// handler will be called with the reply from this specific command when
// the transaction is executed.
func (t *Transaction) Command(name string, args redis.Args, handler ReplyHandler) {
	t.actions = append(t.actions, &Action{
		kind:    commandAction,
		name:    name,
		args:    args,
		handler: handler,
	})
}

// Script adds a script action to the transaction with the given args.
// handler will be called with the reply from this specific script when
// the transaction is executed.
func (t *Transaction) Script(script *redis.Script, args redis.Args, handler ReplyHandler) {
	t.actions = append(t.actions, &Action{
		kind:    scriptAction,
		script:  script,
		args:    args,
		handler: handler,
	})
}

// sendAction writes a to a connection buffer using conn.Send()
func (t *Transaction) sendAction(a *Action) error {
	switch a.kind {
	case commandAction:
		return t.conn.Send(a.name, a.args...)
	case scriptAction:
		return a.script.Send(t.conn, a.args...)
	}
	return nil
}

// doAction writes a to the connection buffer and then immediately
// flushes the buffer and reads the reply via conn.Do()
func (t *Transaction) doAction(a *Action) (interface{}, error) {
	switch a.kind {
	case commandAction:
		return t.conn.Do(a.name, a.args...)
	case scriptAction:
		return a.script.Do(t.conn, a.args...)
	}
	return nil, nil
}

// Exec executes the transaction, sequentially sending each action and
// calling all the action handlers with the corresponding replies.
func (t *Transaction) Exec() error {
	// Return the connection to the pool when we are done
	defer t.conn.Close()

	// If the transaction had an error from a previous command, return it
	// and don't continue
	if t.err != nil {
		return t.err
	}

	if len(t.actions) == 1 && len(t.watching) == 0 {
		// If there is only one command and no keys being watched, no need to use
		// MULTI/EXEC
		a := t.actions[0]
		reply, err := t.doAction(a)
		if err != nil {
			return err
		}
		if a.handler != nil {
			if err := a.handler(reply); err != nil {
				return err
			}
		}
	} else {
		// Send all the commands and scripts at once using MULTI/EXEC
		if err := t.conn.Send("MULTI"); err != nil {
			return err
		}
		for _, a := range t.actions {
			if err := t.sendAction(a); err != nil {
				return err
			}
		}
		// Invoke redis driver to execute the transaction
		replies, err := redis.Values(t.conn.Do("EXEC"))
		if err != nil {
			if err == redis.ErrNil && len(t.watching) > 0 {
				return WatchError{keys: t.watching}
			}
			return err
		}
		// Iterate through the replies, calling the corresponding handler functions
		for i, reply := range replies {
			a := t.actions[i]
			if err, ok := reply.(error); ok {
				return err
			}
			if a.handler != nil {
				if err := a.handler(reply); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

//go:generate go run scripts/main.go

// DeleteModelsBySetIds is a small function wrapper around a Lua script. The
// script will atomically delete the models corresponding to the ids in set
// (not sorted set) identified by setKey and return the number of models that
// were deleted. You can pass in a handler (e.g. NewScanIntHandler) to capture
// the return value of the script. You can use the Name method of a Collection
// to get the name.
func (t *Transaction) DeleteModelsBySetIds(setKey string, collectionName string, handler ReplyHandler) {
	t.Script(deleteModelsBySetIdsScript, redis.Args{setKey, collectionName}, handler)
}

// deleteStringIndex is a small function wrapper around a Lua script. The script
// will atomically remove the existing string index, if any, on the given
// fieldName for the model with the given modelId. You can use the Name method
// of a Collection to get its name. fieldName should be the name as it is stored
// in Redis.
func (t *Transaction) deleteStringIndex(collectionName, modelId, fieldName string) {
	t.Script(deleteStringIndexScript, redis.Args{collectionName, modelId, fieldName}, nil)
}

// ExtractIdsFromFieldIndex is a small function wrapper around a Lua script. The
// script will get all the ids from the sorted set identified by setKey using
// ZRANGEBYSCORE with the given min and max, and then store them in a sorted set
// identified by destKey. The members of the sorted set should be model ids.
// Note that this method will not work on sorted sets that represents string
// indexes because they are stored differently.
func (t *Transaction) ExtractIdsFromFieldIndex(setKey string, destKey string, min interface{}, max interface{}) {
	t.Script(extractIdsFromFieldIndexScript, redis.Args{setKey, destKey, min, max}, nil)
}

// ExtractIdsFromStringIndex is a small function wrapper around a Lua script.
// The script will extract the ids from a sorted set identified by setKey using
// ZRANGEBYLEX with the given min and max, and then store them in a sorted set
// identified by destKey. All the scores for the sorted set should be 0, and the
// members should follow the format <value>\x00<id>, where <value> is the string
// value, \x000 is the NULL ASCII character and <id> is the id of the model
// with that value. As with all string indexes in Zoom, the value cannot contain
// the NULL ASCII character or the DEL character (codepoints 0 and 127
// respectively). Note that the stored ids are sorted in ASCII order according
// to their corresponding string values.
func (t *Transaction) ExtractIdsFromStringIndex(setKey, destKey, min, max string) {
	t.Script(extractIdsFromStringIndexScript, redis.Args{setKey, destKey, min, max}, nil)
}

func (t *Transaction) FindModelsByIdsKey(collection *Collection, idsKey string, fieldNames []string, limit uint, offset uint, reverse bool, models interface{}) {
	if err := collection.checkModelsType(models); err != nil {
		t.setError(fmt.Errorf("zoom: error in FindModelsByIdKey: %s", err.Error()))
		return
	}
	redisNames, err := collection.spec.redisNamesForFieldNames(fieldNames)
	if err != nil {
		t.setError(fmt.Errorf("zoom: error in FindModelsByIdKey: %s", err.Error()))
		return
	}
	sortArgs := collection.spec.sortArgs(idsKey, redisNames, int(limit), offset, reverse)
	t.Command("SORT", sortArgs, newScanModelsHandler(collection.spec, append(fieldNames, "-"), models))
}
<commit_msg>Remove FindModelsByIdsKey.<commit_after>// Copyright 2015 Alex Browne.  All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

// File transaction.go contains code related to the
// transactions abstraction.

package zoom

import (
	"fmt"

	"github.com/garyburd/redigo/redis"
)

// Transaction is an abstraction layer around a Redis transaction.
// Transactions consist of a set of actions which are either Redis
// commands or lua scripts. Transactions feature delayed execution,
// so nothing touches the database until you call Exec.
type Transaction struct {
	conn     redis.Conn
	actions  []*Action
	err      error
	watching []string
}

// Action is a single step in a transaction and must be either a command
// or a script with optional arguments and a reply handler.
type Action struct {
	kind    actionKind
	name    string
	script  *redis.Script
	args    redis.Args
	handler ReplyHandler
}

// actionKind is either a command or a script
type actionKind int

const (
	commandAction actionKind = iota
	scriptAction
)

// NewTransaction instantiates and returns a new transaction.
func (p *Pool) NewTransaction() *Transaction {
	t := &Transaction{
		conn: p.NewConn(),
	}
	return t
}

// SetError sets the err property of the transaction iff it was not already
// set. This will cause exec to fail immediately.
func (t *Transaction) setError(err error) {
	if t.err == nil {
		t.err = err
	}
}

// Watch issues a Redis WATCH command using the key for the given model. If the
// model changes before the transaction is executed, Exec will return a
// WatchError and the commands in the transaction will not be executed. Unlike
// most other transaction methods, Watch does not use delayed execution. Because
// of how the WATCH command works, Watch must send a command to Redis
// immediately. You must call Watch or WatchKey before any other transaction
// methods.
func (t *Transaction) Watch(model Model) error {
	if len(t.actions) != 0 {
		return fmt.Errorf("Cannot call Watch after other commands have been added to the transaction")
	}
	col, err := getCollectionForModel(model)
	if err != nil {
		return err
	}
	key := col.ModelKey(model.ModelId())
	return t.WatchKey(key)
}

// WatchKey issues a Redis WATCH command using the given key. If the key changes
// before the transaction is executed, Exec will return a WatchError and the
// commands in the transaction will not be executed. Unlike most other
// transaction methods, WatchKey does not use delayed execution. Because of how
// the WATCH command works, WatchKey must send a command to Redis immediately.
// You must call Watch or WatchKey before any other transaction methods.
func (t *Transaction) WatchKey(key string) error {
	if len(t.actions) != 0 {
		return fmt.Errorf("Cannot call WatchKey after other commands have been added to the transaction")
	}
	if _, err := t.conn.Do("WATCH", key); err != nil {
		return err
	}
	t.watching = append(t.watching, key)
	return nil
}

// Command adds a command action to the transaction with the given args.
// handler will be called with the reply from this specific command when
// the transaction is executed.
func (t *Transaction) Command(name string, args redis.Args, handler ReplyHandler) {
	t.actions = append(t.actions, &Action{
		kind:    commandAction,
		name:    name,
		args:    args,
		handler: handler,
	})
}

// Script adds a script action to the transaction with the given args.
// handler will be called with the reply from this specific script when
// the transaction is executed.
func (t *Transaction) Script(script *redis.Script, args redis.Args, handler ReplyHandler) {
	t.actions = append(t.actions, &Action{
		kind:    scriptAction,
		script:  script,
		args:    args,
		handler: handler,
	})
}

// sendAction writes a to a connection buffer using conn.Send()
func (t *Transaction) sendAction(a *Action) error {
	switch a.kind {
	case commandAction:
		return t.conn.Send(a.name, a.args...)
	case scriptAction:
		return a.script.Send(t.conn, a.args...)
	}
	return nil
}

// doAction writes a to the connection buffer and then immediately
// flushes the buffer and reads the reply via conn.Do()
func (t *Transaction) doAction(a *Action) (interface{}, error) {
	switch a.kind {
	case commandAction:
		return t.conn.Do(a.name, a.args...)
	case scriptAction:
		return a.script.Do(t.conn, a.args...)
	}
	return nil, nil
}

// Exec executes the transaction, sequentially sending each action and
// calling all the action handlers with the corresponding replies.
func (t *Transaction) Exec() error {
	// Return the connection to the pool when we are done
	defer t.conn.Close()

	// If the transaction had an error from a previous command, return it
	// and don't continue
	if t.err != nil {
		return t.err
	}

	if len(t.actions) == 1 && len(t.watching) == 0 {
		// If there is only one command and no keys being watched, no need to use
		// MULTI/EXEC
		a := t.actions[0]
		reply, err := t.doAction(a)
		if err != nil {
			return err
		}
		if a.handler != nil {
			if err := a.handler(reply); err != nil {
				return err
			}
		}
	} else {
		// Send all the commands and scripts at once using MULTI/EXEC
		if err := t.conn.Send("MULTI"); err != nil {
			return err
		}
		for _, a := range t.actions {
			if err := t.sendAction(a); err != nil {
				return err
			}
		}
		// Invoke redis driver to execute the transaction
		replies, err := redis.Values(t.conn.Do("EXEC"))
		if err != nil {
			if err == redis.ErrNil && len(t.watching) > 0 {
				return WatchError{keys: t.watching}
			}
			return err
		}
		// Iterate through the replies, calling the corresponding handler functions
		for i, reply := range replies {
			a := t.actions[i]
			if err, ok := reply.(error); ok {
				return err
			}
			if a.handler != nil {
				if err := a.handler(reply); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

//go:generate go run scripts/main.go

// DeleteModelsBySetIds is a small function wrapper around a Lua script. The
// script will atomically delete the models corresponding to the ids in set
// (not sorted set) identified by setKey and return the number of models that
// were deleted. You can pass in a handler (e.g. NewScanIntHandler) to capture
// the return value of the script. You can use the Name method of a Collection
// to get the name.
func (t *Transaction) DeleteModelsBySetIds(setKey string, collectionName string, handler ReplyHandler) {
	t.Script(deleteModelsBySetIdsScript, redis.Args{setKey, collectionName}, handler)
}

// deleteStringIndex is a small function wrapper around a Lua script. The script
// will atomically remove the existing string index, if any, on the given
// fieldName for the model with the given modelId. You can use the Name method
// of a Collection to get its name. fieldName should be the name as it is stored
// in Redis.
func (t *Transaction) deleteStringIndex(collectionName, modelId, fieldName string) {
	t.Script(deleteStringIndexScript, redis.Args{collectionName, modelId, fieldName}, nil)
}

// ExtractIdsFromFieldIndex is a small function wrapper around a Lua script. The
// script will get all the ids from the sorted set identified by setKey using
// ZRANGEBYSCORE with the given min and max, and then store them in a sorted set
// identified by destKey. The members of the sorted set should be model ids.
// Note that this method will not work on sorted sets that represents string
// indexes because they are stored differently.
func (t *Transaction) ExtractIdsFromFieldIndex(setKey string, destKey string, min interface{}, max interface{}) {
	t.Script(extractIdsFromFieldIndexScript, redis.Args{setKey, destKey, min, max}, nil)
}

// ExtractIdsFromStringIndex is a small function wrapper around a Lua script.
// The script will extract the ids from a sorted set identified by setKey using
// ZRANGEBYLEX with the given min and max, and then store them in a sorted set
// identified by destKey. All the scores for the sorted set should be 0, and the
// members should follow the format <value>\x00<id>, where <value> is the string
// value, \x000 is the NULL ASCII character and <id> is the id of the model
// with that value. As with all string indexes in Zoom, the value cannot contain
// the NULL ASCII character or the DEL character (codepoints 0 and 127
// respectively). Note that the stored ids are sorted in ASCII order according
// to their corresponding string values.
func (t *Transaction) ExtractIdsFromStringIndex(setKey, destKey, min, max string) {
	t.Script(extractIdsFromStringIndexScript, redis.Args{setKey, destKey, min, max}, nil)
}
<|endoftext|>