<commit_before>package tmpmysql

import (
	"reflect"
	"testing"
)

func TestMySQLServer(t *testing.T) {
	if !IsMySQLInstalled() {
		t.Skip("MySQL not installed")
	}

	server, err := NewMySQLServer("tmpmysqld_test")
	if err != nil {
		t.Fatal(err)
	}
	defer server.Stop()

	if _, err := server.DB.Exec(`
CREATE TABLE things (
  id BIGINT PRIMARY KEY AUTO_INCREMENT,
  name VARCHAR(100) NOT NULL
)
`); err != nil {
		t.Error(err)
	}

	if _, err := server.DB.Exec(`
INSERT INTO things (name) VALUES ("one"), ("two")
`); err != nil {
		t.Error(err)
	}

	rows, err := server.DB.Query("SELECT id, name FROM things")
	if err != nil {
		t.Error(err)
	}
	defer rows.Close()

	actual := make(map[int64]string)

	for rows.Next() {
		var id int64
		var name string
		if err := rows.Scan(&id, &name); err != nil {
			t.Error(err)
		}
		actual[id] = name
	}

	expected := map[int64]string{
		1: "one",
		2: "two",
	}

	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("Was %#v, but expected %#v", actual, expected)
	}
}
<commit_msg>Added an example.<commit_after>package tmpmysql

import (
	"fmt"
	"reflect"
	"testing"
)

func Example() {
	if !IsMySQLInstalled() {
		panic("MySQL not installed")
	}

	server, err := NewMySQLServer("tmpmysqld_test")
	if err != nil {
		panic(err)
	}
	defer server.Stop()

	if _, err := server.DB.Exec(`
CREATE TABLE things (
  id BIGINT PRIMARY KEY AUTO_INCREMENT,
  name VARCHAR(100) NOT NULL
)
`); err != nil {
		panic(err)
	}

	if _, err := server.DB.Exec(`
INSERT INTO things (name) VALUES ("one"), ("two")
`); err != nil {
		panic(err)
	}

	rows, err := server.DB.Query(`SELECT id, name FROM things ORDER BY id`)
	if err != nil {
		panic(err)
	}
	defer rows.Close()

	for rows.Next() {
		var id int64
		var name string
		if err := rows.Scan(&id, &name); err != nil {
			panic(err)
		}
		fmt.Printf("%d=%s\n", id, name)
	}

	// Output:
	// 1=one
	// 2=two
}

func TestMySQLServer(t *testing.T) {
	if !IsMySQLInstalled() {
		t.Skip("MySQL not installed")
	}

	server, err := NewMySQLServer("tmpmysqld_test")
	if err != nil {
		t.Fatal(err)
	}
	defer server.Stop()

	if _, err := server.DB.Exec(`
CREATE TABLE things (
  id BIGINT PRIMARY KEY AUTO_INCREMENT,
  name VARCHAR(100) NOT NULL
)
`); err != nil {
		t.Error(err)
	}

	if _, err := server.DB.Exec(`
INSERT INTO things (name) VALUES ("one"), ("two")
`); err != nil {
		t.Error(err)
	}

	rows, err := server.DB.Query(`SELECT id, name FROM things`)
	if err != nil {
		t.Error(err)
	}
	defer rows.Close()

	actual := make(map[int64]string)

	for rows.Next() {
		var id int64
		var name string
		if err := rows.Scan(&id, &name); err != nil {
			t.Error(err)
		}
		actual[id] = name
	}

	expected := map[int64]string{
		1: "one",
		2: "two",
	}

	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("Was %#v, but expected %#v", actual, expected)
	}
}
<|endoftext|>


<commit_before>package stone

import (
	"crypto/rsa"
	"fmt"
	"strings"
	"time"

	jwt "github.com/dgrijalva/jwt-go"
	"github.com/google/uuid"
	"github.com/mundipagg/boleto-api/certificate"
	"github.com/mundipagg/boleto-api/config"
)

var (
	signKey *rsa.PrivateKey
)

const (
	StoneRealm = "stone"
)

func generateJWT() (string, error) {
	sk, err := certificate.GetCertificateFromStore(config.Get().AzureStorageOpenBankSkName)
	if err != nil {
		return "", err
	}

	signKey, err = jwt.ParseRSAPrivateKeyFromPEM(sk.([]byte))
	if err != nil {
		return "", err
	}

	now := time.Now()

	atClaims := jwt.MapClaims{}
	atClaims["exp"] = now.Add(time.Duration(config.Get().StoneTokenDurationInMinutes) * time.Minute).Unix()
	atClaims["nbf"] = now.Unix()
	atClaims["aud"] = config.Get().StoneAudience
	atClaims["realm"] = StoneRealm
	atClaims["sub"] = config.Get().StoneClientID
	atClaims["clientId"] = config.Get().StoneClientID
	atClaims["iat"] = now.Unix()
	atClaims["jti"] = generateJTIFromTime(now)

	at := jwt.NewWithClaims(jwt.SigningMethodRS256, atClaims)

	token, err := at.SignedString(signKey)
	if err != nil {
		return "", err
	}
	return token, nil
}

func generateJTIFromTime(t time.Time) string {
	id, _ := uuid.NewUUID()
	nowStr := t.Format("2006-01-02T15:04:05.000Z")

	removable := []string{"-", "T", ":", "."}
	for _, ch := range removable {
		nowStr = strings.ReplaceAll(nowStr, ch, "")
	}

	return fmt.Sprintf("%s.%s", nowStr[:17], id.String()[:7])
}
<commit_msg>Removed unused package var<commit_after>package stone

import (
	"fmt"
	"strings"
	"time"

	jwt "github.com/dgrijalva/jwt-go"
	"github.com/google/uuid"
	"github.com/mundipagg/boleto-api/certificate"
	"github.com/mundipagg/boleto-api/config"
)

const (
	StoneRealm = "stone"
)

func generateJWT() (string, error) {
	sk, err := certificate.GetCertificateFromStore(config.Get().AzureStorageOpenBankSkName)
	if err != nil {
		return "", err
	}

	signKey, err := jwt.ParseRSAPrivateKeyFromPEM(sk.([]byte))
	if err != nil {
		return "", err
	}

	now := time.Now()

	atClaims := jwt.MapClaims{}
	atClaims["exp"] = now.Add(time.Duration(config.Get().StoneTokenDurationInMinutes) * time.Minute).Unix()
	atClaims["nbf"] = now.Unix()
	atClaims["aud"] = config.Get().StoneAudience
	atClaims["realm"] = StoneRealm
	atClaims["sub"] = config.Get().StoneClientID
	atClaims["clientId"] = config.Get().StoneClientID
	atClaims["iat"] = now.Unix()
	atClaims["jti"] = generateJTIFromTime(now)

	at := jwt.NewWithClaims(jwt.SigningMethodRS256, atClaims)

	token, err := at.SignedString(signKey)
	if err != nil {
		return "", err
	}
	return token, nil
}

func generateJTIFromTime(t time.Time) string {
	id, _ := uuid.NewUUID()
	nowStr := t.Format("2006-01-02T15:04:05.000Z")

	removable := []string{"-", "T", ":", "."}
	for _, ch := range removable {
		nowStr = strings.ReplaceAll(nowStr, ch, "")
	}

	return fmt.Sprintf("%s.%s", nowStr[:17], id.String()[:7])
}
<|endoftext|>


<commit_before>package sup

import (
	"fmt"

	"golang.org/x/net/context"

	"github.com/Sirupsen/logrus"
	autoctx "github.com/vladimirvivien/automi/context"
)

type ProbeFunc func(interface{}) interface{}

// The Probe is a processor designed for testing and inspecting data flow.
// It captures data in its input channel, apply specified function,
// then outputs the result to its outupt channel
type Probe struct {
	Name    string
	Examine ProbeFunc

	input  <-chan interface{}
	output chan interface{}
	log    *logrus.Entry
}

func (p *Probe) Init(ctx context.Context) error {
	// validation
	if p.Name == "" {
		return fmt.Errorf("Missing name identifier")
	}

	if p.input == nil {
		return fmt.Errorf("Probe [%s] input not set", p.Name)
	}

	p.output = make(chan interface{})

	log, ok := autoctx.GetLogEntry(ctx)
	if !ok {
		log = logrus.WithField("ProcName", p.Name)
		log.Errorf("No valid logger set for %s", p.Name)
	}

	p.log = log.WithFields(logrus.Fields{
		"Component": p.Name,
		"Type":      fmt.Sprintf("%T", p),
	})
	p.log.Info("Component initialized")

	return nil
}

func (p *Probe) Uninit(ctx context.Context) error {
	return nil
}

func (p *Probe) GetName() string {
	return p.Name
}

func (p *Probe) SetInput(in <-chan interface{}) {
	p.input = in
}

func (p *Probe) GetOutput() <-chan interface{} {
	return p.output
}

func (p *Probe) Exec(ctx context.Context) error {
	go func() {
		defer func() {
			close(p.output)
		}()

		// output data
		for item := range p.input {
			if p.Examine != nil {
				p.output <- p.Examine(item)
			}
		}
	}()
	return nil
}
<commit_msg>Added logging statements<commit_after>package sup

import (
	"fmt"

	"golang.org/x/net/context"

	"github.com/Sirupsen/logrus"
	autoctx "github.com/vladimirvivien/automi/context"
)

type ProbeFunc func(interface{}) interface{}

// The Probe is a processor designed for testing and inspecting data flow.
// It captures data in its input channel, apply specified function,
// then outputs the result to its outupt channel
type Probe struct {
	Name    string
	Examine ProbeFunc

	input  <-chan interface{}
	output chan interface{}
	log    *logrus.Entry
}

func (p *Probe) Init(ctx context.Context) error {
	// validation
	if p.Name == "" {
		return fmt.Errorf("Missing name identifier")
	}

	if p.input == nil {
		return fmt.Errorf("Probe [%s] input not set", p.Name)
	}

	p.output = make(chan interface{})

	log, ok := autoctx.GetLogEntry(ctx)
	if !ok {
		log = logrus.WithField("ProcName", p.Name)
		log.Errorf("No valid logger set for %s", p.Name)
	}

	p.log = log.WithFields(logrus.Fields{
		"Component": p.Name,
		"Type":      fmt.Sprintf("%T", p),
	})
	p.log.Info("Component initialized")

	return nil
}

func (p *Probe) Uninit(ctx context.Context) error {
	return nil
}

func (p *Probe) GetName() string {
	return p.Name
}

func (p *Probe) SetInput(in <-chan interface{}) {
	p.input = in
}

func (p *Probe) GetOutput() <-chan interface{} {
	return p.output
}

func (p *Probe) Exec(ctx context.Context) error {
	p.log.Info("Execution started")
	go func() {
		defer func() {
			close(p.output)
			p.log.Info("Execution completed")
		}()

		// output data
		for item := range p.input {
			if p.Examine != nil {
				p.output <- p.Examine(item)
			}
		}
	}()
	return nil
}
<|endoftext|>


<commit_before>package lazycache

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/amarburg/go-fast-png"
	"github.com/spf13/viper"
	"golang.org/x/image/bmp"
 	"github.com/disintegration/imaging"
	"image/jpeg"
	"io"
	"net/http"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

import "github.com/amarburg/go-lazyquicktime"

var leadingNumbers, _ = regexp.Compile("^\\d+")

//go:generate $GOPATH/bin/easyjson -all $GOFILE

// MoovHandlerTiming
type MoovHandlerTiming struct {
	// Don't export start times, so they don't get JSON-encoded
	// Todo, I could clean up this API a bit...
	handlerStart                          time.Time
	Handler, Metadata, Extraction, Encode time.Duration
}

type moovOutputMetadata struct {
	URL       string
	NumFrames uint64
	Duration  float64
	FileSize  int64
}

type QTEntry struct {
	lqt      *lazyquicktime.LazyQuicktime
	metadata moovOutputMetadata
}

type QTStore struct {
	Cache map[string](*QTEntry)
	Mutex sync.Mutex

	Stats struct {
		Requests, Misses int64
	}
}

var qtCache QTStore

func init() {
	qtCache = QTStore{
		Cache: make(map[string](*QTEntry)),
	}
}

func (cache *QTStore) getLQT(node *Node) (*QTEntry, error) {

	cache.Mutex.Lock()
	defer cache.Mutex.Unlock()

	cache.Stats.Requests++

	// Initialize or update as necessary
	Logger.Log("debug", fmt.Sprintf("Querying metadata store for %s", node.Path))
	qte, has := cache.Cache[node.trimPath]

	if !has {
		cache.Stats.Misses++

		//Logger.Log("msg", fmt.Errorf("Initializing LazyFile to %s", node.Path))
		fs, err := node.Fs.FileSource(node.Path)

		if err != nil {
			return nil, fmt.Errorf("Something went boom while opening the HTTP Source!")
		}

		//block, err := lazyfs.OpenBlockStore( fs, 20 )
		// if err != nil {
		// 	return nil, fmt.Errorf("Something went boom while opening the HTTP Source!")
		// }

		//Logger.Log("msg", fmt.Sprintf("Need to pull quicktime information for %s", fs.Path()))
		lqt, err := lazyquicktime.LoadMovMetadata(fs)
		if err != nil {
			return nil, fmt.Errorf("Something went boom while storing the quicktime file: %s", err.Error())
		}

		//Logger.Log("msg", fmt.Sprintf("Updating metadata store for %s", fs.Path()))
		qte = &QTEntry{
			lqt: lqt,
			metadata: moovOutputMetadata{
				FileSize:  lqt.FileSize,
				URL:       node.Path,
				NumFrames: lqt.NumFrames(),
				Duration:  lqt.Duration().Seconds(),
			},
		}

		cache.Cache[node.trimPath] = qte

	} else {
		Logger.Log("msg", fmt.Sprintf("lqt cache has entry for %s", node.Path))
	}

	return qte, nil
}

func MoovHandler(node *Node, path []string, w http.ResponseWriter, req *http.Request) *Node {
	Logger.Log("msg", fmt.Sprintf("Quicktime handler: %s with residual path (%d): (%s)", node.Path, len(path), strings.Join(path, ":")))

	timing := MoovHandlerTiming{
		handlerStart: time.Now(),
	}

	// uri := node.Fs.Uri
	// uri.Path += node.Path

	metadataStart := time.Now()
	qte, err := qtCache.getLQT(node)
	timeTrack(metadataStart, &timing.Metadata)

	if err != nil {
		Logger.Log("msg", err.Error())

		b, _ := json.MarshalIndent(struct {
			URL, Error string
		}{
			URL:   node.Path,
			Error: err.Error(),
		}, "", "  ")

		w.Write(b)
		return nil
	}

	if len(path) == 0 {
		// Leaf node
		startEncode := time.Now()

		Logger.Log("msg", fmt.Sprintf("Returning movie information for %s", node.Path))

		b, err := qte.metadata.MarshalJSON()
		if err != nil {
			fmt.Fprintln(w, "JSON error:", err)
		}

		timeTrack(startEncode, &timing.Encode)
		timeTrack(timing.handlerStart, &timing.Handler)

		w.Header().Set("X-lazycache-timing-handler-ns", strconv.Itoa(int(timing.Handler.Nanoseconds())))
		w.Header().Set("X-lazycache-timing-metadata-ns", strconv.Itoa(int(timing.Metadata.Nanoseconds())))

		w.Write(b)
	} else {

		// Handle any residual path elements (frames, etc) here
		switch strings.ToLower(path[0]) {
		case "frame":
			extractFrame(node, qte, path[1:], w, req, &timing)
		default:
			http.Error(w, fmt.Sprintf("Didn't understand request \"%s\"", path[0]), 500)
		}
	}

	t, _ := timing.MarshalJSON()
	Logger.Log("timing", t)

	return nil
}

func extractFrame(node *Node, qte *QTEntry, path []string, w http.ResponseWriter, req *http.Request, timing *MoovHandlerTiming) {

	if qte == nil || qte.lqt == nil {
		http.Error(w, "Error in extractFrame", 500)
		return
	}

	if len(path) == 0 {
		http.Error(w, fmt.Sprintf("Need to specify frame number"), 500)
		return

	}

	frameNum, err := strconv.Atoi(leadingNumbers.FindString(path[0]))
	if err != nil {
		http.Error(w, fmt.Sprintf("Error parsing frame number \"%s\"", path[0]), 500)
		return
	}

	if uint64(frameNum) > qte.metadata.NumFrames {
		http.Error(w, fmt.Sprintf("Requested frame %d in movie of length %d frames", frameNum, qte.metadata.NumFrames), 400)
		return
	}

	if frameNum < 1 {
		http.Error(w, "Requested frame 0, Quicktime movies start with frame 1", 400)
		return
	}

	// Looks for extension
	extension := filepath.Ext(path[0])

	var contentType string

	switch extension {
	case ".bmp":
		contentType = "image/bmp"
		extension = ".bmp"
	case ".jpg", ".jpeg":
		contentType = "image/jpeg"
		extension = ".jpg"
	case "", ".png":
		extension = ".png"
		contentType = "image/png"
	case ".rgba", ".raw":
		extension = ".rgba"
		contentType = "image/x-raw-rgba"
	default:
		http.Error(w, fmt.Sprintf("Unknown image extension \"%s\"", extension), 500)
		return
	}

	UUID := req.URL.Path + extension
	url, ok := ImageCache.Url(UUID)

	if ok {
		Logger.Log("msg", fmt.Sprintf("Image %s exists in the Image store at %s", UUID, url))
		// Set Content-Type or response
		w.Header().Set("Content-Type", contentType)
		Logger.Log("msg", fmt.Sprintf("Redirecting to %s", url))
		http.Redirect(w, req, url, http.StatusTemporaryRedirect)

	} else {

		startExt := time.Now()
		img, err := qte.lqt.ExtractNRGBA(uint64(frameNum))
		if err != nil {
			http.Error(w, fmt.Sprintf("Error generating image for frame %d: %s", frameNum, err.Error()), 500)
			return
		}
		timeTrack(startExt, &timing.Extraction)

		query := req.URL.Query()

		widthStr, widthValid := query["width"]
		heightStr, heightValid := query["height"]

		Logger.Log("msg", fmt.Sprintf("width = %s, height = %s", widthStr, heightStr))

		if( widthValid && heightValid ) {

			width, _ := strconv.Atoi(widthStr[0])
			height, _ := strconv.Atoi(heightStr[0])

			Logger.Log("msg", fmt.Sprintf("Resizing to %d x %d", width, height))
			resized := imaging.Resize(img, width, height, imaging.Lanczos)
			img = resized

		} else {

			// Check HTTP header for scale information
			hdrWidth := req.Header.Get("X-lazycache-output-width")
			hdrHeight := req.Header.Get("X-lazycache-output-height")

			if( len(hdrWidth) > 0 && len(hdrHeight) > 0 ) {

				width, _ := strconv.Atoi(hdrWidth)
				height, _ := strconv.Atoi(hdrHeight)

				Logger.Log("msg", fmt.Sprintf("Resizing to %d x %d", width, height))
				resized := imaging.Resize(img, width, height, imaging.Lanczos)
				img = resized
			}
		}

		startEncode := time.Now()

		var imgReader *bytes.Reader

		switch contentType {
		case "image/png":

			buffer := new(bytes.Buffer)
			encoder := new(fastpng.Encoder)

			// TODO, allow configuration of PNGs
			if viper.GetBool("public") {
				encoder.CompressionLevel = fastpng.BestCompression
			}

			// {
			// 	CompressionLevel: fastpng.BestSpeed,
			// }

			err = encoder.Encode(buffer, img)
			imgReader = bytes.NewReader(buffer.Bytes())

		case "image/jpeg":
			buffer := new(bytes.Buffer)
			err = jpeg.Encode(buffer, img, &jpeg.Options{Quality: jpeg.DefaultQuality})
			imgReader = bytes.NewReader(buffer.Bytes())

		case "image/bmp":
			buffer := new(bytes.Buffer)
			err = bmp.Encode(buffer, img)
			imgReader = bytes.NewReader(buffer.Bytes())

		case "image/x-raw-rgba":
			if viper.GetBool("allow-raw-output") {
				// stand-in
				//buffer = img.Pix
				imgReader = bytes.NewReader(img.Pix)
			} else {
				http.Error(w, "This server is not configured to produce raw output.", 501)
				return
			}
		}

		timeTrack(startEncode, &timing.Encode)

		//Logger.Log("debug", fmt.Sprintf("%s size %d MB\n", contentType, buffer.Len()/(1024*1024)))

		// write image to Image store
		ImageCache.Store(UUID, imgReader)

		timeTrack(timing.handlerStart, &timing.Handler)

		// Add timing information to HTTP Header
		w.Header().Set("X-lazycache-timing-handler-ns", strconv.Itoa(int(timing.Handler.Nanoseconds())))
		w.Header().Set("X-lazycache-timing-metadata-ns", strconv.Itoa(int(timing.Metadata.Nanoseconds())))
		w.Header().Set("X-lazycache-timing-extraction-ns", strconv.Itoa(int(timing.Extraction.Nanoseconds())))
		w.Header().Set("X-lazycache-timing-encode-ns", strconv.Itoa(int(timing.Encode.Nanoseconds())))

		imgReader.Seek(0, io.SeekStart)
		_, err = imgReader.WriteTo(w)
		if err != nil {
			fmt.Printf("Error writing to HTTP buffer: %s\n", err.Error())
		}

	}

}
<commit_msg>Error message if only width or only height are specified on the command line.<commit_after>package lazycache

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/amarburg/go-fast-png"
	"github.com/spf13/viper"
	"golang.org/x/image/bmp"
 	"github.com/disintegration/imaging"
	"image/jpeg"
	"io"
	"net/http"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

import "github.com/amarburg/go-lazyquicktime"

var leadingNumbers, _ = regexp.Compile("^\\d+")

//go:generate $GOPATH/bin/easyjson -all $GOFILE

// MoovHandlerTiming
type MoovHandlerTiming struct {
	// Don't export start times, so they don't get JSON-encoded
	// Todo, I could clean up this API a bit...
	handlerStart                          time.Time
	Handler, Metadata, Extraction, Encode time.Duration
}

type moovOutputMetadata struct {
	URL       string
	NumFrames uint64
	Duration  float64
	FileSize  int64
}

type QTEntry struct {
	lqt      *lazyquicktime.LazyQuicktime
	metadata moovOutputMetadata
}

type QTStore struct {
	Cache map[string](*QTEntry)
	Mutex sync.Mutex

	Stats struct {
		Requests, Misses int64
	}
}

var qtCache QTStore

func init() {
	qtCache = QTStore{
		Cache: make(map[string](*QTEntry)),
	}
}

func (cache *QTStore) getLQT(node *Node) (*QTEntry, error) {

	cache.Mutex.Lock()
	defer cache.Mutex.Unlock()

	cache.Stats.Requests++

	// Initialize or update as necessary
	Logger.Log("debug", fmt.Sprintf("Querying metadata store for %s", node.Path))
	qte, has := cache.Cache[node.trimPath]

	if !has {
		cache.Stats.Misses++

		//Logger.Log("msg", fmt.Errorf("Initializing LazyFile to %s", node.Path))
		fs, err := node.Fs.FileSource(node.Path)

		if err != nil {
			return nil, fmt.Errorf("Something went boom while opening the HTTP Source!")
		}

		//block, err := lazyfs.OpenBlockStore( fs, 20 )
		// if err != nil {
		// 	return nil, fmt.Errorf("Something went boom while opening the HTTP Source!")
		// }

		//Logger.Log("msg", fmt.Sprintf("Need to pull quicktime information for %s", fs.Path()))
		lqt, err := lazyquicktime.LoadMovMetadata(fs)
		if err != nil {
			return nil, fmt.Errorf("Something went boom while storing the quicktime file: %s", err.Error())
		}

		//Logger.Log("msg", fmt.Sprintf("Updating metadata store for %s", fs.Path()))
		qte = &QTEntry{
			lqt: lqt,
			metadata: moovOutputMetadata{
				FileSize:  lqt.FileSize,
				URL:       node.Path,
				NumFrames: lqt.NumFrames(),
				Duration:  lqt.Duration().Seconds(),
			},
		}

		cache.Cache[node.trimPath] = qte

	} else {
		Logger.Log("msg", fmt.Sprintf("lqt cache has entry for %s", node.Path))
	}

	return qte, nil
}

func MoovHandler(node *Node, path []string, w http.ResponseWriter, req *http.Request) *Node {
	Logger.Log("msg", fmt.Sprintf("Quicktime handler: %s with residual path (%d): (%s)", node.Path, len(path), strings.Join(path, ":")))

	timing := MoovHandlerTiming{
		handlerStart: time.Now(),
	}

	// uri := node.Fs.Uri
	// uri.Path += node.Path

	metadataStart := time.Now()
	qte, err := qtCache.getLQT(node)
	timeTrack(metadataStart, &timing.Metadata)

	if err != nil {
		Logger.Log("msg", err.Error())

		b, _ := json.MarshalIndent(struct {
			URL, Error string
		}{
			URL:   node.Path,
			Error: err.Error(),
		}, "", "  ")

		w.Write(b)
		return nil
	}

	if len(path) == 0 {
		// Leaf node
		startEncode := time.Now()

		Logger.Log("msg", fmt.Sprintf("Returning movie information for %s", node.Path))

		b, err := qte.metadata.MarshalJSON()
		if err != nil {
			fmt.Fprintln(w, "JSON error:", err)
		}

		timeTrack(startEncode, &timing.Encode)
		timeTrack(timing.handlerStart, &timing.Handler)

		w.Header().Set("X-lazycache-timing-handler-ns", strconv.Itoa(int(timing.Handler.Nanoseconds())))
		w.Header().Set("X-lazycache-timing-metadata-ns", strconv.Itoa(int(timing.Metadata.Nanoseconds())))

		w.Write(b)
	} else {

		// Handle any residual path elements (frames, etc) here
		switch strings.ToLower(path[0]) {
		case "frame":
			extractFrame(node, qte, path[1:], w, req, &timing)
		default:
			http.Error(w, fmt.Sprintf("Didn't understand request \"%s\"", path[0]), 500)
		}
	}

	t, _ := timing.MarshalJSON()
	Logger.Log("timing", t)

	return nil
}

func extractFrame(node *Node, qte *QTEntry, path []string, w http.ResponseWriter, req *http.Request, timing *MoovHandlerTiming) {

	if qte == nil || qte.lqt == nil {
		http.Error(w, "Error in extractFrame", 500)
		return
	}

	if len(path) == 0 {
		http.Error(w, fmt.Sprintf("Need to specify frame number"), 500)
		return

	}

	frameNum, err := strconv.Atoi(leadingNumbers.FindString(path[0]))
	if err != nil {
		http.Error(w, fmt.Sprintf("Error parsing frame number \"%s\"", path[0]), 500)
		return
	}

	if uint64(frameNum) > qte.metadata.NumFrames {
		http.Error(w, fmt.Sprintf("Requested frame %d in movie of length %d frames", frameNum, qte.metadata.NumFrames), 400)
		return
	}

	if frameNum < 1 {
		http.Error(w, "Requested frame 0, Quicktime movies start with frame 1", 400)
		return
	}

	// Looks for extension
	extension := filepath.Ext(path[0])

	var contentType string

	switch extension {
	case ".bmp":
		contentType = "image/bmp"
		extension = ".bmp"
	case ".jpg", ".jpeg":
		contentType = "image/jpeg"
		extension = ".jpg"
	case "", ".png":
		extension = ".png"
		contentType = "image/png"
	case ".rgba", ".raw":
		extension = ".rgba"
		contentType = "image/x-raw-rgba"
	default:
		http.Error(w, fmt.Sprintf("Unknown image extension \"%s\"", extension), 500)
		return
	}

	UUID := req.URL.Path + extension
	url, ok := ImageCache.Url(UUID)

	if ok {
		Logger.Log("msg", fmt.Sprintf("Image %s exists in the Image store at %s", UUID, url))
		// Set Content-Type or response
		w.Header().Set("Content-Type", contentType)
		Logger.Log("msg", fmt.Sprintf("Redirecting to %s", url))
		http.Redirect(w, req, url, http.StatusTemporaryRedirect)

	} else {

		startExt := time.Now()
		img, err := qte.lqt.ExtractNRGBA(uint64(frameNum))
		if err != nil {
			http.Error(w, fmt.Sprintf("Error generating image for frame %d: %s", frameNum, err.Error()), 500)
			return
		}
		timeTrack(startExt, &timing.Extraction)

		query := req.URL.Query()

		widthStr, widthValid := query["width"]
		heightStr, heightValid := query["height"]

		if( widthValid && heightValid ) {

			width, _ := strconv.Atoi(widthStr[0])
			height, _ := strconv.Atoi(heightStr[0])

			Logger.Log("msg", fmt.Sprintf("Resizing to %d x %d", width, height))
			resized := imaging.Resize(img, width, height, imaging.Lanczos)
			img = resized

		} else if( widthValid || heightValid ) {

			http.Error(w, fmt.Sprintf("Both width and height must be specified.  Got width = %s and height = %s ", widthStr, heightStr), 500)
			return

		} else {

			// Check HTTP header for scale information
			hdrWidth := req.Header.Get("X-lazycache-output-width")
			hdrHeight := req.Header.Get("X-lazycache-output-height")

			if( len(hdrWidth) > 0 && len(hdrHeight) > 0 ) {

				width, _ := strconv.Atoi(hdrWidth)
				height, _ := strconv.Atoi(hdrHeight)

				Logger.Log("msg", fmt.Sprintf("Resizing to %d x %d", width, height))
				resized := imaging.Resize(img, width, height, imaging.Lanczos)
				img = resized
			}
		}

		startEncode := time.Now()

		var imgReader *bytes.Reader

		switch contentType {
		case "image/png":

			buffer := new(bytes.Buffer)
			encoder := new(fastpng.Encoder)

			// TODO, allow configuration of PNGs
			if viper.GetBool("public") {
				encoder.CompressionLevel = fastpng.BestCompression
			}

			// {
			// 	CompressionLevel: fastpng.BestSpeed,
			// }

			err = encoder.Encode(buffer, img)
			imgReader = bytes.NewReader(buffer.Bytes())

		case "image/jpeg":
			buffer := new(bytes.Buffer)
			err = jpeg.Encode(buffer, img, &jpeg.Options{Quality: jpeg.DefaultQuality})
			imgReader = bytes.NewReader(buffer.Bytes())

		case "image/bmp":
			buffer := new(bytes.Buffer)
			err = bmp.Encode(buffer, img)
			imgReader = bytes.NewReader(buffer.Bytes())

		case "image/x-raw-rgba":
			if viper.GetBool("allow-raw-output") {
				// stand-in
				//buffer = img.Pix
				imgReader = bytes.NewReader(img.Pix)
			} else {
				http.Error(w, "This server is not configured to produce raw output.", 501)
				return
			}
		}

		timeTrack(startEncode, &timing.Encode)

		//Logger.Log("debug", fmt.Sprintf("%s size %d MB\n", contentType, buffer.Len()/(1024*1024)))

		// write image to Image store
		ImageCache.Store(UUID, imgReader)

		timeTrack(timing.handlerStart, &timing.Handler)

		// Add timing information to HTTP Header
		w.Header().Set("X-lazycache-timing-handler-ns", strconv.Itoa(int(timing.Handler.Nanoseconds())))
		w.Header().Set("X-lazycache-timing-metadata-ns", strconv.Itoa(int(timing.Metadata.Nanoseconds())))
		w.Header().Set("X-lazycache-timing-extraction-ns", strconv.Itoa(int(timing.Extraction.Nanoseconds())))
		w.Header().Set("X-lazycache-timing-encode-ns", strconv.Itoa(int(timing.Encode.Nanoseconds())))

		imgReader.Seek(0, io.SeekStart)
		_, err = imgReader.WriteTo(w)
		if err != nil {
			fmt.Printf("Error writing to HTTP buffer: %s\n", err.Error())
		}

	}

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/gorilla/mux"
	"io/ioutil"
	"log"
	"net/http"
)

var (
	ServerVersion string = "pre version"
	APIVersion string = "pre api"
)

type Configuration struct {
	ListeningAddress string
}

func CreateConfig() {
	var Config Configuration
	Config.ListeningAddress = ":80"
	ByteJsonConfig, err := toprettyjson(Config)
	if err != nil {
		panic(err)
	}
	fmt.Println("Writing configuration to ./trackmonserv.conf")
	err = ioutil.WriteFile("./trackmonserv.conf", ByteJsonConfig, 0644)
	if err != nil {
		panic(err)
	}
}

func RootHandler(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusOK)
}

func VersionHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "{\"serverversion\":\"%s\",\"apiversion\":\"%s\"}", ServerVersion, APIVersion)
	w.WriteHeader(http.StatusOK)
}

func main() {
	fmt.Println("trackmon server by Paul Kramme")
	fmt.Println("Please report bugs to https://github.com/trackmon/trackmon-server")

	// Configure flags
	CreateConfigFlag := flag.Bool("createconfig", false, "Creates a standard configuration and exits")
	ConfigLocation := flag.String("config", "./trackmonserv.conf", "Location of config file. Standard is ./trackmonserv")
	flag.Parse()

	// Check flags
	if *CreateConfigFlag == true {
		CreateConfig()
		return
	}

	// Load config
	var Config Configuration
	Configfile, err := ioutil.ReadFile(*ConfigLocation)
	if err != nil {
		fmt.Println("Couldn't find or open config file. Create one with -createconfig")
		panic(err)
	}
	err = fromjson(string(Configfile), &Config)
	if err != nil {
		panic(err)
	}

	// Configure router and server
	r := mux.NewRouter()
	r.HandleFunc("/", RootHandler) // Returnes 200 OK, can be used for health checks or homepage...
	r.HandleFunc("/version", VersionHandler)

	srv := &http.Server{
		Handler: r,
		Addr:    Config.ListeningAddress,
	}

	log.Println("Initialization complete")
	srv.ListenAndServe()
}

func fromjson(src string, v interface{}) error {
	return json.Unmarshal([]byte(src), v)
}

func tojson(v interface{}) ([]byte, error) {
	return json.Marshal(v)
}

func toprettyjson(v interface{}) ([]byte, error) {
	return json.MarshalIndent(v, "", "\t")
}
<commit_msg>Add mux license print<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/gorilla/mux"
	"io/ioutil"
	"log"
	"net/http"
)

var (
	ServerVersion string = "pre version"
	APIVersion string = "pre api"
)

type Configuration struct {
	ListeningAddress string
}

func CreateConfig() {
	var Config Configuration
	Config.ListeningAddress = ":80"
	ByteJsonConfig, err := toprettyjson(Config)
	if err != nil {
		panic(err)
	}
	fmt.Println("Writing configuration to ./trackmonserv.conf")
	err = ioutil.WriteFile("./trackmonserv.conf", ByteJsonConfig, 0644)
	if err != nil {
		panic(err)
	}
}

func RootHandler(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusOK)
}

func VersionHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "{\"serverversion\":\"%s\",\"apiversion\":\"%s\"}", ServerVersion, APIVersion)
	w.WriteHeader(http.StatusOK)
}

func main() {
	fmt.Println("trackmon server by Paul Kramme")
	fmt.Println("Please report bugs to https://github.com/trackmon/trackmon-server")

	// Configure flags
	CreateConfigFlag := flag.Bool("createconfig", false, "Creates a standard configuration and exits")
	ConfigLocation := flag.String("config", "./trackmonserv.conf", "Location of config file. Standard is ./trackmonserv")
	ShowLicenses := flag.Bool("licenses", false, "Shows licenses and exits")
	flag.Parse()

	// Check flags
	if *CreateConfigFlag == true {
		CreateConfig()
		return
	}

	if *ShowLicenses == true {
		fmt.Println("This project uses github.com/gorilla/mux")
		fmt.Print(muxlicense)
		fmt.Println("\n")

		return
	}

	// Load config
	var Config Configuration
	Configfile, err := ioutil.ReadFile(*ConfigLocation)
	if err != nil {
		fmt.Println("Couldn't find or open config file. Create one with -createconfig")
		panic(err)
	}
	err = fromjson(string(Configfile), &Config)
	if err != nil {
		panic(err)
	}

	// Configure router and server
	r := mux.NewRouter()
	r.HandleFunc("/", RootHandler) // Returnes 200 OK, can be used for health checks or homepage...
	r.HandleFunc("/version", VersionHandler)

	srv := &http.Server{
		Handler: r,
		Addr:    Config.ListeningAddress,
	}

	log.Println("Initialization complete")
	srv.ListenAndServe()
}

func fromjson(src string, v interface{}) error {
	return json.Unmarshal([]byte(src), v)
}

func tojson(v interface{}) ([]byte, error) {
	return json.Marshal(v)
}

func toprettyjson(v interface{}) ([]byte, error) {
	return json.MarshalIndent(v, "", "\t")
}

const (
	muxlicense string = `Copyright (c) 2012 Rodrigo Moraes. All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

	 * Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
	 * Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
	 * Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.`
)
<|endoftext|>


<commit_before>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package distsql

import (
	"time"

	"github.com/juju/errors"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/model"
	"github.com/pingcap/tidb/mysql"
	"github.com/pingcap/tidb/terror"
	"github.com/pingcap/tidb/util/types"
	"github.com/pingcap/tipb/go-tipb"
	goctx "golang.org/x/net/context"
)

var (
	errInvalidResp = terror.ClassXEval.New(codeInvalidResp, "invalid response")
)

var (
	_ SelectResult  = &selectResult{}
	_ PartialResult = &partialResult{}
)

// SelectResult is an iterator of coprocessor partial results.
type SelectResult interface {
	// Next gets the next partial result.
	Next() (PartialResult, error)
	// Close closes the iterator.
	Close() error
	// Fetch fetches partial results from client.
	// The caller should call SetFields() before call Fetch().
	Fetch(ctx goctx.Context)
}

// PartialResult is the result from a single region server.
type PartialResult interface {
	// Next returns the next rowData of the sub result.
	// If no more row to return, rowData would be nil.
	Next() (handle int64, rowData []byte, err error)
	// Close closes the partial result.
	Close() error
}

// SelectResult is used to get response rows from SelectRequest.
type selectResult struct {
	label     string
	aggregate bool
	resp      kv.Response

	results chan resultWithErr
	closed  chan struct{}
}

type resultWithErr struct {
	result PartialResult
	err    error
}

func (r *selectResult) Fetch(ctx goctx.Context) {
	go r.fetch(ctx)
}

func (r *selectResult) fetch(ctx goctx.Context) {
	startTime := time.Now()
	defer func() {
		close(r.results)
		duration := time.Since(startTime)
		queryHistgram.WithLabelValues(r.label).Observe(duration.Seconds())
	}()
	for {
		resultSubset, err := r.resp.Next()
		if err != nil {
			r.results <- resultWithErr{err: errors.Trace(err)}
			return
		}
		if resultSubset == nil {
			return
		}
		pr := &partialResult{}
		pr.unmarshal(resultSubset)

		select {
		case r.results <- resultWithErr{result: pr}:
		case <-r.closed:
			// if selectResult called Close() already, make fetch goroutine exit
			return
		case <-ctx.Done():
			return
		}
	}
}

// Next returns the next row.
func (r *selectResult) Next() (PartialResult, error) {
	re := <-r.results
	return re.result, errors.Trace(re.err)
}

// Close closes SelectResult.
func (r *selectResult) Close() error {
	// close this channel tell fetch goroutine to exit
	close(r.closed)
	return r.resp.Close()
}

// partialResult represents a subset of select result.
type partialResult struct {
	resp       *tipb.SelectResponse
	chunkIdx   int
	cursor     int
	dataOffset int64
}

func (pr *partialResult) unmarshal(resultSubset []byte) error {
	pr.resp = new(tipb.SelectResponse)
	err := pr.resp.Unmarshal(resultSubset)
	if err != nil {
		return errors.Trace(err)
	}

	if pr.resp.Error != nil {
		return errInvalidResp.Gen("[%d %s]", pr.resp.Error.GetCode(), pr.resp.Error.GetMsg())
	}

	return nil
}

// Next returns the next row of the sub result.
// If no more row to return, data would be nil.
func (pr *partialResult) Next() (handle int64, data []byte, err error) {
	chunk := pr.getChunk()
	if chunk == nil {
		return 0, nil, nil
	}
	rowMeta := chunk.RowsMeta[pr.cursor]
	data = chunk.RowsData[pr.dataOffset : pr.dataOffset+rowMeta.Length]
	pr.dataOffset += rowMeta.Length
	handle = rowMeta.Handle
	pr.cursor++
	return
}

func (pr *partialResult) getChunk() *tipb.Chunk {
	for {
		if pr.chunkIdx >= len(pr.resp.Chunks) {
			return nil
		}
		chunk := &pr.resp.Chunks[pr.chunkIdx]
		if pr.cursor < len(chunk.RowsMeta) {
			return chunk
		}
		pr.cursor = 0
		pr.dataOffset = 0
		pr.chunkIdx++
	}
}

// Close closes the sub result.
func (pr *partialResult) Close() error {
	return nil
}

// Select do a select request, returns SelectResult.
// concurrency: The max concurrency for underlying coprocessor request.
// keepOrder: If the result should returned in key order. For example if we need keep data in order by
//            scan index, we should set keepOrder to true.
func Select(client kv.Client, ctx goctx.Context, req *tipb.SelectRequest, keyRanges []kv.KeyRange, concurrency int, keepOrder bool) (SelectResult, error) {
	var err error
	defer func() {
		// Add metrics
		if err != nil {
			queryCounter.WithLabelValues(queryFailed).Inc()
		} else {
			queryCounter.WithLabelValues(querySucc).Inc()
		}
	}()

	// Convert tipb.*Request to kv.Request.
	kvReq, err1 := composeRequest(req, keyRanges, concurrency, keepOrder)
	if err1 != nil {
		err = errors.Trace(err1)
		return nil, err
	}

	resp := client.Send(ctx, kvReq)
	if resp == nil {
		err = errors.New("client returns nil response")
		return nil, err
	}
	result := &selectResult{
		resp:    resp,
		results: make(chan resultWithErr, 5),
		closed:  make(chan struct{}),
	}
	// If Aggregates is not nil, we should set result fields latter.
	if len(req.Aggregates) == 0 && len(req.GroupBy) == 0 {
		if req.TableInfo != nil {
			result.label = "table"
		} else {
			result.label = "index"
		}
	} else {
		result.label = "aggregate"
	}
	return result, nil
}

// Convert tipb.Request to kv.Request.
func composeRequest(req *tipb.SelectRequest, keyRanges []kv.KeyRange, concurrency int, keepOrder bool) (*kv.Request, error) {
	kvReq := &kv.Request{
		Concurrency: concurrency,
		KeepOrder:   keepOrder,
		KeyRanges:   keyRanges,
	}
	if req.IndexInfo != nil {
		kvReq.Tp = kv.ReqTypeIndex
	} else {
		kvReq.Tp = kv.ReqTypeSelect
	}
	if req.OrderBy != nil {
		kvReq.Desc = req.OrderBy[0].Desc
	}
	var err error
	kvReq.Data, err = req.Marshal()
	if err != nil {
		return nil, errors.Trace(err)
	}
	return kvReq, nil
}

// XAPI error codes.
const (
	codeInvalidResp = 1
	codeNilResp     = 2
)

// FieldTypeFromPBColumn creates a types.FieldType from tipb.ColumnInfo.
func FieldTypeFromPBColumn(col *tipb.ColumnInfo) *types.FieldType {
	return &types.FieldType{
		Tp:      byte(col.GetTp()),
		Flen:    int(col.GetColumnLen()),
		Decimal: int(col.GetDecimal()),
		Elems:   col.Elems,
		Collate: mysql.Collations[uint8(col.GetCollation())],
	}
}

func columnToProto(c *model.ColumnInfo) *tipb.ColumnInfo {
	pc := &tipb.ColumnInfo{
		ColumnId:  c.ID,
		Collation: collationToProto(c.FieldType.Collate),
		ColumnLen: int32(c.FieldType.Flen),
		Decimal:   int32(c.FieldType.Decimal),
		Flag:      int32(c.Flag),
		Elems:     c.Elems,
	}
	pc.Tp = int32(c.FieldType.Tp)
	return pc
}

func collationToProto(c string) int32 {
	v, ok := mysql.CollationNames[c]
	if ok {
		return int32(v)
	}
	return int32(mysql.DefaultCollationID)
}

// ColumnsToProto converts a slice of model.ColumnInfo to a slice of tipb.ColumnInfo.
func ColumnsToProto(columns []*model.ColumnInfo, pkIsHandle bool) []*tipb.ColumnInfo {
	cols := make([]*tipb.ColumnInfo, 0, len(columns))
	for _, c := range columns {
		col := columnToProto(c)
		if pkIsHandle && mysql.HasPriKeyFlag(c.Flag) {
			col.PkHandle = true
		} else {
			col.PkHandle = false
		}
		cols = append(cols, col)
	}
	return cols
}

// IndexToProto converts a model.IndexInfo to a tipb.IndexInfo.
func IndexToProto(t *model.TableInfo, idx *model.IndexInfo) *tipb.IndexInfo {
	pi := &tipb.IndexInfo{
		TableId: t.ID,
		IndexId: idx.ID,
		Unique:  idx.Unique,
	}
	cols := make([]*tipb.ColumnInfo, 0, len(idx.Columns)+1)
	for _, c := range idx.Columns {
		cols = append(cols, columnToProto(t.Columns[c.Offset]))
	}
	if t.PKIsHandle {
		// Coprocessor needs to know PKHandle column info, so we need to append it.
		for _, col := range t.Columns {
			if mysql.HasPriKeyFlag(col.Flag) {
				colPB := columnToProto(col)
				colPB.PkHandle = true
				cols = append(cols, colPB)
				break
			}
		}
	}
	pi.Columns = cols
	return pi
}
<commit_msg>distsql: return zeroLenData if data length is nil. (#2965)<commit_after>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package distsql

import (
	"time"

	"github.com/juju/errors"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/model"
	"github.com/pingcap/tidb/mysql"
	"github.com/pingcap/tidb/terror"
	"github.com/pingcap/tidb/util/types"
	"github.com/pingcap/tipb/go-tipb"
	goctx "golang.org/x/net/context"
)

var (
	errInvalidResp = terror.ClassXEval.New(codeInvalidResp, "invalid response")
)

var (
	_ SelectResult  = &selectResult{}
	_ PartialResult = &partialResult{}
)

// SelectResult is an iterator of coprocessor partial results.
type SelectResult interface {
	// Next gets the next partial result.
	Next() (PartialResult, error)
	// Close closes the iterator.
	Close() error
	// Fetch fetches partial results from client.
	// The caller should call SetFields() before call Fetch().
	Fetch(ctx goctx.Context)
}

// PartialResult is the result from a single region server.
type PartialResult interface {
	// Next returns the next rowData of the sub result.
	// If no more row to return, rowData would be nil.
	Next() (handle int64, rowData []byte, err error)
	// Close closes the partial result.
	Close() error
}

// SelectResult is used to get response rows from SelectRequest.
type selectResult struct {
	label     string
	aggregate bool
	resp      kv.Response

	results chan resultWithErr
	closed  chan struct{}
}

type resultWithErr struct {
	result PartialResult
	err    error
}

func (r *selectResult) Fetch(ctx goctx.Context) {
	go r.fetch(ctx)
}

func (r *selectResult) fetch(ctx goctx.Context) {
	startTime := time.Now()
	defer func() {
		close(r.results)
		duration := time.Since(startTime)
		queryHistgram.WithLabelValues(r.label).Observe(duration.Seconds())
	}()
	for {
		resultSubset, err := r.resp.Next()
		if err != nil {
			r.results <- resultWithErr{err: errors.Trace(err)}
			return
		}
		if resultSubset == nil {
			return
		}
		pr := &partialResult{}
		pr.unmarshal(resultSubset)

		select {
		case r.results <- resultWithErr{result: pr}:
		case <-r.closed:
			// if selectResult called Close() already, make fetch goroutine exit
			return
		case <-ctx.Done():
			return
		}
	}
}

// Next returns the next row.
func (r *selectResult) Next() (PartialResult, error) {
	re := <-r.results
	return re.result, errors.Trace(re.err)
}

// Close closes SelectResult.
func (r *selectResult) Close() error {
	// close this channel tell fetch goroutine to exit
	close(r.closed)
	return r.resp.Close()
}

// partialResult represents a subset of select result.
type partialResult struct {
	resp       *tipb.SelectResponse
	chunkIdx   int
	cursor     int
	dataOffset int64
}

func (pr *partialResult) unmarshal(resultSubset []byte) error {
	pr.resp = new(tipb.SelectResponse)
	err := pr.resp.Unmarshal(resultSubset)
	if err != nil {
		return errors.Trace(err)
	}

	if pr.resp.Error != nil {
		return errInvalidResp.Gen("[%d %s]", pr.resp.Error.GetCode(), pr.resp.Error.GetMsg())
	}

	return nil
}

var zeroLenData = make([]byte, 0)

// Next returns the next row of the sub result.
// If no more row to return, data would be nil.
func (pr *partialResult) Next() (handle int64, data []byte, err error) {
	chunk := pr.getChunk()
	if chunk == nil {
		return 0, nil, nil
	}
	rowMeta := chunk.RowsMeta[pr.cursor]
	data = chunk.RowsData[pr.dataOffset : pr.dataOffset+rowMeta.Length]
	if data == nil {
		// The caller checks if data is nil to determine finished.
		data = zeroLenData
	}
	pr.dataOffset += rowMeta.Length
	handle = rowMeta.Handle
	pr.cursor++
	return
}

func (pr *partialResult) getChunk() *tipb.Chunk {
	for {
		if pr.chunkIdx >= len(pr.resp.Chunks) {
			return nil
		}
		chunk := &pr.resp.Chunks[pr.chunkIdx]
		if pr.cursor < len(chunk.RowsMeta) {
			return chunk
		}
		pr.cursor = 0
		pr.dataOffset = 0
		pr.chunkIdx++
	}
}

// Close closes the sub result.
func (pr *partialResult) Close() error {
	return nil
}

// Select do a select request, returns SelectResult.
// concurrency: The max concurrency for underlying coprocessor request.
// keepOrder: If the result should returned in key order. For example if we need keep data in order by
//            scan index, we should set keepOrder to true.
func Select(client kv.Client, ctx goctx.Context, req *tipb.SelectRequest, keyRanges []kv.KeyRange, concurrency int, keepOrder bool) (SelectResult, error) {
	var err error
	defer func() {
		// Add metrics
		if err != nil {
			queryCounter.WithLabelValues(queryFailed).Inc()
		} else {
			queryCounter.WithLabelValues(querySucc).Inc()
		}
	}()

	// Convert tipb.*Request to kv.Request.
	kvReq, err1 := composeRequest(req, keyRanges, concurrency, keepOrder)
	if err1 != nil {
		err = errors.Trace(err1)
		return nil, err
	}

	resp := client.Send(ctx, kvReq)
	if resp == nil {
		err = errors.New("client returns nil response")
		return nil, err
	}
	result := &selectResult{
		resp:    resp,
		results: make(chan resultWithErr, 5),
		closed:  make(chan struct{}),
	}
	// If Aggregates is not nil, we should set result fields latter.
	if len(req.Aggregates) == 0 && len(req.GroupBy) == 0 {
		if req.TableInfo != nil {
			result.label = "table"
		} else {
			result.label = "index"
		}
	} else {
		result.label = "aggregate"
	}
	return result, nil
}

// Convert tipb.Request to kv.Request.
func composeRequest(req *tipb.SelectRequest, keyRanges []kv.KeyRange, concurrency int, keepOrder bool) (*kv.Request, error) {
	kvReq := &kv.Request{
		Concurrency: concurrency,
		KeepOrder:   keepOrder,
		KeyRanges:   keyRanges,
	}
	if req.IndexInfo != nil {
		kvReq.Tp = kv.ReqTypeIndex
	} else {
		kvReq.Tp = kv.ReqTypeSelect
	}
	if req.OrderBy != nil {
		kvReq.Desc = req.OrderBy[0].Desc
	}
	var err error
	kvReq.Data, err = req.Marshal()
	if err != nil {
		return nil, errors.Trace(err)
	}
	return kvReq, nil
}

// XAPI error codes.
const (
	codeInvalidResp = 1
	codeNilResp     = 2
)

// FieldTypeFromPBColumn creates a types.FieldType from tipb.ColumnInfo.
func FieldTypeFromPBColumn(col *tipb.ColumnInfo) *types.FieldType {
	return &types.FieldType{
		Tp:      byte(col.GetTp()),
		Flen:    int(col.GetColumnLen()),
		Decimal: int(col.GetDecimal()),
		Elems:   col.Elems,
		Collate: mysql.Collations[uint8(col.GetCollation())],
	}
}

func columnToProto(c *model.ColumnInfo) *tipb.ColumnInfo {
	pc := &tipb.ColumnInfo{
		ColumnId:  c.ID,
		Collation: collationToProto(c.FieldType.Collate),
		ColumnLen: int32(c.FieldType.Flen),
		Decimal:   int32(c.FieldType.Decimal),
		Flag:      int32(c.Flag),
		Elems:     c.Elems,
	}
	pc.Tp = int32(c.FieldType.Tp)
	return pc
}

func collationToProto(c string) int32 {
	v, ok := mysql.CollationNames[c]
	if ok {
		return int32(v)
	}
	return int32(mysql.DefaultCollationID)
}

// ColumnsToProto converts a slice of model.ColumnInfo to a slice of tipb.ColumnInfo.
func ColumnsToProto(columns []*model.ColumnInfo, pkIsHandle bool) []*tipb.ColumnInfo {
	cols := make([]*tipb.ColumnInfo, 0, len(columns))
	for _, c := range columns {
		col := columnToProto(c)
		if pkIsHandle && mysql.HasPriKeyFlag(c.Flag) {
			col.PkHandle = true
		} else {
			col.PkHandle = false
		}
		cols = append(cols, col)
	}
	return cols
}

// IndexToProto converts a model.IndexInfo to a tipb.IndexInfo.
func IndexToProto(t *model.TableInfo, idx *model.IndexInfo) *tipb.IndexInfo {
	pi := &tipb.IndexInfo{
		TableId: t.ID,
		IndexId: idx.ID,
		Unique:  idx.Unique,
	}
	cols := make([]*tipb.ColumnInfo, 0, len(idx.Columns)+1)
	for _, c := range idx.Columns {
		cols = append(cols, columnToProto(t.Columns[c.Offset]))
	}
	if t.PKIsHandle {
		// Coprocessor needs to know PKHandle column info, so we need to append it.
		for _, col := range t.Columns {
			if mysql.HasPriKeyFlag(col.Flag) {
				colPB := columnToProto(col)
				colPB.PkHandle = true
				cols = append(cols, colPB)
				break
			}
		}
	}
	pi.Columns = cols
	return pi
}
<|endoftext|>


<commit_before>package dnsstub

import (
	"crypto/rand"
	"fmt"
	"github.com/miekg/dns"
	"math/big"
	"net"
	"strings"
	"time"
)

type query struct {
	handle int // identifier to match answer with question
	qname  string
	rtype  uint16
}

type answer struct {
	handle int // identifier to match answer with question
	qname  string
	rtype  uint16
	answer *dns.Msg
	rtt    time.Duration
	err    error
}

type StubResolver struct {
	next_handle      int
	queries          chan *query
	answers          chan *answer
	finished_answers []*answer
}

func RandUint16() (uint16, error) {
	var id_max big.Int
	id_max.SetUint64(65536)
	id, err := rand.Int(rand.Reader, &id_max)
	if err != nil {
		return 0, err
	}
	return uint16(id.Uint64()), nil
}

/*
   Send a query to a DNS server, retrying and handling truncation.
*/
func DnsQuery(server string, query *dns.Msg) (*dns.Msg, time.Duration, error) {
	// try to query first in UDP
	dnsClient := new(dns.Client)
	id, err := RandUint16()
	if err != nil {
		return nil, 0, err
	}
	query.Id = id
	var r *dns.Msg
	var rtt time.Duration
	// try a few times with UDP
	for i := 0; i < 3; i++ {
		r, rtt, err = dnsClient.Exchange(query, server)
		if err != nil {
			if err == dns.ErrTruncated {
				break
			}
			if !err.(net.Error).Timeout() {
				return nil, 0, err
			}
		}
		if (r != nil) && (r.Rcode == dns.RcodeSuccess) {
			if r.Truncated {
				break
			}
			return r, rtt, nil
		}
	}
	// if we got a truncation or timeouts, try again in TCP
	dnsClient.Net = "tcp"
	r, rtt, err = dnsClient.Exchange(query, server)
	if err != nil {
		return nil, 0, err
	}
	// return whatever we get in this case, even if an erroneous response
	return r, rtt, nil
}

func stub_resolve(servers []string, queries <-chan *query, answers chan<- *answer) {
	for q := range queries {
		dns_query := new(dns.Msg)
		dns_query.RecursionDesired = true
		dns_query.SetQuestion(q.qname, q.rtype)
		a := new(answer)
		a.handle = q.handle
		a.qname = q.qname
		a.rtype = q.rtype
		a.answer = nil
		for _, server := range servers {
			// look for ':' because that indicates an IPv6 address
			var resolver string
			if strings.ContainsRune(server, ':') {
				resolver = "[" + server + "]:53"
			} else {
				resolver = server + ":53"
			}
			a.answer, a.rtt, a.err = DnsQuery(resolver, dns_query)
			if a.answer != nil {
				break
			}
		}
		answers <- a
	}
}

func Init(concurrency int, server_ips []net.IP) (resolver *StubResolver, err error) {
	stub := new(StubResolver)
	servers := make([]string, 0, 0)
	for _, ip := range server_ips {
		servers = append(servers, ip.String())
	}
	if len(servers) == 0 {
		resolv_conf, err := dns.ClientConfigFromFile("/etc/resolv.conf")
		if err != nil {
			newerr := fmt.Errorf("error reading resolver configuration from '/etc/resolv.conf'; %s", err)
			return nil, newerr
		}
		servers = resolv_conf.Servers
	}
	stub.queries = make(chan *query, concurrency*4)
	stub.answers = make(chan *answer, concurrency*2)
	for i := 0; i < concurrency; i++ {
		go stub_resolve(servers, stub.queries, stub.answers)
	}
	return stub, nil
}

func (resolver *StubResolver) Query(qname string, rtype uint16) (handle int) {
	q := new(query)
	resolver.next_handle += 1
	q.handle = resolver.next_handle
	q.qname = qname
	q.rtype = rtype
	resolver.queries <- q
	return q.handle
}

func (resolver *StubResolver) Wait() (*dns.Msg, time.Duration, string, uint16, error) {
	var a *answer
	// if we have waiting finished answers, return one of them
	if len(resolver.finished_answers) > 0 {
		a = resolver.finished_answers[0]
		resolver.finished_answers = resolver.finished_answers[1:]
		// otherwise wait for an answer to arrive
	} else {
		a = <-resolver.answers
	}
	return a.answer, a.rtt, a.qname, a.rtype, a.err
}

func (resolver *StubResolver) WaitByHandle(handle int) (*dns.Msg, time.Duration, string, uint16, error) {
	// check any existing finished answers to see if we have ours
	for n, a := range resolver.finished_answers {
		if a.handle == handle {
			resolver.finished_answers = append(resolver.finished_answers[:n],
				resolver.finished_answers[n+1:]...)
			return a.answer, a.rtt, a.qname, a.rtype, a.err
		}
	}
	for {
		a := <-resolver.answers
		if a.handle == handle {
			return a.answer, a.rtt, a.qname, a.rtype, a.err
		}
		resolver.finished_answers = append(resolver.finished_answers, a)
	}
}

func (resolver *StubResolver) Close() {
	close(resolver.queries)
	close(resolver.answers)
}

/*
func main() {
	resolver, err := Init(11, nil)
	if err != nil {
		fmt.Printf("Error! %s\n", err)
		return
	}
	resolver.Query("isc.org.", dns.TypeA)
	sleep_time, _ := time.ParseDuration("1s")
	time.Sleep(sleep_time)	// insure that our non-handle query finishes first
	handle := resolver.Query("isc.org.", dns.TypeAAAA)
	answer, _, _, err := resolver.WaitByHandle(handle)
	fmt.Printf("answer: %s\n", answer)
	answer, _, _, err = resolver.Wait()
	fmt.Printf("answer: %s\n", answer)
	resolver.Close()
}
*/
<commit_msg>Change answers to use condition variable instead of channels.<commit_after>package dnsstub

import (
	"crypto/rand"
	"fmt"
	"github.com/miekg/dns"
	"math/big"
	"net"
	"strings"
	"sync"
	"time"
)

type query struct {
	handle int // identifier to match answer with question
	qname  string
	rtype  uint16
}

type answer struct {
	handle int // identifier to match answer with question
	qname  string
	rtype  uint16
	answer *dns.Msg
	rtt    time.Duration
	err    error
}

type StubResolver struct {
	lock             sync.Mutex
	cond             *sync.Cond
	next_handle      int
	queries          chan *query
	finished_answers []*answer
}

func RandUint16() (uint16, error) {
	var id_max big.Int
	id_max.SetUint64(65536)
	id, err := rand.Int(rand.Reader, &id_max)
	if err != nil {
		return 0, err
	}
	return uint16(id.Uint64()), nil
}

/*
   Send a query to a DNS server, retrying and handling truncation.
*/
func DnsQuery(server string, query *dns.Msg) (*dns.Msg, time.Duration, error) {
	// try to query first in UDP
	dnsClient := new(dns.Client)
	id, err := RandUint16()
	if err != nil {
		return nil, 0, err
	}
	query.Id = id
	var r *dns.Msg
	var rtt time.Duration
	// try a few times with UDP
	for i := 0; i < 3; i++ {
		r, rtt, err = dnsClient.Exchange(query, server)
		if err != nil {
			if err == dns.ErrTruncated {
				break
			}
			if !err.(net.Error).Timeout() {
				return nil, 0, err
			}
		}
		if (r != nil) && (r.Rcode == dns.RcodeSuccess) {
			if r.Truncated {
				break
			}
			return r, rtt, nil
		}
	}
	// if we got a truncation or timeouts, try again in TCP
	dnsClient.Net = "tcp"
	r, rtt, err = dnsClient.Exchange(query, server)
	if err != nil {
		return nil, 0, err
	}
	// return whatever we get in this case, even if an erroneous response
	return r, rtt, nil
}

func stub_resolve(resolver *StubResolver, servers []string) {
	for q := range resolver.queries {
		dns_query := new(dns.Msg)
		dns_query.RecursionDesired = true
		dns_query.SetQuestion(q.qname, q.rtype)
		a := new(answer)
		a.handle = q.handle
		a.qname = q.qname
		a.rtype = q.rtype
		a.answer = nil
		for _, server := range servers {
			// look for ':' because that indicates an IPv6 address
			var resolver string
			if strings.ContainsRune(server, ':') {
				resolver = "[" + server + "]:53"
			} else {
				resolver = server + ":53"
			}
			a.answer, a.rtt, a.err = DnsQuery(resolver, dns_query)
			if a.answer != nil {
				break
			}
		}
		resolver.lock.Lock()
		resolver.finished_answers = append(resolver.finished_answers, a)
		resolver.cond.Broadcast()
		resolver.lock.Unlock()
	}
}

func Init(concurrency int, server_ips []net.IP) (resolver *StubResolver, err error) {
	stub := new(StubResolver)
	var servers []string
	for _, ip := range server_ips {
		servers = append(servers, ip.String())
	}
	if len(servers) == 0 {
		resolv_conf, err := dns.ClientConfigFromFile("/etc/resolv.conf")
		if err != nil {
			newerr := fmt.Errorf("error reading resolver configuration from '/etc/resolv.conf'; %s", err)
			return nil, newerr
		}
		servers = resolv_conf.Servers
	}
	stub.queries = make(chan *query, concurrency*4)
	for i := 0; i < concurrency; i++ {
		go stub_resolve(stub, servers)
	}
	stub.cond = sync.NewCond(&stub.lock)
	return stub, nil
}

func (resolver *StubResolver) AsyncQuery(qname string, rtype uint16) (handle int) {
	q := new(query)
	resolver.lock.Lock()
	resolver.next_handle += 1
	q.handle = resolver.next_handle
	resolver.lock.Unlock()
	q.qname = qname
	q.rtype = rtype
	resolver.queries <- q
	return q.handle
}

func (resolver *StubResolver) Wait() (*dns.Msg, time.Duration, string, uint16, error) {
	resolver.lock.Lock()
	defer resolver.lock.Unlock()

	for len(resolver.finished_answers) == 0 {
		resolver.cond.Wait()
	}

	a := resolver.finished_answers[0]
	resolver.finished_answers = resolver.finished_answers[1:]
	return a.answer, a.rtt, a.qname, a.rtype, a.err
}

// Wait for a specific handle.
// Note that mixing Wait() and WaitByHandle() is dangerous because
// a Wait() may read a result before the WaitByHandle() gets it, so
// it may wait forever.
func (resolver *StubResolver) WaitByHandle(handle int) (*dns.Msg, time.Duration, string, uint16, error) {

	resolver.lock.Lock()
	defer resolver.lock.Unlock()

	for {
		for n, a := range resolver.finished_answers {
			if a.handle == handle {
				resolver.finished_answers = append(resolver.finished_answers[:n],
					resolver.finished_answers[n+1:]...)
				return a.answer, a.rtt, a.qname, a.rtype, a.err
			}
		}
		resolver.cond.Wait()
	}
}

func (resolver *StubResolver) SyncQuery(qname string, rtype uint16) (*dns.Msg, time.Duration, error) {
	handle := resolver.AsyncQuery(qname, rtype)
	answer, rtt, _, _, err := resolver.WaitByHandle(handle)
	return answer, rtt, err
}

func (resolver *StubResolver) Close() {
	close(resolver.queries)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"sort"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	"github.com/gholt/ring"
	pb "github.com/pandemicsyn/syndicate/api/proto"

	"log"
	"net"
	"path/filepath"
	"strings"
)

var (
	printVersionInfo = flag.Bool("version", false, "print version/build info")
)
var syndVersion string
var ringVersion string
var goVersion string
var buildDate string

// FatalIf is just a lazy log/panic on error func
func FatalIf(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %v", msg, err)
	}
}

func Filter(vs []string, f func(string) bool) []string {
	vsf := make([]string, 0)
	for _, v := range vs {
		if f(v) {
			vsf = append(vsf, v)
		}
	}
	return vsf
}

func getRingPaths(cfg *Config) (lastBuilder string, lastRing string, err error) {
	_, err = os.Stat(filepath.Join(cfg.RingDir, "oort.builder"))
	if err != nil {
		//TODO: no active builder found, so should we search for the most recent one
		//we can find and load it and hopefully its matching ring?
		return "", "", fmt.Errorf("No builder file found in %s", cfg.RingDir)
	}
	lastBuilder = filepath.Join(cfg.RingDir, "oort.builder")
	_, err = os.Stat(filepath.Join(cfg.RingDir, "oort.ring"))
	if err != nil {
		//TODO: if we don't find a matching oort.ring should we just
		// use oort.builder to make new one ?
		return "", "", fmt.Errorf("No ring file found in %s", cfg.RingDir)
	}
	lastRing = filepath.Join(cfg.RingDir, "oort.ring")
	return lastBuilder, lastRing, nil
}

func findLastRing(cfg *Config) (lastBuilder string, lastRing string, err error) {
	fp, err := os.Open(cfg.RingDir)
	if err != nil {
		return "", "", err
	}
	names, err := fp.Readdirnames(-1)
	fp.Close()
	if err != nil {
		return "", "", err
	}

	fn := Filter(names, func(v string) bool {
		return strings.HasSuffix(v, "-oort.builder")
	})
	sort.Strings(fn)
	if len(fn) != 0 {
		lastBuilder = filepath.Join(cfg.RingDir, fn[len(fn)-1])
	}

	fn = Filter(names, func(v string) bool {
		return strings.HasSuffix(v, "-oort.ring")
	})
	if len(fn) != 0 {
		lastRing = filepath.Join(cfg.RingDir, fn[len(fn)-1])
	}
	return lastBuilder, lastRing, nil
}

func newRingMgrServer(cfg *Config) (*ringmgr, error) {
	var err error
	s := new(ringmgr)
	s.cfg = cfg

	bfile, rfile, err := getRingPaths(cfg)
	if err != nil {
		panic(err)
	}
	_, s.b, err = ring.RingOrBuilder(bfile)
	FatalIf(err, fmt.Sprintf("Builder file (%s) load failed:", bfile))
	s.r, _, err = ring.RingOrBuilder(rfile)
	FatalIf(err, fmt.Sprintf("Ring file (%s) load failed:", rfile))
	log.Println("Ring version is:", s.r.Version())
	//TODO: verify ring version in bytes matches what we expect
	s.rb, s.bb, err = s.loadRingBuilderBytes(s.r.Version())
	FatalIf(err, "Attempting to load ring/builder bytes")

	for _, v := range cfg.NetFilter {
		_, n, err := net.ParseCIDR(v)
		if err != nil {
			FatalIf(err, "Invalid network range provided")
		}
		s.netlimits = append(s.netlimits, n)
	}
	s.tierlimits = cfg.TierFilter
	s.managedNodes = bootstrapManagedNodes(s.r)
	s.changeChan = make(chan *changeMsg, 1)
	go s.RingChangeManager()
	s.slaves = cfg.Slaves
	if len(s.slaves) == 0 {
		log.Println("!! Running without slaves, have no one to register !!")
		return s, nil
	}

	failcount := 0
	for _, slave := range s.slaves {
		if err = s.RegisterSlave(slave); err != nil {
			log.Println("Got error:", err)
			failcount++
		}
	}
	if failcount > (len(s.slaves) / 2) {
		log.Fatalln("More than half of the ring slaves failed to respond. Exiting.")
	}
	return s, nil
}

func newRingDistServer() *ringslave {
	s := new(ringslave)
	return s
}

func main() {
	flag.Parse()
	if *printVersionInfo {
		fmt.Println("syndicate-client:", syndVersion)
		fmt.Println("ring version:", ringVersion)
		fmt.Println("build date:", buildDate)
		fmt.Println("go version:", goVersion)
		return
	}

	cfg, err := loadConfig("/etc/oort/syndicate.toml")
	if err != nil {
		log.Println(err)
		return
	}
	if cfg.Master {
		l, err := net.Listen("tcp", fmt.Sprintf(":%d", cfg.Port))
		FatalIf(err, "Failed to bind to port")
		var opts []grpc.ServerOption
		if cfg.UseTLS {
			creds, err := credentials.NewServerTLSFromFile(cfg.CertFile, cfg.KeyFile)
			FatalIf(err, "Couldn't load cert from file")
			opts = []grpc.ServerOption{grpc.Creds(creds)}
		}
		s := grpc.NewServer(opts...)

		r, err := newRingMgrServer(cfg)
		FatalIf(err, "Couldn't prep ring mgr server")
		pb.RegisterRingMgrServer(s, r)

		log.Printf("Master starting up on %d...\n", cfg.Port)
		s.Serve(l)
	} else {
		l, err := net.Listen("tcp", fmt.Sprintf(":%d", cfg.Port))
		FatalIf(err, "Failed to bind to port")
		var opts []grpc.ServerOption
		if cfg.UseTLS {
			creds, err := credentials.NewServerTLSFromFile(cfg.CertFile, cfg.KeyFile)
			FatalIf(err, "Couldn't load cert from file")
			opts = []grpc.ServerOption{grpc.Creds(creds)}
		}
		s := grpc.NewServer(opts...)

		pb.RegisterRingDistServer(s, newRingDistServer())
		log.Printf("Starting ring slave up on %d...\n", cfg.Port)
		s.Serve(l)
	}
}
<commit_msg>Fix flag parsing bug<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"sort"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	"github.com/gholt/ring"
	pb "github.com/pandemicsyn/syndicate/api/proto"

	"log"
	"net"
	"path/filepath"
	"strings"
)

var (
	printVersionInfo = flag.Bool("version", false, "print version/build info")
)

var syndVersion string
var ringVersion string
var goVersion string
var buildDate string

// FatalIf is just a lazy log/panic on error func
func FatalIf(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %v", msg, err)
	}
}

func Filter(vs []string, f func(string) bool) []string {
	vsf := make([]string, 0)
	for _, v := range vs {
		if f(v) {
			vsf = append(vsf, v)
		}
	}
	return vsf
}

func getRingPaths(cfg *Config) (lastBuilder string, lastRing string, err error) {
	_, err = os.Stat(filepath.Join(cfg.RingDir, "oort.builder"))
	if err != nil {
		//TODO: no active builder found, so should we search for the most recent one
		//we can find and load it and hopefully its matching ring?
		return "", "", fmt.Errorf("No builder file found in %s", cfg.RingDir)
	}
	lastBuilder = filepath.Join(cfg.RingDir, "oort.builder")
	_, err = os.Stat(filepath.Join(cfg.RingDir, "oort.ring"))
	if err != nil {
		//TODO: if we don't find a matching oort.ring should we just
		// use oort.builder to make new one ?
		return "", "", fmt.Errorf("No ring file found in %s", cfg.RingDir)
	}
	lastRing = filepath.Join(cfg.RingDir, "oort.ring")
	return lastBuilder, lastRing, nil
}

func findLastRing(cfg *Config) (lastBuilder string, lastRing string, err error) {
	fp, err := os.Open(cfg.RingDir)
	if err != nil {
		return "", "", err
	}
	names, err := fp.Readdirnames(-1)
	fp.Close()
	if err != nil {
		return "", "", err
	}

	fn := Filter(names, func(v string) bool {
		return strings.HasSuffix(v, "-oort.builder")
	})
	sort.Strings(fn)
	if len(fn) != 0 {
		lastBuilder = filepath.Join(cfg.RingDir, fn[len(fn)-1])
	}

	fn = Filter(names, func(v string) bool {
		return strings.HasSuffix(v, "-oort.ring")
	})
	if len(fn) != 0 {
		lastRing = filepath.Join(cfg.RingDir, fn[len(fn)-1])
	}
	return lastBuilder, lastRing, nil
}

func newRingMgrServer(cfg *Config) (*ringmgr, error) {
	var err error
	s := new(ringmgr)
	s.cfg = cfg

	bfile, rfile, err := getRingPaths(cfg)
	if err != nil {
		panic(err)
	}
	_, s.b, err = ring.RingOrBuilder(bfile)
	FatalIf(err, fmt.Sprintf("Builder file (%s) load failed:", bfile))
	s.r, _, err = ring.RingOrBuilder(rfile)
	FatalIf(err, fmt.Sprintf("Ring file (%s) load failed:", rfile))
	log.Println("Ring version is:", s.r.Version())
	//TODO: verify ring version in bytes matches what we expect
	s.rb, s.bb, err = s.loadRingBuilderBytes(s.r.Version())
	FatalIf(err, "Attempting to load ring/builder bytes")

	for _, v := range cfg.NetFilter {
		_, n, err := net.ParseCIDR(v)
		if err != nil {
			FatalIf(err, "Invalid network range provided")
		}
		s.netlimits = append(s.netlimits, n)
	}
	s.tierlimits = cfg.TierFilter
	s.managedNodes = bootstrapManagedNodes(s.r)
	s.changeChan = make(chan *changeMsg, 1)
	go s.RingChangeManager()
	s.slaves = cfg.Slaves
	if len(s.slaves) == 0 {
		log.Println("!! Running without slaves, have no one to register !!")
		return s, nil
	}

	failcount := 0
	for _, slave := range s.slaves {
		if err = s.RegisterSlave(slave); err != nil {
			log.Println("Got error:", err)
			failcount++
		}
	}
	if failcount > (len(s.slaves) / 2) {
		log.Fatalln("More than half of the ring slaves failed to respond. Exiting.")
	}
	return s, nil
}

func newRingDistServer() *ringslave {
	s := new(ringslave)
	return s
}

func main() {
	cfg, err := loadConfig("/etc/oort/syndicate.toml")
	if err != nil {
		log.Println(err)
		return
	}
	if *printVersionInfo {
		fmt.Println("syndicate-client:", syndVersion)
		fmt.Println("ring version:", ringVersion)
		fmt.Println("build date:", buildDate)
		fmt.Println("go version:", goVersion)
		return
	}
	if cfg.Master {
		l, err := net.Listen("tcp", fmt.Sprintf(":%d", cfg.Port))
		FatalIf(err, "Failed to bind to port")
		var opts []grpc.ServerOption
		if cfg.UseTLS {
			creds, err := credentials.NewServerTLSFromFile(cfg.CertFile, cfg.KeyFile)
			FatalIf(err, "Couldn't load cert from file")
			opts = []grpc.ServerOption{grpc.Creds(creds)}
		}
		s := grpc.NewServer(opts...)

		r, err := newRingMgrServer(cfg)
		FatalIf(err, "Couldn't prep ring mgr server")
		pb.RegisterRingMgrServer(s, r)

		log.Printf("Master starting up on %d...\n", cfg.Port)
		s.Serve(l)
	} else {
		l, err := net.Listen("tcp", fmt.Sprintf(":%d", cfg.Port))
		FatalIf(err, "Failed to bind to port")
		var opts []grpc.ServerOption
		if cfg.UseTLS {
			creds, err := credentials.NewServerTLSFromFile(cfg.CertFile, cfg.KeyFile)
			FatalIf(err, "Couldn't load cert from file")
			opts = []grpc.ServerOption{grpc.Creds(creds)}
		}
		s := grpc.NewServer(opts...)

		pb.RegisterRingDistServer(s, newRingDistServer())
		log.Printf("Starting ring slave up on %d...\n", cfg.Port)
		s.Serve(l)
	}
}
<|endoftext|>


<commit_before>/*
Package detour provides a net.Conn interface which detects blockage
of a site automatically and access it through alternative dialer.

Basically, if a site is not whitelisted, following steps will be taken:
1. Dial proxied dialer a small delay after dialed directly
2. Return to caller if any connection is established
3. Read/write through all connections in parallel
4. Check for blocking in direct connection and closes it if it happens
5. After sucessfully read from a connection, stick with it and close others.
*/
package detour

import (
	"bytes"
	"fmt"
	"io"
	"net"
	"runtime/debug"
	"sync/atomic"
	"time"

	"github.com/getlantern/golog"
)

var TimeoutToConnect = 30 * time.Second

// To avoid unnecessarily proxy not-blocked url, detour will dial proxy connection
// after this small delay
var DelayBeforeDetour = 1 * time.Second

// if DirectAddrCh is set, when a direct connection is closed without any error,
// the connection's remote address (in host:port format) will be send to it
var DirectAddrCh chan string

var (
	log = golog.LoggerFor("detour")
)

type dialFunc func(network, addr string) (net.Conn, error)

type Conn struct {
	// the underlie connections, uses buffered channel as ring queue.
	conns chan conn

	// the chan to receive result of any read operation
	chRead chan ioResult
	// the chan to receive result of any write operation
	chWrite chan ioResult
	// the chan to stop reading/writing when Close() is called
	chClose chan interface{}

	// keep track of the total bytes read from this connection, atomic
	readBytes uint64

	network, addr string
}

// The data structure to pass result of io operation back from underlie connection
type ioResult struct {
	// number of bytes read/wrote
	n int
	// io error, if any
	err error
	// the underlie connection itself
	conn conn
}

type connType int

const (
	connTypeDirect connType = iota
	connTypeDetour connType = iota
)

type conn interface {
	ConnType() connType
	FirstRead(b []byte, ch chan ioResult)
	FollowupRead(b []byte, ch chan ioResult)
	Write(b []byte, ch chan ioResult)
	Close()
}

func typeOf(c conn) string {
	var connTypeDesc = []string{"direct", "detour"}
	return connTypeDesc[c.ConnType()]
}

// Dialer returns a function with same signature of net.Dialer.Dial().
func Dialer(detourDialer dialFunc) dialFunc {
	return func(network, addr string) (net.Conn, error) {
		dc := &Conn{network: network, addr: addr, conns: make(chan conn, 2), chRead: make(chan ioResult), chWrite: make(chan ioResult), chClose: make(chan interface{}, 2)}
		// use buffered channel, as we may send twice to it but only receive once
		chAnyConn := make(chan bool, 1)
		ch := make(chan conn)
		loopCount := 2
		if whitelisted(addr) {
			loopCount = 1
			DialDetour(network, addr, detourDialer, ch)
		} else {
			go func() {
				DialDirect(network, addr, ch)
				time.Sleep(DelayBeforeDetour)
				DialDetour(network, addr, detourDialer, ch)
			}()
		}
		go func() {
			t := time.NewTimer(TimeoutToConnect)
			defer t.Stop()
			for i := 0; i < loopCount; i++ {
				select {
				case c := <-ch:
					if dc.anyDataReceived() {
						log.Debugf("%s connection to %s established too late, close it", typeOf(c), dc.addr)
						c.Close()
						return
					}
					dc.conns <- c
					chAnyConn <- true
				case <-t.C:
					// still no connection made
					chAnyConn <- false
					return
				}
			}
		}()
		// return to caller if any connection available
		if anyConn := <-chAnyConn; anyConn {
			return dc, nil
		}
		return nil, fmt.Errorf("Timeout dialing any connection to %s", addr)
	}
}

func (dc *Conn) anyDataReceived() bool {
	return atomic.LoadUint64(&dc.readBytes) > 0
}

func (dc *Conn) incReadBytes(n int) {
	atomic.AddUint64(&dc.readBytes, uint64(n))
}

// Read() implements the function from net.Conn
func (dc *Conn) Read(b []byte) (n int, err error) {
	log.Tracef("Initiate a read request to %s", dc.addr)
	if dc.anyDataReceived() {
		return dc.followupRead(b)
	}
	for {
		select {
		case conn := <-dc.conns:
			conn.FirstRead(b, dc.chRead)
		case result := <-dc.chRead:
			n, err = result.n, result.err
			if err != nil && err != io.EOF {
				log.Tracef("Read from %s connection to %s failed: %s", typeOf(result.conn), dc.addr, err)
				// skip failed connection
				if len(dc.conns) > 0 {
					continue
				}
				// no more connections, return directly to avoid dead lock
				return n, err
			}
			log.Tracef("Read %d bytes from %s connection to %s", n, typeOf(result.conn), dc.addr)
			dc.incReadBytes(n)
			for i := 0; i < len(dc.conns); i++ {
				c := <-dc.conns
				log.Tracef("Close %s connection to %s", typeOf(c), dc.addr)
				c.Close()
				// direct connection failed
				if c.ConnType() == connTypeDirect {
					log.Tracef("Add %s to whitelist", dc.addr)
					AddToWl(dc.addr, false)
				}
			}
			dc.conns <- result.conn
			return n, err
		}
	}
	return
}

// followUpRead is called by Read() if a connection's state already settled
func (dc *Conn) followupRead(b []byte) (n int, err error) {
	conn := <-dc.conns
	conn.FollowupRead(b, dc.chRead)
	dc.conns <- conn
	result := <-dc.chRead
	dc.incReadBytes(result.n)
	return result.n, result.err
}

// Write() implements the function from net.Conn
func (dc *Conn) Write(b []byte) (n int, err error) {
	log.Tracef("Initiate a write request to %s", dc.addr)
	if dc.anyDataReceived() {
		return dc.followUpWrite(b)
	}
	for {
		select {
		case conn := <-dc.conns:
			if isNonIdempotentRequest(b) {
				dc.conns <- conn
				dc.writeNonIdeomponent(b)
			} else {
				conn.Write(b, dc.chRead)
				dc.conns <- conn
			}
		case result := <-dc.chWrite:
			if n, err = result.n, result.err; err != nil {
				log.Tracef("Error writing %s connection to %s: %s", typeOf(result.conn), dc.addr, err)
				if len(dc.conns) == 0 {
					return
				}
				result.conn.Close()
				continue
			}
			log.Tracef("Wrote %d bytes to %s connection to %s", n, typeOf(result.conn), dc.addr)
			return
		}
	}
	return
}

func (dc *Conn) writeNonIdeomponent(b []byte) {
	log.Tracef("For non ideomponent operation to %s, try write directly first", dc.addr)
	for len(dc.conns) > 0 {
		conn := <-dc.conns
		if conn.ConnType() == connTypeDirect {
			conn.Write(b, dc.chWrite)
			dc.conns <- conn
			return
		}
		dc.conns <- conn
	}
	log.Tracef("No valid direct connection to %s, write to other (detour)", dc.addr)
	for len(dc.conns) > 0 {
		conn := <-dc.conns
		conn.Write(b, dc.chWrite)
		dc.conns <- conn
		return
	}
	return
}

// followUpWrite is called by Write() if a connection's state already settled
func (dc *Conn) followUpWrite(b []byte) (n int, err error) {
	conn := <-dc.conns
	conn.Write(b, dc.chWrite)
	dc.conns <- conn
	result := <-dc.chWrite
	return result.n, result.err
}

// Close() implements the function from net.Conn
func (dc *Conn) Close() error {
	log.Tracef("Closing connection to %s", dc.addr)
	debug.PrintStack()
	dc.chClose <- nil
	dc.chClose <- nil
	for len(dc.conns) > 0 {
		conn := <-dc.conns
		conn.Close()
	}
	return nil
}

// LocalAddr() implements the function from net.Conn
func (dc *Conn) LocalAddr() net.Addr {
	return nil
}

// RemoteAddr() implements the function from net.Conn
func (dc *Conn) RemoteAddr() net.Addr {
	return nil
}

// SetDeadline() implements the function from net.Conn
func (dc *Conn) SetDeadline(t time.Time) error {
	dc.SetReadDeadline(t)
	dc.SetWriteDeadline(t)
	return nil
}

// SetReadDeadline() implements the function from net.Conn
func (dc *Conn) SetReadDeadline(t time.Time) error {
	return nil
}

// SetWriteDeadline() implements the function from net.Conn
func (dc *Conn) SetWriteDeadline(t time.Time) error {
	return nil
}

var nonIdempotentMethods = [][]byte{
	[]byte("POST "),
	[]byte("PATCH "),
}

// ref section 9.1.2 of https://www.ietf.org/rfc/rfc2616.txt.
// checks against non-idemponent methods actually,
// as we consider the https handshake phase to be idemponent.
func isNonIdempotentRequest(b []byte) bool {
	if len(b) > 4 {
		for _, m := range nonIdempotentMethods {
			if bytes.HasPrefix(b, m) {
				return true
			}
		}
	}
	return false
}
<commit_msg>incomplete<commit_after>/*
Package detour provides a net.Conn interface which detects blockage
of a site automatically and access it through alternative dialer.

Basically, if a site is not whitelisted, following steps will be taken:
1. Dial proxied dialer a small delay after dialed directly
2. Return to caller if any connection is established
3. Read/write through all connections in parallel
4. Check for blocking in direct connection and closes it if it happens
5. After sucessfully read from a connection, stick with it and close others.
*/
package detour

import (
	"bytes"
	"fmt"
	"io"
	"net"
	"sync/atomic"
	"time"

	"github.com/getlantern/golog"
)

var TimeoutToConnect = 30 * time.Second

// To avoid unnecessarily proxy not-blocked url, detour will dial proxy connection
// after this small delay
var DelayBeforeDetour = 1 * time.Second

// if DirectAddrCh is set, when a direct connection is closed without any error,
// the connection's remote address (in host:port format) will be send to it
var DirectAddrCh chan string

var (
	log = golog.LoggerFor("detour")
)

type dialFunc func(network, addr string) (net.Conn, error)

type Conn struct {
	// the underlie connections, uses buffered channel as ring queue.
	conns chan conn
	// the channel to notify read/write than a new connection is available
	chDetourConn chan conn

	// the chan to receive result of any read operation
	chRead chan ioResult
	// the chan to receive result of any write operation
	chWrite chan ioResult
	// the chan to stop reading/writing when Close() is called
	chClose chan interface{}

	// keep track of the total bytes read from this connection, atomic
	readBytes uint64

	network, addr string

	writeBuffer *bytes.Buffer
}

// The data structure to pass result of io operation back from underlie connection
type ioResult struct {
	// number of bytes read/wrote
	n int
	// io error, if any
	err error
	// the underlie connection itself
	conn conn
}

type connType int

const (
	connTypeDirect connType = iota
	connTypeDetour connType = iota
)

type conn interface {
	ConnType() connType
	FirstRead(b []byte, ch chan ioResult)
	FollowupRead(b []byte, ch chan ioResult)
	Write(b []byte, ch chan ioResult)
	Close()
}

func typeOf(c conn) string {
	var connTypeDesc = []string{"direct", "detour"}
	return connTypeDesc[c.ConnType()]
}

// Dialer returns a function with same signature of net.Dialer.Dial().
func Dialer(detourDialer dialFunc) dialFunc {
	return func(network, addr string) (net.Conn, error) {
		var b []byte
		dc := &Conn{
			network:      network,
			addr:         addr,
			writeBuffer:  bytes.NewBuffer(b),
			conns:        make(chan conn, 2),
			chDetourConn: make(chan conn, 1),
			chRead:       make(chan ioResult),
			chWrite:      make(chan ioResult),
			chClose:      make(chan interface{}, 2),
		}
		// use buffered channel, as we may send twice to it but only receive once
		chAnyConn := make(chan bool, 1)
		ch := make(chan conn)
		var loopCount uint32 = 1
		if whitelisted(addr) {
			DialDetour(network, addr, detourDialer, ch)
		} else {
			go func() {
				DialDirect(network, addr, ch)
				time.Sleep(DelayBeforeDetour)
				if dc.anyDataReceived() {
					atomic.AddUint32(&loopCount, 1)
					DialDetour(network, addr, detourDialer, ch)
				}
			}()
		}
		go func() {
			t := time.NewTimer(TimeoutToConnect)
			defer t.Stop()
			for i := 0; uint32(i) < atomic.LoadUint32(&loopCount); i++ {
				select {
				case c := <-ch:
					if i == 0 {
						dc.conns <- c
						chAnyConn <- true
					} else {
						if c.ConnType() == connTypeDirect || dc.anyDataReceived() {
							log.Debugf("%s connection to %s established too late, close it", typeOf(c), dc.addr)
							c.Close()
							return
						}
						dc.chDetourConn <- c
					}
				case <-t.C:
					// still no connection made
					chAnyConn <- false
					return
				}
			}
		}()
		// return to caller if any connection available
		if anyConn := <-chAnyConn; anyConn {
			return dc, nil
		}
		return nil, fmt.Errorf("Timeout dialing any connection to %s", addr)
	}
}

func (dc *Conn) anyDataReceived() bool {
	return atomic.LoadUint64(&dc.readBytes) > 0
}

func (dc *Conn) incReadBytes(n int) {
	atomic.AddUint64(&dc.readBytes, uint64(n))
}

// Read() implements the function from net.Conn
func (dc *Conn) Read(b []byte) (n int, err error) {
	log.Tracef("Initiate a read request to %s", dc.addr)
	if dc.anyDataReceived() {
		return dc.followupRead(b)
	}
	conn := <-dc.conns
	conn.FirstRead(b, dc.chRead)
	dc.conns <- conn
	for count := 1; count > 0; count-- {
		select {
		case newConn := <-dc.chDetourConn:
			newConn.FirstRead(b, dc.chRead)
			count++
			dc.chDetourConn <- newConn
		case result := <-dc.chRead:
			log.Tracef("Read back from %s connection", typeOf(result.conn))
			n, err = result.n, result.err
			if err != nil && err != io.EOF {
				log.Tracef("Read from %s connection to %s failed: %s", typeOf(result.conn), dc.addr, err)
				// skip failed connection
				if count > 1 {
					continue
				}
				// no more connections, return directly to avoid dead lock
				return n, err
			}
			log.Tracef("Read %d bytes from %s connection to %s", n, typeOf(result.conn), dc.addr)
			dc.incReadBytes(n)
			return n, err
		}
	}
	return
}

// followUpRead is called by Read() if a connection's state already settled
func (dc *Conn) followupRead(b []byte) (n int, err error) {
	conn := <-dc.conns
	conn.FollowupRead(b, dc.chRead)
	dc.conns <- conn
	result := <-dc.chRead
	dc.incReadBytes(result.n)
	return result.n, result.err
}

// Write() implements the function from net.Conn
func (dc *Conn) Write(b []byte) (n int, err error) {
	log.Tracef("Initiate a write request to %s", dc.addr)
	if dc.anyDataReceived() {
		return dc.followUpWrite(b)
	}
	dc.writeBuffer.Write(b)
	conn := <-dc.conns
	conn.Write(b, dc.chWrite)
	dc.conns <- conn
	for count := 1; count > 0; count-- {
		select {
		case c := <-dc.chDetourConn:
			if isNonIdempotentRequest(b) {
				c.Close()
				return
			}
			count++
			c.Write(dc.writeBuffer.Bytes(), dc.chWrite)
			// add new connection to connections
			dc.conns <- c
			dc.chDetourConn <- c
		case result := <-dc.chWrite:
			if n, err = result.n, result.err; err != nil {
				log.Tracef("Error writing %s connection to %s: %s", typeOf(result.conn), dc.addr, err)
				result.conn.Close()
				if count > 0 {
					continue
				}
				return
			}
			log.Tracef("Wrote %d bytes to %s connection to %s", n, typeOf(result.conn), dc.addr)
			return
		}
	}
	return
}

// followUpWrite is called by Write() if a connection's state already settled
func (dc *Conn) followUpWrite(b []byte) (n int, err error) {
	conn := <-dc.conns
	conn.Write(b, dc.chWrite)
	dc.conns <- conn
	result := <-dc.chWrite
	return result.n, result.err
}

// Close() implements the function from net.Conn
func (dc *Conn) Close() error {
	log.Tracef("Closing connection to %s", dc.addr)
	dc.chClose <- nil
	dc.chClose <- nil
	for len(dc.conns) > 0 {
		conn := <-dc.conns
		conn.Close()
	}
	return nil
}

// LocalAddr() implements the function from net.Conn
func (dc *Conn) LocalAddr() net.Addr {
	return nil
}

// RemoteAddr() implements the function from net.Conn
func (dc *Conn) RemoteAddr() net.Addr {
	return nil
}

// SetDeadline() implements the function from net.Conn
func (dc *Conn) SetDeadline(t time.Time) error {
	dc.SetReadDeadline(t)
	dc.SetWriteDeadline(t)
	return nil
}

// SetReadDeadline() implements the function from net.Conn
func (dc *Conn) SetReadDeadline(t time.Time) error {
	return nil
}

// SetWriteDeadline() implements the function from net.Conn
func (dc *Conn) SetWriteDeadline(t time.Time) error {
	return nil
}

var nonIdempotentMethods = [][]byte{
	[]byte("POST "),
	[]byte("PATCH "),
}

// ref section 9.1.2 of https://www.ietf.org/rfc/rfc2616.txt.
// checks against non-idemponent methods actually,
// as we consider the https handshake phase to be idemponent.
func isNonIdempotentRequest(b []byte) bool {
	if len(b) > 4 {
		for _, m := range nonIdempotentMethods {
			if bytes.HasPrefix(b, m) {
				return true
			}
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package pendingheap

import (
	"go.uber.org/yarpc/api/peer"
	"go.uber.org/yarpc/yarpcconfig"
)

// Spec returns a configuration specification for the round-robin peer list
// implementation, making it possible to select the least recently chosen peer
// with transports that use outbound peer list configuration (like HTTP).
//
//  cfg := yarpcconfig.New()
//  cfg.MustRegisterPeerList(pendingheap.Spec())
//
// This enables the round-robin peer list:
//
//  outbounds:
//    otherservice:
//      unary:
//        http:
//          url: https://host:port/rpc
//          fewest-pending-requests:
//            peers:
//              - 127.0.0.1:8080
//              - 127.0.0.1:8081
func Spec() yarpcconfig.PeerListSpec {
	return yarpcconfig.PeerListSpec{
		Name: "fewest-pending-requests",
		BuildPeerList: func(c struct{}, t peer.Transport, k *yarpcconfig.Kit) (peer.ChooserList, error) {
			return New(t), nil
		},
	}
}
<commit_msg>Remove round-robin references in pendingheap documentation (#1503)<commit_after>// Copyright (c) 2018 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package pendingheap

import (
	"go.uber.org/yarpc/api/peer"
	"go.uber.org/yarpc/yarpcconfig"
)

// Spec returns a configuration specification for the pending heap peer list
// implementation, making it possible to select the least recently chosen peer
// with transports that use outbound peer list configuration (like HTTP).
//
//  cfg := yarpcconfig.New()
//  cfg.MustRegisterPeerList(pendingheap.Spec())
//
// This enables the pending heap peer list:
//
//  outbounds:
//    otherservice:
//      unary:
//        http:
//          url: https://host:port/rpc
//          fewest-pending-requests:
//            peers:
//              - 127.0.0.1:8080
//              - 127.0.0.1:8081
func Spec() yarpcconfig.PeerListSpec {
	return yarpcconfig.PeerListSpec{
		Name: "fewest-pending-requests",
		BuildPeerList: func(c struct{}, t peer.Transport, k *yarpcconfig.Kit) (peer.ChooserList, error) {
			return New(t), nil
		},
	}
}
<|endoftext|>


<commit_before>package permissions

import "testing"

func TestPerm(t *testing.T) {
	userstate := NewUserStateSimple()

	userstate.AddUser("bob", "hunter1", "bob@zombo.com")

	if !userstate.HasUser("bob") {
		t.Error("Error, user bob should exist")
	}

	if userstate.IsConfirmed("bob") {
		t.Error("Error, user bob should not be confirmed right now.")
	}

	userstate.MarkConfirmed("bob")

	if !userstate.IsConfirmed("bob") {
		t.Error("Error, user bob should be marked as confirmed right now.")
	}

	if userstate.IsAdmin("bob") {
		t.Error("Error, user bob should not have admin rights")
	}

	userstate.SetAdminStatus("bob")

	if !userstate.IsAdmin("bob") {
		t.Error("Error, user bob should have admin rights")
	}

	userstate.RemoveUser("bob")

	if userstate.HasUser("bob") {
		t.Error("Error, user bob should not exist")
	}
}

func TestPasswordBasic(t *testing.T) {
	userstate := NewUserStateSimple()

	// assert default password aglo is sha256
	if userstate.GetPasswordAlgo() != "sha256" {
		t.Error("Error, sha256 should be default password algorithm")
	}

	// set password algo
	userstate.SetPasswordAlgo("bcrypt")

	// assert change should be bcrypt
	if userstate.GetPasswordAlgo() != "bcrypt" {
		t.Error("Error, setting password algorithm failed")
	}

}

func TestPasswordAlgoMatching(t *testing.T) {
	userstate := NewUserStateSimple()
	// generate two different password using the same credentials but different algos
	userstate.SetPasswordAlgo("sha256")
	sha256_hash := userstate.HashPassword("testuser@example.com", "textpassword")
	userstate.SetPasswordAlgo("bcrypt")
	bcrypt_hash := userstate.HashPassword("testuser@example.com", "textpassword")

	// they shouldn't match
	if sha256_hash == bcrypt_hash {
		t.Error("Error, different algorithms should not have a password match")
	}
}
<commit_msg>fix: updated failing Password test<commit_after>package permissions

import "testing"

func TestPerm(t *testing.T) {
	userstate := NewUserStateSimple()

	userstate.AddUser("bob", "hunter1", "bob@zombo.com")

	if !userstate.HasUser("bob") {
		t.Error("Error, user bob should exist")
	}

	if userstate.IsConfirmed("bob") {
		t.Error("Error, user bob should not be confirmed right now.")
	}

	userstate.MarkConfirmed("bob")

	if !userstate.IsConfirmed("bob") {
		t.Error("Error, user bob should be marked as confirmed right now.")
	}

	if userstate.IsAdmin("bob") {
		t.Error("Error, user bob should not have admin rights")
	}

	userstate.SetAdminStatus("bob")

	if !userstate.IsAdmin("bob") {
		t.Error("Error, user bob should have admin rights")
	}

	userstate.RemoveUser("bob")

	if userstate.HasUser("bob") {
		t.Error("Error, user bob should not exist")
	}
}

func TestPasswordBasic(t *testing.T) {
	userstate := NewUserStateSimple()

	// assert default password aglo is sha256
	if userstate.GetPasswordAlgo() != "bcrypt" {
		t.Error("Error, bcrypt should be default password algorithm")
	}

	// set password algo
	userstate.SetPasswordAlgo("sha256")

	// assert change should be bcrypt
	if userstate.GetPasswordAlgo() != "sha256" {
		t.Error("Error, setting password algorithm failed")
	}

}

func TestPasswordAlgoMatching(t *testing.T) {
	userstate := NewUserStateSimple()
	// generate two different password using the same credentials but different algos
	userstate.SetPasswordAlgo("sha256")
	sha256_hash := userstate.HashPassword("testuser@example.com", "textpassword")
	userstate.SetPasswordAlgo("bcrypt")
	bcrypt_hash := userstate.HashPassword("testuser@example.com", "textpassword")

	// they shouldn't match
	if sha256_hash == bcrypt_hash {
		t.Error("Error, different algorithms should not have a password match")
	}
}
<|endoftext|>


<commit_before>package permissions

import "testing"

func TestPerm(t *testing.T) {
	userstate := NewUserStateSimple()

	userstate.AddUser("bob", "hunter1", "bob@zombo.com")

	if !userstate.HasUser("bob") {
		t.Error("Error, user bob should exist")
	}

	if userstate.IsConfirmed("bob") {
		t.Error("Error, user bob should not be confirmed right now.")
	}

	userstate.MarkConfirmed("bob")

	if !userstate.IsConfirmed("bob") {
		t.Error("Error, user bob should be marked as confirmed right now.")
	}

	if userstate.IsAdmin("bob") {
		t.Error("Error, user bob should not have admin rights")
	}

	userstate.SetAdminStatus("bob")

	if !userstate.IsAdmin("bob") {
		t.Error("Error, user bob should have admin rights")
	}

	userstate.RemoveUser("bob")

	if userstate.HasUser("bob") {
		t.Error("Error, user bob should not exist")
	}
}

func TestPasswordBasic(t *testing.T) {
	userstate := NewUserStateSimple()

	// assert that the default password algorithm is bcrypt
	if userstate.PasswordAlgo() != "bcrypt" {
		t.Error("Error, bcrypt should be the default password algorithm")
	}

	// set password algo
	userstate.SetPasswordAlgo("sha256")

	// assert change should be sha256
	if userstate.PasswordAlgo() != "sha256" {
		t.Error("Error, setting password algorithm failed")
	}

}

func TestPasswordAlgoMatching(t *testing.T) {
	userstate := NewUserStateSimple()
	// generate two different password using the same credentials but different algos
	userstate.SetPasswordAlgo("sha256")
	sha256_hash := userstate.HashPassword("testuser@example.com", "textpassword")
	userstate.SetPasswordAlgo("bcrypt")
	bcrypt_hash := userstate.HashPassword("testuser@example.com", "textpassword")

	// they shouldn't match
	if sha256_hash == bcrypt_hash {
		t.Error("Error, different algorithms should not have a password match")
	}
}

func TestUserStateKeeper(t *testing.T) {
	userstate := NewUserStateSimple()
	// Check that the userstate qualifies for the UserStateKeeper interface
	var _ UserStateKeeper = userstate
}
<commit_msg>Minor changes<commit_after>package permissions

import "testing"

func TestPerm(t *testing.T) {
	userstate := NewUserStateSimple()

	userstate.AddUser("bob", "hunter1", "bob@zombo.com")

	if !userstate.HasUser("bob") {
		t.Error("Error, user bob should exist")
	}

	if userstate.IsConfirmed("bob") {
		t.Error("Error, user bob should not be confirmed right now.")
	}

	userstate.MarkConfirmed("bob")

	if !userstate.IsConfirmed("bob") {
		t.Error("Error, user bob should be marked as confirmed right now.")
	}

	if userstate.IsAdmin("bob") {
		t.Error("Error, user bob should not have admin rights")
	}

	userstate.SetAdminStatus("bob")

	if !userstate.IsAdmin("bob") {
		t.Error("Error, user bob should have admin rights")
	}

	userstate.RemoveUser("bob")

	if userstate.HasUser("bob") {
		t.Error("Error, user bob should not exist")
	}
}

func TestPasswordBasic(t *testing.T) {
	userstate := NewUserStateSimple()

	// Assert that the default password algorithm is bcrypt
	if userstate.PasswordAlgo() != "bcrypt" {
		t.Error("Error, bcrypt should be the default password algorithm")
	}

	// Set password algorithm
	userstate.SetPasswordAlgo("sha256")

	// Assert that the algorithm is now sha256
	if userstate.PasswordAlgo() != "sha256" {
		t.Error("Error, setting password algorithm failed")
	}

}

func TestPasswordAlgoMatching(t *testing.T) {
	userstate := NewUserStateSimple()
	// generate two different password using the same credentials but different algos
	userstate.SetPasswordAlgo("sha256")
	sha256_hash := userstate.HashPassword("testuser@example.com", "textpassword")
	userstate.SetPasswordAlgo("bcrypt")
	bcrypt_hash := userstate.HashPassword("testuser@example.com", "textpassword")

	// they shouldn't match
	if sha256_hash == bcrypt_hash {
		t.Error("Error, different algorithms should not have a password match")
	}
}

func TestUserStateKeeper(t *testing.T) {
	userstate := NewUserStateSimple()
	// Check that the userstate qualifies for the UserStateKeeper interface
	var _ UserStateKeeper = userstate
}
<|endoftext|>