<commit_before>package record

// /CdsXSnhGV0TQ9B3VZ9IneNK4vk+K9k+

import (
	"encoding/base64"
	"encoding/json"
	"github.com/revel/revel"
	"io/ioutil"
	"replay/app/models/history"
	"strconv"
	"time"
)

var platformURLs map[string]string = map[string]string{
	"NA1":  "http://spectator.na.lol.riotgames.com:80",
	"OC1":  "http://spectator.oc1.lol.riotgames.com:80",
	"EUN1": "http://spectator.eu.lol.riotgames.com:8088",
	"EUW1": "http://spectator.euw1.lol.riotgames.com:80",
}

var version string // Version functions in getters.go
var recording map[string]map[string]string = make(map[string]map[string]string)

func writeRecording(region, gameId, key string, value []byte) {
	currentRecording := recording[region+":"+gameId]
	currentRecording[key] = base64.URLEncoding.EncodeToString(value)
	recording[region+":"+gameId] = currentRecording
}

func writeString(region, gameId, key string, value string) {
	currentRecording := recording[region+":"+gameId]
	currentRecording[key] = value
	recording[region+":"+gameId] = currentRecording
}

func existsRecording(region, gameId, key string) bool {
	if _, exists := recording[region+":"+gameId][key]; exists {
		return true
	}
	return false
}

func writeLastChunkInfo(region, gameId string,
	firstChunk, firstKeyFrame int, chunk ChunkInfo) {

	writeChunk := ChunkInfo{
		NextChunk:       firstChunk,
		CurrentChunk:    firstChunk,
		NextUpdate:      3000,
		StartGameChunk:  chunk.StartGameChunk,
		CurrentKeyFrame: firstKeyFrame,
		EndGameChunk:    chunk.CurrentChunk,
		AvailableSince:  0,
		Duration:        3000,
		EndStartupChunk: chunk.EndStartupChunk,
	}

	result, err := json.Marshal(writeChunk)
	if err != nil {
		panic("Error while encoding first chunk data json?!??!??")
	}

	writeRecording(region, gameId, "firstChunkData", result)

	writeChunk.NextChunk = chunk.CurrentChunk
	writeChunk.CurrentChunk = chunk.CurrentChunk
	writeChunk.CurrentKeyFrame = chunk.CurrentKeyFrame

	result, err = json.Marshal(writeChunk)
	if err != nil {
		panic("Error while encoding last chunk data json?!??!??")
	}

	writeRecording(region, gameId, "lastChunkData", result)
	writeString(region, gameId, "firstChunkNumber", strconv.Itoa(firstChunk))
}

func saveRecording(region, gameId string) {
	savePath := revel.BasePath + "/replays/" + region + "-" + gameId

	result, err := json.Marshal(recording[region+":"+gameId])
	if err != nil {
		panic("Error while encoding recording json?!?!?!?")
	}

	err = ioutil.WriteFile(savePath, result, 0644)
	if err != nil {
		revel.ERROR.Println("Error saving recording!")
	}
}

func recordMetadata(region, gameId string) {
	metadata := getMetadata(region, gameId)

	for {
		chunk := getLastChunkInfo(region, gameId)
		if chunk.CurrentChunk > metadata.StartupChunk {
			break
		}
		time.Sleep(time.Duration(chunk.NextUpdate)*time.Millisecond +
			time.Second)
	}

	metadata = getMetadata(region, gameId)

	// Get the startup frames
	for i := 1; i <= metadata.StartupChunk+1; i++ {
		// revel.INFO.Println("Getting startup chunk:", i)
		for {
			chunk := getLastChunkInfo(region, gameId)
			if i > chunk.CurrentChunk {
				time.Sleep(time.Duration(chunk.NextUpdate)*time.Millisecond +
					time.Second)
				continue
			}
			getChunkFrame(region, gameId, strconv.Itoa(i))
			break
		}
	}
}

func recordFrames(region, gameId string) {
	firstChunk := 0
	firstKeyFrame := 0
	lastChunk := 0
	lastKeyFrame := 0

	for {
		chunk := getLastChunkInfo(region, gameId)

		if firstChunk == 0 {
			if chunk.CurrentChunk > chunk.StartGameChunk {
				firstChunk = chunk.CurrentChunk
			} else {
				firstChunk = chunk.StartGameChunk
			}

			if chunk.CurrentKeyFrame > 0 {
				firstKeyFrame = chunk.CurrentKeyFrame
			} else {
				firstKeyFrame = 1
			}

			lastChunk = chunk.CurrentChunk
			lastKeyFrame = chunk.CurrentKeyFrame
		}

		if chunk.CurrentChunk > lastChunk {
			for i := lastChunk + 1; i <= chunk.CurrentChunk; i++ {
				getChunkFrame(region, gameId, strconv.Itoa(i))
			}
		}

		if chunk.NextChunk < chunk.CurrentChunk && chunk.NextChunk > 0 {
			getChunkFrame(region, gameId, strconv.Itoa(chunk.NextChunk))
		}

		if chunk.CurrentKeyFrame > lastKeyFrame {
			for i := lastKeyFrame + 1; i <= chunk.CurrentKeyFrame; i++ {
				getKeyFrame(region, gameId, strconv.Itoa(chunk.CurrentKeyFrame))
			}
		}

		writeLastChunkInfo(region, gameId, firstChunk, firstKeyFrame, chunk)
		saveRecording(region, gameId)

		lastChunk = chunk.CurrentChunk
		lastKeyFrame = chunk.CurrentKeyFrame

		if chunk.EndGameChunk == chunk.CurrentChunk {
			return
		}

		time.Sleep(time.Duration(chunk.NextUpdate)*time.Millisecond +
			time.Second)
	}
}

func asyncRecord(region, gameId, encryptionKey string) {
	defer func() {
		if r := recover(); r != nil {
			revel.ERROR.Println("Error while recording game ID: " + gameId)
			revel.ERROR.Println(r)
			delete(recording, region+":"+gameId)
		}
	}()

	writeRecording(region, gameId, "encryptionKey", []byte(encryptionKey))

	url := platformURLs[region]
	UpdateVersion(url)

	revel.INFO.Println("Now recording: " + region + ":" + gameId)
	revel.INFO.Println(gameId + "'s Encryption Key: " + encryptionKey)

	recordMetadata(region, gameId)
	recordFrames(region, gameId)

	revel.INFO.Println("Recording complete for: " + region + ":" + gameId)
	delete(recording, region+":"+gameId)
}

func Record(region, gameId, encryptionKey string) bool {
	if _, ok := recording[region+":"+gameId]; ok {
		return false
	} else {
		recording[region+":"+gameId] = make(map[string]string)
	}

	history.StoreGame(region, gameId, encryptionKey)

	go asyncRecord(region, gameId, encryptionKey)
	return true
}
<commit_msg>Fixed not recording first chunk/frame<commit_after>package record

// /CdsXSnhGV0TQ9B3VZ9IneNK4vk+K9k+

import (
	"encoding/base64"
	"encoding/json"
	"github.com/revel/revel"
	"io/ioutil"
	"replay/app/models/history"
	"strconv"
	"time"
)

var platformURLs map[string]string = map[string]string{
	"NA1":  "http://spectator.na.lol.riotgames.com:80",
	"OC1":  "http://spectator.oc1.lol.riotgames.com:80",
	"EUN1": "http://spectator.eu.lol.riotgames.com:8088",
	"EUW1": "http://spectator.euw1.lol.riotgames.com:80",
}

var version string // Version functions in getters.go
var recording map[string]map[string]string = make(map[string]map[string]string)

func writeRecording(region, gameId, key string, value []byte) {
	currentRecording := recording[region+":"+gameId]
	currentRecording[key] = base64.URLEncoding.EncodeToString(value)
	recording[region+":"+gameId] = currentRecording
}

func writeString(region, gameId, key string, value string) {
	currentRecording := recording[region+":"+gameId]
	currentRecording[key] = value
	recording[region+":"+gameId] = currentRecording
}

func existsRecording(region, gameId, key string) bool {
	if _, exists := recording[region+":"+gameId][key]; exists {
		return true
	}
	return false
}

func writeLastChunkInfo(region, gameId string,
	firstChunk, firstKeyFrame int, chunk ChunkInfo) {

	writeChunk := ChunkInfo{
		NextChunk:       firstChunk,
		CurrentChunk:    firstChunk,
		NextUpdate:      3000,
		StartGameChunk:  chunk.StartGameChunk,
		CurrentKeyFrame: firstKeyFrame,
		EndGameChunk:    chunk.CurrentChunk,
		AvailableSince:  0,
		Duration:        3000,
		EndStartupChunk: chunk.EndStartupChunk,
	}

	result, err := json.Marshal(writeChunk)
	if err != nil {
		panic("Error while encoding first chunk data json?!??!??")
	}

	writeRecording(region, gameId, "firstChunkData", result)

	writeChunk.NextChunk = chunk.CurrentChunk
	writeChunk.CurrentChunk = chunk.CurrentChunk
	writeChunk.CurrentKeyFrame = chunk.CurrentKeyFrame

	result, err = json.Marshal(writeChunk)
	if err != nil {
		panic("Error while encoding last chunk data json?!??!??")
	}

	writeRecording(region, gameId, "lastChunkData", result)
	writeString(region, gameId, "firstChunkNumber", strconv.Itoa(firstChunk))
}

func saveRecording(region, gameId string) {
	savePath := revel.BasePath + "/replays/" + region + "-" + gameId

	result, err := json.Marshal(recording[region+":"+gameId])
	if err != nil {
		panic("Error while encoding recording json?!?!?!?")
	}

	err = ioutil.WriteFile(savePath, result, 0644)
	if err != nil {
		revel.ERROR.Println("Error saving recording!")
	}
}

func recordMetadata(region, gameId string) {
	metadata := getMetadata(region, gameId)

	for {
		chunk := getLastChunkInfo(region, gameId)
		if chunk.CurrentChunk > metadata.StartupChunk {
			break
		}
		time.Sleep(time.Duration(chunk.NextUpdate)*time.Millisecond +
			time.Second)
	}

	metadata = getMetadata(region, gameId)

	// Get the startup frames
	for i := 1; i <= metadata.StartupChunk+1; i++ {
		// revel.INFO.Println("Getting startup chunk:", i)
		for {
			chunk := getLastChunkInfo(region, gameId)
			if i > chunk.CurrentChunk {
				time.Sleep(time.Duration(chunk.NextUpdate)*time.Millisecond +
					time.Second)
				continue
			}
			getChunkFrame(region, gameId, strconv.Itoa(i))
			break
		}
	}
}

func recordFrames(region, gameId string) {
	firstChunk := 0
	firstKeyFrame := 0
	lastChunk := 0
	lastKeyFrame := 0

	for {
		chunk := getLastChunkInfo(region, gameId)

		if firstChunk == 0 {
			if chunk.CurrentChunk > chunk.StartGameChunk {
				firstChunk = chunk.CurrentChunk
			} else {
				firstChunk = chunk.StartGameChunk
			}

			if chunk.CurrentKeyFrame > 0 {
				firstKeyFrame = chunk.CurrentKeyFrame
			} else {
				firstKeyFrame = 1
			}

			lastChunk = chunk.CurrentChunk
			lastKeyFrame = chunk.CurrentKeyFrame

			getChunkFrame(region, gameId, chunk.CurrentChunk)
			getKeyFrame(region, gameId, chunk.CurrentKeyFrame)
		}

		if chunk.CurrentChunk > lastChunk {
			for i := lastChunk + 1; i <= chunk.CurrentChunk; i++ {
				getChunkFrame(region, gameId, strconv.Itoa(i))
			}
		}

		if chunk.NextChunk < chunk.CurrentChunk && chunk.NextChunk > 0 {
			getChunkFrame(region, gameId, strconv.Itoa(chunk.NextChunk))
		}

		if chunk.CurrentKeyFrame > lastKeyFrame {
			for i := lastKeyFrame + 1; i <= chunk.CurrentKeyFrame; i++ {
				getKeyFrame(region, gameId, strconv.Itoa(chunk.CurrentKeyFrame))
			}
		}

		writeLastChunkInfo(region, gameId, firstChunk, firstKeyFrame, chunk)
		saveRecording(region, gameId)

		lastChunk = chunk.CurrentChunk
		lastKeyFrame = chunk.CurrentKeyFrame

		if chunk.EndGameChunk == chunk.CurrentChunk {
			return
		}

		time.Sleep(time.Duration(chunk.NextUpdate)*time.Millisecond +
			time.Second)
	}
}

func asyncRecord(region, gameId, encryptionKey string) {
	defer func() {
		if r := recover(); r != nil {
			revel.ERROR.Println("Error while recording game ID: " + gameId)
			revel.ERROR.Println(r)
			delete(recording, region+":"+gameId)
		}
	}()

	writeRecording(region, gameId, "encryptionKey", []byte(encryptionKey))

	url := platformURLs[region]
	UpdateVersion(url)

	revel.INFO.Println("Now recording: " + region + ":" + gameId)
	revel.INFO.Println(gameId + "'s Encryption Key: " + encryptionKey)

	recordMetadata(region, gameId)
	recordFrames(region, gameId)

	revel.INFO.Println("Recording complete for: " + region + ":" + gameId)
	delete(recording, region+":"+gameId)
}

func Record(region, gameId, encryptionKey string) bool {
	if _, ok := recording[region+":"+gameId]; ok {
		return false
	} else {
		recording[region+":"+gameId] = make(map[string]string)
	}

	history.StoreGame(region, gameId, encryptionKey)

	go asyncRecord(region, gameId, encryptionKey)
	return true
}
<|endoftext|>


<commit_before>/*

Package encrypt implement a backend that encrypt/decrypt on the fly (using nacl/secretbox [1])
and store blobs in the "dest" backend.

Links

	[1] godoc.org/code.google.com/p/go.crypto/nacl/secretbox

*/
package encrypt

import (
	"bufio"
	"bytes"
	"crypto/rand"
	"errors"
	"expvar"
	"fmt"
	"io"
	"log"
	"sync"

	"github.com/tsileo/blobstash/backend"

	"github.com/dchest/blake2b"

	"github.com/golang/snappy"
	"golang.org/x/crypto/nacl/secretbox"
)

var (
	bytesUploaded   = expvar.NewMap("encrypt-bytes-uploaded")
	bytesDownloaded = expvar.NewMap("encrypt-bytes-downloaded")
	blobsUploaded   = expvar.NewMap("encrypt-blobs-uploaded")
	blobsDownloaded = expvar.NewMap("encrypt-blobs-downloaded")
)

var headerSize = 86

func GenerateNonce(nonce *[24]byte) (err error) {
	_, err = io.ReadFull(rand.Reader, nonce[:])
	return
}

type EncryptBackend struct {
	dest backend.BlobHandler
	// index map the plain text hash to encrypted hash
	index map[string]string

	// holds the encryption key
	key *[32]byte

	sync.Mutex
}

// New return a backend that encrypt/decrypt blobs on the fly,
// blobs are compressed with snappy before encryption with nacl/secretbox.
// At startup it scan encrypted blobs to discover the plain hash (the hash of the plain-text/unencrypted data).
// Blobs are stored in the following format:
//
// #blobstash/secretbox\n
// [plain hash]\n
// [encrypted data]
//
func New(keyPath string, dest backend.BlobHandler) *EncryptBackend {
	log.Printf("EncryptBackend: starting with dest %v", dest.String())
	if err := LoadKey(keyPath); err != nil {
		panic(err)
	}
	log.Printf("EncryptBackend: loaded key at %v", keyPath)
	b := &EncryptBackend{dest: dest, index: make(map[string]string), key: &Key}
	log.Printf("EncryptBackend: backend id => %v", b.String())
	log.Println("EncryptBackend: scanning blobs to discover plain-text blobs hashes")
	blobsCnt := 0
	// Scan the blobs to discover the plain text blob hashes and build the in-memory index
	hashes := make(chan string)
	errs := make(chan error)
	go func() {
		errs <- b.dest.Enumerate(hashes)
	}()
	for hash := range hashes {
		scanner := b.scanner(hash)
		plainHash, err := scanHash(scanner)
		if err != nil {
			panic(err)
			//return errors.New(fmt.Sprintf("Error reading plain hash from %v, %v", hash, err))
		}
		b.index[plainHash] = hash
		blobsCnt++
	}
	if err := <-errs; err != nil {
		if err == backend.ErrWriteOnly {
			log.Printf("EncryptBackend: no scan in write-only mode")
		} else {
			panic(err)
		}
	}
	log.Printf("EncryptBackend: %v blobs successfully scanned", blobsCnt)
	return b
}

func (backend *EncryptBackend) String() string {
	return fmt.Sprintf("encrypt-%v", backend.dest.String())
}

func (b *EncryptBackend) Put(hash string, rawData []byte) (err error) {
	// #blobstash/secretbox\n
	// data hash\n
	// data
	var nonce [24]byte
	//out := make([]byte, len(data) + secretbox.Overhead + 24 + headerSize)
	if err := GenerateNonce(&nonce); err != nil {
		return err
	}
	// First we compress the data with snappy
	data := snappy.Encode(nil, rawData)

	var out bytes.Buffer
	out.WriteString("#blobstash/secretbox\n")
	out.WriteString(fmt.Sprintf("%v\n", hash))
	encData := make([]byte, len(data)+secretbox.Overhead)
	secretbox.Seal(encData[0:0], data, &nonce, b.key)
	out.Write(nonce[:])
	out.Write(encData)
	encHash := fmt.Sprintf("%x", blake2b.Sum256(out.Bytes()))
	b.dest.Put(encHash, out.Bytes())
	b.Lock()
	b.index[hash] = encHash
	defer b.Unlock()
	blobsUploaded.Add(b.dest.String(), 1)
	bytesUploaded.Add(b.dest.String(), int64(len(out.Bytes())))
	return
}

func (b *EncryptBackend) Delete(hash string) error {
	if err := b.dest.Delete(b.index[hash]); err != nil {
		return err
	}
	delete(b.index, hash)
	return nil
}

func (b *EncryptBackend) Exists(hash string) (bool, error) {
	b.Lock()
	defer b.Unlock()
	_, exists := b.index[hash]
	return exists, nil
}

func (b *EncryptBackend) Done() error {
	return nil
}

func (b *EncryptBackend) scanner(hash string) *bufio.Scanner {
	enc, err := b.dest.Get(hash)
	if err != nil {
		return nil
	}
	buf := bytes.NewBuffer(enc)
	return bufio.NewScanner(buf)
}

func scanHash(scanner *bufio.Scanner) (hash string, err error) {
	if !scanner.Scan() {
		return "", errors.New("No line to read")
	}
	if scanner.Text() != "#blobstash/secretbox" {
		return "", errors.New("bad header")
	}
	if !scanner.Scan() {
		return "", errors.New("ref not found")
	}
	return scanner.Text(), nil
}

func (b *EncryptBackend) Get(hash string) (data []byte, err error) {
	ref, _ := b.index[hash]
	enc, err := b.dest.Get(ref)
	if err != nil {
		return data, err
	}
	box := enc[headerSize:]
	var nonce [24]byte
	encData := make([]byte, len(box)-24)
	copy(nonce[:], box[:24])
	copy(encData[:], box[24:])
	out := make([]byte, len(box)-24)
	out, success := secretbox.Open(nil, encData, &nonce, b.key)
	if !success {
		return data, fmt.Errorf("failed to decrypt blob %v/%v", hash, ref)
	}
	// Decode snappy data
	data, err = snappy.Decode(nil, out)
	if err != nil {
		return data, fmt.Errorf("failed to decode blob %v/%v", hash, ref)
	}
	blobsDownloaded.Add(b.dest.String(), 1)
	bytesDownloaded.Add(b.dest.String(), int64(len(enc)))
	return
}

func (b *EncryptBackend) Enumerate(blobs chan<- string) error {
	defer close(blobs)
	for plainHash, _ := range b.index {
		blobs <- plainHash
	}
	return nil
}

func (b *EncryptBackend) Close() {
	b.dest.Close()
}
<commit_msg>backend: added Config struct for Encrypt backend<commit_after>/*

Package encrypt implement a backend that encrypt/decrypt on the fly (using nacl/secretbox [1])
and store blobs in the "dest" backend.

Links

	[1] godoc.org/code.google.com/p/go.crypto/nacl/secretbox

*/
package encrypt

import (
	"bufio"
	"bytes"
	"crypto/rand"
	"errors"
	"expvar"
	"fmt"
	"io"
	"log"
	"sync"

	"github.com/tsileo/blobstash/backend"

	"github.com/dchest/blake2b"

	"github.com/golang/snappy"
	"golang.org/x/crypto/nacl/secretbox"
)

var (
	bytesUploaded   = expvar.NewMap("encrypt-bytes-uploaded")
	bytesDownloaded = expvar.NewMap("encrypt-bytes-downloaded")
	blobsUploaded   = expvar.NewMap("encrypt-blobs-uploaded")
	blobsDownloaded = expvar.NewMap("encrypt-blobs-downloaded")
)

var headerSize = 86

func GenerateNonce(nonce *[24]byte) (err error) {
	_, err = io.ReadFull(rand.Reader, nonce[:])
	return
}

type Config struct {
	KeyPath     string         `structs:"key_path,omitempty"`
	DestBackend backend.Config `structs:"backend,omitempty"`
}

func (c *Config) Backend() string {
	return "encrypt"
}

func (c *Config) Config() map[string]interface{} {
	return map[string]interface{}{
		"backend-type": c.Backend(),
		"backend-args": c.Map(),
	}
}

func (c *Config) Map() map[string]interface{} {
	return map[string]interface{}{
		"key-path": c.KeyPath,
		"dest":     c.DestBackend.Config(),
	}
}

type EncryptBackend struct {
	dest backend.BlobHandler
	// index map the plain text hash to encrypted hash
	index map[string]string

	// holds the encryption key
	key *[32]byte

	sync.Mutex
}

// New return a backend that encrypt/decrypt blobs on the fly,
// blobs are compressed with snappy before encryption with nacl/secretbox.
// At startup it scan encrypted blobs to discover the plain hash (the hash of the plain-text/unencrypted data).
// Blobs are stored in the following format:
//
// #blobstash/secretbox\n
// [plain hash]\n
// [encrypted data]
//
func New(keyPath string, dest backend.BlobHandler) *EncryptBackend {
	log.Printf("EncryptBackend: starting with dest %v", dest.String())
	if err := LoadKey(keyPath); err != nil {
		panic(err)
	}
	log.Printf("EncryptBackend: loaded key at %v", keyPath)
	b := &EncryptBackend{dest: dest, index: make(map[string]string), key: &Key}
	log.Printf("EncryptBackend: backend id => %v", b.String())
	log.Println("EncryptBackend: scanning blobs to discover plain-text blobs hashes")
	blobsCnt := 0
	// Scan the blobs to discover the plain text blob hashes and build the in-memory index
	hashes := make(chan string)
	errs := make(chan error)
	go func() {
		errs <- b.dest.Enumerate(hashes)
	}()
	for hash := range hashes {
		scanner := b.scanner(hash)
		plainHash, err := scanHash(scanner)
		if err != nil {
			panic(err)
			//return errors.New(fmt.Sprintf("Error reading plain hash from %v, %v", hash, err))
		}
		b.index[plainHash] = hash
		blobsCnt++
	}
	if err := <-errs; err != nil {
		if err == backend.ErrWriteOnly {
			log.Printf("EncryptBackend: no scan in write-only mode")
		} else {
			panic(err)
		}
	}
	log.Printf("EncryptBackend: %v blobs successfully scanned", blobsCnt)
	return b
}

func (backend *EncryptBackend) String() string {
	return fmt.Sprintf("encrypt-%v", backend.dest.String())
}

func (b *EncryptBackend) Put(hash string, rawData []byte) (err error) {
	// #blobstash/secretbox\n
	// data hash\n
	// data
	var nonce [24]byte
	//out := make([]byte, len(data) + secretbox.Overhead + 24 + headerSize)
	if err := GenerateNonce(&nonce); err != nil {
		return err
	}
	// First we compress the data with snappy
	data := snappy.Encode(nil, rawData)

	var out bytes.Buffer
	out.WriteString("#blobstash/secretbox\n")
	out.WriteString(fmt.Sprintf("%v\n", hash))
	encData := make([]byte, len(data)+secretbox.Overhead)
	secretbox.Seal(encData[0:0], data, &nonce, b.key)
	out.Write(nonce[:])
	out.Write(encData)
	encHash := fmt.Sprintf("%x", blake2b.Sum256(out.Bytes()))
	b.dest.Put(encHash, out.Bytes())
	b.Lock()
	b.index[hash] = encHash
	defer b.Unlock()
	blobsUploaded.Add(b.dest.String(), 1)
	bytesUploaded.Add(b.dest.String(), int64(len(out.Bytes())))
	return
}

func (b *EncryptBackend) Delete(hash string) error {
	if err := b.dest.Delete(b.index[hash]); err != nil {
		return err
	}
	delete(b.index, hash)
	return nil
}

func (b *EncryptBackend) Exists(hash string) (bool, error) {
	b.Lock()
	defer b.Unlock()
	_, exists := b.index[hash]
	return exists, nil
}

func (b *EncryptBackend) Done() error {
	return nil
}

func (b *EncryptBackend) scanner(hash string) *bufio.Scanner {
	enc, err := b.dest.Get(hash)
	if err != nil {
		return nil
	}
	buf := bytes.NewBuffer(enc)
	return bufio.NewScanner(buf)
}

func scanHash(scanner *bufio.Scanner) (hash string, err error) {
	if !scanner.Scan() {
		return "", errors.New("No line to read")
	}
	if scanner.Text() != "#blobstash/secretbox" {
		return "", errors.New("bad header")
	}
	if !scanner.Scan() {
		return "", errors.New("ref not found")
	}
	return scanner.Text(), nil
}

func (b *EncryptBackend) Get(hash string) (data []byte, err error) {
	ref, _ := b.index[hash]
	enc, err := b.dest.Get(ref)
	if err != nil {
		return data, err
	}
	box := enc[headerSize:]
	var nonce [24]byte
	encData := make([]byte, len(box)-24)
	copy(nonce[:], box[:24])
	copy(encData[:], box[24:])
	out := make([]byte, len(box)-24)
	out, success := secretbox.Open(nil, encData, &nonce, b.key)
	if !success {
		return data, fmt.Errorf("failed to decrypt blob %v/%v", hash, ref)
	}
	// Decode snappy data
	data, err = snappy.Decode(nil, out)
	if err != nil {
		return data, fmt.Errorf("failed to decode blob %v/%v", hash, ref)
	}
	blobsDownloaded.Add(b.dest.String(), 1)
	bytesDownloaded.Add(b.dest.String(), int64(len(enc)))
	return
}

func (b *EncryptBackend) Enumerate(blobs chan<- string) error {
	defer close(blobs)
	for plainHash, _ := range b.index {
		blobs <- plainHash
	}
	return nil
}

func (b *EncryptBackend) Close() {
	b.dest.Close()
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package lzw implements the Lempel-Ziv-Welch compressed data format,
// described in T. A. Welch, ``A Technique for High-Performance Data
// Compression'', Computer, 17(6) (June 1984), pp 8-19.
//
// In particular, it implements LZW as used by the GIF, TIFF and PDF file
// formats, which means variable-width codes up to 12 bits and the first
// two non-literal codes are a clear code and an EOF code.
package lzw

// TODO(nigeltao): check that TIFF and PDF use LZW in the same way as GIF,
// modulo LSB/MSB packing order.

import (
	"bufio"
	"errors"
	"fmt"
	"io"
)

// Order specifies the bit ordering in an LZW data stream.
type Order int

const (
	// LSB means Least Significant Bits first, as used in the GIF file format.
	LSB Order = iota
	// MSB means Most Significant Bits first, as used in the TIFF and PDF
	// file formats.
	MSB
)

const (
	maxWidth           = 12
	decoderInvalidCode = 0xffff
	flushBuffer        = 1 << maxWidth
)

// decoder is the state from which the readXxx method converts a byte
// stream into a code stream.
type decoder struct {
	r        io.ByteReader
	bits     uint32
	nBits    uint
	width    uint
	read     func(*decoder) (uint16, error) // readLSB or readMSB
	litWidth int                            // width in bits of literal codes
	err      error

	// The first 1<<litWidth codes are literal codes.
	// The next two codes mean clear and EOF.
	// Other valid codes are in the range [lo, hi] where lo := clear + 2,
	// with the upper bound incrementing on each code seen.
	// overflow is the code at which hi overflows the code width.
	// last is the most recently seen code, or decoderInvalidCode.
	clear, eof, hi, overflow, last uint16

	// Each code c in [lo, hi] expands to two or more bytes. For c != hi:
	//   suffix[c] is the last of these bytes.
	//   prefix[c] is the code for all but the last byte.
	//   This code can either be a literal code or another code in [lo, c).
	// The c == hi case is a special case.
	suffix [1 << maxWidth]uint8
	prefix [1 << maxWidth]uint16

	// output is the temporary output buffer.
	// Literal codes are accumulated from the start of the buffer.
	// Non-literal codes decode to a sequence of suffixes that are first
	// written right-to-left from the end of the buffer before being copied
	// to the start of the buffer.
	// It is flushed when it contains >= 1<<maxWidth bytes,
	// so that there is always room to decode an entire code.
	output [2 * 1 << maxWidth]byte
	o      int    // write index into output
	toRead []byte // bytes to return from Read
}

// readLSB returns the next code for "Least Significant Bits first" data.
func (d *decoder) readLSB() (uint16, error) {
	for d.nBits < d.width {
		x, err := d.r.ReadByte()
		if err != nil {
			return 0, err
		}
		d.bits |= uint32(x) << d.nBits
		d.nBits += 8
	}
	code := uint16(d.bits & (1<<d.width - 1))
	d.bits >>= d.width
	d.nBits -= d.width
	return code, nil
}

// readMSB returns the next code for "Most Significant Bits first" data.
func (d *decoder) readMSB() (uint16, error) {
	for d.nBits < d.width {
		x, err := d.r.ReadByte()
		if err != nil {
			return 0, err
		}
		d.bits |= uint32(x) << (24 - d.nBits)
		d.nBits += 8
	}
	code := uint16(d.bits >> (32 - d.width))
	d.bits <<= d.width
	d.nBits -= d.width
	return code, nil
}

func (d *decoder) Read(b []byte) (int, error) {
	for {
		if len(d.toRead) > 0 {
			n := copy(b, d.toRead)
			d.toRead = d.toRead[n:]
			return n, nil
		}
		if d.err != nil {
			return 0, d.err
		}
		d.decode()
	}
}

// decode decompresses bytes from r and leaves them in d.toRead.
// read specifies how to decode bytes into codes.
// litWidth is the width in bits of literal codes.
func (d *decoder) decode() {
	// Loop over the code stream, converting codes into decompressed bytes.
	for {
		code, err := d.read(d)
		if err != nil {
			if err == io.EOF {
				err = io.ErrUnexpectedEOF
			}
			d.err = err
			return
		}
		switch {
		case code < d.clear:
			// We have a literal code.
			d.output[d.o] = uint8(code)
			d.o++
			if d.last != decoderInvalidCode {
				// Save what the hi code expands to.
				d.suffix[d.hi] = uint8(code)
				d.prefix[d.hi] = d.last
			}
		case code == d.clear:
			d.width = 1 + uint(d.litWidth)
			d.hi = d.eof
			d.overflow = 1 << d.width
			d.last = decoderInvalidCode
			continue
		case code == d.eof:
			d.flush()
			d.err = io.EOF
			return
		case code <= d.hi:
			c, i := code, len(d.output)-1
			if code == d.hi {
				// code == hi is a special case which expands to the last expansion
				// followed by the head of the last expansion. To find the head, we walk
				// the prefix chain until we find a literal code.
				c = d.last
				for c >= d.clear {
					c = d.prefix[c]
				}
				d.output[i] = uint8(c)
				i--
				c = d.last
			}
			// Copy the suffix chain into output and then write that to w.
			for c >= d.clear {
				d.output[i] = d.suffix[c]
				i--
				c = d.prefix[c]
			}
			d.output[i] = uint8(c)
			d.o += copy(d.output[d.o:], d.output[i:])
			if d.last != decoderInvalidCode {
				// Save what the hi code expands to.
				d.suffix[d.hi] = uint8(c)
				d.prefix[d.hi] = d.last
			}
		default:
			d.err = errors.New("lzw: invalid code")
			return
		}
		d.last, d.hi = code, d.hi+1
		if d.hi >= d.overflow {
			if d.width == maxWidth {
				d.last = decoderInvalidCode
			} else {
				d.width++
				d.overflow <<= 1
			}
		}
		if d.o >= flushBuffer {
			d.flush()
			return
		}
	}
}

func (d *decoder) flush() {
	d.toRead = d.output[:d.o]
	d.o = 0
}

var errClosed = errors.New("compress/lzw: reader/writer is closed")

func (d *decoder) Close() error {
	d.err = errClosed // in case any Reads come along
	return nil
}

// NewReader creates a new io.ReadCloser.
// Reads from the returned io.ReadCloser read and decompress data from r.
// It is the caller's responsibility to call Close on the ReadCloser when
// finished reading.
// The number of bits to use for literal codes, litWidth, must be in the
// range [2,8] and is typically 8.
func NewReader(r io.Reader, order Order, litWidth int) io.ReadCloser {
	d := new(decoder)
	switch order {
	case LSB:
		d.read = (*decoder).readLSB
	case MSB:
		d.read = (*decoder).readMSB
	default:
		d.err = errors.New("lzw: unknown order")
		return d
	}
	if litWidth < 2 || 8 < litWidth {
		d.err = fmt.Errorf("lzw: litWidth %d out of range", litWidth)
		return d
	}
	if br, ok := r.(io.ByteReader); ok {
		d.r = br
	} else {
		d.r = bufio.NewReader(r)
	}
	d.litWidth = litWidth
	d.width = 1 + uint(litWidth)
	d.clear = uint16(1) << uint(litWidth)
	d.eof, d.hi = d.clear+1, d.clear+1
	d.overflow = uint16(1) << d.width
	d.last = decoderInvalidCode

	return d
}
<commit_msg>compress/lzw: add commentary that TIFF's LZW differs from the standard algorithm.<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package lzw implements the Lempel-Ziv-Welch compressed data format,
// described in T. A. Welch, ``A Technique for High-Performance Data
// Compression'', Computer, 17(6) (June 1984), pp 8-19.
//
// In particular, it implements LZW as used by the GIF and PDF file
// formats, which means variable-width codes up to 12 bits and the first
// two non-literal codes are a clear code and an EOF code.
//
// The TIFF file format uses a similar but incompatible version of the LZW
// algorithm. See the code.google.com/p/go.image/tiff/lzw package for an
// implementation.
package lzw

// TODO(nigeltao): check that PDF uses LZW in the same way as GIF,
// modulo LSB/MSB packing order.

import (
	"bufio"
	"errors"
	"fmt"
	"io"
)

// Order specifies the bit ordering in an LZW data stream.
type Order int

const (
	// LSB means Least Significant Bits first, as used in the GIF file format.
	LSB Order = iota
	// MSB means Most Significant Bits first, as used in the TIFF and PDF
	// file formats.
	MSB
)

const (
	maxWidth           = 12
	decoderInvalidCode = 0xffff
	flushBuffer        = 1 << maxWidth
)

// decoder is the state from which the readXxx method converts a byte
// stream into a code stream.
type decoder struct {
	r        io.ByteReader
	bits     uint32
	nBits    uint
	width    uint
	read     func(*decoder) (uint16, error) // readLSB or readMSB
	litWidth int                            // width in bits of literal codes
	err      error

	// The first 1<<litWidth codes are literal codes.
	// The next two codes mean clear and EOF.
	// Other valid codes are in the range [lo, hi] where lo := clear + 2,
	// with the upper bound incrementing on each code seen.
	// overflow is the code at which hi overflows the code width.
	// last is the most recently seen code, or decoderInvalidCode.
	clear, eof, hi, overflow, last uint16

	// Each code c in [lo, hi] expands to two or more bytes. For c != hi:
	//   suffix[c] is the last of these bytes.
	//   prefix[c] is the code for all but the last byte.
	//   This code can either be a literal code or another code in [lo, c).
	// The c == hi case is a special case.
	suffix [1 << maxWidth]uint8
	prefix [1 << maxWidth]uint16

	// output is the temporary output buffer.
	// Literal codes are accumulated from the start of the buffer.
	// Non-literal codes decode to a sequence of suffixes that are first
	// written right-to-left from the end of the buffer before being copied
	// to the start of the buffer.
	// It is flushed when it contains >= 1<<maxWidth bytes,
	// so that there is always room to decode an entire code.
	output [2 * 1 << maxWidth]byte
	o      int    // write index into output
	toRead []byte // bytes to return from Read
}

// readLSB returns the next code for "Least Significant Bits first" data.
func (d *decoder) readLSB() (uint16, error) {
	for d.nBits < d.width {
		x, err := d.r.ReadByte()
		if err != nil {
			return 0, err
		}
		d.bits |= uint32(x) << d.nBits
		d.nBits += 8
	}
	code := uint16(d.bits & (1<<d.width - 1))
	d.bits >>= d.width
	d.nBits -= d.width
	return code, nil
}

// readMSB returns the next code for "Most Significant Bits first" data.
func (d *decoder) readMSB() (uint16, error) {
	for d.nBits < d.width {
		x, err := d.r.ReadByte()
		if err != nil {
			return 0, err
		}
		d.bits |= uint32(x) << (24 - d.nBits)
		d.nBits += 8
	}
	code := uint16(d.bits >> (32 - d.width))
	d.bits <<= d.width
	d.nBits -= d.width
	return code, nil
}

func (d *decoder) Read(b []byte) (int, error) {
	for {
		if len(d.toRead) > 0 {
			n := copy(b, d.toRead)
			d.toRead = d.toRead[n:]
			return n, nil
		}
		if d.err != nil {
			return 0, d.err
		}
		d.decode()
	}
}

// decode decompresses bytes from r and leaves them in d.toRead.
// read specifies how to decode bytes into codes.
// litWidth is the width in bits of literal codes.
func (d *decoder) decode() {
	// Loop over the code stream, converting codes into decompressed bytes.
	for {
		code, err := d.read(d)
		if err != nil {
			if err == io.EOF {
				err = io.ErrUnexpectedEOF
			}
			d.err = err
			return
		}
		switch {
		case code < d.clear:
			// We have a literal code.
			d.output[d.o] = uint8(code)
			d.o++
			if d.last != decoderInvalidCode {
				// Save what the hi code expands to.
				d.suffix[d.hi] = uint8(code)
				d.prefix[d.hi] = d.last
			}
		case code == d.clear:
			d.width = 1 + uint(d.litWidth)
			d.hi = d.eof
			d.overflow = 1 << d.width
			d.last = decoderInvalidCode
			continue
		case code == d.eof:
			d.flush()
			d.err = io.EOF
			return
		case code <= d.hi:
			c, i := code, len(d.output)-1
			if code == d.hi {
				// code == hi is a special case which expands to the last expansion
				// followed by the head of the last expansion. To find the head, we walk
				// the prefix chain until we find a literal code.
				c = d.last
				for c >= d.clear {
					c = d.prefix[c]
				}
				d.output[i] = uint8(c)
				i--
				c = d.last
			}
			// Copy the suffix chain into output and then write that to w.
			for c >= d.clear {
				d.output[i] = d.suffix[c]
				i--
				c = d.prefix[c]
			}
			d.output[i] = uint8(c)
			d.o += copy(d.output[d.o:], d.output[i:])
			if d.last != decoderInvalidCode {
				// Save what the hi code expands to.
				d.suffix[d.hi] = uint8(c)
				d.prefix[d.hi] = d.last
			}
		default:
			d.err = errors.New("lzw: invalid code")
			return
		}
		d.last, d.hi = code, d.hi+1
		if d.hi >= d.overflow {
			if d.width == maxWidth {
				d.last = decoderInvalidCode
			} else {
				d.width++
				d.overflow <<= 1
			}
		}
		if d.o >= flushBuffer {
			d.flush()
			return
		}
	}
}

func (d *decoder) flush() {
	d.toRead = d.output[:d.o]
	d.o = 0
}

var errClosed = errors.New("compress/lzw: reader/writer is closed")

func (d *decoder) Close() error {
	d.err = errClosed // in case any Reads come along
	return nil
}

// NewReader creates a new io.ReadCloser.
// Reads from the returned io.ReadCloser read and decompress data from r.
// It is the caller's responsibility to call Close on the ReadCloser when
// finished reading.
// The number of bits to use for literal codes, litWidth, must be in the
// range [2,8] and is typically 8.
func NewReader(r io.Reader, order Order, litWidth int) io.ReadCloser {
	d := new(decoder)
	switch order {
	case LSB:
		d.read = (*decoder).readLSB
	case MSB:
		d.read = (*decoder).readMSB
	default:
		d.err = errors.New("lzw: unknown order")
		return d
	}
	if litWidth < 2 || 8 < litWidth {
		d.err = fmt.Errorf("lzw: litWidth %d out of range", litWidth)
		return d
	}
	if br, ok := r.(io.ByteReader); ok {
		d.r = br
	} else {
		d.r = bufio.NewReader(r)
	}
	d.litWidth = litWidth
	d.width = 1 + uint(litWidth)
	d.clear = uint16(1) << uint(litWidth)
	d.eof, d.hi = d.clear+1, d.clear+1
	d.overflow = uint16(1) << d.width
	d.last = decoderInvalidCode

	return d
}
<|endoftext|>


<commit_before><commit_msg>add interactive service constructor<commit_after><|endoftext|>


<commit_before>package calculator

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
)

// in organises input value for Calc
type in struct {
	input float64
	op    string
}

// Test organises values for tests
type Test struct {
	name        string
	in          in
	out         float64
	shouldError bool
}

// TestCalcAdd tests the behavior of Calc with Add operation
func TestCalcAdd(t *testing.T) {
	var tests = []Test{
		Test{
			name: "first input",
			in:   in{input: 5, op: "+"},
			out:  5,
		},
		Test{
			name: "second input",
			in:   in{input: 10, op: "+"},
			out:  15,
		},
		Test{
			name: "third input",
			in:   in{input: 35, op: "+"},
			out:  50,
		},
	}
	c := &Calculator{}
	var result float64
	for _, test := range tests {

		result = c.Do(test.in.input, test.in.op)
		//if test.out != result {
		//	t.Errorf(fmt.Sprintf("%s: %v", test.name, test.in))
		//}
		// Assertion with assert package
		assert.Equal(
			t,
			test.out,
			result,
			fmt.Sprintf("%s: %v", test.name, test.in),
		)
	}
}

// TestCalcMultipleOps tests the behavior of Calc with multiple operations
func TestCalcMultipleOps(t *testing.T) {
	var tests = []Test{
		Test{
			name: "first input for addition",
			in:   in{input: 10, op: "+"},
			out:  10,
		},
		Test{
			name: "second input for multiplication",
			in:   in{input: 5, op: "*"},
			out:  50,
		},
		Test{
			name: "third input for subtraction",
			in:   in{input: 30, op: "-"},
			out:  20,
		},
		Test{
			name: "fourth input for subtraction",
			in:   in{input: 4, op: "/"},
			out:  5,
		},
	}
	c := &Calculator{}
	for _, test := range tests {
		// Assertion with assert package
		assert.Equal(
			t,
			test.out,
			c.Do(test.in.input, test.in.op),
			fmt.Sprintf("%s: %v", test.name, test.in),
		)
	}
}

// TestCalcMultipleOpsWithSubTest tests the behavior of Calc with multiple operations
// with sub-level tests
func TestCalcMultipleOpsWithSubTest(t *testing.T) {
	var tests = []Test{
		Test{
			name: "first input for addition",
			in:   in{input: 10, op: "+"},
			out:  10,
		},
		Test{
			name: "second input for multiplication",
			in:   in{input: 5, op: "*"},
			out:  50,
		},
		Test{
			name: "third input for subtraction",
			in:   in{input: 30, op: "-"},
			out:  20,
		},
		Test{
			name: "fourth input for subtraction",
			in:   in{input: 4, op: "/"},
			out:  5,
		},
	}
	c := &Calculator{}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			assert.Equal(
				t,
				test.out,
				c.Do(test.in.input, test.in.op),
				fmt.Sprintf("%s: %v", test.name, test.in),
			)
		})
	}
}

//// BenchmarkCalc benchmarks Calc
//func BenchmarkCalc(b *testing.B) {
//	test := Test{
//		in:  in{input: 5, op: "+"},
//		out: 5,
//	}
//	for i := 0; i < b.N; i++ {
//		c := &Calculator{}
//		c.Do(test.in.input, test.in.op)
//	}
//}
//
//// BenchmarkCalcMultipleOpsWithSubTest benchmarks with sub-level benchmark
//func BenchmarkCalcMultipleOpsWithSubTest(b *testing.B) {
//	var tests = []Test{
//		Test{
//			name: "input for addition",
//			in:   in{input: 10, op: "+"},
//			out:  10,
//		},
//		Test{
//			name: "input for multiplication",
//			in:   in{input: 5, op: "*"},
//			out:  50,
//		},
//		Test{
//			name: "input for subtraction",
//			in:   in{input: 30, op: "-"},
//			out:  20,
//		},
//		Test{
//			name: "third input for subtraction",
//			in:   in{input: 4, op: "/"},
//			out:  5,
//		},
//	}
//	for _, test := range tests {
//		b.Run(test.name, func(b *testing.B) {
//			for i := 0; i < b.N; i++ {
//				c := &Calculator{}
//				c.Do(test.in.input, test.in.op)
//			}
//		})
//	}
//}
<commit_msg>Modifications in test examples<commit_after>package calculator

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
)

// in organises input value for Calc
type in struct {
	input float64
	op    string
}

// Test organises values for tests
type Test struct {
	name        string
	in          in
	out         float64
	shouldError bool
}

// TestCalcAdd tests the behavior of Calc with Add operation
func TestCalcAdd(t *testing.T) {
	var tests = []Test{
		Test{
			name: "first input",
			in:   in{input: 5, op: "+"},
			out:  5,
		},
		Test{
			name: "second input",
			in:   in{input: 10, op: "+"},
			out:  15,
		},
		Test{
			name: "third input",
			in:   in{input: 35, op: "+"},
			out:  50,
		},
	}
	c := &Calculator{}
	var result float64
	for _, test := range tests {

		result = c.Do(test.in.input, test.in.op)
		//if test.out != result {
		//	t.Errorf(fmt.Sprintf("%s: %v", test.name, test.in))
		//}
		// Assertion with assert package
		assert.Equal(
			t,
			test.out,
			result,
			fmt.Sprintf("%s: %v", test.name, test.in),
		)
	}
}

// TestCalcMultipleOps tests the behavior of Calc with multiple operations
func TestCalcMultipleOps(t *testing.T) {
	var tests = []Test{
		Test{
			name: "first input for addition",
			in:   in{input: 10, op: "+"},
			out:  10,
		},
		Test{
			name: "second input for multiplication",
			in:   in{input: 5, op: "*"},
			out:  50,
		},
		Test{
			name: "third input for subtraction",
			in:   in{input: 30, op: "-"},
			out:  20,
		},
		Test{
			name: "fourth input for subtraction",
			in:   in{input: 4, op: "/"},
			out:  5,
		},
	}
	c := &Calculator{}
	for _, test := range tests {
		// Assertion with assert package
		assert.Equal(
			t,
			test.out,
			c.Do(test.in.input, test.in.op),
			fmt.Sprintf("%s: %v", test.name, test.in),
		)
	}
}

// TestCalcMultipleOpsWithSubTest tests the behavior of Calc with multiple operations
// with sub-level tests
func TestCalcMultipleOpsWithSubTest(t *testing.T) {
	var tests = []Test{
		Test{
			name: "first input for addition",
			in:   in{input: 10, op: "+"},
			out:  10,
		},
		Test{
			name: "second input for multiplication",
			in:   in{input: 5, op: "*"},
			out:  50,
		},
		Test{
			name: "third input for subtraction",
			in:   in{input: 30, op: "-"},
			out:  20,
		},
		Test{
			name: "fourth input for subtraction",
			in:   in{input: 4, op: "/"},
			out:  5,
		},
	}
	c := &Calculator{}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			assert.Equal(
				t,
				test.out,
				c.Do(test.in.input, test.in.op),
				fmt.Sprintf("%s: %v", test.name, test.in),
			)
		})
	}
}

// BenchmarkCalc benchmarks Calc
func BenchmarkCalc(b *testing.B) {
	test := Test{
		in:  in{input: 5, op: "+"},
		out: 5,
	}
	for i := 0; i < b.N; i++ {
		c := &Calculator{}
		c.Do(test.in.input, test.in.op)
	}
}

// BenchmarkCalcMultipleOpsWithSubTest benchmarks with sub-level benchmark
func BenchmarkCalcMultipleOpsWithSubTest(b *testing.B) {
	var tests = []Test{
		Test{
			name: "input for addition",
			in:   in{input: 10, op: "+"},
			out:  10,
		},
		Test{
			name: "input for multiplication",
			in:   in{input: 5, op: "*"},
			out:  50,
		},
		Test{
			name: "input for subtraction",
			in:   in{input: 30, op: "-"},
			out:  20,
		},
		Test{
			name: "third input for subtraction",
			in:   in{input: 4, op: "/"},
			out:  5,
		},
	}
	for _, test := range tests {
		b.Run(test.name, func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				c := &Calculator{}
				c.Do(test.in.input, test.in.op)
			}
		})
	}
}
<|endoftext|>


<commit_before>// Package u5 provides a single utility to fetch the importers of a GoPackage via godoc.org API.
package u5

import (
	"encoding/json"
	"fmt"
	"net/http"
)

// GoPackage represents a Go package.
type GoPackage struct {
	Path     string // Import path of the package.
	Synopsis string // Synopsis of the package.
}

// Importers contains the list of Go packages that import a given Go package.
type Importers struct {
	Results []GoPackage
}

// GetGodocOrgImporters fetches the importers of Go package with specified importPath via godoc.org API.
func GetGodocOrgImporters(importPath string) (*Importers, error) {
	resp, err := http.Get("http://api.godoc.org/importers/" + importPath)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("non-200 status code: %v", resp.StatusCode)
	}
	var importers Importers
	err = json.NewDecoder(resp.Body).Decode(&importers)
	if err != nil {
		return nil, err
	}
	return &importers, nil
}
<commit_msg>u5: Add UserAgent variable.<commit_after>// Package u5 provides a single utility to fetch the importers of a GoPackage via godoc.org API.
package u5

import (
	"encoding/json"
	"fmt"
	"net/http"
)

// GoPackage represents a Go package.
type GoPackage struct {
	Path     string // Import path of the package.
	Synopsis string // Synopsis of the package.
}

// Importers contains the list of Go packages that import a given Go package.
type Importers struct {
	Results []GoPackage
}

// UserAgent is used for outbound requests to godoc.org API, if set to non-empty value.
var UserAgent string

// GetGodocOrgImporters fetches the importers of Go package with specified importPath via godoc.org API.
func GetGodocOrgImporters(importPath string) (*Importers, error) {
	req, err := http.NewRequest("GET", "http://api.godoc.org/importers/"+importPath, nil)
	if err != nil {
		return nil, err
	}
	if UserAgent != "" {
		req.Header.Set("User-Agent", UserAgent)
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("non-200 status code: %v", resp.StatusCode)
	}
	var importers Importers
	err = json.NewDecoder(resp.Body).Decode(&importers)
	if err != nil {
		return nil, err
	}
	return &importers, nil
}
<|endoftext|>


<commit_before>// Copyright 2022 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package concurrentddltest

import (
	"context"
	"fmt"
	"math/rand"
	"testing"
	"time"

	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/meta"
	"github.com/pingcap/tidb/testkit"
	"github.com/pingcap/tidb/util"
	"github.com/stretchr/testify/require"
	"go.uber.org/atomic"
)

func TestConcurrentDDLSwitch(t *testing.T) {
	store := testkit.CreateMockStore(t)

	type table struct {
		columnIdx int
		indexIdx  int
	}

	var tables []*table
	tblCount := 20
	for i := 0; i < tblCount; i++ {
		tables = append(tables, &table{1, 0})
	}

	tk := testkit.NewTestKit(t, store)
	tk.MustExec("use test")
	tk.MustExec("set global tidb_enable_metadata_lock=0")
	tk.MustExec("set @@global.tidb_ddl_reorg_worker_cnt=1")
	tk.MustExec("set @@global.tidb_ddl_reorg_batch_size=32")

	for i := range tables {
		tk.MustExec(fmt.Sprintf("create table t%d (col0 int) partition by range columns (col0) ("+
			"partition p1 values less than (100), "+
			"partition p2 values less than (300), "+
			"partition p3 values less than (500), "+
			"partition p4 values less than (700), "+
			"partition p5 values less than (1000), "+
			"partition p6 values less than maxvalue);",
			i))
		for j := 0; j < 1000; j++ {
			tk.MustExec(fmt.Sprintf("insert into t%d values (%d)", i, j))
		}
	}

	ddls := make([]string, 0, tblCount)
	ddlCount := 100
	for i := 0; i < ddlCount; i++ {
		tblIdx := rand.Intn(tblCount)
		if rand.Intn(2) == 0 {
			ddls = append(ddls, fmt.Sprintf("alter table t%d add index idx%d (col0)", tblIdx, tables[tblIdx].indexIdx))
			tables[tblIdx].indexIdx++
		} else {
			ddls = append(ddls, fmt.Sprintf("alter table t%d add column col%d int", tblIdx, tables[tblIdx].columnIdx))
			tables[tblIdx].columnIdx++
		}
	}

	c := atomic.NewInt32(0)
	ch := make(chan struct{})
	go func() {
		var wg util.WaitGroupWrapper
		for i := range ddls {
			wg.Add(1)
			go func(idx int) {
				tk := testkit.NewTestKit(t, store)
				tk.MustExec("use test")
				tk.MustExec(ddls[idx])
				c.Add(1)
				wg.Done()
			}(i)
		}
		wg.Wait()
		ch <- struct{}{}
	}()

	ticker := time.NewTicker(time.Second)
	count := 0
	done := false
	for !done {
		select {
		case <-ch:
			done = true
		case <-ticker.C:
			var b bool
			var err error
			err = kv.RunInNewTxn(kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL), store, false, func(ctx context.Context, txn kv.Transaction) error {
				b, err = meta.NewMeta(txn).IsConcurrentDDL()
				return err
			})
			require.NoError(t, err)
			rs, err := testkit.NewTestKit(t, store).Exec(fmt.Sprintf("set @@global.tidb_enable_concurrent_ddl=%t", !b))
			if rs != nil {
				require.NoError(t, rs.Close())
			}
			if err == nil {
				count++
				if b {
					tk := testkit.NewTestKit(t, store)
					tk.MustQuery("select count(*) from mysql.tidb_ddl_job").Check(testkit.Rows("0"))
					tk.MustQuery("select count(*) from mysql.tidb_ddl_reorg").Check(testkit.Rows("0"))
				}
			}
		}
	}

	require.Equal(t, int32(ddlCount), c.Load())
	require.Greater(t, count, 0)

	tk = testkit.NewTestKit(t, store)
	tk.MustExec("use test")
	for i, tbl := range tables {
		tk.MustQuery(fmt.Sprintf("select count(*) from information_schema.columns where TABLE_SCHEMA = 'test' and TABLE_NAME = 't%d'", i)).Check(testkit.Rows(fmt.Sprintf("%d", tbl.columnIdx)))
		tk.MustExec(fmt.Sprintf("admin check table t%d", i))
		for j := 0; j < tbl.indexIdx; j++ {
			tk.MustExec(fmt.Sprintf("admin check index t%d idx%d", i, j))
		}
	}
}
<commit_msg>test: remove partition in `TestConcurrentDDLSwitch` (#37962)<commit_after>// Copyright 2022 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package concurrentddltest

import (
	"context"
	"fmt"
	"math/rand"
	"testing"
	"time"

	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/meta"
	"github.com/pingcap/tidb/testkit"
	"github.com/pingcap/tidb/util"
	"github.com/stretchr/testify/require"
	"go.uber.org/atomic"
)

func TestConcurrentDDLSwitch(t *testing.T) {
	store := testkit.CreateMockStore(t)

	type table struct {
		columnIdx int
		indexIdx  int
	}

	var tables []*table
	tblCount := 20
	for i := 0; i < tblCount; i++ {
		tables = append(tables, &table{1, 0})
	}

	tk := testkit.NewTestKit(t, store)
	tk.MustExec("use test")
	tk.MustExec("set global tidb_enable_metadata_lock=0")
	tk.MustExec("set @@global.tidb_ddl_reorg_worker_cnt=1")
	tk.MustExec("set @@global.tidb_ddl_reorg_batch_size=32")

	for i := range tables {
		tk.MustExec(fmt.Sprintf("create table t%d (col0 int)", i))
		for j := 0; j < 1000; j++ {
			tk.MustExec(fmt.Sprintf("insert into t%d values (%d)", i, j))
		}
	}

	ddls := make([]string, 0, tblCount)
	ddlCount := 100
	for i := 0; i < ddlCount; i++ {
		tblIdx := rand.Intn(tblCount)
		if rand.Intn(2) == 0 {
			ddls = append(ddls, fmt.Sprintf("alter table t%d add index idx%d (col0)", tblIdx, tables[tblIdx].indexIdx))
			tables[tblIdx].indexIdx++
		} else {
			ddls = append(ddls, fmt.Sprintf("alter table t%d add column col%d int", tblIdx, tables[tblIdx].columnIdx))
			tables[tblIdx].columnIdx++
		}
	}

	c := atomic.NewInt32(0)
	ch := make(chan struct{})
	go func() {
		var wg util.WaitGroupWrapper
		for i := range ddls {
			wg.Add(1)
			go func(idx int) {
				tk := testkit.NewTestKit(t, store)
				tk.MustExec("use test")
				tk.MustExec(ddls[idx])
				c.Add(1)
				wg.Done()
			}(i)
		}
		wg.Wait()
		ch <- struct{}{}
	}()

	ticker := time.NewTicker(time.Second)
	count := 0
	done := false
	for !done {
		select {
		case <-ch:
			done = true
		case <-ticker.C:
			var b bool
			var err error
			err = kv.RunInNewTxn(kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL), store, false, func(ctx context.Context, txn kv.Transaction) error {
				b, err = meta.NewMeta(txn).IsConcurrentDDL()
				return err
			})
			require.NoError(t, err)
			rs, err := testkit.NewTestKit(t, store).Exec(fmt.Sprintf("set @@global.tidb_enable_concurrent_ddl=%t", !b))
			if rs != nil {
				require.NoError(t, rs.Close())
			}
			if err == nil {
				count++
				if b {
					tk := testkit.NewTestKit(t, store)
					tk.MustQuery("select count(*) from mysql.tidb_ddl_job").Check(testkit.Rows("0"))
					tk.MustQuery("select count(*) from mysql.tidb_ddl_reorg").Check(testkit.Rows("0"))
				}
			}
		}
	}

	require.Equal(t, int32(ddlCount), c.Load())
	require.Greater(t, count, 0)

	tk = testkit.NewTestKit(t, store)
	tk.MustExec("use test")
	for i, tbl := range tables {
		tk.MustQuery(fmt.Sprintf("select count(*) from information_schema.columns where TABLE_SCHEMA = 'test' and TABLE_NAME = 't%d'", i)).Check(testkit.Rows(fmt.Sprintf("%d", tbl.columnIdx)))
		tk.MustExec(fmt.Sprintf("admin check table t%d", i))
		for j := 0; j < tbl.indexIdx; j++ {
			tk.MustExec(fmt.Sprintf("admin check index t%d idx%d", i, j))
		}
	}
}
<|endoftext|>


<commit_before>package connmgr

import (
	"math/rand"
	"sync"
	"testing"

	inet "github.com/libp2p/go-libp2p-net"
)

func randomConns(tb testing.TB) (c [5000]inet.Conn) {
	for i, _ := range c {
		c[i] = randConn(tb, nil)
	}
	return c
}

func BenchmarkLockContention(b *testing.B) {
	conns := randomConns(b)
	cm := NewConnManager(1000, 1000, 0)
	not := cm.Notifee()

	kill := make(chan struct{})
	var wg sync.WaitGroup

	for i := 0; i < 16; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for {
				select {
				case <-kill:
					return
				default:
					_ = cm.GetTagInfo(conns[rand.Intn(3000)].RemotePeer())
				}
			}
		}()
	}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rc := conns[rand.Intn(3000)]
		not.Connected(nil, rc)
		cm.TagPeer(rc.RemotePeer(), "tag", 100)
		cm.UntagPeer(rc.RemotePeer(), "tag")
		not.Disconnected(nil, rc)
	}
	close(kill)
	wg.Wait()
}
<commit_msg>fix bench_test to use TagPeer in parallel goroutines<commit_after>package connmgr

import (
	"math/rand"
	"sync"
	"testing"

	inet "github.com/libp2p/go-libp2p-net"
)

func randomConns(tb testing.TB) (c [5000]inet.Conn) {
	for i, _ := range c {
		c[i] = randConn(tb, nil)
	}
	return c
}

func BenchmarkLockContention(b *testing.B) {
	conns := randomConns(b)
	cm := NewConnManager(1000, 1000, 0)
	not := cm.Notifee()

	kill := make(chan struct{})
	var wg sync.WaitGroup

	for i := 0; i < 16; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for {
				select {
				case <-kill:
					return
				default:
					cm.TagPeer(conns[rand.Intn(len(conns))].RemotePeer(), "another-tag", 1)
				}
			}
		}()
	}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rc := conns[rand.Intn(len(conns))]
		not.Connected(nil, rc)
		cm.TagPeer(rc.RemotePeer(), "tag", 100)
		cm.UntagPeer(rc.RemotePeer(), "tag")
		not.Disconnected(nil, rc)
	}
	close(kill)
	wg.Wait()
}
<|endoftext|>


<commit_before>// Copyright 2019, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import (
	"context"
	"fmt"
	"net/http"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	"go.opentelemetry.io/otel/api/core"
	"go.opentelemetry.io/otel/api/global"
	export "go.opentelemetry.io/otel/sdk/export/metric"
	"go.opentelemetry.io/otel/sdk/export/metric/aggregator"
	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
	"go.opentelemetry.io/otel/sdk/metric/batcher/defaultkeys"
	"go.opentelemetry.io/otel/sdk/metric/controller/push"
	"go.opentelemetry.io/otel/sdk/metric/selector/simple"
)

// Exporter is an implementation of metric.Exporter that sends metrics to
// Prometheus.
type Exporter struct {
	handler http.Handler

	registerer prometheus.Registerer
	gatherer   prometheus.Gatherer

	snapshot export.CheckpointSet
	onError  func(error)

	defaultSummaryQuantiles []float64
}

var _ export.Exporter = &Exporter{}
var _ http.Handler = &Exporter{}

// Config is a set of configs for the tally reporter.
type Config struct {
	// Registry is the prometheus registry that will be used as the default Registerer and
	// Gatherer if these are not specified.
	//
	// If not set a new empty Registry is created.
	Registry *prometheus.Registry

	// Registerer is the prometheus registerer to register
	// metrics with.
	//
	// If not specified the Registry will be used as default.
	Registerer prometheus.Registerer

	// Gatherer is the prometheus gatherer to gather
	// metrics with.
	//
	// If not specified the Registry will be used as default.
	Gatherer prometheus.Gatherer

	// DefaultSummaryQuantiles is the default summary quantiles
	// to use. Use nil to specify the system-default summary quantiles.
	DefaultSummaryQuantiles []float64

	// OnError is a function that handle errors that may occur while exporting metrics.
	// TODO: This should be refactored or even removed once we have a better error handling mechanism.
	OnError func(error)
}

// NewRawExporter returns a new prometheus exporter for prometheus metrics
// for use in a pipeline.
func NewRawExporter(config Config) (*Exporter, error) {
	if config.Registry == nil {
		config.Registry = prometheus.NewRegistry()
	}

	if config.Registerer == nil {
		config.Registerer = config.Registry
	}

	if config.Gatherer == nil {
		config.Gatherer = config.Registry
	}

	if config.OnError == nil {
		config.OnError = func(err error) {
			fmt.Println(err.Error())
		}
	}

	e := &Exporter{
		handler:                 promhttp.HandlerFor(config.Gatherer, promhttp.HandlerOpts{}),
		registerer:              config.Registerer,
		gatherer:                config.Gatherer,
		defaultSummaryQuantiles: config.DefaultSummaryQuantiles,
		onError:                 config.OnError,
	}

	c := newCollector(e)
	if err := config.Registerer.Register(c); err != nil {
		config.OnError(fmt.Errorf("cannot register the collector: %w", err))
	}

	return e, nil
}

// InstallNewPipeline instantiates a NewExportPipeline and registers it globally.
// Typically called as:
// pipeline, hf, err := prometheus.InstallNewPipeline(prometheus.Config{...})
// if err != nil {
// 	...
// }
// http.HandleFunc("/metrics", hf)
// defer pipeline.Stop()
// ... Done
func InstallNewPipeline(config Config) (*push.Controller, http.HandlerFunc, error) {
	controller, hf, err := NewExportPipeline(config)
	if err != nil {
		return controller, hf, err
	}
	global.SetMeterProvider(controller)
	return controller, hf, err
}

// NewExportPipeline sets up a complete export pipeline with the recommended setup,
// chaining a NewRawExporter into the recommended selectors and batchers.
func NewExportPipeline(config Config) (*push.Controller, http.HandlerFunc, error) {
	selector := simple.NewWithExactMeasure()
	exporter, err := NewRawExporter(config)
	if err != nil {
		return nil, nil, err
	}

	// Prometheus needs to use a stateful batcher since counters (and histogram since they are a collection of Counters)
	// are cumulative (i.e., monotonically increasing values) and should not be resetted after each export.
	//
	// Prometheus uses this approach to be resilient to scrape failures.
	// If a Prometheus server tries to scrape metrics from a host and fails for some reason,
	// it could try again on the next scrape and no data would be lost, only resolution.
	//
	// Gauges (or LastValues) and Summaries are an exception to this and have different behaviors.
	batcher := defaultkeys.New(selector, sdkmetric.NewDefaultLabelEncoder(), true)
	pusher := push.New(batcher, exporter, time.Second)
	pusher.Start()

	return pusher, exporter.ServeHTTP, nil
}

// Export exports the provide metric record to prometheus.
func (e *Exporter) Export(_ context.Context, checkpointSet export.CheckpointSet) error {
	e.snapshot = checkpointSet
	return nil
}

// collector implements prometheus.Collector interface.
type collector struct {
	exp *Exporter
}

var _ prometheus.Collector = (*collector)(nil)

func newCollector(exporter *Exporter) *collector {
	return &collector{
		exp: exporter,
	}
}

func (c *collector) Describe(ch chan<- *prometheus.Desc) {
	if c.exp.snapshot == nil {
		return
	}

	c.exp.snapshot.ForEach(func(record export.Record) {
		ch <- c.toDesc(&record)
	})
}

// Collect exports the last calculated CheckpointSet.
//
// Collect is invoked whenever prometheus.Gatherer is also invoked.
// For example, when the HTTP endpoint is invoked by Prometheus.
func (c *collector) Collect(ch chan<- prometheus.Metric) {
	if c.exp.snapshot == nil {
		return
	}

	c.exp.snapshot.ForEach(func(record export.Record) {
		agg := record.Aggregator()
		numberKind := record.Descriptor().NumberKind()
		labels := labelValues(record.Labels())
		desc := c.toDesc(&record)

		// TODO: implement histogram export when the histogram aggregation is done.
		//  https://github.com/open-telemetry/opentelemetry-go/issues/317

		if dist, ok := agg.(aggregator.Distribution); ok {
			// TODO: summaries values are never being resetted.
			//  As measures are recorded, new records starts to have less impact on these summaries.
			//  We should implement an solution that is similar to the Prometheus Clients
			//  using a rolling window for summaries could be a solution.
			//
			//  References:
			// 	https://www.robustperception.io/how-does-a-prometheus-summary-work
			//  https://github.com/prometheus/client_golang/blob/fa4aa9000d2863904891d193dea354d23f3d712a/prometheus/summary.go#L135
			c.exportSummary(ch, dist, numberKind, desc, labels)
		} else if sum, ok := agg.(aggregator.Sum); ok {
			c.exportCounter(ch, sum, numberKind, desc, labels)
		} else if gauge, ok := agg.(aggregator.LastValue); ok {
			c.exportGauge(ch, gauge, numberKind, desc, labels)
		}
	})
}

func (c *collector) exportGauge(ch chan<- prometheus.Metric, gauge aggregator.LastValue, kind core.NumberKind, desc *prometheus.Desc, labels []string) {
	lastValue, _, err := gauge.LastValue()
	if err != nil {
		c.exp.onError(err)
		return
	}

	m, err := prometheus.NewConstMetric(desc, prometheus.GaugeValue, lastValue.CoerceToFloat64(kind), labels...)
	if err != nil {
		c.exp.onError(err)
		return
	}

	ch <- m
}

func (c *collector) exportCounter(ch chan<- prometheus.Metric, sum aggregator.Sum, kind core.NumberKind, desc *prometheus.Desc, labels []string) {
	v, err := sum.Sum()
	if err != nil {
		c.exp.onError(err)
		return
	}

	m, err := prometheus.NewConstMetric(desc, prometheus.CounterValue, v.CoerceToFloat64(kind), labels...)
	if err != nil {
		c.exp.onError(err)
		return
	}

	ch <- m
}

func (c *collector) exportSummary(ch chan<- prometheus.Metric, dist aggregator.Distribution, kind core.NumberKind, desc *prometheus.Desc, labels []string) {
	count, err := dist.Count()
	if err != nil {
		c.exp.onError(err)
		return
	}

	var sum core.Number
	sum, err = dist.Sum()
	if err != nil {
		c.exp.onError(err)
		return
	}

	quantiles := make(map[float64]float64)
	for _, quantile := range c.exp.defaultSummaryQuantiles {
		q, _ := dist.Quantile(quantile)
		quantiles[quantile] = q.CoerceToFloat64(kind)
	}

	m, err := prometheus.NewConstSummary(desc, uint64(count), sum.CoerceToFloat64(kind), quantiles, labels...)
	if err != nil {
		c.exp.onError(err)
		return
	}

	ch <- m
}

func (c *collector) toDesc(metric *export.Record) *prometheus.Desc {
	desc := metric.Descriptor()
	labels := labelsKeys(metric.Labels())
	return prometheus.NewDesc(sanitize(desc.Name()), desc.Description(), labels, nil)
}

func (e *Exporter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	e.handler.ServeHTTP(w, r)
}

func labelsKeys(labels export.Labels) []string {
	keys := make([]string, 0, labels.Len())
	for _, kv := range labels.Ordered() {
		keys = append(keys, sanitize(string(kv.Key)))
	}
	return keys
}

func labelValues(labels export.Labels) []string {
	// TODO(paivagustavo): parse the labels.Encoded() instead of calling `Emit()` directly
	//  this would avoid unnecessary allocations.
	values := make([]string, 0, labels.Len())
	for _, label := range labels.Ordered() {
		values = append(values, label.Value.Emit())
	}
	return values
}
<commit_msg>exporter/metric/prometheus: fix incorrect code format comments for InstallNewPipeline (#482)<commit_after>// Copyright 2019, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import (
	"context"
	"fmt"
	"net/http"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	"go.opentelemetry.io/otel/api/core"
	"go.opentelemetry.io/otel/api/global"
	export "go.opentelemetry.io/otel/sdk/export/metric"
	"go.opentelemetry.io/otel/sdk/export/metric/aggregator"
	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
	"go.opentelemetry.io/otel/sdk/metric/batcher/defaultkeys"
	"go.opentelemetry.io/otel/sdk/metric/controller/push"
	"go.opentelemetry.io/otel/sdk/metric/selector/simple"
)

// Exporter is an implementation of metric.Exporter that sends metrics to
// Prometheus.
type Exporter struct {
	handler http.Handler

	registerer prometheus.Registerer
	gatherer   prometheus.Gatherer

	snapshot export.CheckpointSet
	onError  func(error)

	defaultSummaryQuantiles []float64
}

var _ export.Exporter = &Exporter{}
var _ http.Handler = &Exporter{}

// Config is a set of configs for the tally reporter.
type Config struct {
	// Registry is the prometheus registry that will be used as the default Registerer and
	// Gatherer if these are not specified.
	//
	// If not set a new empty Registry is created.
	Registry *prometheus.Registry

	// Registerer is the prometheus registerer to register
	// metrics with.
	//
	// If not specified the Registry will be used as default.
	Registerer prometheus.Registerer

	// Gatherer is the prometheus gatherer to gather
	// metrics with.
	//
	// If not specified the Registry will be used as default.
	Gatherer prometheus.Gatherer

	// DefaultSummaryQuantiles is the default summary quantiles
	// to use. Use nil to specify the system-default summary quantiles.
	DefaultSummaryQuantiles []float64

	// OnError is a function that handle errors that may occur while exporting metrics.
	// TODO: This should be refactored or even removed once we have a better error handling mechanism.
	OnError func(error)
}

// NewRawExporter returns a new prometheus exporter for prometheus metrics
// for use in a pipeline.
func NewRawExporter(config Config) (*Exporter, error) {
	if config.Registry == nil {
		config.Registry = prometheus.NewRegistry()
	}

	if config.Registerer == nil {
		config.Registerer = config.Registry
	}

	if config.Gatherer == nil {
		config.Gatherer = config.Registry
	}

	if config.OnError == nil {
		config.OnError = func(err error) {
			fmt.Println(err.Error())
		}
	}

	e := &Exporter{
		handler:                 promhttp.HandlerFor(config.Gatherer, promhttp.HandlerOpts{}),
		registerer:              config.Registerer,
		gatherer:                config.Gatherer,
		defaultSummaryQuantiles: config.DefaultSummaryQuantiles,
		onError:                 config.OnError,
	}

	c := newCollector(e)
	if err := config.Registerer.Register(c); err != nil {
		config.OnError(fmt.Errorf("cannot register the collector: %w", err))
	}

	return e, nil
}

// InstallNewPipeline instantiates a NewExportPipeline and registers it globally.
// Typically called as:
//
// 	pipeline, hf, err := prometheus.InstallNewPipeline(prometheus.Config{...})
//
// 	if err != nil {
// 		...
// 	}
// 	http.HandleFunc("/metrics", hf)
// 	defer pipeline.Stop()
// 	... Done
func InstallNewPipeline(config Config) (*push.Controller, http.HandlerFunc, error) {
	controller, hf, err := NewExportPipeline(config)
	if err != nil {
		return controller, hf, err
	}
	global.SetMeterProvider(controller)
	return controller, hf, err
}

// NewExportPipeline sets up a complete export pipeline with the recommended setup,
// chaining a NewRawExporter into the recommended selectors and batchers.
func NewExportPipeline(config Config) (*push.Controller, http.HandlerFunc, error) {
	selector := simple.NewWithExactMeasure()
	exporter, err := NewRawExporter(config)
	if err != nil {
		return nil, nil, err
	}

	// Prometheus needs to use a stateful batcher since counters (and histogram since they are a collection of Counters)
	// are cumulative (i.e., monotonically increasing values) and should not be resetted after each export.
	//
	// Prometheus uses this approach to be resilient to scrape failures.
	// If a Prometheus server tries to scrape metrics from a host and fails for some reason,
	// it could try again on the next scrape and no data would be lost, only resolution.
	//
	// Gauges (or LastValues) and Summaries are an exception to this and have different behaviors.
	batcher := defaultkeys.New(selector, sdkmetric.NewDefaultLabelEncoder(), true)
	pusher := push.New(batcher, exporter, time.Second)
	pusher.Start()

	return pusher, exporter.ServeHTTP, nil
}

// Export exports the provide metric record to prometheus.
func (e *Exporter) Export(_ context.Context, checkpointSet export.CheckpointSet) error {
	e.snapshot = checkpointSet
	return nil
}

// collector implements prometheus.Collector interface.
type collector struct {
	exp *Exporter
}

var _ prometheus.Collector = (*collector)(nil)

func newCollector(exporter *Exporter) *collector {
	return &collector{
		exp: exporter,
	}
}

func (c *collector) Describe(ch chan<- *prometheus.Desc) {
	if c.exp.snapshot == nil {
		return
	}

	c.exp.snapshot.ForEach(func(record export.Record) {
		ch <- c.toDesc(&record)
	})
}

// Collect exports the last calculated CheckpointSet.
//
// Collect is invoked whenever prometheus.Gatherer is also invoked.
// For example, when the HTTP endpoint is invoked by Prometheus.
func (c *collector) Collect(ch chan<- prometheus.Metric) {
	if c.exp.snapshot == nil {
		return
	}

	c.exp.snapshot.ForEach(func(record export.Record) {
		agg := record.Aggregator()
		numberKind := record.Descriptor().NumberKind()
		labels := labelValues(record.Labels())
		desc := c.toDesc(&record)

		// TODO: implement histogram export when the histogram aggregation is done.
		//  https://github.com/open-telemetry/opentelemetry-go/issues/317

		if dist, ok := agg.(aggregator.Distribution); ok {
			// TODO: summaries values are never being resetted.
			//  As measures are recorded, new records starts to have less impact on these summaries.
			//  We should implement an solution that is similar to the Prometheus Clients
			//  using a rolling window for summaries could be a solution.
			//
			//  References:
			// 	https://www.robustperception.io/how-does-a-prometheus-summary-work
			//  https://github.com/prometheus/client_golang/blob/fa4aa9000d2863904891d193dea354d23f3d712a/prometheus/summary.go#L135
			c.exportSummary(ch, dist, numberKind, desc, labels)
		} else if sum, ok := agg.(aggregator.Sum); ok {
			c.exportCounter(ch, sum, numberKind, desc, labels)
		} else if gauge, ok := agg.(aggregator.LastValue); ok {
			c.exportGauge(ch, gauge, numberKind, desc, labels)
		}
	})
}

func (c *collector) exportGauge(ch chan<- prometheus.Metric, gauge aggregator.LastValue, kind core.NumberKind, desc *prometheus.Desc, labels []string) {
	lastValue, _, err := gauge.LastValue()
	if err != nil {
		c.exp.onError(err)
		return
	}

	m, err := prometheus.NewConstMetric(desc, prometheus.GaugeValue, lastValue.CoerceToFloat64(kind), labels...)
	if err != nil {
		c.exp.onError(err)
		return
	}

	ch <- m
}

func (c *collector) exportCounter(ch chan<- prometheus.Metric, sum aggregator.Sum, kind core.NumberKind, desc *prometheus.Desc, labels []string) {
	v, err := sum.Sum()
	if err != nil {
		c.exp.onError(err)
		return
	}

	m, err := prometheus.NewConstMetric(desc, prometheus.CounterValue, v.CoerceToFloat64(kind), labels...)
	if err != nil {
		c.exp.onError(err)
		return
	}

	ch <- m
}

func (c *collector) exportSummary(ch chan<- prometheus.Metric, dist aggregator.Distribution, kind core.NumberKind, desc *prometheus.Desc, labels []string) {
	count, err := dist.Count()
	if err != nil {
		c.exp.onError(err)
		return
	}

	var sum core.Number
	sum, err = dist.Sum()
	if err != nil {
		c.exp.onError(err)
		return
	}

	quantiles := make(map[float64]float64)
	for _, quantile := range c.exp.defaultSummaryQuantiles {
		q, _ := dist.Quantile(quantile)
		quantiles[quantile] = q.CoerceToFloat64(kind)
	}

	m, err := prometheus.NewConstSummary(desc, uint64(count), sum.CoerceToFloat64(kind), quantiles, labels...)
	if err != nil {
		c.exp.onError(err)
		return
	}

	ch <- m
}

func (c *collector) toDesc(metric *export.Record) *prometheus.Desc {
	desc := metric.Descriptor()
	labels := labelsKeys(metric.Labels())
	return prometheus.NewDesc(sanitize(desc.Name()), desc.Description(), labels, nil)
}

func (e *Exporter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	e.handler.ServeHTTP(w, r)
}

func labelsKeys(labels export.Labels) []string {
	keys := make([]string, 0, labels.Len())
	for _, kv := range labels.Ordered() {
		keys = append(keys, sanitize(string(kv.Key)))
	}
	return keys
}

func labelValues(labels export.Labels) []string {
	// TODO(paivagustavo): parse the labels.Encoded() instead of calling `Emit()` directly
	//  this would avoid unnecessary allocations.
	values := make([]string, 0, labels.Len())
	for _, label := range labels.Ordered() {
		values = append(values, label.Value.Emit())
	}
	return values
}
<|endoftext|>


<commit_before>package google

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/davecgh/go-spew/spew"
	"github.com/hashicorp/go-retryablehttp"
	"github.com/pkg/errors"
)

func buildClient() *retryablehttp.Client {
	httpClient := retryablehttp.NewClient()

	// Stop debug logger
	httpClient.Logger = nil
	httpClient.RetryMax = 1
	httpClient.RetryWaitMin = 30 * time.Second
	httpClient.RetryWaitMax = 2 * time.Minute

	return httpClient
}

func buildRequest(langFrom, langTo, inputText string) (*retryablehttp.Request, error) {
	var URL *url.URL
	URL, err := url.Parse("https://translate.googleapis.com/translate_a/single")

	parameters := url.Values{}
	parameters.Add("client", "gtx") // Google translate extension
	parameters.Add("dt", "t")       // Translate text
	parameters.Add("hl", "en")      // Interface language
	parameters.Add("sl", langFrom)  // Source language or "auto"
	parameters.Add("tl", langTo)    // Target language
	parameters.Add("ie", "UTF-8")   // Input encoding
	parameters.Add("oe", "UTF-8")   // Output encoding
	parameters.Add("q", inputText)  // Source text

	URL.RawQuery = parameters.Encode()

	r, err := retryablehttp.NewRequest("GET", URL.String(), nil)
	if err != nil {
		return r, errors.Wrap(err, "Failed to create request")
	}

	return r, err
}

func (q *batchTranslator) translateBatch(items []inputObject) {
	q.lastBatch = time.Now()

	//log.Debug("Items:", spew.Sdump(items))

	log.Infof("processing %d items", len(items))

	if len(items) < 1 {
		log.Debug("nothing to do")
		return
	}

	// TODO: Add support for different language pairs
	err := q.translateItems(items)
	if err != nil {
		// Send error to all items
		for _, i := range items {
			i.outChan <- returnObject{
				text: "",
				err:  err,
			}
		}
	}
}

func (q *batchTranslator) translateItems(items []inputObject) error {
	// Join every input separated by newline
	var reqText string
	for _, i := range items {
		reqText += i.req.Text + "\n"
	}

	r, err := buildRequest(items[0].req.From, items[0].req.To, reqText)
	if err != nil {
		return err
	}

	resp, err := q.client.Do(r)
	if err != nil {
		log.Fatal(err)

		return errors.Wrapf(err, "Failed to do request (%s delay)", q.batchDelay)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		if contents, err := ioutil.ReadAll(resp.Body); err == nil {
			return fmt.Errorf("%s - %s", resp.Status, contents)
		}

		return fmt.Errorf("%s", resp.Status)
	}

	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return errors.Wrap(err, "Failed to read response body")
	}

	response, err := decodeResponse(string(contents))
	if err != nil {
		log.Error("Unknown response: %q", string(contents))
		return errors.Wrap(err, "Failed to decode response json")
	}

	if len(items) != len(response) {
		log.Fatal("Response pairs doesn't match input", spew.Sdump(response), spew.Sdump(items))
	}

	for i, pair := range response {
		if strings.TrimSpace(pair.input) != strings.TrimSpace(items[i].req.Text) {
			items[i].outChan <- returnObject{
				text: pair.input,
				err:  fmt.Errorf("mismatched input text! %q != %q", items[i].req.Text, pair.input),
			}

			continue
		}

		items[i].outChan <- returnObject{
			text: pair.output,
			err:  nil,
		}
	}

	return nil
}
<commit_msg>Fix mising format<commit_after>package google

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/davecgh/go-spew/spew"
	"github.com/hashicorp/go-retryablehttp"
	"github.com/pkg/errors"
)

func buildClient() *retryablehttp.Client {
	httpClient := retryablehttp.NewClient()

	// Stop debug logger
	httpClient.Logger = nil
	httpClient.RetryMax = 1
	httpClient.RetryWaitMin = 30 * time.Second
	httpClient.RetryWaitMax = 2 * time.Minute

	return httpClient
}

func buildRequest(langFrom, langTo, inputText string) (*retryablehttp.Request, error) {
	var URL *url.URL
	URL, err := url.Parse("https://translate.googleapis.com/translate_a/single")

	parameters := url.Values{}
	parameters.Add("client", "gtx") // Google translate extension
	parameters.Add("dt", "t")       // Translate text
	parameters.Add("hl", "en")      // Interface language
	parameters.Add("sl", langFrom)  // Source language or "auto"
	parameters.Add("tl", langTo)    // Target language
	parameters.Add("ie", "UTF-8")   // Input encoding
	parameters.Add("oe", "UTF-8")   // Output encoding
	parameters.Add("q", inputText)  // Source text

	URL.RawQuery = parameters.Encode()

	r, err := retryablehttp.NewRequest("GET", URL.String(), nil)
	if err != nil {
		return r, errors.Wrap(err, "Failed to create request")
	}

	return r, err
}

func (q *batchTranslator) translateBatch(items []inputObject) {
	q.lastBatch = time.Now()

	//log.Debug("Items:", spew.Sdump(items))

	log.Infof("processing %d items", len(items))

	if len(items) < 1 {
		log.Debug("nothing to do")
		return
	}

	// TODO: Add support for different language pairs
	err := q.translateItems(items)
	if err != nil {
		// Send error to all items
		for _, i := range items {
			i.outChan <- returnObject{
				text: "",
				err:  err,
			}
		}
	}
}

func (q *batchTranslator) translateItems(items []inputObject) error {
	// Join every input separated by newline
	var reqText string
	for _, i := range items {
		reqText += i.req.Text + "\n"
	}

	r, err := buildRequest(items[0].req.From, items[0].req.To, reqText)
	if err != nil {
		return err
	}

	resp, err := q.client.Do(r)
	if err != nil {
		log.Fatal(err)

		return errors.Wrapf(err, "Failed to do request (%s delay)", q.batchDelay)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		if contents, err := ioutil.ReadAll(resp.Body); err == nil {
			return fmt.Errorf("%s - %s", resp.Status, contents)
		}

		return fmt.Errorf("%s", resp.Status)
	}

	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return errors.Wrap(err, "Failed to read response body")
	}

	response, err := decodeResponse(string(contents))
	if err != nil {
		log.Errorf("Unknown response: %q", string(contents))
		return errors.Wrap(err, "Failed to decode response json")
	}

	if len(items) != len(response) {
		log.Fatal("Response pairs doesn't match input", spew.Sdump(response), spew.Sdump(items))
	}

	for i, pair := range response {
		if strings.TrimSpace(pair.input) != strings.TrimSpace(items[i].req.Text) {
			items[i].outChan <- returnObject{
				text: pair.input,
				err:  fmt.Errorf("mismatched input text! %q != %q", items[i].req.Text, pair.input),
			}

			continue
		}

		items[i].outChan <- returnObject{
			text: pair.output,
			err:  nil,
		}
	}

	return nil
}
<|endoftext|>