<commit_before>package ics

import (
	"strings"
	"time"
)

type completed struct {
	time.Time
}

func (p *parser) readCompletedProperty() (property, error) {
	v, err := p.readValue()
	if err != nil {
		return nil, err
	}
	var c completed
	c.Time, err = time.ParseInLocation("20060102T150405Z", v, time.UTC)
	if err != nil {
		return nil, err
	}
	return c, nil
}

type dateTimeEnd struct {
	dateTime
}

func (p *parser) readDateTimeOrTime() (t dateTime, err error) {
	as, err := p.readAttributes(tzidparam, valuetypeparam)
	if err != nil {
		return t, err
	}
	var (
		l        *time.Location
		justDate bool
	)
	if tzid, ok := as[tzidparam]; ok {
		l, err = time.LoadLocation(tzid.String())
		if err != nil {
			return t, err
		}
	}
	if v, ok := as[valuetypeparam]; ok {
		val := v.(value)
		switch val {
		case valueDate:
			justDate = true
		case valueDateTime:
			justDate = false
		default:
			return t, ErrUnsupportedValue
		}
	}
	v, err := p.readValue()
	if err != nil {
		return t, err
	}
	if justDate {
		t, err = parseDate(v)
	} else {
		t, err = parseDateTime(v, l)
	}
	return t, err
}

func (p *parser) readDateTimeEndProperty() (property, error) {
	t, err := p.readDateTimeOrTime()
	if err != nil {
		return nil, err
	}
	return dateTimeEnd{t}, nil
}

type dateTimeDue struct {
	dateTime dateTime
}

func (p *parser) readDateTimeDueProperty() (property, error) {
	t, err := p.readDateTimeOrTime()
	if err != nil {
		return nil, err
	}
	return dateTimeDue{t}, nil
}

type dateTimeStart struct {
	dateTime dateTime
}

func (p *parser) readDateTimeStartProperty() (property, error) {
	t, err := p.readDateTimeOrTime()
	if err != nil {
		return nil, err
	}
	return dateTimeStart{t}, nil
}

type duration struct {
	time.Duration
}

func (p *parser) readDurationProperty() (property, error) {
	v, err := p.readValue()
	if err != nil {
		return nil, err
	}
	var d duration
	d.Duration, err = parseDuration(v)
	if err != nil {
		return nil, err
	}
	return d, nil
}

type freeBusyTime struct {
	Typ     freeBusy
	Periods []period
}

type period struct {
	FixedDuration bool
	Start, End    dateTime
}

func parsePeriods(v string, l *time.Location) ([]period, error) {
	periods := make([]period, 0, 1)

	for _, pd := range textSplit(v, ',') {
		parts := strings.Split(pd, "/")
		if len(parts) != 2 {
			return nil, ErrUnsupportedValue
		}
		if parts[0][len(parts[0])-1] != 'Z' {
			return nil, ErrUnsupportedValue
		}
		start, err := parseDateTime(parts[0], l)
		if err != nil {
			return nil, err
		}
		var (
			end           dateTime
			fixedDuration bool
		)
		if parts[1][len(parts[1])-1] == 'Z' {
			end, err = parseDateTime(parts[1], l)
			if err != nil {
				return nil, err
			}
		} else {
			d, err := parseDuration(parts[1])
			if err != nil {
				return nil, err
			}
			if d < 0 {
				return nil, ErrUnsupportedValue
			}
			end = start.Add(d)
			fixedDuration = true
		}
		periods = append(periods, period{fixedDuration, start, end})
	}
	return periods, nil
}

func (p *parser) readFreeBusyTimeProperty() (property, error) {
	as, err := p.readAttributes(fbtypeparam)
	if err != nil {
		return nil, err
	}
	var fb freeBusy
	if f, ok := as[fbtypeparam]; ok {
		fb = f.(freeBusy)
	}
	v, err := p.readValue()
	if err != nil {
		return nil, err
	}
	periods, err := parsePeriods(v, nil)
	if err != nil {
		return nil, err
	}
	return freeBusyTime{
		Typ:     fb,
		Periods: periods,
	}, nil
}

const (
	TTOpaque timeTransparency = iota
	TTTransparent
)

type timeTransparency int

func (p *parser) readTimeTransparencyProperty() (property, error) {
	v, err := p.readValue()
	if err != nil {
		return nil, err
	}
	switch v {
	case "OPAQUE":
		return TTOpaque, nil
	case "TRANSPARENT":
		return TTTransparent, nil
	default:
		return nil, ErrUnsupportedValue
	}
}
<commit_msg>added Validate and Data methods to date/time properties<commit_after>package ics

import (
	"strings"
	"time"
)

type completed struct {
	dateTime
}

func (p *parser) readCompletedProperty() (property, error) {
	v, err := p.readValue()
	if err != nil {
		return nil, err
	}
	var c completed
	c.Time, err = time.ParseInLocation("20060102T150405Z", v, time.UTC)
	if err != nil {
		return nil, err
	}
	return c, nil
}

func (c completed) Validate() bool {
	return c.Location() == time.UTC
}

func (c completed) Data() propertyData {
	return propertyData{
		Name:  completedp,
		Value: c.String(),
	}
}

type dateTimeEnd struct {
	dateTime
}

func (p *parser) readDateTimeOrTime() (t dateTime, err error) {
	as, err := p.readAttributes(tzidparam, valuetypeparam)
	if err != nil {
		return t, err
	}
	var (
		l        *time.Location
		justDate bool
	)
	if tzid, ok := as[tzidparam]; ok {
		l, err = time.LoadLocation(tzid.String())
		if err != nil {
			return t, err
		}
	}
	if v, ok := as[valuetypeparam]; ok {
		val := v.(value)
		switch val {
		case valueDate:
			justDate = true
		case valueDateTime:
			justDate = false
		default:
			return t, ErrUnsupportedValue
		}
	}
	v, err := p.readValue()
	if err != nil {
		return t, err
	}
	if justDate {
		t, err = parseDate(v)
	} else {
		t, err = parseDateTime(v, l)
	}
	return t, err
}

func dateTimeOrTimeData(name string, d dateTime) propertyData {
	params := make(map[string]attribute)
	if d.justDate {
		params[valuetypeparam] = valueDate
	}
	if d.Location() != time.UTC {
		params[tzidparam] = d.Location().String()
	}
	return propertyData{
		Name:   name,
		Params: params,
		Value:  d.String(),
	}
}

func (p *parser) readDateTimeEndProperty() (property, error) {
	t, err := p.readDateTimeOrTime()
	if err != nil {
		return nil, err
	}
	return dateTimeEnd{t}, nil
}

func (d dateTimeEnd) Validate() bool {
	return true
}

func (d dateTimeEnd) Date() propertyData {
	return dateTimeOrTimeData(dtendp, d.dateTime)
}

type dateTimeDue struct {
	dateTime dateTime
}

func (p *parser) readDateTimeDueProperty() (property, error) {
	t, err := p.readDateTimeOrTime()
	if err != nil {
		return nil, err
	}
	return dateTimeDue{t}, nil
}

func (d dateTimeDue) Validate() bool {
	return true
}

func (d dateTimeDue) Data() propertyData {
	return dateTimeOrTimeData(duep, d.dateTime)
}

type dateTimeStart struct {
	dateTime dateTime
}

func (p *parser) readDateTimeStartProperty() (property, error) {
	t, err := p.readDateTimeOrTime()
	if err != nil {
		return nil, err
	}
	return dateTimeStart{t}, nil
}

func (d dateTimeStart) Validate() bool {
	return true
}

func (d dateTimeStart) Data() propertyData {
	return dateTimeOrTimeData(dtstartp, d.dateTime)
}

type duration struct {
	time.Duration
}

func (p *parser) readDurationProperty() (property, error) {
	v, err := p.readValue()
	if err != nil {
		return nil, err
	}
	var d duration
	d.Duration, err = parseDuration(v)
	if err != nil {
		return nil, err
	}
	return d, nil
}

func (d duration) Validate() bool {
	return true
}

func (d duration) Data() propertyData {
	return propertyData{
		Name:  durationp,
		Value: durationString(d.Duration),
	}
}

type freeBusyTime struct {
	Typ     freeBusy
	Periods []period
}

type period struct {
	FixedDuration bool
	Start, End    dateTime
}

func parsePeriods(v string, l *time.Location) ([]period, error) {
	periods := make([]period, 0, 1)

	for _, pd := range textSplit(v, ',') {
		parts := strings.Split(pd, "/")
		if len(parts) != 2 {
			return nil, ErrUnsupportedValue
		}
		if parts[0][len(parts[0])-1] != 'Z' {
			return nil, ErrUnsupportedValue
		}
		start, err := parseDateTime(parts[0], l)
		if err != nil {
			return nil, err
		}
		var (
			end           dateTime
			fixedDuration bool
		)
		if parts[1][len(parts[1])-1] == 'Z' {
			end, err = parseDateTime(parts[1], l)
			if err != nil {
				return nil, err
			}
		} else {
			d, err := parseDuration(parts[1])
			if err != nil {
				return nil, err
			}
			if d < 0 {
				return nil, ErrUnsupportedValue
			}
			end = start.Add(d)
			fixedDuration = true
		}
		periods = append(periods, period{fixedDuration, start, end})
	}
	return periods, nil
}

func (p *parser) readFreeBusyTimeProperty() (property, error) {
	as, err := p.readAttributes(fbtypeparam)
	if err != nil {
		return nil, err
	}
	var fb freeBusy
	if f, ok := as[fbtypeparam]; ok {
		fb = f.(freeBusy)
	}
	v, err := p.readValue()
	if err != nil {
		return nil, err
	}
	periods, err := parsePeriods(v, nil)
	if err != nil {
		return nil, err
	}
	return freeBusyTime{
		Typ:     fb,
		Periods: periods,
	}, nil
}

func (f freeBusyTime) Validate() bool {
	return f.Typ >= fbBusy && f.Typ <= fbBusyTentative
}

func (f freeBusyTime) Data() propertyData {
	params := make(map[string]attribute)
	params[fbtypeparam] = f.Typ
	var val []byte
	for _, period := range f.Periods {
		val = append(val, ',')
		val = append(val, period.Start.String()...)
		val = append(val, '/')
		if period.FixedDuration {
			val = append(val, durationString(period.End.Sub(period.Start.Time))...)
		} else {
			val = append(val, period.End.String()...)
		}
	}
	return propertyData{
		Name:   freebusyp,
		Params: params,
		Value:  string(val),
	}
}

const (
	TTOpaque timeTransparency = iota
	TTTransparent
)

type timeTransparency int

func (p *parser) readTimeTransparencyProperty() (property, error) {
	v, err := p.readValue()
	if err != nil {
		return nil, err
	}
	switch v {
	case "OPAQUE":
		return TTOpaque, nil
	case "TRANSPARENT":
		return TTTransparent, nil
	default:
		return nil, ErrUnsupportedValue
	}
}

func (t timeTransparency) String() string {
	switch t {
	case TTOpaque:
		return "OPAQUE"
	case TTTransparent:
		return "TRANSPARENT"
	default:
		return "UNKNOWN"
	}
}

func (t timeTransparency) Validate() bool {
	switch t {
	case TTOpaque, TTTransparent:
		return true
	default:
		return false
	}
}

func (t timeTransparency) Data() propertyData {
	return propertyData{
		Name:  transpp,
		Value: t.String(),
	}
}
<|endoftext|>


<commit_before>/*
Package fasttext provides a simple wrapper for Facebook
fastText dataset (https://github.com/facebookresearch/fastText/blob/master/pretrained-vectors.md).
It allows fast look-up of word embeddings from persistent data store (Sqlite3).

Installation

	go get -u github.com/ekzhu/go-fasttext

After downloading a .vec data file from the fastText project,
you can initialize the Sqlite3 database (in your code):

	ft := NewFastText("/path/to/sqlite3/file")
	err := ft.BuilDB("/path/to/word/embedding/.vec/file")

This will create a new file on your disk for the Sqlite3 database.
Once the above step is finished, you can start looking up word embeddings
(in your code):

	emb, err := ft.GetEmb("king")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(emb.Word, emb.Vec)

Each word embedding vector is a slice of float64.

Note that you only need to initialize the Sqlite3 database once.
The next time you use it you can skip the call to BuildDB.
*/
package fasttext

import (
	"bufio"
	"database/sql"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"strconv"
	"strings"

	_ "github.com/mattn/go-sqlite3"
)

const (
	// Table name used in Sqlite
	TableName = "fasttext"
)

var (
	ErrNoEmbFound = errors.New("No embedding found for the given word")
	// TODO: parametrize byte order
	ByteOrder = binary.BigEndian
)

type FastText struct {
	db        *sql.DB
	tablename string
	byteOrder binary.ByteOrder
}

// WordEmb is a pair of word and its embedding vector.
type WordEmb struct {
	Word string
	Vec  []float64
}

// Start a new FastText session given the location
// of the Sqlite3 database file.
func NewFastText(dbFilename string) *FastText {
	db, err := sql.Open("sqlite3", dbFilename)
	if err != nil {
		panic(err)
	}
	return &FastText{
		db:        db,
		tablename: TableName,
		byteOrder: ByteOrder,
	}
}

// Close must be called before finishing using FastText
func (ft *FastText) Close() error {
	return ft.db.Close()
}

// GetEmb returns the word embedding of the given word.
func (ft *FastText) GetEmb(word string) (*WordEmb, error) {
	var binVec []byte
	err := ft.db.QueryRow(fmt.Sprintf(`
	SELECT emb FROM %s WHERE word=?;
	`, ft.tablename), word).Scan(&binVec)
	if err == sql.ErrNoRows {
		return nil, ErrNoEmbFound
	}
	if err != nil {
		panic(err)
	}
	vec, err := bytesToVec(binVec, ft.byteOrder)
	if err != nil {
		return nil, err
	}
	return &WordEmb{
		Word: word,
		Vec:  vec,
	}, nil
}

// BuilDB initialize the Sqlite database by importing the word embeddings
// from the .vec file downloaded from
// https://github.com/facebookresearch/fastText/blob/master/pretrained-vectors.md
func (ft *FastText) BuildDB(wordEmbFile io.Reader) error {
	_, err := ft.db.Exec(fmt.Sprintf(`
	CREATE TABLE %s (
		word TEXT UNIQUE,
		emb BLOB
	);
	`, ft.tablename))
	if err != nil {
		return err
	}
	stmt, err := ft.db.Prepare(fmt.Sprintf(`
	INSERT INTO %s(word, emb) VALUES(?, ?);
	`, ft.tablename))
	if err != nil {
		return err
	}
	defer stmt.Close()
	for emb := range readWordEmbdFile(wordEmbFile) {
		binVec := vecToBytes(emb.Vec, ft.byteOrder)
		if _, err := stmt.Exec(emb.Word, binVec); err != nil {
			return err
		}
	}
	// Indexing on words
	_, err = ft.db.Exec(fmt.Sprintf(`
	CREATE INDEX ind_word ON %s(word);
	`, ft.tablename))
	if err != nil {
		return err
	}
	return nil
}

func readWordEmbdFile(wordEmbFile io.Reader) chan *WordEmb {
	out := make(chan *WordEmb)
	go func() {
		defer close(out)
		scanner := bufio.NewScanner(wordEmbFile)
		var embSize int
		var line int
		for scanner.Scan() {
			line++
			data := scanner.Text()
			if embSize == 0 {
				var err error
				embSize, err = strconv.Atoi(strings.Split(data, " ")[1])
				if err != nil {
					panic(err)
				}
				continue
			}
			// Get the word
			items := strings.SplitN(data, " ", 2)
			word := items[0]
			if word == "" {
				word = " "
			}
			// Get the vec
			vecStrs := strings.Split(strings.TrimSpace(items[1]), " ")
			if len(vecStrs) != embSize {
				msg := fmt.Sprintf("Embedding vec size not same: expected %d, got %d. Loc: line %d, word %s",
					embSize, len(vecStrs), line, word)
				panic(msg)
			}
			vec := make([]float64, embSize)
			for i := 0; i < embSize; i++ {
				sf, err := strconv.ParseFloat(vecStrs[i], 64)
				if err != nil {
					panic(err)
				}
				vec[i] = sf
			}
			out <- &WordEmb{
				Word: word,
				Vec:  vec,
			}
		}
		if err := scanner.Err(); err != nil {
			panic(err)
		}
	}()
	return out
}
<commit_msg>add const<commit_after>/*
Package fasttext provides a simple wrapper for Facebook
fastText dataset (https://github.com/facebookresearch/fastText/blob/master/pretrained-vectors.md).
It allows fast look-up of word embeddings from persistent data store (Sqlite3).

Installation

	go get -u github.com/ekzhu/go-fasttext

After downloading a .vec data file from the fastText project,
you can initialize the Sqlite3 database (in your code):

	ft := NewFastText("/path/to/sqlite3/file")
	err := ft.BuilDB("/path/to/word/embedding/.vec/file")

This will create a new file on your disk for the Sqlite3 database.
Once the above step is finished, you can start looking up word embeddings
(in your code):

	emb, err := ft.GetEmb("king")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(emb.Word, emb.Vec)

Each word embedding vector is a slice of float64.

Note that you only need to initialize the Sqlite3 database once.
The next time you use it you can skip the call to BuildDB.
*/
package fasttext

import (
	"bufio"
	"database/sql"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"strconv"
	"strings"

	_ "github.com/mattn/go-sqlite3"
)

const (
	// Table name used in Sqlite
	TableName = "fasttext"
	Dim       = 300
)

var (
	ErrNoEmbFound = errors.New("No embedding found for the given word")
	// TODO: parametrize byte order
	ByteOrder = binary.BigEndian
)

type FastText struct {
	db        *sql.DB
	tablename string
	byteOrder binary.ByteOrder
}

// WordEmb is a pair of word and its embedding vector.
type WordEmb struct {
	Word string
	Vec  []float64
}

// Start a new FastText session given the location
// of the Sqlite3 database file.
func NewFastText(dbFilename string) *FastText {
	db, err := sql.Open("sqlite3", dbFilename)
	if err != nil {
		panic(err)
	}
	return &FastText{
		db:        db,
		tablename: TableName,
		byteOrder: ByteOrder,
	}
}

// Close must be called before finishing using FastText
func (ft *FastText) Close() error {
	return ft.db.Close()
}

// GetEmb returns the word embedding of the given word.
func (ft *FastText) GetEmb(word string) (*WordEmb, error) {
	var binVec []byte
	err := ft.db.QueryRow(fmt.Sprintf(`
	SELECT emb FROM %s WHERE word=?;
	`, ft.tablename), word).Scan(&binVec)
	if err == sql.ErrNoRows {
		return nil, ErrNoEmbFound
	}
	if err != nil {
		panic(err)
	}
	vec, err := bytesToVec(binVec, ft.byteOrder)
	if err != nil {
		return nil, err
	}
	return &WordEmb{
		Word: word,
		Vec:  vec,
	}, nil
}

// BuilDB initialize the Sqlite database by importing the word embeddings
// from the .vec file downloaded from
// https://github.com/facebookresearch/fastText/blob/master/pretrained-vectors.md
func (ft *FastText) BuildDB(wordEmbFile io.Reader) error {
	_, err := ft.db.Exec(fmt.Sprintf(`
	CREATE TABLE %s (
		word TEXT UNIQUE,
		emb BLOB
	);
	`, ft.tablename))
	if err != nil {
		return err
	}
	stmt, err := ft.db.Prepare(fmt.Sprintf(`
	INSERT INTO %s(word, emb) VALUES(?, ?);
	`, ft.tablename))
	if err != nil {
		return err
	}
	defer stmt.Close()
	for emb := range readWordEmbdFile(wordEmbFile) {
		binVec := vecToBytes(emb.Vec, ft.byteOrder)
		if _, err := stmt.Exec(emb.Word, binVec); err != nil {
			return err
		}
	}
	// Indexing on words
	_, err = ft.db.Exec(fmt.Sprintf(`
	CREATE INDEX ind_word ON %s(word);
	`, ft.tablename))
	if err != nil {
		return err
	}
	return nil
}

func readWordEmbdFile(wordEmbFile io.Reader) chan *WordEmb {
	out := make(chan *WordEmb)
	go func() {
		defer close(out)
		scanner := bufio.NewScanner(wordEmbFile)
		var embSize int
		var line int
		for scanner.Scan() {
			line++
			data := scanner.Text()
			if embSize == 0 {
				var err error
				embSize, err = strconv.Atoi(strings.Split(data, " ")[1])
				if err != nil {
					panic(err)
				}
				continue
			}
			// Get the word
			items := strings.SplitN(data, " ", 2)
			word := items[0]
			if word == "" {
				word = " "
			}
			// Get the vec
			vecStrs := strings.Split(strings.TrimSpace(items[1]), " ")
			if len(vecStrs) != embSize {
				msg := fmt.Sprintf("Embedding vec size not same: expected %d, got %d. Loc: line %d, word %s",
					embSize, len(vecStrs), line, word)
				panic(msg)
			}
			vec := make([]float64, embSize)
			for i := 0; i < embSize; i++ {
				sf, err := strconv.ParseFloat(vecStrs[i], 64)
				if err != nil {
					panic(err)
				}
				vec[i] = sf
			}
			out <- &WordEmb{
				Word: word,
				Vec:  vec,
			}
		}
		if err := scanner.Err(); err != nil {
			panic(err)
		}
	}()
	return out
}
<|endoftext|>


<commit_before>// Copyright 2014 Joel Scoble (github.com/mohae) All rights reserved.
// Use of this source code is governed by a BSD-style license that
// can be found in the LICENSE file.
//
package synchronicity

import (
	"bufio"
	"crypto/sha256"
	"fmt"
	"hash"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
)

// This allows for up to 128k of read data. If the file is larger than that,
// a different approach should be done, i.e. don't precompute the hash and use
// other rules for determining difference.
//
var MaxChunks = 16       // Modify directly to change buffered hashes
var chunkSize = int64(8 * 1024) // use 8k chunks as default

// SetChunkSize sets the chunkSize as 1k * i, i.e. 8 == 8k chunkSize
// If the multiplier, i, is <= 0, the default is used, 8.
func SetChunkSize(i int) {
	if i <= 0 {
		i = 8
	}
	chunkSize = int64(1024 * i)
}

const (
	invalid hashType = iota
	SHA256
)

type hashType int // Not really needed atm, but it'll be handy for adding other types.
var useHashType hashType

// SHA256 sized for hashed blocks.
type Hash256 [32]byte

func (h hashType) String() string {
	switch h {
	case SHA256:
		return "sha256"
	case invalid:
		return "invalid"
	}
	return "unknown"
}

func init() {
	useHashType = SHA256
}

type FileData struct {
	Processed bool
	Hashes    []Hash256
	HashType  hashType
	ChunkSize int64 // The chunksize that this was created with.
	MaxChunks int
	CurByte   int64  // for when the while file hasn't been hashed and
	Root      string // the relative root of this file: allows for synch support
	Dir       string // relative path to parent directory of Fi
	Fi        os.FileInfo
}

// Returns a FileData struct for the passed file using the defaults.
// Set any overrides before performing an operation.
func NewFileData(root, dir string, fi os.FileInfo) FileData {
	if dir == "." {
		dir = ""
	}
	fd := FileData{HashType: useHashType, ChunkSize: chunkSize, MaxChunks: MaxChunks, Root: root, Dir: dir, Fi: fi}
	return fd
}

// String is an alias to RelPath
func (fd *FileData) String() string {
	return fd.RelPath()
}

// SetHash computes the hash of the FileData. The path of the file is passed
// because FileData only knows it's name, not its location.
func (fd *FileData) SetHash() error {
	f, hasher, err := fd.getFileHasher()
	if err != nil {
		return err
	}
	if fd.ChunkSize == 0 {
		return fd.hashFile(f, hasher)
	}
	return fd.chunkedHashFile(f, hasher)
}

// getHasher returns a file and the hasher to use it on. If error, return that.
func (fd *FileData) getFileHasher() (f *os.File, hasher hash.Hash, err error) {
	f, err = os.Open(fd.RootPath())
	if err != nil {
		return
	}
	hasher, err = fd.getHasher()
	return
}

// getHasher returns a hasher or an error
func (fd *FileData) getHasher() (hasher hash.Hash, err error) {
	switch fd.HashType {
	case SHA256:
		hasher = sha256.New() //
	default:
		err = fmt.Errorf("%s hash type", fd.HashType.String())
		return
	}
	return
}

// hashFile hashes the entire file.
func (fd *FileData) hashFile(f *os.File, hasher hash.Hash) error {
	_, err := io.Copy(hasher, f)
	if err != nil {
		log.Printf("%s/n", err)
		return err
	}
	h := Hash256{}
	copy(h[:], hasher.Sum(nil))
	fd.Hashes = append(fd.Hashes, Hash256(h))
	return nil
}

// chunkedHashFile reads up to max chunks, or the entire file, whichever comes
// first.
func (fd *FileData) chunkedHashFile(f *os.File, hasher hash.Hash) (err error) {
	reader := bufio.NewReaderSize(f, int(fd.ChunkSize))
	var cnt int
	var bytes int64
	h := Hash256{}
	for cnt = 0; cnt < MaxChunks; cnt++ { // read until EOF || MaxChunks
		n, err := io.CopyN(hasher, reader, int64(fd.ChunkSize))
		if err != nil && err != io.EOF {
			return err
		}
		bytes += n
		copy(h[:], hasher.Sum(nil))
		fd.Hashes = append(fd.Hashes, Hash256(h))
	}
	if err != nil {
		log.Printf("%s\n", err)
		return err
	}
	_ = cnt
	return nil
}

// isEqual compares the current file with the passed file and returns
// whether or not they are equal. If the file length is greater than our
// checksum buffer, the rest of the file is read in chunks, until EOF or
// a difference is found, whichever comes first.
//
// If they are of different lengths, we assume they are different
func (fd *FileData) isEqual(dstFd FileData) (bool, error) {
	if fd.Fi.Size() != dstFd.Fi.Size() {
		return false, nil
	}
	// otherwise, examine the file contents
	f, hasher, err := fd.getFileHasher()
	if err != nil {
		return false, err
	}
	defer f.Close()
	// TODO support adaptive
	chunks := int(fd.Fi.Size()/int64(fd.ChunkSize) + 1)
	if chunks > len(fd.Hashes) {
		return fd.isEqualMixed(chunks, f, hasher, dstFd)
	}

	return fd.isEqualCached(chunks, f, hasher, dstFd)
}

func (fd *FileData) isEqualMixed(chunks int, f *os.File, hasher hash.Hash, dstFd FileData) (bool, error) {
	equal, err := fd.isEqualCached(chunks, f, hasher, dstFd)
	if err != nil {
		return equal, err
	}
	if !equal {
		return equal, nil
	}
	// Otherwise check the file from the current point
	dstF, err := os.Open(dstFd.RootPath())

	// Go to the last read byte
	pos, err := dstF.Seek(dstFd.CurByte, 0)
	if err != nil {
		return false, err
	}
	_ = pos
	dstHasher, err := fd.getHasher()
	if err != nil {
		return false, err
	}
	dH := Hash256{}
	sH := Hash256{}
	// Check until EOF or a difference is found
	for {
		s, err := io.CopyN(hasher, f, fd.ChunkSize)
		if err != nil {
			return false, err
		}
		d, err := io.CopyN(dstHasher, dstF, fd.ChunkSize)
		if err != nil {
			return false, err
		}
		if d != s { // if the bytes copied were different, return false
			return false, nil
		}
		copy(dH[:], dstHasher.Sum(nil))
		copy(sH[:], hasher.Sum(nil))
		if Hash256(dH) != Hash256(sH) {
			return false, nil
		}
	}
	return true, nil
}

// isEqualCached is called when the file fits within the maxChunks.
func (fd *FileData) isEqualCached(chunks int, f *os.File, hasher hash.Hash, dstFd FileData) (bool, error) {
	h := Hash256{}
	for i := 0; i < chunks; i++ {
		_, err := io.CopyN(hasher, f, fd.ChunkSize)
		if err != nil {
			return false, err
		}
		copy(h[:], hasher.Sum(nil))
		if Hash256(h) != dstFd.Hashes[i] {
			return false, nil
		}
	}
	return true, nil
}

// RelPath returns the relative path of the file, this is the file less the
// root information. This allows for easy comparision between two directories.
func (fd *FileData) RelPath() string {
	return filepath.Join(fd.Dir, fd.Fi.Name())
}

// RootPath returns the relative path of the file including its root. A root is
// the directory that Synchronicity considers a root, e.g. one of the
// directories being synched. This is not the FullPath of a file.
func (fd *FileData) RootPath() string {
	return filepath.Join(fd.Root, fd.Dir, fd.Fi.Name())
}

func SetHashType(s string) {
	useHashType = ParseHashType(s)
}

func ParseHashType(s string) hashType {
	s = strings.ToLower(s)
	switch s {
	case "sha256":
		return SHA256
	}
	return invalid
}

<commit_msg>get chunked comparison and handling of larger file comparison working<commit_after>// Copyright 2014 Joel Scoble (github.com/mohae) All rights reserved.
// Use of this source code is governed by a BSD-style license that
// can be found in the LICENSE file.
//
package synchronicity

import (
	"bufio"
	"crypto/sha256"
	"fmt"
	"hash"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
)

// This allows for up to 128k of read data. If the file is larger than that,
// a different approach should be done, i.e. don't precompute the hash and use
// other rules for determining difference.
//
var MaxChunks = 16       // Modify directly to change buffered hashes
var chunkSize = int64(8 * 1024) // use 8k chunks as default

// SetChunkSize sets the chunkSize as 1k * i, i.e. 8 == 8k chunkSize
// If the multiplier, i, is <= 0, the default is used, 8.
func SetChunkSize(i int) {
	if i <= 0 {
		i = 8
	}
	chunkSize = int64(1024 * i)
}

const (
	invalid hashType = iota
	SHA256
)

type hashType int // Not really needed atm, but it'll be handy for adding other types.
var useHashType hashType

// SHA256 sized for hashed blocks.
type Hash256 [32]byte

func (h hashType) String() string {
	switch h {
	case SHA256:
		return "sha256"
	case invalid:
		return "invalid"
	}
	return "unknown"
}

func init() {
	useHashType = SHA256
}

type FileData struct {
	Processed bool
	Hashes    []Hash256
	HashType  hashType
	ChunkSize int64 // The chunksize that this was created with.
	MaxChunks int
	CurByte   int64  // for when the while file hasn't been hashed and
	Root      string // the relative root of this file: allows for synch support
	Dir       string // relative path to parent directory of Fi
	Fi        os.FileInfo
}

// Returns a FileData struct for the passed file using the defaults.
// Set any overrides before performing an operation.
func NewFileData(root, dir string, fi os.FileInfo) FileData {
	if dir == "." {
		dir = ""
	}
	fd := FileData{HashType: useHashType, ChunkSize: chunkSize, MaxChunks: MaxChunks, Root: root, Dir: dir, Fi: fi}
	return fd
}

// String is an alias to RelPath
func (fd *FileData) String() string {
	return fd.RelPath()
}

// SetHash computes the hash of the FileData. The path of the file is passed
// because FileData only knows it's name, not its location.
func (fd *FileData) SetHash() error {
	f, hasher, err := fd.getFileHasher()
	if err != nil {
		return err
	}
	if fd.ChunkSize == 0 {
		return fd.hashFile(f, hasher)
	}
	return fd.chunkedHashFile(f, hasher)
}

// getHasher returns a file and the hasher to use it on. If error, return that.
func (fd *FileData) getFileHasher() (f *os.File, hasher hash.Hash, err error) {
	f, err = os.Open(fd.RootPath())
	if err != nil {
		return
	}
	hasher, err = fd.getHasher()
	return
}

// getHasher returns a hasher or an error
func (fd *FileData) getHasher() (hasher hash.Hash, err error) {
	switch fd.HashType {
	case SHA256:
		hasher = sha256.New() //
	default:
		err = fmt.Errorf("%s hash type", fd.HashType.String())
		return
	}
	return
}

// hashFile hashes the entire file.
func (fd *FileData) hashFile(f *os.File, hasher hash.Hash) error {
	_, err := io.Copy(hasher, f)
	if err != nil {
		log.Printf("%s/n", err)
		return err
	}
	h := Hash256{}
	copy(h[:], hasher.Sum(nil))
	fd.Hashes = append(fd.Hashes, Hash256(h))
	return nil
}

// chunkedHashFile reads up to max chunks, or the entire file, whichever comes
// first.
func (fd *FileData) chunkedHashFile(f *os.File, hasher hash.Hash) (err error) {
	reader := bufio.NewReaderSize(f, int(fd.ChunkSize))
	var cnt int
	var bytes int64
	h := Hash256{}
	for cnt = 0; cnt < MaxChunks; cnt++ { // read until EOF || MaxChunks
		n, err := io.CopyN(hasher, reader, int64(fd.ChunkSize))
		if err != nil && err != io.EOF {
			return err
		}
		bytes += n
		copy(h[:], hasher.Sum(nil))
		fd.Hashes = append(fd.Hashes, Hash256(h))
	}
	if err != nil {
		log.Printf("%s\n", err)
		return err
	}
	_ = cnt
	return nil
}

// isEqual compares the current file with the passed file and returns
// whether or not they are equal. If the file length is greater than our
// checksum buffer, the rest of the file is read in chunks, until EOF or
// a difference is found, whichever comes first.
//
// If they are of different lengths, we assume they are different
func (fd *FileData) isEqual(dstFd FileData) (bool, error) {
	if fd.Fi.Size() != dstFd.Fi.Size() {
		return false, nil
	}
	// otherwise, examine the file contents
	f, hasher, err := fd.getFileHasher()
	if err != nil {
		return false, err
	}
	defer f.Close()
	// TODO support adaptive
	chunks := int(fd.Fi.Size()/int64(fd.ChunkSize) + 1)
	if chunks > len(fd.Hashes) {
		return fd.isEqualMixed(chunks, f, hasher, dstFd)
	}

	return fd.isEqualCached(chunks, f, hasher, dstFd)
}

func (fd *FileData) isEqualMixed(chunks int, f *os.File, hasher hash.Hash, dstFd FileData) (bool, error) {
	equal, err := fd.isEqualCached(dstFd.MaxChunks, f, hasher, dstFd)
	if err != nil {
		return equal, err
	}
	if !equal {
		return equal, nil
	}
	// Otherwise check the file from the current point
	dstF, err := os.Open(dstFd.RootPath())

	// Go to the last read byte
	pos, err := dstF.Seek(dstFd.CurByte, 0)
	if err != nil {
		return false, err
	}
	_ = pos
	dstHasher, err := fd.getHasher()
	if err != nil {
		return false, err
	}
	dH := Hash256{}
	sH := Hash256{}
	// Check until EOF or a difference is found
	for {
		s, err := io.CopyN(hasher, f, fd.ChunkSize)
		if err != nil {
			return false, err
		}
		d, err := io.CopyN(dstHasher, dstF, fd.ChunkSize)
		if err != nil {
			return false, err
		}
		if d != s { // if the bytes copied were different, return false
			return false, nil
		}
		copy(dH[:], dstHasher.Sum(nil))
		copy(sH[:], hasher.Sum(nil))
		if Hash256(dH) != Hash256(sH) {
			return false, nil
		}
	}
	return true, nil
}

// isEqualCached is called when the file fits within the maxChunks.
func (fd *FileData) isEqualCached(chunks int, f *os.File, hasher hash.Hash, dstFd FileData) (bool, error) {
	h := Hash256{}
	for i := 0; i < chunks; i++ {
		_, err := io.CopyN(hasher, f, fd.ChunkSize)
		if err != nil {
			return false, err
		}
		copy(h[:], hasher.Sum(nil))
		if Hash256(h) != dstFd.Hashes[i] {
			return false, nil
		}
	}
	return true, nil
}

// RelPath returns the relative path of the file, this is the file less the
// root information. This allows for easy comparision between two directories.
func (fd *FileData) RelPath() string {
	return filepath.Join(fd.Dir, fd.Fi.Name())
}

// RootPath returns the relative path of the file including its root. A root is
// the directory that Synchronicity considers a root, e.g. one of the
// directories being synched. This is not the FullPath of a file.
func (fd *FileData) RootPath() string {
	return filepath.Join(fd.Root, fd.Dir, fd.Fi.Name())
}

func SetHashType(s string) {
	useHashType = ParseHashType(s)
}

func ParseHashType(s string) hashType {
	s = strings.ToLower(s)
	switch s {
	case "sha256":
		return SHA256
	}
	return invalid
}

<|endoftext|>


<commit_before>/*
 * Copyright 2016 Charith Ellawala
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package gcgrpcpool

import (
	"fmt"
	"sync"

	log "github.com/Sirupsen/logrus"

	"github.com/charithe/gcgrpcpool/gcgrpc"
	"github.com/golang/groupcache"
	"github.com/golang/groupcache/consistenthash"
	pb "github.com/golang/groupcache/groupcachepb"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

const defaultReplicas = 50

type GRPCPool struct {
	self        string
	opts        GRPCPoolOptions
	mu          sync.Mutex
	peers       *consistenthash.Map
	grpcGetters map[string]*grpcGetter
}

type GRPCPoolOptions struct {
	Replicas        int
	HashFn          consistenthash.Hash
	PeerDialOptions []grpc.DialOption
}

func NewGRPCPool(self string, server *grpc.Server) *GRPCPool {
	return NewGRPCPoolOptions(self, server, nil)
}

var grpcPoolCreated bool

func NewGRPCPoolOptions(self string, server *grpc.Server, opts *GRPCPoolOptions) *GRPCPool {
	if grpcPoolCreated {
		panic("NewGRPCPool must be called only once")
	}

	grpcPoolCreated = true

	pool := &GRPCPool{
		self:        self,
		grpcGetters: make(map[string]*grpcGetter),
	}

	if opts != nil {
		pool.opts = *opts
	}

	if pool.opts.Replicas == 0 {
		pool.opts.Replicas = defaultReplicas
	}

	if pool.opts.PeerDialOptions == nil {
		pool.opts.PeerDialOptions = []grpc.DialOption{grpc.WithInsecure()}
	}

	pool.peers = consistenthash.New(pool.opts.Replicas, pool.opts.HashFn)
	groupcache.RegisterPeerPicker(func() groupcache.PeerPicker { return pool })
	gcgrpc.RegisterPeerServer(server, pool)
	return pool
}

func (gp *GRPCPool) Set(peers ...string) {
	gp.mu.Lock()
	defer gp.mu.Unlock()
	gp.peers = consistenthash.New(gp.opts.Replicas, gp.opts.HashFn)
	tempGetters := make(map[string]*grpcGetter, len(peers))
	for _, peer := range peers {
		if getter, exists := gp.grpcGetters[peer]; exists == true {
			tempGetters[peer] = getter
			delete(gp.grpcGetters, peer)
		} else {
			getter, err := newGRPCGetter(peer, gp.opts.PeerDialOptions...)
			if err != nil {
				log.WithError(err).Warnf("Failed to open connection to [%s]", peer)
			} else {
				tempGetters[peer] = getter
				gp.peers.Add(peer)
			}
		}
	}

	for p, g := range gp.grpcGetters {
		g.close()
		delete(gp.grpcGetters, p)
	}

	gp.grpcGetters = tempGetters
}

func (gp *GRPCPool) PickPeer(key string) (groupcache.ProtoGetter, bool) {
	gp.mu.Lock()
	defer gp.mu.Unlock()

	if gp.peers.IsEmpty() {
		return nil, false
	}

	if peer := gp.peers.Get(key); peer != gp.self {
		return gp.grpcGetters[peer], true
	}
	return nil, false
}

func (gp *GRPCPool) Retrieve(ctx context.Context, req *gcgrpc.RetrieveRequest) (*gcgrpc.RetrieveResponse, error) {
	group := groupcache.GetGroup(req.Group)
	if group == nil {
		log.Warnf("Unable to find group [%s]", req.Group)
		return nil, fmt.Errorf("Unable to find group [%s]", req.Group)
	}
	group.Stats.ServerRequests.Add(1)
	var value []byte
	err := group.Get(ctx, req.Key, groupcache.AllocatingByteSliceSink(&value))
	if err != nil {
		log.WithError(err).Warnf("Failed to retrieve [%s]", req)
		return nil, fmt.Errorf("Failed to retrieve [%s]: %v", req, err)
	}

	return &gcgrpc.RetrieveResponse{Value: value}, nil
}

func (gp *GRPCPool) AddPeers(ctx context.Context, peers *gcgrpc.Peers) (*gcgrpc.Ack, error) {
	gp.mu.Lock()
	defer gp.mu.Unlock()
	for _, peer := range peers.PeerAddr {
		if _, exists := gp.grpcGetters[peer]; exists != true {
			getter, err := newGRPCGetter(peer, gp.opts.PeerDialOptions...)
			if err != nil {
				log.WithError(err).Warnf("Failed to open connection to [%s]", peer)
			} else {
				log.Infof("Adding peer [%s]", peer)
				gp.grpcGetters[peer] = getter
				gp.peers.Add(peer)
			}
		}
	}
	return &gcgrpc.Ack{}, nil

}

func (gp *GRPCPool) RemovePeers(ctx context.Context, peers *gcgrpc.Peers) (*gcgrpc.Ack, error) {
	gp.mu.Lock()
	defer gp.mu.Unlock()
	for _, peer := range peers.PeerAddr {
		log.Infof("Removing peer [%s]", peer)
		delete(gp.grpcGetters, peer)
	}
	return &gcgrpc.Ack{}, nil
}

func (gp *GRPCPool) SetPeers(ctx context.Context, peers *gcgrpc.Peers) (*gcgrpc.Ack, error) {
	gp.Set(peers.PeerAddr...)
	return &gcgrpc.Ack{}, nil
}

type grpcGetter struct {
	address string
	conn    *grpc.ClientConn
}

func newGRPCGetter(address string, dialOpts ...grpc.DialOption) (*grpcGetter, error) {
	conn, err := grpc.Dial(address, dialOpts...)
	if err != nil {
		return nil, fmt.Errorf("Failed to connect to [%s]: %v", address, err)
	}
	return &grpcGetter{address: address, conn: conn}, nil
}

func (g *grpcGetter) Get(ctx groupcache.Context, in *pb.GetRequest, out *pb.GetResponse) error {
	client := gcgrpc.NewPeerClient(g.conn)
	resp, err := client.Retrieve(context.Background(), &gcgrpc.RetrieveRequest{Group: *in.Group, Key: *in.Key})
	if err != nil {
		return fmt.Errorf("Failed to GET [%s]: %v", in, err)
	}

	out.Value = resp.Value
	return nil
}

func (g *grpcGetter) close() {
	if g.conn != nil {
		g.conn.Close()
	}
}
<commit_msg>Close connection on peer removal<commit_after>/*
 * Copyright 2016 Charith Ellawala
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package gcgrpcpool

import (
	"fmt"
	"sync"

	log "github.com/Sirupsen/logrus"

	"github.com/charithe/gcgrpcpool/gcgrpc"
	"github.com/golang/groupcache"
	"github.com/golang/groupcache/consistenthash"
	pb "github.com/golang/groupcache/groupcachepb"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

const defaultReplicas = 50

type GRPCPool struct {
	self        string
	opts        GRPCPoolOptions
	mu          sync.Mutex
	peers       *consistenthash.Map
	grpcGetters map[string]*grpcGetter
}

type GRPCPoolOptions struct {
	Replicas        int
	HashFn          consistenthash.Hash
	PeerDialOptions []grpc.DialOption
}

func NewGRPCPool(self string, server *grpc.Server) *GRPCPool {
	return NewGRPCPoolOptions(self, server, nil)
}

var grpcPoolCreated bool

func NewGRPCPoolOptions(self string, server *grpc.Server, opts *GRPCPoolOptions) *GRPCPool {
	if grpcPoolCreated {
		panic("NewGRPCPool must be called only once")
	}

	grpcPoolCreated = true

	pool := &GRPCPool{
		self:        self,
		grpcGetters: make(map[string]*grpcGetter),
	}

	if opts != nil {
		pool.opts = *opts
	}

	if pool.opts.Replicas == 0 {
		pool.opts.Replicas = defaultReplicas
	}

	if pool.opts.PeerDialOptions == nil {
		pool.opts.PeerDialOptions = []grpc.DialOption{grpc.WithInsecure()}
	}

	pool.peers = consistenthash.New(pool.opts.Replicas, pool.opts.HashFn)
	groupcache.RegisterPeerPicker(func() groupcache.PeerPicker { return pool })
	gcgrpc.RegisterPeerServer(server, pool)
	return pool
}

func (gp *GRPCPool) Set(peers ...string) {
	gp.mu.Lock()
	defer gp.mu.Unlock()
	gp.peers = consistenthash.New(gp.opts.Replicas, gp.opts.HashFn)
	tempGetters := make(map[string]*grpcGetter, len(peers))
	for _, peer := range peers {
		if getter, exists := gp.grpcGetters[peer]; exists == true {
			tempGetters[peer] = getter
			delete(gp.grpcGetters, peer)
		} else {
			getter, err := newGRPCGetter(peer, gp.opts.PeerDialOptions...)
			if err != nil {
				log.WithError(err).Warnf("Failed to open connection to [%s]", peer)
			} else {
				tempGetters[peer] = getter
				gp.peers.Add(peer)
			}
		}
	}

	for p, g := range gp.grpcGetters {
		g.close()
		delete(gp.grpcGetters, p)
	}

	gp.grpcGetters = tempGetters
}

func (gp *GRPCPool) PickPeer(key string) (groupcache.ProtoGetter, bool) {
	gp.mu.Lock()
	defer gp.mu.Unlock()

	if gp.peers.IsEmpty() {
		return nil, false
	}

	if peer := gp.peers.Get(key); peer != gp.self {
		return gp.grpcGetters[peer], true
	}
	return nil, false
}

func (gp *GRPCPool) Retrieve(ctx context.Context, req *gcgrpc.RetrieveRequest) (*gcgrpc.RetrieveResponse, error) {
	group := groupcache.GetGroup(req.Group)
	if group == nil {
		log.Warnf("Unable to find group [%s]", req.Group)
		return nil, fmt.Errorf("Unable to find group [%s]", req.Group)
	}
	group.Stats.ServerRequests.Add(1)
	var value []byte
	err := group.Get(ctx, req.Key, groupcache.AllocatingByteSliceSink(&value))
	if err != nil {
		log.WithError(err).Warnf("Failed to retrieve [%s]", req)
		return nil, fmt.Errorf("Failed to retrieve [%s]: %v", req, err)
	}

	return &gcgrpc.RetrieveResponse{Value: value}, nil
}

func (gp *GRPCPool) AddPeers(ctx context.Context, peers *gcgrpc.Peers) (*gcgrpc.Ack, error) {
	gp.mu.Lock()
	defer gp.mu.Unlock()
	for _, peer := range peers.PeerAddr {
		if _, exists := gp.grpcGetters[peer]; exists != true {
			getter, err := newGRPCGetter(peer, gp.opts.PeerDialOptions...)
			if err != nil {
				log.WithError(err).Warnf("Failed to open connection to [%s]", peer)
			} else {
				log.Infof("Adding peer [%s]", peer)
				gp.grpcGetters[peer] = getter
				gp.peers.Add(peer)
			}
		}
	}
	return &gcgrpc.Ack{}, nil

}

func (gp *GRPCPool) RemovePeers(ctx context.Context, peers *gcgrpc.Peers) (*gcgrpc.Ack, error) {
	gp.mu.Lock()
	defer gp.mu.Unlock()
	for _, peer := range peers.PeerAddr {
		if p, exists := gp.grpcGetters[peer]; exists == true {
			log.Infof("Removing peer [%s]", peer)
			p.close()
			delete(gp.grpcGetters, peer)
		}
	}
	return &gcgrpc.Ack{}, nil
}

func (gp *GRPCPool) SetPeers(ctx context.Context, peers *gcgrpc.Peers) (*gcgrpc.Ack, error) {
	gp.Set(peers.PeerAddr...)
	return &gcgrpc.Ack{}, nil
}

type grpcGetter struct {
	address string
	conn    *grpc.ClientConn
}

func newGRPCGetter(address string, dialOpts ...grpc.DialOption) (*grpcGetter, error) {
	conn, err := grpc.Dial(address, dialOpts...)
	if err != nil {
		return nil, fmt.Errorf("Failed to connect to [%s]: %v", address, err)
	}
	return &grpcGetter{address: address, conn: conn}, nil
}

func (g *grpcGetter) Get(ctx groupcache.Context, in *pb.GetRequest, out *pb.GetResponse) error {
	client := gcgrpc.NewPeerClient(g.conn)
	resp, err := client.Retrieve(context.Background(), &gcgrpc.RetrieveRequest{Group: *in.Group, Key: *in.Key})
	if err != nil {
		return fmt.Errorf("Failed to GET [%s]: %v", in, err)
	}

	out.Value = resp.Value
	return nil
}

func (g *grpcGetter) close() {
	if g.conn != nil {
		g.conn.Close()
	}
}
<|endoftext|>


<commit_before>package gen

import (
	"reflect"

	"github.com/leanovate/gopter"
)

// Struct generates a given struct type.
// rt has to be the reflect type of the struct, gens contains a map of field generators.
// Note that the result types of the generators in gen have to match the type of the correspoinding
// field in the struct. Also note that only public fields of a struct can be generated
func Struct(rt reflect.Type, gens map[string]gopter.Gen) gopter.Gen {
	if rt.Kind() == reflect.Ptr {
		rt = rt.Elem()
	}
	if rt.Kind() != reflect.Struct {
		return Fail(rt)
	}
	return func(genParams *gopter.GenParameters) *gopter.GenResult {
		result := reflect.New(rt)

		for name, gen := range gens {
			field, ok := rt.FieldByName(name)
			if !ok {
				continue
			}
			value, ok := gen(genParams).Retrieve()
			if !ok {
				return gopter.NewEmptyResult(rt)
			}
			result.Elem().FieldByIndex(field.Index).Set(reflect.ValueOf(value))
		}

		return gopter.NewGenResult(reflect.Indirect(result).Interface(), gopter.NoShrinker)
	}
}

// StructPtr generates pointers to a given struct type.
// rt has to be the reflect type of the struct, gens contains a map of field generators.
// Note that the result types of the generators in gen have to match the type of the correspoinding
// field in the struct. Also note that only public fields of a struct can be generated
func StructPtr(rt reflect.Type, gens map[string]gopter.Gen) gopter.Gen {
	if rt.Kind() == reflect.Ptr {
		rt = rt.Elem()
	}
	if rt.Kind() != reflect.Struct {
		return Fail(rt)
	}
	return func(genParams *gopter.GenParameters) *gopter.GenResult {
		result := reflect.New(rt)

		for name, gen := range gens {
			field, ok := rt.FieldByName(name)
			if !ok {
				continue
			}
			value, ok := gen(genParams).Retrieve()
			if !ok {
				return gopter.NewEmptyResult(rt)
			}
			result.Elem().FieldByIndex(field.Index).Set(reflect.ValueOf(value))
		}

		return gopter.NewGenResult(result.Interface(), gopter.NoShrinker)
	}
}
<commit_msg>Add gen.Struct(), gen.StructPtr() has remained unchanged<commit_after>package gen

import (
	"reflect"

	"github.com/leanovate/gopter"
)

// Struct generates a given struct type.
// rt has to be the reflect type of the struct, gens contains a map of field generators.
// Note that the result types of the generators in gen have to match the type of the correspoinding
// field in the struct. Also note that only public fields of a struct can be generated
func Struct(rt reflect.Type, gens map[string]gopter.Gen) gopter.Gen {
	if rt.Kind() == reflect.Ptr {
		rt = rt.Elem()
	}
	if rt.Kind() != reflect.Struct {
		return Fail(rt)
	}
	return func(genParams *gopter.GenParameters) *gopter.GenResult {
		result := reflect.New(rt)

		for name, gen := range gens {
			field, ok := rt.FieldByName(name)
			if !ok {
				continue
			}
			value, ok := gen(genParams).Retrieve()
			if !ok {
				return gopter.NewEmptyResult(rt)
			}
			result.Elem().FieldByIndex(field.Index).Set(reflect.ValueOf(value))
		}

		return gopter.NewGenResult(reflect.Indirect(result).Interface(), gopter.NoShrinker)
	}
}

// StructPtr generates pointers to a given struct type.
// Not that SturctPtr does not generate nil, if you want to include nil in your
// testing you should combine gen.PtrOf with gen.Struct.
// rt has to be the reflect type of the struct, gens contains a map of field generators.
// Note that the result types of the generators in gen have to match the type of the correspoinding
// field in the struct. Also note that only public fields of a struct can be generated
func StructPtr(rt reflect.Type, gens map[string]gopter.Gen) gopter.Gen {
	if rt.Kind() == reflect.Ptr {
		rt = rt.Elem()
	}
	if rt.Kind() != reflect.Struct {
		return Fail(rt)
	}
	return func(genParams *gopter.GenParameters) *gopter.GenResult {
		result := reflect.New(rt)

		for name, gen := range gens {
			field, ok := rt.FieldByName(name)
			if !ok {
				continue
			}
			value, ok := gen(genParams).Retrieve()
			if !ok {
				return gopter.NewEmptyResult(rt)
			}
			result.Elem().FieldByIndex(field.Index).Set(reflect.ValueOf(value))
		}

		return gopter.NewGenResult(result.Interface(), gopter.NoShrinker)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 Jan Broer All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"net"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/janeczku/go-redwall/iptables"
)

// create user chains and jump rules in INPUT and FORWARD chain
func initChains() error {

	// create user chains
	if _, err := iptables.Raw("-N", "redwall-main"); err != nil {
		return err
	}

	if _, err := iptables.Raw("-N", "redwall-whitelist"); err != nil {
		return err
	}

	if _, err := iptables.Raw("-N", "redwall-services"); err != nil {
		return err
	}

	// set default policy to ACCEPT
	if _, err := iptables.Raw("-P", "INPUT", "ACCEPT"); err != nil {
		return err
	}

	if _, err := iptables.Raw("-P", "FORWARD", "ACCEPT"); err != nil {
		return err
	}

	// flush INPUT chain
	if _, err := iptables.Raw("-F", "INPUT"); err != nil {
		return err
	}

	// create INPUT chain jump rule
	if _, err := iptables.Raw("-A", "INPUT", "-i", iface, "-j", "redwall-main"); err != nil {
		return err
	}

	// create FORWARD chain jump rule if we shoud filter the docker network
	if filterDocker {
		rule := []string{
			"-i", iface,
			"-o", "docker0",
			"-j", "redwall-main"}
		if !iptables.Exists("filter", "FORWARD", rule...) {
			if _, err := iptables.Raw("-I", "FORWARD", "1", "-i", iface, "-o", "docker0", "-j", "redwall-main"); err != nil {
				return err
			}
		}
	}

	log.Debugf("created redwall user chains")
	return nil
}

// Create the boilerplate rules in redwall-main
func initDefaultRules() error {
	var chain string = "redwall-main"

	// allow established/related conns
	// iptables -A redwall-main -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
	if _, err := iptables.Raw("-A", chain, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"); err != nil {
		return err
	}

	// block null packets
	// iptables -A redwall-main -p tcp --tcp-flags ALL NONE -j DROP
	if _, err := iptables.Raw("-A", chain, "-p", "tcp", "--tcp-flags", "ALL", "NONE", "-j", "DROP"); err != nil {
		return err
	}

	// block XMAS packets
	// iptables -A redwall-main -p tcp --tcp-flags ALL ALL -j DROP
	if _, err := iptables.Raw("-A", chain, "-p", "tcp", "--tcp-flags", "ALL", "ALL", "-j", "DROP"); err != nil {
		return err
	}

	// block invalid packets
	// iptables -A redwall-main -m conntrack --ctstate INVALID -j DROP
	if _, err := iptables.Raw("-A", chain, "-m", "conntrack", "--ctstate", "INVALID", "-j", "DROP"); err != nil {
		return err
	}

	// block remote packets claiming to be from a loopback address.
	// iptables -A redwall-main -s 127.0.0.0/8 -j DROP
	if _, err := iptables.Raw("-A", chain, "-s", "127.0.0.0/8", "-j", "DROP"); err != nil {
		return err
	}

	// allow ICMP ping
	// iptables -A redwall-main -p icmp --icmp-type 8 -m conntrack --ctstate NEW -j ACCEPT
	if _, err := iptables.Raw("-A", chain, "-p", "icmp", "--icmp-type", "8", "-m", "conntrack", "--ctstate", "NEW", "-j", "ACCEPT"); err != nil {
		return err
	}

	// allow SSH
	// iptables -A redwall-main -p tcp --dport 22 --ctstate NEW -j ACCEPT
	if allowSSH {
		if _, err := iptables.Raw("-A", chain, "-p", "tcp", "--dport", "22", "-m", "conntrack", "--ctstate", "NEW", "-j", "ACCEPT"); err != nil {
			return err
		}
	}

	// continue processing in redwall-whitelist
	if _, err := iptables.Raw("-A", chain, "-j", "redwall-whitelist"); err != nil {
		return err
	}

	// continue processing in redwall-services
	if _, err := iptables.Raw("-A", chain, "-j", "redwall-services"); err != nil {
		return err
	}

	// drop all other incoming packets on iface
	if _, err := iptables.Raw("-A", chain, "-j", "DROP"); err != nil {
		return err
	}

	log.Debugf("created boilerplate rules in redwall-main")
	return nil
}

// mitigate ssh bruteforce attacks
func initSSHRules() error {
	var chain string = "redwall-main"
	// iptables -I redwall-main 1 -p tcp --dport 22 -m state --state NEW -m recent --set --name SSH
	_, err := iptables.Raw("-I", chain, "2", "-p", "tcp", "--dport", "22", "-m", "state", "--state", "NEW",
		"-m", "recent", "--set", "--name", "SSH")
	if err != nil {
		return err
	}

	// iptables -I redwall-main 2 -p tcp --dport 22 -m state --state NEW -m recent --update --seconds 60
	// --hitcount 5 --name SSH --rttl -j REJECT --reject-with tcp-reset
	_, err = iptables.Raw("-I", chain, "3", "-p", "tcp", "--dport", "22", "-m", "state", "--state", "NEW",
		"-m", "recent", "--update", "--seconds", "60", "--hitcount", "5", "--name", "SSH", "-rttl", "-j", "REJECT",
		"--reject-with", "tcp-reset")
	if err != nil {
		return err
	}

	log.Debugf("created SSH attack mitigation rules in redwall-main")
	return nil
}

func applyWhitelistRules() error {
	var chain string = "redwall-whitelist"
	ips, err := getRedisSet("firewall:whitelist")
	if err != nil {
		return err
	}

	// flush existing rules in the chain
	if _, err := iptables.Raw("-F", chain); err != nil {
		return fmt.Errorf("flushing iptables chain %q failed: %v", chain, err)
	}
	log.Debugf("flushed iptables chain %q", chain)

	for _, ip := range ips {
		testIP := net.ParseIP(ip)
		testCIDR, _, _ := net.ParseCIDR(ip)
		if testIP.To4() == nil && testCIDR.To4() == nil {
			log.Errorf("error adding whitelist rule: %v is not a valid IPv4 address or network", ip)
			continue
		}

		if _, err := iptables.Raw("-A", chain, "-s", ip, "-j", "ACCEPT"); err != nil {
			return err
		}
	}

	log.Infof("activated whitelist rules: %v", ips)

	return nil
}

func applyServicesRules() error {
	var chain string = "redwall-services"
	ports, err := getRedisSet("firewall:services")
	if err != nil {
		return err
	}

	// flush the existing rules in the chain
	if _, err := iptables.Raw("-F", chain); err != nil {
		return fmt.Errorf("flushing iptables chain %q failed: %v", chain, err)
	}
	log.Debugf("flushed iptables chain %q", chain)

	for _, key := range ports {
		s := strings.Split(key, ":")
		if len(s) < 2 {
			log.Errorf("error adding port rule. invalid rule format: %s", key)
			continue
		}
		proto, port := s[0], s[1]
		if _, err := iptables.Raw("-A", chain, "-p", proto, "--dport", fmt.Sprint(port), "-j", "ACCEPT"); err != nil {
			return err
		}
	}

	log.Infof("activated services rules: %v", ports)

	return nil
}

func flushChain(chain string) {
	if _, err := iptables.Raw("-F", chain); err != nil {
		log.Warningf("flushing iptables chain %q failed: %v", chain, err)
	}
	log.Debugf("flushed iptables chain %q", chain)
}

func deleteChain(chain string) {
	if _, err := iptables.Raw("-X", chain); err != nil {
		log.Warningf("deleting iptables chain %q failed: %v", chain, err)
	}
	log.Debugf("deleted iptables chain %q", chain)
}

func tearDownFirewall() error {

	// flush input chain

	flushChain("INPUT")

	// delete jump rule from forward

	if filterDocker {
		if _, err := iptables.Raw("-D", "FORWARD", "-i", iface, "-o", "docker0", "-j", "redwall-main"); err != nil {
			log.Warningf("failed to remove docker jump rule: %v", err)
		}
		log.Debugf("removed jump rule from FORWARD chain")
	}

	// flush user-defined chains

	flushChain("redwall-main")
	flushChain("redwall-services")
	flushChain("redwall-whitelist")

	// delete user-defined chains

	deleteChain("redwall-main")
	deleteChain("redwall-services")
	deleteChain("redwall-whitelist")

	return nil
}
<commit_msg>Fixed error on creating existing chains<commit_after>/*
Copyright 2015 Jan Broer All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"net"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/janeczku/go-redwall/iptables"
)

// create user chains and jump rules in INPUT and FORWARD chain
func initChains() error {

	// create user chains
	_, err := iptables.NewChain("redwall-main", iptables.Filter, false)
	if  err != nil {
		return err
	}

	_, err = iptables.NewChain("redwall-whitelist", iptables.Filter, false)
	if  err != nil {
		return err
	}

	_, err = iptables.NewChain("redwall-services", iptables.Filter, false)
	if  err != nil {
		return err
	}

	// set default policy to ACCEPT
	if _, err = iptables.Raw("-P", "INPUT", "ACCEPT"); err != nil {
		return err
	}

	if _, err = iptables.Raw("-P", "FORWARD", "ACCEPT"); err != nil {
		return err
	}

	// flush INPUT chain
	if _, err = iptables.Raw("-F", "INPUT"); err != nil {
		return err
	}

	// create INPUT chain jump rule
	if _, err = iptables.Raw("-A", "INPUT", "-i", iface, "-j", "redwall-main"); err != nil {
		return err
	}

	// create FORWARD chain jump rule if we shoud filter the docker network
	if filterDocker {
		rule := []string{
			"-i", iface,
			"-o", "docker0",
			"-j", "redwall-main"}
		if !iptables.Exists("filter", "FORWARD", rule...) {
			if _, err := iptables.Raw("-I", "FORWARD", "1", "-i", iface, "-o", "docker0", "-j", "redwall-main"); err != nil {
				return err
			}
		}
	}

	log.Debugf("created redwall user chains")
	return nil
}

// Create the boilerplate rules in redwall-main
func initDefaultRules() error {
	var chain string = "redwall-main"

	// flush existing rules in the chain
	if _, err := iptables.Raw("-F", chain); err != nil {
		return fmt.Errorf("flushing iptables chain %q failed: %v", chain, err)
	}
	log.Debugf("flushed iptables chain %q", chain)

	// allow established/related conns
	// iptables -A redwall-main -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
	if _, err := iptables.Raw("-A", chain, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"); err != nil {
		return err
	}

	// block null packets
	// iptables -A redwall-main -p tcp --tcp-flags ALL NONE -j DROP
	if _, err := iptables.Raw("-A", chain, "-p", "tcp", "--tcp-flags", "ALL", "NONE", "-j", "DROP"); err != nil {
		return err
	}

	// block XMAS packets
	// iptables -A redwall-main -p tcp --tcp-flags ALL ALL -j DROP
	if _, err := iptables.Raw("-A", chain, "-p", "tcp", "--tcp-flags", "ALL", "ALL", "-j", "DROP"); err != nil {
		return err
	}

	// block invalid packets
	// iptables -A redwall-main -m conntrack --ctstate INVALID -j DROP
	if _, err := iptables.Raw("-A", chain, "-m", "conntrack", "--ctstate", "INVALID", "-j", "DROP"); err != nil {
		return err
	}

	// block remote packets claiming to be from a loopback address.
	// iptables -A redwall-main -s 127.0.0.0/8 -j DROP
	if _, err := iptables.Raw("-A", chain, "-s", "127.0.0.0/8", "-j", "DROP"); err != nil {
		return err
	}

	// allow ICMP ping
	// iptables -A redwall-main -p icmp --icmp-type 8 -m conntrack --ctstate NEW -j ACCEPT
	if _, err := iptables.Raw("-A", chain, "-p", "icmp", "--icmp-type", "8", "-m", "conntrack", "--ctstate", "NEW", "-j", "ACCEPT"); err != nil {
		return err
	}

	// allow SSH
	// iptables -A redwall-main -p tcp --dport 22 --ctstate NEW -j ACCEPT
	if allowSSH {
		if _, err := iptables.Raw("-A", chain, "-p", "tcp", "--dport", "22", "-m", "conntrack", "--ctstate", "NEW", "-j", "ACCEPT"); err != nil {
			return err
		}
	}

	// continue processing in redwall-whitelist
	if _, err := iptables.Raw("-A", chain, "-j", "redwall-whitelist"); err != nil {
		return err
	}

	// continue processing in redwall-services
	if _, err := iptables.Raw("-A", chain, "-j", "redwall-services"); err != nil {
		return err
	}

	// drop all other incoming packets on iface
	if _, err := iptables.Raw("-A", chain, "-j", "DROP"); err != nil {
		return err
	}

	log.Debugf("created boilerplate rules in redwall-main")
	return nil
}

// mitigate ssh bruteforce attacks
func initSSHRules() error {
	var chain string = "redwall-main"
	// iptables -I redwall-main 1 -p tcp --dport 22 -m state --state NEW -m recent --set --name SSH
	_, err := iptables.Raw("-I", chain, "2", "-p", "tcp", "--dport", "22", "-m", "state", "--state", "NEW",
		"-m", "recent", "--set", "--name", "SSH")
	if err != nil {
		return err
	}

	// iptables -I redwall-main 2 -p tcp --dport 22 -m state --state NEW -m recent --update --seconds 60
	// --hitcount 5 --name SSH --rttl -j REJECT --reject-with tcp-reset
	_, err = iptables.Raw("-I", chain, "3", "-p", "tcp", "--dport", "22", "-m", "state", "--state", "NEW",
		"-m", "recent", "--update", "--seconds", "60", "--hitcount", "5", "--name", "SSH", "-rttl", "-j", "REJECT",
		"--reject-with", "tcp-reset")
	if err != nil {
		return err
	}

	log.Debugf("created SSH attack mitigation rules in redwall-main")
	return nil
}

func applyWhitelistRules() error {
	var chain string = "redwall-whitelist"
	ips, err := getRedisSet("firewall:whitelist")
	if err != nil {
		return err
	}

	// flush existing rules in the chain
	if _, err := iptables.Raw("-F", chain); err != nil {
		return fmt.Errorf("flushing iptables chain %q failed: %v", chain, err)
	}
	log.Debugf("flushed iptables chain %q", chain)

	for _, ip := range ips {
		testIP := net.ParseIP(ip)
		testCIDR, _, _ := net.ParseCIDR(ip)
		if testIP.To4() == nil && testCIDR.To4() == nil {
			log.Errorf("error adding whitelist rule: %v is not a valid IPv4 address or network", ip)
			continue
		}

		if _, err := iptables.Raw("-A", chain, "-s", ip, "-j", "ACCEPT"); err != nil {
			return err
		}
	}

	log.Infof("activated whitelist rules: %v", ips)

	return nil
}

func applyServicesRules() error {
	var chain string = "redwall-services"
	ports, err := getRedisSet("firewall:services")
	if err != nil {
		return err
	}

	// flush the existing rules in the chain
	if _, err := iptables.Raw("-F", chain); err != nil {
		return fmt.Errorf("flushing iptables chain %q failed: %v", chain, err)
	}
	log.Debugf("flushed iptables chain %q", chain)

	for _, key := range ports {
		s := strings.Split(key, ":")
		if len(s) < 2 {
			log.Errorf("error adding port rule. invalid rule format: %s", key)
			continue
		}
		proto, port := s[0], s[1]
		if _, err := iptables.Raw("-A", chain, "-p", proto, "--dport", fmt.Sprint(port), "-j", "ACCEPT"); err != nil {
			return err
		}
	}

	log.Infof("activated services rules: %v", ports)

	return nil
}

func flushChain(chain string) {
	if _, err := iptables.Raw("-F", chain); err != nil {
		log.Warningf("flushing iptables chain %q failed: %v", chain, err)
	}
	log.Debugf("flushed iptables chain %q", chain)
}

func deleteChain(chain string) {
	if _, err := iptables.Raw("-X", chain); err != nil {
		log.Warningf("deleting iptables chain %q failed: %v", chain, err)
	}
	log.Debugf("deleted iptables chain %q", chain)
}

func tearDownFirewall() error {

	// flush input chain

	flushChain("INPUT")

	// delete jump rule from forward

	if filterDocker {
		if _, err := iptables.Raw("-D", "FORWARD", "-i", iface, "-o", "docker0", "-j", "redwall-main"); err != nil {
			log.Warningf("failed to remove docker jump rule: %v", err)
		}
		log.Debugf("removed jump rule from FORWARD chain")
	}

	// flush user-defined chains

	flushChain("redwall-main")
	flushChain("redwall-services")
	flushChain("redwall-whitelist")

	// delete user-defined chains

	deleteChain("redwall-main")
	deleteChain("redwall-services")
	deleteChain("redwall-whitelist")

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package grpcproxy

import (
	"github.com/coreos/etcd/clientv3"
	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
	"github.com/coreos/etcd/proxy/grpcproxy/cache"

	"golang.org/x/net/context"
)

type kvProxy struct {
	kv    clientv3.KV
	cache cache.Cache
}

func NewKvProxy(c *clientv3.Client) (pb.KVServer, <-chan struct{}) {
	kv := &kvProxy{
		kv:    c.KV,
		cache: cache.NewCache(cache.DefaultMaxEntries),
	}
	donec := make(chan struct{})
	close(donec)
	return kv, donec
}

func (p *kvProxy) Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeResponse, error) {
	if r.Serializable {
		resp, err := p.cache.Get(r)
		switch err {
		case nil:
			cacheHits.Inc()
			return resp, nil
		case cache.ErrCompacted:
			cacheHits.Inc()
			return nil, err
		}

		cachedMisses.Inc()
	}

	resp, err := p.kv.Do(ctx, RangeRequestToOp(r))
	if err != nil {
		return nil, err
	}

	// cache linearizable as serializable
	req := *r
	req.Serializable = true
	gresp := (*pb.RangeResponse)(resp.Get())
	p.cache.Add(&req, gresp)
	cacheKeys.Set(float64(p.cache.Size()))

	return gresp, nil
}

func (p *kvProxy) Put(ctx context.Context, r *pb.PutRequest) (*pb.PutResponse, error) {
	p.cache.Invalidate(r.Key, nil)
	cacheKeys.Set(float64(p.cache.Size()))

	resp, err := p.kv.Do(ctx, PutRequestToOp(r))
	return (*pb.PutResponse)(resp.Put()), err
}

func (p *kvProxy) DeleteRange(ctx context.Context, r *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, error) {
	p.cache.Invalidate(r.Key, r.RangeEnd)
	cacheKeys.Set(float64(p.cache.Size()))

	resp, err := p.kv.Do(ctx, DelRequestToOp(r))
	return (*pb.DeleteRangeResponse)(resp.Del()), err
}

func (p *kvProxy) txnToCache(reqs []*pb.RequestOp, resps []*pb.ResponseOp) {
	for i := range resps {
		switch tv := resps[i].Response.(type) {
		case *pb.ResponseOp_ResponsePut:
			p.cache.Invalidate(reqs[i].GetRequestPut().Key, nil)
		case *pb.ResponseOp_ResponseDeleteRange:
			rdr := reqs[i].GetRequestDeleteRange()
			p.cache.Invalidate(rdr.Key, rdr.RangeEnd)
		case *pb.ResponseOp_ResponseRange:
			req := *(reqs[i].GetRequestRange())
			req.Serializable = true
			p.cache.Add(&req, tv.ResponseRange)
		}
	}
}

func (p *kvProxy) Txn(ctx context.Context, r *pb.TxnRequest) (*pb.TxnResponse, error) {
	op := TxnRequestToOp(r)
	opResp, err := p.kv.Do(ctx, op)
	if err != nil {
		return nil, err
	}
	resp := opResp.Txn()

	// txn may claim an outdated key is updated; be safe and invalidate
	for _, cmp := range r.Compare {
		p.cache.Invalidate(cmp.Key, cmp.RangeEnd)
	}
	// update any fetched keys
	if resp.Succeeded {
		p.txnToCache(r.Success, resp.Responses)
	} else {
		p.txnToCache(r.Failure, resp.Responses)
	}

	cacheKeys.Set(float64(p.cache.Size()))

	return (*pb.TxnResponse)(resp), nil
}

func (p *kvProxy) Compact(ctx context.Context, r *pb.CompactionRequest) (*pb.CompactionResponse, error) {
	var opts []clientv3.CompactOption
	if r.Physical {
		opts = append(opts, clientv3.WithCompactPhysical())
	}

	resp, err := p.kv.Compact(ctx, r.Revision, opts...)
	if err == nil {
		p.cache.Compact(r.Revision)
	}

	cacheKeys.Set(float64(p.cache.Size()))

	return (*pb.CompactionResponse)(resp), err
}

func requestOpToOp(union *pb.RequestOp) clientv3.Op {
	switch tv := union.Request.(type) {
	case *pb.RequestOp_RequestRange:
		if tv.RequestRange != nil {
			return RangeRequestToOp(tv.RequestRange)
		}
	case *pb.RequestOp_RequestPut:
		if tv.RequestPut != nil {
			return PutRequestToOp(tv.RequestPut)
		}
	case *pb.RequestOp_RequestDeleteRange:
		if tv.RequestDeleteRange != nil {
			return DelRequestToOp(tv.RequestDeleteRange)
		}
	case *pb.RequestOp_RequestTxn:
		if tv.RequestTxn != nil {
			return TxnRequestToOp(tv.RequestTxn)
		}
	}
	panic("unknown request")
}

func RangeRequestToOp(r *pb.RangeRequest) clientv3.Op {
	opts := []clientv3.OpOption{}
	if len(r.RangeEnd) != 0 {
		opts = append(opts, clientv3.WithRange(string(r.RangeEnd)))
	}
	opts = append(opts, clientv3.WithRev(r.Revision))
	opts = append(opts, clientv3.WithLimit(r.Limit))
	opts = append(opts, clientv3.WithSort(
		clientv3.SortTarget(r.SortTarget),
		clientv3.SortOrder(r.SortOrder)),
	)
	opts = append(opts, clientv3.WithMaxCreateRev(r.MaxCreateRevision))
	opts = append(opts, clientv3.WithMinCreateRev(r.MinCreateRevision))
	opts = append(opts, clientv3.WithMaxModRev(r.MaxModRevision))
	opts = append(opts, clientv3.WithMinModRev(r.MinModRevision))
	if r.CountOnly {
		opts = append(opts, clientv3.WithCountOnly())
	}

	if r.Serializable {
		opts = append(opts, clientv3.WithSerializable())
	}

	return clientv3.OpGet(string(r.Key), opts...)
}

func PutRequestToOp(r *pb.PutRequest) clientv3.Op {
	opts := []clientv3.OpOption{}
	opts = append(opts, clientv3.WithLease(clientv3.LeaseID(r.Lease)))
	if r.IgnoreValue {
		opts = append(opts, clientv3.WithIgnoreValue())
	}
	if r.IgnoreLease {
		opts = append(opts, clientv3.WithIgnoreLease())
	}
	return clientv3.OpPut(string(r.Key), string(r.Value), opts...)
}

func DelRequestToOp(r *pb.DeleteRangeRequest) clientv3.Op {
	opts := []clientv3.OpOption{}
	if len(r.RangeEnd) != 0 {
		opts = append(opts, clientv3.WithRange(string(r.RangeEnd)))
	}
	if r.PrevKv {
		opts = append(opts, clientv3.WithPrevKV())
	}
	return clientv3.OpDelete(string(r.Key), opts...)
}

func TxnRequestToOp(r *pb.TxnRequest) clientv3.Op {
	cmps := make([]clientv3.Cmp, len(r.Compare))
	thenops := make([]clientv3.Op, len(r.Success))
	elseops := make([]clientv3.Op, len(r.Failure))
	for i := range r.Compare {
		cmps[i] = (clientv3.Cmp)(*r.Compare[i])
	}
	for i := range r.Success {
		thenops[i] = requestOpToOp(r.Success[i])
	}
	for i := range r.Failure {
		elseops[i] = requestOpToOp(r.Failure[i])
	}
	return clientv3.OpTxn(cmps, thenops, elseops)
}
<commit_msg>grpcproxy: forward PrevKv flag in Put<commit_after>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package grpcproxy

import (
	"github.com/coreos/etcd/clientv3"
	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
	"github.com/coreos/etcd/proxy/grpcproxy/cache"

	"golang.org/x/net/context"
)

type kvProxy struct {
	kv    clientv3.KV
	cache cache.Cache
}

func NewKvProxy(c *clientv3.Client) (pb.KVServer, <-chan struct{}) {
	kv := &kvProxy{
		kv:    c.KV,
		cache: cache.NewCache(cache.DefaultMaxEntries),
	}
	donec := make(chan struct{})
	close(donec)
	return kv, donec
}

func (p *kvProxy) Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeResponse, error) {
	if r.Serializable {
		resp, err := p.cache.Get(r)
		switch err {
		case nil:
			cacheHits.Inc()
			return resp, nil
		case cache.ErrCompacted:
			cacheHits.Inc()
			return nil, err
		}

		cachedMisses.Inc()
	}

	resp, err := p.kv.Do(ctx, RangeRequestToOp(r))
	if err != nil {
		return nil, err
	}

	// cache linearizable as serializable
	req := *r
	req.Serializable = true
	gresp := (*pb.RangeResponse)(resp.Get())
	p.cache.Add(&req, gresp)
	cacheKeys.Set(float64(p.cache.Size()))

	return gresp, nil
}

func (p *kvProxy) Put(ctx context.Context, r *pb.PutRequest) (*pb.PutResponse, error) {
	p.cache.Invalidate(r.Key, nil)
	cacheKeys.Set(float64(p.cache.Size()))

	resp, err := p.kv.Do(ctx, PutRequestToOp(r))
	return (*pb.PutResponse)(resp.Put()), err
}

func (p *kvProxy) DeleteRange(ctx context.Context, r *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, error) {
	p.cache.Invalidate(r.Key, r.RangeEnd)
	cacheKeys.Set(float64(p.cache.Size()))

	resp, err := p.kv.Do(ctx, DelRequestToOp(r))
	return (*pb.DeleteRangeResponse)(resp.Del()), err
}

func (p *kvProxy) txnToCache(reqs []*pb.RequestOp, resps []*pb.ResponseOp) {
	for i := range resps {
		switch tv := resps[i].Response.(type) {
		case *pb.ResponseOp_ResponsePut:
			p.cache.Invalidate(reqs[i].GetRequestPut().Key, nil)
		case *pb.ResponseOp_ResponseDeleteRange:
			rdr := reqs[i].GetRequestDeleteRange()
			p.cache.Invalidate(rdr.Key, rdr.RangeEnd)
		case *pb.ResponseOp_ResponseRange:
			req := *(reqs[i].GetRequestRange())
			req.Serializable = true
			p.cache.Add(&req, tv.ResponseRange)
		}
	}
}

func (p *kvProxy) Txn(ctx context.Context, r *pb.TxnRequest) (*pb.TxnResponse, error) {
	op := TxnRequestToOp(r)
	opResp, err := p.kv.Do(ctx, op)
	if err != nil {
		return nil, err
	}
	resp := opResp.Txn()

	// txn may claim an outdated key is updated; be safe and invalidate
	for _, cmp := range r.Compare {
		p.cache.Invalidate(cmp.Key, cmp.RangeEnd)
	}
	// update any fetched keys
	if resp.Succeeded {
		p.txnToCache(r.Success, resp.Responses)
	} else {
		p.txnToCache(r.Failure, resp.Responses)
	}

	cacheKeys.Set(float64(p.cache.Size()))

	return (*pb.TxnResponse)(resp), nil
}

func (p *kvProxy) Compact(ctx context.Context, r *pb.CompactionRequest) (*pb.CompactionResponse, error) {
	var opts []clientv3.CompactOption
	if r.Physical {
		opts = append(opts, clientv3.WithCompactPhysical())
	}

	resp, err := p.kv.Compact(ctx, r.Revision, opts...)
	if err == nil {
		p.cache.Compact(r.Revision)
	}

	cacheKeys.Set(float64(p.cache.Size()))

	return (*pb.CompactionResponse)(resp), err
}

func requestOpToOp(union *pb.RequestOp) clientv3.Op {
	switch tv := union.Request.(type) {
	case *pb.RequestOp_RequestRange:
		if tv.RequestRange != nil {
			return RangeRequestToOp(tv.RequestRange)
		}
	case *pb.RequestOp_RequestPut:
		if tv.RequestPut != nil {
			return PutRequestToOp(tv.RequestPut)
		}
	case *pb.RequestOp_RequestDeleteRange:
		if tv.RequestDeleteRange != nil {
			return DelRequestToOp(tv.RequestDeleteRange)
		}
	case *pb.RequestOp_RequestTxn:
		if tv.RequestTxn != nil {
			return TxnRequestToOp(tv.RequestTxn)
		}
	}
	panic("unknown request")
}

func RangeRequestToOp(r *pb.RangeRequest) clientv3.Op {
	opts := []clientv3.OpOption{}
	if len(r.RangeEnd) != 0 {
		opts = append(opts, clientv3.WithRange(string(r.RangeEnd)))
	}
	opts = append(opts, clientv3.WithRev(r.Revision))
	opts = append(opts, clientv3.WithLimit(r.Limit))
	opts = append(opts, clientv3.WithSort(
		clientv3.SortTarget(r.SortTarget),
		clientv3.SortOrder(r.SortOrder)),
	)
	opts = append(opts, clientv3.WithMaxCreateRev(r.MaxCreateRevision))
	opts = append(opts, clientv3.WithMinCreateRev(r.MinCreateRevision))
	opts = append(opts, clientv3.WithMaxModRev(r.MaxModRevision))
	opts = append(opts, clientv3.WithMinModRev(r.MinModRevision))
	if r.CountOnly {
		opts = append(opts, clientv3.WithCountOnly())
	}

	if r.Serializable {
		opts = append(opts, clientv3.WithSerializable())
	}

	return clientv3.OpGet(string(r.Key), opts...)
}

func PutRequestToOp(r *pb.PutRequest) clientv3.Op {
	opts := []clientv3.OpOption{}
	opts = append(opts, clientv3.WithLease(clientv3.LeaseID(r.Lease)))
	if r.IgnoreValue {
		opts = append(opts, clientv3.WithIgnoreValue())
	}
	if r.IgnoreLease {
		opts = append(opts, clientv3.WithIgnoreLease())
	}
	if r.PrevKv {
		opts = append(opts, clientv3.WithPrevKV())
	}
	return clientv3.OpPut(string(r.Key), string(r.Value), opts...)
}

func DelRequestToOp(r *pb.DeleteRangeRequest) clientv3.Op {
	opts := []clientv3.OpOption{}
	if len(r.RangeEnd) != 0 {
		opts = append(opts, clientv3.WithRange(string(r.RangeEnd)))
	}
	if r.PrevKv {
		opts = append(opts, clientv3.WithPrevKV())
	}
	return clientv3.OpDelete(string(r.Key), opts...)
}

func TxnRequestToOp(r *pb.TxnRequest) clientv3.Op {
	cmps := make([]clientv3.Cmp, len(r.Compare))
	thenops := make([]clientv3.Op, len(r.Success))
	elseops := make([]clientv3.Op, len(r.Failure))
	for i := range r.Compare {
		cmps[i] = (clientv3.Cmp)(*r.Compare[i])
	}
	for i := range r.Success {
		thenops[i] = requestOpToOp(r.Success[i])
	}
	for i := range r.Failure {
		elseops[i] = requestOpToOp(r.Failure[i])
	}
	return clientv3.OpTxn(cmps, thenops, elseops)
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package main

// CmdDatacenter subcommand
import (
	"fmt"
	"os"

	"github.com/fatih/color"
	"github.com/urfave/cli"
)

// ListServices ...
var ListServices = cli.Command{
	Name:      "list",
	Usage:     "List available services.",
	ArgsUsage: " ",
	Description: `List available services and shows its most relevant information.

   Example:
    $ ernest service list
	`,
	Action: func(c *cli.Context) error {
		m, cfg := setup(c)
		if cfg.Token == "" {
			color.Red("You're not allowed to perform this action, please log in")
			return nil
		}
		services, err := m.ListServices(cfg.Token)
		if err != nil {
			color.Red(err.Error())
			return err
		}

		printServiceList(services)
		return nil
	},
}

// ApplyService command
// Applies changes described on a YAML file to a service
var ApplyService = cli.Command{
	Name:      "apply",
	Aliases:   []string{"a"},
	ArgsUsage: "<file.yml>",
	Usage:     "Builds or changes infrastructure.",
	Description: `Sends a service YAML description file to Ernest to be executed.
   You must be logged in to execute this command.

   If the file is not provided, ernest.yml will be used by default.

   Example:
    $ ernest apply myservice.yml
	`,
	Action: func(c *cli.Context) error {
		file := "ernest.yml"
		if len(c.Args()) == 1 {
			file = c.Args()[0]
		}
		m, cfg := setup(c)
		if cfg.Token == "" {
			color.Red("You're not allowed to perform this action, please log in")
			return nil
		}
		_, err := m.Apply(cfg.Token, file, true)
		if err != nil {
			color.Red(err.Error())
		}
		return nil
	},
}

// DestroyService command
var DestroyService = cli.Command{
	Name:      "destroy",
	Aliases:   []string{"d"},
	ArgsUsage: "<service_name>",
	Usage:     "Destroy a service.",
	Description: `Destroys a service by its name.

   Example:
    $ ernest destroy myservice
  `,
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name:  "force,f",
			Usage: "Force destroy command without asking for permission.",
		},
	},
	Action: func(c *cli.Context) error {
		m, cfg := setup(c)
		if cfg.Token == "" {
			color.Red("You're not allowed to perform this action, please log in")
			return nil
		}

		if len(c.Args()) < 1 {
			color.Red("You should specify an existing service name")
			return nil
		}
		name := c.Args()[0]

		if c.Bool("force") {
			err := m.Destroy(cfg.Token, name, true)
			if err != nil {
				color.Red(err.Error())
			}
		} else {
			fmt.Print("Are you sure? Please type yes or no and then press enter: ")
			if askForConfirmation() {
				err := m.Destroy(cfg.Token, name, true)
				if err != nil {
					color.Red(err.Error())
					return err
				}
			}
		}
		return nil
	},
}

// HistoryService command
// Shows the history of a service, a list of builds
var HistoryService = cli.Command{
	Name:      "history",
	Usage:     "Shows the history of a service, a list of builds",
	ArgsUsage: "<service_name>",
	Description: `Shows the history of a service, a list of builds and its status and basic information.

   Example:
    $ ernest history myservice
	`,
	Action: func(c *cli.Context) error {
		m, cfg := setup(c)
		if cfg.Token == "" {
			color.Red("You're not allowed to perform this action, please log in")
			return nil
		}

		if len(c.Args()) < 1 {
			color.Red("You should specify an existing service name")
			return nil
		}

		serviceName := c.Args()[0]

		services, _ := m.ListBuilds(serviceName, cfg.Token)
		printServiceHistory(services)
		return nil
	},
}

// ResetService command
var ResetService = cli.Command{
	Name:      "reset",
	ArgsUsage: "<service_name>",
	Usage:     "Reset an in progress service.",
	Description: `Reseting a service creation may cause problems, please make sure you know what are you doing.

   Example:
    $ ernest reset myservice
  `,
	Action: func(c *cli.Context) error {
		m, cfg := setup(c)
		if cfg.Token == "" {
			color.Red("You're not allowed to perform this action, please log in")
			return nil
		}

		if len(c.Args()) < 1 {
			color.Red("You should specify the service name")
			return nil
		}
		serviceName := c.Args()[0]
		err := m.ResetService(serviceName, cfg.Token)
		if err != nil {
			color.Red(err.Error())
			return nil
		}
		color.Red("You've successfully resetted the service '" + serviceName + "'")

		return nil
	},
}

// DefinitionService command
// Shows the current definition of a service by its name
var DefinitionService = cli.Command{
	Name:      "definition",
	Aliases:   []string{"s"},
	ArgsUsage: "<service_name>",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "build",
			Value: "",
			Usage: "Build ID",
		},
	},
	Usage: "Show the current definition of a service by its name",
	Description: `Show the current definition of a service by its name getting the definition about the build.

   Example:
    $ ernest service definition myservice
	`,
	Action: func(c *cli.Context) error {
		m, cfg := setup(c)
		if cfg.Token == "" {
			color.Red("You're not allowed to perform this action, please log in")
			return nil
		}

		if len(c.Args()) < 1 {
			color.Red("You should specify the service name")
		}
		serviceName := c.Args()[0]
		if c.String("build") != "" {
			service, err := m.ServiceBuildStatus(cfg.Token, serviceName, c.String("build"))
			if err != nil {
				color.Red(err.Error())
				os.Exit(1)
			}
			fmt.Println(service.Definition)
		} else {
			service, err := m.ServiceStatus(cfg.Token, serviceName)
			if err != nil {
				color.Red(err.Error())
				os.Exit(1)
			}

			fmt.Println(service.Definition)
		}
		return nil
	},
}

// InfoService : Shows detailed information of a service by its name
var InfoService = cli.Command{
	Name:      "info",
	Aliases:   []string{"i"},
	ArgsUsage: "<service_name>",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "build",
			Value: "",
			Usage: "Build ID",
		},
	},
	Usage: "$ ernest service info <my_service> --build <specific build>",
	Description: `Will show detailed information of the last build of a specified service.
	In case you specify --build option you will be able to output the detailed information of specific build of a service.

   Examples:
    $ ernest service definition myservice
    $ ernest service definition myservice --build build1
	`,
	Action: func(c *cli.Context) error {
		var err error
		var service Service

		m, cfg := setup(c)
		if cfg.Token == "" {
			color.Red("You're not allowed to perform this action, please log in")
			return nil
		}

		if len(c.Args()) == 0 {
			color.Red("You should specify an existing service name")
			return nil
		}

		name := c.Args()[0]
		if c.String("build") != "" {
			build := c.String("build")
			service, err = m.ServiceBuildStatus(cfg.Token, name, build)
		} else {
			service, err = m.ServiceStatus(cfg.Token, name)
		}

		if err != nil {
			color.Red(err.Error())
			os.Exit(1)
		}
		printServiceInfo(&service)
		return nil
	},
}

// DiffService : Shows detailed information of a service by its name
var DiffService = cli.Command{
	Name:      "diff",
	Aliases:   []string{"i"},
	ArgsUsage: "<service_name> <build_a> <build_b>",
	Usage:     "$ ernest service diff <service_name> <build_a> <build_b>",
	Description: `Will display the diff between two different builds

   Examples:
    $ ernest service diff 1283018731 9182yuhds12
	`,
	Action: func(c *cli.Context) error {
		var err error

		m, cfg := setup(c)
		if cfg.Token == "" {
			color.Red("You're not allowed to perform this action, please log in")
			return nil
		}

		if len(c.Args()) < 2 {
			color.Red("You should specify the service name and two build ids to compare them")
			return nil
		}

		serviceName := c.Args()[0]
		b1 := c.Args()[1]
		b2 := c.Args()[2]

		build1, err := m.ServiceBuildStatus(cfg.Token, serviceName, b1)
		if err != nil {
			color.Red(err.Error())
			return nil
		}
		build2, err := m.ServiceBuildStatus(cfg.Token, serviceName, b2)
		if err != nil {
			color.Red(err.Error())
			return nil
		}

		if err != nil {
			color.Red(err.Error())
			os.Exit(1)
		}
		printServiceDiff(build1, build2)
		return nil
	},
}

// CmdService ...
var CmdService = cli.Command{
	Name:  "service",
	Usage: "Service related subcommands",
	Subcommands: []cli.Command{
		ListServices,
		ApplyService,
		DestroyService,
		HistoryService,
		ResetService,
		DefinitionService,
		InfoService,
		MonitorService,
		DiffService,
	},
}
<commit_msg>Update service_controller.go - DefinitionService<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package main

// CmdDatacenter subcommand
import (
	"fmt"
	"os"

	"github.com/fatih/color"
	"github.com/urfave/cli"
)

// ListServices ...
var ListServices = cli.Command{
	Name:      "list",
	Usage:     "List available services.",
	ArgsUsage: " ",
	Description: `List available services and shows its most relevant information.

   Example:
    $ ernest service list
	`,
	Action: func(c *cli.Context) error {
		m, cfg := setup(c)
		if cfg.Token == "" {
			color.Red("You're not allowed to perform this action, please log in")
			return nil
		}
		services, err := m.ListServices(cfg.Token)
		if err != nil {
			color.Red(err.Error())
			return err
		}

		printServiceList(services)
		return nil
	},
}

// ApplyService command
// Applies changes described on a YAML file to a service
var ApplyService = cli.Command{
	Name:      "apply",
	Aliases:   []string{"a"},
	ArgsUsage: "<file.yml>",
	Usage:     "Builds or changes infrastructure.",
	Description: `Sends a service YAML description file to Ernest to be executed.
   You must be logged in to execute this command.

   If the file is not provided, ernest.yml will be used by default.

   Example:
    $ ernest apply myservice.yml
	`,
	Action: func(c *cli.Context) error {
		file := "ernest.yml"
		if len(c.Args()) == 1 {
			file = c.Args()[0]
		}
		m, cfg := setup(c)
		if cfg.Token == "" {
			color.Red("You're not allowed to perform this action, please log in")
			return nil
		}
		_, err := m.Apply(cfg.Token, file, true)
		if err != nil {
			color.Red(err.Error())
		}
		return nil
	},
}

// DestroyService command
var DestroyService = cli.Command{
	Name:      "destroy",
	Aliases:   []string{"d"},
	ArgsUsage: "<service_name>",
	Usage:     "Destroy a service.",
	Description: `Destroys a service by its name.

   Example:
    $ ernest destroy myservice
  `,
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name:  "force,f",
			Usage: "Force destroy command without asking for permission.",
		},
	},
	Action: func(c *cli.Context) error {
		m, cfg := setup(c)
		if cfg.Token == "" {
			color.Red("You're not allowed to perform this action, please log in")
			return nil
		}

		if len(c.Args()) < 1 {
			color.Red("You should specify an existing service name")
			return nil
		}
		name := c.Args()[0]

		if c.Bool("force") {
			err := m.Destroy(cfg.Token, name, true)
			if err != nil {
				color.Red(err.Error())
			}
		} else {
			fmt.Print("Are you sure? Please type yes or no and then press enter: ")
			if askForConfirmation() {
				err := m.Destroy(cfg.Token, name, true)
				if err != nil {
					color.Red(err.Error())
					return err
				}
			}
		}
		return nil
	},
}

// HistoryService command
// Shows the history of a service, a list of builds
var HistoryService = cli.Command{
	Name:      "history",
	Usage:     "Shows the history of a service, a list of builds",
	ArgsUsage: "<service_name>",
	Description: `Shows the history of a service, a list of builds and its status and basic information.

   Example:
    $ ernest history myservice
	`,
	Action: func(c *cli.Context) error {
		m, cfg := setup(c)
		if cfg.Token == "" {
			color.Red("You're not allowed to perform this action, please log in")
			return nil
		}

		if len(c.Args()) < 1 {
			color.Red("You should specify an existing service name")
			return nil
		}

		serviceName := c.Args()[0]

		services, _ := m.ListBuilds(serviceName, cfg.Token)
		printServiceHistory(services)
		return nil
	},
}

// ResetService command
var ResetService = cli.Command{
	Name:      "reset",
	ArgsUsage: "<service_name>",
	Usage:     "Reset an in progress service.",
	Description: `Reseting a service creation may cause problems, please make sure you know what are you doing.

   Example:
    $ ernest reset myservice
  `,
	Action: func(c *cli.Context) error {
		m, cfg := setup(c)
		if cfg.Token == "" {
			color.Red("You're not allowed to perform this action, please log in")
			return nil
		}

		if len(c.Args()) < 1 {
			color.Red("You should specify the service name")
			return nil
		}
		serviceName := c.Args()[0]
		err := m.ResetService(serviceName, cfg.Token)
		if err != nil {
			color.Red(err.Error())
			return nil
		}
		color.Red("You've successfully resetted the service '" + serviceName + "'")

		return nil
	},
}

// DefinitionService command
// Shows the current definition of a service by its name
var DefinitionService = cli.Command{
	Name:      "definition",
	Aliases:   []string{"s"},
	ArgsUsage: "<service_name>",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "build",
			Value: "",
			Usage: "Build ID",
		},
	},
	Usage: "Show the current definition of a service by its name",
	Description: `Show the current definition of a service by its name getting the definition about the build.

   Example:
    $ ernest service definition myservice
	`,
	Action: func(c *cli.Context) error {
		m, cfg := setup(c)
		if cfg.Token == "" {
			color.Red("You're not allowed to perform this action, please log in")
			return nil
		}

		if len(c.Args()) < 1 {
			color.Red("You should specify the service name")
			return nil
		}
		serviceName := c.Args()[0]
		if c.String("build") != "" {
			service, err := m.ServiceBuildStatus(cfg.Token, serviceName, c.String("build"))
			if err != nil {
				color.Red(err.Error())
				os.Exit(1)
			}
			fmt.Println(service.Definition)
		} else {
			service, err := m.ServiceStatus(cfg.Token, serviceName)
			if err != nil {
				color.Red(err.Error())
				os.Exit(1)
			}

			fmt.Println(service.Definition)
		}
		return nil
	},
}

// InfoService : Shows detailed information of a service by its name
var InfoService = cli.Command{
	Name:      "info",
	Aliases:   []string{"i"},
	ArgsUsage: "<service_name>",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "build",
			Value: "",
			Usage: "Build ID",
		},
	},
	Usage: "$ ernest service info <my_service> --build <specific build>",
	Description: `Will show detailed information of the last build of a specified service.
	In case you specify --build option you will be able to output the detailed information of specific build of a service.

   Examples:
    $ ernest service definition myservice
    $ ernest service definition myservice --build build1
	`,
	Action: func(c *cli.Context) error {
		var err error
		var service Service

		m, cfg := setup(c)
		if cfg.Token == "" {
			color.Red("You're not allowed to perform this action, please log in")
			return nil
		}

		if len(c.Args()) == 0 {
			color.Red("You should specify an existing service name")
			return nil
		}

		name := c.Args()[0]
		if c.String("build") != "" {
			build := c.String("build")
			service, err = m.ServiceBuildStatus(cfg.Token, name, build)
		} else {
			service, err = m.ServiceStatus(cfg.Token, name)
		}

		if err != nil {
			color.Red(err.Error())
			os.Exit(1)
		}
		printServiceInfo(&service)
		return nil
	},
}

// DiffService : Shows detailed information of a service by its name
var DiffService = cli.Command{
	Name:      "diff",
	Aliases:   []string{"i"},
	ArgsUsage: "<service_name> <build_a> <build_b>",
	Usage:     "$ ernest service diff <service_name> <build_a> <build_b>",
	Description: `Will display the diff between two different builds

   Examples:
    $ ernest service diff 1283018731 9182yuhds12
	`,
	Action: func(c *cli.Context) error {
		var err error

		m, cfg := setup(c)
		if cfg.Token == "" {
			color.Red("You're not allowed to perform this action, please log in")
			return nil
		}

		if len(c.Args()) < 2 {
			color.Red("You should specify the service name and two build ids to compare them")
			return nil
		}

		serviceName := c.Args()[0]
		b1 := c.Args()[1]
		b2 := c.Args()[2]

		build1, err := m.ServiceBuildStatus(cfg.Token, serviceName, b1)
		if err != nil {
			color.Red(err.Error())
			return nil
		}
		build2, err := m.ServiceBuildStatus(cfg.Token, serviceName, b2)
		if err != nil {
			color.Red(err.Error())
			return nil
		}

		if err != nil {
			color.Red(err.Error())
			os.Exit(1)
		}
		printServiceDiff(build1, build2)
		return nil
	},
}

// CmdService ...
var CmdService = cli.Command{
	Name:  "service",
	Usage: "Service related subcommands",
	Subcommands: []cli.Command{
		ListServices,
		ApplyService,
		DestroyService,
		HistoryService,
		ResetService,
		DefinitionService,
		InfoService,
		MonitorService,
		DiffService,
	},
}
<|endoftext|>


<commit_before>package endly

import (
	"github.com/viant/toolbox/url"
	"github.com/pkg/errors"
)

//DockerServiceTagRequest represents docker tag request
type DockerServiceTagRequest struct {
	Target    *url.Resource
	SysPath   []string
	SourceTag *DockerTag
	TargetTag *DockerTag
}

//DockerTag represent a docker tag
type DockerTag struct {
	Username string
	Registry string
	Image    string
	Version  string
}

//DockerServiceTagResponse represents docker tag response
type DockerServiceTagResponse struct {
	Stdout string
}

func (r *DockerServiceTagRequest) Validate() error {
	if r.SourceTag == nil {
		return errors.New("sourceImage was empty")
	}
	if r.TargetTag == nil {
		return errors.New("sourceImage was empty")
	}
	if err := r.SourceTag.Validate();err != nil {
		return err
	}
	if err := r.TargetTag.Validate();err != nil {
		return err
	}
	return nil
}

func (t *DockerTag) Validate() error {
	if t.Image == "" {
		return errors.New("image was empty")
	}
	return nil
}

//String stringify docker tag
func (t *DockerTag) String() string {
	var result = t.Username
	if result == "" {
		result = t.Registry
	}
	if result != "" {
		result += "/"
	}
	result += t.Image
	if t.Version != "" {
		result += ":" + t.Version
	}
	return result
}
<commit_msg>added docker tag<commit_after>package endly

import (
	"github.com/viant/toolbox/url"
	"github.com/pkg/errors"
)

//DockerServiceTagRequest represents docker tag request
type DockerServiceTagRequest struct {
	Target    *url.Resource
	SysPath   []string
	SourceTag *DockerTag
	TargetTag *DockerTag
}

//DockerTag represent a docker tag
type DockerTag struct {
	Username string
	Registry string
	Image    string
	Version  string
}

//DockerServiceTagResponse represents docker tag response
type DockerServiceTagResponse struct {
	Stdout string
}

func (r *DockerServiceTagRequest) Validate() error {
	if r.Target == nil {
		return errors.New("target was empty")
	}
	if r.SourceTag == nil {
		return errors.New("sourceImage was empty")
	}
	if r.TargetTag == nil {
		return errors.New("sourceImage was empty")
	}
	if err := r.SourceTag.Validate(); err != nil {
		return err
	}
	if err := r.TargetTag.Validate(); err != nil {
		return err
	}
	return nil
}

func (t *DockerTag) Validate() error {
	if t.Image == "" {
		return errors.New("image was empty")
	}
	return nil
}

//String stringify docker tag
func (t *DockerTag) String() string {
	var result = t.Username
	if result == "" {
		result = t.Registry
	}
	if result != "" {
		result += "/"
	}
	result += t.Image
	if t.Version != "" {
		result += ":" + t.Version
	}
	return result
}
<|endoftext|>