<commit_before>package main

import (
	"database/sql"
	"flag"
	"fmt"
	"log"
	"os"
	"runtime/debug"
	"time"

	"github.com/FreeFeed/clio-restore/internal/account"
	"github.com/FreeFeed/clio-restore/internal/config"
	"github.com/FreeFeed/clio-restore/internal/dbutil"
	"github.com/FreeFeed/clio-restore/internal/hashtags"
	"github.com/davidmz/mustbe"
	"github.com/lib/pq"
	"gopkg.in/gomail.v2"
)

// Globals
var (
	infoLog  = log.New(os.Stdout, "INFO  ", log.LstdFlags)
	errorLog = log.New(os.Stdout, "ERROR ", log.LstdFlags)
	fatalLog = log.New(os.Stdout, "FATAL ", log.LstdFlags)
)

func main() {
	defer mustbe.Catched(func(err error) {
		fatalLog.Println(err)
		debug.PrintStack()
	})

	flag.Parse()

	conf := mustbe.OKVal(config.Load()).(*config.Config)

	db := mustbe.OKVal(sql.Open("postgres", conf.DbStr)).(*sql.DB)
	mustbe.OK(db.Ping())

	accStore := account.NewStore(db)

	// Looking for users who allow to restore their comments and likes
	var accounts []*account.Account
	mustbe.OK(dbutil.QueryRows(
		db, "select old_username from archives where restore_comments_and_likes", nil,
		func(r dbutil.RowScanner) error {
			var name string
			if err := r.Scan(&name); err != nil {
				return err
			}
			accounts = append(accounts, accStore.Get(name))
			return nil
		},
	))

	infoLog.Printf("Found %d users who allow to restore comments and likes", len(accounts))

	for _, acc := range accounts {
		infoLog.Printf("Processing %q (now %q)", acc.OldUserName, acc.NewUserName)

		if !acc.IsExists() {
			errorLog.Printf("Looks like account with old username %q doesn't exists", acc.OldUserName)
			continue
		}

		var existsComments, existsLikes bool

		mustbe.OK(db.QueryRow(
			`select exists(select 1 from hidden_comments where user_id = $1 or old_username = $2)`,
			acc.UID, acc.OldUserName,
		).Scan(&existsComments))

		mustbe.OK(db.QueryRow(
			`select exists(select 1 from hidden_likes where user_id = $1 or old_username = $2)`,
			acc.UID, acc.OldUserName,
		).Scan(&existsLikes))

		if !existsComments && !existsLikes {
			continue
		}

		dbutil.MustTransact(db, func(tx *sql.Tx) {
			if existsComments {
				infoLog.Printf("Restoring hidden comments of %q (now %q)", acc.OldUserName, acc.NewUserName)
				restoreComments(tx, acc)
			}
			if existsLikes {
				infoLog.Printf("Restoring hidden likes of %q (now %q)", acc.OldUserName, acc.NewUserName)
				restoreLikes(tx, acc)
			}
		})

		if conf.SMTPHost != "" {
			dialer := gomail.NewDialer(conf.SMTPHost, conf.SMTPPort, conf.SMTPUsername, conf.SMTPPassword)
			mail := gomail.NewMessage()
			mail.SetHeader("From", conf.SMTPFrom)
			mail.SetHeader("To", acc.Email, conf.SMTPBcc)
			mail.SetHeader("Subject", "Archive comments restoration request")
			mail.SetBody("text/plain",
				fmt.Sprintf(
					"Comments restoration for FreeFeed user %q (FriendFeed username %q) has been completed.",
					acc.NewUserName, acc.OldUserName,
				),
			)
			if err := dialer.DialAndSend(mail); err != nil {
				errorLog.Printf("Cannot send email to %q: %v", acc.Email, err)
			}
		}
	}
}

const batchSize = 100

func restoreComments(tx *sql.Tx, acc *account.Account) {
	var (
		feeds pq.Int64Array
		count int
	)
	// Feeds to append commented post to
	mustbe.OK(tx.QueryRow(
		`select array_agg(distinct f.id) from
				feeds f join subscriptions s on
					f.user_id = s.user_id and f.name = 'RiverOfNews' or f.uid = s.feed_id
				where s.feed_id = $1`,
		acc.Feeds.Comments.UID,
	).Scan(&feeds))

	processedPosts := make(map[string]bool) // postID is a key

	type commentInfo struct {
		ID     string
		PostID string
		Body   string
	}

	for {
		var comments []commentInfo
		dbutil.MustQueryRows(tx,
			`select hc.comment_id, c.post_id, hc.body from
				hidden_comments hc
				join comments c on c.uid = hc.comment_id
				where hc.user_id = $1 or hc.old_username = $2
				limit $3`,
			dbutil.Args{acc.UID, acc.OldUserName, batchSize},
			func(r dbutil.RowScanner) {
				ci := commentInfo{}
				mustbe.OK(r.Scan(&ci.ID, &ci.PostID, &ci.Body))
				comments = append(comments, ci)
			})
		if len(comments) == 0 {
			break
		}

		for _, ci := range comments {
			mustbe.OKVal(tx.Exec(
				"update comments set (body, user_id, hide_type) = ($1, $2, $3) where uid = $4",
				ci.Body, acc.UID, 0, ci.ID,
			))
			mustbe.OKVal(tx.Exec("delete from hidden_comments where comment_id = $1", ci.ID))

			for _, h := range hashtags.Extract(ci.Body) {
				dbutil.MustInsertWithoutConflict(tx, "hashtag_usages", dbutil.H{
					"hashtag_id": hashtags.GetID(tx, h),
					"entity_id":  ci.ID,
					"type":       "comment",
				})
			}

			if !processedPosts[ci.PostID] && len(feeds) != 0 {
				mustbe.OKVal(tx.Exec(
					"update posts set feed_ids = feed_ids | $1 where uid = $2",
					feeds, ci.PostID,
				))
				processedPosts[ci.PostID] = true
			}
			count++
		}
	}

	mustbe.OKVal(tx.Exec(
		`update user_stats set comments_count = comments_count + $1 where user_id = $2`,
		count, acc.UID,
	))

	infoLog.Printf("Restored %d comments in %d posts", count, len(processedPosts))
}

func restoreLikes(tx *sql.Tx, acc *account.Account) {
	var (
		feeds pq.Int64Array
		count int
	)
	// Feeds to append liked post to
	mustbe.OK(tx.QueryRow(
		`select array_agg(distinct f.id) from
				feeds f join subscriptions s on
					f.user_id = s.user_id and f.name = 'RiverOfNews' or f.uid = s.feed_id
				where s.feed_id = $1`,
		acc.Feeds.Likes.UID,
	).Scan(&feeds))

	type likeInfo struct {
		ID     int
		PostID string
		Date   time.Time
	}

	for {
		var likes []likeInfo

		dbutil.MustQueryRows(tx,
			`select id, post_id, date from hidden_likes
			where user_id = $1 or old_username = $2`,
			dbutil.Args{acc.UID, acc.OldUserName},
			func(r dbutil.RowScanner) {
				li := likeInfo{}
				mustbe.OK(r.Scan(&li.ID, &li.PostID, &li.Date))
				likes = append(likes, li)
			},
		)
		if len(likes) == 0 {
			break
		}

		for _, li := range likes {
			// Probably this post alreaady have like from this user
			// so we should use 'WithoutConflict'
			res := dbutil.MustInsertWithoutConflict(tx, "likes", dbutil.H{
				"post_id":    li.PostID,
				"user_id":    acc.UID,
				"created_at": li.Date,
			})
			rowsAffected := mustbe.OKVal(res.RowsAffected()).(int64)
			mustbe.OKVal(tx.Exec("delete from hidden_likes where id = $1", li.ID))
			if rowsAffected > 0 && len(feeds) != 0 {
				mustbe.OKVal(tx.Exec(
					"update posts set feed_ids = feed_ids | $1 where uid = $2",
					feeds, li.PostID,
				))
				count++
			}
		}
	}

	mustbe.OKVal(tx.Exec(
		`update user_stats set likes_count = likes_count + $1 where user_id = $2`,
		count, acc.UID,
	))

	infoLog.Printf("Restored %d likes", count)
}
<commit_msg>Fix the posts/likes counters bug<commit_after>package main

import (
	"database/sql"
	"flag"
	"fmt"
	"log"
	"os"
	"runtime/debug"
	"time"

	"github.com/FreeFeed/clio-restore/internal/account"
	"github.com/FreeFeed/clio-restore/internal/config"
	"github.com/FreeFeed/clio-restore/internal/dbutil"
	"github.com/FreeFeed/clio-restore/internal/hashtags"
	"github.com/davidmz/mustbe"
	"github.com/lib/pq"
	"gopkg.in/gomail.v2"
)

// Globals
var (
	infoLog  = log.New(os.Stdout, "INFO  ", log.LstdFlags)
	errorLog = log.New(os.Stdout, "ERROR ", log.LstdFlags)
	fatalLog = log.New(os.Stdout, "FATAL ", log.LstdFlags)
)

func main() {
	defer mustbe.Catched(func(err error) {
		fatalLog.Println(err)
		debug.PrintStack()
	})

	flag.Parse()

	conf := mustbe.OKVal(config.Load()).(*config.Config)

	db := mustbe.OKVal(sql.Open("postgres", conf.DbStr)).(*sql.DB)
	mustbe.OK(db.Ping())

	accStore := account.NewStore(db)

	// Looking for users who allow to restore their comments and likes
	var accounts []*account.Account
	mustbe.OK(dbutil.QueryRows(
		db, "select old_username from archives where restore_comments_and_likes", nil,
		func(r dbutil.RowScanner) error {
			var name string
			if err := r.Scan(&name); err != nil {
				return err
			}
			accounts = append(accounts, accStore.Get(name))
			return nil
		},
	))

	infoLog.Printf("Found %d users who allow to restore comments and likes", len(accounts))

	for _, acc := range accounts {
		infoLog.Printf("Processing %q (now %q)", acc.OldUserName, acc.NewUserName)

		if !acc.IsExists() {
			errorLog.Printf("Looks like account with old username %q doesn't exists", acc.OldUserName)
			continue
		}

		var existsComments, existsLikes bool

		mustbe.OK(db.QueryRow(
			`select exists(select 1 from hidden_comments where user_id = $1 or old_username = $2)`,
			acc.UID, acc.OldUserName,
		).Scan(&existsComments))

		mustbe.OK(db.QueryRow(
			`select exists(select 1 from hidden_likes where user_id = $1 or old_username = $2)`,
			acc.UID, acc.OldUserName,
		).Scan(&existsLikes))

		if !existsComments && !existsLikes {
			continue
		}

		dbutil.MustTransact(db, func(tx *sql.Tx) {
			if existsComments {
				infoLog.Printf("Restoring hidden comments of %q (now %q)", acc.OldUserName, acc.NewUserName)
				restoreComments(tx, acc)
			}
			if existsLikes {
				infoLog.Printf("Restoring hidden likes of %q (now %q)", acc.OldUserName, acc.NewUserName)
				restoreLikes(tx, acc)
			}
		})

		if conf.SMTPHost != "" {
			dialer := gomail.NewDialer(conf.SMTPHost, conf.SMTPPort, conf.SMTPUsername, conf.SMTPPassword)
			mail := gomail.NewMessage()
			mail.SetHeader("From", conf.SMTPFrom)
			mail.SetHeader("To", acc.Email, conf.SMTPBcc)
			mail.SetHeader("Subject", "Archive comments restoration request")
			mail.SetBody("text/plain",
				fmt.Sprintf(
					"Comments restoration for FreeFeed user %q (FriendFeed username %q) has been completed.",
					acc.NewUserName, acc.OldUserName,
				),
			)
			if err := dialer.DialAndSend(mail); err != nil {
				errorLog.Printf("Cannot send email to %q: %v", acc.Email, err)
			}
		}
	}
}

const batchSize = 100

func restoreComments(tx *sql.Tx, acc *account.Account) {
	var (
		feeds pq.Int64Array
		count int
	)
	// Feeds to append commented post to
	mustbe.OK(tx.QueryRow(
		`select array_agg(distinct f.id) from
				feeds f join subscriptions s on
					f.user_id = s.user_id and f.name = 'RiverOfNews' or f.uid = s.feed_id
				where s.feed_id = $1`,
		acc.Feeds.Comments.UID,
	).Scan(&feeds))

	processedPosts := make(map[string]bool) // postID is a key

	type commentInfo struct {
		ID     string
		PostID string
		Body   string
	}

	for {
		var comments []commentInfo
		dbutil.MustQueryRows(tx,
			`select hc.comment_id, c.post_id, hc.body from
				hidden_comments hc
				join comments c on c.uid = hc.comment_id
				where hc.user_id = $1 or hc.old_username = $2
				limit $3`,
			dbutil.Args{acc.UID, acc.OldUserName, batchSize},
			func(r dbutil.RowScanner) {
				ci := commentInfo{}
				mustbe.OK(r.Scan(&ci.ID, &ci.PostID, &ci.Body))
				comments = append(comments, ci)
			})
		if len(comments) == 0 {
			break
		}

		for _, ci := range comments {
			mustbe.OKVal(tx.Exec(
				"update comments set (body, user_id, hide_type) = ($1, $2, $3) where uid = $4",
				ci.Body, acc.UID, 0, ci.ID,
			))
			mustbe.OKVal(tx.Exec("delete from hidden_comments where comment_id = $1", ci.ID))

			for _, h := range hashtags.Extract(ci.Body) {
				dbutil.MustInsertWithoutConflict(tx, "hashtag_usages", dbutil.H{
					"hashtag_id": hashtags.GetID(tx, h),
					"entity_id":  ci.ID,
					"type":       "comment",
				})
			}

			if !processedPosts[ci.PostID] {
				if len(feeds) != 0 {
					mustbe.OKVal(tx.Exec(
						"update posts set feed_ids = feed_ids | $1 where uid = $2",
						feeds, ci.PostID,
					))
				}
				processedPosts[ci.PostID] = true
			}
			count++
		}
	}

	mustbe.OKVal(tx.Exec(
		`update user_stats set comments_count = comments_count + $1 where user_id = $2`,
		count, acc.UID,
	))

	infoLog.Printf("Restored %d comments in %d posts", count, len(processedPosts))
}

func restoreLikes(tx *sql.Tx, acc *account.Account) {
	var (
		feeds pq.Int64Array
		count int
	)
	// Feeds to append liked post to
	mustbe.OK(tx.QueryRow(
		`select array_agg(distinct f.id) from
				feeds f join subscriptions s on
					f.user_id = s.user_id and f.name = 'RiverOfNews' or f.uid = s.feed_id
				where s.feed_id = $1`,
		acc.Feeds.Likes.UID,
	).Scan(&feeds))

	type likeInfo struct {
		ID     int
		PostID string
		Date   time.Time
	}

	for {
		var likes []likeInfo

		dbutil.MustQueryRows(tx,
			`select id, post_id, date from hidden_likes
			where user_id = $1 or old_username = $2`,
			dbutil.Args{acc.UID, acc.OldUserName},
			func(r dbutil.RowScanner) {
				li := likeInfo{}
				mustbe.OK(r.Scan(&li.ID, &li.PostID, &li.Date))
				likes = append(likes, li)
			},
		)
		if len(likes) == 0 {
			break
		}

		for _, li := range likes {
			// Probably this post alreaady have like from this user
			// so we should use 'WithoutConflict'
			res := dbutil.MustInsertWithoutConflict(tx, "likes", dbutil.H{
				"post_id":    li.PostID,
				"user_id":    acc.UID,
				"created_at": li.Date,
			})
			rowsAffected := mustbe.OKVal(res.RowsAffected()).(int64)
			mustbe.OKVal(tx.Exec("delete from hidden_likes where id = $1", li.ID))
			if rowsAffected > 0 {
				if len(feeds) != 0 {
					mustbe.OKVal(tx.Exec(
						"update posts set feed_ids = feed_ids | $1 where uid = $2",
						feeds, li.PostID,
					))
				}
				count++
			}
		}
	}

	mustbe.OKVal(tx.Exec(
		`update user_stats set likes_count = likes_count + $1 where user_id = $2`,
		count, acc.UID,
	))

	infoLog.Printf("Restored %d likes", count)
}
<|endoftext|>


<commit_before>// +build !windows,!plan9

package edit

import (
	"testing"
	"time"

	"github.com/elves/elvish/daemon/api"
	"github.com/elves/elvish/eval"
	"github.com/elves/elvish/sys"
	"github.com/kr/pty"
)

var readLineTests = []struct {
	input string
	want  string
}{
	{"\n", ""},
	{"test\n", "test"},
	{"abc\x7fd\n", "abd"},
	{"abc\x17d\n", "d"},
}

var readLineTimeout = 5 * time.Second

func TestReadLine(t *testing.T) {
	master, tty, err := pty.Open()
	if err != nil {
		panic(err)
	}
	// Continually consume tty outputs so that the editor is not blocked on
	// writing.
	go func() {
		var buf [64]byte
		for {
			_, err := master.Read(buf[:])
			if err != nil {
				break
			}
		}
	}()

	ev := eval.NewEvaler(api.NewClient("/invalid"), nil, "", make(map[string]eval.Namespace))

	for _, test := range readLineTests {
		lineChan := make(chan string)
		errChan := make(chan error)
		go func() {
			ed := NewEditor(tty, tty, nil, ev)
			defer ed.Close()
			line, err := ed.ReadLine()
			if err != nil {
				errChan <- err
			} else {
				lineChan <- line
			}
		}()

		_, err := master.WriteString(test.input)
		if err != nil {
			panic(err)
		}

		select {
		case line := <-lineChan:
			if line != test.want {
				t.Errorf("ReadLine() => %q, want %q (input %q)", line, test.want, test.input)
			}
		case err := <-errChan:
			t.Errorf("ReadLine() => error %v (input %q)", err, test.input)
		case <-time.After(readLineTimeout):
			t.Errorf("ReadLine() timed out (input %q)", test.input)
			t.Log("\n" + sys.DumpStack())
		}
	}
}
<commit_msg>edit: In editor_unix_test.go, close files correctly.<commit_after>// +build !windows,!plan9

package edit

import (
	"testing"
	"time"

	"github.com/elves/elvish/daemon/api"
	"github.com/elves/elvish/eval"
	"github.com/elves/elvish/sys"
	"github.com/kr/pty"
)

var readLineTests = []struct {
	input string
	want  string
}{
	{"\n", ""},
	{"test\n", "test"},
	{"abc\x7fd\n", "abd"},
	{"abc\x17d\n", "d"},
}

var readLineTimeout = 5 * time.Second

func TestReadLine(t *testing.T) {
	master, tty, err := pty.Open()
	if err != nil {
		panic(err)
	}
	defer master.Close()
	defer tty.Close()
	// Continually consume tty outputs so that the editor is not blocked on
	// writing.
	go func() {
		var buf [64]byte
		for {
			_, err := master.Read(buf[:])
			if err != nil {
				break
			}
		}
	}()

	ev := eval.NewEvaler(api.NewClient("/invalid"), nil, "", make(map[string]eval.Namespace))

	for _, test := range readLineTests {
		lineChan := make(chan string)
		errChan := make(chan error)
		go func() {
			ed := NewEditor(tty, tty, nil, ev)
			defer ed.Close()
			line, err := ed.ReadLine()
			if err != nil {
				errChan <- err
			} else {
				lineChan <- line
			}
		}()

		_, err := master.WriteString(test.input)
		if err != nil {
			panic(err)
		}

		select {
		case line := <-lineChan:
			if line != test.want {
				t.Errorf("ReadLine() => %q, want %q (input %q)", line, test.want, test.input)
			}
		case err := <-errChan:
			t.Errorf("ReadLine() => error %v (input %q)", err, test.input)
		case <-time.After(readLineTimeout):
			t.Errorf("ReadLine() timed out (input %q)", test.input)
			t.Log("\n" + sys.DumpStack())
		}
	}
}
<|endoftext|>


<commit_before>package memdb

import (
	"database/sql/driver"
	"fmt"

	u "github.com/araddon/gou"
	"github.com/hashicorp/go-memdb"
	"golang.org/x/net/context"

	"github.com/araddon/qlbridge/datasource"
	"github.com/araddon/qlbridge/expr"
	"github.com/araddon/qlbridge/rel"
	"github.com/araddon/qlbridge/schema"
	"github.com/araddon/qlbridge/value"
	"github.com/araddon/qlbridge/vm"
)

const (
	sourceType = "memdb"
)

var (
	_ = u.EMPTY

	// Different Features of this Static Data Source
	_ schema.Source            = (*MemDb)(nil)
	_ schema.SourceTableSchema = (*MemDb)(nil)

	// Connection
	_ schema.Conn         = (*dbConn)(nil)
	_ schema.ConnColumns  = (*dbConn)(nil)
	_ schema.ConnScanner  = (*dbConn)(nil)
	_ schema.ConnUpsert   = (*dbConn)(nil)
	_ schema.ConnDeletion = (*dbConn)(nil)
	_ schema.ConnSeeker   = (*dbConn)(nil)
)

// MemDb implements qlbridge `Source` to allow in-memory native go data
//   to have a Schema and implement and be operated on by Sql Operations
//
// Features
// - ues immuteable radix-tree/db mvcc under the hood
//
type MemDb struct {
	exit           <-chan bool
	*schema.Schema                 // schema
	tbl            *schema.Table   // schema table
	indexes        []*schema.Index // index descriptions
	primaryIndex   string
	db             *memdb.MemDB
	max            int
}
type dbConn struct {
	md     *MemDb
	db     *memdb.MemDB
	txn    *memdb.Txn
	result memdb.ResultIterator
}

// NewMemDbData creates a MemDb with given indexes, columns, and values
func NewMemDbData(name string, data [][]driver.Value, cols []string) (*MemDb, error) {

	m, err := NewMemDb(name, cols)
	if err != nil {
		return nil, err
	}
	// Insert initial values
	conn := newDbConn(m)
	for _, row := range data {
		conn.Put(nil, nil, row)
	}
	conn.Close()
	return m, nil
}

// NewMemDb creates a MemDb with given indexes, columns
func NewMemDb(name string, cols []string) (*MemDb, error) {

	ss := schema.NewSchemaSource(name, sourceType)

	return NewMemDbForSchema(name, ss, cols)
}

// NewMemDbForSchema creates a MemDb with given indexes, columns
func NewMemDbForSchema(name string, ss *schema.SchemaSource, cols []string) (*MemDb, error) {

	if len(cols) < 1 {
		return nil, fmt.Errorf("must have columns provided")
	}

	m := &MemDb{}

	m.tbl = schema.NewTable(name, ss)
	ss.AddTable(m.tbl)
	if ss.Schema == nil {
		m.Schema = schema.NewSchema(name)
		m.Schema.AddSourceSchema(ss)
	} else {
		m.Schema = ss.Schema
	}

	m.tbl.SetColumns(cols)

	err := m.buildDefaultIndexes()
	if err != nil {
		u.Errorf("Default indexes could not be built %v", err)
		return nil, err
	}

	mdbSchema, err := makeMemDbSchema(m)
	if err != nil {
		u.Errorf("Must have valid schema %v", err)
		return nil, err
	}
	db, err := memdb.NewMemDB(mdbSchema)
	if err != nil {
		u.Warnf("could not create db %v", err)
		return nil, err
	}
	m.db = db
	return m, nil
}

// Open a Conn for this source @table name
func (m *MemDb) Open(table string) (schema.Conn, error) { return newDbConn(m), nil }

// Table by name
func (m *MemDb) Table(table string) (*schema.Table, error) { return m.tbl, nil }

// Close this source
func (m *MemDb) Close() error { return nil }

// Tables list, should be single table
func (m *MemDb) Tables() []string { return []string{m.tbl.Name} }

func (m *MemDb) buildDefaultIndexes() error {
	if len(m.indexes) == 0 {
		if len(m.tbl.Columns()) < 1 {
			return fmt.Errorf("must have columns if no index provided")
		}
		u.Debugf("no index provided creating on %q", m.tbl.Columns()[0])
		m.indexes = []*schema.Index{
			{Name: "id", Fields: []string{m.tbl.Columns()[0]}},
		}
	}
	// First ensure we have one primary index
	hasPrimary := false
	for _, idx := range m.indexes {
		if idx.PrimaryKey {
			if idx.Name == "" {
				idx.Name = "id"
			}
			m.primaryIndex = idx.Name
			hasPrimary = true
		}
	}
	if !hasPrimary {
		m.indexes[0].PrimaryKey = true
		m.primaryIndex = m.indexes[0].Name
	}
	return nil
}

//func (m *MemDb) SetColumns(cols []string)                  { m.tbl.SetColumns(cols) }

func newDbConn(mdb *MemDb) *dbConn {
	c := &dbConn{md: mdb, db: mdb.db}
	return c
}
func (m *dbConn) Columns() []string { return m.md.tbl.Columns() }
func (m *dbConn) Close() error      { return nil }
func (m *dbConn) CreateIterator() schema.Iterator {
	m.txn = m.db.Txn(false)
	// Attempt a row scan on the primary index
	result, err := m.txn.Get(m.md.tbl.Name, m.md.primaryIndex)
	if err != nil {
		u.Errorf("error %v", err)
	}
	m.result = result
	return m
}
func (m *dbConn) MesgChan() <-chan schema.Message {
	return datasource.SourceIterChannel(m.CreateIterator(), m.md.exit)
}

func (m *dbConn) Next() schema.Message {
	//u.Infof("Next()")
	if m.txn == nil {
		m.txn = m.db.Txn(false)
	}
	select {
	case <-m.md.exit:
		return nil
	default:
		for {
			if m.result == nil {
				result, err := m.txn.Get(m.md.tbl.Name, m.md.primaryIndex)
				if err != nil {
					u.Errorf("error %v", err)
					return nil
				}
				m.result = result
			}
			raw := m.result.Next()
			if raw == nil {
				return nil
			}
			if msg, ok := raw.(schema.Message); ok {
				return msg //.SqlDriverMessageMap.Copy()
			}
			u.Warnf("error, not correct type: %#v", raw)
			return nil
		}
	}
}

// interface for Upsert.Put()
func (m *dbConn) Put(ctx context.Context, key schema.Key, row interface{}) (schema.Key, error) {

	//u.Infof("%p Put(),  row:%#v", m, row)
	switch rowVals := row.(type) {
	case []driver.Value:
		txn := m.db.Txn(true)
		key, err := m.putValues(txn, rowVals)
		if err != nil {
			txn.Abort()
			return nil, err
		}
		txn.Commit()
		return key, nil
		/*
			case map[string]driver.Value:
				// We need to convert the key:value to []driver.Value so
				// we need to look up column index for each key, and write to vals

				// TODO:   if this is a partial update, we need to look up vals
				row := make([]driver.Value, len(m.Columns()))
				if len(rowVals) < len(m.Columns()) {
					// How do we get the key?
					//m.Get(key)
				}

				for key, val := range rowVals {
					if keyIdx, ok := m.tbl.FieldPositions[key]; ok {
						row[keyIdx] = val
					} else {
						return nil, fmt.Errorf("Found column in Put that doesn't exist in cols: %v", key)
					}
				}
				id := uint64(0)
				if key == nil {
					if row[m.indexCol] == nil {
						// Since we do not have an indexed column to work off of,
						// the ideal would be to get the job builder/planner to do
						// a scan with whatever info we have and feed that in?   Instead
						// of us implementing our own scan?
						u.Warnf("wtf, nil key? %v %v", m.indexCol, row)
						return nil, fmt.Errorf("cannot update on non index column ")
					}
					id = makeId(row[m.indexCol])
				} else {
					id = makeId(key)
					sdm, _ := m.Get(key)
					//u.Debugf("sdm: %#v  err%v", sdm, err)
					if sdm != nil {
						if dmval, ok := sdm.Body().(*datasource.SqlDriverMessageMap); ok {
							for i, val := range dmval.Values() {
								if row[i] == nil {
									row[i] = val
								}
							}
						}
					}
				}
				//u.Debugf("PUT: %#v", row)
				//u.Infof("PUT: %v  key:%v  row:%v", id, key, row)
				sdm := datasource.NewSqlDriverMessageMap(id, row, m.tbl.FieldPositions)
				item := DriverItem{sdm}
				m.bt.ReplaceOrInsert(&item)
				return NewKey(id), nil
		*/
	default:
		u.Warnf("not implemented %T", row)
		return nil, fmt.Errorf("Expected []driver.Value but got %T", row)
	}
}

func (m *dbConn) putValues(txn *memdb.Txn, row []driver.Value) (schema.Key, error) {
	if len(row) != len(m.Columns()) {
		u.Warnf("wrong column ct expected %d got %d for %v", len(m.Columns()), len(row), row)
		return nil, fmt.Errorf("Wrong number of columns, expected %v got %v", len(m.Columns()), len(row))
	}
	id := makeId(row[0])
	msg := &datasource.SqlDriverMessage{Vals: row, IdVal: id}
	txn.Insert(m.md.tbl.Name, msg)
	//u.Debugf("%p  PUT: id:%v IdVal:%v  Id():%v vals:%#v", m, id, sdm.IdVal, sdm.Id(), rowVals)
	return schema.NewKeyUint(id), nil
}

func (m *dbConn) PutMulti(ctx context.Context, keys []schema.Key, objs interface{}) ([]schema.Key, error) {
	txn := m.db.Txn(true)

	switch rows := objs.(type) {
	case [][]driver.Value:
		keys := make([]schema.Key, 0, len(rows))
		for _, row := range rows {
			key, err := m.putValues(txn, row)
			if err != nil {
				txn.Abort()
				return nil, err
			}
			keys = append(keys, key)
		}
		txn.Commit()
		return keys, nil
	}
	return nil, fmt.Errorf("unrecognized put object type: %T", objs)
}

// CanSeek is interface for Seeker, validate if we can perform this query
func (m *dbConn) CanSeek(sql *rel.SqlSelect) bool {
	return true
}

func (m *dbConn) Get(key driver.Value) (schema.Message, error) {
	txn := m.db.Txn(false)
	iter, err := txn.Get(m.md.tbl.Name, m.md.primaryIndex, fmt.Sprintf("%v", key))
	if err != nil {
		txn.Abort()
		u.Errorf("error reading %v because %v", key, err)
		return nil, err
	}
	txn.Commit() // noop

	if item := iter.Next(); item != nil {
		if msg, ok := item.(schema.Message); ok {
			return msg, nil
		}
		u.Warnf("unexpected type %T", item)
	}
	return nil, schema.ErrNotFound // Should not found be an error?
}

// MultiGet to get multiple items by keys
func (m *dbConn) MultiGet(keys []driver.Value) ([]schema.Message, error) {
	return nil, schema.ErrNotImplemented
}

// Interface for Deletion
func (m *dbConn) Delete(key driver.Value) (int, error) {
	txn := m.db.Txn(true)
	err := txn.Delete(m.md.tbl.Name, key)
	if err != nil {
		txn.Abort()
		u.Warnf("could not delete: %v  err=%v", key, err)
		return 0, err
	}
	txn.Commit()
	return 1, nil
}

// Delete using a Where Expression
func (m *dbConn) DeleteExpression(where expr.Node) (int, error) {
	//return 0, fmt.Errorf("not implemented")
	evaluator := vm.Evaluator(where)
	var deletedKeys []schema.Key
	txn := m.db.Txn(true)
	iter, err := txn.Get(m.md.tbl.Name, m.md.primaryIndex)
	if err != nil {
		txn.Abort()
		u.Errorf("could not get values %v", err)
		return 0, err
	}
deleteLoop:
	for {
		item := iter.Next()
		if item == nil {
			break
		}

		msg, ok := item.(datasource.SqlDriverMessage)
		if !ok {
			u.Warnf("wat?  %T   %#v", item, item)
			err = fmt.Errorf("unexpected message type %T", item)
			break
		}
		whereValue, ok := evaluator(msg.ToMsgMap(m.md.tbl.FieldPositions))
		if !ok {
			u.Debugf("could not evaluate where: %v", msg)
		}
		switch whereVal := whereValue.(type) {
		case value.BoolValue:
			if whereVal.Val() == false {
				//this means do NOT delete
			} else {
				// Delete!
				if err = txn.Delete(m.md.tbl.Name, msg); err != nil {
					u.Errorf("could not delete %v", err)
					break deleteLoop
				}
				indexVal := msg.Vals[0]
				deletedKeys = append(deletedKeys, schema.NewKeyUint(makeId(indexVal)))
			}
		case nil:
			// ??
			u.Warnf("this should be fine, couldn't evaluate so don't delete %v", msg)
		default:
			if whereVal.Nil() {
				// Doesn't match, so don't delete
			} else {
				u.Warnf("unknown where eval result? %T", whereVal)
			}
		}
	}
	if err != nil {
		txn.Abort()
		return 0, err
	}
	txn.Commit()
	return len(deletedKeys), nil
}
<commit_msg>Change message type to make downstream happier<commit_after>package memdb

import (
	"database/sql/driver"
	"fmt"

	u "github.com/araddon/gou"
	"github.com/hashicorp/go-memdb"
	"golang.org/x/net/context"

	"github.com/araddon/qlbridge/datasource"
	"github.com/araddon/qlbridge/expr"
	"github.com/araddon/qlbridge/rel"
	"github.com/araddon/qlbridge/schema"
	"github.com/araddon/qlbridge/value"
	"github.com/araddon/qlbridge/vm"
)

const (
	sourceType = "memdb"
)

var (
	_ = u.EMPTY

	// Different Features of this Static Data Source
	_ schema.Source            = (*MemDb)(nil)
	_ schema.SourceTableSchema = (*MemDb)(nil)

	// Connection
	_ schema.Conn         = (*dbConn)(nil)
	_ schema.ConnColumns  = (*dbConn)(nil)
	_ schema.ConnScanner  = (*dbConn)(nil)
	_ schema.ConnUpsert   = (*dbConn)(nil)
	_ schema.ConnDeletion = (*dbConn)(nil)
	_ schema.ConnSeeker   = (*dbConn)(nil)
)

// MemDb implements qlbridge `Source` to allow in-memory native go data
//   to have a Schema and implement and be operated on by Sql Operations
//
// Features
// - ues immuteable radix-tree/db mvcc under the hood
//
type MemDb struct {
	exit           <-chan bool
	*schema.Schema                 // schema
	tbl            *schema.Table   // schema table
	indexes        []*schema.Index // index descriptions
	primaryIndex   string
	db             *memdb.MemDB
	max            int
}
type dbConn struct {
	md     *MemDb
	db     *memdb.MemDB
	txn    *memdb.Txn
	result memdb.ResultIterator
}

// NewMemDbData creates a MemDb with given indexes, columns, and values
func NewMemDbData(name string, data [][]driver.Value, cols []string) (*MemDb, error) {

	m, err := NewMemDb(name, cols)
	if err != nil {
		return nil, err
	}
	// Insert initial values
	conn := newDbConn(m)
	for _, row := range data {
		conn.Put(nil, nil, row)
	}
	conn.Close()
	return m, nil
}

// NewMemDb creates a MemDb with given indexes, columns
func NewMemDb(name string, cols []string) (*MemDb, error) {

	ss := schema.NewSchemaSource(name, sourceType)

	return NewMemDbForSchema(name, ss, cols)
}

// NewMemDbForSchema creates a MemDb with given indexes, columns
func NewMemDbForSchema(name string, ss *schema.SchemaSource, cols []string) (*MemDb, error) {

	if len(cols) < 1 {
		return nil, fmt.Errorf("must have columns provided")
	}

	m := &MemDb{}

	m.tbl = schema.NewTable(name, ss)
	ss.AddTable(m.tbl)
	if ss.Schema == nil {
		m.Schema = schema.NewSchema(name)
		m.Schema.AddSourceSchema(ss)
	} else {
		m.Schema = ss.Schema
	}

	m.tbl.SetColumns(cols)

	err := m.buildDefaultIndexes()
	if err != nil {
		u.Errorf("Default indexes could not be built %v", err)
		return nil, err
	}

	mdbSchema, err := makeMemDbSchema(m)
	if err != nil {
		u.Errorf("Must have valid schema %v", err)
		return nil, err
	}
	db, err := memdb.NewMemDB(mdbSchema)
	if err != nil {
		u.Warnf("could not create db %v", err)
		return nil, err
	}
	m.db = db
	return m, nil
}

// Open a Conn for this source @table name
func (m *MemDb) Open(table string) (schema.Conn, error) { return newDbConn(m), nil }

// Table by name
func (m *MemDb) Table(table string) (*schema.Table, error) { return m.tbl, nil }

// Close this source
func (m *MemDb) Close() error { return nil }

// Tables list, should be single table
func (m *MemDb) Tables() []string { return []string{m.tbl.Name} }

func (m *MemDb) buildDefaultIndexes() error {
	if len(m.indexes) == 0 {
		if len(m.tbl.Columns()) < 1 {
			return fmt.Errorf("must have columns if no index provided")
		}
		u.Debugf("no index provided creating on %q", m.tbl.Columns()[0])
		m.indexes = []*schema.Index{
			{Name: "id", Fields: []string{m.tbl.Columns()[0]}},
		}
	}
	// First ensure we have one primary index
	hasPrimary := false
	for _, idx := range m.indexes {
		if idx.PrimaryKey {
			if idx.Name == "" {
				idx.Name = "id"
			}
			m.primaryIndex = idx.Name
			hasPrimary = true
		}
	}
	if !hasPrimary {
		m.indexes[0].PrimaryKey = true
		m.primaryIndex = m.indexes[0].Name
	}
	return nil
}

//func (m *MemDb) SetColumns(cols []string)                  { m.tbl.SetColumns(cols) }

func newDbConn(mdb *MemDb) *dbConn {
	c := &dbConn{md: mdb, db: mdb.db}
	return c
}
func (m *dbConn) Columns() []string { return m.md.tbl.Columns() }
func (m *dbConn) Close() error      { return nil }
func (m *dbConn) CreateIterator() schema.Iterator {
	m.txn = m.db.Txn(false)
	// Attempt a row scan on the primary index
	result, err := m.txn.Get(m.md.tbl.Name, m.md.primaryIndex)
	if err != nil {
		u.Errorf("error %v", err)
	}
	m.result = result
	return m
}
func (m *dbConn) MesgChan() <-chan schema.Message {
	return datasource.SourceIterChannel(m.CreateIterator(), m.md.exit)
}

func (m *dbConn) Next() schema.Message {
	//u.Infof("Next()")
	if m.txn == nil {
		m.txn = m.db.Txn(false)
	}
	select {
	case <-m.md.exit:
		return nil
	default:
		for {
			if m.result == nil {
				result, err := m.txn.Get(m.md.tbl.Name, m.md.primaryIndex)
				if err != nil {
					u.Errorf("error %v", err)
					return nil
				}
				m.result = result
			}
			raw := m.result.Next()
			if raw == nil {
				return nil
			}
			if msg, ok := raw.(*datasource.SqlDriverMessage); ok {
				return msg.ToMsgMap(m.md.tbl.FieldPositions)
			}
			u.Warnf("error, not correct type: %#v", raw)
			return nil
		}
	}
}

// interface for Upsert.Put()
func (m *dbConn) Put(ctx context.Context, key schema.Key, row interface{}) (schema.Key, error) {

	//u.Infof("%p Put(),  row:%#v", m, row)
	switch rowVals := row.(type) {
	case []driver.Value:
		txn := m.db.Txn(true)
		key, err := m.putValues(txn, rowVals)
		if err != nil {
			txn.Abort()
			return nil, err
		}
		txn.Commit()
		return key, nil
		/*
			case map[string]driver.Value:
				// We need to convert the key:value to []driver.Value so
				// we need to look up column index for each key, and write to vals

				// TODO:   if this is a partial update, we need to look up vals
				row := make([]driver.Value, len(m.Columns()))
				if len(rowVals) < len(m.Columns()) {
					// How do we get the key?
					//m.Get(key)
				}

				for key, val := range rowVals {
					if keyIdx, ok := m.tbl.FieldPositions[key]; ok {
						row[keyIdx] = val
					} else {
						return nil, fmt.Errorf("Found column in Put that doesn't exist in cols: %v", key)
					}
				}
				id := uint64(0)
				if key == nil {
					if row[m.indexCol] == nil {
						// Since we do not have an indexed column to work off of,
						// the ideal would be to get the job builder/planner to do
						// a scan with whatever info we have and feed that in?   Instead
						// of us implementing our own scan?
						u.Warnf("wtf, nil key? %v %v", m.indexCol, row)
						return nil, fmt.Errorf("cannot update on non index column ")
					}
					id = makeId(row[m.indexCol])
				} else {
					id = makeId(key)
					sdm, _ := m.Get(key)
					//u.Debugf("sdm: %#v  err%v", sdm, err)
					if sdm != nil {
						if dmval, ok := sdm.Body().(*datasource.SqlDriverMessageMap); ok {
							for i, val := range dmval.Values() {
								if row[i] == nil {
									row[i] = val
								}
							}
						}
					}
				}
				//u.Debugf("PUT: %#v", row)
				//u.Infof("PUT: %v  key:%v  row:%v", id, key, row)
				sdm := datasource.NewSqlDriverMessageMap(id, row, m.tbl.FieldPositions)
				item := DriverItem{sdm}
				m.bt.ReplaceOrInsert(&item)
				return NewKey(id), nil
		*/
	default:
		u.Warnf("not implemented %T", row)
		return nil, fmt.Errorf("Expected []driver.Value but got %T", row)
	}
}

func (m *dbConn) putValues(txn *memdb.Txn, row []driver.Value) (schema.Key, error) {
	if len(row) != len(m.Columns()) {
		u.Warnf("wrong column ct expected %d got %d for %v", len(m.Columns()), len(row), row)
		return nil, fmt.Errorf("Wrong number of columns, expected %v got %v", len(m.Columns()), len(row))
	}
	id := makeId(row[0])
	msg := &datasource.SqlDriverMessage{Vals: row, IdVal: id}
	txn.Insert(m.md.tbl.Name, msg)
	//u.Debugf("%p  PUT: id:%v IdVal:%v  Id():%v vals:%#v", m, id, sdm.IdVal, sdm.Id(), rowVals)
	return schema.NewKeyUint(id), nil
}

func (m *dbConn) PutMulti(ctx context.Context, keys []schema.Key, objs interface{}) ([]schema.Key, error) {
	txn := m.db.Txn(true)

	switch rows := objs.(type) {
	case [][]driver.Value:
		keys := make([]schema.Key, 0, len(rows))
		for _, row := range rows {
			key, err := m.putValues(txn, row)
			if err != nil {
				txn.Abort()
				return nil, err
			}
			keys = append(keys, key)
		}
		txn.Commit()
		return keys, nil
	}
	return nil, fmt.Errorf("unrecognized put object type: %T", objs)
}

// CanSeek is interface for Seeker, validate if we can perform this query
func (m *dbConn) CanSeek(sql *rel.SqlSelect) bool {
	return true
}

func (m *dbConn) Get(key driver.Value) (schema.Message, error) {
	txn := m.db.Txn(false)
	iter, err := txn.Get(m.md.tbl.Name, m.md.primaryIndex, fmt.Sprintf("%v", key))
	if err != nil {
		txn.Abort()
		u.Errorf("error reading %v because %v", key, err)
		return nil, err
	}
	txn.Commit() // noop

	if item := iter.Next(); item != nil {
		if msg, ok := item.(schema.Message); ok {
			return msg, nil
		}
		u.Warnf("unexpected type %T", item)
	}
	return nil, schema.ErrNotFound // Should not found be an error?
}

// MultiGet to get multiple items by keys
func (m *dbConn) MultiGet(keys []driver.Value) ([]schema.Message, error) {
	return nil, schema.ErrNotImplemented
}

// Interface for Deletion
func (m *dbConn) Delete(key driver.Value) (int, error) {
	txn := m.db.Txn(true)
	err := txn.Delete(m.md.tbl.Name, key)
	if err != nil {
		txn.Abort()
		u.Warnf("could not delete: %v  err=%v", key, err)
		return 0, err
	}
	txn.Commit()
	return 1, nil
}

// Delete using a Where Expression
func (m *dbConn) DeleteExpression(where expr.Node) (int, error) {
	//return 0, fmt.Errorf("not implemented")
	evaluator := vm.Evaluator(where)
	var deletedKeys []schema.Key
	txn := m.db.Txn(true)
	iter, err := txn.Get(m.md.tbl.Name, m.md.primaryIndex)
	if err != nil {
		txn.Abort()
		u.Errorf("could not get values %v", err)
		return 0, err
	}
deleteLoop:
	for {
		item := iter.Next()
		if item == nil {
			break
		}

		msg, ok := item.(datasource.SqlDriverMessage)
		if !ok {
			u.Warnf("wat?  %T   %#v", item, item)
			err = fmt.Errorf("unexpected message type %T", item)
			break
		}
		whereValue, ok := evaluator(msg.ToMsgMap(m.md.tbl.FieldPositions))
		if !ok {
			u.Debugf("could not evaluate where: %v", msg)
		}
		switch whereVal := whereValue.(type) {
		case value.BoolValue:
			if whereVal.Val() == false {
				//this means do NOT delete
			} else {
				// Delete!
				if err = txn.Delete(m.md.tbl.Name, msg); err != nil {
					u.Errorf("could not delete %v", err)
					break deleteLoop
				}
				indexVal := msg.Vals[0]
				deletedKeys = append(deletedKeys, schema.NewKeyUint(makeId(indexVal)))
			}
		case nil:
			// ??
			u.Warnf("this should be fine, couldn't evaluate so don't delete %v", msg)
		default:
			if whereVal.Nil() {
				// Doesn't match, so don't delete
			} else {
				u.Warnf("unknown where eval result? %T", whereVal)
			}
		}
	}
	if err != nil {
		txn.Abort()
		return 0, err
	}
	txn.Commit()
	return len(deletedKeys), nil
}
<|endoftext|>


<commit_before><commit_msg>can now select from the 10 most recent commands (from history)<commit_after><|endoftext|>


<commit_before>/*                          _       _
 *__      _____  __ ___   ___  __ _| |_ ___
 *\ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
 * \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
 *  \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
 *
 * Copyright © 2016 - 2018 Weaviate. All rights reserved.
 * LICENSE: https://github.com/creativesoftwarefdn/weaviate/blob/develop/LICENSE.md
 * AUTHOR: Bob van Luijt (bob@kub.design)
 * See www.creativesoftwarefdn.org for details
 * Contact: @CreativeSofwFdn / bob@kub.design
 */

package graphqlapi

import (
	"github.com/graphql-go/graphql"

	"github.com/creativesoftwarefdn/weaviate/config"
	dbconnector "github.com/creativesoftwarefdn/weaviate/connectors"
	"github.com/creativesoftwarefdn/weaviate/messages"
	"github.com/creativesoftwarefdn/weaviate/schema"
)

type GraphQL struct {
	weaviateGraphQLSchema graphql.Schema
	serverConfig          *config.WeaviateConfig
	databaseSchema        *schema.WeaviateSchema
	dbConnector           *dbconnector.DatabaseConnector
	messaging             *messages.Messaging
}

// The RestAPI handler calls this function to receive the schema.
func (g *GraphQL) Schema() *graphql.Schema {
	return &g.weaviateGraphQLSchema
}

// Initialize the Graphl
func CreateSchema(dbConnector *dbconnector.DatabaseConnector, serverConfig *config.WeaviateConfig, databaseSchema *schema.WeaviateSchema, messaging *messages.Messaging) (GraphQL, error) {
	messaging.InfoMessage("Creating GraphQL schema...")
	var g GraphQL

	// Store for later use.
	g.dbConnector = dbConnector
	g.serverConfig = serverConfig
	g.databaseSchema = databaseSchema
	g.messaging = messaging

	// Now build the graphql schema
	err := g.buildGraphqlSchema()
	return g, err
}
<commit_msg>gh-384: re-applied gofmt to successfully merge<commit_after>/*                          _       _
 *__      _____  __ ___   ___  __ _| |_ ___
 *\ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
 * \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
 *  \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
 *
 * Copyright © 2016 - 2018 Weaviate. All rights reserved.
 * LICENSE: https://github.com/creativesoftwarefdn/weaviate/blob/develop/LICENSE.md
 * AUTHOR: Bob van Luijt (bob@kub.design)
 * See www.creativesoftwarefdn.org for details
 * Contact: @CreativeSofwFdn / bob@kub.design
 */

package graphqlapi

import (
	"github.com/creativesoftwarefdn/weaviate/config"
	dbconnector "github.com/creativesoftwarefdn/weaviate/connectors"
	"github.com/creativesoftwarefdn/weaviate/messages"
	"github.com/creativesoftwarefdn/weaviate/schema"
	"github.com/graphql-go/graphql"
)

type GraphQL struct {
	weaviateGraphQLSchema graphql.Schema
	serverConfig          *config.WeaviateConfig
	databaseSchema        *schema.WeaviateSchema
	dbConnector           *dbconnector.DatabaseConnector
	messaging             *messages.Messaging
}

// The RestAPI handler calls this function to receive the schema.
func (g *GraphQL) Schema() *graphql.Schema {
	return &g.weaviateGraphQLSchema
}

// Initialize the Graphl
func CreateSchema(dbConnector *dbconnector.DatabaseConnector, serverConfig *config.WeaviateConfig, databaseSchema *schema.WeaviateSchema, messaging *messages.Messaging) (GraphQL, error) {
	messaging.InfoMessage("Creating GraphQL schema...")
	var g GraphQL

	// Store for later use.
	g.dbConnector = dbConnector
	g.serverConfig = serverConfig
	g.databaseSchema = databaseSchema
	g.messaging = messaging

	// Now build the graphql schema
	err := g.buildGraphqlSchema()
	return g, err
}
<|endoftext|>


<commit_before>// An example FTP server build on top of graval. graval handles the details
// of the FTP protocol, we just provide a persistence driver for rackspace
// cloud files.
//
// If you're looking to create a custom graval driver, this example is a
// reasonable starting point. I suggest copying this file and changing the
// function bodies as required.
package main

import (
	"errors"
	"github.com/ncw/swift"
	"github.com/yob/graval"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"
)

// A minimal driver for graval that stores all data on rackspace cloudfiles. The
// authentication are ignored, any username and password will work.
//
// This really just exists as a minimal demonstration of the interface graval
// drivers are required to implement.
type SwiftDriver struct{
	connection *swift.Connection
	container  string
	user       string
}

func (driver *SwiftDriver) Authenticate(user string, pass string) bool {
	log.Printf("Authenticate: %s %s", user, pass)
	driver.user = user
	return true
}
func (driver *SwiftDriver) Bytes(path string) (bytes int) {
	log.Printf("Bytes: %s", path)
	bytes = -1
	return
}
func (driver *SwiftDriver) ModifiedTime(path string) (time.Time, error) {
	log.Printf("ModifiedTime: %s", path)
	return time.Now(), nil
}
func (driver *SwiftDriver) ChangeDir(path string) bool {
	log.Printf("ChangeDir: %s", path)
	return false
}
func (driver *SwiftDriver) DirContents(path string) (files []os.FileInfo) {
	path = scoped_path_with_trailing_slash(driver.user, path)
	log.Printf("DirContents: %s", path)
	opts    := &swift.ObjectsOpts{Prefix:path}
	objects, err := driver.connection.ObjectsAll(driver.container, opts)
	if err != nil {
		return // error connecting to cloud files
	}
	for _, object := range objects {
		tail  := strings.Replace(object.Name, path, "", 1)
        basename := strings.Split(tail, "/")[0]
		if object.ContentType == "application/directory" {
			files = append(files, graval.NewDirItem(basename))
		} else  {
			files = append(files, graval.NewFileItem(basename, int(object.Bytes)))
		}
	}
	return
}

func (driver *SwiftDriver) DeleteDir(path string) bool {
	log.Printf("DeleteDir: %s", path)
	return false
}
func (driver *SwiftDriver) DeleteFile(path string) bool {
	log.Printf("DeleteFile: %s", path)
	return false
}
func (driver *SwiftDriver) Rename(fromPath string, toPath string) bool {
	log.Printf("Rename: %s %s", fromPath, toPath)
	return false
}
func (driver *SwiftDriver) MakeDir(path string) bool {
	path = scoped_path_with_trailing_slash(driver.user, path)
	log.Printf("MakeDir: %s", path)
	opts    := &swift.ObjectsOpts{Prefix:path}
	objects, err := driver.connection.ObjectNames(driver.container, opts)
	if err != nil {
		return false // error connection to cloud files
	}
	if len(objects) > 0 {
		return false // the dir already exists
	}
	driver.connection.ObjectPutString(driver.container, path, "", "application/directory")
	return true
}
func (driver *SwiftDriver) GetFile(path string) (data string, err error) {
	log.Printf("GetFile: %d", len(data))
	return "", errors.New("foo")
}
func (driver *SwiftDriver) PutFile(destPath string, data io.Reader) bool {
	log.Printf("PutFile: %s", destPath)
	return false
}

func scoped_path_with_trailing_slash(user string, path string) string {
	path = scoped_path(user, path)
	if !strings.HasSuffix(path, "/") {
		path += "/"
	}
	if path == "/" {
		return ""
	}
	return path
}

func scoped_path(user string, path string) string {
	if path == "/" {
		path = ""
	}
	return filepath.Join("/", user, path)
}

// graval requires a factory that will create a new driver instance for each
// client connection. Generally the factory will be fairly minimal. This is
// a good place to read any required config for your driver.
type SwiftDriverFactory struct{}

func (factory *SwiftDriverFactory) NewDriver() (graval.FTPDriver, error) {
	driver := &SwiftDriver{}
	driver.container  = "rba-uploads"
	driver.connection = &swift.Connection{
		UserName: os.Getenv("UserName"),
		ApiKey:   os.Getenv("ApiKey"),
		AuthUrl:  os.Getenv("AuthUrl"),
	}
	err := driver.connection.Authenticate()
	if err != nil {
		return nil, err
	}
	return driver, nil
}

// it's alive!
func main() {
	factory := &SwiftDriverFactory{}
	ftpServer := graval.NewFTPServer(&graval.FTPServerOpts{ Factory: factory })
	err := ftpServer.ListenAndServe()
	if err != nil {
		log.Fatal("Error starting server!")
		log.Fatal(err)
	}
}
<commit_msg>support uploading new objects to cloudfiles<commit_after>// An example FTP server build on top of graval. graval handles the details
// of the FTP protocol, we just provide a persistence driver for rackspace
// cloud files.
//
// If you're looking to create a custom graval driver, this example is a
// reasonable starting point. I suggest copying this file and changing the
// function bodies as required.
package main

import (
	"errors"
	"github.com/ncw/swift"
	"github.com/yob/graval"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"
)

// A minimal driver for graval that stores all data on rackspace cloudfiles. The
// authentication are ignored, any username and password will work.
//
// This really just exists as a minimal demonstration of the interface graval
// drivers are required to implement.
type SwiftDriver struct{
	connection *swift.Connection
	container  string
	user       string
}

func (driver *SwiftDriver) Authenticate(user string, pass string) bool {
	log.Printf("Authenticate: %s %s", user, pass)
	driver.user = user
	return true
}
func (driver *SwiftDriver) Bytes(path string) (bytes int) {
	log.Printf("Bytes: %s", path)
	bytes = -1
	return
}
func (driver *SwiftDriver) ModifiedTime(path string) (time.Time, error) {
	log.Printf("ModifiedTime: %s", path)
	return time.Now(), nil
}
func (driver *SwiftDriver) ChangeDir(path string) bool {
	log.Printf("ChangeDir: %s", path)
	return false
}
func (driver *SwiftDriver) DirContents(path string) (files []os.FileInfo) {
	path = scoped_path_with_trailing_slash(driver.user, path)
	log.Printf("DirContents: %s", path)
	opts    := &swift.ObjectsOpts{Prefix:path}
	objects, err := driver.connection.ObjectsAll(driver.container, opts)
	if err != nil {
		return // error connecting to cloud files
	}
	for _, object := range objects {
		tail  := strings.Replace(object.Name, path, "", 1)
        basename := strings.Split(tail, "/")[0]
		if object.ContentType == "application/directory" {
			files = append(files, graval.NewDirItem(basename))
		} else  {
			files = append(files, graval.NewFileItem(basename, int(object.Bytes)))
		}
	}
	return
}

func (driver *SwiftDriver) DeleteDir(path string) bool {
	log.Printf("DeleteDir: %s", path)
	return false
}
func (driver *SwiftDriver) DeleteFile(path string) bool {
	log.Printf("DeleteFile: %s", path)
	return false
}
func (driver *SwiftDriver) Rename(fromPath string, toPath string) bool {
	log.Printf("Rename: %s %s", fromPath, toPath)
	return false
}
func (driver *SwiftDriver) MakeDir(path string) bool {
	path = scoped_path_with_trailing_slash(driver.user, path)
	log.Printf("MakeDir: %s", path)
	opts    := &swift.ObjectsOpts{Prefix:path}
	objects, err := driver.connection.ObjectNames(driver.container, opts)
	if err != nil {
		return false // error connection to cloud files
	}
	if len(objects) > 0 {
		return false // the dir already exists
	}
	driver.connection.ObjectPutString(driver.container, path, "", "application/directory")
	return true
}
func (driver *SwiftDriver) GetFile(path string) (data string, err error) {
	log.Printf("GetFile: %d", len(data))
	return "", errors.New("foo")
}
func (driver *SwiftDriver) PutFile(destPath string, data io.Reader) bool {
	destPath = scoped_path(driver.user, destPath)
	log.Printf("PutFile: %s", destPath)
	contents, err := ioutil.ReadAll(data)
	if err != nil {
		return false
	}
	err = driver.connection.ObjectPutBytes(driver.container, destPath, contents, "application/octet-stream")
	if err != nil {
		return false
	}
	return true
}

func scoped_path_with_trailing_slash(user string, path string) string {
	path = scoped_path(user, path)
	if !strings.HasSuffix(path, "/") {
		path += "/"
	}
	if path == "/" {
		return ""
	}
	return path
}

func scoped_path(user string, path string) string {
	if path == "/" {
		path = ""
	}
	return filepath.Join("/", user, path)
}

// graval requires a factory that will create a new driver instance for each
// client connection. Generally the factory will be fairly minimal. This is
// a good place to read any required config for your driver.
type SwiftDriverFactory struct{}

func (factory *SwiftDriverFactory) NewDriver() (graval.FTPDriver, error) {
	driver := &SwiftDriver{}
	driver.container  = "rba-uploads"
	driver.connection = &swift.Connection{
		UserName: os.Getenv("UserName"),
		ApiKey:   os.Getenv("ApiKey"),
		AuthUrl:  os.Getenv("AuthUrl"),
	}
	err := driver.connection.Authenticate()
	if err != nil {
		return nil, err
	}
	return driver, nil
}

// it's alive!
func main() {
	factory := &SwiftDriverFactory{}
	ftpServer := graval.NewFTPServer(&graval.FTPServerOpts{ Factory: factory })
	err := ftpServer.ListenAndServe()
	if err != nil {
		log.Fatal("Error starting server!")
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package nomad

import (
	"testing"
	"time"

	"fmt"

	"github.com/hashicorp/consul/agent/consul/autopilot"
	"github.com/hashicorp/consul/sdk/testutil/retry"
	"github.com/hashicorp/nomad/testutil"
	"github.com/hashicorp/raft"
	"github.com/hashicorp/serf/serf"
)

// wantPeers determines whether the server has the given
// number of voting raft peers.
func wantPeers(s *Server, peers int) error {
	future := s.raft.GetConfiguration()
	if err := future.Error(); err != nil {
		return err
	}

	n := autopilot.NumPeers(future.Configuration())
	if got, want := n, peers; got != want {
		return fmt.Errorf("server %v: got %d peers want %d\n\tservers: %#+v", s.config.NodeName, got, want, future.Configuration().Servers)
	}
	return nil
}

// wantRaft determines if the servers have all of each other in their
// Raft configurations,
func wantRaft(servers []*Server) error {
	// Make sure all the servers are represented in the Raft config,
	// and that there are no extras.
	verifyRaft := func(c raft.Configuration) error {
		want := make(map[raft.ServerID]bool)
		for _, s := range servers {
			want[s.config.RaftConfig.LocalID] = true
		}

		found := make([]raft.ServerID, 0, len(c.Servers))
		for _, s := range c.Servers {
			found = append(found, s.ID)
			if !want[s.ID] {
				return fmt.Errorf("don't want %q", s.ID)
			}
			delete(want, s.ID)
		}

		if len(want) > 0 {
			return fmt.Errorf("didn't find %v in %#+v", want, found)
		}
		return nil
	}

	for _, s := range servers {
		future := s.raft.GetConfiguration()
		if err := future.Error(); err != nil {
			return err
		}
		if err := verifyRaft(future.Configuration()); err != nil {
			return err
		}
	}
	return nil
}

func TestAutopilot_CleanupDeadServer(t *testing.T) {
	t.Parallel()
	for i := 1; i <= 3; i++ {
		t.Run(fmt.Sprintf("raft version: %v", i), func(t *testing.T) {
			testCleanupDeadServer(t, i)
		})
	}
}

func testCleanupDeadServer(t *testing.T, raftVersion int) {
	conf := func(c *Config) {
		c.BootstrapExpect = 3
		c.RaftConfig.ProtocolVersion = raft.ProtocolVersion(raftVersion)
	}

	s1, cleanupS1 := TestServer(t, conf)
	defer cleanupS1()

	s2, cleanupS2 := TestServer(t, conf)
	defer cleanupS2()

	s3, cleanupS3 := TestServer(t, conf)
	defer cleanupS3()

	servers := []*Server{s1, s2, s3}

	// Try to join
	TestJoin(t, s1, s2, s3)

	for _, s := range servers {
		retry.Run(t, func(r *retry.R) { r.Check(wantPeers(s, 3)) })
	}

	// Bring up a new server
	s4, cleanupS4 := TestServer(t, conf)
	defer cleanupS4()

	// Kill a non-leader server
	if leader := waitForStableLeadership(t, servers); leader == s3 {
		s3, s1 = s1, s3
	}
	s3.Shutdown()

	retry.Run(t, func(r *retry.R) {
		alive := 0
		for _, m := range s1.Members() {
			if m.Status == serf.StatusAlive {
				alive++
			}
		}
		if alive != 2 {
			r.Fatalf("expected 2 alive servers but found %v", alive)
		}
	})

	// Join the new server
	TestJoin(t, s1, s2, s4)
	servers[2] = s4

	waitForStableLeadership(t, servers)

	// Make sure the dead server is removed and we're back to 3 total peers
	for _, s := range servers {
		retry.Run(t, func(r *retry.R) { r.Check(wantPeers(s, 3)) })
	}
}

func TestAutopilot_CleanupDeadServerPeriodic(t *testing.T) {
	t.Parallel()

	conf := func(c *Config) {
		c.BootstrapExpect = 5
	}

	s1, cleanupS1 := TestServer(t, conf)
	defer cleanupS1()

	s2, cleanupS2 := TestServer(t, conf)
	defer cleanupS2()

	s3, cleanupS3 := TestServer(t, conf)
	defer cleanupS3()

	s4, cleanupS4 := TestServer(t, conf)
	defer cleanupS4()

	s5, cleanupS5 := TestServer(t, conf)
	defer cleanupS5()

	servers := []*Server{s1, s2, s3, s4, s5}

	// Join the servers to s1, and wait until they are all promoted to
	// voters.
	TestJoin(t, servers...)
	retry.Run(t, func(r *retry.R) {
		r.Check(wantRaft(servers))
		for _, s := range servers {
			r.Check(wantPeers(s, 5))
		}
	})

	// Kill a non-leader server
	if leader := waitForStableLeadership(t, servers); leader == s4 {
		s1, s4 = s4, s1
	}
	s4.Shutdown()

	// Should be removed from the peers automatically
	servers = []*Server{s1, s2, s3, s5}
	retry.Run(t, func(r *retry.R) {
		r.Check(wantRaft(servers))
		for _, s := range servers {
			r.Check(wantPeers(s, 4))
		}
	})
}

func TestAutopilot_RollingUpdate(t *testing.T) {
	t.Parallel()

	conf := func(c *Config) {
		c.BootstrapExpect = 3
		c.RaftConfig.ProtocolVersion = 3
	}

	s1, cleanupS1 := TestServer(t, conf)
	defer cleanupS1()

	s2, cleanupS2 := TestServer(t, conf)
	defer cleanupS2()

	s3, cleanupS3 := TestServer(t, conf)
	defer cleanupS3()

	// Join the servers to s1, and wait until they are all promoted to
	// voters.
	servers := []*Server{s1, s2, s3}
	TestJoin(t, s1, s2, s3)
	retry.Run(t, func(r *retry.R) {
		r.Check(wantRaft(servers))
		for _, s := range servers {
			r.Check(wantPeers(s, 3))
		}
	})

	// Add one more server like we are doing a rolling update.
	t.Logf("adding server s4")
	s4, cleanupS4 := TestServer(t, conf)
	defer cleanupS4()
	TestJoin(t, s1, s4)

	servers = append(servers, s4)
	retry.Run(t, func(r *retry.R) {
		r.Check(wantRaft(servers))
		for _, s := range servers {
			r.Check(wantPeers(s, 4))
		}
	})

	// Now kill one of the "old" nodes like we are doing a rolling update.
	t.Logf("shutting down server s3")
	s3.Shutdown()

	isVoter := func() bool {
		future := s1.raft.GetConfiguration()
		if err := future.Error(); err != nil {
			t.Fatalf("err: %v", err)
		}
		for _, s := range future.Configuration().Servers {
			if string(s.ID) == string(s4.config.NodeID) {
				return s.Suffrage == raft.Voter
			}
		}
		t.Fatalf("didn't find s4")
		return false
	}

	t.Logf("waiting for s4 to stabalize and be promoted")

	// Wait for s4 to stabilize, get promoted to a voter, and for s3 to be
	// removed.
	servers = []*Server{s1, s2, s4}
	retry.Run(t, func(r *retry.R) {
		r.Check(wantRaft(servers))
		for _, s := range servers {
			r.Check(wantPeers(s, 3))
		}
		if !isVoter() {
			r.Fatalf("should be a voter")
		}
	})
}

func TestAutopilot_CleanupStaleRaftServer(t *testing.T) {
	t.Skip("TestAutopilot_CleanupDeadServer is very flaky, removing it for now")
	t.Parallel()

	conf := func(c *Config) {
		c.BootstrapExpect = 3
	}
	s1, cleanupS1 := TestServer(t, conf)
	defer cleanupS1()

	s2, cleanupS2 := TestServer(t, conf)
	defer cleanupS2()

	s3, cleanupS3 := TestServer(t, conf)
	defer cleanupS3()

	s4, cleanupS4 := TestServer(t, func(c *Config) {
		c.BootstrapExpect = 0
	})
	defer cleanupS4()

	servers := []*Server{s1, s2, s3}

	// Join the servers to s1
	TestJoin(t, s1, s2, s3)

	leader := waitForStableLeadership(t, servers)

	// Add s4 to peers directly
	addr := fmt.Sprintf("127.0.0.1:%d", s4.config.RPCAddr.Port)
	future := leader.raft.AddVoter(raft.ServerID(s4.config.NodeID), raft.ServerAddress(addr), 0, 0)
	if err := future.Error(); err != nil {
		t.Fatal(err)
	}

	// Verify we have 4 peers
	peers, err := s1.numPeers()
	if err != nil {
		t.Fatal(err)
	}
	if peers != 4 {
		t.Fatalf("bad: %v", peers)
	}

	// Wait for s4 to be removed
	for _, s := range []*Server{s1, s2, s3} {
		retry.Run(t, func(r *retry.R) { r.Check(wantPeers(s, 3)) })
	}
}

func TestAutopilot_PromoteNonVoter(t *testing.T) {
	t.Parallel()

	s1, cleanupS1 := TestServer(t, func(c *Config) {
		c.RaftConfig.ProtocolVersion = 3
	})
	defer cleanupS1()
	codec := rpcClient(t, s1)
	defer codec.Close()
	testutil.WaitForLeader(t, s1.RPC)

	s2, cleanupS2 := TestServer(t, func(c *Config) {
		c.BootstrapExpect = 0
		c.RaftConfig.ProtocolVersion = 3
	})
	defer cleanupS2()
	TestJoin(t, s1, s2)

	// Make sure we see it as a nonvoter initially. We wait until half
	// the stabilization period has passed.
	retry.Run(t, func(r *retry.R) {
		future := s1.raft.GetConfiguration()
		if err := future.Error(); err != nil {
			r.Fatal(err)
		}

		servers := future.Configuration().Servers
		if len(servers) != 2 {
			r.Fatalf("bad: %v", servers)
		}
		if servers[1].Suffrage != raft.Nonvoter {
			r.Fatalf("bad: %v", servers)
		}
		health := s1.autopilot.GetServerHealth(string(servers[1].ID))
		if health == nil {
			r.Fatalf("nil health, %v", s1.autopilot.GetClusterHealth())
		}
		if !health.Healthy {
			r.Fatalf("bad: %v", health)
		}
		if time.Since(health.StableSince) < s1.config.AutopilotConfig.ServerStabilizationTime/2 {
			r.Fatal("stable period not elapsed")
		}
	})

	// Make sure it ends up as a voter.
	retry.Run(t, func(r *retry.R) {
		future := s1.raft.GetConfiguration()
		if err := future.Error(); err != nil {
			r.Fatal(err)
		}

		servers := future.Configuration().Servers
		if len(servers) != 2 {
			r.Fatalf("bad: %v", servers)
		}
		if servers[1].Suffrage != raft.Voter {
			r.Fatalf("bad: %v", servers)
		}
	})
}
<commit_msg>test: fix flaky TestAutopilot_CleanupDeadServer<commit_after>package nomad

import (
	"testing"
	"time"

	"fmt"

	"github.com/hashicorp/consul/agent/consul/autopilot"
	"github.com/hashicorp/consul/sdk/testutil/retry"
	"github.com/hashicorp/nomad/testutil"
	"github.com/hashicorp/raft"
	"github.com/hashicorp/serf/serf"
)

// wantPeers determines whether the server has the given
// number of voting raft peers.
func wantPeers(s *Server, peers int) error {
	future := s.raft.GetConfiguration()
	if err := future.Error(); err != nil {
		return err
	}

	n := autopilot.NumPeers(future.Configuration())
	if got, want := n, peers; got != want {
		return fmt.Errorf("server %v: got %d peers want %d\n\tservers: %#+v", s.config.NodeName, got, want, future.Configuration().Servers)
	}
	return nil
}

// wantRaft determines if the servers have all of each other in their
// Raft configurations,
func wantRaft(servers []*Server) error {
	// Make sure all the servers are represented in the Raft config,
	// and that there are no extras.
	verifyRaft := func(c raft.Configuration) error {
		want := make(map[raft.ServerID]bool)
		for _, s := range servers {
			want[s.config.RaftConfig.LocalID] = true
		}

		found := make([]raft.ServerID, 0, len(c.Servers))
		for _, s := range c.Servers {
			found = append(found, s.ID)
			if !want[s.ID] {
				return fmt.Errorf("don't want %q", s.ID)
			}
			delete(want, s.ID)
		}

		if len(want) > 0 {
			return fmt.Errorf("didn't find %v in %#+v", want, found)
		}
		return nil
	}

	for _, s := range servers {
		future := s.raft.GetConfiguration()
		if err := future.Error(); err != nil {
			return err
		}
		if err := verifyRaft(future.Configuration()); err != nil {
			return err
		}
	}
	return nil
}

func TestAutopilot_CleanupDeadServer(t *testing.T) {
	t.Parallel()
	t.Run("raft_v2", func(t *testing.T) { testCleanupDeadServer(t, 2) })
	t.Run("raft_v3", func(t *testing.T) { testCleanupDeadServer(t, 3) })
}

func testCleanupDeadServer(t *testing.T, raftVersion int) {
	conf := func(c *Config) {
		c.BootstrapExpect = 3
		c.RaftConfig.ProtocolVersion = raft.ProtocolVersion(raftVersion)
	}

	s1, cleanupS1 := TestServer(t, conf)
	defer cleanupS1()

	s2, cleanupS2 := TestServer(t, conf)
	defer cleanupS2()

	s3, cleanupS3 := TestServer(t, conf)
	defer cleanupS3()

	servers := []*Server{s1, s2, s3}

	// Try to join
	TestJoin(t, servers...)

	for _, s := range servers {
		testutil.WaitForLeader(t, s.RPC)
		retry.Run(t, func(r *retry.R) { r.Check(wantPeers(s, 3)) })
	}

	// Bring up a new server
	s4, cleanupS4 := TestServer(t, conf)
	defer cleanupS4()

	// Kill a non-leader server
	killedIdx := 0
	for i, s := range servers {
		if !s.IsLeader() {
			killedIdx = i
			s.Shutdown()
			break
		}
	}

	retry.Run(t, func(r *retry.R) {
		for i, s := range servers {
			alive := 0
			if i == killedIdx {
				// Skip shutdown server
				continue
			}
			for _, m := range s.Members() {
				if m.Status == serf.StatusAlive {
					alive++
				}
			}

			if alive != 2 {
				r.Fatalf("expected 2 alive servers but found %v", alive)
			}
		}
	})

	// Join the new server
	servers[killedIdx] = s4
	TestJoin(t, servers...)

	waitForStableLeadership(t, servers)

	// Make sure the dead server is removed and we're back to 3 total peers
	for _, s := range servers {
		retry.Run(t, func(r *retry.R) { r.Check(wantPeers(s, 3)) })
	}
}

func TestAutopilot_CleanupDeadServerPeriodic(t *testing.T) {
	t.Parallel()

	conf := func(c *Config) {
		c.BootstrapExpect = 5
	}

	s1, cleanupS1 := TestServer(t, conf)
	defer cleanupS1()

	s2, cleanupS2 := TestServer(t, conf)
	defer cleanupS2()

	s3, cleanupS3 := TestServer(t, conf)
	defer cleanupS3()

	s4, cleanupS4 := TestServer(t, conf)
	defer cleanupS4()

	s5, cleanupS5 := TestServer(t, conf)
	defer cleanupS5()

	servers := []*Server{s1, s2, s3, s4, s5}

	// Join the servers to s1, and wait until they are all promoted to
	// voters.
	TestJoin(t, servers...)
	retry.Run(t, func(r *retry.R) {
		r.Check(wantRaft(servers))
		for _, s := range servers {
			r.Check(wantPeers(s, 5))
		}
	})

	// Kill a non-leader server
	if leader := waitForStableLeadership(t, servers); leader == s4 {
		s1, s4 = s4, s1
	}
	s4.Shutdown()

	// Should be removed from the peers automatically
	servers = []*Server{s1, s2, s3, s5}
	retry.Run(t, func(r *retry.R) {
		r.Check(wantRaft(servers))
		for _, s := range servers {
			r.Check(wantPeers(s, 4))
		}
	})
}

func TestAutopilot_RollingUpdate(t *testing.T) {
	t.Parallel()

	conf := func(c *Config) {
		c.BootstrapExpect = 3
		c.RaftConfig.ProtocolVersion = 3
	}

	s1, cleanupS1 := TestServer(t, conf)
	defer cleanupS1()

	s2, cleanupS2 := TestServer(t, conf)
	defer cleanupS2()

	s3, cleanupS3 := TestServer(t, conf)
	defer cleanupS3()

	// Join the servers to s1, and wait until they are all promoted to
	// voters.
	servers := []*Server{s1, s2, s3}
	TestJoin(t, s1, s2, s3)
	retry.Run(t, func(r *retry.R) {
		r.Check(wantRaft(servers))
		for _, s := range servers {
			r.Check(wantPeers(s, 3))
		}
	})

	// Add one more server like we are doing a rolling update.
	t.Logf("adding server s4")
	s4, cleanupS4 := TestServer(t, conf)
	defer cleanupS4()
	TestJoin(t, s1, s4)

	servers = append(servers, s4)
	retry.Run(t, func(r *retry.R) {
		r.Check(wantRaft(servers))
		for _, s := range servers {
			r.Check(wantPeers(s, 4))
		}
	})

	// Now kill one of the "old" nodes like we are doing a rolling update.
	t.Logf("shutting down server s3")
	s3.Shutdown()

	isVoter := func() bool {
		future := s1.raft.GetConfiguration()
		if err := future.Error(); err != nil {
			t.Fatalf("err: %v", err)
		}
		for _, s := range future.Configuration().Servers {
			if string(s.ID) == string(s4.config.NodeID) {
				return s.Suffrage == raft.Voter
			}
		}
		t.Fatalf("didn't find s4")
		return false
	}

	t.Logf("waiting for s4 to stabalize and be promoted")

	// Wait for s4 to stabilize, get promoted to a voter, and for s3 to be
	// removed.
	servers = []*Server{s1, s2, s4}
	retry.Run(t, func(r *retry.R) {
		r.Check(wantRaft(servers))
		for _, s := range servers {
			r.Check(wantPeers(s, 3))
		}
		if !isVoter() {
			r.Fatalf("should be a voter")
		}
	})
}

func TestAutopilot_CleanupStaleRaftServer(t *testing.T) {
	t.Skip("TestAutopilot_CleanupDeadServer is very flaky, removing it for now")
	t.Parallel()

	conf := func(c *Config) {
		c.BootstrapExpect = 3
	}
	s1, cleanupS1 := TestServer(t, conf)
	defer cleanupS1()

	s2, cleanupS2 := TestServer(t, conf)
	defer cleanupS2()

	s3, cleanupS3 := TestServer(t, conf)
	defer cleanupS3()

	s4, cleanupS4 := TestServer(t, func(c *Config) {
		c.BootstrapExpect = 0
	})
	defer cleanupS4()

	servers := []*Server{s1, s2, s3}

	// Join the servers to s1
	TestJoin(t, s1, s2, s3)

	leader := waitForStableLeadership(t, servers)

	// Add s4 to peers directly
	addr := fmt.Sprintf("127.0.0.1:%d", s4.config.RPCAddr.Port)
	future := leader.raft.AddVoter(raft.ServerID(s4.config.NodeID), raft.ServerAddress(addr), 0, 0)
	if err := future.Error(); err != nil {
		t.Fatal(err)
	}

	// Verify we have 4 peers
	peers, err := s1.numPeers()
	if err != nil {
		t.Fatal(err)
	}
	if peers != 4 {
		t.Fatalf("bad: %v", peers)
	}

	// Wait for s4 to be removed
	for _, s := range []*Server{s1, s2, s3} {
		retry.Run(t, func(r *retry.R) { r.Check(wantPeers(s, 3)) })
	}
}

func TestAutopilot_PromoteNonVoter(t *testing.T) {
	t.Parallel()

	s1, cleanupS1 := TestServer(t, func(c *Config) {
		c.RaftConfig.ProtocolVersion = 3
	})
	defer cleanupS1()
	codec := rpcClient(t, s1)
	defer codec.Close()
	testutil.WaitForLeader(t, s1.RPC)

	s2, cleanupS2 := TestServer(t, func(c *Config) {
		c.BootstrapExpect = 0
		c.RaftConfig.ProtocolVersion = 3
	})
	defer cleanupS2()
	TestJoin(t, s1, s2)

	// Make sure we see it as a nonvoter initially. We wait until half
	// the stabilization period has passed.
	retry.Run(t, func(r *retry.R) {
		future := s1.raft.GetConfiguration()
		if err := future.Error(); err != nil {
			r.Fatal(err)
		}

		servers := future.Configuration().Servers
		if len(servers) != 2 {
			r.Fatalf("bad: %v", servers)
		}
		if servers[1].Suffrage != raft.Nonvoter {
			r.Fatalf("bad: %v", servers)
		}
		health := s1.autopilot.GetServerHealth(string(servers[1].ID))
		if health == nil {
			r.Fatalf("nil health, %v", s1.autopilot.GetClusterHealth())
		}
		if !health.Healthy {
			r.Fatalf("bad: %v", health)
		}
		if time.Since(health.StableSince) < s1.config.AutopilotConfig.ServerStabilizationTime/2 {
			r.Fatal("stable period not elapsed")
		}
	})

	// Make sure it ends up as a voter.
	retry.Run(t, func(r *retry.R) {
		future := s1.raft.GetConfiguration()
		if err := future.Error(); err != nil {
			r.Fatal(err)
		}

		servers := future.Configuration().Servers
		if len(servers) != 2 {
			r.Fatalf("bad: %v", servers)
		}
		if servers[1].Suffrage != raft.Voter {
			r.Fatalf("bad: %v", servers)
		}
	})
}
<|endoftext|>


<commit_before>package geom

// Vector represents a point or a direction vector in a 3D cartesian coordinate system.
type Vector struct {
	// X, Y and Z are the coordinates of a point in each of the three
	// dimensions or alternatively the direction defined by a vector
	// with origin (0,0,0) and end at X, Y and Z.
	X, Y, Z float64
}

// NewVector returns a Vector with coordinates `x`, `y` and `z`.
func NewVector(x, y, z float64) Vector {
	return Vector{
		X: x,
		Y: y,
		Z: z,
	}
}

// Cross returns a Vector, the cross product of v1 and v2.
func Cross(v1, v2 Vector) (v Vector) {
	v.X = v1.Y*v2.Z - v1.Z*v2.Y
	v.Y = v1.Z*v2.X - v1.X*v2.Z
	v.Z = v1.X*v2.Y - v1.Y*v2.X
	return
}

// Dot returns a float64, dot product of v1 and v2.
func Dot(v1, v2 Vector) (d float64) {
	d = v1.X*v2.X + v1.Y*v2.Y + v1.Z*v2.Z
	return
}

// Sub returns a Vector, the result of subtracting v2 from v1.
func Sub(v1, v2 Vector) (v Vector) {
	v.X = v1.X - v2.X
	v.Y = v1.Y - v2.Y
	v.Z = v1.Z - v2.Z
	return
}
<commit_msg>touch up<commit_after>package geom

// Vector represents a point or a direction vector in a 3D cartesian coordinate system.
type Vector struct {
	// X, Y and Z are the coordinates of a point in each of the three
	// dimensions or alternatively the direction defined by a vector
	// with origin (0,0,0) and end at X, Y and Z.
	X, Y, Z float64
}

// NewVector returns a Vector with coordinates `x`, `y` and `z`.
func NewVector(x, y, z float64) Vector {
	return Vector{
		X: x,
		Y: y,
		Z: z,
	}
}

// Cross returns a Vector, the cross product of v1 and v2.
func Cross(v1, v2 Vector) (v Vector) {
	v.X = v1.Y*v2.Z - v1.Z*v2.Y
	v.Y = v1.Z*v2.X - v1.X*v2.Z
	v.Z = v1.X*v2.Y - v1.Y*v2.X
	return
}

// Dot returns a float64, the dot product of v1 and v2.
func Dot(v1, v2 Vector) (d float64) {
	d = v1.X*v2.X + v1.Y*v2.Y + v1.Z*v2.Z
	return
}

// Sub returns a Vector, the result of subtracting v2 from v1.
func Sub(v1, v2 Vector) (v Vector) {
	v.X = v1.X - v2.X
	v.Y = v1.Y - v2.Y
	v.Z = v1.Z - v2.Z
	return
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package gf256 implements arithmetic over the Galois Field GF(256).
package gf256

import "strconv"

type Field struct {
	log [256]byte // log[0] is unused
	exp [510]byte
}

func NewField(poly int) *Field {
	if poly < 0x100 || poly >= 0x200 {
		panic("gf256: invalid polynomial: " + strconv.Itoa(poly))
	}
	var f Field
	x := 1
	for i := 0; i < 255; i++ {
		if x == 1 && i != 0 {
			panic("gf256: reducible polynomial: " + strconv.Itoa(poly))
		}
		f.exp[i] = byte(x)
		f.exp[i+255] = byte(x)
		f.log[x] = byte(i)
		x *= 2
		if x >= 0x100 {
			x ^= poly
		}
	}
	return &f
}

// Add returns the sum of x and y in the field.
func (f *Field) Add(x, y byte) byte {
	return x ^ y
}

// Exp returns the base 2 exponential of e in the field.
// If e < 0, Exp returns 0.
func (f *Field) Exp(e int) byte {
	if e < 0 {
		return 0
	}
	return f.exp[e%255]
}

// Log returns the base 2 logarithm of x in the field.
// If x == 0, Log returns -1.
func (f *Field) Log(x byte) int {
	if x == 0 {
		return -1
	}
	return int(f.log[x])
}

// Inv returns the multiplicative inverse of x in the field.
// If x == 0, Inv returns 0.
func (f *Field) Inv(x byte) byte {
	if x == 0 {
		return 0
	}
	return f.exp[255 - f.log[x]]
}

// Mul returns the product of x and y in the field.
func (f *Field) Mul(x, y byte) byte {
	if x == 0 || y == 0 {
		return 0
	}
	return f.exp[int(f.log[x]) + int(f.log[y])]
}

type Poly []byte

var Zero = Poly{}
var One = Poly{1}

func (z Poly) Norm() Poly {
	i := len(z)
	for i > 0 && z[i-1] == 0 {
		i--
	}
	return z[0:i]
}

func (x Poly) Add(y Poly) Poly {
	if len(x) < len(y) {
		x, y = y, x
	}
	z := make(Poly, len(x))
	for i := range y {
		z[i] = x[i] ^ y[i]
	}
	for i := len(y); i < len(x); i++ {
		z[i] = x[i]
	}
	return z.Norm()
}

func Mono(a byte, i int) Poly {
	p := make(Poly, i+1)
	p[i] = a
	return p
}

func (f *Field) MulPoly(x, y Poly) Poly {
	if len(x) == 0 || len(y) == 0 {
		return nil
	}
	z := make(Poly, len(x)+len(y)-1)
	for i, xi := range x {
		if xi == 0 {
			continue
		}
		for j, yj := range y {
			z[i+j] = z[i+j] ^ f.Mul(xi, yj)
		}
	}
	return z
}

func (f *Field) DivPoly(x, y Poly) (q, r Poly) {
	y = y.Norm()
	if len(y) == 0 {
		panic("divide by zero")
	}

	r = x
	inv := f.Inv(y[len(y)-1])
	for len(r) >= len(y) {
		iq := Mono(f.Mul(r[len(r)-1], inv), len(r)-len(y))
		q = q.Add(iq)
		r = r.Add(f.MulPoly(iq, y))
	}
	return
}

func (p Poly) String() string {
	s := ""
	for i := len(p) - 1; i >= 0; i-- {
		v := p[i]
		if v != 0 {
			if s != "" {
				s += " + "
			}
			if v != 1 {
				s += strconv.Itoa(int(v)) + " "
			}
			s += "x^" + strconv.Itoa(i)
		}
	}
	return s
}

func (f *Field) Gen(e int) Poly {
	p := Poly{1}
	for i := 0; i < e; i++ {
		p = f.MulPoly(p, Poly{f.Exp(i), 1})
	}
	return p
}

// An RSEncoder implements Reed-Solomon encoding
// over a given field using a given number of error correction bytes.
type RSEncoder struct {
	f *Field
	c int
	lgen []byte
	p []byte
}

// NewRSEncoder returns a new Reed-Solomon encoder
// over the given field and number of error correction bytes.
func NewRSEncoder(f *Field, c int) *RSEncoder {
	gen := f.Gen(c)
	for i, j := 0, len(gen)-1; i < j; i, j = i+1, j-1 {
		gen[i], gen[j] = gen[j], gen[i]
	}
	for i, g := range gen {
		if g == 0 {
			panic("gen 0")
		}
		gen[i] = f.log[g]
	}
	return &RSEncoder{f: f, c: c, lgen: gen}
}

// ECC writes to check the error correcting code bytes
// for data using the given Reed-Solomon parameters.
func (rs *RSEncoder) ECC(data []byte, check []byte) {
	if len(check) < rs.c {
		panic("gf256.RSEncoder: invalid check byte length")
	}
	if rs.c == 0 {
		return
	}

	// The check bytes are the remainder after dividing
	// data padded with c zeros by the generator polynomial.

	// p = data padded with c zeros.
	var p []byte
	n := len(data)+rs.c
	if len(rs.p) >= n {
		p = rs.p
	} else {
		p = make([]byte, n)
	}
	copy(p, data)
	for i := len(data); i < len(p); i++ {
		p[i] = 0
	}

	// Divide p by gen, leaving the remainder in p[len(data):].
	// p[0] is the most significant term in p, and
	// gen[0] is the most significant term in the generator.
	// To avoid repeated work, we store various values as
	// lv, not v, where lv = log[v].
	f := rs.f
	lgen := rs.lgen
	linv := 255 - int(lgen[0])
	for i := 0; i < len(data); i++ {
		if p[i] == 0 {
			continue
		}
		q := p[i:]
		// m = p[i] / gen[0]
		lm := int(f.log[p[i]])+linv
		if lm >= 255 {
			lm -= 255
		}
		exp := f.exp[lm:]
		for j, lg := range lgen {
			q[j] ^= exp[lg]
		}
	}
	copy(check, p[len(data):])
	rs.p = p
}
<commit_msg>gofmt<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package gf256 implements arithmetic over the Galois Field GF(256).
package gf256

import "strconv"

type Field struct {
	log [256]byte // log[0] is unused
	exp [510]byte
}

func NewField(poly int) *Field {
	if poly < 0x100 || poly >= 0x200 {
		panic("gf256: invalid polynomial: " + strconv.Itoa(poly))
	}
	var f Field
	x := 1
	for i := 0; i < 255; i++ {
		if x == 1 && i != 0 {
			panic("gf256: reducible polynomial: " + strconv.Itoa(poly))
		}
		f.exp[i] = byte(x)
		f.exp[i+255] = byte(x)
		f.log[x] = byte(i)
		x *= 2
		if x >= 0x100 {
			x ^= poly
		}
	}
	return &f
}

// Add returns the sum of x and y in the field.
func (f *Field) Add(x, y byte) byte {
	return x ^ y
}

// Exp returns the base 2 exponential of e in the field.
// If e < 0, Exp returns 0.
func (f *Field) Exp(e int) byte {
	if e < 0 {
		return 0
	}
	return f.exp[e%255]
}

// Log returns the base 2 logarithm of x in the field.
// If x == 0, Log returns -1.
func (f *Field) Log(x byte) int {
	if x == 0 {
		return -1
	}
	return int(f.log[x])
}

// Inv returns the multiplicative inverse of x in the field.
// If x == 0, Inv returns 0.
func (f *Field) Inv(x byte) byte {
	if x == 0 {
		return 0
	}
	return f.exp[255-f.log[x]]
}

// Mul returns the product of x and y in the field.
func (f *Field) Mul(x, y byte) byte {
	if x == 0 || y == 0 {
		return 0
	}
	return f.exp[int(f.log[x])+int(f.log[y])]
}

type Poly []byte

var Zero = Poly{}
var One = Poly{1}

func (z Poly) Norm() Poly {
	i := len(z)
	for i > 0 && z[i-1] == 0 {
		i--
	}
	return z[0:i]
}

func (x Poly) Add(y Poly) Poly {
	if len(x) < len(y) {
		x, y = y, x
	}
	z := make(Poly, len(x))
	for i := range y {
		z[i] = x[i] ^ y[i]
	}
	for i := len(y); i < len(x); i++ {
		z[i] = x[i]
	}
	return z.Norm()
}

func Mono(a byte, i int) Poly {
	p := make(Poly, i+1)
	p[i] = a
	return p
}

func (f *Field) MulPoly(x, y Poly) Poly {
	if len(x) == 0 || len(y) == 0 {
		return nil
	}
	z := make(Poly, len(x)+len(y)-1)
	for i, xi := range x {
		if xi == 0 {
			continue
		}
		for j, yj := range y {
			z[i+j] = z[i+j] ^ f.Mul(xi, yj)
		}
	}
	return z
}

func (f *Field) DivPoly(x, y Poly) (q, r Poly) {
	y = y.Norm()
	if len(y) == 0 {
		panic("divide by zero")
	}

	r = x
	inv := f.Inv(y[len(y)-1])
	for len(r) >= len(y) {
		iq := Mono(f.Mul(r[len(r)-1], inv), len(r)-len(y))
		q = q.Add(iq)
		r = r.Add(f.MulPoly(iq, y))
	}
	return
}

func (p Poly) String() string {
	s := ""
	for i := len(p) - 1; i >= 0; i-- {
		v := p[i]
		if v != 0 {
			if s != "" {
				s += " + "
			}
			if v != 1 {
				s += strconv.Itoa(int(v)) + " "
			}
			s += "x^" + strconv.Itoa(i)
		}
	}
	return s
}

func (f *Field) Gen(e int) Poly {
	p := Poly{1}
	for i := 0; i < e; i++ {
		p = f.MulPoly(p, Poly{f.Exp(i), 1})
	}
	return p
}

// An RSEncoder implements Reed-Solomon encoding
// over a given field using a given number of error correction bytes.
type RSEncoder struct {
	f    *Field
	c    int
	lgen []byte
	p    []byte
}

// NewRSEncoder returns a new Reed-Solomon encoder
// over the given field and number of error correction bytes.
func NewRSEncoder(f *Field, c int) *RSEncoder {
	gen := f.Gen(c)
	for i, j := 0, len(gen)-1; i < j; i, j = i+1, j-1 {
		gen[i], gen[j] = gen[j], gen[i]
	}
	for i, g := range gen {
		if g == 0 {
			panic("gen 0")
		}
		gen[i] = f.log[g]
	}
	return &RSEncoder{f: f, c: c, lgen: gen}
}

// ECC writes to check the error correcting code bytes
// for data using the given Reed-Solomon parameters.
func (rs *RSEncoder) ECC(data []byte, check []byte) {
	if len(check) < rs.c {
		panic("gf256.RSEncoder: invalid check byte length")
	}
	if rs.c == 0 {
		return
	}

	// The check bytes are the remainder after dividing
	// data padded with c zeros by the generator polynomial.

	// p = data padded with c zeros.
	var p []byte
	n := len(data) + rs.c
	if len(rs.p) >= n {
		p = rs.p
	} else {
		p = make([]byte, n)
	}
	copy(p, data)
	for i := len(data); i < len(p); i++ {
		p[i] = 0
	}

	// Divide p by gen, leaving the remainder in p[len(data):].
	// p[0] is the most significant term in p, and
	// gen[0] is the most significant term in the generator.
	// To avoid repeated work, we store various values as
	// lv, not v, where lv = log[v].
	f := rs.f
	lgen := rs.lgen
	linv := 255 - int(lgen[0])
	for i := 0; i < len(data); i++ {
		if p[i] == 0 {
			continue
		}
		q := p[i:]
		// m = p[i] / gen[0]
		lm := int(f.log[p[i]]) + linv
		if lm >= 255 {
			lm -= 255
		}
		exp := f.exp[lm:]
		for j, lg := range lgen {
			q[j] ^= exp[lg]
		}
	}
	copy(check, p[len(data):])
	rs.p = p
}
<|endoftext|>


<commit_before>package git

import (
	"fmt"
	"os"
	"os/exec"
	"strings"

	"github.com/Sirupsen/logrus"
)

// Commits returns a set of commits.
// If commitrange is a git still range 12345...54321, then it will be isolated set of commits.
// If commitrange is a single commit, all ancestor commits up through the hash provided.
func Commits(commitrange string) ([]CommitEntry, error) {
	cmdArgs := []string{"git", "--no-pager", "log", `--pretty=format:%H`, commitrange}
	if debug() {
		logrus.Infof("[git] cmd: %q", strings.Join(cmdArgs, " "))
	}
	output, err := exec.Command(cmdArgs[0], cmdArgs[1:]...).Output()
	if err != nil {
		logrus.Errorf("mm[git] cmd: %q", strings.Join(cmdArgs, " "))
		return nil, err
	}
	commitHashes := strings.Split(strings.TrimSpace(string(output)), "\n")
	commits := make([]CommitEntry, len(commitHashes))
	for i, commitHash := range commitHashes {
		c, err := LogCommit(commitHash)
		if err != nil {
			return commits, err
		}
		commits[i] = *c
	}
	return commits, nil
}

// FieldNames are for the formating and rendering of the CommitEntry structs.
// Keys here are from git log pretty format "format:..."
var FieldNames = map[string]string{
	"%h":  "abbreviated_commit",
	"%p":  "abbreviated_parent",
	"%t":  "abbreviated_tree",
	"%aD": "author_date",
	"%aE": "author_email",
	"%aN": "author_name",
	"%b":  "body",
	"%H":  "commit",
	"%N":  "commit_notes",
	"%cD": "committer_date",
	"%cE": "committer_email",
	"%cN": "committer_name",
	"%e":  "encoding",
	"%P":  "parent",
	"%D":  "refs",
	"%f":  "sanitized_subject_line",
	"%GS": "signer",
	"%GK": "signer_key",
	"%s":  "subject",
	"%G?": "verification_flag",
}

// Check warns if changes introduce whitespace errors.
// Returns non-zero if any issues are found.
func Check(commit string) ([]byte, error) {
	args := []string{
		"--no-pager", "log", "--check",
		fmt.Sprintf("%s^..%s", commit, commit),
	}
	if exclude := os.Getenv("GIT_CHECK_EXCLUDE"); exclude != "" {
		args = append(args, "--", ".", fmt.Sprintf(":(exclude)%s", exclude))
	}
	cmd := exec.Command("git", args...)
	if debug() {
		logrus.Infof("[git] cmd: %q", strings.Join(cmd.Args, " "))
	}
	cmd.Stderr = os.Stderr
	return cmd.Output()
}

// Show returns the diff of a commit.
//
// NOTE: This could be expensive for very large commits.
func Show(commit string) ([]byte, error) {
	cmd := exec.Command("git", "--no-pager", "show", commit)
	if debug() {
		logrus.Infof("[git] cmd: %q", strings.Join(cmd.Args, " "))
	}
	cmd.Stderr = os.Stderr
	return cmd.Output()
}

// CommitEntry represents a single commit's information from `git`.
// See also FieldNames
type CommitEntry map[string]string

// LogCommit assembles the full information on a commit from its commit hash
func LogCommit(commit string) (*CommitEntry, error) {
	c := CommitEntry{}
	for k, v := range FieldNames {
		cmd := exec.Command("git", "--no-pager", "log", "-1", `--pretty=format:`+k+``, commit)
		if debug() {
			logrus.Infof("[git] cmd: %q", strings.Join(cmd.Args, " "))
		}
		cmd.Stderr = os.Stderr
		out, err := cmd.Output()
		if err != nil {
			logrus.Errorf("[git] cmd: %q", strings.Join(cmd.Args, " "))
			return nil, err
		}
		c[v] = strings.TrimSpace(string(out))
	}

	return &c, nil
}

func debug() bool {
	return len(os.Getenv("DEBUG")) > 0
}

// FetchHeadCommit returns the hash of FETCH_HEAD
func FetchHeadCommit() (string, error) {
	cmdArgs := []string{"git", "--no-pager", "rev-parse", "--verify", "FETCH_HEAD"}
	if debug() {
		logrus.Infof("[git] cmd: %q", strings.Join(cmdArgs, " "))
	}
	output, err := exec.Command(cmdArgs[0], cmdArgs[1:]...).Output()
	if err != nil {
		logrus.Errorf("[git] cmd: %q", strings.Join(cmdArgs, " "))
		return "", err
	}
	return strings.TrimSpace(string(output)), nil
}

// HeadCommit returns the hash of HEAD
func HeadCommit() (string, error) {
	cmdArgs := []string{"git", "--no-pager", "rev-parse", "--verify", "HEAD"}
	if debug() {
		logrus.Infof("[git] cmd: %q", strings.Join(cmdArgs, " "))
	}
	output, err := exec.Command(cmdArgs[0], cmdArgs[1:]...).Output()
	if err != nil {
		logrus.Errorf("[git] cmd: %q", strings.Join(cmdArgs, " "))
		return "", err
	}
	return strings.TrimSpace(string(output)), nil
}
<commit_msg>sirupsen changed name<commit_after>package git

import (
	"fmt"
	"os"
	"os/exec"
	"strings"

	"github.com/sirupsen/logrus"
)

// Commits returns a set of commits.
// If commitrange is a git still range 12345...54321, then it will be isolated set of commits.
// If commitrange is a single commit, all ancestor commits up through the hash provided.
func Commits(commitrange string) ([]CommitEntry, error) {
	cmdArgs := []string{"git", "--no-pager", "log", `--pretty=format:%H`, commitrange}
	if debug() {
		logrus.Infof("[git] cmd: %q", strings.Join(cmdArgs, " "))
	}
	output, err := exec.Command(cmdArgs[0], cmdArgs[1:]...).Output()
	if err != nil {
		logrus.Errorf("mm[git] cmd: %q", strings.Join(cmdArgs, " "))
		return nil, err
	}
	commitHashes := strings.Split(strings.TrimSpace(string(output)), "\n")
	commits := make([]CommitEntry, len(commitHashes))
	for i, commitHash := range commitHashes {
		c, err := LogCommit(commitHash)
		if err != nil {
			return commits, err
		}
		commits[i] = *c
	}
	return commits, nil
}

// FieldNames are for the formating and rendering of the CommitEntry structs.
// Keys here are from git log pretty format "format:..."
var FieldNames = map[string]string{
	"%h":  "abbreviated_commit",
	"%p":  "abbreviated_parent",
	"%t":  "abbreviated_tree",
	"%aD": "author_date",
	"%aE": "author_email",
	"%aN": "author_name",
	"%b":  "body",
	"%H":  "commit",
	"%N":  "commit_notes",
	"%cD": "committer_date",
	"%cE": "committer_email",
	"%cN": "committer_name",
	"%e":  "encoding",
	"%P":  "parent",
	"%D":  "refs",
	"%f":  "sanitized_subject_line",
	"%GS": "signer",
	"%GK": "signer_key",
	"%s":  "subject",
	"%G?": "verification_flag",
}

// Check warns if changes introduce whitespace errors.
// Returns non-zero if any issues are found.
func Check(commit string) ([]byte, error) {
	args := []string{
		"--no-pager", "log", "--check",
		fmt.Sprintf("%s^..%s", commit, commit),
	}
	if exclude := os.Getenv("GIT_CHECK_EXCLUDE"); exclude != "" {
		args = append(args, "--", ".", fmt.Sprintf(":(exclude)%s", exclude))
	}
	cmd := exec.Command("git", args...)
	if debug() {
		logrus.Infof("[git] cmd: %q", strings.Join(cmd.Args, " "))
	}
	cmd.Stderr = os.Stderr
	return cmd.Output()
}

// Show returns the diff of a commit.
//
// NOTE: This could be expensive for very large commits.
func Show(commit string) ([]byte, error) {
	cmd := exec.Command("git", "--no-pager", "show", commit)
	if debug() {
		logrus.Infof("[git] cmd: %q", strings.Join(cmd.Args, " "))
	}
	cmd.Stderr = os.Stderr
	return cmd.Output()
}

// CommitEntry represents a single commit's information from `git`.
// See also FieldNames
type CommitEntry map[string]string

// LogCommit assembles the full information on a commit from its commit hash
func LogCommit(commit string) (*CommitEntry, error) {
	c := CommitEntry{}
	for k, v := range FieldNames {
		cmd := exec.Command("git", "--no-pager", "log", "-1", `--pretty=format:`+k+``, commit)
		if debug() {
			logrus.Infof("[git] cmd: %q", strings.Join(cmd.Args, " "))
		}
		cmd.Stderr = os.Stderr
		out, err := cmd.Output()
		if err != nil {
			logrus.Errorf("[git] cmd: %q", strings.Join(cmd.Args, " "))
			return nil, err
		}
		c[v] = strings.TrimSpace(string(out))
	}

	return &c, nil
}

func debug() bool {
	return len(os.Getenv("DEBUG")) > 0
}

// FetchHeadCommit returns the hash of FETCH_HEAD
func FetchHeadCommit() (string, error) {
	cmdArgs := []string{"git", "--no-pager", "rev-parse", "--verify", "FETCH_HEAD"}
	if debug() {
		logrus.Infof("[git] cmd: %q", strings.Join(cmdArgs, " "))
	}
	output, err := exec.Command(cmdArgs[0], cmdArgs[1:]...).Output()
	if err != nil {
		logrus.Errorf("[git] cmd: %q", strings.Join(cmdArgs, " "))
		return "", err
	}
	return strings.TrimSpace(string(output)), nil
}

// HeadCommit returns the hash of HEAD
func HeadCommit() (string, error) {
	cmdArgs := []string{"git", "--no-pager", "rev-parse", "--verify", "HEAD"}
	if debug() {
		logrus.Infof("[git] cmd: %q", strings.Join(cmdArgs, " "))
	}
	output, err := exec.Command(cmdArgs[0], cmdArgs[1:]...).Output()
	if err != nil {
		logrus.Errorf("[git] cmd: %q", strings.Join(cmdArgs, " "))
		return "", err
	}
	return strings.TrimSpace(string(output)), nil
}
<|endoftext|>