<commit_before>//Package anaconda provides structs and functions for accessing version 1.1
//of the Twitter API.
//
//Successful API queries return native Go structs that can be used immediately,
//with no need for type assertions.
//
//Authentication
//
//If you already have the access token (and secret) for your user (Twitter provides this for your own account on the developer portal), creating the client is simple:
//
//  anaconda.SetConsumerKey("your-consumer-key")
//  anaconda.SetConsumerSecret("your-consumer-secret")
//  api := anaconda.NewTwitterApi("your-access-token", "your-access-token-secret")
//
//
//Queries
//
//Executing queries on an authenticated TwitterApi struct is simple.
//
//  searchResult, _ := api.GetSearch("golang", nil)
//  for _ , tweet := range searchResult {
//      fmt.Print(tweet.Text)
//  }
//
//Certain endpoints allow separate optional parameter; if desired, these can be passed as the final parameter.
//
//  v := url.Values{}
//  v.Set("count", "30")
//  result, err := api.GetSearch("golang", v)
//
//
//Endpoints
//
//Anaconda implements most of the endpoints defined in the Twitter API documentation: https://dev.twitter.com/docs/api/1.1.
//For clarity, in most cases, the function name is simply the name of the HTTP method and the endpoint (e.g., the endpoint `GET /friendships/incoming` is provided by the function `GetFriendshipsIncoming`).
//
//In a few cases, a shortened form has been chosen to make life easier (for example, retweeting is simply the function `Retweet`)
//
//More detailed information about the behavior of each particular endpoint can be found at the official Twitter API documentation.
package anaconda

import (
	"encoding/json"
	"fmt"
	"github.com/garyburd/go-oauth/oauth"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"sync"
	"time"
)

const (
	_GET  = iota
	_POST = iota
)

var oauthClient = oauth.Client{
	TemporaryCredentialRequestURI: "http://api.twitter.com/oauth/request_token",
	ResourceOwnerAuthorizationURI: "http://api.twitter.com/oauth/authenticate",
	TokenRequestURI:               "http://api.twitter.com/oauth/access_token",
}

type TwitterApi struct {
	Credentials *oauth.Credentials
	queryQueue  chan query
	delay       time.Duration
	delay_mutex sync.Mutex
}

type query struct {
	url         string
	form        url.Values
	data        interface{}
	method      int
	response_ch chan response
}

type response struct {
	data interface{}
	err  error
}

type ApiError struct {
	StatusCode int
	Header     http.Header
	Body       string
	URL        *url.URL
}
const DEFAULT_DELAY = 10 * time.Second

//NewTwitterApi takes an user-specific access token and secret and returns a TwitterApi struct for that user.
//The TwitterApi struct can be used for accessing any of the endpoints available.
func NewTwitterApi(access_token string, access_token_secret string) TwitterApi {
	//TODO figure out how much to buffer this channel
	//A non-buffered channel will cause blocking when multiple queries are made at the same time
	queue := make(chan query)
	c := TwitterApi{&oauth.Credentials{Token: access_token, Secret: access_token_secret}, queue, DEFAULT_DELAY, sync.Mutex{}}
	go c.throttledQuery()
	return c
}

//SetConsumerKey will set the application-specific consumer_key used in the initial OAuth process
//This key is listed on https://dev.twitter.com/apps/YOUR_APP_ID/show
func SetConsumerKey(consumer_key string) {
	oauthClient.Credentials.Token = consumer_key
}

//SetConsumerSecret will set the application-specific secret used in the initial OAuth process
//This secret is listed on https://dev.twitter.com/apps/YOUR_APP_ID/show
func SetConsumerSecret(consumer_secret string) {
	oauthClient.Credentials.Secret = consumer_secret
}

//SetDelay will set the delay between throttled queries
//To turn of throttling, set it to 0 seconds
func (c TwitterApi) SetDelay(t time.Duration) {
	c.delay_mutex.Lock()
	c.delay = t
	c.delay_mutex.Unlock()
}

//AuthorizationURL generates the authorization URL for the first part of the OAuth handshake.
//Redirect the user to this URL.
//This assumes that the consumer key has already been set (using SetConsumerKey).
func AuthorizationURL(callback string) (string, *oauth.Credentials, error) {
	tempCred, err := oauthClient.RequestTemporaryCredentials(http.DefaultClient, callback, nil)
	if err != nil {
		return "", nil, err
	}
	return oauthClient.AuthorizationURL(tempCred, nil), tempCred, nil
}

func GetCredentials(tempCred *oauth.Credentials, verifier string) (*oauth.Credentials, url.Values, error) {
	return oauthClient.RequestToken(http.DefaultClient, tempCred, verifier)
}

func cleanValues(v url.Values) url.Values {
	if v == nil {
		return url.Values{}
	}
	return v
}

// apiGet issues a GET request to the Twitter API and decodes the response JSON to data.
func (c TwitterApi) apiGet(urlStr string, form url.Values, data interface{}) error {
	resp, err := oauthClient.Get(http.DefaultClient, c.Credentials, urlStr, form)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	return decodeResponse(resp, data)
}

// apiPost issues a POST request to the Twitter API and decodes the response JSON to data.
func (c TwitterApi) apiPost(urlStr string, form url.Values, data interface{}) error {
	resp, err := oauthClient.Post(http.DefaultClient, c.Credentials, urlStr, form)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	return decodeResponse(resp, data)
}

// decodeResponse decodes the JSON response from the Twitter API.
func decodeResponse(resp *http.Response, data interface{}) error {
	if resp.StatusCode != 200 {
		return NewApiError(resp)
	}
	return json.NewDecoder(resp.Body).Decode(data)
}

func NewApiError(resp *http.Response) *ApiError {
	body, _ := ioutil.ReadAll(resp.Body)

	return &ApiError{
		StatusCode: resp.StatusCode,
		Header:     resp.Header,
		Body:       string(body),
		URL:        resp.Request.URL,
	}
}

func (aerr *ApiError) Error() string {
	return fmt.Sprintf("Get %s returned status %d, %s", aerr.URL, aerr.StatusCode, aerr.Body)
}

// Check to see if an error is a Rate Limiting error. If so, find the next available window in the header.
// Use like so:
//
//    if aerr, ok := err.(*ApiError); ok {
//  	  if isRateLimitError, nextWindow := aerr.RateLimitCheck; isRateLimitError {
//       	time.Sleep(nextWindow.Sub(time.Now()))
//  	  }
//    }
//
func (aerr *ApiError) RateLimitCheck() (isRateLimitError bool, nextWindow time.Time) {
	if aerr.StatusCode == 429 {
		if reset := aerr.Header.Get("X-Rate-Limit-Reset"); reset != "" {
			if resetUnix, err := strconv.ParseInt(reset, 10, 64); err == nil {
				resetTime := time.Unix(resetUnix, 0)

				// Reject any time greater than an hour away
				if resetTime.Sub(time.Now()) > time.Hour {
					return true, time.Now().Add(15 * time.Minute)
				}

				return true, resetTime
			}
		}
	}

	return false, time.Time{}
//query executes a query to the specified url, sending the values specified by form, and decodes the response JSON to data
//method can be either _GET or _POST
func (c TwitterApi) execQuery(urlStr string, form url.Values, data interface{}, method int) error {
	switch method {
	case _GET:
		return c.apiGet(urlStr, form, data)
	case _POST:
		return c.apiPost(urlStr, form, data)
	default:
		return fmt.Errorf("HTTP method not yet supported")
	}
}

//throttledQuery executes queries and automatically throttles them according to SECONDS_PER_QUERY
func (c TwitterApi) throttledQuery() {
	for q := range c.queryQueue {
        now := time.Now()
		url := q.url
		form := q.form
		data := q.data //This is where the actual response will be written
		method := q.method

		response_ch := q.response_ch

		err := c.execQuery(url, form, data, method)
		response_ch <- struct {
			data interface{}
			err  error
		}{data, err}

		//We have to hold a mutex because the delay time may have changed
		c.delay_mutex.Lock()
		delay := c.delay
		c.delay_mutex.Unlock()
		time.Sleep(delay - time.Since(now))
	}
}
<commit_msg>Start parsing errors<commit_after>//Package anaconda provides structs and functions for accessing version 1.1
//of the Twitter API.
//
//Successful API queries return native Go structs that can be used immediately,
//with no need for type assertions.
//
//Authentication
//
//If you already have the access token (and secret) for your user (Twitter provides this for your own account on the developer portal), creating the client is simple:
//
//  anaconda.SetConsumerKey("your-consumer-key")
//  anaconda.SetConsumerSecret("your-consumer-secret")
//  api := anaconda.NewTwitterApi("your-access-token", "your-access-token-secret")
//
//
//Queries
//
//Executing queries on an authenticated TwitterApi struct is simple.
//
//  searchResult, _ := api.GetSearch("golang", nil)
//  for _ , tweet := range searchResult {
//      fmt.Print(tweet.Text)
//  }
//
//Certain endpoints allow separate optional parameter; if desired, these can be passed as the final parameter.
//
//  v := url.Values{}
//  v.Set("count", "30")
//  result, err := api.GetSearch("golang", v)
//
//
//Endpoints
//
//Anaconda implements most of the endpoints defined in the Twitter API documentation: https://dev.twitter.com/docs/api/1.1.
//For clarity, in most cases, the function name is simply the name of the HTTP method and the endpoint (e.g., the endpoint `GET /friendships/incoming` is provided by the function `GetFriendshipsIncoming`).
//
//In a few cases, a shortened form has been chosen to make life easier (for example, retweeting is simply the function `Retweet`)
//
//More detailed information about the behavior of each particular endpoint can be found at the official Twitter API documentation.
package anaconda

import (
	"encoding/json"
	"fmt"
	"github.com/garyburd/go-oauth/oauth"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"sync"
	"time"
)

const (
	_GET  = iota
	_POST = iota
)

var oauthClient = oauth.Client{
	TemporaryCredentialRequestURI: "http://api.twitter.com/oauth/request_token",
	ResourceOwnerAuthorizationURI: "http://api.twitter.com/oauth/authenticate",
	TokenRequestURI:               "http://api.twitter.com/oauth/access_token",
}

type TwitterApi struct {
	Credentials *oauth.Credentials
	queryQueue  chan query
	delay       time.Duration
	delay_mutex sync.Mutex
}

type query struct {
	url         string
	form        url.Values
	data        interface{}
	method      int
	response_ch chan response
}

type response struct {
	data interface{}
	err  error
}

type ApiError struct {
	StatusCode int
	Header     http.Header
	Body       string
	URL        *url.URL
}
const DEFAULT_DELAY = 10 * time.Second

//NewTwitterApi takes an user-specific access token and secret and returns a TwitterApi struct for that user.
//The TwitterApi struct can be used for accessing any of the endpoints available.
func NewTwitterApi(access_token string, access_token_secret string) TwitterApi {
	//TODO figure out how much to buffer this channel
	//A non-buffered channel will cause blocking when multiple queries are made at the same time
	queue := make(chan query)
	c := TwitterApi{&oauth.Credentials{Token: access_token, Secret: access_token_secret}, queue, DEFAULT_DELAY, sync.Mutex{}}
	go c.throttledQuery()
	return c
}

//SetConsumerKey will set the application-specific consumer_key used in the initial OAuth process
//This key is listed on https://dev.twitter.com/apps/YOUR_APP_ID/show
func SetConsumerKey(consumer_key string) {
	oauthClient.Credentials.Token = consumer_key
}

//SetConsumerSecret will set the application-specific secret used in the initial OAuth process
//This secret is listed on https://dev.twitter.com/apps/YOUR_APP_ID/show
func SetConsumerSecret(consumer_secret string) {
	oauthClient.Credentials.Secret = consumer_secret
}

//SetDelay will set the delay between throttled queries
//To turn of throttling, set it to 0 seconds
func (c TwitterApi) SetDelay(t time.Duration) {
	c.delay_mutex.Lock()
	c.delay = t
	c.delay_mutex.Unlock()
}

//AuthorizationURL generates the authorization URL for the first part of the OAuth handshake.
//Redirect the user to this URL.
//This assumes that the consumer key has already been set (using SetConsumerKey).
func AuthorizationURL(callback string) (string, *oauth.Credentials, error) {
	tempCred, err := oauthClient.RequestTemporaryCredentials(http.DefaultClient, callback, nil)
	if err != nil {
		return "", nil, err
	}
	return oauthClient.AuthorizationURL(tempCred, nil), tempCred, nil
}

func GetCredentials(tempCred *oauth.Credentials, verifier string) (*oauth.Credentials, url.Values, error) {
	return oauthClient.RequestToken(http.DefaultClient, tempCred, verifier)
}

func cleanValues(v url.Values) url.Values {
	if v == nil {
		return url.Values{}
	}
	return v
}

// apiGet issues a GET request to the Twitter API and decodes the response JSON to data.
func (c TwitterApi) apiGet(urlStr string, form url.Values, data interface{}) error {
	resp, err := oauthClient.Get(http.DefaultClient, c.Credentials, urlStr, form)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	return decodeResponse(resp, data)
}

// apiPost issues a POST request to the Twitter API and decodes the response JSON to data.
func (c TwitterApi) apiPost(urlStr string, form url.Values, data interface{}) error {
	resp, err := oauthClient.Post(http.DefaultClient, c.Credentials, urlStr, form)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	return decodeResponse(resp, data)
}

// decodeResponse decodes the JSON response from the Twitter API.
func decodeResponse(resp *http.Response, data interface{}) error {
	if resp.StatusCode != 200 {
		p, _ := ioutil.ReadAll(resp.Body)

		var twitterError TwitterErrorResponse
		err := json.Unmarshal([]byte(p), &twitterError)
		if err != nil {
			log.Printf("Could not even parse error. Get %s returned status %d, %s", resp.Request.URL, resp.StatusCode, p)
			panic(err)
		}
		return twitterError
		//return fmt.Errorf("Get %s returned status %d, %s", resp.Request.URL, resp.StatusCode, p)
	}
	return json.NewDecoder(resp.Body).Decode(data)
}

func NewApiError(resp *http.Response) *ApiError {
	body, _ := ioutil.ReadAll(resp.Body)

	return &ApiError{
		StatusCode: resp.StatusCode,
		Header:     resp.Header,
		Body:       string(body),
		URL:        resp.Request.URL,
	}
}

func (aerr *ApiError) Error() string {
	return fmt.Sprintf("Get %s returned status %d, %s", aerr.URL, aerr.StatusCode, aerr.Body)
}

// Check to see if an error is a Rate Limiting error. If so, find the next available window in the header.
// Use like so:
//
//    if aerr, ok := err.(*ApiError); ok {
//  	  if isRateLimitError, nextWindow := aerr.RateLimitCheck; isRateLimitError {
//       	time.Sleep(nextWindow.Sub(time.Now()))
//  	  }
//    }
//
func (aerr *ApiError) RateLimitCheck() (isRateLimitError bool, nextWindow time.Time) {
	if aerr.StatusCode == 429 {
		if reset := aerr.Header.Get("X-Rate-Limit-Reset"); reset != "" {
			if resetUnix, err := strconv.ParseInt(reset, 10, 64); err == nil {
				resetTime := time.Unix(resetUnix, 0)

				// Reject any time greater than an hour away
				if resetTime.Sub(time.Now()) > time.Hour {
					return true, time.Now().Add(15 * time.Minute)
				}

				return true, resetTime
			}
		}
	}

	return false, time.Time{}
//query executes a query to the specified url, sending the values specified by form, and decodes the response JSON to data
//method can be either _GET or _POST
func (c TwitterApi) execQuery(urlStr string, form url.Values, data interface{}, method int) error {
	switch method {
	case _GET:
		return c.apiGet(urlStr, form, data)
	case _POST:
		return c.apiPost(urlStr, form, data)
	default:
		return fmt.Errorf("HTTP method not yet supported")
	}
}

//throttledQuery executes queries and automatically throttles them according to SECONDS_PER_QUERY
func (c TwitterApi) throttledQuery() {
	for q := range c.queryQueue {
        now := time.Now()
		url := q.url
		form := q.form
		data := q.data //This is where the actual response will be written
		method := q.method

		response_ch := q.response_ch

		err := c.execQuery(url, form, data, method)
		response_ch <- struct {
			data interface{}
			err  error
		}{data, err}

		//We have to hold a mutex because the delay time may have changed
		c.delay_mutex.Lock()
		delay := c.delay
		c.delay_mutex.Unlock()
		time.Sleep(delay - time.Since(now))
	}
}
<|endoftext|>


<commit_before>package persist

import (
	"fmt"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	pclient "github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	pfsclient "github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/pfs/db/persist"
	"github.com/pachyderm/pachyderm/src/server/pfs/drive"
	pfsserver "github.com/pachyderm/pachyderm/src/server/pfs/server"

	"github.com/dancannon/gorethink"
	"go.pedge.io/pb/go/google/protobuf"
	"go.pedge.io/proto/server"
	"google.golang.org/grpc"
)

var (
	port int32 = 30651
)

/*
	CommitBranchIndex

	Given a repo and a clock, returns a commit

	Is used in several places to:

	- find a parent commit given the parent's id in the form of an alias (e.g. "master/0")
	- getHeadOfBranch() -- by doing a range query of the form "branchName/0" to "branchName/max" and returning the last result (in this case the head)
	- getIDOfParentcommit() -- by decrementing this commit's clock value, and searching for that new clock
	- getCommitByAmbmiguousID() -- if the commit ID is in the form of an alias, find the commit using the index

*/

func TestCommitBranchIndexBasic(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)

		term := gorethink.DB(dbName).Table(commitTable)
		cursor, err := term.GetAll(commitID).Run(dbClient)
		require.NoError(t, err)
		c := &persist.Commit{}
		err = cursor.One(c)
		require.NoError(t, err)
		require.Equal(t, commitID, c.ID)

		clock := &persist.Clock{Branch: "master", Clock: 0}
		clockID := getClockID(repo.Name, clock).ID
		cursor, err = gorethink.DB(dbName).Table(clockTable).GetAll(clockID).Run(dbClient)
		require.NoError(t, err)
		returnedClock := &persist.Clock{}
		require.NoError(t, cursor.One(returnedClock))
		require.Equal(t, "master", returnedClock.Branch)
		require.Equal(t, uint64(0), returnedClock.Clock)

		key := []interface{}{repo.Name, clock.Branch, clock.Clock}
		cursor, err = term.GetAllByIndex(commitBranchIndex, key).Run(dbClient)
		require.NoError(t, err)
		returnedCommit := &persist.Commit{}
		require.NoError(t, cursor.One(returnedCommit))
		require.Equal(t, commitID, returnedCommit.ID)
	})
}

func TestCommitBranchIndexHeadOfBranch(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit := &pfs.Commit{Repo: repo, ID: commitID}
		require.NoError(t, d.FinishCommit(commit, timestampNow(), false, nil))

		commitID2 := uuid.NewWithoutDashes()
		err = d.StartCommit(
			repo,
			commitID2,
			commitID,
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit2 := &pfs.Commit{Repo: repo, ID: commitID2}
		require.NoError(t, d.FinishCommit(commit2, timestampNow(), false, nil))

		commitID3 := uuid.NewWithoutDashes()
		err = d.StartCommit(
			repo,
			commitID3,
			commitID2,
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit3 := &pfs.Commit{Repo: repo, ID: commitID3}
		require.NoError(t, d.FinishCommit(commit3, timestampNow(), false, nil))

		// Now that the commits are chained together,
		// Grab the head of master branch using the index
		head := &persist.Commit{}
		term := gorethink.DB(dbName).Table(commitTable)
		cursor, err := term.OrderBy(gorethink.OrderByOpts{
			Index: gorethink.Desc(commitBranchIndex),
		}).Between([]interface{}{repo.Name, "master", 0}, []interface{}{repo.Name, "master", gorethink.MaxVal}, gorethink.BetweenOpts{
			RightBound: "open",
		}).Run(dbClient)
		require.NoError(t, err)
		require.NoError(t, cursor.One(head))
		require.Equal(t, commitID3, head.ID)
	})
}

/*
	diffPathIndex

	used nowhere?
	used in ListFile() to list diffs by path
*/

/* diffCommitIndex

Indexed on commitID field in diff row

Used to:

- in FinishCommit() to gather all of the diffs for this commit

*/

func TestDiffCommitIndexBasic(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		file := &pfs.File{
			Commit: &pfs.Commit{
				Repo: repo,
				ID:   commitID,
			},
			Path: "file",
		}
		d.PutFile(file, "", pfs.Delimiter_LINE, 0, strings.NewReader("foo\n"))

		commit := &pfs.Commit{Repo: repo, ID: commitID}
		require.NoError(t, d.FinishCommit(commit, timestampNow(), false, nil))

		cursor, err := gorethink.DB(dbName).Table(diffTable).GetAllByIndex(diffCommitIndex, commitID).Run(dbClient)
		require.NoError(t, err)
		diff := &persist.Diff{}
		require.NoError(t, cursor.One(diff))
		fmt.Printf("got first diff: %v\n", diff)
		require.Equal(t, "file", diff.Path)
		require.Equal(t, 1, len(diff.BlockRefs))

		block := diff.BlockRefs[0]
		fmt.Printf("block: %v\n", block)
		blockSize := block.Upper - block.Lower

		// Was trying to check on a per block level ...
		// But even GetFile() doesn't seem to return the correct results
		// reader, err := d.GetFile(file, nil, 0,
		//	int64(blockSize), &pfs.Commit{Repo: repo, ID: commitID}, 0, false, "")

		reader, err := client.GetBlock(block.Hash, uint64(0), uint64(blockSize))
		require.NoError(t, err)
		var data []byte
		size, err := reader.Read(data)
		fmt.Printf("data=%v, err=%v\n", string(data), err)
		fmt.Printf("size=%v\n", size)
		require.NoError(t, err)
		require.Equal(t, "foo\n", string(data))

	})
}

/* clockBranchIndex

Indexed on:

- repo && branchIndex

Used to:

- in ListBranch() to query the clocks table and return the branches

*/

func TestDiffPrefixIndexBasic(t *testing.T) {

	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		file := &pfs.File{
			Commit: &pfs.Commit{
				Repo: repo,
				ID:   commitID,
			},
			Path: "foo/bar/fizz/buzz",
		}
		d.PutFile(file, "", pfs.Delimiter_LINE, 0, strings.NewReader("aaa\n"))

		commit := &pfs.Commit{Repo: repo, ID: commitID}
		require.NoError(t, d.FinishCommit(commit, timestampNow(), false, nil))

		branchClock := &persist.BranchClock{
			Clocks: []*persist.Clock{
				{
					Branch: "master",
					Clock:  0,
				},
			},
		}
		key := []interface{}{repo.Name, "foo/bar/fizz/buzz", branchClock.ToArray()}
		listIndex := func(row gorethink.Term) interface{} {
			return row.Field("Path").Split("/").DeleteAt(-1).Fold(
				"",
				func(acc, part gorethink.Term) gorethink.Term {
					return acc.Add("/").Add(part)
				},
				gorethink.FoldOpts{
					Emit: func(acc, row, newAcc gorethink.Term) gorethink.Term {
						return newAcc
					},
				}).ConcatMap(
				func(path gorethink.Term) gorethink.Term {
					return row.Field("BranchClocks").Map(
						func(branchClock gorethink.Term) interface{} {
							return []interface{}{row.Field("Repo"), path, persist.BranchClockToArray(branchClock)}
						})
				},
			)
		}
		cursor, err := gorethink.DB(dbName).Table(diffTable).GetAllByIndex(diffPrefixIndex, key).Map(listIndex).Run(dbClient)

		cursor, err = gorethink.DB(dbName).Table(diffTable).Map(listIndex).Run(dbClient)
		require.NoError(t, err)
		diff := &persist.Diff{}
		require.NoError(t, cursor.One(diff))
		fmt.Printf("got first diff: %v\n", diff)
		require.Equal(t, "file", diff.Path)
		require.Equal(t, 1, len(diff.BlockRefs))
	})
}

func timestampNow() *google_protobuf.Timestamp {
	return &google_protobuf.Timestamp{Seconds: time.Now().Unix()}
}

func testSetup(t *testing.T, testCode func(drive.Driver, string, *gorethink.Session, pclient.APIClient)) {
	dbName := "pachyderm_test_" + uuid.NewWithoutDashes()[0:12]
	if err := InitDB(RethinkAddress, dbName); err != nil {
		require.NoError(t, err)
		return
	}
	dbClient, err := gorethink.Connect(gorethink.ConnectOpts{
		Address: RethinkAddress,
		Timeout: connectTimeoutSeconds * time.Second,
	})
	require.NoError(t, err)
	_client, d := startBlockServerAndGetClient(t, dbName)

	testCode(d, dbName, dbClient, _client)

	if err := RemoveDB(RethinkAddress, dbName); err != nil {
		require.NoError(t, err)
		return
	}
}

func runServers(t *testing.T, port int32, blockAPIServer pfsclient.BlockAPIServer) {
	ready := make(chan bool)
	go func() {
		err := protoserver.Serve(
			func(s *grpc.Server) {
				pfsclient.RegisterBlockAPIServer(s, blockAPIServer)
				close(ready)
			},
			protoserver.ServeOptions{Version: version.Version},
			protoserver.ServeEnv{GRPCPort: uint16(port)},
		)
		require.NoError(t, err)
	}()
	<-ready
}

func startBlockServerAndGetClient(t *testing.T, dbName string) (pclient.APIClient, drive.Driver) {

	root := uniqueString("/tmp/pach_test/run")
	var ports []int32
	ports = append(ports, atomic.AddInt32(&port, 1))
	var addresses []string
	for _, port := range ports {
		addresses = append(addresses, fmt.Sprintf("localhost:%d", port))
	}
	var driver drive.Driver
	for i, port := range ports {
		address := addresses[i]
		_driver, err := NewDriver(address, RethinkAddress, dbName)
		driver = _driver
		require.NoError(t, err)
		blockAPIServer, err := pfsserver.NewLocalBlockAPIServer(root)
		require.NoError(t, err)
		runServers(t, port, blockAPIServer)
	}
	clientConn, err := grpc.Dial(addresses[0], grpc.WithInsecure())
	require.NoError(t, err)
	return pclient.APIClient{BlockAPIClient: pfsclient.NewBlockAPIClient(clientConn)}, driver
}

func uniqueString(prefix string) string {
	return prefix + "." + uuid.NewWithoutDashes()[0:12]
}
<commit_msg>Work on diffpathindex test<commit_after>package persist

import (
	"fmt"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	pclient "github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	pfsclient "github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/pfs/db/persist"
	"github.com/pachyderm/pachyderm/src/server/pfs/drive"
	pfsserver "github.com/pachyderm/pachyderm/src/server/pfs/server"

	"github.com/dancannon/gorethink"
	"go.pedge.io/pb/go/google/protobuf"
	"go.pedge.io/proto/server"
	"google.golang.org/grpc"
)

var (
	port int32 = 30651
)

/*
	CommitBranchIndex

	Given a repo and a clock, returns a commit

	Is used in several places to:

	- find a parent commit given the parent's id in the form of an alias (e.g. "master/0")
	- getHeadOfBranch() -- by doing a range query of the form "branchName/0" to "branchName/max" and returning the last result (in this case the head)
	- getIDOfParentcommit() -- by decrementing this commit's clock value, and searching for that new clock
	- getCommitByAmbmiguousID() -- if the commit ID is in the form of an alias, find the commit using the index

*/

func TestCommitBranchIndexBasic(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)

		term := gorethink.DB(dbName).Table(commitTable)
		cursor, err := term.GetAll(commitID).Run(dbClient)
		require.NoError(t, err)
		c := &persist.Commit{}
		err = cursor.One(c)
		require.NoError(t, err)
		require.Equal(t, commitID, c.ID)

		clock := &persist.Clock{Branch: "master", Clock: 0}
		clockID := getClockID(repo.Name, clock).ID
		cursor, err = gorethink.DB(dbName).Table(clockTable).GetAll(clockID).Run(dbClient)
		require.NoError(t, err)
		returnedClock := &persist.Clock{}
		require.NoError(t, cursor.One(returnedClock))
		require.Equal(t, "master", returnedClock.Branch)
		require.Equal(t, uint64(0), returnedClock.Clock)

		key := []interface{}{repo.Name, clock.Branch, clock.Clock}
		cursor, err = term.GetAllByIndex(commitBranchIndex, key).Run(dbClient)
		require.NoError(t, err)
		returnedCommit := &persist.Commit{}
		require.NoError(t, cursor.One(returnedCommit))
		require.Equal(t, commitID, returnedCommit.ID)
	})
}

func TestCommitBranchIndexHeadOfBranch(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit := &pfs.Commit{Repo: repo, ID: commitID}
		require.NoError(t, d.FinishCommit(commit, timestampNow(), false, nil))

		commitID2 := uuid.NewWithoutDashes()
		err = d.StartCommit(
			repo,
			commitID2,
			commitID,
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit2 := &pfs.Commit{Repo: repo, ID: commitID2}
		require.NoError(t, d.FinishCommit(commit2, timestampNow(), false, nil))

		commitID3 := uuid.NewWithoutDashes()
		err = d.StartCommit(
			repo,
			commitID3,
			commitID2,
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit3 := &pfs.Commit{Repo: repo, ID: commitID3}
		require.NoError(t, d.FinishCommit(commit3, timestampNow(), false, nil))

		// Now that the commits are chained together,
		// Grab the head of master branch using the index
		head := &persist.Commit{}
		term := gorethink.DB(dbName).Table(commitTable)
		cursor, err := term.OrderBy(gorethink.OrderByOpts{
			Index: gorethink.Desc(commitBranchIndex),
		}).Between([]interface{}{repo.Name, "master", 0}, []interface{}{repo.Name, "master", gorethink.MaxVal}, gorethink.BetweenOpts{
			RightBound: "open",
		}).Run(dbClient)
		require.NoError(t, err)
		require.NoError(t, cursor.One(head))
		require.Equal(t, commitID3, head.ID)
	})
}

/*
	diffPathIndex

	used nowhere?
	used in ListFile() to list diffs by path
*/

/* diffCommitIndex

Indexed on commitID field in diff row

Used to:

- in FinishCommit() to gather all of the diffs for this commit

*/

func TestDiffCommitIndexBasic(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		file := &pfs.File{
			Commit: &pfs.Commit{
				Repo: repo,
				ID:   commitID,
			},
			Path: "file",
		}
		d.PutFile(file, "", pfs.Delimiter_LINE, 0, strings.NewReader("foo\n"))

		commit := &pfs.Commit{Repo: repo, ID: commitID}
		require.NoError(t, d.FinishCommit(commit, timestampNow(), false, nil))

		cursor, err := gorethink.DB(dbName).Table(diffTable).GetAllByIndex(diffCommitIndex, commitID).Run(dbClient)
		require.NoError(t, err)
		diff := &persist.Diff{}
		require.NoError(t, cursor.One(diff))
		fmt.Printf("got first diff: %v\n", diff)
		require.Equal(t, "file", diff.Path)
		require.Equal(t, 1, len(diff.BlockRefs))

		block := diff.BlockRefs[0]
		fmt.Printf("block: %v\n", block)
		blockSize := block.Upper - block.Lower

		// Was trying to check on a per block level ...
		// But even GetFile() doesn't seem to return the correct results
		// reader, err := d.GetFile(file, nil, 0,
		//	int64(blockSize), &pfs.Commit{Repo: repo, ID: commitID}, 0, false, "")

		reader, err := client.GetBlock(block.Hash, uint64(0), uint64(blockSize))
		require.NoError(t, err)
		var data []byte
		size, err := reader.Read(data)
		fmt.Printf("data=%v, err=%v\n", string(data), err)
		fmt.Printf("size=%v\n", size)
		require.NoError(t, err)
		require.Equal(t, "foo\n", string(data))

	})
}

/* clockBranchIndex

Indexed on:

- repo && branchIndex

Used to:

- in ListBranch() to query the clocks table and return the branches

*/

func TestDiffPathIndexBasic(t *testing.T) {

	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		file := &pfs.File{
			Commit: &pfs.Commit{
				Repo: repo,
				ID:   commitID,
			},
			Path: "foo/bar/fizz/buzz",
		}
		d.PutFile(file, "", pfs.Delimiter_LINE, 0, strings.NewReader("aaa\n"))

		commit := &pfs.Commit{Repo: repo, ID: commitID}
		require.NoError(t, d.FinishCommit(commit, timestampNow(), false, nil))

		branchClock := &persist.BranchClock{
			Clocks: []*persist.Clock{
				{
					Branch: "master",
					Clock:  0,
				},
			},
		}
		key := []interface{}{repo.Name, false, "foo/bar/fizz/buzz", branchClock.ToArray()}
		cursor, err := gorethink.DB(dbName).Table(diffTable).GetAllByIndex(diffPathIndex.Name, key).Map(diffPathIndex.CreateFunction).Run(dbClient)

		cursor, err = gorethink.DB(dbName).Table(diffTable).Map(diffPathIndex.CreateFunction).Run(dbClient)
		require.NoError(t, err)
		diff := &persist.Diff{}
		require.NoError(t, cursor.One(diff))
		fmt.Printf("got first diff: %v\n", diff)
		require.Equal(t, "file", diff.Path)
		require.Equal(t, 1, len(diff.BlockRefs))
	})
}

func timestampNow() *google_protobuf.Timestamp {
	return &google_protobuf.Timestamp{Seconds: time.Now().Unix()}
}

func testSetup(t *testing.T, testCode func(drive.Driver, string, *gorethink.Session, pclient.APIClient)) {
	dbName := "pachyderm_test_" + uuid.NewWithoutDashes()[0:12]
	if err := InitDB(RethinkAddress, dbName); err != nil {
		require.NoError(t, err)
		return
	}
	dbClient, err := gorethink.Connect(gorethink.ConnectOpts{
		Address: RethinkAddress,
		Timeout: connectTimeoutSeconds * time.Second,
	})
	require.NoError(t, err)
	_client, d := startBlockServerAndGetClient(t, dbName)

	testCode(d, dbName, dbClient, _client)

	if err := RemoveDB(RethinkAddress, dbName); err != nil {
		require.NoError(t, err)
		return
	}
}

func runServers(t *testing.T, port int32, blockAPIServer pfsclient.BlockAPIServer) {
	ready := make(chan bool)
	go func() {
		err := protoserver.Serve(
			func(s *grpc.Server) {
				pfsclient.RegisterBlockAPIServer(s, blockAPIServer)
				close(ready)
			},
			protoserver.ServeOptions{Version: version.Version},
			protoserver.ServeEnv{GRPCPort: uint16(port)},
		)
		require.NoError(t, err)
	}()
	<-ready
}

func startBlockServerAndGetClient(t *testing.T, dbName string) (pclient.APIClient, drive.Driver) {

	root := uniqueString("/tmp/pach_test/run")
	var ports []int32
	ports = append(ports, atomic.AddInt32(&port, 1))
	var addresses []string
	for _, port := range ports {
		addresses = append(addresses, fmt.Sprintf("localhost:%d", port))
	}
	var driver drive.Driver
	for i, port := range ports {
		address := addresses[i]
		_driver, err := NewDriver(address, RethinkAddress, dbName)
		driver = _driver
		require.NoError(t, err)
		blockAPIServer, err := pfsserver.NewLocalBlockAPIServer(root)
		require.NoError(t, err)
		runServers(t, port, blockAPIServer)
	}
	clientConn, err := grpc.Dial(addresses[0], grpc.WithInsecure())
	require.NoError(t, err)
	return pclient.APIClient{BlockAPIClient: pfsclient.NewBlockAPIClient(clientConn)}, driver
}

func uniqueString(prefix string) string {
	return prefix + "." + uuid.NewWithoutDashes()[0:12]
}
<|endoftext|>


<commit_before>package pretty

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strings"
	"text/tabwriter"
	"text/template"

	"github.com/fatih/color"
	ppsclient "github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/pretty"
)

// PrintJobHeader prints a job header.
func PrintJobHeader(w io.Writer) {
	fmt.Fprint(w, "ID\tOUTPUT\tSTARTED\tDURATION\tSTATE\t\n")
}

// PrintJobInfo pretty-prints job info.
func PrintJobInfo(w io.Writer, jobInfo *ppsclient.JobInfo) {
	fmt.Fprintf(w, "%s\t", jobInfo.Job.ID)
	if jobInfo.OutputCommit != nil {
		fmt.Fprintf(w, "%s/%s\t", jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.ID)
	} else {
		fmt.Fprintf(w, "-\t")
	}
	fmt.Fprintf(w, "%s\t", pretty.Ago(jobInfo.Started))
	if jobInfo.Finished != nil {
		fmt.Fprintf(w, "%s\t", pretty.Duration(jobInfo.Started, jobInfo.Finished))
	} else {
		fmt.Fprintf(w, "-\t")
	}
	fmt.Fprintf(w, "%s\t\n", jobState(jobInfo.State))
}

// PrintPipelineHeader prints a pipeline header.
func PrintPipelineHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tINPUT\tOUTPUT\tSTATE\t\n")
}

// PrintPipelineInfo pretty-prints pipeline info.
func PrintPipelineInfo(w io.Writer, pipelineInfo *ppsclient.PipelineInfo) {
	fmt.Fprintf(w, "%s\t", pipelineInfo.Pipeline.Name)
	if len(pipelineInfo.Inputs) == 0 {
		fmt.Fprintf(w, "\t")
	} else {
		for i, input := range pipelineInfo.Inputs {
			fmt.Fprintf(w, "%s", input.Repo.Name)
			if i == len(pipelineInfo.Inputs)-1 {
				fmt.Fprintf(w, "\t")
			} else {
				fmt.Fprintf(w, ", ")
			}
		}
	}
	if pipelineInfo.OutputRepo != nil {
		fmt.Fprintf(w, "%s\t", pipelineInfo.OutputRepo.Name)
	} else {
		fmt.Fprintf(w, "\t")
	}
	fmt.Fprintf(w, "%s\t\n", pipelineState(pipelineInfo.State))
}

// PrintJobInputHeader pretty prints a job input header.
func PrintJobInputHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tCOMMIT\tPARTITION\tINCREMENTAL\t\n")
}

// PrintJobInput pretty-prints a job input.
func PrintJobInput(w io.Writer, jobInput *ppsclient.JobInput) {
	fmt.Fprintf(w, "%s\t", jobInput.Commit.Repo.Name)
	fmt.Fprintf(w, "%s\t", jobInput.Commit.ID)
	fmt.Fprintf(w, "%s\t", jobInput.Method.Partition)
	fmt.Fprintf(w, "%t\t\n", jobInput.Method.Incremental)
}

// PrintPipelineInputHeader prints a pipeline input header.
func PrintPipelineInputHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tPARTITION\tINCREMENTAL\t\n")
}

// PrintPipelineInput pretty-prints a pipeline input.
func PrintPipelineInput(w io.Writer, pipelineInput *ppsclient.PipelineInput) {
	fmt.Fprintf(w, "%s\t", pipelineInput.Repo.Name)
	fmt.Fprintf(w, "%s\t", pipelineInput.Method.Partition)
	fmt.Fprintf(w, "%t\t\n", pipelineInput.Method.Incremental)
}

// PrintJobCountsHeader prints a job counts header.
func PrintJobCountsHeader(w io.Writer) {
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_PULLING))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_RUNNING))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_FAILURE))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_SUCCESS))+"\t\n")
}

// PrintDetailedJobInfo pretty-prints detailed job info.
func PrintDetailedJobInfo(jobInfo *ppsclient.JobInfo) error {
	template, err := template.New("JobInfo").Funcs(funcMap).Parse(
		`ID: {{.Job.ID}} {{if .ParentJob}}
Parent: {{.ParentJob.ID}} {{end}}
Started: {{prettyAgo .Started}} {{if .Finished}}
Duration: {{prettyDuration .Started .Finished}} {{end}}
State: {{jobState .State}}
Parallelism: {{.Parallelism}}
Inputs:
{{jobInputs .}}Transform:
{{prettyTransform .Transform}}
`)
	if err != nil {
		return err
	}
	err = template.Execute(os.Stdout, jobInfo)
	if err != nil {
		return err
	}
	return nil
}

// PrintDetailedPipelineInfo pretty-prints detailed pipeline info.
func PrintDetailedPipelineInfo(pipelineInfo *ppsclient.PipelineInfo) error {
	template, err := template.New("PipelineInfo").Funcs(funcMap).Parse(
		`Name: {{.Pipeline.Name}}
Created: {{prettyAgo .CreatedAt}}
State: {{pipelineState .State}}
Parallelism: {{.Parallelism}}
Inputs:
{{pipelineInputs .}}Transform:
{{prettyTransform .Transform}}
{{if .RecentError}} Recent Error: {{.RecentError}} {{end}}
Job Counts:
{{jobCounts .JobCounts}}
`)
	if err != nil {
		return err
	}
	err = template.Execute(os.Stdout, pipelineInfo)
	if err != nil {
		return err
	}
	return nil
}

func jobState(jobState ppsclient.JobState) string {
	switch jobState {
	case ppsclient.JobState_JOB_PULLING:
		return color.New(color.FgYellow).SprintFunc()("pulling")
	case ppsclient.JobState_JOB_RUNNING:
		return color.New(color.FgYellow).SprintFunc()("running")
	case ppsclient.JobState_JOB_FAILURE:
		return color.New(color.FgRed).SprintFunc()("failure")
	case ppsclient.JobState_JOB_SUCCESS:
		return color.New(color.FgGreen).SprintFunc()("success")
	case ppsclient.JobState_JOB_EMPTY:
		return color.New(color.FgGreen).SprintFunc()("empty")
	}
	return "-"
}

func pipelineState(pipelineState ppsclient.PipelineState) string {
	switch pipelineState {
	case ppsclient.PipelineState_PIPELINE_IDLE:
		return color.New(color.FgYellow).SprintFunc()("idle")
	case ppsclient.PipelineState_PIPELINE_RUNNING:
		return color.New(color.FgGreen).SprintFunc()("running")
	case ppsclient.PipelineState_PIPELINE_RESTARTING:
		return color.New(color.FgYellow).SprintFunc()("restarting")
	case ppsclient.PipelineState_PIPELINE_FAILURE:
		return color.New(color.FgRed).SprintFunc()("failure")
	case ppsclient.PipelineState_PIPELINE_STOPPED:
		return color.New(color.FgYellow).SprintFunc()("stopped")
	}
	return "-"
}

func jobInputs(jobInfo *ppsclient.JobInfo) string {
	var buffer bytes.Buffer
	writer := tabwriter.NewWriter(&buffer, 20, 1, 3, ' ', 0)
	PrintJobInputHeader(writer)
	for _, input := range jobInfo.Inputs {
		PrintJobInput(writer, input)
	}
	// can't error because buffer can't error on Write
	writer.Flush()
	return buffer.String()
}

func pipelineInputs(pipelineInfo *ppsclient.PipelineInfo) string {
	var buffer bytes.Buffer
	writer := tabwriter.NewWriter(&buffer, 20, 1, 3, ' ', 0)
	PrintPipelineInputHeader(writer)
	for _, input := range pipelineInfo.Inputs {
		PrintPipelineInput(writer, input)
	}
	// can't error because buffer can't error on Write
	writer.Flush()
	return buffer.String()
}

func jobCounts(counts map[int32]int32) string {
	var buffer bytes.Buffer
	for i := int32(ppsclient.JobState_JOB_PULLING); i <= int32(ppsclient.JobState_JOB_SUCCESS); i++ {
		fmt.Fprintf(&buffer, "%s: %d\t", jobState(ppsclient.JobState(i)), counts[i])
	}
	return buffer.String()
}

func prettyTransform(transform *ppsclient.Transform) (string, error) {
	result, err := json.MarshalIndent(transform, "", "  ")
	if err != nil {
		return "", err
	}
	return pretty.UnescapeHTML(string(result)), nil
}

var funcMap = template.FuncMap{
	"pipelineState":   pipelineState,
	"jobState":        jobState,
	"pipelineInputs":  pipelineInputs,
	"jobInputs":       jobInputs,
	"prettyAgo":       pretty.Ago,
	"prettyDuration":  pretty.Duration,
	"jobCounts":       jobCounts,
	"prettyTransform": prettyTransform,
}
<commit_msg>Incremental isn't a bool anymore.<commit_after>package pretty

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strings"
	"text/tabwriter"
	"text/template"

	"github.com/fatih/color"
	ppsclient "github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/pretty"
)

// PrintJobHeader prints a job header.
func PrintJobHeader(w io.Writer) {
	fmt.Fprint(w, "ID\tOUTPUT\tSTARTED\tDURATION\tSTATE\t\n")
}

// PrintJobInfo pretty-prints job info.
func PrintJobInfo(w io.Writer, jobInfo *ppsclient.JobInfo) {
	fmt.Fprintf(w, "%s\t", jobInfo.Job.ID)
	if jobInfo.OutputCommit != nil {
		fmt.Fprintf(w, "%s/%s\t", jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.ID)
	} else {
		fmt.Fprintf(w, "-\t")
	}
	fmt.Fprintf(w, "%s\t", pretty.Ago(jobInfo.Started))
	if jobInfo.Finished != nil {
		fmt.Fprintf(w, "%s\t", pretty.Duration(jobInfo.Started, jobInfo.Finished))
	} else {
		fmt.Fprintf(w, "-\t")
	}
	fmt.Fprintf(w, "%s\t\n", jobState(jobInfo.State))
}

// PrintPipelineHeader prints a pipeline header.
func PrintPipelineHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tINPUT\tOUTPUT\tSTATE\t\n")
}

// PrintPipelineInfo pretty-prints pipeline info.
func PrintPipelineInfo(w io.Writer, pipelineInfo *ppsclient.PipelineInfo) {
	fmt.Fprintf(w, "%s\t", pipelineInfo.Pipeline.Name)
	if len(pipelineInfo.Inputs) == 0 {
		fmt.Fprintf(w, "\t")
	} else {
		for i, input := range pipelineInfo.Inputs {
			fmt.Fprintf(w, "%s", input.Repo.Name)
			if i == len(pipelineInfo.Inputs)-1 {
				fmt.Fprintf(w, "\t")
			} else {
				fmt.Fprintf(w, ", ")
			}
		}
	}
	if pipelineInfo.OutputRepo != nil {
		fmt.Fprintf(w, "%s\t", pipelineInfo.OutputRepo.Name)
	} else {
		fmt.Fprintf(w, "\t")
	}
	fmt.Fprintf(w, "%s\t\n", pipelineState(pipelineInfo.State))
}

// PrintJobInputHeader pretty prints a job input header.
func PrintJobInputHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tCOMMIT\tPARTITION\tINCREMENTAL\t\n")
}

// PrintJobInput pretty-prints a job input.
func PrintJobInput(w io.Writer, jobInput *ppsclient.JobInput) {
	fmt.Fprintf(w, "%s\t", jobInput.Commit.Repo.Name)
	fmt.Fprintf(w, "%s\t", jobInput.Commit.ID)
	fmt.Fprintf(w, "%s\t", jobInput.Method.Partition)
	fmt.Fprintf(w, "%t\t\n", jobInput.Method.Incremental)
}

// PrintPipelineInputHeader prints a pipeline input header.
func PrintPipelineInputHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tPARTITION\tINCREMENTAL\t\n")
}

// PrintPipelineInput pretty-prints a pipeline input.
func PrintPipelineInput(w io.Writer, pipelineInput *ppsclient.PipelineInput) {
	fmt.Fprintf(w, "%s\t", pipelineInput.Repo.Name)
	fmt.Fprintf(w, "%s\t", pipelineInput.Method.Partition)
	fmt.Fprintf(w, "%s\t\n", pipelineInput.Method.Incremental)
}

// PrintJobCountsHeader prints a job counts header.
func PrintJobCountsHeader(w io.Writer) {
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_PULLING))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_RUNNING))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_FAILURE))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_SUCCESS))+"\t\n")
}

// PrintDetailedJobInfo pretty-prints detailed job info.
func PrintDetailedJobInfo(jobInfo *ppsclient.JobInfo) error {
	template, err := template.New("JobInfo").Funcs(funcMap).Parse(
		`ID: {{.Job.ID}} {{if .ParentJob}}
Parent: {{.ParentJob.ID}} {{end}}
Started: {{prettyAgo .Started}} {{if .Finished}}
Duration: {{prettyDuration .Started .Finished}} {{end}}
State: {{jobState .State}}
Parallelism: {{.Parallelism}}
Inputs:
{{jobInputs .}}Transform:
{{prettyTransform .Transform}}
`)
	if err != nil {
		return err
	}
	err = template.Execute(os.Stdout, jobInfo)
	if err != nil {
		return err
	}
	return nil
}

// PrintDetailedPipelineInfo pretty-prints detailed pipeline info.
func PrintDetailedPipelineInfo(pipelineInfo *ppsclient.PipelineInfo) error {
	template, err := template.New("PipelineInfo").Funcs(funcMap).Parse(
		`Name: {{.Pipeline.Name}}
Created: {{prettyAgo .CreatedAt}}
State: {{pipelineState .State}}
Parallelism: {{.Parallelism}}
Inputs:
{{pipelineInputs .}}Transform:
{{prettyTransform .Transform}}
{{if .RecentError}} Recent Error: {{.RecentError}} {{end}}
Job Counts:
{{jobCounts .JobCounts}}
`)
	if err != nil {
		return err
	}
	err = template.Execute(os.Stdout, pipelineInfo)
	if err != nil {
		return err
	}
	return nil
}

func jobState(jobState ppsclient.JobState) string {
	switch jobState {
	case ppsclient.JobState_JOB_PULLING:
		return color.New(color.FgYellow).SprintFunc()("pulling")
	case ppsclient.JobState_JOB_RUNNING:
		return color.New(color.FgYellow).SprintFunc()("running")
	case ppsclient.JobState_JOB_FAILURE:
		return color.New(color.FgRed).SprintFunc()("failure")
	case ppsclient.JobState_JOB_SUCCESS:
		return color.New(color.FgGreen).SprintFunc()("success")
	case ppsclient.JobState_JOB_EMPTY:
		return color.New(color.FgGreen).SprintFunc()("empty")
	}
	return "-"
}

func pipelineState(pipelineState ppsclient.PipelineState) string {
	switch pipelineState {
	case ppsclient.PipelineState_PIPELINE_IDLE:
		return color.New(color.FgYellow).SprintFunc()("idle")
	case ppsclient.PipelineState_PIPELINE_RUNNING:
		return color.New(color.FgGreen).SprintFunc()("running")
	case ppsclient.PipelineState_PIPELINE_RESTARTING:
		return color.New(color.FgYellow).SprintFunc()("restarting")
	case ppsclient.PipelineState_PIPELINE_FAILURE:
		return color.New(color.FgRed).SprintFunc()("failure")
	case ppsclient.PipelineState_PIPELINE_STOPPED:
		return color.New(color.FgYellow).SprintFunc()("stopped")
	}
	return "-"
}

func jobInputs(jobInfo *ppsclient.JobInfo) string {
	var buffer bytes.Buffer
	writer := tabwriter.NewWriter(&buffer, 20, 1, 3, ' ', 0)
	PrintJobInputHeader(writer)
	for _, input := range jobInfo.Inputs {
		PrintJobInput(writer, input)
	}
	// can't error because buffer can't error on Write
	writer.Flush()
	return buffer.String()
}

func pipelineInputs(pipelineInfo *ppsclient.PipelineInfo) string {
	var buffer bytes.Buffer
	writer := tabwriter.NewWriter(&buffer, 20, 1, 3, ' ', 0)
	PrintPipelineInputHeader(writer)
	for _, input := range pipelineInfo.Inputs {
		PrintPipelineInput(writer, input)
	}
	// can't error because buffer can't error on Write
	writer.Flush()
	return buffer.String()
}

func jobCounts(counts map[int32]int32) string {
	var buffer bytes.Buffer
	for i := int32(ppsclient.JobState_JOB_PULLING); i <= int32(ppsclient.JobState_JOB_SUCCESS); i++ {
		fmt.Fprintf(&buffer, "%s: %d\t", jobState(ppsclient.JobState(i)), counts[i])
	}
	return buffer.String()
}

func prettyTransform(transform *ppsclient.Transform) (string, error) {
	result, err := json.MarshalIndent(transform, "", "  ")
	if err != nil {
		return "", err
	}
	return pretty.UnescapeHTML(string(result)), nil
}

var funcMap = template.FuncMap{
	"pipelineState":   pipelineState,
	"jobState":        jobState,
	"pipelineInputs":  pipelineInputs,
	"jobInputs":       jobInputs,
	"prettyAgo":       pretty.Ago,
	"prettyDuration":  pretty.Duration,
	"jobCounts":       jobCounts,
	"prettyTransform": prettyTransform,
}
<|endoftext|>


<commit_before>package server

import (
	"context"
	"fmt"
	"path"
	"time"

	log "github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kube_watch "k8s.io/apimachinery/pkg/watch"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
	"github.com/pachyderm/pachyderm/src/client/pkg/tracing"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	col "github.com/pachyderm/pachyderm/src/server/pkg/collection"
	"github.com/pachyderm/pachyderm/src/server/pkg/dlock"
	"github.com/pachyderm/pachyderm/src/server/pkg/ppsutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/watch"
)

const (
	masterLockPath = "_master_lock"
)

var (
	failures = map[string]bool{
		"InvalidImageName": true,
		"ErrImagePull":     true,
		"Unschedulable":    true,
	}

	zero     int32 // used to turn down RCs in scaleDownWorkersForPipeline
	falseVal bool  // used to delete RCs in deletePipelineResources and restartPipeline()
)

// The master process is responsible for creating/deleting workers as
// pipelines are created/removed.
func (a *apiServer) master() {
	masterLock := dlock.NewDLock(a.env.GetEtcdClient(), path.Join(a.etcdPrefix, masterLockPath))
	backoff.RetryNotify(func() error {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		ctx, err := masterLock.Lock(ctx)
		if err != nil {
			return err
		}
		defer masterLock.Unlock(ctx)

		// Note: 'pachClient' is unauthenticated. This will use the PPS token (via
		// a.sudo()) to authenticate requests.
		pachClient := a.env.GetPachClient(ctx)
		kubeClient := a.env.GetKubeClient()

		log.Infof("PPS master: launching master process")

		// start pollPipelines in the background to regularly refresh pipelines
		func() {
			a.monitorCancelsMu.Lock()
			defer a.monitorCancelsMu.Unlock()
			a.pollCancel = startMonitorThread(pachClient, "pollPipelines",
				func(pachClient *client.APIClient) {
					a.pollPipelines(pachClient.Ctx())
				})
		}()

		// TODO(msteffen) request only keys, since pipeline_controller.go reads
		// fresh values for each event anyway
		pipelineWatcher, err := a.pipelines.ReadOnly(ctx).Watch()
		if err != nil {
			return errors.Wrapf(err, "error creating watch")
		}
		defer pipelineWatcher.Close()

		// watchChan will be nil if the Watch call below errors, this means
		// that we won't receive events from k8s and won't be able to detect
		// errors in pods. We could just return that error and retry but that
		// prevents pachyderm from creating pipelines when there's an issue
		// talking to k8s.
		var watchChan <-chan kube_watch.Event
		kubePipelineWatch, err := kubeClient.CoreV1().Pods(a.namespace).Watch(
			metav1.ListOptions{
				LabelSelector: metav1.FormatLabelSelector(metav1.SetAsLabelSelector(
					map[string]string{
						"component": "worker",
					})),
				Watch: true,
			})
		if err != nil {
			log.Errorf("failed to watch kuburnetes pods: %v", err)
		} else {
			watchChan = kubePipelineWatch.ResultChan()
			defer kubePipelineWatch.Stop()
		}

		for {
			select {
			case event := <-pipelineWatcher.Watch():
				if event.Err != nil {
					return errors.Wrapf(event.Err, "event err")
				}
				switch event.Type {
				case watch.EventPut:
					pipeline := string(event.Key)
					// Create/Modify/Delete pipeline resources as needed per new state
					if err := a.step(pachClient, pipeline, event.Ver, event.Rev); err != nil {
						log.Errorf("PPS master: %v", err)
					}
				case watch.EventDelete:
					// TODO(msteffen) trace this call
					// This is also called by pollPipelines below, if it discovers
					// dangling monitorPipeline goroutines
					if err := a.deletePipelineResources(pachClient.Ctx(), string(event.Key)); err != nil {
						log.Errorf("PPS master: could not delete pipeline resources for %q: %v", err)
					}
				}
			case event := <-watchChan:
				// if we get an error we restart the watch, k8s watches seem to
				// sometimes get stuck in a loop returning events with Type =
				// "" we treat these as errors since otherwise we get an
				// endless stream of them and can't do anything.
				if event.Type == kube_watch.Error || event.Type == "" {
					if kubePipelineWatch != nil {
						kubePipelineWatch.Stop()
					}
					kubePipelineWatch, err = kubeClient.CoreV1().Pods(a.namespace).Watch(
						metav1.ListOptions{
							LabelSelector: metav1.FormatLabelSelector(metav1.SetAsLabelSelector(
								map[string]string{
									"component": "worker",
								})),
							Watch: true,
						})
					if err != nil {
						log.Errorf("failed to watch kuburnetes pods: %v", err)
						watchChan = nil
					} else {
						watchChan = kubePipelineWatch.ResultChan()
						defer kubePipelineWatch.Stop()
					}
				}
				pod, ok := event.Object.(*v1.Pod)
				if !ok {
					continue
				}
				if pod.Status.Phase == v1.PodFailed {
					log.Errorf("pod failed because: %s", pod.Status.Message)
				}
				pipelineName := pod.ObjectMeta.Annotations["pipelineName"]
				for _, status := range pod.Status.ContainerStatuses {
					if status.State.Waiting != nil && failures[status.State.Waiting.Reason] {
						if err := a.setPipelineCrashing(pachClient.Ctx(), pipelineName, status.State.Waiting.Message); err != nil {
							return err
						}
					}
				}
				for _, condition := range pod.Status.Conditions {
					if condition.Type == v1.PodScheduled &&
						condition.Status != v1.ConditionTrue && failures[condition.Reason] {
						if err := a.setPipelineCrashing(pachClient.Ctx(), pipelineName, condition.Message); err != nil {
							return err
						}
					}
				}
			}
		}
	}, backoff.NewInfiniteBackOff(), func(err error, d time.Duration) error {
		a.cancelAllMonitorsAndCrashingMonitors(nil)
		a.monitorCancelsMu.Lock()
		defer a.monitorCancelsMu.Unlock()
		if a.pollCancel != nil {
			a.pollCancel()
			a.pollCancel = nil
		}
		log.Errorf("PPS master: error running the master process: %v; retrying in %v", err, d)
		return nil
	})
	panic("internal error: PPS master has somehow exited. Restarting pod...")
}

func (a *apiServer) setPipelineFailure(ctx context.Context, pipelineName string, reason string) error {
	return a.setPipelineState(ctx, pipelineName, pps.PipelineState_PIPELINE_FAILURE, reason)
}

func (a *apiServer) setPipelineCrashing(ctx context.Context, pipelineName string, reason string) error {
	return a.setPipelineState(ctx, pipelineName, pps.PipelineState_PIPELINE_CRASHING, reason)
}

func (a *apiServer) deletePipelineResources(ctx context.Context, pipelineName string) (retErr error) {
	log.Infof("PPS master: deleting resources for pipeline %q", pipelineName)
	span, ctx := tracing.AddSpanToAnyExisting(ctx, //lint:ignore SA4006 ctx is unused, but better to have the right ctx in scope so people don't use the wrong one
		"/pps.Master/DeletePipelineResources", "pipeline", pipelineName)
	defer func() {
		tracing.TagAnySpan(span, "err", retErr)
		tracing.FinishAnySpan(span)
	}()

	// Cancel any running monitorPipeline call
	a.cancelMonitor(pipelineName)
	// Same for cancelCrashingMonitor
	a.cancelCrashingMonitor(pipelineName)

	kubeClient := a.env.GetKubeClient()
	// Delete any services associated with op.pipeline
	selector := fmt.Sprintf("%s=%s", pipelineNameLabel, pipelineName)
	opts := &metav1.DeleteOptions{
		OrphanDependents: &falseVal,
	}
	services, err := kubeClient.CoreV1().Services(a.namespace).List(metav1.ListOptions{LabelSelector: selector})
	if err != nil {
		return errors.Wrapf(err, "could not list services")
	}
	for _, service := range services.Items {
		if err := kubeClient.CoreV1().Services(a.namespace).Delete(service.Name, opts); err != nil {
			if !isNotFoundErr(err) {
				return errors.Wrapf(err, "could not delete service %q", service.Name)
			}
		}
	}
	rcs, err := kubeClient.CoreV1().ReplicationControllers(a.namespace).List(metav1.ListOptions{LabelSelector: selector})
	if err != nil {
		return errors.Wrapf(err, "could not list RCs")
	}
	for _, rc := range rcs.Items {
		if err := kubeClient.CoreV1().ReplicationControllers(a.namespace).Delete(rc.Name, opts); err != nil {
			if !isNotFoundErr(err) {
				return errors.Wrapf(err, "could not delete RC %q: %v", rc.Name)
			}
		}
	}
	return nil
}

// setPipelineState is a PPS-master-specific helper that wraps
// ppsutil.SetPipelineState in a trace
func (a *apiServer) setPipelineState(ctx context.Context, pipeline string, state pps.PipelineState, reason string) (retErr error) {
	span, ctx := tracing.AddSpanToAnyExisting(ctx,
		"/pps.Master/SetPipelineState", "pipeline", pipeline, "new-state", state)
	defer func() {
		tracing.TagAnySpan(span, "err", retErr)
		tracing.FinishAnySpan(span)
	}()
	return ppsutil.SetPipelineState(ctx, a.env.GetEtcdClient(), a.pipelines,
		pipeline, nil, state, reason)
}

// transitionPipelineState is similar to setPipelineState, except that it sets
// 'from' and logs a different trace
func (a *apiServer) transitionPipelineState(ctx context.Context, pipeline string, from []pps.PipelineState, to pps.PipelineState, reason string) (retErr error) {
	span, ctx := tracing.AddSpanToAnyExisting(ctx,
		"/pps.Master/TransitionPipelineState", "pipeline", pipeline,
		"from-state", from, "to-state", to)
	defer func() {
		tracing.TagAnySpan(span, "err", retErr)
		tracing.FinishAnySpan(span)
	}()
	return ppsutil.SetPipelineState(ctx, a.env.GetEtcdClient(), a.pipelines,
		pipeline, from, to, reason)
}

func (a *apiServer) pollPipelines(ctx context.Context) {
	start := time.Now()
	for {
		// wait until at least 10s after last loop, so this doesn't ping too
		// frequently (current cap: once/10s)
		select {
		case <-time.After(start.Add(10 * time.Second).Sub(time.Now())):
			// Inner select: confirm that ctx is not done. It's possible that both
			// time.After() and ctx.Done() are ready, and time.After() is randomly
			// chosen over ctx.Done().
			select {
			case <-ctx.Done():
				return
			default:
			}
		case <-ctx.Done():
			return
		}
		start = time.Now()

		etcdPipelines := map[string]bool{}
		pachClient := a.env.GetPachClient(ctx)
		// collect all pipelines in etcd
		if err := a.listPipelinePtr(pachClient, nil, 0,
			func(pipeline string, listPI *pps.EtcdPipelineInfo) error {
				log.Debugf("PPS master: polling pipeline %q", pipeline)
				etcdPipelines[pipeline] = true
				var curPI pps.EtcdPipelineInfo
				_, err := col.NewSTM(ctx, a.env.GetEtcdClient(), func(stm col.STM) error {
					// Read & immediately write
					return a.pipelines.ReadWrite(stm).Update(pipeline, &curPI, func() error { return nil })
				})
				if col.IsErrNotFound(err) {
					log.Warnf("%q polling conflicted with delete, will retry", pipeline)
				} else {
					log.Errorf("could not poll %q: %v", pipeline, err)
				}
				return nil // no error recovery to do here, just keep polling...
			}); err != nil {
			log.Errorf("could not list pipelines for polling: %v\n", err)
			continue // sleep and try again
		}

		// Clean up any orphaned monitorPipeline and monitorCrashingPipeline goros
		a.cancelAllMonitorsAndCrashingMonitors(etcdPipelines)
	}
}
<commit_msg>Add comment to master() backoff<commit_after>package server

import (
	"context"
	"fmt"
	"path"
	"time"

	log "github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kube_watch "k8s.io/apimachinery/pkg/watch"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
	"github.com/pachyderm/pachyderm/src/client/pkg/tracing"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	col "github.com/pachyderm/pachyderm/src/server/pkg/collection"
	"github.com/pachyderm/pachyderm/src/server/pkg/dlock"
	"github.com/pachyderm/pachyderm/src/server/pkg/ppsutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/watch"
)

const (
	masterLockPath = "_master_lock"
)

var (
	failures = map[string]bool{
		"InvalidImageName": true,
		"ErrImagePull":     true,
		"Unschedulable":    true,
	}

	zero     int32 // used to turn down RCs in scaleDownWorkersForPipeline
	falseVal bool  // used to delete RCs in deletePipelineResources and restartPipeline()
)

// The master process is responsible for creating/deleting workers as
// pipelines are created/removed.
func (a *apiServer) master() {
	masterLock := dlock.NewDLock(a.env.GetEtcdClient(), path.Join(a.etcdPrefix, masterLockPath))
	backoff.RetryNotify(func() error {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		ctx, err := masterLock.Lock(ctx)
		if err != nil {
			return err
		}
		defer masterLock.Unlock(ctx)

		// Note: 'pachClient' is unauthenticated. This will use the PPS token (via
		// a.sudo()) to authenticate requests.
		pachClient := a.env.GetPachClient(ctx)
		kubeClient := a.env.GetKubeClient()

		log.Infof("PPS master: launching master process")

		// start pollPipelines in the background to regularly refresh pipelines
		func() {
			a.monitorCancelsMu.Lock()
			defer a.monitorCancelsMu.Unlock()
			a.pollCancel = startMonitorThread(pachClient, "pollPipelines",
				func(pachClient *client.APIClient) {
					a.pollPipelines(pachClient.Ctx())
				})
		}()

		// TODO(msteffen) request only keys, since pipeline_controller.go reads
		// fresh values for each event anyway
		pipelineWatcher, err := a.pipelines.ReadOnly(ctx).Watch()
		if err != nil {
			return errors.Wrapf(err, "error creating watch")
		}
		defer pipelineWatcher.Close()

		// watchChan will be nil if the Watch call below errors, this means
		// that we won't receive events from k8s and won't be able to detect
		// errors in pods. We could just return that error and retry but that
		// prevents pachyderm from creating pipelines when there's an issue
		// talking to k8s.
		var watchChan <-chan kube_watch.Event
		kubePipelineWatch, err := kubeClient.CoreV1().Pods(a.namespace).Watch(
			metav1.ListOptions{
				LabelSelector: metav1.FormatLabelSelector(metav1.SetAsLabelSelector(
					map[string]string{
						"component": "worker",
					})),
				Watch: true,
			})
		if err != nil {
			log.Errorf("failed to watch kuburnetes pods: %v", err)
		} else {
			watchChan = kubePipelineWatch.ResultChan()
			defer kubePipelineWatch.Stop()
		}

		for {
			select {
			case event := <-pipelineWatcher.Watch():
				if event.Err != nil {
					return errors.Wrapf(event.Err, "event err")
				}
				switch event.Type {
				case watch.EventPut:
					pipeline := string(event.Key)
					// Create/Modify/Delete pipeline resources as needed per new state
					if err := a.step(pachClient, pipeline, event.Ver, event.Rev); err != nil {
						log.Errorf("PPS master: %v", err)
					}
				case watch.EventDelete:
					// TODO(msteffen) trace this call
					// This is also called by pollPipelines below, if it discovers
					// dangling monitorPipeline goroutines
					if err := a.deletePipelineResources(pachClient.Ctx(), string(event.Key)); err != nil {
						log.Errorf("PPS master: could not delete pipeline resources for %q: %v", err)
					}
				}
			case event := <-watchChan:
				// if we get an error we restart the watch, k8s watches seem to
				// sometimes get stuck in a loop returning events with Type =
				// "" we treat these as errors since otherwise we get an
				// endless stream of them and can't do anything.
				if event.Type == kube_watch.Error || event.Type == "" {
					if kubePipelineWatch != nil {
						kubePipelineWatch.Stop()
					}
					kubePipelineWatch, err = kubeClient.CoreV1().Pods(a.namespace).Watch(
						metav1.ListOptions{
							LabelSelector: metav1.FormatLabelSelector(metav1.SetAsLabelSelector(
								map[string]string{
									"component": "worker",
								})),
							Watch: true,
						})
					if err != nil {
						log.Errorf("failed to watch kuburnetes pods: %v", err)
						watchChan = nil
					} else {
						watchChan = kubePipelineWatch.ResultChan()
						defer kubePipelineWatch.Stop()
					}
				}
				pod, ok := event.Object.(*v1.Pod)
				if !ok {
					continue
				}
				if pod.Status.Phase == v1.PodFailed {
					log.Errorf("pod failed because: %s", pod.Status.Message)
				}
				pipelineName := pod.ObjectMeta.Annotations["pipelineName"]
				for _, status := range pod.Status.ContainerStatuses {
					if status.State.Waiting != nil && failures[status.State.Waiting.Reason] {
						if err := a.setPipelineCrashing(pachClient.Ctx(), pipelineName, status.State.Waiting.Message); err != nil {
							return err
						}
					}
				}
				for _, condition := range pod.Status.Conditions {
					if condition.Type == v1.PodScheduled &&
						condition.Status != v1.ConditionTrue && failures[condition.Reason] {
						if err := a.setPipelineCrashing(pachClient.Ctx(), pipelineName, condition.Message); err != nil {
							return err
						}
					}
				}
			}
		}
	}, backoff.NewInfiniteBackOff(), func(err error, d time.Duration) error {
		// cancel all monitorPipeline and monitorCrashingPipeline goroutines.
		// Strictly speaking, this should be unnecessary, as the base context for
		// all monitor goros is cancelled by 'defer cancel()' at the beginning of
		// 'RetryNotify' above. However, these cancel calls also block until the
		// monitor goros exit, ensuring that a leftover goro won't interfere with a
		// subsequent iteration
		a.cancelAllMonitorsAndCrashingMonitors(nil)
		a.monitorCancelsMu.Lock()
		defer a.monitorCancelsMu.Unlock()
		if a.pollCancel != nil {
			a.pollCancel()
			a.pollCancel = nil
		}
		log.Errorf("PPS master: error running the master process: %v; retrying in %v", err, d)
		return nil
	})
	panic("internal error: PPS master has somehow exited. Restarting pod...")
}

func (a *apiServer) setPipelineFailure(ctx context.Context, pipelineName string, reason string) error {
	return a.setPipelineState(ctx, pipelineName, pps.PipelineState_PIPELINE_FAILURE, reason)
}

func (a *apiServer) setPipelineCrashing(ctx context.Context, pipelineName string, reason string) error {
	return a.setPipelineState(ctx, pipelineName, pps.PipelineState_PIPELINE_CRASHING, reason)
}

func (a *apiServer) deletePipelineResources(ctx context.Context, pipelineName string) (retErr error) {
	log.Infof("PPS master: deleting resources for pipeline %q", pipelineName)
	span, ctx := tracing.AddSpanToAnyExisting(ctx, //lint:ignore SA4006 ctx is unused, but better to have the right ctx in scope so people don't use the wrong one
		"/pps.Master/DeletePipelineResources", "pipeline", pipelineName)
	defer func() {
		tracing.TagAnySpan(span, "err", retErr)
		tracing.FinishAnySpan(span)
	}()

	// Cancel any running monitorPipeline call
	a.cancelMonitor(pipelineName)
	// Same for cancelCrashingMonitor
	a.cancelCrashingMonitor(pipelineName)

	kubeClient := a.env.GetKubeClient()
	// Delete any services associated with op.pipeline
	selector := fmt.Sprintf("%s=%s", pipelineNameLabel, pipelineName)
	opts := &metav1.DeleteOptions{
		OrphanDependents: &falseVal,
	}
	services, err := kubeClient.CoreV1().Services(a.namespace).List(metav1.ListOptions{LabelSelector: selector})
	if err != nil {
		return errors.Wrapf(err, "could not list services")
	}
	for _, service := range services.Items {
		if err := kubeClient.CoreV1().Services(a.namespace).Delete(service.Name, opts); err != nil {
			if !isNotFoundErr(err) {
				return errors.Wrapf(err, "could not delete service %q", service.Name)
			}
		}
	}
	rcs, err := kubeClient.CoreV1().ReplicationControllers(a.namespace).List(metav1.ListOptions{LabelSelector: selector})
	if err != nil {
		return errors.Wrapf(err, "could not list RCs")
	}
	for _, rc := range rcs.Items {
		if err := kubeClient.CoreV1().ReplicationControllers(a.namespace).Delete(rc.Name, opts); err != nil {
			if !isNotFoundErr(err) {
				return errors.Wrapf(err, "could not delete RC %q: %v", rc.Name)
			}
		}
	}
	return nil
}

// setPipelineState is a PPS-master-specific helper that wraps
// ppsutil.SetPipelineState in a trace
func (a *apiServer) setPipelineState(ctx context.Context, pipeline string, state pps.PipelineState, reason string) (retErr error) {
	span, ctx := tracing.AddSpanToAnyExisting(ctx,
		"/pps.Master/SetPipelineState", "pipeline", pipeline, "new-state", state)
	defer func() {
		tracing.TagAnySpan(span, "err", retErr)
		tracing.FinishAnySpan(span)
	}()
	return ppsutil.SetPipelineState(ctx, a.env.GetEtcdClient(), a.pipelines,
		pipeline, nil, state, reason)
}

// transitionPipelineState is similar to setPipelineState, except that it sets
// 'from' and logs a different trace
func (a *apiServer) transitionPipelineState(ctx context.Context, pipeline string, from []pps.PipelineState, to pps.PipelineState, reason string) (retErr error) {
	span, ctx := tracing.AddSpanToAnyExisting(ctx,
		"/pps.Master/TransitionPipelineState", "pipeline", pipeline,
		"from-state", from, "to-state", to)
	defer func() {
		tracing.TagAnySpan(span, "err", retErr)
		tracing.FinishAnySpan(span)
	}()
	return ppsutil.SetPipelineState(ctx, a.env.GetEtcdClient(), a.pipelines,
		pipeline, from, to, reason)
}

func (a *apiServer) pollPipelines(ctx context.Context) {
	start := time.Now()
	for {
		// wait until at least 10s after last loop, so this doesn't ping too
		// frequently (current cap: once/10s)
		select {
		case <-time.After(start.Add(10 * time.Second).Sub(time.Now())):
			// Inner select: confirm that ctx is not done. It's possible that both
			// time.After() and ctx.Done() are ready, and time.After() is randomly
			// chosen over ctx.Done().
			select {
			case <-ctx.Done():
				return
			default:
			}
		case <-ctx.Done():
			return
		}
		start = time.Now()

		etcdPipelines := map[string]bool{}
		pachClient := a.env.GetPachClient(ctx)
		// collect all pipelines in etcd
		if err := a.listPipelinePtr(pachClient, nil, 0,
			func(pipeline string, listPI *pps.EtcdPipelineInfo) error {
				log.Debugf("PPS master: polling pipeline %q", pipeline)
				etcdPipelines[pipeline] = true
				var curPI pps.EtcdPipelineInfo
				_, err := col.NewSTM(ctx, a.env.GetEtcdClient(), func(stm col.STM) error {
					// Read & immediately write
					return a.pipelines.ReadWrite(stm).Update(pipeline, &curPI, func() error { return nil })
				})
				if col.IsErrNotFound(err) {
					log.Warnf("%q polling conflicted with delete, will retry", pipeline)
				} else {
					log.Errorf("could not poll %q: %v", pipeline, err)
				}
				return nil // no error recovery to do here, just keep polling...
			}); err != nil {
			log.Errorf("could not list pipelines for polling: %v\n", err)
			continue // sleep and try again
		}

		// Clean up any orphaned monitorPipeline and monitorCrashingPipeline goros
		a.cancelAllMonitorsAndCrashingMonitors(etcdPipelines)
	}
}
<|endoftext|>


<commit_before>package server

import (
	"time"

	log "github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	kube_err "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kube_watch "k8s.io/apimachinery/pkg/watch"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	"github.com/pachyderm/pachyderm/src/server/pkg/watch"
)

const pollBackoffTime = 2 * time.Second

// startPipelinePoller starts a new goroutine running pollPipelines
func (m *ppsMaster) startPipelinePoller() {
	m.pollPipelinesMu.Lock()
	defer m.pollPipelinesMu.Unlock()
	m.pollCancel = m.startMonitorThread("pollPipelines", m.pollPipelines)
}

func (m *ppsMaster) cancelPipelinePoller() {
	m.pollPipelinesMu.Lock()
	defer m.pollPipelinesMu.Unlock()
	if m.pollCancel != nil {
		m.pollCancel()
		m.pollCancel = nil
	}
}

// startPipelinePodsPoller starts a new goroutine running pollPipelinePods
func (m *ppsMaster) startPipelinePodsPoller() {
	m.pollPipelinesMu.Lock()
	defer m.pollPipelinesMu.Unlock()
	m.pollPodsCancel = m.startMonitorThread("pollPipelinePods", m.pollPipelinePods)
}

func (m *ppsMaster) cancelPipelinePodsPoller() {
	m.pollPipelinesMu.Lock()
	defer m.pollPipelinesMu.Unlock()
	if m.pollPodsCancel != nil {
		m.pollPodsCancel()
		m.pollPodsCancel = nil
	}
}

// startPipelineEtcdPoller starts a new goroutine running
// pollPipelinesEtcd
func (m *ppsMaster) startPipelineEtcdPoller() {
	m.pollPipelinesMu.Lock()
	defer m.pollPipelinesMu.Unlock()
	m.pollEtcdCancel = m.startMonitorThread("pollPipelinesEtcd", m.pollPipelinesEtcd)
}

func (m *ppsMaster) cancelPipelineEtcdPoller() {
	m.pollPipelinesMu.Lock()
	defer m.pollPipelinesMu.Unlock()
	if m.pollEtcdCancel != nil {
		m.pollEtcdCancel()
		m.pollEtcdCancel = nil
	}
}

//////////////////////////////////////////////////////////////////////////////
//                     PollPipelines Definition                             //
// - As in monitor.go, functions below should not call functions above, to  //
// avoid reentrancy deadlock.                                               //
//////////////////////////////////////////////////////////////////////////////

// pollPipelines generates regular updateEv and deleteEv events for each
// pipeline and sends them to ppsMaster.Run(). By scanning etcd and k8s
// regularly and generating events for them, it prevents pipelines from
// getting orphaned.
func (m *ppsMaster) pollPipelines(pollClient *client.APIClient) {
	ctx := pollClient.Ctx()
	etcdPipelines := map[string]bool{}
	if err := backoff.RetryUntilCancel(ctx, func() error {
		if len(etcdPipelines) == 0 {
			// 1. Get the current set of pipeline RCs.
			//
			// We'll delete any RCs that don't correspond to a live pipeline after
			// querying etcd to determine the set of live pipelines, but we query k8s
			// first to avoid a race (if we were to query etcd first, and
			// CreatePipeline(foo) were to run between querying etcd and querying k8s,
			// then we might delete the RC for brand-new pipeline 'foo'). Even if we
			// do delete a live pipeline's RC, it'll be fixed in the next cycle)
			kc := m.a.env.GetKubeClient().CoreV1().ReplicationControllers(m.a.env.Namespace)
			rcs, err := kc.List(metav1.ListOptions{
				LabelSelector: "suite=pachyderm,pipelineName",
			})
			if err != nil {
				// No sensible error recovery here (e.g .if we can't reach k8s). We'll
				// keep going, and just won't delete any RCs this round.
				log.Errorf("error polling pipeline RCs: %v", err)
			}

			// 2. Replenish 'etcdPipelines' with the set of pipelines currently in
			// etcd. Note that there may be zero, and etcdPipelines may be empty
			if err := m.a.listPipelinePtr(pollClient, nil, 0,
				func(pipeline string, _ *pps.EtcdPipelineInfo) error {
					etcdPipelines[pipeline] = true
					return nil
				}); err != nil {
				// listPipelinePtr results (etcdPipelines) are used by all remaining
				// steps, so if that didn't work, start over and try again
				etcdPipelines = map[string]bool{}
				return errors.Wrap(err, "error polling pipelines")
			}

			// 3. Generate a delete event for orphaned RCs
			if rcs != nil {
				for _, rc := range rcs.Items {
					pipeline, ok := rc.Labels["pipelineName"]
					if !ok {
						return errors.New("'pipelineName' label missing from rc " + rc.Name)
					}
					if !etcdPipelines[pipeline] {
						m.eventCh <- &pipelineEvent{eventType: deleteEv, pipeline: pipeline}
					}
				}
			}

			// 4. Retry if there are no etcd pipelines to read/write
			if len(etcdPipelines) == 0 {
				return backoff.ErrContinue
			}
		}

		// Generate one etcd event for a pipeline (to trigger the pipeline
		// controller) and remove this pipeline from etcdPipelines. Always choose
		// the lexicographically smallest pipeline so that pipelines are always
		// traversed in the same order and the period between polls is stable across
		// all pipelines.
		var pipeline string
		for p := range etcdPipelines {
			if pipeline == "" || p < pipeline {
				pipeline = p
			}
		}

		// always rm 'pipeline', to advance loop
		delete(etcdPipelines, pipeline)

		// generate a pipeline event for 'pipeline'
		log.Debugf("PPS master: polling pipeline %q", pipeline)
		m.eventCh <- &pipelineEvent{eventType: writeEv, pipeline: pipeline}

		// move to next pipeline (after 2s sleep)
		return backoff.ErrContinue
	}, backoff.NewConstantBackOff(pollBackoffTime),
		backoff.NotifyContinue("pollPipelines"),
	); err != nil {
		if ctx.Err() == nil {
			panic("pollPipelines is exiting prematurely which should not happen; restarting pod...")
		}
	}
}

// pollPipelinePods creates a kubernetes watch, and for each event:
//   1) Checks if the event concerns a Pod
//   2) Checks if the Pod belongs to a pipeline (pipelineName annotation is set)
//   3) Checks if the Pod is failing
// If all three conditions are met, then the pipline (in 'pipelineName') is set
// to CRASHING
func (m *ppsMaster) pollPipelinePods(pollClient *client.APIClient) {
	ctx := pollClient.Ctx()
	if err := backoff.RetryUntilCancel(ctx, func() error {
		kubePipelineWatch, err := m.a.env.GetKubeClient().CoreV1().Pods(m.a.namespace).Watch(
			metav1.ListOptions{
				LabelSelector: metav1.FormatLabelSelector(metav1.SetAsLabelSelector(
					map[string]string{
						"component": "worker",
					})),
				Watch: true,
			})
		if err != nil {
			return errors.Wrap(err, "failed to watch kubernetes pods")
		}
		defer kubePipelineWatch.Stop()
		for event := range kubePipelineWatch.ResultChan() {
			// if we get an error we restart the watch
			if event.Type == kube_watch.Error {
				return errors.Wrap(kube_err.FromObject(event.Object), "error while watching kubernetes pods")
			} else if event.Type == "" {
				// k8s watches seem to sometimes get stuck in a loop returning events
				// with Type = "". We treat these as errors as otherwise we get an
				// endless stream of them and can't do anything.
				return errors.New("error while watching kubernetes pods: empty event type")
			}
			pod, ok := event.Object.(*v1.Pod)
			if !ok {
				continue // irrelevant event
			}
			if pod.Status.Phase == v1.PodFailed {
				log.Errorf("pod failed because: %s", pod.Status.Message)
			}
			pipelineName := pod.ObjectMeta.Annotations["pipelineName"]
			for _, status := range pod.Status.ContainerStatuses {
				if status.State.Waiting != nil && failures[status.State.Waiting.Reason] {
					if err := m.a.setPipelineCrashing(ctx, pipelineName, status.State.Waiting.Message); err != nil {
						return errors.Wrap(err, "error moving pipeline to CRASHING")
					}
				}
			}
			for _, condition := range pod.Status.Conditions {
				if condition.Type == v1.PodScheduled &&
					condition.Status != v1.ConditionTrue && failures[condition.Reason] {
					if err := m.a.setPipelineCrashing(ctx, pipelineName, condition.Message); err != nil {
						return errors.Wrap(err, "error moving pipeline to CRASHING")
					}
				}
			}
		}
		return backoff.ErrContinue // keep polling until cancelled (RetryUntilCancel)
	}, &backoff.ZeroBackOff{}, backoff.NotifyContinue("pollPipelinePods"),
	); err != nil && ctx.Err() == nil {
		panic("pollPipelinePods is exiting prematurely which should not happen; restarting pod...")
	}
}

// pollPipelinesEtcd watches the 'pipelines' collection in etcd and sends
// writeEv and deleteEv events to the PPS master when it sees them.
//
// pollPipelinesEtcd is unlike the other poll and monitor goroutines in that it
// sees the result of other poll/monitor goroutines' writes. For example, when
// pollPipelinePods (above) observes that a pipeline is crashing and updates its
// state in etcd, the flow for starting monitorPipelineCrashing is:
//
//  k8s watch ─> pollPipelinePods  ╭──> pollPipelinesEtcd  ╭──> m.run()
//                      │          │            │          │      │
//                      ↓          │            ↓          │      ↓
//                  etcd write─────╯       m.eventCh ──────╯   m.step()
//
// most of the other poll/monitor goroutines actually go through
// pollPipelinesEtcd (by writing to etcd, which is then observed by the etcd
// watch below)
func (m *ppsMaster) pollPipelinesEtcd(pollClient *client.APIClient) {
	ctx := pollClient.Ctx()
	if err := backoff.RetryUntilCancel(ctx, func() error {
		// TODO(msteffen) request only keys, since pipeline_controller.go reads
		// fresh values for each event anyway
		pipelineWatcher, err := m.a.pipelines.ReadOnly(ctx).Watch()
		if err != nil {
			return errors.Wrapf(err, "error creating watch")
		}
		defer pipelineWatcher.Close()

		for event := range pipelineWatcher.Watch() {
			if event.Err != nil {
				return errors.Wrapf(event.Err, "event err")
			}
			switch event.Type {
			case watch.EventPut:
				m.eventCh <- &pipelineEvent{
					eventType: writeEv,
					pipeline:  string(event.Key),
					etcdVer:   event.Ver,
					etcdRev:   event.Rev,
				}
			case watch.EventDelete:
				m.eventCh <- &pipelineEvent{
					eventType: deleteEv,
					pipeline:  string(event.Key),
					etcdVer:   event.Ver,
					etcdRev:   event.Rev,
				}
			}
		}
		return backoff.ErrContinue // reset until ctx is cancelled (RetryUntilCancel)
	}, &backoff.ZeroBackOff{}, backoff.NotifyContinue("pollPipelinesEtcd"),
	); err != nil && ctx.Err() == nil {
		panic("pollPipelinesEtcd is exiting prematurely which should not happen; restarting pod...")
	}
}
<commit_msg>Use exponential backoff around k8s watch<commit_after>package server

import (
	"time"

	log "github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	kube_err "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kube_watch "k8s.io/apimachinery/pkg/watch"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	"github.com/pachyderm/pachyderm/src/server/pkg/watch"
)

const pollBackoffTime = 2 * time.Second

// startPipelinePoller starts a new goroutine running pollPipelines
func (m *ppsMaster) startPipelinePoller() {
	m.pollPipelinesMu.Lock()
	defer m.pollPipelinesMu.Unlock()
	m.pollCancel = m.startMonitorThread("pollPipelines", m.pollPipelines)
}

func (m *ppsMaster) cancelPipelinePoller() {
	m.pollPipelinesMu.Lock()
	defer m.pollPipelinesMu.Unlock()
	if m.pollCancel != nil {
		m.pollCancel()
		m.pollCancel = nil
	}
}

// startPipelinePodsPoller starts a new goroutine running pollPipelinePods
func (m *ppsMaster) startPipelinePodsPoller() {
	m.pollPipelinesMu.Lock()
	defer m.pollPipelinesMu.Unlock()
	m.pollPodsCancel = m.startMonitorThread("pollPipelinePods", m.pollPipelinePods)
}

func (m *ppsMaster) cancelPipelinePodsPoller() {
	m.pollPipelinesMu.Lock()
	defer m.pollPipelinesMu.Unlock()
	if m.pollPodsCancel != nil {
		m.pollPodsCancel()
		m.pollPodsCancel = nil
	}
}

// startPipelineEtcdPoller starts a new goroutine running
// pollPipelinesEtcd
func (m *ppsMaster) startPipelineEtcdPoller() {
	m.pollPipelinesMu.Lock()
	defer m.pollPipelinesMu.Unlock()
	m.pollEtcdCancel = m.startMonitorThread("pollPipelinesEtcd", m.pollPipelinesEtcd)
}

func (m *ppsMaster) cancelPipelineEtcdPoller() {
	m.pollPipelinesMu.Lock()
	defer m.pollPipelinesMu.Unlock()
	if m.pollEtcdCancel != nil {
		m.pollEtcdCancel()
		m.pollEtcdCancel = nil
	}
}

//////////////////////////////////////////////////////////////////////////////
//                     PollPipelines Definition                             //
// - As in monitor.go, functions below should not call functions above, to  //
// avoid reentrancy deadlock.                                               //
//////////////////////////////////////////////////////////////////////////////

// pollPipelines generates regular updateEv and deleteEv events for each
// pipeline and sends them to ppsMaster.Run(). By scanning etcd and k8s
// regularly and generating events for them, it prevents pipelines from
// getting orphaned.
func (m *ppsMaster) pollPipelines(pollClient *client.APIClient) {
	ctx := pollClient.Ctx()
	etcdPipelines := map[string]bool{}
	if err := backoff.RetryUntilCancel(ctx, func() error {
		if len(etcdPipelines) == 0 {
			// 1. Get the current set of pipeline RCs.
			//
			// We'll delete any RCs that don't correspond to a live pipeline after
			// querying etcd to determine the set of live pipelines, but we query k8s
			// first to avoid a race (if we were to query etcd first, and
			// CreatePipeline(foo) were to run between querying etcd and querying k8s,
			// then we might delete the RC for brand-new pipeline 'foo'). Even if we
			// do delete a live pipeline's RC, it'll be fixed in the next cycle)
			kc := m.a.env.GetKubeClient().CoreV1().ReplicationControllers(m.a.env.Namespace)
			rcs, err := kc.List(metav1.ListOptions{
				LabelSelector: "suite=pachyderm,pipelineName",
			})
			if err != nil {
				// No sensible error recovery here (e.g .if we can't reach k8s). We'll
				// keep going, and just won't delete any RCs this round.
				log.Errorf("error polling pipeline RCs: %v", err)
			}

			// 2. Replenish 'etcdPipelines' with the set of pipelines currently in
			// etcd. Note that there may be zero, and etcdPipelines may be empty
			if err := m.a.listPipelinePtr(pollClient, nil, 0,
				func(pipeline string, _ *pps.EtcdPipelineInfo) error {
					etcdPipelines[pipeline] = true
					return nil
				}); err != nil {
				// listPipelinePtr results (etcdPipelines) are used by all remaining
				// steps, so if that didn't work, start over and try again
				etcdPipelines = map[string]bool{}
				return errors.Wrap(err, "error polling pipelines")
			}

			// 3. Generate a delete event for orphaned RCs
			if rcs != nil {
				for _, rc := range rcs.Items {
					pipeline, ok := rc.Labels["pipelineName"]
					if !ok {
						return errors.New("'pipelineName' label missing from rc " + rc.Name)
					}
					if !etcdPipelines[pipeline] {
						m.eventCh <- &pipelineEvent{eventType: deleteEv, pipeline: pipeline}
					}
				}
			}

			// 4. Retry if there are no etcd pipelines to read/write
			if len(etcdPipelines) == 0 {
				return backoff.ErrContinue
			}
		}

		// Generate one etcd event for a pipeline (to trigger the pipeline
		// controller) and remove this pipeline from etcdPipelines. Always choose
		// the lexicographically smallest pipeline so that pipelines are always
		// traversed in the same order and the period between polls is stable across
		// all pipelines.
		var pipeline string
		for p := range etcdPipelines {
			if pipeline == "" || p < pipeline {
				pipeline = p
			}
		}

		// always rm 'pipeline', to advance loop
		delete(etcdPipelines, pipeline)

		// generate a pipeline event for 'pipeline'
		log.Debugf("PPS master: polling pipeline %q", pipeline)
		m.eventCh <- &pipelineEvent{eventType: writeEv, pipeline: pipeline}

		// move to next pipeline (after 2s sleep)
		return backoff.ErrContinue
	}, backoff.NewConstantBackOff(pollBackoffTime),
		backoff.NotifyContinue("pollPipelines"),
	); err != nil {
		if ctx.Err() == nil {
			panic("pollPipelines is exiting prematurely which should not happen; restarting pod...")
		}
	}
}

// pollPipelinePods creates a kubernetes watch, and for each event:
//   1) Checks if the event concerns a Pod
//   2) Checks if the Pod belongs to a pipeline (pipelineName annotation is set)
//   3) Checks if the Pod is failing
// If all three conditions are met, then the pipline (in 'pipelineName') is set
// to CRASHING
func (m *ppsMaster) pollPipelinePods(pollClient *client.APIClient) {
	ctx := pollClient.Ctx()
	if err := backoff.RetryUntilCancel(ctx, func() error {
		kubePipelineWatch, err := m.a.env.GetKubeClient().CoreV1().Pods(m.a.namespace).Watch(
			metav1.ListOptions{
				LabelSelector: metav1.FormatLabelSelector(metav1.SetAsLabelSelector(
					map[string]string{
						"component": "worker",
					})),
				Watch: true,
			})
		if err != nil {
			return errors.Wrap(err, "failed to watch kubernetes pods")
		}
		defer kubePipelineWatch.Stop()
		for event := range kubePipelineWatch.ResultChan() {
			// if we get an error we restart the watch
			if event.Type == kube_watch.Error {
				return errors.Wrap(kube_err.FromObject(event.Object), "error while watching kubernetes pods")
			} else if event.Type == "" {
				// k8s watches seem to sometimes get stuck in a loop returning events
				// with Type = "". We treat these as errors as otherwise we get an
				// endless stream of them and can't do anything.
				return errors.New("error while watching kubernetes pods: empty event type")
			}
			pod, ok := event.Object.(*v1.Pod)
			if !ok {
				continue // irrelevant event
			}
			if pod.Status.Phase == v1.PodFailed {
				log.Errorf("pod failed because: %s", pod.Status.Message)
			}
			pipelineName := pod.ObjectMeta.Annotations["pipelineName"]
			for _, status := range pod.Status.ContainerStatuses {
				if status.State.Waiting != nil && failures[status.State.Waiting.Reason] {
					if err := m.a.setPipelineCrashing(ctx, pipelineName, status.State.Waiting.Message); err != nil {
						return errors.Wrap(err, "error moving pipeline to CRASHING")
					}
				}
			}
			for _, condition := range pod.Status.Conditions {
				if condition.Type == v1.PodScheduled &&
					condition.Status != v1.ConditionTrue && failures[condition.Reason] {
					if err := m.a.setPipelineCrashing(ctx, pipelineName, condition.Message); err != nil {
						return errors.Wrap(err, "error moving pipeline to CRASHING")
					}
				}
			}
		}
		return backoff.ErrContinue // keep polling until cancelled (RetryUntilCancel)
	}, backoff.NewExponentialBackOff(), backoff.NotifyContinue("pollPipelinePods"),
	); err != nil && ctx.Err() == nil {
		panic("pollPipelinePods is exiting prematurely which should not happen; restarting pod...")
	}
}

// pollPipelinesEtcd watches the 'pipelines' collection in etcd and sends
// writeEv and deleteEv events to the PPS master when it sees them.
//
// pollPipelinesEtcd is unlike the other poll and monitor goroutines in that it
// sees the result of other poll/monitor goroutines' writes. For example, when
// pollPipelinePods (above) observes that a pipeline is crashing and updates its
// state in etcd, the flow for starting monitorPipelineCrashing is:
//
//  k8s watch ─> pollPipelinePods  ╭──> pollPipelinesEtcd  ╭──> m.run()
//                      │          │            │          │      │
//                      ↓          │            ↓          │      ↓
//                  etcd write─────╯       m.eventCh ──────╯   m.step()
//
// most of the other poll/monitor goroutines actually go through
// pollPipelinesEtcd (by writing to etcd, which is then observed by the etcd
// watch below)
func (m *ppsMaster) pollPipelinesEtcd(pollClient *client.APIClient) {
	ctx := pollClient.Ctx()
	if err := backoff.RetryUntilCancel(ctx, func() error {
		// TODO(msteffen) request only keys, since pipeline_controller.go reads
		// fresh values for each event anyway
		pipelineWatcher, err := m.a.pipelines.ReadOnly(ctx).Watch()
		if err != nil {
			return errors.Wrapf(err, "error creating watch")
		}
		defer pipelineWatcher.Close()

		for event := range pipelineWatcher.Watch() {
			if event.Err != nil {
				return errors.Wrapf(event.Err, "event err")
			}
			switch event.Type {
			case watch.EventPut:
				m.eventCh <- &pipelineEvent{
					eventType: writeEv,
					pipeline:  string(event.Key),
					etcdVer:   event.Ver,
					etcdRev:   event.Rev,
				}
			case watch.EventDelete:
				m.eventCh <- &pipelineEvent{
					eventType: deleteEv,
					pipeline:  string(event.Key),
					etcdVer:   event.Ver,
					etcdRev:   event.Rev,
				}
			}
		}
		return backoff.ErrContinue // reset until ctx is cancelled (RetryUntilCancel)
	}, &backoff.ZeroBackOff{}, backoff.NotifyContinue("pollPipelinesEtcd"),
	); err != nil && ctx.Err() == nil {
		panic("pollPipelinesEtcd is exiting prematurely which should not happen; restarting pod...")
	}
}
<|endoftext|>


<commit_before>// Licensed to The Moov Authors under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. The Moov Authors licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package ach

// Version Number
const Version = "v1.21.0"
<commit_msg>Release v1.21.1<commit_after>// Licensed to The Moov Authors under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. The Moov Authors licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package ach

// Version Number
const Version = "v1.21.1"
<|endoftext|>


<commit_before>package egoscale

// Version of the library
const Version = "0.9.22"
<commit_msg>prepare release<commit_after>package egoscale

// Version of the library
const Version = "0.9.23"
<|endoftext|>


<commit_before>//
// Copyright © 2016-2019 Guy M. Allard
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package stompngo

/*
	Provide package version information.  A nod to the concept of semver.

	Example:
		fmt.Println("current stompngo version", stompngo.Version())

*/

import (
	"fmt"
)

var (
	pref = "v" // Prefix

	major = "1" // Major

	minor = "0" // Minor

	patch = "13" // Patch

	// mod = "-m.3" // Modification level

	mod = ""
)

func Version() string {
	return fmt.Sprintf("%s%s.%s.%s%s", pref, major, minor, patch, mod)
}
<commit_msg>Start work on next version.<commit_after>//
// Copyright © 2016-2019 Guy M. Allard
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package stompngo

/*
	Provide package version information.  A nod to the concept of semver.

	Example:
		fmt.Println("current stompngo version", stompngo.Version())

*/

import (
	"fmt"
)

var (
	pref = "v" // Prefix

	major = "1" // Major

	minor = "0" // Minor

	patch = "13" // Patch

	mod = "-m.1" // Modification level

	// mod = ""
)

func Version() string {
	return fmt.Sprintf("%s%s.%s.%s%s", pref, major, minor, patch, mod)
}
<|endoftext|>


<commit_before>package main

const VERSION = "0.9.1-alpha1"
<commit_msg>:+1: Bump up the version to 0.9.1<commit_after>package main

const VERSION = "0.9.1"
<|endoftext|>


<commit_before>package cloudstack

const Version string = "v0.1.1"
<commit_msg>v0.1.2<commit_after>package cloudstack

const Version string = "v0.1.2"
<|endoftext|>


<commit_before>package main

const Version = "0.1.15"
<commit_msg>feat(version): bump<commit_after>package main

const Version = "0.1.16"
<|endoftext|>