<commit_before>package gitschemalex

import (
	"database/sql"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"testing"

	_ "github.com/go-sql-driver/mysql"
	"github.com/lestrrat/go-test-mysqld"
)

func TestRunner(t *testing.T) {
	var dsn = "root:@127.0.0.1:3306"
	if ok, _ := strconv.ParseBool(os.Getenv("TRAVIS")); !ok {
		mysqld, err := mysqltest.NewMysqld(nil)
		if err != nil {
			t.Fatal(err)
		}
		defer mysqld.Stop()
		dsn = myqld.DataSource("", "", "", 0)
	}

	db, err := sql.Open("mysql", dsn)
	if err != nil {
		t.Fatal(err)
	}
	defer db.Close()

	if _, err := db.Exec("CREATE DATABASE IF NOT EXISTS `test`"); err != nil {
		t.Fatal(err)
	}

	if _, err := db.Exec("USE `test`"); err != nil {
		t.Fatal(err)
	}

	dir, err := ioutil.TempDir("", "gitschemalex")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	if err := os.Chdir(dir); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "init").Run(); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "config", "user.email", "hoge@example.com").Run(); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "config", "user.name", "hoge").Run(); err != nil {
		t.Fatal(err)
	}

	schema, err := os.Create(filepath.Join(dir, "schema.sql"))
	if err != nil {
		t.Fatal(err)
	}

	// first table

	if _, err := schema.WriteString("CREATE TABLE hoge ( `id` INTEGER NOT NULL, `c` VARCHAR(20) );\n"); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "add", "schema.sql").Run(); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "commit", "-m", "initial commit").Run(); err != nil {
		t.Fatal(err)
	}

	r := &Runner{
		Workspace: dir,
		Deploy:    true,
		DSN:       dsn,
		Table:     "git_schemalex_version",
		Schema:    "schema.sql",
	}
	if err := r.Run(); err != nil {
		t.Fatal(err)
	}

	// deployed

	if _, err := db.Exec("INSERT INTO `hoge` (`id`, `c`) VALUES (1, '2')"); err != nil {
		t.Fatal(err)
	}

	// second table

	if _, err := schema.WriteString("CREATE TABLE fuga ( `id` INTEGER NOT NULL, `c` VARCHAR(20) );\n"); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "add", "schema.sql").Run(); err != nil {
		t.Fatal(err)
	}
	if err := exec.Command("git", "commit", "--author", "hoge <hoge@example.com>", "-m", "second commit").Run(); err != nil {
		t.Fatal(err)
	}

	if err := r.Run(); err != nil {
		t.Fatal(err)
	}

	if _, err := db.Exec("INSERT INTO `fuga` (`id`, `c`) VALUES (1, '2')"); err != nil {
		t.Fatal(err)
	}

	// equal version

	if e, g := ErrEqualVersion, r.Run(); e != g {
		t.Fatal("should %v got %v", e, g)
	}
}
<commit_msg>remove fmt<commit_after>package gitschemalex

import (
	"database/sql"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"testing"

	_ "github.com/go-sql-driver/mysql"
	"github.com/lestrrat/go-test-mysqld"
)

func TestRunner(t *testing.T) {
	var dsn = "root:@127.0.0.1:3306"
	if ok, _ := strconv.ParseBool(os.Getenv("TRAVIS")); !ok {
		mysqld, err := mysqltest.NewMysqld(nil)
		if err != nil {
			t.Fatal(err)
		}
		defer mysqld.Stop()
		dsn = myqld.DataSource("", "", "", 0)
	}

	db, err := sql.Open("mysql", dsn)
	if err != nil {
		t.Fatal(err)
	}
	defer db.Close()

	if _, err := db.Exec("CREATE DATABASE IF NOT EXISTS `test`"); err != nil {
		t.Fatal(err)
	}

	if _, err := db.Exec("USE `test`"); err != nil {
		t.Fatal(err)
	}

	dir, err := ioutil.TempDir("", "gitschemalex")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	if err := os.Chdir(dir); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "init").Run(); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "config", "user.email", "hoge@example.com").Run(); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "config", "user.name", "hoge").Run(); err != nil {
		t.Fatal(err)
	}

	schema, err := os.Create(filepath.Join(dir, "schema.sql"))
	if err != nil {
		t.Fatal(err)
	}

	// first table

	if _, err := schema.WriteString("CREATE TABLE hoge ( `id` INTEGER NOT NULL, `c` VARCHAR(20) );\n"); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "add", "schema.sql").Run(); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "commit", "-m", "initial commit").Run(); err != nil {
		t.Fatal(err)
	}

	r := &Runner{
		Workspace: dir,
		Deploy:    true,
		DSN:       dsn,
		Table:     "git_schemalex_version",
		Schema:    "schema.sql",
	}
	if err := r.Run(); err != nil {
		t.Fatal(err)
	}

	// deployed

	if _, err := db.Exec("INSERT INTO `hoge` (`id`, `c`) VALUES (1, '2')"); err != nil {
		t.Fatal(err)
	}

	// second table

	if _, err := schema.WriteString("CREATE TABLE fuga ( `id` INTEGER NOT NULL, `c` VARCHAR(20) );\n"); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "add", "schema.sql").Run(); err != nil {
		t.Fatal(err)
	}
	if err := exec.Command("git", "commit", "--author", "hoge <hoge@example.com>", "-m", "second commit").Run(); err != nil {
		t.Fatal(err)
	}

	if err := r.Run(); err != nil {
		t.Fatal(err)
	}

	if _, err := db.Exec("INSERT INTO `fuga` (`id`, `c`) VALUES (1, '2')"); err != nil {
		t.Fatal(err)
	}

	// equal version

	if e, g := ErrEqualVersion, r.Run(); e != g {
		t.Fatal("should %v got %v", e, g)
	}
}
<|endoftext|>


<commit_before>package glaze

import (
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path"
	"syscall"
	"time"

	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/spf13/viper"
)

// Application encapsulated all the startup logic for a web app using glaze.
type Application struct {
	port        int
	graceful    bool
	verbose     bool
	listener    gracefulListener
	server      http.Server
	processName string
	accessLog   *os.File
	errorLog    *os.File
}

// Initialize handles all the pre-launch initialization tasks.
func (app *Application) Initialize(processName string, port int, verbose, graceful bool) error {
	app.port = port
	app.verbose = verbose
	app.graceful = graceful

	app.processName = processName

	err := app.parseConfig()
	if nil != err {
		return err
	}

	logPath := viper.GetString("LogPath")

	if _, err := os.Stat(logPath); os.IsNotExist(err) {
		err = os.Mkdir(logPath, os.ModeDir|os.ModePerm)
		if err != nil {
			return err
		}
	}

	// Set up server and listener
	app.server = http.Server{
		Addr:           fmt.Sprintf(":%d", app.port),
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 16,
	}

	var l net.Listener

	if app.graceful {
		f := os.NewFile(3, "")
		l, err = net.FileListener(f)
	} else {
		l, err = net.Listen("tcp", app.server.Addr)
	}
	if err != nil {
		return err
	}

	if app.graceful {
		parent := syscall.Getppid()
		log.Printf("main: Killing parent pid: %v", parent)
		syscall.Kill(parent, syscall.SIGTERM)
	}

	app.listener = gracefulListener{Listener: l, stop: make(chan signal, 1)}

	// Configure the access and error logs
	log.SetPrefix(fmt.Sprintf("[%5d] ", syscall.Getpid()))

	app.accessLog, err = app.CreateLog("access.log")
	if err != nil {
		return err
	}

	app.errorLog, err = app.CreateLog("error.log")
	if err != nil {
		return err
	}

	return nil
}

func (app Application) parseConfig() error {
	// Create configuration
	viper.Set("AppRoot", os.ExpandEnv("."))
	viper.Set("LogPath", os.ExpandEnv("./logs/"))

	viper.SetConfigName("config")
	viper.AddConfigPath(os.ExpandEnv("./config/"))
	err := viper.ReadInConfig()

	if err != nil {
		return err
	}

	viper.Set("FullViewPath", path.Join(viper.GetString("AppRoot"), viper.GetString("ViewPath")))

	return nil
}

func (app Application) logreq(req *http.Request) {
	if app.verbose {
		log.Printf("%v %v from %v", req.Method, req.URL, req.RemoteAddr)
	}
}

// Restart performs a graceful restart of the process
func (app Application) Restart(w http.ResponseWriter, req *http.Request) {
	file := app.listener.File() // this returns a Dup()

	cmd := exec.Command(fmt.Sprintf("./%s", app.processName),
		fmt.Sprintf("-port=%d", app.port),
		fmt.Sprintf("-verbose=%v", app.verbose),
		"-graceful")
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.ExtraFiles = []*os.File{file}

	err := cmd.Start()
	if err != nil {
		log.Fatalf("Graceful restart: Failed to launch, error: %v", err)
	}
}

// CreateLog either opens or creates and opens a log file at the given path, and returns an append handle.
func (app Application) CreateLog(name string) (*os.File, error) {
	fullPath := path.Join(viper.GetString("LogPath"), name)

	if _, err := os.Stat(fullPath); os.IsNotExist(err) {
		_, err := os.Create(fullPath)
		if err != nil {
			return nil, err
		}
	}

	handle, err := os.OpenFile(fullPath, os.O_APPEND|os.O_WRONLY, 0600)

	if err != nil {
		return nil, err
	}

	return handle, nil
}

// ServeWithAutoRouting computes standard paths to every Controller that
// has been added to the Application, and then starts the application server.
func (app Application) ServeWithAutoRouting(controllers []*Controller) error {
	// Create routes from the controllers
	autoRouter := mux.NewRouter()

	for _, controller := range controllers {
		for path, handler := range GenerateRoutes(controller) {
			fmt.Println(path)
			if path == "/public/index" {
				autoRouter.HandleFunc("/", handler)
			} else {
				autoRouter.HandleFunc(path, handler)
			}
		}
	}

	// Tell the router how to handle static assets
	autoRouter.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir("views/static"))))

	err := app.ServeWithHandler(autoRouter)
	if err != nil {
		return err
	}

	return nil
}

// ServeWithHandler starts the server with a client-provided handler.
// If you want to use glaze's auto-routing, use ServeWithAutoRouting instead.
func (app Application) ServeWithHandler(handler http.Handler) error {
	// Wrap the router in a LoggingHandler - this will use accessLog to log every connection to the server.
	loggingRouter := handlers.LoggingHandler(app.accessLog, handler)

	// When we hit the /upgrade URL, perform a graceful restart.
	// TODO - this needs to change, but it's fine for proof-of-concept
	http.HandleFunc("/upgrade", app.Restart)

	// For any other connections, use the loggingRouter
	http.Handle("/", loggingRouter)

	// this goroutine monitors the channel. Can't do this in
	// Accept (below) because once it enters listener.Listener.Accept()
	// it blocks. We unblock it by closing the fd it is trying to
	// accept(2) on.
	go func() {
		_ = <-app.listener.stop
		app.listener.stopped = true
		app.listener.Listener.Close()
	}()

	err := app.server.Serve(app.listener)
	if err != nil {
		return err
	}

	return nil
}

func errorHandler(w http.ResponseWriter, r *http.Request, status int) {
	http.Error(w, "Could not find the requested page", http.StatusInternalServerError)
}
<commit_msg>Removing renamed file<commit_after><|endoftext|>


<commit_before>// Copyright 2015 Mathieu Lonjaret

package main

import (
	//	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	//	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"os/exec"
)

const (
	idstring = "http://golang.org/pkg/http/#ListenAndServe"
	apiURL = "https://api.soundcloud.com"
)

var (
	flagHost         = flag.String("host", "0.0.0.0:8080", "listening port and hostname")
	help             = flag.Bool("h", false, "show this help")
	flagClientSecret = flag.String("client_secret", "", "app oauth2 client secret")
)

func usage() {
	fmt.Fprintf(os.Stderr, "\t likestoplaylist \n")
	flag.PrintDefaults()
	os.Exit(2)
}

var oauthToken string

func makeHandler(fn func(http.ResponseWriter, *http.Request, string)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if e, ok := recover().(error); ok {
				http.Error(w, e.Error(), http.StatusInternalServerError)
				return
			}
		}()
		title := r.URL.Path
		w.Header().Set("Server", idstring)
		fn(w, r, title)
	}
}

type oauth2Response struct {
	Token string `json:"access_token,omitempty"`
	Scope string
}

func getToken(authCode string) (tok string, err error) {
	resp, err := http.PostForm("https://api.soundcloud.com/oauth2/token", url.Values{
		"client_id":     {"eba287aa2c6cef9b2cf5aed6b73d3542"},
		"client_secret": {*flagClientSecret},
		"grant_type":    {"authorization_code"},
		"redirect_uri":  {"http://127.0.0.1:8080/callback"},
		"code":          {authCode},
	})
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	var jsonResp oauth2Response
	if err := json.Unmarshal(body, &jsonResp); err != nil {
		return "", fmt.Errorf("could not decode %q: %v", body, err)
	}
	tok = jsonResp.Token
	if tok == "" {
		return "", fmt.Errorf("No token in response: %q", body)
	}
	return tok, nil
}

func serveError(w http.ResponseWriter, response string, err error) {
	log.Printf("%v", err)
	w.Write([]byte(response))
}

func apiGet(url string) ([]byte, error) {
	resp, err := http.Get(apiURL+url+"?oauth_token=" + oauthToken)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return body, nil
}

func handleCallback(w http.ResponseWriter, r *http.Request) {
	authCode := r.FormValue("code")
	if authCode == "" {
		serveError(w, "no auth code in callback", fmt.Errorf("no auth code in callback"))
		return
	}
	tok, err := getToken(authCode)
	if err != nil {
		serveError(w, "token exchange error", fmt.Errorf("token exchange error: %v", err))
		return
	}
	println("GOT TOKEN: " + tok)
	oauthToken = tok

	body, err := apiGet("/me")
	if err != nil {
		serveError(w, "error getting me", fmt.Errorf("error getting me: %v", err))
		return
	}

	println(string(body))

	body, err = apiGet("/me/favorites")
	if err != nil {
		serveError(w, "error getting my favorites", fmt.Errorf("error getting my favorites: %v", err))
		return
	}

//	println(string(body))
	w.Write("ALL GOOD")
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if *help {
		usage()
	}
	if *flagClientSecret == "" {
		usage()
	}

	nargs := flag.NArg()
	if nargs > 0 {
		usage()
	}

	http.Handle("/", makeHandler(func(w http.ResponseWriter, r *http.Request, title string) {
		println(title)
		w.Write([]byte("OK ALRIGHT"))
		return
	}))
	http.Handle("/callback", makeHandler(func(w http.ResponseWriter, r *http.Request, title string) {
		println(title)
		handleCallback(w, r)
		return
	}))
	errc := make(chan error)
	go func() {
		errc <- http.ListenAndServe(*flagHost, nil)
	}()
	if err := exec.Command("xdg-open", "https://soundcloud.com/connect/?client_id=eba287aa2c6cef9b2cf5aed6b73d3542&redirect_uri=http://127.0.0.1:8080/callback&response_type=code&scope=non-expiring").Run(); err != nil {
		log.Fatal(err)
	}
	log.Fatal(<-errc)
}

/*
https://soundcloud.com/connect/?client_id=eba287aa2c6cef9b2cf5aed6b73d3542&redirect_uri=http://127.0.0.1:8080/callback&response_type=code&scope=non-expiring

code=52a31aaa7ef5dc1f4cee4ebe0eeaef92

curl -F 'client_id=eba287aa2c6cef9b2cf5aed6b73d3542' -F 'grant_type=authorization_code' -F 'redirect_uri=http://127.0.0.1:8080/callback' -F 'code=52a31aaa7ef5dc1f4cee4ebe0eeaef92' https://api.soundcloud.com/oauth2/token

*/
<commit_msg>ok getting favorites<commit_after>// Copyright 2015 Mathieu Lonjaret

package main

import (
	//	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	//	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"os/exec"
)

const (
	idstring = "http://golang.org/pkg/http/#ListenAndServe"
	apiURL = "https://api.soundcloud.com"
)

var (
	flagHost         = flag.String("host", "0.0.0.0:8080", "listening port and hostname")
	help             = flag.Bool("h", false, "show this help")
	flagClientSecret = flag.String("client_secret", "", "app oauth2 client secret")
)

func usage() {
	fmt.Fprintf(os.Stderr, "\t likestoplaylist \n")
	flag.PrintDefaults()
	os.Exit(2)
}

var oauthToken string

func makeHandler(fn func(http.ResponseWriter, *http.Request, string)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if e, ok := recover().(error); ok {
				http.Error(w, e.Error(), http.StatusInternalServerError)
				return
			}
		}()
		title := r.URL.Path
		w.Header().Set("Server", idstring)
		fn(w, r, title)
	}
}

type oauth2Response struct {
	Token string `json:"access_token,omitempty"`
	Scope string
}

func getToken(authCode string) (tok string, err error) {
	resp, err := http.PostForm("https://api.soundcloud.com/oauth2/token", url.Values{
		"client_id":     {"eba287aa2c6cef9b2cf5aed6b73d3542"},
		"client_secret": {*flagClientSecret},
		"grant_type":    {"authorization_code"},
		"redirect_uri":  {"http://127.0.0.1:8080/callback"},
		"code":          {authCode},
	})
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	var jsonResp oauth2Response
	if err := json.Unmarshal(body, &jsonResp); err != nil {
		return "", fmt.Errorf("could not decode %q: %v", body, err)
	}
	tok = jsonResp.Token
	if tok == "" {
		return "", fmt.Errorf("No token in response: %q", body)
	}
	return tok, nil
}

func serveError(w http.ResponseWriter, response string, err error) {
	log.Printf("%v", err)
	w.Write([]byte(response))
}

func apiGet(url string) ([]byte, error) {
	resp, err := http.Get(apiURL+url+"?oauth_token=" + oauthToken)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return body, nil
}

type track struct {
	Kind string
	Id int64
}

func getFavorites() ([]track, error) {
	tracks := make([]track, 1)
	body, err := apiGet("/me/favorites")
	if err != nil {
		return nil, err
	}
	if err := json.Unmarshal(body, &tracks); err != nil {
		return nil, fmt.Errorf("could not decode %q: %v", body, err)
	}
	fmt.Printf("found %d tracks in favorites\n", len(tracks))
	for _,v := range tracks {
		if v.Kind != "track" {
			// TODO(mpl): remove from list? error out?
			fmt.Printf("but %#v is actually not a track\n", v)
		}
	}
	return tracks, nil
}

/*
type playlist struct {
	Id int64
	UserId int
}

{
  "kind": "playlist",
  "id": 405726,
  "created_at": "2010/11/02 09:24:50 +0000",
  "user_id": 3207,
  "duration": 154516,
  "sharing": "public",
  "tag_list": "",
  "permalink": "field-recordings",
  "track_count": 5,
  "streamable": true,
  "downloadable": true,
  "embeddable_by": "me",
  "purchase_url": null,
  "label_id": null,
  "type": "other",
  "playlist_type": "other",
  "ean": "",
  "description": "a couple of field recordings to test http://soundiverse.com.\r\n\r\nrecorded with the fire recorder: http://soundcloud.com/apps/fire",
  "genre": "",
  "release": "",
  "purchase_title": null,
  "label_name": "",
  "title": "Field Recordings",
  "release_year": null,
  "release_month": null,
  "release_day": null,
  "license": "all-rights-reserved",
  "uri": "http://api.soundcloud.com/playlists/405726",
  "permalink_url": "http://soundcloud.com/jwagener/sets/field-recordings",
  "artwork_url": "http://i1.sndcdn.com/artworks-000025801802-1msl1i-large.jpg?5e64f12",
  "user": {
    "id": 3207,
    "kind": "user",
    "permalink": "jwagener",
    "username": "Johannes Wagener",
    "uri": "http://api.soundcloud.com/users/3207",
    "permalink_url": "http://soundcloud.com/jwagener",
    "avatar_url": "http://i1.sndcdn.com/avatars-000014428549-3at7qc-large.jpg?5e64f12"
  },
  "tracks": [
    {
      "kind": "track",
      "id": 6621631,
      "created_at": "2010/11/02 09:08:43 +0000",
      "user_id": 3207,
      "duration": 27099,
      "commentable": true,
      "state": "finished",
      "original_content_size": 2382624,
      "sharing": "public",
      "tag_list": "Fieldrecording geo:lat=52.527544 geo:lon=13.402905",
      "permalink": "coffee-machine",
      "streamable": true,
      "embeddable_by": "all",
      "downloadable": false,
      "purchase_url": null,
      "label_id": null,
      "purchase_title": null,
      "genre": "",
      "title": "coffee machine",
      "description": "",
      "label_name": "",
      "release": "",
      "track_type": "",
      "key_signature": "",
      "isrc": "",
      "video_url": null,
      "bpm": null,
      "release_year": null,
      "release_month": null,
      "release_day": null,
      "original_format": "wav",

func createNewPlaylist(likes []track) error {

}
*/

func handleCallback(w http.ResponseWriter, r *http.Request) {
	authCode := r.FormValue("code")
	if authCode == "" {
		serveError(w, "no auth code in callback", fmt.Errorf("no auth code in callback"))
		return
	}
	tok, err := getToken(authCode)
	if err != nil {
		serveError(w, "token exchange error", fmt.Errorf("token exchange error: %v", err))
		return
	}
	println("GOT TOKEN: " + tok)
	oauthToken = tok

	body, err := apiGet("/me")
	if err != nil {
		serveError(w, "error getting me", fmt.Errorf("error getting me: %v", err))
		return
	}

	println(string(body))

	if _, err := getFavorites(); err != nil {
		serveError(w, "error getting favorites", fmt.Errorf("error getting favorites: %v", err))
		return
	}

	w.Write([]byte("ALL GOOD"))
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if *help {
		usage()
	}
	if *flagClientSecret == "" {
		usage()
	}

	nargs := flag.NArg()
	if nargs > 0 {
		usage()
	}

	http.Handle("/", makeHandler(func(w http.ResponseWriter, r *http.Request, title string) {
		println(title)
		w.Write([]byte("OK ALRIGHT"))
		return
	}))
	http.Handle("/callback", makeHandler(func(w http.ResponseWriter, r *http.Request, title string) {
		println(title)
		handleCallback(w, r)
		return
	}))
	errc := make(chan error)
	go func() {
		errc <- http.ListenAndServe(*flagHost, nil)
	}()
	if err := exec.Command("xdg-open", "https://soundcloud.com/connect/?client_id=eba287aa2c6cef9b2cf5aed6b73d3542&redirect_uri=http://127.0.0.1:8080/callback&response_type=code&scope=non-expiring").Run(); err != nil {
		log.Fatal(err)
	}
	log.Fatal(<-errc)
}

/*
https://soundcloud.com/connect/?client_id=eba287aa2c6cef9b2cf5aed6b73d3542&redirect_uri=http://127.0.0.1:8080/callback&response_type=code&scope=non-expiring

code=52a31aaa7ef5dc1f4cee4ebe0eeaef92

curl -F 'client_id=eba287aa2c6cef9b2cf5aed6b73d3542' -F 'grant_type=authorization_code' -F 'redirect_uri=http://127.0.0.1:8080/callback' -F 'code=52a31aaa7ef5dc1f4cee4ebe0eeaef92' https://api.soundcloud.com/oauth2/token

*/
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package callgraph

import "code.google.com/p/go.tools/go/ssa"

// This file provides various utilities over call graphs, such as
// visitation and path search.

// CalleesOf returns a new set containing all direct callees of the
// caller node.
//
func CalleesOf(caller *Node) map[*Node]bool {
	callees := make(map[*Node]bool)
	for _, e := range caller.Out {
		callees[e.Callee] = true
	}
	return callees
}

// GraphVisitEdges visits all the edges in graph g in depth-first order.
// The edge function is called for each edge in postorder.  If it
// returns non-nil, visitation stops and GraphVisitEdges returns that
// value.
//
func GraphVisitEdges(g *Graph, edge func(*Edge) error) error {
	seen := make(map[*Node]bool)
	var visit func(n *Node) error
	visit = func(n *Node) error {
		if !seen[n] {
			seen[n] = true
			for _, e := range n.Out {
				if err := visit(e.Callee); err != nil {
					return err
				}
				if err := edge(e); err != nil {
					return err
				}
			}
		}
		return nil
	}
	for _, n := range g.Nodes {
		if err := visit(n); err != nil {
			return err
		}
	}
	return nil
}

// PathSearch finds an arbitrary path starting at node start and
// ending at some node for which isEnd() returns true.  On success,
// PathSearch returns the path as an ordered list of edges; on
// failure, it returns nil.
//
func PathSearch(start *Node, isEnd func(*Node) bool) []*Edge {
	stack := make([]*Edge, 0, 32)
	seen := make(map[*Node]bool)
	var search func(n *Node) []*Edge
	search = func(n *Node) []*Edge {
		if !seen[n] {
			seen[n] = true
			if isEnd(n) {
				return stack
			}
			for _, e := range n.Out {
				stack = append(stack, e) // push
				if found := search(e.Callee); found != nil {
					return found
				}
				stack = stack[:len(stack)-1] // pop
			}
		}
		return nil
	}
	return search(start)
}

// DeleteSyntheticNodes removes from call graph g all nodes for
// synthetic functions (except g.Root and package initializers),
// preserving the topology.
func (g *Graph) DeleteSyntheticNodes() {
	// TODO(adonovan): opt: this step results in duplicate
	// edges---approx 10% of the total.  I suspect this is due to
	// some interface method calls dispatching to both (C).f and
	// (*C).f where the latter is a wrapper.
	for fn, cgn := range g.Nodes {
		if cgn == g.Root || fn.Synthetic == "" || isInit(cgn.Func) {
			continue // keep
		}
		for _, eIn := range cgn.In {
			for _, eOut := range cgn.Out {
				AddEdge(eIn.Caller, eIn.Site, eOut.Callee)
			}
		}
		g.DeleteNode(cgn)
	}
}

func isInit(fn *ssa.Function) bool {
	return fn.Pkg != nil && fn.Pkg.Func("init") == fn
}

// DeleteNode removes node n and its edges from the graph g.
// (NB: not efficient for batch deletion.)
func (g *Graph) DeleteNode(n *Node) {
	n.deleteIns()
	n.deleteOuts()
	delete(g.Nodes, n.Func)
}

// deleteIns deletes all incoming edges to n.
func (n *Node) deleteIns() {
	for _, e := range n.In {
		removeOutEdge(e)
	}
	n.In = nil
}

// deleteOuts deletes all outgoing edges from n.
func (n *Node) deleteOuts() {
	for _, e := range n.Out {
		removeInEdge(e)
	}
	n.Out = nil
}

// removeOutEdge removes edge.Caller's outgoing edge 'edge'.
func removeOutEdge(edge *Edge) {
	caller := edge.Caller
	n := len(caller.Out)
	for i, e := range caller.Out {
		if e == edge {
			// Replace it with the final element and shrink the slice.
			caller.Out[i] = caller.Out[n-1]
			caller.Out[n-1] = nil // aid GC
			caller.Out = caller.Out[:n-1]
			return
		}
	}
	panic("edge not found: " + edge.String())
}

// removeInEdge removes edge.Callee's incoming edge 'edge'.
func removeInEdge(edge *Edge) {
	caller := edge.Callee
	n := len(caller.In)
	for i, e := range caller.In {
		if e == edge {
			// Replace it with the final element and shrink the slice.
			caller.In[i] = caller.In[n-1]
			caller.In[n-1] = nil // aid GC
			caller.In = caller.In[:n-1]
			return
		}
	}
	panic("edge not found: " + edge.String())
}
<commit_msg>go/callgraph: fix asymptote trap in DeleteSyntheticNodes.<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package callgraph

import "code.google.com/p/go.tools/go/ssa"

// This file provides various utilities over call graphs, such as
// visitation and path search.

// CalleesOf returns a new set containing all direct callees of the
// caller node.
//
func CalleesOf(caller *Node) map[*Node]bool {
	callees := make(map[*Node]bool)
	for _, e := range caller.Out {
		callees[e.Callee] = true
	}
	return callees
}

// GraphVisitEdges visits all the edges in graph g in depth-first order.
// The edge function is called for each edge in postorder.  If it
// returns non-nil, visitation stops and GraphVisitEdges returns that
// value.
//
func GraphVisitEdges(g *Graph, edge func(*Edge) error) error {
	seen := make(map[*Node]bool)
	var visit func(n *Node) error
	visit = func(n *Node) error {
		if !seen[n] {
			seen[n] = true
			for _, e := range n.Out {
				if err := visit(e.Callee); err != nil {
					return err
				}
				if err := edge(e); err != nil {
					return err
				}
			}
		}
		return nil
	}
	for _, n := range g.Nodes {
		if err := visit(n); err != nil {
			return err
		}
	}
	return nil
}

// PathSearch finds an arbitrary path starting at node start and
// ending at some node for which isEnd() returns true.  On success,
// PathSearch returns the path as an ordered list of edges; on
// failure, it returns nil.
//
func PathSearch(start *Node, isEnd func(*Node) bool) []*Edge {
	stack := make([]*Edge, 0, 32)
	seen := make(map[*Node]bool)
	var search func(n *Node) []*Edge
	search = func(n *Node) []*Edge {
		if !seen[n] {
			seen[n] = true
			if isEnd(n) {
				return stack
			}
			for _, e := range n.Out {
				stack = append(stack, e) // push
				if found := search(e.Callee); found != nil {
					return found
				}
				stack = stack[:len(stack)-1] // pop
			}
		}
		return nil
	}
	return search(start)
}

// DeleteSyntheticNodes removes from call graph g all nodes for
// synthetic functions (except g.Root and package initializers),
// preserving the topology.  In effect, calls to synthetic wrappers
// are "inlined".
//
func (g *Graph) DeleteSyntheticNodes() {
	// Measurements on the standard library and go.tools show that
	// resulting graph has ~15% fewer nodes and 4-8% fewer edges
	// than the input.
	//
	// Inlining a wrapper of in-degree m, out-degree n adds m*n
	// and removes m+n edges.  Since most wrappers are monomorphic
	// (n=1) this results in a slight reduction.  Polymorphic
	// wrappers (n>1), e.g. from embedding an interface value
	// inside a struct to satisfy some interface, cause an
	// increase in the graph, but they seem to be uncommon.

	// Hash all existing edges to avoid creating duplicates.
	edges := make(map[Edge]bool)
	for _, cgn := range g.Nodes {
		for _, e := range cgn.Out {
			edges[*e] = true
		}
	}
	for fn, cgn := range g.Nodes {
		if cgn == g.Root || fn.Synthetic == "" || isInit(cgn.Func) {
			continue // keep
		}
		for _, eIn := range cgn.In {
			for _, eOut := range cgn.Out {
				newEdge := Edge{eIn.Caller, eIn.Site, eOut.Callee}
				if edges[newEdge] {
					continue // don't add duplicate
				}
				AddEdge(eIn.Caller, eIn.Site, eOut.Callee)
				edges[newEdge] = true
			}
		}
		g.DeleteNode(cgn)
	}
}

func isInit(fn *ssa.Function) bool {
	return fn.Pkg != nil && fn.Pkg.Func("init") == fn
}

// DeleteNode removes node n and its edges from the graph g.
// (NB: not efficient for batch deletion.)
func (g *Graph) DeleteNode(n *Node) {
	n.deleteIns()
	n.deleteOuts()
	delete(g.Nodes, n.Func)
}

// deleteIns deletes all incoming edges to n.
func (n *Node) deleteIns() {
	for _, e := range n.In {
		removeOutEdge(e)
	}
	n.In = nil
}

// deleteOuts deletes all outgoing edges from n.
func (n *Node) deleteOuts() {
	for _, e := range n.Out {
		removeInEdge(e)
	}
	n.Out = nil
}

// removeOutEdge removes edge.Caller's outgoing edge 'edge'.
func removeOutEdge(edge *Edge) {
	caller := edge.Caller
	n := len(caller.Out)
	for i, e := range caller.Out {
		if e == edge {
			// Replace it with the final element and shrink the slice.
			caller.Out[i] = caller.Out[n-1]
			caller.Out[n-1] = nil // aid GC
			caller.Out = caller.Out[:n-1]
			return
		}
	}
	panic("edge not found: " + edge.String())
}

// removeInEdge removes edge.Callee's incoming edge 'edge'.
func removeInEdge(edge *Edge) {
	caller := edge.Callee
	n := len(caller.In)
	for i, e := range caller.In {
		if e == edge {
			// Replace it with the final element and shrink the slice.
			caller.In[i] = caller.In[n-1]
			caller.In[n-1] = nil // aid GC
			caller.In = caller.In[:n-1]
			return
		}
	}
	panic("edge not found: " + edge.String())
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"log"
	"math/big"
	"sort"
	"strconv"
	"strings"
)

func ToIntOrPanic(content string) int {
	v, err := strconv.ParseInt(content, 10, 64)
	if err != nil {
		log.Fatalf("Cannot parse %v", err.Error())
	}
	return int(v)
}

// Trim and parse Int.
func TrInt(content string) int {
	return ToIntOrPanic(strings.TrimSpace(content))
}

func ToString(v int) string {
	return fmt.Sprintf("%d", v)
}

func ToStringBig(v *big.Int) string {
	return fmt.Sprintf("%v", v)
}

func ParseMatrix(content string) [][]string {
	rows := SplitByNewline(content)
	matrix := make([][]string, len(rows))

	for i, row := range rows {
		matrix[i] = SplitByWhitespaceTrim(row)
	}
	return matrix
}

func ParseIntArray(array []string) []int {
	result := make([]int, len(array))
	for i, str := range array {
		v, err := strconv.ParseInt(str, 10, 64)
		if err != nil {
			log.Fatalf("Cannot parse value '%s", str)
			return []int{}
		}
		result[i] = int(v)
	}
	return result
}

func SplitByCommaTrim(content string) []string {
	values := strings.FieldsFunc(content, func(c rune) bool { return c == ',' })
	result := make([]string, len(values))
	for i, v := range values {
		result[i] = strings.TrimSpace(v)
	}
	return result
}

func SplitTrim(content string, char rune) []string {
	values := strings.FieldsFunc(content, func(c rune) bool { return c == char })
	result := make([]string, len(values))
	for i, v := range values {
		result[i] = strings.TrimSpace(v)
	}
	return result
}

func SplitByWhitespaceTrim(content string) []string {
	values := strings.Fields(content)
	result := make([]string, len(values))
	for i, v := range values {
		result[i] = strings.TrimSpace(v)
	}
	return result
}

func SplitByNewline(content string) []string {
	values := strings.FieldsFunc(content, func(c rune) bool { return c == '\n' })
	result := make([]string, len(values))
	for i, v := range values {
		result[i] = strings.TrimSpace(v)
	}
	return result
}

func SortChars(str string) string {
	s := strings.Split(str, "")
	sort.Strings(s)
	return strings.Join(s, "")
}
<commit_msg>Use scanner to read in lines from string.<commit_after>package common

import (
	"bufio"
	"fmt"
	"log"
	"math/big"
	"sort"
	"strconv"
	"strings"
)

func ToIntOrPanic(content string) int {
	v, err := strconv.ParseInt(content, 10, 64)
	if err != nil {
		log.Fatalf("Cannot parse %v", err.Error())
	}
	return int(v)
}

// Trim and parse Int.
func TrInt(content string) int {
	return ToIntOrPanic(strings.TrimSpace(content))
}

func ToString(v int) string {
	return fmt.Sprintf("%d", v)
}

func ToStringBig(v *big.Int) string {
	return fmt.Sprintf("%v", v)
}

func ParseMatrix(content string) [][]string {
	rows := SplitByNewline(content)
	matrix := make([][]string, len(rows))

	for i, row := range rows {
		matrix[i] = SplitByWhitespaceTrim(row)
	}
	return matrix
}

func ParseIntArray(array []string) []int {
	result := make([]int, len(array))
	for i, str := range array {
		v, err := strconv.ParseInt(str, 10, 64)
		if err != nil {
			log.Fatalf("Cannot parse value '%s", str)
			return []int{}
		}
		result[i] = int(v)
	}
	return result
}

func SplitByCommaTrim(content string) []string {
	values := strings.FieldsFunc(content, func(c rune) bool { return c == ',' })
	result := make([]string, len(values))
	for i, v := range values {
		result[i] = strings.TrimSpace(v)
	}
	return result
}

func SplitTrim(content string, char rune) []string {
	values := strings.FieldsFunc(content, func(c rune) bool { return c == char })
	result := make([]string, len(values))
	for i, v := range values {
		result[i] = strings.TrimSpace(v)
	}
	return result
}

func SplitByWhitespaceTrim(content string) []string {
	values := strings.Fields(content)
	result := make([]string, len(values))
	for i, v := range values {
		result[i] = strings.TrimSpace(v)
	}
	return result
}

func SplitByNewline(content string) []string {
	var result []string
	sc := bufio.NewScanner(strings.NewReader(content))
	for sc.Scan() {
		result = append(result, sc.Text())
	}
	return result
}

func SortChars(str string) string {
	s := strings.Split(str, "")
	sort.Strings(s)
	return strings.Join(s, "")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/streadway/amqp"
)

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
		panic(fmt.Sprintf("%s: %s", msg, err))
	}
}

func main() {
	conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/")
	failOnError(err, "Failed to connect to RabbitMQ")
	defer conn.Close()

	ch, err := conn.Channel()
	failOnError(err, "Failed to open a channel")
	defer ch.Close()

	err = ch.ExchangeDeclare(
		"logs_topic", // name
		"topic",      // type
		true,         // durable
		false,        // auto-deleted
		false,        // internal
		false,        // no-wait
		nil,          // arguments
	)
	failOnError(err, "Failed to declare an exchange")

	body := bodyFrom(os.Args)
	err = ch.Publish(
		"logs_topic",          // exchange
		severityFrom(os.Args), // routing key
		false, // mandatory
		false, // immediate
		amqp.Publishing{
			ContentType: "text/plain",
			Body:        []byte(body),
		})
	failOnError(err, "Failed to publish a message")

	log.Printf(" [x] Sent %s", body)
}

func bodyFrom(args []string) string {
	var s string
	if (len(args) < 3) || os.Args[2] == "" {
		s = "hello"
	} else {
		s = strings.Join(args[2:], " ")
	}
	return s
}

func severityFrom(args []string) string {
	var s string
	if (len(args) < 2) || os.Args[1] == "" {
		s = "anonymous.info"
	} else {
		s = os.Args[1]
	}
	return s
}
<commit_msg>emit_log_topic.go: Removed unreachable panic()<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/streadway/amqp"
)

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
	}
}

func main() {
	conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/")
	failOnError(err, "Failed to connect to RabbitMQ")
	defer conn.Close()

	ch, err := conn.Channel()
	failOnError(err, "Failed to open a channel")
	defer ch.Close()

	err = ch.ExchangeDeclare(
		"logs_topic", // name
		"topic",      // type
		true,         // durable
		false,        // auto-deleted
		false,        // internal
		false,        // no-wait
		nil,          // arguments
	)
	failOnError(err, "Failed to declare an exchange")

	body := bodyFrom(os.Args)
	err = ch.Publish(
		"logs_topic",          // exchange
		severityFrom(os.Args), // routing key
		false, // mandatory
		false, // immediate
		amqp.Publishing{
			ContentType: "text/plain",
			Body:        []byte(body),
		})
	failOnError(err, "Failed to publish a message")

	log.Printf(" [x] Sent %s", body)
}

func bodyFrom(args []string) string {
	var s string
	if (len(args) < 3) || os.Args[2] == "" {
		s = "hello"
	} else {
		s = strings.Join(args[2:], " ")
	}
	return s
}

func severityFrom(args []string) string {
	var s string
	if (len(args) < 2) || os.Args[1] == "" {
		s = "anonymous.info"
	} else {
		s = os.Args[1]
	}
	return s
}
<|endoftext|>


<commit_before>package fakes

import (
	"context"
	"fmt"
	"testing"

	"github.com/bazelbuild/remote-apis-sdks/go/pkg/client"
	"github.com/bazelbuild/remote-apis-sdks/go/pkg/digest"
	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	regrpc "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
	repb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
	oppb "google.golang.org/genproto/googleapis/longrunning"
)

// Exec implements the complete RE execution interface for a single execution, returning a fixed
// result or an error.
type Exec struct {
	// Execution will check the action cache first, and update the action cache upon completion.
	ac *ActionCache
	// The action will be fetched from the CAS at start of execution, and outputs will be put in the
	// CAS upon execution completion.
	cas *CAS
	// Fake result of an execution.
	// The returned completed result, if any.
	ActionResult *repb.ActionResult
	// Returned completed execution status, if not Ok.
	Status *status.Status
	// Whether action was fake-fetched from the action cache upon execution (simulates a race between
	// two executions).
	Cached bool
	// Any blobs that will be put in the CAS after the fake execution completes.
	OutputBlobs [][]byte
	// Number of Execute calls.
	numExecCalls int
	// Used for errors.
	t testing.TB
	// The digest of the fake action.
	adg digest.Digest
}

// NewExec returns a new empty Exec.
func NewExec(t testing.TB, ac *ActionCache, cas *CAS) *Exec {
	c := &Exec{t: t, ac: ac, cas: cas}
	c.Clear()
	return c
}

// Clear removes all preset results from the fake.
func (s *Exec) Clear() {
	s.ActionResult = nil
	s.Status = nil
	s.Cached = false
	s.OutputBlobs = nil
	s.numExecCalls = 0
}

// ExecuteCalls returns the total number of Execute calls.
func (s *Exec) ExecuteCalls() int {
	return s.numExecCalls
}

func (s *Exec) fakeExecution(dg digest.Digest, skipCacheLookup bool) (*oppb.Operation, error) {
	ar := s.ActionResult
	st := s.Status
	cached := s.Cached
	// Check action cache first, unless instructed not to.
	if !skipCacheLookup {
		cr := s.ac.Get(dg)
		if cr != nil {
			ar = cr
			st = nil
			cached = true
		}
	}
	// Fetch action from CAS.
	blob, ok := s.cas.Get(dg)
	if !ok {
		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("action blob with digest %v not in the cas", dg))
	}
	apb := &repb.Action{}
	if err := proto.Unmarshal(blob, apb); err != nil {
		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("error unmarshalling %v as Action", blob))
	}
	if !apb.DoNotCache {
		s.ac.Put(dg, ar)
	}
	for _, out := range s.OutputBlobs {
		s.cas.Put(out)
	}
	execResp := &repb.ExecuteResponse{
		Result:       ar,
		Status:       st.Proto(),
		CachedResult: cached,
	}
	any, err := ptypes.MarshalAny(execResp)
	if err != nil {
		return nil, err
	}
	return &oppb.Operation{
		Name:   "fake",
		Done:   true,
		Result: &oppb.Operation_Response{Response: any},
	}, nil
}

// GetCapabilities returns the fake capabilities.
func (c *Exec) GetCapabilities(ctx context.Context, req *repb.GetCapabilitiesRequest) (res *repb.ServerCapabilities, err error) {
	dgFn := digest.GetDigestFunction()
	res = &repb.ServerCapabilities{
		ExecutionCapabilities: &repb.ExecutionCapabilities{
			DigestFunction: dgFn,
			ExecEnabled:    true,
		},
		CacheCapabilities: &repb.CacheCapabilities{
			DigestFunctions: []repb.DigestFunction_Value{dgFn},
			ActionCacheUpdateCapabilities: &repb.ActionCacheUpdateCapabilities{
				UpdateEnabled: true,
			},
			MaxBatchTotalSizeBytes:      client.DefaultMaxBatchSize,
			SymlinkAbsolutePathStrategy: repb.SymlinkAbsolutePathStrategy_DISALLOWED,
		},
	}
	return res, nil
}

// Execute returns the saved result ActionResult, or a Status. It also puts it in the action cache
// unless the execute request specified
func (s *Exec) Execute(req *repb.ExecuteRequest, stream regrpc.Execution_ExecuteServer) (err error) {
	dg, err := digest.NewFromProto(req.ActionDigest)
	if err != nil {
		return status.Error(codes.InvalidArgument, fmt.Sprintf("invalid digest received: %v", req.ActionDigest))
	}
	if dg != s.adg {
		s.t.Errorf("unexpected action digest received by fake: expected %v, got %v", s.adg, dg)
		return status.Error(codes.InvalidArgument, fmt.Sprintf("unexpected digest received: %v", req.ActionDigest))
	}
	if op, err := s.fakeExecution(dg, req.SkipCacheLookup); err != nil {
		return err
	} else if err = stream.Send(op); err != nil {
		return err
	}
	s.numExecCalls++
	return nil
}

// WaitExecution is not implemented on this fake.
func (s *Exec) WaitExecution(req *repb.WaitExecutionRequest, stream regrpc.Execution_WaitExecutionServer) (err error) {
	return status.Error(codes.Unimplemented, "method WaitExecution not implemented by test fake")
}
<commit_msg>Make numExecCalls increment atomic (#383)<commit_after>package fakes

import (
	"context"
	"fmt"
	"sync/atomic"
	"testing"

	"github.com/bazelbuild/remote-apis-sdks/go/pkg/client"
	"github.com/bazelbuild/remote-apis-sdks/go/pkg/digest"
	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	regrpc "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
	repb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
	oppb "google.golang.org/genproto/googleapis/longrunning"
)

// Exec implements the complete RE execution interface for a single execution, returning a fixed
// result or an error.
type Exec struct {
	// Execution will check the action cache first, and update the action cache upon completion.
	ac *ActionCache
	// The action will be fetched from the CAS at start of execution, and outputs will be put in the
	// CAS upon execution completion.
	cas *CAS
	// Fake result of an execution.
	// The returned completed result, if any.
	ActionResult *repb.ActionResult
	// Returned completed execution status, if not Ok.
	Status *status.Status
	// Whether action was fake-fetched from the action cache upon execution (simulates a race between
	// two executions).
	Cached bool
	// Any blobs that will be put in the CAS after the fake execution completes.
	OutputBlobs [][]byte
	// Number of Execute calls.
	numExecCalls int32
	// Used for errors.
	t testing.TB
	// The digest of the fake action.
	adg digest.Digest
}

// NewExec returns a new empty Exec.
func NewExec(t testing.TB, ac *ActionCache, cas *CAS) *Exec {
	c := &Exec{t: t, ac: ac, cas: cas}
	c.Clear()
	return c
}

// Clear removes all preset results from the fake.
func (s *Exec) Clear() {
	s.ActionResult = nil
	s.Status = nil
	s.Cached = false
	s.OutputBlobs = nil
	atomic.StoreInt32(&s.numExecCalls, 0)
}

// ExecuteCalls returns the total number of Execute calls.
func (s *Exec) ExecuteCalls() int {
	return int(atomic.LoadInt32(&s.numExecCalls))
}

func (s *Exec) fakeExecution(dg digest.Digest, skipCacheLookup bool) (*oppb.Operation, error) {
	ar := s.ActionResult
	st := s.Status
	cached := s.Cached
	// Check action cache first, unless instructed not to.
	if !skipCacheLookup {
		cr := s.ac.Get(dg)
		if cr != nil {
			ar = cr
			st = nil
			cached = true
		}
	}
	// Fetch action from CAS.
	blob, ok := s.cas.Get(dg)
	if !ok {
		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("action blob with digest %v not in the cas", dg))
	}
	apb := &repb.Action{}
	if err := proto.Unmarshal(blob, apb); err != nil {
		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("error unmarshalling %v as Action", blob))
	}
	if !apb.DoNotCache {
		s.ac.Put(dg, ar)
	}
	for _, out := range s.OutputBlobs {
		s.cas.Put(out)
	}
	execResp := &repb.ExecuteResponse{
		Result:       ar,
		Status:       st.Proto(),
		CachedResult: cached,
	}
	any, err := ptypes.MarshalAny(execResp)
	if err != nil {
		return nil, err
	}
	return &oppb.Operation{
		Name:   "fake",
		Done:   true,
		Result: &oppb.Operation_Response{Response: any},
	}, nil
}

// GetCapabilities returns the fake capabilities.
func (c *Exec) GetCapabilities(ctx context.Context, req *repb.GetCapabilitiesRequest) (res *repb.ServerCapabilities, err error) {
	dgFn := digest.GetDigestFunction()
	res = &repb.ServerCapabilities{
		ExecutionCapabilities: &repb.ExecutionCapabilities{
			DigestFunction: dgFn,
			ExecEnabled:    true,
		},
		CacheCapabilities: &repb.CacheCapabilities{
			DigestFunctions: []repb.DigestFunction_Value{dgFn},
			ActionCacheUpdateCapabilities: &repb.ActionCacheUpdateCapabilities{
				UpdateEnabled: true,
			},
			MaxBatchTotalSizeBytes:      client.DefaultMaxBatchSize,
			SymlinkAbsolutePathStrategy: repb.SymlinkAbsolutePathStrategy_DISALLOWED,
		},
	}
	return res, nil
}

// Execute returns the saved result ActionResult, or a Status. It also puts it in the action cache
// unless the execute request specified
func (s *Exec) Execute(req *repb.ExecuteRequest, stream regrpc.Execution_ExecuteServer) (err error) {
	dg, err := digest.NewFromProto(req.ActionDigest)
	if err != nil {
		return status.Error(codes.InvalidArgument, fmt.Sprintf("invalid digest received: %v", req.ActionDigest))
	}
	if dg != s.adg {
		s.t.Errorf("unexpected action digest received by fake: expected %v, got %v", s.adg, dg)
		return status.Error(codes.InvalidArgument, fmt.Sprintf("unexpected digest received: %v", req.ActionDigest))
	}
	if op, err := s.fakeExecution(dg, req.SkipCacheLookup); err != nil {
		return err
	} else if err = stream.Send(op); err != nil {
		return err
	}
	atomic.AddInt32(&s.numExecCalls, 1)
	return nil
}

// WaitExecution is not implemented on this fake.
func (s *Exec) WaitExecution(req *repb.WaitExecutionRequest, stream regrpc.Execution_WaitExecutionServer) (err error) {
	return status.Error(codes.Unimplemented, "method WaitExecution not implemented by test fake")
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"os/exec"
	"strings"
	"time"

	"github.com/brotherlogic/goserver/utils"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	pbb "github.com/brotherlogic/buildserver/proto"
	pbfc "github.com/brotherlogic/filecopier/proto"
	pb "github.com/brotherlogic/gobuildslave/proto"
	pbvt "github.com/brotherlogic/versiontracker/proto"
)

const (
	pendWait = time.Minute
)

var (
	ackQueueLen = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "gobuildslave_ackqueuelen",
		Help: "The size of the ack queue",
	})
)

func (s *Server) procAcks() {
	for job := range s.ackChan {
		ackQueueLen.Set(float64(len(s.ackChan)))
		ctx, cancel := utils.ManualContext("gobuildslaveack", time.Minute)
		conn, err := s.FDialSpecificServer(ctx, "versiontracker", s.Registry.GetIdentifier())
		if err != nil {
			s.DLog(ctx, fmt.Sprintf("Dial error: (%v), %v\n", job.GetJob(), err))
			s.ackChan <- job
		} else {

			client := pbvt.NewVersionTrackerServiceClient(conn)
			_, err = client.NewJob(ctx, &pbvt.NewJobRequest{Version: &pbb.Version{Job: job.GetJob()}})
			if err != nil {
				s.ackChan <- job
			}
			conn.Close()
		}
		cancel()

		// Don't rush the system
		time.Sleep(time.Second * 10)
	}
}

func (s *Server) runTransition(ctx context.Context, job *pb.JobAssignment) {
	s.DLog(ctx, fmt.Sprintf("TRANS: %v\n", job))
	startState := job.State
	job.LastUpdateTime = time.Now().Unix()
	switch job.State {
	case pb.State_WARMUP:
		s.ackChan <- job
		job.State = pb.State_ACKNOWLEDGED
	case pb.State_ACKNOWLEDGED:
		key := s.scheduleBuild(ctx, job)
		job.SubState = fmt.Sprintf("SCHED: %v @ %v", key, time.Now())
		if !job.Job.Bootstrap {
			if key != "" {
				job.State = pb.State_BUILT
				job.RunningVersion = key
			} else {
				// Bootstrap this job since we don't have an initial version
				if job.Job.PartialBootstrap {
					job.Job.Bootstrap = true
				}
			}
		} else {
			job.CommandKey = key
			job.State = pb.State_BUILDING
		}
	case pb.State_BUILDING:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILD(%v): %v", job.CommandKey, s.scheduler.getState(job.CommandKey))
		s.stateMutex.Unlock()
		s.scheduler.wait(job.CommandKey)
		job.State = pb.State_BUILT
	case pb.State_BUILT:
		job.SubState = "Getting Output"
		output, _ := s.scheduler.getOutput(job.CommandKey)
		job.SubState = "Entering Lock"
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILT(%v): (%v): %v", job.CommandKey, len(output), output)
		s.stateMutex.Unlock()
		if job.Job.Bootstrap && len(output) > 0 {
			if job.BuildFail == 5 {
				job.SubState = "Sending Report"
				s.deliverCrashReport(ctx, job, output)
				job.BuildFail = 0
			}
			job.BuildFail++
			job.State = pb.State_DIED
		} else {
			job.State = pb.State_VERSION_CHECK
			// Don't check version on pb jobs
			if job.GetJob().GetPartialBootstrap() {
				job.BuildFail = 0
				job.SubState = "Scheduling A Run"
				key := s.scheduleRun(job)
				job.CommandKey = key
				job.StartTime = time.Now().Unix()
				job.State = pb.State_PENDING
				if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
					s.pendingMap[time.Now().Weekday()] = make(map[string]int)
				}
				s.pendingMap[time.Now().Weekday()][job.Job.Name]++
			}
		}
		job.SubState = "Out of case"
	case pb.State_VERSION_CHECK:
		s.doCopy(job)
		s.loadCurrentVersions()
		version, err := s.getLatestVersion(ctx, job.GetJob().Name, job.GetJob().GetGoPath())
		if err != nil {
			s.Log(fmt.Sprintf("Error getting version: %v", err))
			break
		}

		res, err := exec.Command("md5sum", fmt.Sprintf("/home/simon/gobuild/bin/%v", job.GetJob().GetName())).Output()
		if err != nil {
			s.Log(fmt.Sprintf("Error reading md5sum: %v", err))
		}
		elems := strings.Fields(string(res))
		s.versionsMutex.Lock()
		defer s.versionsMutex.Unlock()
		if elems[0] != version.GetVersion() {
			s.Log(fmt.Sprintf("Bad version found in the wild", fmt.Sprintf("Bad version on %v for %v -> %v vs %v", s.Registry.Identifier, job.GetJob().GetName(), elems[0], s.versions[job.GetJob().GetName()].Version)))
			job.SubState = fmt.Sprintf("Dealing With Version Mismatch: %v", job.BuildFail)

			// Don't let the job sit here
			job.BuildFail++
			if job.BuildFail > 10 {
				job.State = pb.State_ACKNOWLEDGED

			}
			break
		}

		if time.Unix(version.GetVersionDate(), 0).Sub(time.Unix(s.versions[job.GetJob().GetName()].GetVersionDate(), 0)) < time.Hour*24 {
			job.BuildFail = 0
			job.SubState = "Scheduling of the Run"
			key := s.scheduleRun(job)
			job.CommandKey = key
			job.StartTime = time.Now().Unix()
			job.State = pb.State_PENDING
			if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
				s.pendingMap[time.Now().Weekday()] = make(map[string]int)
			}
			s.pendingMap[time.Now().Weekday()][job.Job.Name]++
		}

	case pb.State_PENDING:
		if job.Job.PartialBootstrap && job.Job.Bootstrap {
			job.Job.Bootstrap = false
		}
		s.stateMutex.Lock()
		out, _ := s.scheduler.getOutput(job.CommandKey)
		s.stateMap[job.Job.Name] = fmt.Sprintf("OUTPUT = %v", out)
		s.stateMutex.Unlock()
		if time.Now().Add(-time.Minute).Unix() > job.StartTime {
			var err error
			if job.Job.Name == "discovery" {
				err = s.runOnChange()
			}
			code := status.Convert(err).Code()

			//Unavailable allows the job to die here
			if code == codes.OK || code == codes.Unavailable {
				// Validate that the job is alive
				if !s.isJobAlive(ctx, job) {
					s.Log(fmt.Sprintf("Job %v is not alive", job))
				}
				job.State = pb.State_RUNNING
			} else {
				s.Log(fmt.Sprintf("Cannot reregister: %v", err))
			}
		}
	case pb.State_RUNNING:
		output, errout := s.scheduler.getOutput(job.CommandKey)
		output2, _ := s.scheduler.getErrOutput(job.CommandKey)
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("ROUTPUT = %v, %v", output, s.scheduler.getStatus(job.CommandKey))
		job.Status = s.scheduler.getStatus(job.CommandKey)
		s.stateMutex.Unlock()
		if len(job.CommandKey) > 0 {
			s.scheduler.wait(job.CommandKey)
			s.stateMap[job.Job.Name] = fmt.Sprintf("ONLOCk = (%v, %v)", job, output)
			s.stateMutex.Lock()
			s.stateMap[job.Job.Name] = fmt.Sprintf("COMPLETE = (%v, %v)", job, output)
			s.stateMutex.Unlock()
			s.deliverCrashReport(ctx, job, fmt.Sprintf("%v%v", output, output2))
			job.State = pb.State_DIED
		}

		if s.Registry != nil {
			entry, err := s.FFindSpecificServer(ctx, job.Job.Name, s.Registry.Identifier)
			if err != nil {
				if job.DiscoverCount > 30 {
					output2, errout2 := s.scheduler.getErrOutput(job.CommandKey)
					s.RaiseIssue("Cannot Discover Running Server", fmt.Sprintf("%v on %v is not discoverable, despite running (%v) the output says %v (%v), %v, %v", job.Job.Name, s.Registry.Identifier, err, output, errout, output2, errout2))
				}
				job.DiscoverCount++
				s.Log(fmt.Sprintf("Missing discover for %+v, %v -> %v", job, entry, err))
			} else {
				job.Port = entry.GetPort()
				job.DiscoverCount = 0
			}
		}

		// Restart this job if we need to
		if !job.Job.Bootstrap {
			if time.Now().Sub(time.Unix(job.LastVersionPull, 0)) > time.Minute*5 {
				version, err := s.getVersion(ctx, job.Job)
				job.LastVersionPull = time.Now().Unix()

				if err == nil && version.Version != job.RunningVersion {
					s.stateMutex.Lock()
					s.stateMap[job.Job.Name] = fmt.Sprintf("VERSION_MISMATCH = %v,%v", version, job.RunningVersion)
					s.stateMutex.Unlock()
					s.scheduler.killJob(job.CommandKey)
				}
			}
		}
	case pb.State_BRINK_OF_DEATH:
		if s.version.confirm(ctx, job.Job.Name) {
			s.scheduler.killJob(job.CommandKey)
			job.State = pb.State_ACKNOWLEDGED
		}
	case pb.State_DIED:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("DIED %v", job.CommandKey)
		s.stateMutex.Unlock()
		job.State = pb.State_ACKNOWLEDGED
	}

	if job.State != startState {
		job.LastTransitionTime = time.Now().Unix()
	}
}

type translator interface {
	build(job *pb.Job) *exec.Cmd
	run(job *pb.Job) *exec.Cmd
}

type checker interface {
	isAlive(ctx context.Context, job *pb.JobAssignment) bool
}

func (s *Server) getVersion(ctx context.Context, job *pb.Job) (*pbb.Version, error) {
	version, err := s.builder.build(ctx, job)
	if err != nil {
		return &pbb.Version{}, err
	}

	return version, nil
}

func updateJob(err error, job *pb.JobAssignment, resp *pbfc.CopyResponse) {
	if err == nil {
		job.QueuePos = resp.IndexInQueue
	}

}

// scheduleBuild builds out the job, returning the current version
func (s *Server) scheduleBuild(ctx context.Context, job *pb.JobAssignment) string {
	if job.Job.Bootstrap {
		c := s.translator.build(job.Job)
		// Block builds to prevent clashes
		return s.scheduler.Schedule(&rCommand{command: c, base: job.Job.Name, block: true})
	}

	val, err := s.builder.build(ctx, job.Job)
	if err != nil {
		s.DLog(ctx, fmt.Sprintf("BUILD Error: %v\n", err))
		return ""
	}
	return val.Version
}

func (s *Server) doCopy(job *pb.JobAssignment) {
	//Copy over any existing new versions
	key := s.scheduler.Schedule(&rCommand{command: exec.Command("mv", "$GOPATH/bin/"+job.GetJob().GetName()+".new", "$GOPATH/bin/"+job.GetJob().GetName()), base: job.GetJob().GetName()})
	s.scheduler.wait(key)
}

func (s *Server) scheduleRun(job *pb.JobAssignment) string {
	// Wait a while before starting the job JIC
	time.Sleep(time.Second * 2)
	c := s.translator.run(job.GetJob())
	return s.scheduler.Schedule(&rCommand{command: c, base: job.GetJob().GetName()})
}
<commit_msg>Include substate<commit_after>package main

import (
	"context"
	"fmt"
	"os/exec"
	"strings"
	"time"

	"github.com/brotherlogic/goserver/utils"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	pbb "github.com/brotherlogic/buildserver/proto"
	pbfc "github.com/brotherlogic/filecopier/proto"
	pb "github.com/brotherlogic/gobuildslave/proto"
	pbvt "github.com/brotherlogic/versiontracker/proto"
)

const (
	pendWait = time.Minute
)

var (
	ackQueueLen = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "gobuildslave_ackqueuelen",
		Help: "The size of the ack queue",
	})
)

func (s *Server) procAcks() {
	for job := range s.ackChan {
		ackQueueLen.Set(float64(len(s.ackChan)))
		ctx, cancel := utils.ManualContext("gobuildslaveack", time.Minute)
		conn, err := s.FDialSpecificServer(ctx, "versiontracker", s.Registry.GetIdentifier())
		if err != nil {
			s.DLog(ctx, fmt.Sprintf("Dial error: (%v), %v\n", job.GetJob(), err))
			s.ackChan <- job
		} else {

			client := pbvt.NewVersionTrackerServiceClient(conn)
			_, err = client.NewJob(ctx, &pbvt.NewJobRequest{Version: &pbb.Version{Job: job.GetJob()}})
			if err != nil {
				s.ackChan <- job
			}
			conn.Close()
		}
		cancel()

		// Don't rush the system
		time.Sleep(time.Second * 10)
	}
}

func (s *Server) runTransition(ctx context.Context, job *pb.JobAssignment) {
	s.DLog(ctx, fmt.Sprintf("TRANS: %v\n", job))
	startState := job.State
	job.LastUpdateTime = time.Now().Unix()
	switch job.State {
	case pb.State_WARMUP:
		s.ackChan <- job
		job.State = pb.State_ACKNOWLEDGED
	case pb.State_ACKNOWLEDGED:
		key := s.scheduleBuild(ctx, job)
		job.SubState = fmt.Sprintf("SCHED: %v @ %v", key, time.Now())
		if !job.Job.Bootstrap {
			if key != "" {
				job.State = pb.State_BUILT
				job.RunningVersion = key
			} else {
				// Bootstrap this job since we don't have an initial version
				if job.Job.PartialBootstrap {
					job.Job.Bootstrap = true
				}
			}
		} else {
			job.CommandKey = key
			job.State = pb.State_BUILDING
		}
	case pb.State_BUILDING:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILD(%v): %v", job.CommandKey, s.scheduler.getState(job.CommandKey))
		s.stateMutex.Unlock()
		s.scheduler.wait(job.CommandKey)
		job.State = pb.State_BUILT
	case pb.State_BUILT:
		job.SubState = "Getting Output"
		output, _ := s.scheduler.getOutput(job.CommandKey)
		job.SubState = "Entering Lock"
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILT(%v): (%v): %v", job.CommandKey, len(output), output)
		s.stateMutex.Unlock()
		if job.Job.Bootstrap && len(output) > 0 {
			if job.BuildFail == 5 {
				job.SubState = "Sending Report"
				s.deliverCrashReport(ctx, job, output)
				job.BuildFail = 0
			}
			job.BuildFail++
			job.State = pb.State_DIED
		} else {
			job.State = pb.State_VERSION_CHECK
			// Don't check version on pb jobs
			if job.GetJob().GetPartialBootstrap() {
				job.BuildFail = 0
				job.SubState = "Scheduling A Run"
				key := s.scheduleRun(job)
				job.CommandKey = key
				job.StartTime = time.Now().Unix()
				job.State = pb.State_PENDING
				if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
					s.pendingMap[time.Now().Weekday()] = make(map[string]int)
				}
				s.pendingMap[time.Now().Weekday()][job.Job.Name]++
			}
		}
		job.SubState = "Out of case"
	case pb.State_VERSION_CHECK:
		s.doCopy(job)
		s.loadCurrentVersions()
		version, err := s.getLatestVersion(ctx, job.GetJob().Name, job.GetJob().GetGoPath())
		if err != nil {
			s.Log(fmt.Sprintf("Error getting version: %v", err))
			break
		}

		res, err := exec.Command("md5sum", fmt.Sprintf("/home/simon/gobuild/bin/%v", job.GetJob().GetName())).Output()
		if err != nil {
			s.Log(fmt.Sprintf("Error reading md5sum: %v", err))
		}
		elems := strings.Fields(string(res))
		s.versionsMutex.Lock()
		defer s.versionsMutex.Unlock()
		if elems[0] != version.GetVersion() {
			s.Log(fmt.Sprintf("Bad version found in the wild", fmt.Sprintf("Bad version on %v for %v -> %v vs %v", s.Registry.Identifier, job.GetJob().GetName(), elems[0], s.versions[job.GetJob().GetName()].Version)))
			job.SubState = fmt.Sprintf("Dealing With Version Mismatch: %v", job.BuildFail)

			// Don't let the job sit here
			job.BuildFail++
			if job.BuildFail > 10 {
				job.State = pb.State_ACKNOWLEDGED

			}
			break
		}

		if time.Unix(version.GetVersionDate(), 0).Sub(time.Unix(s.versions[job.GetJob().GetName()].GetVersionDate(), 0)) < time.Hour*24 {
			job.BuildFail = 0
			job.SubState = "Scheduling of the Run"
			key := s.scheduleRun(job)
			job.CommandKey = key
			job.StartTime = time.Now().Unix()
			job.State = pb.State_PENDING
			if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
				s.pendingMap[time.Now().Weekday()] = make(map[string]int)
			}
			s.pendingMap[time.Now().Weekday()][job.Job.Name]++
		} else {
			job.SubState = fmt.Sprintf("%v -> %v is too old (%v)", time.Unix(version.GetVersionDate(), 0), time.Unix(s.versions[job.GetJob().GetName()].GetVersionDate(), 0), time.Unix(version.GetVersionDate(), 0).Sub(time.Unix(s.versions[job.GetJob().GetName()].GetVersionDate(), 0)))
		}

	case pb.State_PENDING:
		if job.Job.PartialBootstrap && job.Job.Bootstrap {
			job.Job.Bootstrap = false
		}
		s.stateMutex.Lock()
		out, _ := s.scheduler.getOutput(job.CommandKey)
		s.stateMap[job.Job.Name] = fmt.Sprintf("OUTPUT = %v", out)
		s.stateMutex.Unlock()
		if time.Now().Add(-time.Minute).Unix() > job.StartTime {
			var err error
			if job.Job.Name == "discovery" {
				err = s.runOnChange()
			}
			code := status.Convert(err).Code()

			//Unavailable allows the job to die here
			if code == codes.OK || code == codes.Unavailable {
				// Validate that the job is alive
				if !s.isJobAlive(ctx, job) {
					s.Log(fmt.Sprintf("Job %v is not alive", job))
				}
				job.State = pb.State_RUNNING
			} else {
				s.Log(fmt.Sprintf("Cannot reregister: %v", err))
			}
		}
	case pb.State_RUNNING:
		output, errout := s.scheduler.getOutput(job.CommandKey)
		output2, _ := s.scheduler.getErrOutput(job.CommandKey)
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("ROUTPUT = %v, %v", output, s.scheduler.getStatus(job.CommandKey))
		job.Status = s.scheduler.getStatus(job.CommandKey)
		s.stateMutex.Unlock()
		if len(job.CommandKey) > 0 {
			s.scheduler.wait(job.CommandKey)
			s.stateMap[job.Job.Name] = fmt.Sprintf("ONLOCk = (%v, %v)", job, output)
			s.stateMutex.Lock()
			s.stateMap[job.Job.Name] = fmt.Sprintf("COMPLETE = (%v, %v)", job, output)
			s.stateMutex.Unlock()
			s.deliverCrashReport(ctx, job, fmt.Sprintf("%v%v", output, output2))
			job.State = pb.State_DIED
		}

		if s.Registry != nil {
			entry, err := s.FFindSpecificServer(ctx, job.Job.Name, s.Registry.Identifier)
			if err != nil {
				if job.DiscoverCount > 30 {
					output2, errout2 := s.scheduler.getErrOutput(job.CommandKey)
					s.RaiseIssue("Cannot Discover Running Server", fmt.Sprintf("%v on %v is not discoverable, despite running (%v) the output says %v (%v), %v, %v", job.Job.Name, s.Registry.Identifier, err, output, errout, output2, errout2))
				}
				job.DiscoverCount++
				s.Log(fmt.Sprintf("Missing discover for %+v, %v -> %v", job, entry, err))
			} else {
				job.Port = entry.GetPort()
				job.DiscoverCount = 0
			}
		}

		// Restart this job if we need to
		if !job.Job.Bootstrap {
			if time.Now().Sub(time.Unix(job.LastVersionPull, 0)) > time.Minute*5 {
				version, err := s.getVersion(ctx, job.Job)
				job.LastVersionPull = time.Now().Unix()

				if err == nil && version.Version != job.RunningVersion {
					s.stateMutex.Lock()
					s.stateMap[job.Job.Name] = fmt.Sprintf("VERSION_MISMATCH = %v,%v", version, job.RunningVersion)
					s.stateMutex.Unlock()
					s.scheduler.killJob(job.CommandKey)
				}
			}
		}
	case pb.State_BRINK_OF_DEATH:
		if s.version.confirm(ctx, job.Job.Name) {
			s.scheduler.killJob(job.CommandKey)
			job.State = pb.State_ACKNOWLEDGED
		}
	case pb.State_DIED:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("DIED %v", job.CommandKey)
		s.stateMutex.Unlock()
		job.State = pb.State_ACKNOWLEDGED
	}

	if job.State != startState {
		job.LastTransitionTime = time.Now().Unix()
	}
}

type translator interface {
	build(job *pb.Job) *exec.Cmd
	run(job *pb.Job) *exec.Cmd
}

type checker interface {
	isAlive(ctx context.Context, job *pb.JobAssignment) bool
}

func (s *Server) getVersion(ctx context.Context, job *pb.Job) (*pbb.Version, error) {
	version, err := s.builder.build(ctx, job)
	if err != nil {
		return &pbb.Version{}, err
	}

	return version, nil
}

func updateJob(err error, job *pb.JobAssignment, resp *pbfc.CopyResponse) {
	if err == nil {
		job.QueuePos = resp.IndexInQueue
	}

}

// scheduleBuild builds out the job, returning the current version
func (s *Server) scheduleBuild(ctx context.Context, job *pb.JobAssignment) string {
	if job.Job.Bootstrap {
		c := s.translator.build(job.Job)
		// Block builds to prevent clashes
		return s.scheduler.Schedule(&rCommand{command: c, base: job.Job.Name, block: true})
	}

	val, err := s.builder.build(ctx, job.Job)
	if err != nil {
		s.DLog(ctx, fmt.Sprintf("BUILD Error: %v\n", err))
		return ""
	}
	return val.Version
}

func (s *Server) doCopy(job *pb.JobAssignment) {
	//Copy over any existing new versions
	key := s.scheduler.Schedule(&rCommand{command: exec.Command("mv", "$GOPATH/bin/"+job.GetJob().GetName()+".new", "$GOPATH/bin/"+job.GetJob().GetName()), base: job.GetJob().GetName()})
	s.scheduler.wait(key)
}

func (s *Server) scheduleRun(job *pb.JobAssignment) string {
	// Wait a while before starting the job JIC
	time.Sleep(time.Second * 2)
	c := s.translator.run(job.GetJob())
	return s.scheduler.Schedule(&rCommand{command: c, base: job.GetJob().GetName()})
}
<|endoftext|>


<commit_before>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Tamir Duberstein (tamird@gmail.com)

package main

import (
	"fmt"
	"os"
	"regexp"
	"testing"

	"github.com/google/go-github/github"
)

func TestRunGH(t *testing.T) {
	f, err := os.Open("testdata/fatal")
	if err != nil {
		t.Fatal(err)
	}

	const (
		expOwner = "cockroachdb"
		expRepo  = "cockroach"
		pkg      = "foo/bar/baz"
		sha      = "abcd123"
		issueID  = 1337
	)

	issueBodyRe := regexp.MustCompile(fmt.Sprintf(`(?s)\ASHA: https://github.com/cockroachdb/cockroach/commits/%s

Stress build found a failed test:

.*
F161007 00:27:33\.243126 449 storage/store\.go:2446  \[s3\] \[n3,s3,r1:/M{in-ax}\]: could not remove placeholder after preemptive snapshot
goroutine 449 \[running\]:
`, sha))

	if val, ok := os.LookupEnv(teamcityVCSNumberEnv); ok {
		defer os.Setenv(teamcityVCSNumberEnv, val)
	} else {
		defer os.Unsetenv(teamcityVCSNumberEnv)
	}

	if err := os.Setenv(teamcityVCSNumberEnv, sha); err != nil {
		t.Fatal(err)
	}

	if err := runGH(
		f,
		func(owner string, repo string, issue *github.IssueRequest) (*github.Issue, *github.Response, error) {
			if owner != expOwner {
				t.Fatalf("got %s, expected %s", owner, expOwner)
			}
			if repo != expRepo {
				t.Fatalf("got %s, expected %s", repo, expRepo)
			}
			if expected := fmt.Sprintf("%s: %s failed under stress", pkg, "TestRaftRemoveRace"); *issue.Title != expected {
				t.Fatalf("got %s, expected %s", *issue.Title, expected)
			}
			if !issueBodyRe.MatchString(*issue.Body) {
				t.Fatalf("got:\n%s\nexpected:\n%s", *issue.Body, issueBodyRe)
			}
			return &github.Issue{ID: github.Int(issueID)}, nil, nil
		},
	); err != nil {
		t.Fatal(err)
	}
}
<commit_msg>cmd: check an error<commit_after>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Tamir Duberstein (tamird@gmail.com)

package main

import (
	"fmt"
	"os"
	"regexp"
	"testing"

	"github.com/google/go-github/github"
)

func TestRunGH(t *testing.T) {
	f, err := os.Open("testdata/fatal")
	if err != nil {
		t.Fatal(err)
	}

	const (
		expOwner = "cockroachdb"
		expRepo  = "cockroach"
		pkg      = "foo/bar/baz"
		sha      = "abcd123"
		issueID  = 1337
	)

	issueBodyRe := regexp.MustCompile(fmt.Sprintf(`(?s)\ASHA: https://github.com/cockroachdb/cockroach/commits/%s

Stress build found a failed test:

.*
F161007 00:27:33\.243126 449 storage/store\.go:2446  \[s3\] \[n3,s3,r1:/M{in-ax}\]: could not remove placeholder after preemptive snapshot
goroutine 449 \[running\]:
`, sha))

	if val, ok := os.LookupEnv(teamcityVCSNumberEnv); ok {
		defer func() {
			if err := os.Setenv(teamcityVCSNumberEnv, val); err != nil {
				t.Error(err)
			}
		}()
	} else {
		defer func() {
			if err := os.Unsetenv(teamcityVCSNumberEnv); err != nil {
				t.Error(err)
			}
		}()
	}

	if err := os.Setenv(teamcityVCSNumberEnv, sha); err != nil {
		t.Fatal(err)
	}

	if err := runGH(
		f,
		func(owner string, repo string, issue *github.IssueRequest) (*github.Issue, *github.Response, error) {
			if owner != expOwner {
				t.Fatalf("got %s, expected %s", owner, expOwner)
			}
			if repo != expRepo {
				t.Fatalf("got %s, expected %s", repo, expRepo)
			}
			if expected := fmt.Sprintf("%s: %s failed under stress", pkg, "TestRaftRemoveRace"); *issue.Title != expected {
				t.Fatalf("got %s, expected %s", *issue.Title, expected)
			}
			if !issueBodyRe.MatchString(*issue.Body) {
				t.Fatalf("got:\n%s\nexpected:\n%s", *issue.Body, issueBodyRe)
			}
			return &github.Issue{ID: github.Int(issueID)}, nil, nil
		},
	); err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pool

import (
	"context"
	"fmt"
	"sync"
	"time"

	peerpb "github.com/cilium/cilium/api/v1/peer"
	peerTypes "github.com/cilium/cilium/pkg/hubble/peer/types"
	poolTypes "github.com/cilium/cilium/pkg/hubble/relay/pool/types"
	"github.com/cilium/cilium/pkg/lock"

	"github.com/sirupsen/logrus"
	"google.golang.org/grpc/connectivity"
)

type peer struct {
	mu lock.Mutex
	peerTypes.Peer
	conn            poolTypes.ClientConn
	connAttempts    int
	nextConnAttempt time.Time
}

// PeerManager manages a pool of peers (Peer) and associated gRPC connections.
// Peers and peer change notifications are obtained from a peer gRPC service.
type PeerManager struct {
	opts    options
	offline chan string
	wg      sync.WaitGroup
	stop    chan struct{}
	mu      lock.RWMutex
	peers   map[string]*peer
}

// NewPeerManager creates a new manager that connects to a peer gRPC service to
// manage peers and a connection to every peer's gRPC API.
func NewPeerManager(options ...Option) (*PeerManager, error) {
	opts := defaultOptions
	for _, opt := range options {
		if err := opt(&opts); err != nil {
			return nil, fmt.Errorf("failed to apply option: %v", err)
		}
	}
	return &PeerManager{
		peers:   make(map[string]*peer),
		offline: make(chan string, 100),
		stop:    make(chan struct{}),
		opts:    opts,
	}, nil
}

// Start starts the manager.
func (m *PeerManager) Start() {
	m.wg.Add(2)
	go func() {
		defer m.wg.Done()
		m.watchNotifications()
	}()
	go func() {
		defer m.wg.Done()
		m.manageConnections()
	}()
}

func (m *PeerManager) watchNotifications() {
	ctx, cancel := context.WithCancel(context.Background())
connect:
	for {
		cl, err := m.opts.peerClientBuilder.Client(m.opts.peerServiceAddress)
		if err != nil {
			m.opts.log.WithFields(logrus.Fields{
				"error":  err,
				"target": m.opts.peerServiceAddress,
			}).Warning("Failed to create peer client for peers synchronization; will try again after the timeout has expired")
			select {
			case <-m.stop:
				cancel()
				return
			case <-time.After(m.opts.retryTimeout):
				continue
			}
		}
		client, err := cl.Notify(ctx, &peerpb.NotifyRequest{})
		if err != nil {
			cl.Close()
			m.opts.log.WithFields(logrus.Fields{
				"error":              err,
				"connection timeout": m.opts.retryTimeout,
			}).Warning("Failed to create peer notify client for peers change notification; will try again after the timeout has expired")
			select {
			case <-m.stop:
				cancel()
				return
			case <-time.After(m.opts.retryTimeout):
				continue
			}
		}
		for {
			select {
			case <-m.stop:
				cl.Close()
				cancel()
				return
			default:
			}
			cn, err := client.Recv()
			if err != nil {
				cl.Close()
				m.opts.log.WithFields(logrus.Fields{
					"error":              err,
					"connection timeout": m.opts.retryTimeout,
				}).Warning("Error while receiving peer change notification; will try again after the timeout has expired")
				select {
				case <-m.stop:
					cancel()
					return
				case <-time.After(m.opts.retryTimeout):
					continue connect
				}
			}
			m.opts.log.WithField("change notification", cn).Debug("Received peer change notification")
			p := peerTypes.FromChangeNotification(cn)
			switch cn.GetType() {
			case peerpb.ChangeNotificationType_PEER_ADDED:
				m.add(p)
			case peerpb.ChangeNotificationType_PEER_DELETED:
				m.remove(p)
			case peerpb.ChangeNotificationType_PEER_UPDATED:
				m.update(p)
			}
		}
	}
}

func (m *PeerManager) manageConnections() {
	for {
		select {
		case <-m.stop:
			return
		case name := <-m.offline:
			m.mu.RLock()
			p := m.peers[name]
			m.mu.RUnlock()
			m.wg.Add(1)
			go func() {
				defer m.wg.Done()
				// a connection request has been made, make sure to attempt a connection
				m.connect(p, true)
			}()
		case <-time.After(m.opts.connCheckInterval):
			m.mu.RLock()
			now := time.Now()
			for _, p := range m.peers {
				p.mu.Lock()
				if p.conn != nil {
					switch p.conn.GetState() {
					case connectivity.Connecting, connectivity.Idle, connectivity.Ready, connectivity.Shutdown:
						p.mu.Unlock()
						continue
					}
				}
				switch {
				case p.nextConnAttempt.IsZero(), p.nextConnAttempt.Before(now):
					p.mu.Unlock()
					m.wg.Add(1)
					go func() {
						defer m.wg.Done()
						m.connect(p, false)
					}()
				default:
					p.mu.Unlock()
				}
			}
			m.mu.RUnlock()
		}
	}
}

// Stop stops the manaager.
func (m *PeerManager) Stop() {
	close(m.stop)
	m.wg.Wait()
}

// List implements observer.PeerLister.List.
func (m *PeerManager) List() []poolTypes.Peer {
	m.mu.RLock()
	defer m.mu.RUnlock()
	if len(m.peers) == 0 {
		return nil
	}
	peers := make([]poolTypes.Peer, 0, len(m.peers))
	for _, v := range m.peers {
		// note: there shouldn't be null entries in the map
		peers = append(peers, poolTypes.Peer{
			Peer: peerTypes.Peer{
				Name:          v.Name,
				Address:       v.Address,
				TLSEnabled:    v.TLSEnabled,
				TLSServerName: v.TLSServerName,
			},
			Conn: v.conn,
		})
	}
	return peers
}

// ReportOffline implements observer.PeerReporter.ReportOffline.
func (m *PeerManager) ReportOffline(name string) {
	m.wg.Add(1)
	go func() {
		defer m.wg.Done()
		m.mu.RLock()
		p, ok := m.peers[name]
		m.mu.RUnlock()
		if !ok {
			return
		}
		p.mu.Lock()
		if p.conn != nil {
			switch p.conn.GetState() {
			case connectivity.Connecting, connectivity.Idle, connectivity.Ready:
				// it looks like it's actually online or being brought online
				p.mu.Unlock()
				return
			}
		}
		p.mu.Unlock()
		select {
		case <-m.stop:
		case m.offline <- name:
		}
	}()
}

func (m *PeerManager) add(hp *peerTypes.Peer) {
	if hp == nil {
		return
	}
	p := &peer{Peer: *hp}
	m.mu.Lock()
	m.peers[p.Name] = p
	m.mu.Unlock()
	select {
	case <-m.stop:
	case m.offline <- p.Name:
	}
}

func (m *PeerManager) remove(hp *peerTypes.Peer) {
	if hp == nil {
		return
	}
	m.mu.Lock()
	if p, ok := m.peers[hp.Name]; ok {
		m.disconnect(p)
		delete(m.peers, hp.Name)
	}
	m.mu.Unlock()
}

func (m *PeerManager) update(hp *peerTypes.Peer) {
	if hp == nil {
		return
	}
	p := &peer{Peer: *hp}
	m.mu.Lock()
	if old, ok := m.peers[p.Name]; ok {
		m.disconnect(old)
	}
	m.peers[p.Name] = p
	m.mu.Unlock()
	select {
	case <-m.stop:
	case m.offline <- p.Name:
	}
}

func (m *PeerManager) connect(p *peer, ignoreBackoff bool) {
	if p == nil {
		return
	}
	p.mu.Lock()
	defer p.mu.Unlock()
	now := time.Now()
	if p.Address == nil || (p.nextConnAttempt.After(now) && !ignoreBackoff) {
		return
	}
	if p.conn != nil {
		switch p.conn.GetState() {
		case connectivity.Connecting, connectivity.Idle, connectivity.Ready:
			return // no need to attempt to connect
		default:
			if err := p.conn.Close(); err != nil {
				m.opts.log.WithFields(logrus.Fields{
					"error": err,
				}).Warningf("Failed to properly close gRPC client connection to peer %s", p.Name)
			}
			p.conn = nil
		}
	}

	m.opts.log.WithFields(logrus.Fields{
		"address": p.Address,
	}).Debugf("Connecting peer %s...", p.Name)
	//FIXME: provide hostname to ClientConn
	conn, err := m.opts.clientConnBuilder.ClientConn(p.Address.String(), "")
	if err != nil {
		duration := m.opts.backoff.Duration(p.connAttempts)
		p.nextConnAttempt = now.Add(duration)
		p.connAttempts++
		m.opts.log.WithFields(logrus.Fields{
			"address": p.Address,
			"error":   err,
		}).Warningf("Failed to create gRPC client connection to peer %s; next attempt after %s", p.Name, duration)
	} else {
		p.nextConnAttempt = time.Time{}
		p.connAttempts = 0
		p.conn = conn
		m.opts.log.Debugf("Peer %s connected", p.Name)
	}
}

func (m *PeerManager) disconnect(p *peer) {
	if p == nil {
		return
	}
	p.mu.Lock()
	defer p.mu.Unlock()
	if p.conn == nil {
		return
	}
	m.opts.log.Debugf("Disconnecting peer %s...", p.Name)
	if err := p.conn.Close(); err != nil {
		m.opts.log.WithFields(logrus.Fields{
			"error": err,
		}).Warningf("Failed to properly close gRPC client connection to peer %s", p.Name)
	}
	p.conn = nil
	m.opts.log.Debugf("Peer %s disconnected", p.Name)
}
<commit_msg>hubble/relay: pass TLS server name to peer conn builder<commit_after>// Copyright 2020 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pool

import (
	"context"
	"fmt"
	"sync"
	"time"

	peerpb "github.com/cilium/cilium/api/v1/peer"
	peerTypes "github.com/cilium/cilium/pkg/hubble/peer/types"
	poolTypes "github.com/cilium/cilium/pkg/hubble/relay/pool/types"
	"github.com/cilium/cilium/pkg/lock"

	"github.com/sirupsen/logrus"
	"google.golang.org/grpc/connectivity"
)

type peer struct {
	mu lock.Mutex
	peerTypes.Peer
	conn            poolTypes.ClientConn
	connAttempts    int
	nextConnAttempt time.Time
}

// PeerManager manages a pool of peers (Peer) and associated gRPC connections.
// Peers and peer change notifications are obtained from a peer gRPC service.
type PeerManager struct {
	opts    options
	offline chan string
	wg      sync.WaitGroup
	stop    chan struct{}
	mu      lock.RWMutex
	peers   map[string]*peer
}

// NewPeerManager creates a new manager that connects to a peer gRPC service to
// manage peers and a connection to every peer's gRPC API.
func NewPeerManager(options ...Option) (*PeerManager, error) {
	opts := defaultOptions
	for _, opt := range options {
		if err := opt(&opts); err != nil {
			return nil, fmt.Errorf("failed to apply option: %v", err)
		}
	}
	return &PeerManager{
		peers:   make(map[string]*peer),
		offline: make(chan string, 100),
		stop:    make(chan struct{}),
		opts:    opts,
	}, nil
}

// Start starts the manager.
func (m *PeerManager) Start() {
	m.wg.Add(2)
	go func() {
		defer m.wg.Done()
		m.watchNotifications()
	}()
	go func() {
		defer m.wg.Done()
		m.manageConnections()
	}()
}

func (m *PeerManager) watchNotifications() {
	ctx, cancel := context.WithCancel(context.Background())
connect:
	for {
		cl, err := m.opts.peerClientBuilder.Client(m.opts.peerServiceAddress)
		if err != nil {
			m.opts.log.WithFields(logrus.Fields{
				"error":  err,
				"target": m.opts.peerServiceAddress,
			}).Warning("Failed to create peer client for peers synchronization; will try again after the timeout has expired")
			select {
			case <-m.stop:
				cancel()
				return
			case <-time.After(m.opts.retryTimeout):
				continue
			}
		}
		client, err := cl.Notify(ctx, &peerpb.NotifyRequest{})
		if err != nil {
			cl.Close()
			m.opts.log.WithFields(logrus.Fields{
				"error":              err,
				"connection timeout": m.opts.retryTimeout,
			}).Warning("Failed to create peer notify client for peers change notification; will try again after the timeout has expired")
			select {
			case <-m.stop:
				cancel()
				return
			case <-time.After(m.opts.retryTimeout):
				continue
			}
		}
		for {
			select {
			case <-m.stop:
				cl.Close()
				cancel()
				return
			default:
			}
			cn, err := client.Recv()
			if err != nil {
				cl.Close()
				m.opts.log.WithFields(logrus.Fields{
					"error":              err,
					"connection timeout": m.opts.retryTimeout,
				}).Warning("Error while receiving peer change notification; will try again after the timeout has expired")
				select {
				case <-m.stop:
					cancel()
					return
				case <-time.After(m.opts.retryTimeout):
					continue connect
				}
			}
			m.opts.log.WithField("change notification", cn).Debug("Received peer change notification")
			p := peerTypes.FromChangeNotification(cn)
			switch cn.GetType() {
			case peerpb.ChangeNotificationType_PEER_ADDED:
				m.add(p)
			case peerpb.ChangeNotificationType_PEER_DELETED:
				m.remove(p)
			case peerpb.ChangeNotificationType_PEER_UPDATED:
				m.update(p)
			}
		}
	}
}

func (m *PeerManager) manageConnections() {
	for {
		select {
		case <-m.stop:
			return
		case name := <-m.offline:
			m.mu.RLock()
			p := m.peers[name]
			m.mu.RUnlock()
			m.wg.Add(1)
			go func() {
				defer m.wg.Done()
				// a connection request has been made, make sure to attempt a connection
				m.connect(p, true)
			}()
		case <-time.After(m.opts.connCheckInterval):
			m.mu.RLock()
			now := time.Now()
			for _, p := range m.peers {
				p.mu.Lock()
				if p.conn != nil {
					switch p.conn.GetState() {
					case connectivity.Connecting, connectivity.Idle, connectivity.Ready, connectivity.Shutdown:
						p.mu.Unlock()
						continue
					}
				}
				switch {
				case p.nextConnAttempt.IsZero(), p.nextConnAttempt.Before(now):
					p.mu.Unlock()
					m.wg.Add(1)
					go func() {
						defer m.wg.Done()
						m.connect(p, false)
					}()
				default:
					p.mu.Unlock()
				}
			}
			m.mu.RUnlock()
		}
	}
}

// Stop stops the manaager.
func (m *PeerManager) Stop() {
	close(m.stop)
	m.wg.Wait()
}

// List implements observer.PeerLister.List.
func (m *PeerManager) List() []poolTypes.Peer {
	m.mu.RLock()
	defer m.mu.RUnlock()
	if len(m.peers) == 0 {
		return nil
	}
	peers := make([]poolTypes.Peer, 0, len(m.peers))
	for _, v := range m.peers {
		// note: there shouldn't be null entries in the map
		peers = append(peers, poolTypes.Peer{
			Peer: peerTypes.Peer{
				Name:          v.Name,
				Address:       v.Address,
				TLSEnabled:    v.TLSEnabled,
				TLSServerName: v.TLSServerName,
			},
			Conn: v.conn,
		})
	}
	return peers
}

// ReportOffline implements observer.PeerReporter.ReportOffline.
func (m *PeerManager) ReportOffline(name string) {
	m.wg.Add(1)
	go func() {
		defer m.wg.Done()
		m.mu.RLock()
		p, ok := m.peers[name]
		m.mu.RUnlock()
		if !ok {
			return
		}
		p.mu.Lock()
		if p.conn != nil {
			switch p.conn.GetState() {
			case connectivity.Connecting, connectivity.Idle, connectivity.Ready:
				// it looks like it's actually online or being brought online
				p.mu.Unlock()
				return
			}
		}
		p.mu.Unlock()
		select {
		case <-m.stop:
		case m.offline <- name:
		}
	}()
}

func (m *PeerManager) add(hp *peerTypes.Peer) {
	if hp == nil {
		return
	}
	p := &peer{Peer: *hp}
	m.mu.Lock()
	m.peers[p.Name] = p
	m.mu.Unlock()
	select {
	case <-m.stop:
	case m.offline <- p.Name:
	}
}

func (m *PeerManager) remove(hp *peerTypes.Peer) {
	if hp == nil {
		return
	}
	m.mu.Lock()
	if p, ok := m.peers[hp.Name]; ok {
		m.disconnect(p)
		delete(m.peers, hp.Name)
	}
	m.mu.Unlock()
}

func (m *PeerManager) update(hp *peerTypes.Peer) {
	if hp == nil {
		return
	}
	p := &peer{Peer: *hp}
	m.mu.Lock()
	if old, ok := m.peers[p.Name]; ok {
		m.disconnect(old)
	}
	m.peers[p.Name] = p
	m.mu.Unlock()
	select {
	case <-m.stop:
	case m.offline <- p.Name:
	}
}

func (m *PeerManager) connect(p *peer, ignoreBackoff bool) {
	if p == nil {
		return
	}
	p.mu.Lock()
	defer p.mu.Unlock()
	now := time.Now()
	if p.Address == nil || (p.nextConnAttempt.After(now) && !ignoreBackoff) {
		return
	}
	if p.conn != nil {
		switch p.conn.GetState() {
		case connectivity.Connecting, connectivity.Idle, connectivity.Ready:
			return // no need to attempt to connect
		default:
			if err := p.conn.Close(); err != nil {
				m.opts.log.WithFields(logrus.Fields{
					"error": err,
				}).Warningf("Failed to properly close gRPC client connection to peer %s", p.Name)
			}
			p.conn = nil
		}
	}

	m.opts.log.WithFields(logrus.Fields{
		"address": p.Address,
	}).Debugf("Connecting peer %s...", p.Name)
	conn, err := m.opts.clientConnBuilder.ClientConn(p.Address.String(), p.TLSServerName)
	if err != nil {
		duration := m.opts.backoff.Duration(p.connAttempts)
		p.nextConnAttempt = now.Add(duration)
		p.connAttempts++
		m.opts.log.WithFields(logrus.Fields{
			"address": p.Address,
			"error":   err,
		}).Warningf("Failed to create gRPC client connection to peer %s; next attempt after %s", p.Name, duration)
	} else {
		p.nextConnAttempt = time.Time{}
		p.connAttempts = 0
		p.conn = conn
		m.opts.log.Debugf("Peer %s connected", p.Name)
	}
}

func (m *PeerManager) disconnect(p *peer) {
	if p == nil {
		return
	}
	p.mu.Lock()
	defer p.mu.Unlock()
	if p.conn == nil {
		return
	}
	m.opts.log.Debugf("Disconnecting peer %s...", p.Name)
	if err := p.conn.Close(); err != nil {
		m.opts.log.WithFields(logrus.Fields{
			"error": err,
		}).Warningf("Failed to properly close gRPC client connection to peer %s", p.Name)
	}
	p.conn = nil
	m.opts.log.Debugf("Peer %s disconnected", p.Name)
}
<|endoftext|>


<commit_before>package mgr

import (
	"github.com/stretchr/testify/suite"
	"github.com/golang/mock/gomock"
	"capsulecd/pkg/mgr/mock"
	"capsulecd/pkg/config/mock"
	"capsulecd/pkg/pipeline"
	"testing"
	"io/ioutil"
	"github.com/stretchr/testify/require"
	"os"
	"path"
	"capsulecd/pkg/metadata"
	"capsulecd/pkg/utils"
	"capsulecd/pkg/mgr"
)

// Define the suite, and absorb the built-in basic suite
// functionality from testify - including a T() method which
// returns the current testing context
type MgrRubyBundlerTestSuite struct {
	suite.Suite
	MockCtrl     *gomock.Controller
	Mgr          *mock_mgr.MockInterface
	Config       *mock_config.MockInterface
	PipelineData *pipeline.Data
}

// Make sure that VariableThatShouldStartAtFive is set to five
// before each test
func (suite *MgrRubyBundlerTestSuite) SetupTest() {
	suite.MockCtrl = gomock.NewController(suite.T())

	suite.PipelineData = new(pipeline.Data)

	suite.Config = mock_config.NewMockInterface(suite.MockCtrl)
	suite.Mgr = mock_mgr.NewMockInterface(suite.MockCtrl)

}

func (suite *MgrRubyBundlerTestSuite) TearDownTest() {
	suite.MockCtrl.Finish()
}

// In order for 'go test' to run this suite, we need to create
// a normal test function and pass our suite to suite.Run
func TestMgrRubyBundler_TestSuite(t *testing.T) {
	suite.Run(t, new(MgrRubyBundlerTestSuite))
}

func (suite *MgrRubyBundlerTestSuite) TestMgrRubyBundlerTestSuite_DependenciesStep() {
	//setup
	//suite.Config.EXPECT().SetDefault(gomock.Any(), gomock.Any()).MinTimes(1)

	//copy cookbook fixture into a temp directory.
	parentPath, err := ioutil.TempDir("", "")
	require.NoError(suite.T(), err)
	defer os.RemoveAll(parentPath)
	suite.PipelineData.GitParentPath = parentPath
	suite.PipelineData.GitLocalPath = path.Join(parentPath, "gem_analogj_test")
	cerr := utils.CopyDir(path.Join("testdata", "ruby", "gem_analogj_test"), suite.PipelineData.GitLocalPath)
	require.NoError(suite.T(), cerr)
	cperr := utils.CopyFile(path.Join("testdata", "ruby", "gem_analogj_test-0.1.4.gem"), path.Join(suite.PipelineData.GitLocalPath, "-.gem"))
	require.NoError(suite.T(), cperr)

	mgrRubyBundler, err := mgr.Create("bundler", suite.PipelineData, suite.Config, nil)
	require.NoError(suite.T(), err)
	currentVersion := new(metadata.RubyMetadata)
	nextVersion := new(metadata.RubyMetadata)

	//test
	berr := mgrRubyBundler.MgrDependenciesStep(currentVersion, nextVersion)

	//assert
	require.NoError(suite.T(), berr)
	require.True(suite.T(), utils.FileExists(path.Join(suite.PipelineData.GitLocalPath, "Gemfile.lock")))

}

func (suite *MgrRubyBundlerTestSuite) TestMgrRubyBundlerTestSuite_MgrDistStep_WithoutCredentials() {
	//setup
	//suite.Config.EXPECT().SetDefault(gomock.Any(), gomock.Any()).MinTimes(1)
	suite.Config.EXPECT().IsSet("rubygems_api_key").MinTimes(1).Return(false)

	mgrRubyBundler, err := mgr.Create("bundler", suite.PipelineData, suite.Config, nil)
	require.NoError(suite.T(), err)
	currentVersion := new(metadata.RubyMetadata)
	nextVersion := new(metadata.RubyMetadata)

	//test
	berr := mgrRubyBundler.MgrDistStep(currentVersion, nextVersion)

	//assert
	require.Error(suite.T(), berr)
}<commit_msg>fix import cycle.<commit_after>package mgr_test

import (
	"github.com/stretchr/testify/suite"
	"github.com/golang/mock/gomock"
	"capsulecd/pkg/mgr/mock"
	"capsulecd/pkg/config/mock"
	"capsulecd/pkg/pipeline"
	"testing"
	"io/ioutil"
	"github.com/stretchr/testify/require"
	"os"
	"path"
	"capsulecd/pkg/metadata"
	"capsulecd/pkg/utils"
	"capsulecd/pkg/mgr"
)

// Define the suite, and absorb the built-in basic suite
// functionality from testify - including a T() method which
// returns the current testing context
type MgrRubyBundlerTestSuite struct {
	suite.Suite
	MockCtrl     *gomock.Controller
	Mgr          *mock_mgr.MockInterface
	Config       *mock_config.MockInterface
	PipelineData *pipeline.Data
}

// Make sure that VariableThatShouldStartAtFive is set to five
// before each test
func (suite *MgrRubyBundlerTestSuite) SetupTest() {
	suite.MockCtrl = gomock.NewController(suite.T())

	suite.PipelineData = new(pipeline.Data)

	suite.Config = mock_config.NewMockInterface(suite.MockCtrl)
	suite.Mgr = mock_mgr.NewMockInterface(suite.MockCtrl)

}

func (suite *MgrRubyBundlerTestSuite) TearDownTest() {
	suite.MockCtrl.Finish()
}

// In order for 'go test' to run this suite, we need to create
// a normal test function and pass our suite to suite.Run
func TestMgrRubyBundler_TestSuite(t *testing.T) {
	suite.Run(t, new(MgrRubyBundlerTestSuite))
}

func (suite *MgrRubyBundlerTestSuite) TestMgrRubyBundlerTestSuite_DependenciesStep() {
	//setup
	//suite.Config.EXPECT().SetDefault(gomock.Any(), gomock.Any()).MinTimes(1)

	//copy cookbook fixture into a temp directory.
	parentPath, err := ioutil.TempDir("", "")
	require.NoError(suite.T(), err)
	defer os.RemoveAll(parentPath)
	suite.PipelineData.GitParentPath = parentPath
	suite.PipelineData.GitLocalPath = path.Join(parentPath, "gem_analogj_test")
	cerr := utils.CopyDir(path.Join("testdata", "ruby", "gem_analogj_test"), suite.PipelineData.GitLocalPath)
	require.NoError(suite.T(), cerr)
	cperr := utils.CopyFile(path.Join("testdata", "ruby", "gem_analogj_test-0.1.4.gem"), path.Join(suite.PipelineData.GitLocalPath, "-.gem"))
	require.NoError(suite.T(), cperr)

	mgrRubyBundler, err := mgr.Create("bundler", suite.PipelineData, suite.Config, nil)
	require.NoError(suite.T(), err)
	currentVersion := new(metadata.RubyMetadata)
	nextVersion := new(metadata.RubyMetadata)

	//test
	berr := mgrRubyBundler.MgrDependenciesStep(currentVersion, nextVersion)

	//assert
	require.NoError(suite.T(), berr)
	require.True(suite.T(), utils.FileExists(path.Join(suite.PipelineData.GitLocalPath, "Gemfile.lock")))

}

func (suite *MgrRubyBundlerTestSuite) TestMgrRubyBundlerTestSuite_MgrDistStep_WithoutCredentials() {
	//setup
	//suite.Config.EXPECT().SetDefault(gomock.Any(), gomock.Any()).MinTimes(1)
	suite.Config.EXPECT().IsSet("rubygems_api_key").MinTimes(1).Return(false)

	mgrRubyBundler, err := mgr.Create("bundler", suite.PipelineData, suite.Config, nil)
	require.NoError(suite.T(), err)
	currentVersion := new(metadata.RubyMetadata)
	nextVersion := new(metadata.RubyMetadata)

	//test
	berr := mgrRubyBundler.MgrDistStep(currentVersion, nextVersion)

	//assert
	require.Error(suite.T(), berr)
}<|endoftext|>