<commit_before>package social

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"

	"github.com/grafana/grafana/pkg/models"

	"golang.org/x/oauth2"
)

type GenericOAuth struct {
	*oauth2.Config
	allowedDomains       []string
	allowedOrganizations []string
	apiUrl               string
	allowSignup          bool
	teamIds              []int
}

func (s *GenericOAuth) Type() int {
	return int(models.GENERIC)
}

func (s *GenericOAuth) IsEmailAllowed(email string) bool {
	return isEmailAllowed(email, s.allowedDomains)
}

func (s *GenericOAuth) IsSignupAllowed() bool {
	return s.allowSignup
}

func (s *GenericOAuth) IsTeamMember(client *http.Client) bool {
	if len(s.teamIds) == 0 {
		return true
	}

	teamMemberships, err := s.FetchTeamMemberships(client)
	if err != nil {
		return false
	}

	for _, teamId := range s.teamIds {
		for _, membershipId := range teamMemberships {
			if teamId == membershipId {
				return true
			}
		}
	}

	return false
}

func (s *GenericOAuth) IsOrganizationMember(client *http.Client) bool {
	if len(s.allowedOrganizations) == 0 {
		return true
	}

	organizations, err := s.FetchOrganizations(client)
	if err != nil {
		return false
	}

	for _, allowedOrganization := range s.allowedOrganizations {
		for _, organization := range organizations {
			if organization == allowedOrganization {
				return true
			}
		}
	}

	return false
}

func (s *GenericOAuth) FetchPrivateEmail(client *http.Client) (string, error) {
	type Record struct {
		Email       string `json:"email"`
		Primary     bool   `json:"primary"`
		IsPrimary   bool   `json:"is_primary"`
		Verified    bool   `json:"verified"`
		IsConfirmed bool   `json:"is_confirmed"`
	}

	response, err := HttpGet(client, fmt.Sprintf(s.apiUrl+"/emails"))
	if err != nil {
		return "", fmt.Errorf("Error getting email address: %s", err)
	}

	var records []Record

	err = json.Unmarshal(response.Body, &records)
	if err != nil {
		var data struct {
			Values []Record `json:"values"`
		}

		err = json.Unmarshal(response.Body, &data)
		if err != nil {
			return "", fmt.Errorf("Error getting email address: %s", err)
		}

		records = data.Values
	}

	var email = ""
	for _, record := range records {
		if record.Primary || record.IsPrimary {
			email = record.Email
			break
		}
	}

	return email, nil
}

func (s *GenericOAuth) FetchTeamMemberships(client *http.Client) ([]int, error) {
	type Record struct {
		Id int `json:"id"`
	}

	response, err := HttpGet(client, fmt.Sprintf(s.apiUrl+"/teams"))
	if err != nil {
		return nil, fmt.Errorf("Error getting team memberships: %s", err)
	}

	var records []Record

	err = json.Unmarshal(response.Body, &records)
	if err != nil {
		return nil, fmt.Errorf("Error getting team memberships: %s", err)
	}

	var ids = make([]int, len(records))
	for i, record := range records {
		ids[i] = record.Id
	}

	return ids, nil
}

func (s *GenericOAuth) FetchOrganizations(client *http.Client) ([]string, error) {
	type Record struct {
		Login string `json:"login"`
	}

	response, err := HttpGet(client, fmt.Sprintf(s.apiUrl+"/orgs"))
	if err != nil {
		return nil, fmt.Errorf("Error getting organizations: %s", err)
	}

	var records []Record

	err = json.Unmarshal(response.Body, &records)
	if err != nil {
		return nil, fmt.Errorf("Error getting organizations: %s", err)
	}

	var logins = make([]string, len(records))
	for i, record := range records {
		logins[i] = record.Login
	}

	return logins, nil
}

func (s *GenericOAuth) UserInfo(client *http.Client) (*BasicUserInfo, error) {
	var data struct {
		Name        string              `json:"name"`
		DisplayName string              `json:"display_name"`
		Login       string              `json:"login"`
		Username    string              `json:"username"`
		Email       string              `json:"email"`
		Attributes  map[string][]string `json:"attributes"`
	}

	response, err := HttpGet(client, s.apiUrl)
	if err != nil {
		return nil, fmt.Errorf("Error getting user info: %s", err)
	}

	err = json.Unmarshal(response.Body, &data)
	if err != nil {
		return nil, fmt.Errorf("Error getting user info: %s", err)
	}

	userInfo := &BasicUserInfo{
		Name:  data.Name,
		Login: data.Login,
		Email: data.Email,
	}

	if userInfo.Email == "" && data.Attributes["email:primary"] != nil {
		userInfo.Email = data.Attributes["email:primary"][0]
	}

	if userInfo.Email == "" {
		userInfo.Email, err = s.FetchPrivateEmail(client)
		if err != nil {
			return nil, err
		}
	}

	if userInfo.Name == "" && data.DisplayName != "" {
		userInfo.Name = data.DisplayName
	}

	if userInfo.Login == "" && data.Username != "" {
		userInfo.Login = data.Username
	}

	if userInfo.Login == "" {
		userInfo.Login = data.Email
	}

	if !s.IsTeamMember(client) {
		return nil, errors.New("User not a member of one of the required teams")
	}

	if !s.IsOrganizationMember(client) {
		return nil, errors.New("User not a member of one of the required organizations")
	}

	return userInfo, nil
}
<commit_msg>Generic Oauth Support for ADFS (#9242)<commit_after>package social

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/mail"

	"github.com/grafana/grafana/pkg/models"

	"golang.org/x/oauth2"
)

type GenericOAuth struct {
	*oauth2.Config
	allowedDomains       []string
	allowedOrganizations []string
	apiUrl               string
	allowSignup          bool
	teamIds              []int
}

func (s *GenericOAuth) Type() int {
	return int(models.GENERIC)
}

func (s *GenericOAuth) IsEmailAllowed(email string) bool {
	return isEmailAllowed(email, s.allowedDomains)
}

func (s *GenericOAuth) IsSignupAllowed() bool {
	return s.allowSignup
}

func (s *GenericOAuth) IsTeamMember(client *http.Client) bool {
	if len(s.teamIds) == 0 {
		return true
	}

	teamMemberships, err := s.FetchTeamMemberships(client)
	if err != nil {
		return false
	}

	for _, teamId := range s.teamIds {
		for _, membershipId := range teamMemberships {
			if teamId == membershipId {
				return true
			}
		}
	}

	return false
}

func (s *GenericOAuth) IsOrganizationMember(client *http.Client) bool {
	if len(s.allowedOrganizations) == 0 {
		return true
	}

	organizations, err := s.FetchOrganizations(client)
	if err != nil {
		return false
	}

	for _, allowedOrganization := range s.allowedOrganizations {
		for _, organization := range organizations {
			if organization == allowedOrganization {
				return true
			}
		}
	}

	return false
}

func (s *GenericOAuth) FetchPrivateEmail(client *http.Client) (string, error) {
	type Record struct {
		Email       string `json:"email"`
		Primary     bool   `json:"primary"`
		IsPrimary   bool   `json:"is_primary"`
		Verified    bool   `json:"verified"`
		IsConfirmed bool   `json:"is_confirmed"`
	}

	response, err := HttpGet(client, fmt.Sprintf(s.apiUrl+"/emails"))
	if err != nil {
		return "", fmt.Errorf("Error getting email address: %s", err)
	}

	var records []Record

	err = json.Unmarshal(response.Body, &records)
	if err != nil {
		var data struct {
			Values []Record `json:"values"`
		}

		err = json.Unmarshal(response.Body, &data)
		if err != nil {
			return "", fmt.Errorf("Error getting email address: %s", err)
		}

		records = data.Values
	}

	var email = ""
	for _, record := range records {
		if record.Primary || record.IsPrimary {
			email = record.Email
			break
		}
	}

	return email, nil
}

func (s *GenericOAuth) FetchTeamMemberships(client *http.Client) ([]int, error) {
	type Record struct {
		Id int `json:"id"`
	}

	response, err := HttpGet(client, fmt.Sprintf(s.apiUrl+"/teams"))
	if err != nil {
		return nil, fmt.Errorf("Error getting team memberships: %s", err)
	}

	var records []Record

	err = json.Unmarshal(response.Body, &records)
	if err != nil {
		return nil, fmt.Errorf("Error getting team memberships: %s", err)
	}

	var ids = make([]int, len(records))
	for i, record := range records {
		ids[i] = record.Id
	}

	return ids, nil
}

func (s *GenericOAuth) FetchOrganizations(client *http.Client) ([]string, error) {
	type Record struct {
		Login string `json:"login"`
	}

	response, err := HttpGet(client, fmt.Sprintf(s.apiUrl+"/orgs"))
	if err != nil {
		return nil, fmt.Errorf("Error getting organizations: %s", err)
	}

	var records []Record

	err = json.Unmarshal(response.Body, &records)
	if err != nil {
		return nil, fmt.Errorf("Error getting organizations: %s", err)
	}

	var logins = make([]string, len(records))
	for i, record := range records {
		logins[i] = record.Login
	}

	return logins, nil
}

type UserInfoJson struct {
	Name        string              `json:"name"`
	DisplayName string              `json:"display_name"`
	Login       string              `json:"login"`
	Username    string              `json:"username"`
	Email       string              `json:"email"`
	Upn         string              `json:"upn"`
	Attributes  map[string][]string `json:"attributes"`
}

func (s *GenericOAuth) UserInfo(client *http.Client) (*BasicUserInfo, error) {
	var data UserInfoJson

	response, err := HttpGet(client, s.apiUrl)
	if err != nil {
		return nil, fmt.Errorf("Error getting user info: %s", err)
	}

	err = json.Unmarshal(response.Body, &data)
	if err != nil {
		return nil, fmt.Errorf("Error getting user info: %s", err)
	}

	name, err := s.extractName(data)
	if err != nil {
		return nil, err
	}

	email, err := s.extractEmail(data, client)
	if err != nil {
		return nil, err
	}

	login, err := s.extractLogin(data, email)
	if err != nil {
		return nil, err
	}

	userInfo := &BasicUserInfo{
		Name:  name,
		Login: login,
		Email: email,
	}

	if !s.IsTeamMember(client) {
		return nil, errors.New("User not a member of one of the required teams")
	}

	if !s.IsOrganizationMember(client) {
		return nil, errors.New("User not a member of one of the required organizations")
	}

	return userInfo, nil
}

func (s *GenericOAuth) extractEmail(data UserInfoJson, client *http.Client) (string, error) {
	if data.Email != "" {
		return data.Email, nil
	}

	if data.Attributes["email:primary"] != nil {
		return data.Attributes["email:primary"][0], nil
	}

	if data.Upn != "" {
		emailAddr, emailErr := mail.ParseAddress(data.Upn)
		if emailErr == nil {
			return emailAddr.Address, nil
		}
	}

	return s.FetchPrivateEmail(client)
}

func (s *GenericOAuth) extractLogin(data UserInfoJson, email string) (string, error) {
	if data.Login != "" {
		return data.Login, nil
	}

	if data.Username != "" {
		return data.Username, nil
	}

	return email, nil
}

func (s *GenericOAuth) extractName(data UserInfoJson) (string, error) {
	if data.Name != "" {
		return data.Name, nil
	}

	if data.DisplayName != "" {
		return data.DisplayName, nil
	}

	return "", nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/golang/glog"
	yaml "gopkg.in/yaml.v2"

	"strings"

	"k8s.io/publishing-bot/cmd/publishing-bot/config"
)

const (
	depCommit        = "7c44971bbb9f0ed87db40b601f2d9fe4dffb750d"
	godepCommit      = "tags/v80"
	DefaultGoVersion = "1.11.1"
)

var (
	SystemGoPath = os.Getenv("GOPATH")
	BaseRepoPath = filepath.Join(SystemGoPath, "src", "k8s.io")
)

func Usage() {
	fmt.Fprintf(os.Stderr, `
Usage: %s [-config <config-yaml-file>] [-source-repo <repo>] [-source-org <org>] [-rules-file <file> ] [-skip-godep|skip-dep] [-target-org <org>]

Command line flags override config values.
`, os.Args[0])
	flag.PrintDefaults()
}

func main() {
	configFilePath := flag.String("config", "", "the config file in yaml format")
	githubHost := flag.String("github-host", "", "the address of github (defaults to github.com)")
	basePackage := flag.String("base-package", "", "the name of the package base (defaults to k8s.io when source repo is kubernetes, "+
		"otherwise github-host/target-org)")
	repoName := flag.String("source-repo", "", "the name of the source repository (eg. kubernetes)")
	repoOrg := flag.String("source-org", "", "the name of the source repository organization, (eg. kubernetes)")
	rulesFile := flag.String("rules-file", "", "the file with repository rules")
	targetOrg := flag.String("target-org", "", `the target organization to publish into (e.g. "k8s-publishing-bot")`)
	skipGodep := flag.Bool("skip-godep", false, `skip godeps installation and godeps-restore`)
	skipDep := flag.Bool("skip-dep", false, `skip 'dep'' installation`)

	flag.Usage = Usage
	flag.Parse()

	cfg := config.Config{}
	if *configFilePath != "" {
		bs, err := ioutil.ReadFile(*configFilePath)
		if err != nil {
			glog.Fatalf("Failed to load config file from %q: %v", *configFilePath, err)
		}
		if err := yaml.Unmarshal(bs, &cfg); err != nil {
			glog.Fatalf("Failed to parse config file at %q: %v", *configFilePath, err)
		}
	}

	if *targetOrg != "" {
		cfg.TargetOrg = *targetOrg
	}
	if *repoName != "" {
		cfg.SourceRepo = *repoName
	}
	if *repoOrg != "" {
		cfg.SourceOrg = *repoOrg
	}
	if *githubHost != "" {
		cfg.GithubHost = *githubHost
	}
	if *basePackage != "" {
		cfg.BasePackage = *basePackage
	}

	if cfg.GithubHost == "" {
		cfg.GithubHost = "github.com"
	}
	// defaulting when base package is not specified
	if cfg.BasePackage == "" {
		if cfg.SourceRepo == "kubernetes" {
			cfg.BasePackage = "k8s.io"
		} else {
			cfg.BasePackage = filepath.Join(cfg.GithubHost, cfg.TargetOrg)
		}
	}

	BaseRepoPath = filepath.Join(SystemGoPath, "src", cfg.BasePackage)

	if *rulesFile != "" {
		cfg.RulesFile = *rulesFile
	}

	if len(cfg.SourceRepo) == 0 || len(cfg.SourceOrg) == 0 {
		glog.Fatalf("source-org and source-repo cannot be empty")
	}

	if len(cfg.TargetOrg) == 0 {
		glog.Fatalf("Target organization cannot be empty")
	}

	// If RULE_FILE_PATH is detected, check if the source repository include rules files.
	if len(os.Getenv("RULE_FILE_PATH")) > 0 {
		cfg.RulesFile = filepath.Join(BaseRepoPath, cfg.SourceRepo, os.Getenv("RULE_FILE_PATH"))
	}

	if len(cfg.RulesFile) == 0 {
		glog.Fatalf("No rules file provided")
	}
	rules, err := config.LoadRules(cfg.RulesFile)
	if err != nil {
		glog.Fatalf("Failed to load rules: %v", err)
	}

	goVersions := []string{DefaultGoVersion}
	for _, rule := range rules.Rules {
		for _, branch := range rule.Branches {
			if branch.GoVersion != "" {
				found := false
				for _, v := range goVersions {
					if v == branch.GoVersion {
						found = true
					}
				}
				if !found {
					goVersions = append(goVersions, branch.GoVersion)
				}
			}
		}
	}
	for _, v := range goVersions {
		installGoVersion(v, filepath.Join(SystemGoPath, "go-"+v))
	}
	goLink, target := filepath.Join(SystemGoPath, "go"), filepath.Join(SystemGoPath, "go-"+DefaultGoVersion)
	os.Remove(goLink)
	if err := os.Symlink(target, goLink); err != nil {
		glog.Fatalf("Failed to link %s to %s: %s", goLink, target, err)
	}

	if err := os.MkdirAll(BaseRepoPath, os.ModePerm); err != nil {
		glog.Fatalf("Failed to create source repo directory %s: %v", BaseRepoPath, err)
	}

	if !*skipGodep {
		installGodeps()
	}
	if !*skipDep {
		installDep()
	}

	cloneSourceRepo(cfg, *skipGodep)
	for _, rule := range rules.Rules {
		cloneForkRepo(cfg, rule.DestinationRepository)
	}
}

func installGoVersion(v string, pth string) {
	if s, err := os.Stat(pth); err != nil && !os.IsNotExist(err) {
		glog.Fatal(err)
	} else if err == nil {
		if s.IsDir() {
			glog.Infof("Found existing go %s at %s", v, pth)
			return
		}
		glog.Fatalf("Expected %s to be a directory", pth)
	}

	glog.Infof("Installing go %s to %s", v, pth)
	tmpPath, err := ioutil.TempDir(SystemGoPath, "go-tmp-")
	if err != nil {
		glog.Fatal(err)
	}
	defer os.RemoveAll(tmpPath)
	cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("curl -SLf https://storage.googleapis.com/golang/go%s.linux-amd64.tar.gz | tar -xz --strip 1 -C %s", v, tmpPath))
	cmd.Dir = tmpPath
	run(cmd)
	if err := os.Rename(tmpPath, pth); err != nil {
		glog.Fatal(err)
	}
}

func cloneForkRepo(cfg config.Config, repoName string) {
	forkRepoLocation := fmt.Sprintf("https://%s/%s/%s", cfg.GithubHost, cfg.TargetOrg, repoName)
	repoDir := filepath.Join(BaseRepoPath, repoName)

	if _, err := os.Stat(repoDir); err == nil {
		glog.Infof("Fork repository %q already cloned to %s, resetting remote URL ...", repoName, repoDir)
		setUrlCmd := exec.Command("git", "remote", "set-url", "origin", forkRepoLocation)
		setUrlCmd.Dir = repoDir
		run(setUrlCmd)
		os.Remove(filepath.Join(repoDir, ".git", "index.lock"))
		return
	}

	glog.Infof("Cloning fork repository %s ...", forkRepoLocation)
	run(exec.Command("git", "clone", forkRepoLocation))

	// TODO: This can be set as an env variable for the container
	setUsernameCmd := exec.Command("git", "config", "user.name", os.Getenv("GIT_COMMITTER_NAME"))
	setUsernameCmd.Dir = repoDir
	run(setUsernameCmd)

	// TODO: This can be set as an env variable for the container
	setEmailCmd := exec.Command("git", "config", "user.email", os.Getenv("GIT_COMMITTER_EMAIL"))
	setEmailCmd.Dir = repoDir
	run(setEmailCmd)
}

func installGodeps() {
	if _, err := exec.LookPath("godep"); err == nil {
		glog.Infof("Already installed: godep")
		return
	}
	glog.Infof("Installing github.com/tools/godep#%s ...", godepCommit)
	run(exec.Command("go", "get", "github.com/tools/godep"))

	godepDir := filepath.Join(SystemGoPath, "src", "github.com", "tools", "godep")
	godepCheckoutCmd := exec.Command("git", "checkout", godepCommit)
	godepCheckoutCmd.Dir = godepDir
	run(godepCheckoutCmd)

	godepInstallCmd := exec.Command("go", "install", "./...")
	godepInstallCmd.Dir = godepDir
	run(godepInstallCmd)
}

func installDep() {
	if _, err := exec.LookPath("dep"); err == nil {
		glog.Infof("Already installed: dep")
		return
	}
	glog.Infof("Installing github.com/golang/dep#%s ...", depCommit)
	depGoGetCmd := exec.Command("go", "get", "github.com/golang/dep")
	run(depGoGetCmd)

	depDir := filepath.Join(SystemGoPath, "src", "github.com", "golang", "dep")
	depCheckoutCmd := exec.Command("git", "checkout", depCommit)
	depCheckoutCmd.Dir = depDir
	run(depCheckoutCmd)

	depInstallCmd := exec.Command("go", "install", "./cmd/dep")
	depInstallCmd.Dir = depDir
	run(depInstallCmd)
}

// run wraps the cmd.Run() command and sets the standard output and common environment variables.
// if the c.Dir is not set, the BaseRepoPath will be used as a base directory for the command.
func run(c *exec.Cmd) {
	c.Stdout = os.Stdout
	c.Stderr = os.Stderr
	if len(c.Dir) == 0 {
		c.Dir = BaseRepoPath
	}
	if err := c.Run(); err != nil {
		glog.Fatalf("Command %q failed: %v", strings.Join(c.Args, " "), err)
	}
}

func cloneSourceRepo(cfg config.Config, runGodepRestore bool) {
	if _, err := os.Stat(filepath.Join(BaseRepoPath, cfg.SourceRepo)); err == nil {
		glog.Infof("Source repository %q already cloned, skipping", cfg.SourceRepo)
		return
	}

	repoLocation := fmt.Sprintf("https://%s/%s/%s", cfg.GithubHost, cfg.SourceOrg, cfg.SourceRepo)
	glog.Infof("Cloning source repository %s ...", repoLocation)
	cloneCmd := exec.Command("git", "clone", repoLocation)
	run(cloneCmd)

	if runGodepRestore {
		glog.Infof("Running hack/godep-restore.sh ...")
		restoreCmd := exec.Command("bash", "-x", "hack/godep-restore.sh")
		restoreCmd.Dir = filepath.Join(BaseRepoPath, cfg.SourceRepo)
		run(restoreCmd)
	}
}
<commit_msg>Bump default go version to 1.11.2<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/golang/glog"
	yaml "gopkg.in/yaml.v2"

	"k8s.io/publishing-bot/cmd/publishing-bot/config"
)

const (
	depCommit        = "7c44971bbb9f0ed87db40b601f2d9fe4dffb750d"
	godepCommit      = "tags/v80"
	DefaultGoVersion = "1.11.2"
)

var (
	SystemGoPath = os.Getenv("GOPATH")
	BaseRepoPath = filepath.Join(SystemGoPath, "src", "k8s.io")
)

func Usage() {
	fmt.Fprintf(os.Stderr, `
Usage: %s [-config <config-yaml-file>] [-source-repo <repo>] [-source-org <org>] [-rules-file <file> ] [-skip-godep|skip-dep] [-target-org <org>]

Command line flags override config values.
`, os.Args[0])
	flag.PrintDefaults()
}

func main() {
	configFilePath := flag.String("config", "", "the config file in yaml format")
	githubHost := flag.String("github-host", "", "the address of github (defaults to github.com)")
	basePackage := flag.String("base-package", "", "the name of the package base (defaults to k8s.io when source repo is kubernetes, "+
		"otherwise github-host/target-org)")
	repoName := flag.String("source-repo", "", "the name of the source repository (eg. kubernetes)")
	repoOrg := flag.String("source-org", "", "the name of the source repository organization, (eg. kubernetes)")
	rulesFile := flag.String("rules-file", "", "the file with repository rules")
	targetOrg := flag.String("target-org", "", `the target organization to publish into (e.g. "k8s-publishing-bot")`)
	skipGodep := flag.Bool("skip-godep", false, `skip godeps installation and godeps-restore`)
	skipDep := flag.Bool("skip-dep", false, `skip 'dep'' installation`)

	flag.Usage = Usage
	flag.Parse()

	cfg := config.Config{}
	if *configFilePath != "" {
		bs, err := ioutil.ReadFile(*configFilePath)
		if err != nil {
			glog.Fatalf("Failed to load config file from %q: %v", *configFilePath, err)
		}
		if err := yaml.Unmarshal(bs, &cfg); err != nil {
			glog.Fatalf("Failed to parse config file at %q: %v", *configFilePath, err)
		}
	}

	if *targetOrg != "" {
		cfg.TargetOrg = *targetOrg
	}
	if *repoName != "" {
		cfg.SourceRepo = *repoName
	}
	if *repoOrg != "" {
		cfg.SourceOrg = *repoOrg
	}
	if *githubHost != "" {
		cfg.GithubHost = *githubHost
	}
	if *basePackage != "" {
		cfg.BasePackage = *basePackage
	}

	if cfg.GithubHost == "" {
		cfg.GithubHost = "github.com"
	}
	// defaulting when base package is not specified
	if cfg.BasePackage == "" {
		if cfg.SourceRepo == "kubernetes" {
			cfg.BasePackage = "k8s.io"
		} else {
			cfg.BasePackage = filepath.Join(cfg.GithubHost, cfg.TargetOrg)
		}
	}

	BaseRepoPath = filepath.Join(SystemGoPath, "src", cfg.BasePackage)

	if *rulesFile != "" {
		cfg.RulesFile = *rulesFile
	}

	if len(cfg.SourceRepo) == 0 || len(cfg.SourceOrg) == 0 {
		glog.Fatalf("source-org and source-repo cannot be empty")
	}

	if len(cfg.TargetOrg) == 0 {
		glog.Fatalf("Target organization cannot be empty")
	}

	// If RULE_FILE_PATH is detected, check if the source repository include rules files.
	if len(os.Getenv("RULE_FILE_PATH")) > 0 {
		cfg.RulesFile = filepath.Join(BaseRepoPath, cfg.SourceRepo, os.Getenv("RULE_FILE_PATH"))
	}

	if len(cfg.RulesFile) == 0 {
		glog.Fatalf("No rules file provided")
	}
	rules, err := config.LoadRules(cfg.RulesFile)
	if err != nil {
		glog.Fatalf("Failed to load rules: %v", err)
	}

	goVersions := []string{DefaultGoVersion}
	for _, rule := range rules.Rules {
		for _, branch := range rule.Branches {
			if branch.GoVersion != "" {
				found := false
				for _, v := range goVersions {
					if v == branch.GoVersion {
						found = true
					}
				}
				if !found {
					goVersions = append(goVersions, branch.GoVersion)
				}
			}
		}
	}
	for _, v := range goVersions {
		installGoVersion(v, filepath.Join(SystemGoPath, "go-"+v))
	}
	goLink, target := filepath.Join(SystemGoPath, "go"), filepath.Join(SystemGoPath, "go-"+DefaultGoVersion)
	os.Remove(goLink)
	if err := os.Symlink(target, goLink); err != nil {
		glog.Fatalf("Failed to link %s to %s: %s", goLink, target, err)
	}

	if err := os.MkdirAll(BaseRepoPath, os.ModePerm); err != nil {
		glog.Fatalf("Failed to create source repo directory %s: %v", BaseRepoPath, err)
	}

	if !*skipGodep {
		installGodeps()
	}
	if !*skipDep {
		installDep()
	}

	cloneSourceRepo(cfg, *skipGodep)
	for _, rule := range rules.Rules {
		cloneForkRepo(cfg, rule.DestinationRepository)
	}
}

func installGoVersion(v string, pth string) {
	if s, err := os.Stat(pth); err != nil && !os.IsNotExist(err) {
		glog.Fatal(err)
	} else if err == nil {
		if s.IsDir() {
			glog.Infof("Found existing go %s at %s", v, pth)
			return
		}
		glog.Fatalf("Expected %s to be a directory", pth)
	}

	glog.Infof("Installing go %s to %s", v, pth)
	tmpPath, err := ioutil.TempDir(SystemGoPath, "go-tmp-")
	if err != nil {
		glog.Fatal(err)
	}
	defer os.RemoveAll(tmpPath)
	cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("curl -SLf https://storage.googleapis.com/golang/go%s.linux-amd64.tar.gz | tar -xz --strip 1 -C %s", v, tmpPath))
	cmd.Dir = tmpPath
	run(cmd)
	if err := os.Rename(tmpPath, pth); err != nil {
		glog.Fatal(err)
	}
}

func cloneForkRepo(cfg config.Config, repoName string) {
	forkRepoLocation := fmt.Sprintf("https://%s/%s/%s", cfg.GithubHost, cfg.TargetOrg, repoName)
	repoDir := filepath.Join(BaseRepoPath, repoName)

	if _, err := os.Stat(repoDir); err == nil {
		glog.Infof("Fork repository %q already cloned to %s, resetting remote URL ...", repoName, repoDir)
		setUrlCmd := exec.Command("git", "remote", "set-url", "origin", forkRepoLocation)
		setUrlCmd.Dir = repoDir
		run(setUrlCmd)
		os.Remove(filepath.Join(repoDir, ".git", "index.lock"))
		return
	}

	glog.Infof("Cloning fork repository %s ...", forkRepoLocation)
	run(exec.Command("git", "clone", forkRepoLocation))

	// TODO: This can be set as an env variable for the container
	setUsernameCmd := exec.Command("git", "config", "user.name", os.Getenv("GIT_COMMITTER_NAME"))
	setUsernameCmd.Dir = repoDir
	run(setUsernameCmd)

	// TODO: This can be set as an env variable for the container
	setEmailCmd := exec.Command("git", "config", "user.email", os.Getenv("GIT_COMMITTER_EMAIL"))
	setEmailCmd.Dir = repoDir
	run(setEmailCmd)
}

func installGodeps() {
	if _, err := exec.LookPath("godep"); err == nil {
		glog.Infof("Already installed: godep")
		return
	}
	glog.Infof("Installing github.com/tools/godep#%s ...", godepCommit)
	run(exec.Command("go", "get", "github.com/tools/godep"))

	godepDir := filepath.Join(SystemGoPath, "src", "github.com", "tools", "godep")
	godepCheckoutCmd := exec.Command("git", "checkout", godepCommit)
	godepCheckoutCmd.Dir = godepDir
	run(godepCheckoutCmd)

	godepInstallCmd := exec.Command("go", "install", "./...")
	godepInstallCmd.Dir = godepDir
	run(godepInstallCmd)
}

func installDep() {
	if _, err := exec.LookPath("dep"); err == nil {
		glog.Infof("Already installed: dep")
		return
	}
	glog.Infof("Installing github.com/golang/dep#%s ...", depCommit)
	depGoGetCmd := exec.Command("go", "get", "github.com/golang/dep")
	run(depGoGetCmd)

	depDir := filepath.Join(SystemGoPath, "src", "github.com", "golang", "dep")
	depCheckoutCmd := exec.Command("git", "checkout", depCommit)
	depCheckoutCmd.Dir = depDir
	run(depCheckoutCmd)

	depInstallCmd := exec.Command("go", "install", "./cmd/dep")
	depInstallCmd.Dir = depDir
	run(depInstallCmd)
}

// run wraps the cmd.Run() command and sets the standard output and common environment variables.
// if the c.Dir is not set, the BaseRepoPath will be used as a base directory for the command.
func run(c *exec.Cmd) {
	c.Stdout = os.Stdout
	c.Stderr = os.Stderr
	if len(c.Dir) == 0 {
		c.Dir = BaseRepoPath
	}
	if err := c.Run(); err != nil {
		glog.Fatalf("Command %q failed: %v", strings.Join(c.Args, " "), err)
	}
}

func cloneSourceRepo(cfg config.Config, runGodepRestore bool) {
	if _, err := os.Stat(filepath.Join(BaseRepoPath, cfg.SourceRepo)); err == nil {
		glog.Infof("Source repository %q already cloned, skipping", cfg.SourceRepo)
		return
	}

	repoLocation := fmt.Sprintf("https://%s/%s/%s", cfg.GithubHost, cfg.SourceOrg, cfg.SourceRepo)
	glog.Infof("Cloning source repository %s ...", repoLocation)
	cloneCmd := exec.Command("git", "clone", repoLocation)
	run(cloneCmd)

	if runGodepRestore {
		glog.Infof("Running hack/godep-restore.sh ...")
		restoreCmd := exec.Command("bash", "-x", "hack/godep-restore.sh")
		restoreCmd.Dir = filepath.Join(BaseRepoPath, cfg.SourceRepo)
		run(restoreCmd)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package runtime

import (
	"fmt"
	"runtime"
	"sync"
	"time"

	"github.com/golang/glog"
)

var (
	// ReallyCrash controls the behavior of HandleCrash and now defaults
	// true. It's still exposed so components can optionally set to false
	// to restore prior behavior.
	ReallyCrash = true
)

// PanicHandlers is a list of functions which will be invoked when a panic happens.
var PanicHandlers = []func(interface{}){logPanic}

// HandleCrash simply catches a crash and logs an error. Meant to be called via
// defer.  Additional context-specific handlers can be provided, and will be
// called in case of panic.  HandleCrash actually crashes, after calling the
// handlers and logging the panic message.
//
// TODO: remove this function. We are switching to a world where it's safe for
// apiserver to panic, since it will be restarted by kubelet. At the beginning
// of the Kubernetes project, nothing was going to restart apiserver and so
// catching panics was important. But it's actually much simpler for montoring
// software if we just exit when an unexpected panic happens.
func HandleCrash(additionalHandlers ...func(interface{})) {
	if r := recover(); r != nil {
		for _, fn := range PanicHandlers {
			fn(r)
		}
		for _, fn := range additionalHandlers {
			fn(r)
		}
		if ReallyCrash {
			// Actually proceed to panic.
			panic(r)
		}
	}
}

// logPanic logs the caller tree when a panic occurs.
func logPanic(r interface{}) {
	callers := getCallers(r)
	glog.Errorf("Observed a panic: %#v (%v)\n%v", r, r, callers)
}

func getCallers(r interface{}) string {
	callers := ""
	for i := 0; true; i++ {
		_, file, line, ok := runtime.Caller(i)
		if !ok {
			break
		}
		callers = callers + fmt.Sprintf("%v:%v\n", file, line)
	}

	return callers
}

// ErrorHandlers is a list of functions which will be invoked when an unreturnable
// error occurs.
// TODO(lavalamp): for testability, this and the below HandleError function
// should be packaged up into a testable and reusable object.
var ErrorHandlers = []func(error){
	logError,
	(&rudimentaryErrorBackoff{
		lastErrorTime: time.Now(),
		minPeriod:     500 * time.Millisecond,
	}).OnError,
}

// HandlerError is a method to invoke when a non-user facing piece of code cannot
// return an error and needs to indicate it has been ignored. Invoking this method
// is preferable to logging the error - the default behavior is to log but the
// errors may be sent to a remote server for analysis.
func HandleError(err error) {
	// this is sometimes called with a nil error.  We probably shouldn't fail and should do nothing instead
	if err == nil {
		return
	}

	for _, fn := range ErrorHandlers {
		fn(err)
	}
}

// logError prints an error with the call stack of the location it was reported
func logError(err error) {
	glog.ErrorDepth(2, err)
}

type rudimentaryErrorBackoff struct {
	minPeriod time.Duration // immutable
	// TODO(lavalamp): use the clock for testability. Need to move that
	// package for that to be accessible here.
	lastErrorTimeLock sync.Mutex
	lastErrorTime     time.Time
}

// OnError will block if it is called more often than the embedded period time.
// This will prevent overly tight hot error loops.
func (r *rudimentaryErrorBackoff) OnError(error) {
	r.lastErrorTimeLock.Lock()
	defer r.lastErrorTimeLock.Unlock()
	d := time.Since(r.lastErrorTime)
	if d < r.minPeriod {
		time.Sleep(r.minPeriod - d)
	}
	r.lastErrorTime = time.Now()
}

// GetCaller returns the caller of the function that calls it.
func GetCaller() string {
	var pc [1]uintptr
	runtime.Callers(3, pc[:])
	f := runtime.FuncForPC(pc[0])
	if f == nil {
		return fmt.Sprintf("Unable to find caller")
	}
	return f.Name()
}

// RecoverFromPanic replaces the specified error with an error containing the
// original error, and  the call tree when a panic occurs. This enables error
// handlers to handle errors and panics the same way.
func RecoverFromPanic(err *error) {
	if r := recover(); r != nil {
		callers := getCallers(r)

		*err = fmt.Errorf(
			"recovered from panic %q. (err=%v) Call stack:\n%v",
			r,
			*err,
			callers)
	}
}
<commit_msg>Adjust global log limit to 1ms<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package runtime

import (
	"fmt"
	"runtime"
	"sync"
	"time"

	"github.com/golang/glog"
)

var (
	// ReallyCrash controls the behavior of HandleCrash and now defaults
	// true. It's still exposed so components can optionally set to false
	// to restore prior behavior.
	ReallyCrash = true
)

// PanicHandlers is a list of functions which will be invoked when a panic happens.
var PanicHandlers = []func(interface{}){logPanic}

// HandleCrash simply catches a crash and logs an error. Meant to be called via
// defer.  Additional context-specific handlers can be provided, and will be
// called in case of panic.  HandleCrash actually crashes, after calling the
// handlers and logging the panic message.
//
// TODO: remove this function. We are switching to a world where it's safe for
// apiserver to panic, since it will be restarted by kubelet. At the beginning
// of the Kubernetes project, nothing was going to restart apiserver and so
// catching panics was important. But it's actually much simpler for montoring
// software if we just exit when an unexpected panic happens.
func HandleCrash(additionalHandlers ...func(interface{})) {
	if r := recover(); r != nil {
		for _, fn := range PanicHandlers {
			fn(r)
		}
		for _, fn := range additionalHandlers {
			fn(r)
		}
		if ReallyCrash {
			// Actually proceed to panic.
			panic(r)
		}
	}
}

// logPanic logs the caller tree when a panic occurs.
func logPanic(r interface{}) {
	callers := getCallers(r)
	glog.Errorf("Observed a panic: %#v (%v)\n%v", r, r, callers)
}

func getCallers(r interface{}) string {
	callers := ""
	for i := 0; true; i++ {
		_, file, line, ok := runtime.Caller(i)
		if !ok {
			break
		}
		callers = callers + fmt.Sprintf("%v:%v\n", file, line)
	}

	return callers
}

// ErrorHandlers is a list of functions which will be invoked when an unreturnable
// error occurs.
// TODO(lavalamp): for testability, this and the below HandleError function
// should be packaged up into a testable and reusable object.
var ErrorHandlers = []func(error){
	logError,
	(&rudimentaryErrorBackoff{
		lastErrorTime: time.Now(),
		// 1ms was the number folks were able to stomach as a global rate limit.
		// If you need to log errors more than 1000 times a second you
		// should probably consider fixing your code instead. :)
		minPeriod: time.Millisecond,
	}).OnError,
}

// HandlerError is a method to invoke when a non-user facing piece of code cannot
// return an error and needs to indicate it has been ignored. Invoking this method
// is preferable to logging the error - the default behavior is to log but the
// errors may be sent to a remote server for analysis.
func HandleError(err error) {
	// this is sometimes called with a nil error.  We probably shouldn't fail and should do nothing instead
	if err == nil {
		return
	}

	for _, fn := range ErrorHandlers {
		fn(err)
	}
}

// logError prints an error with the call stack of the location it was reported
func logError(err error) {
	glog.ErrorDepth(2, err)
}

type rudimentaryErrorBackoff struct {
	minPeriod time.Duration // immutable
	// TODO(lavalamp): use the clock for testability. Need to move that
	// package for that to be accessible here.
	lastErrorTimeLock sync.Mutex
	lastErrorTime     time.Time
}

// OnError will block if it is called more often than the embedded period time.
// This will prevent overly tight hot error loops.
func (r *rudimentaryErrorBackoff) OnError(error) {
	r.lastErrorTimeLock.Lock()
	defer r.lastErrorTimeLock.Unlock()
	d := time.Since(r.lastErrorTime)
	if d < r.minPeriod {
		time.Sleep(r.minPeriod - d)
	}
	r.lastErrorTime = time.Now()
}

// GetCaller returns the caller of the function that calls it.
func GetCaller() string {
	var pc [1]uintptr
	runtime.Callers(3, pc[:])
	f := runtime.FuncForPC(pc[0])
	if f == nil {
		return fmt.Sprintf("Unable to find caller")
	}
	return f.Name()
}

// RecoverFromPanic replaces the specified error with an error containing the
// original error, and  the call tree when a panic occurs. This enables error
// handlers to handle errors and panics the same way.
func RecoverFromPanic(err *error) {
	if r := recover(); r != nil {
		callers := getCallers(r)

		*err = fmt.Errorf(
			"recovered from panic %q. (err=%v) Call stack:\n%v",
			r,
			*err,
			callers)
	}
}
<|endoftext|>


<commit_before>package redis

import (
	"fmt"
	"net"
	"strings"
	"time"

	"golang.org/x/net/context"

	"github.com/CodisLabs/codis/pkg/utils/errors"
	"github.com/CodisLabs/codis/pkg/utils/log"

	redigo "github.com/garyburd/redigo/redis"
)

type Sentinel struct {
	context.Context
	Cancel context.CancelFunc

	product, auth string
}

func NewSentinel(product string) *Sentinel {
	return NewSentinelWithAuth(product, "")
}

func NewSentinelWithAuth(product, auth string) *Sentinel {
	s := &Sentinel{product: product, auth: auth}
	s.Context, s.Cancel = context.WithCancel(context.Background())
	return s
}

func (s *Sentinel) IsCancelled() bool {
	select {
	case <-s.Context.Done():
		return true
	default:
		return false
	}
}

func (s *Sentinel) AfterSeconds(n int) {
	if n == 0 {
		return
	}
	select {
	case <-s.Context.Done():
	case <-time.After(time.Second * time.Duration(n)):
	}
}

func (s *Sentinel) MasterName(gid int) string {
	return fmt.Sprintf("%s-%d", s.product, gid)
}

func (s *Sentinel) newSentinelClient(sentinel string, timeout time.Duration) (*Client, error) {
	return NewClient(sentinel, "", timeout)
}

func (s *Sentinel) SubscribeOne(ctx context.Context, sentinel string) (bool, error) {
	c, err := s.newSentinelClient(sentinel, time.Minute*30)
	if err != nil {
		return false, err
	}
	defer c.Close()

	var ech = make(chan error, 1)
	go func() (err error) {
		defer func() {
			ech <- err
		}()
		if err := c.Flush("SUBSCRIBE", "+switch-master"); err != nil {
			return err
		}
		for {
			switch r, err := redigo.Strings(c.Receive()); {
			case err != nil:
				return errors.Trace(err)
			case len(r) != 3:
				return errors.Errorf("invalid response = %v", r)
			case strings.HasPrefix(r[2], s.product):
				return nil
			}
		}
	}()

	select {
	case <-ctx.Done():
		return false, nil
	case err := <-ech:
		if err != nil {
			return false, err
		}
		return true, nil
	}
}

func (s *Sentinel) SubscribeMulti(ctx context.Context, sentinels []string) bool {
	if len(sentinels) == 0 {
		return false
	}
	nctx, cancel := context.WithCancel(ctx)
	defer cancel()

	var results = make(chan bool, len(sentinels))
	for i := range sentinels {
		go func(sentinel string) {
			notified, err := s.SubscribeOne(nctx, sentinel)
			if err != nil {
				log.WarnErrorf(err, "sentinel %s subscribe failed", sentinel)
			}
			if notified {
				log.Warnf("sentinel %s event +switch-master", sentinel)
			}
			results <- notified
		}(sentinels[i])
	}

	var majority = 1 + len(sentinels)/2

	for i := 0; i < majority; i++ {
		select {
		case <-ctx.Done():
			return false
		case notified := <-results:
			if notified {
				return true
			}
		}
	}
	return false
}

func (s *Sentinel) getServerRole(addr string) (string, error) {
	c, err := NewClient(addr, s.auth, time.Second*5)
	if err != nil {
		return "", err
	}
	defer c.Close()
	return c.Role()
}

func (s *Sentinel) MastersOne(ctx context.Context, sentinel string, groupIds map[int]bool) (map[int]string, error) {
	c, err := s.newSentinelClient(sentinel, time.Second*10)
	if err != nil {
		return nil, err
	}
	defer c.Close()

	var gmp = make(map[int]string)
	var ech = make(chan error, 1)
	go func() (err error) {
		defer func() {
			ech <- err
		}()
		for gid := range groupIds {
			switch r, err := redigo.Strings(c.Do("SENTINEL", "get-master-addr-by-name", s.MasterName(gid))); {
			case err != nil:
				return errors.Trace(err)
			case len(r) == 2:
				var addr = fmt.Sprintf("%s:%s", r[0], r[1])
				if role, err := s.getServerRole(addr); err != nil {
					log.WarnErrorf(err, "sentinel get role of %s failed", addr)
				} else if role == "MASTER" {
					gmp[gid] = addr
				}
			case len(r) != 0:
				return errors.Errorf("invalid response = %v", r)
			}
		}
		return nil
	}()

	select {
	case <-ctx.Done():
		return nil, nil
	case err := <-ech:
		if err != nil {
			return nil, err
		}
		return gmp, nil
	}
}

func (s *Sentinel) MastersMulti(ctx context.Context, sentinels []string, groupIds map[int]bool) map[int]string {
	if len(sentinels) == 0 || len(groupIds) == 0 {
		return map[int]string{}
	}
	nctx, cancel := context.WithCancel(ctx)
	defer cancel()

	var results = make(chan map[int]string, len(sentinels))
	for i := range sentinels {
		go func(sentinel string) {
			m, err := s.MastersOne(nctx, sentinel, groupIds)
			if err != nil {
				log.WarnErrorf(err, "sentinel %s masters failed", sentinel)
			}
			results <- m
		}(sentinels[i])
	}

	var masters = make(map[int]string)
	var counter = make(map[int]int)
	for i := 0; i < len(sentinels); i++ {
		select {
		case <-ctx.Done():
			return nil
		case m := <-results:
			if m != nil {
				for gid, addr := range m {
					if masters[gid] == addr {
						counter[gid]++
						continue
					}
					switch counter[gid] {
					case 0:
						masters[gid] = addr
						counter[gid]++
					case 1:
						delete(masters, gid)
						fallthrough
					default:
						counter[gid]--
					}
				}
			}
		}
	}
	return masters
}

func (s *Sentinel) MonitorOne(sentinel string, masters map[int]string, quorum int, overwrite bool) error {
	c, err := s.newSentinelClient(sentinel, time.Second*10)
	if err != nil {
		return err
	}
	defer c.Close()
	for gid, master := range masters {
		host, port, err := net.SplitHostPort(master)
		if err != nil {
			return errors.Trace(err)
		}
		name := s.MasterName(gid)
		if overwrite {
			_, err := c.Do("SENTINEL", "remove", name)
			if err != nil {
				return err
			}
		}
		if _, err := redigo.String(c.Do("SENTINEL", "monitor", name, host, port, quorum)); err != nil {
			return errors.Trace(err)
		}
		if s.auth == "" {
			continue
		}
		if _, err := redigo.String(c.Do("SENTINEL", "set", name, "auth-pass", s.auth)); err != nil {
			return errors.Trace(err)
		}
	}
	return nil
}

func (s *Sentinel) Monitor(sentinels []string, masters map[int]string, quorum int, overwrite bool) error {
	if len(sentinels) == 0 {
		return nil
	}

	var results = make(chan error, len(sentinels))
	for i := range sentinels {
		go func(sentinel string) {
			err := s.MonitorOne(sentinel, masters, quorum, overwrite)
			if err != nil {
				log.WarnErrorf(err, "sentinel %s monitor failed", sentinel)
			}
			results <- err
		}(sentinels[i])
	}

	for i := 0; i < len(sentinels); i++ {
		if err := <-results; err != nil {
			return err
		}
	}
	return nil
}

func (s *Sentinel) RemoveMonitor(sentinel string, groups ...int) error {
	c, err := s.newSentinelClient(sentinel, time.Second*10)
	if err != nil {
		return err
	}
	defer c.Close()
	for gid := range groups {
		_, err := c.Do("SENTINEL", "remove", s.MasterName(gid))
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>sentinel: handle unexcepted response<commit_after>package redis

import (
	"fmt"
	"net"
	"strings"
	"time"

	"golang.org/x/net/context"

	"github.com/CodisLabs/codis/pkg/utils/errors"
	"github.com/CodisLabs/codis/pkg/utils/log"

	redigo "github.com/garyburd/redigo/redis"
)

type Sentinel struct {
	context.Context
	Cancel context.CancelFunc

	product, auth string
}

func NewSentinel(product string) *Sentinel {
	return NewSentinelWithAuth(product, "")
}

func NewSentinelWithAuth(product, auth string) *Sentinel {
	s := &Sentinel{product: product, auth: auth}
	s.Context, s.Cancel = context.WithCancel(context.Background())
	return s
}

func (s *Sentinel) IsCancelled() bool {
	select {
	case <-s.Context.Done():
		return true
	default:
		return false
	}
}

func (s *Sentinel) AfterSeconds(n int) {
	if n == 0 {
		return
	}
	select {
	case <-s.Context.Done():
	case <-time.After(time.Second * time.Duration(n)):
	}
}

func (s *Sentinel) MasterName(gid int) string {
	return fmt.Sprintf("%s-%d", s.product, gid)
}

func (s *Sentinel) newSentinelClient(sentinel string, timeout time.Duration) (*Client, error) {
	return NewClient(sentinel, "", timeout)
}

func (s *Sentinel) SubscribeOne(ctx context.Context, sentinel string) (bool, error) {
	c, err := s.newSentinelClient(sentinel, time.Minute*30)
	if err != nil {
		return false, err
	}
	defer c.Close()

	var ech = make(chan error, 1)
	go func() (err error) {
		defer func() {
			ech <- err
		}()
		if err := c.Flush("SUBSCRIBE", "+switch-master"); err != nil {
			return err
		}
		for {
			reply, err := redigo.Values(c.Receive())
			if err != nil {
				return errors.Trace(err)
			}
			if msg, err := redigo.String(reply[0], nil); err != nil {
				return errors.Trace(err)
			} else if msg != "message" {
				continue
			}
			if evt, err := redigo.String(reply[1], nil); err != nil {
				return errors.Trace(err)
			} else if evt != "+switch-master" {
				continue
			}
			if len(reply) != 3 {
				return errors.Errorf("invalid response = %v", reply)
			}
			name, err := redigo.String(reply[2], nil)
			if err != nil {
				return errors.Trace(err)
			}
			if strings.HasPrefix(name, s.product) {
				return nil
			}
		}
	}()

	select {
	case <-ctx.Done():
		return false, nil
	case err := <-ech:
		if err != nil {
			return false, err
		}
		return true, nil
	}
}

func (s *Sentinel) SubscribeMulti(ctx context.Context, sentinels []string) bool {
	if len(sentinels) == 0 {
		return false
	}
	nctx, cancel := context.WithCancel(ctx)
	defer cancel()

	var results = make(chan bool, len(sentinels))
	for i := range sentinels {
		go func(sentinel string) {
			notified, err := s.SubscribeOne(nctx, sentinel)
			if err != nil {
				log.WarnErrorf(err, "sentinel %s subscribe failed", sentinel)
			}
			if notified {
				log.Warnf("sentinel %s event +switch-master", sentinel)
			}
			results <- notified
		}(sentinels[i])
	}

	var majority = 1 + len(sentinels)/2

	for i := 0; i < majority; i++ {
		select {
		case <-ctx.Done():
			return false
		case notified := <-results:
			if notified {
				return true
			}
		}
	}
	return false
}

func (s *Sentinel) getServerRole(addr string) (string, error) {
	c, err := NewClient(addr, s.auth, time.Second*5)
	if err != nil {
		return "", err
	}
	defer c.Close()
	return c.Role()
}

func (s *Sentinel) MastersOne(ctx context.Context, sentinel string, groupIds map[int]bool) (map[int]string, error) {
	c, err := s.newSentinelClient(sentinel, time.Second*10)
	if err != nil {
		return nil, err
	}
	defer c.Close()

	var gmp = make(map[int]string)
	var ech = make(chan error, 1)
	go func() (err error) {
		defer func() {
			ech <- err
		}()
		for gid := range groupIds {
			reply, err := c.Do("SENTINEL", "get-master-addr-by-name", s.MasterName(gid))
			if err != nil {
				return errors.Trace(err)
			}
			if reply == nil {
				continue
			}
			r, err := redigo.Strings(reply, nil)
			if err != nil {
				return errors.Trace(err)
			}
			if len(r) != 2 {
				return errors.Errorf("invalid response = %v", r)
			}
			var addr = fmt.Sprintf("%s:%s", r[0], r[1])
			if role, err := s.getServerRole(addr); err != nil {
				log.WarnErrorf(err, "sentinel get role of %s failed", addr)
			} else if role == "MASTER" {
				gmp[gid] = addr
			}
		}
		return nil
	}()

	select {
	case <-ctx.Done():
		return nil, nil
	case err := <-ech:
		if err != nil {
			return nil, err
		}
		return gmp, nil
	}
}

func (s *Sentinel) MastersMulti(ctx context.Context, sentinels []string, groupIds map[int]bool) map[int]string {
	if len(sentinels) == 0 || len(groupIds) == 0 {
		return map[int]string{}
	}
	nctx, cancel := context.WithCancel(ctx)
	defer cancel()

	var results = make(chan map[int]string, len(sentinels))
	for i := range sentinels {
		go func(sentinel string) {
			m, err := s.MastersOne(nctx, sentinel, groupIds)
			if err != nil {
				log.WarnErrorf(err, "sentinel %s masters failed", sentinel)
			}
			results <- m
		}(sentinels[i])
	}

	var masters = make(map[int]string)
	var counter = make(map[int]int)
	for i := 0; i < len(sentinels); i++ {
		select {
		case <-ctx.Done():
			return nil
		case m := <-results:
			if m != nil {
				for gid, addr := range m {
					if masters[gid] == addr {
						counter[gid]++
						continue
					}
					switch counter[gid] {
					case 0:
						masters[gid] = addr
						counter[gid]++
					case 1:
						delete(masters, gid)
						fallthrough
					default:
						counter[gid]--
					}
				}
			}
		}
	}
	return masters
}

func (s *Sentinel) MonitorOne(sentinel string, masters map[int]string, quorum int, overwrite bool) error {
	c, err := s.newSentinelClient(sentinel, time.Second*10)
	if err != nil {
		return err
	}
	defer c.Close()
	for gid, master := range masters {
		host, port, err := net.SplitHostPort(master)
		if err != nil {
			return errors.Trace(err)
		}
		name := s.MasterName(gid)
		if overwrite {
			_, err := c.Do("SENTINEL", "remove", name)
			if err != nil {
				return err
			}
		}
		if _, err := redigo.String(c.Do("SENTINEL", "monitor", name, host, port, quorum)); err != nil {
			return errors.Trace(err)
		}
		if s.auth == "" {
			continue
		}
		if _, err := redigo.String(c.Do("SENTINEL", "set", name, "auth-pass", s.auth)); err != nil {
			return errors.Trace(err)
		}
	}
	return nil
}

func (s *Sentinel) Monitor(sentinels []string, masters map[int]string, quorum int, overwrite bool) error {
	if len(sentinels) == 0 {
		return nil
	}

	var results = make(chan error, len(sentinels))
	for i := range sentinels {
		go func(sentinel string) {
			err := s.MonitorOne(sentinel, masters, quorum, overwrite)
			if err != nil {
				log.WarnErrorf(err, "sentinel %s monitor failed", sentinel)
			}
			results <- err
		}(sentinels[i])
	}

	for i := 0; i < len(sentinels); i++ {
		if err := <-results; err != nil {
			return err
		}
	}
	return nil
}

func (s *Sentinel) RemoveMonitor(sentinel string, groups ...int) error {
	c, err := s.newSentinelClient(sentinel, time.Second*10)
	if err != nil {
		return err
	}
	defer c.Close()
	for gid := range groups {
		_, err := c.Do("SENTINEL", "remove", s.MasterName(gid))
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"context"
	"fmt"
	"net/http"
	"os/exec"
	"strings"
	"time"

	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

// Github represents a control version repository to
// interact with github.com
type Github struct {
	client *github.Client
	owner  string
	repo   string
	url    string
}

const (
	timeoutShortRequest = 10 * time.Second
	timeoutLongRequest  = 20 * time.Second
)

// newGithub returns an object of type Github
func newGithub(url, token string) (CVR, error) {
	url = strings.TrimSpace(url)

	ownerRepo := strings.SplitAfter(url, "/"+githubDomain+"/")

	// at least we need two tokens
	if len(ownerRepo) < 2 {
		return nil, fmt.Errorf("missing owner and repo %s", url)
	}

	ownerRepo = strings.Split(ownerRepo[1], "/")

	// at least we need two tokens: owner and repo
	if len(ownerRepo) < 2 {
		return nil, fmt.Errorf("failed to get owner and repo %s", url)
	}

	if len(ownerRepo[0]) == 0 {
		return nil, fmt.Errorf("missing owner in url %s", url)
	}

	if len(ownerRepo[1]) == 0 {
		return nil, fmt.Errorf("missing repository in url %s", url)
	}

	// create a new http client using the token
	var client *http.Client
	if token != "" {
		ts := oauth2.StaticTokenSource(
			&oauth2.Token{AccessToken: token},
		)
		client = oauth2.NewClient(context.Background(), ts)
	}

	return &Github{
		client: github.NewClient(client),
		owner:  ownerRepo[0],
		repo:   ownerRepo[1],
		url:    url,
	}, nil
}

// getProjectSlug returns the domain, owner and repo name separated by '/'
func (g *Github) getProjectSlug() string {
	return fmt.Sprintf("%s/%s/%s", githubDomain, g.owner, g.repo)
}

// getPullRequestCommits returns the commits of a pull request
func (g *Github) getPullRequestCommits(pr int) ([]repoCommit, error) {
	ctx, cancel := context.WithTimeout(context.Background(), timeoutShortRequest)
	defer cancel()

	// get all commits of the pull request
	listCommits, _, err := g.client.PullRequests.ListCommits(ctx, g.owner, g.repo, pr, nil)
	if err != nil {
		return nil, err
	}

	var commits []repoCommit
	for _, c := range listCommits {
		if c == nil {
			return nil, fmt.Errorf("failed to get all commits of the pull request %d", pr)
		}

		if c.SHA == nil {
			return nil, fmt.Errorf("failed to get commit SHA of the pull request %d", pr)
		}
		sha := *c.SHA

		if c.Commit == nil || c.Commit.Committer == nil || c.Commit.Committer.Date == nil {
			return nil, fmt.Errorf("failed to get commit time of the pull request %d", pr)
		}
		time := *c.Commit.Committer.Date

		commits = append(commits,
			repoCommit{
				sha:  sha,
				time: time,
			},
		)
	}

	return commits, nil
}

// getOpenPullRequests returns the open pull requests
func (g *Github) getOpenPullRequests() ([]int, error) {
	ctx, cancel := context.WithTimeout(context.Background(), timeoutShortRequest)
	defer cancel()

	pullRequests, _, err := g.client.PullRequests.List(ctx, g.owner, g.repo, nil)
	if err != nil {
		return nil, fmt.Errorf("failed to list pull requests %s", err)
	}

	prs := []int{}

	for _, pr := range pullRequests {
		if pr == nil || pr.Number == nil {
			continue
		}

		prs = append(prs, *pr.Number)
	}

	return prs, nil
}

// getBranchInfo returns a specific branch
func (g *Github) getBranchInfo(branch string) (repoBranchInfo, error) {
	i := repoBranchInfo{}

	ctx, cancel := context.WithTimeout(context.Background(), timeoutShortRequest)
	defer cancel()

	b, _, err := g.client.Repositories.GetBranch(ctx, g.owner, g.repo, branch)
	if err != nil {
		return i, err
	}

	if b.Commit == nil && b.Commit.SHA == nil {
		return i, fmt.Errorf("failed to get commit sha of branch %s", branch)
	}

	return repoBranchInfo{
		sha: *b.Commit.SHA,
	}, nil
}

// getPullRequestInfo return information about a pull request
func (g *Github) getPullRequestInfo(pr int) (pullRequestInfo, error) {
	i := pullRequestInfo{}

	ctx, cancel := context.WithTimeout(context.Background(), timeoutShortRequest)
	defer cancel()

	pullRequest, _, err := g.client.PullRequests.Get(ctx, g.owner, g.repo, pr)
	if err != nil {
		return i, err
	}

	// check the integrity of the pullRuest object before use it
	if pullRequest == nil {
		return i, fmt.Errorf("failed to get pull request %d", pr)
	}

	if pullRequest.User == nil || pullRequest.User.Login == nil {
		return i, fmt.Errorf("failed to get the author of the pull request %d", pr)
	}

	author := *pullRequest.User.Login

	// do not fail if we don't know if the pull request is
	// mergeable, just test it
	mergeable := true
	if pullRequest.Mergeable != nil {
		mergeable = *pullRequest.Mergeable
	}

	// include the state to check it later
	state := ""
	if pullRequest.State != nil {
		state = *pullRequest.State
	}

	if pullRequest.Head == nil || pullRequest.Head.Ref == nil {
		return i, fmt.Errorf("failed to get the branch name of the pull request %d", pr)
	}

	branch := *pullRequest.Head.Ref

	return pullRequestInfo{
		branch:    branch,
		author:    author,
		mergeable: mergeable,
		state:     state,
	}, nil
}

// getLatestPullRequestComment returns the latest comment of a specific
// user in the specific pr. If user is an empty string then any user
// could be the author of the latest comment. If body is an empty
// string an error is returned.
func (g *Github) getLatestPullRequestComment(pr int, user, body string) (RepoComment, error) {
	c := RepoComment{}

	if len(body) == 0 {
		return c, fmt.Errorf("body cannot be an empty string")
	}

	ctx, cancel := context.WithTimeout(context.Background(), timeoutLongRequest)
	defer cancel()

	comments, _, err := g.client.Issues.ListComments(ctx, g.owner, g.repo, pr, nil)
	if err != nil {
		return c, err
	}

	for i := len(comments) - 1; i >= 0; i-- {
		c := comments[i]
		if len(user) != 0 {
			if *c.User.Login != user {
				continue
			}
		}

		if *c.Body == body {
			return RepoComment{
				User:    user,
				Comment: body,
				time:    *c.CreatedAt,
			}, nil
		}
	}

	return c, fmt.Errorf("comment '%s' not found", body)
}

func (g *Github) downloadRepo(workingDir string) error {
	var stderr bytes.Buffer

	// clone the project
	cmd := exec.Command("git", "clone", g.url, ".")
	cmd.Dir = workingDir
	cmd.Stderr = &stderr
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("failed to run git clone %s %s", stderr.String(), err)
	}

	return nil
}

func (g *Github) checkoutBranch(branch string, workingDir string) error {
	var stderr bytes.Buffer

	// checkout the branch
	stderr.Reset()
	cmd := exec.Command("git", "checkout", branch)
	cmd.Dir = workingDir
	cmd.Stderr = &stderr
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("failed to run git checkout %s %s", stderr.String(), err)
	}

	return nil
}

func (g *Github) downloadBranch(branch string, workingDir string) error {
	if err := g.downloadRepo(workingDir); err != nil {
		return err
	}

	if err := g.checkoutBranch(branch, workingDir); err != nil {
		return err
	}

	return nil
}

func (g *Github) downloadPullRequest(pr int, branch string, workingDir string) error {
	var stderr bytes.Buffer

	if err := g.downloadRepo(workingDir); err != nil {
		return err
	}

	// fetch the branch
	stderr.Reset()
	cmd := exec.Command("git", "fetch", "origin", fmt.Sprintf("pull/%d/head:%s", pr, branch))
	cmd.Dir = workingDir
	cmd.Stderr = &stderr
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("failed to run git fetch %s %s", stderr.String(), err)
	}

	if err := g.checkoutBranch(branch, workingDir); err != nil {
		return err
	}

	return nil
}

// createComment creates a comment in the specific pr
func (g *Github) createComment(pr int, comment string) error {
	ctx, cancel := context.WithTimeout(context.Background(), timeoutLongRequest)
	defer cancel()

	c := &github.IssueComment{Body: &comment}

	_, _, err := g.client.Issues.CreateComment(ctx, g.owner, g.repo, pr, c)

	return err
}

// isMember returns true if the user is member of the organization, else false
func (g *Github) isMember(user string) (bool, error) {
	ctx, cancel := context.WithTimeout(context.Background(), timeoutShortRequest)
	defer cancel()

	ret, _, err := g.client.Organizations.IsMember(ctx, g.owner, user)

	return ret, err
}
<commit_msg>localCI: validate go-github variables before use them<commit_after>// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"context"
	"fmt"
	"net/http"
	"os/exec"
	"strings"
	"time"

	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

// Github represents a control version repository to
// interact with github.com
type Github struct {
	client *github.Client
	owner  string
	repo   string
	url    string
}

const (
	timeoutShortRequest = 10 * time.Second
	timeoutLongRequest  = 20 * time.Second
)

// newGithub returns an object of type Github
func newGithub(url, token string) (CVR, error) {
	url = strings.TrimSpace(url)

	ownerRepo := strings.SplitAfter(url, "/"+githubDomain+"/")

	// at least we need two tokens
	if len(ownerRepo) < 2 {
		return nil, fmt.Errorf("missing owner and repo %s", url)
	}

	ownerRepo = strings.Split(ownerRepo[1], "/")

	// at least we need two tokens: owner and repo
	if len(ownerRepo) < 2 {
		return nil, fmt.Errorf("failed to get owner and repo %s", url)
	}

	if len(ownerRepo[0]) == 0 {
		return nil, fmt.Errorf("missing owner in url %s", url)
	}

	if len(ownerRepo[1]) == 0 {
		return nil, fmt.Errorf("missing repository in url %s", url)
	}

	// create a new http client using the token
	var client *http.Client
	if token != "" {
		ts := oauth2.StaticTokenSource(
			&oauth2.Token{AccessToken: token},
		)
		client = oauth2.NewClient(context.Background(), ts)
	}

	return &Github{
		client: github.NewClient(client),
		owner:  ownerRepo[0],
		repo:   ownerRepo[1],
		url:    url,
	}, nil
}

// getProjectSlug returns the domain, owner and repo name separated by '/'
func (g *Github) getProjectSlug() string {
	return fmt.Sprintf("%s/%s/%s", githubDomain, g.owner, g.repo)
}

// getPullRequestCommits returns the commits of a pull request
func (g *Github) getPullRequestCommits(pr int) ([]repoCommit, error) {
	ctx, cancel := context.WithTimeout(context.Background(), timeoutShortRequest)
	defer cancel()

	// get all commits of the pull request
	listCommits, _, err := g.client.PullRequests.ListCommits(ctx, g.owner, g.repo, pr, nil)
	if err != nil {
		return nil, err
	}

	var commits []repoCommit
	for _, c := range listCommits {
		if c == nil {
			return nil, fmt.Errorf("failed to get all commits of the pull request %d", pr)
		}

		if c.SHA == nil {
			return nil, fmt.Errorf("failed to get commit SHA of the pull request %d", pr)
		}
		sha := *c.SHA

		if c.Commit == nil || c.Commit.Committer == nil || c.Commit.Committer.Date == nil {
			return nil, fmt.Errorf("failed to get commit time of the pull request %d", pr)
		}
		time := *c.Commit.Committer.Date

		commits = append(commits,
			repoCommit{
				sha:  sha,
				time: time,
			},
		)
	}

	return commits, nil
}

// getOpenPullRequests returns the open pull requests
func (g *Github) getOpenPullRequests() ([]int, error) {
	ctx, cancel := context.WithTimeout(context.Background(), timeoutShortRequest)
	defer cancel()

	pullRequests, _, err := g.client.PullRequests.List(ctx, g.owner, g.repo, nil)
	if err != nil {
		return nil, fmt.Errorf("failed to list pull requests %s", err)
	}

	prs := []int{}

	for _, pr := range pullRequests {
		if pr == nil || pr.Number == nil {
			continue
		}

		prs = append(prs, *pr.Number)
	}

	return prs, nil
}

// getBranchInfo returns a specific branch
func (g *Github) getBranchInfo(branch string) (repoBranchInfo, error) {
	i := repoBranchInfo{}

	ctx, cancel := context.WithTimeout(context.Background(), timeoutShortRequest)
	defer cancel()

	b, _, err := g.client.Repositories.GetBranch(ctx, g.owner, g.repo, branch)
	if err != nil {
		return i, err
	}

	if b.Commit == nil && b.Commit.SHA == nil {
		return i, fmt.Errorf("failed to get commit sha of branch %s", branch)
	}

	return repoBranchInfo{
		sha: *b.Commit.SHA,
	}, nil
}

// getPullRequestInfo return information about a pull request
func (g *Github) getPullRequestInfo(pr int) (pullRequestInfo, error) {
	i := pullRequestInfo{}

	ctx, cancel := context.WithTimeout(context.Background(), timeoutShortRequest)
	defer cancel()

	pullRequest, _, err := g.client.PullRequests.Get(ctx, g.owner, g.repo, pr)
	if err != nil {
		return i, err
	}

	// check the integrity of the pullRuest object before use it
	if pullRequest == nil {
		return i, fmt.Errorf("failed to get pull request %d", pr)
	}

	if pullRequest.User == nil || pullRequest.User.Login == nil {
		return i, fmt.Errorf("failed to get the author of the pull request %d", pr)
	}

	author := *pullRequest.User.Login

	// do not fail if we don't know if the pull request is
	// mergeable, just test it
	mergeable := true
	if pullRequest.Mergeable != nil {
		mergeable = *pullRequest.Mergeable
	}

	// include the state to check it later
	state := ""
	if pullRequest.State != nil {
		state = *pullRequest.State
	}

	if pullRequest.Head == nil || pullRequest.Head.Ref == nil {
		return i, fmt.Errorf("failed to get the branch name of the pull request %d", pr)
	}

	branch := *pullRequest.Head.Ref

	return pullRequestInfo{
		branch:    branch,
		author:    author,
		mergeable: mergeable,
		state:     state,
	}, nil
}

// getLatestPullRequestComment returns the latest comment of a specific
// user in the specific pr. If user is an empty string then any user
// could be the author of the latest comment. If body is an empty
// string an error is returned.
func (g *Github) getLatestPullRequestComment(pr int, user, body string) (RepoComment, error) {
	c := RepoComment{}

	if len(body) == 0 {
		return c, fmt.Errorf("body cannot be an empty string")
	}

	ctx, cancel := context.WithTimeout(context.Background(), timeoutLongRequest)
	defer cancel()

	comments, _, err := g.client.Issues.ListComments(ctx, g.owner, g.repo, pr, nil)
	if err != nil {
		return c, err
	}

	for i := len(comments) - 1; i >= 0; i-- {
		c := comments[i]
		if len(user) != 0 {
			if c.User == nil || c.User.Login == nil || *c.User.Login != user {
				continue
			}
		}

		if c.CreatedAt == nil {
			continue
		}

		if c.Body != nil && *c.Body == body {
			return RepoComment{
				User:    user,
				Comment: body,
				time:    *c.CreatedAt,
			}, nil
		}
	}

	return c, fmt.Errorf("comment '%s' not found", body)
}

func (g *Github) downloadRepo(workingDir string) error {
	var stderr bytes.Buffer

	// clone the project
	cmd := exec.Command("git", "clone", g.url, ".")
	cmd.Dir = workingDir
	cmd.Stderr = &stderr
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("failed to run git clone %s %s", stderr.String(), err)
	}

	return nil
}

func (g *Github) checkoutBranch(branch string, workingDir string) error {
	var stderr bytes.Buffer

	// checkout the branch
	stderr.Reset()
	cmd := exec.Command("git", "checkout", branch)
	cmd.Dir = workingDir
	cmd.Stderr = &stderr
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("failed to run git checkout %s %s", stderr.String(), err)
	}

	return nil
}

func (g *Github) downloadBranch(branch string, workingDir string) error {
	if err := g.downloadRepo(workingDir); err != nil {
		return err
	}

	if err := g.checkoutBranch(branch, workingDir); err != nil {
		return err
	}

	return nil
}

func (g *Github) downloadPullRequest(pr int, branch string, workingDir string) error {
	var stderr bytes.Buffer

	if err := g.downloadRepo(workingDir); err != nil {
		return err
	}

	// fetch the branch
	stderr.Reset()
	cmd := exec.Command("git", "fetch", "origin", fmt.Sprintf("pull/%d/head:%s", pr, branch))
	cmd.Dir = workingDir
	cmd.Stderr = &stderr
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("failed to run git fetch %s %s", stderr.String(), err)
	}

	if err := g.checkoutBranch(branch, workingDir); err != nil {
		return err
	}

	return nil
}

// createComment creates a comment in the specific pr
func (g *Github) createComment(pr int, comment string) error {
	ctx, cancel := context.WithTimeout(context.Background(), timeoutLongRequest)
	defer cancel()

	c := &github.IssueComment{Body: &comment}

	_, _, err := g.client.Issues.CreateComment(ctx, g.owner, g.repo, pr, c)

	return err
}

// isMember returns true if the user is member of the organization, else false
func (g *Github) isMember(user string) (bool, error) {
	ctx, cancel := context.WithTimeout(context.Background(), timeoutShortRequest)
	defer cancel()

	ret, _, err := g.client.Organizations.IsMember(ctx, g.owner, user)

	return ret, err
}
<|endoftext|>


<commit_before>package move

import (
	"encoding/base64"
	"runtime"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/workers/mails"
)

func init() {
	jobs.AddWorker(&jobs.WorkerConfig{
		WorkerType:   "export",
		Concurrency:  runtime.NumCPU(),
		MaxExecCount: 1,
		Timeout:      10 * 60 * time.Second,
		WorkerFunc:   ExportWorker,
	})
}

// ExportWorker is the worker responsible for creating an export of the
// instance.
func ExportWorker(c *jobs.WorkerContext) error {
	i, err := instance.Get(c.Domain())
	if err != nil {
		return err
	}

	exportDoc, err := Export(i, SystemArchiver())
	if err != nil {
		return err
	}

	link := i.SubDomain(consts.SettingsSlug)
	link.Fragment = "/exports/" + base64.URLEncoding.EncodeToString(exportDoc.GenerateAuthMessage(i))
	mail := mails.Options{
		Mode:           mails.ModeNoReply,
		TemplateName:   "archiver",
		TemplateValues: map[string]string{"ArchiveLink": link.String()},
	}

	msg, err := jobs.NewMessage(&mail)
	if err != nil {
		return err
	}

	_, err = jobs.System().PushJob(&jobs.JobRequest{
		Domain:     i.Domain,
		WorkerType: "sendmail",
		Message:    msg,
	})
	return err
}
<commit_msg>Export link contains is /exports/:id?mac=<commit_after>package move

import (
	"encoding/base64"
	"net/url"
	"runtime"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/workers/mails"
)

func init() {
	jobs.AddWorker(&jobs.WorkerConfig{
		WorkerType:   "export",
		Concurrency:  runtime.NumCPU(),
		MaxExecCount: 1,
		Timeout:      10 * 60 * time.Second,
		WorkerFunc:   ExportWorker,
	})
}

// ExportWorker is the worker responsible for creating an export of the
// instance.
func ExportWorker(c *jobs.WorkerContext) error {
	i, err := instance.Get(c.Domain())
	if err != nil {
		return err
	}

	exportDoc, err := Export(i, SystemArchiver())
	if err != nil {
		return err
	}

	mac := base64.URLEncoding.EncodeToString(exportDoc.GenerateAuthMessage(i))
	link := i.SubDomain(consts.SettingsSlug)
	link.Fragment = "/exports/" + exportDoc.ID()
	link.RawPath = url.Values{"mac": {mac}}.Encode()
	mail := mails.Options{
		Mode:           mails.ModeNoReply,
		TemplateName:   "archiver",
		TemplateValues: map[string]string{"ArchiveLink": link.String()},
	}

	msg, err := jobs.NewMessage(&mail)
	if err != nil {
		return err
	}

	_, err = jobs.System().PushJob(&jobs.JobRequest{
		Domain:     i.Domain,
		WorkerType: "sendmail",
		Message:    msg,
	})
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"compress/zlib"
	"encoding/ascii85"
	"flag"
	"fmt"
	"github.com/bnagy/pdflex"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strconv"
	"strings"
)

const MAXDATA = 1024

var xref = []byte("xref")
var startxref = []byte("startxref")
var trailer = []byte("trailer")
var pref85 = []byte("<~")
var suff85 = []byte("~>")

func fixStartXref(in []byte) []byte {

	sxrIdx := bytes.LastIndex(in, startxref)
	if sxrIdx < 0 {
		return in
	}

	xrIdx := bytes.LastIndex(in[:sxrIdx], xref)
	if xrIdx < 0 {
		return in
	}

	scratch := []byte{}
	scratch = append(scratch, in[:sxrIdx]...)
	scratch = append(scratch, []byte(fmt.Sprintf("startxref\n%d\n%%%%EOF\n", xrIdx))...)
	return scratch
}

func fixXrefs(in []byte) []byte {
	out := new(bytes.Buffer)
	tr := bytes.LastIndex(in, trailer)
	xr := bytes.LastIndex(in[:tr], xref)
	xrSection := in[xr:tr]
	normalized := strings.Replace(string(xrSection), "\r", "\n", -1)
	normalized = strings.Replace(normalized, "\n\n", "\n", -1)
	scanner := bufio.NewScanner(strings.NewReader(normalized))
	scanner.Scan()
	ff := strings.Fields(scanner.Text())
	if ff[0] != "xref" {
		log.Fatalf("Corrupt xref section\n%#v\n", string(xrSection))
	}
	fmt.Fprintln(out, scanner.Text())
	scanner.Scan()
	ff = strings.Fields(scanner.Text())
	if len(ff) < 2 {
		log.Fatalf("Corrupt xref section\n%#v\n", string(xrSection))
	}
	expected, err := strconv.Atoi(ff[1])
	if err != nil {
		log.Fatalf("Corrupt xref section\n%#v\n", string(xrSection))
	}

	for i := 0; i < expected; i++ {
		if !scanner.Scan() {
			log.Fatalf("Short xref section\n")
		}
		ff := strings.Fields(scanner.Text())
		if ff[2] == "n" {
			idx := -1
			idx = bytes.Index(in, []byte(fmt.Sprintf("\n%d 0 obj", i)))
			if idx < 0 {
				idx = bytes.Index(in, []byte(fmt.Sprintf("\r%d 0 obj", i)))
			}
			if idx >= 0 {
				fmt.Fprintf(out, "%.10d %s %s\n", idx, ff[1], ff[2])
				continue
			}
		}
		// in all other cases
		fmt.Fprintln(out, scanner.Text())
	}
	head := in[:xr]
	tail := in[tr:]
	final := append(head, out.Bytes()...)
	final = append(final, tail...)
	return fixStartXref(final)
}

func inflate(s string) (string, error) {
	in := strings.NewReader(s)
	decom, err := zlib.NewReader(in)
	if err != nil {
		return "", err
	}
	var b bytes.Buffer
	io.Copy(&b, decom)
	decom.Close()
	return b.String(), err
}

func deflate(s string) (string, error) {
	var b bytes.Buffer
	w := zlib.NewWriter(&b)
	_, err := w.Write([]byte(s))
	w.Close()
	if err != nil {
		return "", err
	}
	return b.String(), nil
}

func un85(s string) (string, error) {
	in := []byte(s)
	in = bytes.TrimPrefix(in, pref85)
	in = bytes.TrimSuffix(in, suff85)
	out := make([]byte, len(in))

	n, _, err := ascii85.Decode(out, in, true)
	if err != nil {
		return "", err
	}

	return string(out[:n]), nil
}

func re85(s string) (string, error) {
	var b bytes.Buffer
	w := ascii85.NewEncoder(&b)
	_, err := w.Write([]byte(s))
	w.Close()
	if err != nil {
		return "", err
	}
	return b.String(), nil
}

func main() {

	flag.Usage = func() {
		fmt.Fprintf(
			os.Stderr,
			"  Usage: %s file [file file ...]\n",
			path.Base(os.Args[0]),
		)
	}

	for _, arg := range os.Args[1:] {

		raw, err := ioutil.ReadFile(arg)
		if err != nil {
			os.Exit(1)
		}

		l := pdflex.NewLexer(arg, string(raw))
		var out bytes.Buffer
		zipped := false
		asc85 := false
		for i := l.NextItem(); i.Typ != pdflex.ItemEOF; i = l.NextItem() {
			if i.Typ == pdflex.ItemStreamBody {
				s := i.Val

				if asc85 {

					s, err = un85(s)
					if err != nil {
						log.Fatalf("Failed to un85: %s", err)
					}
				}

				if zipped {

					s2, err := inflate(s)

					if err != nil {
						log.Fatalf("%d Failed to inflate internal stream: %s", i.Pos, err)
					}
					if len(s2) > 0 {
						s = s2
					}
				}

				if len(s) > MAXDATA {
					s = s[:MAXDATA]
				} else {
					// write the original string
					out.WriteString(i.Val)
					zipped = false
					asc85 = false
					continue
				}

				if zipped {
					s, err = deflate(s)
					if err != nil {
						log.Fatalf("Error zipping truncated string: %s", err)
					}
				}
				if asc85 {
					s, err = re85(s)
					if err != nil {
						log.Fatalf("Error Ascii85ing zipped string: %s", err)
					}
				}
				out.WriteString(s)
				zipped = false
				asc85 = false
			} else {
				if i.Typ == pdflex.ItemName && i.Val == "/FlateDecode" {

					zipped = true
				}
				if i.Typ == pdflex.ItemName && i.Val == "/ASCII85Decode" {
					asc85 = true
				}
				out.WriteString(i.Val)
			}

			if i.Typ == pdflex.ItemError {
				break
			}

		}

		fixed := fixXrefs(out.Bytes())
		fmt.Println(string(fixed))
	}

}
<commit_msg>kick into beta<commit_after>package main

import (
	"bufio"
	"bytes"
	"compress/zlib"
	"encoding/ascii85"
	"flag"
	"fmt"
	"github.com/bnagy/pdflex"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strconv"
	"strings"
)

const MAXDATA = 1024

var xref = []byte("xref")
var startxref = []byte("startxref")
var trailer = []byte("trailer")
var pref85 = []byte("<~")
var suff85 = []byte("~>")

var (
	flagStrict *bool = flag.Bool("strict", false, "Abort on xref parsing errors etc")
	flagMax    *int  = flag.Int("max", MAXDATA, "Trim streams whose size is greater than this value")
)

func fixStartXref(in []byte) []byte {

	sxrIdx := bytes.LastIndex(in, startxref)
	if sxrIdx < 0 {
		return in
	}

	xrIdx := bytes.LastIndex(in[:sxrIdx], xref)
	if xrIdx < 0 {
		return in
	}

	scratch := []byte{}
	scratch = append(scratch, in[:sxrIdx]...)
	scratch = append(scratch, []byte(fmt.Sprintf("startxref\n%d\n%%%%EOF", xrIdx))...)
	return scratch
}

func fixXrefs(in []byte) []byte {

	out := new(bytes.Buffer)
	tr := bytes.LastIndex(in, trailer)
	xr := bytes.LastIndex(in[:tr], xref)
	if xr < 0 || tr < 0 || tr < xr {
		return in
	}
	xrSection := in[xr:tr]

	// Try to normalize the xrefs so lines are delimited by one \n. This works
	// for \n, \r, \r\n...
	normalized := strings.Replace(string(xrSection), "\r", "\n", -1)
	normalized = strings.Replace(normalized, "\n\n", "\n", -1)

	// Validate / parse the header rows
	scanner := bufio.NewScanner(strings.NewReader(normalized))
	scanner.Scan()
	ff := strings.Fields(scanner.Text())
	if ff[0] != "xref" {
		if *flagStrict {
			log.Fatalf("[STRICT] Corrupt xref section\n%#v\n", string(xrSection))
		}
		return in
	}
	fmt.Fprintln(out, scanner.Text())
	scanner.Scan()
	ff = strings.Fields(scanner.Text())
	if len(ff) < 2 {
		if *flagStrict {
			log.Fatalf("[STRICT] Corrupt xref section\n%#v\n", string(xrSection))
		}
		return in
	}
	expected, err := strconv.Atoi(ff[1])
	if err != nil {
		if *flagStrict {
			log.Fatalf("[STRICT] Corrupt xref section\n%#v\n", string(xrSection))
		}
		return in
	}

	// Parse the xrefs entries and try to fix up indirect ref offsets
	for i := 0; i < expected; i++ {
		if !scanner.Scan() {
			if *flagStrict {
				log.Fatalf("[STRICT] Short xref section\n")
			}
			return in
		}
		ff := strings.Fields(scanner.Text())
		// Get indirect refs like 0000037118 00000 n
		// Don't know what 0000000389 00001 f refs are
		if ff[2] == "n" {
			idx := -1
			// Do it this way instead of using a regex, because the multi-line
			// regexes / anchors seem wonky for PDFs that use \r as a "bare"
			// line delimeter
			idx = bytes.Index(in, []byte(fmt.Sprintf("\n%d 0 obj", i)))
			if idx < 0 {
				idx = bytes.Index(in, []byte(fmt.Sprintf("\r%d 0 obj", i)))
			}
			if idx >= 0 {
				fmt.Fprintf(out, "%.10d %s %s\n", idx, ff[1], ff[2])
				continue
			}
		}
		// Not an indirect ref OR couldn't find that obj declaration
		// Emit this line unmodified
		fmt.Fprintln(out, scanner.Text())
	}

	// Replace the xrefs with our modified version. The length might be
	// slightly different if we squeezed one or more \r\n into \n.
	final := append(in[:xr], out.Bytes()...)
	final = append(final, in[tr:]...)
	return fixStartXref(final)
}

func inflate(s string) (string, error) {

	in := strings.NewReader(s)
	decom, err := zlib.NewReader(in)
	if err != nil {
		return "", err
	}

	var b bytes.Buffer
	_, err = io.Copy(&b, decom)
	decom.Close()

	return b.String(), err
}

func deflate(s string) (string, error) {

	var b bytes.Buffer
	w := zlib.NewWriter(&b)
	_, err := w.Write([]byte(s))
	w.Close()
	if err != nil {
		return "", err
	}

	return b.String(), nil
}

func un85(s string) (string, error) {

	in := []byte(s)
	// Caller is expected to trim <~ ~> if present
	in = bytes.TrimPrefix(in, pref85)
	in = bytes.TrimSuffix(in, suff85)
	out := make([]byte, 0, len(in))

	n, _, err := ascii85.Decode(out, in, true)
	if err != nil {
		return "", err
	}

	return string(out[:n]), nil
}

func re85(s string) (string, error) {
	var b bytes.Buffer
	w := ascii85.NewEncoder(&b)
	_, err := w.Write([]byte(s))
	w.Close()
	if err != nil {
		return "", err
	}
	return b.String(), nil
}

func main() {

	flag.Usage = func() {
		fmt.Fprintf(
			os.Stderr,
			"  Usage: %s file [file file ...]\n",
			path.Base(os.Args[0]),
		)
	}

	flag.Parse()
	if *flagMax < 0 {
		flag.Usage()
		os.Exit(1)
	}

	for _, arg := range os.Args[1:] {

		raw, err := ioutil.ReadFile(arg)
		if err != nil {
			os.Exit(1)
		}

		l := pdflex.NewLexer(arg, string(raw))
		var out bytes.Buffer
		zipped := false
		asc85 := false
		for i := l.NextItem(); i.Typ != pdflex.ItemEOF; i = l.NextItem() {
			if i.Typ == pdflex.ItemStreamBody {
				s := i.Val

				if asc85 {
					s, err = un85(s)
					if err != nil {
						log.Fatalf("[STRICT] Failed to un85: %s", err)
					}
				}

				if zipped {
					s2, err := inflate(s)
					if err != nil && *flagStrict {
						log.Fatalf("[STRICT] Error unzipping internal stream: %s", err)
					}
					// If not struct, we ignore any errors here. If it's
					// unexpected EOF we'll get partial unzipped data, so use
					// that for truncation. Other errors will read a zero
					// length string, in which case we fall back to truncating
					// the original (corrupt) zipped stream.
					if len(s2) > 0 {
						s = s2
					}
				}

				if len(s) > *flagMax {
					s = s[:*flagMax]
				} else {
					// write the original string
					out.WriteString(i.Val)
					zipped = false
					asc85 = false
					continue
				}

				if zipped {
					s, err = deflate(s)
					if err != nil {
						// should never happen, strict mode or not
						log.Fatalf("Error zipping truncated string: %s", err)
					}
				}
				if asc85 {
					s, err = re85(s)
					if err != nil {
						// ditto
						log.Fatalf("Error Ascii85ing zipped string: %s", err)
					}
				}
				out.WriteString(s)
				zipped = false
				asc85 = false

			} else {

				if i.Typ == pdflex.ItemName && i.Val == "/FlateDecode" {
					zipped = true
				}
				if i.Typ == pdflex.ItemName && i.Val == "/ASCII85Decode" {
					asc85 = true
				}
				out.WriteString(i.Val)
			}

			if i.Typ == pdflex.ItemError {
				break
			}
		}

		fixed := fixXrefs(out.Bytes())
		newfn := strings.TrimSuffix(path.Base(arg), path.Ext(arg)) + "-small" + path.Ext(arg)
		fmt.Println(newfn)
		newfn = path.Join(path.Dir(arg), newfn)
		err = ioutil.WriteFile(newfn, fixed, 0600)
		if err != nil {
			log.Fatalf("Unable to write %s: %s", newfn, err)
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"

	mt "buddin.us/musictheory"
)

const help = `transpose: tool for music pitch transposition

Usage:

transpose -pitch <pitch> -interval [<polarity>]<interval>

Pitches:

Pitches are expressed in scientific pitch notation: pitch class + octave. The valid pitch classes are:

C, C#, Db, D, D#, Eb, E, F, F#, Gb, G, G#, Ab, A, Bb, B

Double flats (bb) and double sharps (x) are also valid.

Intervals:

Name	    Values
----------  -------
Perfect     P, perf
Major       M, maj
Minor       m, min
Augmented   A, aug
Diminished  d, dim

Providing a negative interval (e.g. "-d5") will transpose down by that interval.

Examples:

transpose -pitch C3 -interval m3
transpose -pitch F#4 -interval +P5
transpose -pitch Bb5 -interval -m2

Flags:
`

var (
	pitchFlag    = flag.String("pitch", "", "pitch")
	intervalFlag = flag.String("interval", "", "interval")
	namingFlag   = flag.String("naming", "asc", "ascending or descending naming strategy")
	helpFlag     = flag.Bool("help", false, "show usage message")
)

func main() {
	flag.Parse()

	if len(flag.Args()) > 0 {
		fmt.Fprintln(os.Stderr, "transpose: too many arguments")
		os.Exit(1)
	}

	if *helpFlag || *pitchFlag == "" || *intervalFlag == "" {
		fmt.Println(help)
		flag.PrintDefaults()
		return
	}

	tonic, err := mt.ParsePitch(*pitchFlag)
	if err != nil {
		fmt.Fprintf(os.Stderr, "parse pitch: %v\n", err)
		os.Exit(1)
	}
	interval, err := mt.ParseInterval(*intervalFlag)
	if err != nil {
		fmt.Fprintf(os.Stderr, "parse interval: %v\n", err)
		os.Exit(1)
	}

	var strategy mt.ModifierStrategy
	switch *namingFlag {
	case "asc":
		strategy = mt.AscNames
	case "desc":
		strategy = mt.DescNames
	default:
		fmt.Fprintf(os.Stderr, "unknown naming strategy: %v\n", *namingFlag)
		os.Exit(1)
	}

	fmt.Println(tonic.Transpose(*interval).(mt.Pitch).Name(strategy))
}
<commit_msg>Describe how intervals are expressed.<commit_after>package main

import (
	"flag"
	"fmt"
	"os"

	mt "buddin.us/musictheory"
)

const help = `transpose: tool for music pitch transposition

Usage:

transpose -pitch <pitch> -interval [<polarity>]<interval>

Pitches:

Pitches are expressed in scientific pitch notation: pitch class + octave. The valid pitch classes are:

C, C#, Db, D, D#, Eb, E, F, F#, Gb, G, G#, Ab, A, Bb, B

Double flats (bb) and double sharps (x) are also valid.

Intervals:

Intervals are expressed as a quality + step.

Name	    Values
----------  -------
Perfect     P, perf
Major       M, maj
Minor       m, min
Augmented   A, aug
Diminished  d, dim

Providing a negative interval (e.g. "-d5") will transpose down by that interval.

Examples:

transpose -pitch C3 -interval m3
transpose -pitch F#4 -interval +P5
transpose -pitch Bb5 -interval -m2

Flags:
`

var (
	pitchFlag    = flag.String("pitch", "", "pitch")
	intervalFlag = flag.String("interval", "", "interval")
	namingFlag   = flag.String("naming", "asc", "ascending or descending naming strategy")
	helpFlag     = flag.Bool("help", false, "show usage message")
)

func main() {
	flag.Parse()

	if len(flag.Args()) > 0 {
		fmt.Fprintln(os.Stderr, "transpose: too many arguments")
		os.Exit(1)
	}

	if *helpFlag || *pitchFlag == "" || *intervalFlag == "" {
		fmt.Println(help)
		flag.PrintDefaults()
		return
	}

	tonic, err := mt.ParsePitch(*pitchFlag)
	if err != nil {
		fmt.Fprintf(os.Stderr, "parse pitch: %v\n", err)
		os.Exit(1)
	}
	interval, err := mt.ParseInterval(*intervalFlag)
	if err != nil {
		fmt.Fprintf(os.Stderr, "parse interval: %v\n", err)
		os.Exit(1)
	}

	var strategy mt.ModifierStrategy
	switch *namingFlag {
	case "asc":
		strategy = mt.AscNames
	case "desc":
		strategy = mt.DescNames
	default:
		fmt.Fprintf(os.Stderr, "unknown naming strategy: %v\n", *namingFlag)
		os.Exit(1)
	}

	fmt.Println(tonic.Transpose(*interval).(mt.Pitch).Name(strategy))
}
<|endoftext|>