<commit_before>package discovery

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	etcd "github.com/coreos/etcd/client"
	"github.com/matt-deboer/etcdcd/pkg/platform"
)

// Discovery provides correct startup details for etcd with respect to
// known vs. expected cluster membership
type Discovery struct {
	ConfigFile           string
	Platform             string
	ClientPort           int
	ServerPort           int
	ClientScheme         string
	ServerScheme         string
	MaxTries             int
	ProxyMode            bool
	MasterFilter         string
	DryRun               bool
	IgnoreNamingMismatch bool
}

func findMemberByName(members []etcd.Member, name string) *etcd.Member {
	for _, member := range members {
		if name == member.Name {
			return &member
		}
	}
	return nil
}

func containsMember(members []etcd.Member, member etcd.Member) bool {
	for _, m := range members {
		for _, peerURL := range m.PeerURLs {
			for _, memberPeerURL := range member.PeerURLs {
				if peerURL == memberPeerURL {
					return true
				}
			}
		}
	}
	return false
}

// DiscoverEnvironment produces an environment hash
func (d *Discovery) DiscoverEnvironment() (map[string]string, error) {

	p, err := platform.Get(d.Platform, d.ConfigFile)
	if p == nil {
		if err != nil {
			return nil, err
		}
		return nil, errors.New("No such platform: " + d.Platform)
	}

	var expectedMembers []etcd.Member
	for tries := 0; tries < d.MaxTries && len(expectedMembers) == 0; tries++ {
		if members, err := p.ExpectedMembers(d.MasterFilter, d.ClientScheme,
			d.ClientPort, d.ServerScheme, d.ServerPort); err == nil {
			for _, m := range members {
				// have to cast here because of golang type-system--ugh!
				expectedMembers = append(expectedMembers, etcd.Member(m))
			}
		} else {
			return nil, err
		}
		sleepTime := (2 * time.Second)
		if log.GetLevel() >= log.DebugLevel {
			log.Debugf("Failed to resolve expected members; sleeping for %s", sleepTime)
		}
		time.Sleep(sleepTime)
	}

	if len(expectedMembers) == 0 {
		log.Fatal("Failed to determine expected members")
	} else if log.GetLevel() >= log.DebugLevel {
		log.Debugf("Expected cluster members: %v#", expectedMembers)
	}

	var currentMembers []etcd.Member
	ctx := context.Background()

	var membersAPI etcd.MembersAPI
	for tries := 0; tries <= d.MaxTries; tries++ {
		for _, master := range expectedMembers {

			cfg := etcd.Config{
				Endpoints: master.ClientURLs,
				Transport: etcd.DefaultTransport,
				// set timeout per request to fail fast when the target endpoint is unavailable
				HeaderTimeoutPerRequest: time.Second,
			}
			etcdClient, err := etcd.New(cfg)
			if err != nil {
				log.Warnf("Error connecting to %s %v, %v", master.Name, master.ClientURLs, err)
				continue
			}

			membersAPI = etcd.NewMembersAPI(etcdClient)
			currentMembers, err = membersAPI.List(ctx)
			if err != nil {
				log.Warnf("Error listing members %s %v, %v", master.Name, master.ClientURLs, err)
				continue
			}
			if log.GetLevel() >= log.DebugLevel {
				log.Debugf("Actual cluster members: %#v", currentMembers)
			}

			break
		}
		if len(currentMembers) == 0 {
			// TODO: what's our timeout here?
			sleepTime := (3 * time.Second)
			if log.GetLevel() >= log.DebugLevel {
				log.Debugf("Failed to resolve members; sleeping for %s", sleepTime)
			}
			time.Sleep(sleepTime)
		} else {
			break
		}
	}

	environment := map[string]string{}
	environment["ETCD_NAME"] = p.LocalInstanceName()
	environment["ETCD_INITIAL_CLUSTER"] = initialClusterString(expectedMembers)

	localMaster := findMemberByName(expectedMembers, p.LocalInstanceName())
	if localMaster != nil {
		if log.GetLevel() >= log.DebugLevel {
			log.Debugf("Local master: %#v", *localMaster)
		}
		// this instance is an expected master
		if len(currentMembers) > 0 {
			// there is an existing cluster
			if err = d.assertSaneClusterState(expectedMembers, currentMembers); err != nil {
				log.Fatal(err)
			}

			d.evictBadPeers(membersAPI, expectedMembers, currentMembers)
			log.Infof("Joining existing cluster as a master")
			// TODO: what if we encounter a state where not of the expected masters are
			// members of the current cluster?
			if err := d.joinExistingCluster(membersAPI, *localMaster); err != nil {
				log.Fatal(err)
			}
			environment["ETCD_INITIAL_CLUSTER_STATE"] = "existing"
		} else {
			log.Infof("Creating a new cluster")
			environment["ETCD_INITIAL_CLUSTER_STATE"] = "new"
		}
	} else if d.ProxyMode {
		log.Infof("Proxying existing cluster")
		environment["ETCD_INITIAL_CLUSTER_STATE"] = "existing"
		environment["ETCD_PROXY"] = "on"
	} else {
		return nil, fmt.Errorf(
			"Invalid cluster configuration: localhost (%s) is not an expected master, and not in proxy mode",
			p.LocalInstanceName())
	}
	return environment, nil
}

func initialClusterString(members []etcd.Member) string {
	if len(members) > 0 {
		initialCluster := make([]string, 0, len(members))
		for _, m := range members {
			member := fmt.Sprintf("%s=%s", m.Name, m.PeerURLs[0])
			initialCluster = append(initialCluster, member)
		}
		return strings.Join(initialCluster, ",")
	}
	return ""
}

// Check for mismatched names between expected and current members with
// matching peer URLs; also check for lack of intersection between
// expected and current members--indicating an invalid current (or expected)
// cluster state
func (d *Discovery) assertSaneClusterState(expectedMembers []etcd.Member, currentMembers []etcd.Member) error {
	partialMatchCount := 0
	for _, current := range currentMembers {
		for _, expected := range expectedMembers {
			matchingPeerURL := ""
			for _, expectedPeerURL := range expected.PeerURLs {
				for _, currentPeerURL := range current.PeerURLs {
					if expectedPeerURL == currentPeerURL {
						matchingPeerURL = expectedPeerURL
						partialMatchCount++
						break
					}
				}
				if len(matchingPeerURL) > 0 {
					break
				}
			}
			if len(matchingPeerURL) > 0 && current.Name != expected.Name {
				if !d.IgnoreNamingMismatch {
					return fmt.Errorf("Expected peer %s with peer URL %s already exists with a different name: %s",
						expected.Name, matchingPeerURL, current.Name)
				} else if log.GetLevel() >= log.DebugLevel {
					log.Debugf("Ignoring expected peer %s with peer URL %s already exists with a different name: %s",
						expected.Name, matchingPeerURL, current.Name)
				}
			}
		}
	}
	if partialMatchCount == 0 && len(expectedMembers) > 0 && len(currentMembers) > 0 {
		expectedJSON, _ := json.Marshal(expectedMembers)
		currentJSON, _ := json.Marshal(currentMembers)
		return fmt.Errorf("Invalid cluster state: found no intersection between peer URLs of expected members %s and current members %s",
			expectedJSON, currentJSON)
	}

	return nil
}

func (d *Discovery) evictBadPeers(membersAPI etcd.MembersAPI, expectedMembers []etcd.Member, currentMembers []etcd.Member) {

	for _, peer := range currentMembers {
		if !containsMember(expectedMembers, peer) {
			msg := fmt.Sprintf("Ejecting bad peer %s %v from the cluster:", peer.Name, peer.PeerURLs)
			if d.DryRun {
				log.Infof("DRY_RUN: would have ejected peer %s %v from the cluster", peer.Name, peer.PeerURLs)
			} else {
				for tries := 0; tries < d.MaxTries; tries++ {
					err := membersAPI.Remove(context.Background(), peer.ID)
					if err == nil {
						log.Infof("%s DONE", msg)
						break
					} else if (tries + 1) == d.MaxTries {
						log.Errorf("%s ERROR: %v", msg, err)
					}
				}
			}
		}
	}
}

func (d *Discovery) joinExistingCluster(membersAPI etcd.MembersAPI, localMember etcd.Member) error {
	msg := "Joining existing cluster: "
	for tries := 0; tries < d.MaxTries; tries++ {
		if d.DryRun {
			log.Infof("DRY_RUN: would have added %s %v to the cluster", localMember.Name, localMember.PeerURLs)
		} else {
			_, err := membersAPI.Add(context.Background(), localMember.PeerURLs[0])
			if err == nil {
				log.Infof("%s DONE", msg)
				break
			} else if (tries + 1) == d.MaxTries {
				log.Errorf("%s ERROR: %v", msg, err)
				return err
			}
		}
	}
	return nil
}
<commit_msg>re-resolve members api<commit_after>package discovery

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	etcd "github.com/coreos/etcd/client"
	"github.com/matt-deboer/etcdcd/pkg/platform"
)

// Discovery provides correct startup details for etcd with respect to
// known vs. expected cluster membership
type Discovery struct {
	ConfigFile           string
	Platform             string
	ClientPort           int
	ServerPort           int
	ClientScheme         string
	ServerScheme         string
	MaxTries             int
	ProxyMode            bool
	MasterFilter         string
	DryRun               bool
	IgnoreNamingMismatch bool
}

func findMemberByName(members []etcd.Member, name string) *etcd.Member {
	for _, member := range members {
		if name == member.Name {
			return &member
		}
	}
	return nil
}

func containsMember(members []etcd.Member, member etcd.Member) bool {
	for _, m := range members {
		for _, peerURL := range m.PeerURLs {
			for _, memberPeerURL := range member.PeerURLs {
				if peerURL == memberPeerURL {
					return true
				}
			}
		}
	}
	return false
}

// DiscoverEnvironment produces an environment hash
func (d *Discovery) DiscoverEnvironment() (map[string]string, error) {

	p, err := platform.Get(d.Platform, d.ConfigFile)
	if p == nil {
		if err != nil {
			return nil, err
		}
		return nil, errors.New("No such platform: " + d.Platform)
	}

	var expectedMembers []etcd.Member
	for tries := 0; tries < d.MaxTries && len(expectedMembers) == 0; tries++ {
		if members, err := p.ExpectedMembers(d.MasterFilter, d.ClientScheme,
			d.ClientPort, d.ServerScheme, d.ServerPort); err == nil {
			for _, m := range members {
				// have to cast here because of golang type-system--ugh!
				expectedMembers = append(expectedMembers, etcd.Member(m))
			}
		} else {
			return nil, err
		}
		sleepTime := (2 * time.Second)
		if log.GetLevel() >= log.DebugLevel {
			log.Debugf("Failed to resolve expected members; sleeping for %s", sleepTime)
		}
		time.Sleep(sleepTime)
	}

	if len(expectedMembers) == 0 {
		log.Fatal("Failed to determine expected members")
	} else if log.GetLevel() >= log.DebugLevel {
		log.Debugf("Expected cluster members: %v#", expectedMembers)
	}

	membersAPI, currentMembers, err := d.resolveMembersAndAPI(expectedMembers)

	environment := map[string]string{}
	environment["ETCD_NAME"] = p.LocalInstanceName()
	environment["ETCD_INITIAL_CLUSTER"] = initialClusterString(expectedMembers)

	localMaster := findMemberByName(expectedMembers, p.LocalInstanceName())
	if localMaster != nil {
		if log.GetLevel() >= log.DebugLevel {
			log.Debugf("Local master: %#v", *localMaster)
		}
		// this instance is an expected master
		if len(currentMembers) > 0 {
			// there is an existing cluster
			if err = d.assertSaneClusterState(expectedMembers, currentMembers); err != nil {
				log.Fatal(err)
			}

			d.evictBadPeers(membersAPI, expectedMembers, currentMembers)
			log.Infof("Joining existing cluster as a master")
			// TODO: what if we encounter a state where not of the expected masters are
			// members of the current cluster?
			if err := d.joinExistingCluster(membersAPI, expectedMembers, *localMaster); err != nil {
				log.Fatal(err)
			}
			environment["ETCD_INITIAL_CLUSTER_STATE"] = "existing"
		} else {
			log.Infof("Creating a new cluster")
			environment["ETCD_INITIAL_CLUSTER_STATE"] = "new"
		}
	} else if d.ProxyMode {
		log.Infof("Proxying existing cluster")
		environment["ETCD_INITIAL_CLUSTER_STATE"] = "existing"
		environment["ETCD_PROXY"] = "on"
	} else {
		return nil, fmt.Errorf(
			"Invalid cluster configuration: localhost (%s) is not an expected master, and not in proxy mode",
			p.LocalInstanceName())
	}
	return environment, nil
}

func initialClusterString(members []etcd.Member) string {
	if len(members) > 0 {
		initialCluster := make([]string, 0, len(members))
		for _, m := range members {
			member := fmt.Sprintf("%s=%s", m.Name, m.PeerURLs[0])
			initialCluster = append(initialCluster, member)
		}
		return strings.Join(initialCluster, ",")
	}
	return ""
}

// Check for mismatched names between expected and current members with
// matching peer URLs; also check for lack of intersection between
// expected and current members--indicating an invalid current (or expected)
// cluster state
func (d *Discovery) assertSaneClusterState(expectedMembers []etcd.Member, currentMembers []etcd.Member) error {
	partialMatchCount := 0
	for _, current := range currentMembers {
		for _, expected := range expectedMembers {
			matchingPeerURL := ""
			for _, expectedPeerURL := range expected.PeerURLs {
				for _, currentPeerURL := range current.PeerURLs {
					if expectedPeerURL == currentPeerURL {
						matchingPeerURL = expectedPeerURL
						partialMatchCount++
						break
					}
				}
				if len(matchingPeerURL) > 0 {
					break
				}
			}
			if len(matchingPeerURL) > 0 && current.Name != expected.Name {
				if !d.IgnoreNamingMismatch {
					return fmt.Errorf("Expected peer %s with peer URL %s already exists with a different name: %s",
						expected.Name, matchingPeerURL, current.Name)
				} else if log.GetLevel() >= log.DebugLevel {
					log.Debugf("Ignoring expected peer %s with peer URL %s already exists with a different name: %s",
						expected.Name, matchingPeerURL, current.Name)
				}
			}
		}
	}
	if partialMatchCount == 0 && len(expectedMembers) > 0 && len(currentMembers) > 0 {
		expectedJSON, _ := json.Marshal(expectedMembers)
		currentJSON, _ := json.Marshal(currentMembers)
		return fmt.Errorf("Invalid cluster state: found no intersection between peer URLs of expected members %s and current members %s",
			expectedJSON, currentJSON)
	}

	return nil
}

func (d *Discovery) evictBadPeers(membersAPI etcd.MembersAPI, expectedMembers []etcd.Member, currentMembers []etcd.Member) {

	for _, peer := range currentMembers {
		if !containsMember(expectedMembers, peer) {
			msg := fmt.Sprintf("Ejecting bad peer %s %v from the cluster:", peer.Name, peer.PeerURLs)
			if d.DryRun {
				log.Infof("DRY_RUN: would have ejected peer %s %v from the cluster", peer.Name, peer.PeerURLs)
			} else {
				for tries := 0; tries < d.MaxTries; tries++ {
					err := membersAPI.Remove(context.Background(), peer.ID)
					if err == nil {
						log.Infof("%s DONE", msg)
						break
					} else if (tries + 1) == d.MaxTries {
						log.Errorf("%s ERROR: %v", msg, err)
					}
				}
			}
		}
	}
}

func (d *Discovery) joinExistingCluster(membersAPI etcd.MembersAPI,
	expectedMembers []etcd.Member, localMember etcd.Member) error {

	msg := "Joining existing cluster: "
	for tries := 0; tries < d.MaxTries; tries++ {
		if d.DryRun {
			log.Infof("DRY_RUN: would have added %s %v to the cluster", localMember.Name, localMember.PeerURLs)
		} else {
			_, err := membersAPI.Add(context.Background(), localMember.PeerURLs[0])
			if err == nil {
				log.Infof("%s DONE", msg)
				break
			} else if (tries + 1) == d.MaxTries {
				log.Errorf("%s ERROR: %v", msg, err)
				return err
			}
			membersAPI, _, err = d.resolveMembersAndAPI(expectedMembers)
			if err != nil {
				log.Errorf("%s ERROR: %v", msg, err)
				return err
			}
		}
	}
	return nil
}

func (d *Discovery) resolveMembersAndAPI(expectedMembers []etcd.Member) (etcd.MembersAPI, []etcd.Member, error) {

	ctx := context.Background()
	var currentMembers []etcd.Member
	var membersAPI etcd.MembersAPI
	var lastErr error
	for tries := 0; tries <= d.MaxTries; tries++ {
		for _, member := range expectedMembers {

			cfg := etcd.Config{
				Endpoints: member.ClientURLs,
				Transport: etcd.DefaultTransport,
				// set timeout per request to fail fast when the target endpoint is unavailable
				HeaderTimeoutPerRequest: time.Second,
			}
			etcdClient, err := etcd.New(cfg)
			if err != nil {
				log.Warnf("Error connecting to %s %v, %v", member.Name, member.ClientURLs, err)
				lastErr = err
				continue
			}

			membersAPI = etcd.NewMembersAPI(etcdClient)
			currentMembers, err = membersAPI.List(ctx)
			if err != nil {
				log.Warnf("Error listing members %s %v, %v", member.Name, member.ClientURLs, err)
				lastErr = err
				continue
			}
			if log.GetLevel() >= log.DebugLevel {
				log.Debugf("Actual cluster members: %#v", currentMembers)
			}
			return membersAPI, currentMembers, nil
			break
		}
		if len(currentMembers) == 0 {
			// TODO: what's our timeout here?
			sleepTime := (1 * time.Second)
			if log.GetLevel() >= log.DebugLevel {
				log.Debugf("Failed to resolve members; sleeping for %s", sleepTime)
			}
			time.Sleep(sleepTime)
		} else {
			break
		}
	}
	return nil, nil, lastErr
}
<|endoftext|>


<commit_before>package github

import (
	"net/http"

	"github.com/dgrijalva/jwt-go"
	"github.com/hellofresh/janus/pkg/config"
	"github.com/hellofresh/janus/pkg/jwt/provider"
)

func init() {
	provider.Register("github", &Provider{})
}

// Provider abstracts the authentication for github
type Provider struct {
	provider.Verifier
}

// Build acts like the constructor for a provider
func (gp *Provider) Build(config config.Credentials) provider.Provider {
	client := NewClient()

	return &Provider{
		Verifier: provider.NewVerifierBasket(
			NewTeamVerifier(teamConfigsToTeam(config.Github.Teams), client),
			NewOrganizationVerifier(config.Github.Organizations, client),
		),
	}
}

// GetClaims returns a JWT Map Claim
func (gp *Provider) GetClaims(httpClient *http.Client) (jwt.MapClaims, error) {
	client := NewClient()

	user, err := client.CurrentUser(httpClient)
	if err != nil {
		return nil, err
	}

	return jwt.MapClaims{
		"sub": *user.Login,
	}, nil
}

func teamConfigsToTeam(configTeams map[string]string) []Team {
	teams := []Team{}
	for org, team := range configTeams {
		teams = append(teams, Team{
			Name:         team,
			Organization: org,
		})
	}
	return teams
}
<commit_msg>Check github permission<commit_after>package github

import (
	"net/http"
	"sync"

	"github.com/dgrijalva/jwt-go"
	"github.com/google/go-github/github"
	"github.com/hellofresh/janus/pkg/config"
	"github.com/hellofresh/janus/pkg/jwt/provider"
	"github.com/pkg/errors"
)

const (
	janusOwners = "janus-owners"
)

func init() {
	provider.Register("github", &Provider{})
}

// Provider abstracts the authentication for github
type Provider struct {
	provider.Verifier

	teams []Team
}

// Build acts like the constructor for a provider
func (gp *Provider) Build(config config.Credentials) provider.Provider {
	client := NewClient()
	teams := gp.teamConfigsToTeam(config.Github.Teams)

	return &Provider{
		Verifier: provider.NewVerifierBasket(
			NewTeamVerifier(teams, client),
			NewOrganizationVerifier(config.Github.Organizations, client),
		),
		teams: teams,
	}
}

// GetClaims returns a JWT Map Claim
func (gp *Provider) GetClaims(httpClient *http.Client) (jwt.MapClaims, error) {
	client := NewClient()

	var (
		wg            sync.WaitGroup
		user          *github.User
		usersOrgTeams OrganizationTeams
		errs          []error
	)

	wg.Add(2)

	go func() {
		res, err := client.CurrentUser(httpClient)
		if err != nil {
			errs = append(errs, errors.Wrap(err, "failed to get github users"))
		} else {
			user = res
		}
		wg.Done()
	}()

	go func() {
		res, err := client.Teams(httpClient)
		if err != nil {
			errs = append(errs, errors.Wrap(err, "failed to get github teams"))
		} else {
			usersOrgTeams = res
		}
		wg.Done()
	}()

	wg.Wait()

	if len(errs) > 0 {
		return nil, errs[0]
	}

	return jwt.MapClaims{
		"sub":      *user.Login,
		"is_admin": gp.isAdmin(usersOrgTeams),
	}, nil
}

func (gp *Provider) teamConfigsToTeam(configTeams map[string]string) []Team {
	teams := []Team{}
	for org, team := range configTeams {
		teams = append(teams, Team{
			Name:         team,
			Organization: org,
		})
	}
	return teams
}

func (gp *Provider) isAdmin(usersOrgTeams OrganizationTeams) bool {
	for _, team := range gp.teams {
		if teams, ok := usersOrgTeams[team.Organization]; ok {
			for _, teamUserBelongsTo := range teams {
				if teamUserBelongsTo == janusOwners {
					return true
				}
			}
		}
	}
	return false
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rules

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"text/template"

	"github.com/ghodss/yaml"
	"k8s.io/helm/pkg/chartutil"
	"k8s.io/helm/pkg/engine"
	"k8s.io/helm/pkg/lint/support"
	"k8s.io/helm/pkg/timeconv"
)

// Templates lints the templates in the Linter.
func Templates(linter *support.Linter) {
	path := "templates/"
	templatesPath := filepath.Join(linter.ChartDir, path)

	templatesDirExist := linter.RunLinterRule(support.WarningSev, path, validateTemplatesDir(templatesPath))

	// Templates directory is optional for now
	if !templatesDirExist {
		return
	}

	// Load chart and parse templates, based on tiller/release_server
	chart, err := chartutil.Load(linter.ChartDir)

	chartLoaded := linter.RunLinterRule(support.ErrorSev, path, err)

	if !chartLoaded {
		return
	}

	options := chartutil.ReleaseOptions{Name: "testRelease", Time: timeconv.Now(), Namespace: "testNamespace"}
	valuesToRender, err := chartutil.ToRenderValues(chart, chart.Values, options)
	if err != nil {
		// FIXME: This seems to generate a duplicate, but I can't find where the first
		// error is coming from.
		//linter.RunLinterRule(support.ErrorSev, err)
		return
	}
	renderedContentMap, err := engine.New().Render(chart, valuesToRender)

	renderOk := linter.RunLinterRule(support.ErrorSev, path, err)

	if !renderOk {
		return
	}

	/* Iterate over all the templates to check:
	- It is a .yaml file
	- All the values in the template file is defined
	- {{}} include | quote
	- Generated content is a valid Yaml file
	- Metadata.Namespace is not set
	*/
	for _, template := range chart.Templates {
		fileName, preExecutedTemplate := template.Name, template.Data
		path = fileName

		linter.RunLinterRule(support.ErrorSev, path, validateAllowedExtension(fileName))

		// We only apply the following lint rules to yaml files
		if filepath.Ext(fileName) != ".yaml" {
			continue
		}

		// Check that all the templates have a matching value
		linter.RunLinterRule(support.WarningSev, path, validateNoMissingValues(templatesPath, valuesToRender, preExecutedTemplate))

		// NOTE, disabled for now, Refs https://github.com/kubernetes/helm/issues/1037
		// linter.RunLinterRule(support.WarningSev, path, validateQuotes(string(preExecutedTemplate)))

		renderedContent := renderedContentMap[filepath.Join(chart.GetMetadata().Name, fileName)]
		var yamlStruct K8sYamlStruct
		// Even though K8sYamlStruct only defines Metadata namespace, an error in any other
		// key will be raised as well
		err := yaml.Unmarshal([]byte(renderedContent), &yamlStruct)

		validYaml := linter.RunLinterRule(support.ErrorSev, path, validateYamlContent(err))

		if !validYaml {
			continue
		}

		linter.RunLinterRule(support.ErrorSev, path, validateNoNamespace(yamlStruct))
	}
}

// Validation functions
func validateTemplatesDir(templatesPath string) error {
	if fi, err := os.Stat(templatesPath); err != nil {
		return errors.New("directory not found")
	} else if err == nil && !fi.IsDir() {
		return errors.New("not a directory")
	}
	return nil
}

func validateAllowedExtension(fileName string) error {
	ext := filepath.Ext(fileName)
	validExtensions := []string{".yaml", ".tpl", ".txt"}

	for _, b := range validExtensions {
		if b == ext {
			return nil
		}
	}

	return fmt.Errorf("file extension '%s' not valid. Valid extensions are .yaml, .tpl, or .txt", ext)
}

// validateNoMissingValues checks that all the {{}} functions returns a non empty value (<no value> or "")
// and return an error otherwise.
func validateNoMissingValues(templatesPath string, chartValues chartutil.Values, templateContent []byte) error {
	// 1 - Load Main and associated templates
	// Main template that we will parse dynamically
	tmpl := template.New("tpl").Funcs(engine.FuncMap())
	// If the templatesPath includes any *.tpl files we will parse and import them as associated templates
	associatedTemplates, err := filepath.Glob(filepath.Join(templatesPath, "*.tpl"))

	if len(associatedTemplates) > 0 {
		tmpl, err = tmpl.ParseFiles(associatedTemplates...)
		if err != nil {
			return err
		}
	}

	var buf bytes.Buffer
	var emptyValues []string

	// 2 - Extract every function and execute them against the loaded values
	// Supported {{ .Chart.Name }}, {{ .Chart.Name | quote }}
	r, _ := regexp.Compile(`{{[\w.\s|"'-]+}}`)
	functions := r.FindAllString(string(templateContent), -1)

	skipRegex, _ := regexp.Compile(`if|else|end`)

	// Iterate over the {{ FOO }} templates, executing them against the chartValues
	// We do individual templates parsing so we keep the reference for the key (str) that we want it to be interpolated.
	for _, str := range functions {
		if skipRegex.MatchString(str) {
			continue
		}
		newtmpl, err := tmpl.Parse(str)
		if err != nil {
			return err
		}

		err = newtmpl.ExecuteTemplate(&buf, "tpl", chartValues)

		if err != nil {
			return err
		}

		renderedValue := buf.String()

		if renderedValue == "<no value>" || renderedValue == "" {
			emptyValues = append(emptyValues, str)
		}
		buf.Reset()
	}

	if len(emptyValues) > 0 {
		return fmt.Errorf("these substitution functions are returning no value: %v", emptyValues)
	}
	return nil
}

func validateYamlContent(err error) error {
	if err != nil {
		return fmt.Errorf("unable to parse YAML\n\t%s", err)
	}
	return nil
}

func validateNoNamespace(yamlStruct K8sYamlStruct) error {
	if yamlStruct.Metadata.Namespace != "" {
		return errors.New("namespace option is currently NOT supported")
	}
	return nil
}

// K8sYamlStruct stubs a Kubernetes YAML file.
// Need to access for now to Namespace only
type K8sYamlStruct struct {
	Metadata struct {
		Namespace string
	}
}
<commit_msg>fix(linter): remove spurious namespace error<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rules

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"text/template"

	"github.com/ghodss/yaml"
	"k8s.io/helm/pkg/chartutil"
	"k8s.io/helm/pkg/engine"
	"k8s.io/helm/pkg/lint/support"
	"k8s.io/helm/pkg/timeconv"
)

// Templates lints the templates in the Linter.
func Templates(linter *support.Linter) {
	path := "templates/"
	templatesPath := filepath.Join(linter.ChartDir, path)

	templatesDirExist := linter.RunLinterRule(support.WarningSev, path, validateTemplatesDir(templatesPath))

	// Templates directory is optional for now
	if !templatesDirExist {
		return
	}

	// Load chart and parse templates, based on tiller/release_server
	chart, err := chartutil.Load(linter.ChartDir)

	chartLoaded := linter.RunLinterRule(support.ErrorSev, path, err)

	if !chartLoaded {
		return
	}

	options := chartutil.ReleaseOptions{Name: "testRelease", Time: timeconv.Now(), Namespace: "testNamespace"}
	valuesToRender, err := chartutil.ToRenderValues(chart, chart.Values, options)
	if err != nil {
		// FIXME: This seems to generate a duplicate, but I can't find where the first
		// error is coming from.
		//linter.RunLinterRule(support.ErrorSev, err)
		return
	}
	renderedContentMap, err := engine.New().Render(chart, valuesToRender)

	renderOk := linter.RunLinterRule(support.ErrorSev, path, err)

	if !renderOk {
		return
	}

	/* Iterate over all the templates to check:
	- It is a .yaml file
	- All the values in the template file is defined
	- {{}} include | quote
	- Generated content is a valid Yaml file
	- Metadata.Namespace is not set
	*/
	for _, template := range chart.Templates {
		fileName, preExecutedTemplate := template.Name, template.Data
		path = fileName

		linter.RunLinterRule(support.ErrorSev, path, validateAllowedExtension(fileName))

		// We only apply the following lint rules to yaml files
		if filepath.Ext(fileName) != ".yaml" {
			continue
		}

		// Check that all the templates have a matching value
		linter.RunLinterRule(support.WarningSev, path, validateNoMissingValues(templatesPath, valuesToRender, preExecutedTemplate))

		// NOTE, disabled for now, Refs https://github.com/kubernetes/helm/issues/1037
		// linter.RunLinterRule(support.WarningSev, path, validateQuotes(string(preExecutedTemplate)))

		renderedContent := renderedContentMap[filepath.Join(chart.GetMetadata().Name, fileName)]
		var yamlStruct K8sYamlStruct
		// Even though K8sYamlStruct only defines Metadata namespace, an error in any other
		// key will be raised as well
		err := yaml.Unmarshal([]byte(renderedContent), &yamlStruct)

		validYaml := linter.RunLinterRule(support.ErrorSev, path, validateYamlContent(err))

		if !validYaml {
			continue
		}
	}
}

// Validation functions
func validateTemplatesDir(templatesPath string) error {
	if fi, err := os.Stat(templatesPath); err != nil {
		return errors.New("directory not found")
	} else if err == nil && !fi.IsDir() {
		return errors.New("not a directory")
	}
	return nil
}

func validateAllowedExtension(fileName string) error {
	ext := filepath.Ext(fileName)
	validExtensions := []string{".yaml", ".tpl", ".txt"}

	for _, b := range validExtensions {
		if b == ext {
			return nil
		}
	}

	return fmt.Errorf("file extension '%s' not valid. Valid extensions are .yaml, .tpl, or .txt", ext)
}

// validateNoMissingValues checks that all the {{}} functions returns a non empty value (<no value> or "")
// and return an error otherwise.
func validateNoMissingValues(templatesPath string, chartValues chartutil.Values, templateContent []byte) error {
	// 1 - Load Main and associated templates
	// Main template that we will parse dynamically
	tmpl := template.New("tpl").Funcs(engine.FuncMap())
	// If the templatesPath includes any *.tpl files we will parse and import them as associated templates
	associatedTemplates, err := filepath.Glob(filepath.Join(templatesPath, "*.tpl"))

	if len(associatedTemplates) > 0 {
		tmpl, err = tmpl.ParseFiles(associatedTemplates...)
		if err != nil {
			return err
		}
	}

	var buf bytes.Buffer
	var emptyValues []string

	// 2 - Extract every function and execute them against the loaded values
	// Supported {{ .Chart.Name }}, {{ .Chart.Name | quote }}
	r, _ := regexp.Compile(`{{[\w.\s|"'-]+}}`)
	functions := r.FindAllString(string(templateContent), -1)

	skipRegex, _ := regexp.Compile(`if|else|end`)

	// Iterate over the {{ FOO }} templates, executing them against the chartValues
	// We do individual templates parsing so we keep the reference for the key (str) that we want it to be interpolated.
	for _, str := range functions {
		if skipRegex.MatchString(str) {
			continue
		}
		newtmpl, err := tmpl.Parse(str)
		if err != nil {
			return err
		}

		err = newtmpl.ExecuteTemplate(&buf, "tpl", chartValues)

		if err != nil {
			return err
		}

		renderedValue := buf.String()

		if renderedValue == "<no value>" || renderedValue == "" {
			emptyValues = append(emptyValues, str)
		}
		buf.Reset()
	}

	if len(emptyValues) > 0 {
		return fmt.Errorf("these substitution functions are returning no value: %v", emptyValues)
	}
	return nil
}

func validateYamlContent(err error) error {
	if err != nil {
		return fmt.Errorf("unable to parse YAML\n\t%s", err)
	}
	return nil
}

// K8sYamlStruct stubs a Kubernetes YAML file.
// Need to access for now to Namespace only
type K8sYamlStruct struct {
	Metadata struct {
		Namespace string
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	"fmt"
	"net"
	"os/exec"
	"reflect"
	"regexp"
	"strings"

	"github.com/docker/machine/libmachine"
	"github.com/docker/machine/libmachine/host"
	"github.com/pkg/errors"
	"k8s.io/klog/v2"
	"k8s.io/minikube/pkg/drivers/kic/oci"
	"k8s.io/minikube/pkg/minikube/driver"
	"k8s.io/minikube/pkg/minikube/machine"
)

// HostIP gets the ip address to be used for mapping host -> VM and VM -> host
func HostIP(host *host.Host, clusterName string) (net.IP, error) {
	switch host.DriverName {
	case driver.Docker:
		return oci.RoutableHostIPFromInside(oci.Docker, clusterName, host.Name)
	case driver.Podman:
		return oci.RoutableHostIPFromInside(oci.Podman, clusterName, host.Name)
	case driver.SSH:
		ip, err := host.Driver.GetIP()
		if err != nil {
			return []byte{}, errors.Wrap(err, "Error getting VM/Host IP address")
		}
		return net.ParseIP(ip), nil
	case driver.KVM2:
		// `host.Driver.GetIP` returns dhcp lease info for a given network(=`virsh net-dhcp-leases minikube-net`)
		vmIPString, err := host.Driver.GetIP()
		if err != nil {
			return []byte{}, errors.Wrap(err, "Error getting VM/Host IP address")
		}
		vmIP := net.ParseIP(vmIPString).To4()
		if vmIP == nil {
			// We need the network ip address for minikube-net. It's the start address of the returned subnet.
			return []byte{}, errors.Wrap(err, "Error converting VM/Host IP address to IPv4 address")
		}
		return net.IPv4(vmIP[0], vmIP[1], vmIP[2], byte(1)), nil
	case driver.QEMU:
		fallthrough
	case driver.QEMU2:
		ipString, err := host.Driver.GetIP()
		if err != nil {
			return []byte{}, errors.Wrap(err, "Error getting IP address")
		}
		if ipString == "127.0.0.1" {
			// user network case
			return net.ParseIP("10.0.2.2"), nil
		} else {
			// socket_vmnet network case
			return net.ParseIP("192.168.105.1"), nil
		}
	case driver.HyperV:
		v := reflect.ValueOf(host.Driver).Elem()
		var hypervVirtualSwitch string
		// We don't have direct access to hyperv.Driver so use reflection to retrieve the virtual switch name
		for i := 0; i < v.NumField(); i++ {
			if v.Type().Field(i).Name == "VSwitch" {
				hypervVirtualSwitch = v.Field(i).Interface().(string)

				break
			}
		}
		if hypervVirtualSwitch == "" {
			return nil, errors.New("No virtual switch found")
		}
		ip, err := getIPForInterface(fmt.Sprintf("vEthernet (%s)", hypervVirtualSwitch))
		if err != nil {
			return []byte{}, errors.Wrap(err, fmt.Sprintf("ip for interface (%s)", hypervVirtualSwitch))
		}

		return ip, nil
	case driver.VirtualBox:
		vBoxManageCmd := driver.VBoxManagePath()
		out, err := exec.Command(vBoxManageCmd, "showvminfo", host.Name, "--machinereadable").Output()
		if err != nil {
			return []byte{}, errors.Wrap(err, "vboxmanage")
		}
		re := regexp.MustCompile(`hostonlyadapter2="(.*?)"`)
		iface := re.FindStringSubmatch(string(out))[1]
		ipList, err := exec.Command(vBoxManageCmd, "list", "hostonlyifs").Output()
		if err != nil {
			return []byte{}, errors.Wrap(err, "Error getting VM/Host IP address")
		}
		re = regexp.MustCompile(`(?sm)Name:\s*` + iface + `\s*$.+?IPAddress:\s*(\S+)`)
		ip := re.FindStringSubmatch(string(ipList))[1]

		return net.ParseIP(ip), nil
	case driver.Parallels:
		bin := "prlsrvctl"
		var binPath string
		if fullPath, err := exec.LookPath(bin); err != nil {
			binPath = fullPath
		} else {
			binPath = bin
		}
		out, err := exec.Command(binPath, "net", "info", "Shared").Output()
		if err != nil {
			return []byte{}, errors.Wrap(err, "Error reading the info of Parallels Shared network interface")
		}
		re := regexp.MustCompile(`IPv4 address: (.*)`)
		ipMatch := re.FindStringSubmatch(string(out))
		if len(ipMatch) < 2 {
			return []byte{}, errors.Wrap(err, "Error getting the IP address of Parallels Shared network interface")
		}
		ip := ipMatch[1]

		return net.ParseIP(ip), nil
	case driver.HyperKit:
		return net.ParseIP("192.168.64.1"), nil
	case driver.VMware:
		vmIPString, err := host.Driver.GetIP()
		if err != nil {
			return []byte{}, errors.Wrap(err, "Error getting VM IP address")
		}
		vmIP := net.ParseIP(vmIPString).To4()
		if vmIP == nil {
			return []byte{}, errors.Wrap(err, "Error converting VM IP address to IPv4 address")
		}
		return net.IPv4(vmIP[0], vmIP[1], vmIP[2], byte(1)), nil
	case driver.None:
		return net.ParseIP("127.0.0.1"), nil
	default:
		return []byte{}, fmt.Errorf("HostIP not yet implemented for %q driver", host.DriverName)
	}
}

// DriverIP gets the ip address of the current minikube cluster
func DriverIP(api libmachine.API, machineName string) (net.IP, error) {
	host, err := machine.LoadHost(api, machineName)
	if err != nil {
		return nil, err
	}

	ipStr, err := host.Driver.GetIP()
	if err != nil {
		return nil, errors.Wrap(err, "getting IP")
	}
	if driver.IsKIC(host.DriverName) {
		ipStr = oci.DefaultBindIPV4
	}
	ip := net.ParseIP(ipStr)
	if ip == nil {
		return nil, fmt.Errorf("parsing IP: %s", ipStr)
	}
	return ip, nil
}

// Based on code from http://stackoverflow.com/questions/23529663/how-to-get-all-addresses-and-masks-from-local-interfaces-in-go
func getIPForInterface(name string) (net.IP, error) {
	klog.Infof("getIPForInterface: searching for %q", name)
	ints, err := net.Interfaces()
	if err != nil {
		return nil, err
	}

	var i net.Interface
	for _, in := range ints {
		if strings.HasPrefix(strings.ToLower(in.Name), strings.ToLower(name)) {
			klog.Infof("found prefix matching interface for %q: %q", name, in.Name)
			i = in

			break
		}
		klog.Infof("%q does not match prefix %q", in.Name, name)
	}

	// Didn't find prefix, let's try any substring
	if i.Name == "" {
		for _, in := range ints {
			if strings.Contains(strings.ToLower(in.Name), strings.ToLower(name)) {
				klog.Infof("found substring matching interface for %q: %q", name, in.Name)
				i = in

				break
			}
			klog.Infof("%q does not match substring %q", in.Name, name)
		}
	}

	// We found nothing, fail out
	if i.Name == "" {
		return nil, errors.Errorf("Could not find interface %s inside %+v", name, ints)
	}

	klog.Infof("Found interface: %+v\n", i)
	addrs, _ := i.Addrs()
	for _, a := range addrs {
		klog.Infof("interface addr: %+v", a)
		if ipnet, ok := a.(*net.IPNet); ok {
			if ip := ipnet.IP.To4(); ip != nil {
				return ip, nil
			}
		}
	}
	return nil, errors.Errorf("Unable to find a IPv4 address for interface %q", name)
}
<commit_msg>fix lint<commit_after>/*
Copyright 2020 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	"fmt"
	"net"
	"os/exec"
	"reflect"
	"regexp"
	"strings"

	"github.com/docker/machine/libmachine"
	"github.com/docker/machine/libmachine/host"
	"github.com/pkg/errors"
	"k8s.io/klog/v2"
	"k8s.io/minikube/pkg/drivers/kic/oci"
	"k8s.io/minikube/pkg/minikube/driver"
	"k8s.io/minikube/pkg/minikube/machine"
)

// HostIP gets the ip address to be used for mapping host -> VM and VM -> host
func HostIP(host *host.Host, clusterName string) (net.IP, error) {
	switch host.DriverName {
	case driver.Docker:
		return oci.RoutableHostIPFromInside(oci.Docker, clusterName, host.Name)
	case driver.Podman:
		return oci.RoutableHostIPFromInside(oci.Podman, clusterName, host.Name)
	case driver.SSH:
		ip, err := host.Driver.GetIP()
		if err != nil {
			return []byte{}, errors.Wrap(err, "Error getting VM/Host IP address")
		}
		return net.ParseIP(ip), nil
	case driver.KVM2:
		// `host.Driver.GetIP` returns dhcp lease info for a given network(=`virsh net-dhcp-leases minikube-net`)
		vmIPString, err := host.Driver.GetIP()
		if err != nil {
			return []byte{}, errors.Wrap(err, "Error getting VM/Host IP address")
		}
		vmIP := net.ParseIP(vmIPString).To4()
		if vmIP == nil {
			// We need the network ip address for minikube-net. It's the start address of the returned subnet.
			return []byte{}, errors.Wrap(err, "Error converting VM/Host IP address to IPv4 address")
		}
		return net.IPv4(vmIP[0], vmIP[1], vmIP[2], byte(1)), nil
	case driver.QEMU:
		fallthrough
	case driver.QEMU2:
		ipString, err := host.Driver.GetIP()
		if err != nil {
			return []byte{}, errors.Wrap(err, "Error getting IP address")
		}
		if ipString == "127.0.0.1" {
			// user network case
			return net.ParseIP("10.0.2.2"), nil
		}
		// socket_vmnet network case
		return net.ParseIP("192.168.105.1"), nil
	case driver.HyperV:
		v := reflect.ValueOf(host.Driver).Elem()
		var hypervVirtualSwitch string
		// We don't have direct access to hyperv.Driver so use reflection to retrieve the virtual switch name
		for i := 0; i < v.NumField(); i++ {
			if v.Type().Field(i).Name == "VSwitch" {
				hypervVirtualSwitch = v.Field(i).Interface().(string)

				break
			}
		}
		if hypervVirtualSwitch == "" {
			return nil, errors.New("No virtual switch found")
		}
		ip, err := getIPForInterface(fmt.Sprintf("vEthernet (%s)", hypervVirtualSwitch))
		if err != nil {
			return []byte{}, errors.Wrap(err, fmt.Sprintf("ip for interface (%s)", hypervVirtualSwitch))
		}

		return ip, nil
	case driver.VirtualBox:
		vBoxManageCmd := driver.VBoxManagePath()
		out, err := exec.Command(vBoxManageCmd, "showvminfo", host.Name, "--machinereadable").Output()
		if err != nil {
			return []byte{}, errors.Wrap(err, "vboxmanage")
		}
		re := regexp.MustCompile(`hostonlyadapter2="(.*?)"`)
		iface := re.FindStringSubmatch(string(out))[1]
		ipList, err := exec.Command(vBoxManageCmd, "list", "hostonlyifs").Output()
		if err != nil {
			return []byte{}, errors.Wrap(err, "Error getting VM/Host IP address")
		}
		re = regexp.MustCompile(`(?sm)Name:\s*` + iface + `\s*$.+?IPAddress:\s*(\S+)`)
		ip := re.FindStringSubmatch(string(ipList))[1]

		return net.ParseIP(ip), nil
	case driver.Parallels:
		bin := "prlsrvctl"
		var binPath string
		if fullPath, err := exec.LookPath(bin); err != nil {
			binPath = fullPath
		} else {
			binPath = bin
		}
		out, err := exec.Command(binPath, "net", "info", "Shared").Output()
		if err != nil {
			return []byte{}, errors.Wrap(err, "Error reading the info of Parallels Shared network interface")
		}
		re := regexp.MustCompile(`IPv4 address: (.*)`)
		ipMatch := re.FindStringSubmatch(string(out))
		if len(ipMatch) < 2 {
			return []byte{}, errors.Wrap(err, "Error getting the IP address of Parallels Shared network interface")
		}
		ip := ipMatch[1]

		return net.ParseIP(ip), nil
	case driver.HyperKit:
		return net.ParseIP("192.168.64.1"), nil
	case driver.VMware:
		vmIPString, err := host.Driver.GetIP()
		if err != nil {
			return []byte{}, errors.Wrap(err, "Error getting VM IP address")
		}
		vmIP := net.ParseIP(vmIPString).To4()
		if vmIP == nil {
			return []byte{}, errors.Wrap(err, "Error converting VM IP address to IPv4 address")
		}
		return net.IPv4(vmIP[0], vmIP[1], vmIP[2], byte(1)), nil
	case driver.None:
		return net.ParseIP("127.0.0.1"), nil
	default:
		return []byte{}, fmt.Errorf("HostIP not yet implemented for %q driver", host.DriverName)
	}
}

// DriverIP gets the ip address of the current minikube cluster
func DriverIP(api libmachine.API, machineName string) (net.IP, error) {
	host, err := machine.LoadHost(api, machineName)
	if err != nil {
		return nil, err
	}

	ipStr, err := host.Driver.GetIP()
	if err != nil {
		return nil, errors.Wrap(err, "getting IP")
	}
	if driver.IsKIC(host.DriverName) {
		ipStr = oci.DefaultBindIPV4
	}
	ip := net.ParseIP(ipStr)
	if ip == nil {
		return nil, fmt.Errorf("parsing IP: %s", ipStr)
	}
	return ip, nil
}

// Based on code from http://stackoverflow.com/questions/23529663/how-to-get-all-addresses-and-masks-from-local-interfaces-in-go
func getIPForInterface(name string) (net.IP, error) {
	klog.Infof("getIPForInterface: searching for %q", name)
	ints, err := net.Interfaces()
	if err != nil {
		return nil, err
	}

	var i net.Interface
	for _, in := range ints {
		if strings.HasPrefix(strings.ToLower(in.Name), strings.ToLower(name)) {
			klog.Infof("found prefix matching interface for %q: %q", name, in.Name)
			i = in

			break
		}
		klog.Infof("%q does not match prefix %q", in.Name, name)
	}

	// Didn't find prefix, let's try any substring
	if i.Name == "" {
		for _, in := range ints {
			if strings.Contains(strings.ToLower(in.Name), strings.ToLower(name)) {
				klog.Infof("found substring matching interface for %q: %q", name, in.Name)
				i = in

				break
			}
			klog.Infof("%q does not match substring %q", in.Name, name)
		}
	}

	// We found nothing, fail out
	if i.Name == "" {
		return nil, errors.Errorf("Could not find interface %s inside %+v", name, ints)
	}

	klog.Infof("Found interface: %+v\n", i)
	addrs, _ := i.Addrs()
	for _, a := range addrs {
		klog.Infof("interface addr: %+v", a)
		if ipnet, ok := a.(*net.IPNet); ok {
			if ip := ipnet.IP.To4(); ip != nil {
				return ip, nil
			}
		}
	}
	return nil, errors.Errorf("Unable to find a IPv4 address for interface %q", name)
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package transform

import (
	"context"

	"github.com/google/gapid/gapis/api"
)

// Transformer is the interface that wraps the basic Transform method.
type Transformer interface {
	// Transform takes a given command and identifier and Writes out a possibly
	// transformed set of commands to the output.
	// Transform must not modify cmd in any way.
	Transform(ctx context.Context, id api.CmdID, cmd api.Cmd, output Writer)
	// Flush is called at the end of a command stream to cause Transformers
	// that cache commands to send any they have stored into the output.
	Flush(ctx context.Context, output Writer)
	// Preloop is called at the beginning of the loop if the trace is going to
	// be looped.
	PreLoop(ctx context.Context, output Writer)
	// PostLoop is called at the end of the loop if the trace is going to
	// be looped.
	PostLoop(ctx context.Context, output Writer)
}

// Writer is the interface which consumes the output of an Transformer.
// It also keeps track of state changes caused by all commands written to it.
// Conceptually, each Writer object contains its own separate State object,
// which is modified when MutateAndWrite is called.
// This allows the transform to access the state both before and after the
// mutation of state happens. It is also possible to omit/insert commands.
// In practice, single state object can be shared by all transforms for
// performance (i.e. the mutation is done only once at the very end).
// This potentially allows state changes to leak upstream so care is needed.
// There is a configuration flag to switch between the shared/separate modes.
type Writer interface {
	// State returns the state object associated with this writer.
	State() *api.GlobalState
	// MutateAndWrite mutates the state object associated with this writer,
	// and it passes the command to further consumers.
	MutateAndWrite(ctx context.Context, id api.CmdID, cmd api.Cmd)
	//Notify next tranformer it's ready to start loop the trace.
	NotifyPreLoop(ctx context.Context)
	//Notify next tranformer it's the end of the loop.
	NotifyPostLoop(ctx context.Context)
}
<commit_msg>Fix typo (#2941)<commit_after>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package transform

import (
	"context"

	"github.com/google/gapid/gapis/api"
)

// Transformer is the interface that wraps the basic Transform method.
type Transformer interface {
	// Transform takes a given command and identifier and Writes out a possibly
	// transformed set of commands to the output.
	// Transform must not modify cmd in any way.
	Transform(ctx context.Context, id api.CmdID, cmd api.Cmd, output Writer)
	// Flush is called at the end of a command stream to cause Transformers
	// that cache commands to send any they have stored into the output.
	Flush(ctx context.Context, output Writer)
	// Preloop is called at the beginning of the loop if the trace is going to
	// be looped.
	PreLoop(ctx context.Context, output Writer)
	// PostLoop is called at the end of the loop if the trace is going to
	// be looped.
	PostLoop(ctx context.Context, output Writer)
}

// Writer is the interface which consumes the output of an Transformer.
// It also keeps track of state changes caused by all commands written to it.
// Conceptually, each Writer object contains its own separate State object,
// which is modified when MutateAndWrite is called.
// This allows the transform to access the state both before and after the
// mutation of state happens. It is also possible to omit/insert commands.
// In practice, single state object can be shared by all transforms for
// performance (i.e. the mutation is done only once at the very end).
// This potentially allows state changes to leak upstream so care is needed.
// There is a configuration flag to switch between the shared/separate modes.
type Writer interface {
	// State returns the state object associated with this writer.
	State() *api.GlobalState
	// MutateAndWrite mutates the state object associated with this writer,
	// and it passes the command to further consumers.
	MutateAndWrite(ctx context.Context, id api.CmdID, cmd api.Cmd)
	//Notify next transformer it's ready to start loop the trace.
	NotifyPreLoop(ctx context.Context)
	//Notify next transformer it's the end of the loop.
	NotifyPostLoop(ctx context.Context)
}
<|endoftext|>


<commit_before>/*
  Copyright 2012 Sergey Cherepanov (https://github.com/cheggaaa)

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
*/

package cnst

const (
	VERSION = "0.3.11"
	SIGN    = "Anteater " + VERSION
)

<commit_msg>0.3.12<commit_after>/*
  Copyright 2012 Sergey Cherepanov (https://github.com/cheggaaa)

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
*/

package cnst

const (
	VERSION = "0.3.12"
	SIGN    = "Anteater " + VERSION
)

<|endoftext|>


<commit_before>// Copyright 2015 Reborndb Org. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package models

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"

	"github.com/reborndb/reborn/pkg/utils"

	"github.com/juju/errors"
	"github.com/ngaut/go-zookeeper/zk"
	"github.com/ngaut/log"
	"github.com/ngaut/zkhelper"
)

const (
	SERVER_TYPE_MASTER  string = "master"
	SERVER_TYPE_SLAVE   string = "slave"
	SERVER_TYPE_OFFLINE string = "offline"
)

// redis server instance
type Server struct {
	Type    string `json:"type"`
	GroupId int    `json:"group_id"`
	Addr    string `json:"addr"`
}

// redis server group
type ServerGroup struct {
	Id          int       `json:"id"`
	ProductName string    `json:"product_name"`
	Servers     []*Server `json:"servers"`
}

func (s *Server) String() string {
	b, _ := json.MarshalIndent(s, "", "  ")
	return string(b)
}

func (sg *ServerGroup) String() string {
	b, _ := json.MarshalIndent(sg, "", "  ")
	return string(b) + "\n"
}

func GetServer(coordConn zkhelper.Conn, coordPath string) (*Server, error) {
	data, _, err := coordConn.Get(coordPath)
	if err != nil {
		return nil, errors.Trace(err)
	}
	srv := Server{}
	if err := json.Unmarshal(data, &srv); err != nil {
		return nil, errors.Trace(err)
	}
	return &srv, nil
}

func NewServer(serverType string, addr string) *Server {
	return &Server{
		Type:    serverType,
		GroupId: INVALID_ID,
		Addr:    addr,
	}
}

func NewServerGroup(productName string, id int) *ServerGroup {
	return &ServerGroup{
		Id:          id,
		ProductName: productName,
	}
}

func GroupExists(coordConn zkhelper.Conn, productName string, groupId int) (bool, error) {
	coordPath := fmt.Sprintf("/zk/reborn/db_%s/servers/group_%d", productName, groupId)
	exists, _, err := coordConn.Exists(coordPath)
	if err != nil {
		return false, errors.Trace(err)
	}
	return exists, nil
}

func GetGroup(coordConn zkhelper.Conn, productName string, groupId int) (*ServerGroup, error) {
	exists, err := GroupExists(coordConn, productName, groupId)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if !exists {
		return nil, errors.NotFoundf("group %d", groupId)
	}

	group := &ServerGroup{
		ProductName: productName,
		Id:          groupId,
	}

	group.Servers, err = group.GetServers(coordConn)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return group, nil
}

func ServerGroups(coordConn zkhelper.Conn, productName string) ([]*ServerGroup, error) {
	var ret []*ServerGroup
	root := fmt.Sprintf("/zk/reborn/db_%s/servers", productName)
	groups, _, err := coordConn.Children(root)
	// if ErrNoNode, we may return an empty slice like ProxyList
	if err != nil && !zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
		return nil, errors.Trace(err)
	}

	// Buggy :X
	//zkhelper.ChildrenRecursive(*coordConn, root)

	for _, group := range groups {
		// parse group_1 => 1
		groupId, err := strconv.Atoi(strings.Split(group, "_")[1])
		if err != nil {
			return nil, errors.Trace(err)
		}
		g, err := GetGroup(coordConn, productName, groupId)
		if err != nil {
			return nil, errors.Trace(err)
		}
		ret = append(ret, g)
	}
	return ret, nil
}

func (sg *ServerGroup) Master(coordConn zkhelper.Conn) (*Server, error) {
	servers, err := sg.GetServers(coordConn)
	if err != nil {
		return nil, errors.Trace(err)
	}
	for _, s := range servers {
		// TODO check if there are two masters
		if s.Type == SERVER_TYPE_MASTER {
			return s, nil
		}
	}
	return nil, nil
}

func (sg *ServerGroup) Remove(coordConn zkhelper.Conn) error {
	// check if this group is not used by any slot
	slots, err := Slots(coordConn, sg.ProductName)
	if err != nil {
		return errors.Trace(err)
	}

	for _, slot := range slots {
		if slot.GroupId == sg.Id {
			return errors.AlreadyExistsf("group %d is using by slot %d", slot.GroupId, slot.Id)
		}
	}

	// do delete
	coordPath := fmt.Sprintf("/zk/reborn/db_%s/servers/group_%d", sg.ProductName, sg.Id)
	err = zkhelper.DeleteRecursive(coordConn, coordPath, -1)

	// we know that there's no slots affected, so this action doesn't need proxy confirm
	err = NewAction(coordConn, sg.ProductName, ACTION_TYPE_SERVER_GROUP_REMOVE, sg, "", false)
	return errors.Trace(err)
}

func (sg *ServerGroup) RemoveServer(coordConn zkhelper.Conn, addr string) error {
	coordPath := fmt.Sprintf("/zk/reborn/db_%s/servers/group_%d/%s", sg.ProductName, sg.Id, addr)
	data, _, err := coordConn.Get(coordPath)
	if err != nil {
		return errors.Trace(err)
	}

	var s Server
	err = json.Unmarshal(data, &s)
	if err != nil {
		return errors.Trace(err)
	}
	log.Info(s)
	if s.Type == SERVER_TYPE_MASTER {
		return errors.New("cannot remove master, use promote first")
	}

	err = coordConn.Delete(coordPath, -1)
	if err != nil {
		return errors.Trace(err)
	}

	// update server list
	for i := 0; i < len(sg.Servers); i++ {
		if sg.Servers[i].Addr == s.Addr {
			sg.Servers = append(sg.Servers[:i], sg.Servers[i+1:]...)
			break
		}
	}

	// remove slave won't need proxy confirm
	err = NewAction(coordConn, sg.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, sg, "", false)
	return errors.Trace(err)
}

func (sg *ServerGroup) Promote(conn zkhelper.Conn, addr string, auth string) error {
	var s *Server
	exists := false
	for i := 0; i < len(sg.Servers); i++ {
		if sg.Servers[i].Addr == addr {
			s = sg.Servers[i]
			exists = true
			break
		}
	}

	if !exists {
		return errors.NotFoundf("no such addr %s", addr)
	}

	err := utils.SlaveNoOne(s.Addr, auth)
	if err != nil {
		return errors.Trace(err)
	}

	// set origin master offline
	master, err := sg.Master(conn)
	if err != nil {
		return errors.Trace(err)
	}

	// old master may be nil
	if master != nil {
		master.Type = SERVER_TYPE_OFFLINE
		err = sg.AddServer(conn, master, auth)
		if err != nil {
			return errors.Trace(err)
		}
	}

	// promote new server to master
	s.Type = SERVER_TYPE_MASTER
	err = sg.AddServer(conn, s, auth)
	return errors.Trace(err)
}

func (sg *ServerGroup) Create(coordConn zkhelper.Conn) error {
	if sg.Id < 0 {
		return errors.NotSupportedf("invalid server group id %d", sg.Id)
	}
	coordPath := fmt.Sprintf("/zk/reborn/db_%s/servers/group_%d", sg.ProductName, sg.Id)
	_, err := zkhelper.CreateOrUpdate(coordConn, coordPath, "", 0, zkhelper.DefaultDirACLs(), true)
	if err != nil {
		return errors.Trace(err)
	}
	err = NewAction(coordConn, sg.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, sg, "", false)
	if err != nil {
		return errors.Trace(err)
	}

	// set no server slots' group id to this server group, no need to return error
	slots, err := NoGroupSlots(coordConn, sg.ProductName)
	if err == nil && len(slots) > 0 {
		SetSlots(coordConn, sg.ProductName, slots, sg.Id, SLOT_STATUS_ONLINE)
	}

	return nil
}

func (sg *ServerGroup) Exists(coordConn zkhelper.Conn) (bool, error) {
	coordPath := fmt.Sprintf("/zk/reborn/db_%s/servers/group_%d", sg.ProductName, sg.Id)
	b, err := zkhelper.NodeExists(coordConn, coordPath)
	if err != nil {
		return false, errors.Trace(err)
	}
	return b, nil
}

var ErrNodeExists = errors.New("node already exists")

func (sg *ServerGroup) AddServer(coordConn zkhelper.Conn, s *Server, auth string) error {
	s.GroupId = sg.Id

	servers, err := sg.GetServers(coordConn)
	if err != nil {
		return errors.Trace(err)
	}
	var masterAddr string
	for _, server := range servers {
		if server.Type == SERVER_TYPE_MASTER {
			masterAddr = server.Addr
		}
	}

	// make sure there is only one master
	if s.Type == SERVER_TYPE_MASTER && len(masterAddr) > 0 {
		return errors.Trace(ErrNodeExists)
	}

	// if this group has no server. auto promote this server to master
	if len(servers) == 0 {
		s.Type = SERVER_TYPE_MASTER
	}

	val, err := json.Marshal(s)
	if err != nil {
		return errors.Trace(err)
	}

	coordPath := fmt.Sprintf("/zk/reborn/db_%s/servers/group_%d/%s", sg.ProductName, sg.Id, s.Addr)
	_, err = zkhelper.CreateOrUpdate(coordConn, coordPath, string(val), 0, zkhelper.DefaultFileACLs(), true)

	// update servers
	servers, err = sg.GetServers(coordConn)
	if err != nil {
		return errors.Trace(err)
	}
	sg.Servers = servers

	if s.Type == SERVER_TYPE_MASTER {
		err = NewAction(coordConn, sg.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, sg, "", true)
		if err != nil {
			return errors.Trace(err)
		}
	} else if s.Type == SERVER_TYPE_SLAVE && len(masterAddr) > 0 {
		// send command slaveof to slave
		err := utils.SlaveOf(s.Addr, masterAddr, auth)
		if err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}

func (sg *ServerGroup) GetServers(coordConn zkhelper.Conn) ([]*Server, error) {
	var ret []*Server
	root := fmt.Sprintf("/zk/reborn/db_%s/servers/group_%d", sg.ProductName, sg.Id)
	nodes, _, err := coordConn.Children(root)
	if err != nil {
		return nil, errors.Trace(err)
	}
	for _, node := range nodes {
		nodePath := root + "/" + node
		s, err := GetServer(coordConn, nodePath)
		if err != nil {
			return nil, errors.Trace(err)
		}
		ret = append(ret, s)
	}
	return ret, nil
}
<commit_msg>use slot info to check current server when adding<commit_after>// Copyright 2015 Reborndb Org. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package models

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"

	"github.com/reborndb/reborn/pkg/utils"

	"github.com/juju/errors"
	"github.com/ngaut/go-zookeeper/zk"
	"github.com/ngaut/log"
	"github.com/ngaut/zkhelper"
)

const (
	SERVER_TYPE_MASTER  string = "master"
	SERVER_TYPE_SLAVE   string = "slave"
	SERVER_TYPE_OFFLINE string = "offline"
)

// redis server instance
type Server struct {
	Type    string `json:"type"`
	GroupId int    `json:"group_id"`
	Addr    string `json:"addr"`
}

// redis server group
type ServerGroup struct {
	Id          int       `json:"id"`
	ProductName string    `json:"product_name"`
	Servers     []*Server `json:"servers"`
}

func (s *Server) String() string {
	b, _ := json.MarshalIndent(s, "", "  ")
	return string(b)
}

func (sg *ServerGroup) String() string {
	b, _ := json.MarshalIndent(sg, "", "  ")
	return string(b) + "\n"
}

func GetServer(coordConn zkhelper.Conn, coordPath string) (*Server, error) {
	data, _, err := coordConn.Get(coordPath)
	if err != nil {
		return nil, errors.Trace(err)
	}
	srv := Server{}
	if err := json.Unmarshal(data, &srv); err != nil {
		return nil, errors.Trace(err)
	}
	return &srv, nil
}

func NewServer(serverType string, addr string) *Server {
	return &Server{
		Type:    serverType,
		GroupId: INVALID_ID,
		Addr:    addr,
	}
}

func NewServerGroup(productName string, id int) *ServerGroup {
	return &ServerGroup{
		Id:          id,
		ProductName: productName,
	}
}

func GroupExists(coordConn zkhelper.Conn, productName string, groupId int) (bool, error) {
	coordPath := fmt.Sprintf("/zk/reborn/db_%s/servers/group_%d", productName, groupId)
	exists, _, err := coordConn.Exists(coordPath)
	if err != nil {
		return false, errors.Trace(err)
	}
	return exists, nil
}

func GetGroup(coordConn zkhelper.Conn, productName string, groupId int) (*ServerGroup, error) {
	exists, err := GroupExists(coordConn, productName, groupId)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if !exists {
		return nil, errors.NotFoundf("group %d", groupId)
	}

	group := &ServerGroup{
		ProductName: productName,
		Id:          groupId,
	}

	group.Servers, err = group.GetServers(coordConn)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return group, nil
}

func ServerGroups(coordConn zkhelper.Conn, productName string) ([]*ServerGroup, error) {
	var ret []*ServerGroup
	root := fmt.Sprintf("/zk/reborn/db_%s/servers", productName)
	groups, _, err := coordConn.Children(root)
	// if ErrNoNode, we may return an empty slice like ProxyList
	if err != nil && !zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
		return nil, errors.Trace(err)
	}

	// Buggy :X
	//zkhelper.ChildrenRecursive(*coordConn, root)

	for _, group := range groups {
		// parse group_1 => 1
		groupId, err := strconv.Atoi(strings.Split(group, "_")[1])
		if err != nil {
			return nil, errors.Trace(err)
		}
		g, err := GetGroup(coordConn, productName, groupId)
		if err != nil {
			return nil, errors.Trace(err)
		}
		ret = append(ret, g)
	}
	return ret, nil
}

func (sg *ServerGroup) Master(coordConn zkhelper.Conn) (*Server, error) {
	servers, err := sg.GetServers(coordConn)
	if err != nil {
		return nil, errors.Trace(err)
	}
	for _, s := range servers {
		// TODO check if there are two masters
		if s.Type == SERVER_TYPE_MASTER {
			return s, nil
		}
	}
	return nil, nil
}

func (sg *ServerGroup) Remove(coordConn zkhelper.Conn) error {
	// check if this group is not used by any slot
	slots, err := Slots(coordConn, sg.ProductName)
	if err != nil {
		return errors.Trace(err)
	}

	for _, slot := range slots {
		if slot.GroupId == sg.Id {
			return errors.AlreadyExistsf("group %d is using by slot %d", slot.GroupId, slot.Id)
		}
	}

	// do delete
	coordPath := fmt.Sprintf("/zk/reborn/db_%s/servers/group_%d", sg.ProductName, sg.Id)
	err = zkhelper.DeleteRecursive(coordConn, coordPath, -1)

	// we know that there's no slots affected, so this action doesn't need proxy confirm
	err = NewAction(coordConn, sg.ProductName, ACTION_TYPE_SERVER_GROUP_REMOVE, sg, "", false)
	return errors.Trace(err)
}

func (sg *ServerGroup) RemoveServer(coordConn zkhelper.Conn, addr string) error {
	coordPath := fmt.Sprintf("/zk/reborn/db_%s/servers/group_%d/%s", sg.ProductName, sg.Id, addr)
	data, _, err := coordConn.Get(coordPath)
	if err != nil {
		return errors.Trace(err)
	}

	var s Server
	err = json.Unmarshal(data, &s)
	if err != nil {
		return errors.Trace(err)
	}
	log.Info(s)
	if s.Type == SERVER_TYPE_MASTER {
		return errors.New("cannot remove master, use promote first")
	}

	err = coordConn.Delete(coordPath, -1)
	if err != nil {
		return errors.Trace(err)
	}

	// update server list
	for i := 0; i < len(sg.Servers); i++ {
		if sg.Servers[i].Addr == s.Addr {
			sg.Servers = append(sg.Servers[:i], sg.Servers[i+1:]...)
			break
		}
	}

	// remove slave won't need proxy confirm
	err = NewAction(coordConn, sg.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, sg, "", false)
	return errors.Trace(err)
}

func (sg *ServerGroup) Promote(conn zkhelper.Conn, addr string, auth string) error {
	var s *Server
	exists := false
	for i := 0; i < len(sg.Servers); i++ {
		if sg.Servers[i].Addr == addr {
			s = sg.Servers[i]
			exists = true
			break
		}
	}

	if !exists {
		return errors.NotFoundf("no such addr %s", addr)
	}

	err := utils.SlaveNoOne(s.Addr, auth)
	if err != nil {
		return errors.Trace(err)
	}

	// set origin master offline
	master, err := sg.Master(conn)
	if err != nil {
		return errors.Trace(err)
	}

	// old master may be nil
	if master != nil {
		master.Type = SERVER_TYPE_OFFLINE
		err = sg.AddServer(conn, master, auth)
		if err != nil {
			return errors.Trace(err)
		}
	}

	// promote new server to master
	s.Type = SERVER_TYPE_MASTER
	err = sg.AddServer(conn, s, auth)
	return errors.Trace(err)
}

func (sg *ServerGroup) Create(coordConn zkhelper.Conn) error {
	if sg.Id < 0 {
		return errors.NotSupportedf("invalid server group id %d", sg.Id)
	}
	coordPath := fmt.Sprintf("/zk/reborn/db_%s/servers/group_%d", sg.ProductName, sg.Id)
	_, err := zkhelper.CreateOrUpdate(coordConn, coordPath, "", 0, zkhelper.DefaultDirACLs(), true)
	if err != nil {
		return errors.Trace(err)
	}
	err = NewAction(coordConn, sg.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, sg, "", false)
	if err != nil {
		return errors.Trace(err)
	}

	// set no server slots' group id to this server group, no need to return error
	slots, err := NoGroupSlots(coordConn, sg.ProductName)
	if err == nil && len(slots) > 0 {
		SetSlots(coordConn, sg.ProductName, slots, sg.Id, SLOT_STATUS_ONLINE)
	}

	return nil
}

func (sg *ServerGroup) Exists(coordConn zkhelper.Conn) (bool, error) {
	coordPath := fmt.Sprintf("/zk/reborn/db_%s/servers/group_%d", sg.ProductName, sg.Id)
	b, err := zkhelper.NodeExists(coordConn, coordPath)
	if err != nil {
		return false, errors.Trace(err)
	}
	return b, nil
}

var ErrNodeExists = errors.New("node already exists")

func (sg *ServerGroup) AddServer(coordConn zkhelper.Conn, s *Server, auth string) error {
	// we only support reborn-server and qdb-server
	// origin redis has no slot_info command
	// atm, we can use this command to check whether server is alive or not.
	if _, err := utils.SlotsInfo(s.Addr, 0, 0, auth); err != nil {
		return errors.Trace(err)
	}

	s.GroupId = sg.Id

	servers, err := sg.GetServers(coordConn)
	if err != nil {
		return errors.Trace(err)
	}
	var masterAddr string
	for _, server := range servers {
		if server.Type == SERVER_TYPE_MASTER {
			masterAddr = server.Addr
		}
	}

	// make sure there is only one master
	if s.Type == SERVER_TYPE_MASTER && len(masterAddr) > 0 {
		return errors.Trace(ErrNodeExists)
	}

	// if this group has no server. auto promote this server to master
	if len(servers) == 0 {
		s.Type = SERVER_TYPE_MASTER
	}

	val, err := json.Marshal(s)
	if err != nil {
		return errors.Trace(err)
	}

	coordPath := fmt.Sprintf("/zk/reborn/db_%s/servers/group_%d/%s", sg.ProductName, sg.Id, s.Addr)
	_, err = zkhelper.CreateOrUpdate(coordConn, coordPath, string(val), 0, zkhelper.DefaultFileACLs(), true)

	// update servers
	servers, err = sg.GetServers(coordConn)
	if err != nil {
		return errors.Trace(err)
	}
	sg.Servers = servers

	if s.Type == SERVER_TYPE_MASTER {
		err = NewAction(coordConn, sg.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, sg, "", true)
		if err != nil {
			return errors.Trace(err)
		}
	} else if s.Type == SERVER_TYPE_SLAVE && len(masterAddr) > 0 {
		// send command slaveof to slave
		err := utils.SlaveOf(s.Addr, masterAddr, auth)
		if err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}

func (sg *ServerGroup) GetServers(coordConn zkhelper.Conn) ([]*Server, error) {
	var ret []*Server
	root := fmt.Sprintf("/zk/reborn/db_%s/servers/group_%d", sg.ProductName, sg.Id)
	nodes, _, err := coordConn.Children(root)
	if err != nil {
		return nil, errors.Trace(err)
	}
	for _, node := range nodes {
		nodePath := root + "/" + node
		s, err := GetServer(coordConn, nodePath)
		if err != nil {
			return nil, errors.Trace(err)
		}
		ret = append(ret, s)
	}
	return ret, nil
}
<|endoftext|>


<commit_before>// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fsbridge

import (
	"io"

	"gvisor.dev/gvisor/pkg/abi/linux"
	"gvisor.dev/gvisor/pkg/context"
	"gvisor.dev/gvisor/pkg/fspath"
	"gvisor.dev/gvisor/pkg/sentry/kernel/auth"
	"gvisor.dev/gvisor/pkg/sentry/memmap"
	"gvisor.dev/gvisor/pkg/sentry/vfs"
	"gvisor.dev/gvisor/pkg/usermem"
)

// VFSFile implements File interface over vfs.FileDescription.
//
// +stateify savable
type VFSFile struct {
	file *vfs.FileDescription
}

var _ File = (*VFSFile)(nil)

// NewVFSFile creates a new File over fs.File.
func NewVFSFile(file *vfs.FileDescription) File {
	return &VFSFile{file: file}
}

// PathnameWithDeleted implements File.
func (f *VFSFile) PathnameWithDeleted(ctx context.Context) string {
	root := vfs.RootFromContext(ctx)
	defer root.DecRef(ctx)

	vfsObj := f.file.VirtualDentry().Mount().Filesystem().VirtualFilesystem()
	name, _ := vfsObj.PathnameWithDeleted(ctx, root, f.file.VirtualDentry())
	return name
}

// ReadFull implements File.
func (f *VFSFile) ReadFull(ctx context.Context, dst usermem.IOSequence, offset int64) (int64, error) {
	var total int64
	for dst.NumBytes() > 0 {
		n, err := f.file.PRead(ctx, dst, offset+total, vfs.ReadOptions{})
		total += n
		if err == io.EOF && total != 0 {
			return total, io.ErrUnexpectedEOF
		} else if err != nil {
			return total, err
		}
		dst = dst.DropFirst64(n)
	}
	return total, nil
}

// ConfigureMMap implements File.
func (f *VFSFile) ConfigureMMap(ctx context.Context, opts *memmap.MMapOpts) error {
	return f.file.ConfigureMMap(ctx, opts)
}

// Type implements File.
func (f *VFSFile) Type(ctx context.Context) (linux.FileMode, error) {
	stat, err := f.file.Stat(ctx, vfs.StatOptions{})
	if err != nil {
		return 0, err
	}
	return linux.FileMode(stat.Mode).FileType(), nil
}

// IncRef implements File.
func (f *VFSFile) IncRef() {
	f.file.IncRef()
}

// DecRef implements File.
func (f *VFSFile) DecRef(ctx context.Context) {
	f.file.DecRef(ctx)
}

// FileDescription returns the FileDescription represented by f. It does not
// take an additional reference on the returned FileDescription.
func (f *VFSFile) FileDescription() *vfs.FileDescription {
	return f.file
}

// fsLookup implements Lookup interface using fs.File.
//
// +stateify savable
type vfsLookup struct {
	mntns *vfs.MountNamespace

	root       vfs.VirtualDentry
	workingDir vfs.VirtualDentry
}

var _ Lookup = (*vfsLookup)(nil)

// NewVFSLookup creates a new Lookup using VFS2.
func NewVFSLookup(mntns *vfs.MountNamespace, root, workingDir vfs.VirtualDentry) Lookup {
	return &vfsLookup{
		mntns:      mntns,
		root:       root,
		workingDir: workingDir,
	}
}

// OpenPath implements Lookup.
//
// remainingTraversals is not configurable in VFS2, all callers are using the
// default anyways.
func (l *vfsLookup) OpenPath(ctx context.Context, pathname string, opts vfs.OpenOptions, _ *uint, resolveFinal bool) (File, error) {
	vfsObj := l.mntns.Root().Mount().Filesystem().VirtualFilesystem()
	creds := auth.CredentialsFromContext(ctx)
	path := fspath.Parse(pathname)
	pop := &vfs.PathOperation{
		Root:               l.root,
		Start:              l.workingDir,
		Path:               path,
		FollowFinalSymlink: resolveFinal,
	}
	if path.Absolute {
		pop.Start = l.root
	}
	fd, err := vfsObj.OpenAt(ctx, creds, pop, &opts)
	if err != nil {
		return nil, err
	}
	return &VFSFile{file: fd}, nil
}
<commit_msg>[vfs2] Don't leak reference from Mountnamespace.Root().<commit_after>// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fsbridge

import (
	"io"

	"gvisor.dev/gvisor/pkg/abi/linux"
	"gvisor.dev/gvisor/pkg/context"
	"gvisor.dev/gvisor/pkg/fspath"
	"gvisor.dev/gvisor/pkg/sentry/kernel/auth"
	"gvisor.dev/gvisor/pkg/sentry/memmap"
	"gvisor.dev/gvisor/pkg/sentry/vfs"
	"gvisor.dev/gvisor/pkg/usermem"
)

// VFSFile implements File interface over vfs.FileDescription.
//
// +stateify savable
type VFSFile struct {
	file *vfs.FileDescription
}

var _ File = (*VFSFile)(nil)

// NewVFSFile creates a new File over fs.File.
func NewVFSFile(file *vfs.FileDescription) File {
	return &VFSFile{file: file}
}

// PathnameWithDeleted implements File.
func (f *VFSFile) PathnameWithDeleted(ctx context.Context) string {
	root := vfs.RootFromContext(ctx)
	defer root.DecRef(ctx)

	vfsObj := f.file.VirtualDentry().Mount().Filesystem().VirtualFilesystem()
	name, _ := vfsObj.PathnameWithDeleted(ctx, root, f.file.VirtualDentry())
	return name
}

// ReadFull implements File.
func (f *VFSFile) ReadFull(ctx context.Context, dst usermem.IOSequence, offset int64) (int64, error) {
	var total int64
	for dst.NumBytes() > 0 {
		n, err := f.file.PRead(ctx, dst, offset+total, vfs.ReadOptions{})
		total += n
		if err == io.EOF && total != 0 {
			return total, io.ErrUnexpectedEOF
		} else if err != nil {
			return total, err
		}
		dst = dst.DropFirst64(n)
	}
	return total, nil
}

// ConfigureMMap implements File.
func (f *VFSFile) ConfigureMMap(ctx context.Context, opts *memmap.MMapOpts) error {
	return f.file.ConfigureMMap(ctx, opts)
}

// Type implements File.
func (f *VFSFile) Type(ctx context.Context) (linux.FileMode, error) {
	stat, err := f.file.Stat(ctx, vfs.StatOptions{})
	if err != nil {
		return 0, err
	}
	return linux.FileMode(stat.Mode).FileType(), nil
}

// IncRef implements File.
func (f *VFSFile) IncRef() {
	f.file.IncRef()
}

// DecRef implements File.
func (f *VFSFile) DecRef(ctx context.Context) {
	f.file.DecRef(ctx)
}

// FileDescription returns the FileDescription represented by f. It does not
// take an additional reference on the returned FileDescription.
func (f *VFSFile) FileDescription() *vfs.FileDescription {
	return f.file
}

// fsLookup implements Lookup interface using fs.File.
//
// +stateify savable
type vfsLookup struct {
	mntns *vfs.MountNamespace

	root       vfs.VirtualDentry
	workingDir vfs.VirtualDentry
}

var _ Lookup = (*vfsLookup)(nil)

// NewVFSLookup creates a new Lookup using VFS2.
func NewVFSLookup(mntns *vfs.MountNamespace, root, workingDir vfs.VirtualDentry) Lookup {
	return &vfsLookup{
		mntns:      mntns,
		root:       root,
		workingDir: workingDir,
	}
}

// OpenPath implements Lookup.
//
// remainingTraversals is not configurable in VFS2, all callers are using the
// default anyways.
func (l *vfsLookup) OpenPath(ctx context.Context, pathname string, opts vfs.OpenOptions, _ *uint, resolveFinal bool) (File, error) {
	vfsObj := l.root.Mount().Filesystem().VirtualFilesystem()
	creds := auth.CredentialsFromContext(ctx)
	path := fspath.Parse(pathname)
	pop := &vfs.PathOperation{
		Root:               l.root,
		Start:              l.workingDir,
		Path:               path,
		FollowFinalSymlink: resolveFinal,
	}
	if path.Absolute {
		pop.Start = l.root
	}
	fd, err := vfsObj.OpenAt(ctx, creds, pop, &opts)
	if err != nil {
		return nil, err
	}
	return &VFSFile{file: fd}, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package storage

import (
	"time"

	"github.com/pkg/errors"
	"golang.org/x/net/context"

	"github.com/cockroachdb/cockroach/pkg/config"
	"github.com/cockroachdb/cockroach/pkg/gossip"
	"github.com/cockroachdb/cockroach/pkg/internal/client"
	"github.com/cockroachdb/cockroach/pkg/roachpb"
	"github.com/cockroachdb/cockroach/pkg/util/hlc"
	"github.com/cockroachdb/cockroach/pkg/util/log"
)

const (
	// splitQueueMaxSize is the max size of the split queue.
	splitQueueMaxSize = 100
	// splitQueueTimerDuration is the duration between splits of queued ranges.
	splitQueueTimerDuration = 0 // zero duration to process splits greedily.
)

// splitQueue manages a queue of ranges slated to be split due to size
// or along intersecting zone config boundaries.
type splitQueue struct {
	*baseQueue
	db *client.DB
}

// newSplitQueue returns a new instance of splitQueue.
func newSplitQueue(store *Store, db *client.DB, gossip *gossip.Gossip) *splitQueue {
	sq := &splitQueue{
		db: db,
	}
	sq.baseQueue = newBaseQueue(
		"split", sq, store, gossip,
		queueConfig{
			maxSize:              splitQueueMaxSize,
			needsLease:           true,
			acceptsUnsplitRanges: true,
			successes:            store.metrics.SplitQueueSuccesses,
			failures:             store.metrics.SplitQueueFailures,
			pending:              store.metrics.SplitQueuePending,
			processingNanos:      store.metrics.SplitQueueProcessingNanos,
		},
	)
	return sq
}

// shouldQueue determines whether a range should be queued for
// splitting. This is true if the range is intersected by a zone config
// prefix or if the range's size in bytes exceeds the limit for the zone.
func (sq *splitQueue) shouldQueue(
	ctx context.Context, now hlc.Timestamp, repl *Replica, sysCfg config.SystemConfig,
) (shouldQ bool, priority float64) {
	desc := repl.Desc()
	if len(sysCfg.ComputeSplitKeys(desc.StartKey, desc.EndKey)) > 0 {
		// Set priority to 1 in the event the range is split by zone configs.
		priority = 1
		shouldQ = true
	}

	// Add priority based on the size of range compared to the max
	// size for the zone it's in.
	zone, err := sysCfg.GetZoneConfigForKey(desc.StartKey)
	if err != nil {
		log.Error(ctx, err)
		return
	}

	if ratio := float64(repl.GetMVCCStats().Total()) / float64(zone.RangeMaxBytes); ratio > 1 {
		priority += ratio
		shouldQ = true
	}
	return
}

// process synchronously invokes admin split for each proposed split key.
func (sq *splitQueue) process(
	ctx context.Context, now hlc.Timestamp, r *Replica, sysCfg config.SystemConfig,
) error {
	// First handle case of splitting due to zone config maps.
	desc := r.Desc()
	splitKeys := sysCfg.ComputeSplitKeys(desc.StartKey, desc.EndKey)
	if len(splitKeys) > 0 {
		log.Infof(ctx, "splitting at keys %v", splitKeys)
		for _, splitKey := range splitKeys {
			if err := sq.db.AdminSplit(ctx, splitKey.AsRawKey()); err != nil {
				return errors.Errorf("unable to split %s at key %q: %s", r, splitKey, err)
			}
		}
		return nil
	}

	// Next handle case of splitting due to size.
	zone, err := sysCfg.GetZoneConfigForKey(desc.StartKey)
	if err != nil {
		return err
	}
	size := r.GetMVCCStats().Total()
	// FIXME: why is this implementation not the same as the one above?
	if float64(size)/float64(zone.RangeMaxBytes) > 1 {
		log.Infof(ctx, "splitting size=%d max=%d", size, zone.RangeMaxBytes)
		if _, pErr := client.SendWrappedWith(ctx, r, roachpb.Header{
			Timestamp: now,
		}, &roachpb.AdminSplitRequest{
			Span: roachpb.Span{Key: desc.StartKey.AsRawKey()},
		}); pErr != nil {
			return pErr.GoError()
		}
	}
	return nil
}

// timer returns interval between processing successive queued splits.
func (*splitQueue) timer() time.Duration {
	return splitQueueTimerDuration
}

// purgatoryChan returns nil.
func (*splitQueue) purgatoryChan() <-chan struct{} {
	return nil
}
<commit_msg>storage: unify range-splitting implementations in split queue<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package storage

import (
	"time"

	"github.com/pkg/errors"
	"golang.org/x/net/context"

	"github.com/cockroachdb/cockroach/pkg/config"
	"github.com/cockroachdb/cockroach/pkg/gossip"
	"github.com/cockroachdb/cockroach/pkg/internal/client"
	"github.com/cockroachdb/cockroach/pkg/roachpb"
	"github.com/cockroachdb/cockroach/pkg/util/hlc"
	"github.com/cockroachdb/cockroach/pkg/util/log"
)

const (
	// splitQueueMaxSize is the max size of the split queue.
	splitQueueMaxSize = 100
	// splitQueueTimerDuration is the duration between splits of queued ranges.
	splitQueueTimerDuration = 0 // zero duration to process splits greedily.
)

// splitQueue manages a queue of ranges slated to be split due to size
// or along intersecting zone config boundaries.
type splitQueue struct {
	*baseQueue
	db *client.DB
}

// newSplitQueue returns a new instance of splitQueue.
func newSplitQueue(store *Store, db *client.DB, gossip *gossip.Gossip) *splitQueue {
	sq := &splitQueue{
		db: db,
	}
	sq.baseQueue = newBaseQueue(
		"split", sq, store, gossip,
		queueConfig{
			maxSize:              splitQueueMaxSize,
			needsLease:           true,
			acceptsUnsplitRanges: true,
			successes:            store.metrics.SplitQueueSuccesses,
			failures:             store.metrics.SplitQueueFailures,
			pending:              store.metrics.SplitQueuePending,
			processingNanos:      store.metrics.SplitQueueProcessingNanos,
		},
	)
	return sq
}

// shouldQueue determines whether a range should be queued for
// splitting. This is true if the range is intersected by a zone config
// prefix or if the range's size in bytes exceeds the limit for the zone.
func (sq *splitQueue) shouldQueue(
	ctx context.Context, now hlc.Timestamp, repl *Replica, sysCfg config.SystemConfig,
) (shouldQ bool, priority float64) {
	desc := repl.Desc()
	if len(sysCfg.ComputeSplitKeys(desc.StartKey, desc.EndKey)) > 0 {
		// Set priority to 1 in the event the range is split by zone configs.
		priority = 1
		shouldQ = true
	}

	// Add priority based on the size of range compared to the max
	// size for the zone it's in.
	zone, err := sysCfg.GetZoneConfigForKey(desc.StartKey)
	if err != nil {
		log.Error(ctx, err)
		return
	}

	if ratio := float64(repl.GetMVCCStats().Total()) / float64(zone.RangeMaxBytes); ratio > 1 {
		priority += ratio
		shouldQ = true
	}
	return
}

// process synchronously invokes admin split for each proposed split key.
func (sq *splitQueue) process(
	ctx context.Context, now hlc.Timestamp, r *Replica, sysCfg config.SystemConfig,
) error {
	// First handle case of splitting due to zone config maps.
	desc := r.Desc()
	splitKeys := sysCfg.ComputeSplitKeys(desc.StartKey, desc.EndKey)
	if len(splitKeys) > 0 {
		log.Infof(ctx, "splitting at keys %v", splitKeys)
		for _, splitKey := range splitKeys {
			if _, pErr := r.AdminSplit(
				ctx,
				roachpb.AdminSplitRequest{
					SplitKey: splitKey.AsRawKey(),
				},
				desc,
			); pErr != nil {
				return errors.Wrapf(pErr.GoError(), "unable to split %s at key %q", r, splitKey)
			}
		}
		return nil
	}

	// Next handle case of splitting due to size.
	zone, err := sysCfg.GetZoneConfigForKey(desc.StartKey)
	if err != nil {
		return err
	}
	size := r.GetMVCCStats().Total()
	if float64(size)/float64(zone.RangeMaxBytes) > 1 {
		log.Infof(ctx, "splitting size=%d max=%d", size, zone.RangeMaxBytes)
		if _, pErr := r.AdminSplit(
			ctx,
			roachpb.AdminSplitRequest{},
			desc,
		); pErr != nil {
			return pErr.GoError()
		}
	}
	return nil
}

// timer returns interval between processing successive queued splits.
func (*splitQueue) timer() time.Duration {
	return splitQueueTimerDuration
}

// purgatoryChan returns nil.
func (*splitQueue) purgatoryChan() <-chan struct{} {
	return nil
}
<|endoftext|>