<commit_before>package main

import (
	log "github.com/Sirupsen/logrus"
	"github.com/spf13/pflag"

	"github.com/jtblin/kube2iam/iam"
	"github.com/jtblin/kube2iam/iptables"
	"github.com/jtblin/kube2iam/server"
	"github.com/jtblin/kube2iam/version"
)

// addFlags adds the command line flags.
func addFlags(s *server.Server, fs *pflag.FlagSet) {
	fs.StringVar(&s.APIServer, "api-server", s.APIServer, "Endpoint for the api server")
	fs.StringVar(&s.APIToken, "api-token", s.APIToken, "Token to authenticate with the api server")
	fs.StringVar(&s.AppPort, "app-port", s.AppPort, "Http port")
	fs.StringVar(&s.BaseRoleARN, "base-role-arn", s.BaseRoleARN, "Base role ARN")
	fs.BoolVar(&s.Debug, "debug", s.Debug, "Enable debug features")
	fs.StringVar(&s.DefaultIAMRole, "default-role", s.DefaultIAMRole, "Fallback role to use when annotation is not set")
	fs.StringVar(&s.IAMRoleKey, "iam-role-key", s.IAMRoleKey, "Pod annotation key used to retrieve the IAM role")
	fs.BoolVar(&s.Insecure, "insecure", false, "Kubernetes server should be accessed without verifying the TLS. Testing only")
	fs.StringVar(&s.MetadataAddress, "metadata-addr", s.MetadataAddress, "Address for the ec2 metadata")
	fs.BoolVar(&s.AddIPTablesRule, "iptables", false, "Add iptables rule (also requires --host-ip)")
	fs.BoolVar(&s.AutoDiscoverBaseArn, "auto-discover-base-arn", false, "Queries EC2 Metadata to determine the base ARN")
	fs.BoolVar(&s.AutoDiscoverDefaultRole, "auto-discover-default-role", false, "Queries EC2 Metadata to determine the default Iam Role and base ARN, cannot be used with --default-role, overwrites any previous setting for --base-role-arn")
	fs.StringVar(&s.HostInterface, "host-interface", "docker0", "Host interface for proxying AWS metadata")
	fs.BoolVar(&s.NamespaceRestriction, "namespace-restrictions", false, "Enable namespace restrictions")
	fs.StringVar(&s.NamespaceKey, "namespace-key", s.NamespaceKey, "Namespace annotation key used to retrieve the IAM roles allowed (value in annotation should be json array)")
	fs.StringVar(&s.HostIP, "host-ip", s.HostIP, "IP address of host")
	fs.DurationVar(&s.BackoffMaxInterval, "backoff-max-interval", s.BackoffMaxInterval, "Max interval for backoff when querying for role.")
	fs.DurationVar(&s.BackoffMaxElapsedTime, "backoff-max-elapsed-time", s.BackoffMaxElapsedTime, "Max elapsed time for backoff when querying for role.")
	fs.BoolVar(&s.Verbose, "verbose", false, "Verbose")
	fs.BoolVar(&s.Version, "version", false, "Print the version and exits")
}

func main() {
	s := server.NewServer()
	addFlags(s, pflag.CommandLine)
	pflag.Parse()

	log.SetLevel(log.InfoLevel)
	if s.Verbose {
		log.SetLevel(log.DebugLevel)
	}

	if s.Version {
		version.PrintVersionAndExit()
	}

	if s.AutoDiscoverBaseArn {
		if s.BaseRoleARN != "" {
			log.Fatal("--auto-discover-base-arn cannot be used if --base-role-arn is specified")
		}
		arn, err := iam.GetBaseArn()
		if err != nil {
			log.Fatalf("%s", err)
		}
		log.Infof("base ARN autodetected, %s", arn)
		s.BaseRoleARN = arn
	}

	if s.AutoDiscoverDefaultRole {
		if s.DefaultIAMRole != "" {
			log.Fatalf("You cannot use --default-role and --auto-discover-default-role at the same time")
		}
		arn, err := iam.GetBaseArn()
		if err != nil {
			log.Fatalf("%s", err)
		}
		s.BaseRoleARN = arn
		instanceIAMRole, err := iam.GetInstanceIamRole()
		if err != nil {
			log.Fatalf("%s", err)
		}
		s.DefaultIAMRole = instanceIAMRole
		log.Infof("Using instance IAMRole %s%s as default", s.BaseRoleARN, s.DefaultIAMRole)
	}

	if s.AddIPTablesRule {
		if err := iptables.AddRule(s.AppPort, s.MetadataAddress, s.HostInterface, s.HostIP); err != nil {
			log.Fatalf("%s", err)
		}
	}

	if err := s.Run(s.APIServer, s.APIToken, s.Insecure); err != nil {
		log.Fatalf("%s", err)
	}
}
<commit_msg>Fix #71 - validate base arn format<commit_after>package main

import (
	"regexp"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/spf13/pflag"

	"github.com/jtblin/kube2iam/iam"
	"github.com/jtblin/kube2iam/iptables"
	"github.com/jtblin/kube2iam/server"
	"github.com/jtblin/kube2iam/version"
)

// addFlags adds the command line flags.
func addFlags(s *server.Server, fs *pflag.FlagSet) {
	fs.StringVar(&s.APIServer, "api-server", s.APIServer, "Endpoint for the api server")
	fs.StringVar(&s.APIToken, "api-token", s.APIToken, "Token to authenticate with the api server")
	fs.StringVar(&s.AppPort, "app-port", s.AppPort, "Http port")
	fs.StringVar(&s.BaseRoleARN, "base-role-arn", s.BaseRoleARN, "Base role ARN")
	fs.BoolVar(&s.Debug, "debug", s.Debug, "Enable debug features")
	fs.StringVar(&s.DefaultIAMRole, "default-role", s.DefaultIAMRole, "Fallback role to use when annotation is not set")
	fs.StringVar(&s.IAMRoleKey, "iam-role-key", s.IAMRoleKey, "Pod annotation key used to retrieve the IAM role")
	fs.BoolVar(&s.Insecure, "insecure", false, "Kubernetes server should be accessed without verifying the TLS. Testing only")
	fs.StringVar(&s.MetadataAddress, "metadata-addr", s.MetadataAddress, "Address for the ec2 metadata")
	fs.BoolVar(&s.AddIPTablesRule, "iptables", false, "Add iptables rule (also requires --host-ip)")
	fs.BoolVar(&s.AutoDiscoverBaseArn, "auto-discover-base-arn", false, "Queries EC2 Metadata to determine the base ARN")
	fs.BoolVar(&s.AutoDiscoverDefaultRole, "auto-discover-default-role", false, "Queries EC2 Metadata to determine the default Iam Role and base ARN, cannot be used with --default-role, overwrites any previous setting for --base-role-arn")
	fs.StringVar(&s.HostInterface, "host-interface", "docker0", "Host interface for proxying AWS metadata")
	fs.BoolVar(&s.NamespaceRestriction, "namespace-restrictions", false, "Enable namespace restrictions")
	fs.StringVar(&s.NamespaceKey, "namespace-key", s.NamespaceKey, "Namespace annotation key used to retrieve the IAM roles allowed (value in annotation should be json array)")
	fs.StringVar(&s.HostIP, "host-ip", s.HostIP, "IP address of host")
	fs.DurationVar(&s.BackoffMaxInterval, "backoff-max-interval", s.BackoffMaxInterval, "Max interval for backoff when querying for role.")
	fs.DurationVar(&s.BackoffMaxElapsedTime, "backoff-max-elapsed-time", s.BackoffMaxElapsedTime, "Max elapsed time for backoff when querying for role.")
	fs.BoolVar(&s.Verbose, "verbose", false, "Verbose")
	fs.BoolVar(&s.Version, "version", false, "Print the version and exits")
}

var arnRegexp = regexp.MustCompile(`^arn:\w*:iam:\w*:\w*:role\/?$`)

func main() {
	s := server.NewServer()
	addFlags(s, pflag.CommandLine)
	pflag.Parse()

	log.SetLevel(log.InfoLevel)
	if s.Verbose {
		log.SetLevel(log.DebugLevel)
	}

	if s.Version {
		version.PrintVersionAndExit()
	}

	if s.BaseRoleARN != "" {
		if !arnRegexp.MatchString(s.BaseRoleARN) {
			log.Fatalf("Invalid --base-role-arn specified, expected: %s", arnRegexp.String())
		}
		if !strings.HasSuffix(s.BaseRoleARN, "/") {
			s.BaseRoleARN += "/"
		}
	}

	if s.AutoDiscoverBaseArn {
		if s.BaseRoleARN != "" {
			log.Fatal("--auto-discover-base-arn cannot be used if --base-role-arn is specified")
		}
		arn, err := iam.GetBaseArn()
		if err != nil {
			log.Fatalf("%s", err)
		}
		log.Infof("base ARN autodetected, %s", arn)
		s.BaseRoleARN = arn
	}

	if s.AutoDiscoverDefaultRole {
		if s.DefaultIAMRole != "" {
			log.Fatalf("You cannot use --default-role and --auto-discover-default-role at the same time")
		}
		arn, err := iam.GetBaseArn()
		if err != nil {
			log.Fatalf("%s", err)
		}
		s.BaseRoleARN = arn
		instanceIAMRole, err := iam.GetInstanceIamRole()
		if err != nil {
			log.Fatalf("%s", err)
		}
		s.DefaultIAMRole = instanceIAMRole
		log.Infof("Using instance IAMRole %s%s as default", s.BaseRoleARN, s.DefaultIAMRole)
	}

	if s.AddIPTablesRule {
		if err := iptables.AddRule(s.AppPort, s.MetadataAddress, s.HostInterface, s.HostIP); err != nil {
			log.Fatalf("%s", err)
		}
	}

	if err := s.Run(s.APIServer, s.APIToken, s.Insecure); err != nil {
		log.Fatalf("%s", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"
)

const (
	postLayerURI               = "/v1/layers"
	getLayerVulnerabilitiesURI = "/v1/layers/%s/vulnerabilities?minimumPriority=%s"
	httpPort                   = 9279
)

type APIVulnerabilitiesResponse struct {
	Vulnerabilities []APIVulnerability
}

type APIVulnerability struct {
	ID, Link, Priority, Description string
}

func main() {
	endpoint := flag.String("endpoint", "http://127.0.0.1:6060", "Address to Clair API")
	myAddress := flag.String("my-address", "127.0.0.1", "Address from the point of view of Clair")
	minimumPriority := flag.String("minimum-priority", "Low", "Minimum vulnerability vulnerability to show")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [options] image-id\n\nOptions:\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()

	if len(flag.Args()) != 1 {
		flag.Usage()
		os.Exit(1)
	}
	imageName := flag.Args()[0]

	// Save image
	fmt.Printf("Saving %s\n", imageName)
	path, err := save(imageName)
	defer os.RemoveAll(path)
	if err != nil {
		log.Fatalf("- Could not save image: %s\n", err)
	}

	// Retrieve history
	fmt.Println("Getting image's history")
	layerIDs, err := history(imageName)
	if err != nil || len(layerIDs) == 0 {
		log.Fatalf("- Could not get image's history: %s\n", err)
	}

	// Setup a simple HTTP server if Clair is not local
	if !strings.Contains(*endpoint, "127.0.0.1") && !strings.Contains(*endpoint, "localhost") {
		go func(path string) {
			allowedHost := strings.TrimPrefix(*endpoint, "http://")
			portIndex := strings.Index(allowedHost, ":")
			if portIndex >= 0 {
				allowedHost = allowedHost[:portIndex]
			}

			fmt.Printf("Setting up HTTP server (allowing: %s)\n", allowedHost)

			err := http.ListenAndServe(":"+strconv.Itoa(httpPort), restrictedFileServer(path, allowedHost))
			if err != nil {
				log.Fatalf("- An error occurs with the HTTP Server: %s\n", err)
			}
		}(path)

		path = "http://" + *myAddress + ":" + strconv.Itoa(httpPort)
		time.Sleep(200 * time.Millisecond)
	}

	// Analyze layers
	fmt.Printf("Analyzing %d layers\n", len(layerIDs))
	for i := 0; i < len(layerIDs); i++ {
		fmt.Printf("- Analyzing %s\n", layerIDs[i])

		var err error
		if i > 0 {
			err = analyzeLayer(*endpoint, path+"/"+layerIDs[i]+"/layer.tar", layerIDs[i], layerIDs[i-1])
		} else {
			err = analyzeLayer(*endpoint, path+"/"+layerIDs[i]+"/layer.tar", layerIDs[i], "")
		}
		if err != nil {
			log.Fatalf("- Could not analyze layer: %s\n", err)
		}
	}

	// Get vulnerabilities
	fmt.Println("Getting image's vulnerabilities")
	vulnerabilities, err := getVulnerabilities(*endpoint, layerIDs[len(layerIDs)-1], *minimumPriority)
	if err != nil {
		log.Fatalf("- Could not get vulnerabilities: %s\n", err)
	}
	if len(vulnerabilities) == 0 {
		fmt.Println("Bravo, your image looks SAFE !")
	}
	for _, vulnerability := range vulnerabilities {
		fmt.Printf("- # %s\n", vulnerability.ID)
		fmt.Printf("  - Priority:    %s\n", vulnerability.Priority)
		fmt.Printf("  - Link:        %s\n", vulnerability.Link)
		fmt.Printf("  - Description: %s\n", vulnerability.Description)
	}
}

func save(imageName string) (string, error) {
	path, err := ioutil.TempDir("", "analyze-local-image-")
	if err != nil {
		return "", err
	}

	var stderr bytes.Buffer
	save := exec.Command("docker", "save", imageName)
	save.Stderr = &stderr
	extract := exec.Command("tar", "xf", "-", "-C"+path)
	extract.Stderr = &stderr
	pipe, err := extract.StdinPipe()
	if err != nil {
		return "", err
	}
	save.Stdout = pipe

	err = extract.Start()
	if err != nil {
		return "", errors.New(stderr.String())
	}
	err = save.Run()
	if err != nil {
		return "", errors.New(stderr.String())
	}

	return path, nil
}

func history(imageName string) ([]string, error) {
	var stderr bytes.Buffer
	cmd := exec.Command("docker", "history", "-q", "--no-trunc", imageName)
	cmd.Stderr = &stderr
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return []string{}, err
	}

	err = cmd.Start()
	if err != nil {
		return []string{}, errors.New(stderr.String())
	}

	var layers []string
	scanner := bufio.NewScanner(stdout)
	for scanner.Scan() {
		layers = append(layers, scanner.Text())
	}

	for i := len(layers)/2 - 1; i >= 0; i-- {
		opp := len(layers) - 1 - i
		layers[i], layers[opp] = layers[opp], layers[i]
	}

	return layers, nil
}

func analyzeLayer(endpoint, path, layerID, parentLayerID string) error {
	payload := struct{ ID, Path, ParentID string }{ID: layerID, Path: path, ParentID: parentLayerID}
	jsonPayload, err := json.Marshal(payload)
	if err != nil {
		return err
	}

	request, err := http.NewRequest("POST", endpoint+postLayerURI, bytes.NewBuffer(jsonPayload))
	if err != nil {
		return err
	}
	request.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	response, err := client.Do(request)
	if err != nil {
		return err
	}
	defer response.Body.Close()

	if response.StatusCode != 201 {
		body, _ := ioutil.ReadAll(response.Body)
		return fmt.Errorf("Got response %d with message %s", response.StatusCode, string(body))
	}

	return nil
}

func getVulnerabilities(endpoint, layerID, minimumPriority string) ([]APIVulnerability, error) {
	response, err := http.Get(endpoint + fmt.Sprintf(getLayerVulnerabilitiesURI, layerID, minimumPriority))
	if err != nil {
		return []APIVulnerability{}, err
	}
	defer response.Body.Close()

	if response.StatusCode != 200 {
		body, _ := ioutil.ReadAll(response.Body)
		return []APIVulnerability{}, fmt.Errorf("Got response %d with message %s", response.StatusCode, string(body))
	}

	var apiResponse APIVulnerabilitiesResponse
	err = json.NewDecoder(response.Body).Decode(&apiResponse)
	if err != nil {
		return []APIVulnerability{}, err
	}

	return apiResponse.Vulnerabilities, nil
}

func restrictedFileServer(path, allowedHost string) http.Handler {
	fc := func(w http.ResponseWriter, r *http.Request) {
		if r.Host == allowedHost {
			http.FileServer(http.Dir(path)).ServeHTTP(w, r)
			return
		}
		w.WriteHeader(403)
	}
	return http.HandlerFunc(fc)
}
<commit_msg>contrib: Wait for extraction to finish before continuing.<commit_after>package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"
)

const (
	postLayerURI               = "/v1/layers"
	getLayerVulnerabilitiesURI = "/v1/layers/%s/vulnerabilities?minimumPriority=%s"
	httpPort                   = 9279
)

type APIVulnerabilitiesResponse struct {
	Vulnerabilities []APIVulnerability
}

type APIVulnerability struct {
	ID, Link, Priority, Description string
}

func main() {
	endpoint := flag.String("endpoint", "http://127.0.0.1:6060", "Address to Clair API")
	myAddress := flag.String("my-address", "127.0.0.1", "Address from the point of view of Clair")
	minimumPriority := flag.String("minimum-priority", "Low", "Minimum vulnerability vulnerability to show")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [options] image-id\n\nOptions:\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()

	if len(flag.Args()) != 1 {
		flag.Usage()
		os.Exit(1)
	}
	imageName := flag.Args()[0]

	// Save image
	fmt.Printf("Saving %s\n", imageName)
	path, err := save(imageName)
	defer os.RemoveAll(path)
	if err != nil {
		log.Fatalf("- Could not save image: %s\n", err)
	}

	// Retrieve history
	fmt.Println("Getting image's history")
	layerIDs, err := history(imageName)
	if err != nil || len(layerIDs) == 0 {
		log.Fatalf("- Could not get image's history: %s\n", err)
	}

	// Setup a simple HTTP server if Clair is not local
	if !strings.Contains(*endpoint, "127.0.0.1") && !strings.Contains(*endpoint, "localhost") {
		go func(path string) {
			allowedHost := strings.TrimPrefix(*endpoint, "http://")
			portIndex := strings.Index(allowedHost, ":")
			if portIndex >= 0 {
				allowedHost = allowedHost[:portIndex]
			}

			fmt.Printf("Setting up HTTP server (allowing: %s)\n", allowedHost)

			err := http.ListenAndServe(":"+strconv.Itoa(httpPort), restrictedFileServer(path, allowedHost))
			if err != nil {
				log.Fatalf("- An error occurs with the HTTP Server: %s\n", err)
			}
		}(path)

		path = "http://" + *myAddress + ":" + strconv.Itoa(httpPort)
		time.Sleep(200 * time.Millisecond)
	}

	// Analyze layers
	fmt.Printf("Analyzing %d layers\n", len(layerIDs))
	for i := 0; i < len(layerIDs); i++ {
		fmt.Printf("- Analyzing %s\n", layerIDs[i])

		var err error
		if i > 0 {
			err = analyzeLayer(*endpoint, path+"/"+layerIDs[i]+"/layer.tar", layerIDs[i], layerIDs[i-1])
		} else {
			err = analyzeLayer(*endpoint, path+"/"+layerIDs[i]+"/layer.tar", layerIDs[i], "")
		}
		if err != nil {
			log.Fatalf("- Could not analyze layer: %s\n", err)
		}
	}

	// Get vulnerabilities
	fmt.Println("Getting image's vulnerabilities")
	vulnerabilities, err := getVulnerabilities(*endpoint, layerIDs[len(layerIDs)-1], *minimumPriority)
	if err != nil {
		log.Fatalf("- Could not get vulnerabilities: %s\n", err)
	}
	if len(vulnerabilities) == 0 {
		fmt.Println("Bravo, your image looks SAFE !")
	}
	for _, vulnerability := range vulnerabilities {
		fmt.Printf("- # %s\n", vulnerability.ID)
		fmt.Printf("  - Priority:    %s\n", vulnerability.Priority)
		fmt.Printf("  - Link:        %s\n", vulnerability.Link)
		fmt.Printf("  - Description: %s\n", vulnerability.Description)
	}
}

func save(imageName string) (string, error) {
	path, err := ioutil.TempDir("", "analyze-local-image-")
	if err != nil {
		return "", err
	}

	var stderr bytes.Buffer
	save := exec.Command("docker", "save", imageName)
	save.Stderr = &stderr
	extract := exec.Command("tar", "xf", "-", "-C"+path)
	extract.Stderr = &stderr
	pipe, err := extract.StdinPipe()
	if err != nil {
		return "", err
	}
	save.Stdout = pipe

	err = extract.Start()
	if err != nil {
		return "", errors.New(stderr.String())
	}
	err = save.Run()
	if err != nil {
		return "", errors.New(stderr.String())
	}
	err = pipe.Close()
	if err != nil {
		return "", err
	}
	err = extract.Wait()
	if err != nil {
		return "", errors.New(stderr.String())
	}

	return path, nil
}

func history(imageName string) ([]string, error) {
	var stderr bytes.Buffer
	cmd := exec.Command("docker", "history", "-q", "--no-trunc", imageName)
	cmd.Stderr = &stderr
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return []string{}, err
	}

	err = cmd.Start()
	if err != nil {
		return []string{}, errors.New(stderr.String())
	}

	var layers []string
	scanner := bufio.NewScanner(stdout)
	for scanner.Scan() {
		layers = append(layers, scanner.Text())
	}

	for i := len(layers)/2 - 1; i >= 0; i-- {
		opp := len(layers) - 1 - i
		layers[i], layers[opp] = layers[opp], layers[i]
	}

	return layers, nil
}

func analyzeLayer(endpoint, path, layerID, parentLayerID string) error {
	payload := struct{ ID, Path, ParentID string }{ID: layerID, Path: path, ParentID: parentLayerID}
	jsonPayload, err := json.Marshal(payload)
	if err != nil {
		return err
	}

	request, err := http.NewRequest("POST", endpoint+postLayerURI, bytes.NewBuffer(jsonPayload))
	if err != nil {
		return err
	}
	request.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	response, err := client.Do(request)
	if err != nil {
		return err
	}
	defer response.Body.Close()

	if response.StatusCode != 201 {
		body, _ := ioutil.ReadAll(response.Body)
		return fmt.Errorf("Got response %d with message %s", response.StatusCode, string(body))
	}

	return nil
}

func getVulnerabilities(endpoint, layerID, minimumPriority string) ([]APIVulnerability, error) {
	response, err := http.Get(endpoint + fmt.Sprintf(getLayerVulnerabilitiesURI, layerID, minimumPriority))
	if err != nil {
		return []APIVulnerability{}, err
	}
	defer response.Body.Close()

	if response.StatusCode != 200 {
		body, _ := ioutil.ReadAll(response.Body)
		return []APIVulnerability{}, fmt.Errorf("Got response %d with message %s", response.StatusCode, string(body))
	}

	var apiResponse APIVulnerabilitiesResponse
	err = json.NewDecoder(response.Body).Decode(&apiResponse)
	if err != nil {
		return []APIVulnerability{}, err
	}

	return apiResponse.Vulnerabilities, nil
}

func restrictedFileServer(path, allowedHost string) http.Handler {
	fc := func(w http.ResponseWriter, r *http.Request) {
		if r.Host == allowedHost {
			http.FileServer(http.Dir(path)).ServeHTTP(w, r)
			return
		}
		w.WriteHeader(403)
	}
	return http.HandlerFunc(fc)
}
<|endoftext|>


<commit_before>package gcc

import (
	"container/list"
	"sync"
	"time"

	"github.com/pion/interceptor"
	"github.com/pion/logging"
	"github.com/pion/rtp"
)

type item struct {
	header     *rtp.Header
	payload    *[]byte
	size       int
	attributes interceptor.Attributes
}

// LeakyBucketPacer implements a leaky bucket pacing algorithm
type LeakyBucketPacer struct {
	log logging.LeveledLogger

	f                 float64
	targetBitrate     int
	targetBitrateLock sync.Mutex

	pacingInterval time.Duration

	qLock sync.RWMutex
	queue *list.List
	done  chan struct{}

	ssrcToWriter map[uint32]interceptor.RTPWriter
	writerLock   sync.RWMutex

	pool *sync.Pool
}

// NewLeakyBucketPacer initializes a new LeakyBucketPacer
func NewLeakyBucketPacer(initialBitrate int) *LeakyBucketPacer {
	p := &LeakyBucketPacer{
		log:            logging.NewDefaultLoggerFactory().NewLogger("pacer"),
		f:              1.5,
		targetBitrate:  initialBitrate,
		pacingInterval: 5 * time.Millisecond,
		qLock:          sync.RWMutex{},
		queue:          list.New(),
		done:           make(chan struct{}),
		ssrcToWriter:   map[uint32]interceptor.RTPWriter{},
		pool:           &sync.Pool{},
	}
	p.pool = &sync.Pool{
		New: func() interface{} {
			b := make([]byte, 1460)
			return &b
		},
	}

	go p.Run()
	return p
}

// AddStream adds a new stream and its corresponding writer to the pacer
func (p *LeakyBucketPacer) AddStream(ssrc uint32, writer interceptor.RTPWriter) {
	p.writerLock.Lock()
	defer p.writerLock.Unlock()
	p.ssrcToWriter[ssrc] = writer
}

// SetTargetBitrate updates the target bitrate at which the pacer is allowed to
// send packets. The pacer may exceed this limit by p.f
func (p *LeakyBucketPacer) SetTargetBitrate(rate int) {
	p.targetBitrateLock.Lock()
	defer p.targetBitrateLock.Unlock()
	p.targetBitrate = int(p.f * float64(rate))
}

func (p *LeakyBucketPacer) getTargetBitrate() int {
	p.targetBitrateLock.Lock()
	defer p.targetBitrateLock.Unlock()

	return p.targetBitrate
}

// Write sends a packet with header and payload the a previously registered
// stream.
func (p *LeakyBucketPacer) Write(header *rtp.Header, payload []byte, attributes interceptor.Attributes) (int, error) {
	buf := p.pool.Get().(*[]byte)
	copy(*buf, payload)
	hdr := header.Clone()

	p.qLock.Lock()
	p.queue.PushBack(&item{
		header:     &hdr,
		payload:    buf,
		size:       len(payload),
		attributes: attributes,
	})
	p.qLock.Unlock()

	return header.MarshalSize() + len(payload), nil
}

// Run starts the LeakyBucketPacer
func (p *LeakyBucketPacer) Run() {
	ticker := time.NewTicker(p.pacingInterval)

	lastSent := time.Now()
	for {
		select {
		case <-p.done:
			return
		case now := <-ticker.C:
			budget := int(float64(now.Sub(lastSent).Milliseconds()) * float64(p.getTargetBitrate()) / 8000.0)
			p.qLock.Lock()
			for p.queue.Len() != 0 && budget > 0 {
				p.log.Infof("budget=%v, len(queue)=%v, targetBitrate=%v", budget, p.queue.Len(), p.getTargetBitrate())
				next := p.queue.Remove(p.queue.Front()).(*item)
				p.qLock.Unlock()

				p.writerLock.RLock()
				writer, ok := p.ssrcToWriter[next.header.SSRC]
				p.writerLock.RUnlock()
				if !ok {
					p.log.Warnf("no writer found for ssrc: %v", next.header.SSRC)
					p.pool.Put(next.payload)
					p.qLock.Lock()
					continue
				}

				n, err := writer.Write(next.header, (*next.payload)[:next.size], next.attributes)
				if err != nil {
					p.log.Errorf("failed to write packet: %v", err)
				}
				lastSent = now
				budget -= n

				p.pool.Put(next.payload)
				p.qLock.Lock()
			}
			p.qLock.Unlock()
		}
	}
}

// Close closes the LeakyBucketPacer
func (p *LeakyBucketPacer) Close() error {
	close(p.done)
	return nil
}
<commit_msg>Fix LeakyBucketPacker.Run not stopping Ticker<commit_after>package gcc

import (
	"container/list"
	"sync"
	"time"

	"github.com/pion/interceptor"
	"github.com/pion/logging"
	"github.com/pion/rtp"
)

type item struct {
	header     *rtp.Header
	payload    *[]byte
	size       int
	attributes interceptor.Attributes
}

// LeakyBucketPacer implements a leaky bucket pacing algorithm
type LeakyBucketPacer struct {
	log logging.LeveledLogger

	f                 float64
	targetBitrate     int
	targetBitrateLock sync.Mutex

	pacingInterval time.Duration

	qLock sync.RWMutex
	queue *list.List
	done  chan struct{}

	ssrcToWriter map[uint32]interceptor.RTPWriter
	writerLock   sync.RWMutex

	pool *sync.Pool
}

// NewLeakyBucketPacer initializes a new LeakyBucketPacer
func NewLeakyBucketPacer(initialBitrate int) *LeakyBucketPacer {
	p := &LeakyBucketPacer{
		log:            logging.NewDefaultLoggerFactory().NewLogger("pacer"),
		f:              1.5,
		targetBitrate:  initialBitrate,
		pacingInterval: 5 * time.Millisecond,
		qLock:          sync.RWMutex{},
		queue:          list.New(),
		done:           make(chan struct{}),
		ssrcToWriter:   map[uint32]interceptor.RTPWriter{},
		pool:           &sync.Pool{},
	}
	p.pool = &sync.Pool{
		New: func() interface{} {
			b := make([]byte, 1460)
			return &b
		},
	}

	go p.Run()
	return p
}

// AddStream adds a new stream and its corresponding writer to the pacer
func (p *LeakyBucketPacer) AddStream(ssrc uint32, writer interceptor.RTPWriter) {
	p.writerLock.Lock()
	defer p.writerLock.Unlock()
	p.ssrcToWriter[ssrc] = writer
}

// SetTargetBitrate updates the target bitrate at which the pacer is allowed to
// send packets. The pacer may exceed this limit by p.f
func (p *LeakyBucketPacer) SetTargetBitrate(rate int) {
	p.targetBitrateLock.Lock()
	defer p.targetBitrateLock.Unlock()
	p.targetBitrate = int(p.f * float64(rate))
}

func (p *LeakyBucketPacer) getTargetBitrate() int {
	p.targetBitrateLock.Lock()
	defer p.targetBitrateLock.Unlock()

	return p.targetBitrate
}

// Write sends a packet with header and payload the a previously registered
// stream.
func (p *LeakyBucketPacer) Write(header *rtp.Header, payload []byte, attributes interceptor.Attributes) (int, error) {
	buf := p.pool.Get().(*[]byte)
	copy(*buf, payload)
	hdr := header.Clone()

	p.qLock.Lock()
	p.queue.PushBack(&item{
		header:     &hdr,
		payload:    buf,
		size:       len(payload),
		attributes: attributes,
	})
	p.qLock.Unlock()

	return header.MarshalSize() + len(payload), nil
}

// Run starts the LeakyBucketPacer
func (p *LeakyBucketPacer) Run() {
	ticker := time.NewTicker(p.pacingInterval)
	defer ticker.Stop()

	lastSent := time.Now()
	for {
		select {
		case <-p.done:
			return
		case now := <-ticker.C:
			budget := int(float64(now.Sub(lastSent).Milliseconds()) * float64(p.getTargetBitrate()) / 8000.0)
			p.qLock.Lock()
			for p.queue.Len() != 0 && budget > 0 {
				p.log.Infof("budget=%v, len(queue)=%v, targetBitrate=%v", budget, p.queue.Len(), p.getTargetBitrate())
				next := p.queue.Remove(p.queue.Front()).(*item)
				p.qLock.Unlock()

				p.writerLock.RLock()
				writer, ok := p.ssrcToWriter[next.header.SSRC]
				p.writerLock.RUnlock()
				if !ok {
					p.log.Warnf("no writer found for ssrc: %v", next.header.SSRC)
					p.pool.Put(next.payload)
					p.qLock.Lock()
					continue
				}

				n, err := writer.Write(next.header, (*next.payload)[:next.size], next.attributes)
				if err != nil {
					p.log.Errorf("failed to write packet: %v", err)
				}
				lastSent = now
				budget -= n

				p.pool.Put(next.payload)
				p.qLock.Lock()
			}
			p.qLock.Unlock()
		}
	}
}

// Close closes the LeakyBucketPacer
func (p *LeakyBucketPacer) Close() error {
	close(p.done)
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 the Heptio Ark contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package restore

import (
	"encoding/json"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	corev1api "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/apimachinery/pkg/runtime"

	api "github.com/heptio/ark/pkg/apis/ark/v1"
	"github.com/heptio/ark/pkg/util/collections"
)

const annotationLastAppliedConfig = "kubectl.kubernetes.io/last-applied-configuration"

type serviceAction struct {
	log logrus.FieldLogger
}

func NewServiceAction(logger logrus.FieldLogger) ItemAction {
	return &serviceAction{log: logger}
}

func (a *serviceAction) AppliesTo() (ResourceSelector, error) {
	return ResourceSelector{
		IncludedResources: []string{"services"},
	}, nil
}

func (a *serviceAction) Execute(obj runtime.Unstructured, restore *api.Restore) (runtime.Unstructured, error, error) {
	spec, err := collections.GetMap(obj.UnstructuredContent(), "spec")
	if err != nil {
		return nil, nil, err
	}

	// Since clusterIP is an optional key, we can ignore 'not found' errors. Also assuming it was a string already.
	if val, _ := collections.GetString(spec, "clusterIP"); val != "None" {
		delete(spec, "clusterIP")
	}

	preservedPorts, err := getPreservedPorts(obj)
	if err != nil {
		return nil, nil, err
	}

	ports, err := collections.GetSlice(obj.UnstructuredContent(), "spec.ports")
	serviceType, _ := collections.GetString(spec, "type")
	if err != nil && serviceType != "ExternalName" {
		return nil, nil, err
	}

	for _, port := range ports {
		p := port.(map[string]interface{})
		var name string
		if nameVal, ok := p["name"]; ok {
			name = nameVal.(string)
		}
		if preservedPorts[name] {
			continue
		}
		delete(p, "nodePort")
	}

	return obj, nil, nil
}

func getPreservedPorts(obj runtime.Unstructured) (map[string]bool, error) {
	preservedPorts := map[string]bool{}
	metadata, err := meta.Accessor(obj)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	if lac, ok := metadata.GetAnnotations()[annotationLastAppliedConfig]; ok {
		var svc corev1api.Service
		if err := json.Unmarshal([]byte(lac), &svc); err != nil {
			return nil, errors.WithStack(err)
		}
		for _, port := range svc.Spec.Ports {
			if port.NodePort > 0 {
				preservedPorts[port.Name] = true
			}
		}
	}
	return preservedPorts, nil
}
<commit_msg>move code dealing with node ports into a separate function<commit_after>/*
Copyright 2017 the Heptio Ark contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package restore

import (
	"encoding/json"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	corev1api "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/apimachinery/pkg/runtime"

	api "github.com/heptio/ark/pkg/apis/ark/v1"
	"github.com/heptio/ark/pkg/util/collections"
)

const annotationLastAppliedConfig = "kubectl.kubernetes.io/last-applied-configuration"

type serviceAction struct {
	log logrus.FieldLogger
}

func NewServiceAction(logger logrus.FieldLogger) ItemAction {
	return &serviceAction{log: logger}
}

func (a *serviceAction) AppliesTo() (ResourceSelector, error) {
	return ResourceSelector{
		IncludedResources: []string{"services"},
	}, nil
}

func (a *serviceAction) Execute(obj runtime.Unstructured, restore *api.Restore) (runtime.Unstructured, error, error) {
	spec, err := collections.GetMap(obj.UnstructuredContent(), "spec")
	if err != nil {
		return nil, nil, err
	}

	// Since clusterIP is an optional key, we can ignore 'not found' errors. Also assuming it was a string already.
	if val, _ := collections.GetString(spec, "clusterIP"); val != "None" {
		delete(spec, "clusterIP")
	}

	err = deleteNodePorts(obj, &spec)
	if err != nil {
		return nil, nil, err
	}
	return obj, nil, nil
}

func getPreservedPorts(obj runtime.Unstructured) (map[string]bool, error) {
	preservedPorts := map[string]bool{}
	metadata, err := meta.Accessor(obj)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	if lac, ok := metadata.GetAnnotations()[annotationLastAppliedConfig]; ok {
		var svc corev1api.Service
		if err := json.Unmarshal([]byte(lac), &svc); err != nil {
			return nil, errors.WithStack(err)
		}
		for _, port := range svc.Spec.Ports {
			if port.NodePort > 0 {
				preservedPorts[port.Name] = true
			}
		}
	}
	return preservedPorts, nil
}

func deleteNodePorts(obj runtime.Unstructured, spec *map[string]interface{}) error {
	preservedPorts, err := getPreservedPorts(obj)
	if err != nil {
		return err
	}

	ports, err := collections.GetSlice(obj.UnstructuredContent(), "spec.ports")
	serviceType, _ := collections.GetString(*spec, "type")
	if err != nil && serviceType != "ExternalName" {
		return err
	}

	for _, port := range ports {
		p := port.(map[string]interface{})
		var name string
		if nameVal, ok := p["name"]; ok {
			name = nameVal.(string)
		}
		if preservedPorts[name] {
			continue
		}
		delete(p, "nodePort")
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package iptables

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"sync"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	utilexec "github.com/GoogleCloudPlatform/kubernetes/pkg/util/exec"
	"github.com/golang/glog"
)

// An injectable interface for running iptables commands.  Implementations must be goroutine-safe.
type Interface interface {
	// EnsureChain checks if the specified chain exists and, if not, creates it.  If the chain existed, return true.
	EnsureChain(table Table, chain Chain) (bool, error)
	// FlushChain clears the specified chain.
	FlushChain(table Table, chain Chain) error
	// EnsureRule checks if the specified rule is present and, if not, creates it.  If the rule existed, return true.
	EnsureRule(table Table, chain Chain, args ...string) (bool, error)
	// DeleteRule checks if the specified rule is present and, if so, deletes it.
	DeleteRule(table Table, chain Chain, args ...string) error
	// IsIpv6 returns true if this is managing ipv6 tables
	IsIpv6() bool
}

type Protocol byte

const (
	ProtocolIpv4 Protocol = iota + 1
	ProtocolIpv6
)

type Table string

const (
	TableNAT Table = "nat"
)

type Chain string

const (
	ChainPostrouting Chain = "POSTROUTING"
	ChainPrerouting  Chain = "PREROUTING"
	ChainOutput      Chain = "OUTPUT"
)

// runner implements Interface in terms of exec("iptables").
type runner struct {
	mu       sync.Mutex
	exec     utilexec.Interface
	protocol Protocol
}

// New returns a new Interface which will exec iptables.
func New(exec utilexec.Interface, protocol Protocol) Interface {
	return &runner{exec: exec, protocol: protocol}
}

// EnsureChain is part of Interface.
func (runner *runner) EnsureChain(table Table, chain Chain) (bool, error) {
	fullArgs := makeFullArgs(table, chain)

	runner.mu.Lock()
	defer runner.mu.Unlock()

	out, err := runner.run(opCreateChain, fullArgs)
	if err != nil {
		if ee, ok := err.(utilexec.ExitError); ok {
			if ee.Exited() && ee.ExitStatus() == 1 {
				return true, nil
			}
		}
		return false, fmt.Errorf("error creating chain %q: %v: %s", chain, err, out)
	}
	return false, nil
}

// FlushChain is part of Interface.
func (runner *runner) FlushChain(table Table, chain Chain) error {
	fullArgs := makeFullArgs(table, chain)

	runner.mu.Lock()
	defer runner.mu.Unlock()

	out, err := runner.run(opFlushChain, fullArgs)
	if err != nil {
		return fmt.Errorf("error flushing chain %q: %v: %s", chain, err, out)
	}
	return nil
}

// EnsureRule is part of Interface.
func (runner *runner) EnsureRule(table Table, chain Chain, args ...string) (bool, error) {
	fullArgs := makeFullArgs(table, chain, args...)

	runner.mu.Lock()
	defer runner.mu.Unlock()

	exists, err := runner.checkRule(table, chain, args...)
	if err != nil {
		return false, err
	}
	if exists {
		return true, nil
	}
	out, err := runner.run(opAppendRule, fullArgs)
	if err != nil {
		return false, fmt.Errorf("error appending rule: %v: %s", err, out)
	}
	return false, nil
}

// DeleteRule is part of Interface.
func (runner *runner) DeleteRule(table Table, chain Chain, args ...string) error {
	fullArgs := makeFullArgs(table, chain, args...)

	runner.mu.Lock()
	defer runner.mu.Unlock()

	exists, err := runner.checkRule(table, chain, args...)
	if err != nil {
		return err
	}
	if !exists {
		return nil
	}
	out, err := runner.run(opDeleteRule, fullArgs)
	if err != nil {
		return fmt.Errorf("error deleting rule: %v: %s", err, out)
	}
	return nil
}

func (runner *runner) IsIpv6() bool {
	return runner.protocol == ProtocolIpv6
}

func (runner *runner) iptablesCommand() string {
	if runner.IsIpv6() {
		return "ip6tables"
	} else {
		return "iptables"
	}
}

func (runner *runner) run(op operation, args []string) ([]byte, error) {
	iptablesCmd := runner.iptablesCommand()

	fullArgs := append([]string{string(op)}, args...)
	glog.V(1).Infof("running iptables %s %v", string(op), args)
	return runner.exec.Command(iptablesCmd, fullArgs...).CombinedOutput()
	// Don't log err here - callers might not think it is an error.
}

// Returns (bool, nil) if it was able to check the existence of the rule, or
// (<undefined>, error) if the process of checking failed.
func (runner *runner) checkRule(table Table, chain Chain, args ...string) (bool, error) {
	checkPresent, err := getIptablesHasCheckCommand(runner.exec)
	if err != nil {
		glog.Warning("Error checking iptables version, assuming version at least 1.4.11: %v", err)
		checkPresent = true
	}
	if checkPresent {
		return runner.checkRuleUsingCheck(makeFullArgs(table, chain, args...))
	} else {
		return runner.checkRuleWithoutCheck(table, chain, args...)
	}
}

// Executes the rule check without using the "-C" flag, instead parsing iptables-save.
// Present for compatibility with <1.4.11 versions of iptables.
func (runner *runner) checkRuleWithoutCheck(table Table, chain Chain, args ...string) (bool, error) {
	glog.V(1).Infof("running iptables-save -t %s", string(table))
	out, err := runner.exec.Command("iptables-save", "-t", string(table)).CombinedOutput()
	if err != nil {
		return false, fmt.Errorf("error checking rule: %v", err)
	}

	argset := util.NewStringSet(args...)

	for _, line := range strings.Split(string(out), "\n") {
		var fields = strings.Fields(line)

		// Check that this is a rule for the correct chain, and that it has
		// the correct number of argument (+2 for "-A <chain name>")
		if strings.HasPrefix(line, fmt.Sprintf("-A %s", string(chain))) && len(fields) == len(args)+2 {
			// TODO: This misses reorderings e.g. "-x foo ! -y bar" will match "! -x foo -y bar"
			if util.NewStringSet(fields...).IsSuperset(argset) {
				return true, nil
			}
			glog.V(5).Infof("DBG: fields is not a superset of args: fields=%v  args=%v", fields, args)
		}
	}

	return false, nil
}

// Executes the rule check using the "-C" flag
func (runner *runner) checkRuleUsingCheck(args []string) (bool, error) {
	out, err := runner.run(opCheckRule, args)
	if err == nil {
		return true, nil
	}
	if ee, ok := err.(utilexec.ExitError); ok {
		// iptables uses exit(1) to indicate a failure of the operation,
		// as compared to a malformed commandline, for example.
		if ee.Exited() && ee.ExitStatus() == 1 {
			return false, nil
		}
	}
	return false, fmt.Errorf("error checking rule: %v: %s", err, out)
}

type operation string

const (
	opCreateChain operation = "-N"
	opFlushChain  operation = "-F"
	opAppendRule  operation = "-A"
	opCheckRule   operation = "-C"
	opDeleteRule  operation = "-D"
)

func makeFullArgs(table Table, chain Chain, args ...string) []string {
	return append([]string{string(chain), "-t", string(table)}, args...)
}

// Checks if iptables has the "-C" flag
func getIptablesHasCheckCommand(exec utilexec.Interface) (bool, error) {
	vstring, err := getIptablesVersionString(exec)
	if err != nil {
		return false, err
	}

	v1, v2, v3, err := extractIptablesVersion(vstring)
	if err != nil {
		return false, err
	}

	return iptablesHasCheckCommand(v1, v2, v3), nil
}

// getIptablesVersion returns the first three components of the iptables version.
// e.g. "iptables v1.3.66" would return (1, 3, 66, nil)
func extractIptablesVersion(str string) (int, int, int, error) {
	versionMatcher := regexp.MustCompile("v([0-9]+)\\.([0-9]+)\\.([0-9]+)")
	result := versionMatcher.FindStringSubmatch(str)
	if result == nil {
		return 0, 0, 0, fmt.Errorf("no iptables version found in string: %s", str)
	}

	v1, err := strconv.Atoi(result[1])
	if err != nil {
		return 0, 0, 0, err
	}

	v2, err := strconv.Atoi(result[2])
	if err != nil {
		return 0, 0, 0, err
	}

	v3, err := strconv.Atoi(result[3])
	if err != nil {
		return 0, 0, 0, err
	}

	return v1, v2, v3, nil
}

// Runs "iptables --version" to get the version string
func getIptablesVersionString(exec utilexec.Interface) (string, error) {
	bytes, err := exec.Command("iptables", "--version").CombinedOutput()
	if err != nil {
		return "", err
	}

	return string(bytes), nil
}

// Checks if an iptables version is after 1.4.11, when --check was added
func iptablesHasCheckCommand(v1 int, v2 int, v3 int) bool {
	if v1 > 1 {
		return true
	}
	if v1 == 1 && v2 > 4 {
		return true
	}
	if v1 == 1 && v2 == 4 && v3 >= 11 {
		return true
	}
	return false
}
<commit_msg>strip quotes in back-compat iptables<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package iptables

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"sync"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	utilexec "github.com/GoogleCloudPlatform/kubernetes/pkg/util/exec"
	"github.com/golang/glog"
)

// An injectable interface for running iptables commands.  Implementations must be goroutine-safe.
type Interface interface {
	// EnsureChain checks if the specified chain exists and, if not, creates it.  If the chain existed, return true.
	EnsureChain(table Table, chain Chain) (bool, error)
	// FlushChain clears the specified chain.
	FlushChain(table Table, chain Chain) error
	// EnsureRule checks if the specified rule is present and, if not, creates it.  If the rule existed, return true.
	EnsureRule(table Table, chain Chain, args ...string) (bool, error)
	// DeleteRule checks if the specified rule is present and, if so, deletes it.
	DeleteRule(table Table, chain Chain, args ...string) error
	// IsIpv6 returns true if this is managing ipv6 tables
	IsIpv6() bool
}

type Protocol byte

const (
	ProtocolIpv4 Protocol = iota + 1
	ProtocolIpv6
)

type Table string

const (
	TableNAT Table = "nat"
)

type Chain string

const (
	ChainPostrouting Chain = "POSTROUTING"
	ChainPrerouting  Chain = "PREROUTING"
	ChainOutput      Chain = "OUTPUT"
)

// runner implements Interface in terms of exec("iptables").
type runner struct {
	mu       sync.Mutex
	exec     utilexec.Interface
	protocol Protocol
}

// New returns a new Interface which will exec iptables.
func New(exec utilexec.Interface, protocol Protocol) Interface {
	return &runner{exec: exec, protocol: protocol}
}

// EnsureChain is part of Interface.
func (runner *runner) EnsureChain(table Table, chain Chain) (bool, error) {
	fullArgs := makeFullArgs(table, chain)

	runner.mu.Lock()
	defer runner.mu.Unlock()

	out, err := runner.run(opCreateChain, fullArgs)
	if err != nil {
		if ee, ok := err.(utilexec.ExitError); ok {
			if ee.Exited() && ee.ExitStatus() == 1 {
				return true, nil
			}
		}
		return false, fmt.Errorf("error creating chain %q: %v: %s", chain, err, out)
	}
	return false, nil
}

// FlushChain is part of Interface.
func (runner *runner) FlushChain(table Table, chain Chain) error {
	fullArgs := makeFullArgs(table, chain)

	runner.mu.Lock()
	defer runner.mu.Unlock()

	out, err := runner.run(opFlushChain, fullArgs)
	if err != nil {
		return fmt.Errorf("error flushing chain %q: %v: %s", chain, err, out)
	}
	return nil
}

// EnsureRule is part of Interface.
func (runner *runner) EnsureRule(table Table, chain Chain, args ...string) (bool, error) {
	fullArgs := makeFullArgs(table, chain, args...)

	runner.mu.Lock()
	defer runner.mu.Unlock()

	exists, err := runner.checkRule(table, chain, args...)
	if err != nil {
		return false, err
	}
	if exists {
		return true, nil
	}
	out, err := runner.run(opAppendRule, fullArgs)
	if err != nil {
		return false, fmt.Errorf("error appending rule: %v: %s", err, out)
	}
	return false, nil
}

// DeleteRule is part of Interface.
func (runner *runner) DeleteRule(table Table, chain Chain, args ...string) error {
	fullArgs := makeFullArgs(table, chain, args...)

	runner.mu.Lock()
	defer runner.mu.Unlock()

	exists, err := runner.checkRule(table, chain, args...)
	if err != nil {
		return err
	}
	if !exists {
		return nil
	}
	out, err := runner.run(opDeleteRule, fullArgs)
	if err != nil {
		return fmt.Errorf("error deleting rule: %v: %s", err, out)
	}
	return nil
}

func (runner *runner) IsIpv6() bool {
	return runner.protocol == ProtocolIpv6
}

func (runner *runner) iptablesCommand() string {
	if runner.IsIpv6() {
		return "ip6tables"
	} else {
		return "iptables"
	}
}

func (runner *runner) run(op operation, args []string) ([]byte, error) {
	iptablesCmd := runner.iptablesCommand()

	fullArgs := append([]string{string(op)}, args...)
	glog.V(1).Infof("running iptables %s %v", string(op), args)
	return runner.exec.Command(iptablesCmd, fullArgs...).CombinedOutput()
	// Don't log err here - callers might not think it is an error.
}

// Returns (bool, nil) if it was able to check the existence of the rule, or
// (<undefined>, error) if the process of checking failed.
func (runner *runner) checkRule(table Table, chain Chain, args ...string) (bool, error) {
	checkPresent, err := getIptablesHasCheckCommand(runner.exec)
	if err != nil {
		glog.Warning("Error checking iptables version, assuming version at least 1.4.11: %v", err)
		checkPresent = true
	}
	if checkPresent {
		return runner.checkRuleUsingCheck(makeFullArgs(table, chain, args...))
	} else {
		return runner.checkRuleWithoutCheck(table, chain, args...)
	}
}

// Executes the rule check without using the "-C" flag, instead parsing iptables-save.
// Present for compatibility with <1.4.11 versions of iptables.  This is full
// of hack and half-measures.  We should nix this ASAP.
func (runner *runner) checkRuleWithoutCheck(table Table, chain Chain, args ...string) (bool, error) {
	glog.V(1).Infof("running iptables-save -t %s", string(table))
	out, err := runner.exec.Command("iptables-save", "-t", string(table)).CombinedOutput()
	if err != nil {
		return false, fmt.Errorf("error checking rule: %v", err)
	}

	// Sadly, iptables has inconsistent quoting rules for comments.
	// Just unquote any arg that is wrapped in quotes.
	argsCopy := make([]string, len(args))
	copy(argsCopy, args)
	for i := range argsCopy {
		unquote(&argsCopy[i])
	}
	argset := util.NewStringSet(argsCopy...)

	for _, line := range strings.Split(string(out), "\n") {
		var fields = strings.Fields(line)

		// Check that this is a rule for the correct chain, and that it has
		// the correct number of argument (+2 for "-A <chain name>")
		if !strings.HasPrefix(line, fmt.Sprintf("-A %s", string(chain))) || len(fields) != len(args)+2 {
			continue
		}

		// Sadly, iptables has inconsistent quoting rules for comments.
		// Just unquote any arg that is wrapped in quotes.
		for i := range fields {
			unquote(&fields[i])
		}

		// TODO: This misses reorderings e.g. "-x foo ! -y bar" will match "! -x foo -y bar"
		if util.NewStringSet(fields...).IsSuperset(argset) {
			return true, nil
		}
		glog.V(5).Infof("DBG: fields is not a superset of args: fields=%v  args=%v", fields, args)
	}

	return false, nil
}

func unquote(strp *string) {
	if len(*strp) >= 2 && (*strp)[0] == '"' && (*strp)[len(*strp)-1] == '"' {
		*strp = strings.TrimPrefix(strings.TrimSuffix(*strp, `"`), `"`)
	}
}

// Executes the rule check using the "-C" flag
func (runner *runner) checkRuleUsingCheck(args []string) (bool, error) {
	out, err := runner.run(opCheckRule, args)
	if err == nil {
		return true, nil
	}
	if ee, ok := err.(utilexec.ExitError); ok {
		// iptables uses exit(1) to indicate a failure of the operation,
		// as compared to a malformed commandline, for example.
		if ee.Exited() && ee.ExitStatus() == 1 {
			return false, nil
		}
	}
	return false, fmt.Errorf("error checking rule: %v: %s", err, out)
}

type operation string

const (
	opCreateChain operation = "-N"
	opFlushChain  operation = "-F"
	opAppendRule  operation = "-A"
	opCheckRule   operation = "-C"
	opDeleteRule  operation = "-D"
)

func makeFullArgs(table Table, chain Chain, args ...string) []string {
	return append([]string{string(chain), "-t", string(table)}, args...)
}

// Checks if iptables has the "-C" flag
func getIptablesHasCheckCommand(exec utilexec.Interface) (bool, error) {
	vstring, err := getIptablesVersionString(exec)
	if err != nil {
		return false, err
	}

	v1, v2, v3, err := extractIptablesVersion(vstring)
	if err != nil {
		return false, err
	}

	return iptablesHasCheckCommand(v1, v2, v3), nil
}

// getIptablesVersion returns the first three components of the iptables version.
// e.g. "iptables v1.3.66" would return (1, 3, 66, nil)
func extractIptablesVersion(str string) (int, int, int, error) {
	versionMatcher := regexp.MustCompile("v([0-9]+)\\.([0-9]+)\\.([0-9]+)")
	result := versionMatcher.FindStringSubmatch(str)
	if result == nil {
		return 0, 0, 0, fmt.Errorf("no iptables version found in string: %s", str)
	}

	v1, err := strconv.Atoi(result[1])
	if err != nil {
		return 0, 0, 0, err
	}

	v2, err := strconv.Atoi(result[2])
	if err != nil {
		return 0, 0, 0, err
	}

	v3, err := strconv.Atoi(result[3])
	if err != nil {
		return 0, 0, 0, err
	}

	return v1, v2, v3, nil
}

// Runs "iptables --version" to get the version string
func getIptablesVersionString(exec utilexec.Interface) (string, error) {
	bytes, err := exec.Command("iptables", "--version").CombinedOutput()
	if err != nil {
		return "", err
	}

	return string(bytes), nil
}

// Checks if an iptables version is after 1.4.11, when --check was added
func iptablesHasCheckCommand(v1 int, v2 int, v3 int) bool {
	if v1 > 1 {
		return true
	}
	if v1 == 1 && v2 > 4 {
		return true
	}
	if v1 == 1 && v2 == 4 && v3 >= 11 {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package models

import (
	"fmt"
	"time"

	"github.com/netlify/gotrue/storage"
	"github.com/pkg/errors"
	uuid "github.com/satori/go.uuid"
)

type AuditAction string
type auditLogType string

const (
	LoginAction                 AuditAction = "login"
	LogoutAction                AuditAction = "logout"
	InviteAcceptedAction        AuditAction = "invite_accepted"
	UserSignedUpAction          AuditAction = "user_signedup"
	UserInvitedAction           AuditAction = "user_invited"
	UserDeletedAction           AuditAction = "user_deleted"
	UserModifiedAction          AuditAction = "user_modified"
	UserRecoveryRequestedAction AuditAction = "user_recovery_requested"
	TokenRevokedAction          AuditAction = "token_revoked"
	TokenRefreshedAction        AuditAction = "token_refreshed"

	account auditLogType = "account"
	team    auditLogType = "team"
	token   auditLogType = "token"
	user    auditLogType = "user"
)

var actionLogTypeMap = map[AuditAction]auditLogType{
	LoginAction:                 account,
	LogoutAction:                account,
	InviteAcceptedAction:        account,
	UserSignedUpAction:          team,
	UserInvitedAction:           team,
	UserDeletedAction:           team,
	TokenRevokedAction:          token,
	TokenRefreshedAction:        token,
	UserModifiedAction:          user,
	UserRecoveryRequestedAction: user,
}

// AuditLogEntry is the database model for audit log entries.
type AuditLogEntry struct {
	InstanceID uuid.UUID `json:"-" db:"instance_id"`
	ID         uuid.UUID `json:"id" db:"id"`

	Payload JSONMap `json:"payload" db:"payload"`

	CreatedAt time.Time `json:"created_at" db:"created_at"`
}

func NewAuditLogEntry(tx *storage.Connection, instanceID uuid.UUID, actor *User, action AuditAction, traits map[string]interface{}) error {
	id, err := uuid.NewV4()
	if err != nil {
		return errors.Wrap(err, "Error generating unique id")
	}
	l := AuditLogEntry{
		InstanceID: instanceID,
		ID:         id,
		Payload: JSONMap{
			"timestamp":   time.Now().UTC().Format(time.RFC3339),
			"actor_id":    actor.ID,
			"actor_email": actor.Email,
			"action":      action,
			"log_type":    actionLogTypeMap[action],
		},
	}

	if name, ok := actor.UserMetaData["full_name"]; ok {
		l.Payload["actor_name"] = name
	}

	if traits != nil {
		l.Payload["traits"] = traits
	}

	return errors.Wrap(tx.Create(&l), "Database error creating audit log entry")
}

func FindAuditLogEntries(tx *storage.Connection, instanceID uuid.UUID, filterColumns []string, filterValue string, pageParams *Pagination) ([]*AuditLogEntry, error) {
	q := tx.Q().Where("instance_id = ?", instanceID)

	if len(filterColumns) > 0 && filterValue != "" {
		lf := "%" + filterValue + "%"
		for _, col := range filterColumns {
			q = q.Where(fmt.Sprintf("payload->>'$.%s' COLLATE utf8mb4_unicode_ci LIKE ?", col), lf)
		}
	}

	logs := []*AuditLogEntry{}
	var err error
	if pageParams != nil {
		err = q.Paginate(int(pageParams.Page), int(pageParams.PerPage)).All(&logs)
		pageParams.Count = uint64(q.Paginator.TotalEntriesSize)
	} else {
		err = q.All(&logs)
	}

	return logs, err
}
<commit_msg>Sort audit log events to show the most recent first.<commit_after>package models

import (
	"fmt"
	"time"

	"github.com/netlify/gotrue/storage"
	"github.com/pkg/errors"
	uuid "github.com/satori/go.uuid"
)

type AuditAction string
type auditLogType string

const (
	LoginAction                 AuditAction = "login"
	LogoutAction                AuditAction = "logout"
	InviteAcceptedAction        AuditAction = "invite_accepted"
	UserSignedUpAction          AuditAction = "user_signedup"
	UserInvitedAction           AuditAction = "user_invited"
	UserDeletedAction           AuditAction = "user_deleted"
	UserModifiedAction          AuditAction = "user_modified"
	UserRecoveryRequestedAction AuditAction = "user_recovery_requested"
	TokenRevokedAction          AuditAction = "token_revoked"
	TokenRefreshedAction        AuditAction = "token_refreshed"

	account auditLogType = "account"
	team    auditLogType = "team"
	token   auditLogType = "token"
	user    auditLogType = "user"
)

var actionLogTypeMap = map[AuditAction]auditLogType{
	LoginAction:                 account,
	LogoutAction:                account,
	InviteAcceptedAction:        account,
	UserSignedUpAction:          team,
	UserInvitedAction:           team,
	UserDeletedAction:           team,
	TokenRevokedAction:          token,
	TokenRefreshedAction:        token,
	UserModifiedAction:          user,
	UserRecoveryRequestedAction: user,
}

// AuditLogEntry is the database model for audit log entries.
type AuditLogEntry struct {
	InstanceID uuid.UUID `json:"-" db:"instance_id"`
	ID         uuid.UUID `json:"id" db:"id"`

	Payload JSONMap `json:"payload" db:"payload"`

	CreatedAt time.Time `json:"created_at" db:"created_at"`
}

func NewAuditLogEntry(tx *storage.Connection, instanceID uuid.UUID, actor *User, action AuditAction, traits map[string]interface{}) error {
	id, err := uuid.NewV4()
	if err != nil {
		return errors.Wrap(err, "Error generating unique id")
	}
	l := AuditLogEntry{
		InstanceID: instanceID,
		ID:         id,
		Payload: JSONMap{
			"timestamp":   time.Now().UTC().Format(time.RFC3339),
			"actor_id":    actor.ID,
			"actor_email": actor.Email,
			"action":      action,
			"log_type":    actionLogTypeMap[action],
		},
	}

	if name, ok := actor.UserMetaData["full_name"]; ok {
		l.Payload["actor_name"] = name
	}

	if traits != nil {
		l.Payload["traits"] = traits
	}

	return errors.Wrap(tx.Create(&l), "Database error creating audit log entry")
}

func FindAuditLogEntries(tx *storage.Connection, instanceID uuid.UUID, filterColumns []string, filterValue string, pageParams *Pagination) ([]*AuditLogEntry, error) {
	q := tx.Q().Order("created_at desc").Where("instance_id = ?", instanceID)

	if len(filterColumns) > 0 && filterValue != "" {
		lf := "%" + filterValue + "%"
		for _, col := range filterColumns {
			q = q.Where(fmt.Sprintf("payload->>'$.%s' COLLATE utf8mb4_unicode_ci LIKE ?", col), lf)
		}
	}

	logs := []*AuditLogEntry{}
	var err error
	if pageParams != nil {
		err = q.Paginate(int(pageParams.Page), int(pageParams.PerPage)).All(&logs)
		pageParams.Count = uint64(q.Paginator.TotalEntriesSize)
	} else {
		err = q.All(&logs)
	}

	return logs, err
}
<|endoftext|>


<commit_before>package firewall

import (
	"fmt"
	"github.com/centurylinkcloud/clc-go-cli/models"
)

type CreateReq struct {
	SourceAccountAlias string `valid:"required" URIParam:"yes"`
	DataCenter         string `valid:"required" URIParam:"yes"`
	DestinationAccount string `valid:"required"`
	Source             []string
	Destination        []string
	Ports              []string
}

type CreateRes struct {
	Links []models.LinkEntity
}

func (c *CreateReq) Validate() error {
	err := validateSource(c.Source)
	if err != nil {
		return err
	}
	err = validateDestination(c.Destination)
	if err != nil {
		return err
	}
	return nil
}

func validateSource(source []string) error {
	if len(source) == 0 {
		return fmt.Errorf("source: non-zero value required.")
	}
	return nil
}

func validateDestination(dest []string) error {
	if len(dest) == 0 {
		return fmt.Errorf("destination: non-zero value required.")
	}
	return nil
}
<commit_msg>Check that ports is not empty as well<commit_after>package firewall

import (
	"fmt"
	"github.com/centurylinkcloud/clc-go-cli/models"
)

type CreateReq struct {
	SourceAccountAlias string `valid:"required" URIParam:"yes"`
	DataCenter         string `valid:"required" URIParam:"yes"`
	DestinationAccount string `valid:"required"`
	Source             []string
	Destination        []string
	Ports              []string
}

type CreateRes struct {
	Links []models.LinkEntity
}

func (c *CreateReq) Validate() error {
	err := validateSource(c.Source)
	if err != nil {
		return err
	}
	err = validateDestination(c.Destination)
	if err != nil {
		return err
	}
	err = validatePorts(c.Ports)
	if err != nil {
		return err
	}
	return nil
}

func validateSource(source []string) error {
	if len(source) == 0 {
		return fmt.Errorf("source: non-zero value required.")
	}
	return nil
}

func validateDestination(dest []string) error {
	if len(dest) == 0 {
		return fmt.Errorf("destination: non-zero value required.")
	}
	return nil
}

func validatePorts(ports []string) error {
	if len(ports) == 0 {
		return fmt.Errorf("ports: non-zero value required.")
	}
	return nil
}
<|endoftext|>


<commit_before><commit_msg>Update GitLab api to v4.<commit_after><|endoftext|>


<commit_before>// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package migrations

import (
	"fmt"

	"xorm.io/xorm"
)

func addBranchProtectionCanPushAndEnableWhitelist(x *xorm.Engine) error {
	type ProtectedBranch struct {
		CanPush                   bool    `xorm:"NOT NULL DEFAULT false"`
		EnableApprovalsWhitelist  bool    `xorm:"NOT NULL DEFAULT false"`
		ApprovalsWhitelistUserIDs []int64 `xorm:"JSON TEXT"`
		ApprovalsWhitelistTeamIDs []int64 `xorm:"JSON TEXT"`
		RequiredApprovals         int64   `xorm:"NOT NULL DEFAULT 0"`
	}

	type User struct {
		ID   int64 `xorm:"pk autoincr"`
		Type int

		// Permissions
		IsAdmin      bool
		IsRestricted bool `xorm:"NOT NULL DEFAULT false"`
		Visibility   int  `xorm:"NOT NULL DEFAULT 0"`
	}

	type Review struct {
		ID       int64 `xorm:"pk autoincr"`
		Official bool  `xorm:"NOT NULL DEFAULT false"`

		ReviewerID int64 `xorm:"index"`
		IssueID    int64 `xorm:"index"`
	}

	if err := x.Sync2(new(ProtectedBranch)); err != nil {
		return err
	}

	if err := x.Sync2(new(Review)); err != nil {
		return err
	}

	const (
		// ReviewTypeApprove approves changes
		ReviewTypeApprove int = 1
		// ReviewTypeReject gives feedback blocking merge
		ReviewTypeReject int = 3

		// VisibleTypePublic Visible for everyone
		VisibleTypePublic int = 0
		// VisibleTypePrivate Visible only for organization's members
		VisibleTypePrivate int = 2

		// UnitTypeCode is unit type code
		UnitTypeCode int = 1

		// AccessModeNone no access
		AccessModeNone int = 0
		// AccessModeRead read access
		AccessModeRead int = 1
		// AccessModeWrite write access
		AccessModeWrite int = 2
		// AccessModeOwner owner access
		AccessModeOwner int = 4
	)

	// Repository represents a git repository.
	type Repository struct {
		ID      int64 `xorm:"pk autoincr"`
		OwnerID int64 `xorm:"UNIQUE(s) index"`

		IsPrivate bool `xorm:"INDEX"`
	}

	type PullRequest struct {
		ID int64 `xorm:"pk autoincr"`

		BaseRepoID int64 `xorm:"INDEX"`
		BaseBranch string
	}

	// RepoUnit describes all units of a repository
	type RepoUnit struct {
		ID     int64
		RepoID int64 `xorm:"INDEX(s)"`
		Type   int   `xorm:"INDEX(s)"`
	}

	type Permission struct {
		AccessMode int
		Units      []*RepoUnit
		UnitsMode  map[int]int
	}

	type TeamUser struct {
		ID     int64 `xorm:"pk autoincr"`
		TeamID int64 `xorm:"UNIQUE(s)"`
		UID    int64 `xorm:"UNIQUE(s)"`
	}

	type Collaboration struct {
		ID     int64 `xorm:"pk autoincr"`
		RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
		UserID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
		Mode   int   `xorm:"DEFAULT 2 NOT NULL"`
	}

	type Access struct {
		ID     int64 `xorm:"pk autoincr"`
		UserID int64 `xorm:"UNIQUE(s)"`
		RepoID int64 `xorm:"UNIQUE(s)"`
		Mode   int
	}

	type TeamUnit struct {
		ID     int64 `xorm:"pk autoincr"`
		OrgID  int64 `xorm:"INDEX"`
		TeamID int64 `xorm:"UNIQUE(s)"`
		Type   int   `xorm:"UNIQUE(s)"`
	}

	// Team represents a organization team.
	type Team struct {
		ID        int64 `xorm:"pk autoincr"`
		OrgID     int64 `xorm:"INDEX"`
		Authorize int
	}

	// getUserRepoPermission static function based on models.IsOfficialReviewer at 5d78792385
	getUserRepoPermission := func(sess *xorm.Session, repo *Repository, user *User) (Permission, error) {
		var perm Permission

		repoOwner := new(User)
		has, err := sess.ID(repo.OwnerID).Get(repoOwner)
		if err != nil || !has {
			return perm, err
		}

		// Prevent strangers from checking out public repo of private orginization
		// Allow user if they are collaborator of a repo within a private orginization but not a member of the orginization itself
		hasOrgVisible := true
		// Not SignedUser
		if user == nil {
			hasOrgVisible = repoOwner.Visibility == VisibleTypePublic
		} else if !user.IsAdmin {
			hasMemberWithUserID, err := sess.
				Where("uid=?", user.ID).
				And("org_id=?", repoOwner.ID).
				Table("org_user").
				Exist()
			if err != nil {
				hasOrgVisible = false
			}
			if (repoOwner.Visibility == VisibleTypePrivate || user.IsRestricted) && !hasMemberWithUserID {
				hasOrgVisible = false
			}
		}

		isCollaborator, err := sess.Get(&Collaboration{RepoID: repo.ID, UserID: user.ID})
		if err != nil {
			return perm, err
		}

		if repoOwner.Type == 1 && !hasOrgVisible && !isCollaborator {
			perm.AccessMode = AccessModeNone
			return perm, err
		}

		var units []*RepoUnit
		if err := sess.Where("repo_id = ?", repo.ID).Find(&units); err != nil {
			return perm, err
		}
		perm.Units = units

		// anonymous visit public repo
		if user == nil {
			perm.AccessMode = AccessModeRead
			return perm, err
		}

		// Admin or the owner has super access to the repository
		if user.IsAdmin || user.ID == repo.OwnerID {
			perm.AccessMode = AccessModeOwner
			return perm, err
		}

		accessLevel := func(user *User, repo *Repository) (int, error) {
			mode := AccessModeNone
			var userID int64
			restricted := false

			if user != nil {
				userID = user.ID
				restricted = user.IsRestricted
			}

			if !restricted && !repo.IsPrivate {
				mode = AccessModeRead
			}

			if userID == 0 {
				return mode, nil
			}

			if userID == repo.OwnerID {
				return AccessModeOwner, nil
			}

			a := &Access{UserID: userID, RepoID: repo.ID}
			if has, err := sess.Get(a); !has || err != nil {
				return mode, err
			}
			return a.Mode, nil
		}

		// plain user
		perm.AccessMode, err = accessLevel(user, repo)
		if err != nil {
			return perm, err
		}

		// If Owner is no Org
		if repoOwner.Type != 1 {
			return perm, err
		}

		perm.UnitsMode = make(map[int]int)

		// Collaborators on organization
		if isCollaborator {
			for _, u := range units {
				perm.UnitsMode[u.Type] = perm.AccessMode
			}
		}

		// get units mode from teams
		var teams []*Team
		err = sess.
			Join("INNER", "team_user", "team_user.team_id = team.id").
			Join("INNER", "team_repo", "team_repo.team_id = team.id").
			Where("team.org_id = ?", repo.OwnerID).
			And("team_user.uid=?", user.ID).
			And("team_repo.repo_id=?", repo.ID).
			Find(&teams)
		if err != nil {
			return perm, err
		}

		// if user in an owner team
		for _, team := range teams {
			if team.Authorize >= AccessModeOwner {
				perm.AccessMode = AccessModeOwner
				perm.UnitsMode = nil
				return perm, err
			}
		}

		for _, u := range units {
			var found bool
			for _, team := range teams {

				var teamU []*TeamUnit
				var unitEnabled bool
				err = sess.Where("team_id = ?", team.ID).Find(&teamU)

				for _, tu := range teamU {
					if tu.Type == u.Type {
						unitEnabled = true
						break
					}
				}

				if unitEnabled {
					m := perm.UnitsMode[u.Type]
					if m < team.Authorize {
						perm.UnitsMode[u.Type] = team.Authorize
					}
					found = true
				}
			}

			// for a public repo on an organization, a non-restricted user has read permission on non-team defined units.
			if !found && !repo.IsPrivate && !user.IsRestricted {
				if _, ok := perm.UnitsMode[u.Type]; !ok {
					perm.UnitsMode[u.Type] = AccessModeRead
				}
			}
		}

		// remove no permission units
		perm.Units = make([]*RepoUnit, 0, len(units))
		for t := range perm.UnitsMode {
			for _, u := range units {
				if u.Type == t {
					perm.Units = append(perm.Units, u)
				}
			}
		}

		return perm, err
	}

	// isOfficialReviewer static function based on 5d78792385
	isOfficialReviewer := func(sess *xorm.Session, issueID int64, reviewer *User) (bool, error) {
		pr := new(PullRequest)
		has, err := sess.ID(issueID).Get(pr)
		if err != nil {
			return false, err
		} else if !has {
			return false, fmt.Errorf("PullRequest for issueID %d not exist", issueID)
		}

		baseRepo := new(Repository)
		has, err = sess.ID(pr.BaseRepoID).Get(baseRepo)
		if err != nil {
			return false, err
		} else if !has {
			return false, fmt.Errorf("baseRepo with id %d not exist", pr.BaseRepoID)
		}
		protectedBranch := new(ProtectedBranch)
		has, err = sess.Where("repo_id=? AND branch_name=?", baseRepo.ID, pr.BaseBranch).Get(protectedBranch)
		if err != nil {
			return false, err
		}
		if !has {
			return false, nil
		}

		if !protectedBranch.EnableApprovalsWhitelist {

			perm, err := getUserRepoPermission(sess, baseRepo, reviewer)
			if err != nil {
				return false, err
			}
			if perm.UnitsMode == nil {
				for _, u := range perm.Units {
					if u.Type == UnitTypeCode {
						return AccessModeWrite <= perm.AccessMode, nil
					}
				}
				return false, nil
			}
			return AccessModeWrite <= perm.UnitsMode[UnitTypeCode], nil
		}
		for _, id := range protectedBranch.ApprovalsWhitelistUserIDs {
			if id == reviewer.ID {
				return true, nil
			}
		}

		// isUserInTeams
		return sess.Where("uid=?", reviewer.ID).In("team_id", protectedBranch.ApprovalsWhitelistTeamIDs).Exist(new(TeamUser))
	}

	sess := x.NewSession()
	defer sess.Close()

	if _, err := sess.Exec("UPDATE `protected_branch` SET `enable_whitelist` = ? WHERE enable_whitelist IS NULL", false); err != nil {
		return err
	}
	if _, err := sess.Exec("UPDATE `protected_branch` SET `can_push` = `enable_whitelist`"); err != nil {
		return err
	}
	if _, err := sess.Exec("UPDATE `protected_branch` SET `enable_approvals_whitelist` = ? WHERE `required_approvals` > ?", true, 0); err != nil {
		return err
	}

	var pageSize int64 = 20
	qresult, err := sess.QueryInterface("SELECT max(id) as max_id FROM issue")
	if err != nil {
		return err
	}
	var totalIssues int64
	totalIssues, ok := qresult[0]["max_id"].(int64)
	if !ok {
		// If there are no issues at all we ignore it
		return nil
	}
	totalPages := totalIssues / pageSize

	// Find latest review of each user in each pull request, and set official field if appropriate
	reviews := []*Review{}
	var page int64
	for page = 0; page <= totalPages; page++ {
		if err := sess.SQL("SELECT * FROM review WHERE id IN (SELECT max(id) as id FROM review WHERE issue_id > ? AND issue_id <= ? AND type in (?, ?) GROUP BY issue_id, reviewer_id)",
			page*pageSize, (page+1)*pageSize, ReviewTypeApprove, ReviewTypeReject).
			Find(&reviews); err != nil {
			return err
		}

		for _, review := range reviews {
			reviewer := new(User)
			has, err := sess.ID(review.ReviewerID).Get(reviewer)
			if err != nil || !has {
				// Error might occur if user doesn't exist, ignore it.
				continue
			}

			official, err := isOfficialReviewer(sess, review.IssueID, reviewer)
			if err != nil {
				// Branch might not be proteced or other error, ignore it.
				continue
			}
			review.Official = official

			if _, err := sess.ID(review.ID).Cols("official").Update(review); err != nil {
				return err
			}
		}

	}

	return sess.Commit()
}
<commit_msg>Fix bug on migration 111 (#12736)<commit_after>// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package migrations

import (
	"fmt"

	"xorm.io/xorm"
)

func addBranchProtectionCanPushAndEnableWhitelist(x *xorm.Engine) error {
	type ProtectedBranch struct {
		CanPush                   bool    `xorm:"NOT NULL DEFAULT false"`
		EnableApprovalsWhitelist  bool    `xorm:"NOT NULL DEFAULT false"`
		ApprovalsWhitelistUserIDs []int64 `xorm:"JSON TEXT"`
		ApprovalsWhitelistTeamIDs []int64 `xorm:"JSON TEXT"`
		RequiredApprovals         int64   `xorm:"NOT NULL DEFAULT 0"`
	}

	type User struct {
		ID   int64 `xorm:"pk autoincr"`
		Type int

		// Permissions
		IsAdmin      bool
		IsRestricted bool `xorm:"NOT NULL DEFAULT false"`
		Visibility   int  `xorm:"NOT NULL DEFAULT 0"`
	}

	type Review struct {
		ID       int64 `xorm:"pk autoincr"`
		Official bool  `xorm:"NOT NULL DEFAULT false"`

		ReviewerID int64 `xorm:"index"`
		IssueID    int64 `xorm:"index"`
	}

	if err := x.Sync2(new(ProtectedBranch)); err != nil {
		return err
	}

	if err := x.Sync2(new(Review)); err != nil {
		return err
	}

	const (
		// ReviewTypeApprove approves changes
		ReviewTypeApprove int = 1
		// ReviewTypeReject gives feedback blocking merge
		ReviewTypeReject int = 3

		// VisibleTypePublic Visible for everyone
		VisibleTypePublic int = 0
		// VisibleTypePrivate Visible only for organization's members
		VisibleTypePrivate int = 2

		// UnitTypeCode is unit type code
		UnitTypeCode int = 1

		// AccessModeNone no access
		AccessModeNone int = 0
		// AccessModeRead read access
		AccessModeRead int = 1
		// AccessModeWrite write access
		AccessModeWrite int = 2
		// AccessModeOwner owner access
		AccessModeOwner int = 4
	)

	// Repository represents a git repository.
	type Repository struct {
		ID      int64 `xorm:"pk autoincr"`
		OwnerID int64 `xorm:"UNIQUE(s) index"`

		IsPrivate bool `xorm:"INDEX"`
	}

	type PullRequest struct {
		ID int64 `xorm:"pk autoincr"`

		BaseRepoID int64 `xorm:"INDEX"`
		BaseBranch string
	}

	// RepoUnit describes all units of a repository
	type RepoUnit struct {
		ID     int64
		RepoID int64 `xorm:"INDEX(s)"`
		Type   int   `xorm:"INDEX(s)"`
	}

	type Permission struct {
		AccessMode int
		Units      []*RepoUnit
		UnitsMode  map[int]int
	}

	type TeamUser struct {
		ID     int64 `xorm:"pk autoincr"`
		TeamID int64 `xorm:"UNIQUE(s)"`
		UID    int64 `xorm:"UNIQUE(s)"`
	}

	type Collaboration struct {
		ID     int64 `xorm:"pk autoincr"`
		RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
		UserID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
		Mode   int   `xorm:"DEFAULT 2 NOT NULL"`
	}

	type Access struct {
		ID     int64 `xorm:"pk autoincr"`
		UserID int64 `xorm:"UNIQUE(s)"`
		RepoID int64 `xorm:"UNIQUE(s)"`
		Mode   int
	}

	type TeamUnit struct {
		ID     int64 `xorm:"pk autoincr"`
		OrgID  int64 `xorm:"INDEX"`
		TeamID int64 `xorm:"UNIQUE(s)"`
		Type   int   `xorm:"UNIQUE(s)"`
	}

	// Team represents a organization team.
	type Team struct {
		ID        int64 `xorm:"pk autoincr"`
		OrgID     int64 `xorm:"INDEX"`
		Authorize int
	}

	// getUserRepoPermission static function based on models.IsOfficialReviewer at 5d78792385
	getUserRepoPermission := func(sess *xorm.Session, repo *Repository, user *User) (Permission, error) {
		var perm Permission

		repoOwner := new(User)
		has, err := sess.ID(repo.OwnerID).Get(repoOwner)
		if err != nil || !has {
			return perm, err
		}

		// Prevent strangers from checking out public repo of private orginization
		// Allow user if they are collaborator of a repo within a private orginization but not a member of the orginization itself
		hasOrgVisible := true
		// Not SignedUser
		if user == nil {
			hasOrgVisible = repoOwner.Visibility == VisibleTypePublic
		} else if !user.IsAdmin {
			hasMemberWithUserID, err := sess.
				Where("uid=?", user.ID).
				And("org_id=?", repoOwner.ID).
				Table("org_user").
				Exist()
			if err != nil {
				hasOrgVisible = false
			}
			if (repoOwner.Visibility == VisibleTypePrivate || user.IsRestricted) && !hasMemberWithUserID {
				hasOrgVisible = false
			}
		}

		isCollaborator, err := sess.Get(&Collaboration{RepoID: repo.ID, UserID: user.ID})
		if err != nil {
			return perm, err
		}

		if repoOwner.Type == 1 && !hasOrgVisible && !isCollaborator {
			perm.AccessMode = AccessModeNone
			return perm, err
		}

		var units []*RepoUnit
		if err := sess.Where("repo_id = ?", repo.ID).Find(&units); err != nil {
			return perm, err
		}
		perm.Units = units

		// anonymous visit public repo
		if user == nil {
			perm.AccessMode = AccessModeRead
			return perm, err
		}

		// Admin or the owner has super access to the repository
		if user.IsAdmin || user.ID == repo.OwnerID {
			perm.AccessMode = AccessModeOwner
			return perm, err
		}

		accessLevel := func(user *User, repo *Repository) (int, error) {
			mode := AccessModeNone
			var userID int64
			restricted := false

			if user != nil {
				userID = user.ID
				restricted = user.IsRestricted
			}

			if !restricted && !repo.IsPrivate {
				mode = AccessModeRead
			}

			if userID == 0 {
				return mode, nil
			}

			if userID == repo.OwnerID {
				return AccessModeOwner, nil
			}

			a := &Access{UserID: userID, RepoID: repo.ID}
			if has, err := sess.Get(a); !has || err != nil {
				return mode, err
			}
			return a.Mode, nil
		}

		// plain user
		perm.AccessMode, err = accessLevel(user, repo)
		if err != nil {
			return perm, err
		}

		// If Owner is no Org
		if repoOwner.Type != 1 {
			return perm, err
		}

		perm.UnitsMode = make(map[int]int)

		// Collaborators on organization
		if isCollaborator {
			for _, u := range units {
				perm.UnitsMode[u.Type] = perm.AccessMode
			}
		}

		// get units mode from teams
		var teams []*Team
		err = sess.
			Join("INNER", "team_user", "team_user.team_id = team.id").
			Join("INNER", "team_repo", "team_repo.team_id = team.id").
			Where("team.org_id = ?", repo.OwnerID).
			And("team_user.uid=?", user.ID).
			And("team_repo.repo_id=?", repo.ID).
			Find(&teams)
		if err != nil {
			return perm, err
		}

		// if user in an owner team
		for _, team := range teams {
			if team.Authorize >= AccessModeOwner {
				perm.AccessMode = AccessModeOwner
				perm.UnitsMode = nil
				return perm, err
			}
		}

		for _, u := range units {
			var found bool
			for _, team := range teams {

				var teamU []*TeamUnit
				var unitEnabled bool
				err = sess.Where("team_id = ?", team.ID).Find(&teamU)

				for _, tu := range teamU {
					if tu.Type == u.Type {
						unitEnabled = true
						break
					}
				}

				if unitEnabled {
					m := perm.UnitsMode[u.Type]
					if m < team.Authorize {
						perm.UnitsMode[u.Type] = team.Authorize
					}
					found = true
				}
			}

			// for a public repo on an organization, a non-restricted user has read permission on non-team defined units.
			if !found && !repo.IsPrivate && !user.IsRestricted {
				if _, ok := perm.UnitsMode[u.Type]; !ok {
					perm.UnitsMode[u.Type] = AccessModeRead
				}
			}
		}

		// remove no permission units
		perm.Units = make([]*RepoUnit, 0, len(units))
		for t := range perm.UnitsMode {
			for _, u := range units {
				if u.Type == t {
					perm.Units = append(perm.Units, u)
				}
			}
		}

		return perm, err
	}

	// isOfficialReviewer static function based on 5d78792385
	isOfficialReviewer := func(sess *xorm.Session, issueID int64, reviewer *User) (bool, error) {
		pr := new(PullRequest)
		has, err := sess.ID(issueID).Get(pr)
		if err != nil {
			return false, err
		} else if !has {
			return false, fmt.Errorf("PullRequest for issueID %d not exist", issueID)
		}

		baseRepo := new(Repository)
		has, err = sess.ID(pr.BaseRepoID).Get(baseRepo)
		if err != nil {
			return false, err
		} else if !has {
			return false, fmt.Errorf("baseRepo with id %d not exist", pr.BaseRepoID)
		}
		protectedBranch := new(ProtectedBranch)
		has, err = sess.Where("repo_id=? AND branch_name=?", baseRepo.ID, pr.BaseBranch).Get(protectedBranch)
		if err != nil {
			return false, err
		}
		if !has {
			return false, nil
		}

		if !protectedBranch.EnableApprovalsWhitelist {

			perm, err := getUserRepoPermission(sess, baseRepo, reviewer)
			if err != nil {
				return false, err
			}
			if perm.UnitsMode == nil {
				for _, u := range perm.Units {
					if u.Type == UnitTypeCode {
						return AccessModeWrite <= perm.AccessMode, nil
					}
				}
				return false, nil
			}
			return AccessModeWrite <= perm.UnitsMode[UnitTypeCode], nil
		}
		for _, id := range protectedBranch.ApprovalsWhitelistUserIDs {
			if id == reviewer.ID {
				return true, nil
			}
		}

		// isUserInTeams
		return sess.Where("uid=?", reviewer.ID).In("team_id", protectedBranch.ApprovalsWhitelistTeamIDs).Exist(new(TeamUser))
	}

	if _, err := x.Exec("UPDATE `protected_branch` SET `enable_whitelist` = ? WHERE enable_whitelist IS NULL", false); err != nil {
		return err
	}
	if _, err := x.Exec("UPDATE `protected_branch` SET `can_push` = `enable_whitelist`"); err != nil {
		return err
	}
	if _, err := x.Exec("UPDATE `protected_branch` SET `enable_approvals_whitelist` = ? WHERE `required_approvals` > ?", true, 0); err != nil {
		return err
	}

	var pageSize int64 = 20
	qresult, err := x.QueryInterface("SELECT max(id) as max_id FROM issue")
	if err != nil {
		return err
	}
	var totalIssues int64
	totalIssues, ok := qresult[0]["max_id"].(int64)
	if !ok {
		// If there are no issues at all we ignore it
		return nil
	}
	totalPages := totalIssues / pageSize

	// Find latest review of each user in each pull request, and set official field if appropriate
	reviews := []*Review{}

	var executeBody = func(page, pageSize int64) error {
		if err := x.SQL("SELECT * FROM review WHERE id IN (SELECT max(id) as id FROM review WHERE issue_id > ? AND issue_id <= ? AND type in (?, ?) GROUP BY issue_id, reviewer_id)",
			page*pageSize, (page+1)*pageSize, ReviewTypeApprove, ReviewTypeReject).
			Find(&reviews); err != nil {
			return err
		}

		if len(reviews) == 0 {
			return nil
		}

		sess := x.NewSession()
		defer sess.Close()

		if err := sess.Begin(); err != nil {
			return err
		}

		var updated int
		for _, review := range reviews {
			reviewer := new(User)
			has, err := sess.ID(review.ReviewerID).Get(reviewer)
			if err != nil || !has {
				// Error might occur if user doesn't exist, ignore it.
				continue
			}

			official, err := isOfficialReviewer(sess, review.IssueID, reviewer)
			if err != nil {
				// Branch might not be proteced or other error, ignore it.
				continue
			}
			review.Official = official
			updated++
			if _, err := sess.ID(review.ID).Cols("official").Update(review); err != nil {
				return err
			}
		}

		if updated > 0 {
			return sess.Commit()
		}
		return nil
	}

	var page int64
	for page = 0; page <= totalPages; page++ {
		if err := executeBody(page, pageSize); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package migrations

import (
	"fmt"
	"path/filepath"
	"strings"

	"code.gitea.io/gitea/modules/git"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/setting"

	"xorm.io/xorm"
)

// Copy paste from models/repo.go because we cannot import models package
func repoPath(userName, repoName string) string {
	return filepath.Join(userPath(userName), strings.ToLower(repoName)+".git")
}

func userPath(userName string) string {
	return filepath.Join(setting.RepoRootPath, strings.ToLower(userName))
}

func fixPublisherIDforTagReleases(x *xorm.Engine) error {
	type Release struct {
		ID          int64
		RepoID      int64
		Sha1        string
		TagName     string
		PublisherID int64
	}

	type Repository struct {
		ID        int64
		OwnerID   int64
		OwnerName string
		Name      string
	}

	type User struct {
		ID    int64
		Name  string
		Email string
	}

	const batchSize = 100
	sess := x.NewSession()
	defer sess.Close()

	var (
		repo    *Repository
		gitRepo *git.Repository
		user    *User
	)
	defer func() {
		if gitRepo != nil {
			gitRepo.Close()
		}
	}()
	for start := 0; ; start += batchSize {
		releases := make([]*Release, 0, batchSize)

		if err := sess.Begin(); err != nil {
			return err
		}

		if err := sess.Limit(batchSize, start).
			Where("publisher_id = 0 OR publisher_id is null").
			Asc("repo_id", "id").Where("is_tag=?", true).
			Find(&releases); err != nil {
			return err
		}

		if len(releases) == 0 {
			break
		}

		for _, release := range releases {
			if repo == nil || repo.ID != release.RepoID {
				if gitRepo != nil {
					gitRepo.Close()
					gitRepo = nil
				}
				repo = new(Repository)
				has, err := sess.ID(release.RepoID).Get(repo)
				if err != nil {
					log.Error("Error whilst loading repository[%d] for release[%d] with tag name %s", release.RepoID, release.ID, release.TagName)
					return err
				} else if !has {
					log.Warn("Release[%d] is orphaned and refers to non-existing repository %d", release.ID, release.RepoID)
					log.Warn("This release should be deleted")
					continue
				}

				if repo.OwnerName == "" {
					// v120.go migration may not have been run correctly - we'll just replicate it here
					// because this appears to be a common-ish problem.
					if _, err := sess.Exec("UPDATE repository SET owner_name = (SELECT name FROM `user` WHERE `user`.id = repository.owner_id)"); err != nil {
						log.Error("Error whilst updating repository[%d] owner name", repo.ID)
						return err
					}

					if _, err := sess.ID(release.RepoID).Get(repo); err != nil {
						log.Error("Error whilst loading repository[%d] for release[%d] with tag name %s", release.RepoID, release.ID, release.TagName)
						return err
					}
				}
				gitRepo, err = git.OpenRepository(repoPath(repo.OwnerName, repo.Name))
				if err != nil {
					log.Error("Error whilst opening git repo for %-v", repo)
					return err
				}
			}

			commit, err := gitRepo.GetTagCommit(release.TagName)
			if err != nil {
				if git.IsErrNotExist(err) {
					log.Warn("Unable to find commit %s for Tag: %s in %-v. Cannot update publisher ID.", err.(*git.ErrNotExist).ID, release.TagName, repo)
					continue
				}
				log.Error("Error whilst getting commit for Tag: %s in %-v.", release.TagName, repo)
				return fmt.Errorf("GetTagCommit: %v", err)
			}

			if user == nil || !strings.EqualFold(user.Email, commit.Author.Email) {
				user = new(User)
				_, err = sess.Where("email=?", commit.Author.Email).Get(user)
				if err != nil {
					log.Error("Error whilst getting commit author by email: %s for Tag: %s in %-v.", commit.Author.Email, release.TagName, repo)
					return err
				}

				user.Email = commit.Author.Email
			}

			if user.ID <= 0 {
				continue
			}

			release.PublisherID = user.ID
			if _, err := sess.ID(release.ID).Cols("publisher_id").Update(release); err != nil {
				log.Error("Error whilst updating publisher[%d] for release[%d] with tag name %s", release.PublisherID, release.ID, release.TagName)
				return err
			}
		}
		if gitRepo != nil {
			gitRepo.Close()
		}

		if err := sess.Commit(); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Resolve panic on failed interface conversion in migration v156 (#15604)<commit_after>// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package migrations

import (
	"fmt"
	"path/filepath"
	"strings"

	"code.gitea.io/gitea/modules/git"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/setting"

	"xorm.io/xorm"
)

// Copy paste from models/repo.go because we cannot import models package
func repoPath(userName, repoName string) string {
	return filepath.Join(userPath(userName), strings.ToLower(repoName)+".git")
}

func userPath(userName string) string {
	return filepath.Join(setting.RepoRootPath, strings.ToLower(userName))
}

func fixPublisherIDforTagReleases(x *xorm.Engine) error {
	type Release struct {
		ID          int64
		RepoID      int64
		Sha1        string
		TagName     string
		PublisherID int64
	}

	type Repository struct {
		ID        int64
		OwnerID   int64
		OwnerName string
		Name      string
	}

	type User struct {
		ID    int64
		Name  string
		Email string
	}

	const batchSize = 100
	sess := x.NewSession()
	defer sess.Close()

	var (
		repo    *Repository
		gitRepo *git.Repository
		user    *User
	)
	defer func() {
		if gitRepo != nil {
			gitRepo.Close()
		}
	}()
	for start := 0; ; start += batchSize {
		releases := make([]*Release, 0, batchSize)

		if err := sess.Begin(); err != nil {
			return err
		}

		if err := sess.Limit(batchSize, start).
			Where("publisher_id = 0 OR publisher_id is null").
			Asc("repo_id", "id").Where("is_tag=?", true).
			Find(&releases); err != nil {
			return err
		}

		if len(releases) == 0 {
			break
		}

		for _, release := range releases {
			if repo == nil || repo.ID != release.RepoID {
				if gitRepo != nil {
					gitRepo.Close()
					gitRepo = nil
				}
				repo = new(Repository)
				has, err := sess.ID(release.RepoID).Get(repo)
				if err != nil {
					log.Error("Error whilst loading repository[%d] for release[%d] with tag name %s", release.RepoID, release.ID, release.TagName)
					return err
				} else if !has {
					log.Warn("Release[%d] is orphaned and refers to non-existing repository %d", release.ID, release.RepoID)
					log.Warn("This release should be deleted")
					continue
				}

				if repo.OwnerName == "" {
					// v120.go migration may not have been run correctly - we'll just replicate it here
					// because this appears to be a common-ish problem.
					if _, err := sess.Exec("UPDATE repository SET owner_name = (SELECT name FROM `user` WHERE `user`.id = repository.owner_id)"); err != nil {
						log.Error("Error whilst updating repository[%d] owner name", repo.ID)
						return err
					}

					if _, err := sess.ID(release.RepoID).Get(repo); err != nil {
						log.Error("Error whilst loading repository[%d] for release[%d] with tag name %s", release.RepoID, release.ID, release.TagName)
						return err
					}
				}
				gitRepo, err = git.OpenRepository(repoPath(repo.OwnerName, repo.Name))
				if err != nil {
					log.Error("Error whilst opening git repo for %-v", repo)
					return err
				}
			}

			commit, err := gitRepo.GetTagCommit(release.TagName)
			if err != nil {
				if git.IsErrNotExist(err) {
					log.Warn("Unable to find commit %s for Tag: %s in %-v. Cannot update publisher ID.", err.(git.ErrNotExist).ID, release.TagName, repo)
					continue
				}
				log.Error("Error whilst getting commit for Tag: %s in %-v.", release.TagName, repo)
				return fmt.Errorf("GetTagCommit: %v", err)
			}

			if user == nil || !strings.EqualFold(user.Email, commit.Author.Email) {
				user = new(User)
				_, err = sess.Where("email=?", commit.Author.Email).Get(user)
				if err != nil {
					log.Error("Error whilst getting commit author by email: %s for Tag: %s in %-v.", commit.Author.Email, release.TagName, repo)
					return err
				}

				user.Email = commit.Author.Email
			}

			if user.ID <= 0 {
				continue
			}

			release.PublisherID = user.ID
			if _, err := sess.ID(release.ID).Cols("publisher_id").Update(release); err != nil {
				log.Error("Error whilst updating publisher[%d] for release[%d] with tag name %s", release.PublisherID, release.ID, release.TagName)
				return err
			}
		}
		if gitRepo != nil {
			gitRepo.Close()
		}

		if err := sess.Commit(); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vfs

import (
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/golang/glog"
	"os"
	"sync"
	"time"
)

type S3Context struct {
	mutex           sync.Mutex
	clients         map[string]*s3.S3
	bucketLocations map[string]string
}

func NewS3Context() *S3Context {
	return &S3Context{
		clients:         make(map[string]*s3.S3),
		bucketLocations: make(map[string]string),
	}
}

func (s *S3Context) getClient(region string) (*s3.S3, error) {
	s.mutex.Lock()
	defer s.mutex.Unlock()

	s3Client := s.clients[region]
	if s3Client == nil {
		config := aws.NewConfig().WithRegion(region)
		config = config.WithCredentialsChainVerboseErrors(true)

		session := session.New()
		s3Client = s3.New(session, config)
	}

	s.clients[region] = s3Client

	return s3Client, nil
}

func (s *S3Context) getRegionForBucket(bucket string) (string, error) {
	region := func() string {
		s.mutex.Lock()
		defer s.mutex.Unlock()
		return s.bucketLocations[bucket]
	}()

	if region != "" {
		return region, nil
	}

	// Probe to find correct region for bucket
	awsRegion := os.Getenv("AWS_REGION")
	if awsRegion == "" {
		awsRegion = "us-east-1"
	}

	response, err := bruteforceBucketLocation(&awsRegion, &bucket)

	if err != nil {
		return "", err
	}

	if response.LocationConstraint == nil {
		// US Classic does not return a region
		region = "us-east-1"
	} else {
		region = *response.LocationConstraint
		// Another special case: "EU" can mean eu-west-1
		if region == "EU" {
			region = "eu-west-1"
		}
	}
	glog.V(2).Infof("Found bucket %q in region %q", bucket, region)

	s.mutex.Lock()
	defer s.mutex.Unlock()
	s.bucketLocations[bucket] = region

	return region, nil
}

/*
Amazon's S3 API provides the GetBucketLocation call to determine the region in which a bucket is located.
This call can however only be used globally by the owner of the bucket, as mentioned on the documentation page.

For S3 buckets that are shared across multiple AWS accounts using bucket policies the call will only work if it is sent
to the correct region in the first place.

This method will attempt to "bruteforce" the bucket location by sending a request to every available region and picking
out the first result.

See also: https://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/GetBucketLocationRequest
*/
func bruteforceBucketLocation(region *string, bucket *string) (*s3.GetBucketLocationOutput, error) {
	session, _ := session.NewSession(&aws.Config{Region: region})
	regions, err := ec2.New(session).DescribeRegions(nil)

	if err != nil {
		return nil, fmt.Errorf("Unable to list AWS regions: %v", err)
	}

	request := &s3.GetBucketLocationInput{
		Bucket: bucket,
	}

	glog.V(2).Infof("Querying S3 for bucket location for %s", *bucket)

	out := make(chan *s3.GetBucketLocationOutput)
	for _, region := range regions.Regions {
		go func(regionName string) {
			s3Client := s3.New(session, &aws.Config{Region: aws.String(regionName)})
			result, bucketError := s3Client.GetBucketLocation(request)

			if bucketError == nil {
				out <- result
			}
		}(*region.RegionName)
	}

	select {
	case bucketLocation := <-out:
		return bucketLocation, nil
	case <-time.After(5 * time.Second):
		return nil, fmt.Errorf("Could not retrieve location for AWS bucket %s", *bucket)
	}
}
<commit_msg>fix s3context: Attempt a normal S3 call before bruteforcing location<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vfs

import (
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/golang/glog"
	"os"
	"sync"
	"time"
)

type S3Context struct {
	mutex           sync.Mutex
	clients         map[string]*s3.S3
	bucketLocations map[string]string
}

func NewS3Context() *S3Context {
	return &S3Context{
		clients:         make(map[string]*s3.S3),
		bucketLocations: make(map[string]string),
	}
}

func (s *S3Context) getClient(region string) (*s3.S3, error) {
	s.mutex.Lock()
	defer s.mutex.Unlock()

	s3Client := s.clients[region]
	if s3Client == nil {
		config := aws.NewConfig().WithRegion(region)
		config = config.WithCredentialsChainVerboseErrors(true)

		session := session.New()
		s3Client = s3.New(session, config)
	}

	s.clients[region] = s3Client

	return s3Client, nil
}

func (s *S3Context) getRegionForBucket(bucket string) (string, error) {
	region := func() string {
		s.mutex.Lock()
		defer s.mutex.Unlock()
		return s.bucketLocations[bucket]
	}()

	if region != "" {
		return region, nil
	}

	// Probe to find correct region for bucket
	awsRegion := os.Getenv("AWS_REGION")
	if awsRegion == "" {
		awsRegion = "us-east-1"
	}

	request := &s3.GetBucketLocationInput{
		Bucket: &bucket,
	}
	var response *s3.GetBucketLocationOutput

	s3Client, err := s.getClient(awsRegion)

	// Attempt one GetBucketLocation call the "normal" way (i.e. as the bucket owner)
	response, err = s3Client.GetBucketLocation(request)

	// and fallback to brute-forcing if it fails
	if err != nil {
		response, err = bruteforceBucketLocation(&awsRegion, request)
	}

	if err != nil {
		return "", err
	}

	if response.LocationConstraint == nil {
		// US Classic does not return a region
		region = "us-east-1"
	} else {
		region = *response.LocationConstraint
		// Another special case: "EU" can mean eu-west-1
		if region == "EU" {
			region = "eu-west-1"
		}
	}
	glog.V(2).Infof("Found bucket %q in region %q", bucket, region)

	s.mutex.Lock()
	defer s.mutex.Unlock()
	s.bucketLocations[bucket] = region

	return region, nil
}

/*
Amazon's S3 API provides the GetBucketLocation call to determine the region in which a bucket is located.
This call can however only be used globally by the owner of the bucket, as mentioned on the documentation page.

For S3 buckets that are shared across multiple AWS accounts using bucket policies the call will only work if it is sent
to the correct region in the first place.

This method will attempt to "bruteforce" the bucket location by sending a request to every available region and picking
out the first result.

See also: https://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/GetBucketLocationRequest
*/
func bruteforceBucketLocation(region *string, request *s3.GetBucketLocationInput) (*s3.GetBucketLocationOutput, error) {
	session, _ := session.NewSession(&aws.Config{Region: region})
	regions, err := ec2.New(session).DescribeRegions(nil)

	if err != nil {
		return nil, fmt.Errorf("Unable to list AWS regions: %v", err)
	}

	glog.V(2).Infof("Querying S3 for bucket location for %s", *request.Bucket)

	out := make(chan *s3.GetBucketLocationOutput)
	for _, region := range regions.Regions {
		go func(regionName string) {
			s3Client := s3.New(session, &aws.Config{Region: aws.String(regionName)})
			result, bucketError := s3Client.GetBucketLocation(request)

			if bucketError == nil {
				out <- result
			}
		}(*region.RegionName)
	}

	select {
	case bucketLocation := <-out:
		return bucketLocation, nil
	case <-time.After(5 * time.Second):
		return nil, fmt.Errorf("Could not retrieve location for AWS bucket %s", *request.Bucket)
	}
}
<|endoftext|>