<commit_before>package shared

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"time"

	"github.com/gorilla/websocket"
)

func RFC3493Dialer(network, address string) (net.Conn, error) {
	host, port, err := net.SplitHostPort(address)
	if err != nil {
		return nil, err
	}

	addrs, err := net.LookupHost(host)
	if err != nil {
		return nil, err
	}
	for _, a := range addrs {
		c, err := net.DialTimeout(network, net.JoinHostPort(a, port), 10*time.Second)
		if err != nil {
			continue
		}
		return c, err
	}
	return nil, fmt.Errorf("Unable to connect to: " + address)
}

func initTLSConfig() *tls.Config {
	return &tls.Config{
		MinVersion: tls.VersionTLS12,
		MaxVersion: tls.VersionTLS12,
		CipherSuites: []uint16{
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA},
		PreferServerCipherSuites: true,
	}
}

func finalizeTLSConfig(tlsConfig *tls.Config, tlsRemoteCert *x509.Certificate) {
	// Trusted certificates
	if tlsRemoteCert != nil {
		caCertPool := tlsConfig.RootCAs
		if caCertPool == nil {
			caCertPool = x509.NewCertPool()
		}

		// Make it a valid RootCA
		tlsRemoteCert.IsCA = true
		tlsRemoteCert.KeyUsage = x509.KeyUsageCertSign

		// Setup the pool
		caCertPool.AddCert(tlsRemoteCert)
		tlsConfig.RootCAs = caCertPool

		// Set the ServerName
		if tlsRemoteCert.DNSNames != nil {
			tlsConfig.ServerName = tlsRemoteCert.DNSNames[0]
		}
	}

	tlsConfig.BuildNameToCertificate()
}

func GetTLSConfig(tlsClientCertFile string, tlsClientKeyFile string, tlsClientCAFile string, tlsRemoteCert *x509.Certificate) (*tls.Config, error) {
	tlsConfig := initTLSConfig()

	// Client authentication
	if tlsClientCertFile != "" && tlsClientKeyFile != "" {
		cert, err := tls.LoadX509KeyPair(tlsClientCertFile, tlsClientKeyFile)
		if err != nil {
			return nil, err
		}

		tlsConfig.Certificates = []tls.Certificate{cert}
	}

	if tlsClientCAFile != "" {
		caCertificates, err := ioutil.ReadFile(tlsClientCAFile)
		if err != nil {
			return nil, err
		}

		caPool := x509.NewCertPool()
		caPool.AppendCertsFromPEM(caCertificates)

		tlsConfig.RootCAs = caPool
	}

	finalizeTLSConfig(tlsConfig, tlsRemoteCert)
	return tlsConfig, nil
}

func GetTLSConfigMem(tlsClientCert string, tlsClientKey string, tlsClientCA string, tlsRemoteCertPEM string) (*tls.Config, error) {
	tlsConfig := initTLSConfig()

	// Client authentication
	if tlsClientCert != "" && tlsClientKey != "" {
		cert, err := tls.X509KeyPair([]byte(tlsClientCert), []byte(tlsClientKey))
		if err != nil {
			return nil, err
		}

		tlsConfig.Certificates = []tls.Certificate{cert}
	}

	var tlsRemoteCert *x509.Certificate
	if tlsRemoteCertPEM != "" {
		// Ignore any content outside of the PEM bytes we care about
		certBlock, _ := pem.Decode([]byte(tlsRemoteCertPEM))
		if certBlock == nil {
			return nil, fmt.Errorf("Invalid remote certificate")
		}

		var err error
		tlsRemoteCert, err = x509.ParseCertificate(certBlock.Bytes)
		if err != nil {
			return nil, err
		}
	}

	if tlsClientCA != "" {
		caPool := x509.NewCertPool()
		caPool.AppendCertsFromPEM([]byte(tlsClientCA))

		tlsConfig.RootCAs = caPool
	}

	finalizeTLSConfig(tlsConfig, tlsRemoteCert)

	return tlsConfig, nil
}

func IsLoopback(iface *net.Interface) bool {
	return int(iface.Flags&net.FlagLoopback) > 0
}

func WebsocketSendStream(conn *websocket.Conn, r io.Reader, bufferSize int) chan bool {
	ch := make(chan bool)

	if r == nil {
		close(ch)
		return ch
	}

	go func(conn *websocket.Conn, r io.Reader) {
		in := ReaderToChannel(r, bufferSize)
		for {
			buf, ok := <-in
			if !ok {
				break
			}

			w, err := conn.NextWriter(websocket.BinaryMessage)
			if err != nil {
				Debugf("Got error getting next writer %s", err)
				break
			}

			_, err = w.Write(buf)
			w.Close()
			if err != nil {
				Debugf("Got err writing %s", err)
				break
			}
		}
		conn.WriteMessage(websocket.TextMessage, []byte{})
		ch <- true
	}(conn, r)

	return ch
}

func WebsocketRecvStream(w io.Writer, conn *websocket.Conn) chan bool {
	ch := make(chan bool)

	go func(w io.Writer, conn *websocket.Conn) {
		for {
			mt, r, err := conn.NextReader()
			if mt == websocket.CloseMessage {
				Debugf("Got close message for reader")
				break
			}

			if mt == websocket.TextMessage {
				Debugf("got message barrier")
				break
			}

			if err != nil {
				Debugf("Got error getting next reader %s, %s", err, w)
				break
			}

			buf, err := ioutil.ReadAll(r)
			if err != nil {
				Debugf("Got error writing to writer %s", err)
				break
			}

			if w == nil {
				continue
			}

			i, err := w.Write(buf)
			if i != len(buf) {
				Debugf("Didn't write all of buf")
				break
			}
			if err != nil {
				Debugf("Error writing buf %s", err)
				break
			}
		}
		ch <- true
	}(w, conn)

	return ch
}

// WebsocketMirror allows mirroring a reader to a websocket and taking the
// result and writing it to a writer. This function allows for multiple
// mirrorings and correctly negotiates stream endings. However, it means any
// websocket.Conns passed to it are live when it returns, and must be closed
// explicitly.
func WebsocketMirror(conn *websocket.Conn, w io.WriteCloser, r io.ReadCloser) (chan bool, chan bool) {
	readDone := make(chan bool, 1)
	writeDone := make(chan bool, 1)
	go func(conn *websocket.Conn, w io.WriteCloser) {
		for {
			mt, r, err := conn.NextReader()
			if err != nil {
				Debugf("Got error getting next reader %s, %s", err, w)
				break
			}

			if mt == websocket.CloseMessage {
				Debugf("Got close message for reader")
				break
			}

			if mt == websocket.TextMessage {
				Debugf("Got message barrier, resetting stream")
				break
			}

			buf, err := ioutil.ReadAll(r)
			if err != nil {
				Debugf("Got error writing to writer %s", err)
				break
			}
			i, err := w.Write(buf)
			if i != len(buf) {
				Debugf("Didn't write all of buf")
				break
			}
			if err != nil {
				Debugf("Error writing buf %s", err)
				break
			}
		}
		writeDone <- true
		w.Close()
	}(conn, w)

	go func(conn *websocket.Conn, r io.ReadCloser) {
		/* For now, we don't need to adjust buffer sizes in
		 * WebsocketMirror, since it's used for interactive things like
		 * exec.
		 */
		in := ReaderToChannel(r, -1)
		for {
			buf, ok := <-in
			if !ok {
				r.Close()
				Debugf("sending write barrier")
				conn.WriteMessage(websocket.TextMessage, []byte{})
				readDone <- true
				return
			}
			w, err := conn.NextWriter(websocket.BinaryMessage)
			if err != nil {
				Debugf("Got error getting next writer %s", err)
				break
			}

			_, err = w.Write(buf)
			w.Close()
			if err != nil {
				Debugf("Got err writing %s", err)
				break
			}
		}
		closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")
		conn.WriteMessage(websocket.CloseMessage, closeMsg)
		readDone <- true
		r.Close()
	}(conn, r)

	return readDone, writeDone
}
<commit_msg>shared/network: Debugf() --> LogDebugf()<commit_after>package shared

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"time"

	"github.com/gorilla/websocket"
)

func RFC3493Dialer(network, address string) (net.Conn, error) {
	host, port, err := net.SplitHostPort(address)
	if err != nil {
		return nil, err
	}

	addrs, err := net.LookupHost(host)
	if err != nil {
		return nil, err
	}
	for _, a := range addrs {
		c, err := net.DialTimeout(network, net.JoinHostPort(a, port), 10*time.Second)
		if err != nil {
			continue
		}
		return c, err
	}
	return nil, fmt.Errorf("Unable to connect to: " + address)
}

func initTLSConfig() *tls.Config {
	return &tls.Config{
		MinVersion: tls.VersionTLS12,
		MaxVersion: tls.VersionTLS12,
		CipherSuites: []uint16{
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA},
		PreferServerCipherSuites: true,
	}
}

func finalizeTLSConfig(tlsConfig *tls.Config, tlsRemoteCert *x509.Certificate) {
	// Trusted certificates
	if tlsRemoteCert != nil {
		caCertPool := tlsConfig.RootCAs
		if caCertPool == nil {
			caCertPool = x509.NewCertPool()
		}

		// Make it a valid RootCA
		tlsRemoteCert.IsCA = true
		tlsRemoteCert.KeyUsage = x509.KeyUsageCertSign

		// Setup the pool
		caCertPool.AddCert(tlsRemoteCert)
		tlsConfig.RootCAs = caCertPool

		// Set the ServerName
		if tlsRemoteCert.DNSNames != nil {
			tlsConfig.ServerName = tlsRemoteCert.DNSNames[0]
		}
	}

	tlsConfig.BuildNameToCertificate()
}

func GetTLSConfig(tlsClientCertFile string, tlsClientKeyFile string, tlsClientCAFile string, tlsRemoteCert *x509.Certificate) (*tls.Config, error) {
	tlsConfig := initTLSConfig()

	// Client authentication
	if tlsClientCertFile != "" && tlsClientKeyFile != "" {
		cert, err := tls.LoadX509KeyPair(tlsClientCertFile, tlsClientKeyFile)
		if err != nil {
			return nil, err
		}

		tlsConfig.Certificates = []tls.Certificate{cert}
	}

	if tlsClientCAFile != "" {
		caCertificates, err := ioutil.ReadFile(tlsClientCAFile)
		if err != nil {
			return nil, err
		}

		caPool := x509.NewCertPool()
		caPool.AppendCertsFromPEM(caCertificates)

		tlsConfig.RootCAs = caPool
	}

	finalizeTLSConfig(tlsConfig, tlsRemoteCert)
	return tlsConfig, nil
}

func GetTLSConfigMem(tlsClientCert string, tlsClientKey string, tlsClientCA string, tlsRemoteCertPEM string) (*tls.Config, error) {
	tlsConfig := initTLSConfig()

	// Client authentication
	if tlsClientCert != "" && tlsClientKey != "" {
		cert, err := tls.X509KeyPair([]byte(tlsClientCert), []byte(tlsClientKey))
		if err != nil {
			return nil, err
		}

		tlsConfig.Certificates = []tls.Certificate{cert}
	}

	var tlsRemoteCert *x509.Certificate
	if tlsRemoteCertPEM != "" {
		// Ignore any content outside of the PEM bytes we care about
		certBlock, _ := pem.Decode([]byte(tlsRemoteCertPEM))
		if certBlock == nil {
			return nil, fmt.Errorf("Invalid remote certificate")
		}

		var err error
		tlsRemoteCert, err = x509.ParseCertificate(certBlock.Bytes)
		if err != nil {
			return nil, err
		}
	}

	if tlsClientCA != "" {
		caPool := x509.NewCertPool()
		caPool.AppendCertsFromPEM([]byte(tlsClientCA))

		tlsConfig.RootCAs = caPool
	}

	finalizeTLSConfig(tlsConfig, tlsRemoteCert)

	return tlsConfig, nil
}

func IsLoopback(iface *net.Interface) bool {
	return int(iface.Flags&net.FlagLoopback) > 0
}

func WebsocketSendStream(conn *websocket.Conn, r io.Reader, bufferSize int) chan bool {
	ch := make(chan bool)

	if r == nil {
		close(ch)
		return ch
	}

	go func(conn *websocket.Conn, r io.Reader) {
		in := ReaderToChannel(r, bufferSize)
		for {
			buf, ok := <-in
			if !ok {
				break
			}

			w, err := conn.NextWriter(websocket.BinaryMessage)
			if err != nil {
				LogDebugf("Got error getting next writer %s", err)
				break
			}

			_, err = w.Write(buf)
			w.Close()
			if err != nil {
				LogDebugf("Got err writing %s", err)
				break
			}
		}
		conn.WriteMessage(websocket.TextMessage, []byte{})
		ch <- true
	}(conn, r)

	return ch
}

func WebsocketRecvStream(w io.Writer, conn *websocket.Conn) chan bool {
	ch := make(chan bool)

	go func(w io.Writer, conn *websocket.Conn) {
		for {
			mt, r, err := conn.NextReader()
			if mt == websocket.CloseMessage {
				LogDebugf("Got close message for reader")
				break
			}

			if mt == websocket.TextMessage {
				LogDebugf("got message barrier")
				break
			}

			if err != nil {
				LogDebugf("Got error getting next reader %s, %s", err, w)
				break
			}

			buf, err := ioutil.ReadAll(r)
			if err != nil {
				LogDebugf("Got error writing to writer %s", err)
				break
			}

			if w == nil {
				continue
			}

			i, err := w.Write(buf)
			if i != len(buf) {
				LogDebugf("Didn't write all of buf")
				break
			}
			if err != nil {
				LogDebugf("Error writing buf %s", err)
				break
			}
		}
		ch <- true
	}(w, conn)

	return ch
}

// WebsocketMirror allows mirroring a reader to a websocket and taking the
// result and writing it to a writer. This function allows for multiple
// mirrorings and correctly negotiates stream endings. However, it means any
// websocket.Conns passed to it are live when it returns, and must be closed
// explicitly.
func WebsocketMirror(conn *websocket.Conn, w io.WriteCloser, r io.ReadCloser) (chan bool, chan bool) {
	readDone := make(chan bool, 1)
	writeDone := make(chan bool, 1)
	go func(conn *websocket.Conn, w io.WriteCloser) {
		for {
			mt, r, err := conn.NextReader()
			if err != nil {
				LogDebugf("Got error getting next reader %s, %s", err, w)
				break
			}

			if mt == websocket.CloseMessage {
				LogDebugf("Got close message for reader")
				break
			}

			if mt == websocket.TextMessage {
				LogDebugf("Got message barrier, resetting stream")
				break
			}

			buf, err := ioutil.ReadAll(r)
			if err != nil {
				LogDebugf("Got error writing to writer %s", err)
				break
			}
			i, err := w.Write(buf)
			if i != len(buf) {
				LogDebugf("Didn't write all of buf")
				break
			}
			if err != nil {
				LogDebugf("Error writing buf %s", err)
				break
			}
		}
		writeDone <- true
		w.Close()
	}(conn, w)

	go func(conn *websocket.Conn, r io.ReadCloser) {
		/* For now, we don't need to adjust buffer sizes in
		 * WebsocketMirror, since it's used for interactive things like
		 * exec.
		 */
		in := ReaderToChannel(r, -1)
		for {
			buf, ok := <-in
			if !ok {
				r.Close()
				LogDebugf("sending write barrier")
				conn.WriteMessage(websocket.TextMessage, []byte{})
				readDone <- true
				return
			}
			w, err := conn.NextWriter(websocket.BinaryMessage)
			if err != nil {
				LogDebugf("Got error getting next writer %s", err)
				break
			}

			_, err = w.Write(buf)
			w.Close()
			if err != nil {
				LogDebugf("Got err writing %s", err)
				break
			}
		}
		closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")
		conn.WriteMessage(websocket.CloseMessage, closeMsg)
		readDone <- true
		r.Close()
	}(conn, r)

	return readDone, writeDone
}
<|endoftext|>


<commit_before>package delta_test

import (
	"strings"
	"testing"
	"time"

	"github.com/GeoNet/delta/meta"
)

var testFeatures = map[string]func([]meta.Feature) func(t *testing.T){
	"check for duplicated site features": func(features []meta.Feature) func(t *testing.T) {
		return func(t *testing.T) {

			for i := 0; i < len(features); i++ {
				for j := i + 1; j < len(features); j++ {
					if features[i].Station != features[j].Station {
						continue
					}
					if features[i].Location != features[j].Location {
						continue
					}
					if features[i].SubLocation != features[j].SubLocation {
						continue
					}
					if features[i].End.Before(features[j].Start) {
						continue
					}
					t.Errorf("site feature overlap: " + features[i].Station + "/" + features[i].Location + "/" + features[i].SubLocation)
				}
			}
		}
	},
}

var testFeatures_Stations = map[string]func([]meta.Feature, []meta.Station) func(t *testing.T){
	"check for duplicated features": func(features []meta.Feature, stations []meta.Station) func(t *testing.T) {
		return func(t *testing.T) {
			stas := make(map[string]meta.Station)
			for _, s := range stations {
				stas[s.Code] = s
			}
			for _, c := range features {
				if _, ok := stas[c.Station]; !ok {
					t.Error("error: unable to find feature station: " + c.Station)
				}
			}
			for _, c := range features {
				if s, ok := stas[c.Station]; ok {
					switch {
					case c.Start.Before(s.Start):
						t.Log("warning: feature start mismatch: " + strings.Join([]string{
							c.Station,
							c.Location,
							c.Start.String(),
							"before",
							s.Start.String(),
						}, " "))
					case s.End.Before(time.Now()) && c.End.After(s.End):
						t.Log("warning: feature end mismatch: " + strings.Join([]string{
							c.Station,
							c.Location,
							c.End.String(),
							"after",
							s.End.String(),
						}, " "))
					}
				}
			}
		}
	},
}

var testFeatures_Sites = map[string]func([]meta.Feature, []meta.Site) func(t *testing.T){
	"check for duplicated feature sites": func(features []meta.Feature, sites []meta.Site) func(t *testing.T) {
		return func(t *testing.T) {
			list := make(map[string]meta.Site)
			for _, s := range sites {
				list[s.Station+"/"+s.Location] = s
			}
			for _, c := range features {
				if _, ok := list[c.Station+"/"+c.Location]; !ok {
					t.Error("error: unable to find feature site: " + c.Station + "/" + c.Location)
				}
			}
			for _, c := range features {
				if s, ok := list[c.Station+"/"+c.Location]; ok {
					switch {
					case c.Start.Before(s.Start):
						t.Log("warning: feature start mismatch: " + strings.Join([]string{
							c.Station,
							c.Location,
							c.Start.String(),
							"before",
							s.Start.String(),
						}, " "))
					case s.End.Before(time.Now()) && c.End.After(s.End):
						t.Log("warning: feature end mismatch: " + strings.Join([]string{
							c.Station,
							c.Location,
							c.End.String(),
							"after",
							s.End.String(),
						}, " "))
					}
				}
			}
		}
	},
}

func TestFeatures(t *testing.T) {

	var features meta.FeatureList
	loadListFile(t, "../environment/features.csv", &features)

	for k, fn := range testFeatures {
		t.Run(k, fn(features))
	}
}

func TestFeatures_Stations(t *testing.T) {

	var features meta.FeatureList
	loadListFile(t, "../environment/features.csv", &features)

	var stations meta.StationList
	loadListFile(t, "../network/stations.csv", &stations)

	for k, fn := range testFeatures_Stations {
		t.Run(k, fn(features, stations))
	}
}

func TestFeatures_Sites(t *testing.T) {

	var features meta.FeatureList
	loadListFile(t, "../environment/features.csv", &features)

	var sites meta.SiteList
	loadListFile(t, "../network/sites.csv", &sites)

	for k, fn := range testFeatures_Sites {
		t.Run(k, fn(features, sites))
	}
}
<commit_msg>add features properties to overlap checks<commit_after>package delta_test

import (
	"strings"
	"testing"
	"time"

	"github.com/GeoNet/delta/meta"
)

var testFeatures = map[string]func([]meta.Feature) func(t *testing.T){
	"check for duplicated site features": func(features []meta.Feature) func(t *testing.T) {
		return func(t *testing.T) {

			for i := 0; i < len(features); i++ {
				for j := i + 1; j < len(features); j++ {
					if features[i].Station != features[j].Station {
						continue
					}
					if features[i].Location != features[j].Location {
						continue
					}
					if features[i].SubLocation != features[j].SubLocation {
						continue
					}
					if features[i].Property != features[j].Property {
						continue
					}
					if features[i].End.Before(features[j].Start) {
						continue
					}
					t.Errorf("site feature overlap: " + features[i].Station + "/" + features[i].Location + "/" + features[i].SubLocation)
				}
			}
		}
	},
}

var testFeatures_Stations = map[string]func([]meta.Feature, []meta.Station) func(t *testing.T){
	"check for duplicated features": func(features []meta.Feature, stations []meta.Station) func(t *testing.T) {
		return func(t *testing.T) {
			stas := make(map[string]meta.Station)
			for _, s := range stations {
				stas[s.Code] = s
			}
			for _, c := range features {
				if _, ok := stas[c.Station]; !ok {
					t.Error("error: unable to find feature station: " + c.Station)
				}
			}
			for _, c := range features {
				if s, ok := stas[c.Station]; ok {
					switch {
					case c.Start.Before(s.Start):
						t.Log("warning: feature start mismatch: " + strings.Join([]string{
							c.Station,
							c.Location,
							c.Start.String(),
							"before",
							s.Start.String(),
						}, " "))
					case s.End.Before(time.Now()) && c.End.After(s.End):
						t.Log("warning: feature end mismatch: " + strings.Join([]string{
							c.Station,
							c.Location,
							c.End.String(),
							"after",
							s.End.String(),
						}, " "))
					}
				}
			}
		}
	},
}

var testFeatures_Sites = map[string]func([]meta.Feature, []meta.Site) func(t *testing.T){
	"check for duplicated feature sites": func(features []meta.Feature, sites []meta.Site) func(t *testing.T) {
		return func(t *testing.T) {
			list := make(map[string]meta.Site)
			for _, s := range sites {
				list[s.Station+"/"+s.Location] = s
			}
			for _, c := range features {
				if _, ok := list[c.Station+"/"+c.Location]; !ok {
					t.Error("error: unable to find feature site: " + c.Station + "/" + c.Location)
				}
			}
			for _, c := range features {
				if s, ok := list[c.Station+"/"+c.Location]; ok {
					switch {
					case c.Start.Before(s.Start):
						t.Log("warning: feature start mismatch: " + strings.Join([]string{
							c.Station,
							c.Location,
							c.Start.String(),
							"before",
							s.Start.String(),
						}, " "))
					case s.End.Before(time.Now()) && c.End.After(s.End):
						t.Log("warning: feature end mismatch: " + strings.Join([]string{
							c.Station,
							c.Location,
							c.End.String(),
							"after",
							s.End.String(),
						}, " "))
					}
				}
			}
		}
	},
}

func TestFeatures(t *testing.T) {

	var features meta.FeatureList
	loadListFile(t, "../environment/features.csv", &features)

	for k, fn := range testFeatures {
		t.Run(k, fn(features))
	}
}

func TestFeatures_Stations(t *testing.T) {

	var features meta.FeatureList
	loadListFile(t, "../environment/features.csv", &features)

	var stations meta.StationList
	loadListFile(t, "../network/stations.csv", &stations)

	for k, fn := range testFeatures_Stations {
		t.Run(k, fn(features, stations))
	}
}

func TestFeatures_Sites(t *testing.T) {

	var features meta.FeatureList
	loadListFile(t, "../environment/features.csv", &features)

	var sites meta.SiteList
	loadListFile(t, "../network/sites.csv", &sites)

	for k, fn := range testFeatures_Sites {
		t.Run(k, fn(features, sites))
	}
}
<|endoftext|>


<commit_before>//
// DISCLAIMER
//
// Copyright 2017 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
//

package test

import (
	"context"
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"testing"
	"time"

	"github.com/arangodb-helper/arangodb/client"
	"github.com/arangodb-helper/arangodb/service"
)

const (
	basePort = service.DefaultMasterPort
)

var (
	// Custom httpClient which allows insecure HTTPS connections.
	httpClient = &http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			DialContext: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
				DualStack: true,
			}).DialContext,
			MaxIdleConns:        100,
			IdleConnTimeout:     90 * time.Second,
			TLSHandshakeTimeout: 10 * time.Second,
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
			ExpectContinueTimeout: 1 * time.Second,
		},
	}
)

// insecureStarterEndpoint creates an insecure (HTTP) endpoint for a starter
// at localhost using default base port + given offset.
func insecureStarterEndpoint(portOffset int) string {
	return fmt.Sprintf("http://localhost:%d", basePort+portOffset)
}

// secureStarterEndpoint creates a secure (HTTPS) endpoint for a starter
// at localhost using default base port + given offset.
func secureStarterEndpoint(portOffset int) string {
	return fmt.Sprintf("https://localhost:%d", basePort+portOffset)
}

// testCluster runs a series of tests to verify a good cluster.
func testCluster(t *testing.T, starterEndpoint string, isSecure bool) client.API {
	c := NewStarterClient(t, starterEndpoint)
	testProcesses(t, c, "cluster", starterEndpoint, isSecure, false)
	return c
}

// testSingle runs a series of tests to verify a good single server.
func testSingle(t *testing.T, starterEndpoint string, isSecure bool) client.API {
	c := NewStarterClient(t, starterEndpoint)
	testProcesses(t, c, "single", starterEndpoint, isSecure, false)
	return c
}

// testResilientSingle runs a series of tests to verify good resilientsingle servers.
func testResilientSingle(t *testing.T, starterEndpoint string, isSecure bool, expectAgencyOnly bool) client.API {
	c := NewStarterClient(t, starterEndpoint)
	testProcesses(t, c, "resilientsingle", starterEndpoint, isSecure, expectAgencyOnly)
	return c
}

// testProcesses runs a series of tests to verify a good series of database servers.
func testProcesses(t *testing.T, c client.API, mode, starterEndpoint string, isSecure bool, expectAgencyOnly bool) {
	ctx := context.Background()

	// Fetch version
	if info, err := c.Version(ctx); err != nil {
		t.Errorf("Failed to get starter version: %s", describe(err))
	} else {
		if isVerbose {
			t.Logf("Found starter version %s, %s", info.Version, info.Build)
		}
	}

	// Fetch server processes
	processes, err := c.Processes(ctx)
	if err != nil {
		t.Fatalf("Failed to get server processes: %s", describe(err))
	}

	// Check agent
	if sp, ok := processes.ServerByType(client.ServerTypeAgent); ok {
		if sp.IsSecure != isSecure {
			t.Errorf("Invalid IsSecure on agent. Expected %v, got %v", isSecure, sp.IsSecure)
		}
		if mode == "single" {
			t.Errorf("Found agent, not allowed in single mode")
		} else {
			if isVerbose {
				t.Logf("Found agent at %s:%d", sp.IP, sp.Port)
			}
			testArangodReachable(t, sp)
		}
	}

	// Check coordinator
	if sp, ok := processes.ServerByType(client.ServerTypeCoordinator); ok {
		if sp.IsSecure != isSecure {
			t.Errorf("Invalid IsSecure on coordinator. Expected %v, got %v", isSecure, sp.IsSecure)
		}
		if mode == "single" || mode == "resilientsingle" {
			t.Errorf("Found coordinator, not allowed in single|resilientsingle mode")
		} else {
			if isVerbose {
				t.Logf("Found coordinator at %s:%d", sp.IP, sp.Port)
			}
			testArangodReachable(t, sp)
		}
	} else if mode == "cluster" {
		t.Errorf("No coordinator found in %s", starterEndpoint)
	}

	// Check dbserver
	if sp, ok := processes.ServerByType(client.ServerTypeDBServer); ok {
		if sp.IsSecure != isSecure {
			t.Errorf("Invalid IsSecure on dbserver. Expected %v, got %v", isSecure, sp.IsSecure)
		}
		if mode == "single" || mode == "resilientsingle" {
			t.Errorf("Found dbserver, not allowed in single|resilientsingle mode")
		} else {
			if isVerbose {
				t.Logf("Found dbserver at %s:%d", sp.IP, sp.Port)
			}
			testArangodReachable(t, sp)
		}
	} else if mode == "cluster" {
		t.Errorf("No dbserver found in %s", starterEndpoint)
	}

	// Check single
	if sp, ok := processes.ServerByType(client.ServerTypeSingle); ok {
		if sp.IsSecure != isSecure {
			t.Errorf("Invalid IsSecure on single. Expected %v, got %v", isSecure, sp.IsSecure)
		}
		if mode == "cluster" || expectAgencyOnly {
			t.Errorf("Found single, not allowed in cluster mode")
		} else {
			if isVerbose {
				t.Logf("Found single at %s:%d", sp.IP, sp.Port)
			}
			testArangodReachable(t, sp)
		}
	} else if (mode == "single" || mode == "resilientsingle") && !expectAgencyOnly {
		t.Errorf("No single found in %s", starterEndpoint)
	}
}

// testArangodReachable tries to call some HTTP API methods of the given server process to make sure
// it is reachable.
func testArangodReachable(t *testing.T, sp client.ServerProcess) {
	scheme := "http"
	if sp.IsSecure {
		scheme = "https"
	}
	url := fmt.Sprintf("%s://%s:%d/_api/version", scheme, sp.IP, sp.Port)
	_, err := httpClient.Get(url)
	if err != nil {
		t.Errorf("Failed to reach arangod at %s:%d", sp.IP, sp.Port)
	}
}
<commit_msg>Show more error info on failure<commit_after>//
// DISCLAIMER
//
// Copyright 2017 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
//

package test

import (
	"context"
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"testing"
	"time"

	"github.com/arangodb-helper/arangodb/client"
	"github.com/arangodb-helper/arangodb/service"
)

const (
	basePort = service.DefaultMasterPort
)

var (
	// Custom httpClient which allows insecure HTTPS connections.
	httpClient = &http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			DialContext: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
				DualStack: true,
			}).DialContext,
			MaxIdleConns:        100,
			IdleConnTimeout:     90 * time.Second,
			TLSHandshakeTimeout: 10 * time.Second,
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
			ExpectContinueTimeout: 1 * time.Second,
		},
	}
)

// insecureStarterEndpoint creates an insecure (HTTP) endpoint for a starter
// at localhost using default base port + given offset.
func insecureStarterEndpoint(portOffset int) string {
	return fmt.Sprintf("http://localhost:%d", basePort+portOffset)
}

// secureStarterEndpoint creates a secure (HTTPS) endpoint for a starter
// at localhost using default base port + given offset.
func secureStarterEndpoint(portOffset int) string {
	return fmt.Sprintf("https://localhost:%d", basePort+portOffset)
}

// testCluster runs a series of tests to verify a good cluster.
func testCluster(t *testing.T, starterEndpoint string, isSecure bool) client.API {
	c := NewStarterClient(t, starterEndpoint)
	testProcesses(t, c, "cluster", starterEndpoint, isSecure, false)
	return c
}

// testSingle runs a series of tests to verify a good single server.
func testSingle(t *testing.T, starterEndpoint string, isSecure bool) client.API {
	c := NewStarterClient(t, starterEndpoint)
	testProcesses(t, c, "single", starterEndpoint, isSecure, false)
	return c
}

// testResilientSingle runs a series of tests to verify good resilientsingle servers.
func testResilientSingle(t *testing.T, starterEndpoint string, isSecure bool, expectAgencyOnly bool) client.API {
	c := NewStarterClient(t, starterEndpoint)
	testProcesses(t, c, "resilientsingle", starterEndpoint, isSecure, expectAgencyOnly)
	return c
}

// testProcesses runs a series of tests to verify a good series of database servers.
func testProcesses(t *testing.T, c client.API, mode, starterEndpoint string, isSecure bool, expectAgencyOnly bool) {
	ctx := context.Background()

	// Fetch version
	if info, err := c.Version(ctx); err != nil {
		t.Errorf("Failed to get starter version: %s", describe(err))
	} else {
		if isVerbose {
			t.Logf("Found starter version %s, %s", info.Version, info.Build)
		}
	}

	// Fetch server processes
	processes, err := c.Processes(ctx)
	if err != nil {
		t.Fatalf("Failed to get server processes: %s", describe(err))
	}

	// Check agent
	if sp, ok := processes.ServerByType(client.ServerTypeAgent); ok {
		if sp.IsSecure != isSecure {
			t.Errorf("Invalid IsSecure on agent. Expected %v, got %v", isSecure, sp.IsSecure)
		}
		if mode == "single" {
			t.Errorf("Found agent, not allowed in single mode")
		} else {
			if isVerbose {
				t.Logf("Found agent at %s:%d", sp.IP, sp.Port)
			}
			testArangodReachable(t, sp)
		}
	}

	// Check coordinator
	if sp, ok := processes.ServerByType(client.ServerTypeCoordinator); ok {
		if sp.IsSecure != isSecure {
			t.Errorf("Invalid IsSecure on coordinator. Expected %v, got %v", isSecure, sp.IsSecure)
		}
		if mode == "single" || mode == "resilientsingle" {
			t.Errorf("Found coordinator, not allowed in single|resilientsingle mode")
		} else {
			if isVerbose {
				t.Logf("Found coordinator at %s:%d", sp.IP, sp.Port)
			}
			testArangodReachable(t, sp)
		}
	} else if mode == "cluster" {
		t.Errorf("No coordinator found in %s", starterEndpoint)
	}

	// Check dbserver
	if sp, ok := processes.ServerByType(client.ServerTypeDBServer); ok {
		if sp.IsSecure != isSecure {
			t.Errorf("Invalid IsSecure on dbserver. Expected %v, got %v", isSecure, sp.IsSecure)
		}
		if mode == "single" || mode == "resilientsingle" {
			t.Errorf("Found dbserver, not allowed in single|resilientsingle mode")
		} else {
			if isVerbose {
				t.Logf("Found dbserver at %s:%d", sp.IP, sp.Port)
			}
			testArangodReachable(t, sp)
		}
	} else if mode == "cluster" {
		t.Errorf("No dbserver found in %s", starterEndpoint)
	}

	// Check single
	if sp, ok := processes.ServerByType(client.ServerTypeSingle); ok {
		if sp.IsSecure != isSecure {
			t.Errorf("Invalid IsSecure on single. Expected %v, got %v", isSecure, sp.IsSecure)
		}
		if mode == "cluster" || expectAgencyOnly {
			t.Errorf("Found single, not allowed in cluster mode")
		} else {
			if isVerbose {
				t.Logf("Found single at %s:%d", sp.IP, sp.Port)
			}
			testArangodReachable(t, sp)
		}
	} else if (mode == "single" || mode == "resilientsingle") && !expectAgencyOnly {
		t.Errorf("No single found in %s", starterEndpoint)
	}
}

// testArangodReachable tries to call some HTTP API methods of the given server process to make sure
// it is reachable.
func testArangodReachable(t *testing.T, sp client.ServerProcess) {
	scheme := "http"
	if sp.IsSecure {
		scheme = "https"
	}
	url := fmt.Sprintf("%s://%s:%d/_api/version", scheme, sp.IP, sp.Port)
	_, err := httpClient.Get(url)
	if err != nil {
		t.Errorf("Failed to reach arangod at %s:%d (%#v)", sp.IP, sp.Port, err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"context"
	"net/http"
	"net/url"
	"strings"
	"sync/atomic"
	"time"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/httpstream"
	"k8s.io/apimachinery/pkg/util/httpstream/spdy"
	utilnet "k8s.io/apimachinery/pkg/util/net"
	"k8s.io/apimachinery/pkg/util/proxy"
	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
	endpointmetrics "k8s.io/apiserver/pkg/endpoints/metrics"
	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
	genericfeatures "k8s.io/apiserver/pkg/features"
	utilfeature "k8s.io/apiserver/pkg/util/feature"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/transport"
	"k8s.io/klog"
	apiregistrationv1api "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
	apiregistrationv1apihelper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper"
)

const (
	aggregatorComponent string = "aggregator"

	aggregatedDiscoveryTimeout = 5 * time.Second
)

// proxyHandler provides a http.Handler which will proxy traffic to locations
// specified by items implementing Redirector.
type proxyHandler struct {
	// localDelegate is used to satisfy local APIServices
	localDelegate http.Handler

	// proxyClientCert/Key are the client cert used to identify this proxy. Backing APIServices use
	// this to confirm the proxy's identity
	proxyClientCert []byte
	proxyClientKey  []byte
	proxyTransport  *http.Transport

	// Endpoints based routing to map from cluster IP to routable IP
	serviceResolver ServiceResolver

	handlingInfo atomic.Value
}

type proxyHandlingInfo struct {
	// local indicates that this APIService is locally satisfied
	local bool

	// name is the name of the APIService
	name string
	// restConfig holds the information for building a roundtripper
	restConfig *restclient.Config
	// transportBuildingError is an error produced while building the transport.  If this
	// is non-nil, it will be reported to clients.
	transportBuildingError error
	// proxyRoundTripper is the re-useable portion of the transport.  It does not vary with any request.
	proxyRoundTripper http.RoundTripper
	// serviceName is the name of the service this handler proxies to
	serviceName string
	// namespace is the namespace the service lives in
	serviceNamespace string
	// serviceAvailable indicates this APIService is available or not
	serviceAvailable bool
	// servicePort is the port of the service this handler proxies to
	servicePort int32
}

func proxyError(w http.ResponseWriter, req *http.Request, error string, code int) {
	http.Error(w, error, code)

	ctx := req.Context()
	info, ok := genericapirequest.RequestInfoFrom(ctx)
	if !ok {
		klog.Warning("no RequestInfo found in the context")
		return
	}
	// TODO: record long-running request differently? The long-running check func does not necessarily match the one of the aggregated apiserver
	endpointmetrics.Record(req, info, aggregatorComponent, "", code, 0, 0)
}

func (r *proxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	value := r.handlingInfo.Load()
	if value == nil {
		r.localDelegate.ServeHTTP(w, req)
		return
	}
	handlingInfo := value.(proxyHandlingInfo)
	if handlingInfo.local {
		if r.localDelegate == nil {
			http.Error(w, "", http.StatusNotFound)
			return
		}
		r.localDelegate.ServeHTTP(w, req)
		return
	}

	if !handlingInfo.serviceAvailable {
		proxyError(w, req, "service unavailable", http.StatusServiceUnavailable)
		return
	}

	if handlingInfo.transportBuildingError != nil {
		proxyError(w, req, handlingInfo.transportBuildingError.Error(), http.StatusInternalServerError)
		return
	}

	user, ok := genericapirequest.UserFrom(req.Context())
	if !ok {
		proxyError(w, req, "missing user", http.StatusInternalServerError)
		return
	}

	// write a new location based on the existing request pointed at the target service
	location := &url.URL{}
	location.Scheme = "https"
	rloc, err := r.serviceResolver.ResolveEndpoint(handlingInfo.serviceNamespace, handlingInfo.serviceName, handlingInfo.servicePort)
	if err != nil {
		klog.Errorf("error resolving %s/%s: %v", handlingInfo.serviceNamespace, handlingInfo.serviceName, err)
		proxyError(w, req, "service unavailable", http.StatusServiceUnavailable)
		return
	}
	location.Host = rloc.Host
	location.Path = req.URL.Path
	location.RawQuery = req.URL.Query().Encode()

	newReq, cancelFn := newRequestForProxy(location, req)
	defer cancelFn()

	if handlingInfo.proxyRoundTripper == nil {
		proxyError(w, req, "", http.StatusNotFound)
		return
	}

	// we need to wrap the roundtripper in another roundtripper which will apply the front proxy headers
	proxyRoundTripper, upgrade, err := maybeWrapForConnectionUpgrades(handlingInfo.restConfig, handlingInfo.proxyRoundTripper, req)
	if err != nil {
		proxyError(w, req, err.Error(), http.StatusInternalServerError)
		return
	}
	proxyRoundTripper = transport.NewAuthProxyRoundTripper(user.GetName(), user.GetGroups(), user.GetExtra(), proxyRoundTripper)

	// if we are upgrading, then the upgrade path tries to use this request with the TLS config we provide, but it does
	// NOT use the roundtripper.  Its a direct call that bypasses the round tripper.  This means that we have to
	// attach the "correct" user headers to the request ahead of time.  After the initial upgrade, we'll be back
	// at the roundtripper flow, so we only have to muck with this request, but we do have to do it.
	if upgrade {
		transport.SetAuthProxyHeaders(newReq, user.GetName(), user.GetGroups(), user.GetExtra())
	}

	handler := proxy.NewUpgradeAwareHandler(location, proxyRoundTripper, true, upgrade, &responder{w: w})
	handler.ServeHTTP(w, newReq)
}

// newRequestForProxy returns a shallow copy of the original request with a context that may include a timeout for discovery requests
func newRequestForProxy(location *url.URL, req *http.Request) (*http.Request, context.CancelFunc) {
	newCtx := context.Background()
	cancelFn := func() {}

	// if the original request has a deadline, we should honor that deadline for our proxied request
	if deadline, ok := req.Context().Deadline(); ok {
		newCtx, cancelFn = context.WithDeadline(newCtx, deadline)

		// trim leading and trailing slashes. Then "/apis/group/version" requests are for discovery, so if we have exactly three
		// segments that we are going to proxy, we have a discovery request.
	} else if len(strings.Split(strings.Trim(req.URL.Path, "/"), "/")) == 3 {
		// discovery requests are used by kubectl and others to determine which resources a server has.  This is a cheap call that
		// should be fast for every aggregated apiserver.  Latency for aggregation is expected to be low (as for all extensions)
		// so forcing a short timeout here helps responsiveness of all clients.
		newCtx, cancelFn = context.WithTimeout(newCtx, aggregatedDiscoveryTimeout)
	}

	// WithContext creates a shallow clone of the request with the new context.
	newReq := req.WithContext(newCtx)
	newReq.Header = utilnet.CloneHeader(req.Header)
	newReq.URL = location
	newReq.Host = location.Host

	return newReq, cancelFn
}

// maybeWrapForConnectionUpgrades wraps the roundtripper for upgrades.  The bool indicates if it was wrapped
func maybeWrapForConnectionUpgrades(restConfig *restclient.Config, rt http.RoundTripper, req *http.Request) (http.RoundTripper, bool, error) {
	if !httpstream.IsUpgradeRequest(req) {
		return rt, false, nil
	}

	tlsConfig, err := restclient.TLSConfigFor(restConfig)
	if err != nil {
		return nil, true, err
	}
	followRedirects := utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StreamingProxyRedirects)
	requireSameHostRedirects := utilfeature.DefaultFeatureGate.Enabled(genericfeatures.ValidateProxyRedirects)
	upgradeRoundTripper := spdy.NewRoundTripper(tlsConfig, followRedirects, requireSameHostRedirects)
	wrappedRT, err := restclient.HTTPWrappersForConfig(restConfig, upgradeRoundTripper)
	if err != nil {
		return nil, true, err
	}

	return wrappedRT, true, nil
}

// responder implements rest.Responder for assisting a connector in writing objects or errors.
type responder struct {
	w http.ResponseWriter
}

// TODO this should properly handle content type negotiation
// if the caller asked for protobuf and you write JSON bad things happen.
func (r *responder) Object(statusCode int, obj runtime.Object) {
	responsewriters.WriteRawJSON(statusCode, obj, r.w)
}

func (r *responder) Error(_ http.ResponseWriter, _ *http.Request, err error) {
	http.Error(r.w, err.Error(), http.StatusInternalServerError)
}

// these methods provide locked access to fields

func (r *proxyHandler) updateAPIService(apiService *apiregistrationv1api.APIService) {
	if apiService.Spec.Service == nil {
		r.handlingInfo.Store(proxyHandlingInfo{local: true})
		return
	}

	newInfo := proxyHandlingInfo{
		name: apiService.Name,
		restConfig: &restclient.Config{
			TLSClientConfig: restclient.TLSClientConfig{
				Insecure:   apiService.Spec.InsecureSkipTLSVerify,
				ServerName: apiService.Spec.Service.Name + "." + apiService.Spec.Service.Namespace + ".svc",
				CertData:   r.proxyClientCert,
				KeyData:    r.proxyClientKey,
				CAData:     apiService.Spec.CABundle,
			},
		},
		serviceName:      apiService.Spec.Service.Name,
		serviceNamespace: apiService.Spec.Service.Namespace,
		servicePort:      *apiService.Spec.Service.Port,
		serviceAvailable: apiregistrationv1apihelper.IsAPIServiceConditionTrue(apiService, apiregistrationv1api.Available),
	}
	if r.proxyTransport != nil && r.proxyTransport.DialContext != nil {
		newInfo.restConfig.Dial = r.proxyTransport.DialContext
	}
	newInfo.proxyRoundTripper, newInfo.transportBuildingError = restclient.TransportFor(newInfo.restConfig)
	if newInfo.transportBuildingError != nil {
		klog.Warning(newInfo.transportBuildingError.Error())
	}
	r.handlingInfo.Store(newInfo)
}
<commit_msg>add env var to allow disabling the aggregated discovery timeout<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"context"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"sync/atomic"
	"time"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/httpstream"
	"k8s.io/apimachinery/pkg/util/httpstream/spdy"
	utilnet "k8s.io/apimachinery/pkg/util/net"
	"k8s.io/apimachinery/pkg/util/proxy"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
	endpointmetrics "k8s.io/apiserver/pkg/endpoints/metrics"
	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
	genericfeatures "k8s.io/apiserver/pkg/features"
	utilfeature "k8s.io/apiserver/pkg/util/feature"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/transport"
	"k8s.io/klog"
	apiregistrationv1api "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
	apiregistrationv1apihelper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper"
)

const (
	aggregatorComponent string = "aggregator"

	aggregatedDiscoveryTimeout = 5 * time.Second
)

var (
	// TODO this should be unconditionally true once we remove the env var override
	enableAggregatedDiscoveryTimeout = true
)

func init() {
	disableAggregatedDiscoveryTimeout, err := strconv.ParseBool(os.Getenv("DEPRECATED_DISABLE_AGGREGATOR_DISCOVERY_TIMEOUT"))
	if err != nil {
		utilruntime.HandleError(err)
	}
	enableAggregatedDiscoveryTimeout = !disableAggregatedDiscoveryTimeout
}

// proxyHandler provides a http.Handler which will proxy traffic to locations
// specified by items implementing Redirector.
type proxyHandler struct {
	// localDelegate is used to satisfy local APIServices
	localDelegate http.Handler

	// proxyClientCert/Key are the client cert used to identify this proxy. Backing APIServices use
	// this to confirm the proxy's identity
	proxyClientCert []byte
	proxyClientKey  []byte
	proxyTransport  *http.Transport

	// Endpoints based routing to map from cluster IP to routable IP
	serviceResolver ServiceResolver

	handlingInfo atomic.Value
}

type proxyHandlingInfo struct {
	// local indicates that this APIService is locally satisfied
	local bool

	// name is the name of the APIService
	name string
	// restConfig holds the information for building a roundtripper
	restConfig *restclient.Config
	// transportBuildingError is an error produced while building the transport.  If this
	// is non-nil, it will be reported to clients.
	transportBuildingError error
	// proxyRoundTripper is the re-useable portion of the transport.  It does not vary with any request.
	proxyRoundTripper http.RoundTripper
	// serviceName is the name of the service this handler proxies to
	serviceName string
	// namespace is the namespace the service lives in
	serviceNamespace string
	// serviceAvailable indicates this APIService is available or not
	serviceAvailable bool
	// servicePort is the port of the service this handler proxies to
	servicePort int32
}

func proxyError(w http.ResponseWriter, req *http.Request, error string, code int) {
	http.Error(w, error, code)

	ctx := req.Context()
	info, ok := genericapirequest.RequestInfoFrom(ctx)
	if !ok {
		klog.Warning("no RequestInfo found in the context")
		return
	}
	// TODO: record long-running request differently? The long-running check func does not necessarily match the one of the aggregated apiserver
	endpointmetrics.Record(req, info, aggregatorComponent, "", code, 0, 0)
}

func (r *proxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	value := r.handlingInfo.Load()
	if value == nil {
		r.localDelegate.ServeHTTP(w, req)
		return
	}
	handlingInfo := value.(proxyHandlingInfo)
	if handlingInfo.local {
		if r.localDelegate == nil {
			http.Error(w, "", http.StatusNotFound)
			return
		}
		r.localDelegate.ServeHTTP(w, req)
		return
	}

	if !handlingInfo.serviceAvailable {
		proxyError(w, req, "service unavailable", http.StatusServiceUnavailable)
		return
	}

	if handlingInfo.transportBuildingError != nil {
		proxyError(w, req, handlingInfo.transportBuildingError.Error(), http.StatusInternalServerError)
		return
	}

	user, ok := genericapirequest.UserFrom(req.Context())
	if !ok {
		proxyError(w, req, "missing user", http.StatusInternalServerError)
		return
	}

	// write a new location based on the existing request pointed at the target service
	location := &url.URL{}
	location.Scheme = "https"
	rloc, err := r.serviceResolver.ResolveEndpoint(handlingInfo.serviceNamespace, handlingInfo.serviceName, handlingInfo.servicePort)
	if err != nil {
		klog.Errorf("error resolving %s/%s: %v", handlingInfo.serviceNamespace, handlingInfo.serviceName, err)
		proxyError(w, req, "service unavailable", http.StatusServiceUnavailable)
		return
	}
	location.Host = rloc.Host
	location.Path = req.URL.Path
	location.RawQuery = req.URL.Query().Encode()

	newReq, cancelFn := newRequestForProxy(location, req)
	defer cancelFn()

	if handlingInfo.proxyRoundTripper == nil {
		proxyError(w, req, "", http.StatusNotFound)
		return
	}

	// we need to wrap the roundtripper in another roundtripper which will apply the front proxy headers
	proxyRoundTripper, upgrade, err := maybeWrapForConnectionUpgrades(handlingInfo.restConfig, handlingInfo.proxyRoundTripper, req)
	if err != nil {
		proxyError(w, req, err.Error(), http.StatusInternalServerError)
		return
	}
	proxyRoundTripper = transport.NewAuthProxyRoundTripper(user.GetName(), user.GetGroups(), user.GetExtra(), proxyRoundTripper)

	// if we are upgrading, then the upgrade path tries to use this request with the TLS config we provide, but it does
	// NOT use the roundtripper.  Its a direct call that bypasses the round tripper.  This means that we have to
	// attach the "correct" user headers to the request ahead of time.  After the initial upgrade, we'll be back
	// at the roundtripper flow, so we only have to muck with this request, but we do have to do it.
	if upgrade {
		transport.SetAuthProxyHeaders(newReq, user.GetName(), user.GetGroups(), user.GetExtra())
	}

	handler := proxy.NewUpgradeAwareHandler(location, proxyRoundTripper, true, upgrade, &responder{w: w})
	handler.ServeHTTP(w, newReq)
}

// newRequestForProxy returns a shallow copy of the original request with a context that may include a timeout for discovery requests
func newRequestForProxy(location *url.URL, req *http.Request) (*http.Request, context.CancelFunc) {
	newCtx := context.Background()
	cancelFn := func() {}

	// if the original request has a deadline, we should honor that deadline for our proxied request
	if deadline, ok := req.Context().Deadline(); ok {
		newCtx, cancelFn = context.WithDeadline(newCtx, deadline)

		// trim leading and trailing slashes. Then "/apis/group/version" requests are for discovery, so if we have exactly three
		// segments that we are going to proxy, we have a discovery request.
	} else if enableAggregatedDiscoveryTimeout && len(strings.Split(strings.Trim(req.URL.Path, "/"), "/")) == 3 {
		// discovery requests are used by kubectl and others to determine which resources a server has.  This is a cheap call that
		// should be fast for every aggregated apiserver.  Latency for aggregation is expected to be low (as for all extensions)
		// so forcing a short timeout here helps responsiveness of all clients.
		newCtx, cancelFn = context.WithTimeout(newCtx, aggregatedDiscoveryTimeout)
	}

	// WithContext creates a shallow clone of the request with the new context.
	newReq := req.WithContext(newCtx)
	newReq.Header = utilnet.CloneHeader(req.Header)
	newReq.URL = location
	newReq.Host = location.Host

	return newReq, cancelFn
}

// maybeWrapForConnectionUpgrades wraps the roundtripper for upgrades.  The bool indicates if it was wrapped
func maybeWrapForConnectionUpgrades(restConfig *restclient.Config, rt http.RoundTripper, req *http.Request) (http.RoundTripper, bool, error) {
	if !httpstream.IsUpgradeRequest(req) {
		return rt, false, nil
	}

	tlsConfig, err := restclient.TLSConfigFor(restConfig)
	if err != nil {
		return nil, true, err
	}
	followRedirects := utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StreamingProxyRedirects)
	requireSameHostRedirects := utilfeature.DefaultFeatureGate.Enabled(genericfeatures.ValidateProxyRedirects)
	upgradeRoundTripper := spdy.NewRoundTripper(tlsConfig, followRedirects, requireSameHostRedirects)
	wrappedRT, err := restclient.HTTPWrappersForConfig(restConfig, upgradeRoundTripper)
	if err != nil {
		return nil, true, err
	}

	return wrappedRT, true, nil
}

// responder implements rest.Responder for assisting a connector in writing objects or errors.
type responder struct {
	w http.ResponseWriter
}

// TODO this should properly handle content type negotiation
// if the caller asked for protobuf and you write JSON bad things happen.
func (r *responder) Object(statusCode int, obj runtime.Object) {
	responsewriters.WriteRawJSON(statusCode, obj, r.w)
}

func (r *responder) Error(_ http.ResponseWriter, _ *http.Request, err error) {
	http.Error(r.w, err.Error(), http.StatusInternalServerError)
}

// these methods provide locked access to fields

func (r *proxyHandler) updateAPIService(apiService *apiregistrationv1api.APIService) {
	if apiService.Spec.Service == nil {
		r.handlingInfo.Store(proxyHandlingInfo{local: true})
		return
	}

	newInfo := proxyHandlingInfo{
		name: apiService.Name,
		restConfig: &restclient.Config{
			TLSClientConfig: restclient.TLSClientConfig{
				Insecure:   apiService.Spec.InsecureSkipTLSVerify,
				ServerName: apiService.Spec.Service.Name + "." + apiService.Spec.Service.Namespace + ".svc",
				CertData:   r.proxyClientCert,
				KeyData:    r.proxyClientKey,
				CAData:     apiService.Spec.CABundle,
			},
		},
		serviceName:      apiService.Spec.Service.Name,
		serviceNamespace: apiService.Spec.Service.Namespace,
		servicePort:      *apiService.Spec.Service.Port,
		serviceAvailable: apiregistrationv1apihelper.IsAPIServiceConditionTrue(apiService, apiregistrationv1api.Available),
	}
	if r.proxyTransport != nil && r.proxyTransport.DialContext != nil {
		newInfo.restConfig.Dial = r.proxyTransport.DialContext
	}
	newInfo.proxyRoundTripper, newInfo.transportBuildingError = restclient.TransportFor(newInfo.restConfig)
	if newInfo.transportBuildingError != nil {
		klog.Warning(newInfo.transportBuildingError.Error())
	}
	r.handlingInfo.Store(newInfo)
}
<|endoftext|>


<commit_before>package templates

const (
	MainHelpTemplate = `{{.Long | trim}}
{{if or .Runnable .HasSubCommands}}{{.UsageString}}{{end}}
`

	MainUsageTemplate = `{{ $cmd := . }}
Usage: {{if .Runnable}}
  {{.UseLine}}{{if .HasFlags}} [options]{{end}}{{end}}{{if .HasSubCommands}}
  {{ .CommandPath}} [command]{{end}}{{if gt .Aliases 0}}

Aliases:
  {{.NameAndAliases}}{{end}}
{{ if .HasSubCommands}}
Available Commands: {{range .Commands}}{{if .Runnable}}
  {{rpad .Use .UsagePadding }} {{.Short}}{{end}}{{end}}
{{end}}
{{ if .HasLocalFlags}}Options:
{{.LocalFlags.FlagUsages}}{{end}}
{{ if .HasAnyPersistentFlags}}Global Options:
{{.AllPersistentFlags.FlagUsages}}{{end}}{{ if .HasSubCommands }}
Use "{{.Root.Name}} help [command]" for more information about that command.{{end}}`

	CliHelpTemplate = `{{.Long | trim}}
{{if or .Runnable .HasSubCommands}}{{.UsageString}}{{end}}
`
	CliUsageTemplate = `{{ $cmd := . }}{{ if .HasSubCommands}}
Available Commands: {{range .Commands}}{{if .Runnable}}{{if ne .Name "options"}}
 {{rpad .Use .UsagePadding }} {{.Short}}{{end}}{{end}}{{end}}
{{end}}
{{ if .HasLocalFlags}}Options:
{{.LocalFlags.FlagUsages}}{{end}}{{ if .HasSubCommands }}
Use "{{.Root.Name}} help [command]" for more information about that command.{{end}}{{ if .HasAnyPersistentFlags}}
Use "{{.Root.Name}} options" for a list of global command-line options.{{end}}`

	OptionsUsageTemplate = `{{ if .HasAnyPersistentFlags}}The following options can be passed to any command:

{{.AllPersistentFlags.FlagUsages}}{{end}}`
)
<commit_msg>Remove the usage from the list of available commands in help<commit_after>package templates

const (
	MainHelpTemplate = `{{.Long | trim}}
{{if or .Runnable .HasSubCommands}}{{.UsageString}}{{end}}
`

	MainUsageTemplate = `{{ $cmd := . }}
Usage: {{if .Runnable}}
  {{.UseLine}}{{if .HasFlags}} [options]{{end}}{{end}}{{if .HasSubCommands}}
  {{ .CommandPath}} [command]{{end}}{{if gt .Aliases 0}}

Aliases:
  {{.NameAndAliases}}{{end}}
{{ if .HasSubCommands}}
Available Commands: {{range .Commands}}{{if .Runnable}}
  {{rpad .Use .UsagePadding }} {{.Short}}{{end}}{{end}}
{{end}}
{{ if .HasLocalFlags}}Options:
{{.LocalFlags.FlagUsages}}{{end}}
{{ if .HasAnyPersistentFlags}}Global Options:
{{.AllPersistentFlags.FlagUsages}}{{end}}{{ if .HasSubCommands }}
Use "{{.Root.Name}} help [command]" for more information about that command.{{end}}`

	CliHelpTemplate = `{{.Long | trim}}
{{if or .Runnable .HasSubCommands}}{{.UsageString}}{{end}}
`
	CliUsageTemplate = `{{ $cmd := . }}{{ if .HasSubCommands}}
Available Commands: {{range .Commands}}{{if .Runnable}}{{if ne .Name "options"}}
 {{rpad .Name 20 }} {{.Short}}{{end}}{{end}}{{end}}
{{end}}
{{ if .HasLocalFlags}}Options:
{{.LocalFlags.FlagUsages}}{{end}}{{ if .HasSubCommands }}
Use "{{.Root.Name}} help [command]" for more information about that command.{{end}}{{ if .HasAnyPersistentFlags}}
Use "{{.Root.Name}} options" for a list of global command-line options.{{end}}`

	OptionsUsageTemplate = `{{ if .HasAnyPersistentFlags}}The following options can be passed to any command:

{{.AllPersistentFlags.FlagUsages}}{{end}}`
)
<|endoftext|>


<commit_before>//  Copyright 2016 Red Hat, Inc.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package k8sutil

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"

	"k8s.io/client-go/1.5/kubernetes"
	"k8s.io/client-go/1.5/pkg/util/homedir"
	"github.com/kardianos/osext"
)

// ResolveKubectlBinary resolves the binary to use such as 'kubectl' or 'oc'
func ResolveKubectlBinary(kubeclient *kubernetes.Clientset) (string, error) {
	isOpenshift := isOpenShiftCluster(kubeclient)
	kubeBinary := "kubectl"
	if isOpenshift {
		kubeBinary = "oc"
	}

	if runtime.GOOS == "windows" && !strings.HasSuffix(kubeBinary, ".exe") {
		kubeBinary += ".exe"
	}

	name, err := resolveBinaryLocation(kubeBinary)
	if err != nil {
	  return "", err
	}
	if len(name) == 0 {
		return "", fmt.Errorf("Could not find binary %s on your PATH. Is it installed?", kubeBinary)
	}
	return name, nil
}

func isOpenShiftCluster(kubeclient *kubernetes.Clientset) bool {
	// TODO
	return false
}

// lets find the executable on the PATH or in the fabric8 directory
func resolveBinaryLocation(executable string) (string, error) {
	path, err := exec.LookPath(executable)
	if err != nil || fileNotExist(path) {
		home := os.Getenv("HOME")
		if home == "" {
			fmt.Printf("No $HOME environment variable found")
		}
		writeFileLocation := getFabric8BinLocation()

		// lets try in the fabric8 folder
		path = filepath.Join(writeFileLocation, executable)
		if fileNotExist(path) {
			path = executable
			// lets try in the folder where we found the gofabric8 executable
			folder, err := osext.ExecutableFolder()
			if err != nil {
				return "", fmt.Errorf("Failed to find executable folder: %v\n", err)
			} else {
				path = filepath.Join(folder, executable)
				if fileNotExist(path) {
					fmt.Printf("Could not find executable at %v\n", path)
					path = executable
				}
			}
		}
	}
	return path, nil
}

func findExecutable(file string) error {
	d, err := os.Stat(file)
	if err != nil {
		return err
	}
	if m := d.Mode(); !m.IsDir() {
		return nil
	}
	return os.ErrPermission
}

func fileNotExist(path string) bool {
	return findExecutable(path) != nil
}

func getFabric8BinLocation() string {
	home := homedir.HomeDir()
	if home == "" {
		fmt.Printf("No user home environment variable found for OS %s\n", runtime.GOOS)
	}
	return filepath.Join(home, ".fabric8", "bin")
}
<commit_msg>Use oc instead of kubectl when talking to OpenShift<commit_after>//  Copyright 2016 Red Hat, Inc.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package k8sutil

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"

	"k8s.io/client-go/1.5/kubernetes"
	"k8s.io/client-go/1.5/pkg/util/homedir"
	"github.com/kardianos/osext"
)

// ResolveKubectlBinary resolves the binary to use such as 'kubectl' or 'oc'
func ResolveKubectlBinary(kubeclient *kubernetes.Clientset) (string, error) {
	isOpenshift := isOpenShiftCluster(kubeclient)
	kubeBinary := "kubectl"
	if isOpenshift {
		kubeBinary = "oc"
	}

	if runtime.GOOS == "windows" && !strings.HasSuffix(kubeBinary, ".exe") {
		kubeBinary += ".exe"
	}

	name, err := resolveBinaryLocation(kubeBinary)
	if err != nil {
	  return "", err
	}
	if len(name) == 0 {
		return "", fmt.Errorf("Could not find binary %s on your PATH. Is it installed?", kubeBinary)
	}
	return name, nil
}

func isOpenShiftCluster(kubeclient *kubernetes.Clientset) bool {
	// The presence of "/oapi" on the API server is our hacky way of
	// determining if we're talking to OpenShift
	err := kubeclient.Core().GetRESTClient().Get().AbsPath("/oapi").Do().Error()
	if err != nil {
		return false
	}
	return true
}

// lets find the executable on the PATH or in the fabric8 directory
func resolveBinaryLocation(executable string) (string, error) {
	path, err := exec.LookPath(executable)
	if err != nil || fileNotExist(path) {
		home := os.Getenv("HOME")
		if home == "" {
			fmt.Printf("No $HOME environment variable found")
		}
		writeFileLocation := getFabric8BinLocation()

		// lets try in the fabric8 folder
		path = filepath.Join(writeFileLocation, executable)
		if fileNotExist(path) {
			path = executable
			// lets try in the folder where we found the gofabric8 executable
			folder, err := osext.ExecutableFolder()
			if err != nil {
				return "", fmt.Errorf("Failed to find executable folder: %v\n", err)
			} else {
				path = filepath.Join(folder, executable)
				if fileNotExist(path) {
					fmt.Printf("Could not find executable at %v\n", path)
					path = executable
				}
			}
		}
	}
	return path, nil
}

func findExecutable(file string) error {
	d, err := os.Stat(file)
	if err != nil {
		return err
	}
	if m := d.Mode(); !m.IsDir() {
		return nil
	}
	return os.ErrPermission
}

func fileNotExist(path string) bool {
	return findExecutable(path) != nil
}

func getFabric8BinLocation() string {
	home := homedir.HomeDir()
	if home == "" {
		fmt.Printf("No user home environment variable found for OS %s\n", runtime.GOOS)
	}
	return filepath.Join(home, ".fabric8", "bin")
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/golang/glog"
	"k8s.io/minikube/pkg/drivers/kic/oci"
	"k8s.io/minikube/pkg/minikube/localpath"
	"k8s.io/minikube/pkg/util/lock"
)

var keywords = []string{"start", "stop", "status", "delete", "config", "open", "profile", "addons", "cache", "logs"}

// IsValid checks if the profile has the essential info needed for a profile
func (p *Profile) IsValid() bool {
	if p.Config == nil {
		return false
	}
	if p.Config == nil {
		return false
	}
	if p.Config.VMDriver == "" {
		return false
	}
	for _, n := range p.Config.Nodes {
		if n.KubernetesVersion == "" {
			return false
		}
	}
	return true
}

// PrimaryControlPlane gets the node specific config for the first created control plane
func PrimaryControlPlane(cc MachineConfig) (Node, error) {
	for _, n := range cc.Nodes {
		if n.ControlPlane {
			return n, nil
		}
	}

	return Node{}, errors.New("could not find master node")
}

// ProfileNameInReservedKeywords checks if the profile is an internal keywords
func ProfileNameInReservedKeywords(name string) bool {
	for _, v := range keywords {
		if strings.EqualFold(v, name) {
			return true
		}
	}
	return false
}

// ProfileExists returns true if there is a profile config (regardless of being valid)
func ProfileExists(name string, miniHome ...string) bool {
	miniPath := localpath.MiniPath()
	if len(miniHome) > 0 {
		miniPath = miniHome[0]
	}

	p := profileFilePath(name, miniPath)
	_, err := os.Stat(p)
	return err == nil
}

// CreateEmptyProfile creates an empty profile and stores in $MINIKUBE_HOME/profiles/<profilename>/config.json
func CreateEmptyProfile(name string, miniHome ...string) error {
	cfg := &MachineConfig{}
	return SaveProfile(name, cfg, miniHome...)
}

// SaveProfile creates an profile out of the cfg and stores in $MINIKUBE_HOME/profiles/<profilename>/config.json
func SaveProfile(name string, cfg *MachineConfig, miniHome ...string) error {
	data, err := json.MarshalIndent(cfg, "", "    ")
	if err != nil {
		return err
	}
	path := profileFilePath(name, miniHome...)
	glog.Infof("Saving config to %s ...", path)
	if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
		return err
	}

	// If no config file exists, don't worry about swapping paths
	if _, err := os.Stat(path); os.IsNotExist(err) {
		if err := lock.WriteFile(path, data, 0600); err != nil {
			return err
		}
		return nil
	}

	tf, err := ioutil.TempFile(filepath.Dir(path), "config.json.tmp")
	if err != nil {
		return err
	}
	defer os.Remove(tf.Name())

	if err = ioutil.WriteFile(tf.Name(), data, 0600); err != nil {
		return err
	}

	if err = tf.Close(); err != nil {
		return err
	}

	if err = os.Remove(path); err != nil {
		return err
	}

	if err = os.Rename(tf.Name(), path); err != nil {
		return err
	}
	return nil
}

// DeleteProfile deletes a profile and removes the profile dir
func DeleteProfile(profile string, miniHome ...string) error {
	miniPath := localpath.MiniPath()
	if len(miniHome) > 0 {
		miniPath = miniHome[0]
	}
	return os.RemoveAll(ProfileFolderPath(profile, miniPath))
}

// ListProfiles returns all valid and invalid (if any) minikube profiles
// invalidPs are the profiles that have a directory or config file but not usable
// invalidPs would be suggested to be deleted
func ListProfiles(miniHome ...string) (validPs []*Profile, inValidPs []*Profile, err error) {

	// try to get profiles list based on left over evidences such as directory
	pDirs, err := profileDirs(miniHome...)
	if err != nil {
		return nil, nil, err
	}
	// try to get profiles list based on all contrainers created by docker driver
	cs, err := oci.ListOwnedContainers(oci.Docker)
	if err == nil {
		pDirs = append(pDirs, cs...)
	}
	for _, n := range pDirs {
		p, err := LoadProfile(n, miniHome...)
		if err != nil {
			inValidPs = append(inValidPs, p)
			continue
		}
		if !p.IsValid() {
			inValidPs = append(inValidPs, p)
			continue
		}
		validPs = append(validPs, p)
	}
	return validPs, inValidPs, nil
}

// LoadProfile loads type Profile based on its name
func LoadProfile(name string, miniHome ...string) (*Profile, error) {
	cfg, err := DefaultLoader.LoadConfigFromFile(name, miniHome...)
	p := &Profile{
		Name:   name,
		Config: cfg,
	}
	return p, err
}

// profileDirs gets all the folders in the user's profiles folder regardless of valid or invalid config
func profileDirs(miniHome ...string) (dirs []string, err error) {
	miniPath := localpath.MiniPath()
	if len(miniHome) > 0 {
		miniPath = miniHome[0]
	}
	pRootDir := filepath.Join(miniPath, "profiles")
	items, err := ioutil.ReadDir(pRootDir)
	for _, f := range items {
		if f.IsDir() {
			dirs = append(dirs, f.Name())
		}
	}
	return dirs, err
}

// profileFilePath returns path of profile config file
func profileFilePath(profile string, miniHome ...string) string {
	miniPath := localpath.MiniPath()
	if len(miniHome) > 0 {
		miniPath = miniHome[0]
	}

	return filepath.Join(miniPath, "profiles", profile, "config.json")
}

// ProfileFolderPath returns path of profile folder
func ProfileFolderPath(profile string, miniHome ...string) string {
	miniPath := localpath.MiniPath()
	if len(miniHome) > 0 {
		miniPath = miniHome[0]
	}
	return filepath.Join(miniPath, "profiles", profile)
}
<commit_msg>remove duplicate profile names<commit_after>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/golang/glog"
	"k8s.io/minikube/pkg/drivers/kic/oci"
	"k8s.io/minikube/pkg/minikube/localpath"
	"k8s.io/minikube/pkg/util/lock"
)

var keywords = []string{"start", "stop", "status", "delete", "config", "open", "profile", "addons", "cache", "logs"}

// IsValid checks if the profile has the essential info needed for a profile
func (p *Profile) IsValid() bool {
	if p.Config == nil {
		return false
	}
	if p.Config == nil {
		return false
	}
	if p.Config.VMDriver == "" {
		return false
	}
	for _, n := range p.Config.Nodes {
		if n.KubernetesVersion == "" {
			return false
		}
	}
	return true
}

// PrimaryControlPlane gets the node specific config for the first created control plane
func PrimaryControlPlane(cc MachineConfig) (Node, error) {
	for _, n := range cc.Nodes {
		if n.ControlPlane {
			return n, nil
		}
	}

	return Node{}, errors.New("could not find master node")
}

// ProfileNameInReservedKeywords checks if the profile is an internal keywords
func ProfileNameInReservedKeywords(name string) bool {
	for _, v := range keywords {
		if strings.EqualFold(v, name) {
			return true
		}
	}
	return false
}

// ProfileExists returns true if there is a profile config (regardless of being valid)
func ProfileExists(name string, miniHome ...string) bool {
	miniPath := localpath.MiniPath()
	if len(miniHome) > 0 {
		miniPath = miniHome[0]
	}

	p := profileFilePath(name, miniPath)
	_, err := os.Stat(p)
	return err == nil
}

// CreateEmptyProfile creates an empty profile and stores in $MINIKUBE_HOME/profiles/<profilename>/config.json
func CreateEmptyProfile(name string, miniHome ...string) error {
	cfg := &MachineConfig{}
	return SaveProfile(name, cfg, miniHome...)
}

// SaveProfile creates an profile out of the cfg and stores in $MINIKUBE_HOME/profiles/<profilename>/config.json
func SaveProfile(name string, cfg *MachineConfig, miniHome ...string) error {
	data, err := json.MarshalIndent(cfg, "", "    ")
	if err != nil {
		return err
	}
	path := profileFilePath(name, miniHome...)
	glog.Infof("Saving config to %s ...", path)
	if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
		return err
	}

	// If no config file exists, don't worry about swapping paths
	if _, err := os.Stat(path); os.IsNotExist(err) {
		if err := lock.WriteFile(path, data, 0600); err != nil {
			return err
		}
		return nil
	}

	tf, err := ioutil.TempFile(filepath.Dir(path), "config.json.tmp")
	if err != nil {
		return err
	}
	defer os.Remove(tf.Name())

	if err = ioutil.WriteFile(tf.Name(), data, 0600); err != nil {
		return err
	}

	if err = tf.Close(); err != nil {
		return err
	}

	if err = os.Remove(path); err != nil {
		return err
	}

	if err = os.Rename(tf.Name(), path); err != nil {
		return err
	}
	return nil
}

// DeleteProfile deletes a profile and removes the profile dir
func DeleteProfile(profile string, miniHome ...string) error {
	miniPath := localpath.MiniPath()
	if len(miniHome) > 0 {
		miniPath = miniHome[0]
	}
	return os.RemoveAll(ProfileFolderPath(profile, miniPath))
}

// ListProfiles returns all valid and invalid (if any) minikube profiles
// invalidPs are the profiles that have a directory or config file but not usable
// invalidPs would be suggested to be deleted
func ListProfiles(miniHome ...string) (validPs []*Profile, inValidPs []*Profile, err error) {

	// try to get profiles list based on left over evidences such as directory
	pDirs, err := profileDirs(miniHome...)
	if err != nil {
		return nil, nil, err
	}
	// try to get profiles list based on all contrainers created by docker driver
	cs, err := oci.ListOwnedContainers(oci.Docker)
	if err == nil {
		pDirs = append(pDirs, cs...)
	}
	pDirs = removeDups(pDirs)
	for _, n := range pDirs {
		p, err := LoadProfile(n, miniHome...)
		if err != nil {
			inValidPs = append(inValidPs, p)
			continue
		}
		if !p.IsValid() {
			inValidPs = append(inValidPs, p)
			continue
		}
		validPs = append(validPs, p)
	}
	return validPs, inValidPs, nil
}

// removeDups removes duplicates
func removeDups(profiles []string) []string {
	// Use map to record duplicates as we find them.
	seen := map[string]bool{}
	result := []string{}

	for n := range profiles {
		if seen[profiles[n]] {
			// Do not add duplicate.
		} else {
			// Record this element as an encountered element.
			seen[profiles[n]] = true
			// Append to result slice.
			result = append(result, profiles[n])
		}
	}
	// Return the new slice.
	return result
}

// LoadProfile loads type Profile based on its name
func LoadProfile(name string, miniHome ...string) (*Profile, error) {
	cfg, err := DefaultLoader.LoadConfigFromFile(name, miniHome...)
	p := &Profile{
		Name:   name,
		Config: cfg,
	}
	return p, err
}

// profileDirs gets all the folders in the user's profiles folder regardless of valid or invalid config
func profileDirs(miniHome ...string) (dirs []string, err error) {
	miniPath := localpath.MiniPath()
	if len(miniHome) > 0 {
		miniPath = miniHome[0]
	}
	pRootDir := filepath.Join(miniPath, "profiles")
	items, err := ioutil.ReadDir(pRootDir)
	for _, f := range items {
		if f.IsDir() {
			dirs = append(dirs, f.Name())
		}
	}
	return dirs, err
}

// profileFilePath returns path of profile config file
func profileFilePath(profile string, miniHome ...string) string {
	miniPath := localpath.MiniPath()
	if len(miniHome) > 0 {
		miniPath = miniHome[0]
	}

	return filepath.Join(miniPath, "profiles", profile, "config.json")
}

// ProfileFolderPath returns path of profile folder
func ProfileFolderPath(profile string, miniHome ...string) string {
	miniPath := localpath.MiniPath()
	if len(miniHome) > 0 {
		miniPath = miniHome[0]
	}
	return filepath.Join(miniPath, "profiles", profile)
}
<|endoftext|>


<commit_before>package scwversion

import (
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestInit(t *testing.T) {
	Convey("Testing init()", t, func() {
		So(VERSION, ShouldNotEqual, "")
		So(GITCOMMIT, ShouldNotEqual, "")
	})
}
<commit_msg>Disable version test for GITCOMMIT<commit_after>package scwversion

import (
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestInit(t *testing.T) {
	Convey("Testing init()", t, func() {
		So(VERSION, ShouldNotEqual, "")
		// So(GITCOMMIT, ShouldNotEqual, "")
	})
}
<|endoftext|>


<commit_before>package sqlgen

import (
	"strings"

	"github.com/rbastic/dyndao/schema"
)

func (s *SQLBuilder) GetUUID() string {
	if s.IsMYSQL {
		return "uuid()"
	} else if s.IsORACLE {
		return "SYS_GUID()"
	} else if s.IsPOSTGRES {
		return "gen_random_uuid()"
	} else {
		panic("TODO - Unimplemented")
	}
}

// ColumnsAliasedJoin is currently used by orm.CrossJoin(). Given a slice
// of column names and a prefix, it will use the prefix to reference the
// column and alias the column as {PREFIX}_{COLUMNNAME}. It returns two
// slices, one contains the SELECT columns (Prefix.Name as PrefixedName),
// and the other contains just the column names.
func (s *SQLBuilder) ColumnsAliasedJoin(cols []string, prefix string) ([]string, []string) {
	newCols := make([]string, len(cols))
	aliasCols := make([]string, len(cols))
	for i, v := range cols {
		suffixSTR := strings.ToUpper(prefix) + "_" + v
		newCols[i] = prefix + "." + v + " AS " + suffixSTR
		aliasCols[i] = suffixSTR
	}
	return newCols, aliasCols
}

func (s *SQLBuilder) ColumnsAliased(cols []string, prefix string) []string {
	newCols := make([]string, len(cols))
	for i, v := range cols {
		newCols[i] = prefix + "." + v
	}
	return newCols
}

func (s *SQLBuilder) JSONColumns(schTbl *schema.Table, tblPrefix string, jsonPathAliases map[string]string) ([]string, []string, error) {
	return s.MapJSONPaths(s, schTbl, tblPrefix, jsonPathAliases)
}

func (s *SQLBuilder) JSONWhere(schTbl *schema.Table, tblPrefix string, jsonPathAlias string) string {
	return s.MapJSONWhere(s, schTbl, tblPrefix, jsonPathAlias)
}
<commit_msg>IsROACH gen_random_uuid()<commit_after>package sqlgen

import (
	"strings"

	"github.com/rbastic/dyndao/schema"
)

func (s *SQLBuilder) GetUUID() string {
	if s.IsMYSQL {
		return "uuid()"
	} else if s.IsORACLE {
		return "SYS_GUID()"
	} else if s.IsPOSTGRES {
		return "gen_random_uuid()"
	} else if s.IsROACH {
		return "gen_random_uuid()"
	} else {
		panic("TODO - Unimplemented")
	}
}

// ColumnsAliasedJoin is currently used by orm.CrossJoin(). Given a slice
// of column names and a prefix, it will use the prefix to reference the
// column and alias the column as {PREFIX}_{COLUMNNAME}. It returns two
// slices, one contains the SELECT columns (Prefix.Name as PrefixedName),
// and the other contains just the column names.
func (s *SQLBuilder) ColumnsAliasedJoin(cols []string, prefix string) ([]string, []string) {
	newCols := make([]string, len(cols))
	aliasCols := make([]string, len(cols))
	for i, v := range cols {
		suffixSTR := strings.ToUpper(prefix) + "_" + v
		newCols[i] = prefix + "." + v + " AS " + suffixSTR
		aliasCols[i] = suffixSTR
	}
	return newCols, aliasCols
}

func (s *SQLBuilder) ColumnsAliased(cols []string, prefix string) []string {
	newCols := make([]string, len(cols))
	for i, v := range cols {
		newCols[i] = prefix + "." + v
	}
	return newCols
}

func (s *SQLBuilder) JSONColumns(schTbl *schema.Table, tblPrefix string, jsonPathAliases map[string]string) ([]string, []string, error) {
	return s.MapJSONPaths(s, schTbl, tblPrefix, jsonPathAliases)
}

func (s *SQLBuilder) JSONWhere(schTbl *schema.Table, tblPrefix string, jsonPathAlias string) string {
	return s.MapJSONWhere(s, schTbl, tblPrefix, jsonPathAlias)
}
<|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package models

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"time"

	"github.com/gogits/gogs/modules/httplib"
	"github.com/gogits/gogs/modules/log"
	"github.com/gogits/gogs/modules/setting"
	"github.com/gogits/gogs/modules/uuid"
)

var (
	ErrWebhookNotExist = errors.New("Webhook does not exist")
)

type HookContentType int

const (
	JSON HookContentType = iota + 1
	FORM
)

// HookEvent represents events that will delivery hook.
type HookEvent struct {
	PushOnly bool `json:"push_only"`
}

// Webhook represents a web hook object.
type Webhook struct {
	Id           int64
	RepoId       int64
	Url          string `xorm:"TEXT"`
	ContentType  HookContentType
	Secret       string `xorm:"TEXT"`
	Events       string `xorm:"TEXT"`
	*HookEvent   `xorm:"-"`
	IsSsl        bool
	IsActive     bool
	HookTaskType HookTaskType
	Meta         string `xorm:"TEXT"` // store hook-specific attributes
	OrgId        int64
}

// GetEvent handles conversion from Events to HookEvent.
func (w *Webhook) GetEvent() {
	w.HookEvent = &HookEvent{}
	if err := json.Unmarshal([]byte(w.Events), w.HookEvent); err != nil {
		log.Error(4, "webhook.GetEvent(%d): %v", w.Id, err)
	}
}

func (w *Webhook) GetSlackHook() *Slack {
	s := &Slack{}
	if err := json.Unmarshal([]byte(w.Meta), s); err != nil {
		log.Error(4, "webhook.GetSlackHook(%d): %v", w.Id, err)
	}
	return s
}

// UpdateEvent handles conversion from HookEvent to Events.
func (w *Webhook) UpdateEvent() error {
	data, err := json.Marshal(w.HookEvent)
	w.Events = string(data)
	return err
}

// HasPushEvent returns true if hook enbaled push event.
func (w *Webhook) HasPushEvent() bool {
	if w.PushOnly {
		return true
	}
	return false
}

// CreateWebhook creates a new web hook.
func CreateWebhook(w *Webhook) error {
	_, err := x.Insert(w)
	return err
}

// GetWebhookById returns webhook by given ID.
func GetWebhookById(hookId int64) (*Webhook, error) {
	w := &Webhook{Id: hookId}
	has, err := x.Get(w)
	if err != nil {
		return nil, err
	} else if !has {
		return nil, ErrWebhookNotExist
	}
	return w, nil
}

// GetActiveWebhooksByRepoId returns all active webhooks of repository.
func GetActiveWebhooksByRepoId(repoId int64) (ws []*Webhook, err error) {
	err = x.Find(&ws, &Webhook{RepoId: repoId, IsActive: true})
	return ws, err
}

// GetWebhooksByRepoId returns all webhooks of repository.
func GetWebhooksByRepoId(repoId int64) (ws []*Webhook, err error) {
	err = x.Find(&ws, &Webhook{RepoId: repoId})
	return ws, err
}

// UpdateWebhook updates information of webhook.
func UpdateWebhook(w *Webhook) error {
	_, err := x.Id(w.Id).AllCols().Update(w)
	return err
}

// DeleteWebhook deletes webhook of repository.
func DeleteWebhook(hookId int64) error {
	_, err := x.Delete(&Webhook{Id: hookId})
	return err
}

// GetWebhooksByOrgId returns all webhooks for an organization.
func GetWebhooksByOrgId(orgId int64) (ws []*Webhook, err error) {
	err = x.Find(&ws, &Webhook{OrgId: orgId})
	return ws, err
}

// GetActiveWebhooksByOrgId returns all active webhooks for an organization.
func GetActiveWebhooksByOrgId(orgId int64) (ws []*Webhook, err error) {
	err = x.Find(&ws, &Webhook{OrgId: orgId, IsActive: true})
	return ws, err
}

//   ___ ___                __   ___________              __
//  /   |   \  ____   ____ |  | _\__    ___/____    _____|  | __
// /    ~    \/  _ \ /  _ \|  |/ / |    |  \__  \  /  ___/  |/ /
// \    Y    (  <_> |  <_> )    <  |    |   / __ \_\___ \|    <
//  \___|_  / \____/ \____/|__|_ \ |____|  (____  /____  >__|_ \
//        \/                    \/              \/     \/     \/

type HookTaskType int

const (
	GOGS HookTaskType = iota + 1
	SLACK
)

type HookEventType string

const (
	PUSH HookEventType = "push"
)

type PayloadAuthor struct {
	Name  string `json:"name"`
	Email string `json:"email"`
}

type PayloadCommit struct {
	Id      string         `json:"id"`
	Message string         `json:"message"`
	Url     string         `json:"url"`
	Author  *PayloadAuthor `json:"author"`
}

type PayloadRepo struct {
	Id          int64          `json:"id"`
	Name        string         `json:"name"`
	Url         string         `json:"url"`
	Description string         `json:"description"`
	Website     string         `json:"website"`
	Watchers    int            `json:"watchers"`
	Owner       *PayloadAuthor `json:"author"`
	Private     bool           `json:"private"`
}

type BasePayload interface {
	GetJSONPayload() ([]byte, error)
}

// Payload represents a payload information of hook.
type Payload struct {
	Secret     string           `json:"secret"`
	Ref        string           `json:"ref"`
	Commits    []*PayloadCommit `json:"commits"`
	Repo       *PayloadRepo     `json:"repository"`
	Pusher     *PayloadAuthor   `json:"pusher"`
	Before     string           `json:"before"`
	After      string           `json:"after"`
	CompareUrl string           `json:"compare_url"`
}

func (p Payload) GetJSONPayload() ([]byte, error) {
	data, err := json.Marshal(p)
	if err != nil {
		return []byte{}, err
	}
	return data, nil
}

// HookTask represents a hook task.
type HookTask struct {
	Id             int64
	Uuid           string
	Type           HookTaskType
	Url            string
	BasePayload    `xorm:"-"`
	PayloadContent string `xorm:"TEXT"`
	ContentType    HookContentType
	EventType      HookEventType
	IsSsl          bool
	IsDelivered    bool
	IsSucceed      bool
}

// CreateHookTask creates a new hook task,
// it handles conversion from Payload to PayloadContent.
func CreateHookTask(t *HookTask) error {
	data, err := t.BasePayload.GetJSONPayload()
	if err != nil {
		return err
	}
	t.Uuid = uuid.NewV4().String()
	t.PayloadContent = string(data)
	_, err = x.Insert(t)
	return err
}

// UpdateHookTask updates information of hook task.
func UpdateHookTask(t *HookTask) error {
	_, err := x.AllCols().Update(t)
	return err
}

// DeliverHooks checks and delivers undelivered hooks.
func DeliverHooks() {
	timeout := time.Duration(setting.WebhookDeliverTimeout) * time.Second
	x.Where("is_delivered=?", false).Iterate(new(HookTask),
		func(idx int, bean interface{}) error {
			t := bean.(*HookTask)
			req := httplib.Post(t.Url).SetTimeout(timeout, timeout).
				Header("X-Gogs-Delivery", t.Uuid).
				Header("X-Gogs-Event", string(t.EventType))

			switch t.ContentType {
			case JSON:
				req = req.Header("Content-Type", "application/json").Body(t.PayloadContent)
			case FORM:
				req.Param("payload", t.PayloadContent)
			}

			t.IsDelivered = true

			// TODO: record response.
			switch t.Type {
			case GOGS:
				{
					if _, err := req.Response(); err != nil {
						log.Error(4, "Delivery: %v", err)
					} else {
						t.IsSucceed = true
					}
				}
			case SLACK:
				{
					if res, err := req.Response(); err != nil {
						log.Error(4, "Delivery: %v", err)
					} else {
						defer res.Body.Close()
						contents, err := ioutil.ReadAll(res.Body)
						if err != nil {
							log.Error(4, "%s", err)
						} else {
							if string(contents) != "ok" {
								log.Error(4, "slack failed with: %s", string(contents))
							} else {
								t.IsSucceed = true
							}
						}
					}
				}
			}

			if err := UpdateHookTask(t); err != nil {
				log.Error(4, "UpdateHookTask: %v", err)
				return nil
			}

			log.Trace("Hook delivered(%s): %s", t.Uuid, t.PayloadContent)
			return nil
		})
}
<commit_msg>update specific hook_task, not all<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package models

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"time"

	"github.com/gogits/gogs/modules/httplib"
	"github.com/gogits/gogs/modules/log"
	"github.com/gogits/gogs/modules/setting"
	"github.com/gogits/gogs/modules/uuid"
)

var (
	ErrWebhookNotExist = errors.New("Webhook does not exist")
)

type HookContentType int

const (
	JSON HookContentType = iota + 1
	FORM
)

// HookEvent represents events that will delivery hook.
type HookEvent struct {
	PushOnly bool `json:"push_only"`
}

// Webhook represents a web hook object.
type Webhook struct {
	Id           int64
	RepoId       int64
	Url          string `xorm:"TEXT"`
	ContentType  HookContentType
	Secret       string `xorm:"TEXT"`
	Events       string `xorm:"TEXT"`
	*HookEvent   `xorm:"-"`
	IsSsl        bool
	IsActive     bool
	HookTaskType HookTaskType
	Meta         string `xorm:"TEXT"` // store hook-specific attributes
	OrgId        int64
}

// GetEvent handles conversion from Events to HookEvent.
func (w *Webhook) GetEvent() {
	w.HookEvent = &HookEvent{}
	if err := json.Unmarshal([]byte(w.Events), w.HookEvent); err != nil {
		log.Error(4, "webhook.GetEvent(%d): %v", w.Id, err)
	}
}

func (w *Webhook) GetSlackHook() *Slack {
	s := &Slack{}
	if err := json.Unmarshal([]byte(w.Meta), s); err != nil {
		log.Error(4, "webhook.GetSlackHook(%d): %v", w.Id, err)
	}
	return s
}

// UpdateEvent handles conversion from HookEvent to Events.
func (w *Webhook) UpdateEvent() error {
	data, err := json.Marshal(w.HookEvent)
	w.Events = string(data)
	return err
}

// HasPushEvent returns true if hook enbaled push event.
func (w *Webhook) HasPushEvent() bool {
	if w.PushOnly {
		return true
	}
	return false
}

// CreateWebhook creates a new web hook.
func CreateWebhook(w *Webhook) error {
	_, err := x.Insert(w)
	return err
}

// GetWebhookById returns webhook by given ID.
func GetWebhookById(hookId int64) (*Webhook, error) {
	w := &Webhook{Id: hookId}
	has, err := x.Get(w)
	if err != nil {
		return nil, err
	} else if !has {
		return nil, ErrWebhookNotExist
	}
	return w, nil
}

// GetActiveWebhooksByRepoId returns all active webhooks of repository.
func GetActiveWebhooksByRepoId(repoId int64) (ws []*Webhook, err error) {
	err = x.Find(&ws, &Webhook{RepoId: repoId, IsActive: true})
	return ws, err
}

// GetWebhooksByRepoId returns all webhooks of repository.
func GetWebhooksByRepoId(repoId int64) (ws []*Webhook, err error) {
	err = x.Find(&ws, &Webhook{RepoId: repoId})
	return ws, err
}

// UpdateWebhook updates information of webhook.
func UpdateWebhook(w *Webhook) error {
	_, err := x.Id(w.Id).AllCols().Update(w)
	return err
}

// DeleteWebhook deletes webhook of repository.
func DeleteWebhook(hookId int64) error {
	_, err := x.Delete(&Webhook{Id: hookId})
	return err
}

// GetWebhooksByOrgId returns all webhooks for an organization.
func GetWebhooksByOrgId(orgId int64) (ws []*Webhook, err error) {
	err = x.Find(&ws, &Webhook{OrgId: orgId})
	return ws, err
}

// GetActiveWebhooksByOrgId returns all active webhooks for an organization.
func GetActiveWebhooksByOrgId(orgId int64) (ws []*Webhook, err error) {
	err = x.Find(&ws, &Webhook{OrgId: orgId, IsActive: true})
	return ws, err
}

//   ___ ___                __   ___________              __
//  /   |   \  ____   ____ |  | _\__    ___/____    _____|  | __
// /    ~    \/  _ \ /  _ \|  |/ / |    |  \__  \  /  ___/  |/ /
// \    Y    (  <_> |  <_> )    <  |    |   / __ \_\___ \|    <
//  \___|_  / \____/ \____/|__|_ \ |____|  (____  /____  >__|_ \
//        \/                    \/              \/     \/     \/

type HookTaskType int

const (
	GOGS HookTaskType = iota + 1
	SLACK
)

type HookEventType string

const (
	PUSH HookEventType = "push"
)

type PayloadAuthor struct {
	Name  string `json:"name"`
	Email string `json:"email"`
}

type PayloadCommit struct {
	Id      string         `json:"id"`
	Message string         `json:"message"`
	Url     string         `json:"url"`
	Author  *PayloadAuthor `json:"author"`
}

type PayloadRepo struct {
	Id          int64          `json:"id"`
	Name        string         `json:"name"`
	Url         string         `json:"url"`
	Description string         `json:"description"`
	Website     string         `json:"website"`
	Watchers    int            `json:"watchers"`
	Owner       *PayloadAuthor `json:"author"`
	Private     bool           `json:"private"`
}

type BasePayload interface {
	GetJSONPayload() ([]byte, error)
}

// Payload represents a payload information of hook.
type Payload struct {
	Secret     string           `json:"secret"`
	Ref        string           `json:"ref"`
	Commits    []*PayloadCommit `json:"commits"`
	Repo       *PayloadRepo     `json:"repository"`
	Pusher     *PayloadAuthor   `json:"pusher"`
	Before     string           `json:"before"`
	After      string           `json:"after"`
	CompareUrl string           `json:"compare_url"`
}

func (p Payload) GetJSONPayload() ([]byte, error) {
	data, err := json.Marshal(p)
	if err != nil {
		return []byte{}, err
	}
	return data, nil
}

// HookTask represents a hook task.
type HookTask struct {
	Id             int64
	Uuid           string
	Type           HookTaskType
	Url            string
	BasePayload    `xorm:"-"`
	PayloadContent string `xorm:"TEXT"`
	ContentType    HookContentType
	EventType      HookEventType
	IsSsl          bool
	IsDelivered    bool
	IsSucceed      bool
}

// CreateHookTask creates a new hook task,
// it handles conversion from Payload to PayloadContent.
func CreateHookTask(t *HookTask) error {
	data, err := t.BasePayload.GetJSONPayload()
	if err != nil {
		return err
	}
	t.Uuid = uuid.NewV4().String()
	t.PayloadContent = string(data)
	_, err = x.Insert(t)
	return err
}

// UpdateHookTask updates information of hook task.
func UpdateHookTask(t *HookTask) error {
	_, err := x.Id(t.Id).AllCols().Update(t)
	return err
}

// DeliverHooks checks and delivers undelivered hooks.
func DeliverHooks() {
	timeout := time.Duration(setting.WebhookDeliverTimeout) * time.Second
	x.Where("is_delivered=?", false).Iterate(new(HookTask),
		func(idx int, bean interface{}) error {
			t := bean.(*HookTask)
			req := httplib.Post(t.Url).SetTimeout(timeout, timeout).
				Header("X-Gogs-Delivery", t.Uuid).
				Header("X-Gogs-Event", string(t.EventType))

			switch t.ContentType {
			case JSON:
				req = req.Header("Content-Type", "application/json").Body(t.PayloadContent)
			case FORM:
				req.Param("payload", t.PayloadContent)
			}

			t.IsDelivered = true

			// TODO: record response.
			switch t.Type {
			case GOGS:
				{
					if _, err := req.Response(); err != nil {
						log.Error(4, "Delivery: %v", err)
					} else {
						t.IsSucceed = true
					}
				}
			case SLACK:
				{
					if res, err := req.Response(); err != nil {
						log.Error(4, "Delivery: %v", err)
					} else {
						defer res.Body.Close()
						contents, err := ioutil.ReadAll(res.Body)
						if err != nil {
							log.Error(4, "%s", err)
						} else {
							if string(contents) != "ok" {
								log.Error(4, "slack failed with: %s", string(contents))
							} else {
								t.IsSucceed = true
							}
						}
					}
				}
			}

			if err := UpdateHookTask(t); err != nil {
				log.Error(4, "UpdateHookTask: %v", err)
				return nil
			}

			log.Trace("Hook delivered(%s): %s", t.Uuid, t.PayloadContent)
			return nil
		})
}
<|endoftext|>


<commit_before>package testutil

import "fmt"

// Checks whether two slices contain the same elements.
func ShouldSetEqual(actual interface{}, expected ...interface{}) (msg string) {
	actualSlice, expectedSlice := actual.([]string), expected[0].([]string)

	msg = fmt.Sprintf("Expected:\t%q\nActual:\t%q\n(Should have the same elements)", expectedSlice, actualSlice)

	if len(actualSlice) != len(expectedSlice) {
		return
	}

	actualSet := make(map[string]bool)
	for _, a := range actualSlice {
		actualSet[a] = true
	}

	for _, e := range expectedSlice {
		if actualSet[e] != true {
			return
		}
	}

	expectedSet := make(map[string]bool)
	for _, e := range expectedSlice {
		expectedSet[e] = true
	}

	for _, a := range actualSlice {
		if expectedSet[a] != true {
			return
		}
	}

	return ""
}

func ShouldConsistOf(actual interface{}, expected ...interface{}) (msg string) {
	actualSlice, expectedSlice := actual.([]string), []string{}

	for _, v := range expected {
		expectedSlice = append(expectedSlice, v.(string))
	}

	msg = fmt.Sprintf("Expected:\t%q\nActual:\t%q\n(Should consist of the given elements", expectedSlice, actualSlice)

	if len(actualSlice) != len(expectedSlice) {
		return
	}

	for i, v := range actualSlice {
		if v != expectedSlice[i] {
			return
		}
	}

	return ""
}
<commit_msg>Add ShouldBeSuperSetOf assertion to testutil<commit_after>package testutil

import "fmt"

// Checks whether two slices contain the same elements.
func ShouldSetEqual(actual interface{}, expected ...interface{}) (msg string) {
	actualSlice, expectedSlice := actual.([]string), expected[0].([]string)

	msg = fmt.Sprintf("Expected:\t%q\nActual:\t%q\n(Should have the same elements)", expectedSlice, actualSlice)

	if len(actualSlice) != len(expectedSlice) {
		return
	}

	actualSet := make(map[string]bool)
	for _, a := range actualSlice {
		actualSet[a] = true
	}

	for _, e := range expectedSlice {
		if actualSet[e] != true {
			return
		}
	}

	expectedSet := make(map[string]bool)
	for _, e := range expectedSlice {
		expectedSet[e] = true
	}

	for _, a := range actualSlice {
		if expectedSet[a] != true {
			return
		}
	}

	return ""
}

func ShouldConsistOf(actual interface{}, expected ...interface{}) (msg string) {
	actualSlice, expectedSlice := actual.([]string), []string{}

	for _, v := range expected {
		expectedSlice = append(expectedSlice, v.(string))
	}

	msg = fmt.Sprintf("Expected:\t%q\nActual:\t%q\n(Should consist of the given elements)", expectedSlice, actualSlice)

	if len(actualSlice) != len(expectedSlice) {
		return
	}

	for i, v := range actualSlice {
		if v != expectedSlice[i] {
			return
		}
	}

	return ""
}

func ShouldBeSuperSetOf(super interface{}, sub ...interface{}) (msg string) {
	superSlice, subSlice := super.([]string), sub[0].([]string)

	msg = fmt.Sprintf("SubSet:\t%q\nSuperSet:\t%q\n(Should contain every element of the given slice)", subSlice, superSlice)

	if len(superSlice) < len(subSlice) {
		return
	}

	superSet := make(map[string]bool)
	for _, e := range superSlice {
		superSet[e] = true
	}

	for _, a := range subSlice {
		if _, ok := superSet[a]; ok != true {
			return
		}
	}

	return ""
}
<|endoftext|>