<commit_before>package main

import (
	"code.google.com/p/go.net/websocket"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"queue"
	"runtime"
	"strconv"
	"strings"
	"time"
)

const debug = true
const maxSizeHealthStatusQueue = 1000 // Magnificent Server Log Store Size, displays the last 1000 seconds in the plot

type Msg struct {
	MessageId string
	Content   string
	TimeStamp int64
}

// Client connection consists of the websocket and the client ip
type Client struct {
	websocket *websocket.Conn
	clientIP  string
}

type ServiceMonitor struct {
	errChan           chan error // unbuffered channel
	errChanWebsock    chan error // unbuffered channel
	activeClients     map[string]Client
	healthStatusChan  chan bool
	alertChan         chan string
	newClientChan     chan Client
	alertQueue        *queue.Queue
	healthStatusQueue *queue.Queue
}

func NewServiceMonitor() *ServiceMonitor {
	m := ServiceMonitor{}
	m.activeClients = make(map[string]Client)
	m.errChan = make(chan error)
	m.healthStatusChan = make(chan bool, 10)
	m.alertChan = make(chan string, 10)
	m.newClientChan = make(chan Client, 10)
	m.alertQueue = queue.NewQueue()
	m.healthStatusQueue = queue.NewQueue()
	return &m
}

func IntToString(value int) string {
	return strconv.FormatInt(int64(value), 10)
}

func StringToFloat(value string) float64 {
	result, _ := strconv.ParseFloat(value, 64)
	return result
}

func StringToInt(value string) int {
	result, _ := strconv.ParseInt(value, 10, 64)
	return int(result)
}

func FloatToString(value float64) string {
	return strconv.FormatFloat(value, 'f', -1, 64)
}

func BoolToString(value bool) string {
	if value {
		return "1"
	}
	return "0"
}

func (m *ServiceMonitor) sendClientMsg(msg *Msg, ip string) {
	var err error
	var Message = websocket.JSON

	if err = Message.Send(m.activeClients[ip].websocket, msg); err != nil {
		// we could not send the message to a peer
		log.Println("Could not send message to:", ip, err.Error())
		log.Println("Client disconnected:", ip)
		delete(m.activeClients, ip)
	}
}

func (m *ServiceMonitor) sendBroadcastMsg(msg *Msg) {
	var err error
	var Message = websocket.JSON

	for ip, _ := range m.activeClients {
		if err = Message.Send(m.activeClients[ip].websocket, msg); err != nil {
			// we could not send the message to a peer
			log.Println("Could not send message to:", ip, err.Error())
			log.Println("Client disconnected:", ip)
			delete(m.activeClients, ip)
		}
	}
}

func (m *ServiceMonitor) sendQueueData(ip string) {
	for i := 0; i < m.healthStatusQueue.Len(); i++ {
		e, found := m.healthStatusQueue.Get(i)

		if found {
			if msg, ok := e.(*Msg); ok {
				m.sendClientMsg(msg, ip)
			}
		}
	}

	for i := 0; i < m.alertQueue.Len(); i++ {
		e, found := m.alertQueue.Get(i)

		if found {
			if msg, ok := e.(*Msg); ok {
				m.sendClientMsg(msg, ip)
			}
		}
	}
}

// this routine handles all outgoing websocket messages
func (m *ServiceMonitor) pushDataToClients() {
	for {
		select {
		// a new Client is connecting
		case newClient := <-m.newClientChan:
			// send current Queue data to the new connecting client
			m.activeClients[newClient.clientIP] = newClient
			m.sendQueueData(newClient.clientIP)

		// broadcast a new health status message to all clients
		// newHealthStatus == 1 ... deamon failure
		// newHealthStatus == 0 ... deamon ok
		case newHealthStatus := <-m.healthStatusChan:
			msg := Msg{"Plot", BoolToString(newHealthStatus), time.Now().UnixNano() / int64(time.Millisecond)}
			m.sendBroadcastMsg(&msg)
			// add msg to HealthStatusQueue
			if m.healthStatusQueue.Len() < maxSizeHealthStatusQueue {
				m.healthStatusQueue.Push(&msg)
			} else {
				m.healthStatusQueue.Pop()
				m.healthStatusQueue.Push(&msg)
			}

		// broadcast an alert message to all clients
		case newAlert := <-m.alertChan:
			msg := Msg{"Alert", newAlert, time.Now().UnixNano() / int64(time.Millisecond)}
			m.sendBroadcastMsg(&msg)
			// add msg to alertQueue
			m.alertQueue.Push(&msg)
		}
	}
}

// reference: https://github.com/Niessy/websocket-golang-chat
// WebSocket server to handle clients
func (m *ServiceMonitor) WebSocketServer(ws *websocket.Conn) {
	var err error

	// cleanup on server side
	defer func() {
		if err = ws.Close(); err != nil {
			log.Println("Websocket could not be closed", err.Error())
		}
	}()

	client := ws.Request().RemoteAddr
	if debug {
		log.Println("New client connected:", client)
	}

	m.newClientChan <- Client{ws, client}

	// wait for errChan, so the websocket stays open otherwise it'll close
	err = <-m.errChanWebsock
}

// handler for the main page
func HomeHandler(response http.ResponseWriter, request *http.Request) {
	response.Header().Set("Content-type", "text/html")
	webpage, err := ioutil.ReadFile("home.html")

	if err != nil {
		http.Error(response, fmt.Sprintf("home.html file error %v", err), 500)
	}

	fmt.Fprint(response, string(webpage))
}

func (m *ServiceMonitor) parseResponse(resp string) {
	if !strings.Contains(resp, "Magnificent!") {
		m.healthStatusChan <- false
	} else {
		m.healthStatusChan <- true
		m.alertChan <- "Deamon has failed"
	}
}

func (m *ServiceMonitor) requestDeamonStatus(url string) {
	response, err := http.Get(url)
	if err != nil {
		if debug {
			log.Println("requestDeamonStatus failed:", err.Error())
			return
		}
	}

	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		if debug {
			log.Println("requestDeamonStatus failed:", err.Error())
			return
		}
	}

	resp := string(contents)
	m.parseResponse(resp)
}

func (m *ServiceMonitor) monitorDeamon(url string, time_interval time.Duration) {
	for {
		go m.requestDeamonStatus(url)
		time.Sleep(time_interval * time.Second)
	}
}

func (m *ServiceMonitor) startHTTPServer() {
	http.Handle("/", http.HandlerFunc(HomeHandler))
	http.Handle("/sock", websocket.Handler(m.WebSocketServer))

	err := http.ListenAndServe(":8080", nil)
	m.errChanWebsock <- err
	m.errChan <- err
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	m := NewServiceMonitor()

	go m.startHTTPServer()
	go m.pushDataToClients()
	go m.monitorDeamon("http://localhost:12345/", 1)

	err := <-m.errChan
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}
}
<commit_msg>updated monitor<commit_after>package main

import (
	"code.google.com/p/go.net/websocket"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"queue"
	"runtime"
	"strconv"
	"strings"
	"time"
)

const debug = true
const maxSizeHealthStatusQueue = 600 // Magnificent Server Log Store Size, displays the last 600 seconds in the plot

type Msg struct {
	MessageId string
	Content   string
	TimeStamp int64
}

// Client connection consists of the websocket and the client ip
type Client struct {
	websocket *websocket.Conn
	clientIP  string
}

type ServiceMonitor struct {
	errChan           chan error // unbuffered channel
	errChanWebsock    chan error // unbuffered channel
	activeClients     map[string]Client
	healthStatusChan  chan bool
	alertChan         chan string
	newClientChan     chan Client
	alertQueue        *queue.Queue
	healthStatusQueue *queue.Queue
}

func NewServiceMonitor() *ServiceMonitor {
	m := ServiceMonitor{}
	m.activeClients = make(map[string]Client)
	m.errChan = make(chan error)
	m.healthStatusChan = make(chan bool, 10)
	m.alertChan = make(chan string, 10)
	m.newClientChan = make(chan Client, 10)
	m.alertQueue = queue.NewQueue()
	m.healthStatusQueue = queue.NewQueue()
	return &m
}

func IntToString(value int) string {
	return strconv.FormatInt(int64(value), 10)
}

func StringToFloat(value string) float64 {
	result, _ := strconv.ParseFloat(value, 64)
	return result
}

func StringToInt(value string) int {
	result, _ := strconv.ParseInt(value, 10, 64)
	return int(result)
}

func FloatToString(value float64) string {
	return strconv.FormatFloat(value, 'f', -1, 64)
}

func BoolToString(value bool) string {
	if value {
		return "1"
	}
	return "0"
}

func (m *ServiceMonitor) sendClientMsg(msg *Msg, ip string) {
	var err error
	var Message = websocket.JSON

	if err = Message.Send(m.activeClients[ip].websocket, msg); err != nil {
		// we could not send the message to a peer
		log.Println("Could not send message to:", ip, err.Error())
		log.Println("Client disconnected:", ip)
		delete(m.activeClients, ip)
	}
}

func (m *ServiceMonitor) sendBroadcastMsg(msg *Msg) {
	var err error
	var Message = websocket.JSON

	for ip, _ := range m.activeClients {
		if err = Message.Send(m.activeClients[ip].websocket, msg); err != nil {
			// we could not send the message to a peer
			log.Println("Could not send message to:", ip, err.Error())
			log.Println("Client disconnected:", ip)
			delete(m.activeClients, ip)
		}
	}
}

func (m *ServiceMonitor) sendQueueData(ip string) {
	for i := 0; i < m.healthStatusQueue.Len(); i++ {
		e, found := m.healthStatusQueue.Get(i)

		if found {
			if msg, ok := e.(*Msg); ok {
				m.sendClientMsg(msg, ip)
			}
		}
	}

	for i := 0; i < m.alertQueue.Len(); i++ {
		e, found := m.alertQueue.Get(i)

		if found {
			if msg, ok := e.(*Msg); ok {
				m.sendClientMsg(msg, ip)
			}
		}
	}
}

// this routine handles all outgoing websocket messages
func (m *ServiceMonitor) pushDataToClients() {
	for {
		select {
		// a new Client is connecting
		case newClient := <-m.newClientChan:
			// send current Queue data to the new connecting client
			m.activeClients[newClient.clientIP] = newClient
			m.sendQueueData(newClient.clientIP)

		// broadcast a new health status message to all clients
		// newHealthStatus == 1 ... deamon failure
		// newHealthStatus == 0 ... deamon ok
		case newHealthStatus := <-m.healthStatusChan:
			msg := Msg{"Plot", BoolToString(newHealthStatus), time.Now().UnixNano() / int64(time.Millisecond)}
			m.sendBroadcastMsg(&msg)
			// add msg to HealthStatusQueue
			if m.healthStatusQueue.Len() < maxSizeHealthStatusQueue {
				m.healthStatusQueue.Push(&msg)
			} else {
				m.healthStatusQueue.Pop()
				m.healthStatusQueue.Push(&msg)
			}

		// broadcast an alert message to all clients
		case newAlert := <-m.alertChan:
			msg := Msg{"Alert", newAlert, time.Now().UnixNano() / int64(time.Millisecond)}
			m.sendBroadcastMsg(&msg)
			// add msg to alertQueue
			m.alertQueue.Push(&msg)
		}
	}
}

// reference: https://github.com/Niessy/websocket-golang-chat
// WebSocket server to handle clients
func (m *ServiceMonitor) WebSocketServer(ws *websocket.Conn) {
	var err error

	// cleanup on server side
	defer func() {
		if err = ws.Close(); err != nil {
			log.Println("Websocket could not be closed", err.Error())
		}
	}()

	client := ws.Request().RemoteAddr
	if debug {
		log.Println("New client connected:", client)
	}

	m.newClientChan <- Client{ws, client}

	// wait for errChan, so the websocket stays open otherwise it'll close
	err = <-m.errChanWebsock
}

// handler for the main page
func HomeHandler(response http.ResponseWriter, request *http.Request) {
	response.Header().Set("Content-type", "text/html")
	webpage, err := ioutil.ReadFile("home.html")

	if err != nil {
		http.Error(response, fmt.Sprintf("home.html file error %v", err), 500)
	}

	fmt.Fprint(response, string(webpage))
}

func (m *ServiceMonitor) parseResponse(resp string) {
	if !strings.Contains(resp, "Magnificent!") {
		m.healthStatusChan <- false
	} else {
		m.healthStatusChan <- true
		m.alertChan <- "Deamon has failed"
	}
}

func (m *ServiceMonitor) requestDeamonStatus(url string) {
	response, err := http.Get(url)
	if err != nil {
		if debug {
			log.Println("requestDeamonStatus failed:", err.Error())
			return
		}
	}

	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		if debug {
			log.Println("requestDeamonStatus failed:", err.Error())
			return
		}
	}

	resp := string(contents)
	m.parseResponse(resp)
}

func (m *ServiceMonitor) monitorDeamon(url string, time_interval time.Duration) {
	for {
		go m.requestDeamonStatus(url)
		time.Sleep(time_interval * time.Second)
	}
}

func (m *ServiceMonitor) startHTTPServer() {
	http.Handle("/", http.HandlerFunc(HomeHandler))
	http.Handle("/sock", websocket.Handler(m.WebSocketServer))

	err := http.ListenAndServe(":8080", nil)
	m.errChanWebsock <- err
	m.errChan <- err
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	m := NewServiceMonitor()

	go m.startHTTPServer()
	go m.pushDataToClients()
	go m.monitorDeamon("http://localhost:12345/", 1)

	err := <-m.errChan
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// +build performance

/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package performance

import (
	"context"
	"fmt"
	"time"

	pkgTest "github.com/knative/pkg/test"
	"github.com/knative/pkg/test/logging"
	"github.com/knative/serving/test"
	"github.com/knative/test-infra/shared/prometheus"
	"github.com/knative/test-infra/shared/testgrid"
	"istio.io/fortio/fhttp"
	"istio.io/fortio/periodic"

	// Mysteriously required to support GCP auth (required by k8s libs). Apparently just importing it is enough. @_@ side effects @_@. https://github.com/kubernetes/client-go/issues/242
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
)

const (
	istioNS      = "istio-system"
	monitoringNS = "knative-monitoring"
	gateway      = "istio-ingressgateway"
	// Property name used by testgrid.
	perfLatency = "perf_latency"
	duration    = 1 * time.Minute
)

type PerformanceClient struct {
	E2EClients *test.Clients
	PromClient *prometheus.PromProxy
}

// Setup creates all the clients that we need to interact with in our tests
func Setup(ctx context.Context, logger *logging.BaseLogger, promReqd bool) (*PerformanceClient, error) {
	clients, err := test.NewClients(pkgTest.Flags.Kubeconfig, pkgTest.Flags.Cluster, test.ServingNamespace)
	if err != nil {
		return nil, err
	}

	var p *prometheus.PromProxy
	if promReqd {
		logger.Infof("Creating prometheus proxy client")
		p = &prometheus.PromProxy{Namespace: monitoringNS}
		p.Setup(ctx, logger)
	}
	return &PerformanceClient{E2EClients: clients, PromClient: p}, nil
}

// Teardown cleans up resources used
func TearDown(client *PerformanceClient, logger *logging.BaseLogger, names test.ResourceNames) {
	if client.E2EClients != nil && client.E2EClients.ServingClient != nil {
		client.E2EClients.ServingClient.Delete([]string{names.Route}, []string{names.Config}, []string{names.Service})
	}

	if client.PromClient != nil {
		client.PromClient.Teardown(logger)
	}
}

// RunLoadTest runs the load test with fortio and returns the response
func RunLoadTest(duration time.Duration, nThreads, nConnections int, url, domain string) (*fhttp.HTTPRunnerResults, error) {
	o := fhttp.NewHTTPOptions(url)
	o.NumConnections = nConnections
	o.AddAndValidateExtraHeader(fmt.Sprintf("Host: %s", domain))

	opts := fhttp.HTTPRunnerOptions{
		RunnerOptions: periodic.RunnerOptions{
			Duration:    duration,
			NumThreads:  nThreads,
			Percentiles: []float64{50.0, 90.0, 99.0},
		},
		HTTPOptions:        *o,
		AllowInitialErrors: true,
	}

	return fhttp.RunHTTPTest(&opts)
}

// CreatePerfTestCase creates a perf test case with the provided name and value
func CreatePerfTestCase(tcVal float32, tcName, tName string) testgrid.TestCase {
	tp := []testgrid.TestProperty{{Name: perfLatency, Value: tcVal}}
	tc := testgrid.TestCase{
		ClassName:  tName,
		Name:       fmt.Sprintf("%s/%s", tName, tcName),
		Properties: testgrid.TestProperties{Property: tp}}
	return tc
}
<commit_msg>Make arg names clearer for perf testcase (#2760)<commit_after>// +build performance

/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package performance

import (
	"context"
	"fmt"
	"time"

	pkgTest "github.com/knative/pkg/test"
	"github.com/knative/pkg/test/logging"
	"github.com/knative/serving/test"
	"github.com/knative/test-infra/shared/prometheus"
	"github.com/knative/test-infra/shared/testgrid"
	"istio.io/fortio/fhttp"
	"istio.io/fortio/periodic"

	// Mysteriously required to support GCP auth (required by k8s libs). Apparently just importing it is enough. @_@ side effects @_@. https://github.com/kubernetes/client-go/issues/242
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
)

const (
	istioNS      = "istio-system"
	monitoringNS = "knative-monitoring"
	gateway      = "istio-ingressgateway"
	// Property name used by testgrid.
	perfLatency = "perf_latency"
	duration    = 1 * time.Minute
)

type PerformanceClient struct {
	E2EClients *test.Clients
	PromClient *prometheus.PromProxy
}

// Setup creates all the clients that we need to interact with in our tests
func Setup(ctx context.Context, logger *logging.BaseLogger, promReqd bool) (*PerformanceClient, error) {
	clients, err := test.NewClients(pkgTest.Flags.Kubeconfig, pkgTest.Flags.Cluster, test.ServingNamespace)
	if err != nil {
		return nil, err
	}

	var p *prometheus.PromProxy
	if promReqd {
		logger.Infof("Creating prometheus proxy client")
		p = &prometheus.PromProxy{Namespace: monitoringNS}
		p.Setup(ctx, logger)
	}
	return &PerformanceClient{E2EClients: clients, PromClient: p}, nil
}

// Teardown cleans up resources used
func TearDown(client *PerformanceClient, logger *logging.BaseLogger, names test.ResourceNames) {
	if client.E2EClients != nil && client.E2EClients.ServingClient != nil {
		client.E2EClients.ServingClient.Delete([]string{names.Route}, []string{names.Config}, []string{names.Service})
	}

	if client.PromClient != nil {
		client.PromClient.Teardown(logger)
	}
}

// RunLoadTest runs the load test with fortio and returns the response
func RunLoadTest(duration time.Duration, nThreads, nConnections int, url, domain string) (*fhttp.HTTPRunnerResults, error) {
	o := fhttp.NewHTTPOptions(url)
	o.NumConnections = nConnections
	o.AddAndValidateExtraHeader(fmt.Sprintf("Host: %s", domain))

	opts := fhttp.HTTPRunnerOptions{
		RunnerOptions: periodic.RunnerOptions{
			Duration:    duration,
			NumThreads:  nThreads,
			Percentiles: []float64{50.0, 90.0, 99.0},
		},
		HTTPOptions:        *o,
		AllowInitialErrors: true,
	}

	return fhttp.RunHTTPTest(&opts)
}

// CreatePerfTestCase creates a perf test case with the provided name and value
func CreatePerfTestCase(metricValue float32, metricName, testName string) testgrid.TestCase {
	tp := []testgrid.TestProperty{{Name: perfLatency, Value: metricValue}}
	tc := testgrid.TestCase{
		ClassName:  testName,
		Name:       fmt.Sprintf("%s/%s", testName, metricName),
		Properties: testgrid.TestProperties{Property: tp}}
	return tc
}
<|endoftext|>


<commit_before>package isolation_segments

import (
	"encoding/json"
	"fmt"
	"os"

	. "github.com/cloudfoundry/cf-acceptance-tests/cats_suite_helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gexec"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
	"github.com/cloudfoundry-incubator/cf-test-helpers/workflowhelpers"
	"github.com/cloudfoundry/cf-acceptance-tests/helpers/app_helpers"
	"github.com/cloudfoundry/cf-acceptance-tests/helpers/assets"
	"github.com/cloudfoundry/cf-acceptance-tests/helpers/config"
	"github.com/cloudfoundry/cf-acceptance-tests/helpers/random_name"
	//	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
	//	"github.com/cloudfoundry/cf-acceptance-tests/helpers/app_helpers"
)

const (
	SHARED_ISOLATION_SEGMENT_GUID = "933b4c58-120b-499a-b85d-4b6fc9e2903b"
	binaryHi                      = "Hello from a binary"
)

func assignIsolationSegment(orgGuid, isGuid string) {
	Eventually(cf.Cf("curl",
		fmt.Sprintf("/v3/isolation_segments/%s/relationships/organizations", isGuid),
		"-X",
		"POST",
		"-d",
		fmt.Sprintf(`{"data":[{ "guid":"%s" }]}`, orgGuid)),
		Config.DefaultTimeoutDuration()).Should(Exit(0))
}

func setDefaultIsolationSegment(orgGuid, isGuid string) {
	Eventually(cf.Cf("curl",
		fmt.Sprintf("/v2/organizations/%s", orgGuid),
		"-X",
		"PUT",
		"-d",
		fmt.Sprintf(`{"default_isolation_segment_guid":"%s"}`, isGuid)),
		Config.DefaultTimeoutDuration()).Should(Exit(0))
}

func getV3Guid(response []byte) string {
	type resource struct {
		Guid string `json:"guid"`
	}
	var GetResponse struct {
		Resources []resource `json:"resources"`
	}

	err := json.Unmarshal(response, &GetResponse)
	Expect(err).ToNot(HaveOccurred())

	if len(GetResponse.Resources) == 0 {
		Fail("No guid found for response")
	}

	return GetResponse.Resources[0].Guid
}

func getV2Guid(response []byte) string {
	type metadata struct {
		Guid string `json:"guid"`
	}
	type resource struct {
		Metadata metadata `json:"metadata"`
	}
	var GetResponse struct {
		Resources []resource `json:"resources"`
	}

	err := json.Unmarshal(response, &GetResponse)
	Expect(err).ToNot(HaveOccurred())

	if len(GetResponse.Resources) == 0 {
		Fail("No guid found for response")
	}

	return GetResponse.Resources[0].Metadata.Guid
}

func getIsolationSegmentGuid(name string) string {
	session := cf.Cf("curl", fmt.Sprintf("/v3/isolation_segments?names=%s", name))
	bytes := session.Wait(Config.DefaultTimeoutDuration()).Out.Contents()
	return getV3Guid(bytes)
}

func isolationSegmentExists(name string) bool {
	session := cf.Cf("curl", fmt.Sprintf("/v3/isolation_segments?names=%s", name))
	bytes := session.Wait(Config.DefaultTimeoutDuration()).Out.Contents()
	type resource struct {
		Guid string `json:"guid"`
	}
	var GetResponse struct {
		Resources []resource `json:"resources"`
	}

	err := json.Unmarshal(bytes, &GetResponse)
	Expect(err).ToNot(HaveOccurred())
	return len(GetResponse.Resources) > 0
}

func createIsolationSegment(name string) string {
	session := cf.Cf("curl", "/v3/isolation_segments", "-X", "POST", "-d", fmt.Sprintf(`{"name":"%s"}`, name))
	bytes := session.Wait(Config.DefaultTimeoutDuration()).Out.Contents()

	var isolation_segment struct {
		Guid string `json:"guid"`
	}
	err := json.Unmarshal(bytes, &isolation_segment)
	Expect(err).ToNot(HaveOccurred())

	return isolation_segment.Guid
}

func deleteIsolationSegment(guid string) {
	Eventually(cf.Cf("curl", fmt.Sprintf("/v3/isolation_segments/%s", guid), "-X", "DELETE"), Config.DefaultTimeoutDuration()).Should(Exit(0))
}

var _ = IsolationSegmentsDescribe("IsolationSegments", func() {
	var orgGuid string
	var testSetup *workflowhelpers.ReproducibleTestSuiteSetup
	var isGuid string
	var shouldDeleteIsolationSegment bool

	BeforeEach(func() {
		// New up a organization since we will be assigning isolation segments.
		// This has a potential to cause other tests to fail if running in parallel mode.
		cfg, _ := config.NewCatsConfig(os.Getenv("CONFIG"))
		testSetup = workflowhelpers.NewTestSuiteSetup(cfg)
		testSetup.Setup()

		session := cf.Cf("curl", fmt.Sprintf("/v2/organizations?q=name:%s", testSetup.GetOrganizationName()))
		bytes := session.Wait(Config.DefaultTimeoutDuration()).Out.Contents()
		orgGuid = getV2Guid(bytes)

		shouldDeleteIsolationSegment = true
	})

	AfterEach(func() {
		testSetup.Teardown()

		if isGuid != "" && shouldDeleteIsolationSegment {
			workflowhelpers.AsUser(testSetup.AdminUserContext(), testSetup.ShortTimeout(), func() {
				deleteIsolationSegment(isGuid)
			})
		}
	})

	Context("When an organization has the shared segment as its default", func() {
		BeforeEach(func() {
			assignIsolationSegment(orgGuid, SHARED_ISOLATION_SEGMENT_GUID)
		})

		It("can push an app to a space with no assigned segment", func() {
			appName := random_name.CATSRandomName("APP")
			Eventually(cf.Cf(
				"push", appName,
				"-p", assets.NewAssets().Binary,
				"--no-start",
				"-m", DEFAULT_MEMORY_LIMIT,
				"-b", "binary_buildpack",
				"-d", Config.GetAppsDomain(),
				"-c", "./app"),
				Config.CfPushTimeoutDuration()).Should(Exit(0))

			app_helpers.EnableDiego(appName)
			Eventually(cf.Cf("start", appName), Config.CfPushTimeoutDuration()).Should(Exit(0))
			Eventually(helpers.CurlingAppRoot(Config, appName), Config.DefaultTimeoutDuration()).Should(ContainSubstring(binaryHi))
		})
	})

	Context("when the user-provided Isolation Segment has an associated cell", func() {
		BeforeEach(func() {
			workflowhelpers.AsUser(testSetup.AdminUserContext(), testSetup.ShortTimeout(), func() {
				name := Config.GetIsolationSegmentName()
				if !isolationSegmentExists(name) {
					isGuid = createIsolationSegment(name)
					shouldDeleteIsolationSegment = true
				} else {
					isGuid = getIsolationSegmentGuid(name)
					shouldDeleteIsolationSegment = false
				}
				assignIsolationSegment(orgGuid, isGuid)
				setDefaultIsolationSegment(orgGuid, isGuid)
			})
		})

		It("can push an app to an org where the default is the user-provided isolation segment", func() {
			appName := random_name.CATSRandomName("APP")
			Eventually(cf.Cf(
				"push", appName,
				"-p", assets.NewAssets().Binary,
				"--no-start",
				"-m", DEFAULT_MEMORY_LIMIT,
				"-b", "binary_buildpack",
				"-d", Config.GetAppsDomain(),
				"-c", "./app"),
				Config.CfPushTimeoutDuration()).Should(Exit(0))

			app_helpers.EnableDiego(appName)
			Eventually(cf.Cf("start", appName), Config.CfPushTimeoutDuration()).Should(Exit(0))
			Eventually(helpers.CurlingAppRoot(Config, appName), Config.DefaultTimeoutDuration()).Should(ContainSubstring(binaryHi))
		})
	})

	Context("When the Isolation Segment has no associated cells", func() {
		BeforeEach(func() {
			workflowhelpers.AsUser(testSetup.AdminUserContext(), testSetup.ShortTimeout(), func() {
				isGuid = createIsolationSegment(random_name.CATSRandomName("IsolationSegment"))
				assignIsolationSegment(orgGuid, isGuid)
				setDefaultIsolationSegment(orgGuid, isGuid)
			})
		})

		It("fails to start an app in the Isolation Segment", func() {
			appName := random_name.CATSRandomName("APP")
			Eventually(cf.Cf(
				"push", appName,
				"-p", assets.NewAssets().Binary,
				"--no-start",
				"-m", DEFAULT_MEMORY_LIMIT,
				"-b", "binary_buildpack",
				"-d", Config.GetAppsDomain(),
				"-c", "./app"),
				Config.CfPushTimeoutDuration()).Should(Exit(0))

			app_helpers.EnableDiego(appName)
			Eventually(cf.Cf("start", appName), Config.CfPushTimeoutDuration()).Should(Exit(1))
		})
	})
})
<commit_msg>Use v3 Org endpoints in IS tests<commit_after>package isolation_segments

import (
	"encoding/json"
	"fmt"
	"os"

	. "github.com/cloudfoundry/cf-acceptance-tests/cats_suite_helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gexec"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
	"github.com/cloudfoundry-incubator/cf-test-helpers/workflowhelpers"
	"github.com/cloudfoundry/cf-acceptance-tests/helpers/app_helpers"
	"github.com/cloudfoundry/cf-acceptance-tests/helpers/assets"
	"github.com/cloudfoundry/cf-acceptance-tests/helpers/config"
	"github.com/cloudfoundry/cf-acceptance-tests/helpers/random_name"
	//	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
	//	"github.com/cloudfoundry/cf-acceptance-tests/helpers/app_helpers"
)

const (
	SHARED_ISOLATION_SEGMENT_GUID = "933b4c58-120b-499a-b85d-4b6fc9e2903b"
	binaryHi                      = "Hello from a binary"
)

func assignIsolationSegment(orgGuid, isGuid string) {
	Eventually(cf.Cf("curl",
		fmt.Sprintf("/v3/isolation_segments/%s/relationships/organizations", isGuid),
		"-X",
		"POST",
		"-d",
		fmt.Sprintf(`{"data":[{ "guid":"%s" }]}`, orgGuid)),
		Config.DefaultTimeoutDuration()).Should(Exit(0))
}

func setDefaultIsolationSegment(orgGuid, isGuid string) {
	Eventually(cf.Cf("curl",
		fmt.Sprintf("/v3/organizations/%s/relationships/default_isolation_segment", orgGuid),
		"-X",
		"PATCH",
		"-d",
		fmt.Sprintf(`{"data":{"guid":"%s"}}`, isGuid)),
		Config.DefaultTimeoutDuration()).Should(Exit(0))
}

func getV3Guid(response []byte) string {
	type resource struct {
		Guid string `json:"guid"`
	}
	var GetResponse struct {
		Resources []resource `json:"resources"`
	}

	err := json.Unmarshal(response, &GetResponse)
	Expect(err).ToNot(HaveOccurred())

	if len(GetResponse.Resources) == 0 {
		Fail("No guid found for response")
	}

	return GetResponse.Resources[0].Guid
}

func getV2Guid(response []byte) string {
	type metadata struct {
		Guid string `json:"guid"`
	}
	type resource struct {
		Metadata metadata `json:"metadata"`
	}
	var GetResponse struct {
		Resources []resource `json:"resources"`
	}

	err := json.Unmarshal(response, &GetResponse)
	Expect(err).ToNot(HaveOccurred())

	if len(GetResponse.Resources) == 0 {
		Fail("No guid found for response")
	}

	return GetResponse.Resources[0].Metadata.Guid
}

func getIsolationSegmentGuid(name string) string {
	session := cf.Cf("curl", fmt.Sprintf("/v3/isolation_segments?names=%s", name))
	bytes := session.Wait(Config.DefaultTimeoutDuration()).Out.Contents()
	return getV3Guid(bytes)
}

func isolationSegmentExists(name string) bool {
	session := cf.Cf("curl", fmt.Sprintf("/v3/isolation_segments?names=%s", name))
	bytes := session.Wait(Config.DefaultTimeoutDuration()).Out.Contents()
	type resource struct {
		Guid string `json:"guid"`
	}
	var GetResponse struct {
		Resources []resource `json:"resources"`
	}

	err := json.Unmarshal(bytes, &GetResponse)
	Expect(err).ToNot(HaveOccurred())
	return len(GetResponse.Resources) > 0
}

func createIsolationSegment(name string) string {
	session := cf.Cf("curl", "/v3/isolation_segments", "-X", "POST", "-d", fmt.Sprintf(`{"name":"%s"}`, name))
	bytes := session.Wait(Config.DefaultTimeoutDuration()).Out.Contents()

	var isolation_segment struct {
		Guid string `json:"guid"`
	}
	err := json.Unmarshal(bytes, &isolation_segment)
	Expect(err).ToNot(HaveOccurred())

	return isolation_segment.Guid
}

func deleteIsolationSegment(guid string) {
	Eventually(cf.Cf("curl", fmt.Sprintf("/v3/isolation_segments/%s", guid), "-X", "DELETE"), Config.DefaultTimeoutDuration()).Should(Exit(0))
}

var _ = IsolationSegmentsDescribe("IsolationSegments", func() {
	var orgGuid string
	var testSetup *workflowhelpers.ReproducibleTestSuiteSetup
	var isGuid string
	var shouldDeleteIsolationSegment bool

	BeforeEach(func() {
		// New up a organization since we will be assigning isolation segments.
		// This has a potential to cause other tests to fail if running in parallel mode.
		cfg, _ := config.NewCatsConfig(os.Getenv("CONFIG"))
		testSetup = workflowhelpers.NewTestSuiteSetup(cfg)
		testSetup.Setup()

		session := cf.Cf("curl", fmt.Sprintf("/v3/organizations?names=%s", testSetup.GetOrganizationName()))
		bytes := session.Wait(Config.DefaultTimeoutDuration()).Out.Contents()
		orgGuid = getV3Guid(bytes)

		shouldDeleteIsolationSegment = true
	})

	AfterEach(func() {
		testSetup.Teardown()

		if isGuid != "" && shouldDeleteIsolationSegment {
			workflowhelpers.AsUser(testSetup.AdminUserContext(), testSetup.ShortTimeout(), func() {
				deleteIsolationSegment(isGuid)
			})
		}
	})

	Context("When an organization has the shared segment as its default", func() {
		BeforeEach(func() {
			assignIsolationSegment(orgGuid, SHARED_ISOLATION_SEGMENT_GUID)
		})

		It("can push an app to a space with no assigned segment", func() {
			appName := random_name.CATSRandomName("APP")
			Eventually(cf.Cf(
				"push", appName,
				"-p", assets.NewAssets().Binary,
				"--no-start",
				"-m", DEFAULT_MEMORY_LIMIT,
				"-b", "binary_buildpack",
				"-d", Config.GetAppsDomain(),
				"-c", "./app"),
				Config.CfPushTimeoutDuration()).Should(Exit(0))

			app_helpers.EnableDiego(appName)
			Eventually(cf.Cf("start", appName), Config.CfPushTimeoutDuration()).Should(Exit(0))
			Eventually(helpers.CurlingAppRoot(Config, appName), Config.DefaultTimeoutDuration()).Should(ContainSubstring(binaryHi))
		})
	})

	Context("when the user-provided Isolation Segment has an associated cell", func() {
		BeforeEach(func() {
			workflowhelpers.AsUser(testSetup.AdminUserContext(), testSetup.ShortTimeout(), func() {
				name := Config.GetIsolationSegmentName()
				if !isolationSegmentExists(name) {
					isGuid = createIsolationSegment(name)
					shouldDeleteIsolationSegment = true
				} else {
					isGuid = getIsolationSegmentGuid(name)
					shouldDeleteIsolationSegment = false
				}
				assignIsolationSegment(orgGuid, isGuid)
				setDefaultIsolationSegment(orgGuid, isGuid)
			})
		})

		It("can push an app to an org where the default is the user-provided isolation segment", func() {
			appName := random_name.CATSRandomName("APP")
			Eventually(cf.Cf(
				"push", appName,
				"-p", assets.NewAssets().Binary,
				"--no-start",
				"-m", DEFAULT_MEMORY_LIMIT,
				"-b", "binary_buildpack",
				"-d", Config.GetAppsDomain(),
				"-c", "./app"),
				Config.CfPushTimeoutDuration()).Should(Exit(0))

			app_helpers.EnableDiego(appName)
			Eventually(cf.Cf("start", appName), Config.CfPushTimeoutDuration()).Should(Exit(0))
			Eventually(helpers.CurlingAppRoot(Config, appName), Config.DefaultTimeoutDuration()).Should(ContainSubstring(binaryHi))
		})
	})

	Context("When the Isolation Segment has no associated cells", func() {
		BeforeEach(func() {
			workflowhelpers.AsUser(testSetup.AdminUserContext(), testSetup.ShortTimeout(), func() {
				isGuid = createIsolationSegment(random_name.CATSRandomName("IsolationSegment"))
				assignIsolationSegment(orgGuid, isGuid)
				setDefaultIsolationSegment(orgGuid, isGuid)
			})
		})

		It("fails to start an app in the Isolation Segment", func() {
			appName := random_name.CATSRandomName("APP")
			Eventually(cf.Cf(
				"push", appName,
				"-p", assets.NewAssets().Binary,
				"--no-start",
				"-m", DEFAULT_MEMORY_LIMIT,
				"-b", "binary_buildpack",
				"-d", Config.GetAppsDomain(),
				"-c", "./app"),
				Config.CfPushTimeoutDuration()).Should(Exit(0))

			app_helpers.EnableDiego(appName)
			Eventually(cf.Cf("start", appName), Config.CfPushTimeoutDuration()).Should(Exit(1))
		})
	})
})
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"exec"
	"flag"
	"fmt"
	"html"
	"http"
	"io"
	"json"
	"log"
	"os"
	"path/filepath"
	"strings"
)

const (
	watershed = "http://api.oswatershed.org/api/0.1/package.json?package=%s&cb=go"
	debian    = "http://packages.debian.org/%s/%s"
	drelease  = "wheezy"
)

const (
	ARCHLINUX = iota
)

var (
	isTest          = flag.Bool("test", false, "run tests")
	isCheckTemplate = flag.Bool("ct", false, "check templates *currently* only checks homepage, license")
	isHome          = flag.Bool("home", false, "open package homepage in browser")
	isBroken        = flag.Bool("b", false, "print out broken upstream versions")
	isCheck         = flag.Bool("c", false, "check upstream version against our versions")
	isSync          = flag.Bool("s", false, "sync oswatershed and srpkgs cache")
	isLongDesc      = flag.Bool("ld", false, "get long description from debian packages")
	srcPath         = flag.String("path", "/home/strings/github/vanilla/srcpkgs/", "path to srcpkgs")
	browser         = flag.String("browser", "chromium", "broswer to use")
)

type Distro struct {
	Version string
}

type Package struct {
	Name    string "package"
	Latest  string
	Vanilla string
	Distros []Distro
	Check   bool
}

func init() {
	log.SetPrefix("")
	log.SetFlags(log.Lshortfile)
}

func main() {
	flag.Parse()
	if *isTest {
		test()
		return
	}
	if *isCheckTemplate {
		err := checkTemplates()
		if err != nil {
			log.Print(err)
		}
		return
	}
	if *isHome {
		pack := flag.Arg(0)
		if pack == "" {
			flag.Usage()
			return
		}
		err := home(pack)
		if err != nil {
			log.Print(err)
		}
		return
	}
	if *isSync {
		err := sync()
		if err != nil {
			log.Print(err)
		}
		return
	}
	if *isBroken {
		err := broken()
		if err != nil {
			fmt.Println(err)
		}
		return
	}
	if *isCheck {
		err := checkVersions()
		if err != nil {
			fmt.Println(err)
		}
		return
	}
	if *isLongDesc {
		err := longDesc(flag.Arg(0))
		if err != nil {
			fmt.Println(err)
		}
		return
	}
	if len(flag.Args()) != 1 {
		flag.Usage()
		return
	}
	pack, err := latest(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(pack.Latest)
}

func test() {
	checkFatal(checkTemplates())
	checkFatal(home("bash"))
	checkFatal(checkVersions())
	//checkFatal(sync())
}

func checkFatal(err os.Error) {
	if err != nil {
		log.Fatal(err)
	}
}

func checkTemplates() os.Error {
	templates, err := GetTemplates(*srcPath)
	if err != nil {
		return err
	}
	for _, t := range templates {
		if t.License == "" || t.Homepage == "" {
			fmt.Println(t.Pkgname)
		}
	}
	return nil
}

func home(pack string) os.Error {
	file := filepath.Join(*srcPath, pack, "template")
	t, err := NewTemplate(file)
	if err != nil {
		return err
	}
	fmt.Println("opening", t.Homepage)
	err = exec.Command(*browser, t.Homepage).Run()
	if err != nil {
		return err
	}
	return nil
}

func broken() os.Error {
	cache, err := loadCache()
	if err != nil {
		return err
	}
	for _, p := range *cache {
		if !p.Check {
			fmt.Printf("%v\n", p.Name)
		}
	}
	return nil

}

func checkVersions() os.Error {
	cache, err := loadCache()
	if err != nil {
		return err
	}
	fmt.Printf("%-20.20s %-20.20s %-20.20s %-20.20s\n", "name", "vanilla", "upstream", "archlinux")
	for _, p := range *cache {
		if p.Latest != p.Vanilla && p.Check {
			fmt.Printf("%-20.20s %-20.20s %-20.20s %-20.20s\n", p.Name, p.Vanilla, p.Latest, p.Distros[ARCHLINUX].Version)
		}
	}
	fmt.Printf("%-20.20s %-20.20s %-20.20s %-20.20s\n", "name", "vanilla", "upstream", "archlinux")
	return nil
}

func sync() os.Error {
	cache := map[string]*Package{}
	templates, err := GetTemplates(*srcPath)
	checkFatal(err)
	nrange := 0
	for _, t := range templates {
		nrange++
		pack, err := latest(t.Pkgname)
		if err != nil {
			p := new(Package)
			p.Name = t.Pkgname
			p.Check = false
			p.Vanilla = t.Version
			fmt.Printf("%04.0v %-20.20s %s\n", nrange, t.Pkgname, "error")
			cache[t.Pkgname] = p
			continue
		}
		pack.Vanilla = t.Version
		pack.Check = true
		cache[t.Pkgname] = pack
		fmt.Printf("%04.0v %-20.20s %-10.10s\n", nrange, t.Pkgname, pack.Latest)
	}
	f, err := os.Create(*srcPath + "/upstream.json")
	if err != nil {
		return err
	}
	defer f.Close()
	b, err := json.Marshal(cache)
	if err != nil {
		return err
	}
	buf := new(bytes.Buffer)
	err = json.Indent(buf, b, "", "\t")
	if err != nil {
		return err
	}
	_, err = io.Copy(f, buf)
	if err != nil {
		return err
	}
	return nil
}

func loadCache() (*map[string]*Package, os.Error) {
	cache := &map[string]*Package{}
	fd, err := os.Open(*srcPath + "upstream.json")
	if err != nil {
		return nil, err
	}
	defer fd.Close()
	err = json.NewDecoder(fd).Decode(cache)
	if err != nil {
		return nil, err
	}
	return cache, nil
}

func getVar(template string, shvar string) (string, os.Error) {
	os.Setenv("XBPS_SRCPKGDIR", *srcPath)
	fd, err := os.Open(template)
	if err != nil {
		return "", err
	}
	defer fd.Close()
	buf := new(bytes.Buffer)
	buf.WriteString("Add_dependency(){ :\n }\n")
	io.Copy(buf, fd)
	buf.WriteString("echo $" + shvar)
	cmd := exec.Command("sh")
	cmd.Stdin = buf
	output, err := cmd.CombinedOutput()
	if err != nil {
		println(template)
		log.Println(err, string(output))
		return "", err
	}
	svar := strings.Replace(string(output), "\n", "", -1)
	if svar == "" {
		return "error", fmt.Errorf("%s not found in %s", shvar, template)
	}
	return svar, err
}

func longDesc(name string) os.Error {
	client := new(http.Client)
	url := fmt.Sprintf(debian, drelease, name)
	res, err := client.Do(request(url))
	if err != nil {
		log.Fatal(err)
		return err
	}
	if res.StatusCode != 200 {
		return fmt.Errorf("Http GET error %s", res.Status)
	}
	buf := new(bytes.Buffer)
	_, err = io.Copy(buf, res.Body)
	if err != nil {
		return err
	}
	doc, err := html.Parse(buf)
	if err != nil {
		return err
	}
	//TODO: move this to a proper function
	var f func(*html.Node)
	f = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "div" {
			if len(n.Attr) > 0 {
				if n.Attr[0].Val == "pdesc" {
					fmt.Printf("%s\n\n", n.Child[1].Child[0].Data)
					fmt.Printf("*long desc*\n%s", n.Child[1].Child[2].Child[0].Data)
				}
			}
		}
		for _, c := range n.Child {
			f(c)
		}
	}
	f(doc)
	return nil
}

func latest(name string) (*Package, os.Error) {
	client := new(http.Client)
	url := fmt.Sprintf(watershed, name)
	res, err := client.Do(request(url))
	if err != nil {
		log.Fatal(err)
		return nil, err
	}
	if res.StatusCode != 200 {
		return nil, fmt.Errorf("Http GET error %s", res.Status)
	}
	buf := new(bytes.Buffer)
	io.Copy(buf, res.Body)
	p := new(Package)
	b := buf.Bytes()
	err = json.Unmarshal(b[3:len(b)-1], p)
	if err != nil {
		fmt.Println(err)
		return p, err
	}
	return p, nil
}

func request(url string) *http.Request {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Fatal(err)
		return nil
	}
	req.ProtoMajor = 1
	req.ProtoMinor = 1
	req.TransferEncoding = []string{"chunked"}
	//FIXME: Debian doesnt always return compressed
	//req.Header.Set("Accept-Encoding", "gzip,deflate")
	req.Header.Set("Connection", "keep-alive")
	return req
}
<commit_msg>fix typo in flag usage<commit_after>package main

import (
	"bytes"
	"exec"
	"flag"
	"fmt"
	"html"
	"http"
	"io"
	"json"
	"log"
	"os"
	"path/filepath"
	"strings"
)

const (
	watershed = "http://api.oswatershed.org/api/0.1/package.json?package=%s&cb=go"
	debian    = "http://packages.debian.org/%s/%s"
	drelease  = "wheezy"
)

const (
	ARCHLINUX = iota
)

var (
	isTest          = flag.Bool("test", false, "run tests")
	isCheckTemplate = flag.Bool("ct", false, "check templates *currently* only checks homepage, license")
	isHome          = flag.Bool("home", false, "open package homepage in browser")
	isBroken        = flag.Bool("b", false, "print out broken upstream versions")
	isCheck         = flag.Bool("c", false, "check upstream version against our versions")
	isSync          = flag.Bool("s", false, "sync oswatershed and srpkgs cache")
	isLongDesc      = flag.Bool("ld", false, "get long description from debian packages")
	srcPath         = flag.String("path", "/home/strings/github/vanilla/srcpkgs/", "path to srcpkgs")
	browser         = flag.String("browser", "chromium", "browser to use")
)

type Distro struct {
	Version string
}

type Package struct {
	Name    string "package"
	Latest  string
	Vanilla string
	Distros []Distro
	Check   bool
}

func init() {
	log.SetPrefix("")
	log.SetFlags(log.Lshortfile)
}

func main() {
	flag.Parse()
	if *isTest {
		test()
		return
	}
	if *isCheckTemplate {
		err := checkTemplates()
		if err != nil {
			log.Print(err)
		}
		return
	}
	if *isHome {
		pack := flag.Arg(0)
		if pack == "" {
			flag.Usage()
			return
		}
		err := home(pack)
		if err != nil {
			log.Print(err)
		}
		return
	}
	if *isSync {
		err := sync()
		if err != nil {
			log.Print(err)
		}
		return
	}
	if *isBroken {
		err := broken()
		if err != nil {
			fmt.Println(err)
		}
		return
	}
	if *isCheck {
		err := checkVersions()
		if err != nil {
			fmt.Println(err)
		}
		return
	}
	if *isLongDesc {
		err := longDesc(flag.Arg(0))
		if err != nil {
			fmt.Println(err)
		}
		return
	}
	if len(flag.Args()) != 1 {
		flag.Usage()
		return
	}
	pack, err := latest(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(pack.Latest)
}

func test() {
	checkFatal(checkTemplates())
	checkFatal(home("bash"))
	checkFatal(checkVersions())
	//checkFatal(sync())
}

func checkFatal(err os.Error) {
	if err != nil {
		log.Fatal(err)
	}
}

func checkTemplates() os.Error {
	templates, err := GetTemplates(*srcPath)
	if err != nil {
		return err
	}
	for _, t := range templates {
		if t.License == "" || t.Homepage == "" {
			fmt.Println(t.Pkgname)
		}
	}
	return nil
}

func home(pack string) os.Error {
	file := filepath.Join(*srcPath, pack, "template")
	t, err := NewTemplate(file)
	if err != nil {
		return err
	}
	fmt.Println("opening", t.Homepage)
	err = exec.Command(*browser, t.Homepage).Run()
	if err != nil {
		return err
	}
	return nil
}

func broken() os.Error {
	cache, err := loadCache()
	if err != nil {
		return err
	}
	for _, p := range *cache {
		if !p.Check {
			fmt.Printf("%v\n", p.Name)
		}
	}
	return nil

}

func checkVersions() os.Error {
	cache, err := loadCache()
	if err != nil {
		return err
	}
	fmt.Printf("%-20.20s %-20.20s %-20.20s %-20.20s\n", "name", "vanilla", "upstream", "archlinux")
	for _, p := range *cache {
		if p.Latest != p.Vanilla && p.Check {
			fmt.Printf("%-20.20s %-20.20s %-20.20s %-20.20s\n", p.Name, p.Vanilla, p.Latest, p.Distros[ARCHLINUX].Version)
		}
	}
	fmt.Printf("%-20.20s %-20.20s %-20.20s %-20.20s\n", "name", "vanilla", "upstream", "archlinux")
	return nil
}

func sync() os.Error {
	cache := map[string]*Package{}
	templates, err := GetTemplates(*srcPath)
	checkFatal(err)
	nrange := 0
	for _, t := range templates {
		nrange++
		pack, err := latest(t.Pkgname)
		if err != nil {
			p := new(Package)
			p.Name = t.Pkgname
			p.Check = false
			p.Vanilla = t.Version
			fmt.Printf("%04.0v %-20.20s %s\n", nrange, t.Pkgname, "error")
			cache[t.Pkgname] = p
			continue
		}
		pack.Vanilla = t.Version
		pack.Check = true
		cache[t.Pkgname] = pack
		fmt.Printf("%04.0v %-20.20s %-10.10s\n", nrange, t.Pkgname, pack.Latest)
	}
	f, err := os.Create(*srcPath + "/upstream.json")
	if err != nil {
		return err
	}
	defer f.Close()
	b, err := json.Marshal(cache)
	if err != nil {
		return err
	}
	buf := new(bytes.Buffer)
	err = json.Indent(buf, b, "", "\t")
	if err != nil {
		return err
	}
	_, err = io.Copy(f, buf)
	if err != nil {
		return err
	}
	return nil
}

func loadCache() (*map[string]*Package, os.Error) {
	cache := &map[string]*Package{}
	fd, err := os.Open(*srcPath + "upstream.json")
	if err != nil {
		return nil, err
	}
	defer fd.Close()
	err = json.NewDecoder(fd).Decode(cache)
	if err != nil {
		return nil, err
	}
	return cache, nil
}

func getVar(template string, shvar string) (string, os.Error) {
	os.Setenv("XBPS_SRCPKGDIR", *srcPath)
	fd, err := os.Open(template)
	if err != nil {
		return "", err
	}
	defer fd.Close()
	buf := new(bytes.Buffer)
	buf.WriteString("Add_dependency(){ :\n }\n")
	io.Copy(buf, fd)
	buf.WriteString("echo $" + shvar)
	cmd := exec.Command("sh")
	cmd.Stdin = buf
	output, err := cmd.CombinedOutput()
	if err != nil {
		println(template)
		log.Println(err, string(output))
		return "", err
	}
	svar := strings.Replace(string(output), "\n", "", -1)
	if svar == "" {
		return "error", fmt.Errorf("%s not found in %s", shvar, template)
	}
	return svar, err
}

func longDesc(name string) os.Error {
	client := new(http.Client)
	url := fmt.Sprintf(debian, drelease, name)
	res, err := client.Do(request(url))
	if err != nil {
		log.Fatal(err)
		return err
	}
	if res.StatusCode != 200 {
		return fmt.Errorf("Http GET error %s", res.Status)
	}
	buf := new(bytes.Buffer)
	_, err = io.Copy(buf, res.Body)
	if err != nil {
		return err
	}
	doc, err := html.Parse(buf)
	if err != nil {
		return err
	}
	//TODO: move this to a proper function
	var f func(*html.Node)
	f = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "div" {
			if len(n.Attr) > 0 {
				if n.Attr[0].Val == "pdesc" {
					fmt.Printf("%s\n\n", n.Child[1].Child[0].Data)
					fmt.Printf("*long desc*\n%s", n.Child[1].Child[2].Child[0].Data)
				}
			}
		}
		for _, c := range n.Child {
			f(c)
		}
	}
	f(doc)
	return nil
}

func latest(name string) (*Package, os.Error) {
	client := new(http.Client)
	url := fmt.Sprintf(watershed, name)
	res, err := client.Do(request(url))
	if err != nil {
		log.Fatal(err)
		return nil, err
	}
	if res.StatusCode != 200 {
		return nil, fmt.Errorf("Http GET error %s", res.Status)
	}
	buf := new(bytes.Buffer)
	io.Copy(buf, res.Body)
	p := new(Package)
	b := buf.Bytes()
	err = json.Unmarshal(b[3:len(b)-1], p)
	if err != nil {
		fmt.Println(err)
		return p, err
	}
	return p, nil
}

func request(url string) *http.Request {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Fatal(err)
		return nil
	}
	req.ProtoMajor = 1
	req.ProtoMinor = 1
	req.TransferEncoding = []string{"chunked"}
	//FIXME: Debian doesnt always return compressed
	//req.Header.Set("Accept-Encoding", "gzip,deflate")
	req.Header.Set("Connection", "keep-alive")
	return req
}
<|endoftext|>


<commit_before>package gowebdav

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
	"path"
	"strings"
)

func (c *Client) req(method, path string, body io.Reader, intercept func(*http.Request)) (req *http.Response, err error) {
	// Tee the body, because if authorization fails we will need to read from it again.
	var r *http.Request
	var ba bytes.Buffer
	bb := io.TeeReader(body, &ba)

	if body == nil {
		r, err = http.NewRequest(method, PathEscape(Join(c.root, path)), nil)
	} else {
		r, err = http.NewRequest(method, PathEscape(Join(c.root, path)), bb)
	}

	if err != nil {
		return nil, err
	}

	c.auth.Authorize(c, method, path)

	for k, vals := range c.headers {
		for _, v := range vals {
			r.Header.Add(k, v)
		}
	}

	if intercept != nil {
		intercept(r)
	}

	rs, err := c.c.Do(r)
	if err != nil {
		return nil, err
	}

	if rs.StatusCode == 401 && c.auth.Type() == "NoAuth" {
		if strings.Index(rs.Header.Get("Www-Authenticate"), "Digest") > -1 {
			c.auth = &DigestAuth{c.auth.User(), c.auth.Pass(), digestParts(rs)}
		} else if strings.Index(rs.Header.Get("Www-Authenticate"), "Basic") > -1 {
			c.auth = &BasicAuth{c.auth.User(), c.auth.Pass()}
		} else {
			return rs, newPathError("Authorize", c.root, rs.StatusCode)
		}

		if body == nil {
			return c.req(method, path, nil, intercept)
		} else {
			return c.req(method, path, &ba, intercept)
		}

	} else if rs.StatusCode == 401 {
		return rs, newPathError("Authorize", c.root, rs.StatusCode)
	}

	return rs, err
}

func (c *Client) mkcol(path string) int {
	rs, err := c.req("MKCOL", path, nil, nil)
	if err != nil {
		return 400
	}
	defer rs.Body.Close()

	if rs.StatusCode == 201 || rs.StatusCode == 405 {
		return 201
	}

	return rs.StatusCode
}

func (c *Client) options(path string) (*http.Response, error) {
	return c.req("OPTIONS", path, nil, func(rq *http.Request) {
		rq.Header.Add("Depth", "0")
	})
}

func (c *Client) propfind(path string, self bool, body string, resp interface{}, parse func(resp interface{}) error) error {
	rs, err := c.req("PROPFIND", path, strings.NewReader(body), func(rq *http.Request) {
		if self {
			rq.Header.Add("Depth", "0")
		} else {
			rq.Header.Add("Depth", "1")
		}
		rq.Header.Add("Content-Type", "text/xml;charset=UTF-8")
		rq.Header.Add("Accept", "application/xml,text/xml")
		rq.Header.Add("Accept-Charset", "utf-8")
		// TODO add support for 'gzip,deflate;q=0.8,q=0.7'
		rq.Header.Add("Accept-Encoding", "")
	})
	if err != nil {
		return err
	}
	defer rs.Body.Close()

	if rs.StatusCode != 207 {
		return fmt.Errorf("%s - %s %s", rs.Status, "PROPFIND", path)
	}

	return parseXML(rs.Body, resp, parse)
}

func (c *Client) doCopyMove(method string, oldpath string, newpath string, overwrite bool) (int, io.ReadCloser) {
	rs, err := c.req(method, oldpath, nil, func(rq *http.Request) {
		rq.Header.Add("Destination", Join(c.root, newpath))
		if overwrite {
			rq.Header.Add("Overwrite", "T")
		} else {
			rq.Header.Add("Overwrite", "F")
		}
	})
	if err != nil {
		return 400, nil
	}
	return rs.StatusCode, rs.Body
}

func (c *Client) copymove(method string, oldpath string, newpath string, overwrite bool) error {
	s, data := c.doCopyMove(method, oldpath, newpath, overwrite)
	defer data.Close()

	switch s {
	case 201, 204:
		return nil

	case 207:
		// TODO handle multistat errors, worst case ...
		log(fmt.Sprintf(" TODO handle %s - %s multistatus result %s", method, oldpath, String(data)))

	case 409:
		err := c.createParentCollection(newpath)
		if err != nil {
			return err
		}

		return c.copymove(method, oldpath, newpath, overwrite)
	}

	return newPathError(method, oldpath, s)
}

func (c *Client) put(path string, stream io.Reader) int {
	rs, err := c.req("PUT", path, stream, nil)
	if err != nil {
		return 400
	}
	defer rs.Body.Close()

	return rs.StatusCode
}

func (c *Client) createParentCollection(itemPath string) (err error) {
	parentPath := path.Dir(itemPath)
	return c.MkdirAll(parentPath, 0755)
}
<commit_msg>use 'application/xml' instead of 'text/xml'. related with (1) in #15<commit_after>package gowebdav

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
	"path"
	"strings"
)

func (c *Client) req(method, path string, body io.Reader, intercept func(*http.Request)) (req *http.Response, err error) {
	// Tee the body, because if authorization fails we will need to read from it again.
	var r *http.Request
	var ba bytes.Buffer
	bb := io.TeeReader(body, &ba)

	if body == nil {
		r, err = http.NewRequest(method, PathEscape(Join(c.root, path)), nil)
	} else {
		r, err = http.NewRequest(method, PathEscape(Join(c.root, path)), bb)
	}

	if err != nil {
		return nil, err
	}

	c.auth.Authorize(c, method, path)

	for k, vals := range c.headers {
		for _, v := range vals {
			r.Header.Add(k, v)
		}
	}

	if intercept != nil {
		intercept(r)
	}

	rs, err := c.c.Do(r)
	if err != nil {
		return nil, err
	}

	if rs.StatusCode == 401 && c.auth.Type() == "NoAuth" {
		if strings.Index(rs.Header.Get("Www-Authenticate"), "Digest") > -1 {
			c.auth = &DigestAuth{c.auth.User(), c.auth.Pass(), digestParts(rs)}
		} else if strings.Index(rs.Header.Get("Www-Authenticate"), "Basic") > -1 {
			c.auth = &BasicAuth{c.auth.User(), c.auth.Pass()}
		} else {
			return rs, newPathError("Authorize", c.root, rs.StatusCode)
		}

		if body == nil {
			return c.req(method, path, nil, intercept)
		} else {
			return c.req(method, path, &ba, intercept)
		}

	} else if rs.StatusCode == 401 {
		return rs, newPathError("Authorize", c.root, rs.StatusCode)
	}

	return rs, err
}

func (c *Client) mkcol(path string) int {
	rs, err := c.req("MKCOL", path, nil, nil)
	if err != nil {
		return 400
	}
	defer rs.Body.Close()

	if rs.StatusCode == 201 || rs.StatusCode == 405 {
		return 201
	}

	return rs.StatusCode
}

func (c *Client) options(path string) (*http.Response, error) {
	return c.req("OPTIONS", path, nil, func(rq *http.Request) {
		rq.Header.Add("Depth", "0")
	})
}

func (c *Client) propfind(path string, self bool, body string, resp interface{}, parse func(resp interface{}) error) error {
	rs, err := c.req("PROPFIND", path, strings.NewReader(body), func(rq *http.Request) {
		if self {
			rq.Header.Add("Depth", "0")
		} else {
			rq.Header.Add("Depth", "1")
		}
		rq.Header.Add("Content-Type", "application/xml;charset=UTF-8")
		rq.Header.Add("Accept", "application/xml,text/xml")
		rq.Header.Add("Accept-Charset", "utf-8")
		// TODO add support for 'gzip,deflate;q=0.8,q=0.7'
		rq.Header.Add("Accept-Encoding", "")
	})
	if err != nil {
		return err
	}
	defer rs.Body.Close()

	if rs.StatusCode != 207 {
		return fmt.Errorf("%s - %s %s", rs.Status, "PROPFIND", path)
	}

	return parseXML(rs.Body, resp, parse)
}

func (c *Client) doCopyMove(method string, oldpath string, newpath string, overwrite bool) (int, io.ReadCloser) {
	rs, err := c.req(method, oldpath, nil, func(rq *http.Request) {
		rq.Header.Add("Destination", Join(c.root, newpath))
		if overwrite {
			rq.Header.Add("Overwrite", "T")
		} else {
			rq.Header.Add("Overwrite", "F")
		}
	})
	if err != nil {
		return 400, nil
	}
	return rs.StatusCode, rs.Body
}

func (c *Client) copymove(method string, oldpath string, newpath string, overwrite bool) error {
	s, data := c.doCopyMove(method, oldpath, newpath, overwrite)
	defer data.Close()

	switch s {
	case 201, 204:
		return nil

	case 207:
		// TODO handle multistat errors, worst case ...
		log(fmt.Sprintf(" TODO handle %s - %s multistatus result %s", method, oldpath, String(data)))

	case 409:
		err := c.createParentCollection(newpath)
		if err != nil {
			return err
		}

		return c.copymove(method, oldpath, newpath, overwrite)
	}

	return newPathError(method, oldpath, s)
}

func (c *Client) put(path string, stream io.Reader) int {
	rs, err := c.req("PUT", path, stream, nil)
	if err != nil {
		return 400
	}
	defer rs.Body.Close()

	return rs.StatusCode
}

func (c *Client) createParentCollection(itemPath string) (err error) {
	parentPath := path.Dir(itemPath)
	return c.MkdirAll(parentPath, 0755)
}
<|endoftext|>


<commit_before>// +build linux

/*

directory structure

  .
  |-- repositoryName
	  |-- scratch
		  |-- shardNum // the read-only read created on InitRepository, this is where to start branching
      |-- commitID
	      |-- shardNum // this is where subvolumes are

*/

package btrfs

/*
#include <stdlib.h>
#include <dirent.h>
#include <btrfs/ioctl.h>
*/
import "C"

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"syscall"
	"time"
	"unsafe"

	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pfs/drive"
	"github.com/peter-edge/go-google-protobuf"
	"github.com/satori/go.uuid"
)

const (
	metadataDir = ".pfs"
)

var (
	callStrings = map[uintptr]string{
		C.BTRFS_IOC_SUBVOL_CREATE:  "btrfs subvolume create",
		C.BTRFS_IOC_SNAP_CREATE_V2: "btrfs subvolume snapshot",
	}
)

type driver struct {
	rootDir string
}

func newDriver(rootDir string) *driver {
	return &driver{rootDir}
}

func (d *driver) InitRepository(repository *pfs.Repository, shards map[int]bool) error {
	if err := os.MkdirAll(d.repositoryPath(repository), 0700); err != nil {
		return err
	}
	initialCommit := &pfs.Commit{
		Repository: repository,
		Id:         drive.InitialCommitID,
	}
	if err := os.MkdirAll(d.commitPathNoShard(initialCommit), 0700); err != nil {
		return err
	}
	for shard := range shards {
		if err := ffiSubvolumeCreate(d.commitPath(initialCommit, shard)); err != nil {
			return err
		}
		if err := os.Mkdir(d.filePath(&pfs.Path{Commit: initialCommit, Path: metadataDir}, shard), 0700); err != nil {
			return err
		}
		if err := d.setReadOnly(initialCommit, shard, true); err != nil {
			return err
		}
	}
	return nil
}

func (d *driver) GetFile(path *pfs.Path, shard int) (drive.ReaderAtCloser, error) {
	return os.Open(d.filePath(path, shard))
}

func (d *driver) GetFileInfo(path *pfs.Path, shard int) (_ *pfs.FileInfo, ok bool, _ error) {
	filePath, err := d.stat(path, shard)
	if err != nil && os.IsNotExist(err) {
		return nil, false, nil
	}
	if err != nil {
		return nil, false, err
	}
	return filePath, true, nil
}

func (d *driver) MakeDirectory(path *pfs.Path, shards map[int]bool) error {
	// TODO(pedge): if PutFile fails here or on another shard, the directories
	// will still exist and be returned from ListFiles, we want to do this
	// iteratively and with rollback
	// TODO(pedge): check that commit exists and is a write commit
	for shard := range shards {
		if err := d.checkWrite(path.Commit, shard); err != nil {
			return err
		}
		if err := os.MkdirAll(d.filePath(path, shard), 0700); err != nil {
			return err
		}
	}
	return nil
}

func (d *driver) PutFile(path *pfs.Path, shard int, offset int64, reader io.Reader) error {
	if err := d.checkWrite(path.Commit, shard); err != nil {
		return err
	}
	file, err := os.OpenFile(d.filePath(path, shard), os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	defer file.Close()
	if _, err := file.Seek(offset, 0); err != nil { // 0 means relative to start
		return err
	}
	_, err = bufio.NewReader(reader).WriteTo(file)
	return err
}

func (d *driver) ListFiles(path *pfs.Path, shard int) (_ []*pfs.FileInfo, retErr error) {
	filePath := d.filePath(path, shard)
	stat, err := os.Stat(filePath)
	if err != nil {
		return nil, err
	}
	if !stat.IsDir() {
		return nil, fmt.Errorf("%s is not a directory", filePath)
	}
	dir, err := os.Open(filePath)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err := dir.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	var fileInfos []*pfs.FileInfo
	// TODO(pedge): constant
	for names, err := dir.Readdirnames(100); err != io.EOF; names, err = dir.Readdirnames(100) {
		if err != nil {
			return nil, err
		}
		for _, name := range names {
			if inMetadataDir(name) {
				continue
			}
			fileInfo, err := d.stat(
				&pfs.Path{
					Commit: path.Commit,
					Path:   filepath.Join(path.Path, name),
				},
				shard,
			)
			if err != nil {
				return nil, err
			}
			fileInfos = append(fileInfos, fileInfo)
		}
	}
	return fileInfos, nil
}

func (d *driver) stat(path *pfs.Path, shard int) (*pfs.FileInfo, error) {
	stat, err := os.Stat(d.filePath(path, shard))
	if err != nil {
		return nil, err
	}
	fileType := pfs.FileType_FILE_TYPE_OTHER
	if stat.Mode().IsRegular() {
		fileType = pfs.FileType_FILE_TYPE_REGULAR
	}
	if stat.Mode().IsDir() {
		fileType = pfs.FileType_FILE_TYPE_DIR
	}
	return &pfs.FileInfo{
		Path:      path,
		FileType:  fileType,
		SizeBytes: uint64(stat.Size()),
		Perm:      uint32(stat.Mode() & os.ModePerm),
		LastModified: &google_protobuf.Timestamp{
			Seconds: stat.ModTime().UnixNano() / int64(time.Second),
			Nanos:   int32(stat.ModTime().UnixNano() % int64(time.Second)),
		},
	}, nil
}

func (d *driver) Branch(commit *pfs.Commit, newCommit *pfs.Commit, shards map[int]bool) (*pfs.Commit, error) {
	if newCommit == nil {
		newCommit = &pfs.Commit{
			Repository: commit.Repository,
			Id:         newCommitID(),
		}
	}
	if err := os.MkdirAll(d.commitPathNoShard(newCommit), 0700); err != nil {
		return nil, err
	}
	for shard := range shards {
		if err := d.checkReadOnly(commit, shard); err != nil {
			return nil, err
		}
		commitPath := d.commitPath(commit, shard)
		newCommitPath := d.commitPath(newCommit, shard)
		if err := ffiSubvolumeSnapshot(commitPath, newCommitPath); err != nil {
			return nil, err
		}
		if err := ioutil.WriteFile(d.filePath(&pfs.Path{Commit: newCommit, Path: filepath.Join(metadataDir, "parent")}, shard), []byte(commit.Id), 0600); err != nil {
			return nil, err
		}
		if err := d.setReadOnly(newCommit, shard, false); err != nil {
			return nil, err
		}
	}
	return newCommit, nil
}

func (d *driver) Commit(commit *pfs.Commit, shards map[int]bool) error {
	for shard := range shards {
		if err := d.checkWrite(commit, shard); err != nil {
			return err
		}
		if err := d.setReadOnly(commit, shard, true); err != nil {
			return err
		}
	}
	return nil
}

func (d *driver) PullDiff(commit *pfs.Commit, shard int) (io.Reader, error) {
	return nil, nil
}

func (d *driver) PushDiff(commit *pfs.Commit, shard int, reader io.Reader) error {
	return nil
}

func (d *driver) GetCommitInfo(commit *pfs.Commit, shard int) (_ *pfs.CommitInfo, ok bool, _ error) {
	_, err := os.Stat(d.commitPath(commit, shard))
	if err != nil && os.IsNotExist(err) {
		return nil, false, nil
	}
	if err != nil {
		return nil, false, err
	}
	parent, err := d.getParent(commit, shard)
	if err != nil {
		return nil, false, err
	}
	readOnly, err := d.getReadOnly(commit, shard)
	if err != nil {
		return nil, false, err
	}
	commitType := pfs.CommitType_COMMIT_TYPE_WRITE
	if readOnly {
		commitType = pfs.CommitType_COMMIT_TYPE_READ
	}
	return &pfs.CommitInfo{
		Commit:       commit,
		CommitType:   commitType,
		ParentCommit: parent,
	}, true, nil
}

func (d *driver) ListCommits(repository *pfs.Repository, shard int) (_ []*pfs.CommitInfo, retErr error) {

	dir, err := os.Open(d.repositoryPath(repository))
	if err != nil {
		return nil, err
	}
	defer func() {
		if err := dir.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	var commitInfos []*pfs.CommitInfo
	// TODO(pedge): constant
	for names, err := dir.Readdirnames(100); err != io.EOF; names, err = dir.Readdirnames(100) {
		if err != nil {
			return nil, err
		}
		for _, name := range names {
			commitInfo, ok, err := d.GetCommitInfo(
				&pfs.Commit{
					Repository: repository,
					Id:         path.Base(name),
				},
				shard,
			)
			if !ok {
				// This is a really weird error to get since we got this commit
				// name by listing commits. This is probably indicative of a
				// race condition.
				return nil, fmt.Errorf("Commit not found.")
			}
			if err != nil {
				return nil, err
			}
			commitInfos = append(commitInfos, commitInfo)
		}
	}
	return commitInfos, nil
}

func (d *driver) getParent(commit *pfs.Commit, shard int) (*pfs.Commit, error) {
	if commit.Id == drive.InitialCommitID {
		return nil, nil
	}
	data, err := ioutil.ReadFile(d.filePath(&pfs.Path{Commit: commit, Path: filepath.Join(metadataDir, "parent")}, shard))
	if err != nil {
		return nil, err
	}
	return &pfs.Commit{
		Repository: commit.Repository,
		Id:         string(data),
	}, nil
}

func (d *driver) checkReadOnly(commit *pfs.Commit, shard int) error {
	ok, err := d.getReadOnly(commit, shard)
	if err != nil {
		return err
	}
	if !ok {
		return fmt.Errorf("%+v is not a read only commit", commit)
	}
	return nil
}

func (d *driver) checkWrite(commit *pfs.Commit, shard int) error {
	ok, err := d.getReadOnly(commit, shard)
	if err != nil {
		return err
	}
	if ok {
		return fmt.Errorf("%+v is not a write commit", commit)
	}
	return nil
}

func (d *driver) getReadOnly(commit *pfs.Commit, shard int) (bool, error) {
	data, err := ioutil.ReadFile(d.filePath(&pfs.Path{Commit: commit, Path: filepath.Join(metadataDir, "readonly")}, shard))
	if err != nil {
		return false, err
	}
	switch string(data) {
	case "0":
		return false, nil
	case "1":
		return true, nil
	default:
		return false, fmt.Errorf("unknown data for readonly metadata file: %s", string(data))
	}
}

func (d *driver) setReadOnly(commit *pfs.Commit, shard int, readOnly bool) error {
	data := []byte("0")
	if readOnly {
		data = []byte("1")
	}
	return ioutil.WriteFile(d.filePath(&pfs.Path{Commit: commit, Path: filepath.Join(metadataDir, "readonly")}, shard), data, 0400)
}

func (d *driver) repositoryPath(repository *pfs.Repository) string {
	return filepath.Join(d.rootDir, repository.Name)
}

func (d *driver) commitPathNoShard(commit *pfs.Commit) string {
	return filepath.Join(d.repositoryPath(commit.Repository), commit.Id)
}

func (d *driver) commitPath(commit *pfs.Commit, shard int) string {
	return filepath.Join(d.commitPathNoShard(commit), fmt.Sprintf("%d", shard))
}

func (d *driver) filePath(path *pfs.Path, shard int) string {
	return filepath.Join(d.commitPath(path.Commit, shard), path.Path)
}

func newCommitID() string {
	return strings.Replace(uuid.NewV4().String(), "-", "", -1)
}

func inMetadataDir(name string) bool {
	parts := strings.Split(name, "/")
	return (len(parts) > 0 && parts[0] == metadataDir)
}

func ffiSubvolumeCreate(path string) error {
	var args C.struct_btrfs_ioctl_vol_args
	for i, c := range []byte(filepath.Base(path)) {
		args.name[i] = C.char(c)
	}
	return ffiIoctl(filepath.Dir(path), C.BTRFS_IOC_SUBVOL_CREATE, uintptr(unsafe.Pointer(&args)))
}

func ffiSubvolumeSnapshot(src string, dest string) error {
	srcDir, err := ffiOpenDir(src)
	if err != nil {
		return err
	}
	defer ffiCloseDir(srcDir)
	var args C.struct_btrfs_ioctl_vol_args_v2
	args.fd = C.__s64(ffiGetDirFd(srcDir))
	for i, c := range []byte(filepath.Base(dest)) {
		args.name[i] = C.char(c)
	}
	return ffiIoctl(filepath.Dir(dest), C.BTRFS_IOC_SNAP_CREATE_V2, uintptr(unsafe.Pointer(&args)))
}

func ffiIoctl(path string, call uintptr, args uintptr) error {
	dir, err := ffiOpenDir(path)
	if err != nil {
		return err
	}
	defer ffiCloseDir(dir)
	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, ffiGetDirFd(dir), call, args)
	if errno != 0 {
		return fmt.Errorf("%s failed for %s: %v", callStrings[call], path, errno.Error())
	}
	return nil
}

func ffiOpenDir(path string) (*C.DIR, error) {
	Cpath := C.CString(path)
	defer C.free(unsafe.Pointer(Cpath))
	dir := C.opendir(Cpath)
	if dir == nil {
		return nil, fmt.Errorf("cannot open dir: %s", path)
	}
	return dir, nil
}

func ffiCloseDir(dir *C.DIR) {
	if dir != nil {
		C.closedir(dir)
	}
}

func ffiGetDirFd(dir *C.DIR) uintptr {
	return uintptr(C.dirfd(dir))
}
<commit_msg>Go back to exec interfaces.<commit_after>// +build linux

/*

directory structure

  .
  |-- repositoryName
	  |-- scratch
		  |-- shardNum // the read-only read created on InitRepository, this is where to start branching
      |-- commitID
	      |-- shardNum // this is where subvolumes are

*/

package btrfs

/*
#include <stdlib.h>
#include <dirent.h>
#include <btrfs/ioctl.h>
*/
import "C"

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pfs/drive"
	"github.com/pachyderm/pachyderm/src/pkg/executil"
	"github.com/peter-edge/go-google-protobuf"
	"github.com/satori/go.uuid"
)

const (
	metadataDir = ".pfs"
)

var (
	callStrings = map[uintptr]string{
		C.BTRFS_IOC_SUBVOL_CREATE:  "btrfs subvolume create",
		C.BTRFS_IOC_SNAP_CREATE_V2: "btrfs subvolume snapshot",
	}
)

type driver struct {
	rootDir string
}

func newDriver(rootDir string) *driver {
	return &driver{rootDir}
}

func (d *driver) InitRepository(repository *pfs.Repository, shards map[int]bool) error {
	if err := os.MkdirAll(d.repositoryPath(repository), 0700); err != nil {
		return err
	}
	initialCommit := &pfs.Commit{
		Repository: repository,
		Id:         drive.InitialCommitID,
	}
	if err := os.MkdirAll(d.commitPathNoShard(initialCommit), 0700); err != nil {
		return err
	}
	for shard := range shards {
		if err := execSubvolumeCreate(d.commitPath(initialCommit, shard)); err != nil {
			return err
		}
		if err := os.Mkdir(d.filePath(&pfs.Path{Commit: initialCommit, Path: metadataDir}, shard), 0700); err != nil {
			return err
		}
		if err := d.setReadOnly(initialCommit, shard, true); err != nil {
			return err
		}
	}
	return nil
}

func (d *driver) GetFile(path *pfs.Path, shard int) (drive.ReaderAtCloser, error) {
	return os.Open(d.filePath(path, shard))
}

func (d *driver) GetFileInfo(path *pfs.Path, shard int) (_ *pfs.FileInfo, ok bool, _ error) {
	filePath, err := d.stat(path, shard)
	if err != nil && os.IsNotExist(err) {
		return nil, false, nil
	}
	if err != nil {
		return nil, false, err
	}
	return filePath, true, nil
}

func (d *driver) MakeDirectory(path *pfs.Path, shards map[int]bool) error {
	// TODO(pedge): if PutFile fails here or on another shard, the directories
	// will still exist and be returned from ListFiles, we want to do this
	// iteratively and with rollback
	// TODO(pedge): check that commit exists and is a write commit
	for shard := range shards {
		if err := d.checkWrite(path.Commit, shard); err != nil {
			return err
		}
		if err := os.MkdirAll(d.filePath(path, shard), 0700); err != nil {
			return err
		}
	}
	return nil
}

func (d *driver) PutFile(path *pfs.Path, shard int, offset int64, reader io.Reader) error {
	if err := d.checkWrite(path.Commit, shard); err != nil {
		return err
	}
	file, err := os.OpenFile(d.filePath(path, shard), os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	defer file.Close()
	if _, err := file.Seek(offset, 0); err != nil { // 0 means relative to start
		return err
	}
	_, err = bufio.NewReader(reader).WriteTo(file)
	return err
}

func (d *driver) ListFiles(path *pfs.Path, shard int) (_ []*pfs.FileInfo, retErr error) {
	filePath := d.filePath(path, shard)
	stat, err := os.Stat(filePath)
	if err != nil {
		return nil, err
	}
	if !stat.IsDir() {
		return nil, fmt.Errorf("%s is not a directory", filePath)
	}
	dir, err := os.Open(filePath)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err := dir.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	var fileInfos []*pfs.FileInfo
	// TODO(pedge): constant
	for names, err := dir.Readdirnames(100); err != io.EOF; names, err = dir.Readdirnames(100) {
		if err != nil {
			return nil, err
		}
		for _, name := range names {
			if inMetadataDir(name) {
				continue
			}
			fileInfo, err := d.stat(
				&pfs.Path{
					Commit: path.Commit,
					Path:   filepath.Join(path.Path, name),
				},
				shard,
			)
			if err != nil {
				return nil, err
			}
			fileInfos = append(fileInfos, fileInfo)
		}
	}
	return fileInfos, nil
}

func (d *driver) stat(path *pfs.Path, shard int) (*pfs.FileInfo, error) {
	stat, err := os.Stat(d.filePath(path, shard))
	if err != nil {
		return nil, err
	}
	fileType := pfs.FileType_FILE_TYPE_OTHER
	if stat.Mode().IsRegular() {
		fileType = pfs.FileType_FILE_TYPE_REGULAR
	}
	if stat.Mode().IsDir() {
		fileType = pfs.FileType_FILE_TYPE_DIR
	}
	return &pfs.FileInfo{
		Path:      path,
		FileType:  fileType,
		SizeBytes: uint64(stat.Size()),
		Perm:      uint32(stat.Mode() & os.ModePerm),
		LastModified: &google_protobuf.Timestamp{
			Seconds: stat.ModTime().UnixNano() / int64(time.Second),
			Nanos:   int32(stat.ModTime().UnixNano() % int64(time.Second)),
		},
	}, nil
}

func (d *driver) Branch(commit *pfs.Commit, newCommit *pfs.Commit, shards map[int]bool) (*pfs.Commit, error) {
	if newCommit == nil {
		newCommit = &pfs.Commit{
			Repository: commit.Repository,
			Id:         newCommitID(),
		}
	}
	if err := os.MkdirAll(d.commitPathNoShard(newCommit), 0700); err != nil {
		return nil, err
	}
	for shard := range shards {
		if err := d.checkReadOnly(commit, shard); err != nil {
			return nil, err
		}
		commitPath := d.commitPath(commit, shard)
		newCommitPath := d.commitPath(newCommit, shard)
		if err := execSubvolumeSnapshot(commitPath, newCommitPath); err != nil {
			return nil, err
		}
		if err := ioutil.WriteFile(d.filePath(&pfs.Path{Commit: newCommit, Path: filepath.Join(metadataDir, "parent")}, shard), []byte(commit.Id), 0600); err != nil {
			return nil, err
		}
		if err := d.setReadOnly(newCommit, shard, false); err != nil {
			return nil, err
		}
	}
	return newCommit, nil
}

func (d *driver) Commit(commit *pfs.Commit, shards map[int]bool) error {
	for shard := range shards {
		if err := d.checkWrite(commit, shard); err != nil {
			return err
		}
		if err := d.setReadOnly(commit, shard, true); err != nil {
			return err
		}
	}
	return nil
}

func (d *driver) PullDiff(commit *pfs.Commit, shard int) (io.Reader, error) {
	return nil, nil
}

func (d *driver) PushDiff(commit *pfs.Commit, shard int, reader io.Reader) error {
	return nil
}

func (d *driver) GetCommitInfo(commit *pfs.Commit, shard int) (_ *pfs.CommitInfo, ok bool, _ error) {
	_, err := os.Stat(d.commitPath(commit, shard))
	if err != nil && os.IsNotExist(err) {
		return nil, false, nil
	}
	if err != nil {
		return nil, false, err
	}
	parent, err := d.getParent(commit, shard)
	if err != nil {
		return nil, false, err
	}
	readOnly, err := d.getReadOnly(commit, shard)
	if err != nil {
		return nil, false, err
	}
	commitType := pfs.CommitType_COMMIT_TYPE_WRITE
	if readOnly {
		commitType = pfs.CommitType_COMMIT_TYPE_READ
	}
	return &pfs.CommitInfo{
		Commit:       commit,
		CommitType:   commitType,
		ParentCommit: parent,
	}, true, nil
}

func (d *driver) ListCommits(repository *pfs.Repository, shard int) (_ []*pfs.CommitInfo, retErr error) {

	dir, err := os.Open(d.repositoryPath(repository))
	if err != nil {
		return nil, err
	}
	defer func() {
		if err := dir.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	var commitInfos []*pfs.CommitInfo
	// TODO(pedge): constant
	for names, err := dir.Readdirnames(100); err != io.EOF; names, err = dir.Readdirnames(100) {
		if err != nil {
			return nil, err
		}
		for _, name := range names {
			commitInfo, ok, err := d.GetCommitInfo(
				&pfs.Commit{
					Repository: repository,
					Id:         path.Base(name),
				},
				shard,
			)
			if !ok {
				// This is a really weird error to get since we got this commit
				// name by listing commits. This is probably indicative of a
				// race condition.
				return nil, fmt.Errorf("Commit not found.")
			}
			if err != nil {
				return nil, err
			}
			commitInfos = append(commitInfos, commitInfo)
		}
	}
	return commitInfos, nil
}

func (d *driver) getParent(commit *pfs.Commit, shard int) (*pfs.Commit, error) {
	if commit.Id == drive.InitialCommitID {
		return nil, nil
	}
	data, err := ioutil.ReadFile(d.filePath(&pfs.Path{Commit: commit, Path: filepath.Join(metadataDir, "parent")}, shard))
	if err != nil {
		return nil, err
	}
	return &pfs.Commit{
		Repository: commit.Repository,
		Id:         string(data),
	}, nil
}

func (d *driver) checkReadOnly(commit *pfs.Commit, shard int) error {
	ok, err := d.getReadOnly(commit, shard)
	if err != nil {
		return err
	}
	if !ok {
		return fmt.Errorf("%+v is not a read only commit", commit)
	}
	return nil
}

func (d *driver) checkWrite(commit *pfs.Commit, shard int) error {
	ok, err := d.getReadOnly(commit, shard)
	if err != nil {
		return err
	}
	if ok {
		return fmt.Errorf("%+v is not a write commit", commit)
	}
	return nil
}

func (d *driver) getReadOnly(commit *pfs.Commit, shard int) (bool, error) {
	data, err := ioutil.ReadFile(d.filePath(&pfs.Path{Commit: commit, Path: filepath.Join(metadataDir, "readonly")}, shard))
	if err != nil {
		return false, err
	}
	switch string(data) {
	case "0":
		return false, nil
	case "1":
		return true, nil
	default:
		return false, fmt.Errorf("unknown data for readonly metadata file: %s", string(data))
	}
}

func (d *driver) setReadOnly(commit *pfs.Commit, shard int, readOnly bool) error {
	data := []byte("0")
	if readOnly {
		data = []byte("1")
	}
	return ioutil.WriteFile(d.filePath(&pfs.Path{Commit: commit, Path: filepath.Join(metadataDir, "readonly")}, shard), data, 0400)
}

func (d *driver) repositoryPath(repository *pfs.Repository) string {
	return filepath.Join(d.rootDir, repository.Name)
}

func (d *driver) commitPathNoShard(commit *pfs.Commit) string {
	return filepath.Join(d.repositoryPath(commit.Repository), commit.Id)
}

func (d *driver) commitPath(commit *pfs.Commit, shard int) string {
	return filepath.Join(d.commitPathNoShard(commit), fmt.Sprintf("%d", shard))
}

func (d *driver) filePath(path *pfs.Path, shard int) string {
	return filepath.Join(d.commitPath(path.Commit, shard), path.Path)
}

func newCommitID() string {
	return strings.Replace(uuid.NewV4().String(), "-", "", -1)
}

func inMetadataDir(name string) bool {
	parts := strings.Split(name, "/")
	return (len(parts) > 0 && parts[0] == metadataDir)
}

func execSubvolumeCreate(path string) error {
	return executil.Run("btrfs", "subvolume", "create", path)
}

func execSubvolumeSnapshot(src string, dest string) error {
	return executil.Run("btrfs", "subvolume", "snapshot", src, dest)
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package x509

import (
	"strings"
	"time"
	"unicode/utf8"
)

type InvalidReason int

const (
	// NotAuthorizedToSign results when a certificate is signed by another
	// which isn't marked as a CA certificate.
	NotAuthorizedToSign InvalidReason = iota
	// Expired results when a certificate has expired, based on the time
	// given in the VerifyOptions.
	Expired
	// CANotAuthorizedForThisName results when an intermediate or root
	// certificate has a name constraint which doesn't include the name
	// being checked.
	CANotAuthorizedForThisName
)

// CertificateInvalidError results when an odd error occurs. Users of this
// library probably want to handle all these errors uniformly.
type CertificateInvalidError struct {
	Cert   *Certificate
	Reason InvalidReason
}

func (e CertificateInvalidError) Error() string {
	switch e.Reason {
	case NotAuthorizedToSign:
		return "x509: certificate is not authorized to sign other other certificates"
	case Expired:
		return "x509: certificate has expired or is not yet valid"
	case CANotAuthorizedForThisName:
		return "x509: a root or intermediate certificate is not authorized to sign in this domain"
	}
	return "x509: unknown error"
}

// HostnameError results when the set of authorized names doesn't match the
// requested name.
type HostnameError struct {
	Certificate *Certificate
	Host        string
}

func (h HostnameError) Error() string {
	var valid string
	c := h.Certificate
	if len(c.DNSNames) > 0 {
		valid = strings.Join(c.DNSNames, ", ")
	} else {
		valid = c.Subject.CommonName
	}
	return "certificate is valid for " + valid + ", not " + h.Host
}

// UnknownAuthorityError results when the certificate issuer is unknown
type UnknownAuthorityError struct {
	cert *Certificate
}

func (e UnknownAuthorityError) Error() string {
	return "x509: certificate signed by unknown authority"
}

// VerifyOptions contains parameters for Certificate.Verify. It's a structure
// because other PKIX verification APIs have ended up needing many options.
type VerifyOptions struct {
	DNSName       string
	Intermediates *CertPool
	Roots         *CertPool
	CurrentTime   time.Time // if zero, the current time is used
}

const (
	leafCertificate = iota
	intermediateCertificate
	rootCertificate
)

// isValid performs validity checks on the c.
func (c *Certificate) isValid(certType int, opts *VerifyOptions) error {
	now := opts.CurrentTime
	if now.IsZero() {
		now = time.Now()
	}
	if now.Before(c.NotBefore) || now.After(c.NotAfter) {
		return CertificateInvalidError{c, Expired}
	}

	if len(c.PermittedDNSDomains) > 0 {
		for _, domain := range c.PermittedDNSDomains {
			if opts.DNSName == domain ||
				(strings.HasSuffix(opts.DNSName, domain) &&
					len(opts.DNSName) >= 1+len(domain) &&
					opts.DNSName[len(opts.DNSName)-len(domain)-1] == '.') {
				continue
			}

			return CertificateInvalidError{c, CANotAuthorizedForThisName}
		}
	}

	// KeyUsage status flags are ignored. From Engineering Security, Peter
	// Gutmann: A European government CA marked its signing certificates as
	// being valid for encryption only, but no-one noticed. Another
	// European CA marked its signature keys as not being valid for
	// signatures. A different CA marked its own trusted root certificate
	// as being invalid for certificate signing.  Another national CA
	// distributed a certificate to be used to encrypt data for the
	// country’s tax authority that was marked as only being usable for
	// digital signatures but not for encryption. Yet another CA reversed
	// the order of the bit flags in the keyUsage due to confusion over
	// encoding endianness, essentially setting a random keyUsage in
	// certificates that it issued. Another CA created a self-invalidating
	// certificate by adding a certificate policy statement stipulating
	// that the certificate had to be used strictly as specified in the
	// keyUsage, and a keyUsage containing a flag indicating that the RSA
	// encryption key could only be used for Diffie-Hellman key agreement.

	if certType == intermediateCertificate && (!c.BasicConstraintsValid || !c.IsCA) {
		return CertificateInvalidError{c, NotAuthorizedToSign}
	}

	return nil
}

// Verify attempts to verify c by building one or more chains from c to a
// certificate in opts.roots, using certificates in opts.Intermediates if
// needed. If successful, it returns one or chains where the first element of
// the chain is c and the last element is from opts.Roots.
//
// WARNING: this doesn't do any revocation checking.
func (c *Certificate) Verify(opts VerifyOptions) (chains [][]*Certificate, err error) {
	err = c.isValid(leafCertificate, &opts)
	if err != nil {
		return
	}
	if len(opts.DNSName) > 0 {
		err = c.VerifyHostname(opts.DNSName)
		if err != nil {
			return
		}
	}
	return c.buildChains(make(map[int][][]*Certificate), []*Certificate{c}, &opts)
}

func appendToFreshChain(chain []*Certificate, cert *Certificate) []*Certificate {
	n := make([]*Certificate, len(chain)+1)
	copy(n, chain)
	n[len(chain)] = cert
	return n
}

func (c *Certificate) buildChains(cache map[int][][]*Certificate, currentChain []*Certificate, opts *VerifyOptions) (chains [][]*Certificate, err error) {
	for _, rootNum := range opts.Roots.findVerifiedParents(c) {
		root := opts.Roots.certs[rootNum]
		err = root.isValid(rootCertificate, opts)
		if err != nil {
			continue
		}
		chains = append(chains, appendToFreshChain(currentChain, root))
	}

nextIntermediate:
	for _, intermediateNum := range opts.Intermediates.findVerifiedParents(c) {
		intermediate := opts.Intermediates.certs[intermediateNum]
		for _, cert := range currentChain {
			if cert == intermediate {
				continue nextIntermediate
			}
		}
		err = intermediate.isValid(intermediateCertificate, opts)
		if err != nil {
			continue
		}
		var childChains [][]*Certificate
		childChains, ok := cache[intermediateNum]
		if !ok {
			childChains, err = intermediate.buildChains(cache, appendToFreshChain(currentChain, intermediate), opts)
			cache[intermediateNum] = childChains
		}
		chains = append(chains, childChains...)
	}

	if len(chains) > 0 {
		err = nil
	}

	if len(chains) == 0 && err == nil {
		err = UnknownAuthorityError{c}
	}

	return
}

func matchHostnames(pattern, host string) bool {
	if len(pattern) == 0 || len(host) == 0 {
		return false
	}

	patternParts := strings.Split(pattern, ".")
	hostParts := strings.Split(host, ".")

	if len(patternParts) != len(hostParts) {
		return false
	}

	for i, patternPart := range patternParts {
		if patternPart == "*" {
			continue
		}
		if patternPart != hostParts[i] {
			return false
		}
	}

	return true
}

// toLowerCaseASCII returns a lower-case version of in. See RFC 6125 6.4.1. We use
// an explicitly ASCII function to avoid any sharp corners resulting from
// performing Unicode operations on DNS labels.
func toLowerCaseASCII(in string) string {
	// If the string is already lower-case then there's nothing to do.
	isAlreadyLowerCase := true
	for _, c := range in {
		if c == utf8.RuneError {
			// If we get a UTF-8 error then there might be
			// upper-case ASCII bytes in the invalid sequence.
			isAlreadyLowerCase = false
			break
		}
		if 'A' <= c && c <= 'Z' {
			isAlreadyLowerCase = false
			break
		}
	}

	if isAlreadyLowerCase {
		return in
	}

	out := []byte(in)
	for i, c := range out {
		if 'A' <= c && c <= 'Z' {
			out[i] += 'a' - 'A'
		}
	}
	return string(out)
}

// VerifyHostname returns nil if c is a valid certificate for the named host.
// Otherwise it returns an error describing the mismatch.
func (c *Certificate) VerifyHostname(h string) error {
	lowered := toLowerCaseASCII(h)

	if len(c.DNSNames) > 0 {
		for _, match := range c.DNSNames {
			if matchHostnames(toLowerCaseASCII(match), lowered) {
				return nil
			}
		}
		// If Subject Alt Name is given, we ignore the common name.
	} else if matchHostnames(toLowerCaseASCII(c.Subject.CommonName), lowered) {
		return nil
	}

	return HostnameError{c, h}
}
<commit_msg>crypto/x509: fix typo in Verify documentation<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package x509

import (
	"strings"
	"time"
	"unicode/utf8"
)

type InvalidReason int

const (
	// NotAuthorizedToSign results when a certificate is signed by another
	// which isn't marked as a CA certificate.
	NotAuthorizedToSign InvalidReason = iota
	// Expired results when a certificate has expired, based on the time
	// given in the VerifyOptions.
	Expired
	// CANotAuthorizedForThisName results when an intermediate or root
	// certificate has a name constraint which doesn't include the name
	// being checked.
	CANotAuthorizedForThisName
)

// CertificateInvalidError results when an odd error occurs. Users of this
// library probably want to handle all these errors uniformly.
type CertificateInvalidError struct {
	Cert   *Certificate
	Reason InvalidReason
}

func (e CertificateInvalidError) Error() string {
	switch e.Reason {
	case NotAuthorizedToSign:
		return "x509: certificate is not authorized to sign other other certificates"
	case Expired:
		return "x509: certificate has expired or is not yet valid"
	case CANotAuthorizedForThisName:
		return "x509: a root or intermediate certificate is not authorized to sign in this domain"
	}
	return "x509: unknown error"
}

// HostnameError results when the set of authorized names doesn't match the
// requested name.
type HostnameError struct {
	Certificate *Certificate
	Host        string
}

func (h HostnameError) Error() string {
	var valid string
	c := h.Certificate
	if len(c.DNSNames) > 0 {
		valid = strings.Join(c.DNSNames, ", ")
	} else {
		valid = c.Subject.CommonName
	}
	return "certificate is valid for " + valid + ", not " + h.Host
}

// UnknownAuthorityError results when the certificate issuer is unknown
type UnknownAuthorityError struct {
	cert *Certificate
}

func (e UnknownAuthorityError) Error() string {
	return "x509: certificate signed by unknown authority"
}

// VerifyOptions contains parameters for Certificate.Verify. It's a structure
// because other PKIX verification APIs have ended up needing many options.
type VerifyOptions struct {
	DNSName       string
	Intermediates *CertPool
	Roots         *CertPool
	CurrentTime   time.Time // if zero, the current time is used
}

const (
	leafCertificate = iota
	intermediateCertificate
	rootCertificate
)

// isValid performs validity checks on the c.
func (c *Certificate) isValid(certType int, opts *VerifyOptions) error {
	now := opts.CurrentTime
	if now.IsZero() {
		now = time.Now()
	}
	if now.Before(c.NotBefore) || now.After(c.NotAfter) {
		return CertificateInvalidError{c, Expired}
	}

	if len(c.PermittedDNSDomains) > 0 {
		for _, domain := range c.PermittedDNSDomains {
			if opts.DNSName == domain ||
				(strings.HasSuffix(opts.DNSName, domain) &&
					len(opts.DNSName) >= 1+len(domain) &&
					opts.DNSName[len(opts.DNSName)-len(domain)-1] == '.') {
				continue
			}

			return CertificateInvalidError{c, CANotAuthorizedForThisName}
		}
	}

	// KeyUsage status flags are ignored. From Engineering Security, Peter
	// Gutmann: A European government CA marked its signing certificates as
	// being valid for encryption only, but no-one noticed. Another
	// European CA marked its signature keys as not being valid for
	// signatures. A different CA marked its own trusted root certificate
	// as being invalid for certificate signing.  Another national CA
	// distributed a certificate to be used to encrypt data for the
	// country’s tax authority that was marked as only being usable for
	// digital signatures but not for encryption. Yet another CA reversed
	// the order of the bit flags in the keyUsage due to confusion over
	// encoding endianness, essentially setting a random keyUsage in
	// certificates that it issued. Another CA created a self-invalidating
	// certificate by adding a certificate policy statement stipulating
	// that the certificate had to be used strictly as specified in the
	// keyUsage, and a keyUsage containing a flag indicating that the RSA
	// encryption key could only be used for Diffie-Hellman key agreement.

	if certType == intermediateCertificate && (!c.BasicConstraintsValid || !c.IsCA) {
		return CertificateInvalidError{c, NotAuthorizedToSign}
	}

	return nil
}

// Verify attempts to verify c by building one or more chains from c to a
// certificate in opts.roots, using certificates in opts.Intermediates if
// needed. If successful, it returns one or more chains where the first
// element of the chain is c and the last element is from opts.Roots.
//
// WARNING: this doesn't do any revocation checking.
func (c *Certificate) Verify(opts VerifyOptions) (chains [][]*Certificate, err error) {
	err = c.isValid(leafCertificate, &opts)
	if err != nil {
		return
	}
	if len(opts.DNSName) > 0 {
		err = c.VerifyHostname(opts.DNSName)
		if err != nil {
			return
		}
	}
	return c.buildChains(make(map[int][][]*Certificate), []*Certificate{c}, &opts)
}

func appendToFreshChain(chain []*Certificate, cert *Certificate) []*Certificate {
	n := make([]*Certificate, len(chain)+1)
	copy(n, chain)
	n[len(chain)] = cert
	return n
}

func (c *Certificate) buildChains(cache map[int][][]*Certificate, currentChain []*Certificate, opts *VerifyOptions) (chains [][]*Certificate, err error) {
	for _, rootNum := range opts.Roots.findVerifiedParents(c) {
		root := opts.Roots.certs[rootNum]
		err = root.isValid(rootCertificate, opts)
		if err != nil {
			continue
		}
		chains = append(chains, appendToFreshChain(currentChain, root))
	}

nextIntermediate:
	for _, intermediateNum := range opts.Intermediates.findVerifiedParents(c) {
		intermediate := opts.Intermediates.certs[intermediateNum]
		for _, cert := range currentChain {
			if cert == intermediate {
				continue nextIntermediate
			}
		}
		err = intermediate.isValid(intermediateCertificate, opts)
		if err != nil {
			continue
		}
		var childChains [][]*Certificate
		childChains, ok := cache[intermediateNum]
		if !ok {
			childChains, err = intermediate.buildChains(cache, appendToFreshChain(currentChain, intermediate), opts)
			cache[intermediateNum] = childChains
		}
		chains = append(chains, childChains...)
	}

	if len(chains) > 0 {
		err = nil
	}

	if len(chains) == 0 && err == nil {
		err = UnknownAuthorityError{c}
	}

	return
}

func matchHostnames(pattern, host string) bool {
	if len(pattern) == 0 || len(host) == 0 {
		return false
	}

	patternParts := strings.Split(pattern, ".")
	hostParts := strings.Split(host, ".")

	if len(patternParts) != len(hostParts) {
		return false
	}

	for i, patternPart := range patternParts {
		if patternPart == "*" {
			continue
		}
		if patternPart != hostParts[i] {
			return false
		}
	}

	return true
}

// toLowerCaseASCII returns a lower-case version of in. See RFC 6125 6.4.1. We use
// an explicitly ASCII function to avoid any sharp corners resulting from
// performing Unicode operations on DNS labels.
func toLowerCaseASCII(in string) string {
	// If the string is already lower-case then there's nothing to do.
	isAlreadyLowerCase := true
	for _, c := range in {
		if c == utf8.RuneError {
			// If we get a UTF-8 error then there might be
			// upper-case ASCII bytes in the invalid sequence.
			isAlreadyLowerCase = false
			break
		}
		if 'A' <= c && c <= 'Z' {
			isAlreadyLowerCase = false
			break
		}
	}

	if isAlreadyLowerCase {
		return in
	}

	out := []byte(in)
	for i, c := range out {
		if 'A' <= c && c <= 'Z' {
			out[i] += 'a' - 'A'
		}
	}
	return string(out)
}

// VerifyHostname returns nil if c is a valid certificate for the named host.
// Otherwise it returns an error describing the mismatch.
func (c *Certificate) VerifyHostname(h string) error {
	lowered := toLowerCaseASCII(h)

	if len(c.DNSNames) > 0 {
		for _, match := range c.DNSNames {
			if matchHostnames(toLowerCaseASCII(match), lowered) {
				return nil
			}
		}
		// If Subject Alt Name is given, we ignore the common name.
	} else if matchHostnames(toLowerCaseASCII(c.Subject.CommonName), lowered) {
		return nil
	}

	return HostnameError{c, h}
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"testing"
	"text/template"
)

type crashTest struct {
	Cgo bool
}

// This test is a separate program, because it is testing
// both main (m0) and non-main threads (m).

func testCrashHandler(t *testing.T, ct *crashTest) {
	if runtime.GOOS == "freebsd" {
		// TODO(brainman): do not know why this test fails on freebsd
		t.Logf("skipping test on %q", runtime.GOOS)
		return
	}

	st := template.Must(template.New("crashSource").Parse(crashSource))

	dir, err := ioutil.TempDir("", "go-build")
	if err != nil {
		t.Fatalf("failed to create temp directory: %v", err)
	}
	defer os.RemoveAll(dir)

	src := filepath.Join(dir, "main.go")
	f, err := os.Create(src)
	if err != nil {
		t.Fatalf("failed to create %v: %v", src, err)
	}
	err = st.Execute(f, ct)
	if err != nil {
		f.Close()
		t.Fatalf("failed to execute template: %v", err)
	}
	f.Close()

	got, err := exec.Command("go", "run", src).CombinedOutput()
	if err != nil {
		t.Fatalf("program exited with error: %v\n%v", err, string(got))
	}
	want := "main: recovered done\nnew-thread: recovered done\nsecond-new-thread: recovered done\nmain-again: recovered done\n"
	if string(got) != string(want) {
		t.Fatalf("expected %q, but got %q", string(want), string(got))
	}
}

func TestCrashHandler(t *testing.T) {
	testCrashHandler(t, &crashTest{Cgo: false})
}

const crashSource = `
package main

import (
	"fmt"
	"runtime"
)

{{if .Cgo}}
import "C"
{{end}}

func test(name string) {
	defer func() {
		if x := recover(); x != nil {
			fmt.Printf(" recovered")
		}
		fmt.Printf(" done\n")
	}()
	fmt.Printf("%s:", name)
	var s *string
	_ = *s
	fmt.Print("SHOULD NOT BE HERE")
}

func testInNewThread(name string) {
	c := make(chan bool)
	go func() {
		runtime.LockOSThread()
		test(name)
		c <- true
	}()
	<-c
}

func main() {
	runtime.LockOSThread()
	test("main")
	testInNewThread("new-thread")
	testInNewThread("second-new-thread")
	test("main-again")
}
`
<commit_msg>runtime: disable crash handler test on netbsd<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"testing"
	"text/template"
)

type crashTest struct {
	Cgo bool
}

// This test is a separate program, because it is testing
// both main (m0) and non-main threads (m).

func testCrashHandler(t *testing.T, ct *crashTest) {
	if runtime.GOOS == "freebsd" || runtime.GOOS == "netbsd" {
		// TODO(brainman): do not know why this test fails on freebsd
		// TODO(jsing): figure out why this causes delayed failures
		// on NetBSD - http://golang.org/issue/3954
		t.Logf("skipping test on %q", runtime.GOOS)
		return
	}

	st := template.Must(template.New("crashSource").Parse(crashSource))

	dir, err := ioutil.TempDir("", "go-build")
	if err != nil {
		t.Fatalf("failed to create temp directory: %v", err)
	}
	defer os.RemoveAll(dir)

	src := filepath.Join(dir, "main.go")
	f, err := os.Create(src)
	if err != nil {
		t.Fatalf("failed to create %v: %v", src, err)
	}
	err = st.Execute(f, ct)
	if err != nil {
		f.Close()
		t.Fatalf("failed to execute template: %v", err)
	}
	f.Close()

	got, err := exec.Command("go", "run", src).CombinedOutput()
	if err != nil {
		t.Fatalf("program exited with error: %v\n%v", err, string(got))
	}
	want := "main: recovered done\nnew-thread: recovered done\nsecond-new-thread: recovered done\nmain-again: recovered done\n"
	if string(got) != string(want) {
		t.Fatalf("expected %q, but got %q", string(want), string(got))
	}
}

func TestCrashHandler(t *testing.T) {
	testCrashHandler(t, &crashTest{Cgo: false})
}

const crashSource = `
package main

import (
	"fmt"
	"runtime"
)

{{if .Cgo}}
import "C"
{{end}}

func test(name string) {
	defer func() {
		if x := recover(); x != nil {
			fmt.Printf(" recovered")
		}
		fmt.Printf(" done\n")
	}()
	fmt.Printf("%s:", name)
	var s *string
	_ = *s
	fmt.Print("SHOULD NOT BE HERE")
}

func testInNewThread(name string) {
	c := make(chan bool)
	go func() {
		runtime.LockOSThread()
		test(name)
		c <- true
	}()
	<-c
}

func main() {
	runtime.LockOSThread()
	test("main")
	testInNewThread("new-thread")
	testInNewThread("second-new-thread")
	test("main-again")
}
`
<|endoftext|>