<commit_before>package web

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"net/http/httputil"
	"regexp"
	"sort"
	"strings"
	"time"

	log "github.com/golang/glog"
	"github.com/gorilla/mux"
	"github.com/micro/cli"
	"github.com/micro/go-micro"
	"github.com/micro/go-micro/cmd"
	"github.com/micro/go-micro/registry"
	"github.com/micro/go-micro/selector"
	"github.com/micro/micro/internal/handler"
	"github.com/micro/micro/internal/server"
	"github.com/serenize/snaker"
)

var (
	re = regexp.MustCompile("^[a-zA-Z0-9]+$")
	// Default address to bind to
	Address = ":8082"
	// The namespace to serve
	// Example:
	// Namespace + /[Service]/foo/bar
	// Host: Namespace.Service Endpoint: /foo/bar
	Namespace = "go.micro.web"
	// Base path sent to web service.
	// This is stripped from the request path
	// Allows the web service to define absolute paths
	BasePathHeader = "X-Micro-Web-Base-Path"

	CORS = map[string]bool{"*": true}
)

type srv struct {
	*mux.Router
}

func (s *srv) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if origin := r.Header.Get("Origin"); CORS[origin] {
		w.Header().Set("Access-Control-Allow-Origin", origin)
	} else if len(origin) > 0 && CORS["*"] {
		w.Header().Set("Access-Control-Allow-Origin", origin)
	}

	w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
	w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token")
	w.Header().Set("Access-Control-Allow-Credentials", "true")

	if r.Method == "OPTIONS" {
		return
	}

	s.Router.ServeHTTP(w, r)
}

func (s *srv) proxy() http.Handler {
	sel := selector.NewSelector(
		selector.Registry((*cmd.DefaultOptions().Registry)),
	)

	director := func(r *http.Request) {
		kill := func() {
			r.URL.Host = ""
			r.URL.Path = ""
			r.URL.Scheme = ""
		}

		parts := strings.Split(r.URL.Path, "/")
		if len(parts) < 2 {
			kill()
			return
		}
		if !re.MatchString(parts[1]) {
			kill()
			return
		}
		next, err := sel.Select(Namespace + "." + parts[1])
		if err != nil {
			kill()
			return
		}
		r.URL.Scheme = "http"
		s, err := next()
		if err != nil {
			kill()
			return
		}

		r.Header.Set(BasePathHeader, "/"+parts[1])
		r.URL.Host = fmt.Sprintf("%s:%d", s.Address, s.Port)
		r.URL.Path = "/" + strings.Join(parts[2:], "/")
	}

	return &proxy{
		Default:  &httputil.ReverseProxy{Director: director},
		Director: director,
	}
	/*
		return &httputil.ReverseProxy{
			Director: director,
		}
	*/
}

func format(v *registry.Value) string {
	if v == nil || len(v.Values) == 0 {
		return "{}"
	}
	var f []string
	for _, k := range v.Values {
		f = append(f, formatEndpoint(k, 0))
	}
	return fmt.Sprintf("{\n%s}", strings.Join(f, ""))
}

func formatEndpoint(v *registry.Value, r int) string {
	// default format is tabbed plus the value plus new line
	fparts := []string{"", "%s %s", "\n"}
	for i := 0; i < r+1; i++ {
		fparts[0] += "\t"
	}
	// its just a primitive of sorts so return
	if len(v.Values) == 0 {
		return fmt.Sprintf(strings.Join(fparts, ""), snaker.CamelToSnake(v.Name), v.Type)
	}

	// this thing has more things, it's complex
	fparts[1] += " {"

	vals := []interface{}{snaker.CamelToSnake(v.Name), v.Type}

	for _, val := range v.Values {
		fparts = append(fparts, "%s")
		vals = append(vals, formatEndpoint(val, r+1))
	}

	// at the end
	l := len(fparts) - 1
	for i := 0; i < r+1; i++ {
		fparts[l] += "\t"
	}
	fparts = append(fparts, "}\n")

	return fmt.Sprintf(strings.Join(fparts, ""), vals...)
}

func faviconHandler(w http.ResponseWriter, r *http.Request) {
	return
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	services, err := (*cmd.DefaultOptions().Registry).ListServices()
	if err != nil {
		http.Error(w, "Error occurred:"+err.Error(), 500)
		return
	}

	var webServices []string
	for _, s := range services {
		if strings.Index(s.Name, Namespace) == 0 && len(strings.TrimPrefix(s.Name, Namespace)) > 0 {
			webServices = append(webServices, strings.Replace(s.Name, Namespace+".", "", 1))
		}
	}

	sort.Strings(webServices)

	type templateData struct {
		HasWebServices bool
		WebServices    []string
	}

	data := templateData{len(webServices) > 0, webServices}
	render(w, r, indexTemplate, data)
}

func registryHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	svc := r.Form.Get("service")

	if len(svc) > 0 {
		s, err := (*cmd.DefaultOptions().Registry).GetService(svc)
		if err != nil {
			http.Error(w, "Error occurred:"+err.Error(), 500)
			return
		}

		if len(s) == 0 {
			http.Error(w, "Not found", 404)
			return
		}

		if r.Header.Get("Content-Type") == "application/json" {
			b, err := json.Marshal(map[string]interface{}{
				"services": s,
			})
			if err != nil {
				http.Error(w, "Error occurred:"+err.Error(), 500)
				return
			}
			w.Header().Set("Content-Type", "application/json")
			w.Write(b)
			return
		}

		render(w, r, serviceTemplate, s)
		return
	}

	services, err := (*cmd.DefaultOptions().Registry).ListServices()
	if err != nil {
		http.Error(w, "Error occurred:"+err.Error(), 500)
		return
	}

	sort.Sort(sortedServices{services})

	if r.Header.Get("Content-Type") == "application/json" {
		b, err := json.Marshal(map[string]interface{}{
			"services": services,
		})
		if err != nil {
			http.Error(w, "Error occurred:"+err.Error(), 500)
			return
		}
		w.Header().Set("Content-Type", "application/json")
		w.Write(b)
		return
	}

	render(w, r, registryTemplate, services)
}

func queryHandler(w http.ResponseWriter, r *http.Request) {
	render(w, r, queryTemplate, nil)
}

func render(w http.ResponseWriter, r *http.Request, tmpl string, data interface{}) {
	t, err := template.New("template").Funcs(template.FuncMap{
		"format": format,
	}).Parse(layoutTemplate)
	if err != nil {
		http.Error(w, "Error occurred:"+err.Error(), 500)
		return
	}
	t, err = t.Parse(tmpl)
	if err != nil {
		http.Error(w, "Error occurred:"+err.Error(), 500)
		return
	}
	if err := t.ExecuteTemplate(w, "layout", data); err != nil {
		http.Error(w, "Error occurred:"+err.Error(), 500)
	}
}

func run(ctx *cli.Context) {
	r := mux.NewRouter()
	s := &srv{r}
	s.HandleFunc("/registry", registryHandler)
	s.HandleFunc("/rpc", handler.RPC)
	s.HandleFunc("/query", queryHandler)
	s.HandleFunc("/favicon.ico", faviconHandler)
	s.PathPrefix("/{service:[a-zA-Z0-9]+}").Handler(s.proxy())
	s.HandleFunc("/", indexHandler)

	var opts []server.Option

	if ctx.GlobalBool("enable_tls") {
		cert := ctx.GlobalString("tls_cert_file")
		key := ctx.GlobalString("tls_key_file")

		if len(cert) > 0 && len(key) > 0 {
			certs, err := tls.LoadX509KeyPair(cert, key)
			if err != nil {
				fmt.Println(err.Error())
				return
			}
			config := &tls.Config{
				Certificates: []tls.Certificate{certs},
			}
			opts = append(opts, server.EnableTLS(true))
			opts = append(opts, server.TLSConfig(config))
		} else {
			fmt.Println("Enable TLS specified without certificate and key files")
			return
		}
	}

	srv := server.NewServer(Address)
	srv.Init(opts...)
	srv.Handle("/", s)

	// Initialise Server
	service := micro.NewService(
		micro.Name("go.micro.web"),
		micro.RegisterTTL(
			time.Duration(ctx.GlobalInt("register_ttl"))*time.Second,
		),
		micro.RegisterInterval(
			time.Duration(ctx.GlobalInt("register_interval"))*time.Second,
		),
	)

	if err := srv.Start(); err != nil {
		log.Fatal(err)
	}

	// Run server
	if err := service.Run(); err != nil {
		log.Fatal(err)
	}

	if err := srv.Stop(); err != nil {
		log.Fatal(err)
	}
}

func Commands() []cli.Command {
	return []cli.Command{
		{
			Name:  "web",
			Usage: "Run the micro web app",
			Action: func(c *cli.Context) {
				run(c)
			},
		},
	}
}
<commit_msg>Why wont you die<commit_after>package web

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"net/http/httputil"
	"regexp"
	"sort"
	"strings"
	"time"

	log "github.com/golang/glog"
	"github.com/gorilla/mux"
	"github.com/micro/cli"
	"github.com/micro/go-micro"
	"github.com/micro/go-micro/cmd"
	"github.com/micro/go-micro/registry"
	"github.com/micro/go-micro/selector"
	"github.com/micro/micro/internal/handler"
	"github.com/micro/micro/internal/server"
	"github.com/serenize/snaker"
)

var (
	re = regexp.MustCompile("^[a-zA-Z0-9]+$")
	// Default address to bind to
	Address = ":8082"
	// The namespace to serve
	// Example:
	// Namespace + /[Service]/foo/bar
	// Host: Namespace.Service Endpoint: /foo/bar
	Namespace = "go.micro.web"
	// Base path sent to web service.
	// This is stripped from the request path
	// Allows the web service to define absolute paths
	BasePathHeader = "X-Micro-Web-Base-Path"

	CORS = map[string]bool{"*": true}
)

type srv struct {
	*mux.Router
}

func (s *srv) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if origin := r.Header.Get("Origin"); CORS[origin] {
		w.Header().Set("Access-Control-Allow-Origin", origin)
	} else if len(origin) > 0 && CORS["*"] {
		w.Header().Set("Access-Control-Allow-Origin", origin)
	}

	w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
	w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token")
	w.Header().Set("Access-Control-Allow-Credentials", "true")

	if r.Method == "OPTIONS" {
		return
	}

	s.Router.ServeHTTP(w, r)
}

func (s *srv) proxy() http.Handler {
	sel := selector.NewSelector(
		selector.Registry((*cmd.DefaultOptions().Registry)),
	)

	director := func(r *http.Request) {
		kill := func() {
			r.URL.Host = ""
			r.URL.Path = ""
			r.URL.Scheme = ""
			r.Host = ""
			r.RequestURI = ""
		}

		parts := strings.Split(r.URL.Path, "/")
		if len(parts) < 2 {
			kill()
			return
		}
		if !re.MatchString(parts[1]) {
			kill()
			return
		}
		next, err := sel.Select(Namespace + "." + parts[1])
		if err != nil {
			kill()
			return
		}

		s, err := next()
		if err != nil {
			kill()
			return
		}

		r.Header.Set(BasePathHeader, "/"+parts[1])
		r.URL.Host = fmt.Sprintf("%s:%d", s.Address, s.Port)
		r.URL.Path = "/" + strings.Join(parts[2:], "/")
		r.URL.Scheme = "http"
	}

	return &proxy{
		Default:  &httputil.ReverseProxy{Director: director},
		Director: director,
	}
	/*
		return &httputil.ReverseProxy{
			Director: director,
		}
	*/
}

func format(v *registry.Value) string {
	if v == nil || len(v.Values) == 0 {
		return "{}"
	}
	var f []string
	for _, k := range v.Values {
		f = append(f, formatEndpoint(k, 0))
	}
	return fmt.Sprintf("{\n%s}", strings.Join(f, ""))
}

func formatEndpoint(v *registry.Value, r int) string {
	// default format is tabbed plus the value plus new line
	fparts := []string{"", "%s %s", "\n"}
	for i := 0; i < r+1; i++ {
		fparts[0] += "\t"
	}
	// its just a primitive of sorts so return
	if len(v.Values) == 0 {
		return fmt.Sprintf(strings.Join(fparts, ""), snaker.CamelToSnake(v.Name), v.Type)
	}

	// this thing has more things, it's complex
	fparts[1] += " {"

	vals := []interface{}{snaker.CamelToSnake(v.Name), v.Type}

	for _, val := range v.Values {
		fparts = append(fparts, "%s")
		vals = append(vals, formatEndpoint(val, r+1))
	}

	// at the end
	l := len(fparts) - 1
	for i := 0; i < r+1; i++ {
		fparts[l] += "\t"
	}
	fparts = append(fparts, "}\n")

	return fmt.Sprintf(strings.Join(fparts, ""), vals...)
}

func faviconHandler(w http.ResponseWriter, r *http.Request) {
	return
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	services, err := (*cmd.DefaultOptions().Registry).ListServices()
	if err != nil {
		http.Error(w, "Error occurred:"+err.Error(), 500)
		return
	}

	var webServices []string
	for _, s := range services {
		if strings.Index(s.Name, Namespace) == 0 && len(strings.TrimPrefix(s.Name, Namespace)) > 0 {
			webServices = append(webServices, strings.Replace(s.Name, Namespace+".", "", 1))
		}
	}

	sort.Strings(webServices)

	type templateData struct {
		HasWebServices bool
		WebServices    []string
	}

	data := templateData{len(webServices) > 0, webServices}
	render(w, r, indexTemplate, data)
}

func registryHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	svc := r.Form.Get("service")

	if len(svc) > 0 {
		s, err := (*cmd.DefaultOptions().Registry).GetService(svc)
		if err != nil {
			http.Error(w, "Error occurred:"+err.Error(), 500)
			return
		}

		if len(s) == 0 {
			http.Error(w, "Not found", 404)
			return
		}

		if r.Header.Get("Content-Type") == "application/json" {
			b, err := json.Marshal(map[string]interface{}{
				"services": s,
			})
			if err != nil {
				http.Error(w, "Error occurred:"+err.Error(), 500)
				return
			}
			w.Header().Set("Content-Type", "application/json")
			w.Write(b)
			return
		}

		render(w, r, serviceTemplate, s)
		return
	}

	services, err := (*cmd.DefaultOptions().Registry).ListServices()
	if err != nil {
		http.Error(w, "Error occurred:"+err.Error(), 500)
		return
	}

	sort.Sort(sortedServices{services})

	if r.Header.Get("Content-Type") == "application/json" {
		b, err := json.Marshal(map[string]interface{}{
			"services": services,
		})
		if err != nil {
			http.Error(w, "Error occurred:"+err.Error(), 500)
			return
		}
		w.Header().Set("Content-Type", "application/json")
		w.Write(b)
		return
	}

	render(w, r, registryTemplate, services)
}

func queryHandler(w http.ResponseWriter, r *http.Request) {
	render(w, r, queryTemplate, nil)
}

func render(w http.ResponseWriter, r *http.Request, tmpl string, data interface{}) {
	t, err := template.New("template").Funcs(template.FuncMap{
		"format": format,
	}).Parse(layoutTemplate)
	if err != nil {
		http.Error(w, "Error occurred:"+err.Error(), 500)
		return
	}
	t, err = t.Parse(tmpl)
	if err != nil {
		http.Error(w, "Error occurred:"+err.Error(), 500)
		return
	}
	if err := t.ExecuteTemplate(w, "layout", data); err != nil {
		http.Error(w, "Error occurred:"+err.Error(), 500)
	}
}

func run(ctx *cli.Context) {
	r := mux.NewRouter()
	s := &srv{r}
	s.HandleFunc("/registry", registryHandler)
	s.HandleFunc("/rpc", handler.RPC)
	s.HandleFunc("/query", queryHandler)
	s.HandleFunc("/favicon.ico", faviconHandler)
	s.PathPrefix("/{service:[a-zA-Z0-9]+}").Handler(s.proxy())
	s.HandleFunc("/", indexHandler)

	var opts []server.Option

	if ctx.GlobalBool("enable_tls") {
		cert := ctx.GlobalString("tls_cert_file")
		key := ctx.GlobalString("tls_key_file")

		if len(cert) > 0 && len(key) > 0 {
			certs, err := tls.LoadX509KeyPair(cert, key)
			if err != nil {
				fmt.Println(err.Error())
				return
			}
			config := &tls.Config{
				Certificates: []tls.Certificate{certs},
			}
			opts = append(opts, server.EnableTLS(true))
			opts = append(opts, server.TLSConfig(config))
		} else {
			fmt.Println("Enable TLS specified without certificate and key files")
			return
		}
	}

	srv := server.NewServer(Address)
	srv.Init(opts...)
	srv.Handle("/", s)

	// Initialise Server
	service := micro.NewService(
		micro.Name("go.micro.web"),
		micro.RegisterTTL(
			time.Duration(ctx.GlobalInt("register_ttl"))*time.Second,
		),
		micro.RegisterInterval(
			time.Duration(ctx.GlobalInt("register_interval"))*time.Second,
		),
	)

	if err := srv.Start(); err != nil {
		log.Fatal(err)
	}

	// Run server
	if err := service.Run(); err != nil {
		log.Fatal(err)
	}

	if err := srv.Stop(); err != nil {
		log.Fatal(err)
	}
}

func Commands() []cli.Command {
	return []cli.Command{
		{
			Name:  "web",
			Usage: "Run the micro web app",
			Action: func(c *cli.Context) {
				run(c)
			},
		},
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Command test_serviced is an implementation of the test_service service.
package main

import (
	"fmt"
	"log"

	"v.io/v23"
	"v.io/x/ref/lib/flags"
	"v.io/x/ref/lib/signals"
	_ "v.io/x/ref/runtime/factories/generic"
)

func main() {
	flags.SetDefaultHostPort("127.0.0.1:0")
	ctx, shutdown := v23.Init()
	defer shutdown()

	ctx, s, err := v23.WithNewDispatchingServer(ctx, "test_service", NewDispatcher())
	if err != nil {
		log.Fatalf("failure creating server: %v", err)
	}
	endpoint := s.Status().Endpoints[0]
	fmt.Printf("Listening at: %v\n", endpoint)
	<-signals.ShutdownOnSignals(ctx)
}
<commit_msg>v.io: use test.V23Init instead of v23.Init in test code<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Command test_serviced is an implementation of the test_service service.
package main

import (
	"fmt"
	"log"

	"v.io/v23"
	"v.io/x/ref/lib/flags"
	"v.io/x/ref/lib/signals"
	_ "v.io/x/ref/runtime/factories/generic"
	"v.io/x/ref/test"
)

func main() {
	flags.SetDefaultHostPort("127.0.0.1:0")
	ctx, shutdown := test.V23Init()
	defer shutdown()

	ctx, s, err := v23.WithNewDispatchingServer(ctx, "test_service", NewDispatcher())
	if err != nil {
		log.Fatalf("failure creating server: %v", err)
	}
	endpoint := s.Status().Endpoints[0]
	fmt.Printf("Listening at: %v\n", endpoint)
	<-signals.ShutdownOnSignals(ctx)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/rand"
	"encoding/base64"
	"encoding/gob"
	"io"
	"log"
	"net"
	"os"
	"strconv"

	"github.com/dotcloud/docker/term"
	"github.com/flynn/go-discover/discover"
	"github.com/flynn/lorne/types"
	"github.com/flynn/sampi/client"
	"github.com/flynn/sampi/types"
	"github.com/titanous/go-dockerclient"
)

func main() {
	disc, err := discover.NewClient()
	if err != nil {
		log.Fatal(err)
	}

	scheduler, err := client.New()
	if err != nil {
		log.Fatal(err)
	}

	state, err := scheduler.State()
	if err != nil {
		log.Fatal(err)
	}

	var firstHost string
	for k := range state {
		firstHost = k
		break
	}
	if firstHost == "" {
		log.Fatal("no hosts")
	}

	id := randomID()

	services := disc.Services("flynn-lorne-attach." + firstHost)
	conn, err := net.Dial("tcp", services.OnlineAddrs()[0])
	if err != nil {
		log.Fatal(err)
	}
	ws, _ := term.GetWinsize(os.Stdin.Fd())
	err = gob.NewEncoder(conn).Encode(&lorne.AttachReq{
		JobID:  id,
		Flags:  lorne.AttachFlagStdout | lorne.AttachFlagStderr | lorne.AttachFlagStdin | lorne.AttachFlagStream,
		Height: int(ws.Height),
		Width:  int(ws.Width),
	})
	if err != nil {
		log.Fatal(err)
	}
	attachState := make([]byte, 1)
	if _, err := conn.Read(attachState); err != nil {
		log.Fatal(err)
	}
	switch attachState[0] {
	case lorne.AttachError:
		log.Fatal("attach error")
	}

	schedReq := &sampi.ScheduleReq{
		Incremental: true,
		HostJobs: map[string][]*sampi.Job{firstHost: {{ID: id, Config: &docker.Config{
			Image:        "titanous/redis",
			Cmd:          []string{"/bin/bash", "-i"},
			Tty:          true,
			AttachStdin:  true,
			AttachStdout: true,
			AttachStderr: true,
			OpenStdin:    true,
			StdinOnce:    true,
			Env: []string{
				"COLUMNS=" + strconv.Itoa(int(ws.Width)),
				"LINES=" + strconv.Itoa(int(ws.Height)),
				"TERM=" + os.Getenv("TERM"),
			},
		}}}},
	}
	if _, err := scheduler.Schedule(schedReq); err != nil {
		log.Fatal(err)
	}

	if _, err := conn.Read(attachState); err != nil {
		log.Fatal(err)
	}

	oldState, err := term.SetRawTerminal(os.Stdin.Fd())
	if err != nil {
		log.Fatal(err)
	}
	go io.Copy(conn, os.Stdin)
	if _, err := io.Copy(os.Stdout, conn); err != nil {
		log.Fatal(err)
	}
	term.RestoreTerminal(os.Stdin.Fd(), oldState)
}

func randomID() string {
	b := make([]byte, 16)
	enc := make([]byte, 24)
	_, err := io.ReadFull(rand.Reader, b)
	if err != nil {
		panic(err) // This shouldn't ever happen, right?
	}
	base64.URLEncoding.Encode(enc, b)
	return string(bytes.TrimRight(enc, "="))
}
<commit_msg>host/sampi: example: Fix discover API<commit_after>package main

import (
	"bytes"
	"crypto/rand"
	"encoding/base64"
	"encoding/gob"
	"io"
	"log"
	"net"
	"os"
	"strconv"

	"github.com/dotcloud/docker/term"
	"github.com/flynn/go-discover/discover"
	"github.com/flynn/lorne/types"
	"github.com/flynn/sampi/client"
	"github.com/flynn/sampi/types"
	"github.com/titanous/go-dockerclient"
)

func main() {
	disc, err := discover.NewClient()
	if err != nil {
		log.Fatal(err)
	}

	scheduler, err := client.New()
	if err != nil {
		log.Fatal(err)
	}

	state, err := scheduler.State()
	if err != nil {
		log.Fatal(err)
	}

	var firstHost string
	for k := range state {
		firstHost = k
		break
	}
	if firstHost == "" {
		log.Fatal("no hosts")
	}

	id := randomID()

	services, err := disc.Services("flynn-lorne-attach." + firstHost)
	if err != nil {
		log.Fatal(err)
	}
	conn, err := net.Dial("tcp", services.OnlineAddrs()[0])
	if err != nil {
		log.Fatal(err)
	}
	ws, _ := term.GetWinsize(os.Stdin.Fd())
	err = gob.NewEncoder(conn).Encode(&lorne.AttachReq{
		JobID:  id,
		Flags:  lorne.AttachFlagStdout | lorne.AttachFlagStderr | lorne.AttachFlagStdin | lorne.AttachFlagStream,
		Height: int(ws.Height),
		Width:  int(ws.Width),
	})
	if err != nil {
		log.Fatal(err)
	}
	attachState := make([]byte, 1)
	if _, err := conn.Read(attachState); err != nil {
		log.Fatal(err)
	}
	switch attachState[0] {
	case lorne.AttachError:
		log.Fatal("attach error")
	}

	schedReq := &sampi.ScheduleReq{
		Incremental: true,
		HostJobs: map[string][]*sampi.Job{firstHost: {{ID: id, Config: &docker.Config{
			Image:        "titanous/redis",
			Cmd:          []string{"/bin/bash", "-i"},
			Tty:          true,
			AttachStdin:  true,
			AttachStdout: true,
			AttachStderr: true,
			OpenStdin:    true,
			StdinOnce:    true,
			Env: []string{
				"COLUMNS=" + strconv.Itoa(int(ws.Width)),
				"LINES=" + strconv.Itoa(int(ws.Height)),
				"TERM=" + os.Getenv("TERM"),
			},
		}}}},
	}
	if _, err := scheduler.Schedule(schedReq); err != nil {
		log.Fatal(err)
	}

	if _, err := conn.Read(attachState); err != nil {
		log.Fatal(err)
	}

	oldState, err := term.SetRawTerminal(os.Stdin.Fd())
	if err != nil {
		log.Fatal(err)
	}
	go io.Copy(conn, os.Stdin)
	if _, err := io.Copy(os.Stdout, conn); err != nil {
		log.Fatal(err)
	}
	term.RestoreTerminal(os.Stdin.Fd(), oldState)
}

func randomID() string {
	b := make([]byte, 16)
	enc := make([]byte, 24)
	_, err := io.ReadFull(rand.Reader, b)
	if err != nil {
		panic(err) // This shouldn't ever happen, right?
	}
	base64.URLEncoding.Encode(enc, b)
	return string(bytes.TrimRight(enc, "="))
}
<|endoftext|>


<commit_before>package isolated

import (
	"code.cloudfoundry.org/cli/integration/helpers"
	"fmt"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("unset-space-role command", func() {
	var (
		privilegedUsername string
		orgName            string
		spaceName          string
	)

	BeforeEach(func() {
		privilegedUsername = helpers.LoginCF()
		orgName = helpers.NewOrgName()
		spaceName = helpers.NewSpaceName()
		helpers.CreateOrgAndSpace(orgName, spaceName)
	})

	AfterEach(func() {
		helpers.QuickDeleteOrg(orgName)
	})

	Describe("help text and argument validation", func() {
		When("--help flag is unset", func() {
			It("Displays command usage to output", func() {
				session := helpers.CF("unset-space-role", "--help")
				Eventually(session).Should(Say("NAME:"))
				Eventually(session).Should(Say("unset-space-role - Remove a space role from a user"))
				Eventually(session).Should(Say("USAGE:"))
				Eventually(session).Should(Say("cf unset-space-role USERNAME ORG SPACE ROLE"))
				Eventually(session).Should(Say(`cf unset-space-role USERNAME ORG SPACE ROLE \[--client\]`))
				Eventually(session).Should(Say(`cf unset-space-role USERNAME ORG SPACE ROLE \[--origin ORIGIN\]`))
				Eventually(session).Should(Say("ROLES:"))
				Eventually(session).Should(Say("SpaceManager - Invite and manage users, and enable features for a given space"))
				Eventually(session).Should(Say("SpaceDeveloper - Create and manage apps and services, and see logs and reports"))
				Eventually(session).Should(Say("SpaceAuditor - View logs, reports, and settings on this space"))
				Eventually(session).Should(Say("OPTIONS:"))
				Eventually(session).Should(Say(`--client\s+Remove space role from a client-id of a \(non-user\) service account`))
				Eventually(session).Should(Say(`--origin\s+Indicates the identity provider to be used for authentication`))
				Eventually(session).Should(Say("SEE ALSO:"))
				Eventually(session).Should(Say("set-space-role, space-users"))
				Eventually(session).Should(Exit(0))
			})
		})

		When("the role type does not exist", func() {
			It("prints a useful error, prints help text, and exits 1", func() {
				session := helpers.CF("unset-space-role", "some-user", "some-org", "some-space", "NotARealRole")
				Eventually(session.Err).Should(Say(`Incorrect Usage: ROLE must be "SpaceManager", "SpaceDeveloper" and "SpaceAuditor"`))
				Eventually(session).Should(Say(`NAME:`))
				Eventually(session).Should(Exit(1))
			})
		})

		When("too few arguments are passed", func() {
			It("prints a useful error, prints help text, and exits 1", func() {
				session := helpers.CF("unset-space-role", "not-enough", "arguments")
				Eventually(session.Err).Should(Say("Incorrect Usage: the required arguments `SPACE` and `ROLE` were not provided"))
				Eventually(session).Should(Say(`NAME:`))
				Eventually(session).Should(Exit(1))
			})
		})

		When("too many arguments are passed", func() {
			It("prints a useful error, prints help text, and exits 1", func() {
				session := helpers.CF("unset-space-role", "some-user", "some-org", "some-space", "SpaceAuditor", "some-extra-argument")
				Eventually(session.Err).Should(Say(`Incorrect Usage: unexpected argument "some-extra-argument"`))
				Eventually(session).Should(Say(`NAME:`))
				Eventually(session).Should(Exit(1))
			})
		})
	})

	When("logged in as a privileged user", func() {
		When("the --client flag is passed", func() {
			var clientID string

			BeforeEach(func() {
				clientID, _ = helpers.SkipIfClientCredentialsNotSet()
				session := helpers.CF("curl", "-X", "POST", "v3/users", "-d", fmt.Sprintf(`{"guid":"%s"}`, clientID))
				Eventually(session).Should(Exit(0))
			})

			When("the client exists and is affiliated with the active user's org", func() {
				BeforeEach(func() {
					session := helpers.CF("set-space-role", clientID, orgName, spaceName, "SpaceAuditor", "--client")
					Eventually(session).Should(Exit(0))
					privilegedUsername = helpers.SwitchToSpaceRole(orgName, spaceName, "SpaceManager")
				})

				It("unsets the space role for the client", func() {
					session := helpers.CF("unset-space-role", clientID, orgName, spaceName, "SpaceAuditor", "--client")
					Eventually(session).Should(Say("Removing role SpaceAuditor from user %s in org %s / space %s as %s...", clientID, orgName, spaceName, privilegedUsername))
					Eventually(session).Should(Say("OK"))
					Eventually(session).Should(Exit(0))
				})

			})

			When("the active user lacks permissions to look up clients", func() {
				BeforeEach(func() {
					helpers.SwitchToSpaceRole(orgName, spaceName, "SpaceManager")
				})

				It("prints an appropriate error and exits 1", func() {
					session := helpers.CF("unset-space-role", "cf_smoke_tests", orgName, spaceName, "SpaceAuditor", "--client")
					Eventually(session).Should(Say("FAILED"))
					Eventually(session.Err).Should(Say("User '%s' does not exist.", "cf_smoke_tests"))
					Eventually(session).Should(Exit(1))
				})
			})

			When("the targeted client does not exist", func() {
				var badClientID string

				BeforeEach(func() {
					badClientID = "nonexistent-client"
				})

				It("fails with an appropriate error message", func() {
					session := helpers.CF("unset-space-role", badClientID, orgName, spaceName, "SpaceAuditor", "--client")
					Eventually(session).Should(Say("FAILED"))
					Eventually(session.Err).Should(Say("User 'nonexistent-client' does not exist."))
					Eventually(session).Should(Exit(1))
				})
			})
		})

		When("the user exists", func() {
			var username string

			BeforeEach(func() {
				username, _ = helpers.CreateUser()
				session := helpers.CF("set-space-role", username, orgName, spaceName, "spaceauditor")
				Eventually(session).Should(Exit(0))
			})

			When("the passed role type is lowercase", func() {
				It("unsets the space role for the user", func() {
					session := helpers.CF("unset-space-role", "-v", username, orgName, spaceName, "spaceauditor")
					Eventually(session).Should(Say("Removing role SpaceAuditor from user %s in org %s / space %s as %s...", username, orgName, spaceName, privilegedUsername))
					Eventually(session).Should(Say("OK"))
					Eventually(session).Should(Exit(0))
				})
			})

			It("unsets the space role for the user", func() {
				session := helpers.CF("unset-space-role", username, orgName, spaceName, "SpaceAuditor")
				Eventually(session).Should(Say("Removing role SpaceAuditor from user %s in org %s / space %s as %s...", username, orgName, spaceName, privilegedUsername))
				Eventually(session).Should(Say("OK"))
				Eventually(session).Should(Exit(0))
			})

			When("the user does not have the role to delete", func() {
				It("is idempotent", func() {
					session := helpers.CF("unset-space-role", username, orgName, spaceName, "SpaceDeveloper")
					Eventually(session).Should(Say("Removing role SpaceDeveloper from user %s in org %s / space %s as %s...", username, orgName, spaceName, privilegedUsername))
					Eventually(session).Should(Exit(0))
				})
			})

			When("the org does not exist", func() {
				It("prints an appropriate error and exits 1", func() {
					session := helpers.CF("unset-space-role", username, "invalid-org", spaceName, "SpaceAuditor")
					Eventually(session).Should(Say("FAILED"))
					Eventually(session.Err).Should(Say("Organization 'invalid-org' not found."))
					Eventually(session).Should(Exit(1))
				})
			})

			When("the space does not exist", func() {
				It("prints an appropriate error and exits 1", func() {
					session := helpers.CF("unset-space-role", username, orgName, "invalid-space", "SpaceAuditor")
					Eventually(session).Should(Say("FAILED"))
					Eventually(session.Err).Should(Say("Space 'invalid-space' not found."))
					Eventually(session).Should(Exit(1))
				})
			})
		})

		When("the user does not exist", func() {
			It("prints an appropriate error and exits 1", func() {
				session := helpers.CF("unset-space-role", "not-exists", orgName, spaceName, "SpaceAuditor")
				Eventually(session).Should(Say("Removing role SpaceAuditor from user not-exists in org %s / space %s as %s...", orgName, spaceName, privilegedUsername))
				Eventually(session.Err).Should(Say("User 'not-exists' does not exist."))
				Eventually(session).Should(Say("FAILED"))
				Eventually(session).Should(Exit(1))
			})
		})
	})

	When("the logged in user does not have permission to write to the space", func() {
		var username string

		BeforeEach(func() {
			username, _ = helpers.CreateUser()
			session := helpers.CF("set-space-role", username, orgName, spaceName, "SpaceAuditor")
			Eventually(session).Should(Exit(0))
			helpers.SwitchToSpaceRole(orgName, spaceName, "SpaceAuditor")
		})

		It("prints out the error message from CC API and exits 1", func() {
			session := helpers.CF("unset-space-role", username, orgName, spaceName, "SpaceAuditor")
			Eventually(session).Should(Say("FAILED"))
			Eventually(session.Err).Should(Say("You are not authorized to perform the requested action"))
			Eventually(session).Should(Exit(1))
		})
	})

	When("the logged in user has insufficient permissions to see the user", func() {
		var username string

		BeforeEach(func() {
			username, _ = helpers.CreateUser()
			helpers.SwitchToSpaceRole(orgName, spaceName, "SpaceManager")
		})

		It("prints out the error message from CC API and exits 1", func() {
			session := helpers.CF("unset-space-role", username, orgName, spaceName, "SpaceAuditor", "-v")
			Eventually(session).Should(Say("FAILED"))
			Eventually(session.Err).Should(Say("User '%s' does not exist.", username))
			Eventually(session).Should(Exit(1))
		})
	})
})
<commit_msg>🐞<commit_after>package isolated

import (
	"fmt"

	"code.cloudfoundry.org/cli/integration/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("unset-space-role command", func() {
	var (
		privilegedUsername string
		orgName            string
		spaceName          string
	)

	BeforeEach(func() {
		privilegedUsername = helpers.LoginCF()
		orgName = helpers.NewOrgName()
		spaceName = helpers.NewSpaceName()
		helpers.CreateOrgAndSpace(orgName, spaceName)
	})

	AfterEach(func() {
		helpers.QuickDeleteOrg(orgName)
	})

	Describe("help text and argument validation", func() {
		When("--help flag is unset", func() {
			It("Displays command usage to output", func() {
				session := helpers.CF("unset-space-role", "--help")
				Eventually(session).Should(Say("NAME:"))
				Eventually(session).Should(Say("unset-space-role - Remove a space role from a user"))
				Eventually(session).Should(Say("USAGE:"))
				Eventually(session).Should(Say("cf unset-space-role USERNAME ORG SPACE ROLE"))
				Eventually(session).Should(Say(`cf unset-space-role USERNAME ORG SPACE ROLE \[--client\]`))
				Eventually(session).Should(Say(`cf unset-space-role USERNAME ORG SPACE ROLE \[--origin ORIGIN\]`))
				Eventually(session).Should(Say("ROLES:"))
				Eventually(session).Should(Say("SpaceManager - Invite and manage users, and enable features for a given space"))
				Eventually(session).Should(Say("SpaceDeveloper - Create and manage apps and services, and see logs and reports"))
				Eventually(session).Should(Say("SpaceAuditor - View logs, reports, and settings on this space"))
				Eventually(session).Should(Say("OPTIONS:"))
				Eventually(session).Should(Say(`--client\s+Remove space role from a client-id of a \(non-user\) service account`))
				Eventually(session).Should(Say(`--origin\s+Indicates the identity provider to be used for authentication`))
				Eventually(session).Should(Say("SEE ALSO:"))
				Eventually(session).Should(Say("set-space-role, space-users"))
				Eventually(session).Should(Exit(0))
			})
		})

		When("the role type does not exist", func() {
			It("prints a useful error, prints help text, and exits 1", func() {
				session := helpers.CF("unset-space-role", "some-user", "some-org", "some-space", "NotARealRole")
				Eventually(session.Err).Should(Say(`Incorrect Usage: ROLE must be "SpaceManager", "SpaceDeveloper" and "SpaceAuditor"`))
				Eventually(session).Should(Say(`NAME:`))
				Eventually(session).Should(Exit(1))
			})
		})

		When("too few arguments are passed", func() {
			It("prints a useful error, prints help text, and exits 1", func() {
				session := helpers.CF("unset-space-role", "not-enough", "arguments")
				Eventually(session.Err).Should(Say("Incorrect Usage: the required arguments `SPACE` and `ROLE` were not provided"))
				Eventually(session).Should(Say(`NAME:`))
				Eventually(session).Should(Exit(1))
			})
		})

		When("too many arguments are passed", func() {
			It("prints a useful error, prints help text, and exits 1", func() {
				session := helpers.CF("unset-space-role", "some-user", "some-org", "some-space", "SpaceAuditor", "some-extra-argument")
				Eventually(session.Err).Should(Say(`Incorrect Usage: unexpected argument "some-extra-argument"`))
				Eventually(session).Should(Say(`NAME:`))
				Eventually(session).Should(Exit(1))
			})
		})
	})

	When("logged in as a privileged user", func() {
		When("the --client flag is passed", func() {
			var clientID string

			BeforeEach(func() {
				clientID, _ = helpers.SkipIfClientCredentialsNotSet()
				session := helpers.CF("curl", "-X", "POST", "v3/users", "-d", fmt.Sprintf(`{"guid":"%s"}`, clientID))
				Eventually(session).Should(Exit(0))
			})

			When("the client exists and is affiliated with the active user's org", func() {
				BeforeEach(func() {
					session := helpers.CF("set-space-role", clientID, orgName, spaceName, "SpaceAuditor", "--client")
					Eventually(session).Should(Exit(0))
					privilegedUsername = helpers.SwitchToSpaceRole(orgName, spaceName, "SpaceManager")
				})

				It("unsets the space role for the client", func() {
					session := helpers.CF("unset-space-role", clientID, orgName, spaceName, "SpaceAuditor", "--client")
					Eventually(session).Should(Say("Removing role SpaceAuditor from user %s in org %s / space %s as %s...", clientID, orgName, spaceName, privilegedUsername))
					Eventually(session).Should(Say("OK"))
					Eventually(session).Should(Exit(0))
				})

			})

			When("the active user lacks permissions to look up clients", func() {
				BeforeEach(func() {
					helpers.SwitchToSpaceRole(orgName, spaceName, "SpaceManager")
				})

				It("prints an appropriate error and exits 1", func() {
					session := helpers.CF("unset-space-role", "cf_smoke_tests", orgName, spaceName, "SpaceAuditor", "--client")
					Eventually(session).Should(Say("FAILED"))
					Eventually(session.Err).Should(Say("User '%s' does not exist.", "cf_smoke_tests"))
					Eventually(session).Should(Exit(1))
				})
			})

			When("the targeted client does not exist", func() {
				var badClientID string

				BeforeEach(func() {
					badClientID = "nonexistent-client"
				})

				It("fails with an appropriate error message", func() {
					session := helpers.CF("unset-space-role", badClientID, orgName, spaceName, "SpaceAuditor", "--client")
					Eventually(session).Should(Say("FAILED"))
					Eventually(session.Err).Should(Say("User 'nonexistent-client' does not exist."))
					Eventually(session).Should(Exit(1))
				})
			})
		})

		When("the user exists", func() {
			var username string

			BeforeEach(func() {
				username, _ = helpers.CreateUser()
				session := helpers.CF("set-space-role", username, orgName, spaceName, "spaceauditor")
				Eventually(session).Should(Exit(0))
			})

			When("the passed role type is lowercase", func() {
				It("unsets the space role for the user", func() {
					session := helpers.CF("unset-space-role", "-v", username, orgName, spaceName, "spaceauditor")
					Eventually(session).Should(Say("Removing role SpaceAuditor from user %s in org %s / space %s as %s...", username, orgName, spaceName, privilegedUsername))
					Eventually(session).Should(Say("OK"))
					Eventually(session).Should(Exit(0))
				})
			})

			It("unsets the space role for the user", func() {
				session := helpers.CF("unset-space-role", username, orgName, spaceName, "SpaceAuditor")
				Eventually(session).Should(Say("Removing role SpaceAuditor from user %s in org %s / space %s as %s...", username, orgName, spaceName, privilegedUsername))
				Eventually(session).Should(Say("OK"))
				Eventually(session).Should(Exit(0))
			})

			When("the user does not have the role to delete", func() {
				It("is idempotent", func() {
					session := helpers.CF("unset-space-role", username, orgName, spaceName, "SpaceDeveloper")
					Eventually(session).Should(Say("Removing role SpaceDeveloper from user %s in org %s / space %s as %s...", username, orgName, spaceName, privilegedUsername))
					Eventually(session).Should(Exit(0))
				})
			})

			When("the org does not exist", func() {
				It("prints an appropriate error and exits 1", func() {
					session := helpers.CF("unset-space-role", username, "invalid-org", spaceName, "SpaceAuditor")
					Eventually(session).Should(Say("FAILED"))
					Eventually(session.Err).Should(Say("Organization 'invalid-org' not found."))
					Eventually(session).Should(Exit(1))
				})
			})

			When("the space does not exist", func() {
				It("prints an appropriate error and exits 1", func() {
					session := helpers.CF("unset-space-role", username, orgName, "invalid-space", "SpaceAuditor")
					Eventually(session).Should(Say("FAILED"))
					Eventually(session.Err).Should(Say("Space 'invalid-space' not found."))
					Eventually(session).Should(Exit(1))
				})
			})
		})

		When("the user does not exist", func() {
			It("prints an appropriate error and exits 1", func() {
				session := helpers.CF("unset-space-role", "not-exists", orgName, spaceName, "SpaceAuditor")
				Eventually(session).Should(Say("Removing role SpaceAuditor from user not-exists in org %s / space %s as %s...", orgName, spaceName, privilegedUsername))
				Eventually(session.Err).Should(Say("User 'not-exists' does not exist."))
				Eventually(session).Should(Say("FAILED"))
				Eventually(session).Should(Exit(1))
			})
		})
	})

	When("the logged in user does not have permission to write to the space", func() {
		var username string

		BeforeEach(func() {
			username, _ = helpers.CreateUser()
			session := helpers.CF("set-space-role", username, orgName, spaceName, "SpaceAuditor")
			Eventually(session).Should(Exit(0))
			helpers.SwitchToSpaceRole(orgName, spaceName, "SpaceAuditor")
		})

		It("prints out the error message from CC API and exits 1", func() {
			session := helpers.CF("unset-space-role", username, orgName, spaceName, "SpaceAuditor")
			Eventually(session).Should(Say("FAILED"))
			Eventually(session.Err).Should(Say("You are not authorized to perform the requested action"))
			Eventually(session).Should(Exit(1))
		})
	})

	When("the logged in user has insufficient permissions to see the user", func() {
		var username string

		BeforeEach(func() {
			username, _ = helpers.CreateUser()
			helpers.SwitchToSpaceRole(orgName, spaceName, "SpaceManager")
		})

		It("prints out the error message from CC API and exits 1", func() {
			session := helpers.CF("unset-space-role", username, orgName, spaceName, "SpaceAuditor", "-v")
			Eventually(session).Should(Say("FAILED"))
			Eventually(session.Err).Should(Say("User '%s' does not exist.", username))
			Eventually(session).Should(Exit(1))
		})
	})
})
<|endoftext|>


<commit_before>// Package tools contains other helper functions too small to justify their own package
// NOTE: Subject to change, do not rely on this package from outside git-lfs source
package tools

import (
	"bufio"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"

	"github.com/git-lfs/git-lfs/filepathfilter"
)

// FileOrDirExists determines if a file/dir exists, returns IsDir() results too.
func FileOrDirExists(path string) (exists bool, isDir bool) {
	fi, err := os.Stat(path)
	if err != nil {
		return false, false
	} else {
		return true, fi.IsDir()
	}
}

// FileExists determines if a file (NOT dir) exists.
func FileExists(path string) bool {
	ret, isDir := FileOrDirExists(path)
	return ret && !isDir
}

// DirExists determines if a dir (NOT file) exists.
func DirExists(path string) bool {
	ret, isDir := FileOrDirExists(path)
	return ret && isDir
}

// FileExistsOfSize determines if a file exists and is of a specific size.
func FileExistsOfSize(path string, sz int64) bool {
	fi, err := os.Stat(path)

	if err != nil {
		return false
	}

	return !fi.IsDir() && fi.Size() == sz
}

// ResolveSymlinks ensures that if the path supplied is a symlink, it is
// resolved to the actual concrete path
func ResolveSymlinks(path string) string {
	if len(path) == 0 {
		return path
	}

	if resolved, err := filepath.EvalSymlinks(path); err == nil {
		return resolved
	}
	return path
}

// RenameFileCopyPermissions moves srcfile to destfile, replacing destfile if
// necessary and also copying the permissions of destfile if it already exists
func RenameFileCopyPermissions(srcfile, destfile string) error {
	info, err := os.Stat(destfile)
	if os.IsNotExist(err) {
		// no original file
	} else if err != nil {
		return err
	} else {
		if err := os.Chmod(srcfile, info.Mode()); err != nil {
			return fmt.Errorf("can't set filemode on file %q: %v", srcfile, err)
		}
	}

	if err := os.Rename(srcfile, destfile); err != nil {
		return fmt.Errorf("cannot replace %q with %q: %v", destfile, srcfile, err)
	}
	return nil
}

// CleanPaths splits the given `paths` argument by the delimiter argument, and
// then "cleans" that path according to the path.Clean function (see
// https://golang.org/pkg/path#Clean).
// Note always cleans to '/' path separators regardless of platform (git friendly)
func CleanPaths(paths, delim string) (cleaned []string) {
	// If paths is an empty string, splitting it will yield [""], which will
	// become the path ".". To avoid this, bail out if trimmed paths
	// argument is empty.
	if paths = strings.TrimSpace(paths); len(paths) == 0 {
		return
	}

	for _, part := range strings.Split(paths, delim) {
		part = strings.TrimSpace(part)

		cleaned = append(cleaned, path.Clean(part))
	}

	return cleaned
}

// VerifyFileHash reads a file and verifies whether the SHA is correct
// Returns an error if there is a problem
func VerifyFileHash(oid, path string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()

	h := NewLfsContentHash()
	_, err = io.Copy(h, f)
	if err != nil {
		return err
	}

	calcOid := hex.EncodeToString(h.Sum(nil))
	if calcOid != oid {
		return fmt.Errorf("File %q has an invalid hash %s, expected %s", path, calcOid, oid)
	}

	return nil
}

// Returned from FastWalk with parent directory context
// This is needed because FastWalk can provide paths out of order so the
// parent dir cannot be implied
type fastWalkInfo struct {
	ParentDir string
	Info      os.FileInfo
}

type FastWalkCallback func(parentDir string, info os.FileInfo, err error)

// FastWalkGitRepo is a more optimal implementation of filepath.Walk for a Git repo
// It differs in the following ways:
//  * Provides a channel of information instead of using a callback func
//  * Uses goroutines to parallelise large dirs and descent into subdirs
//  * Does not provide sorted output; parents will always be before children but
//    there are no other guarantees. Use parentDir in the fastWalkInfo struct to
//    determine absolute path rather than tracking it yourself like filepath.Walk
//  * Automatically ignores any .git directories
//  * Respects .gitignore contents and skips ignored files/dirs
func FastWalkGitRepo(dir string, cb FastWalkCallback) {
	// Ignore all git metadata including subrepos
	excludePaths := []filepathfilter.Pattern{
		filepathfilter.NewPattern(".git"),
		filepathfilter.NewPattern(filepath.Join("**", ".git")),
	}

	fileCh, errCh := fastWalkWithExcludeFiles(dir, ".gitignore", excludePaths)
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		for file := range fileCh {
			cb(file.ParentDir, file.Info, nil)
		}
		wg.Done()
	}()

	for err := range errCh {
		cb("", nil, err)
	}

	wg.Wait()
}

// fastWalkWithExcludeFiles walks the contents of a dir, respecting
// include/exclude patterns and also loading new exlude patterns from files
// named excludeFilename in directories walked
func fastWalkWithExcludeFiles(dir, excludeFilename string,
	excludePaths []filepathfilter.Pattern) (<-chan fastWalkInfo, <-chan error) {
	fiChan := make(chan fastWalkInfo, 256)
	errChan := make(chan error, 10)

	go fastWalkFromRoot(dir, excludeFilename, excludePaths, fiChan, errChan)
	return fiChan, errChan
}

func fastWalkFromRoot(dir string, excludeFilename string,
	excludePaths []filepathfilter.Pattern, fiChan chan<- fastWalkInfo, errChan chan<- error) {

	dirFi, err := os.Stat(dir)
	if err != nil {
		errChan <- err
		return
	}

	// This waitgroup will be incremented for each nested goroutine
	var waitg sync.WaitGroup
	fastWalkFileOrDir(filepath.Dir(dir), dirFi, excludeFilename, excludePaths, fiChan, errChan, &waitg)
	waitg.Wait()
	close(fiChan)
	close(errChan)
}

// fastWalkFileOrDir is the main recursive implementation of fast walk
// Sends the file/dir and any contents to the channel so long as it passes the
// include/exclude filter. If a dir, parses any excludeFilename found and updates
// the excludePaths with its content before (parallel) recursing into contents
// Also splits large directories into multiple goroutines.
// Increments waitg.Add(1) for each new goroutine launched internally
func fastWalkFileOrDir(parentDir string, itemFi os.FileInfo, excludeFilename string,
	excludePaths []filepathfilter.Pattern, fiChan chan<- fastWalkInfo, errChan chan<- error,
	waitg *sync.WaitGroup) {

	fullPath := filepath.Join(parentDir, itemFi.Name())

	if !filepathfilter.NewFromPatterns(nil, excludePaths).Allows(fullPath) {
		return
	}

	fiChan <- fastWalkInfo{ParentDir: parentDir, Info: itemFi}

	if !itemFi.IsDir() {
		// Nothing more to do if this is not a dir
		return
	}

	if len(excludeFilename) > 0 {
		possibleExcludeFile := filepath.Join(fullPath, excludeFilename)
		if FileExists(possibleExcludeFile) {
			var err error
			excludePaths, err = loadExcludeFilename(possibleExcludeFile, fullPath, excludePaths)
			if err != nil {
				errChan <- err
			}
		}
	}

	// The absolute optimal way to scan would be File.Readdirnames but we
	// still need the Stat() to know whether something is a dir, so use
	// File.Readdir instead. Means we can provide os.FileInfo to callers like
	// filepath.Walk as a bonus.
	df, err := os.Open(fullPath)
	if err != nil {
		errChan <- err
		return
	}
	defer df.Close()

	// The number of items in a dir we process in each goroutine
	jobSize := 100
	for children, err := df.Readdir(jobSize); err == nil; children, err = df.Readdir(jobSize) {
		// Parallelise all dirs, and chop large dirs into batches
		waitg.Add(1)
		go func(subitems []os.FileInfo) {
			for _, childFi := range subitems {
				fastWalkFileOrDir(fullPath, childFi, excludeFilename, excludePaths, fiChan, errChan, waitg)
			}
			waitg.Done()
		}(children)

	}
	if err != nil && err != io.EOF {
		errChan <- err
	}
}

// loadExcludeFilename reads the given file in gitignore format and returns a
// revised array of exclude paths if there are any changes.
// If any changes are made a copy of the array is taken so the original is not
// modified
func loadExcludeFilename(filename, parentDir string, excludePaths []filepathfilter.Pattern) ([]filepathfilter.Pattern, error) {
	f, err := os.OpenFile(filename, os.O_RDONLY, 0644)
	if err != nil {
		return excludePaths, err
	}
	defer f.Close()

	retPaths := excludePaths
	modified := false

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		// Skip blanks, comments and negations (not supported right now)
		if len(line) == 0 || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "!") {
			continue
		}

		if !modified {
			// copy on write
			retPaths = make([]filepathfilter.Pattern, len(excludePaths))
			copy(retPaths, excludePaths)
			modified = true
		}

		path := line
		// Add pattern in context if exclude has separator, or no wildcard
		// Allow for both styles of separator at this point
		if strings.ContainsAny(path, "/\\") ||
			!strings.Contains(path, "*") {
			path = filepath.Join(parentDir, line)
		}
		retPaths = append(retPaths, filepathfilter.NewPattern(path))
	}

	return retPaths, nil
}
<commit_msg>update comments<commit_after>// Package tools contains other helper functions too small to justify their own package
// NOTE: Subject to change, do not rely on this package from outside git-lfs source
package tools

import (
	"bufio"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"

	"github.com/git-lfs/git-lfs/filepathfilter"
)

// FileOrDirExists determines if a file/dir exists, returns IsDir() results too.
func FileOrDirExists(path string) (exists bool, isDir bool) {
	fi, err := os.Stat(path)
	if err != nil {
		return false, false
	} else {
		return true, fi.IsDir()
	}
}

// FileExists determines if a file (NOT dir) exists.
func FileExists(path string) bool {
	ret, isDir := FileOrDirExists(path)
	return ret && !isDir
}

// DirExists determines if a dir (NOT file) exists.
func DirExists(path string) bool {
	ret, isDir := FileOrDirExists(path)
	return ret && isDir
}

// FileExistsOfSize determines if a file exists and is of a specific size.
func FileExistsOfSize(path string, sz int64) bool {
	fi, err := os.Stat(path)

	if err != nil {
		return false
	}

	return !fi.IsDir() && fi.Size() == sz
}

// ResolveSymlinks ensures that if the path supplied is a symlink, it is
// resolved to the actual concrete path
func ResolveSymlinks(path string) string {
	if len(path) == 0 {
		return path
	}

	if resolved, err := filepath.EvalSymlinks(path); err == nil {
		return resolved
	}
	return path
}

// RenameFileCopyPermissions moves srcfile to destfile, replacing destfile if
// necessary and also copying the permissions of destfile if it already exists
func RenameFileCopyPermissions(srcfile, destfile string) error {
	info, err := os.Stat(destfile)
	if os.IsNotExist(err) {
		// no original file
	} else if err != nil {
		return err
	} else {
		if err := os.Chmod(srcfile, info.Mode()); err != nil {
			return fmt.Errorf("can't set filemode on file %q: %v", srcfile, err)
		}
	}

	if err := os.Rename(srcfile, destfile); err != nil {
		return fmt.Errorf("cannot replace %q with %q: %v", destfile, srcfile, err)
	}
	return nil
}

// CleanPaths splits the given `paths` argument by the delimiter argument, and
// then "cleans" that path according to the path.Clean function (see
// https://golang.org/pkg/path#Clean).
// Note always cleans to '/' path separators regardless of platform (git friendly)
func CleanPaths(paths, delim string) (cleaned []string) {
	// If paths is an empty string, splitting it will yield [""], which will
	// become the path ".". To avoid this, bail out if trimmed paths
	// argument is empty.
	if paths = strings.TrimSpace(paths); len(paths) == 0 {
		return
	}

	for _, part := range strings.Split(paths, delim) {
		part = strings.TrimSpace(part)

		cleaned = append(cleaned, path.Clean(part))
	}

	return cleaned
}

// VerifyFileHash reads a file and verifies whether the SHA is correct
// Returns an error if there is a problem
func VerifyFileHash(oid, path string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()

	h := NewLfsContentHash()
	_, err = io.Copy(h, f)
	if err != nil {
		return err
	}

	calcOid := hex.EncodeToString(h.Sum(nil))
	if calcOid != oid {
		return fmt.Errorf("File %q has an invalid hash %s, expected %s", path, calcOid, oid)
	}

	return nil
}

// FastWalkCallback is the signature for the callback given to FastWalkGitRepo()
type FastWalkCallback func(parentDir string, info os.FileInfo, err error)

// FastWalkGitRepo is a more optimal implementation of filepath.Walk for a Git
// repo. The callback guaranteed to be called sequentially. The function returns
// once all files and errors have triggered callbacks.
// It differs in the following ways:
//  * Uses goroutines to parallelise large dirs and descent into subdirs
//  * Does not provide sorted output; parents will always be before children but
//    there are no other guarantees. Use parentDir argument in the callback to
//    determine absolute path rather than tracking it yourself
//  * Automatically ignores any .git directories
//  * Respects .gitignore contents and skips ignored files/dirs
func FastWalkGitRepo(dir string, cb FastWalkCallback) {
	// Ignore all git metadata including subrepos
	excludePaths := []filepathfilter.Pattern{
		filepathfilter.NewPattern(".git"),
		filepathfilter.NewPattern(filepath.Join("**", ".git")),
	}

	fileCh, errCh := fastWalkWithExcludeFiles(dir, ".gitignore", excludePaths)
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		for file := range fileCh {
			cb(file.ParentDir, file.Info, nil)
		}
		wg.Done()
	}()

	for err := range errCh {
		cb("", nil, err)
	}

	wg.Wait()
}

// Returned from FastWalk with parent directory context
// This is needed because FastWalk can provide paths out of order so the
// parent dir cannot be implied
type fastWalkInfo struct {
	ParentDir string
	Info      os.FileInfo
}

// fastWalkWithExcludeFiles walks the contents of a dir, respecting
// include/exclude patterns and also loading new exlude patterns from files
// named excludeFilename in directories walked
func fastWalkWithExcludeFiles(dir, excludeFilename string,
	excludePaths []filepathfilter.Pattern) (<-chan fastWalkInfo, <-chan error) {
	fiChan := make(chan fastWalkInfo, 256)
	errChan := make(chan error, 10)

	go fastWalkFromRoot(dir, excludeFilename, excludePaths, fiChan, errChan)
	return fiChan, errChan
}

func fastWalkFromRoot(dir string, excludeFilename string,
	excludePaths []filepathfilter.Pattern, fiChan chan<- fastWalkInfo, errChan chan<- error) {

	dirFi, err := os.Stat(dir)
	if err != nil {
		errChan <- err
		return
	}

	// This waitgroup will be incremented for each nested goroutine
	var waitg sync.WaitGroup
	fastWalkFileOrDir(filepath.Dir(dir), dirFi, excludeFilename, excludePaths, fiChan, errChan, &waitg)
	waitg.Wait()
	close(fiChan)
	close(errChan)
}

// fastWalkFileOrDir is the main recursive implementation of fast walk
// Sends the file/dir and any contents to the channel so long as it passes the
// include/exclude filter. If a dir, parses any excludeFilename found and updates
// the excludePaths with its content before (parallel) recursing into contents
// Also splits large directories into multiple goroutines.
// Increments waitg.Add(1) for each new goroutine launched internally
func fastWalkFileOrDir(parentDir string, itemFi os.FileInfo, excludeFilename string,
	excludePaths []filepathfilter.Pattern, fiChan chan<- fastWalkInfo, errChan chan<- error,
	waitg *sync.WaitGroup) {

	fullPath := filepath.Join(parentDir, itemFi.Name())

	if !filepathfilter.NewFromPatterns(nil, excludePaths).Allows(fullPath) {
		return
	}

	fiChan <- fastWalkInfo{ParentDir: parentDir, Info: itemFi}

	if !itemFi.IsDir() {
		// Nothing more to do if this is not a dir
		return
	}

	if len(excludeFilename) > 0 {
		possibleExcludeFile := filepath.Join(fullPath, excludeFilename)
		if FileExists(possibleExcludeFile) {
			var err error
			excludePaths, err = loadExcludeFilename(possibleExcludeFile, fullPath, excludePaths)
			if err != nil {
				errChan <- err
			}
		}
	}

	// The absolute optimal way to scan would be File.Readdirnames but we
	// still need the Stat() to know whether something is a dir, so use
	// File.Readdir instead. Means we can provide os.FileInfo to callers like
	// filepath.Walk as a bonus.
	df, err := os.Open(fullPath)
	if err != nil {
		errChan <- err
		return
	}
	defer df.Close()

	// The number of items in a dir we process in each goroutine
	jobSize := 100
	for children, err := df.Readdir(jobSize); err == nil; children, err = df.Readdir(jobSize) {
		// Parallelise all dirs, and chop large dirs into batches
		waitg.Add(1)
		go func(subitems []os.FileInfo) {
			for _, childFi := range subitems {
				fastWalkFileOrDir(fullPath, childFi, excludeFilename, excludePaths, fiChan, errChan, waitg)
			}
			waitg.Done()
		}(children)

	}
	if err != nil && err != io.EOF {
		errChan <- err
	}
}

// loadExcludeFilename reads the given file in gitignore format and returns a
// revised array of exclude paths if there are any changes.
// If any changes are made a copy of the array is taken so the original is not
// modified
func loadExcludeFilename(filename, parentDir string, excludePaths []filepathfilter.Pattern) ([]filepathfilter.Pattern, error) {
	f, err := os.OpenFile(filename, os.O_RDONLY, 0644)
	if err != nil {
		return excludePaths, err
	}
	defer f.Close()

	retPaths := excludePaths
	modified := false

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		// Skip blanks, comments and negations (not supported right now)
		if len(line) == 0 || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "!") {
			continue
		}

		if !modified {
			// copy on write
			retPaths = make([]filepathfilter.Pattern, len(excludePaths))
			copy(retPaths, excludePaths)
			modified = true
		}

		path := line
		// Add pattern in context if exclude has separator, or no wildcard
		// Allow for both styles of separator at this point
		if strings.ContainsAny(path, "/\\") ||
			!strings.Contains(path, "*") {
			path = filepath.Join(parentDir, line)
		}
		retPaths = append(retPaths, filepathfilter.NewPattern(path))
	}

	return retPaths, nil
}
<|endoftext|>


<commit_before>// Package quantile computes approximate quantiles over an unbounded data
// stream within low memory and CPU bounds.
//
// A small amount of accuracy is traded to achieve the above properties.
//
// Multiple streams can be merged before calling Query to generate a single set
// of results. This is meaningful when the streams represent the same type of
// data. See Merge and Samples.
//
// For more detailed information about the algorithm used, see:
//
// Effective Computation of Biased Quantiles over Data Streams
//
// http://www.cs.rutgers.edu/~muthu/bquant.pdf
package quantile

import (
	"container/list"
	"math"
	"sort"
)

// Sample holds an observed value and meta information for compression. JSON
// tags have been added for convenience.
type Sample struct {
	Value float64 `json:",string"`
	Width float64 `json:",string"`
	Delta float64 `json:",string"`
}

// Samples represents a slice of samples. It implements sort.Interface.
type Samples []Sample

func (a Samples) Len() int {
	return len(a)
}

func (a Samples) Less(i, j int) bool {
	return a[i].Value < a[j].Value
}

func (a Samples) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

type invariant func(s *stream, r float64) float64

// NewBiased returns an initialized Stream for high-biased quantiles (e.g.
// 50th, 90th, 99th) not known a priori with ﬁner error guarantees for the
// higher ranks of the data distribution.
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
func NewBiased() *Stream {
	ƒ := func(s *stream, r float64) float64 {
		return 2 * s.epsilon * r
	}
	return newStream(ƒ)
}

// NewTargeted returns an initialized Stream concerned with a particular set of
// quantile values that are supplied a priori. Knowing these a priori reduces
// space and computation time.
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
func NewTargeted(quantiles ...float64) *Stream {
	ƒ := func(s *stream, r float64) float64 {
		var m float64 = math.MaxFloat64
		var f float64
		for _, q := range quantiles {
			if q*s.n <= r {
				f = (2 * s.epsilon * r) / q
			} else {
				f = (2 * s.epsilon * (s.n - r)) / (1 - q)
			}
			m = math.Min(m, f)
		}
		return m
	}
	return newStream(ƒ)
}

// Stream computes quantiles for a stream of float64s. It is not thread-safe.
type Stream struct {
	*stream
	b Samples
}

func newStream(ƒ invariant) *Stream {
	const defaultEpsilon = 0.01
	x := &stream{epsilon: defaultEpsilon, ƒ: ƒ, l: list.New()}
	return &Stream{x, make(Samples, 0, 500)}
}

// Insert inserts v into the stream.
func (s *Stream) Insert(v float64) {
	s.insert(Sample{Value: v, Width: 1})
}

func (s *Stream) insert(sample Sample) {
	s.b = append(s.b, sample)
	if len(s.b) == cap(s.b) {
		s.flush()
		s.compress()
	}
}

// Query returns the computed qth percentiles value. If s was created with
// NewTargeted, and q is not in the set of quantiles provided a priori, Query
// will return an unspecified result.
func (s *Stream) Query(q float64) float64 {
	if s.flushed() {
		// Fast path when there hasn't been enough data for a flush;
		// this also yeilds better accuracy for small sets of data.
		l := len(s.b)
		if l == 0 {
			return 0
		}
		i := int(float64(l) * q)
		if i > 0 {
			i -= 1
		}
		return s.b[i].Value
	}
	s.flush()
	return s.stream.query(q)
}

// Merge merges samples into the underlying streams samples. This is handy when
// merging multiple streams from separate threads, database shards, etc.
func (s *Stream) Merge(samples Samples) {
	s.stream.merge(samples)
}

// Reset reinitializes and clears the list reusing the samples buffer memory.
func (s *Stream) Reset() {
	s.stream.reset()
	s.b = s.b[:0]
}

// Samples returns stream samples held by s.
func (s *Stream) Samples() Samples {
	if !s.flushed() {
		return s.b
	}
	return s.stream.samples()
}

// Count returns the total number of samples observed in the stream
// since initialization.
func (s *Stream) Count() int {
	return len(s.b) + s.stream.count()
}

func (s *Stream) flush() {
	sort.Sort(s.b)
	s.stream.merge(s.b)
	s.b = s.b[:0]
}

func (s *Stream) flushed() bool {
	return s.stream.l.Len() == 0
}

type stream struct {
	epsilon float64
	n       float64
	l       *list.List
	ƒ       invariant
}

// SetEpsilon sets the error epsilon for the Stream. The default epsilon is
// 0.01 and is usually satisfactory. If needed, this must be called before all
// Inserts.
// To learn more, see: http://www.cs.rutgers.edu/~muthu/bquant.pdf
func (s *stream) SetEpsilon(epsilon float64) {
	s.epsilon = epsilon
}

func (s *stream) reset() {
	s.l.Init()
	s.n = 0
}

func (s *stream) insert(v float64) {
	fn := s.mergeFunc()
	fn(v, 1)
}

func (s *stream) merge(samples Samples) {
	fn := s.mergeFunc()
	for _, s := range samples {
		fn(s.Value, s.Width)
	}
}

func (s *stream) mergeFunc() func(v, w float64) {
	// NOTE: I used a goto over defer because it bought me a few extra
	// nanoseconds. I know. I know.
	var r float64
	e := s.l.Front()
	return func(v, w float64) {
		for ; e != nil; e = e.Next() {
			c := e.Value.(*Sample)
			if c.Value > v {
				sm := &Sample{v, w, math.Floor(s.ƒ(s, r)) - 1}
				s.l.InsertBefore(sm, e)
				goto inserted
			}
			r += c.Width
		}
		s.l.PushBack(&Sample{v, w, 0})
	inserted:
		s.n += w
	}
}

func (s *stream) count() int {
	return int(s.n)
}

func (s *stream) query(q float64) float64 {
	e := s.l.Front()
	t := math.Ceil(q * s.n)
	t += math.Ceil(s.ƒ(s, t) / 2)
	p := e.Value.(*Sample)
	e = e.Next()
	r := float64(0)
	for e != nil {
		c := e.Value.(*Sample)
		if r+c.Width+c.Delta > t {
			return p.Value
		}
		r += p.Width
		p = c
		e = e.Next()
	}
	return p.Value
}

func (s *stream) compress() {
	if s.l.Len() < 2 {
		return
	}
	e := s.l.Back()
	x := e.Value.(*Sample)
	r := s.n - 1 - x.Width
	e = e.Prev()
	for e != nil {
		c := e.Value.(*Sample)
		if c.Width+x.Width+x.Delta <= s.ƒ(s, r) {
			x.Width += c.Width
			o := e
			e = e.Prev()
			s.l.Remove(o)
		} else {
			x = c
			e = e.Prev()
		}
		r -= c.Width
	}
}

func (s *stream) samples() Samples {
	samples := make(Samples, 0, s.l.Len())
	for e := s.l.Front(); e != nil; e = e.Next() {
		samples = append(samples, *e.Value.(*Sample))
	}
	return samples
}
<commit_msg>take care<commit_after>// Package quantile computes approximate quantiles over an unbounded data
// stream within low memory and CPU bounds.
//
// A small amount of accuracy is traded to achieve the above properties.
//
// Multiple streams can be merged before calling Query to generate a single set
// of results. This is meaningful when the streams represent the same type of
// data. See Merge and Samples.
//
// For more detailed information about the algorithm used, see:
//
// Effective Computation of Biased Quantiles over Data Streams
//
// http://www.cs.rutgers.edu/~muthu/bquant.pdf
package quantile

import (
	"container/list"
	"math"
	"sort"
)

// Sample holds an observed value and meta information for compression. JSON
// tags have been added for convenience.
type Sample struct {
	Value float64 `json:",string"`
	Width float64 `json:",string"`
	Delta float64 `json:",string"`
}

// Samples represents a slice of samples. It implements sort.Interface.
type Samples []Sample

func (a Samples) Len() int {
	return len(a)
}

func (a Samples) Less(i, j int) bool {
	return a[i].Value < a[j].Value
}

func (a Samples) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

type invariant func(s *stream, r float64) float64

// NewBiased returns an initialized Stream for high-biased quantiles (e.g.
// 50th, 90th, 99th) not known a priori with ﬁner error guarantees for the
// higher ranks of the data distribution.
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
func NewBiased() *Stream {
	ƒ := func(s *stream, r float64) float64 {
		return 2 * s.epsilon * r
	}
	return newStream(ƒ)
}

// NewTargeted returns an initialized Stream concerned with a particular set of
// quantile values that are supplied a priori. Knowing these a priori reduces
// space and computation time.
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
func NewTargeted(quantiles ...float64) *Stream {
	ƒ := func(s *stream, r float64) float64 {
		var m float64 = math.MaxFloat64
		var f float64
		for _, q := range quantiles {
			if q*s.n <= r {
				f = (2 * s.epsilon * r) / q
			} else {
				f = (2 * s.epsilon * (s.n - r)) / (1 - q)
			}
			m = math.Min(m, f)
		}
		return m
	}
	return newStream(ƒ)
}

// Stream computes quantiles for a stream of float64s. It is not thread-safe by
// design. Take care when using across multiple goroutines.
type Stream struct {
	*stream
	b Samples
}

func newStream(ƒ invariant) *Stream {
	const defaultEpsilon = 0.01
	x := &stream{epsilon: defaultEpsilon, ƒ: ƒ, l: list.New()}
	return &Stream{x, make(Samples, 0, 500)}
}

// Insert inserts v into the stream.
func (s *Stream) Insert(v float64) {
	s.insert(Sample{Value: v, Width: 1})
}

func (s *Stream) insert(sample Sample) {
	s.b = append(s.b, sample)
	if len(s.b) == cap(s.b) {
		s.flush()
		s.compress()
	}
}

// Query returns the computed qth percentiles value. If s was created with
// NewTargeted, and q is not in the set of quantiles provided a priori, Query
// will return an unspecified result.
func (s *Stream) Query(q float64) float64 {
	if s.flushed() {
		// Fast path when there hasn't been enough data for a flush;
		// this also yeilds better accuracy for small sets of data.
		l := len(s.b)
		if l == 0 {
			return 0
		}
		i := int(float64(l) * q)
		if i > 0 {
			i -= 1
		}
		return s.b[i].Value
	}
	s.flush()
	return s.stream.query(q)
}

// Merge merges samples into the underlying streams samples. This is handy when
// merging multiple streams from separate threads, database shards, etc.
func (s *Stream) Merge(samples Samples) {
	s.stream.merge(samples)
}

// Reset reinitializes and clears the list reusing the samples buffer memory.
func (s *Stream) Reset() {
	s.stream.reset()
	s.b = s.b[:0]
}

// Samples returns stream samples held by s.
func (s *Stream) Samples() Samples {
	if !s.flushed() {
		return s.b
	}
	return s.stream.samples()
}

// Count returns the total number of samples observed in the stream
// since initialization.
func (s *Stream) Count() int {
	return len(s.b) + s.stream.count()
}

func (s *Stream) flush() {
	sort.Sort(s.b)
	s.stream.merge(s.b)
	s.b = s.b[:0]
}

func (s *Stream) flushed() bool {
	return s.stream.l.Len() == 0
}

type stream struct {
	epsilon float64
	n       float64
	l       *list.List
	ƒ       invariant
}

// SetEpsilon sets the error epsilon for the Stream. The default epsilon is
// 0.01 and is usually satisfactory. If needed, this must be called before all
// Inserts.
// To learn more, see: http://www.cs.rutgers.edu/~muthu/bquant.pdf
func (s *stream) SetEpsilon(epsilon float64) {
	s.epsilon = epsilon
}

func (s *stream) reset() {
	s.l.Init()
	s.n = 0
}

func (s *stream) insert(v float64) {
	fn := s.mergeFunc()
	fn(v, 1)
}

func (s *stream) merge(samples Samples) {
	fn := s.mergeFunc()
	for _, s := range samples {
		fn(s.Value, s.Width)
	}
}

func (s *stream) mergeFunc() func(v, w float64) {
	// NOTE: I used a goto over defer because it bought me a few extra
	// nanoseconds. I know. I know.
	var r float64
	e := s.l.Front()
	return func(v, w float64) {
		for ; e != nil; e = e.Next() {
			c := e.Value.(*Sample)
			if c.Value > v {
				sm := &Sample{v, w, math.Floor(s.ƒ(s, r)) - 1}
				s.l.InsertBefore(sm, e)
				goto inserted
			}
			r += c.Width
		}
		s.l.PushBack(&Sample{v, w, 0})
	inserted:
		s.n += w
	}
}

func (s *stream) count() int {
	return int(s.n)
}

func (s *stream) query(q float64) float64 {
	e := s.l.Front()
	t := math.Ceil(q * s.n)
	t += math.Ceil(s.ƒ(s, t) / 2)
	p := e.Value.(*Sample)
	e = e.Next()
	r := float64(0)
	for e != nil {
		c := e.Value.(*Sample)
		if r+c.Width+c.Delta > t {
			return p.Value
		}
		r += p.Width
		p = c
		e = e.Next()
	}
	return p.Value
}

func (s *stream) compress() {
	if s.l.Len() < 2 {
		return
	}
	e := s.l.Back()
	x := e.Value.(*Sample)
	r := s.n - 1 - x.Width
	e = e.Prev()
	for e != nil {
		c := e.Value.(*Sample)
		if c.Width+x.Width+x.Delta <= s.ƒ(s, r) {
			x.Width += c.Width
			o := e
			e = e.Prev()
			s.l.Remove(o)
		} else {
			x = c
			e = e.Prev()
		}
		r -= c.Width
	}
}

func (s *stream) samples() Samples {
	samples := make(Samples, 0, s.l.Len())
	for e := s.l.Front(); e != nil; e = e.Next() {
		samples = append(samples, *e.Value.(*Sample))
	}
	return samples
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !appengine

package main

import (
	"flag"
	"fmt"
	"go/build"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"code.google.com/p/go.talks/pkg/socket"

	// Imports so that go build/install automatically installs them.
	_ "code.google.com/p/go-tour/pic"
	_ "code.google.com/p/go-tour/tree"
	_ "code.google.com/p/go-tour/wc"
)

const (
	basePkg    = "code.google.com/p/go-tour/"
	socketPath = "/socket"
)

var (
	httpListen  = flag.String("http", "127.0.0.1:3999", "host:port to listen on")
	htmlOutput  = flag.Bool("html", false, "render program output as HTML")
	openBrowser = flag.Bool("openbrowser", true, "open browser automatically")
)

var (
	// a source of numbers, for naming temporary files
	uniq = make(chan int)

	// GOPATH containing the tour packages
	gopath = os.Getenv("GOPATH")

	httpAddr string
)

func isRoot(path string) bool {
	_, err := os.Stat(filepath.Join(path, "tour.article"))
	return err == nil
}

func findRoot() (string, error) {
	ctx := build.Default
	p, err := ctx.Import(basePkg, "", build.FindOnly)
	if err == nil && isRoot(p.Dir) {
		return p.Dir, nil
	}
	tourRoot := filepath.Join(runtime.GOROOT(), "misc", "tour")
	ctx.GOPATH = tourRoot
	p, err = ctx.Import(basePkg, "", build.FindOnly)
	if err == nil && isRoot(tourRoot) {
		gopath = tourRoot
		return tourRoot, nil
	}
	return "", fmt.Errorf("could not find go-tour content; check $GOROOT and $GOPATH")
}

func main() {
	flag.Parse()

	// source of unique numbers
	go func() {
		for i := 0; ; i++ {
			uniq <- i
		}
	}()

	// find and serve the go tour files
	root, err := findRoot()
	if err != nil {
		log.Fatalf("Couldn't find tour files: %v", err)
	}

	log.Println("Serving content from", root)

	host, port, err := net.SplitHostPort(*httpListen)
	if err != nil {
		log.Fatal(err)
	}
	if host == "" {
		host = "localhost"
	}
	if host != "127.0.0.1" && host != "localhost" {
		log.Print(localhostWarning)
	}
	httpAddr = host + ":" + port

	if err := initTour(root); err != nil {
		log.Fatal(err)
	}

	fs := http.FileServer(http.Dir(root))
	http.Handle("/favicon.ico", fs)
	http.Handle("/static/", fs)
	http.Handle("/talks/", fs)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/" {
			if err := renderTour(w); err != nil {
				log.Println(err)
			}
			return
		}
		http.Error(w, "not found", 404)
	})

	http.Handle(socketPath, socket.Handler)

	err = serveScripts(filepath.Join(root, "js"), "socket.js")
	if err != nil {
		log.Fatal(err)
	}

	go func() {
		url := "http://" + httpAddr
		if waitServer(url) && *openBrowser && startBrowser(url) {
			log.Printf("A browser window should open. If not, please visit %s", url)
		} else {
			log.Printf("Please open your web browser and visit %s", url)
		}
	}()
	log.Fatal(http.ListenAndServe(httpAddr, nil))
}

const localhostWarning = `
WARNING!  WARNING!  WARNING!

I appear to be listening on an address that is not localhost.
Anyone with access to this address and port will have access
to this machine as the user running gotour.

If you don't understand this message, hit Control-C to terminate this process.

WARNING!  WARNING!  WARNING!
`

type response struct {
	Output string `json:"output"`
	Errors string `json:"compile_errors"`
}

// environ returns an execution environment containing only GO* variables
// and replacing GOPATH with the value of the global var gopath.
func environ() (env []string) {
	for _, v := range os.Environ() {
		if !strings.HasPrefix(v, "GO") {
			continue
		}
		if strings.HasPrefix(v, "GOPATH=") {
			v = "GOPATH=" + gopath
		}
		env = append(env, v)
	}
	return
}

// waitServer waits some time for the http Server to start
// serving url and returns whether it starts
func waitServer(url string) bool {
	tries := 20
	for tries > 0 {
		resp, err := http.Get(url)
		if err == nil {
			resp.Body.Close()
			return true
		}
		time.Sleep(100 * time.Millisecond)
		tries--
	}
	return false
}

// startBrowser tries to open the URL in a browser, and returns
// whether it succeed.
func startBrowser(url string) bool {
	// try to start the browser
	var args []string
	switch runtime.GOOS {
	case "darwin":
		args = []string{"open"}
	case "windows":
		args = []string{"cmd", "/c", "start"}
	default:
		args = []string{"xdg-open"}
	}
	cmd := exec.Command(args[0], append(args[1:], url)...)
	return cmd.Start() == nil
}

// prepContent for the local tour simply returns the content as-is.
func prepContent(r io.Reader) io.Reader { return r }

// socketAddr returns the WebSocket handler address.
func socketAddr() string { return "ws://" + httpAddr + socketPath }
<commit_msg>Apply codereview.appspot.com/8567043<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !appengine

package main

import (
	"flag"
	"fmt"
	"go/build"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"code.google.com/p/go.talks/pkg/socket"

	// Imports so that go build/install automatically installs them.
	_ "code.google.com/p/go-tour/pic"
	_ "code.google.com/p/go-tour/tree"
	_ "code.google.com/p/go-tour/wc"
)

const (
	basePkg    = "code.google.com/p/go-tour/"
	socketPath = "/socket"
)

var (
	httpListen  = flag.String("http", "127.0.0.1:3999", "host:port to listen on")
	htmlOutput  = flag.Bool("html", false, "render program output as HTML")
	openBrowser = flag.Bool("openbrowser", true, "open browser automatically")
)

var (
	// a source of numbers, for naming temporary files
	uniq = make(chan int)

	// GOPATH containing the tour packages
	gopath = os.Getenv("GOPATH")

	httpAddr string
)

func isRoot(path string) bool {
	_, err := os.Stat(filepath.Join(path, "tour.article"))
	return err == nil
}

func findRoot() (string, error) {
	ctx := build.Default
	p, err := ctx.Import(basePkg, "", build.FindOnly)
	if err == nil && isRoot(p.Dir) {
		return p.Dir, nil
	}
	tourRoot := filepath.Join(runtime.GOROOT(), "misc", "tour")
	ctx.GOPATH = tourRoot
	p, err = ctx.Import(basePkg, "", build.FindOnly)
	if err == nil && isRoot(tourRoot) {
		gopath = tourRoot
		return tourRoot, nil
	}
	return "", fmt.Errorf("could not find go-tour content; check $GOROOT and $GOPATH")
}

func main() {
	flag.Parse()

	// source of unique numbers
	go func() {
		for i := 0; ; i++ {
			uniq <- i
		}
	}()

	// find and serve the go tour files
	root, err := findRoot()
	if err != nil {
		log.Fatalf("Couldn't find tour files: %v", err)
	}

	log.Println("Serving content from", root)

	host, port, err := net.SplitHostPort(*httpListen)
	if err != nil {
		log.Fatal(err)
	}
	if host == "" {
		host = "localhost"
	}
	if host != "127.0.0.1" && host != "localhost" {
		log.Print(localhostWarning)
	}
	httpAddr = host + ":" + port

	if err := initTour(root); err != nil {
		log.Fatal(err)
	}

	fs := http.FileServer(http.Dir(root))
	http.Handle("/favicon.ico", fs)
	http.Handle("/static/", fs)
	http.Handle("/talks/", fs)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/" {
			if err := renderTour(w); err != nil {
				log.Println(err)
			}
			return
		}
		http.Error(w, "not found", 404)
	})

	http.Handle(socketPath, socket.Handler)

	err = serveScripts(filepath.Join(root, "js"), "socket.js")
	if err != nil {
		log.Fatal(err)
	}

	go func() {
		url := "http://" + httpAddr
		if waitServer(url) && *openBrowser && startBrowser(url) {
			log.Printf("A browser window should open. If not, please visit %s", url)
		} else {
			log.Printf("Please open your web browser and visit %s", url)
		}
	}()
	log.Fatal(http.ListenAndServe(httpAddr, nil))
}

const localhostWarning = `
WARNING!  WARNING!  WARNING!

I appear to be listening on an address that is not localhost.
Anyone with access to this address and port will have access
to this machine as the user running gotour.

If you don't understand this message, hit Control-C to terminate this process.

WARNING!  WARNING!  WARNING!
`

type response struct {
	Output string `json:"output"`
	Errors string `json:"compile_errors"`
}

func init() {
	socket.Environ = environ
}

// environ returns an execution environment containing only GO* variables
// and replacing GOPATH with the value of the global var gopath.
func environ() (env []string) {
	for _, v := range os.Environ() {
		if !strings.HasPrefix(v, "GO") {
			continue
		}
		if strings.HasPrefix(v, "GOPATH=") {
			v = "GOPATH=" + gopath
		}
		env = append(env, v)
	}
	return
}

// waitServer waits some time for the http Server to start
// serving url and returns whether it starts
func waitServer(url string) bool {
	tries := 20
	for tries > 0 {
		resp, err := http.Get(url)
		if err == nil {
			resp.Body.Close()
			return true
		}
		time.Sleep(100 * time.Millisecond)
		tries--
	}
	return false
}

// startBrowser tries to open the URL in a browser, and returns
// whether it succeed.
func startBrowser(url string) bool {
	// try to start the browser
	var args []string
	switch runtime.GOOS {
	case "darwin":
		args = []string{"open"}
	case "windows":
		args = []string{"cmd", "/c", "start"}
	default:
		args = []string{"xdg-open"}
	}
	cmd := exec.Command(args[0], append(args[1:], url)...)
	return cmd.Start() == nil
}

// prepContent for the local tour simply returns the content as-is.
func prepContent(r io.Reader) io.Reader { return r }

// socketAddr returns the WebSocket handler address.
func socketAddr() string { return "ws://" + httpAddr + socketPath }
<|endoftext|>


<commit_before>package regexp_test

import (
	"flag"
	"fmt"
	"regexp"
	"testing"

	"github.com/openconfig/goyang/pkg/yang"

	"gooctest"
)

var ocdir string

// YANGLeaf is a structure sued to describe a particular leaf of YANG schema.
type YANGLeaf struct {
	module string
	name   string
}

// RegexpTest specifies a test case for a particular regular expression check.
type RegexpTest struct {
	inData    string
	wantMatch bool
}

// TestRegexps tests mock input data against a set of leaves that have patterns
// specified for them. It ensures that the regexp compiles as a POSIX regular
// expression according to the OpenConfig style guide.
func TestRegexps(t *testing.T) {
	tests := []struct {
		name     string
		modules  []string
		leaf     YANGLeaf
		testData []RegexpTest
	}{{
		name:    "ipv4 address",
		modules: []string{"testdata/test.yang"},
		leaf:    YANGLeaf{"regexp-test", "ipv4-address"},
		testData: []RegexpTest{
			RegexpTest{`1.1.1.1`, true},
			RegexpTest{`1.1.1.256`, false},
			RegexpTest{`256.1.1.1%eth0`, false},
		},
	}, {
		name:    "union ip address",
		modules: []string{"testdata/test.yang"},
		leaf:    YANGLeaf{"regexp-test", "ip-address"},
		testData: []RegexpTest{
			RegexpTest{`255.255.255.255`, true},
			RegexpTest{`2001:db8::1`, true},
			RegexpTest{"invalid-data", false},
			RegexpTest{`::1`, true},
		},
	}, {
		name:    "bgp-standard-community",
		modules: []string{"testdata/test.yang"},
		leaf:    YANGLeaf{"regexp-test", "bgp-std-community"},
		testData: []RegexpTest{
			RegexpTest{`15169:42`, true},
			RegexpTest{`6643:21438`, true},
			RegexpTest{`29636:4444`, true},
			RegexpTest{`65535:65535`, true},
			RegexpTest{`0:0`, true},
			RegexpTest{`65536:1`, false},
			RegexpTest{`1:65536`, false},
			RegexpTest{`425353:comm`, false},
		},
	}, {
		name:    "ipv4-prefix",
		modules: []string{"testdata/test.yang"},
		leaf:    YANGLeaf{"regexp-test", "ipv4-prefix"},
		testData: []RegexpTest{
			RegexpTest{`0.0.0.0/0`, true},
			RegexpTest{`255.255.255.255/32`, true},
			RegexpTest{`256.0.0.0/31`, false},
			RegexpTest{`1.2.3.0/24`, true},
			RegexpTest{`1.2.3.4/33`, false},
		},
	}, {
		name:    "ipv6-prefix",
		modules: []string{"testdata/test.yang"},
		leaf:    YANGLeaf{"regexp-test", "ipv6-prefix"},
		testData: []RegexpTest{
			RegexpTest{`::/0`, true},
			RegexpTest{`FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF/128`, true},
			RegexpTest{`FFFF:FFFF:FFFF:NOTVALID:FFFF:FFFF:FFFF:FFFF/64`, false},
			RegexpTest{`2001:DB8::/32`, true},
			RegexpTest{`2001:4C20::/129`, false},
		},
	}, {
		name:    "ip-prefix",
		modules: []string{"testdata/test.yang"},
		leaf:    YANGLeaf{"regexp-test", "ip-prefix"},
		testData: []RegexpTest{
			RegexpTest{"0.0.0.0/0", true},
			RegexpTest{"192.0.2.1/32", true},
			RegexpTest{"192.0.2.2/33", false},
			RegexpTest{"FE80::CAFE/128", true},
			RegexpTest{"FE81::CAFE:DEAD:BEEF/129", false},
		},
	}, {
		name:    "ipv6-address",
		modules: []string{"testdata/test.yang"},
		leaf:    YANGLeaf{"regexp-test", "ipv6-address"},
		testData: []RegexpTest{
			RegexpTest{"2620::1000:3202:23e:e1ff:fec7:7112", true},
			RegexpTest{"fe80::23e:e1ff:fec7:7112", true},
			RegexpTest{"fe80::23e:NOTVALID:fec7:7112", false},
			RegexpTest{"FFFF::NOTE::FFFF", false},
			RegexpTest{"FFFF::1::42", false},
		},
	}, {
		name:    "bgp-extended-community",
		modules: []string{"testdata/test.yang"},
		leaf:    YANGLeaf{"regexp-test", "bgp-ext-community"},
		testData: []RegexpTest{
			// Type 1 extended communities (2b AS: 4b integer)
			RegexpTest{`29636:10`, true},
			RegexpTest{`5413:4294967295`, true},
			RegexpTest{`4445:0`, true},
			RegexpTest{`1273:4294967296`, false},
			RegexpTest{`2856:400`, true},
			RegexpTest{`5400:invalid`, false},
			RegexpTest{`i6643:10`, false},
			RegexpTest{`15169:22432`, true},
			// Type 2 extended communities: (4b IP: 2b integer)
			RegexpTest{`1.1.1.1:4294967296`, false},
			RegexpTest{`1.2.3.4.5:10`, false},
			RegexpTest{`82.42.12.35:65535`, true},
			RegexpTest{`82.42.12.35:66536`, false},
			RegexpTest{`254.254.256.254:10`, false},
			RegexpTest{`0.0.0.0:200`, true},
			RegexpTest{`leading192.0.2.1:65535`, false},
			// 4b AS : 2b integer
			RegexpTest{`4294967296:65535`, false},
			RegexpTest{`4294967295:65535`, true},
			RegexpTest{`0:65535`, true},
			RegexpTest{`4294967295:0`, true},
			RegexpTest{`4294967296:0`, false},
			// Route Target Type 1 - route-target:<2b AS>:<4b local>
			RegexpTest{`route-target:64`, false},
			RegexpTest{`route-target:65535:10`, true},
			RegexpTest{`route-TARGET:65535:10`, false},
			RegexpTest{`route-target:15169:4294967296`, false},
			RegexpTest{`route-target:15169:4294967295`, true},
			// Route Target Type 2 - route-target:<ipv4>:<2b local>
			RegexpTest{`route-target:256.0.2.36:10`, false},
			RegexpTest{`route-target:192.0.2.1:10`, true},
			RegexpTest{`route-target:192.0.2.1:65536`, false},
			// Route Target w/ 4B AS:<2b local>
			RegexpTest{`route-target:4294967295:10`, true},
			RegexpTest{`route-target:4294967296:10`, false},
			RegexpTest{`route-target:5413:65535`, true},
			// Route Origin Type 1 - route-target:<2b AS>:<4b local>
			RegexpTest{`route-origin:53`, false},
			RegexpTest{`route-origin:65535:10`, true},
			RegexpTest{`route-ORIGINTRAIL:65535:10`, false},
			RegexpTest{`route-origin:15169:4294967296`, false},
			RegexpTest{`route-origin:15169:4294967295`, true},
			// Route Origin Type 2 - route-target:<ipv4>:<2b local>
			RegexpTest{`route-origin:512.0.2.36:10`, false},
			RegexpTest{`route-origin:10.18.253.24:10`, true},
			RegexpTest{`route-origin:192.168.1.1:65536`, false},
			// Route Origin w/ 4B AS:<2b local>
			RegexpTest{`route-origin:4294967295:5353`, true},
			RegexpTest{`route-origin:4294967296:9009`, false},
			RegexpTest{`route-origin:5413:65535`, true},
		},
	}}

	for _, tt := range tests {
		yangE, errs := gooctest.ProcessModules(tt.modules, []string{ocdir})
		if len(errs) != 0 {
			t.Fatalf("%s: could not parse modules: %v", tt.name, errs)
		}

		mod, modok := yangE[tt.leaf.module]
		if !modok {
			t.Fatalf("%s: could not find expected module: %s (%v)", tt.name, tt.leaf.module, yangE)
		}

		leaf, leafok := mod.Dir[tt.leaf.name]
		if !leafok {
			t.Fatalf("%s: could not find expected leaf: %s", tt.name, tt.leaf.name)
		}

		if len(leaf.Errors) != 0 {
			t.Errorf("%s: leaf had associated errors: %v", tt.name, leaf.Errors)
			continue
		}

		for _, tc := range tt.testData {
			var gotMatch bool
			if len(leaf.Type.Type) == 0 {
				_, gotMatch = checkPattern(tc.inData, leaf.Type.Pattern)
			} else {
				// Handle unions
				results := make([]bool, 0)
				for _, membertype := range leaf.Type.Type {
					// Only do the test when there is a pattern specified against the
					// type as it may not be a string.
					if membertype.Kind != yang.Ystring || len(membertype.Pattern) == 0 {
						continue
					}
					matchedAllForType := true
					_, matchedAllForType = checkPattern(tc.inData, membertype.Pattern)
					results = append(results, matchedAllForType)
				}

				gotMatch = false
				for _, r := range results {
					if r == true {
						gotMatch = true
					}
				}
			}

			if gotMatch != tc.wantMatch {
				t.Errorf("%s: string %s did not have expected result: %v",
					tt.name, tc.inData, tc.wantMatch)
			}
		}
	}
}

// checkPattern builds and compils
func checkPattern(testData string, patterns []string) (compileErr error, matched bool) {
	for _, pattern := range patterns {
		if r, err := regexp.CompilePOSIX(fmt.Sprintf("^%s$", pattern)); err != nil {
			return
		} else {
			matched = r.MatchString(testData)
		}
	}
	return
}

// init sets up the test, particularly parsing the OpenConfig path which is
// supplied as a command line argument.
func init() {
	flag.StringVar(&ocdir, "ocdir", "../..", "Path to OpenConfig models repo")
	flag.Parse()
}
<commit_msg>fix attempt<commit_after>package regexp_test

import (
	"fmt"
	"log"
	"os"
	"regexp"
	"testing"

	"github.com/openconfig/goyang/pkg/yang"

	"gooctest"
)

var ocdir string

// YANGLeaf is a structure sued to describe a particular leaf of YANG schema.
type YANGLeaf struct {
	module string
	name   string
}

// RegexpTest specifies a test case for a particular regular expression check.
type RegexpTest struct {
	inData    string
	wantMatch bool
}

// TestRegexps tests mock input data against a set of leaves that have patterns
// specified for them. It ensures that the regexp compiles as a POSIX regular
// expression according to the OpenConfig style guide.
func TestRegexps(t *testing.T) {
	tests := []struct {
		name     string
		modules  []string
		leaf     YANGLeaf
		testData []RegexpTest
	}{{
		name:    "ipv4 address",
		modules: []string{"testdata/test.yang"},
		leaf:    YANGLeaf{"regexp-test", "ipv4-address"},
		testData: []RegexpTest{
			RegexpTest{`1.1.1.1`, true},
			RegexpTest{`1.1.1.256`, false},
			RegexpTest{`256.1.1.1%eth0`, false},
		},
	}, {
		name:    "union ip address",
		modules: []string{"testdata/test.yang"},
		leaf:    YANGLeaf{"regexp-test", "ip-address"},
		testData: []RegexpTest{
			RegexpTest{`255.255.255.255`, true},
			RegexpTest{`2001:db8::1`, true},
			RegexpTest{"invalid-data", false},
			RegexpTest{`::1`, true},
		},
	}, {
		name:    "bgp-standard-community",
		modules: []string{"testdata/test.yang"},
		leaf:    YANGLeaf{"regexp-test", "bgp-std-community"},
		testData: []RegexpTest{
			RegexpTest{`15169:42`, true},
			RegexpTest{`6643:21438`, true},
			RegexpTest{`29636:4444`, true},
			RegexpTest{`65535:65535`, true},
			RegexpTest{`0:0`, true},
			RegexpTest{`65536:1`, false},
			RegexpTest{`1:65536`, false},
			RegexpTest{`425353:comm`, false},
		},
	}, {
		name:    "ipv4-prefix",
		modules: []string{"testdata/test.yang"},
		leaf:    YANGLeaf{"regexp-test", "ipv4-prefix"},
		testData: []RegexpTest{
			RegexpTest{`0.0.0.0/0`, true},
			RegexpTest{`255.255.255.255/32`, true},
			RegexpTest{`256.0.0.0/31`, false},
			RegexpTest{`1.2.3.0/24`, true},
			RegexpTest{`1.2.3.4/33`, false},
		},
	}, {
		name:    "ipv6-prefix",
		modules: []string{"testdata/test.yang"},
		leaf:    YANGLeaf{"regexp-test", "ipv6-prefix"},
		testData: []RegexpTest{
			RegexpTest{`::/0`, true},
			RegexpTest{`FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF/128`, true},
			RegexpTest{`FFFF:FFFF:FFFF:NOTVALID:FFFF:FFFF:FFFF:FFFF/64`, false},
			RegexpTest{`2001:DB8::/32`, true},
			RegexpTest{`2001:4C20::/129`, false},
		},
	}, {
		name:    "ip-prefix",
		modules: []string{"testdata/test.yang"},
		leaf:    YANGLeaf{"regexp-test", "ip-prefix"},
		testData: []RegexpTest{
			RegexpTest{"0.0.0.0/0", true},
			RegexpTest{"192.0.2.1/32", true},
			RegexpTest{"192.0.2.2/33", false},
			RegexpTest{"FE80::CAFE/128", true},
			RegexpTest{"FE81::CAFE:DEAD:BEEF/129", false},
		},
	}, {
		name:    "ipv6-address",
		modules: []string{"testdata/test.yang"},
		leaf:    YANGLeaf{"regexp-test", "ipv6-address"},
		testData: []RegexpTest{
			RegexpTest{"2620::1000:3202:23e:e1ff:fec7:7112", true},
			RegexpTest{"fe80::23e:e1ff:fec7:7112", true},
			RegexpTest{"fe80::23e:NOTVALID:fec7:7112", false},
			RegexpTest{"FFFF::NOTE::FFFF", false},
			RegexpTest{"FFFF::1::42", false},
		},
	}, {
		name:    "bgp-extended-community",
		modules: []string{"testdata/test.yang"},
		leaf:    YANGLeaf{"regexp-test", "bgp-ext-community"},
		testData: []RegexpTest{
			// Type 1 extended communities (2b AS: 4b integer)
			RegexpTest{`29636:10`, true},
			RegexpTest{`5413:4294967295`, true},
			RegexpTest{`4445:0`, true},
			RegexpTest{`1273:4294967296`, false},
			RegexpTest{`2856:400`, true},
			RegexpTest{`5400:invalid`, false},
			RegexpTest{`i6643:10`, false},
			RegexpTest{`15169:22432`, true},
			// Type 2 extended communities: (4b IP: 2b integer)
			RegexpTest{`1.1.1.1:4294967296`, false},
			RegexpTest{`1.2.3.4.5:10`, false},
			RegexpTest{`82.42.12.35:65535`, true},
			RegexpTest{`82.42.12.35:66536`, false},
			RegexpTest{`254.254.256.254:10`, false},
			RegexpTest{`0.0.0.0:200`, true},
			RegexpTest{`leading192.0.2.1:65535`, false},
			// 4b AS : 2b integer
			RegexpTest{`4294967296:65535`, false},
			RegexpTest{`4294967295:65535`, true},
			RegexpTest{`0:65535`, true},
			RegexpTest{`4294967295:0`, true},
			RegexpTest{`4294967296:0`, false},
			// Route Target Type 1 - route-target:<2b AS>:<4b local>
			RegexpTest{`route-target:64`, false},
			RegexpTest{`route-target:65535:10`, true},
			RegexpTest{`route-TARGET:65535:10`, false},
			RegexpTest{`route-target:15169:4294967296`, false},
			RegexpTest{`route-target:15169:4294967295`, true},
			// Route Target Type 2 - route-target:<ipv4>:<2b local>
			RegexpTest{`route-target:256.0.2.36:10`, false},
			RegexpTest{`route-target:192.0.2.1:10`, true},
			RegexpTest{`route-target:192.0.2.1:65536`, false},
			// Route Target w/ 4B AS:<2b local>
			RegexpTest{`route-target:4294967295:10`, true},
			RegexpTest{`route-target:4294967296:10`, false},
			RegexpTest{`route-target:5413:65535`, true},
			// Route Origin Type 1 - route-target:<2b AS>:<4b local>
			RegexpTest{`route-origin:53`, false},
			RegexpTest{`route-origin:65535:10`, true},
			RegexpTest{`route-ORIGINTRAIL:65535:10`, false},
			RegexpTest{`route-origin:15169:4294967296`, false},
			RegexpTest{`route-origin:15169:4294967295`, true},
			// Route Origin Type 2 - route-target:<ipv4>:<2b local>
			RegexpTest{`route-origin:512.0.2.36:10`, false},
			RegexpTest{`route-origin:10.18.253.24:10`, true},
			RegexpTest{`route-origin:192.168.1.1:65536`, false},
			// Route Origin w/ 4B AS:<2b local>
			RegexpTest{`route-origin:4294967295:5353`, true},
			RegexpTest{`route-origin:4294967296:9009`, false},
			RegexpTest{`route-origin:5413:65535`, true},
		},
	}}

	for _, tt := range tests {
		yangE, errs := gooctest.ProcessModules(tt.modules, []string{ocdir})
		if len(errs) != 0 {
			t.Fatalf("%s: could not parse modules: %v", tt.name, errs)
		}

		mod, modok := yangE[tt.leaf.module]
		if !modok {
			t.Fatalf("%s: could not find expected module: %s (%v)", tt.name, tt.leaf.module, yangE)
		}

		leaf, leafok := mod.Dir[tt.leaf.name]
		if !leafok {
			t.Fatalf("%s: could not find expected leaf: %s", tt.name, tt.leaf.name)
		}

		if len(leaf.Errors) != 0 {
			t.Errorf("%s: leaf had associated errors: %v", tt.name, leaf.Errors)
			continue
		}

		for _, tc := range tt.testData {
			var gotMatch bool
			if len(leaf.Type.Type) == 0 {
				_, gotMatch = checkPattern(tc.inData, leaf.Type.Pattern)
			} else {
				// Handle unions
				results := make([]bool, 0)
				for _, membertype := range leaf.Type.Type {
					// Only do the test when there is a pattern specified against the
					// type as it may not be a string.
					if membertype.Kind != yang.Ystring || len(membertype.Pattern) == 0 {
						continue
					}
					matchedAllForType := true
					_, matchedAllForType = checkPattern(tc.inData, membertype.Pattern)
					results = append(results, matchedAllForType)
				}

				gotMatch = false
				for _, r := range results {
					if r == true {
						gotMatch = true
					}
				}
			}

			if gotMatch != tc.wantMatch {
				t.Errorf("%s: string %s did not have expected result: %v",
					tt.name, tc.inData, tc.wantMatch)
			}
		}
	}
}

// checkPattern builds and compils
func checkPattern(testData string, patterns []string) (compileErr error, matched bool) {
	for _, pattern := range patterns {
		if r, err := regexp.CompilePOSIX(fmt.Sprintf("^%s$", pattern)); err != nil {
			return
		} else {
			matched = r.MatchString(testData)
		}
	}
	return
}

// init sets up the test, particularly parsing the OpenConfig path which is
// supplied as a command line argument.
func init() {
	ocdir = os.Getenv("OCDIR")
	if ocdir == "" {
		log.Fatal("missing environment variable $OCDIR for specifying model root directory")
	}
}
<|endoftext|>