<commit_before>package main

import (
	"flag"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"strings"
	"sync"
	"syscall"
)

func main() {
	var dependencies []StaticLibrary
	wg := new(sync.WaitGroup)

	// set parallel numbers
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Parse flags
	version := flag.String("v", NGINX_VERSION, "nginx version")
	nginxConfigurePath := flag.String("c", "", "configuration file for building nginx")
	modulesConfPath := flag.String("m", "", "configuration file for 3rd party modules")
	workParentDir := flag.String("d", "", "working directory")
	jobs := flag.Int("j", runtime.NumCPU(), "jobs to build nginx")
	verbose := flag.Bool("verbose", false, "verbose mode")
	pcreStatic := flag.Bool("pcre", false, "embedded PCRE statically")
	pcreVersion := flag.String("pcreversion", PCRE_VERSION, "PCRE version")
	openSSLStatic := flag.Bool("openssl", false, "embedded OpenSSL statically")
	openSSLVersion := flag.String("opensslversion", OPENSSL_VERSION, "OpenSSL version")
	zlibStatic := flag.Bool("zlib", false, "embedded zlib statically")
	zlibVersion := flag.String("zlibversion", ZLIB_VERSION, "zlib version")
	clear := flag.Bool("clear", false, "remove entries in working directory")
	versionPrint := flag.Bool("version", false, "print nginx-build versions")
	versionsPrint := flag.Bool("versions", false, "print nginx versions")
	openResty := flag.Bool("openresty", false, "download openresty instead of nginx")
	tengine := flag.Bool("tengine", false, "download tengine instead of nginx")
	openRestyVersion := flag.String("openrestyversion", OPENRESTY_VERSION, "openresty version")
	tengineVersion := flag.String("tengineversion", TENGINE_VERSION, "tengine version")
	configureOnly := flag.Bool("configureonly", false, "configuring nginx only not building")

	var configureOptions ConfigureOptions
	argsString := makeArgsString()
	argsBool := makeArgsBool()
	var multiflag StringFlag

	for k, v := range argsString {
		if k == "add-module" {
			flag.Var(&multiflag, k, v.Desc)
		} else {
			v.Value = flag.String(k, "", v.Desc)
			argsString[k] = v
		}
	}

	for k, v := range argsBool {
		v.Enabled = flag.Bool(k, false, v.Desc)
		argsBool[k] = v
	}

	flag.Parse()

	// Allow multiple flags for `--add-module`
	{
		tmp := argsString["add-module"]
		tmp_ := multiflag.String()
		tmp.Value = &tmp_
		argsString["add-module"] = tmp
	}

	configureOptions.Values = argsString
	configureOptions.Bools = argsBool

	if *versionPrint {
		printNginxBuildVersion()
		return
	}

	if *versionsPrint {
		printNginxVersions()
		return
	}

	printFirstMsg()

	// set verbose mode
	VerboseEnabled = *verbose

	var nginxBuilder Builder
	if *openResty && *tengine {
		log.Fatal("select one between '-openresty' and '-tengine'.")
	}
	if *openResty {
		nginxBuilder = makeBuilder(COMPONENT_OPENRESTY, *openRestyVersion)
	} else if *tengine {
		nginxBuilder = makeBuilder(COMPONENT_TENGINE, *tengineVersion)
	} else {
		nginxBuilder = makeBuilder(COMPONENT_NGINX, *version)
	}
	pcreBuilder := makeBuilder(COMPONENT_PCRE, *pcreVersion)
	openSSLBuilder := makeBuilder(COMPONENT_OPENSSL, *openSSLVersion)
	zlibBuilder := makeBuilder(COMPONENT_ZLIB, *zlibVersion)

	// change default umask
	_ = syscall.Umask(0)

	versionCheck(*version)

	nginxConfigure, err := fileGetContents(*nginxConfigurePath)
	if err != nil {
		log.Fatal(err)
	}
	nginxConfigure = normalizeConfigure(nginxConfigure)

	modules3rd, err := loadModules3rdFile(*modulesConfPath)
	if err != nil {
		log.Fatal(err)
	}

	if len(*workParentDir) == 0 {
		log.Fatal("set working directory with -d")
	}

	if !fileExists(*workParentDir) {
		err := os.Mkdir(*workParentDir, 0755)
		if err != nil {
			log.Fatalf("Failed to create working directory(%s) does not exist.", *workParentDir)
		}
	}

	var workDir string
	if *openResty {
		workDir = *workParentDir + "/openresty/" + *openRestyVersion
	} else if *tengine {
		workDir = *workParentDir + "/tengine/" + *tengineVersion
	} else {
		workDir = *workParentDir + "/nginx/" + *version
	}
	if *clear {
		err := clearWorkDir(workDir)
		if err != nil {
			log.Fatal(err)
		}
	}

	if !fileExists(workDir) {
		err := os.MkdirAll(workDir, 0755)
		if err != nil {
			log.Fatalf("Failed to create working directory(%s) does not exist.", workDir)
		}
	}

	rootDir := saveCurrentDir()
	// cd workDir
	err = os.Chdir(workDir)
	if err != nil {
		log.Fatal(err)
	}

	if *pcreStatic {
		wg.Add(1)
		go downloadAndExtractParallel(&pcreBuilder, wg)
	}

	if *openSSLStatic {
		wg.Add(1)
		go downloadAndExtractParallel(&openSSLBuilder, wg)
	}

	if *zlibStatic {
		wg.Add(1)
		go downloadAndExtractParallel(&zlibBuilder, wg)
	}

	wg.Add(1)
	go downloadAndExtractParallel(&nginxBuilder, wg)

	if len(modules3rd) > 0 {
		wg.Add(len(modules3rd))
		for _, m := range modules3rd {
			go downloadAndExtractModule3rdParallel(m, wg)
		}

	}

	// wait until all downloading processes by goroutine finish
	wg.Wait()

	if len(modules3rd) > 0 {
		for _, m := range modules3rd {
			provideModule3rd(&m)
		}
	}

	// cd workDir/nginx-${version}
	os.Chdir(nginxBuilder.sourcePath())

	if *pcreStatic {
		dependencies = append(dependencies, makeStaticLibrary(&pcreBuilder))
	}

	if *openSSLStatic {
		dependencies = append(dependencies, makeStaticLibrary(&openSSLBuilder))
	}

	if *zlibStatic {
		dependencies = append(dependencies, makeStaticLibrary(&zlibBuilder))
	}

	log.Printf("Generate configure script for %s.....", nginxBuilder.sourcePath())

	if *pcreStatic && pcreBuilder.isIncludeWithOption(nginxConfigure) {
		log.Printf(pcreBuilder.warnMsgWithLibrary())
	}

	if *openSSLStatic && openSSLBuilder.isIncludeWithOption(nginxConfigure) {
		log.Printf(openSSLBuilder.warnMsgWithLibrary())

	}

	if *zlibStatic && zlibBuilder.isIncludeWithOption(nginxConfigure) {
		log.Printf(zlibBuilder.warnMsgWithLibrary())
	}

	if strings.Contains(nginxConfigure, "--add-module=") {
		log.Println("[warn]Using '--add-module' is discouraged. Instead give ini-file with '-m' to 'nginx-build'")
	}

	configureScript := configureGen(nginxConfigure, modules3rd, dependencies, configureOptions, rootDir)

	err = ioutil.WriteFile("./nginx-configure", []byte(configureScript), 0655)
	if err != nil {
		log.Fatalf("Failed to generate configure script for %s", nginxBuilder.sourcePath())
	}

	log.Printf("Configure %s.....", nginxBuilder.sourcePath())

	err = configureNginx()
	if err != nil {
		log.Printf("Failed to configure %s\n", nginxBuilder.sourcePath())
		printFatalMsg(err, "nginx-configure.log")
	}

	if *configureOnly {
		printLastMsg(workDir, nginxBuilder.sourcePath(), *openResty, *configureOnly)
		return
	}

	log.Printf("Build %s.....", nginxBuilder.sourcePath())

	if *openSSLStatic {
		// Workarounds for protecting a failure of building nginx with static-linked OpenSSL.

		// Unfortunately a build of OpenSSL fails when multi-CPUs are used.
		*jobs = 1

		// Sometimes machine hardware name('uname -m') is different
		// from machine processor architecture name('uname -p') on Mac.
		// Specifically, `uname -p` is 'i386' and `uname -m` is 'x86_64'.
		// In this case, a build of OpenSSL fails.
		// So it needs to convince OpenSSL with KERNEL_BITS.
		if runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
			os.Setenv("KERNEL_BITS", "64")
		}
	}

	err = buildNginx(*jobs)
	if err != nil {
		log.Printf("Failed to build %s\n", nginxBuilder.sourcePath())
		printFatalMsg(err, "nginx-build.log")
	}

	printLastMsg(workDir, nginxBuilder.sourcePath(), *openResty, *configureOnly)

	// cd rootDir
	os.Chdir(rootDir)
}
<commit_msg>removed runtime.GOMAXPROCS(runtime.NumCPU()).<commit_after>package main

import (
	"flag"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"strings"
	"sync"
	"syscall"
)

func main() {
	var dependencies []StaticLibrary
	wg := new(sync.WaitGroup)

	// Parse flags
	version := flag.String("v", NGINX_VERSION, "nginx version")
	nginxConfigurePath := flag.String("c", "", "configuration file for building nginx")
	modulesConfPath := flag.String("m", "", "configuration file for 3rd party modules")
	workParentDir := flag.String("d", "", "working directory")
	jobs := flag.Int("j", runtime.NumCPU(), "jobs to build nginx")
	verbose := flag.Bool("verbose", false, "verbose mode")
	pcreStatic := flag.Bool("pcre", false, "embedded PCRE statically")
	pcreVersion := flag.String("pcreversion", PCRE_VERSION, "PCRE version")
	openSSLStatic := flag.Bool("openssl", false, "embedded OpenSSL statically")
	openSSLVersion := flag.String("opensslversion", OPENSSL_VERSION, "OpenSSL version")
	zlibStatic := flag.Bool("zlib", false, "embedded zlib statically")
	zlibVersion := flag.String("zlibversion", ZLIB_VERSION, "zlib version")
	clear := flag.Bool("clear", false, "remove entries in working directory")
	versionPrint := flag.Bool("version", false, "print nginx-build versions")
	versionsPrint := flag.Bool("versions", false, "print nginx versions")
	openResty := flag.Bool("openresty", false, "download openresty instead of nginx")
	tengine := flag.Bool("tengine", false, "download tengine instead of nginx")
	openRestyVersion := flag.String("openrestyversion", OPENRESTY_VERSION, "openresty version")
	tengineVersion := flag.String("tengineversion", TENGINE_VERSION, "tengine version")
	configureOnly := flag.Bool("configureonly", false, "configuring nginx only not building")

	var configureOptions ConfigureOptions
	argsString := makeArgsString()
	argsBool := makeArgsBool()
	var multiflag StringFlag

	for k, v := range argsString {
		if k == "add-module" {
			flag.Var(&multiflag, k, v.Desc)
		} else {
			v.Value = flag.String(k, "", v.Desc)
			argsString[k] = v
		}
	}

	for k, v := range argsBool {
		v.Enabled = flag.Bool(k, false, v.Desc)
		argsBool[k] = v
	}

	flag.Parse()

	// Allow multiple flags for `--add-module`
	{
		tmp := argsString["add-module"]
		tmp_ := multiflag.String()
		tmp.Value = &tmp_
		argsString["add-module"] = tmp
	}

	configureOptions.Values = argsString
	configureOptions.Bools = argsBool

	if *versionPrint {
		printNginxBuildVersion()
		return
	}

	if *versionsPrint {
		printNginxVersions()
		return
	}

	printFirstMsg()

	// set verbose mode
	VerboseEnabled = *verbose

	var nginxBuilder Builder
	if *openResty && *tengine {
		log.Fatal("select one between '-openresty' and '-tengine'.")
	}
	if *openResty {
		nginxBuilder = makeBuilder(COMPONENT_OPENRESTY, *openRestyVersion)
	} else if *tengine {
		nginxBuilder = makeBuilder(COMPONENT_TENGINE, *tengineVersion)
	} else {
		nginxBuilder = makeBuilder(COMPONENT_NGINX, *version)
	}
	pcreBuilder := makeBuilder(COMPONENT_PCRE, *pcreVersion)
	openSSLBuilder := makeBuilder(COMPONENT_OPENSSL, *openSSLVersion)
	zlibBuilder := makeBuilder(COMPONENT_ZLIB, *zlibVersion)

	// change default umask
	_ = syscall.Umask(0)

	versionCheck(*version)

	nginxConfigure, err := fileGetContents(*nginxConfigurePath)
	if err != nil {
		log.Fatal(err)
	}
	nginxConfigure = normalizeConfigure(nginxConfigure)

	modules3rd, err := loadModules3rdFile(*modulesConfPath)
	if err != nil {
		log.Fatal(err)
	}

	if len(*workParentDir) == 0 {
		log.Fatal("set working directory with -d")
	}

	if !fileExists(*workParentDir) {
		err := os.Mkdir(*workParentDir, 0755)
		if err != nil {
			log.Fatalf("Failed to create working directory(%s) does not exist.", *workParentDir)
		}
	}

	var workDir string
	if *openResty {
		workDir = *workParentDir + "/openresty/" + *openRestyVersion
	} else if *tengine {
		workDir = *workParentDir + "/tengine/" + *tengineVersion
	} else {
		workDir = *workParentDir + "/nginx/" + *version
	}
	if *clear {
		err := clearWorkDir(workDir)
		if err != nil {
			log.Fatal(err)
		}
	}

	if !fileExists(workDir) {
		err := os.MkdirAll(workDir, 0755)
		if err != nil {
			log.Fatalf("Failed to create working directory(%s) does not exist.", workDir)
		}
	}

	rootDir := saveCurrentDir()
	// cd workDir
	err = os.Chdir(workDir)
	if err != nil {
		log.Fatal(err)
	}

	if *pcreStatic {
		wg.Add(1)
		go downloadAndExtractParallel(&pcreBuilder, wg)
	}

	if *openSSLStatic {
		wg.Add(1)
		go downloadAndExtractParallel(&openSSLBuilder, wg)
	}

	if *zlibStatic {
		wg.Add(1)
		go downloadAndExtractParallel(&zlibBuilder, wg)
	}

	wg.Add(1)
	go downloadAndExtractParallel(&nginxBuilder, wg)

	if len(modules3rd) > 0 {
		wg.Add(len(modules3rd))
		for _, m := range modules3rd {
			go downloadAndExtractModule3rdParallel(m, wg)
		}

	}

	// wait until all downloading processes by goroutine finish
	wg.Wait()

	if len(modules3rd) > 0 {
		for _, m := range modules3rd {
			provideModule3rd(&m)
		}
	}

	// cd workDir/nginx-${version}
	os.Chdir(nginxBuilder.sourcePath())

	if *pcreStatic {
		dependencies = append(dependencies, makeStaticLibrary(&pcreBuilder))
	}

	if *openSSLStatic {
		dependencies = append(dependencies, makeStaticLibrary(&openSSLBuilder))
	}

	if *zlibStatic {
		dependencies = append(dependencies, makeStaticLibrary(&zlibBuilder))
	}

	log.Printf("Generate configure script for %s.....", nginxBuilder.sourcePath())

	if *pcreStatic && pcreBuilder.isIncludeWithOption(nginxConfigure) {
		log.Printf(pcreBuilder.warnMsgWithLibrary())
	}

	if *openSSLStatic && openSSLBuilder.isIncludeWithOption(nginxConfigure) {
		log.Printf(openSSLBuilder.warnMsgWithLibrary())

	}

	if *zlibStatic && zlibBuilder.isIncludeWithOption(nginxConfigure) {
		log.Printf(zlibBuilder.warnMsgWithLibrary())
	}

	if strings.Contains(nginxConfigure, "--add-module=") {
		log.Println("[warn]Using '--add-module' is discouraged. Instead give ini-file with '-m' to 'nginx-build'")
	}

	configureScript := configureGen(nginxConfigure, modules3rd, dependencies, configureOptions, rootDir)

	err = ioutil.WriteFile("./nginx-configure", []byte(configureScript), 0655)
	if err != nil {
		log.Fatalf("Failed to generate configure script for %s", nginxBuilder.sourcePath())
	}

	log.Printf("Configure %s.....", nginxBuilder.sourcePath())

	err = configureNginx()
	if err != nil {
		log.Printf("Failed to configure %s\n", nginxBuilder.sourcePath())
		printFatalMsg(err, "nginx-configure.log")
	}

	if *configureOnly {
		printLastMsg(workDir, nginxBuilder.sourcePath(), *openResty, *configureOnly)
		return
	}

	log.Printf("Build %s.....", nginxBuilder.sourcePath())

	if *openSSLStatic {
		// Workarounds for protecting a failure of building nginx with static-linked OpenSSL.

		// Unfortunately a build of OpenSSL fails when multi-CPUs are used.
		*jobs = 1

		// Sometimes machine hardware name('uname -m') is different
		// from machine processor architecture name('uname -p') on Mac.
		// Specifically, `uname -p` is 'i386' and `uname -m` is 'x86_64'.
		// In this case, a build of OpenSSL fails.
		// So it needs to convince OpenSSL with KERNEL_BITS.
		if runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
			os.Setenv("KERNEL_BITS", "64")
		}
	}

	err = buildNginx(*jobs)
	if err != nil {
		log.Printf("Failed to build %s\n", nginxBuilder.sourcePath())
		printFatalMsg(err, "nginx-build.log")
	}

	printLastMsg(workDir, nginxBuilder.sourcePath(), *openResty, *configureOnly)

	// cd rootDir
	os.Chdir(rootDir)
}
<|endoftext|>


<commit_before>// Package gcp implements a simple interface with the Google Cloud Platform
// for storing blobs in buckets and performing other types of maintenance on GCP.
package gcp

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"

	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	storage "google.golang.org/api/storage/v1"
)

const (
	scope = storage.DevstorageFullControlScope
)

// ACLs for writing data to Cloud Store.
// Definitions according to https://github.com/google/google-api-go-client/blob/master/storage/v1/storage-gen.go:
//   "authenticatedRead" - Project team owners get OWNER access, and
//       allAuthenticatedUsers get READER access.
//   "private" - Project team owners get OWNER access.
//   "projectPrivate" - Project team members get access according to
//       their roles.
//   "publicRead" - Project team owners get OWNER access, and allUsers
//       get READER access.
//   "publicReadWrite" - Project team owners get OWNER access, and
//       allUsers get WRITER access.
type WriteACL string

const (
	PublicRead        WriteACL = "publicRead"
	AuthenticatedRead WriteACL = "authenticatedRead"
	Private           WriteACL = "private"
	ProjectPrivate    WriteACL = "projectPrivate"
	PublicReadWrite   WriteACL = "publicReadWrite"
	DefaultWriteACL   WriteACL = PublicRead
)

// Interface is how GCP clients talk to GCP.
type Interface interface {
	// PutLocalFile copies a local file to GCP using ref as its
	// name. It returns a direct link for downloading the file
	// from GCP.
	PutLocalFile(srcLocalFilename string, ref string) (refLink string, error error)

	// Get returns a link for downloading ref from GCP.
	Get(ref string) (link string, error error)

	// Put stores the contents given as ref on GCP.
	Put(ref string, contents []byte) (refLink string, error error)

	// List returns all the filenames stored inside a given path
	// prefix.  If successful, it returns two parallel slices
	// containing for each file its name and a URL-encoded link to
	// it.
	List(prefix string) (name []string, link []string, err error)

	// Delete permanently removes all storage space associated
	// with a ref.
	Delete(ref string) error

	// Connect connects with the Google Cloud Platform.
	Connect()
}

// GCP is an implementation of Interface that connects to a live GCP instance.
type GCP struct {
	client          *http.Client
	service         *storage.Service
	projectId       string
	bucketName      string
	defaultWriteACL WriteACL
}

// Guarantee we implement the interface.
var _ Interface = (*GCP)(nil)

// New creates a new GCP instance associated with the given project id and bucket name.
func New(projectId, bucketName string, defaultWriteACL WriteACL) *GCP {
	gcp := &GCP{}
	gcp.projectId = projectId
	gcp.bucketName = bucketName
	switch defaultWriteACL {
	case PublicRead, AuthenticatedRead, Private, ProjectPrivate, PublicReadWrite:
		gcp.defaultWriteACL = defaultWriteACL
	default:
		gcp.defaultWriteACL = DefaultWriteACL
	}
	gcp.Connect()
	return gcp
}

func (gcp *GCP) PutLocalFile(srcLocalFilename string, ref string) (refLink string, error error) {
	// Insert an object into a bucket.
	object := &storage.Object{Name: ref}
	file, err := os.Open(srcLocalFilename)
	if err != nil {
		log.Printf("Error opening: %v", err)
		return "", err
	}
	defer file.Close()
	acl := string(gcp.defaultWriteACL)
	res, err := gcp.service.Objects.Insert(gcp.bucketName, object).Media(file).PredefinedAcl(acl).Do()
	if err == nil {
		log.Printf("Created object %v at location %v\n", res.Name, res.SelfLink)
	} else {
		log.Printf("Objects.Insert failed: %v", err)
	}
	return res.MediaLink, err
}

func (gcp *GCP) Get(ref string) (link string, error error) {
	// Get the link of the blob
	res, err := gcp.service.Objects.Get(gcp.bucketName, ref).Do()
	if err != nil {
		return "", err
	}
	fmt.Printf("The media download link for %v/%v is %v.\n\n", gcp.bucketName, res.Name, res.MediaLink)
	return res.MediaLink, nil
}

func (gcp *GCP) Put(ref string, contents []byte) (refLink string, error error) {
	// TODO(edpin): this is not super safe, given the file has
	// permissions 0666. But for now the contents are always
	// public on cloud store, so it doesn't matter.
	f, err := ioutil.TempFile("", "upload-gcp-blob-")
	if err != nil {
		return "", err
	}
	defer f.Close()
	name := f.Name()
	defer os.Remove(name)
	n, err := f.Write(contents)
	if err != nil || n != len(contents) {
		return "", err
	}
	link, err := gcp.PutLocalFile(name, ref)

	return link, err
}

func (gcp *GCP) List(prefix string) (name []string, link []string, err error) {
	nextPageToken := ""
	for {
		moreNames, moreLinks, nextPageToken, err := gcp.innerList(prefix, nextPageToken)
		if err != nil {
			return nil, nil, err
		}
		name = append(name, moreNames...)
		link = append(link, moreLinks...)
		if nextPageToken == "" {
			break
		}
	}
	return
}

// innerList is an internal function that does what List does, except
// it accepts a continuation token and possibly returns one if there
// are more objects to retrieve.
func (gcp *GCP) innerList(prefix, pageToken string) (name []string, link []string, nextPageToken string, error error) {
	objs, err := gcp.service.Objects.List(gcp.bucketName).Prefix(prefix).Fields("items(name,mediaLink),nextPageToken").PageToken(pageToken).Do()
	if err != nil {
		return nil, nil, "", err
	}
	// objs.Items is a slice of Objects.

	// Allocate space for all returned objects in this call.
	name = make([]string, len(objs.Items))
	link = make([]string, len(objs.Items))

	for i, o := range objs.Items {
		name[i] = o.Name
		link[i] = o.MediaLink
	}

	return name, link, objs.NextPageToken, nil
}

func (gcp *GCP) Delete(ref string) error {
	err := gcp.service.Objects.Delete(gcp.bucketName, ref).Do()
	if err != nil {
		return err
	}
	return nil
}

func (gcp *GCP) Connect() {
	if gcp.projectId == "" {
		log.Fatalf("Project argument is required.")
	}
	if gcp.bucketName == "" {
		log.Fatalf("Bucket argument is required.")
	}

	// Authentication is provided by the gcloud tool when running locally, and
	// by the associated service account when running on Compute Engine.
	client, err := google.DefaultClient(context.Background(), scope)
	if err != nil {
		log.Fatalf("Unable to get default client: %v", err)
	}
	service, err := storage.New(client)
	if err != nil {
		log.Fatalf("Unable to create storage service: %v", err)
	}
	// Initialize the object
	gcp.client = client
	gcp.service = service
}
<commit_msg>cloud/gcp: fixed bug in error reporting<commit_after>// Package gcp implements a simple interface with the Google Cloud Platform
// for storing blobs in buckets and performing other types of maintenance on GCP.
package gcp

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"

	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	storage "google.golang.org/api/storage/v1"
)

const (
	scope = storage.DevstorageFullControlScope
)

// ACLs for writing data to Cloud Store.
// Definitions according to https://github.com/google/google-api-go-client/blob/master/storage/v1/storage-gen.go:
//   "authenticatedRead" - Project team owners get OWNER access, and
//       allAuthenticatedUsers get READER access.
//   "private" - Project team owners get OWNER access.
//   "projectPrivate" - Project team members get access according to
//       their roles.
//   "publicRead" - Project team owners get OWNER access, and allUsers
//       get READER access.
//   "publicReadWrite" - Project team owners get OWNER access, and
//       allUsers get WRITER access.
type WriteACL string

const (
	PublicRead        WriteACL = "publicRead"
	AuthenticatedRead WriteACL = "authenticatedRead"
	Private           WriteACL = "private"
	ProjectPrivate    WriteACL = "projectPrivate"
	PublicReadWrite   WriteACL = "publicReadWrite"
	DefaultWriteACL   WriteACL = PublicRead
)

// Interface is how GCP clients talk to GCP.
type Interface interface {
	// PutLocalFile copies a local file to GCP using ref as its
	// name. It returns a direct link for downloading the file
	// from GCP.
	PutLocalFile(srcLocalFilename string, ref string) (refLink string, error error)

	// Get returns a link for downloading ref from GCP.
	Get(ref string) (link string, error error)

	// Put stores the contents given as ref on GCP.
	Put(ref string, contents []byte) (refLink string, error error)

	// List returns all the filenames stored inside a given path
	// prefix.  If successful, it returns two parallel slices
	// containing for each file its name and a URL-encoded link to
	// it.
	List(prefix string) (name []string, link []string, err error)

	// Delete permanently removes all storage space associated
	// with a ref.
	Delete(ref string) error

	// Connect connects with the Google Cloud Platform.
	Connect()
}

// GCP is an implementation of Interface that connects to a live GCP instance.
type GCP struct {
	client          *http.Client
	service         *storage.Service
	projectId       string
	bucketName      string
	defaultWriteACL WriteACL
}

// Guarantee we implement the interface.
var _ Interface = (*GCP)(nil)

// New creates a new GCP instance associated with the given project id and bucket name.
func New(projectId, bucketName string, defaultWriteACL WriteACL) *GCP {
	gcp := &GCP{}
	gcp.projectId = projectId
	gcp.bucketName = bucketName
	switch defaultWriteACL {
	case PublicRead, AuthenticatedRead, Private, ProjectPrivate, PublicReadWrite:
		gcp.defaultWriteACL = defaultWriteACL
	default:
		gcp.defaultWriteACL = DefaultWriteACL
	}
	gcp.Connect()
	return gcp
}

func (gcp *GCP) PutLocalFile(srcLocalFilename string, ref string) (refLink string, error error) {
	// Insert an object into a bucket.
	object := &storage.Object{Name: ref}
	file, err := os.Open(srcLocalFilename)
	if err != nil {
		log.Printf("Error opening: %v", err)
		return "", err
	}
	defer file.Close()
	acl := string(gcp.defaultWriteACL)
	res, err := gcp.service.Objects.Insert(gcp.bucketName, object).Media(file).PredefinedAcl(acl).Do()
	if err == nil {
		log.Printf("Created object %v at location %v\n", res.Name, res.SelfLink)
	} else {
		log.Printf("Objects.Insert failed: %v", err)
		return "", err
	}
	return res.MediaLink, err
}

func (gcp *GCP) Get(ref string) (link string, error error) {
	// Get the link of the blob
	res, err := gcp.service.Objects.Get(gcp.bucketName, ref).Do()
	if err != nil {
		return "", err
	}
	fmt.Printf("The media download link for %v/%v is %v.\n\n", gcp.bucketName, res.Name, res.MediaLink)
	return res.MediaLink, nil
}

func (gcp *GCP) Put(ref string, contents []byte) (refLink string, error error) {
	// TODO(edpin): this is not super safe, given the file has
	// permissions 0666. But for now the contents are always
	// public on cloud store, so it doesn't matter.
	f, err := ioutil.TempFile("", "upload-gcp-blob-")
	if err != nil {
		return "", err
	}
	defer f.Close()
	name := f.Name()
	defer os.Remove(name)
	n, err := f.Write(contents)
	if err != nil || n != len(contents) {
		return "", err
	}
	link, err := gcp.PutLocalFile(name, ref)

	return link, err
}

func (gcp *GCP) List(prefix string) (name []string, link []string, err error) {
	nextPageToken := ""
	for {
		moreNames, moreLinks, nextPageToken, err := gcp.innerList(prefix, nextPageToken)
		if err != nil {
			return nil, nil, err
		}
		name = append(name, moreNames...)
		link = append(link, moreLinks...)
		if nextPageToken == "" {
			break
		}
	}
	return
}

// innerList is an internal function that does what List does, except
// it accepts a continuation token and possibly returns one if there
// are more objects to retrieve.
func (gcp *GCP) innerList(prefix, pageToken string) (name []string, link []string, nextPageToken string, error error) {
	objs, err := gcp.service.Objects.List(gcp.bucketName).Prefix(prefix).Fields("items(name,mediaLink),nextPageToken").PageToken(pageToken).Do()
	if err != nil {
		return nil, nil, "", err
	}
	// objs.Items is a slice of Objects.

	// Allocate space for all returned objects in this call.
	name = make([]string, len(objs.Items))
	link = make([]string, len(objs.Items))

	for i, o := range objs.Items {
		name[i] = o.Name
		link[i] = o.MediaLink
	}

	return name, link, objs.NextPageToken, nil
}

func (gcp *GCP) Delete(ref string) error {
	err := gcp.service.Objects.Delete(gcp.bucketName, ref).Do()
	if err != nil {
		return err
	}
	return nil
}

func (gcp *GCP) Connect() {
	if gcp.projectId == "" {
		log.Fatalf("Project argument is required.")
	}
	if gcp.bucketName == "" {
		log.Fatalf("Bucket argument is required.")
	}

	// Authentication is provided by the gcloud tool when running locally, and
	// by the associated service account when running on Compute Engine.
	client, err := google.DefaultClient(context.Background(), scope)
	if err != nil {
		log.Fatalf("Unable to get default client: %v", err)
	}
	service, err := storage.New(client)
	if err != nil {
		log.Fatalf("Unable to create storage service: %v", err)
	}
	// Initialize the object
	gcp.client = client
	gcp.service = service
}
<|endoftext|>


<commit_before>package radosAPI

// Usage represents the response of usage requests
type Usage struct {
	Entries []struct {
		Buckets []struct {
			Bucket     string `json:"bucket"`
			Categories []struct {
				BytesReceived int    `json:"bytes_received"`
				BytesSent     int    `json:"bytes_sent"`
				Category      string `json:"category"`
				Ops           int    `json:"ops"`
				SuccessfulOps int    `json:"successful_ops"`
			} `json:"categories"`
			Epoch int    `json:"epoch"`
			Time  string `json:"time"`
		} `json:"buckets"`
		Owner string `json:"owner"`
	} `json:"entries"`
	Summary []struct {
		Categories []struct {
			BytesReceived int    `json:"bytes_received"`
			BytesSent     int    `json:"bytes_sent"`
			Category      string `json:"category"`
			Ops           int    `json:"ops"`
			SuccessfulOps int    `json:"successful_ops"`
		} `json:"categories"`
		Total struct {
			BytesReceived int `json:"bytes_received"`
			BytesSent     int `json:"bytes_sent"`
			Ops           int `json:"ops"`
			SuccessfulOps int `json:"successful_ops"`
		} `json:"total"`
		User string `json:"user"`
	} `json:"summary"`
}
<commit_msg>Add apiError struct to handle errors<commit_after>package radosAPI

type apiError struct {
	Code string `json:"Code"`
}

// Usage represents the response of usage requests
type Usage struct {
	Entries []struct {
		Buckets []struct {
			Bucket     string `json:"bucket"`
			Categories []struct {
				BytesReceived int    `json:"bytes_received"`
				BytesSent     int    `json:"bytes_sent"`
				Category      string `json:"category"`
				Ops           int    `json:"ops"`
				SuccessfulOps int    `json:"successful_ops"`
			} `json:"categories"`
			Epoch int    `json:"epoch"`
			Time  string `json:"time"`
		} `json:"buckets"`
		Owner string `json:"owner"`
	} `json:"entries"`
	Summary []struct {
		Categories []struct {
			BytesReceived int    `json:"bytes_received"`
			BytesSent     int    `json:"bytes_sent"`
			Category      string `json:"category"`
			Ops           int    `json:"ops"`
			SuccessfulOps int    `json:"successful_ops"`
		} `json:"categories"`
		Total struct {
			BytesReceived int `json:"bytes_received"`
			BytesSent     int `json:"bytes_sent"`
			Ops           int `json:"ops"`
			SuccessfulOps int `json:"successful_ops"`
		} `json:"total"`
		User string `json:"user"`
	} `json:"summary"`
}
<|endoftext|>


<commit_before>// Copyright 2019 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package build

import (
	"fmt"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/osutil"
)

type freebsd struct{}

func (ctx freebsd) build(params *Params) error {
	confDir := fmt.Sprintf("%v/sys/%v/conf/", params.KernelDir, params.TargetArch)
	confFile := "SYZKALLER"

	config := params.Config
	if config == nil {
		config = []byte(`
include "./GENERIC"

ident		SYZKALLER
options 	COVERAGE
options 	KCOV

options 	KERN_TLS
options 	TCPHPTS
options 	RATELIMIT

options 	DEBUG_VFS_LOCKS
options 	DIAGNOSTIC
`)
	}
	if err := osutil.WriteFile(filepath.Join(confDir, confFile), config); err != nil {
		return err
	}

	objPrefix := filepath.Join(params.KernelDir, "obj")
	if err := ctx.make(params.KernelDir, objPrefix, "kernel-toolchain", "-DNO_CLEAN"); err != nil {
		return err
	}
	if err := ctx.make(params.KernelDir, objPrefix, "buildkernel", "WITH_EXTRA_TCP_STACKS=",
		fmt.Sprintf("KERNCONF=%v", confFile)); err != nil {
		return err
	}

	kernelObjDir := filepath.Join(objPrefix, params.KernelDir,
		fmt.Sprintf("%v.%v", params.TargetArch, params.TargetArch), "sys", confFile)
	for _, s := range []struct{ dir, src, dst string }{
		{params.UserspaceDir, "image", "image"},
		{params.UserspaceDir, "key", "key"},
		{kernelObjDir, "kernel.full", "obj/kernel.full"},
	} {
		fullSrc := filepath.Join(s.dir, s.src)
		fullDst := filepath.Join(params.OutputDir, s.dst)
		if err := osutil.CopyFile(fullSrc, fullDst); err != nil {
			return fmt.Errorf("failed to copy %v -> %v: %v", fullSrc, fullDst, err)
		}
	}

	script := fmt.Sprintf(`
set -eux
md=$(sudo mdconfig -a -t vnode image)
partn=$(gpart show /dev/${md} | awk '/freebsd-ufs/{print $3}' | head -n 1)
tmpdir=$(mktemp -d)
sudo mount /dev/${md}p${partn} $tmpdir

sudo MAKEOBJDIRPREFIX=%s make -C %s installkernel WITH_EXTRA_TCP_STACKS= KERNCONF=%s DESTDIR=$tmpdir

cat | sudo tee ${tmpdir}/boot/loader.conf.local <<__EOF__
ipsec_load="YES"
pf_load="YES"
sctp_load="YES"
tcp_bbr_load="YES"
tcp_rack_load="YES"
sem_load="YES"
mqueuefs_load="YES"
cryptodev_load="YES"
cc_cdg_load="YES"
cc_chd_load="YES"
cc_cubic_load="YES"
cc_dctcp_load="YES"
cc_hd_load="YES"
cc_htcp_load="YES"
cc_vegas_load="YES"

kern.ipc.tls.enable="1"
__EOF__

cat | sudo tee -a ${tmpdir}/etc/sysctl.conf <<__EOF__
net.inet.sctp.udp_tunneling_port=9899
net.inet.tcp.udp_tunneling_port=9811
__EOF__

sudo umount $tmpdir
sudo mdconfig -d -u ${md#md}
`, objPrefix, params.KernelDir, confFile)

	if debugOut, err := osutil.RunCmd(10*time.Minute, params.OutputDir, "/bin/sh", "-c", script); err != nil {
		return fmt.Errorf("error copying kernel: %v\n%v", err, debugOut)
	}
	return nil
}

func (ctx freebsd) clean(kernelDir, targetArch string) error {
	objPrefix := filepath.Join(kernelDir, "obj")
	return ctx.make(kernelDir, objPrefix, "cleanworld")
}

func (ctx freebsd) make(kernelDir, objPrefix string, makeArgs ...string) error {
	args := append([]string{
		fmt.Sprintf("MAKEOBJDIRPREFIX=%v", objPrefix),
		"make",
		"-C", kernelDir,
		"-j", strconv.Itoa(runtime.NumCPU()),
	}, makeArgs...)
	_, err := osutil.RunCmd(3*time.Hour, kernelDir, "sh", "-c", strings.Join(args, " "))
	return err
}
<commit_msg>pkg/build: enable RedZone on FreeBSD (#2584)<commit_after>// Copyright 2019 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package build

import (
	"fmt"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/osutil"
)

type freebsd struct{}

func (ctx freebsd) build(params *Params) error {
	confDir := fmt.Sprintf("%v/sys/%v/conf/", params.KernelDir, params.TargetArch)
	confFile := "SYZKALLER"

	config := params.Config
	if config == nil {
		config = []byte(`
include "./GENERIC"

ident		SYZKALLER
options 	COVERAGE
options 	KCOV

options 	KERN_TLS
options 	TCPHPTS
options 	RATELIMIT

options 	DEBUG_VFS_LOCKS
options 	DIAGNOSTIC
options 	DEBUG_REDZONE
`)
	}
	if err := osutil.WriteFile(filepath.Join(confDir, confFile), config); err != nil {
		return err
	}

	objPrefix := filepath.Join(params.KernelDir, "obj")
	if err := ctx.make(params.KernelDir, objPrefix, "kernel-toolchain", "-DNO_CLEAN"); err != nil {
		return err
	}
	if err := ctx.make(params.KernelDir, objPrefix, "buildkernel", "WITH_EXTRA_TCP_STACKS=",
		fmt.Sprintf("KERNCONF=%v", confFile)); err != nil {
		return err
	}

	kernelObjDir := filepath.Join(objPrefix, params.KernelDir,
		fmt.Sprintf("%v.%v", params.TargetArch, params.TargetArch), "sys", confFile)
	for _, s := range []struct{ dir, src, dst string }{
		{params.UserspaceDir, "image", "image"},
		{params.UserspaceDir, "key", "key"},
		{kernelObjDir, "kernel.full", "obj/kernel.full"},
	} {
		fullSrc := filepath.Join(s.dir, s.src)
		fullDst := filepath.Join(params.OutputDir, s.dst)
		if err := osutil.CopyFile(fullSrc, fullDst); err != nil {
			return fmt.Errorf("failed to copy %v -> %v: %v", fullSrc, fullDst, err)
		}
	}

	script := fmt.Sprintf(`
set -eux
md=$(sudo mdconfig -a -t vnode image)
partn=$(gpart show /dev/${md} | awk '/freebsd-ufs/{print $3}' | head -n 1)
tmpdir=$(mktemp -d)
sudo mount /dev/${md}p${partn} $tmpdir

sudo MAKEOBJDIRPREFIX=%s make -C %s installkernel WITH_EXTRA_TCP_STACKS= KERNCONF=%s DESTDIR=$tmpdir

cat | sudo tee ${tmpdir}/boot/loader.conf.local <<__EOF__
ipsec_load="YES"
pf_load="YES"
sctp_load="YES"
tcp_bbr_load="YES"
tcp_rack_load="YES"
sem_load="YES"
mqueuefs_load="YES"
cryptodev_load="YES"
cc_cdg_load="YES"
cc_chd_load="YES"
cc_cubic_load="YES"
cc_dctcp_load="YES"
cc_hd_load="YES"
cc_htcp_load="YES"
cc_vegas_load="YES"

kern.ipc.tls.enable="1"
__EOF__

cat | sudo tee -a ${tmpdir}/etc/sysctl.conf <<__EOF__
net.inet.sctp.udp_tunneling_port=9899
net.inet.tcp.udp_tunneling_port=9811
vm.redzone.panic=1
__EOF__

sudo umount $tmpdir
sudo mdconfig -d -u ${md#md}
`, objPrefix, params.KernelDir, confFile)

	if debugOut, err := osutil.RunCmd(10*time.Minute, params.OutputDir, "/bin/sh", "-c", script); err != nil {
		return fmt.Errorf("error copying kernel: %v\n%v", err, debugOut)
	}
	return nil
}

func (ctx freebsd) clean(kernelDir, targetArch string) error {
	objPrefix := filepath.Join(kernelDir, "obj")
	return ctx.make(kernelDir, objPrefix, "cleanworld")
}

func (ctx freebsd) make(kernelDir, objPrefix string, makeArgs ...string) error {
	args := append([]string{
		fmt.Sprintf("MAKEOBJDIRPREFIX=%v", objPrefix),
		"make",
		"-C", kernelDir,
		"-j", strconv.Itoa(runtime.NumCPU()),
	}, makeArgs...)
	_, err := osutil.RunCmd(3*time.Hour, kernelDir, "sh", "-c", strings.Join(args, " "))
	return err
}
<|endoftext|>


<commit_before>// Copyright 2018 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package build

import (
	"fmt"
	"path/filepath"
	"time"

	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/sys/targets"
)

type fuchsia struct{}

func (fu fuchsia) build(targetArch, vmType, kernelDir, outputDir, compiler, userspaceDir,
	cmdlineFile, sysctlFile string, config []byte) error {
	sysTarget := targets.Get("fuchsia", targetArch)
	if sysTarget == nil {
		return fmt.Errorf("unsupported fuchsia arch %v", targetArch)
	}
	arch := sysTarget.KernelHeaderArch
	if _, err := osutil.RunCmd(10*time.Minute, kernelDir, "scripts/fx", "set", arch,
		"--packages", "garnet/packages/products/sshd"); err != nil {
		return err
	}
	if _, err := osutil.RunCmd(time.Hour, kernelDir, "scripts/fx", "full-build"); err != nil {
		return err
	}
	for src, dst := range map[string]string{
		"out/" + arch + "/images/fvm.blk":                "image",
		"out/" + arch + "/ssh-keys/id_ed25519":           "key",
		"out/build-zircon/build-" + arch + "/zircon.elf": "obj/zircon.elf",
		"out/build-zircon/build-" + arch + "/zircon.bin": "kernel",
		"out/" + arch + "/bootdata-blob.bin":             "initrd",
	} {
		fullSrc := filepath.Join(kernelDir, filepath.FromSlash(src))
		fullDst := filepath.Join(outputDir, filepath.FromSlash(dst))
		if err := osutil.CopyFile(fullSrc, fullDst); err != nil {
			return fmt.Errorf("faied to copy %v: %v", src, err)
		}
	}
	return nil
}

func (fu fuchsia) clean(kernelDir string) error {
	// Let's assume that fx always properly handles build without cleaning (until proven otherwise).
	return nil
}
<commit_msg>pkg/build: always do clean build for fuchsia<commit_after>// Copyright 2018 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package build

import (
	"fmt"
	"path/filepath"
	"time"

	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/sys/targets"
)

type fuchsia struct{}

func (fu fuchsia) build(targetArch, vmType, kernelDir, outputDir, compiler, userspaceDir,
	cmdlineFile, sysctlFile string, config []byte) error {
	sysTarget := targets.Get("fuchsia", targetArch)
	if sysTarget == nil {
		return fmt.Errorf("unsupported fuchsia arch %v", targetArch)
	}
	arch := sysTarget.KernelHeaderArch
	if _, err := osutil.RunCmd(time.Hour, kernelDir, "scripts/fx", "clean-build", arch,
		"--packages", "garnet/packages/products/sshd"); err != nil {
		return err
	}
	for src, dst := range map[string]string{
		"out/" + arch + "/images/fvm.blk":                "image",
		"out/" + arch + "/ssh-keys/id_ed25519":           "key",
		"out/build-zircon/build-" + arch + "/zircon.elf": "obj/zircon.elf",
		"out/build-zircon/build-" + arch + "/zircon.bin": "kernel",
		"out/" + arch + "/bootdata-blob.bin":             "initrd",
	} {
		fullSrc := filepath.Join(kernelDir, filepath.FromSlash(src))
		fullDst := filepath.Join(outputDir, filepath.FromSlash(dst))
		if err := osutil.CopyFile(fullSrc, fullDst); err != nil {
			return fmt.Errorf("faied to copy %v: %v", src, err)
		}
	}
	return nil
}

func (fu fuchsia) clean(kernelDir string) error {
	// We always do clean build because incremental build is frequently broken.
	// So no need to clean separately.
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strings"
	"text/tabwriter"
	"time"

	clientapi "github.com/cilium/cilium/api/v1/client"
	"github.com/cilium/cilium/api/v1/models"
	"github.com/cilium/cilium/pkg/defaults"
	"github.com/cilium/cilium/pkg/ip"

	runtime_client "github.com/go-openapi/runtime/client"
	"github.com/go-openapi/strfmt"
)

type Client struct {
	clientapi.Cilium
}

// DefaultSockPath returns deafult UNIX domain socket path or
// path set using CILIUM_SOCK env variable
func DefaultSockPath() string {
	// Check if environment variable points to socket
	e := os.Getenv(defaults.SockPathEnv)
	if e == "" {
		// If unset, fall back to default value
		e = defaults.SockPath
	}
	return "unix://" + e

}

func configureTransport(tr *http.Transport, proto, addr string) *http.Transport {
	if tr == nil {
		tr = &http.Transport{}
	}

	if proto == "unix" {
		// No need for compression in local communications.
		tr.DisableCompression = true
		tr.Dial = func(_, _ string) (net.Conn, error) {
			return net.Dial(proto, addr)
		}
	} else {
		tr.Proxy = http.ProxyFromEnvironment
		tr.Dial = (&net.Dialer{}).Dial
	}

	return tr
}

// NewDefaultClient creates a client with default parameters connecting to UNIX domain socket.
func NewDefaultClient() (*Client, error) {
	return NewClient("")
}

// NewClient creates a client for the given `host`.
// If host is nil then use SockPath provided by CILIUM_SOCK
// or the cilium default SockPath
func NewClient(host string) (*Client, error) {
	if host == "" {
		host = DefaultSockPath()
	}
	tmp := strings.SplitN(host, "://", 2)
	if len(tmp) != 2 {
		return nil, fmt.Errorf("invalid host format '%s'", host)
	}

	switch tmp[0] {
	case "tcp":
		if _, err := url.Parse("tcp://" + tmp[1]); err != nil {
			return nil, err
		}
		host = "http://" + tmp[1]
	case "unix":
		host = tmp[1]
	}

	transport := configureTransport(nil, tmp[0], host)
	httpClient := &http.Client{Transport: transport}
	clientTrans := runtime_client.NewWithClient(tmp[1], clientapi.DefaultBasePath,
		clientapi.DefaultSchemes, httpClient)
	return &Client{*clientapi.New(clientTrans, strfmt.Default)}, nil
}

// Hint tries to improve the error message displayed to the user.
func Hint(err error) error {
	if err == nil {
		return err
	}
	e, _ := url.PathUnescape(err.Error())
	if strings.Contains(err.Error(), defaults.SockPath) {
		return fmt.Errorf("%s\nIs the agent running?", e)
	}
	return fmt.Errorf("%s", e)
}

func timeSince(since time.Time) string {
	out := "never"
	if !since.IsZero() {
		// Poor man's implementtion of time.Truncate(). Can be refined
		// when we rebase to go 1.9
		t := time.Since(since)
		t -= t % time.Second
		out = t.String() + " ago"
	}

	return out
}

func stateUnhealthy(state string) bool {
	return state == models.StatusStateWarning ||
		state == models.StatusStateFailure
}

func statusUnhealthy(s *models.Status) bool {
	if s != nil {
		return stateUnhealthy(s.State)
	}
	return false
}

// FormatStatusResponseBrief writes a one-line status to the writer. If
// everything ok, this is "ok", otherwise a message of the form "error in ..."
func FormatStatusResponseBrief(w io.Writer, sr *models.StatusResponse) {
	msg := ""

	switch {
	case statusUnhealthy(sr.Cilium):
		msg = fmt.Sprintf("cilium: %s", sr.Cilium.Msg)
	case statusUnhealthy(sr.ContainerRuntime):
		msg = fmt.Sprintf("container runtime: %s", sr.ContainerRuntime.Msg)
	case statusUnhealthy(sr.Kvstore):
		msg = fmt.Sprintf("kvstore: %s", sr.Kvstore.Msg)
	case sr.Kubernetes != nil && stateUnhealthy(sr.Kubernetes.State):
		msg = fmt.Sprintf("kubernetes: %s", sr.Kubernetes.Msg)
	case sr.Cluster != nil && statusUnhealthy(sr.Cluster.CiliumHealth):
		msg = fmt.Sprintf("cilium-health: %s", sr.Cluster.CiliumHealth.Msg)
	}

	// Only bother looking at controller failures if everything else is ok
	if msg == "" {
		for _, ctrl := range sr.Controllers {
			if ctrl.Status == nil {
				continue
			}
			if ctrl.Status.LastFailureMsg != "" {
				msg = fmt.Sprintf("controller %s: %s",
					ctrl.Name, ctrl.Status.LastFailureMsg)
				break
			}
		}
	}

	if msg == "" {
		fmt.Fprintf(w, "OK\n")
	} else {
		fmt.Fprintf(w, "error in %s\n", msg)
	}
}

// FormatStatusResponse writes a StatusResponse as a string to the writer.
//
// The parameters 'allAddresses', 'allControllers', 'allNodes', respectively,
// cause all details about that aspect of the status to be printed to the
// terminal. For each of these, if they are false then only a summary will be
// printed, with perhaps some detail if there are errors.
func FormatStatusResponse(w io.Writer, sr *models.StatusResponse, allAddresses, allControllers, allNodes, allRedirects bool) {
	if sr.Kvstore != nil {
		fmt.Fprintf(w, "KVStore:\t%s\t%s\n", sr.Kvstore.State, sr.Kvstore.Msg)
	}
	if sr.ContainerRuntime != nil {
		fmt.Fprintf(w, "ContainerRuntime:\t%s\t%s\n",
			sr.ContainerRuntime.State, sr.ContainerRuntime.Msg)
	}
	if sr.Kubernetes != nil {
		fmt.Fprintf(w, "Kubernetes:\t%s\t%s\n", sr.Kubernetes.State, sr.Kubernetes.Msg)
		if sr.Kubernetes.State != models.K8sStatusStateDisabled {
			fmt.Fprintf(w, "Kubernetes APIs:\t[\"%s\"]\n", strings.Join(sr.Kubernetes.K8sAPIVersions, "\", \""))
		}
	}
	if sr.Cilium != nil {
		fmt.Fprintf(w, "Cilium:\t%s\t%s\n", sr.Cilium.State, sr.Cilium.Msg)
	}

	if sr.Stale != nil {
		sortedProbes := make([]string, 0, len(sr.Stale))
		for probe := range sr.Stale {
			sortedProbes = append(sortedProbes, probe)
		}
		sort.Strings(sortedProbes)

		stalesStr := make([]string, 0, len(sr.Stale))
		for _, probe := range sortedProbes {
			stalesStr = append(stalesStr, fmt.Sprintf("%q since %s", probe, sr.Stale[probe]))
		}

		fmt.Fprintf(w, "Stale status:\t%s\n", strings.Join(stalesStr, ", "))
	}

	if nm := sr.NodeMonitor; nm != nil {
		fmt.Fprintf(w, "NodeMonitor:\tListening for events on %d CPUs with %dx%d of shared memory\n",
			nm.Cpus, nm.Npages, nm.Pagesize)
		if nm.Lost != 0 || nm.Unknown != 0 {
			fmt.Fprintf(w, "\t%d events lost, %d unknown notifications\n", nm.Lost, nm.Unknown)
		}
	} else {
		fmt.Fprintf(w, "NodeMonitor:\tDisabled\n")
	}

	var localNode *models.NodeElement
	if sr.Cluster != nil {
		if sr.Cluster.CiliumHealth != nil {
			ch := sr.Cluster.CiliumHealth
			fmt.Fprintf(w, "Cilium health daemon:\t%s\t%s\n", ch.State, ch.Msg)
		}
		for _, node := range sr.Cluster.Nodes {
			if node.Name == sr.Cluster.Self {
				localNode = node
			} else {
				continue
			}
		}
	}

	if sr.IPAM != nil {
		var v4CIDR, v6CIDR string
		if localNode != nil {
			if nIPs := ip.CountIPsInCIDR(localNode.PrimaryAddress.IPV4.AllocRange); nIPs > 0 {
				v4CIDR = fmt.Sprintf("/%d", nIPs)
			}
			if nIPs := ip.CountIPsInCIDR(localNode.PrimaryAddress.IPV6.AllocRange); nIPs > 0 {
				v6CIDR = fmt.Sprintf("/%d", nIPs)
			}
		}
		fmt.Fprintf(w, "IPv4 address pool:\t%d%s allocated\n", len(sr.IPAM.IPV4), v4CIDR)
		if allAddresses {
			for _, ipv4 := range sr.IPAM.IPV4 {
				fmt.Fprintf(w, "  %s\n", ipv4)
			}
		}
		fmt.Fprintf(w, "IPv6 address pool:\t%d%s allocated\n", len(sr.IPAM.IPV6), v6CIDR)
		if allAddresses {
			for _, ipv6 := range sr.IPAM.IPV6 {
				fmt.Fprintf(w, "  %s\n", ipv6)
			}
		}
	}

	if sr.Controllers != nil {
		nFailing, out := 0, []string{"  Name\tLast success\tLast error\tCount\tMessage\n"}
		for _, ctrl := range sr.Controllers {
			status := ctrl.Status
			if status == nil {
				continue
			}

			if status.ConsecutiveFailureCount > 0 {
				nFailing++
			} else if !allControllers {
				continue
			}

			failSince := timeSince(time.Time(status.LastFailureTimestamp))
			successSince := timeSince(time.Time(status.LastSuccessTimestamp))

			err := "no error"
			if status.LastFailureMsg != "" {
				err = status.LastFailureMsg
			}

			out = append(out, fmt.Sprintf("  %s\t%s\t%s\t%d\t%s\t\n",
				ctrl.Name, successSince, failSince, status.ConsecutiveFailureCount, err))
		}

		nOK := len(sr.Controllers) - nFailing
		fmt.Fprintf(w, "Controller Status:\t%d/%d healthy\n", nOK, len(sr.Controllers))
		if len(out) > 1 {
			tab := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0)
			sort.Strings(out)
			for _, s := range out {
				fmt.Fprint(tab, s)
			}
			tab.Flush()
		}

	}

	if sr.Proxy != nil {
		fmt.Fprintf(w, "Proxy Status:\tOK, ip %s, port-range %s\n",
			sr.Proxy.IP, sr.Proxy.PortRange)
	} else {
		fmt.Fprintf(w, "Proxy Status:\tNo managed proxy redirect\n")
	}
}
<commit_msg>pkg/client: sort kubernetes API versions for CLI output<commit_after>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strings"
	"text/tabwriter"
	"time"

	clientapi "github.com/cilium/cilium/api/v1/client"
	"github.com/cilium/cilium/api/v1/models"
	"github.com/cilium/cilium/pkg/defaults"
	"github.com/cilium/cilium/pkg/ip"

	runtime_client "github.com/go-openapi/runtime/client"
	"github.com/go-openapi/strfmt"
)

type Client struct {
	clientapi.Cilium
}

// DefaultSockPath returns deafult UNIX domain socket path or
// path set using CILIUM_SOCK env variable
func DefaultSockPath() string {
	// Check if environment variable points to socket
	e := os.Getenv(defaults.SockPathEnv)
	if e == "" {
		// If unset, fall back to default value
		e = defaults.SockPath
	}
	return "unix://" + e

}

func configureTransport(tr *http.Transport, proto, addr string) *http.Transport {
	if tr == nil {
		tr = &http.Transport{}
	}

	if proto == "unix" {
		// No need for compression in local communications.
		tr.DisableCompression = true
		tr.Dial = func(_, _ string) (net.Conn, error) {
			return net.Dial(proto, addr)
		}
	} else {
		tr.Proxy = http.ProxyFromEnvironment
		tr.Dial = (&net.Dialer{}).Dial
	}

	return tr
}

// NewDefaultClient creates a client with default parameters connecting to UNIX domain socket.
func NewDefaultClient() (*Client, error) {
	return NewClient("")
}

// NewClient creates a client for the given `host`.
// If host is nil then use SockPath provided by CILIUM_SOCK
// or the cilium default SockPath
func NewClient(host string) (*Client, error) {
	if host == "" {
		host = DefaultSockPath()
	}
	tmp := strings.SplitN(host, "://", 2)
	if len(tmp) != 2 {
		return nil, fmt.Errorf("invalid host format '%s'", host)
	}

	switch tmp[0] {
	case "tcp":
		if _, err := url.Parse("tcp://" + tmp[1]); err != nil {
			return nil, err
		}
		host = "http://" + tmp[1]
	case "unix":
		host = tmp[1]
	}

	transport := configureTransport(nil, tmp[0], host)
	httpClient := &http.Client{Transport: transport}
	clientTrans := runtime_client.NewWithClient(tmp[1], clientapi.DefaultBasePath,
		clientapi.DefaultSchemes, httpClient)
	return &Client{*clientapi.New(clientTrans, strfmt.Default)}, nil
}

// Hint tries to improve the error message displayed to the user.
func Hint(err error) error {
	if err == nil {
		return err
	}
	e, _ := url.PathUnescape(err.Error())
	if strings.Contains(err.Error(), defaults.SockPath) {
		return fmt.Errorf("%s\nIs the agent running?", e)
	}
	return fmt.Errorf("%s", e)
}

func timeSince(since time.Time) string {
	out := "never"
	if !since.IsZero() {
		// Poor man's implementtion of time.Truncate(). Can be refined
		// when we rebase to go 1.9
		t := time.Since(since)
		t -= t % time.Second
		out = t.String() + " ago"
	}

	return out
}

func stateUnhealthy(state string) bool {
	return state == models.StatusStateWarning ||
		state == models.StatusStateFailure
}

func statusUnhealthy(s *models.Status) bool {
	if s != nil {
		return stateUnhealthy(s.State)
	}
	return false
}

// FormatStatusResponseBrief writes a one-line status to the writer. If
// everything ok, this is "ok", otherwise a message of the form "error in ..."
func FormatStatusResponseBrief(w io.Writer, sr *models.StatusResponse) {
	msg := ""

	switch {
	case statusUnhealthy(sr.Cilium):
		msg = fmt.Sprintf("cilium: %s", sr.Cilium.Msg)
	case statusUnhealthy(sr.ContainerRuntime):
		msg = fmt.Sprintf("container runtime: %s", sr.ContainerRuntime.Msg)
	case statusUnhealthy(sr.Kvstore):
		msg = fmt.Sprintf("kvstore: %s", sr.Kvstore.Msg)
	case sr.Kubernetes != nil && stateUnhealthy(sr.Kubernetes.State):
		msg = fmt.Sprintf("kubernetes: %s", sr.Kubernetes.Msg)
	case sr.Cluster != nil && statusUnhealthy(sr.Cluster.CiliumHealth):
		msg = fmt.Sprintf("cilium-health: %s", sr.Cluster.CiliumHealth.Msg)
	}

	// Only bother looking at controller failures if everything else is ok
	if msg == "" {
		for _, ctrl := range sr.Controllers {
			if ctrl.Status == nil {
				continue
			}
			if ctrl.Status.LastFailureMsg != "" {
				msg = fmt.Sprintf("controller %s: %s",
					ctrl.Name, ctrl.Status.LastFailureMsg)
				break
			}
		}
	}

	if msg == "" {
		fmt.Fprintf(w, "OK\n")
	} else {
		fmt.Fprintf(w, "error in %s\n", msg)
	}
}

// FormatStatusResponse writes a StatusResponse as a string to the writer.
//
// The parameters 'allAddresses', 'allControllers', 'allNodes', respectively,
// cause all details about that aspect of the status to be printed to the
// terminal. For each of these, if they are false then only a summary will be
// printed, with perhaps some detail if there are errors.
func FormatStatusResponse(w io.Writer, sr *models.StatusResponse, allAddresses, allControllers, allNodes, allRedirects bool) {
	if sr.Kvstore != nil {
		fmt.Fprintf(w, "KVStore:\t%s\t%s\n", sr.Kvstore.State, sr.Kvstore.Msg)
	}
	if sr.ContainerRuntime != nil {
		fmt.Fprintf(w, "ContainerRuntime:\t%s\t%s\n",
			sr.ContainerRuntime.State, sr.ContainerRuntime.Msg)
	}
	if sr.Kubernetes != nil {
		fmt.Fprintf(w, "Kubernetes:\t%s\t%s\n", sr.Kubernetes.State, sr.Kubernetes.Msg)
		if sr.Kubernetes.State != models.K8sStatusStateDisabled {
			sort.Strings(sr.Kubernetes.K8sAPIVersions)
			fmt.Fprintf(w, "Kubernetes APIs:\t[\"%s\"]\n", strings.Join(sr.Kubernetes.K8sAPIVersions, "\", \""))
		}
	}
	if sr.Cilium != nil {
		fmt.Fprintf(w, "Cilium:\t%s\t%s\n", sr.Cilium.State, sr.Cilium.Msg)
	}

	if sr.Stale != nil {
		sortedProbes := make([]string, 0, len(sr.Stale))
		for probe := range sr.Stale {
			sortedProbes = append(sortedProbes, probe)
		}
		sort.Strings(sortedProbes)

		stalesStr := make([]string, 0, len(sr.Stale))
		for _, probe := range sortedProbes {
			stalesStr = append(stalesStr, fmt.Sprintf("%q since %s", probe, sr.Stale[probe]))
		}

		fmt.Fprintf(w, "Stale status:\t%s\n", strings.Join(stalesStr, ", "))
	}

	if nm := sr.NodeMonitor; nm != nil {
		fmt.Fprintf(w, "NodeMonitor:\tListening for events on %d CPUs with %dx%d of shared memory\n",
			nm.Cpus, nm.Npages, nm.Pagesize)
		if nm.Lost != 0 || nm.Unknown != 0 {
			fmt.Fprintf(w, "\t%d events lost, %d unknown notifications\n", nm.Lost, nm.Unknown)
		}
	} else {
		fmt.Fprintf(w, "NodeMonitor:\tDisabled\n")
	}

	var localNode *models.NodeElement
	if sr.Cluster != nil {
		if sr.Cluster.CiliumHealth != nil {
			ch := sr.Cluster.CiliumHealth
			fmt.Fprintf(w, "Cilium health daemon:\t%s\t%s\n", ch.State, ch.Msg)
		}
		for _, node := range sr.Cluster.Nodes {
			if node.Name == sr.Cluster.Self {
				localNode = node
			} else {
				continue
			}
		}
	}

	if sr.IPAM != nil {
		var v4CIDR, v6CIDR string
		if localNode != nil {
			if nIPs := ip.CountIPsInCIDR(localNode.PrimaryAddress.IPV4.AllocRange); nIPs > 0 {
				v4CIDR = fmt.Sprintf("/%d", nIPs)
			}
			if nIPs := ip.CountIPsInCIDR(localNode.PrimaryAddress.IPV6.AllocRange); nIPs > 0 {
				v6CIDR = fmt.Sprintf("/%d", nIPs)
			}
		}
		fmt.Fprintf(w, "IPv4 address pool:\t%d%s allocated\n", len(sr.IPAM.IPV4), v4CIDR)
		if allAddresses {
			for _, ipv4 := range sr.IPAM.IPV4 {
				fmt.Fprintf(w, "  %s\n", ipv4)
			}
		}
		fmt.Fprintf(w, "IPv6 address pool:\t%d%s allocated\n", len(sr.IPAM.IPV6), v6CIDR)
		if allAddresses {
			for _, ipv6 := range sr.IPAM.IPV6 {
				fmt.Fprintf(w, "  %s\n", ipv6)
			}
		}
	}

	if sr.Controllers != nil {
		nFailing, out := 0, []string{"  Name\tLast success\tLast error\tCount\tMessage\n"}
		for _, ctrl := range sr.Controllers {
			status := ctrl.Status
			if status == nil {
				continue
			}

			if status.ConsecutiveFailureCount > 0 {
				nFailing++
			} else if !allControllers {
				continue
			}

			failSince := timeSince(time.Time(status.LastFailureTimestamp))
			successSince := timeSince(time.Time(status.LastSuccessTimestamp))

			err := "no error"
			if status.LastFailureMsg != "" {
				err = status.LastFailureMsg
			}

			out = append(out, fmt.Sprintf("  %s\t%s\t%s\t%d\t%s\t\n",
				ctrl.Name, successSince, failSince, status.ConsecutiveFailureCount, err))
		}

		nOK := len(sr.Controllers) - nFailing
		fmt.Fprintf(w, "Controller Status:\t%d/%d healthy\n", nOK, len(sr.Controllers))
		if len(out) > 1 {
			tab := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0)
			sort.Strings(out)
			for _, s := range out {
				fmt.Fprint(tab, s)
			}
			tab.Flush()
		}

	}

	if sr.Proxy != nil {
		fmt.Fprintf(w, "Proxy Status:\tOK, ip %s, port-range %s\n",
			sr.Proxy.IP, sr.Proxy.PortRange)
	} else {
		fmt.Fprintf(w, "Proxy Status:\tNo managed proxy redirect\n")
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package config defines the environment baased configuration for this project.
// Each server has a unique config type.
package config

import (
	"context"
	"fmt"
	"time"

	"github.com/google/exposure-notifications-server/pkg/secrets"

	"github.com/sethvargo/go-envconfig/pkg/envconfig"
)

// Validatable indicates that a type can be validated.
type Validatable interface {
	Validate() error
}

// ProcessWith creates a new config with the given lookuper for parsing config.
func ProcessWith(ctx context.Context, spec Validatable, l envconfig.Lookuper) error {
	// Build a list of mutators. This list will grow as we initialize more of the
	// configuration, such as the secret manager.
	var mutatorFuncs []envconfig.MutatorFunc

	{
		// Load the secret manager configuration first - this needs to be loaded first
		// because other processors may need secrets.
		var smConfig secrets.Config
		if err := envconfig.ProcessWith(ctx, &smConfig, l); err != nil {
			return fmt.Errorf("unable to process secret configuration: %w", err)
		}

		sm, err := secrets.SecretManagerFor(ctx, smConfig.SecretManagerType)
		if err != nil {
			return fmt.Errorf("unable to connect to secret manager: %w", err)
		}

		// Enable caching, if a TTL was provided.
		if ttl := smConfig.SecretCacheTTL; ttl > 0 {
			sm, err = secrets.WrapCacher(ctx, sm, ttl)
			if err != nil {
				return fmt.Errorf("unable to create secret manager cache: %w", err)
			}
		}

		// Update the mutators to process secrets.
		mutatorFuncs = append(mutatorFuncs, secrets.Resolver(sm, &smConfig))
	}

	// Parse the main configuration.
	if err := envconfig.ProcessWith(ctx, &spec, l, mutatorFuncs...); err != nil {
		return err
	}

	if err := spec.Validate(); err != nil {
		return err
	}

	return nil
}

func checkPositiveDuration(d time.Duration, name string) error {
	if d < 0 {
		return fmt.Errorf("%v must be a positive duration, got: %v", name, d)
	}
	return nil
}
<commit_msg>Too many addresses of things... (#84)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package config defines the environment baased configuration for this project.
// Each server has a unique config type.
package config

import (
	"context"
	"fmt"
	"time"

	"github.com/google/exposure-notifications-server/pkg/secrets"

	"github.com/sethvargo/go-envconfig/pkg/envconfig"
)

// Validatable indicates that a type can be validated.
type Validatable interface {
	Validate() error
}

// ProcessWith creates a new config with the given lookuper for parsing config.
func ProcessWith(ctx context.Context, spec Validatable, l envconfig.Lookuper) error {
	// Build a list of mutators. This list will grow as we initialize more of the
	// configuration, such as the secret manager.
	var mutatorFuncs []envconfig.MutatorFunc

	{
		// Load the secret manager configuration first - this needs to be loaded first
		// because other processors may need secrets.
		var smConfig secrets.Config
		if err := envconfig.ProcessWith(ctx, &smConfig, l); err != nil {
			return fmt.Errorf("unable to process secret configuration: %w", err)
		}

		sm, err := secrets.SecretManagerFor(ctx, smConfig.SecretManagerType)
		if err != nil {
			return fmt.Errorf("unable to connect to secret manager: %w", err)
		}

		// Enable caching, if a TTL was provided.
		if ttl := smConfig.SecretCacheTTL; ttl > 0 {
			sm, err = secrets.WrapCacher(ctx, sm, ttl)
			if err != nil {
				return fmt.Errorf("unable to create secret manager cache: %w", err)
			}
		}

		// Update the mutators to process secrets.
		mutatorFuncs = append(mutatorFuncs, secrets.Resolver(sm, &smConfig))
	}

	// Parse the main configuration.
	if err := envconfig.ProcessWith(ctx, spec, l, mutatorFuncs...); err != nil {
		return err
	}

	if err := spec.Validate(); err != nil {
		return err
	}

	return nil
}

func checkPositiveDuration(d time.Duration, name string) error {
	if d < 0 {
		return fmt.Errorf("%v must be a positive duration, got: %v", name, d)
	}
	return nil
}
<|endoftext|>


<commit_before>package paxos

func learner(quorum uint64, ins chan Msg) string {
	var round uint64 = 1
	votes := make(map[string]uint64) // maps values to number of votes
	voted := make(map[int]bool)      // maps nodes to vote status

	for in := range ins {
		if in.Cmd() != Vote {
			continue
		}

		mRound, v := VoteParts(in)

		switch {
		case mRound < round:
			continue
		case mRound > round:
			round = mRound
			votes = make(map[string]uint64)
			voted = make(map[int]bool)
			fallthrough
		case mRound == round:
			if voted[in.From()] {
				continue
			}
			votes[v]++
			voted[in.From()] = true

			if votes[v] >= quorum {
				return v // winner!
			}
		}
	}
	return ""
}
<commit_msg>looks nicer this way<commit_after>package paxos

func learner(quorum uint64, ins chan Msg) string {
	round := uint64(1)
	votes := make(map[string]uint64) // maps values to number of votes
	voted := make(map[int]bool)      // maps nodes to vote status

	for in := range ins {
		if in.Cmd() != Vote {
			continue
		}

		mRound, v := VoteParts(in)

		switch {
		case mRound < round:
			continue
		case mRound > round:
			round = mRound
			votes = make(map[string]uint64)
			voted = make(map[int]bool)
			fallthrough
		case mRound == round:
			if voted[in.From()] {
				continue
			}
			votes[v]++
			voted[in.From()] = true

			if votes[v] >= quorum {
				return v // winner!
			}
		}
	}
	return ""
}
<|endoftext|>


<commit_before>package paxos

import (
	"borg/util"
)

const (
	mFrom = iota
	mCmd
	mClusterVersion // 2 - 9
	mSeqn = 10 // - 17
	mBody = 18
	baseLen = mBody
)

const (
	Nop = iota
	Invite
	Rsvp
	Nominate
	Vote
)

const (
	inviteLen   = 8
	rsvpLen     = 16 // not including v
	nominateLen = 8  // not including v
	voteLen     = 8  // not including v
)

func NewInvite(crnd uint64) Msg {
	m := make(Msg, baseLen + inviteLen)
	m[mCmd] = Invite
	util.Packui64(m.Body()[0:8], crnd)
	return m
}

// Returns the info for `m`. If `m` is not an invite, the result is undefined.
func InviteParts(m Msg) (crnd uint64) {
	return util.Unpackui64(m.Body())
}

func NewNominate(crnd uint64, v string) Msg {
	m := make(Msg, baseLen+nominateLen+len(v))
	m[mCmd] = Nominate
	util.Packui64(m.Body()[0:8], crnd)
	copy(m.Body()[nominateLen:], []byte(v))
	return m
}

// Returns the info for `m`. If `m` is not a nominate, the result is undefined.
func NominateParts(m Msg) (crnd uint64, v string) {
	crnd = util.Unpackui64(m.Body()[0:8])
	v = string(m.Body()[8:])
	return
}

func NewRsvp(i, vrnd uint64, vval string) Msg {
	m := make(Msg, baseLen+rsvpLen+len(vval))
	m[mCmd] = Rsvp
	util.Packui64(m.Body()[0:8], i)
	util.Packui64(m.Body()[8:16], vrnd)
	copy(m.Body()[rsvpLen:], []byte(vval))
	return m
}

// Returns the info for `m`. If `m` is not an rsvp, the result is undefined.
func RsvpParts(m Msg) (i, vrnd uint64, vval string) {
	i = util.Unpackui64(m.Body()[0:8])
	vrnd = util.Unpackui64(m.Body()[8:16])
	vval = string(m.Body()[16:])
	return
}

func NewVote(i uint64, vval string) Msg {
	m := make(Msg, baseLen+voteLen+len(vval))
	m[mCmd] = Vote
	util.Packui64(m.Body()[0:8], i)
	copy(m.Body()[voteLen:], []byte(vval))
	return m
}

// Returns the info for `m`. If `m` is not a vote, the result is undefined.
func VoteParts(m Msg) (i uint64, vval string) {
	i = util.Unpackui64(m.Body()[0:8])
	vval = string(m.Body()[8:])
	return
}

// In-memory format:
//
//     0    -- index of sender
//     1    -- cmd
//     2..9 -- seqn
//     10.. -- body -- format depends on command
//
// Wire format is same as in-memory format, but without the first byte (the
// sender index). Here it is for clarity:
//
//     0    -- cmd
//     1..8 -- seqn
//     9..  -- body -- format depends on command
type Msg []byte

func (m Msg) From() int {
	return int(m[mFrom])
}

func (m Msg) Cmd() int {
	return int(m[mCmd])
}

func (m Msg) ClusterVersion() uint64 {
	return util.Unpackui64(m[mClusterVersion : mClusterVersion+8])
}

func (m Msg) Seqn() uint64 {
	return util.Unpackui64(m[mSeqn : mSeqn+8])
}

func (m Msg) Body() []byte {
	return m[mBody:]
}

func (m Msg) SetFrom(from byte) {
	m[mFrom] = from
}

func (m Msg) SetClusterVersion(ver uint64) {
	util.Packui64(m[mClusterVersion:mClusterVersion+8], ver)
}

func (m Msg) SetSeqn(seqn uint64) {
	util.Packui64(m[mSeqn:mSeqn+8], seqn)
}

func (m Msg) Ok() bool {
	if len(m) < 2 {
		return false
	}
	switch m.Cmd() {
	case Invite:
		return len(m.Body()) == inviteLen
	case Rsvp:
		return len(m.Body()) >= rsvpLen
	case Nominate:
		return len(m.Body()) >= nominateLen
	case Vote:
		return len(m.Body()) >= voteLen
	}
	return false
}

func (m Msg) WireBytes() []byte {
	return m[1:]
}
<commit_msg>update comments<commit_after>package paxos

import (
	"borg/util"
)

// In-memory format:
//
//     0      -- index of sender
//     1      -- cmd
//     2..9   -- cluster version
//     10..17 -- seqn
//     18..   -- body -- format depends on command
//
// Wire format is same as in-memory format, but without the first byte (the
// sender index). Here it is for clarity:
//
//     0     -- cmd
//     1..8  -- cluster version
//     9..15 -- seqn
//     16..  -- body -- format depends on command
type Msg []byte

const (
	mFrom = iota
	mCmd
	mClusterVersion // 2 - 9
	mSeqn = 10 // - 17
	mBody = 18
	baseLen = mBody
)

const (
	Nop = iota
	Invite
	Rsvp
	Nominate
	Vote
)

const (
	inviteLen   = 8
	rsvpLen     = 16 // not including v
	nominateLen = 8  // not including v
	voteLen     = 8  // not including v
)

func NewInvite(crnd uint64) Msg {
	m := make(Msg, baseLen + inviteLen)
	m[mCmd] = Invite
	util.Packui64(m.Body()[0:8], crnd)
	return m
}

// Returns the info for `m`. If `m` is not an invite, the result is undefined.
func InviteParts(m Msg) (crnd uint64) {
	return util.Unpackui64(m.Body())
}

func NewNominate(crnd uint64, v string) Msg {
	m := make(Msg, baseLen+nominateLen+len(v))
	m[mCmd] = Nominate
	util.Packui64(m.Body()[0:8], crnd)
	copy(m.Body()[nominateLen:], []byte(v))
	return m
}

// Returns the info for `m`. If `m` is not a nominate, the result is undefined.
func NominateParts(m Msg) (crnd uint64, v string) {
	crnd = util.Unpackui64(m.Body()[0:8])
	v = string(m.Body()[8:])
	return
}

func NewRsvp(i, vrnd uint64, vval string) Msg {
	m := make(Msg, baseLen+rsvpLen+len(vval))
	m[mCmd] = Rsvp
	util.Packui64(m.Body()[0:8], i)
	util.Packui64(m.Body()[8:16], vrnd)
	copy(m.Body()[rsvpLen:], []byte(vval))
	return m
}

// Returns the info for `m`. If `m` is not an rsvp, the result is undefined.
func RsvpParts(m Msg) (i, vrnd uint64, vval string) {
	i = util.Unpackui64(m.Body()[0:8])
	vrnd = util.Unpackui64(m.Body()[8:16])
	vval = string(m.Body()[16:])
	return
}

func NewVote(i uint64, vval string) Msg {
	m := make(Msg, baseLen+voteLen+len(vval))
	m[mCmd] = Vote
	util.Packui64(m.Body()[0:8], i)
	copy(m.Body()[voteLen:], []byte(vval))
	return m
}

// Returns the info for `m`. If `m` is not a vote, the result is undefined.
func VoteParts(m Msg) (i uint64, vval string) {
	i = util.Unpackui64(m.Body()[0:8])
	vval = string(m.Body()[8:])
	return
}

func (m Msg) From() int {
	return int(m[mFrom])
}

func (m Msg) Cmd() int {
	return int(m[mCmd])
}

func (m Msg) ClusterVersion() uint64 {
	return util.Unpackui64(m[mClusterVersion : mClusterVersion+8])
}

func (m Msg) Seqn() uint64 {
	return util.Unpackui64(m[mSeqn : mSeqn+8])
}

func (m Msg) Body() []byte {
	return m[mBody:]
}

func (m Msg) SetFrom(from byte) {
	m[mFrom] = from
}

func (m Msg) SetClusterVersion(ver uint64) {
	util.Packui64(m[mClusterVersion:mClusterVersion+8], ver)
}

func (m Msg) SetSeqn(seqn uint64) {
	util.Packui64(m[mSeqn:mSeqn+8], seqn)
}

func (m Msg) Ok() bool {
	if len(m) < 2 {
		return false
	}
	switch m.Cmd() {
	case Invite:
		return len(m.Body()) == inviteLen
	case Rsvp:
		return len(m.Body()) >= rsvpLen
	case Nominate:
		return len(m.Body()) >= nominateLen
	case Vote:
		return len(m.Body()) >= voteLen
	}
	return false
}

func (m Msg) WireBytes() []byte {
	return m[1:]
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"io"
	"os"
	"sync"
	"text/tabwriter"
	"time"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/progress"
	"github.com/containerd/containerd/remotes"
	digest "github.com/opencontainers/go-digest"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/urfave/cli"
)

var fetchCommand = cli.Command{
	Name:      "fetch",
	Usage:     "fetch all content for an image into containerd",
	ArgsUsage: "[flags] <remote> <object>",
	Description: `Fetch an image into containerd.

This command ensures that containerd has all the necessary resources to build
an image's rootfs and convert the configuration to a runtime format supported
by containerd.

This command uses the same syntax, of remote and object, as 'dist
fetch-object'. We may want to make this nicer, but agnostism is preferred for
the moment.

Right now, the responsibility of the daemon and the cli aren't quite clear. Do
not use this implementation as a guide. The end goal should be having metadata,
content and snapshots ready for a direct use via the 'ctr run'.

Most of this is experimental and there are few leaps to make this work.`,
	Flags: registryFlags,
	Action: func(clicontext *cli.Context) error {
		var (
			ref = clicontext.Args().First()
		)
		ctx, cancel := appContext(clicontext)
		defer cancel()

		_, err := fetch(ctx, ref, clicontext)
		return err
	},
}

func fetch(ctx context.Context, ref string, clicontext *cli.Context) (containerd.Image, error) {
	client, err := newClient(clicontext)
	if err != nil {
		return nil, err
	}

	resolver, err := getResolver(ctx, clicontext)
	if err != nil {
		return nil, err
	}

	ongoing := newJobs(ref)

	pctx, stopProgress := context.WithCancel(ctx)
	progress := make(chan struct{})

	go func() {
		showProgress(pctx, ongoing, client.ContentStore(), os.Stdout)
		close(progress)
	}()

	h := images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
		if desc.MediaType != images.MediaTypeDockerSchema1Manifest {
			ongoing.add(desc)
		}
		return nil, nil
	})

	log.G(pctx).WithField("image", ref).Debug("fetching")

	img, err := client.Pull(pctx, ref, containerd.WithResolver(resolver), containerd.WithImageHandler(h), containerd.WithSchema1Conversion)
	stopProgress()
	if err != nil {
		return nil, err
	}

	<-progress
	return img, nil
}

func showProgress(ctx context.Context, ongoing *jobs, cs content.Store, out io.Writer) {
	var (
		ticker   = time.NewTicker(100 * time.Millisecond)
		fw       = progress.NewWriter(out)
		start    = time.Now()
		statuses = map[string]statusInfo{}
		done     bool
	)
	defer ticker.Stop()

outer:
	for {
		select {
		case <-ticker.C:
			fw.Flush()

			tw := tabwriter.NewWriter(fw, 1, 8, 1, ' ', 0)

			resolved := "resolved"
			if !ongoing.isResolved() {
				resolved = "resolving"
			}
			statuses[ongoing.name] = statusInfo{
				Ref:    ongoing.name,
				Status: resolved,
			}
			keys := []string{ongoing.name}

			activeSeen := map[string]struct{}{}
			if !done {
				active, err := cs.ListStatuses(ctx, "")
				if err != nil {
					log.G(ctx).WithError(err).Error("active check failed")
					continue
				}
				// update status of active entries!
				for _, active := range active {
					statuses[active.Ref] = statusInfo{
						Ref:       active.Ref,
						Status:    "downloading",
						Offset:    active.Offset,
						Total:     active.Total,
						StartedAt: active.StartedAt,
						UpdatedAt: active.UpdatedAt,
					}
					activeSeen[active.Ref] = struct{}{}
				}
			}

			// now, update the items in jobs that are not in active
			for _, j := range ongoing.jobs() {
				key := remotes.MakeRefKey(ctx, j)
				keys = append(keys, key)
				if _, ok := activeSeen[key]; ok {
					continue
				}

				status, ok := statuses[key]
				if !done && (!ok || status.Status == "downloading") {
					info, err := cs.Info(ctx, j.Digest)
					if err != nil {
						if !errdefs.IsNotFound(err) {
							log.G(ctx).WithError(err).Errorf("failed to get content info")
							continue outer
						} else {
							statuses[key] = statusInfo{
								Ref:    key,
								Status: "waiting",
							}
						}
					} else if info.CreatedAt.After(start) {
						statuses[key] = statusInfo{
							Ref:       key,
							Status:    "done",
							Offset:    info.Size,
							Total:     info.Size,
							UpdatedAt: info.CreatedAt,
						}
					} else {
						statuses[key] = statusInfo{
							Ref:    key,
							Status: "exists",
						}
					}
				} else if done {
					if ok {
						if status.Status != "done" && status.Status != "exists" {
							status.Status = "done"
							statuses[key] = status
						}
					} else {
						statuses[key] = statusInfo{
							Ref:    key,
							Status: "done",
						}
					}
				}
			}

			var ordered []statusInfo
			for _, key := range keys {
				ordered = append(ordered, statuses[key])
			}

			display(tw, ordered, start)
			tw.Flush()

			if done {
				fw.Flush()
				return
			}
		case <-ctx.Done():
			done = true // allow ui to update once more
		}
	}
}

// jobs provides a way of identifying the download keys for a particular task
// encountering during the pull walk.
//
// This is very minimal and will probably be replaced with something more
// featured.
type jobs struct {
	name     string
	added    map[digest.Digest]struct{}
	descs    []ocispec.Descriptor
	mu       sync.Mutex
	resolved bool
}

func newJobs(name string) *jobs {
	return &jobs{
		name:  name,
		added: map[digest.Digest]struct{}{},
	}
}

func (j *jobs) add(desc ocispec.Descriptor) {
	j.mu.Lock()
	defer j.mu.Unlock()
	j.resolved = true

	if _, ok := j.added[desc.Digest]; ok {
		return
	}
	j.descs = append(j.descs, desc)
	j.added[desc.Digest] = struct{}{}
}

func (j *jobs) jobs() []ocispec.Descriptor {
	j.mu.Lock()
	defer j.mu.Unlock()

	var descs []ocispec.Descriptor
	return append(descs, j.descs...)
}

func (j *jobs) isResolved() bool {
	j.mu.Lock()
	defer j.mu.Unlock()
	return j.resolved
}

type statusInfo struct {
	Ref       string
	Status    string
	Offset    int64
	Total     int64
	StartedAt time.Time
	UpdatedAt time.Time
}

func display(w io.Writer, statuses []statusInfo, start time.Time) {
	var total int64
	for _, status := range statuses {
		total += status.Offset
		switch status.Status {
		case "downloading", "uploading":
			var bar progress.Bar
			if status.Total > 0.0 {
				bar = progress.Bar(float64(status.Offset) / float64(status.Total))
			}
			fmt.Fprintf(w, "%s:\t%s\t%40r\t%8.8s/%s\t\n",
				status.Ref,
				status.Status,
				bar,
				progress.Bytes(status.Offset), progress.Bytes(status.Total))
		case "resolving", "waiting":
			bar := progress.Bar(0.0)
			fmt.Fprintf(w, "%s:\t%s\t%40r\t\n",
				status.Ref,
				status.Status,
				bar)
		default:
			bar := progress.Bar(1.0)
			fmt.Fprintf(w, "%s:\t%s\t%40r\t\n",
				status.Ref,
				status.Status,
				bar)
		}
	}

	fmt.Fprintf(w, "elapsed: %-4.1fs\ttotal: %7.6v\t(%v)\t\n",
		time.Since(start).Seconds(),
		// TODO(stevvooe): These calculations are actually way off.
		// Need to account for previously downloaded data. These
		// will basically be right for a download the first time
		// but will be skewed if restarting, as it includes the
		// data into the start time before.
		progress.Bytes(total),
		progress.NewBytesPerSecond(total, time.Since(start)))
}
<commit_msg>comment: dist -> ctr<commit_after>package main

import (
	"context"
	"fmt"
	"io"
	"os"
	"sync"
	"text/tabwriter"
	"time"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/progress"
	"github.com/containerd/containerd/remotes"
	digest "github.com/opencontainers/go-digest"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/urfave/cli"
)

var fetchCommand = cli.Command{
	Name:      "fetch",
	Usage:     "fetch all content for an image into containerd",
	ArgsUsage: "[flags] <remote> <object>",
	Description: `Fetch an image into containerd.

This command ensures that containerd has all the necessary resources to build
an image's rootfs and convert the configuration to a runtime format supported
by containerd.

This command uses the same syntax, of remote and object, as 'ctr fetch-object'.
We may want to make this nicer, but agnostism is preferred for the moment.

Right now, the responsibility of the daemon and the cli aren't quite clear. Do
not use this implementation as a guide. The end goal should be having metadata,
content and snapshots ready for a direct use via the 'ctr run'.

Most of this is experimental and there are few leaps to make this work.`,
	Flags: registryFlags,
	Action: func(clicontext *cli.Context) error {
		var (
			ref = clicontext.Args().First()
		)
		ctx, cancel := appContext(clicontext)
		defer cancel()

		_, err := fetch(ctx, ref, clicontext)
		return err
	},
}

func fetch(ctx context.Context, ref string, clicontext *cli.Context) (containerd.Image, error) {
	client, err := newClient(clicontext)
	if err != nil {
		return nil, err
	}

	resolver, err := getResolver(ctx, clicontext)
	if err != nil {
		return nil, err
	}

	ongoing := newJobs(ref)

	pctx, stopProgress := context.WithCancel(ctx)
	progress := make(chan struct{})

	go func() {
		showProgress(pctx, ongoing, client.ContentStore(), os.Stdout)
		close(progress)
	}()

	h := images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
		if desc.MediaType != images.MediaTypeDockerSchema1Manifest {
			ongoing.add(desc)
		}
		return nil, nil
	})

	log.G(pctx).WithField("image", ref).Debug("fetching")

	img, err := client.Pull(pctx, ref, containerd.WithResolver(resolver), containerd.WithImageHandler(h), containerd.WithSchema1Conversion)
	stopProgress()
	if err != nil {
		return nil, err
	}

	<-progress
	return img, nil
}

func showProgress(ctx context.Context, ongoing *jobs, cs content.Store, out io.Writer) {
	var (
		ticker   = time.NewTicker(100 * time.Millisecond)
		fw       = progress.NewWriter(out)
		start    = time.Now()
		statuses = map[string]statusInfo{}
		done     bool
	)
	defer ticker.Stop()

outer:
	for {
		select {
		case <-ticker.C:
			fw.Flush()

			tw := tabwriter.NewWriter(fw, 1, 8, 1, ' ', 0)

			resolved := "resolved"
			if !ongoing.isResolved() {
				resolved = "resolving"
			}
			statuses[ongoing.name] = statusInfo{
				Ref:    ongoing.name,
				Status: resolved,
			}
			keys := []string{ongoing.name}

			activeSeen := map[string]struct{}{}
			if !done {
				active, err := cs.ListStatuses(ctx, "")
				if err != nil {
					log.G(ctx).WithError(err).Error("active check failed")
					continue
				}
				// update status of active entries!
				for _, active := range active {
					statuses[active.Ref] = statusInfo{
						Ref:       active.Ref,
						Status:    "downloading",
						Offset:    active.Offset,
						Total:     active.Total,
						StartedAt: active.StartedAt,
						UpdatedAt: active.UpdatedAt,
					}
					activeSeen[active.Ref] = struct{}{}
				}
			}

			// now, update the items in jobs that are not in active
			for _, j := range ongoing.jobs() {
				key := remotes.MakeRefKey(ctx, j)
				keys = append(keys, key)
				if _, ok := activeSeen[key]; ok {
					continue
				}

				status, ok := statuses[key]
				if !done && (!ok || status.Status == "downloading") {
					info, err := cs.Info(ctx, j.Digest)
					if err != nil {
						if !errdefs.IsNotFound(err) {
							log.G(ctx).WithError(err).Errorf("failed to get content info")
							continue outer
						} else {
							statuses[key] = statusInfo{
								Ref:    key,
								Status: "waiting",
							}
						}
					} else if info.CreatedAt.After(start) {
						statuses[key] = statusInfo{
							Ref:       key,
							Status:    "done",
							Offset:    info.Size,
							Total:     info.Size,
							UpdatedAt: info.CreatedAt,
						}
					} else {
						statuses[key] = statusInfo{
							Ref:    key,
							Status: "exists",
						}
					}
				} else if done {
					if ok {
						if status.Status != "done" && status.Status != "exists" {
							status.Status = "done"
							statuses[key] = status
						}
					} else {
						statuses[key] = statusInfo{
							Ref:    key,
							Status: "done",
						}
					}
				}
			}

			var ordered []statusInfo
			for _, key := range keys {
				ordered = append(ordered, statuses[key])
			}

			display(tw, ordered, start)
			tw.Flush()

			if done {
				fw.Flush()
				return
			}
		case <-ctx.Done():
			done = true // allow ui to update once more
		}
	}
}

// jobs provides a way of identifying the download keys for a particular task
// encountering during the pull walk.
//
// This is very minimal and will probably be replaced with something more
// featured.
type jobs struct {
	name     string
	added    map[digest.Digest]struct{}
	descs    []ocispec.Descriptor
	mu       sync.Mutex
	resolved bool
}

func newJobs(name string) *jobs {
	return &jobs{
		name:  name,
		added: map[digest.Digest]struct{}{},
	}
}

func (j *jobs) add(desc ocispec.Descriptor) {
	j.mu.Lock()
	defer j.mu.Unlock()
	j.resolved = true

	if _, ok := j.added[desc.Digest]; ok {
		return
	}
	j.descs = append(j.descs, desc)
	j.added[desc.Digest] = struct{}{}
}

func (j *jobs) jobs() []ocispec.Descriptor {
	j.mu.Lock()
	defer j.mu.Unlock()

	var descs []ocispec.Descriptor
	return append(descs, j.descs...)
}

func (j *jobs) isResolved() bool {
	j.mu.Lock()
	defer j.mu.Unlock()
	return j.resolved
}

type statusInfo struct {
	Ref       string
	Status    string
	Offset    int64
	Total     int64
	StartedAt time.Time
	UpdatedAt time.Time
}

func display(w io.Writer, statuses []statusInfo, start time.Time) {
	var total int64
	for _, status := range statuses {
		total += status.Offset
		switch status.Status {
		case "downloading", "uploading":
			var bar progress.Bar
			if status.Total > 0.0 {
				bar = progress.Bar(float64(status.Offset) / float64(status.Total))
			}
			fmt.Fprintf(w, "%s:\t%s\t%40r\t%8.8s/%s\t\n",
				status.Ref,
				status.Status,
				bar,
				progress.Bytes(status.Offset), progress.Bytes(status.Total))
		case "resolving", "waiting":
			bar := progress.Bar(0.0)
			fmt.Fprintf(w, "%s:\t%s\t%40r\t\n",
				status.Ref,
				status.Status,
				bar)
		default:
			bar := progress.Bar(1.0)
			fmt.Fprintf(w, "%s:\t%s\t%40r\t\n",
				status.Ref,
				status.Status,
				bar)
		}
	}

	fmt.Fprintf(w, "elapsed: %-4.1fs\ttotal: %7.6v\t(%v)\t\n",
		time.Since(start).Seconds(),
		// TODO(stevvooe): These calculations are actually way off.
		// Need to account for previously downloaded data. These
		// will basically be right for a download the first time
		// but will be skewed if restarting, as it includes the
		// data into the start time before.
		progress.Bytes(total),
		progress.NewBytesPerSecond(total, time.Since(start)))
}
<|endoftext|>


<commit_before>// tun_linux.go: Open a Tunnel (L3 virtual interface) using the Universal TUN/TAP device driver.
package tun

import (
	"context"
	"fmt"
	"net"
	"os"
	"unsafe"

	"golang.org/x/sys/unix"

	"github.com/datawire/dlib/dexec"
	"github.com/telepresenceio/telepresence/v2/pkg/tun/buffer"
)

const devicePath = "/dev/net/tun"

type Device struct {
	*os.File
	name  string
	index uint32
}

// OpenTun creates a new TUN device and ensures that it is up and running.
func OpenTun() (*Device, error) {
	// https://www.kernel.org/doc/Documentation/networking/tuntap.txt

	fd, err := unix.Open(devicePath, unix.O_RDWR, 0)
	if err != nil {
		return nil, fmt.Errorf("failed to open TUN device %s: %v", devicePath, err)
	}
	unix.CloseOnExec(fd)
	defer func() {
		if err != nil {
			_ = unix.Close(fd)
		}
	}()

	var flagsRequest struct {
		name  [unix.IFNAMSIZ]byte
		flags int16
	}
	copy(flagsRequest.name[:], "tel%d")
	flagsRequest.flags = unix.IFF_TUN | unix.IFF_NO_PI

	err = unix.IoctlSetInt(fd, unix.TUNSETIFF, int(uintptr(unsafe.Pointer(&flagsRequest))))
	if err != nil {
		return nil, err
	}

	var name string
	for i := 0; i < unix.IFNAMSIZ; i++ {
		if flagsRequest.name[i] == 0 {
			name = string(flagsRequest.name[0:i])
			break
		}
	}
	if name == "" {
		name = string(flagsRequest.name[:])
	}

	htons := func(value uint16) uint16 {
		test := uint16(1)
		if (*[2]byte)(unsafe.Pointer(&test))[0] == 1 {
			// this machine is little endian, swap bytes
			value = value&0xff<<8 | value&0xff00>>8
		}
		return value
	}

	// Passing a network ordered short here is required.
	provisioningSocket, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(htons(unix.ETH_P_ALL)))
	if err != nil {
		return nil, err
	}
	if err = unix.BindToDevice(provisioningSocket, name); err != nil {
		return nil, err
	}
	flagsRequest.flags |= unix.IFF_UP | unix.IFF_RUNNING
	if err = ioctl(provisioningSocket, unix.SIOCSIFFLAGS, unsafe.Pointer(&flagsRequest)); err != nil {
		return nil, err
	}

	index, err := getInterfaceIndex(provisioningSocket, name)
	if err != nil {
		return nil, err
	}

	// Set non-blocking so that Read() doesn't hang for several seconds when the
	// fd is Closed. Read() will still wait for data to arrive.
	//
	// See: https://github.com/golang/go/issues/30426#issuecomment-470044803
	_ = unix.SetNonblock(fd, true)
	return &Device{File: os.NewFile(uintptr(fd), devicePath), name: name, index: index}, nil
}

func (t *Device) AddSubnet(ctx context.Context, subnet *net.IPNet) error {
	return dexec.CommandContext(ctx, "ip", "a", "add", subnet.String(), "dev", t.name).Run()
}

// RemoveSubnet removes a subnet from this TUN device and also removes the route for that subnet which
// is associated with the device.
func (t *Device) RemoveSubnet(ctx context.Context, subnet *net.IPNet) error {
	return dexec.CommandContext(ctx, "ip", "a", "del", subnet.String(), "dev", t.name).Run()
}

// Index returns the index of this device
func (t *Device) Index() uint32 {
	return t.index
}

func (t *Device) SetMTU(mtu int) error {
	return withSocket(unix.AF_INET, func(fd int) error {
		var mtuRequest struct {
			name [unix.IFNAMSIZ]byte
			mtu  uint32
		}
		copy(mtuRequest.name[:], t.name)
		mtuRequest.mtu = uint32(mtu)
		err := ioctl(fd, unix.SIOCSIFMTU, unsafe.Pointer(&mtuRequest))
		if err != nil {
			err = fmt.Errorf("set MTU on %s failed: %w", t.name, err)
		}
		return err
	})
}

// Read reads as many bytes as possible into the given buffer.Data and returns the
// number of bytes actually read
func (t *Device) Read(into *buffer.Data) (int, error) {
	return t.File.Read(into.Raw())
}

// Write writes bytes from the given buffer.Data and returns the number of bytes
// actually written.
func (t *Device) Write(from *buffer.Data) (int, error) {
	return t.File.Write(from.Raw())
}

func getInterfaceIndex(fd int, name string) (uint32, error) {
	var indexRequest struct {
		name  [unix.IFNAMSIZ]byte
		index uint32
	}
	copy(indexRequest.name[:], name)
	if err := ioctl(fd, unix.SIOCGIFINDEX, unsafe.Pointer(&indexRequest)); err != nil {
		return 0, fmt.Errorf("get interface index on %s failed: %w", name, err)
	}
	return indexRequest.index, nil
}
<commit_msg>Add close of provisioning socket when opening Linux TUN-device.<commit_after>// tun_linux.go: Open a Tunnel (L3 virtual interface) using the Universal TUN/TAP device driver.
package tun

import (
	"context"
	"fmt"
	"net"
	"os"
	"unsafe"

	"golang.org/x/sys/unix"

	"github.com/datawire/dlib/dexec"
	"github.com/telepresenceio/telepresence/v2/pkg/tun/buffer"
)

const devicePath = "/dev/net/tun"

type Device struct {
	*os.File
	name  string
	index uint32
}

// OpenTun creates a new TUN device and ensures that it is up and running.
func OpenTun() (*Device, error) {
	// https://www.kernel.org/doc/Documentation/networking/tuntap.txt

	fd, err := unix.Open(devicePath, unix.O_RDWR, 0)
	if err != nil {
		return nil, fmt.Errorf("failed to open TUN device %s: %v", devicePath, err)
	}
	unix.CloseOnExec(fd)
	defer func() {
		if err != nil {
			_ = unix.Close(fd)
		}
	}()

	var flagsRequest struct {
		name  [unix.IFNAMSIZ]byte
		flags int16
	}
	copy(flagsRequest.name[:], "tel%d")
	flagsRequest.flags = unix.IFF_TUN | unix.IFF_NO_PI

	err = unix.IoctlSetInt(fd, unix.TUNSETIFF, int(uintptr(unsafe.Pointer(&flagsRequest))))
	if err != nil {
		return nil, err
	}

	var name string
	for i := 0; i < unix.IFNAMSIZ; i++ {
		if flagsRequest.name[i] == 0 {
			name = string(flagsRequest.name[0:i])
			break
		}
	}
	if name == "" {
		name = string(flagsRequest.name[:])
	}

	htons := func(value uint16) uint16 {
		test := uint16(1)
		if (*[2]byte)(unsafe.Pointer(&test))[0] == 1 {
			// this machine is little endian, swap bytes
			value = value&0xff<<8 | value&0xff00>>8
		}
		return value
	}

	// Passing a network ordered short here is required.
	provisioningSocket, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(htons(unix.ETH_P_ALL)))
	if err != nil {
		return nil, err
	}
	defer unix.Close(provisioningSocket)

	if err = unix.BindToDevice(provisioningSocket, name); err != nil {
		return nil, err
	}
	flagsRequest.flags |= unix.IFF_UP | unix.IFF_RUNNING
	if err = ioctl(provisioningSocket, unix.SIOCSIFFLAGS, unsafe.Pointer(&flagsRequest)); err != nil {
		return nil, err
	}

	index, err := getInterfaceIndex(provisioningSocket, name)
	if err != nil {
		return nil, err
	}

	// Set non-blocking so that Read() doesn't hang for several seconds when the
	// fd is Closed. Read() will still wait for data to arrive.
	//
	// See: https://github.com/golang/go/issues/30426#issuecomment-470044803
	_ = unix.SetNonblock(fd, true)
	return &Device{File: os.NewFile(uintptr(fd), devicePath), name: name, index: index}, nil
}

func (t *Device) AddSubnet(ctx context.Context, subnet *net.IPNet) error {
	return dexec.CommandContext(ctx, "ip", "a", "add", subnet.String(), "dev", t.name).Run()
}

// RemoveSubnet removes a subnet from this TUN device and also removes the route for that subnet which
// is associated with the device.
func (t *Device) RemoveSubnet(ctx context.Context, subnet *net.IPNet) error {
	return dexec.CommandContext(ctx, "ip", "a", "del", subnet.String(), "dev", t.name).Run()
}

// Index returns the index of this device
func (t *Device) Index() uint32 {
	return t.index
}

func (t *Device) SetMTU(mtu int) error {
	return withSocket(unix.AF_INET, func(fd int) error {
		var mtuRequest struct {
			name [unix.IFNAMSIZ]byte
			mtu  uint32
		}
		copy(mtuRequest.name[:], t.name)
		mtuRequest.mtu = uint32(mtu)
		err := ioctl(fd, unix.SIOCSIFMTU, unsafe.Pointer(&mtuRequest))
		if err != nil {
			err = fmt.Errorf("set MTU on %s failed: %w", t.name, err)
		}
		return err
	})
}

// Read reads as many bytes as possible into the given buffer.Data and returns the
// number of bytes actually read
func (t *Device) Read(into *buffer.Data) (int, error) {
	return t.File.Read(into.Raw())
}

// Write writes bytes from the given buffer.Data and returns the number of bytes
// actually written.
func (t *Device) Write(from *buffer.Data) (int, error) {
	return t.File.Write(from.Raw())
}

func getInterfaceIndex(fd int, name string) (uint32, error) {
	var indexRequest struct {
		name  [unix.IFNAMSIZ]byte
		index uint32
	}
	copy(indexRequest.name[:], name)
	if err := ioctl(fd, unix.SIOCGIFINDEX, unsafe.Pointer(&indexRequest)); err != nil {
		return 0, fmt.Errorf("get interface index on %s failed: %w", name, err)
	}
	return indexRequest.index, nil
}
<|endoftext|>