<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"parseargs"
	"strings"
)

const (
	versionNumber   = "0.15"
	author          = "B. VAUDOUR"
	description     = "Tool in command-line for KaOS Community Packages"
	longDescription = `Provides a tool to make the use of KaOS Community Packages.

With this tool, you can search, get and install a package from KaOS Community Packages.`
	defaultEditor  = "vim"
	searchHead     = "application/vnd.github.v3.text-match+json"
	searchHeadType = "Accept"
	searchBase     = "https://api.github.com/search/repositories?q=%v+user:KaOS-Community-Packages+fork:true"
	urlBase        = "https://github.com/KaOS-Community-Packages/%v.git"
	urlPkgbuild    = "https://raw.githubusercontent.com/KaOS-Community-Packages/%v/master/PKGBUILD"
	kcp_lock       = "kcp.lock"
)

var editor string
var tmpDir string
var fGet, fInstall, fSearch *string
var fHelp, fVersion, fFast, fDeps *bool
var p *parseargs.Parser

func init() {
	editor = os.Getenv("EDITOR")
	if editor == "" {
		editor = defaultEditor
	}
	tmpDir = os.TempDir()

	p = parseargs.New(description, versionNumber)
	p.Set(parseargs.SYNOPSIS, "[OPTIONS] [APP]")
	p.Set(parseargs.AUTHOR, author)
	p.Set(parseargs.LONGDESCRIPTION, longDescription)
	g := p.InsertGroup()
	fHelp = p.Bool("-h", "--help", "print this help")
	fVersion = g.Bool("-v", "--version", "print version")
	fSearch = g.String("-s", "--search", "search an app in KCP", "APP", "")
	fFast = p.Bool("", "--fast", "in conjonction with --search, don't print version")
	fGet = g.String("-g", "--get", "get needed files to build app", "APP", "")
	fInstall = g.String("-i", "--install", "install an app from KCP", "APP", "")
	fDeps = p.Bool("", "--asdeps", "in conjonction with --install, install as a dependence")
	p.Link("--asdeps", "-i")
	p.Link("--fast", "-s")
}

func printError(msg string) {
	fmt.Printf("\033[1;31m%v\033[m\n", msg)
}

func question(msg string, value bool) bool {
	var defaultVal string
	if value {
		defaultVal = "[Y/n]"
	} else {
		defaultVal = "[y/N]"
	}
	fmt.Printf("\033[1;33m%v %v \033[m", msg, defaultVal)
	var response string
	if _, err := fmt.Scanf("%v", &response); err != nil {
		return value
	}
	if len(response) == 0 {
		return value
	}
	response = strings.ToLower(response)
	if strings.HasPrefix(response, "y") {
		return true
	} else if strings.HasPrefix(response, "n") {
		return false
	}
	return value
}

func edit(filename string) {
	cmd := exec.Command(editor, filename)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Run()
}

func checkUser() {
	if e := os.Geteuid(); e == 0 {
		printError("Don't launch this program as root!")
		os.Exit(1)
	}
}

func editPkgbuild() {
	if question("Do you want to edit PKGBUILD?", true) {
		edit("PKGBUILD")
	}
}

func getPackage(app string) {
	url := fmt.Sprintf(urlBase, app)
	cmd := exec.Command("git", "clone", url)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		os.Exit(1)
	}
}

func launchRequest(search string, withHeader bool) []byte {
	client := &http.Client{}
	req, err := http.NewRequest("GET", search, nil)
	if err != nil {
		return []byte{}
	}
	if withHeader {
		req.Header.Add(searchHeadType, searchHead)
	}
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println(err)
	}
	b, err := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()
	return b
}

func getVersion(result []byte) string {
	var pkgver, pkgrel string
	l := strings.Split(string(result), "\n")
	for _, e := range l {
		e := strings.TrimSpace(e)
		if len(e) <= 7 {
			continue
		}
		if e[:7] == "pkgver=" {
			pkgver = e[7:]
		} else if e[:7] == "pkgrel=" {
			pkgrel = e[7:]
		}
	}
	if pkgver != "" && pkgrel != "" {
		return pkgver + "-" + pkgrel
	}
	return "<unknown>"
}

func checkInstalled(app string) string {
	var localVersion string
	cmd := exec.Command("pacman", "-Q", app)
	if out, err := cmd.Output(); err == nil {
		o := strings.Fields(string(out))
		if len(o) >= 2 {
			localVersion = o[1]
		}
	}
	return localVersion
}

func searchPackage(app string, fast bool) {
	search := fmt.Sprintf(searchBase, app)
	var f interface{}
	if err := json.Unmarshal(launchRequest(search, true), &f); err != nil {
		return
	}
	j := f.(map[string]interface{})["items"]
	result := j.([]interface{})
	for _, a := range result {
		e := a.(map[string]interface{})
		n, d, s := e["name"], e["description"], e["stargazers_count"]
		i := checkInstalled(fmt.Sprintf("%v", n))
		if fast {
			if i != "" {
				fmt.Printf("\033[1m%v\033[m \033[1;36m[installed: %v]\033[m \033[1;34m(%v)\033[m\n", n, i, s)
			} else {
				fmt.Printf("\033[1m%v\033[m\033 \033[1;34m(%v)\033[m\n", n, s)
			}
		} else {
			v := string(getVersion(launchRequest(fmt.Sprintf(urlPkgbuild, n), false)))
			if i != "" {
				if v == i {
					i = " [installed]"
				} else {
					i = fmt.Sprintf(" [installed: %v]", i)
				}
			}
			fmt.Printf("\033[1m%v\033[m \033[1;32m%v\033[m\033[1;36m%v\033[m \033[1;34m(%v)\033[m\n", n, v, i, s)
		}
		fmt.Println("\t", d)
	}
}

func installPackage(app string, asdeps bool) {
	os.Chdir(tmpDir)
	lck := tmpDir + string(os.PathSeparator) + kcp_lock
	_, e := os.Open(lck)
	if e == nil {
		fmt.Println("\033[1;31mAnother instance is running!\033[m")
		os.Exit(1)
	}
	os.Create(lck)
	wDir := tmpDir + string(os.PathSeparator) + app
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		os.RemoveAll(wDir)
		os.Remove(lck)
		os.Exit(1)
	}()
	getPackage(app)
	defer os.RemoveAll(wDir)
	if err := os.Chdir(wDir); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	editPkgbuild()
	var cmd *exec.Cmd
	if asdeps {
		cmd = exec.Command("makepkg", "-si", "--asdeps")
	} else {
		cmd = exec.Command("makepkg", "-si")
	}
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Run()
}

func main() {
	checkUser()
	err := p.Parse(os.Args)
	switch {
	case err != nil:
		fmt.Println(err)
		p.PrintHelp()
	case *fHelp:
		p.PrintHelp()
	case *fVersion:
		p.PrintVersion()
	case *fGet != "":
		getPackage(*fGet)
	case *fSearch != "":
		searchPackage(*fSearch, *fFast)
	case *fInstall != "":
		installPackage(*fInstall, *fDeps)
	default:
		p.PrintHelp()
	}
}
<commit_msg>minor correct<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"parseargs"
	"strings"
)

const (
	versionNumber   = "0.15"
	author          = "B. VAUDOUR"
	description     = "Tool in command-line for KaOS Community Packages"
	longDescription = `Provides a tool to make the use of KaOS Community Packages.

With this tool, you can search, get and install a package from KaOS Community Packages.`
	defaultEditor  = "vim"
	searchHead     = "application/vnd.github.v3.text-match+json"
	searchHeadType = "Accept"
	searchBase     = "https://api.github.com/search/repositories?q=%v+user:KaOS-Community-Packages+fork:true"
	urlBase        = "https://github.com/KaOS-Community-Packages/%v.git"
	urlPkgbuild    = "https://raw.githubusercontent.com/KaOS-Community-Packages/%v/master/PKGBUILD"
	kcp_lock       = "kcp.lock"
)

var editor string
var tmpDir string
var fGet, fInstall, fSearch *string
var fHelp, fVersion, fFast, fDeps *bool
var p *parseargs.Parser

func init() {
	editor = os.Getenv("EDITOR")
	if editor == "" {
		editor = defaultEditor
	}
	tmpDir = os.TempDir()

	p = parseargs.New(description, versionNumber)
	p.Set(parseargs.SYNOPSIS, "[OPTIONS] [APP]")
	p.Set(parseargs.AUTHOR, author)
	p.Set(parseargs.LONGDESCRIPTION, longDescription)
	g := p.InsertGroup()
	fHelp = p.Bool("-h", "--help", "print this help")
	fVersion = g.Bool("-v", "--version", "print version")
	fSearch = g.String("-s", "--search", "search an app in KCP", "APP", "")
	fFast = p.Bool("", "--fast", "in conjonction with --search, don't print version")
	fGet = g.String("-g", "--get", "get needed files to build app", "APP", "")
	fInstall = g.String("-i", "--install", "install an app from KCP", "APP", "")
	fDeps = p.Bool("", "--asdeps", "in conjonction with --install, install as a dependence")
	p.Link("--asdeps", "-i")
	p.Link("--fast", "-s")
}

func printError(msg string) {
	fmt.Printf("\033[1;31m%v\033[m\n", msg)
}

func question(msg string, value bool) bool {
	var defaultVal string
	if value {
		defaultVal = "[Y/n]"
	} else {
		defaultVal = "[y/N]"
	}
	fmt.Printf("\033[1;33m%v %v \033[m", msg, defaultVal)
	var response string
	if _, err := fmt.Scanf("%v", &response); err != nil {
		return value
	}
	if len(response) == 0 {
		return value
	}
	response = strings.ToLower(response)
	if strings.HasPrefix(response, "y") {
		return true
	} else if strings.HasPrefix(response, "n") {
		return false
	}
	return value
}

func edit(filename string) {
	cmd := exec.Command(editor, filename)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Run()
}

func checkUser() {
	if e := os.Geteuid(); e == 0 {
		printError("Don't launch this program as root!")
		os.Exit(1)
	}
}

func editPkgbuild() {
	if question("Do you want to edit PKGBUILD?", true) {
		edit("PKGBUILD")
	}
}

func getPackage(app string) {
	url := fmt.Sprintf(urlBase, app)
	cmd := exec.Command("git", "clone", url)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		os.Exit(1)
	}
}

func launchRequest(search string, withHeader bool) []byte {
	client := &http.Client{}
	req, err := http.NewRequest("GET", search, nil)
	if err != nil {
		return []byte{}
	}
	if withHeader {
		req.Header.Add(searchHeadType, searchHead)
	}
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println(err)
	}
	b, err := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()
	return b
}

func getVersion(result []byte) string {
	var pkgver, pkgrel string
	l := strings.Split(string(result), "\n")
	for _, e := range l {
		e := strings.TrimSpace(e)
		if len(e) <= 7 {
			continue
		}
		if e[:7] == "pkgver=" {
			pkgver = e[7:]
		} else if e[:7] == "pkgrel=" {
			pkgrel = e[7:]
		}
	}
	if pkgver != "" && pkgrel != "" {
		return pkgver + "-" + pkgrel
	}
	return "<unknown>"
}

func checkInstalled(app string) string {
	var localVersion string
	cmd := exec.Command("pacman", "-Q", app)
	if out, err := cmd.Output(); err == nil {
		o := strings.Fields(string(out))
		if len(o) >= 2 {
			localVersion = o[1]
		}
	}
	return localVersion
}

func searchPackage(app string, fast bool) {
	search := fmt.Sprintf(searchBase, app)
	var f interface{}
	if err := json.Unmarshal(launchRequest(search, true), &f); err != nil {
		return
	}
	j := f.(map[string]interface{})["items"]
	result := j.([]interface{})
	for _, a := range result {
		e := a.(map[string]interface{})
		n, d, s := e["name"], e["description"], e["stargazers_count"]
		i := checkInstalled(fmt.Sprintf("%v", n))
		if fast {
			if i != "" {
				fmt.Printf("\033[1m%v\033[m \033[1;36m[installed: %v]\033[m \033[1;34m(%v)\033[m\n", n, i, s)
			} else {
				fmt.Printf("\033[1m%v\033[m\033 \033[1;34m(%v)\033[m\n", n, s)
			}
		} else {
			v := string(getVersion(launchRequest(fmt.Sprintf(urlPkgbuild, n), false)))
			if i != "" {
				if v == i {
					i = " [installed]"
				} else {
					i = fmt.Sprintf(" [installed: %v]", i)
				}
			}
			fmt.Printf("\033[1m%v\033[m \033[1;32m%v\033[m\033[1;36m%v\033[m \033[1;34m(%v)\033[m\n", n, v, i, s)
		}
		fmt.Println("\t", d)
	}
}

func installPackage(app string, asdeps bool) {
	os.Chdir(tmpDir)
	lck := tmpDir + string(os.PathSeparator) + kcp_lock
	_, e := os.Open(lck)
	if e == nil {
		fmt.Println("\033[1;31mAnother instance is running!\033[m")
		os.Exit(1)
	}
	os.Create(lck)
	wDir := tmpDir + string(os.PathSeparator) + app
	end := func() {
		os.RemoveAll(wDir)
		os.Remove(lck)
	}
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		end()
		os.Exit(1)
	}()
	getPackage(app)
	defer end()
	if err := os.Chdir(wDir); err != nil {
		fmt.Println(err)
		end()
		os.Exit(1)
	}
	editPkgbuild()
	var cmd *exec.Cmd
	if asdeps {
		cmd = exec.Command("makepkg", "-si", "--asdeps")
	} else {
		cmd = exec.Command("makepkg", "-si")
	}
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Run()
}

func main() {
	checkUser()
	err := p.Parse(os.Args)
	switch {
	case err != nil:
		fmt.Println(err)
		p.PrintHelp()
	case *fHelp:
		p.PrintHelp()
	case *fVersion:
		p.PrintVersion()
	case *fGet != "":
		getPackage(*fGet)
	case *fSearch != "":
		searchPackage(*fSearch, *fFast)
	case *fInstall != "":
		installPackage(*fInstall, *fDeps)
	default:
		p.PrintHelp()
	}
}
<|endoftext|>


<commit_before>// perf-ingest listens to a PubSub Topic for new files that appear
// in a storage bucket and then ingests those files into BigTable.
package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"strings"
	"time"

	"cloud.google.com/go/bigtable"
	"cloud.google.com/go/pubsub"
	"cloud.google.com/go/storage"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/git/gitinfo"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/metrics2"
	"go.skia.org/infra/go/paramtools"
	"go.skia.org/infra/go/query"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/go/vcsinfo"
	"go.skia.org/infra/perf/go/btts"
	"go.skia.org/infra/perf/go/config"
	"go.skia.org/infra/perf/go/ingestcommon"
	"google.golang.org/api/option"
)

// flags
var (
	configName = flag.String("config_name", "nano", "Name of the perf ingester config to use.")
	local      = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
	port       = flag.String("port", ":8000", "HTTP service address (e.g., ':8000')")
	promPort   = flag.String("prom_port", ":20000", "Metrics service address (e.g., ':10110')")
)

const (
	// MAX_PARALLEL_RECEIVES is the number of Go routines we want to run. Determined experimentally.
	MAX_PARALLEL_RECEIVES = 1
)

var (
	// NonRecoverableError is returned if the error is non-recoverable and we
	// should Ack the PubSub message. This might happen if, for example, a
	// non-JSON file gets dropped in the bucket.
	NonRecoverableError = errors.New("Non-recoverable ingestion error.")
)

// getParamsAndValues returns two parallel slices, each slice contains the
// params and then the float for a single value of a trace. It also returns the
// consolidated ParamSet built from all the Params.
func getParamsAndValues(b *ingestcommon.BenchData) ([]paramtools.Params, []float32, paramtools.ParamSet) {
	params := []paramtools.Params{}
	values := []float32{}
	ps := paramtools.ParamSet{}
	for testName, allConfigs := range b.Results {
		for configName, result := range allConfigs {
			key := paramtools.Params(b.Key).Copy()
			key["test"] = testName
			key["config"] = configName
			key.Add(paramtools.Params(b.Options))

			// If there is an options map inside the result add it to the params.
			if resultOptions, ok := result["options"]; ok {
				if opts, ok := resultOptions.(map[string]interface{}); ok {
					for k, vi := range opts {
						// Ignore the very long and not useful GL_ values, we can retrieve
						// them later via ptracestore.Details.
						if strings.HasPrefix(k, "GL_") {
							continue
						}
						if s, ok := vi.(string); ok {
							key[k] = s
						}
					}
				}
			}

			for k, vi := range result {
				if k == "options" || k == "samples" {
					continue
				}
				key["sub_result"] = k
				floatVal, ok := vi.(float64)
				if !ok {
					sklog.Errorf("Found a non-float64 in %v", result)
					continue
				}

				key = query.ForceValid(key)
				params = append(params, key.Copy())
				values = append(values, float32(floatVal))
				ps.AddParams(key)
			}
		}
	}
	ps.Normalize()
	return params, values, ps
}

// processSingleFile parses the contents of a single JSON file and writes the values into BigTable.
func processSingleFile(ctx context.Context, store *btts.BigTableTraceStore, vcs vcsinfo.VCS, name string, r io.Reader, timestamp time.Time) error {
	benchData, err := ingestcommon.ParseBenchDataFromReader(r)
	if err != nil {
		sklog.Errorf("Failed to read or parse data: %s", err)
		return NonRecoverableError
	}

	params, values, paramset := getParamsAndValues(benchData)
	index, err := vcs.IndexOf(ctx, benchData.Hash)
	if err != nil {
		if err := vcs.Update(context.Background(), true, false); err != nil {
			return fmt.Errorf("Could not ingest, failed to pull: %s", err)
		}
		index, err = vcs.IndexOf(ctx, benchData.Hash)
		if err != nil {
			sklog.Errorf("Could not ingest, hash not found even after pulling %q: %s", benchData.Hash, err)
			return NonRecoverableError
		}
	}
	tileKey := store.TileKey(int32(index))
	ops, err := store.UpdateOrderedParamSet(tileKey, paramset)
	if err != nil {
		return fmt.Errorf("Could not ingest, failed to update OPS: %s", err)
	}
	encoded := map[string]float32{}
	for i, p := range params {
		key, err := ops.EncodeParamsAsString(p)
		if err != nil {
			sklog.Errorf("Could not ingest, failed OPS encoding: %s", err)
			return NonRecoverableError
		}
		encoded[key] = values[i]
	}
	return store.WriteTraces(int32(index), encoded, name, timestamp)
}

// Event is used to deserialize the PubSub data.
//
// The PubSub event data is a JSON serialized storage.ObjectAttrs object.
// See https://cloud.google.com/storage/docs/pubsub-notifications#payload
type Event struct {
	Bucket string `json:"bucket"`
	Name   string `json:"name"`
}

func main() {
	common.InitWithMust(
		"perf-ingest",
		common.PrometheusOpt(promPort),
		common.MetricsLoggingOpt(),
	)

	// nackCounter is the number files we weren't able to ingest.
	nackCounter := metrics2.GetCounter("nack", nil)

	ctx := context.Background()
	cfg, ok := config.PERF_BIGTABLE_CONFIGS[*configName]
	if !ok {
		sklog.Fatalf("Invalid --config value: %q", *configName)
	}
	hostname, err := os.Hostname()
	if err != nil {
		sklog.Fatalf("Failed to get hostname: %s", err)
	}
	ts, err := auth.NewDefaultTokenSource(*local, bigtable.Scope, storage.ScopeReadOnly, pubsub.ScopePubSub)
	if err != nil {
		sklog.Fatalf("Failed to create TokenSource: %s", err)
	}

	client := httputils.DefaultClientConfig().WithTokenSource(ts).WithoutRetries().Client()
	gcsClient, err := storage.NewClient(ctx, option.WithHTTPClient(client))
	if err != nil {
		sklog.Fatalf("Failed to create GCS client: %s", err)
	}
	pubSubClient, err := pubsub.NewClient(ctx, cfg.Project, option.WithTokenSource(ts))
	if err != nil {
		sklog.Fatal(err)
	}

	// When running in production we have every instance use the same topic name so that
	// they load-balance pulling items from the topic.
	subName := fmt.Sprintf("%s-%s", cfg.Topic, "prod")
	if *local {
		// When running locally create a new topic for every host.
		subName = fmt.Sprintf("%s-%s", cfg.Topic, hostname)
	}
	sub := pubSubClient.Subscription(subName)
	ok, err = sub.Exists(ctx)
	if err != nil {
		sklog.Fatalf("Failed checking subscription existence: %s", err)
	}
	if !ok {
		sub, err = pubSubClient.CreateSubscription(ctx, subName, pubsub.SubscriptionConfig{
			Topic: pubSubClient.Topic(cfg.Topic),
		})
		if err != nil {
			sklog.Fatalf("Failed creating subscription: %s", err)
		}
	}

	// How many Go routines should be processing messages?
	sub.ReceiveSettings.MaxOutstandingMessages = MAX_PARALLEL_RECEIVES
	sub.ReceiveSettings.NumGoroutines = MAX_PARALLEL_RECEIVES

	vcs, err := gitinfo.CloneOrUpdate(ctx, cfg.GitUrl, "/tmp/skia_ingest_checkout", true)
	if err != nil {
		sklog.Fatal(err)
	}

	store, err := btts.NewBigTableTraceStoreFromConfig(ctx, cfg, ts, true)
	if err != nil {
		sklog.Fatal(err)
	}

	// Process all incoming PubSub requests.
	go func() {
		for {
			// Wait for PubSub events.
			err := sub.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) {
				// Set success to true if we should Ack the PubSub message, otherwise
				// the message will be Nack'd, and PubSub will try to send the message
				// again.
				success := false
				defer func() {
					if success {
						msg.Ack()
					} else {
						nackCounter.Inc(1)
						msg.Nack()
					}
				}()
				// Decode the event, which is a GCS event that a file was written.
				var event Event
				if err := json.Unmarshal(msg.Data, &event); err != nil {
					sklog.Error(err)
					return
				}
				// Load the file.
				obj := gcsClient.Bucket(event.Bucket).Object(event.Name)
				attrs, err := obj.Attrs(ctx)
				if err != nil {
					sklog.Error(err)
					return
				}
				reader, err := obj.NewReader(ctx)
				if err != nil {
					sklog.Error(err)
					return
				}
				defer util.Close(reader)
				sklog.Info(attrs.Name)
				// Pull data out of file and write it into BigTable.
				fullName := fmt.Sprintf("gs://%s/%s", event.Bucket, event.Name)
				err = processSingleFile(ctx, store, vcs, fullName, reader, attrs.Created)
				if err := reader.Close(); err != nil {
					sklog.Errorf("Failed to close: %s", err)
				}
				if err == NonRecoverableError {
					success = true
				} else if err != nil {
					sklog.Errorf("Failed to write results: %s", err)
					return
				}
				success = true
			})
			if err != nil {
				sklog.Errorf("Failed receiving pubsub message: %s", err)
			}
		}
	}()

	// Set up the http handler to indicate ready-ness and start serving.
	http.HandleFunc("/ready", httputils.ReadyHandleFunc)
	log.Fatal(http.ListenAndServe(*port, nil))
}
<commit_msg>perf-ingest - Cache the results from git.<commit_after>// perf-ingest listens to a PubSub Topic for new files that appear
// in a storage bucket and then ingests those files into BigTable.
package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"strings"
	"sync"
	"time"

	"cloud.google.com/go/bigtable"
	"cloud.google.com/go/pubsub"
	"cloud.google.com/go/storage"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/git/gitinfo"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/metrics2"
	"go.skia.org/infra/go/paramtools"
	"go.skia.org/infra/go/query"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/go/vcsinfo"
	"go.skia.org/infra/perf/go/btts"
	"go.skia.org/infra/perf/go/config"
	"go.skia.org/infra/perf/go/ingestcommon"
	"google.golang.org/api/option"
)

// flags
var (
	configName = flag.String("config_name", "nano", "Name of the perf ingester config to use.")
	local      = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
	port       = flag.String("port", ":8000", "HTTP service address (e.g., ':8000')")
	promPort   = flag.String("prom_port", ":20000", "Metrics service address (e.g., ':10110')")
)

const (
	// MAX_PARALLEL_RECEIVES is the number of Go routines we want to run. Determined experimentally.
	MAX_PARALLEL_RECEIVES = 1
)

var (
	// mutex protects hashCache.
	mutex = sync.Mutex{}
	// hashCache is a cache of results from calling vcs.IndexOf().
	hashCache = map[string]int{}
)

var (
	// NonRecoverableError is returned if the error is non-recoverable and we
	// should Ack the PubSub message. This might happen if, for example, a
	// non-JSON file gets dropped in the bucket.
	NonRecoverableError = errors.New("Non-recoverable ingestion error.")
)

// getParamsAndValues returns two parallel slices, each slice contains the
// params and then the float for a single value of a trace. It also returns the
// consolidated ParamSet built from all the Params.
func getParamsAndValues(b *ingestcommon.BenchData) ([]paramtools.Params, []float32, paramtools.ParamSet) {
	params := []paramtools.Params{}
	values := []float32{}
	ps := paramtools.ParamSet{}
	for testName, allConfigs := range b.Results {
		for configName, result := range allConfigs {
			key := paramtools.Params(b.Key).Copy()
			key["test"] = testName
			key["config"] = configName
			key.Add(paramtools.Params(b.Options))

			// If there is an options map inside the result add it to the params.
			if resultOptions, ok := result["options"]; ok {
				if opts, ok := resultOptions.(map[string]interface{}); ok {
					for k, vi := range opts {
						// Ignore the very long and not useful GL_ values, we can retrieve
						// them later via ptracestore.Details.
						if strings.HasPrefix(k, "GL_") {
							continue
						}
						if s, ok := vi.(string); ok {
							key[k] = s
						}
					}
				}
			}

			for k, vi := range result {
				if k == "options" || k == "samples" {
					continue
				}
				key["sub_result"] = k
				floatVal, ok := vi.(float64)
				if !ok {
					sklog.Errorf("Found a non-float64 in %v", result)
					continue
				}

				key = query.ForceValid(key)
				params = append(params, key.Copy())
				values = append(values, float32(floatVal))
				ps.AddParams(key)
			}
		}
	}
	ps.Normalize()
	return params, values, ps
}

func indexFromCache(hash string) (int, bool) {
	mutex.Lock()
	defer mutex.Unlock()

	index, ok := hashCache[hash]
	return index, ok
}

func indexToCache(hash string, index int) {
	mutex.Lock()
	defer mutex.Unlock()

	hashCache[hash] = index
}

// processSingleFile parses the contents of a single JSON file and writes the values into BigTable.
func processSingleFile(ctx context.Context, store *btts.BigTableTraceStore, vcs vcsinfo.VCS, name string, r io.Reader, timestamp time.Time) error {
	benchData, err := ingestcommon.ParseBenchDataFromReader(r)
	if err != nil {
		sklog.Errorf("Failed to read or parse data: %s", err)
		return NonRecoverableError
	}

	params, values, paramset := getParamsAndValues(benchData)
	index, ok := indexFromCache(benchData.Hash)
	if !ok {
		var err error
		index, err = vcs.IndexOf(ctx, benchData.Hash)
		if err != nil {
			if err := vcs.Update(context.Background(), true, false); err != nil {
				return fmt.Errorf("Could not ingest, failed to pull: %s", err)
			}
			index, err = vcs.IndexOf(ctx, benchData.Hash)
			if err != nil {
				sklog.Errorf("Could not ingest, hash not found even after pulling %q: %s", benchData.Hash, err)
				return NonRecoverableError
			}
		}
		indexToCache(benchData.Hash, index)
	}
	tileKey := store.TileKey(int32(index))
	ops, err := store.UpdateOrderedParamSet(tileKey, paramset)
	if err != nil {
		return fmt.Errorf("Could not ingest, failed to update OPS: %s", err)
	}
	encoded := map[string]float32{}
	for i, p := range params {
		key, err := ops.EncodeParamsAsString(p)
		if err != nil {
			sklog.Errorf("Could not ingest, failed OPS encoding: %s", err)
			return NonRecoverableError
		}
		encoded[key] = values[i]
	}
	return store.WriteTraces(int32(index), encoded, name, timestamp)
}

// Event is used to deserialize the PubSub data.
//
// The PubSub event data is a JSON serialized storage.ObjectAttrs object.
// See https://cloud.google.com/storage/docs/pubsub-notifications#payload
type Event struct {
	Bucket string `json:"bucket"`
	Name   string `json:"name"`
}

func main() {
	common.InitWithMust(
		"perf-ingest",
		common.PrometheusOpt(promPort),
		common.MetricsLoggingOpt(),
	)

	// nackCounter is the number files we weren't able to ingest.
	nackCounter := metrics2.GetCounter("nack", nil)

	ctx := context.Background()
	cfg, ok := config.PERF_BIGTABLE_CONFIGS[*configName]
	if !ok {
		sklog.Fatalf("Invalid --config value: %q", *configName)
	}
	hostname, err := os.Hostname()
	if err != nil {
		sklog.Fatalf("Failed to get hostname: %s", err)
	}
	ts, err := auth.NewDefaultTokenSource(*local, bigtable.Scope, storage.ScopeReadOnly, pubsub.ScopePubSub)
	if err != nil {
		sklog.Fatalf("Failed to create TokenSource: %s", err)
	}

	client := httputils.DefaultClientConfig().WithTokenSource(ts).WithoutRetries().Client()
	gcsClient, err := storage.NewClient(ctx, option.WithHTTPClient(client))
	if err != nil {
		sklog.Fatalf("Failed to create GCS client: %s", err)
	}
	pubSubClient, err := pubsub.NewClient(ctx, cfg.Project, option.WithTokenSource(ts))
	if err != nil {
		sklog.Fatal(err)
	}

	// When running in production we have every instance use the same topic name so that
	// they load-balance pulling items from the topic.
	subName := fmt.Sprintf("%s-%s", cfg.Topic, "prod")
	if *local {
		// When running locally create a new topic for every host.
		subName = fmt.Sprintf("%s-%s", cfg.Topic, hostname)
	}
	sub := pubSubClient.Subscription(subName)
	ok, err = sub.Exists(ctx)
	if err != nil {
		sklog.Fatalf("Failed checking subscription existence: %s", err)
	}
	if !ok {
		sub, err = pubSubClient.CreateSubscription(ctx, subName, pubsub.SubscriptionConfig{
			Topic: pubSubClient.Topic(cfg.Topic),
		})
		if err != nil {
			sklog.Fatalf("Failed creating subscription: %s", err)
		}
	}

	// How many Go routines should be processing messages?
	sub.ReceiveSettings.MaxOutstandingMessages = MAX_PARALLEL_RECEIVES
	sub.ReceiveSettings.NumGoroutines = MAX_PARALLEL_RECEIVES

	vcs, err := gitinfo.CloneOrUpdate(ctx, cfg.GitUrl, "/tmp/skia_ingest_checkout", true)
	if err != nil {
		sklog.Fatal(err)
	}

	store, err := btts.NewBigTableTraceStoreFromConfig(ctx, cfg, ts, true)
	if err != nil {
		sklog.Fatal(err)
	}

	// Process all incoming PubSub requests.
	go func() {
		for {
			// Wait for PubSub events.
			err := sub.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) {
				// Set success to true if we should Ack the PubSub message, otherwise
				// the message will be Nack'd, and PubSub will try to send the message
				// again.
				success := false
				defer func() {
					if success {
						msg.Ack()
					} else {
						nackCounter.Inc(1)
						msg.Nack()
					}
				}()
				// Decode the event, which is a GCS event that a file was written.
				var event Event
				if err := json.Unmarshal(msg.Data, &event); err != nil {
					sklog.Error(err)
					return
				}
				// Load the file.
				obj := gcsClient.Bucket(event.Bucket).Object(event.Name)
				attrs, err := obj.Attrs(ctx)
				if err != nil {
					sklog.Error(err)
					return
				}
				reader, err := obj.NewReader(ctx)
				if err != nil {
					sklog.Error(err)
					return
				}
				defer util.Close(reader)
				sklog.Info(attrs.Name)
				// Pull data out of file and write it into BigTable.
				fullName := fmt.Sprintf("gs://%s/%s", event.Bucket, event.Name)
				err = processSingleFile(ctx, store, vcs, fullName, reader, attrs.Created)
				if err := reader.Close(); err != nil {
					sklog.Errorf("Failed to close: %s", err)
				}
				if err == NonRecoverableError {
					success = true
				} else if err != nil {
					sklog.Errorf("Failed to write results: %s", err)
					return
				}
				success = true
			})
			if err != nil {
				sklog.Errorf("Failed receiving pubsub message: %s", err)
			}
		}
	}()

	// Set up the http handler to indicate ready-ness and start serving.
	http.HandleFunc("/ready", httputils.ReadyHandleFunc)
	log.Fatal(http.ListenAndServe(*port, nil))
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"io"
	"sort"

	"k8s.io/kubernetes/pkg/client/restclient"
	kclientcmd "k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
	clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api"
	kcmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"

	"github.com/openshift/origin/pkg/client"
	cliconfig "github.com/openshift/origin/pkg/cmd/cli/config"
	"github.com/openshift/origin/pkg/cmd/util/clientcmd"
	"github.com/openshift/origin/pkg/project/api"

	"github.com/spf13/cobra"
)

type ProjectsOptions struct {
	Config       clientcmdapi.Config
	ClientConfig *restclient.Config
	Client       *client.Client
	Out          io.Writer
	PathOptions  *kclientcmd.PathOptions

	// internal strings
	CommandName string

	DisplayShort bool
}

// SortByProjectName is sort
type SortByProjectName []api.Project

func (p SortByProjectName) Len() int {
	return len(p)
}
func (p SortByProjectName) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}
func (p SortByProjectName) Less(i, j int) bool {
	return p[i].Name < p[j].Name
}

const (
	projectsLong = `
Display information about the current active project and existing projects on the server.

For advanced configuration, or to manage the contents of your config file, use the 'config'
command.`
)

// NewCmdProjects implements the OpenShift cli rollback command
func NewCmdProjects(fullName string, f *clientcmd.Factory, out io.Writer) *cobra.Command {
	options := &ProjectsOptions{}

	cmd := &cobra.Command{
		Use:   "projects",
		Short: "Display existing projects",
		Long:  projectsLong,
		Run: func(cmd *cobra.Command, args []string) {
			options.PathOptions = cliconfig.NewPathOptions(cmd)

			if err := options.Complete(f, args, fullName, out); err != nil {
				kcmdutil.CheckErr(kcmdutil.UsageError(cmd, err.Error()))
			}

			if err := options.RunProjects(); err != nil {
				kcmdutil.CheckErr(err)
			}
		},
	}

	cmd.Flags().BoolVarP(&options.DisplayShort, "short", "q", false, "If true, display only the project names")
	return cmd
}

func (o *ProjectsOptions) Complete(f *clientcmd.Factory, args []string, commandName string, out io.Writer) error {
	if len(args) > 0 {
		return fmt.Errorf("no arguments should be passed")
	}

	o.CommandName = commandName

	var err error
	o.Config, err = f.OpenShiftClientConfig.RawConfig()
	if err != nil {
		return err
	}

	o.ClientConfig, err = f.OpenShiftClientConfig.ClientConfig()
	if err != nil {
		return err
	}

	o.Client, _, err = f.Clients()
	if err != nil {
		return err
	}

	o.Out = out

	return nil
}

// RunProjects lists all projects a user belongs to
func (o ProjectsOptions) RunProjects() error {
	config := o.Config
	clientCfg := o.ClientConfig
	out := o.Out

	currentContext := config.Contexts[config.CurrentContext]
	currentProject := currentContext.Namespace

	var currentProjectExists bool
	var currentProjectErr error

	client := o.Client

	if len(currentProject) > 0 {
		if _, currentProjectErr := client.Projects().Get(currentProject); currentProjectErr == nil {
			currentProjectExists = true
		}
	}

	defaultContextName := cliconfig.GetContextNickname(currentContext.Namespace, currentContext.Cluster, currentContext.AuthInfo)

	var msg string
	projects, err := getProjects(client)
	if err == nil {
		switch len(projects) {
		case 0:
			msg += "You are not a member of any projects. You can request a project to be created with the 'new-project' command."
		case 1:
			if o.DisplayShort {
				msg += fmt.Sprintf("%s", api.DisplayNameAndNameForProject(&projects[0]))
			} else {
				msg += fmt.Sprintf("You have one project on this server: %q.", api.DisplayNameAndNameForProject(&projects[0]))
			}
		default:
			asterisk := ""
			count := 0
			if !o.DisplayShort {
				msg += fmt.Sprintf("You have access to the following projects and can switch between them with '%s project <projectname>':\n", o.CommandName)
			}

			sort.Sort(SortByProjectName(projects))
			for _, project := range projects {
				count = count + 1
				displayName := project.Annotations["openshift.io/display-name"]
				linebreak := "\n"
				if len(displayName) == 0 {
					displayName = project.Annotations["displayName"]
				}

				if currentProjectExists && !o.DisplayShort {
					asterisk = "    "
					if currentProject == project.Name {
						asterisk = "  * "
					}
				}
				if len(displayName) > 0 && displayName != project.Name && !o.DisplayShort {
					msg += fmt.Sprintf("\n"+asterisk+"%s - %s", project.Name, displayName)
				} else {
					if o.DisplayShort && count == 1 {
						linebreak = ""
					}
					msg += fmt.Sprintf(linebreak+asterisk+"%s", project.Name)
				}
			}
		}
		fmt.Println(msg)

		if len(projects) > 0 && !o.DisplayShort {
			if !currentProjectExists {
				if clientcmd.IsForbidden(currentProjectErr) {
					fmt.Printf("you do not have rights to view project %q. Please switch to an existing one.", currentProject)
				}
				return currentProjectErr
			}

			// if they specified a project name and got a generated context, then only show the information they care about.  They won't recognize
			// a context name they didn't choose
			if config.CurrentContext == defaultContextName {
				fmt.Fprintf(out, "\nUsing project %q on server %q.\n", currentProject, clientCfg.Host)
			} else {
				fmt.Fprintf(out, "\nUsing project %q from context named %q on server %q.\n", currentProject, config.CurrentContext, clientCfg.Host)
			}
		}
		return nil
	}

	return err
}
<commit_msg>It is better to add "\n" in printf<commit_after>package cmd

import (
	"fmt"
	"io"
	"sort"

	"k8s.io/kubernetes/pkg/client/restclient"
	kclientcmd "k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
	clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api"
	kcmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"

	"github.com/openshift/origin/pkg/client"
	cliconfig "github.com/openshift/origin/pkg/cmd/cli/config"
	"github.com/openshift/origin/pkg/cmd/util/clientcmd"
	"github.com/openshift/origin/pkg/project/api"

	"github.com/spf13/cobra"
)

type ProjectsOptions struct {
	Config       clientcmdapi.Config
	ClientConfig *restclient.Config
	Client       *client.Client
	Out          io.Writer
	PathOptions  *kclientcmd.PathOptions

	// internal strings
	CommandName string

	DisplayShort bool
}

// SortByProjectName is sort
type SortByProjectName []api.Project

func (p SortByProjectName) Len() int {
	return len(p)
}
func (p SortByProjectName) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}
func (p SortByProjectName) Less(i, j int) bool {
	return p[i].Name < p[j].Name
}

const (
	projectsLong = `
Display information about the current active project and existing projects on the server.

For advanced configuration, or to manage the contents of your config file, use the 'config'
command.`
)

// NewCmdProjects implements the OpenShift cli rollback command
func NewCmdProjects(fullName string, f *clientcmd.Factory, out io.Writer) *cobra.Command {
	options := &ProjectsOptions{}

	cmd := &cobra.Command{
		Use:   "projects",
		Short: "Display existing projects",
		Long:  projectsLong,
		Run: func(cmd *cobra.Command, args []string) {
			options.PathOptions = cliconfig.NewPathOptions(cmd)

			if err := options.Complete(f, args, fullName, out); err != nil {
				kcmdutil.CheckErr(kcmdutil.UsageError(cmd, err.Error()))
			}

			if err := options.RunProjects(); err != nil {
				kcmdutil.CheckErr(err)
			}
		},
	}

	cmd.Flags().BoolVarP(&options.DisplayShort, "short", "q", false, "If true, display only the project names")
	return cmd
}

func (o *ProjectsOptions) Complete(f *clientcmd.Factory, args []string, commandName string, out io.Writer) error {
	if len(args) > 0 {
		return fmt.Errorf("no arguments should be passed")
	}

	o.CommandName = commandName

	var err error
	o.Config, err = f.OpenShiftClientConfig.RawConfig()
	if err != nil {
		return err
	}

	o.ClientConfig, err = f.OpenShiftClientConfig.ClientConfig()
	if err != nil {
		return err
	}

	o.Client, _, err = f.Clients()
	if err != nil {
		return err
	}

	o.Out = out

	return nil
}

// RunProjects lists all projects a user belongs to
func (o ProjectsOptions) RunProjects() error {
	config := o.Config
	clientCfg := o.ClientConfig
	out := o.Out

	currentContext := config.Contexts[config.CurrentContext]
	currentProject := currentContext.Namespace

	var currentProjectExists bool
	var currentProjectErr error

	client := o.Client

	if len(currentProject) > 0 {
		if _, currentProjectErr := client.Projects().Get(currentProject); currentProjectErr == nil {
			currentProjectExists = true
		}
	}

	defaultContextName := cliconfig.GetContextNickname(currentContext.Namespace, currentContext.Cluster, currentContext.AuthInfo)

	var msg string
	projects, err := getProjects(client)
	if err == nil {
		switch len(projects) {
		case 0:
			msg += "You are not a member of any projects. You can request a project to be created with the 'new-project' command."
		case 1:
			if o.DisplayShort {
				msg += fmt.Sprintf("%s", api.DisplayNameAndNameForProject(&projects[0]))
			} else {
				msg += fmt.Sprintf("You have one project on this server: %q.", api.DisplayNameAndNameForProject(&projects[0]))
			}
		default:
			asterisk := ""
			count := 0
			if !o.DisplayShort {
				msg += fmt.Sprintf("You have access to the following projects and can switch between them with '%s project <projectname>':\n", o.CommandName)
			}

			sort.Sort(SortByProjectName(projects))
			for _, project := range projects {
				count = count + 1
				displayName := project.Annotations["openshift.io/display-name"]
				linebreak := "\n"
				if len(displayName) == 0 {
					displayName = project.Annotations["displayName"]
				}

				if currentProjectExists && !o.DisplayShort {
					asterisk = "    "
					if currentProject == project.Name {
						asterisk = "  * "
					}
				}
				if len(displayName) > 0 && displayName != project.Name && !o.DisplayShort {
					msg += fmt.Sprintf("\n"+asterisk+"%s - %s", project.Name, displayName)
				} else {
					if o.DisplayShort && count == 1 {
						linebreak = ""
					}
					msg += fmt.Sprintf(linebreak+asterisk+"%s", project.Name)
				}
			}
		}
		fmt.Println(msg)

		if len(projects) > 0 && !o.DisplayShort {
			if !currentProjectExists {
				if clientcmd.IsForbidden(currentProjectErr) {
					fmt.Printf("You do not have rights to view project %q. Please switch to an existing one.\n", currentProject)
				}
				return currentProjectErr
			}

			// if they specified a project name and got a generated context, then only show the information they care about.  They won't recognize
			// a context name they didn't choose
			if config.CurrentContext == defaultContextName {
				fmt.Fprintf(out, "\nUsing project %q on server %q.\n", currentProject, clientCfg.Host)
			} else {
				fmt.Fprintf(out, "\nUsing project %q from context named %q on server %q.\n", currentProject, config.CurrentContext, clientCfg.Host)
			}
		}
		return nil
	}

	return err
}
<|endoftext|>


<commit_before>package handler

import (
	"encoding/json"
	"net/http"

	"github.com/skygeario/skygear-server/pkg/core/skyerr"
)

type RequestPayload interface {
	Validate() error
}

type EmptyRequestPayload struct{}

func (p EmptyRequestPayload) Validate() error {
	return nil
}

const BodyMaxSize = 1024 * 1024 * 10

func DecodeJSONBody(r *http.Request, w http.ResponseWriter, payload interface{}) error {
	if contentType := r.Header.Get("Content-Type"); contentType != "application/json" {
		return skyerr.NewBadRequest("request content type is invalid")
	}
	body := http.MaxBytesReader(w, r.Body, BodyMaxSize)
	defer body.Close()
	if err := json.NewDecoder(body).Decode(payload); err != nil {
		return skyerr.NewBadRequest("failed to decode the request payload")
	}
	return nil
}
<commit_msg>Add ParseJSONBody<commit_after>package handler

import (
	"encoding/json"
	"io"
	"net/http"

	"github.com/skygeario/skygear-server/pkg/core/skyerr"
)

type RequestPayload interface {
	Validate() error
}

type EmptyRequestPayload struct{}

func (p EmptyRequestPayload) Validate() error {
	return nil
}

const BodyMaxSize = 1024 * 1024 * 10

func DecodeJSONBody(r *http.Request, w http.ResponseWriter, payload interface{}) error {
	return ParseJSONBody(r, w, func(r io.Reader, p interface{}) error {
		if err := json.NewDecoder(r).Decode(payload); err != nil {
			return skyerr.NewBadRequest("failed to decode the request payload")
		}
		return nil
	}, payload)
}

func ParseJSONBody(r *http.Request, w http.ResponseWriter, parse func(io.Reader, interface{}) error, payload interface{}) error {
	if contentType := r.Header.Get("Content-Type"); contentType != "application/json" {
		return skyerr.NewBadRequest("request content type is invalid")
	}
	body := http.MaxBytesReader(w, r.Body, BodyMaxSize)
	defer body.Close()
	return parse(body, payload)
}
<|endoftext|>


<commit_before>package crawler

import (
	"io"
	"net/http"
	"os"
	"testing"

	"labix.org/v2/mgo/bson"

	"github.com/janicduplessis/resultscrawler/pkg/store"
)

type FakeClient struct {
	Data io.ReadCloser
}

func (c *FakeClient) Do(req *http.Request) (*http.Response, error) {
	return &http.Response{
		Body: c.Data,
	}, nil
}

func TestCrawlerNewResults(t *testing.T) {
	crawler := getCrawler(t, "test/results.html")
	results := crawler.Run(getTestUser())
	if len(results) <= 0 {
		t.Error("Found no results. Expected results")
	}
	for _, res := range results {
		if res.Err != nil {
			t.Errorf("Expected no errors. Found error: %s", res.Err)
		}
	}
}

func TestCrawlerErrorNoResults(t *testing.T) {
	crawler := getCrawler(t, "test/no_results.html")
	results := crawler.Run(getTestUser())
	if len(results) <= 0 {
		t.Error("Found no results. Expected results")
	}
	for _, res := range results {
		if res.Err != ErrNoResults {
			t.Errorf("Expected ErrNoResults. Found: %s", res.Err)
		}
	}
}

func TestCrawlerErrorInvalidCodeNip(t *testing.T) {
	crawler := getCrawler(t, "test/invalid_code_or_nip.html")
	results := crawler.Run(getTestUser())
	if len(results) <= 0 {
		t.Error("Found no results. Expected results")
	}
	for _, res := range results {
		if res.Err != ErrInvalidCodeNip {
			t.Errorf("Expected ErrInvalidCodeNip. Found: %s", res.Err)
		}
	}
}

func TestCrawlerErrorInvalidClassGroup(t *testing.T) {
	crawler := getCrawler(t, "test/invalid_class_or_group.html")
	results := crawler.Run(getTestUser())
	if len(results) <= 0 {
		t.Error("Found no results. Expected results")
	}
	for _, res := range results {
		if res.Err != ErrInvalidGroupClass {
			t.Errorf("Expected ErrInvalidGroupClass. Found: %s", res.Err)
		}
	}
}

func TestCrawlerErrorNotRegistered(t *testing.T) {
	crawler := getCrawler(t, "test/not_registered_for_class.html")
	results := crawler.Run(getTestUser())
	if len(results) <= 0 {
		t.Error("Found no results. Expected results")
	}
	for _, res := range results {
		if res.Err != ErrNotRegistered {
			t.Errorf("Expected ErrNotRegistered. Found: %s", res.Err)
		}
	}
}

func getTestUser() *crawlerUser {
	return &crawlerUser{
		ID:    bson.NewObjectId(),
		Code:  "aaaaaa",
		Nip:   "zzzzzzz",
		Email: "test@test.com",
		Classes: []store.Class{
			store.Class{
				Name:    "Class1",
				Group:   "20",
				Year:    "2014",
				Results: []store.Result{},
			},
		},
	}
}

func getCrawler(t *testing.T, fileToCrawl string) *Crawler {
	data, err := os.Open(fileToCrawl)
	if err != nil {
		t.Errorf("Error opening test file %s. Err: %s", fileToCrawl, err)
	}
	client := &FakeClient{
		Data: data,
	}
	return NewCrawler(client)
}

func init() {
	// Working directory is different in test so we have to fix the path of
	// the template file.
	msgTemplatePath = "../../crawler/msgtemplate.html"
}
<commit_msg>fix tests<commit_after>package crawler

import (
	"io"
	"net/http"
	"os"
	"testing"

	"labix.org/v2/mgo/bson"

	"github.com/janicduplessis/resultscrawler/pkg/api"
)

type FakeClient struct {
	Data io.ReadCloser
}

func (c *FakeClient) Do(req *http.Request) (*http.Response, error) {
	return &http.Response{
		Body: c.Data,
	}, nil
}

func TestCrawlerNewResults(t *testing.T) {
	crawler := getCrawler(t, "test/results.html")
	results := crawler.Run(getTestUser())
	if len(results) <= 0 {
		t.Error("Found no results. Expected results")
	}
	for _, res := range results {
		if res.Err != nil {
			t.Errorf("Expected no errors. Found error: %s", res.Err)
		}
	}
}

func TestCrawlerErrorNoResults(t *testing.T) {
	crawler := getCrawler(t, "test/no_results.html")
	results := crawler.Run(getTestUser())
	if len(results) <= 0 {
		t.Error("Found no results. Expected results")
	}
	for _, res := range results {
		if res.Err != ErrNoResults {
			t.Errorf("Expected ErrNoResults. Found: %s", res.Err)
		}
	}
}

func TestCrawlerErrorInvalidCodeNip(t *testing.T) {
	crawler := getCrawler(t, "test/invalid_code_or_nip.html")
	results := crawler.Run(getTestUser())
	if len(results) <= 0 {
		t.Error("Found no results. Expected results")
	}
	for _, res := range results {
		if res.Err != ErrInvalidCodeNip {
			t.Errorf("Expected ErrInvalidCodeNip. Found: %s", res.Err)
		}
	}
}

func TestCrawlerErrorInvalidClassGroup(t *testing.T) {
	crawler := getCrawler(t, "test/invalid_class_or_group.html")
	results := crawler.Run(getTestUser())
	if len(results) <= 0 {
		t.Error("Found no results. Expected results")
	}
	for _, res := range results {
		if res.Err != ErrInvalidGroupClass {
			t.Errorf("Expected ErrInvalidGroupClass. Found: %s", res.Err)
		}
	}
}

func TestCrawlerErrorNotRegistered(t *testing.T) {
	crawler := getCrawler(t, "test/not_registered_for_class.html")
	results := crawler.Run(getTestUser())
	if len(results) <= 0 {
		t.Error("Found no results. Expected results")
	}
	for _, res := range results {
		if res.Err != ErrNotRegistered {
			t.Errorf("Expected ErrNotRegistered. Found: %s", res.Err)
		}
	}
}

func getTestUser() *crawlerUser {
	return &crawlerUser{
		ID:    bson.NewObjectId().Hex(),
		Code:  "aaaaaa",
		Nip:   "zzzzzzz",
		Email: "test@test.com",
		Classes: []api.Class{
			api.Class{
				Name:    "Class1",
				Group:   "20",
				Year:    "2014",
				Results: []api.Result{},
			},
		},
	}
}

func getCrawler(t *testing.T, fileToCrawl string) *Crawler {
	data, err := os.Open(fileToCrawl)
	if err != nil {
		t.Errorf("Error opening test file %s. Err: %s", fileToCrawl, err)
	}
	client := &FakeClient{
		Data: data,
	}
	return NewCrawler(client)
}

func init() {
	// Working directory is different in test so we have to fix the path of
	// the template file.
	msgTemplatePath = "../../crawler/msgtemplate.html"
}
<|endoftext|>


<commit_before>// Copyright 2015 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package csource

import (
	"fmt"
	"math/rand"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/google/syzkaller/prog"
	_ "github.com/google/syzkaller/sys"
)

func TestGenerate(t *testing.T) {
	t.Parallel()
	for _, target := range prog.AllTargets() {
		switch target.OS {
		case "netbsd", "windows":
			continue
		}
		target := target
		t.Run(target.OS+"/"+target.Arch, func(t *testing.T) {
			if target.OS == "linux" && target.Arch == "arm" {
				// This currently fails (at least with my arm-linux-gnueabihf-gcc-4.8) with:
				// Assembler messages:
				// Error: alignment too large: 15 assumed
				t.Skip("broken")
			}
			if target.OS == "linux" && target.Arch == "386" {
				// Currently fails on travis with:
				// fatal error: asm/unistd.h: No such file or directory
				t.Skip("broken")
			}
			t.Parallel()
			testTarget(t, target)
		})
	}
}

func testTarget(t *testing.T, target *prog.Target) {
	seed := int64(time.Now().UnixNano())
	rs := rand.NewSource(seed)
	t.Logf("seed=%v", seed)
	r := rand.New(rs)
	progs := []*prog.Prog{target.GenerateSimpleProg()}
	if p := target.GenerateAllSyzProg(rs); len(p.Calls) != 0 {
		progs = append(progs, p)
	}
	if !testing.Short() {
		progs = append(progs, target.Generate(rs, 10, nil))
	}
	opts := allOptionsSingle(target.OS)
	opts = append(opts, Options{
		Threaded:  true,
		Collide:   true,
		Repeat:    true,
		Procs:     2,
		Sandbox:   "none",
		Repro:     true,
		UseTmpDir: true,
	})
	allPermutations := allOptionsPermutations(target.OS)
	if testing.Short() {
		for i := 0; i < 16; i++ {
			opts = append(opts, allPermutations[r.Intn(len(allPermutations))])
		}
	} else {
		opts = allPermutations
	}
	for pi, p := range progs {
		for opti, opts := range opts {
			p, opts := p, opts
			t.Run(fmt.Sprintf("%v/%v", pi, opti), func(t *testing.T) {
				t.Parallel()
				testOne(t, p, opts)
			})
		}
	}
}

func testOne(t *testing.T, p *prog.Prog, opts Options) {
	src, err := Write(p, opts)
	if err != nil {
		t.Logf("opts: %+v\nprogram:\n%s\n", opts, p.Serialize())
		t.Fatalf("%v", err)
	}
	bin, err := Build(p.Target, src)
	if err != nil {
		if strings.Contains(err.Error(), "no target compiler") {
			t.Skip(err)
		}
		t.Logf("opts: %+v\nprogram:\n%s\n", opts, p.Serialize())
		t.Fatalf("%v", err)
	}
	defer os.Remove(bin)
}
<commit_msg>pkg/csource: turn off 32-bit tests<commit_after>// Copyright 2015 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package csource

import (
	"fmt"
	"math/rand"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/google/syzkaller/prog"
	_ "github.com/google/syzkaller/sys"
)

func TestGenerate(t *testing.T) {
	t.Parallel()
	for _, target := range prog.AllTargets() {
		switch target.OS {
		case "netbsd", "windows":
			continue
		}
		target := target
		t.Run(target.OS+"/"+target.Arch, func(t *testing.T) {
			if target.OS == "linux" && target.Arch == "arm" {
				// This currently fails (at least with my arm-linux-gnueabihf-gcc-4.8) with:
				// Assembler messages:
				// Error: alignment too large: 15 assumed
				t.Skip("broken")
			}
			if target.OS == "linux" && target.Arch == "386" {
				// Currently fails on travis with:
				// fatal error: asm/unistd.h: No such file or directory
				t.Skip("broken")
			}
			if target.OS == "test" && target.PtrSize == 4 {
				// The same reason as linux/32.
				t.Skip("broken")
			}
			t.Parallel()
			testTarget(t, target)
		})
	}
}

func testTarget(t *testing.T, target *prog.Target) {
	seed := int64(time.Now().UnixNano())
	rs := rand.NewSource(seed)
	t.Logf("seed=%v", seed)
	r := rand.New(rs)
	progs := []*prog.Prog{target.GenerateSimpleProg()}
	if p := target.GenerateAllSyzProg(rs); len(p.Calls) != 0 {
		progs = append(progs, p)
	}
	if !testing.Short() {
		progs = append(progs, target.Generate(rs, 10, nil))
	}
	opts := allOptionsSingle(target.OS)
	opts = append(opts, Options{
		Threaded:  true,
		Collide:   true,
		Repeat:    true,
		Procs:     2,
		Sandbox:   "none",
		Repro:     true,
		UseTmpDir: true,
	})
	allPermutations := allOptionsPermutations(target.OS)
	if testing.Short() {
		for i := 0; i < 16; i++ {
			opts = append(opts, allPermutations[r.Intn(len(allPermutations))])
		}
	} else {
		opts = allPermutations
	}
	for pi, p := range progs {
		for opti, opts := range opts {
			p, opts := p, opts
			t.Run(fmt.Sprintf("%v/%v", pi, opti), func(t *testing.T) {
				t.Parallel()
				testOne(t, p, opts)
			})
		}
	}
}

func testOne(t *testing.T, p *prog.Prog, opts Options) {
	src, err := Write(p, opts)
	if err != nil {
		t.Logf("opts: %+v\nprogram:\n%s\n", opts, p.Serialize())
		t.Fatalf("%v", err)
	}
	bin, err := Build(p.Target, src)
	if err != nil {
		if strings.Contains(err.Error(), "no target compiler") {
			t.Skip(err)
		}
		t.Logf("opts: %+v\nprogram:\n%s\n", opts, p.Serialize())
		t.Fatalf("%v", err)
	}
	defer os.Remove(bin)
}
<|endoftext|>


<commit_before>// Copyright 2016-2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ipcache

import (
	"fmt"
	"net"
	"unsafe"

	"github.com/cilium/cilium/common/types"
	"github.com/cilium/cilium/pkg/bpf"
	"github.com/cilium/cilium/pkg/logging"
	"github.com/cilium/cilium/pkg/logging/logfields"

	"golang.org/x/sys/unix"
)

var log = logging.DefaultLogger.WithField(logfields.LogSubsys, "map-ipcache")

const (
	// MaxEntries is the maximum number of keys that can be present in the
	// RemoteEndpointMap.
	MaxEntries = 512000

	// Name is the canonical name for the IPCache map on the filesystem.
	Name = "cilium_ipcache"

	// maxPrefixLengths is an approximation of how many different CIDR
	// prefix lengths may be supported by the BPF datapath without causing
	// BPF code generation to exceed the verifier instruction limit.
	// It applies to Linux versions that lack support for LPM, ie < v4.11.
	//
	// This is based upon the defines in bpf/lxc_config.h, which in turn
	// are derived by building the bpf/ directory and running the script
	// test/bpf/verifier-test.sh, then adjusting the number of unique
	// prefix lengths until the script passes.
	maxPrefixLengths6 = 4
	maxPrefixLengths4 = 18
)

// Key implements the bpf.MapKey interface.
//
// Must be in sync with struct bpf_ipcache_key in <bpf/lib/eps.h>
type Key struct {
	Prefixlen uint32
	Pad1      uint16
	Pad2      uint8
	Family    uint8
	IP        types.IPv6 // represents both IPv6 and IPv4 (in the lowest four bytes)
}

// GetKeyPtr returns the unsafe pointer to the BPF key
func (k *Key) GetKeyPtr() unsafe.Pointer { return unsafe.Pointer(k) }

// NewValue returns a new empty instance of the structure representing the BPF
// map value
func (k Key) NewValue() bpf.MapValue { return &RemoteEndpointInfo{} }

func getStaticPrefixBits() uint32 {
	staticMatchSize := unsafe.Sizeof(Key{})
	staticMatchSize -= unsafe.Sizeof(Key{}.Prefixlen)
	staticMatchSize -= unsafe.Sizeof(Key{}.IP)
	return uint32(staticMatchSize) * 8
}

func (k Key) String() string {
	prefixLen := k.Prefixlen - getStaticPrefixBits()
	switch k.Family {
	case bpf.EndpointKeyIPv4:
		ipStr := net.IP(k.IP[:net.IPv4len]).String()
		return fmt.Sprintf("%s/%d", ipStr, prefixLen)
	case bpf.EndpointKeyIPv6:
		ipStr := k.IP.String()
		return fmt.Sprintf("%s/%d", ipStr, prefixLen)
	}
	return fmt.Sprintf("<unknown>")
}

// getPrefixLen determines the length that should be set inside the Key so that
// the lookup prefix is correct in the BPF map key. The specified 'prefixBits'
// indicates the number of bits in the IP that must match to match the entry in
// the BPF ipcache.
func getPrefixLen(prefixBits int) uint32 {
	return getStaticPrefixBits() + uint32(prefixBits)
}

// NewKey returns an Key based on the provided IP address and mask. The address
// family is automatically detected
func NewKey(ip net.IP, mask net.IPMask) Key {
	result := Key{}

	ones, _ := mask.Size()
	if ip4 := ip.To4(); ip4 != nil {
		if mask == nil {
			ones = net.IPv4len * 8
		}
		result.Prefixlen = getPrefixLen(ones)
		result.Family = bpf.EndpointKeyIPv4
		copy(result.IP[:], ip4)
	} else {
		if mask == nil {
			ones = net.IPv6len * 8
		}
		result.Prefixlen = getPrefixLen(ones)
		result.Family = bpf.EndpointKeyIPv6
		copy(result.IP[:], ip)
	}

	return result
}

// RemoteEndpointInfo implements the bpf.MapValue interface. It contains the
// security identity of a remote endpoint.
type RemoteEndpointInfo struct {
	SecurityIdentity uint32
	TunnelEndpoint   [4]byte
}

func (v *RemoteEndpointInfo) String() string {
	return fmt.Sprintf("%d", v.SecurityIdentity)
}

// GetValuePtr returns the unsafe pointer to the BPF value.
func (v *RemoteEndpointInfo) GetValuePtr() unsafe.Pointer { return unsafe.Pointer(v) }

// Map represents an IPCache BPF map.
type Map struct {
	bpf.Map
}

// NewMap instantiates a Map.
func NewMap(name string) *Map {
	return &Map{
		Map: *bpf.NewMap(
			name,
			bpf.BPF_MAP_TYPE_LPM_TRIE,
			int(unsafe.Sizeof(Key{})),
			int(unsafe.Sizeof(RemoteEndpointInfo{})),
			MaxEntries,
			bpf.BPF_F_NO_PREALLOC,
			func(key []byte, value []byte) (bpf.MapKey, bpf.MapValue, error) {
				k, v := Key{}, RemoteEndpointInfo{}

				if err := bpf.ConvertKeyValue(key, value, &k, &v); err != nil {
					return nil, nil, err
				}
				return &k, &v, nil
			},
		).WithCache(),
	}
}

// Delete removes a key from the ipcache BPF map
func (m *Map) Delete(k bpf.MapKey) error {
	// Older kernels do not support deletion of LPM map entries so zero out
	// the entry instead of attempting a deletion
	err, errno := m.DeleteWithErrno(k)
	if errno == unix.ENOSYS {
		return m.Update(k, &RemoteEndpointInfo{})
	}

	return err
}

// GetMaxPrefixLengths determines how many unique prefix lengths are supported
// simultaneously based on the underlying BPF map type in use.
func (m *Map) GetMaxPrefixLengths(ipv6 bool) (count int) {
	if IPCache.MapType == bpf.BPF_MAP_TYPE_LPM_TRIE {
		if ipv6 {
			return net.IPv6len*8 + 1
		} else {
			return net.IPv4len*8 + 1
		}
	}
	if ipv6 {
		return maxPrefixLengths6
	}
	return maxPrefixLengths4
}

// BackedByLPM returns true if the IPCache is backed by a proper LPM
// implementation (provided by Linux kernels 4.11 or later), false otherwise.
func BackedByLPM() bool {
	return IPCache.MapType == bpf.BPF_MAP_TYPE_LPM_TRIE
}

var (
	// IPCache is a mapping of all endpoint IPs in the cluster which this
	// Cilium agent is a part of to their corresponding security identities.
	// It is a singleton; there is only one such map per agent.
	IPCache = NewMap(Name)
)

func init() {
	err := bpf.OpenAfterMount(&IPCache.Map)
	if err != nil {
		log.WithError(err).Error("unable to open map")
	}
}

// Reopen attempts to close and re-open the IPCache map at the standard path
// on the filesystem.
func Reopen() error {
	return IPCache.Map.Reopen()
}
<commit_msg>maps/ipcache: Add mechanism to detect delete support<commit_after>// Copyright 2016-2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ipcache

import (
	"fmt"
	"net"
	"sync"
	"unsafe"

	"github.com/cilium/cilium/common/types"
	"github.com/cilium/cilium/pkg/bpf"
	"github.com/cilium/cilium/pkg/logging"
	"github.com/cilium/cilium/pkg/logging/logfields"

	"golang.org/x/sys/unix"
)

var log = logging.DefaultLogger.WithField(logfields.LogSubsys, "map-ipcache")

const (
	// MaxEntries is the maximum number of keys that can be present in the
	// RemoteEndpointMap.
	MaxEntries = 512000

	// Name is the canonical name for the IPCache map on the filesystem.
	Name = "cilium_ipcache"

	// maxPrefixLengths is an approximation of how many different CIDR
	// prefix lengths may be supported by the BPF datapath without causing
	// BPF code generation to exceed the verifier instruction limit.
	// It applies to Linux versions that lack support for LPM, ie < v4.11.
	//
	// This is based upon the defines in bpf/lxc_config.h, which in turn
	// are derived by building the bpf/ directory and running the script
	// test/bpf/verifier-test.sh, then adjusting the number of unique
	// prefix lengths until the script passes.
	maxPrefixLengths6 = 4
	maxPrefixLengths4 = 18
)

// Key implements the bpf.MapKey interface.
//
// Must be in sync with struct bpf_ipcache_key in <bpf/lib/eps.h>
type Key struct {
	Prefixlen uint32
	Pad1      uint16
	Pad2      uint8
	Family    uint8
	IP        types.IPv6 // represents both IPv6 and IPv4 (in the lowest four bytes)
}

// GetKeyPtr returns the unsafe pointer to the BPF key
func (k *Key) GetKeyPtr() unsafe.Pointer { return unsafe.Pointer(k) }

// NewValue returns a new empty instance of the structure representing the BPF
// map value
func (k Key) NewValue() bpf.MapValue { return &RemoteEndpointInfo{} }

func getStaticPrefixBits() uint32 {
	staticMatchSize := unsafe.Sizeof(Key{})
	staticMatchSize -= unsafe.Sizeof(Key{}.Prefixlen)
	staticMatchSize -= unsafe.Sizeof(Key{}.IP)
	return uint32(staticMatchSize) * 8
}

func (k Key) String() string {
	prefixLen := k.Prefixlen - getStaticPrefixBits()
	switch k.Family {
	case bpf.EndpointKeyIPv4:
		ipStr := net.IP(k.IP[:net.IPv4len]).String()
		return fmt.Sprintf("%s/%d", ipStr, prefixLen)
	case bpf.EndpointKeyIPv6:
		ipStr := k.IP.String()
		return fmt.Sprintf("%s/%d", ipStr, prefixLen)
	}
	return fmt.Sprintf("<unknown>")
}

// getPrefixLen determines the length that should be set inside the Key so that
// the lookup prefix is correct in the BPF map key. The specified 'prefixBits'
// indicates the number of bits in the IP that must match to match the entry in
// the BPF ipcache.
func getPrefixLen(prefixBits int) uint32 {
	return getStaticPrefixBits() + uint32(prefixBits)
}

// NewKey returns an Key based on the provided IP address and mask. The address
// family is automatically detected
func NewKey(ip net.IP, mask net.IPMask) Key {
	result := Key{}

	ones, _ := mask.Size()
	if ip4 := ip.To4(); ip4 != nil {
		if mask == nil {
			ones = net.IPv4len * 8
		}
		result.Prefixlen = getPrefixLen(ones)
		result.Family = bpf.EndpointKeyIPv4
		copy(result.IP[:], ip4)
	} else {
		if mask == nil {
			ones = net.IPv6len * 8
		}
		result.Prefixlen = getPrefixLen(ones)
		result.Family = bpf.EndpointKeyIPv6
		copy(result.IP[:], ip)
	}

	return result
}

// RemoteEndpointInfo implements the bpf.MapValue interface. It contains the
// security identity of a remote endpoint.
type RemoteEndpointInfo struct {
	SecurityIdentity uint32
	TunnelEndpoint   [4]byte
}

func (v *RemoteEndpointInfo) String() string {
	return fmt.Sprintf("%d", v.SecurityIdentity)
}

// GetValuePtr returns the unsafe pointer to the BPF value.
func (v *RemoteEndpointInfo) GetValuePtr() unsafe.Pointer { return unsafe.Pointer(v) }

// Map represents an IPCache BPF map.
type Map struct {
	bpf.Map

	// detectDeleteSupport is used to initialize 'supportsDelete' the first
	// time that a delete is issued from the datapath.
	detectDeleteSupport sync.Once

	// deleteSupport is set to 'true' initially, then is updated to set
	// whether the underlying kernel supports delete operations on the map
	// the first time that supportsDelete() is called.
	deleteSupport bool
}

// NewMap instantiates a Map.
func NewMap(name string) *Map {
	return &Map{
		Map: *bpf.NewMap(
			name,
			bpf.BPF_MAP_TYPE_LPM_TRIE,
			int(unsafe.Sizeof(Key{})),
			int(unsafe.Sizeof(RemoteEndpointInfo{})),
			MaxEntries,
			bpf.BPF_F_NO_PREALLOC,
			func(key []byte, value []byte) (bpf.MapKey, bpf.MapValue, error) {
				k, v := Key{}, RemoteEndpointInfo{}

				if err := bpf.ConvertKeyValue(key, value, &k, &v); err != nil {
					return nil, nil, err
				}
				return &k, &v, nil
			},
		).WithCache(),
		deleteSupport: true,
	}
}

// delete removes a key from the ipcache BPF map, and returns whether the
// kernel supports the delete operation (true) or not (false), and any error
// that may have occurred while attempting to delete the entry.
//
// If "overwrite" is true, then if delete is not supported the entry's value
// will be overwritten with zeroes to signify that it's an invalid entry.
func (m *Map) delete(k bpf.MapKey, overwrite bool) (bool, error) {
	// Older kernels do not support deletion of LPM map entries so zero out
	// the entry instead of attempting a deletion
	err, errno := m.DeleteWithErrno(k)
	if errno == unix.ENOSYS {
		if overwrite {
			return false, m.Update(k, &RemoteEndpointInfo{})
		}
		return false, nil
	}

	return true, err
}

// Delete removes a key from the ipcache BPF map
func (m *Map) Delete(k bpf.MapKey) error {
	_, err := m.delete(k, true)
	return err
}

// GetMaxPrefixLengths determines how many unique prefix lengths are supported
// simultaneously based on the underlying BPF map type in use.
func (m *Map) GetMaxPrefixLengths(ipv6 bool) (count int) {
	if IPCache.MapType == bpf.BPF_MAP_TYPE_LPM_TRIE {
		if ipv6 {
			return net.IPv6len*8 + 1
		} else {
			return net.IPv4len*8 + 1
		}
	}
	if ipv6 {
		return maxPrefixLengths6
	}
	return maxPrefixLengths4
}

func (m *Map) supportsDelete() bool {
	m.detectDeleteSupport.Do(func() {
		// Entry is invalid because IPCache needs a family specified.
		invalidEntry := &Key{}
		m.deleteSupport, _ = m.delete(invalidEntry, false)
		log.Debugf("Detected IPCache delete operation support: %t", m.deleteSupport)
		if !m.deleteSupport {
			log.Infof("Periodic IPCache map swap will occur due to lack of kernel support for LPM delete operation. Upgrade to Linux 4.15 or higher to avoid this.")
		}
	})
	return m.deleteSupport
}

// SupportsDelete determines whether the underlying kernel map type supports
// the delete operation.
func SupportsDelete() bool {
	return IPCache.supportsDelete()
}

// BackedByLPM returns true if the IPCache is backed by a proper LPM
// implementation (provided by Linux kernels 4.11 or later), false otherwise.
func BackedByLPM() bool {
	return IPCache.MapType == bpf.BPF_MAP_TYPE_LPM_TRIE
}

var (
	// IPCache is a mapping of all endpoint IPs in the cluster which this
	// Cilium agent is a part of to their corresponding security identities.
	// It is a singleton; there is only one such map per agent.
	IPCache = NewMap(Name)
)

func init() {
	err := bpf.OpenAfterMount(&IPCache.Map)
	if err != nil {
		log.WithError(err).Error("unable to open map")
	}
}

// Reopen attempts to close and re-open the IPCache map at the standard path
// on the filesystem.
func Reopen() error {
	return IPCache.Map.Reopen()
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rest

import (
	"context"
	"fmt"

	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	genericvalidation "k8s.io/apimachinery/pkg/api/validation"
	"k8s.io/apimachinery/pkg/api/validation/path"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/validation/field"
	"k8s.io/apiserver/pkg/admission"
	"k8s.io/apiserver/pkg/features"
	utilfeature "k8s.io/apiserver/pkg/util/feature"
)

// RESTUpdateStrategy defines the minimum validation, accepted input, and
// name generation behavior to update an object that follows Kubernetes
// API conventions. A resource may have many UpdateStrategies, depending on
// the call pattern in use.
type RESTUpdateStrategy interface {
	runtime.ObjectTyper
	// NamespaceScoped returns true if the object must be within a namespace.
	NamespaceScoped() bool
	// AllowCreateOnUpdate returns true if the object can be created by a PUT.
	AllowCreateOnUpdate() bool
	// PrepareForUpdate is invoked on update before validation to normalize
	// the object.  For example: remove fields that are not to be persisted,
	// sort order-insensitive list fields, etc.  This should not remove fields
	// whose presence would be considered a validation error.
	PrepareForUpdate(ctx context.Context, obj, old runtime.Object)
	// ValidateUpdate is invoked after default fields in the object have been
	// filled in before the object is persisted.  This method should not mutate
	// the object.
	ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList
	// Canonicalize allows an object to be mutated into a canonical form. This
	// ensures that code that operates on these objects can rely on the common
	// form for things like comparison.  Canonicalize is invoked after
	// validation has succeeded but before the object has been persisted.
	// This method may mutate the object.
	Canonicalize(obj runtime.Object)
	// AllowUnconditionalUpdate returns true if the object can be updated
	// unconditionally (irrespective of the latest resource version), when
	// there is no resource version specified in the object.
	AllowUnconditionalUpdate() bool
}

// TODO: add other common fields that require global validation.
func validateCommonFields(obj, old runtime.Object, strategy RESTUpdateStrategy) (field.ErrorList, error) {
	allErrs := field.ErrorList{}
	objectMeta, err := meta.Accessor(obj)
	if err != nil {
		return nil, fmt.Errorf("failed to get new object metadata: %v", err)
	}
	oldObjectMeta, err := meta.Accessor(old)
	if err != nil {
		return nil, fmt.Errorf("failed to get old object metadata: %v", err)
	}
	allErrs = append(allErrs, genericvalidation.ValidateObjectMetaAccessor(objectMeta, strategy.NamespaceScoped(), path.ValidatePathSegmentName, field.NewPath("metadata"))...)
	allErrs = append(allErrs, genericvalidation.ValidateObjectMetaAccessorUpdate(objectMeta, oldObjectMeta, field.NewPath("metadata"))...)

	return allErrs, nil
}

// BeforeUpdate ensures that common operations for all resources are performed on update. It only returns
// errors that can be converted to api.Status. It will invoke update validation with the provided existing
// and updated objects.
func BeforeUpdate(strategy RESTUpdateStrategy, ctx context.Context, obj, old runtime.Object) error {
	objectMeta, kind, kerr := objectMetaAndKind(strategy, obj)
	if kerr != nil {
		return kerr
	}
	if strategy.NamespaceScoped() {
		if !ValidNamespace(ctx, objectMeta) {
			return errors.NewBadRequest("the namespace of the provided object does not match the namespace sent on the request")
		}
	} else {
		objectMeta.SetNamespace(metav1.NamespaceNone)
	}
	// Ensure requests cannot update generation
	oldMeta, err := meta.Accessor(old)
	if err != nil {
		return err
	}
	objectMeta.SetGeneration(oldMeta.GetGeneration())

	// Ensure Initializers are not set unless the feature is enabled
	if !utilfeature.DefaultFeatureGate.Enabled(features.Initializers) {
		oldMeta.SetInitializers(nil)
		objectMeta.SetInitializers(nil)
	}

	strategy.PrepareForUpdate(ctx, obj, old)

	// ClusterName is ignored and should not be saved
	objectMeta.SetClusterName("")

	// Ensure some common fields, like UID, are validated for all resources.
	errs, err := validateCommonFields(obj, old, strategy)
	if err != nil {
		return errors.NewInternalError(err)
	}

	errs = append(errs, strategy.ValidateUpdate(ctx, obj, old)...)
	if len(errs) > 0 {
		return errors.NewInvalid(kind.GroupKind(), objectMeta.GetName(), errs)
	}

	strategy.Canonicalize(obj)

	return nil
}

// TransformFunc is a function to transform and return newObj
type TransformFunc func(ctx context.Context, newObj runtime.Object, oldObj runtime.Object) (transformedNewObj runtime.Object, err error)

// defaultUpdatedObjectInfo implements UpdatedObjectInfo
type defaultUpdatedObjectInfo struct {
	// obj is the updated object
	obj runtime.Object

	// transformers is an optional list of transforming functions that modify or
	// replace obj using information from the context, old object, or other sources.
	transformers []TransformFunc
}

// DefaultUpdatedObjectInfo returns an UpdatedObjectInfo impl based on the specified object.
func DefaultUpdatedObjectInfo(obj runtime.Object, transformers ...TransformFunc) UpdatedObjectInfo {
	return &defaultUpdatedObjectInfo{obj, transformers}
}

// Preconditions satisfies the UpdatedObjectInfo interface.
func (i *defaultUpdatedObjectInfo) Preconditions() *metav1.Preconditions {
	// Attempt to get the UID out of the object
	accessor, err := meta.Accessor(i.obj)
	if err != nil {
		// If no UID can be read, no preconditions are possible
		return nil
	}

	// If empty, no preconditions needed
	uid := accessor.GetUID()
	if len(uid) == 0 {
		return nil
	}

	return &metav1.Preconditions{UID: &uid}
}

// UpdatedObject satisfies the UpdatedObjectInfo interface.
// It returns a copy of the held obj, passed through any configured transformers.
func (i *defaultUpdatedObjectInfo) UpdatedObject(ctx context.Context, oldObj runtime.Object) (runtime.Object, error) {
	var err error
	// Start with the configured object
	newObj := i.obj

	// If the original is non-nil (might be nil if the first transformer builds the object from the oldObj), make a copy,
	// so we don't return the original. BeforeUpdate can mutate the returned object, doing things like clearing ResourceVersion.
	// If we're re-called, we need to be able to return the pristine version.
	if newObj != nil {
		newObj = newObj.DeepCopyObject()
	}

	// Allow any configured transformers to update the new object
	for _, transformer := range i.transformers {
		newObj, err = transformer(ctx, newObj, oldObj)
		if err != nil {
			return nil, err
		}
	}

	return newObj, nil
}

// wrappedUpdatedObjectInfo allows wrapping an existing objInfo and
// chaining additional transformations/checks on the result of UpdatedObject()
type wrappedUpdatedObjectInfo struct {
	// obj is the updated object
	objInfo UpdatedObjectInfo

	// transformers is an optional list of transforming functions that modify or
	// replace obj using information from the context, old object, or other sources.
	transformers []TransformFunc
}

// WrapUpdatedObjectInfo returns an UpdatedObjectInfo impl that delegates to
// the specified objInfo, then calls the passed transformers
func WrapUpdatedObjectInfo(objInfo UpdatedObjectInfo, transformers ...TransformFunc) UpdatedObjectInfo {
	return &wrappedUpdatedObjectInfo{objInfo, transformers}
}

// Preconditions satisfies the UpdatedObjectInfo interface.
func (i *wrappedUpdatedObjectInfo) Preconditions() *metav1.Preconditions {
	return i.objInfo.Preconditions()
}

// UpdatedObject satisfies the UpdatedObjectInfo interface.
// It delegates to the wrapped objInfo and passes the result through any configured transformers.
func (i *wrappedUpdatedObjectInfo) UpdatedObject(ctx context.Context, oldObj runtime.Object) (runtime.Object, error) {
	newObj, err := i.objInfo.UpdatedObject(ctx, oldObj)
	if err != nil {
		return newObj, err
	}

	// Allow any configured transformers to update the new object or error
	for _, transformer := range i.transformers {
		newObj, err = transformer(ctx, newObj, oldObj)
		if err != nil {
			return nil, err
		}
	}

	return newObj, nil
}

// AdmissionToValidateObjectUpdateFunc converts validating admission to a rest validate object update func
func AdmissionToValidateObjectUpdateFunc(admit admission.Interface, staticAttributes admission.Attributes) ValidateObjectUpdateFunc {
	validatingAdmission, ok := admit.(admission.ValidationInterface)
	if !ok {
		return func(obj, old runtime.Object) error { return nil }
	}
	return func(obj, old runtime.Object) error {
		finalAttributes := admission.NewAttributesRecord(
			obj,
			old,
			staticAttributes.GetKind(),
			staticAttributes.GetNamespace(),
			staticAttributes.GetName(),
			staticAttributes.GetResource(),
			staticAttributes.GetSubresource(),
			staticAttributes.GetOperation(),
			staticAttributes.GetUserInfo(),
		)
		if !validatingAdmission.Handles(finalAttributes.GetOperation()) {
			return nil
		}
		return validatingAdmission.Validate(finalAttributes)
	}
}
<commit_msg>Deprecate repair-malformed-updates flag, move object meta mutation into BeforeCreate<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rest

import (
	"context"
	"fmt"

	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	genericvalidation "k8s.io/apimachinery/pkg/api/validation"
	"k8s.io/apimachinery/pkg/api/validation/path"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/validation/field"
	"k8s.io/apiserver/pkg/admission"
	"k8s.io/apiserver/pkg/features"
	utilfeature "k8s.io/apiserver/pkg/util/feature"
)

// RESTUpdateStrategy defines the minimum validation, accepted input, and
// name generation behavior to update an object that follows Kubernetes
// API conventions. A resource may have many UpdateStrategies, depending on
// the call pattern in use.
type RESTUpdateStrategy interface {
	runtime.ObjectTyper
	// NamespaceScoped returns true if the object must be within a namespace.
	NamespaceScoped() bool
	// AllowCreateOnUpdate returns true if the object can be created by a PUT.
	AllowCreateOnUpdate() bool
	// PrepareForUpdate is invoked on update before validation to normalize
	// the object.  For example: remove fields that are not to be persisted,
	// sort order-insensitive list fields, etc.  This should not remove fields
	// whose presence would be considered a validation error.
	PrepareForUpdate(ctx context.Context, obj, old runtime.Object)
	// ValidateUpdate is invoked after default fields in the object have been
	// filled in before the object is persisted.  This method should not mutate
	// the object.
	ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList
	// Canonicalize allows an object to be mutated into a canonical form. This
	// ensures that code that operates on these objects can rely on the common
	// form for things like comparison.  Canonicalize is invoked after
	// validation has succeeded but before the object has been persisted.
	// This method may mutate the object.
	Canonicalize(obj runtime.Object)
	// AllowUnconditionalUpdate returns true if the object can be updated
	// unconditionally (irrespective of the latest resource version), when
	// there is no resource version specified in the object.
	AllowUnconditionalUpdate() bool
}

// TODO: add other common fields that require global validation.
func validateCommonFields(obj, old runtime.Object, strategy RESTUpdateStrategy) (field.ErrorList, error) {
	allErrs := field.ErrorList{}
	objectMeta, err := meta.Accessor(obj)
	if err != nil {
		return nil, fmt.Errorf("failed to get new object metadata: %v", err)
	}
	oldObjectMeta, err := meta.Accessor(old)
	if err != nil {
		return nil, fmt.Errorf("failed to get old object metadata: %v", err)
	}
	allErrs = append(allErrs, genericvalidation.ValidateObjectMetaAccessor(objectMeta, strategy.NamespaceScoped(), path.ValidatePathSegmentName, field.NewPath("metadata"))...)
	allErrs = append(allErrs, genericvalidation.ValidateObjectMetaAccessorUpdate(objectMeta, oldObjectMeta, field.NewPath("metadata"))...)

	return allErrs, nil
}

// BeforeUpdate ensures that common operations for all resources are performed on update. It only returns
// errors that can be converted to api.Status. It will invoke update validation with the provided existing
// and updated objects.
func BeforeUpdate(strategy RESTUpdateStrategy, ctx context.Context, obj, old runtime.Object) error {
	objectMeta, kind, kerr := objectMetaAndKind(strategy, obj)
	if kerr != nil {
		return kerr
	}
	if strategy.NamespaceScoped() {
		if !ValidNamespace(ctx, objectMeta) {
			return errors.NewBadRequest("the namespace of the provided object does not match the namespace sent on the request")
		}
	} else {
		objectMeta.SetNamespace(metav1.NamespaceNone)
	}
	// Ensure requests cannot update generation
	oldMeta, err := meta.Accessor(old)
	if err != nil {
		return err
	}
	objectMeta.SetGeneration(oldMeta.GetGeneration())

	// Ensure Initializers are not set unless the feature is enabled
	if !utilfeature.DefaultFeatureGate.Enabled(features.Initializers) {
		oldMeta.SetInitializers(nil)
		objectMeta.SetInitializers(nil)
	}

	strategy.PrepareForUpdate(ctx, obj, old)

	// ClusterName is ignored and should not be saved
	objectMeta.SetClusterName("")
	// Use the existing UID if none is provided
	if len(objectMeta.GetUID()) == 0 {
		objectMeta.SetUID(oldMeta.GetUID())
	}
	// ignore changes to timestamp
	if oldCreationTime := oldMeta.GetCreationTimestamp(); !oldCreationTime.IsZero() {
		objectMeta.SetCreationTimestamp(oldMeta.GetCreationTimestamp())
	}
	// an update can never remove/change a deletion timestamp
	if !oldMeta.GetDeletionTimestamp().IsZero() {
		objectMeta.SetDeletionTimestamp(oldMeta.GetDeletionTimestamp())
	}
	// an update can never remove/change grace period seconds
	if oldMeta.GetDeletionGracePeriodSeconds() != nil && objectMeta.GetDeletionGracePeriodSeconds() == nil {
		objectMeta.SetDeletionGracePeriodSeconds(oldMeta.GetDeletionGracePeriodSeconds())
	}

	// Ensure some common fields, like UID, are validated for all resources.
	errs, err := validateCommonFields(obj, old, strategy)
	if err != nil {
		return errors.NewInternalError(err)
	}

	errs = append(errs, strategy.ValidateUpdate(ctx, obj, old)...)
	if len(errs) > 0 {
		return errors.NewInvalid(kind.GroupKind(), objectMeta.GetName(), errs)
	}

	strategy.Canonicalize(obj)

	return nil
}

// TransformFunc is a function to transform and return newObj
type TransformFunc func(ctx context.Context, newObj runtime.Object, oldObj runtime.Object) (transformedNewObj runtime.Object, err error)

// defaultUpdatedObjectInfo implements UpdatedObjectInfo
type defaultUpdatedObjectInfo struct {
	// obj is the updated object
	obj runtime.Object

	// transformers is an optional list of transforming functions that modify or
	// replace obj using information from the context, old object, or other sources.
	transformers []TransformFunc
}

// DefaultUpdatedObjectInfo returns an UpdatedObjectInfo impl based on the specified object.
func DefaultUpdatedObjectInfo(obj runtime.Object, transformers ...TransformFunc) UpdatedObjectInfo {
	return &defaultUpdatedObjectInfo{obj, transformers}
}

// Preconditions satisfies the UpdatedObjectInfo interface.
func (i *defaultUpdatedObjectInfo) Preconditions() *metav1.Preconditions {
	// Attempt to get the UID out of the object
	accessor, err := meta.Accessor(i.obj)
	if err != nil {
		// If no UID can be read, no preconditions are possible
		return nil
	}

	// If empty, no preconditions needed
	uid := accessor.GetUID()
	if len(uid) == 0 {
		return nil
	}

	return &metav1.Preconditions{UID: &uid}
}

// UpdatedObject satisfies the UpdatedObjectInfo interface.
// It returns a copy of the held obj, passed through any configured transformers.
func (i *defaultUpdatedObjectInfo) UpdatedObject(ctx context.Context, oldObj runtime.Object) (runtime.Object, error) {
	var err error
	// Start with the configured object
	newObj := i.obj

	// If the original is non-nil (might be nil if the first transformer builds the object from the oldObj), make a copy,
	// so we don't return the original. BeforeUpdate can mutate the returned object, doing things like clearing ResourceVersion.
	// If we're re-called, we need to be able to return the pristine version.
	if newObj != nil {
		newObj = newObj.DeepCopyObject()
	}

	// Allow any configured transformers to update the new object
	for _, transformer := range i.transformers {
		newObj, err = transformer(ctx, newObj, oldObj)
		if err != nil {
			return nil, err
		}
	}

	return newObj, nil
}

// wrappedUpdatedObjectInfo allows wrapping an existing objInfo and
// chaining additional transformations/checks on the result of UpdatedObject()
type wrappedUpdatedObjectInfo struct {
	// obj is the updated object
	objInfo UpdatedObjectInfo

	// transformers is an optional list of transforming functions that modify or
	// replace obj using information from the context, old object, or other sources.
	transformers []TransformFunc
}

// WrapUpdatedObjectInfo returns an UpdatedObjectInfo impl that delegates to
// the specified objInfo, then calls the passed transformers
func WrapUpdatedObjectInfo(objInfo UpdatedObjectInfo, transformers ...TransformFunc) UpdatedObjectInfo {
	return &wrappedUpdatedObjectInfo{objInfo, transformers}
}

// Preconditions satisfies the UpdatedObjectInfo interface.
func (i *wrappedUpdatedObjectInfo) Preconditions() *metav1.Preconditions {
	return i.objInfo.Preconditions()
}

// UpdatedObject satisfies the UpdatedObjectInfo interface.
// It delegates to the wrapped objInfo and passes the result through any configured transformers.
func (i *wrappedUpdatedObjectInfo) UpdatedObject(ctx context.Context, oldObj runtime.Object) (runtime.Object, error) {
	newObj, err := i.objInfo.UpdatedObject(ctx, oldObj)
	if err != nil {
		return newObj, err
	}

	// Allow any configured transformers to update the new object or error
	for _, transformer := range i.transformers {
		newObj, err = transformer(ctx, newObj, oldObj)
		if err != nil {
			return nil, err
		}
	}

	return newObj, nil
}

// AdmissionToValidateObjectUpdateFunc converts validating admission to a rest validate object update func
func AdmissionToValidateObjectUpdateFunc(admit admission.Interface, staticAttributes admission.Attributes) ValidateObjectUpdateFunc {
	validatingAdmission, ok := admit.(admission.ValidationInterface)
	if !ok {
		return func(obj, old runtime.Object) error { return nil }
	}
	return func(obj, old runtime.Object) error {
		finalAttributes := admission.NewAttributesRecord(
			obj,
			old,
			staticAttributes.GetKind(),
			staticAttributes.GetNamespace(),
			staticAttributes.GetName(),
			staticAttributes.GetResource(),
			staticAttributes.GetSubresource(),
			staticAttributes.GetOperation(),
			staticAttributes.GetUserInfo(),
		)
		if !validatingAdmission.Handles(finalAttributes.GetOperation()) {
			return nil
		}
		return validatingAdmission.Validate(finalAttributes)
	}
}
<|endoftext|>