<commit_before>package ui

import (
	"encoding/json"
	"html/template"
	"net/http"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"github.com/elazarl/go-bindata-assetfs"
	"github.com/gorilla/mux"

	"github.com/supergiant/supergiant/bindata"
	"github.com/supergiant/supergiant/pkg/client"
	"github.com/supergiant/supergiant/pkg/core"
	"github.com/supergiant/supergiant/pkg/model"
)

var templates = make(map[string]*template.Template)

func init() {
	// AssetNames() is like a filepath Glob of our generated assets
	viewFiles := bindata.AssetNames()
	var partials []string
	var layouts []string
	var views []string
	for _, viewFile := range viewFiles {
		if !strings.HasPrefix(viewFile, "ui/views") {
			continue // don't want to load any non-HTML assets into this
		}
		if strings.HasPrefix(viewFile, "ui/views/partials") {
			partials = append(partials, viewFile)
		} else if strings.HasPrefix(viewFile, "ui/views/layouts") {
			layouts = append(layouts, viewFile)
		} else {
			views = append(views, viewFile)
		}
	}

	for _, view := range views {
		// https://golang.org/src/html/template/template.go?s=11938:12007#L354
		var t *template.Template
		for _, view := range append(layouts, append(partials, view)...) {
			name := filepath.Base(view)
			if t == nil {
				t = template.New(name)
			}
			var tmpl *template.Template
			if name == t.Name() {
				tmpl = t
			} else {
				tmpl = t.New(name)
			}

			src, err := bindata.Asset(view)
			if err != nil {
				panic(err)
			}

			if _, err = tmpl.Parse(string(src)); err != nil {
				panic(err)
			}
		}

		key := regexp.MustCompile(`([^/]+)\.html$`).FindStringSubmatch(view)[1]
		templates[key] = t
	}
}

func NewRouter(c *core.Core, baseRouter *mux.Router) *mux.Router {
	base := baseRouter.StrictSlash(true)

	// Redirect / to /ui
	base.HandleFunc("/", uiRedirect).Methods("GET")

	r := base.PathPrefix("/ui").Subrouter()

	// Assets
	assetDir := &assetfs.AssetFS{Asset: bindata.Asset, AssetDir: bindata.AssetDir, AssetInfo: bindata.AssetInfo}
	r.PathPrefix("/assets/").Handler(http.FileServer(assetDir))

	r.HandleFunc("/", restrictedHandler(c, Root)).Methods("GET")

	r.HandleFunc("/sessions/new", openHandler(c, NewSession)).Methods("GET")
	r.HandleFunc("/sessions", openHandler(c, CreateSession)).Methods("POST")
	r.HandleFunc("/sessions/{id}", openHandler(c, GetSession)).Methods("GET")

	r.HandleFunc("/sessions", restrictedHandler(c, ListSessions)).Methods("GET")

	r.HandleFunc("/users/new", restrictedHandler(c, NewUser)).Methods("GET")
	r.HandleFunc("/users", restrictedHandler(c, CreateUser)).Methods("POST")
	r.HandleFunc("/users", restrictedHandler(c, ListUsers)).Methods("GET")
	r.HandleFunc("/users/{id}", restrictedHandler(c, GetUser)).Methods("GET")
	r.HandleFunc("/users/{id}/edit", restrictedHandler(c, EditUser)).Methods("GET")
	r.HandleFunc("/users/{id}", restrictedHandler(c, UpdateUser)).Methods("POST")

	r.HandleFunc("/cloud_accounts/new", restrictedHandler(c, NewCloudAccount)).Methods("GET")
	r.HandleFunc("/cloud_accounts", restrictedHandler(c, CreateCloudAccount)).Methods("POST")
	r.HandleFunc("/cloud_accounts", restrictedHandler(c, ListCloudAccounts)).Methods("GET")
	r.HandleFunc("/cloud_accounts/{id}", restrictedHandler(c, GetCloudAccount)).Methods("GET")

	r.HandleFunc("/kubes/new", restrictedHandler(c, NewKube)).Methods("GET")
	r.HandleFunc("/kubes", restrictedHandler(c, CreateKube)).Methods("POST")
	r.HandleFunc("/kubes", restrictedHandler(c, ListKubes)).Methods("GET")
	r.HandleFunc("/kubes/{id}", restrictedHandler(c, GetKube)).Methods("GET")

	r.HandleFunc("/nodes/new", restrictedHandler(c, NewNode)).Methods("GET")
	r.HandleFunc("/nodes", restrictedHandler(c, CreateNode)).Methods("POST")
	r.HandleFunc("/nodes", restrictedHandler(c, ListNodes)).Methods("GET")
	r.HandleFunc("/nodes/{id}", restrictedHandler(c, GetNode)).Methods("GET")

	r.HandleFunc("/load_balancers/new", restrictedHandler(c, NewLoadBalancer)).Methods("GET")
	r.HandleFunc("/load_balancers", restrictedHandler(c, CreateLoadBalancer)).Methods("POST")
	r.HandleFunc("/load_balancers", restrictedHandler(c, ListLoadBalancers)).Methods("GET")
	r.HandleFunc("/load_balancers/{id}", restrictedHandler(c, GetLoadBalancer)).Methods("GET")
	r.HandleFunc("/load_balancers/{id}/edit", restrictedHandler(c, EditLoadBalancer)).Methods("GET")
	r.HandleFunc("/load_balancers/{id}", restrictedHandler(c, UpdateLoadBalancer)).Methods("POST")

	r.HandleFunc("/pods", restrictedHandler(c, ListPods)).Methods("GET")
	r.HandleFunc("/pods/new", restrictedHandler(c, NewPod)).Methods("GET")

	r.HandleFunc("/services", restrictedHandler(c, ListServices)).Methods("GET")
	r.HandleFunc("/services/new", restrictedHandler(c, NewService)).Methods("GET")

	r.HandleFunc("/volumes", restrictedHandler(c, ListVolumes)).Methods("GET")
	// r.HandleFunc("/volumes/new", restrictedHandler(c, NewVolume)).Methods("GET")

	// TODO these are currently legacy, for redirects after Create and such
	r.HandleFunc("/kube_resources", restrictedHandler(c, ListKubeResources)).Methods("GET")
	r.HandleFunc("/kube_resources/new", restrictedHandler(c, NewKubeResource)).Methods("GET")
	// TODO this is so gross
	r.HandleFunc("/{kind:(kube_resource|pod|service|volume)}s", restrictedHandler(c, CreateKubeResource)).Methods("POST")
	r.HandleFunc("/{kind:(kube_resource|pod|service|volume)}s/{id}", restrictedHandler(c, GetKubeResource)).Methods("GET")
	r.HandleFunc("/{kind:(kube_resource|pod|service|volume)}s/{id}/edit", restrictedHandler(c, EditKubeResource)).Methods("GET")
	r.HandleFunc("/{kind:(kube_resource|pod|service|volume)}s/{id}", restrictedHandler(c, UpdateKubeResource)).Methods("POST")

	r.HandleFunc("/helm_repos/new", restrictedHandler(c, NewHelmRepo)).Methods("GET")
	r.HandleFunc("/helm_repos", restrictedHandler(c, CreateHelmRepo)).Methods("POST")
	r.HandleFunc("/helm_repos", restrictedHandler(c, ListHelmRepos)).Methods("GET")
	r.HandleFunc("/helm_repos/{id}", restrictedHandler(c, GetHelmRepo)).Methods("GET")

	r.HandleFunc("/helm_charts/new", restrictedHandler(c, NewHelmChart)).Methods("GET")
	r.HandleFunc("/helm_charts", restrictedHandler(c, CreateHelmChart)).Methods("POST")
	r.HandleFunc("/helm_charts", restrictedHandler(c, ListHelmCharts)).Methods("GET")
	r.HandleFunc("/helm_charts/{id}", restrictedHandler(c, GetHelmChart)).Methods("GET")

	r.HandleFunc("/helm_releases/new", restrictedHandler(c, NewHelmRelease)).Methods("GET")
	r.HandleFunc("/helm_releases", restrictedHandler(c, CreateHelmRelease)).Methods("POST")
	r.HandleFunc("/helm_releases", restrictedHandler(c, ListHelmReleases)).Methods("GET")
	r.HandleFunc("/helm_releases/{id}", restrictedHandler(c, GetHelmRelease)).Methods("GET")

	return baseRouter
}

func restrictedHandler(c *core.Core, fn func(*client.Client, http.ResponseWriter, *http.Request) error) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		// Load Client by Session ID stored in cookie. If either cookie or client
		// does not exist, redirect to login page with 401.
		var sessionID string
		var client *client.Client

		if sessionCookie, err := r.Cookie(core.SessionCookieName); err == nil {
			sessionID = sessionCookie.Value
			client = c.Sessions.Client(sessionID)
		}

		if client == nil {
			http.Redirect(w, r, "/ui/sessions/new", http.StatusFound) // can't do 401 here unless you want browser behavior
			return
		}
		if err := fn(client, w, r); err != nil {

			status := http.StatusInternalServerError
			if strings.Contains(err.Error(), "404") {
				status = http.StatusNotFound
			}

			w.WriteHeader(status)
			w.Write([]byte(err.Error()))
		}
	}
}

func openHandler(c *core.Core, fn func(*client.Client, http.ResponseWriter, *http.Request) error) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		// Unauthenticated client
		if err := fn(c.APIClient("", ""), w, r); err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			w.Write([]byte(err.Error()))
		}
	}
}

//------------------------------------------------------------------------------

func parseID(r *http.Request) (*int64, error) {
	id, err := strconv.Atoi(mux.Vars(r)["id"])
	if err != nil {
		return nil, err
	}
	id64 := int64(id)
	return &id64, nil
}

func renderTemplate(sg *client.Client, w http.ResponseWriter, name string, data map[string]interface{}) error {

	// TODO
	if mi := data["model"]; mi != nil {
		if m, isModel := mi.(model.Model); isModel {
			model.ZeroPrivateFields(m)
			data["model"] = m
		}
	}

	modelJSON, _ := json.Marshal(data["model"])
	data["modelJSON"] = string(modelJSON)

	if fields, ok := data["fields"]; ok {
		fieldsJSON, _ := json.Marshal(fields)
		data["fieldsJSON"] = string(fieldsJSON)
	}

	// Version number
	data["supergiantVersion"] = sg.Version

	// Session ID (for JS-based logout)
	data["sessionID"] = sg.AuthToken

	if err := templates[name].ExecuteTemplate(w, "layout", data); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return err
	}
	return nil
}

func unmarshalFormInto(r *http.Request, out interface{}) error {
	if err := r.ParseForm(); err != nil {
		return err
	}
	return json.Unmarshal([]byte(r.PostForm.Get("json_input")), out)
}

//------------------------------------------------------------------------------

func uiRedirect(w http.ResponseWriter, r *http.Request) {
	http.Redirect(w, r, "/ui", http.StatusFound)
}

//------------------------------------------------------------------------------

func Root(sg *client.Client, w http.ResponseWriter, r *http.Request) error {
	http.Redirect(w, r, "/ui/cloud_accounts", http.StatusFound)
	return nil
}
<commit_msg>Fix 'Only non-capturing groups are accepted' panic (#524)<commit_after>package ui

import (
	"encoding/json"
	"html/template"
	"net/http"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"github.com/elazarl/go-bindata-assetfs"
	"github.com/gorilla/mux"

	"github.com/supergiant/supergiant/bindata"
	"github.com/supergiant/supergiant/pkg/client"
	"github.com/supergiant/supergiant/pkg/core"
	"github.com/supergiant/supergiant/pkg/model"
)

var templates = make(map[string]*template.Template)

func init() {
	// AssetNames() is like a filepath Glob of our generated assets
	viewFiles := bindata.AssetNames()
	var partials []string
	var layouts []string
	var views []string
	for _, viewFile := range viewFiles {
		if !strings.HasPrefix(viewFile, "ui/views") {
			continue // don't want to load any non-HTML assets into this
		}
		if strings.HasPrefix(viewFile, "ui/views/partials") {
			partials = append(partials, viewFile)
		} else if strings.HasPrefix(viewFile, "ui/views/layouts") {
			layouts = append(layouts, viewFile)
		} else {
			views = append(views, viewFile)
		}
	}

	for _, view := range views {
		// https://golang.org/src/html/template/template.go?s=11938:12007#L354
		var t *template.Template
		for _, view := range append(layouts, append(partials, view)...) {
			name := filepath.Base(view)
			if t == nil {
				t = template.New(name)
			}
			var tmpl *template.Template
			if name == t.Name() {
				tmpl = t
			} else {
				tmpl = t.New(name)
			}

			src, err := bindata.Asset(view)
			if err != nil {
				panic(err)
			}

			if _, err = tmpl.Parse(string(src)); err != nil {
				panic(err)
			}
		}

		key := regexp.MustCompile(`([^/]+)\.html$`).FindStringSubmatch(view)[1]
		templates[key] = t
	}
}

func NewRouter(c *core.Core, baseRouter *mux.Router) *mux.Router {
	base := baseRouter.StrictSlash(true)

	// Redirect / to /ui
	base.HandleFunc("/", uiRedirect).Methods("GET")

	r := base.PathPrefix("/ui").Subrouter()

	// Assets
	assetDir := &assetfs.AssetFS{Asset: bindata.Asset, AssetDir: bindata.AssetDir, AssetInfo: bindata.AssetInfo}
	r.PathPrefix("/assets/").Handler(http.FileServer(assetDir))

	r.HandleFunc("/", restrictedHandler(c, Root)).Methods("GET")

	r.HandleFunc("/sessions/new", openHandler(c, NewSession)).Methods("GET")
	r.HandleFunc("/sessions", openHandler(c, CreateSession)).Methods("POST")
	r.HandleFunc("/sessions/{id}", openHandler(c, GetSession)).Methods("GET")

	r.HandleFunc("/sessions", restrictedHandler(c, ListSessions)).Methods("GET")

	r.HandleFunc("/users/new", restrictedHandler(c, NewUser)).Methods("GET")
	r.HandleFunc("/users", restrictedHandler(c, CreateUser)).Methods("POST")
	r.HandleFunc("/users", restrictedHandler(c, ListUsers)).Methods("GET")
	r.HandleFunc("/users/{id}", restrictedHandler(c, GetUser)).Methods("GET")
	r.HandleFunc("/users/{id}/edit", restrictedHandler(c, EditUser)).Methods("GET")
	r.HandleFunc("/users/{id}", restrictedHandler(c, UpdateUser)).Methods("POST")

	r.HandleFunc("/cloud_accounts/new", restrictedHandler(c, NewCloudAccount)).Methods("GET")
	r.HandleFunc("/cloud_accounts", restrictedHandler(c, CreateCloudAccount)).Methods("POST")
	r.HandleFunc("/cloud_accounts", restrictedHandler(c, ListCloudAccounts)).Methods("GET")
	r.HandleFunc("/cloud_accounts/{id}", restrictedHandler(c, GetCloudAccount)).Methods("GET")

	r.HandleFunc("/kubes/new", restrictedHandler(c, NewKube)).Methods("GET")
	r.HandleFunc("/kubes", restrictedHandler(c, CreateKube)).Methods("POST")
	r.HandleFunc("/kubes", restrictedHandler(c, ListKubes)).Methods("GET")
	r.HandleFunc("/kubes/{id}", restrictedHandler(c, GetKube)).Methods("GET")

	r.HandleFunc("/nodes/new", restrictedHandler(c, NewNode)).Methods("GET")
	r.HandleFunc("/nodes", restrictedHandler(c, CreateNode)).Methods("POST")
	r.HandleFunc("/nodes", restrictedHandler(c, ListNodes)).Methods("GET")
	r.HandleFunc("/nodes/{id}", restrictedHandler(c, GetNode)).Methods("GET")

	r.HandleFunc("/load_balancers/new", restrictedHandler(c, NewLoadBalancer)).Methods("GET")
	r.HandleFunc("/load_balancers", restrictedHandler(c, CreateLoadBalancer)).Methods("POST")
	r.HandleFunc("/load_balancers", restrictedHandler(c, ListLoadBalancers)).Methods("GET")
	r.HandleFunc("/load_balancers/{id}", restrictedHandler(c, GetLoadBalancer)).Methods("GET")
	r.HandleFunc("/load_balancers/{id}/edit", restrictedHandler(c, EditLoadBalancer)).Methods("GET")
	r.HandleFunc("/load_balancers/{id}", restrictedHandler(c, UpdateLoadBalancer)).Methods("POST")

	r.HandleFunc("/pods", restrictedHandler(c, ListPods)).Methods("GET")
	r.HandleFunc("/pods/new", restrictedHandler(c, NewPod)).Methods("GET")

	r.HandleFunc("/services", restrictedHandler(c, ListServices)).Methods("GET")
	r.HandleFunc("/services/new", restrictedHandler(c, NewService)).Methods("GET")

	r.HandleFunc("/volumes", restrictedHandler(c, ListVolumes)).Methods("GET")
	// r.HandleFunc("/volumes/new", restrictedHandler(c, NewVolume)).Methods("GET")

	// TODO these are currently legacy, for redirects after Create and such
	r.HandleFunc("/kube_resources", restrictedHandler(c, ListKubeResources)).Methods("GET")
	r.HandleFunc("/kube_resources/new", restrictedHandler(c, NewKubeResource)).Methods("GET")
	// TODO this is so gross
	r.HandleFunc("/{kind:(?:kube_resource|pod|service|volume)}s", restrictedHandler(c, CreateKubeResource)).Methods("POST")
	r.HandleFunc("/{kind:(?:kube_resource|pod|service|volume)}s/{id}", restrictedHandler(c, GetKubeResource)).Methods("GET")
	r.HandleFunc("/{kind:(?:kube_resource|pod|service|volume)}s/{id}/edit", restrictedHandler(c, EditKubeResource)).Methods("GET")
	r.HandleFunc("/{kind:(?:kube_resource|pod|service|volume)}s/{id}", restrictedHandler(c, UpdateKubeResource)).Methods("POST")

	r.HandleFunc("/helm_repos/new", restrictedHandler(c, NewHelmRepo)).Methods("GET")
	r.HandleFunc("/helm_repos", restrictedHandler(c, CreateHelmRepo)).Methods("POST")
	r.HandleFunc("/helm_repos", restrictedHandler(c, ListHelmRepos)).Methods("GET")
	r.HandleFunc("/helm_repos/{id}", restrictedHandler(c, GetHelmRepo)).Methods("GET")

	r.HandleFunc("/helm_charts/new", restrictedHandler(c, NewHelmChart)).Methods("GET")
	r.HandleFunc("/helm_charts", restrictedHandler(c, CreateHelmChart)).Methods("POST")
	r.HandleFunc("/helm_charts", restrictedHandler(c, ListHelmCharts)).Methods("GET")
	r.HandleFunc("/helm_charts/{id}", restrictedHandler(c, GetHelmChart)).Methods("GET")

	r.HandleFunc("/helm_releases/new", restrictedHandler(c, NewHelmRelease)).Methods("GET")
	r.HandleFunc("/helm_releases", restrictedHandler(c, CreateHelmRelease)).Methods("POST")
	r.HandleFunc("/helm_releases", restrictedHandler(c, ListHelmReleases)).Methods("GET")
	r.HandleFunc("/helm_releases/{id}", restrictedHandler(c, GetHelmRelease)).Methods("GET")

	return baseRouter
}

func restrictedHandler(c *core.Core, fn func(*client.Client, http.ResponseWriter, *http.Request) error) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		// Load Client by Session ID stored in cookie. If either cookie or client
		// does not exist, redirect to login page with 401.
		var sessionID string
		var client *client.Client

		if sessionCookie, err := r.Cookie(core.SessionCookieName); err == nil {
			sessionID = sessionCookie.Value
			client = c.Sessions.Client(sessionID)
		}

		if client == nil {
			http.Redirect(w, r, "/ui/sessions/new", http.StatusFound) // can't do 401 here unless you want browser behavior
			return
		}
		if err := fn(client, w, r); err != nil {

			status := http.StatusInternalServerError
			if strings.Contains(err.Error(), "404") {
				status = http.StatusNotFound
			}

			w.WriteHeader(status)
			w.Write([]byte(err.Error()))
		}
	}
}

func openHandler(c *core.Core, fn func(*client.Client, http.ResponseWriter, *http.Request) error) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		// Unauthenticated client
		if err := fn(c.APIClient("", ""), w, r); err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			w.Write([]byte(err.Error()))
		}
	}
}

//------------------------------------------------------------------------------

func parseID(r *http.Request) (*int64, error) {
	id, err := strconv.Atoi(mux.Vars(r)["id"])
	if err != nil {
		return nil, err
	}
	id64 := int64(id)
	return &id64, nil
}

func renderTemplate(sg *client.Client, w http.ResponseWriter, name string, data map[string]interface{}) error {

	// TODO
	if mi := data["model"]; mi != nil {
		if m, isModel := mi.(model.Model); isModel {
			model.ZeroPrivateFields(m)
			data["model"] = m
		}
	}

	modelJSON, _ := json.Marshal(data["model"])
	data["modelJSON"] = string(modelJSON)

	if fields, ok := data["fields"]; ok {
		fieldsJSON, _ := json.Marshal(fields)
		data["fieldsJSON"] = string(fieldsJSON)
	}

	// Version number
	data["supergiantVersion"] = sg.Version

	// Session ID (for JS-based logout)
	data["sessionID"] = sg.AuthToken

	if err := templates[name].ExecuteTemplate(w, "layout", data); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return err
	}
	return nil
}

func unmarshalFormInto(r *http.Request, out interface{}) error {
	if err := r.ParseForm(); err != nil {
		return err
	}
	return json.Unmarshal([]byte(r.PostForm.Get("json_input")), out)
}

//------------------------------------------------------------------------------

func uiRedirect(w http.ResponseWriter, r *http.Request) {
	http.Redirect(w, r, "/ui", http.StatusFound)
}

//------------------------------------------------------------------------------

func Root(sg *client.Client, w http.ResponseWriter, r *http.Request) error {
	http.Redirect(w, r, "/ui/cloud_accounts", http.StatusFound)
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2019 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package vcs

import (
	"bytes"
	"io"
	"net/mail"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/email"
	"github.com/google/syzkaller/pkg/osutil"
)

type linux struct {
	*git
}

func newLinux(dir string) *linux {
	ignoreCC := map[string]bool{
		"stable@vger.kernel.org": true,
	}
	return &linux{
		git: newGit(dir, ignoreCC),
	}
}

func (ctx *linux) Bisect(bad, good string, trace io.Writer, pred func() (BisectResult, error)) ([]*Commit, error) {
	commits, err := ctx.git.Bisect(bad, good, trace, pred)
	if len(commits) == 1 {
		ctx.addMaintainers(commits[0])
	}
	return commits, err
}

func (ctx *linux) addMaintainers(com *Commit) {
	if len(com.CC) > 3 {
		return
	}
	list := ctx.getMaintainers(com.Hash, false)
	if len(list) < 3 {
		list = ctx.getMaintainers(com.Hash, true)
	}
	com.CC = email.MergeEmailLists(com.CC, list)
}

func (ctx *linux) getMaintainers(hash string, blame bool) []string {
	args := "git show " + hash + " | " +
		filepath.FromSlash("scripts/get_maintainer.pl") + " --no-n --no-rolestats"
	if blame {
		args += " --git-blame"
	}
	output, err := osutil.RunCmd(time.Minute, ctx.git.dir, "bash", "-c", args)
	if err != nil {
		return nil
	}
	var list []string
	for _, line := range strings.Split(string(output), "\n") {
		addr, err := mail.ParseAddress(line)
		if err != nil {
			continue
		}
		list = append(list, strings.ToLower(addr.Address))
	}
	return list
}

func (ctx *linux) PreviousReleaseTags(commit string) ([]string, error) {
	return ctx.previousReleaseTags(commit, false)
}

func (ctx *linux) previousReleaseTags(commit string, self bool) ([]string, error) {
	var tags []string
	if self {
		output, err := runSandboxed(ctx.dir, "git", "tag", "--points-at", commit, "--merged", commit, "v*.*")
		if err != nil {
			return nil, err
		}
		tags, err = gitParseReleaseTags(output)
		if err != nil {
			return nil, err
		}
	}
	output, err := runSandboxed(ctx.dir, "git", "tag", "--no-contains", commit, "--merged", commit, "v*.*")
	if err != nil {
		return nil, err
	}
	tags1, err := gitParseReleaseTags(output)
	if err != nil {
		return nil, err
	}
	tags = append(tags, tags1...)
	for i, tag := range tags {
		if tag == "v4.0" {
			// Initially we tried to stop at 3.8 because:
			// v3.8 does not work with modern perl, and as we go further in history
			// make stops to work, then binutils, glibc, etc. So we stop at v3.8.
			// Up to that point we only need an ancient gcc.
			//
			// But kernels don't boot starting from 4.0 and back.
			// That was fixed by 99124e4db5b7b70daeaaf1d88a6a8078a0004c6e,
			// and it can be cherry-picked into 3.14..4.0 but it conflicts for 3.13 and older.
			//
			// But starting from 4.0 our user-space binaries start crashing with assorted errors
			// which suggests process memory corruption by kernel. So for now we stop at 4.1.
			tags = tags[:i]
			break
		}
	}
	return tags, nil
}

func gitParseReleaseTags(output []byte) ([]string, error) {
	var tags []string
	for _, tag := range bytes.Split(output, []byte{'\n'}) {
		if releaseTagRe.Match(tag) && gitReleaseTagToInt(string(tag)) != 0 {
			tags = append(tags, string(tag))
		}
	}
	sort.Slice(tags, func(i, j int) bool {
		return gitReleaseTagToInt(tags[i]) > gitReleaseTagToInt(tags[j])
	})
	return tags, nil
}

func gitReleaseTagToInt(tag string) uint64 {
	matches := releaseTagRe.FindStringSubmatchIndex(tag)
	v1, err := strconv.ParseUint(tag[matches[2]:matches[3]], 10, 64)
	if err != nil {
		return 0
	}
	v2, err := strconv.ParseUint(tag[matches[4]:matches[5]], 10, 64)
	if err != nil {
		return 0
	}
	var v3 uint64
	if matches[6] != -1 {
		v3, err = strconv.ParseUint(tag[matches[6]:matches[7]], 10, 64)
		if err != nil {
			return 0
		}
	}
	return v1*1e6 + v2*1e3 + v3
}

func (ctx *linux) EnvForCommit(commit string, kernelConfig []byte) (*BisectEnv, error) {
	tagList, err := ctx.previousReleaseTags(commit, true)
	if err != nil {
		return nil, err
	}
	tags := make(map[string]bool)
	for _, tag := range tagList {
		tags[tag] = true
	}
	env := &BisectEnv{
		Compiler:     "gcc-" + linuxCompilerVersion(tags),
		KernelConfig: linuxDisableConfigs(kernelConfig, tags),
	}
	// v4.0 doesn't boot with our config nor with defconfig, it halts on an interrupt in x86_64_start_kernel.
	if !tags["v4.1"] {
		if _, err := runSandboxed(ctx.dir, "git", "cherry-pick", "--no-commit", "99124e4db5b7b70daeaaf1d88a6a8078a0004c6e"); err != nil {
			return nil, err
		}
	}
	return env, nil
}

func linuxCompilerVersion(tags map[string]bool) string {
	switch {
	case tags["v4.12"]:
		return "8.1.0"
	case tags["v4.11"]:
		return "7.3.0"
	default:
		return "5.5.0"
	}
}

func linuxDisableConfigs(config []byte, tags map[string]bool) []byte {
	prereq := map[string]string{
		// Kernel is boot broken before 4.15 due to double-free in vudc_probe:
		// https://lkml.org/lkml/2018/9/7/648
		// Fixed by e28fd56ad5273be67d0fae5bedc7e1680e729952.
		"CONFIG_USBIP_VUDC": "v4.15",
		// Setup of network devices is broken before v4.12 with a "WARNING in hsr_get_node".
		// Fixed by 675c8da049fd6556eb2d6cdd745fe812752f07a8.
		"CONFIG_HSR": "v4.12",
		// Setup of network devices is broken before v4.12 with a "WARNING: ODEBUG bug in __sk_destruct"
		// coming from smc_release.
		"CONFIG_SMC": "v4.12",
		// Kernel is boot broken before 4.10 with a lockdep warning in vhci_hcd_probe.
		"CONFIG_USBIP_VHCI_HCD": "v4.10",
		"CONFIG_BT_HCIVHCI":     "v4.10",
		// Setup of network devices is broken before v4.7 with a deadlock involving team.
		"CONFIG_NET_TEAM": "v4.7",
		// Setup of network devices is broken before v4.5 with a warning in batadv_tvlv_container_remove.
		"CONFIG_BATMAN_ADV": "v4.5",
		// First, we disable coverage in pkg/bisect because it fails machine testing starting from 4.7.
		// Second, at 6689da155bdcd17abfe4d3a8b1e245d9ed4b5f2c CONFIG_KCOV selects CONFIG_GCC_PLUGIN_SANCOV
		// (why?), which is build broken for hundreds of revisions.
		"CONFIG_KCOV": "disable-always",
	}
	for cfg, tag := range prereq {
		if !tags[tag] {
			config = bytes.Replace(config, []byte(cfg+"=y"), []byte("# "+cfg+" is not set"), -1)
		}
	}
	return config
}
<commit_msg>pkg/vcs: fix too long line<commit_after>// Copyright 2019 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package vcs

import (
	"bytes"
	"io"
	"net/mail"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/email"
	"github.com/google/syzkaller/pkg/osutil"
)

type linux struct {
	*git
}

func newLinux(dir string) *linux {
	ignoreCC := map[string]bool{
		"stable@vger.kernel.org": true,
	}
	return &linux{
		git: newGit(dir, ignoreCC),
	}
}

func (ctx *linux) PreviousReleaseTags(commit string) ([]string, error) {
	return ctx.previousReleaseTags(commit, false)
}

func (ctx *linux) previousReleaseTags(commit string, self bool) ([]string, error) {
	var tags []string
	if self {
		output, err := runSandboxed(ctx.dir, "git", "tag", "--points-at", commit, "--merged", commit, "v*.*")
		if err != nil {
			return nil, err
		}
		tags, err = gitParseReleaseTags(output)
		if err != nil {
			return nil, err
		}
	}
	output, err := runSandboxed(ctx.dir, "git", "tag", "--no-contains", commit, "--merged", commit, "v*.*")
	if err != nil {
		return nil, err
	}
	tags1, err := gitParseReleaseTags(output)
	if err != nil {
		return nil, err
	}
	tags = append(tags, tags1...)
	for i, tag := range tags {
		if tag == "v4.0" {
			// Initially we tried to stop at 3.8 because:
			// v3.8 does not work with modern perl, and as we go further in history
			// make stops to work, then binutils, glibc, etc. So we stop at v3.8.
			// Up to that point we only need an ancient gcc.
			//
			// But kernels don't boot starting from 4.0 and back.
			// That was fixed by 99124e4db5b7b70daeaaf1d88a6a8078a0004c6e,
			// and it can be cherry-picked into 3.14..4.0 but it conflicts for 3.13 and older.
			//
			// But starting from 4.0 our user-space binaries start crashing with assorted errors
			// which suggests process memory corruption by kernel. So for now we stop at 4.1.
			tags = tags[:i]
			break
		}
	}
	return tags, nil
}

func gitParseReleaseTags(output []byte) ([]string, error) {
	var tags []string
	for _, tag := range bytes.Split(output, []byte{'\n'}) {
		if releaseTagRe.Match(tag) && gitReleaseTagToInt(string(tag)) != 0 {
			tags = append(tags, string(tag))
		}
	}
	sort.Slice(tags, func(i, j int) bool {
		return gitReleaseTagToInt(tags[i]) > gitReleaseTagToInt(tags[j])
	})
	return tags, nil
}

func gitReleaseTagToInt(tag string) uint64 {
	matches := releaseTagRe.FindStringSubmatchIndex(tag)
	v1, err := strconv.ParseUint(tag[matches[2]:matches[3]], 10, 64)
	if err != nil {
		return 0
	}
	v2, err := strconv.ParseUint(tag[matches[4]:matches[5]], 10, 64)
	if err != nil {
		return 0
	}
	var v3 uint64
	if matches[6] != -1 {
		v3, err = strconv.ParseUint(tag[matches[6]:matches[7]], 10, 64)
		if err != nil {
			return 0
		}
	}
	return v1*1e6 + v2*1e3 + v3
}

func (ctx *linux) EnvForCommit(commit string, kernelConfig []byte) (*BisectEnv, error) {
	tagList, err := ctx.previousReleaseTags(commit, true)
	if err != nil {
		return nil, err
	}
	tags := make(map[string]bool)
	for _, tag := range tagList {
		tags[tag] = true
	}
	env := &BisectEnv{
		Compiler:     "gcc-" + linuxCompilerVersion(tags),
		KernelConfig: linuxDisableConfigs(kernelConfig, tags),
	}
	// v4.0 doesn't boot with our config nor with defconfig, it halts on an interrupt in x86_64_start_kernel.
	if !tags["v4.1"] {
		_, err := runSandboxed(ctx.dir, "git", "cherry-pick",
			"--no-commit", "99124e4db5b7b70daeaaf1d88a6a8078a0004c6e")
		if err != nil {
			return nil, err
		}
	}
	return env, nil
}

func linuxCompilerVersion(tags map[string]bool) string {
	switch {
	case tags["v4.12"]:
		return "8.1.0"
	case tags["v4.11"]:
		return "7.3.0"
	default:
		return "5.5.0"
	}
}

func linuxDisableConfigs(config []byte, tags map[string]bool) []byte {
	prereq := map[string]string{
		// Kernel is boot broken before 4.15 due to double-free in vudc_probe:
		// https://lkml.org/lkml/2018/9/7/648
		// Fixed by e28fd56ad5273be67d0fae5bedc7e1680e729952.
		"CONFIG_USBIP_VUDC": "v4.15",
		// Setup of network devices is broken before v4.12 with a "WARNING in hsr_get_node".
		// Fixed by 675c8da049fd6556eb2d6cdd745fe812752f07a8.
		"CONFIG_HSR": "v4.12",
		// Setup of network devices is broken before v4.12 with a "WARNING: ODEBUG bug in __sk_destruct"
		// coming from smc_release.
		"CONFIG_SMC": "v4.12",
		// Kernel is boot broken before 4.10 with a lockdep warning in vhci_hcd_probe.
		"CONFIG_USBIP_VHCI_HCD": "v4.10",
		"CONFIG_BT_HCIVHCI":     "v4.10",
		// Setup of network devices is broken before v4.7 with a deadlock involving team.
		"CONFIG_NET_TEAM": "v4.7",
		// Setup of network devices is broken before v4.5 with a warning in batadv_tvlv_container_remove.
		"CONFIG_BATMAN_ADV": "v4.5",
		// First, we disable coverage in pkg/bisect because it fails machine testing starting from 4.7.
		// Second, at 6689da155bdcd17abfe4d3a8b1e245d9ed4b5f2c CONFIG_KCOV selects CONFIG_GCC_PLUGIN_SANCOV
		// (why?), which is build broken for hundreds of revisions.
		"CONFIG_KCOV": "disable-always",
	}
	for cfg, tag := range prereq {
		if !tags[tag] {
			config = bytes.Replace(config, []byte(cfg+"=y"), []byte("# "+cfg+" is not set"), -1)
		}
	}
	return config
}

func (ctx *linux) Bisect(bad, good string, trace io.Writer, pred func() (BisectResult, error)) ([]*Commit, error) {
	commits, err := ctx.git.Bisect(bad, good, trace, pred)
	if len(commits) == 1 {
		ctx.addMaintainers(commits[0])
	}
	return commits, err
}

func (ctx *linux) addMaintainers(com *Commit) {
	if len(com.CC) > 3 {
		return
	}
	list := ctx.getMaintainers(com.Hash, false)
	if len(list) < 3 {
		list = ctx.getMaintainers(com.Hash, true)
	}
	com.CC = email.MergeEmailLists(com.CC, list)
}

func (ctx *linux) getMaintainers(hash string, blame bool) []string {
	args := "git show " + hash + " | " +
		filepath.FromSlash("scripts/get_maintainer.pl") + " --no-n --no-rolestats"
	if blame {
		args += " --git-blame"
	}
	output, err := osutil.RunCmd(time.Minute, ctx.git.dir, "bash", "-c", args)
	if err != nil {
		return nil
	}
	var list []string
	for _, line := range strings.Split(string(output), "\n") {
		addr, err := mail.ParseAddress(line)
		if err != nil {
			continue
		}
		list = append(list, strings.ToLower(addr.Address))
	}
	return list
}
<|endoftext|>


<commit_before>// Copyright 2015 Dorival de Moraes Pedroso. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package goga

import (
	"github.com/cpmech/gosl/io"
	"github.com/cpmech/gosl/la"
	"github.com/cpmech/gosl/rnd"
)

// Func_t defines a type for a generic function to be used as a gene value
type Func_t func(ind *Individual) string

// Individual implements one individual in a population
type Individual struct {

	// data
	ObjValue float64 // objective value
	Fitness  float64 // fitness
	Nfloats  int     // number of floats
	Nbases   int     // number of bases to split Floats

	// chromosome
	Ints    []int     // integers
	Floats  []float64 // floats [nfloats * nbases]
	Strings []string  // strings
	Keys    []byte    // 1D bytes
	Bytes   [][]byte  // 2D bytes
	Funcs   []Func_t  // functions
}

// NewIndividual allocates a new individual
//  Input:
//   nbases -- used to split genes of floats into smaller parts
//   slices -- slices of ints, floats, strings, bytes, []bytes, and/or Func_t
//  Notes:
//   1) the slices in 'genes' can all be combined to define genes with mixed data;
//   2) the slices can also be nil, except for one of them.
func NewIndividual(nbases int, slices ...interface{}) (o *Individual) {
	o = new(Individual)
	for _, slice := range slices {
		switch s := slice.(type) {
		case []int:
			o.Ints = make([]int, len(s))
			copy(o.Ints, s)

		case []float64:
			o.Nfloats = len(s)
			o.Nbases = nbases
			if o.Nbases > 1 {
				o.Floats = SimpleChromo(s, nbases)
			} else {
				o.Floats = make([]float64, o.Nfloats*o.Nbases)
				copy(o.Floats, s)
			}

		case []string:
			o.Strings = make([]string, len(s))
			copy(o.Strings, s)

		case []byte:
			o.Keys = make([]byte, len(s))
			copy(o.Keys, s)

		case [][]byte:
			o.Bytes = make([][]byte, len(s))
			for i, x := range s {
				o.Bytes[i] = make([]byte, len(x))
				copy(o.Bytes[i], x)
			}

		case []Func_t:
			o.Funcs = make([]Func_t, len(s))
			copy(o.Funcs, s)
		}
	}
	return
}

// GetCopy returns a copy of this individual
func (o Individual) GetCopy() (x *Individual) {

	x = new(Individual)
	x.ObjValue = o.ObjValue
	x.Fitness = o.Fitness
	x.Nfloats = o.Nfloats
	x.Nbases = o.Nbases

	if o.Ints != nil {
		x.Ints = make([]int, len(o.Ints))
		copy(x.Ints, o.Ints)
	}

	if o.Floats != nil {
		x.Floats = make([]float64, len(o.Floats))
		copy(x.Floats, o.Floats)
	}

	if o.Strings != nil {
		x.Strings = make([]string, len(o.Strings))
		copy(x.Strings, o.Strings)
	}

	if o.Keys != nil {
		x.Keys = make([]byte, len(o.Keys))
		copy(x.Keys, o.Keys)
	}

	if o.Bytes != nil {
		x.Bytes = make([][]byte, len(o.Bytes))
		for i, b := range o.Bytes {
			x.Bytes[i] = make([]byte, len(b))
			copy(x.Bytes[i], b)
		}
	}

	if o.Funcs != nil {
		x.Funcs = make([]Func_t, len(o.Funcs))
		copy(x.Funcs, o.Funcs)
	}
	return
}

// genetic algorithm routines //////////////////////////////////////////////////////////////////////

// crossover functions
type IntCxFunc_t func(a, b, A, B []int, ncuts int, cuts []int, pc float64) (ends []int)
type FltCxFunc_t func(a, b, A, B []float64, ncuts int, cuts []int, pc float64) (ends []int)
type StrCxFunc_t func(a, b, A, B []string, ncuts int, cuts []int, pc float64) (ends []int)
type KeyCxFunc_t func(a, b, A, B []byte, ncuts int, cuts []int, pc float64) (ends []int)
type BytCxFunc_t func(a, b, A, B [][]byte, ncuts int, cuts []int, pc float64) (ends []int)
type FunCxFunc_t func(a, b, A, B []Func_t, ncuts int, cuts []int, pc float64) (ends []int)

// Crossover performs the crossover between chromosomes of two individuals A and B
// resulting in the chromosomes of other two individuals a and b
//  Input:
//   A and B -- parents
//   ncuts   -- number of cuts. keys are: 'int', 'flt', 'str', 'key', 'byt', 'fun'
//              ncuts can be nil if 'cuts' is provided
//   cuts    -- positions for cuts in the augmented/whole chromosome
//              len(cuts) == 6: {int, flt, str, key, byt, fun
//              cuts == nil indicates ncuts is to be used instead
//   probs   -- probabilities. use nil for default values
//   cxfucns -- crossover functions. use nil for default ones
//  Output:
//   a and b -- offspring
func Crossover(a, b, A, B *Individual, ncuts map[string]int, cuts map[string][]int, probs map[string]float64, cxfuncs ...interface{}) {

	// default values
	pc := func(t string) float64 {
		if val, ok := probs[t]; ok {
			return val
		}
		return 0.8
	}

	// default functions
	intcxf := IntCrossover
	fltcxf := FltCrossover
	strcxf := StrCrossover
	keycxf := KeyCrossover
	bytcxf := BytCrossover
	funcxf := FunCrossover

	// perform crossover
	if A.Ints != nil {
		intcxf(a.Ints, b.Ints, A.Ints, B.Ints, ncuts["int"], cuts["int"], pc("int"))
	}
	if A.Floats != nil {
		fltcxf(a.Floats, b.Floats, A.Floats, B.Floats, ncuts["flt"], cuts["flt"], pc("flt"))
	}
	if A.Strings != nil {
		strcxf(a.Strings, b.Strings, A.Strings, B.Strings, ncuts["str"], cuts["str"], pc("str"))
	}
	if A.Keys != nil {
		keycxf(a.Keys, b.Keys, A.Keys, B.Keys, ncuts["key"], cuts["key"], pc("key"))
	}
	if A.Bytes != nil {
		bytcxf(a.Bytes, b.Bytes, A.Bytes, B.Bytes, ncuts["byt"], cuts["byt"], pc("byt"))
	}
	if A.Funcs != nil {
		funcxf(a.Funcs, b.Funcs, A.Funcs, B.Funcs, ncuts["fun"], cuts["fun"], pc("fun"))
	}
}

// mutation functions
type IntMutFunc_t func(a []int, pm float64)
type FltMutFunc_t func(a []float64, pm float64)
type StrMutFunc_t func(a []string, pm float64)
type KeyMutFunc_t func(a []byte, pm float64)
type BytMutFunc_t func(a [][]byte, pm float64)
type FunMutFunc_t func(a []Func_t, pm float64)

// handle bases ////////////////////////////////////////////////////////////////////////////////////

// SetFloat returns the float corresponding to gene 'i'
//  igene -- is the index of gene/float in [0, Nfloats]
func (o *Individual) SetFloat(igene int, x float64) {
	if o.Nbases > 1 {
		values := make([]float64, o.Nbases)
		rnd.Float64s(values, 0, 1)
		sum := la.VecAccum(values)
		for j := 0; j < o.Nbases; j++ {
			o.Floats[igene*o.Nbases+j] = x * values[j] / sum
		}
		return
	}
	o.Floats[igene] = x
}

// GetFloat returns the float corresponding to gene 'i'
//  igene -- is the index of gene/float in [0, Nfloats]
func (o Individual) GetFloat(igene int) (x float64) {
	if o.Nbases > 1 {
		for j := 0; j < o.Nbases; j++ {
			x += o.Floats[igene*o.Nbases+j]
		}
		return
	}
	return o.Floats[igene]
}

// output //////////////////////////////////////////////////////////////////////////////////////////

// GetStringSizes returns the sizes of strings representing each gene type
//  sizes -- [6][...] sizes of strings for {int, flt, string, byte, bytes, func}
func (o *Individual) GetStringSizes() (sizes [][]int) {

	sizes = make([][]int, 6)
	if o.Ints != nil {
		sizes[0] = make([]int, len(o.Ints))
		for i, x := range o.Ints {
			sizes[0][i] = imax(sizes[0][i], len(io.Sf("%v", x)))
		}
	}

	if o.Floats != nil {
		sizes[1] = make([]int, o.Nfloats)
		for i := 0; i < o.Nfloats; i++ {
			x := o.Floats[i]
			if o.Nbases > 1 {
				x = 0
				for j := 0; j < o.Nbases; j++ {
					x += o.Floats[i*o.Nbases+j]
				}
			}
			sizes[1][i] = imax(sizes[1][i], len(io.Sf("%v", x)))
		}
	}

	if o.Strings != nil {
		sizes[2] = make([]int, len(o.Strings))
		for i, x := range o.Strings {
			sizes[2][i] = imax(sizes[2][i], len(io.Sf("%v", x)))
		}
	}

	if o.Keys != nil {
		sizes[3] = make([]int, len(o.Keys))
		for i, x := range o.Keys {
			sizes[3][i] = imax(sizes[3][i], len(io.Sf("%v", x)))
		}
	}

	if o.Bytes != nil {
		sizes[4] = make([]int, len(o.Bytes))
		for i, x := range o.Bytes {
			sizes[4][i] = imax(sizes[4][i], len(io.Sf("%v", string(x))))
		}
	}

	if o.Funcs != nil {
		sizes[5] = make([]int, len(o.Funcs))
		for i, x := range o.Funcs {
			sizes[5][i] = imax(sizes[5][i], len(io.Sf("%v", x(o))))
		}
	}
	return
}

// Output returns a string representation of this individual
//  fmts -- [6][...] formats of strings for {int, flt, string, byte, bytes, func}
//          use fmts == nil to choose default ones
func (o *Individual) Output(fmts [][]string) (l string) {

	if fmts == nil {
		fmts = [][]string{{" %d"}, {" %g"}, {" %q"}, {" %x"}, {" %q"}, {" %q"}}
	}

	fmt := func(itype, idx int) (s string) {
		s = fmts[itype][0]
		if idx < len(fmts[itype]) {
			s = fmts[itype][idx]
		}
		return
	}

	for i, x := range o.Ints {
		l += io.Sf(fmt(0, i), x)
	}

	for i := 0; i < o.Nfloats; i++ {
		x := o.Floats[i]
		if o.Nbases > 1 {
			x = 0
			for j := 0; j < o.Nbases; j++ {
				x += o.Floats[i*o.Nbases+j]
			}
		}
		l += io.Sf(fmt(1, i), x)
	}

	for i, x := range o.Strings {
		l += io.Sf(fmt(2, i), x)
	}

	for i, x := range o.Keys {
		l += io.Sf(fmt(3, i), x)
	}

	for i, x := range o.Bytes {
		l += io.Sf(fmt(4, i), string(x))
	}

	for i, x := range o.Funcs {
		l += io.Sf(fmt(5, i), x(o))
	}

	return
}
<commit_msg>reading of functions in crossover fixed<commit_after>// Copyright 2015 Dorival de Moraes Pedroso. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package goga

import (
	"github.com/cpmech/gosl/io"
	"github.com/cpmech/gosl/la"
	"github.com/cpmech/gosl/rnd"
)

// Func_t defines a type for a generic function to be used as a gene value
type Func_t func(ind *Individual) string

// Individual implements one individual in a population
type Individual struct {

	// data
	ObjValue float64 // objective value
	Fitness  float64 // fitness
	Nfloats  int     // number of floats
	Nbases   int     // number of bases to split Floats

	// chromosome
	Ints    []int     // integers
	Floats  []float64 // floats [nfloats * nbases]
	Strings []string  // strings
	Keys    []byte    // 1D bytes
	Bytes   [][]byte  // 2D bytes
	Funcs   []Func_t  // functions
}

// NewIndividual allocates a new individual
//  Input:
//   nbases -- used to split genes of floats into smaller parts
//   slices -- slices of ints, floats, strings, bytes, []bytes, and/or Func_t
//  Notes:
//   1) the slices in 'genes' can all be combined to define genes with mixed data;
//   2) the slices can also be nil, except for one of them.
func NewIndividual(nbases int, slices ...interface{}) (o *Individual) {
	o = new(Individual)
	for _, slice := range slices {
		switch s := slice.(type) {
		case []int:
			o.Ints = make([]int, len(s))
			copy(o.Ints, s)

		case []float64:
			o.Nfloats = len(s)
			o.Nbases = nbases
			if o.Nbases > 1 {
				o.Floats = SimpleChromo(s, nbases)
			} else {
				o.Floats = make([]float64, o.Nfloats*o.Nbases)
				copy(o.Floats, s)
			}

		case []string:
			o.Strings = make([]string, len(s))
			copy(o.Strings, s)

		case []byte:
			o.Keys = make([]byte, len(s))
			copy(o.Keys, s)

		case [][]byte:
			o.Bytes = make([][]byte, len(s))
			for i, x := range s {
				o.Bytes[i] = make([]byte, len(x))
				copy(o.Bytes[i], x)
			}

		case []Func_t:
			o.Funcs = make([]Func_t, len(s))
			copy(o.Funcs, s)
		}
	}
	return
}

// GetCopy returns a copy of this individual
func (o Individual) GetCopy() (x *Individual) {

	x = new(Individual)
	x.ObjValue = o.ObjValue
	x.Fitness = o.Fitness
	x.Nfloats = o.Nfloats
	x.Nbases = o.Nbases

	if o.Ints != nil {
		x.Ints = make([]int, len(o.Ints))
		copy(x.Ints, o.Ints)
	}

	if o.Floats != nil {
		x.Floats = make([]float64, len(o.Floats))
		copy(x.Floats, o.Floats)
	}

	if o.Strings != nil {
		x.Strings = make([]string, len(o.Strings))
		copy(x.Strings, o.Strings)
	}

	if o.Keys != nil {
		x.Keys = make([]byte, len(o.Keys))
		copy(x.Keys, o.Keys)
	}

	if o.Bytes != nil {
		x.Bytes = make([][]byte, len(o.Bytes))
		for i, b := range o.Bytes {
			x.Bytes[i] = make([]byte, len(b))
			copy(x.Bytes[i], b)
		}
	}

	if o.Funcs != nil {
		x.Funcs = make([]Func_t, len(o.Funcs))
		copy(x.Funcs, o.Funcs)
	}
	return
}

// genetic algorithm routines //////////////////////////////////////////////////////////////////////

// crossover functions
type IntCxFunc_t func(a, b, A, B []int, ncuts int, cuts []int, pc float64) (ends []int)
type FltCxFunc_t func(a, b, A, B []float64, ncuts int, cuts []int, pc float64) (ends []int)
type StrCxFunc_t func(a, b, A, B []string, ncuts int, cuts []int, pc float64) (ends []int)
type KeyCxFunc_t func(a, b, A, B []byte, ncuts int, cuts []int, pc float64) (ends []int)
type BytCxFunc_t func(a, b, A, B [][]byte, ncuts int, cuts []int, pc float64) (ends []int)
type FunCxFunc_t func(a, b, A, B []Func_t, ncuts int, cuts []int, pc float64) (ends []int)

// Crossover performs the crossover between chromosomes of two individuals A and B
// resulting in the chromosomes of other two individuals a and b
//  Input:
//   A and B -- parents
//   ncuts   -- number of cuts. keys are: 'int', 'flt', 'str', 'key', 'byt', 'fun'
//              ncuts can be nil if 'cuts' is provided
//   cuts    -- positions for cuts in the augmented/whole chromosome
//              len(cuts) == 6: {int, flt, str, key, byt, fun
//              cuts == nil indicates ncuts is to be used instead
//   probs   -- probabilities. use nil for default values
//   cxfucns -- crossover functions. use nil for default ones
//  Output:
//   a and b -- offspring
func Crossover(a, b, A, B *Individual, ncuts map[string]int, cuts map[string][]int, probs map[string]float64, cxfuncs ...interface{}) {

	// default values
	pc := func(t string) float64 {
		if val, ok := probs[t]; ok {
			return val
		}
		return 0.8
	}

	// default functions
	intcxf := IntCrossover
	fltcxf := FltCrossover
	strcxf := StrCrossover
	keycxf := KeyCrossover
	bytcxf := BytCrossover
	funcxf := FunCrossover
	for _, fcn := range cxfuncs {
		switch f := fcn.(type) {
		case IntCxFunc_t:
			intcxf = f
		case FltCxFunc_t:
			fltcxf = f
		case StrCxFunc_t:
			strcxf = f
		case KeyCxFunc_t:
			keycxf = f
		case BytCxFunc_t:
			bytcxf = f
		case FunCxFunc_t:
			funcxf = f
		}
	}

	// perform crossover
	if A.Ints != nil {
		intcxf(a.Ints, b.Ints, A.Ints, B.Ints, ncuts["int"], cuts["int"], pc("int"))
	}
	if A.Floats != nil {
		fltcxf(a.Floats, b.Floats, A.Floats, B.Floats, ncuts["flt"], cuts["flt"], pc("flt"))
	}
	if A.Strings != nil {
		strcxf(a.Strings, b.Strings, A.Strings, B.Strings, ncuts["str"], cuts["str"], pc("str"))
	}
	if A.Keys != nil {
		keycxf(a.Keys, b.Keys, A.Keys, B.Keys, ncuts["key"], cuts["key"], pc("key"))
	}
	if A.Bytes != nil {
		bytcxf(a.Bytes, b.Bytes, A.Bytes, B.Bytes, ncuts["byt"], cuts["byt"], pc("byt"))
	}
	if A.Funcs != nil {
		funcxf(a.Funcs, b.Funcs, A.Funcs, B.Funcs, ncuts["fun"], cuts["fun"], pc("fun"))
	}
}

// mutation functions
type IntMutFunc_t func(a []int, pm float64)
type FltMutFunc_t func(a []float64, pm float64)
type StrMutFunc_t func(a []string, pm float64)
type KeyMutFunc_t func(a []byte, pm float64)
type BytMutFunc_t func(a [][]byte, pm float64)
type FunMutFunc_t func(a []Func_t, pm float64)

// handle bases ////////////////////////////////////////////////////////////////////////////////////

// SetFloat returns the float corresponding to gene 'i'
//  igene -- is the index of gene/float in [0, Nfloats]
func (o *Individual) SetFloat(igene int, x float64) {
	if o.Nbases > 1 {
		values := make([]float64, o.Nbases)
		rnd.Float64s(values, 0, 1)
		sum := la.VecAccum(values)
		for j := 0; j < o.Nbases; j++ {
			o.Floats[igene*o.Nbases+j] = x * values[j] / sum
		}
		return
	}
	o.Floats[igene] = x
}

// GetFloat returns the float corresponding to gene 'i'
//  igene -- is the index of gene/float in [0, Nfloats]
func (o Individual) GetFloat(igene int) (x float64) {
	if o.Nbases > 1 {
		for j := 0; j < o.Nbases; j++ {
			x += o.Floats[igene*o.Nbases+j]
		}
		return
	}
	return o.Floats[igene]
}

// output //////////////////////////////////////////////////////////////////////////////////////////

// GetStringSizes returns the sizes of strings representing each gene type
//  sizes -- [6][...] sizes of strings for {int, flt, string, byte, bytes, func}
func (o *Individual) GetStringSizes() (sizes [][]int) {

	sizes = make([][]int, 6)
	if o.Ints != nil {
		sizes[0] = make([]int, len(o.Ints))
		for i, x := range o.Ints {
			sizes[0][i] = imax(sizes[0][i], len(io.Sf("%v", x)))
		}
	}

	if o.Floats != nil {
		sizes[1] = make([]int, o.Nfloats)
		for i := 0; i < o.Nfloats; i++ {
			x := o.Floats[i]
			if o.Nbases > 1 {
				x = 0
				for j := 0; j < o.Nbases; j++ {
					x += o.Floats[i*o.Nbases+j]
				}
			}
			sizes[1][i] = imax(sizes[1][i], len(io.Sf("%v", x)))
		}
	}

	if o.Strings != nil {
		sizes[2] = make([]int, len(o.Strings))
		for i, x := range o.Strings {
			sizes[2][i] = imax(sizes[2][i], len(io.Sf("%v", x)))
		}
	}

	if o.Keys != nil {
		sizes[3] = make([]int, len(o.Keys))
		for i, x := range o.Keys {
			sizes[3][i] = imax(sizes[3][i], len(io.Sf("%v", x)))
		}
	}

	if o.Bytes != nil {
		sizes[4] = make([]int, len(o.Bytes))
		for i, x := range o.Bytes {
			sizes[4][i] = imax(sizes[4][i], len(io.Sf("%v", string(x))))
		}
	}

	if o.Funcs != nil {
		sizes[5] = make([]int, len(o.Funcs))
		for i, x := range o.Funcs {
			sizes[5][i] = imax(sizes[5][i], len(io.Sf("%v", x(o))))
		}
	}
	return
}

// Output returns a string representation of this individual
//  fmts -- [6][...] formats of strings for {int, flt, string, byte, bytes, func}
//          use fmts == nil to choose default ones
func (o *Individual) Output(fmts [][]string) (l string) {

	if fmts == nil {
		fmts = [][]string{{" %d"}, {" %g"}, {" %q"}, {" %x"}, {" %q"}, {" %q"}}
	}

	fmt := func(itype, idx int) (s string) {
		s = fmts[itype][0]
		if idx < len(fmts[itype]) {
			s = fmts[itype][idx]
		}
		return
	}

	for i, x := range o.Ints {
		l += io.Sf(fmt(0, i), x)
	}

	for i := 0; i < o.Nfloats; i++ {
		x := o.Floats[i]
		if o.Nbases > 1 {
			x = 0
			for j := 0; j < o.Nbases; j++ {
				x += o.Floats[i*o.Nbases+j]
			}
		}
		l += io.Sf(fmt(1, i), x)
	}

	for i, x := range o.Strings {
		l += io.Sf(fmt(2, i), x)
	}

	for i, x := range o.Keys {
		l += io.Sf(fmt(3, i), x)
	}

	for i, x := range o.Bytes {
		l += io.Sf(fmt(4, i), string(x))
	}

	for i, x := range o.Funcs {
		l += io.Sf(fmt(5, i), x(o))
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2016 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package input

import (
	"errors"
	"io/ioutil"
	"path/filepath"
	"strings"

	"github.com/caixw/apidoc/locale"
)

// 所有支持的语言模型定义
//
// NOTE: 应该保持键名为非大写，按字母顺序排列，方便查找。
// langs 应该和 langExts 保持一一对应关系。
var langs = map[string][]blocker{
	// C#
	"c#": cStyle,

	// c/c++
	"c++": cStyle,

	// d
	"d": cStyle,

	// erlang
	"erlang": []blocker{
		&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
		&block{Type: blockTypeSComment, Begin: `%`},
	},

	// golang
	"go": []blocker{
		&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
		&block{Type: blockTypeString, Begin: "`", End: "`"},
		&block{Type: blockTypeSComment, Begin: `//`},
		&block{Type: blockTypeMComment, Begin: `/*`, End: `*/`},
	},

	// groovy
	"groovy": []blocker{
		&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
		&block{Type: blockTypeString, Begin: "'", End: "'", Escape: `\`},
		&block{Type: blockTypeString, Begin: "'''", End: "'''", Escape: `\`},
		&block{Type: blockTypeSComment, Begin: `//`},
		&block{Type: blockTypeMComment, Begin: `/*`, End: `*/`},
	},

	// java
	"java": cStyle,

	// javascript
	"javascript": []blocker{
		&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
		&block{Type: blockTypeString, Begin: "'", End: "'", Escape: `\`},
		&block{Type: blockTypeSComment, Begin: `//`},
		&block{Type: blockTypeMComment, Begin: `/*`, End: `*/`},
		// NOTE: js 中若出现 /*abc/.test() 应该是先优先注释的。放最后，优先匹配 // 和 /*
		&block{Type: blockTypeString, Begin: "/", End: "/", Escape: `\`}, // 正则表达式
	},

	// pascal/delphi
	"pascal": []blocker{
		newPascalStringBlock('\''),
		newPascalStringBlock('"'),
		&block{Type: blockTypeMComment, Begin: "{", End: "}"},
		&block{Type: blockTypeMComment, Begin: "(*", End: "*)"},
	},

	// perl
	"perl": []blocker{
		&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
		&block{Type: blockTypeString, Begin: "'", End: "'", Escape: `\`},
		&block{Type: blockTypeSComment, Begin: `#`},
		&block{Type: blockTypeMComment, Begin: "\n=pod\n", End: "\n=cut\n"},
	},

	// python
	"python": []blocker{
		&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
		&block{Type: blockTypeSComment, Begin: `#`},
	},

	// php
	"php": []blocker{
		&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
		&block{Type: blockTypeString, Begin: "'", End: "'", Escape: `\`},
		&block{Type: blockTypeSComment, Begin: `//`},
		&block{Type: blockTypeMComment, Begin: `/*`, End: `*/`},
	},

	// ruby
	"ruby": []blocker{
		&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
		&block{Type: blockTypeString, Begin: "'", End: "'", Escape: `\`},
		&block{Type: blockTypeSComment, Begin: `#`},
		&block{Type: blockTypeMComment, Begin: "\n=begin\n", End: "\n=end\n"},
	},

	// rust
	"rust": []blocker{
		&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
		&block{Type: blockTypeSComment, Begin: `///`}, // 需要在 // 之前定义
		&block{Type: blockTypeSComment, Begin: `//`},
		&block{Type: blockTypeMComment, Begin: `/*`, End: `*/`},
	},

	// scala
	"scala": cStyle,

	// swift
	"swift": []blocker{
		&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
		&block{Type: blockTypeSComment, Begin: `//`},
		newSwiftNestMCommentBlock("/*", "*/"),
	},
}

var cStyle = []blocker{
	&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
	&block{Type: blockTypeSComment, Begin: `//`},
	&block{Type: blockTypeMComment, Begin: `/*`, End: `*/`},
}

// 各语言默认支持的文件扩展名。
//
// NOTE: 键名与 langs 中的键一一对应。
// 键值为可能的文件扩展名列表，应该使用非大写状态。
// `go test` 会作大量名称是否规范的检测。
var langExts = map[string][]string{
	"c#":         []string{".cs"},
	"c++":        []string{".h", ".c", ".cpp", ".cxx", "hpp"},
	"d":          []string{".d"},
	"erlang":     []string{".erl", "hrl"},
	"go":         []string{".go"},
	"groovy":     []string{".groovy"},
	"java":       []string{".java"},
	"javascript": []string{".js"},
	"pascal":     []string{".pas", ".pp"},
	"perl":       []string{".perl", ".prl", ".pl"},
	"php":        []string{".php"},
	"python":     []string{".py"},
	"ruby":       []string{".rb"},
	"rust":       []string{".rs"},
	"scala":      []string{".scala"},
	"swift":      []string{".swift"},
}

// Langs 返回所有支持的语言
func Langs() []string {
	ret := make([]string, 0, len(langs))
	for l := range langs {
		ret = append(ret, l)
	}

	return ret
}

// DetectDirLang 检测指定目录下的语言类型。
//
// 检测依据为根据扩展名来做统计，数量最大且被支持的获胜。
// 不会分析子目录。
func DetectDirLang(dir string) (string, error) {
	fs, err := ioutil.ReadDir(dir)
	if err != nil {
		return "", err
	}

	// langsMap 记录每个支持的语言对应的文件数量
	langsMap := make(map[string]int, len(fs))
	for _, f := range fs { // 遍历所有的文件
		if f.IsDir() {
			continue
		}

		ext := strings.ToLower(filepath.Ext(f.Name()))
		lang := getLangByExt(ext)
		if len(lang) > 0 {
			langsMap[lang]++
		}
	}

	if len(langsMap) == 0 {
		return "", errors.New(locale.Sprintf(locale.ErrNotFoundSupportedLang))
	}

	lang := ""
	cnt := 0
	for k, v := range langsMap {
		if v >= cnt {
			lang = k
			cnt = v
		}
	}

	if len(lang) > 0 {
		return lang, nil
	}
	return "", errors.New(locale.Sprintf(locale.ErrNotFoundSupportedLang))
}

// 根据扩展名获取其对应的语言名称。
// 若返回空值，则表示没有找到对应的。
func getLangByExt(ext string) string {
	ext = strings.ToLower(ext)
	for lang, exts := range langExts {
		for _, elem := range exts {
			if elem == ext {
				return lang
			}
		}
	}
	return ""
}

// 是否支持该语言
func langIsSupported(lang string) bool {
	// 由测试函数保证 langs 和 langExts 拥有相同的键名。
	_, found := langs[lang]
	return found
}
<commit_msg>Add python docstring support<commit_after>// Copyright 2016 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package input

import (
	"errors"
	"io/ioutil"
	"path/filepath"
	"strings"

	"github.com/caixw/apidoc/locale"
)

// 所有支持的语言模型定义
//
// NOTE: 应该保持键名为非大写，按字母顺序排列，方便查找。
// langs 应该和 langExts 保持一一对应关系。
var langs = map[string][]blocker{
	// C#
	"c#": cStyle,

	// c/c++
	"c++": cStyle,

	// d
	"d": cStyle,

	// erlang
	"erlang": []blocker{
		&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
		&block{Type: blockTypeSComment, Begin: `%`},
	},

	// golang
	"go": []blocker{
		&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
		&block{Type: blockTypeString, Begin: "`", End: "`"},
		&block{Type: blockTypeSComment, Begin: `//`},
		&block{Type: blockTypeMComment, Begin: `/*`, End: `*/`},
	},

	// groovy
	"groovy": []blocker{
		&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
		&block{Type: blockTypeString, Begin: "'", End: "'", Escape: `\`},
		&block{Type: blockTypeString, Begin: "'''", End: "'''", Escape: `\`},
		&block{Type: blockTypeSComment, Begin: `//`},
		&block{Type: blockTypeMComment, Begin: `/*`, End: `*/`},
	},

	// java
	"java": cStyle,

	// javascript
	"javascript": []blocker{
		&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
		&block{Type: blockTypeString, Begin: "'", End: "'", Escape: `\`},
		&block{Type: blockTypeSComment, Begin: `//`},
		&block{Type: blockTypeMComment, Begin: `/*`, End: `*/`},
		// NOTE: js 中若出现 /*abc/.test() 应该是先优先注释的。放最后，优先匹配 // 和 /*
		&block{Type: blockTypeString, Begin: "/", End: "/", Escape: `\`}, // 正则表达式
	},

	// pascal/delphi
	"pascal": []blocker{
		newPascalStringBlock('\''),
		newPascalStringBlock('"'),
		&block{Type: blockTypeMComment, Begin: "{", End: "}"},
		&block{Type: blockTypeMComment, Begin: "(*", End: "*)"},
	},

	// perl
	"perl": []blocker{
		&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
		&block{Type: blockTypeString, Begin: "'", End: "'", Escape: `\`},
		&block{Type: blockTypeSComment, Begin: `#`},
		&block{Type: blockTypeMComment, Begin: "\n=pod\n", End: "\n=cut\n"},
	},

	// python
	"python": []blocker{
		&block{Type: blockTypeMComment, Begin: `"""`, End: `"""`},
		&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
		&block{Type: blockTypeSComment, Begin: `#`},
	},

	// php
	"php": []blocker{
		&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
		&block{Type: blockTypeString, Begin: "'", End: "'", Escape: `\`},
		&block{Type: blockTypeSComment, Begin: `//`},
		&block{Type: blockTypeMComment, Begin: `/*`, End: `*/`},
	},

	// ruby
	"ruby": []blocker{
		&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
		&block{Type: blockTypeString, Begin: "'", End: "'", Escape: `\`},
		&block{Type: blockTypeSComment, Begin: `#`},
		&block{Type: blockTypeMComment, Begin: "\n=begin\n", End: "\n=end\n"},
	},

	// rust
	"rust": []blocker{
		&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
		&block{Type: blockTypeSComment, Begin: `///`}, // 需要在 // 之前定义
		&block{Type: blockTypeSComment, Begin: `//`},
		&block{Type: blockTypeMComment, Begin: `/*`, End: `*/`},
	},

	// scala
	"scala": cStyle,

	// swift
	"swift": []blocker{
		&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
		&block{Type: blockTypeSComment, Begin: `//`},
		newSwiftNestMCommentBlock("/*", "*/"),
	},
}

var cStyle = []blocker{
	&block{Type: blockTypeString, Begin: `"`, End: `"`, Escape: `\`},
	&block{Type: blockTypeSComment, Begin: `//`},
	&block{Type: blockTypeMComment, Begin: `/*`, End: `*/`},
}

// 各语言默认支持的文件扩展名。
//
// NOTE: 键名与 langs 中的键一一对应。
// 键值为可能的文件扩展名列表，应该使用非大写状态。
// `go test` 会作大量名称是否规范的检测。
var langExts = map[string][]string{
	"c#":         []string{".cs"},
	"c++":        []string{".h", ".c", ".cpp", ".cxx", "hpp"},
	"d":          []string{".d"},
	"erlang":     []string{".erl", "hrl"},
	"go":         []string{".go"},
	"groovy":     []string{".groovy"},
	"java":       []string{".java"},
	"javascript": []string{".js"},
	"pascal":     []string{".pas", ".pp"},
	"perl":       []string{".perl", ".prl", ".pl"},
	"php":        []string{".php"},
	"python":     []string{".py"},
	"ruby":       []string{".rb"},
	"rust":       []string{".rs"},
	"scala":      []string{".scala"},
	"swift":      []string{".swift"},
}

// Langs 返回所有支持的语言
func Langs() []string {
	ret := make([]string, 0, len(langs))
	for l := range langs {
		ret = append(ret, l)
	}

	return ret
}

// DetectDirLang 检测指定目录下的语言类型。
//
// 检测依据为根据扩展名来做统计，数量最大且被支持的获胜。
// 不会分析子目录。
func DetectDirLang(dir string) (string, error) {
	fs, err := ioutil.ReadDir(dir)
	if err != nil {
		return "", err
	}

	// langsMap 记录每个支持的语言对应的文件数量
	langsMap := make(map[string]int, len(fs))
	for _, f := range fs { // 遍历所有的文件
		if f.IsDir() {
			continue
		}

		ext := strings.ToLower(filepath.Ext(f.Name()))
		lang := getLangByExt(ext)
		if len(lang) > 0 {
			langsMap[lang]++
		}
	}

	if len(langsMap) == 0 {
		return "", errors.New(locale.Sprintf(locale.ErrNotFoundSupportedLang))
	}

	lang := ""
	cnt := 0
	for k, v := range langsMap {
		if v >= cnt {
			lang = k
			cnt = v
		}
	}

	if len(lang) > 0 {
		return lang, nil
	}
	return "", errors.New(locale.Sprintf(locale.ErrNotFoundSupportedLang))
}

// 根据扩展名获取其对应的语言名称。
// 若返回空值，则表示没有找到对应的。
func getLangByExt(ext string) string {
	ext = strings.ToLower(ext)
	for lang, exts := range langExts {
		for _, elem := range exts {
			if elem == ext {
				return lang
			}
		}
	}
	return ""
}

// 是否支持该语言
func langIsSupported(lang string) bool {
	// 由测试函数保证 langs 和 langExts 拥有相同的键名。
	_, found := langs[lang]
	return found
}
<|endoftext|>


<commit_before>// This is free and unencumbered software released into the public
// domain.  For more information, see <http://unlicense.org> or the
// accompanying UNLICENSE file.

// +build !linux !go1.8

package plugin

import (
	"strings"

	"github.com/nelsam/gxui/themes/basic"
	"github.com/nelsam/vidar/commander/bind"
	"github.com/nelsam/vidar/plugin/comments"
	"github.com/nelsam/vidar/plugin/godef"
	"github.com/nelsam/vidar/plugin/goimports"
	"github.com/nelsam/vidar/plugin/license"
)

type GolangHook struct {
	Theme *basic.Theme
}

func (h GolangHook) Name() string {
	return "golang-hook"
}

func (h GolangHook) CommandName() string {
	return "open-file"
}

func (h GolangHook) FileBindables(path string) []bind.Bindable {
	if !strings.HasSuffix(path, ".go") {
		return nil
	}
	return []bind.Bindable{
		comments.NewToggle(),
		godef.New(h.Theme),
		goimports.New(h.Theme),
		goimports.OnSave{},
		license.NewHeaderUpdate(h.Theme),
	}
}
<commit_msg>Add syntax highlighting for non-plugin architectures<commit_after>// This is free and unencumbered software released into the public
// domain.  For more information, see <http://unlicense.org> or the
// accompanying UNLICENSE file.

// +build !linux !go1.8

package plugin

import (
	"strings"

	"github.com/nelsam/gxui/themes/basic"
	"github.com/nelsam/vidar/commander/bind"
	"github.com/nelsam/vidar/plugin/comments"
	"github.com/nelsam/vidar/plugin/godef"
	"github.com/nelsam/vidar/plugin/goimports"
	"github.com/nelsam/vidar/plugin/gosyntax"
	"github.com/nelsam/vidar/plugin/license"
)

type GolangHook struct {
	Theme *basic.Theme
}

func (h GolangHook) Name() string {
	return "golang-hook"
}

func (h GolangHook) CommandName() string {
	return "open-file"
}

func (h GolangHook) FileBindables(path string) []bind.Bindable {
	if !strings.HasSuffix(path, ".go") {
		return nil
	}
	return []bind.Bindable{
		comments.NewToggle(),
		godef.New(h.Theme),
		goimports.New(h.Theme),
		goimports.OnSave{},
		gosyntax.New(),
		license.NewHeaderUpdate(h.Theme),
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e_node

import (
	"path"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/kubernetes/pkg/kubelet/cm"
	"k8s.io/kubernetes/test/e2e/framework"

	"github.com/golang/glog"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// getResourceList returns a ResourceList with the
// specified cpu and memory resource values
func getResourceList(cpu, memory string) v1.ResourceList {
	res := v1.ResourceList{}
	if cpu != "" {
		res[v1.ResourceCPU] = resource.MustParse(cpu)
	}
	if memory != "" {
		res[v1.ResourceMemory] = resource.MustParse(memory)
	}
	return res
}

// getResourceRequirements returns a ResourceRequirements object
func getResourceRequirements(requests, limits v1.ResourceList) v1.ResourceRequirements {
	res := v1.ResourceRequirements{}
	res.Requests = requests
	res.Limits = limits
	return res
}

const (
	// Kubelet internal cgroup name for node allocatable cgroup.
	defaultNodeAllocatableCgroup = "kubepods"
	// Kubelet internal cgroup name for burstable tier
	burstableCgroup = "burstable"
	// Kubelet internal cgroup name for burstable tier
	bestEffortCgroup = "burstable"
)

// makePodToVerifyCgroups returns a pod that verifies the existence of the specified cgroups.
func makePodToVerifyCgroups(cgroupNames []cm.CgroupName) *v1.Pod {
	// convert the names to their literal cgroupfs forms...
	cgroupFsNames := []string{}
	for _, cgroupName := range cgroupNames {
		// Add top level cgroup used to enforce node allocatable.
		cgroupName = cm.CgroupName(path.Join(defaultNodeAllocatableCgroup, string(cgroupName)))
		if framework.TestContext.KubeletConfig.CgroupDriver == "systemd" {
			cgroupFsNames = append(cgroupFsNames, cm.ConvertCgroupNameToSystemd(cgroupName, true))
		} else {
			cgroupFsNames = append(cgroupFsNames, string(cgroupName))
		}
	}
	glog.Infof("expecting %v cgroups to be found", cgroupFsNames)
	// build the pod command to either verify cgroups exist
	command := ""
	for _, cgroupFsName := range cgroupFsNames {
		localCommand := "if [ ! -d /tmp/memory/" + cgroupFsName + " ] || [ ! -d /tmp/cpu/" + cgroupFsName + " ]; then exit 1; fi; "
		command += localCommand
	}

	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: "pod" + string(uuid.NewUUID()),
		},
		Spec: v1.PodSpec{
			RestartPolicy: v1.RestartPolicyNever,
			Containers: []v1.Container{
				{
					Image:   busyboxImage,
					Name:    "container" + string(uuid.NewUUID()),
					Command: []string{"sh", "-c", command},
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      "sysfscgroup",
							MountPath: "/tmp",
						},
					},
				},
			},
			Volumes: []v1.Volume{
				{
					Name: "sysfscgroup",
					VolumeSource: v1.VolumeSource{
						HostPath: &v1.HostPathVolumeSource{Path: "/sys/fs/cgroup"},
					},
				},
			},
		},
	}
	return pod
}

// makePodToVerifyCgroupRemoved verfies the specified cgroup does not exist.
func makePodToVerifyCgroupRemoved(cgroupName cm.CgroupName) *v1.Pod {
	cgroupFsName := string(cgroupName)
	if framework.TestContext.KubeletConfig.CgroupDriver == "systemd" {
		cgroupFsName = cm.ConvertCgroupNameToSystemd(cm.CgroupName(cgroupName), true)
	}
	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: "pod" + string(uuid.NewUUID()),
		},
		Spec: v1.PodSpec{
			RestartPolicy: v1.RestartPolicyOnFailure,
			Containers: []v1.Container{
				{
					Image:   busyboxImage,
					Name:    "container" + string(uuid.NewUUID()),
					Command: []string{"sh", "-c", "for i in `seq 1 10`; do if [ ! -d /tmp/memory/" + cgroupFsName + " ] && [ ! -d /tmp/cpu/" + cgroupFsName + " ]; then exit 0; else sleep 10; fi; done; exit 1"},
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      "sysfscgroup",
							MountPath: "/tmp",
						},
					},
				},
			},
			Volumes: []v1.Volume{
				{
					Name: "sysfscgroup",
					VolumeSource: v1.VolumeSource{
						HostPath: &v1.HostPathVolumeSource{Path: "/sys/fs/cgroup"},
					},
				},
			},
		},
	}
	return pod
}

var _ = framework.KubeDescribe("Kubelet Cgroup Manager", func() {
	f := framework.NewDefaultFramework("kubelet-cgroup-manager")
	Describe("QOS containers", func() {
		Context("On enabling QOS cgroup hierarchy", func() {
			It("Top level QoS containers should have been created", func() {
				if !framework.TestContext.KubeletConfig.CgroupsPerQOS {
					return
				}
				cgroupsToVerify := []cm.CgroupName{cm.CgroupName(burstableCgroup), cm.CgroupName(bestEffortCgroup)}
				pod := makePodToVerifyCgroups(cgroupsToVerify)
				f.PodClient().Create(pod)
				err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
				Expect(err).NotTo(HaveOccurred())
			})
		})
	})

	Describe("Pod containers", func() {
		Context("On scheduling a Guaranteed Pod", func() {
			It("Pod containers should have been created under the cgroup-root", func() {
				if !framework.TestContext.KubeletConfig.CgroupsPerQOS {
					return
				}
				var (
					guaranteedPod *v1.Pod
					podUID        string
				)
				By("Creating a Guaranteed pod in Namespace", func() {
					guaranteedPod = f.PodClient().Create(&v1.Pod{
						ObjectMeta: metav1.ObjectMeta{
							Name:      "pod" + string(uuid.NewUUID()),
							Namespace: f.Namespace.Name,
						},
						Spec: v1.PodSpec{
							Containers: []v1.Container{
								{
									Image:     framework.GetPauseImageName(f.ClientSet),
									Name:      "container" + string(uuid.NewUUID()),
									Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("100m", "100Mi")),
								},
							},
						},
					})
					podUID = string(guaranteedPod.UID)
				})
				By("Checking if the pod cgroup was created", func() {
					cgroupsToVerify := []cm.CgroupName{cm.CgroupName("pod" + podUID)}
					pod := makePodToVerifyCgroups(cgroupsToVerify)
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
				By("Checking if the pod cgroup was deleted", func() {
					gp := int64(1)
					Expect(f.PodClient().Delete(guaranteedPod.Name, &metav1.DeleteOptions{GracePeriodSeconds: &gp})).NotTo(HaveOccurred())
					pod := makePodToVerifyCgroupRemoved(cm.CgroupName("pod" + podUID))
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
			})
		})
		Context("On scheduling a BestEffort Pod", func() {
			It("Pod containers should have been created under the BestEffort cgroup", func() {
				if !framework.TestContext.KubeletConfig.CgroupsPerQOS {
					return
				}
				var (
					podUID        string
					bestEffortPod *v1.Pod
				)
				By("Creating a BestEffort pod in Namespace", func() {
					bestEffortPod = f.PodClient().Create(&v1.Pod{
						ObjectMeta: metav1.ObjectMeta{
							Name:      "pod" + string(uuid.NewUUID()),
							Namespace: f.Namespace.Name,
						},
						Spec: v1.PodSpec{
							Containers: []v1.Container{
								{
									Image:     framework.GetPauseImageName(f.ClientSet),
									Name:      "container" + string(uuid.NewUUID()),
									Resources: getResourceRequirements(getResourceList("", ""), getResourceList("", "")),
								},
							},
						},
					})
					podUID = string(bestEffortPod.UID)
				})
				By("Checking if the pod cgroup was created", func() {
					cgroupsToVerify := []cm.CgroupName{cm.CgroupName("besteffort/pod" + podUID)}
					pod := makePodToVerifyCgroups(cgroupsToVerify)
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
				By("Checking if the pod cgroup was deleted", func() {
					gp := int64(1)
					Expect(f.PodClient().Delete(bestEffortPod.Name, &metav1.DeleteOptions{GracePeriodSeconds: &gp})).NotTo(HaveOccurred())
					pod := makePodToVerifyCgroupRemoved(cm.CgroupName("besteffort/pod" + podUID))
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
			})
		})
		Context("On scheduling a Burstable Pod", func() {
			It("Pod containers should have been created under the Burstable cgroup", func() {
				if !framework.TestContext.KubeletConfig.CgroupsPerQOS {
					return
				}
				var (
					podUID       string
					burstablePod *v1.Pod
				)
				By("Creating a Burstable pod in Namespace", func() {
					burstablePod = f.PodClient().Create(&v1.Pod{
						ObjectMeta: metav1.ObjectMeta{
							Name:      "pod" + string(uuid.NewUUID()),
							Namespace: f.Namespace.Name,
						},
						Spec: v1.PodSpec{
							Containers: []v1.Container{
								{
									Image:     framework.GetPauseImageName(f.ClientSet),
									Name:      "container" + string(uuid.NewUUID()),
									Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("200m", "200Mi")),
								},
							},
						},
					})
					podUID = string(burstablePod.UID)
				})
				By("Checking if the pod cgroup was created", func() {
					cgroupsToVerify := []cm.CgroupName{cm.CgroupName("burstable/pod" + podUID)}
					pod := makePodToVerifyCgroups(cgroupsToVerify)
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
				By("Checking if the pod cgroup was deleted", func() {
					gp := int64(1)
					Expect(f.PodClient().Delete(burstablePod.Name, &metav1.DeleteOptions{GracePeriodSeconds: &gp})).NotTo(HaveOccurred())
					pod := makePodToVerifyCgroupRemoved(cm.CgroupName("burstable/pod" + podUID))
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
			})
		})
	})
})
<commit_msg>e2e-node:the value of bestEffortCgroup is wrong<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e_node

import (
	"path"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/kubernetes/pkg/kubelet/cm"
	"k8s.io/kubernetes/test/e2e/framework"

	"github.com/golang/glog"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// getResourceList returns a ResourceList with the
// specified cpu and memory resource values
func getResourceList(cpu, memory string) v1.ResourceList {
	res := v1.ResourceList{}
	if cpu != "" {
		res[v1.ResourceCPU] = resource.MustParse(cpu)
	}
	if memory != "" {
		res[v1.ResourceMemory] = resource.MustParse(memory)
	}
	return res
}

// getResourceRequirements returns a ResourceRequirements object
func getResourceRequirements(requests, limits v1.ResourceList) v1.ResourceRequirements {
	res := v1.ResourceRequirements{}
	res.Requests = requests
	res.Limits = limits
	return res
}

const (
	// Kubelet internal cgroup name for node allocatable cgroup.
	defaultNodeAllocatableCgroup = "kubepods"
	// Kubelet internal cgroup name for burstable tier
	burstableCgroup = "burstable"
	// Kubelet internal cgroup name for besteffort tier
	bestEffortCgroup = "besteffort"
)

// makePodToVerifyCgroups returns a pod that verifies the existence of the specified cgroups.
func makePodToVerifyCgroups(cgroupNames []cm.CgroupName) *v1.Pod {
	// convert the names to their literal cgroupfs forms...
	cgroupFsNames := []string{}
	for _, cgroupName := range cgroupNames {
		// Add top level cgroup used to enforce node allocatable.
		cgroupName = cm.CgroupName(path.Join(defaultNodeAllocatableCgroup, string(cgroupName)))
		if framework.TestContext.KubeletConfig.CgroupDriver == "systemd" {
			cgroupFsNames = append(cgroupFsNames, cm.ConvertCgroupNameToSystemd(cgroupName, true))
		} else {
			cgroupFsNames = append(cgroupFsNames, string(cgroupName))
		}
	}
	glog.Infof("expecting %v cgroups to be found", cgroupFsNames)
	// build the pod command to either verify cgroups exist
	command := ""
	for _, cgroupFsName := range cgroupFsNames {
		localCommand := "if [ ! -d /tmp/memory/" + cgroupFsName + " ] || [ ! -d /tmp/cpu/" + cgroupFsName + " ]; then exit 1; fi; "
		command += localCommand
	}

	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: "pod" + string(uuid.NewUUID()),
		},
		Spec: v1.PodSpec{
			RestartPolicy: v1.RestartPolicyNever,
			Containers: []v1.Container{
				{
					Image:   busyboxImage,
					Name:    "container" + string(uuid.NewUUID()),
					Command: []string{"sh", "-c", command},
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      "sysfscgroup",
							MountPath: "/tmp",
						},
					},
				},
			},
			Volumes: []v1.Volume{
				{
					Name: "sysfscgroup",
					VolumeSource: v1.VolumeSource{
						HostPath: &v1.HostPathVolumeSource{Path: "/sys/fs/cgroup"},
					},
				},
			},
		},
	}
	return pod
}

// makePodToVerifyCgroupRemoved verfies the specified cgroup does not exist.
func makePodToVerifyCgroupRemoved(cgroupName cm.CgroupName) *v1.Pod {
	cgroupFsName := string(cgroupName)
	if framework.TestContext.KubeletConfig.CgroupDriver == "systemd" {
		cgroupFsName = cm.ConvertCgroupNameToSystemd(cm.CgroupName(cgroupName), true)
	}
	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: "pod" + string(uuid.NewUUID()),
		},
		Spec: v1.PodSpec{
			RestartPolicy: v1.RestartPolicyOnFailure,
			Containers: []v1.Container{
				{
					Image:   busyboxImage,
					Name:    "container" + string(uuid.NewUUID()),
					Command: []string{"sh", "-c", "for i in `seq 1 10`; do if [ ! -d /tmp/memory/" + cgroupFsName + " ] && [ ! -d /tmp/cpu/" + cgroupFsName + " ]; then exit 0; else sleep 10; fi; done; exit 1"},
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      "sysfscgroup",
							MountPath: "/tmp",
						},
					},
				},
			},
			Volumes: []v1.Volume{
				{
					Name: "sysfscgroup",
					VolumeSource: v1.VolumeSource{
						HostPath: &v1.HostPathVolumeSource{Path: "/sys/fs/cgroup"},
					},
				},
			},
		},
	}
	return pod
}

var _ = framework.KubeDescribe("Kubelet Cgroup Manager", func() {
	f := framework.NewDefaultFramework("kubelet-cgroup-manager")
	Describe("QOS containers", func() {
		Context("On enabling QOS cgroup hierarchy", func() {
			It("Top level QoS containers should have been created", func() {
				if !framework.TestContext.KubeletConfig.CgroupsPerQOS {
					return
				}
				cgroupsToVerify := []cm.CgroupName{cm.CgroupName(burstableCgroup), cm.CgroupName(bestEffortCgroup)}
				pod := makePodToVerifyCgroups(cgroupsToVerify)
				f.PodClient().Create(pod)
				err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
				Expect(err).NotTo(HaveOccurred())
			})
		})
	})

	Describe("Pod containers", func() {
		Context("On scheduling a Guaranteed Pod", func() {
			It("Pod containers should have been created under the cgroup-root", func() {
				if !framework.TestContext.KubeletConfig.CgroupsPerQOS {
					return
				}
				var (
					guaranteedPod *v1.Pod
					podUID        string
				)
				By("Creating a Guaranteed pod in Namespace", func() {
					guaranteedPod = f.PodClient().Create(&v1.Pod{
						ObjectMeta: metav1.ObjectMeta{
							Name:      "pod" + string(uuid.NewUUID()),
							Namespace: f.Namespace.Name,
						},
						Spec: v1.PodSpec{
							Containers: []v1.Container{
								{
									Image:     framework.GetPauseImageName(f.ClientSet),
									Name:      "container" + string(uuid.NewUUID()),
									Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("100m", "100Mi")),
								},
							},
						},
					})
					podUID = string(guaranteedPod.UID)
				})
				By("Checking if the pod cgroup was created", func() {
					cgroupsToVerify := []cm.CgroupName{cm.CgroupName("pod" + podUID)}
					pod := makePodToVerifyCgroups(cgroupsToVerify)
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
				By("Checking if the pod cgroup was deleted", func() {
					gp := int64(1)
					Expect(f.PodClient().Delete(guaranteedPod.Name, &metav1.DeleteOptions{GracePeriodSeconds: &gp})).NotTo(HaveOccurred())
					pod := makePodToVerifyCgroupRemoved(cm.CgroupName("pod" + podUID))
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
			})
		})
		Context("On scheduling a BestEffort Pod", func() {
			It("Pod containers should have been created under the BestEffort cgroup", func() {
				if !framework.TestContext.KubeletConfig.CgroupsPerQOS {
					return
				}
				var (
					podUID        string
					bestEffortPod *v1.Pod
				)
				By("Creating a BestEffort pod in Namespace", func() {
					bestEffortPod = f.PodClient().Create(&v1.Pod{
						ObjectMeta: metav1.ObjectMeta{
							Name:      "pod" + string(uuid.NewUUID()),
							Namespace: f.Namespace.Name,
						},
						Spec: v1.PodSpec{
							Containers: []v1.Container{
								{
									Image:     framework.GetPauseImageName(f.ClientSet),
									Name:      "container" + string(uuid.NewUUID()),
									Resources: getResourceRequirements(getResourceList("", ""), getResourceList("", "")),
								},
							},
						},
					})
					podUID = string(bestEffortPod.UID)
				})
				By("Checking if the pod cgroup was created", func() {
					cgroupsToVerify := []cm.CgroupName{cm.CgroupName("besteffort/pod" + podUID)}
					pod := makePodToVerifyCgroups(cgroupsToVerify)
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
				By("Checking if the pod cgroup was deleted", func() {
					gp := int64(1)
					Expect(f.PodClient().Delete(bestEffortPod.Name, &metav1.DeleteOptions{GracePeriodSeconds: &gp})).NotTo(HaveOccurred())
					pod := makePodToVerifyCgroupRemoved(cm.CgroupName("besteffort/pod" + podUID))
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
			})
		})
		Context("On scheduling a Burstable Pod", func() {
			It("Pod containers should have been created under the Burstable cgroup", func() {
				if !framework.TestContext.KubeletConfig.CgroupsPerQOS {
					return
				}
				var (
					podUID       string
					burstablePod *v1.Pod
				)
				By("Creating a Burstable pod in Namespace", func() {
					burstablePod = f.PodClient().Create(&v1.Pod{
						ObjectMeta: metav1.ObjectMeta{
							Name:      "pod" + string(uuid.NewUUID()),
							Namespace: f.Namespace.Name,
						},
						Spec: v1.PodSpec{
							Containers: []v1.Container{
								{
									Image:     framework.GetPauseImageName(f.ClientSet),
									Name:      "container" + string(uuid.NewUUID()),
									Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("200m", "200Mi")),
								},
							},
						},
					})
					podUID = string(burstablePod.UID)
				})
				By("Checking if the pod cgroup was created", func() {
					cgroupsToVerify := []cm.CgroupName{cm.CgroupName("burstable/pod" + podUID)}
					pod := makePodToVerifyCgroups(cgroupsToVerify)
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
				By("Checking if the pod cgroup was deleted", func() {
					gp := int64(1)
					Expect(f.PodClient().Delete(burstablePod.Name, &metav1.DeleteOptions{GracePeriodSeconds: &gp})).NotTo(HaveOccurred())
					pod := makePodToVerifyCgroupRemoved(cm.CgroupName("burstable/pod" + podUID))
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
			})
		})
	})
})
<|endoftext|>


<commit_before>package main

import (
	"compress/bzip2"
	"database/sql"
	"flag"
	"fmt"
	"github.com/semanticize/st/hash"
	"github.com/semanticize/st/hash/countmin"
	"github.com/semanticize/st/nlp"
	"github.com/semanticize/st/storage"
	"github.com/semanticize/st/dumpparser/wikidump"
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"sync"
)

func init() {
	if os.Getenv("GOMAXPROCS") == "" {
		// Four is about the number of cores that we can put to useful work
		// when the disk is fast.
		runtime.GOMAXPROCS(min(runtime.NumCPU(), 4))
	}
}

func open(path string) (r io.ReadCloser, err error) {
	r, err = os.Open(path)
	if err == nil && filepath.Ext(path) == ".bz2" {
		r = struct {
			io.Reader
			io.Closer
		}{bzip2.NewReader(r), r}
	}
	return
}

var download = flag.String("download", "",
	"download Wikipedia dump (e.g., 'enwiki')")
var nrows = flag.Uint("nrows", 16, "number of rows in count-min sketch")
var ncols = flag.Uint("ncols", 262144, "number of columns in count-min sketch")
var maxNGram = flag.Uint("ngram", storage.DefaultMaxNGram,
	"max. length of n-grams")

func main() {
	log.SetPrefix("dumpparser ")

	var err error
	check := func() {
		if err != nil {
			log.Fatal(err)
		}
	}

	flag.Parse()
	args := flag.Args()

	var dbpath, inputpath string
	if *download != "" {
		if len(args) != 1 {
			fmt.Fprintf(os.Stderr,
				"usage: %s -download=wikiname model.db\n", os.Args[0])
			os.Exit(1)
		}
		inputpath, err = wikidump.Download(*download, true)
		check()
		dbpath = args[0]
	} else {
		if len(args) != 2 {
			fmt.Fprintf(os.Stderr, "usage: %s wikidump model.db\n", os.Args[0])
			os.Exit(1)
		}
		inputpath, dbpath = args[0], args[1]
	}

	f, err := open(inputpath)
	check()
	defer f.Close()

	log.Printf("Creating database at %s", dbpath)
	db, err := storage.MakeDB(dbpath, true,
		&storage.Settings{inputpath, *maxNGram})
	check()

	// The numbers here are completely arbitrary.
	nworkers := runtime.GOMAXPROCS(0)
	articles := make(chan *wikidump.Page, 10*nworkers)
	links := make(chan map[wikidump.Link]int, 10*nworkers)
	redirects := make(chan *wikidump.Redirect, 100)

	var wg sync.WaitGroup

	// Collect redirects.
	wg.Add(1)
	redirmap := make(map[string]string)
	go func() {
		for r := range redirects {
			redirmap[r.Title] = r.Target
		}
		wg.Done()
	}()

	// Clean up and tokenize articles, extract links, count n-grams.
	maxN := int(*maxNGram)
	counters := make([]*countmin.Sketch, nworkers)

	var worker sync.WaitGroup
	worker.Add(nworkers)
	log.Printf("%d workers", nworkers)
	for i := 0; i < nworkers; i++ {
		counters[i], err = countmin.New(int(*nrows), int(*ncols))
		check()

		go func(ngramcount *countmin.Sketch) {
			for a := range articles {
				text := wikidump.Cleanup(a.Text)
				links <- wikidump.ExtractLinks(text)

				tokens := nlp.Tokenize(text)
				for _, h := range hash.NGrams(tokens, 1, maxN) {
					ngramcount.Add1(h)
				}
			}
			worker.Done()
		}(counters[i])
	}

	wg.Add(1)
	go func() {
		worker.Wait()
		close(links)

		for i := 1; i < nworkers; i++ {
			counters[0].Sum(counters[i])
		}
		counters = counters[:1]

		wg.Done()
	}()

	// Collect links and store them in the database.
	wg.Add(1)
	done := make(chan struct{})
	go func() {
		if slerr := storeLinks(db, links, maxN); slerr != nil {
			panic(slerr)
		}
		wg.Done()
	}()

	go wikidump.GetPages(f, articles, redirects)

	wg.Wait()
	close(done)

	log.Printf("Processing %d redirects", len(redirmap))
	storage.ProcessRedirects(db, redirmap)

	err = storage.StoreCM(db, counters[0])
	check()

	log.Println("Finalizing database")
	err = storage.Finalize(db)
	check()
	err = db.Close()
	check()
}

func storeLinks(db *sql.DB, links <-chan map[wikidump.Link]int,
	maxN int) (err error) {

	insTitle, err := db.Prepare(`insert or ignore into titles values (NULL, ?)`)
	if err != nil {
		return
	}
	insLink, err := db.Prepare(
		`insert or ignore into linkstats values
		 (?, (select id from titles where title = ?), 0)`)
	if err != nil {
		return
	}
	update, err := db.Prepare(
		`update linkstats set count = count + ?
		 where ngramhash = ?
		 and targetid = (select id from titles where title =?)`)
	if err != nil {
		return
	}

	for linkFreq := range links {
		for link, freq := range linkFreq {
			tokens := nlp.Tokenize(link.Anchor)
			n := min(maxN, len(tokens))
			hashes := hash.NGrams(tokens, n, n)
			count := float64(freq)
			if len(hashes) > 1 {
				count = 1 / float64(len(hashes))
			}
			for _, h := range hashes {
				_, err = insTitle.Exec(link.Target)
				if err != nil {
					return
				}
				_, err = insLink.Exec(h, link.Target)
				if err != nil {
					return
				}
				_, err = update.Exec(count, h, link.Target)
				if err != nil {
					return
				}
			}
		}
	}
	return
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}
<commit_msg>buffered reader in dumpparser<commit_after>package main

import (
	"bufio"
	"compress/bzip2"
	"database/sql"
	"flag"
	"fmt"
	"github.com/semanticize/st/hash"
	"github.com/semanticize/st/hash/countmin"
	"github.com/semanticize/st/nlp"
	"github.com/semanticize/st/storage"
	"github.com/semanticize/st/dumpparser/wikidump"
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"sync"
)

func init() {
	if os.Getenv("GOMAXPROCS") == "" {
		// Four is about the number of cores that we can put to useful work
		// when the disk is fast.
		runtime.GOMAXPROCS(min(runtime.NumCPU(), 4))
	}
}

func open(path string) (r io.ReadCloser, err error) {
	rf, err := os.Open(path)
	if err != nil {
		return
	}
	r = struct {
		*bufio.Reader
		io.Closer
	}{bufio.NewReader(rf), rf}
	if filepath.Ext(path) == ".bz2" {
		r = struct {
			io.Reader
			io.Closer
		}{bzip2.NewReader(r), rf}
	}
	return
}

var download = flag.String("download", "",
	"download Wikipedia dump (e.g., 'enwiki')")
var nrows = flag.Uint("nrows", 16, "number of rows in count-min sketch")
var ncols = flag.Uint("ncols", 262144, "number of columns in count-min sketch")
var maxNGram = flag.Uint("ngram", storage.DefaultMaxNGram,
	"max. length of n-grams")

func main() {
	log.SetPrefix("dumpparser ")

	var err error
	check := func() {
		if err != nil {
			log.Fatal(err)
		}
	}

	flag.Parse()
	args := flag.Args()

	var dbpath, inputpath string
	if *download != "" {
		if len(args) != 1 {
			fmt.Fprintf(os.Stderr,
				"usage: %s -download=wikiname model.db\n", os.Args[0])
			os.Exit(1)
		}
		inputpath, err = wikidump.Download(*download, true)
		check()
		dbpath = args[0]
	} else {
		if len(args) != 2 {
			fmt.Fprintf(os.Stderr, "usage: %s wikidump model.db\n", os.Args[0])
			os.Exit(1)
		}
		inputpath, dbpath = args[0], args[1]
	}

	f, err := open(inputpath)
	check()
	defer f.Close()

	log.Printf("Creating database at %s", dbpath)
	db, err := storage.MakeDB(dbpath, true,
		&storage.Settings{inputpath, *maxNGram})
	check()

	// The numbers here are completely arbitrary.
	nworkers := runtime.GOMAXPROCS(0)
	articles := make(chan *wikidump.Page, 10*nworkers)
	links := make(chan map[wikidump.Link]int, 10*nworkers)
	redirects := make(chan *wikidump.Redirect, 100)

	var wg sync.WaitGroup

	// Collect redirects.
	wg.Add(1)
	redirmap := make(map[string]string)
	go func() {
		for r := range redirects {
			redirmap[r.Title] = r.Target
		}
		wg.Done()
	}()

	// Clean up and tokenize articles, extract links, count n-grams.
	maxN := int(*maxNGram)
	counters := make([]*countmin.Sketch, nworkers)

	var worker sync.WaitGroup
	worker.Add(nworkers)
	log.Printf("%d workers", nworkers)
	for i := 0; i < nworkers; i++ {
		counters[i], err = countmin.New(int(*nrows), int(*ncols))
		check()

		go func(ngramcount *countmin.Sketch) {
			for a := range articles {
				text := wikidump.Cleanup(a.Text)
				links <- wikidump.ExtractLinks(text)

				tokens := nlp.Tokenize(text)
				for _, h := range hash.NGrams(tokens, 1, maxN) {
					ngramcount.Add1(h)
				}
			}
			worker.Done()
		}(counters[i])
	}

	wg.Add(1)
	go func() {
		worker.Wait()
		close(links)

		for i := 1; i < nworkers; i++ {
			counters[0].Sum(counters[i])
		}
		counters = counters[:1]

		wg.Done()
	}()

	// Collect links and store them in the database.
	wg.Add(1)
	done := make(chan struct{})
	go func() {
		if slerr := storeLinks(db, links, maxN); slerr != nil {
			panic(slerr)
		}
		wg.Done()
	}()

	go wikidump.GetPages(f, articles, redirects)

	wg.Wait()
	close(done)

	log.Printf("Processing %d redirects", len(redirmap))
	storage.ProcessRedirects(db, redirmap)

	err = storage.StoreCM(db, counters[0])
	check()

	log.Println("Finalizing database")
	err = storage.Finalize(db)
	check()
	err = db.Close()
	check()
}

func storeLinks(db *sql.DB, links <-chan map[wikidump.Link]int,
	maxN int) (err error) {

	insTitle, err := db.Prepare(`insert or ignore into titles values (NULL, ?)`)
	if err != nil {
		return
	}
	insLink, err := db.Prepare(
		`insert or ignore into linkstats values
		 (?, (select id from titles where title = ?), 0)`)
	if err != nil {
		return
	}
	update, err := db.Prepare(
		`update linkstats set count = count + ?
		 where ngramhash = ?
		 and targetid = (select id from titles where title =?)`)
	if err != nil {
		return
	}

	for linkFreq := range links {
		for link, freq := range linkFreq {
			tokens := nlp.Tokenize(link.Anchor)
			n := min(maxN, len(tokens))
			hashes := hash.NGrams(tokens, n, n)
			count := float64(freq)
			if len(hashes) > 1 {
				count = 1 / float64(len(hashes))
			}
			for _, h := range hashes {
				_, err = insTitle.Exec(link.Target)
				if err != nil {
					return
				}
				_, err = insLink.Exec(h, link.Target)
				if err != nil {
					return
				}
				_, err = update.Exec(count, h, link.Target)
				if err != nil {
					return
				}
			}
		}
	}
	return
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}
<|endoftext|>