<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The coordinator runs on GCE and coordinates builds in Docker containers.
package main

import (
	"bytes"
	"crypto/hmac"
	"crypto/md5"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"sort"
	"strings"
	"sync"
	"time"
)

var (
	masterKeyFile = flag.String("masterkey", "", "Path to builder master key. Else fetched using GCE project attribute 'builder-master-key'.")
	maxBuilds     = flag.Int("maxbuilds", 6, "Max concurrent builds")

	// Debug flags:
	addTemp = flag.Bool("temp", false, "Append -temp to all builders.")
	just    = flag.String("just", "", "If non-empty, run single build in the foreground. Requires rev.")
	rev     = flag.String("rev", "", "Revision to build.")
)

var (
	startTime = time.Now()
	builders  = map[string]buildConfig{} // populated once at startup
	donec     = make(chan builderRev)    // reports of finished builders

	statusMu sync.Mutex
	status   = map[builderRev]*buildStatus{}
)

type imageInfo struct {
	url string // of tar file

	mu      sync.Mutex
	lastMod string
}

var images = map[string]*imageInfo{
	"gobuilders/linux-x86-base": {url: "https://storage.googleapis.com/go-builder-data/docker-linux.base.tar.gz"},
	"gobuilders/linux-x86-nacl": {url: "https://storage.googleapis.com/go-builder-data/docker-linux.nacl.tar.gz"},
}

type buildConfig struct {
	name  string   // "linux-amd64-race"
	image string   // Docker image to use to build
	cmd   string   // optional -cmd flag (relative to go/src/)
	env   []string // extra environment ("key=value") pairs
}

func main() {
	flag.Parse()
	addBuilder(buildConfig{name: "linux-386"})
	addBuilder(buildConfig{name: "linux-386-387", env: []string{"GO386=387"}})
	addBuilder(buildConfig{name: "linux-amd64"})
	addBuilder(buildConfig{name: "linux-amd64-nocgo", env: []string{"CGO_ENABLED=0"}})
	addBuilder(buildConfig{name: "linux-amd64-race"})
	addBuilder(buildConfig{name: "nacl-386"})
	addBuilder(buildConfig{name: "nacl-amd64p32"})

	if (*just != "") != (*rev != "") {
		log.Fatalf("--just and --rev must be used together")
	}
	if *just != "" {
		conf, ok := builders[*just]
		if !ok {
			log.Fatalf("unknown builder %q", *just)
		}
		cmd := exec.Command("docker", append([]string{"run"}, conf.dockerRunArgs(*rev)...)...)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err != nil {
			log.Fatalf("Build failed: %v", err)
		}
		return
	}

	http.HandleFunc("/", handleStatus)
	http.HandleFunc("/logs", handleLogs)
	go http.ListenAndServe(":80", nil)

	workc := make(chan builderRev)
	for name := range builders {
		go findWorkLoop(name, workc)
	}

	ticker := time.NewTicker(1 * time.Minute)
	for {
		select {
		case work := <-workc:
			log.Printf("workc received %+v; len(status) = %v, maxBuilds = %v; cur = %p", work, len(status), *maxBuilds, status[work])
			mayBuild := mayBuildRev(work)
			if mayBuild {
				out, _ := exec.Command("docker", "ps").Output()
				numBuilds := bytes.Count(out, []byte("\n")) - 1
				log.Printf("num current docker builds: %d", numBuilds)
				if numBuilds > *maxBuilds {
					mayBuild = false
				}
			}
			if mayBuild {
				if st, err := startBuilding(builders[work.name], work.rev); err == nil {
					setStatus(work, st)
					log.Printf("%v now building in %v", work, st.container)
				} else {
					log.Printf("Error starting to build %v: %v", work, err)
				}
			}
		case done := <-donec:
			log.Printf("%v done", done)
			setStatus(done, nil)
		case <-ticker.C:
			if numCurrentBuilds() == 0 && time.Now().After(startTime.Add(10*time.Minute)) {
				// TODO: halt the whole machine to kill the VM or something
			}
		}
	}
}

func numCurrentBuilds() int {
	statusMu.Lock()
	defer statusMu.Unlock()
	return len(status)
}

func mayBuildRev(work builderRev) bool {
	statusMu.Lock()
	defer statusMu.Unlock()
	return len(status) < *maxBuilds && status[work] == nil
}

func setStatus(work builderRev, st *buildStatus) {
	statusMu.Lock()
	defer statusMu.Unlock()
	if st == nil {
		delete(status, work)
	} else {
		status[work] = st
	}
}

func getStatus(work builderRev) *buildStatus {
	statusMu.Lock()
	defer statusMu.Unlock()
	return status[work]
}

type byAge []*buildStatus

func (s byAge) Len() int           { return len(s) }
func (s byAge) Less(i, j int) bool { return s[i].start.Before(s[j].start) }
func (s byAge) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

func handleStatus(w http.ResponseWriter, r *http.Request) {
	var active []*buildStatus
	statusMu.Lock()
	for _, st := range status {
		active = append(active, st)
	}
	statusMu.Unlock()

	fmt.Fprintf(w, "
Go build coordinator
%d of max %d builds running:

", len(status), *maxBuilds)
	sort.Sort(byAge(active))
	for _, st := range active {
		fmt.Fprintf(w, "%-22s hg %s in container %s, %v ago\n", st.name, st.rev, st.name, st.rev,
			st.container, time.Now().Sub(st.start))
	}
	fmt.Fprintf(w, "
")
}

func handleLogs(w http.ResponseWriter, r *http.Request) {
	st := getStatus(builderRev{r.FormValue("name"), r.FormValue("rev")})
	if st == nil {
		fmt.Fprintf(w, "
not building
")
		return
	}
	out, err := exec.Command("docker", "logs", st.container).CombinedOutput()
	if err != nil {
		log.Print(err)
		http.Error(w, "Error fetching logs. Already finished?", 500)
		return
	}
	key := builderKey(st.name)
	logs := strings.Replace(string(out), key, "BUILDERKEY", -1)
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	io.WriteString(w, logs)
}

func findWorkLoop(builderName string, work chan<- builderRev) {
	// TODO: make this better
	for {
		rev, err := findWork(builderName)
		if err != nil {
			log.Printf("Finding work for %s: %v", builderName, err)
		} else if rev != "" {
			work <- builderRev{builderName, rev}
		}
		time.Sleep(60 * time.Second)
	}
}

func findWork(builderName string) (rev string, err error) {
	var jres struct {
		Response struct {
			Kind string
			Data struct {
				Hash        string
				PerfResults []string
			}
		}
	}
	res, err := http.Get("https://build.golang.org/todo?builder=" + builderName + "&kind=build-go-commit")
	if err != nil {
		return
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return "", fmt.Errorf("unexpected http status %d", res.StatusCode)
	}
	err = json.NewDecoder(res.Body).Decode(&jres)
	if jres.Response.Kind == "build-go-commit" {
		rev = jres.Response.Data.Hash
	}
	return rev, err
}

type builderRev struct {
	name, rev string
}

// returns the part after "docker run"
func (conf buildConfig) dockerRunArgs(rev string) (args []string) {
	if key := builderKey(conf.name); key != "" {
		tmpKey := "/tmp/" + conf.name + ".buildkey"
		if _, err := os.Stat(tmpKey); err != nil {
			if err := ioutil.WriteFile(tmpKey, []byte(key), 0600); err != nil {
				log.Fatal(err)
			}
		}
		args = append(args, "-v", tmpKey+":/.gobuildkey")
	}
	for _, pair := range conf.env {
		args = append(args, "-e", pair)
	}
	args = append(args,
		conf.image,
		"/usr/local/bin/builder",
		"-rev="+rev,
		"-buildroot=/",
		"-v",
	)
	if conf.cmd != "" {
		args = append(args, "-cmd", conf.cmd)
	}
	args = append(args, conf.name)
	return
}

func addBuilder(c buildConfig) {
	if c.name == "" {
		panic("empty name")
	}
	if *addTemp {
		c.name += "-temp"
	}
	if _, dup := builders[c.name]; dup {
		panic("dup name")
	}
	if strings.HasPrefix(c.name, "nacl-") {
		if c.image == "" {
			c.image = "gobuilders/linux-x86-nacl"
		}
		if c.cmd == "" {
			c.cmd = "/usr/local/bin/build-command.pl"
		}
	}
	if strings.HasPrefix(c.name, "linux-") && c.image == "" {
		c.image = "gobuilders/linux-x86-base"
	}
	if c.image == "" {
		panic("empty image")
	}
	builders[c.name] = c
}

func condUpdateImage(img string) error {
	ii := images[img]
	if ii == nil {
		log.Fatalf("Image %q not described.", img)
	}
	ii.mu.Lock()
	defer ii.mu.Unlock()
	res, err := http.Head(ii.url)
	if err != nil {
		return fmt.Errorf("Error checking %s: %v", ii.url, err)
	}
	if res.StatusCode != 200 {
		return fmt.Errorf("Error checking %s: %v", ii.url, res.Status)
	}
	if res.Header.Get("Last-Modified") == ii.lastMod {
		return nil
	}

	res, err = http.Get(ii.url)
	if err != nil || res.StatusCode != 200 {
		return fmt.Errorf("Get after Head failed for %s: %v, %v", ii.url, err, res)
	}
	defer res.Body.Close()

	log.Printf("Running: docker load of %s\n", ii.url)
	cmd := exec.Command("docker", "load")
	cmd.Stdin = res.Body

	var out bytes.Buffer
	cmd.Stdout = &out
	cmd.Stderr = &out

	if cmd.Run(); err != nil {
		log.Printf("Failed to pull latest %s from %s and pipe into docker load: %v, %s", img, ii.url, err, out.Bytes())
		return err
	}
	ii.lastMod = res.Header.Get("Last-Modified")
	return nil
}

func startBuilding(conf buildConfig, rev string) (*buildStatus, error) {
	if err := condUpdateImage(conf.image); err != nil {
		log.Printf("Failed to setup container for %v %v: %v", conf.name, rev, err)
		return nil, err
	}

	cmd := exec.Command("docker", append([]string{"run", "-d"}, conf.dockerRunArgs(rev)...)...)
	all, err := cmd.CombinedOutput()
	log.Printf("Docker run for %v %v = err:%v, output:%s", conf.name, rev, err, all)
	if err != nil {
		return nil, err
	}
	container := strings.TrimSpace(string(all))
	go func() {
		all, err := exec.Command("docker", "wait", container).CombinedOutput()
		log.Printf("docker wait %s: %v, %s", container, err, strings.TrimSpace(string(all)))
		donec <- builderRev{conf.name, rev}
		exec.Command("docker", "rm", container).Run()
	}()
	return &buildStatus{
		builderRev: builderRev{
			name: conf.name,
			rev:  rev,
		},
		container: container,
		start:     time.Now(),
	}, nil
}

type buildStatus struct {
	builderRev
	container string
	start     time.Time

	mu sync.Mutex
	// ...
}

func builderKey(builder string) string {
	master := masterKey()
	if len(master) == 0 {
		return ""
	}
	h := hmac.New(md5.New, master)
	io.WriteString(h, builder)
	return fmt.Sprintf("%x", h.Sum(nil))
}

func masterKey() []byte {
	keyOnce.Do(loadKey)
	return masterKeyCache
}

var (
	keyOnce        sync.Once
	masterKeyCache []byte
)

func loadKey() {
	if *masterKeyFile != "" {
		b, err := ioutil.ReadFile(*masterKeyFile)
		if err != nil {
			log.Fatal(err)
		}
		masterKeyCache = bytes.TrimSpace(b)
		return
	}
	req, _ := http.NewRequest("GET", "http://metadata.google.internal/computeMetadata/v1/project/attributes/builder-master-key", nil)
	req.Header.Set("Metadata-Flavor", "Google")
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Fatal("No builder master key available")
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		log.Fatalf("No builder-master-key project attribute available.")
	}
	slurp, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}
	masterKeyCache = bytes.TrimSpace(slurp)
}
<commit_msg>dashboard/coordinator: set USER for the -nocgo builder<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The coordinator runs on GCE and coordinates builds in Docker containers.
package main

import (
	"bytes"
	"crypto/hmac"
	"crypto/md5"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"sort"
	"strings"
	"sync"
	"time"
)

var (
	masterKeyFile = flag.String("masterkey", "", "Path to builder master key. Else fetched using GCE project attribute 'builder-master-key'.")
	maxBuilds     = flag.Int("maxbuilds", 6, "Max concurrent builds")

	// Debug flags:
	addTemp = flag.Bool("temp", false, "Append -temp to all builders.")
	just    = flag.String("just", "", "If non-empty, run single build in the foreground. Requires rev.")
	rev     = flag.String("rev", "", "Revision to build.")
)

var (
	startTime = time.Now()
	builders  = map[string]buildConfig{} // populated once at startup
	donec     = make(chan builderRev)    // reports of finished builders

	statusMu sync.Mutex
	status   = map[builderRev]*buildStatus{}
)

type imageInfo struct {
	url string // of tar file

	mu      sync.Mutex
	lastMod string
}

var images = map[string]*imageInfo{
	"gobuilders/linux-x86-base": {url: "https://storage.googleapis.com/go-builder-data/docker-linux.base.tar.gz"},
	"gobuilders/linux-x86-nacl": {url: "https://storage.googleapis.com/go-builder-data/docker-linux.nacl.tar.gz"},
}

type buildConfig struct {
	name  string   // "linux-amd64-race"
	image string   // Docker image to use to build
	cmd   string   // optional -cmd flag (relative to go/src/)
	env   []string // extra environment ("key=value") pairs
}

func main() {
	flag.Parse()
	addBuilder(buildConfig{name: "linux-386"})
	addBuilder(buildConfig{name: "linux-386-387", env: []string{"GO386=387"}})
	addBuilder(buildConfig{name: "linux-amd64"})
	addBuilder(buildConfig{name: "linux-amd64-nocgo", env: []string{"CGO_ENABLED=0", "USER=root"}})
	addBuilder(buildConfig{name: "linux-amd64-race"})
	addBuilder(buildConfig{name: "nacl-386"})
	addBuilder(buildConfig{name: "nacl-amd64p32"})

	if (*just != "") != (*rev != "") {
		log.Fatalf("--just and --rev must be used together")
	}
	if *just != "" {
		conf, ok := builders[*just]
		if !ok {
			log.Fatalf("unknown builder %q", *just)
		}
		cmd := exec.Command("docker", append([]string{"run"}, conf.dockerRunArgs(*rev)...)...)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err != nil {
			log.Fatalf("Build failed: %v", err)
		}
		return
	}

	http.HandleFunc("/", handleStatus)
	http.HandleFunc("/logs", handleLogs)
	go http.ListenAndServe(":80", nil)

	workc := make(chan builderRev)
	for name := range builders {
		go findWorkLoop(name, workc)
	}

	ticker := time.NewTicker(1 * time.Minute)
	for {
		select {
		case work := <-workc:
			log.Printf("workc received %+v; len(status) = %v, maxBuilds = %v; cur = %p", work, len(status), *maxBuilds, status[work])
			mayBuild := mayBuildRev(work)
			if mayBuild {
				out, _ := exec.Command("docker", "ps").Output()
				numBuilds := bytes.Count(out, []byte("\n")) - 1
				log.Printf("num current docker builds: %d", numBuilds)
				if numBuilds > *maxBuilds {
					mayBuild = false
				}
			}
			if mayBuild {
				if st, err := startBuilding(builders[work.name], work.rev); err == nil {
					setStatus(work, st)
					log.Printf("%v now building in %v", work, st.container)
				} else {
					log.Printf("Error starting to build %v: %v", work, err)
				}
			}
		case done := <-donec:
			log.Printf("%v done", done)
			setStatus(done, nil)
		case <-ticker.C:
			if numCurrentBuilds() == 0 && time.Now().After(startTime.Add(10*time.Minute)) {
				// TODO: halt the whole machine to kill the VM or something
			}
		}
	}
}

func numCurrentBuilds() int {
	statusMu.Lock()
	defer statusMu.Unlock()
	return len(status)
}

func mayBuildRev(work builderRev) bool {
	statusMu.Lock()
	defer statusMu.Unlock()
	return len(status) < *maxBuilds && status[work] == nil
}

func setStatus(work builderRev, st *buildStatus) {
	statusMu.Lock()
	defer statusMu.Unlock()
	if st == nil {
		delete(status, work)
	} else {
		status[work] = st
	}
}

func getStatus(work builderRev) *buildStatus {
	statusMu.Lock()
	defer statusMu.Unlock()
	return status[work]
}

type byAge []*buildStatus

func (s byAge) Len() int           { return len(s) }
func (s byAge) Less(i, j int) bool { return s[i].start.Before(s[j].start) }
func (s byAge) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

func handleStatus(w http.ResponseWriter, r *http.Request) {
	var active []*buildStatus
	statusMu.Lock()
	for _, st := range status {
		active = append(active, st)
	}
	statusMu.Unlock()

	fmt.Fprintf(w, "
Go build coordinator
%d of max %d builds running:

", len(status), *maxBuilds)
	sort.Sort(byAge(active))
	for _, st := range active {
		fmt.Fprintf(w, "%-22s hg %s in container %s, %v ago\n", st.name, st.rev, st.name, st.rev,
			st.container, time.Now().Sub(st.start))
	}
	fmt.Fprintf(w, "
")
}

func handleLogs(w http.ResponseWriter, r *http.Request) {
	st := getStatus(builderRev{r.FormValue("name"), r.FormValue("rev")})
	if st == nil {
		fmt.Fprintf(w, "
not building
")
		return
	}
	out, err := exec.Command("docker", "logs", st.container).CombinedOutput()
	if err != nil {
		log.Print(err)
		http.Error(w, "Error fetching logs. Already finished?", 500)
		return
	}
	key := builderKey(st.name)
	logs := strings.Replace(string(out), key, "BUILDERKEY", -1)
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	io.WriteString(w, logs)
}

func findWorkLoop(builderName string, work chan<- builderRev) {
	// TODO: make this better
	for {
		rev, err := findWork(builderName)
		if err != nil {
			log.Printf("Finding work for %s: %v", builderName, err)
		} else if rev != "" {
			work <- builderRev{builderName, rev}
		}
		time.Sleep(60 * time.Second)
	}
}

func findWork(builderName string) (rev string, err error) {
	var jres struct {
		Response struct {
			Kind string
			Data struct {
				Hash        string
				PerfResults []string
			}
		}
	}
	res, err := http.Get("https://build.golang.org/todo?builder=" + builderName + "&kind=build-go-commit")
	if err != nil {
		return
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return "", fmt.Errorf("unexpected http status %d", res.StatusCode)
	}
	err = json.NewDecoder(res.Body).Decode(&jres)
	if jres.Response.Kind == "build-go-commit" {
		rev = jres.Response.Data.Hash
	}
	return rev, err
}

type builderRev struct {
	name, rev string
}

// returns the part after "docker run"
func (conf buildConfig) dockerRunArgs(rev string) (args []string) {
	if key := builderKey(conf.name); key != "" {
		tmpKey := "/tmp/" + conf.name + ".buildkey"
		if _, err := os.Stat(tmpKey); err != nil {
			if err := ioutil.WriteFile(tmpKey, []byte(key), 0600); err != nil {
				log.Fatal(err)
			}
		}
		args = append(args, "-v", tmpKey+":/.gobuildkey")
	}
	for _, pair := range conf.env {
		args = append(args, "-e", pair)
	}
	args = append(args,
		conf.image,
		"/usr/local/bin/builder",
		"-rev="+rev,
		"-buildroot=/",
		"-v",
	)
	if conf.cmd != "" {
		args = append(args, "-cmd", conf.cmd)
	}
	args = append(args, conf.name)
	return
}

func addBuilder(c buildConfig) {
	if c.name == "" {
		panic("empty name")
	}
	if *addTemp {
		c.name += "-temp"
	}
	if _, dup := builders[c.name]; dup {
		panic("dup name")
	}
	if strings.HasPrefix(c.name, "nacl-") {
		if c.image == "" {
			c.image = "gobuilders/linux-x86-nacl"
		}
		if c.cmd == "" {
			c.cmd = "/usr/local/bin/build-command.pl"
		}
	}
	if strings.HasPrefix(c.name, "linux-") && c.image == "" {
		c.image = "gobuilders/linux-x86-base"
	}
	if c.image == "" {
		panic("empty image")
	}
	builders[c.name] = c
}

func condUpdateImage(img string) error {
	ii := images[img]
	if ii == nil {
		log.Fatalf("Image %q not described.", img)
	}
	ii.mu.Lock()
	defer ii.mu.Unlock()
	res, err := http.Head(ii.url)
	if err != nil {
		return fmt.Errorf("Error checking %s: %v", ii.url, err)
	}
	if res.StatusCode != 200 {
		return fmt.Errorf("Error checking %s: %v", ii.url, res.Status)
	}
	if res.Header.Get("Last-Modified") == ii.lastMod {
		return nil
	}

	res, err = http.Get(ii.url)
	if err != nil || res.StatusCode != 200 {
		return fmt.Errorf("Get after Head failed for %s: %v, %v", ii.url, err, res)
	}
	defer res.Body.Close()

	log.Printf("Running: docker load of %s\n", ii.url)
	cmd := exec.Command("docker", "load")
	cmd.Stdin = res.Body

	var out bytes.Buffer
	cmd.Stdout = &out
	cmd.Stderr = &out

	if cmd.Run(); err != nil {
		log.Printf("Failed to pull latest %s from %s and pipe into docker load: %v, %s", img, ii.url, err, out.Bytes())
		return err
	}
	ii.lastMod = res.Header.Get("Last-Modified")
	return nil
}

func startBuilding(conf buildConfig, rev string) (*buildStatus, error) {
	if err := condUpdateImage(conf.image); err != nil {
		log.Printf("Failed to setup container for %v %v: %v", conf.name, rev, err)
		return nil, err
	}

	cmd := exec.Command("docker", append([]string{"run", "-d"}, conf.dockerRunArgs(rev)...)...)
	all, err := cmd.CombinedOutput()
	log.Printf("Docker run for %v %v = err:%v, output:%s", conf.name, rev, err, all)
	if err != nil {
		return nil, err
	}
	container := strings.TrimSpace(string(all))
	go func() {
		all, err := exec.Command("docker", "wait", container).CombinedOutput()
		log.Printf("docker wait %s: %v, %s", container, err, strings.TrimSpace(string(all)))
		donec <- builderRev{conf.name, rev}
		exec.Command("docker", "rm", container).Run()
	}()
	return &buildStatus{
		builderRev: builderRev{
			name: conf.name,
			rev:  rev,
		},
		container: container,
		start:     time.Now(),
	}, nil
}

type buildStatus struct {
	builderRev
	container string
	start     time.Time

	mu sync.Mutex
	// ...
}

func builderKey(builder string) string {
	master := masterKey()
	if len(master) == 0 {
		return ""
	}
	h := hmac.New(md5.New, master)
	io.WriteString(h, builder)
	return fmt.Sprintf("%x", h.Sum(nil))
}

func masterKey() []byte {
	keyOnce.Do(loadKey)
	return masterKeyCache
}

var (
	keyOnce        sync.Once
	masterKeyCache []byte
)

func loadKey() {
	if *masterKeyFile != "" {
		b, err := ioutil.ReadFile(*masterKeyFile)
		if err != nil {
			log.Fatal(err)
		}
		masterKeyCache = bytes.TrimSpace(b)
		return
	}
	req, _ := http.NewRequest("GET", "http://metadata.google.internal/computeMetadata/v1/project/attributes/builder-master-key", nil)
	req.Header.Set("Metadata-Flavor", "Google")
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Fatal("No builder master key available")
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		log.Fatalf("No builder-master-key project attribute available.")
	}
	slurp, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}
	masterKeyCache = bytes.TrimSpace(slurp)
}
<|endoftext|>


<commit_before>// +build ignore

/*
Copyright 2015 The Camlistore Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Command build-binaries builds camlistored and the Camlistore tools.
// It should be run in a docker container.
package main // import "camlistore.org/misc/docker/release"

import (
	"archive/tar"
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"runtime"
	"strings"
)

var (
	flagRev     = flag.String("rev", "", "Camlistore revision to build (tag or commit hash). For development purposes, you can instead specify the path to a local Camlistore source tree from which to build, with the form \"WIP:/path/to/dir\".")
	flagVersion = flag.String("version", "", "The optional version number (e.g. 0.9) that will be stamped into the binaries, in addition to the revision.")
	outDir      = flag.String("outdir", "/OUT/", "Output directory, where the binaries will be written")
	buildOS     = flag.String("os", runtime.GOOS, "Operating system to build for.")
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage:\n")
	fmt.Fprintf(os.Stderr, "%s --rev=camlistore_revision\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "%s --rev=WIP:/path/to/camli/source/dir\n", os.Args[0])
	flag.PrintDefaults()
	example(os.Args[0])
	os.Exit(1)
}

func example(program string) {
	fmt.Fprintf(os.Stderr, "Examples:\n")
	fmt.Fprintf(os.Stderr, "\tdocker run --rm --volume=/tmp/camli-build/camlistore.org:/OUT camlistore/go %s --rev=4e8413c5012c\n", program)
	fmt.Fprintf(os.Stderr, "\tdocker run --rm --volume=/tmp/camli-build/camlistore.org:/OUT --volume=~/camlistore.org:/IN camlistore/go %s --rev=WIP:/IN\n", program)
}

func isWIP() bool {
	return strings.HasPrefix(*flagRev, "WIP")
}

// localCamliSource returns the path to the local Camlistore source tree
// that should be specified in *flagRev if *flagRev starts with "WIP:",
// empty string otherwise.
func localCamliSource() string {
	if !isWIP() {
		return ""
	}
	return strings.TrimPrefix(*flagRev, "WIP:")
}

func rev() string {
	if isWIP() {
		return "WORKINPROGRESS"
	}
	return *flagRev
}

func version() string {
	if *flagVersion != "" {
		return fmt.Sprintf("%v (git rev %v)", *flagVersion, rev())
	}
	return rev()
}

func getCamliSrc() {
	if localCamliSource() != "" {
		mirrorCamliSrc(localCamliSource())
	} else {
		fetchCamliSrc()
	}
	// we insert the version in the VERSION file, so make.go does no need git
	// in the container to detect the Camlistore version.
	check(os.Chdir("/gopath/src/camlistore.org"))
	check(ioutil.WriteFile("VERSION", []byte(version()), 0777))
}

func mirrorCamliSrc(srcDir string) {
	check(os.MkdirAll("/gopath/src", 0777))
	cmd := exec.Command("cp", "-a", srcDir, "/gopath/src/camlistore.org")
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		log.Fatalf("Error mirroring camlistore source from %v: %v", srcDir, err)
	}
}

func fetchCamliSrc() {
	check(os.MkdirAll("/gopath/src/camlistore.org", 0777))
	check(os.Chdir("/gopath/src/camlistore.org"))

	res, err := http.Get("https://camlistore.googlesource.com/camlistore/+archive/" + *flagRev + ".tar.gz")
	check(err)
	defer res.Body.Close()
	gz, err := gzip.NewReader(res.Body)
	check(err)
	defer gz.Close()
	tr := tar.NewReader(gz)
	for {
		h, err := tr.Next()
		if err == io.EOF {
			break
		}
		check(err)
		if h.Typeflag == tar.TypeDir {
			check(os.MkdirAll(h.Name, os.FileMode(h.Mode)))
			continue
		}
		f, err := os.Create(h.Name)
		check(err)
		n, err := io.Copy(f, tr)
		if err != nil && err != io.EOF {
			log.Fatal(err)
		}
		if n != h.Size {
			log.Fatalf("Error when creating %v: wanted %v bytes, got %v bytes", h.Name, h.Size, n)
		}
		check(f.Close())
	}
	// TODO(mpl): remove once https://camlistore-review.googlesource.com/8146 is in.
	check(os.Chmod("app/publisher/js/gensearchtypes.sh", 0500))
}

func build() {
	check(os.Chdir("/gopath/src/camlistore.org"))
	oldPath := os.Getenv("PATH")
	os.Setenv("GOPATH", "/gopath")
	os.Setenv("PATH", "/usr/local/go/bin:"+oldPath)
	cmd := exec.Command("go", "run", "make.go", "--os", *buildOS)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		log.Fatalf("Error building all Camlistore binaries for %v in go container: %v", *buildOS, err)
	}
	srcDir := "bin"
	if *buildOS != "linux" {
		// TODO(mpl): probably bail early if GOARCH != amd64. Or do we want to distribute for other arches?
		srcDir = path.Join(srcDir, *buildOS+"_amd64")
	}
	cmd = exec.Command("mv", srcDir, path.Join(*outDir, "/bin"))
	if err := cmd.Run(); err != nil {
		log.Fatalf("Error copying Camlistore binaries to %v: %v", path.Join(*outDir, "/bin"), err)
	}
}

func checkArgs() {
	if flag.NArg() != 0 {
		usage()
	}
	if *flagRev == "" {
		fmt.Fprintf(os.Stderr, "Usage error: --rev is required.\n")
		usage()
	}
}

func main() {
	flag.Usage = usage
	flag.Parse()
	// TODO(mpl): find another way to detect whether we're on docker, as
	// /.dockerinit does not exist anymore apparently? No time for now.
	checkArgs()

	getCamliSrc()
	build()
}

func check(err error) {
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>misc/docker: gensearchtypes.sh does not exist anymore<commit_after>// +build ignore

/*
Copyright 2015 The Camlistore Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Command build-binaries builds camlistored and the Camlistore tools.
// It should be run in a docker container.
package main // import "camlistore.org/misc/docker/release"

import (
	"archive/tar"
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"runtime"
	"strings"
)

var (
	flagRev     = flag.String("rev", "", "Camlistore revision to build (tag or commit hash). For development purposes, you can instead specify the path to a local Camlistore source tree from which to build, with the form \"WIP:/path/to/dir\".")
	flagVersion = flag.String("version", "", "The optional version number (e.g. 0.9) that will be stamped into the binaries, in addition to the revision.")
	outDir      = flag.String("outdir", "/OUT/", "Output directory, where the binaries will be written")
	buildOS     = flag.String("os", runtime.GOOS, "Operating system to build for.")
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage:\n")
	fmt.Fprintf(os.Stderr, "%s --rev=camlistore_revision\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "%s --rev=WIP:/path/to/camli/source/dir\n", os.Args[0])
	flag.PrintDefaults()
	example(os.Args[0])
	os.Exit(1)
}

func example(program string) {
	fmt.Fprintf(os.Stderr, "Examples:\n")
	fmt.Fprintf(os.Stderr, "\tdocker run --rm --volume=/tmp/camli-build/camlistore.org:/OUT camlistore/go %s --rev=4e8413c5012c\n", program)
	fmt.Fprintf(os.Stderr, "\tdocker run --rm --volume=/tmp/camli-build/camlistore.org:/OUT --volume=~/camlistore.org:/IN camlistore/go %s --rev=WIP:/IN\n", program)
}

func isWIP() bool {
	return strings.HasPrefix(*flagRev, "WIP")
}

// localCamliSource returns the path to the local Camlistore source tree
// that should be specified in *flagRev if *flagRev starts with "WIP:",
// empty string otherwise.
func localCamliSource() string {
	if !isWIP() {
		return ""
	}
	return strings.TrimPrefix(*flagRev, "WIP:")
}

func rev() string {
	if isWIP() {
		return "WORKINPROGRESS"
	}
	return *flagRev
}

func version() string {
	if *flagVersion != "" {
		return fmt.Sprintf("%v (git rev %v)", *flagVersion, rev())
	}
	return rev()
}

func getCamliSrc() {
	if localCamliSource() != "" {
		mirrorCamliSrc(localCamliSource())
	} else {
		fetchCamliSrc()
	}
	// we insert the version in the VERSION file, so make.go does no need git
	// in the container to detect the Camlistore version.
	check(os.Chdir("/gopath/src/camlistore.org"))
	check(ioutil.WriteFile("VERSION", []byte(version()), 0777))
}

func mirrorCamliSrc(srcDir string) {
	check(os.MkdirAll("/gopath/src", 0777))
	cmd := exec.Command("cp", "-a", srcDir, "/gopath/src/camlistore.org")
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		log.Fatalf("Error mirroring camlistore source from %v: %v", srcDir, err)
	}
}

func fetchCamliSrc() {
	check(os.MkdirAll("/gopath/src/camlistore.org", 0777))
	check(os.Chdir("/gopath/src/camlistore.org"))

	res, err := http.Get("https://camlistore.googlesource.com/camlistore/+archive/" + *flagRev + ".tar.gz")
	check(err)
	defer res.Body.Close()
	gz, err := gzip.NewReader(res.Body)
	check(err)
	defer gz.Close()
	tr := tar.NewReader(gz)
	for {
		h, err := tr.Next()
		if err == io.EOF {
			break
		}
		check(err)
		if h.Typeflag == tar.TypeDir {
			check(os.MkdirAll(h.Name, os.FileMode(h.Mode)))
			continue
		}
		f, err := os.Create(h.Name)
		check(err)
		n, err := io.Copy(f, tr)
		if err != nil && err != io.EOF {
			log.Fatal(err)
		}
		if n != h.Size {
			log.Fatalf("Error when creating %v: wanted %v bytes, got %v bytes", h.Name, h.Size, n)
		}
		check(f.Close())
	}
}

func build() {
	check(os.Chdir("/gopath/src/camlistore.org"))
	oldPath := os.Getenv("PATH")
	os.Setenv("GOPATH", "/gopath")
	os.Setenv("PATH", "/usr/local/go/bin:"+oldPath)
	cmd := exec.Command("go", "run", "make.go", "--os", *buildOS)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		log.Fatalf("Error building all Camlistore binaries for %v in go container: %v", *buildOS, err)
	}
	srcDir := "bin"
	if *buildOS != "linux" {
		// TODO(mpl): probably bail early if GOARCH != amd64. Or do we want to distribute for other arches?
		srcDir = path.Join(srcDir, *buildOS+"_amd64")
	}
	cmd = exec.Command("mv", srcDir, path.Join(*outDir, "/bin"))
	if err := cmd.Run(); err != nil {
		log.Fatalf("Error copying Camlistore binaries to %v: %v", path.Join(*outDir, "/bin"), err)
	}
}

func checkArgs() {
	if flag.NArg() != 0 {
		usage()
	}
	if *flagRev == "" {
		fmt.Fprintf(os.Stderr, "Usage error: --rev is required.\n")
		usage()
	}
}

func main() {
	flag.Usage = usage
	flag.Parse()
	// TODO(mpl): find another way to detect whether we're on docker, as
	// /.dockerinit does not exist anymore apparently? No time for now.
	checkArgs()

	getCamliSrc()
	build()
}

func check(err error) {
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "net/http"
    "log"
    "strconv"
    "encoding/json"
    "strings"
)

// handlers for the api calls

func handler(w http.ResponseWriter, r *http.Request) {
    fmt.Fprintf(w, "Hi there, I love %s! \n", r.URL.Path[1:])
}

func handler_status(w http.ResponseWriter, r *http.Request) {
    //TODO: check for notifications here?

    fmt.Fprintf(w, "looks like we are ok")
}

func handler_app_link(w http.ResponseWriter, r *http.Request) {
    // TODO: redirect to the app store link
    fmt.Fprintf(w, "app store link:")

}

func handler_group_all(w http.ResponseWriter, r *http.Request) {

    all_groups := get_group_all()

    text, err := json.Marshal(all_groups)

    if (err != nil) {
      // rip
      return
    }

    fmt.Fprintf(w, string(text))
}

func handler_group_one(w http.ResponseWriter, r *http.Request) {
    if r.Method != "GET" {
      // only allow POST requests
      return
    }

    params := strings.Split(r.URL.Path,"/")

    group_id := params[len(params)-1]
    group_id_int, err := strconv.Atoi(group_id)

    group := get_group_with_id(group_id_int)

    text, err := json.Marshal(group)

    if (err != nil) {
      log.Fatal(err.Error())
      return
    }

    // show the group info
    fmt.Fprintf(w, string(text))
}

func handler_group_new(w http.ResponseWriter, r *http.Request) {
    if r.Method != "POST" {
      // only allow POST requests
      return
    }

    r.ParseForm()
    group_name := r.PostFormValue("group_name")

    create_group(group_name)
}

func handler_group_delete(w http.ResponseWriter, r *http.Request) {
    if r.Method != "POST" {
      // only allow POST requests
      return
    }

    r.ParseForm()
    group_id := r.PostFormValue("group_id")
    group_id_int, err := strconv.Atoi(group_id)

    if (err != nil) {
      log.Fatal(err.Error())
      return
    }

    delete_group(group_id_int)
}

func handler_group_join(w http.ResponseWriter, r *http.Request) {
    if r.Method != "POST" {
      // only allow POST requests
      return
    }

    r.ParseForm()
    group_id := r.PostFormValue("group_id")
    group_id_int, err := strconv.Atoi(group_id)

    user_id := r.PostFormValue("user_id")
    user_id_int, err := strconv.Atoi(user_id)

    if (err != nil) {
      log.Fatal(err.Error())
      return
    }

    join_group(group_id_int, user_id_int)
}

func handler_group_kick(w http.ResponseWriter, r *http.Request) {
    if r.Method != "POST" {
      // only allow POST requests
      return
    }

    decoder := json.NewDecoder(r.Body)

    datas := []struct {
      GroupId int
      UserId int
    }{}

    err := decoder.Decode(&datas)

    if(err != nil) {
      fmt.Fprintf(w, string(err.Error()))
    }

    data := datas[0]

    kick_user(data.GroupId, data.UserId)
}

func handler_user_new_or_rename(w http.ResponseWriter, r *http.Request) {
    if r.Method != "POST" {
      // only allow POST requests
      return
    }

    decoder := json.NewDecoder(r.Body)
    var users []User
    err := decoder.Decode(&users)

    if(err != nil) {
      fmt.Fprintf(w, string(err.Error()))
    }

    user := users[0]

    if(user.UserId > -1) {
      // rename a user

      user := rename_user(user.UserId, user.Name)

      text, err := json.Marshal([]User{user})
      if (err != nil) {
        // rip
        return
      }
      // prints id of the created user
      fmt.Fprintf(w, string(text))
      return
    }

    // if not -1, we rename
    if (err != nil) {
      log.Fatal(err.Error())
      return
    }

    new_user := create_user(user.Name)

    text, err := json.Marshal(new_user)
    if (err != nil) {
      // rip
      return
    }
    // prints id of the created user
    fmt.Fprintf(w, string(text))
}

func handler_group_messages(w http.ResponseWriter, r *http.Request) {

    decoder := json.NewDecoder(r.Body)

    datas := []struct {
      GroupId int
    }{}

    err := decoder.Decode(&datas)

    if(err != nil) {
      fmt.Fprintf(w, string(err.Error()))
    }

    data := datas[0]

    all_messages := get_messages(data.GroupId)

    text, err := json.Marshal(all_messages)

    if (err != nil) {
      // rip
      return
    }

    fmt.Fprintf(w, string(text))
}

func handler_group_messages_send(w http.ResponseWriter, r *http.Request) {
    if r.Method != "POST" {
      // only allow POST requests
      return
    }

    r.ParseForm()
    body := r.PostFormValue("body")

    group_id := r.PostFormValue("group_id")
    group_id_int, err := strconv.Atoi(group_id)

    user_id := r.PostFormValue("user_id")
    user_id_int, err := strconv.Atoi(user_id)

    if (err != nil) {
      log.Fatal(err.Error())
      return
    }

    create_message(body, group_id_int, user_id_int)
}

func getURLParam(path string, prefix string) []string {
  // currently returns trailing param, int only
  s := strings.SplitAfter(path, prefix)
  return s
}

// startup

func main() {
    var ssl_path = "/var/www/group_chat_project/web/private/ssl/"
    var crt = ssl_path+"both_cert.crt"
    var key = ssl_path+"amber_nopass.key"

    // handler definitions for various api calls
    http.HandleFunc("/", handler) // default
    http.HandleFunc("/status", handler_status)
    http.HandleFunc("/app", handler_app_link)

    // group info
    http.HandleFunc("/group/all", handler_group_all)
    http.HandleFunc("/group/", handler_group_one)

    // creates new group
    http.HandleFunc("/group/new", handler_group_new)

    // deletes group
    http.HandleFunc("/group/delete", handler_group_delete)

    // get messages in a group
    http.HandleFunc("/group/messages", handler_group_messages)

    // create message in a group
    http.HandleFunc("/group/messages/send", handler_group_messages_send)

    // // send message to group
    // http.HandleFunc("/group/send", handler_group_send)
    //
    // add person to group
    http.HandleFunc("/group/join", handler_group_join)

    // remove person from group
    http.HandleFunc("/group/kick", handler_group_kick)

    // create new user
    http.HandleFunc("/user/new", handler_user_new_or_rename)

    err := http.ListenAndServeTLS(":443", crt, key, nil)

    if err != nil {
        log.Fatal("ListenAndServe: ", err)
    }

}
<commit_msg>json messages for group, GET request<commit_after>package main

import (
    "fmt"
    "net/http"
    "log"
    "strconv"
    "encoding/json"
    "strings"
)

// handlers for the api calls

func handler(w http.ResponseWriter, r *http.Request) {
    fmt.Fprintf(w, "Hi there, I love %s! \n", r.URL.Path[1:])
}

func handler_status(w http.ResponseWriter, r *http.Request) {
    //TODO: check for notifications here?

    fmt.Fprintf(w, "looks like we are ok")
}

func handler_app_link(w http.ResponseWriter, r *http.Request) {
    // TODO: redirect to the app store link
    fmt.Fprintf(w, "app store link:")

}

func handler_group_all(w http.ResponseWriter, r *http.Request) {

    all_groups := get_group_all()

    text, err := json.Marshal(all_groups)

    if (err != nil) {
      // rip
      return
    }

    fmt.Fprintf(w, string(text))
}

func handler_group_one(w http.ResponseWriter, r *http.Request) {
    if r.Method != "GET" {
      // only allow POST requests
      return
    }

    params := strings.Split(r.URL.Path,"/")

    group_id := params[len(params)-1] // gets trailing part
    group_id_int, err := strconv.Atoi(group_id)

    group := get_group_with_id(group_id_int)

    text, err := json.Marshal(group)

    if (err != nil) {
      log.Fatal(err.Error())
      return
    }

    // show the group info
    fmt.Fprintf(w, string(text))
}

func handler_group_new(w http.ResponseWriter, r *http.Request) {
    if r.Method != "POST" {
      // only allow POST requests
      return
    }

    r.ParseForm()
    group_name := r.PostFormValue("group_name")

    create_group(group_name)
}

func handler_group_delete(w http.ResponseWriter, r *http.Request) {
    if r.Method != "POST" {
      // only allow POST requests
      return
    }

    r.ParseForm()
    group_id := r.PostFormValue("group_id")
    group_id_int, err := strconv.Atoi(group_id)

    if (err != nil) {
      log.Fatal(err.Error())
      return
    }

    delete_group(group_id_int)
}

func handler_group_join(w http.ResponseWriter, r *http.Request) {
    if r.Method != "POST" {
      // only allow POST requests
      return
    }

    r.ParseForm()
    group_id := r.PostFormValue("group_id")
    group_id_int, err := strconv.Atoi(group_id)

    user_id := r.PostFormValue("user_id")
    user_id_int, err := strconv.Atoi(user_id)

    if (err != nil) {
      log.Fatal(err.Error())
      return
    }

    join_group(group_id_int, user_id_int)
}

func handler_group_kick(w http.ResponseWriter, r *http.Request) {
    if r.Method != "POST" {
      // only allow POST requests
      return
    }

    decoder := json.NewDecoder(r.Body)

    datas := []struct {
      GroupId int
      UserId int
    }{}

    err := decoder.Decode(&datas)

    if(err != nil) {
      fmt.Fprintf(w, string(err.Error()))
    }

    data := datas[0]

    kick_user(data.GroupId, data.UserId)
}

func handler_user_new_or_rename(w http.ResponseWriter, r *http.Request) {
    if r.Method != "POST" {
      // only allow POST requests
      return
    }

    decoder := json.NewDecoder(r.Body)
    var users []User
    err := decoder.Decode(&users)

    if(err != nil) {
      fmt.Fprintf(w, string(err.Error()))
    }

    user := users[0]

    if(user.UserId > -1) {
      // rename a user

      user := rename_user(user.UserId, user.Name)

      text, err := json.Marshal([]User{user})
      if (err != nil) {
        // rip
        return
      }
      // prints id of the created user
      fmt.Fprintf(w, string(text))
      return
    }

    // if not -1, we rename
    if (err != nil) {
      log.Fatal(err.Error())
      return
    }

    new_user := create_user(user.Name)

    text, err := json.Marshal(new_user)
    if (err != nil) {
      // rip
      return
    }
    // prints id of the created user
    fmt.Fprintf(w, string(text))
}

func handler_group_messages(w http.ResponseWriter, r *http.Request) {
    // GET request

    params := strings.Split(r.URL.Path,"/")
    group_id := params[len(params)-1] // gets trailing part
    group_id_int, err := strconv.Atoi(group_id)

    all_messages := get_messages(group_id_int)

    text, err := json.Marshal(all_messages)

    if (err != nil) {
      // rip
      return
    }

    fmt.Fprintf(w, string(text))
}

func handler_group_messages_send(w http.ResponseWriter, r *http.Request) {
    if r.Method != "POST" {
      // only allow POST requests
      return
    }

    r.ParseForm()
    body := r.PostFormValue("body")

    group_id := r.PostFormValue("group_id")
    group_id_int, err := strconv.Atoi(group_id)

    user_id := r.PostFormValue("user_id")
    user_id_int, err := strconv.Atoi(user_id)

    if (err != nil) {
      log.Fatal(err.Error())
      return
    }

    create_message(body, group_id_int, user_id_int)
}

func getURLParam(path string, prefix string) []string {
  // currently returns trailing param, int only
  s := strings.SplitAfter(path, prefix)
  return s
}

// startup

func main() {
    var ssl_path = "/var/www/group_chat_project/web/private/ssl/"
    var crt = ssl_path+"both_cert.crt"
    var key = ssl_path+"amber_nopass.key"

    // handler definitions for various api calls
    http.HandleFunc("/", handler) // default
    http.HandleFunc("/status", handler_status)
    http.HandleFunc("/app", handler_app_link)

    // group info
    http.HandleFunc("/group/all", handler_group_all)
    http.HandleFunc("/group/", handler_group_one)

    // creates new group
    http.HandleFunc("/group/new", handler_group_new)

    // deletes group
    http.HandleFunc("/group/delete", handler_group_delete)

    // get messages in a group
    http.HandleFunc("/group/messages/", handler_group_messages)

    // create message in a group
    http.HandleFunc("/group/messages/send", handler_group_messages_send)

    // // send message to group
    // http.HandleFunc("/group/send", handler_group_send)
    //
    // add person to group
    http.HandleFunc("/group/join", handler_group_join)

    // remove person from group
    http.HandleFunc("/group/kick", handler_group_kick)

    // create new user
    http.HandleFunc("/user/new", handler_user_new_or_rename)

    err := http.ListenAndServeTLS(":443", crt, key, nil)

    if err != nil {
        log.Fatal("ListenAndServe: ", err)
    }

}
<|endoftext|>


<commit_before>package migrations

import (
	"fmt"
	"gopkg.in/go-pg/migrations.v4"
)

var Initialization = migrations.Migration{
	Version: 1,
	Up: func(db migrations.DB) error {
		fmt.Println("Upgrading Initialization migration")
		queries := []string{
			`CREATE TABLE public.organizations (
			 	id SERIAL NOT NULL,
				name CHARACTER VARYING(200) NOT NULL
			)`,
			`CREATE TABLE public.users (
				id SERIAL NOT NULL,
				username CHARACTER VARYING(200) NOT NULL,
				email CHARACTER VARYING(200) NOT NULL,
				password CHARACTER VARYING(200) NOT NULL
			)`,
			`CREATE TABLE public.customers (
				id SERIAL NOT NULL,
				company_name CHARACTER VARYING(200) NOT NULL,
				first_name CHARACTER VARYING(200) NOT NULL,
				last_name CHARACTER VARYING(200) NOT NULL,
				email CHARACTER VARYING(200) NOT NULL,
				phone_number CHARACTER VARYING(200) NOT NULL,
				vat_number CHARACTER VARYING(200) NOT NULL
			)`,
			`CREATE TABLE public.addresses (
				id SERIAL NOT NULL,
				address CHARACTER VARYING(255) NOT NULL,
				postal_code CHARACTER VARYING(10) NOT NULL,
				city CHARACTER VARYING(200) NOT NULL,
				country CHARACTER VARYING(200) NOT NULL,
			)`,
		}
		for _, q := range queries {
			_, err := db.Exec(q)
			if err != nil {
				return err
			}
		}
		return nil
	},
	Down: func(db migrations.DB) error {
		fmt.Println("Downgrading Initialization migration")
		queries := []string{
			`DROP TABLE users`,
			`DROP TABLE organizations`,
			`DROP TABLE customers`,
			`DROP TABLE addresses`,
		}
		for _, q := range queries {
			_, err := db.Exec(q)
			if err != nil {
				return err
			}
		}
		return nil
	},
}
<commit_msg>Text type all the way<commit_after>package migrations

import (
	"fmt"
	"gopkg.in/go-pg/migrations.v4"
)

var Initialization = migrations.Migration{
	Version: 1,
	Up: func(db migrations.DB) error {
		fmt.Println("Upgrading Initialization migration")
		queries := []string{
			`CREATE TABLE public.organizations (
			 	id SERIAL NOT NULL,
				name TEXT NOT NULL
			)`,
			`CREATE TABLE public.users (
				id SERIAL NOT NULL,
				username TEXT NOT NULL,
				email TEXT NOT NULL,
				password TEXT NOT NULL
			)`,
			`CREATE TABLE public.customers (
				id SERIAL NOT NULL,
				company_name TEXT NOT NULL,
				first_name TEXT NOT NULL,
				last_name TEXT NOT NULL,
				email TEXT NOT NULL,
				phone_number TEXT NOT NULL,
				vat_number TEXT NOT NULL
			)`,
			`CREATE TABLE public.addresses (
				id SERIAL NOT NULL,
				address TEXT NOT NULL,
				postal_code TEXT NOT NULL,
				city TEXT NOT NULL,
				country TEXT NOT NULL,
			)`,
		}
		for _, q := range queries {
			_, err := db.Exec(q)
			if err != nil {
				return err
			}
		}
		return nil
	},
	Down: func(db migrations.DB) error {
		fmt.Println("Downgrading Initialization migration")
		queries := []string{
			`DROP TABLE users`,
			`DROP TABLE organizations`,
			`DROP TABLE customers`,
			`DROP TABLE addresses`,
		}
		for _, q := range queries {
			_, err := db.Exec(q)
			if err != nil {
				return err
			}
		}
		return nil
	},
}
<|endoftext|>


<commit_before>// CookieJar - A contestant's algorithm toolbox
// Copyright 2013 Peter Szilagyi. All rights reserved.
//
// CookieJar is dual licensed: you can redistribute it and/or modify it under
// the terms of the GNU General Public License as published by the Free Software
// Foundation, either version 3 of the License, or (at your option) any later
// version.
//
// The toolbox is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
// more details.
//
// Alternatively, the CookieJar toolbox may be used in accordance with the terms
// and conditions contained in a signed written agreement between you and the
// author(s).
//
// Author: peterke@gmail.com (Peter Szilagyi)

// Command depsmerge implements a command to retrieve and merge all dependencies
// of a package into a single file.
package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"go/ast"
	"go/parser"
	"go/printer"
	"go/token"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

// Package description from the go list command
type Package struct {
	Name     string
	Dir      string
	Standard bool
	Deps     []string
	GoFiles  []string
}

// Loads the details of the Go package.
func details(name string) (*Package, error) {
	// Create the command to retrieve the package infos
	cmd := exec.Command("go", "list", "-e", "-json", name)

	// Retrieve the output, redirect the errors
	out, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	cmd.Stderr = os.Stderr

	// Start executing and parse the results
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	defer cmd.Process.Kill()

	info := new(Package)
	if err := json.NewDecoder(out).Decode(&info); err != nil {
		return nil, err
	}
	// Clean up and return
	if err := cmd.Wait(); err != nil {
		return nil, err
	}
	return info, nil
}

// Collects all the imported packages of a file.
func dependencies(path string) (map[string][]string, error) {
	// Retrieve the dependencies of the source file
	info, err := details(path)
	if err != nil {
		return nil, err
	}
	// Iterate over every dependency and gather the sources
	sources := make(map[string][]string)
	for _, dep := range info.Deps {
		// Retrieve the dependency details
		pkgInfo, err := details(dep)
		if err != nil {
			return nil, err
		}
		// Gather external library sources
		if !pkgInfo.Standard {
			for _, src := range pkgInfo.GoFiles {
				sources[pkgInfo.Name] = append(sources[pkgInfo.Name], filepath.Join(pkgInfo.Dir, src))
			}
		}
	}
	return sources, nil
}

// Parses a source file and scopes all global declarations.
func rewrite(src string, pkg string, deps []string) (string, error) {
	fileSet := token.NewFileSet()
	tree, err := parser.ParseFile(fileSet, src, nil, parser.ParseComments)
	if err != nil {
		return "", err
	}
	// Scope all top level declarations if not main file
	if pkg != "" {
		for _, decl := range tree.Decls {
			switch v := decl.(type) {
			case *ast.FuncDecl:
				if v.Recv == nil {
					rename(tree, v.Name.String(), pkg+"•"+v.Name.String())
				}
			case *ast.GenDecl:
				for _, spec := range v.Specs {
					switch spec := spec.(type) {
					case *ast.ValueSpec:
						for _, name := range spec.Names {
							rename(tree, name.String(), pkg+"•"+name.String())
						}
					case *ast.TypeSpec:
						rename(tree, spec.Name.String(), pkg+"•"+spec.Name.String())
					default:
						fmt.Println("Unknown spec:", spec)
					}
				}
			default:
				fmt.Println(v)
			}
		}
	}
	// Generate the new source contents
	out := bytes.NewBuffer(nil)
	for _, decl := range tree.Decls {
		if gen, ok := decl.(*ast.GenDecl); !ok || gen.Tok != token.IMPORT {
			if err := printer.Fprint(out, fileSet, decl); err != nil {
				return "", err
			}
		}
		fmt.Fprintf(out, "\n\n")
	}
	blob := out.Bytes()

	// Scope all externally imported dependencies
	var fail error
	ast.Inspect(tree, func(node ast.Node) bool {
		if imp, ok := node.(*ast.GenDecl); ok && imp.Tok == token.IMPORT {
			for _, spec := range imp.Specs {
				if spec, ok := spec.(*ast.ImportSpec); ok {
					// Figure out the correct name of the import
					path := strings.Trim(spec.Path.Value, "\"")
					if info, err := details(path); err != nil {
						fail = err
						return false
					} else if !info.Standard {
						// Add scope to external import
						scoper := regexp.MustCompile("\\b" + info.Name + "\\.(.+)")
						blob = scoper.ReplaceAll(blob, []byte(info.Name+"•$1"))
					}
				}
			}
		}
		return true
	})
	if fail != nil {
		return "", fail
	}
	fmt.Println(string(blob))
	return "", nil
}

// Renames a top level declaration to something else.
func rename(tree *ast.File, old, new string) {
	// Rename top-level declarations
	for _, decl := range tree.Decls {
		switch decl := decl.(type) {
		case *ast.FuncDecl:
			// If a top level function matches, rename
			if decl.Recv == nil && decl.Name.Name == old {
				decl.Name.Name = new
				decl.Name.Obj.Name = new
			}
		case *ast.GenDecl:
			// Iterate over all the generic declaration
			for _, spec := range decl.Specs {
				switch spec := spec.(type) {
				case *ast.ValueSpec:
					// If a top level variable matches, rename
					for _, name := range spec.Names {
						if name.Name == old {
							name.Name = new
							name.Obj.Name = new
						}
					}
				case *ast.TypeSpec:
					if spec.Name.Name == old {
						spec.Name.Name = new
						spec.Name.Obj.Name = new
					}
				}
			}
		}
	}
	// Walk the AST and rename all internal occurrences
	stack := []ast.Node{}
	ast.Inspect(tree, func(node ast.Node) bool {
		// Keep a traversal stack if need to reference parent
		if node == nil {
			stack = stack[:len(stack)-1]
			return true
		}
		stack = append(stack, node)

		// Look for identifiers to rename
		id, ok := node.(*ast.Ident)
		if ok && id.Obj == nil && id.Name == old {
			// If selected identifier, leave it alone
			if _, ok := stack[len(stack)-2].(*ast.SelectorExpr); ok {
				return true
			}
			id.Name = new
		}
		if ok && id.Obj != nil && id.Name == old && id.Obj.Name == new {
			id.Name = id.Obj.Name
		}
		return true
	})
}

func main() {
	deps, err := dependencies(os.Args[1])
	if err != nil {
		log.Fatalf("Failed to parse dependency chain: %v.", err)
	}
	pkgs := []string{}
	for pkg, _ := range deps {
		pkgs = append(pkgs, pkg)
	}
	// Rewrite all the dependencies
	for pkg, sources := range deps {
		for _, src := range sources {
			fmt.Println(rewrite(src, pkg, pkgs))
		}
	}
	// Rewrite the main file itself
	fmt.Println(rewrite(os.Args[1], "", pkgs))
}
<commit_msg>Initial trivial flatten, change separator.<commit_after>// CookieJar - A contestant's algorithm toolbox
// Copyright 2013 Peter Szilagyi. All rights reserved.
//
// CookieJar is dual licensed: you can redistribute it and/or modify it under
// the terms of the GNU General Public License as published by the Free Software
// Foundation, either version 3 of the License, or (at your option) any later
// version.
//
// The toolbox is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
// more details.
//
// Alternatively, the CookieJar toolbox may be used in accordance with the terms
// and conditions contained in a signed written agreement between you and the
// author(s).
//
// Author: peterke@gmail.com (Peter Szilagyi)

// Command depsmerge implements a command to retrieve and merge all dependencies
// of a package into a single file.
package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"go/ast"
	"go/format"
	"go/parser"
	"go/printer"
	"go/token"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

// Package description from the go list command
type Package struct {
	Name     string
	Dir      string
	Standard bool
	Deps     []string
	GoFiles  []string
}

// Loads the details of the Go package.
func details(name string) (*Package, error) {
	// Create the command to retrieve the package infos
	cmd := exec.Command("go", "list", "-e", "-json", name)

	// Retrieve the output, redirect the errors
	out, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	cmd.Stderr = os.Stderr

	// Start executing and parse the results
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	defer cmd.Process.Kill()

	info := new(Package)
	if err := json.NewDecoder(out).Decode(&info); err != nil {
		return nil, err
	}
	// Clean up and return
	if err := cmd.Wait(); err != nil {
		return nil, err
	}
	return info, nil
}

// Collects all the imported packages of a file.
func dependencies(path string) (map[string][]string, error) {
	// Retrieve the dependencies of the source file
	info, err := details(path)
	if err != nil {
		return nil, err
	}
	// Iterate over every dependency and gather the sources
	sources := make(map[string][]string)
	for _, dep := range info.Deps {
		// Retrieve the dependency details
		pkgInfo, err := details(dep)
		if err != nil {
			return nil, err
		}
		// Gather external library sources
		if !pkgInfo.Standard {
			for _, src := range pkgInfo.GoFiles {
				sources[pkgInfo.Name] = append(sources[pkgInfo.Name], filepath.Join(pkgInfo.Dir, src))
			}
		}
	}
	return sources, nil
}

// Parses a source file and scopes all global declarations.
func rewrite(src string, pkg string) ([]byte, error) {
	fileSet := token.NewFileSet()
	tree, err := parser.ParseFile(fileSet, src, nil, parser.ParseComments)
	if err != nil {
		return nil, err
	}
	// Scope all top level declarations if not main file
	if pkg != "" {
		for _, decl := range tree.Decls {
			switch v := decl.(type) {
			case *ast.FuncDecl:
				if v.Recv == nil {
					rename(tree, v.Name.String(), pkg+"ᴥ"+v.Name.String())
				}
			case *ast.GenDecl:
				for _, spec := range v.Specs {
					switch spec := spec.(type) {
					case *ast.ValueSpec:
						for _, name := range spec.Names {
							rename(tree, name.String(), pkg+"ᴥ"+name.String())
						}
					case *ast.TypeSpec:
						rename(tree, spec.Name.String(), pkg+"ᴥ"+spec.Name.String())
					default:
						fmt.Println("Unknown spec:", spec)
					}
				}
			default:
				fmt.Println(v)
			}
		}
	}
	// Generate the new source contents
	out := bytes.NewBuffer(nil)
	for _, decl := range tree.Decls {
		if gen, ok := decl.(*ast.GenDecl); !ok || gen.Tok != token.IMPORT {
			if err := printer.Fprint(out, fileSet, decl); err != nil {
				return nil, err
			}
		}
		fmt.Fprintf(out, "\n\n")
	}
	blob := out.Bytes()

	// Scope all externally imported dependencies
	var fail error
	ast.Inspect(tree, func(node ast.Node) bool {
		if imp, ok := node.(*ast.GenDecl); ok && imp.Tok == token.IMPORT {
			for _, spec := range imp.Specs {
				if spec, ok := spec.(*ast.ImportSpec); ok {
					// Figure out the correct name of the import
					path := strings.Trim(spec.Path.Value, "\"")
					if info, err := details(path); err != nil {
						fail = err
						return false
					} else if !info.Standard {
						// Add scope to external import
						scoper := regexp.MustCompile("\\b" + info.Name + "\\.(.+)")
						blob = scoper.ReplaceAll(blob, []byte(info.Name+"ᴥ$1"))
					}
				}
			}
		}
		return true
	})
	if fail != nil {
		return nil, fail
	}
	return blob, nil
}

// Renames a top level declaration to something else.
func rename(tree *ast.File, old, new string) {
	// Rename top-level declarations
	for _, decl := range tree.Decls {
		switch decl := decl.(type) {
		case *ast.FuncDecl:
			// If a top level function matches, rename
			if decl.Recv == nil && decl.Name.Name == old {
				decl.Name.Name = new
				decl.Name.Obj.Name = new
			}
		case *ast.GenDecl:
			// Iterate over all the generic declaration
			for _, spec := range decl.Specs {
				switch spec := spec.(type) {
				case *ast.ValueSpec:
					// If a top level variable matches, rename
					for _, name := range spec.Names {
						if name.Name == old {
							name.Name = new
							name.Obj.Name = new
						}
					}
				case *ast.TypeSpec:
					if spec.Name.Name == old {
						spec.Name.Name = new
						spec.Name.Obj.Name = new
					}
				}
			}
		}
	}
	// Walk the AST and rename all internal occurrences
	stack := []ast.Node{}
	ast.Inspect(tree, func(node ast.Node) bool {
		// Keep a traversal stack if need to reference parent
		if node == nil {
			stack = stack[:len(stack)-1]
			return true
		}
		stack = append(stack, node)

		// Look for identifiers to rename
		id, ok := node.(*ast.Ident)
		if ok && id.Obj == nil && id.Name == old {
			// If selected identifier, leave it alone
			if _, ok := stack[len(stack)-2].(*ast.SelectorExpr); ok {
				return true
			}
			id.Name = new
		}
		if ok && id.Obj != nil && id.Name == old && id.Obj.Name == new {
			id.Name = id.Obj.Name
		}
		return true
	})
}

// Flattens a main file with rewritten dependencies.
func flatten(main []byte, deps [][]byte) ([]byte, error) {
	// Dump all code pieces into a buffer
	buffer := new(bytes.Buffer)
	fmt.Fprintf(buffer, "%s\n\n", main)
	for _, dep := range deps {
		fmt.Fprintf(buffer, "%s\n\n", dep)
	}
	// Format the blob to Go standards
	blob, err := format.Source(buffer.Bytes())
	if err != nil {
		return nil, err
	}
	return blob, nil
}

func main() {
	deps, err := dependencies(os.Args[1])
	if err != nil {
		log.Fatalf("Failed to parse dependency chain: %v.", err)
	}
	// Rewrite all the dependencies
	pieces := make([][]byte, 0, len(deps))
	for pkg, sources := range deps {
		for _, src := range sources {
			blob, err := rewrite(src, pkg)
			if err != nil {
				log.Fatalf("Failed to rewrite dependency %s: %v.", src, err)
			}
			pieces = append(pieces, blob)
		}
	}
	// Rewrite the main file itself, append all dependencies
	main, err := rewrite(os.Args[1], "")
	if err != nil {
		log.Fatalf("Failed to rewrite main file %s: %v.", os.Args[1], err)
	}
	blob, err := flatten(main, pieces)
	if err != nil {
		log.Fatalf("Failed to flatten and format sources: %v.", err)
	}
	fmt.Println(string(blob))
}
<|endoftext|>


<commit_before>// handler package implements a library for handling run lambda requests from
// the worker server.
package handler

import (
	"container/list"
	"fmt"
	"log"
	"os"
	"path"
	"sync"
	"sync/atomic"
	"time"

	"github.com/open-lambda/open-lambda/worker/config"
	"github.com/open-lambda/open-lambda/worker/handler/state"
	"github.com/open-lambda/open-lambda/worker/import-cache"
	"github.com/open-lambda/open-lambda/worker/registry"

	sb "github.com/open-lambda/open-lambda/worker/sandbox"
)

// HandlerSet represents a collection of Handlers of a worker server. It
// manages the Handler by HandlerLRU.
type HandlerManagerSet struct {
	mutex      sync.Mutex
	hmMap      map[string]*HandlerManager
	regMgr     registry.RegistryManager
	sbFactory  sb.SandboxFactory
	cacheMgr   *cache.CacheManager
	config     *config.Config
	lru        *HandlerLRU
	workerDir  string
	maxRunners int
	hhits      *int64
	ihits      *int64
	misses     *int64
}

type HandlerManager struct {
	name        string
	mutex       sync.Mutex
	hms         *HandlerManagerSet
	handlers    *list.List
	hElements   map[*Handler]*list.Element
	workingDir  string
	maxHandlers int
	lastPull    *time.Time
	code        []byte
	codeDir     string
	pkgs        []string
}

// Handler handles requests to run a lambda on a worker server. It handles
// concurrency and communicates with the sandbox manager to change the
// state of the container that servers the lambda.
type Handler struct {
	name    string
	id      string
	mutex   sync.Mutex
	hm      *HandlerManager
	sandbox sb.Sandbox
	fs      *cache.ForkServer
	hostDir string
	runners int
	usage   int
}

// NewHandlerSet creates an empty HandlerSet
func NewHandlerManagerSet(opts *config.Config) (hms *HandlerManagerSet, err error) {
	rm, err := registry.InitRegistryManager(opts)
	if err != nil {
		return nil, err
	}

	sf, err := sb.InitSandboxFactory(opts)
	if err != nil {
		return nil, err
	}

	cm, err := cache.InitCacheManager(opts)
	if err != nil {
		return nil, err
	}

	var hhits int64 = 0
	var ihits int64 = 0
	var misses int64 = 0
	hms = &HandlerManagerSet{
		hmMap:      make(map[string]*HandlerManager),
		regMgr:     rm,
		sbFactory:  sf,
		cacheMgr:   cm,
		config:     opts,
		workerDir:  opts.Worker_dir,
		maxRunners: opts.Max_runners,
		hhits:      &hhits,
		ihits:      &ihits,
		misses:     &misses,
	}

	hms.lru = NewHandlerLRU(hms, opts.Handler_cache_size) //kb

	/*
		if cm != nil {
			go handlerSet.killOrphans()
		}
	*/

	return hms, nil
}

// Get always returns a Handler, creating one if necessarily.
func (hms *HandlerManagerSet) Get(name string) (h *Handler, err error) {
	start := time.Now()
	defer log.Printf("get handler took %v\n", time.Since(start))

	hms.mutex.Lock()

	hm := hms.hmMap[name]

	if hm == nil {
		workingDir := path.Join(hms.workerDir, "handlers", name)
		hms.hmMap[name] = &HandlerManager{
			name:       name,
			hms:        hms,
			handlers:   list.New(),
			hElements:  make(map[*Handler]*list.Element),
			workingDir: workingDir,
			pkgs:       []string{},
		}

		hm = hms.hmMap[name]
	}

	// find or create handler
	hm.mutex.Lock()
	if hms.config.Handler_cache_size == 0 || hm.handlers.Front() == nil {
		h = &Handler{
			name:    name,
			hm:      hm,
			runners: 1,
		}
	} else {
		hEle := hm.handlers.Front()
		h = hEle.Value.(*Handler)

		// remove from lru if necessary
		h.mutex.Lock()
		if h.runners == 0 {
			hms.lru.Remove(h)
		}

		h.runners += 1

		if h.hm.hms.maxRunners != 0 && h.runners == h.hm.hms.maxRunners {
			hm.handlers.Remove(hEle)
			delete(hm.hElements, h)
		}
		h.mutex.Unlock()
	}
	// not perfect, but removal from the LRU needs to be atomic
	// with respect to the LRU and the HandlerManager
	hms.mutex.Unlock()

	// get code if needed
	if hm.lastPull == nil {
		codeDir, pkgs, err := hms.regMgr.Pull(hm.name)
		if err != nil {
			return nil, err
		}

		now := time.Now()
		hm.lastPull = &now
		hm.codeDir = codeDir
		hm.pkgs = pkgs
	}
	hm.mutex.Unlock()

	return h, nil
}

/*
func (h *HandlerSet) killOrphans() {
	var toDelete string
	for {
		if toDelete != "" {
			h.mutex.Lock()
			handler := h.handlers[toDelete]
			delete(h.handlers, toDelete)
			h.mutex.Unlock()
			go handler.nuke()
		}
		toDelete = ""
		for _, handler := range h.handlers {
			handler.mutex.Lock()
			if handler.fs != nil && handler.fs.Dead {
				toDelete = handler.name
			}
			time.Sleep(50 * time.Microsecond)
			handler.mutex.Unlock()
		}
	}
}
*/

// Dump prints the name and state of the Handlers currently in the HandlerSet.
func (hms *HandlerManagerSet) Dump() {
	hms.mutex.Lock()
	defer hms.mutex.Unlock()

	log.Printf("HANDLERS:\n")
	for name, hm := range hms.hmMap {
		hm.mutex.Lock()
		log.Printf(" %v: %d", name, hm.maxHandlers)
		for e := hm.handlers.Front(); e != nil; e = e.Next() {
			h := e.Value.(*Handler)
			state, _ := h.sandbox.State()
			log.Printf(" > %v: %v\n", h.id, state.String())
		}
		hm.mutex.Unlock()
	}
}

func (hms *HandlerManagerSet) Cleanup() {
	hms.mutex.Lock()
	defer hms.mutex.Unlock()

	for _, hm := range hms.hmMap {
		for e := hm.handlers.Front(); e != nil; e = e.Next() {
			e.Value.(*Handler).nuke()
		}
	}

	hms.sbFactory.Cleanup()

	if hms.cacheMgr != nil {
		hms.cacheMgr.Cleanup()
	}
}

// must be called with handler lock
func (hm *HandlerManager) AddHandler(h *Handler) {
	hms := hm.hms

	// if we finish first
	// no deadlock can occur here despite taking the locks in the
	// opposite order because hm -> h in Get has no reference
	// in the handler list
	if hms.maxRunners != 0 && h.runners == hms.maxRunners-1 {
		hm.mutex.Lock()
		hm.hElements[h] = hm.handlers.PushFront(h)
		hm.maxHandlers = max(hm.maxHandlers, hm.handlers.Len())
		hm.mutex.Unlock()
	}
}

// RunStart runs the lambda handled by this Handler. It checks if the code has
// been pulled, sandbox been created, and sandbox been started. The channel of
// the sandbox of this lambda is returned.
func (h *Handler) RunStart() (ch *sb.SandboxChannel, err error) {
	start := time.Now()
	defer log.Printf("start server took %v\n", time.Since(start))

	h.mutex.Lock()
	defer h.mutex.Unlock()

	hm := h.hm
	hms := h.hm.hms

	// create sandbox if needed
	if h.sandbox == nil {
		sandbox, err := hms.sbFactory.Create(hm.codeDir, hm.workingDir)
		if err != nil {
			return nil, err
		}

		h.sandbox = sandbox
		h.id = h.sandbox.ID()
		h.hostDir = path.Join(hm.workingDir, h.id)
		if sbState, err := h.sandbox.State(); err != nil {
			return nil, err
		} else if sbState == state.Stopped {
			if err := h.sandbox.Start(); err != nil {
				return nil, err
			}
		} else if sbState == state.Paused {
			if err := h.sandbox.Unpause(); err != nil {
				return nil, err
			}
		}

		hit := false
		if hms.cacheMgr == nil || hms.cacheMgr.Full() {
			err := h.sandbox.RunServer()
			if err != nil {
				return nil, err
			}
		} else {
			containerSB, ok := h.sandbox.(sb.ContainerSandbox)
			if !ok {
				return nil, fmt.Errorf("forkenter only supported with ContainerSandbox")
			}
			if h.fs, hit, err = hms.cacheMgr.Provision(containerSB, h.hostDir, hm.pkgs); err != nil {
				return nil, err
			}

		}

		if hit {
			atomic.AddInt64(hms.ihits, 1)
		} else {
			atomic.AddInt64(hms.misses, 1)
		}

		sockPath := fmt.Sprintf("%s/ol.sock", h.hostDir)

		// wait up to 20s for server to initialize
		start := time.Now()
		for ok := true; ok; ok = os.IsNotExist(err) {
			_, err = os.Stat(sockPath)
			if hms.config.Sandbox == "olcontainer" && (hms.cacheMgr == nil || hms.cacheMgr.Full()) {
				time.Sleep(10 * time.Microsecond)
				if err := h.sandbox.RunServer(); err != nil {
					return nil, err
				}
			}
			if time.Since(start).Seconds() > 600 {
				return nil, fmt.Errorf("handler server failed to initialize after 20s")
			}
			time.Sleep(50 * time.Microsecond)
		}

		// we are up so we can add ourselves for reuse
		if hms.maxRunners == 0 || h.runners < hms.maxRunners {
			hm.mutex.Lock()
			hm.hElements[h] = hm.handlers.PushFront(h)
			hm.maxHandlers = max(hm.maxHandlers, hm.handlers.Len())
			hm.mutex.Unlock()
		}

	} else if sbState, _ := h.sandbox.State(); sbState == state.Paused {
		// unpause if paused
		atomic.AddInt64(hms.hhits, 1)
		if err := h.sandbox.Unpause(); err != nil {
			return nil, err
		}
	} else {
		atomic.AddInt64(hms.hhits, 1)
	}

	log.Printf("handler hits: %v, import hits: %v, misses: %v", *hms.hhits, *hms.ihits, *hms.misses)
	return h.sandbox.Channel()
}

// RunFinish notifies that a request to run the lambda has completed. If no
// request is being run in its sandbox, sandbox will be paused and the handler
// be added to the HandlerLRU.
func (h *Handler) RunFinish() {
	h.mutex.Lock()
	defer h.mutex.Unlock()

	hm := h.hm
	hms := h.hm.hms

	h.runners -= 1

	// are we the last?
	if h.runners == 0 {
		if err := h.sandbox.Pause(); err != nil {
			// TODO(tyler): better way to handle this?  If
			// we can't pause, the handler gets to keep
			// running for free...
			log.Printf("Could not pause %v: %v!  Error: %v\n", h.name, h.id, err)
		}

		hm.AddHandler(h)
		hms.lru.Add(h)
	} else {
		hm.AddHandler(h)
	}
}

func (h *Handler) nuke() {
	if err := h.sandbox.Unpause(); err != nil {
		log.Printf("failed to unpause sandbox :: %v", err.Error())
	}
	if err := h.sandbox.Stop(); err != nil {
		log.Printf("failed to stop sandbox :: %v", err.Error())
	}
	if err := h.sandbox.Remove(); err != nil {
		log.Printf("failed to remove sandbox :: %v", err.Error())
	}
}

// Sandbox returns the sandbox of this Handler.
func (h *Handler) Sandbox() sb.Sandbox {
	return h.sandbox
}

func max(i1, i2 int) int {
	if i1 < i2 {
		return i2
	}
	return i1
}
<commit_msg>handler: only add to the LRU if the handler fits in the cache<commit_after>// handler package implements a library for handling run lambda requests from
// the worker server.
package handler

import (
	"container/list"
	"errors"
	"fmt"
	"log"
	"os"
	"path"
	"sync"
	"sync/atomic"
	"time"

	"github.com/open-lambda/open-lambda/worker/config"
	"github.com/open-lambda/open-lambda/worker/handler/state"
	"github.com/open-lambda/open-lambda/worker/import-cache"
	"github.com/open-lambda/open-lambda/worker/registry"

	sb "github.com/open-lambda/open-lambda/worker/sandbox"
)

// HandlerSet represents a collection of Handlers of a worker server. It
// manages the Handler by HandlerLRU.
type HandlerManagerSet struct {
	mutex      sync.Mutex
	hmMap      map[string]*HandlerManager
	regMgr     registry.RegistryManager
	sbFactory  sb.SandboxFactory
	cacheMgr   *cache.CacheManager
	config     *config.Config
	lru        *HandlerLRU
	workerDir  string
	maxRunners int
	hhits      *int64
	ihits      *int64
	misses     *int64
}

type HandlerManager struct {
	name        string
	mutex       sync.Mutex
	hms         *HandlerManagerSet
	handlers    *list.List
	hElements   map[*Handler]*list.Element
	workingDir  string
	maxHandlers int
	lastPull    *time.Time
	code        []byte
	codeDir     string
	pkgs        []string
}

// Handler handles requests to run a lambda on a worker server. It handles
// concurrency and communicates with the sandbox manager to change the
// state of the container that servers the lambda.
type Handler struct {
	name    string
	id      string
	mutex   sync.Mutex
	hm      *HandlerManager
	sandbox sb.Sandbox
	fs      *cache.ForkServer
	hostDir string
	runners int
	usage   int
}

// NewHandlerSet creates an empty HandlerSet
func NewHandlerManagerSet(opts *config.Config) (hms *HandlerManagerSet, err error) {
	rm, err := registry.InitRegistryManager(opts)
	if err != nil {
		return nil, err
	}

	sf, err := sb.InitSandboxFactory(opts)
	if err != nil {
		return nil, err
	}

	cm, err := cache.InitCacheManager(opts)
	if err != nil {
		return nil, err
	}

	var hhits int64 = 0
	var ihits int64 = 0
	var misses int64 = 0
	hms = &HandlerManagerSet{
		hmMap:      make(map[string]*HandlerManager),
		regMgr:     rm,
		sbFactory:  sf,
		cacheMgr:   cm,
		config:     opts,
		workerDir:  opts.Worker_dir,
		maxRunners: opts.Max_runners,
		hhits:      &hhits,
		ihits:      &ihits,
		misses:     &misses,
	}

	hms.lru = NewHandlerLRU(hms, opts.Handler_cache_size) //kb

	/*
		if cm != nil {
			go handlerSet.killOrphans()
		}
	*/

	return hms, nil
}

// Get always returns a Handler, creating one if necessarily.
func (hms *HandlerManagerSet) Get(name string) (h *Handler, err error) {
	start := time.Now()
	defer log.Printf("get handler took %v\n", time.Since(start))

	hms.mutex.Lock()

	hm := hms.hmMap[name]

	if hm == nil {
		workingDir := path.Join(hms.workerDir, "handlers", name)
		hms.hmMap[name] = &HandlerManager{
			name:       name,
			hms:        hms,
			handlers:   list.New(),
			hElements:  make(map[*Handler]*list.Element),
			workingDir: workingDir,
			pkgs:       []string{},
		}

		hm = hms.hmMap[name]
	}

	// find or create handler
	hm.mutex.Lock()
	if hm.handlers.Front() == nil {
		h = &Handler{
			name:    name,
			hm:      hm,
			runners: 1,
		}
	} else {
		hEle := hm.handlers.Front()
		h = hEle.Value.(*Handler)

		// remove from lru if necessary
		h.mutex.Lock()
		if h.runners == 0 {
			hms.lru.Remove(h)
		}

		h.runners += 1

		if h.hm.hms.maxRunners != 0 && h.runners == h.hm.hms.maxRunners {
			hm.handlers.Remove(hEle)
			delete(hm.hElements, h)
		}
		h.mutex.Unlock()
	}
	// not perfect, but removal from the LRU needs to be atomic
	// with respect to the LRU and the HandlerManager
	hms.mutex.Unlock()

	// get code if needed
	if hm.lastPull == nil {
		codeDir, pkgs, err := hms.regMgr.Pull(hm.name)
		if err != nil {
			return nil, err
		}

		now := time.Now()
		hm.lastPull = &now
		hm.codeDir = codeDir
		hm.pkgs = pkgs
	}
	hm.mutex.Unlock()

	return h, nil
}

/*
func (h *HandlerSet) killOrphans() {
	var toDelete string
	for {
		if toDelete != "" {
			h.mutex.Lock()
			handler := h.handlers[toDelete]
			delete(h.handlers, toDelete)
			h.mutex.Unlock()
			go handler.nuke()
		}
		toDelete = ""
		for _, handler := range h.handlers {
			handler.mutex.Lock()
			if handler.fs != nil && handler.fs.Dead {
				toDelete = handler.name
			}
			time.Sleep(50 * time.Microsecond)
			handler.mutex.Unlock()
		}
	}
}
*/

// Dump prints the name and state of the Handlers currently in the HandlerSet.
func (hms *HandlerManagerSet) Dump() {
	hms.mutex.Lock()
	defer hms.mutex.Unlock()

	log.Printf("HANDLERS:\n")
	for name, hm := range hms.hmMap {
		hm.mutex.Lock()
		log.Printf(" %v: %d", name, hm.maxHandlers)
		for e := hm.handlers.Front(); e != nil; e = e.Next() {
			h := e.Value.(*Handler)
			state, _ := h.sandbox.State()
			log.Printf(" > %v: %v\n", h.id, state.String())
		}
		hm.mutex.Unlock()
	}
}

func (hms *HandlerManagerSet) Cleanup() {
	hms.mutex.Lock()
	defer hms.mutex.Unlock()

	for _, hm := range hms.hmMap {
		for e := hm.handlers.Front(); e != nil; e = e.Next() {
			e.Value.(*Handler).nuke()
		}
	}

	hms.sbFactory.Cleanup()

	if hms.cacheMgr != nil {
		hms.cacheMgr.Cleanup()
	}
}

// must be called with handler lock
func (hm *HandlerManager) AddHandler(h *Handler) {
	hms := hm.hms

	// if we finish first
	// no deadlock can occur here despite taking the locks in the
	// opposite order because hm -> h in Get has no reference
	// in the handler list
	if hms.maxRunners != 0 && h.runners == hms.maxRunners-1 {
		hm.mutex.Lock()
		hm.hElements[h] = hm.handlers.PushFront(h)
		hm.maxHandlers = max(hm.maxHandlers, hm.handlers.Len())
		hm.mutex.Unlock()
	}
}

func (hm *HandlerManager) TryRemoveHandler(h *Handler) error {
	hm.mutex.Lock()
	defer hm.mutex.Unlock()
	h.mutex.Lock()
	defer h.mutex.Unlock()

	// someone has come in and has started running
	if h.runners > 0 {
		return errors.New("concurrent runner entered system")
	}

	// remove reference to handler in HandlerManager
	// this ensures h is the last reference to the Handler
	if hEle := hm.hElements[h]; hEle != nil {
		hm.handlers.Remove(hEle)
		delete(hm.hElements, h)
	}

	return nil
}

// RunStart runs the lambda handled by this Handler. It checks if the code has
// been pulled, sandbox been created, and sandbox been started. The channel of
// the sandbox of this lambda is returned.
func (h *Handler) RunStart() (ch *sb.SandboxChannel, err error) {
	start := time.Now()
	defer log.Printf("start server took %v\n", time.Since(start))

	h.mutex.Lock()
	defer h.mutex.Unlock()

	hm := h.hm
	hms := h.hm.hms

	// create sandbox if needed
	if h.sandbox == nil {
		sandbox, err := hms.sbFactory.Create(hm.codeDir, hm.workingDir)
		if err != nil {
			return nil, err
		}

		h.sandbox = sandbox
		h.id = h.sandbox.ID()
		h.hostDir = path.Join(hm.workingDir, h.id)
		if sbState, err := h.sandbox.State(); err != nil {
			return nil, err
		} else if sbState == state.Stopped {
			if err := h.sandbox.Start(); err != nil {
				return nil, err
			}
		} else if sbState == state.Paused {
			if err := h.sandbox.Unpause(); err != nil {
				return nil, err
			}
		}

		hit := false
		if hms.cacheMgr == nil || hms.cacheMgr.Full() {
			err := h.sandbox.RunServer()
			if err != nil {
				return nil, err
			}
		} else {
			containerSB, ok := h.sandbox.(sb.ContainerSandbox)
			if !ok {
				return nil, fmt.Errorf("forkenter only supported with ContainerSandbox")
			}
			if h.fs, hit, err = hms.cacheMgr.Provision(containerSB, h.hostDir, hm.pkgs); err != nil {
				return nil, err
			}

		}

		if hit {
			atomic.AddInt64(hms.ihits, 1)
		} else {
			atomic.AddInt64(hms.misses, 1)
		}

		sockPath := fmt.Sprintf("%s/ol.sock", h.hostDir)

		// wait up to 20s for server to initialize
		start := time.Now()
		for ok := true; ok; ok = os.IsNotExist(err) {
			_, err = os.Stat(sockPath)
			if hms.config.Sandbox == "olcontainer" && (hms.cacheMgr == nil || hms.cacheMgr.Full()) {
				time.Sleep(10 * time.Microsecond)
				if err := h.sandbox.RunServer(); err != nil {
					return nil, err
				}
			}
			if time.Since(start).Seconds() > 600 {
				return nil, fmt.Errorf("handler server failed to initialize after 20s")
			}
			time.Sleep(50 * time.Microsecond)
		}

		// we are up so we can add ourselves for reuse
		if hms.maxRunners == 0 || h.runners < hms.maxRunners {
			hm.mutex.Lock()
			hm.hElements[h] = hm.handlers.PushFront(h)
			hm.maxHandlers = max(hm.maxHandlers, hm.handlers.Len())
			hm.mutex.Unlock()
		}

	} else if sbState, _ := h.sandbox.State(); sbState == state.Paused {
		// unpause if paused
		atomic.AddInt64(hms.hhits, 1)
		if err := h.sandbox.Unpause(); err != nil {
			return nil, err
		}
	} else {
		atomic.AddInt64(hms.hhits, 1)
	}

	log.Printf("handler hits: %v, import hits: %v, misses: %v", *hms.hhits, *hms.ihits, *hms.misses)
	return h.sandbox.Channel()
}

// RunFinish notifies that a request to run the lambda has completed. If no
// request is being run in its sandbox, sandbox will be paused and the handler
// be added to the HandlerLRU.
func (h *Handler) RunFinish() {
	h.mutex.Lock()

	hm := h.hm
	hms := h.hm.hms

	h.runners -= 1

	// are we the last?
	if h.runners == 0 {
		if err := h.sandbox.Pause(); err != nil {
			// TODO(tyler): better way to handle this?  If
			// we can't pause, the handler gets to keep
			// running for free...
			log.Printf("Could not pause %v: %v!  Error: %v\n", h.name, h.id, err)
		}

		if handlerUsage(h) > hms.lru.soft_limit {
			h.mutex.Unlock()

			// we were potentially the last runner
			// try to remove us from the handler manager
			if err := hm.TryRemoveHandler(h); err == nil {
				// we were the last one so... bye
				go h.nuke()
			}
			return
		}

		hm.AddHandler(h)
		hms.lru.Add(h)
	} else {
		hm.AddHandler(h)
	}

	h.mutex.Unlock()
}

func (h *Handler) nuke() {
	if err := h.sandbox.Unpause(); err != nil {
		log.Printf("failed to unpause sandbox :: %v", err.Error())
	}
	if err := h.sandbox.Stop(); err != nil {
		log.Printf("failed to stop sandbox :: %v", err.Error())
	}
	if err := h.sandbox.Remove(); err != nil {
		log.Printf("failed to remove sandbox :: %v", err.Error())
	}
}

// Sandbox returns the sandbox of this Handler.
func (h *Handler) Sandbox() sb.Sandbox {
	return h.sandbox
}

func max(i1, i2 int) int {
	if i1 < i2 {
		return i2
	}
	return i1
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

import (
	"fmt"
	"strings"

	"github.com/spf13/pflag"

	"k8s.io/apimachinery/pkg/runtime"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apiserver/pkg/admission"
	"k8s.io/apiserver/pkg/admission/initializer"
	admissionmetrics "k8s.io/apiserver/pkg/admission/metrics"
	"k8s.io/apiserver/pkg/admission/plugin/initialization"
	"k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle"
	mutatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/mutating"
	validatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/validating"
	apiserverapi "k8s.io/apiserver/pkg/apis/apiserver"
	apiserverapiv1alpha1 "k8s.io/apiserver/pkg/apis/apiserver/v1alpha1"
	"k8s.io/apiserver/pkg/server"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

var configScheme = runtime.NewScheme()

func init() {
	utilruntime.Must(apiserverapi.AddToScheme(configScheme))
	utilruntime.Must(apiserverapiv1alpha1.AddToScheme(configScheme))
}

// AdmissionOptions holds the admission options
type AdmissionOptions struct {
	// RecommendedPluginOrder holds an ordered list of plugin names we recommend to use by default
	RecommendedPluginOrder []string
	// DefaultOffPlugins is a set of plugin names that is disabled by default
	DefaultOffPlugins sets.String

	// EnablePlugins indicates plugins to be enabled passed through `--enable-admission-plugins`.
	EnablePlugins []string
	// DisablePlugins indicates plugins to be disabled passed through `--disable-admission-plugins`.
	DisablePlugins []string
	// ConfigFile is the file path with admission control configuration.
	ConfigFile string
	// Plugins contains all registered plugins.
	Plugins *admission.Plugins
}

// NewAdmissionOptions creates a new instance of AdmissionOptions
// Note:
//  In addition it calls RegisterAllAdmissionPlugins to register
//  all generic admission plugins.
//
//  Provides the list of RecommendedPluginOrder that holds sane values
//  that can be used by servers that don't care about admission chain.
//  Servers that do care can overwrite/append that field after creation.
func NewAdmissionOptions() *AdmissionOptions {
	options := &AdmissionOptions{
		Plugins: admission.NewPlugins(),
		// This list is mix of mutating admission plugins and validating
		// admission plugins. The apiserver always runs the validating ones
		// after all the mutating ones, so their relative order in this list
		// doesn't matter.
		RecommendedPluginOrder: []string{lifecycle.PluginName, initialization.PluginName, mutatingwebhook.PluginName, validatingwebhook.PluginName},
		DefaultOffPlugins:      sets.NewString(initialization.PluginName),
	}
	server.RegisterAllAdmissionPlugins(options.Plugins)
	return options
}

// AddFlags adds flags related to admission for a specific APIServer to the specified FlagSet
func (a *AdmissionOptions) AddFlags(fs *pflag.FlagSet) {
	if a == nil {
		return
	}

	fs.StringSliceVar(&a.EnablePlugins, "enable-admission-plugins", a.EnablePlugins, ""+
		"admission plugins that should be enabled in addition to default enabled ones. "+
		"Comma-delimited list of admission plugins: "+strings.Join(a.Plugins.Registered(), ", ")+". "+
		"The order of plugins in this flag does not matter.")
	fs.StringSliceVar(&a.DisablePlugins, "disable-admission-plugins", a.DisablePlugins, ""+
		"admission plugins that should be disabled although they are in the default enabled plugins list. "+
		"Comma-delimited list of admission plugins: "+strings.Join(a.Plugins.Registered(), ", ")+". "+
		"The order of plugins in this flag does not matter.")
	fs.StringVar(&a.ConfigFile, "admission-control-config-file", a.ConfigFile,
		"File with admission control configuration.")
}

// ApplyTo adds the admission chain to the server configuration.
// In case admission plugin names were not provided by a custer-admin they will be prepared from the recommended/default values.
// In addition the method lazily initializes a generic plugin that is appended to the list of pluginInitializers
// note this method uses:
//  genericconfig.Authorizer
func (a *AdmissionOptions) ApplyTo(
	c *server.Config,
	informers informers.SharedInformerFactory,
	kubeAPIServerClientConfig *rest.Config,
	scheme *runtime.Scheme,
	pluginInitializers ...admission.PluginInitializer,
) error {
	if a == nil {
		return nil
	}

	// Admission need scheme to construct admission initializer.
	if scheme == nil {
		return fmt.Errorf("admission depends on a scheme, it cannot be nil")
	}

	// Admission depends on CoreAPI to set SharedInformerFactory and ClientConfig.
	if informers == nil {
		return fmt.Errorf("admission depends on a Kubernetes core API shared informer, it cannot be nil")
	}

	pluginNames := a.enabledPluginNames()

	pluginsConfigProvider, err := admission.ReadAdmissionConfiguration(pluginNames, a.ConfigFile, configScheme)
	if err != nil {
		return fmt.Errorf("failed to read plugin config: %v", err)
	}

	clientset, err := kubernetes.NewForConfig(kubeAPIServerClientConfig)
	if err != nil {
		return err
	}
	genericInitializer := initializer.New(clientset, informers, c.Authorization.Authorizer, scheme)
	initializersChain := admission.PluginInitializers{}
	pluginInitializers = append(pluginInitializers, genericInitializer)
	initializersChain = append(initializersChain, pluginInitializers...)

	admissionChain, err := a.Plugins.NewFromPlugins(pluginNames, pluginsConfigProvider, initializersChain, admission.DecoratorFunc(admissionmetrics.WithControllerMetrics))
	if err != nil {
		return err
	}

	c.AdmissionControl = admissionmetrics.WithStepMetrics(admissionChain)
	return nil
}

// Validate verifies flags passed to AdmissionOptions.
func (a *AdmissionOptions) Validate() []error {
	if a == nil {
		return nil
	}

	errs := []error{}

	registeredPlugins := sets.NewString(a.Plugins.Registered()...)
	for _, name := range a.EnablePlugins {
		if !registeredPlugins.Has(name) {
			errs = append(errs, fmt.Errorf("enable-admission-plugins plugin %q is unknown", name))
		}
	}

	for _, name := range a.DisablePlugins {
		if !registeredPlugins.Has(name) {
			errs = append(errs, fmt.Errorf("disable-admission-plugins plugin %q is unknown", name))
		}
	}

	enablePlugins := sets.NewString(a.EnablePlugins...)
	disablePlugins := sets.NewString(a.DisablePlugins...)
	if len(enablePlugins.Intersection(disablePlugins).List()) > 0 {
		errs = append(errs, fmt.Errorf("%v in enable-admission-plugins and disable-admission-plugins "+
			"overlapped", enablePlugins.Intersection(disablePlugins).List()))
	}

	// Verify RecommendedPluginOrder.
	recommendPlugins := sets.NewString(a.RecommendedPluginOrder...)
	intersections := registeredPlugins.Intersection(recommendPlugins)
	if !intersections.Equal(recommendPlugins) {
		// Developer error, this should never run in.
		errs = append(errs, fmt.Errorf("plugins %v in RecommendedPluginOrder are not registered",
			recommendPlugins.Difference(intersections).List()))
	}
	if !intersections.Equal(registeredPlugins) {
		// Developer error, this should never run in.
		errs = append(errs, fmt.Errorf("plugins %v registered are not in RecommendedPluginOrder",
			registeredPlugins.Difference(intersections).List()))
	}

	return errs
}

// enabledPluginNames makes use of RecommendedPluginOrder, DefaultOffPlugins,
// EnablePlugins, DisablePlugins fields
// to prepare a list of ordered plugin names that are enabled.
func (a *AdmissionOptions) enabledPluginNames() []string {
	allOffPlugins := append(a.DefaultOffPlugins.List(), a.DisablePlugins...)
	disabledPlugins := sets.NewString(allOffPlugins...)
	enabledPlugins := sets.NewString(a.EnablePlugins...)
	disabledPlugins = disabledPlugins.Difference(enabledPlugins)

	orderedPlugins := []string{}
	for _, plugin := range a.RecommendedPluginOrder {
		if !disabledPlugins.Has(plugin) {
			orderedPlugins = append(orderedPlugins, plugin)
		}
	}

	return orderedPlugins
}
<commit_msg>list the default enabled admission plugins<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

import (
	"fmt"
	"strings"

	"github.com/spf13/pflag"

	"k8s.io/apimachinery/pkg/runtime"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apiserver/pkg/admission"
	"k8s.io/apiserver/pkg/admission/initializer"
	admissionmetrics "k8s.io/apiserver/pkg/admission/metrics"
	"k8s.io/apiserver/pkg/admission/plugin/initialization"
	"k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle"
	mutatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/mutating"
	validatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/validating"
	apiserverapi "k8s.io/apiserver/pkg/apis/apiserver"
	apiserverapiv1alpha1 "k8s.io/apiserver/pkg/apis/apiserver/v1alpha1"
	"k8s.io/apiserver/pkg/server"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

var configScheme = runtime.NewScheme()

func init() {
	utilruntime.Must(apiserverapi.AddToScheme(configScheme))
	utilruntime.Must(apiserverapiv1alpha1.AddToScheme(configScheme))
}

// AdmissionOptions holds the admission options
type AdmissionOptions struct {
	// RecommendedPluginOrder holds an ordered list of plugin names we recommend to use by default
	RecommendedPluginOrder []string
	// DefaultOffPlugins is a set of plugin names that is disabled by default
	DefaultOffPlugins sets.String

	// EnablePlugins indicates plugins to be enabled passed through `--enable-admission-plugins`.
	EnablePlugins []string
	// DisablePlugins indicates plugins to be disabled passed through `--disable-admission-plugins`.
	DisablePlugins []string
	// ConfigFile is the file path with admission control configuration.
	ConfigFile string
	// Plugins contains all registered plugins.
	Plugins *admission.Plugins
}

// NewAdmissionOptions creates a new instance of AdmissionOptions
// Note:
//  In addition it calls RegisterAllAdmissionPlugins to register
//  all generic admission plugins.
//
//  Provides the list of RecommendedPluginOrder that holds sane values
//  that can be used by servers that don't care about admission chain.
//  Servers that do care can overwrite/append that field after creation.
func NewAdmissionOptions() *AdmissionOptions {
	options := &AdmissionOptions{
		Plugins: admission.NewPlugins(),
		// This list is mix of mutating admission plugins and validating
		// admission plugins. The apiserver always runs the validating ones
		// after all the mutating ones, so their relative order in this list
		// doesn't matter.
		RecommendedPluginOrder: []string{lifecycle.PluginName, initialization.PluginName, mutatingwebhook.PluginName, validatingwebhook.PluginName},
		DefaultOffPlugins:      sets.NewString(initialization.PluginName),
	}
	server.RegisterAllAdmissionPlugins(options.Plugins)
	return options
}

// AddFlags adds flags related to admission for a specific APIServer to the specified FlagSet
func (a *AdmissionOptions) AddFlags(fs *pflag.FlagSet) {
	if a == nil {
		return
	}

	fs.StringSliceVar(&a.EnablePlugins, "enable-admission-plugins", a.EnablePlugins, ""+
		"admission plugins that should be enabled in addition to default enabled ones ("+
		strings.Join(a.defaultEnabledPluginNames(), ", ")+"). "+
		"Comma-delimited list of admission plugins: "+strings.Join(a.Plugins.Registered(), ", ")+". "+
		"The order of plugins in this flag does not matter.")
	fs.StringSliceVar(&a.DisablePlugins, "disable-admission-plugins", a.DisablePlugins, ""+
		"admission plugins that should be disabled although they are in the default enabled plugins list ("+
		strings.Join(a.defaultEnabledPluginNames(), ", ")+"). "+
		"Comma-delimited list of admission plugins: "+strings.Join(a.Plugins.Registered(), ", ")+". "+
		"The order of plugins in this flag does not matter.")
	fs.StringVar(&a.ConfigFile, "admission-control-config-file", a.ConfigFile,
		"File with admission control configuration.")
}

// ApplyTo adds the admission chain to the server configuration.
// In case admission plugin names were not provided by a custer-admin they will be prepared from the recommended/default values.
// In addition the method lazily initializes a generic plugin that is appended to the list of pluginInitializers
// note this method uses:
//  genericconfig.Authorizer
func (a *AdmissionOptions) ApplyTo(
	c *server.Config,
	informers informers.SharedInformerFactory,
	kubeAPIServerClientConfig *rest.Config,
	scheme *runtime.Scheme,
	pluginInitializers ...admission.PluginInitializer,
) error {
	if a == nil {
		return nil
	}

	// Admission need scheme to construct admission initializer.
	if scheme == nil {
		return fmt.Errorf("admission depends on a scheme, it cannot be nil")
	}

	// Admission depends on CoreAPI to set SharedInformerFactory and ClientConfig.
	if informers == nil {
		return fmt.Errorf("admission depends on a Kubernetes core API shared informer, it cannot be nil")
	}

	pluginNames := a.enabledPluginNames()

	pluginsConfigProvider, err := admission.ReadAdmissionConfiguration(pluginNames, a.ConfigFile, configScheme)
	if err != nil {
		return fmt.Errorf("failed to read plugin config: %v", err)
	}

	clientset, err := kubernetes.NewForConfig(kubeAPIServerClientConfig)
	if err != nil {
		return err
	}
	genericInitializer := initializer.New(clientset, informers, c.Authorization.Authorizer, scheme)
	initializersChain := admission.PluginInitializers{}
	pluginInitializers = append(pluginInitializers, genericInitializer)
	initializersChain = append(initializersChain, pluginInitializers...)

	admissionChain, err := a.Plugins.NewFromPlugins(pluginNames, pluginsConfigProvider, initializersChain, admission.DecoratorFunc(admissionmetrics.WithControllerMetrics))
	if err != nil {
		return err
	}

	c.AdmissionControl = admissionmetrics.WithStepMetrics(admissionChain)
	return nil
}

// Validate verifies flags passed to AdmissionOptions.
func (a *AdmissionOptions) Validate() []error {
	if a == nil {
		return nil
	}

	errs := []error{}

	registeredPlugins := sets.NewString(a.Plugins.Registered()...)
	for _, name := range a.EnablePlugins {
		if !registeredPlugins.Has(name) {
			errs = append(errs, fmt.Errorf("enable-admission-plugins plugin %q is unknown", name))
		}
	}

	for _, name := range a.DisablePlugins {
		if !registeredPlugins.Has(name) {
			errs = append(errs, fmt.Errorf("disable-admission-plugins plugin %q is unknown", name))
		}
	}

	enablePlugins := sets.NewString(a.EnablePlugins...)
	disablePlugins := sets.NewString(a.DisablePlugins...)
	if len(enablePlugins.Intersection(disablePlugins).List()) > 0 {
		errs = append(errs, fmt.Errorf("%v in enable-admission-plugins and disable-admission-plugins "+
			"overlapped", enablePlugins.Intersection(disablePlugins).List()))
	}

	// Verify RecommendedPluginOrder.
	recommendPlugins := sets.NewString(a.RecommendedPluginOrder...)
	intersections := registeredPlugins.Intersection(recommendPlugins)
	if !intersections.Equal(recommendPlugins) {
		// Developer error, this should never run in.
		errs = append(errs, fmt.Errorf("plugins %v in RecommendedPluginOrder are not registered",
			recommendPlugins.Difference(intersections).List()))
	}
	if !intersections.Equal(registeredPlugins) {
		// Developer error, this should never run in.
		errs = append(errs, fmt.Errorf("plugins %v registered are not in RecommendedPluginOrder",
			registeredPlugins.Difference(intersections).List()))
	}

	return errs
}

// enabledPluginNames makes use of RecommendedPluginOrder, DefaultOffPlugins,
// EnablePlugins, DisablePlugins fields
// to prepare a list of ordered plugin names that are enabled.
func (a *AdmissionOptions) enabledPluginNames() []string {
	allOffPlugins := append(a.DefaultOffPlugins.List(), a.DisablePlugins...)
	disabledPlugins := sets.NewString(allOffPlugins...)
	enabledPlugins := sets.NewString(a.EnablePlugins...)
	disabledPlugins = disabledPlugins.Difference(enabledPlugins)

	orderedPlugins := []string{}
	for _, plugin := range a.RecommendedPluginOrder {
		if !disabledPlugins.Has(plugin) {
			orderedPlugins = append(orderedPlugins, plugin)
		}
	}

	return orderedPlugins
}

//Return names of plugins which are enabled by default
func (a *AdmissionOptions) defaultEnabledPluginNames() []string {
	defaultOnPluginNames := []string{}
	for _, pluginName := range a.RecommendedPluginOrder {
		if !a.DefaultOffPlugins.Has(pluginName) {
			defaultOnPluginNames = append(defaultOnPluginNames, pluginName)
		}
	}

	return defaultOnPluginNames
}
<|endoftext|>