<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/pkg"
	"github.com/coreos/coreos-cloudinit/system"
)

const (
	version               = "0.8.5"
	datasourceInterval    = 100 * time.Millisecond
	datasourceMaxInterval = 30 * time.Second
	datasourceTimeout     = 5 * time.Minute
)

var (
	printVersion  bool
	ignoreFailure bool
	sources       struct {
		file            string
		configDrive     string
		metadataService bool
		url             string
		procCmdLine     bool
	}
	convertNetconf string
	workspace      string
	sshKeyName     string
)

func init() {
	flag.BoolVar(&printVersion, "version", false, "Print the version and exit")
	flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")
	flag.StringVar(&sources.file, "from-file", "", "Read user-data from provided file")
	flag.StringVar(&sources.configDrive, "from-configdrive", "", "Read data from provided cloud-drive directory")
	flag.BoolVar(&sources.metadataService, "from-metadata-service", false, "Download data from metadata service")
	flag.StringVar(&sources.url, "from-url", "", "Download user-data from provided url")
	flag.BoolVar(&sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", datasource.ProcCmdlineLocation, datasource.ProcCmdlineCloudConfigFlag))
	flag.StringVar(&convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files (requires the -from-configdrive flag)")
	flag.StringVar(&workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")
	flag.StringVar(&sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")
}

func main() {
	flag.Parse()

	die := func() {
		if ignoreFailure {
			os.Exit(0)
		}
		os.Exit(1)
	}

	if printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	if convertNetconf != "" && sources.configDrive == "" {
		fmt.Println("-convert-netconf flag requires -from-configdrive")
		os.Exit(1)
	}

	switch convertNetconf {
	case "":
	case "debian":
	default:
		fmt.Printf("Invalid option to -convert-netconf: '%s'. Supported options: 'debian'\n", convertNetconf)
		os.Exit(1)
	}

	dss := getDatasources()
	if len(dss) == 0 {
		fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-metadata-service, --from-url or --from-proc-cmdline")
		os.Exit(1)
	}

	ds := selectDatasource(dss)
	if ds == nil {
		fmt.Println("No datasources available in time")
		die()
	}

	fmt.Printf("Fetching user-data from datasource of type %q\n", ds.Type())
	userdataBytes, err := ds.FetchUserdata()
	if err != nil {
		fmt.Printf("Failed fetching user-data from datasource: %v\n", err)
		die()
	}

	fmt.Printf("Fetching meta-data from datasource of type %q\n", ds.Type())
	metadataBytes, err := ds.FetchMetadata()
	if err != nil {
		fmt.Printf("Failed fetching meta-data from datasource: %v\n", err)
		die()
	}

	// Extract IPv4 addresses from metadata if possible
	var subs map[string]string
	if len(metadataBytes) > 0 {
		subs, err = initialize.ExtractIPsFromMetadata(metadataBytes)
		if err != nil {
			fmt.Printf("Failed extracting IPs from meta-data: %v\n", err)
			die()
		}
	}

	// Apply environment to user-data
	env := initialize.NewEnvironment("/", ds.ConfigRoot(), workspace, convertNetconf, sshKeyName, subs)
	userdata := env.Apply(string(userdataBytes))

	var ccm, ccu *initialize.CloudConfig
	var script *system.Script
	if ccm, err = initialize.ParseMetaData(string(metadataBytes)); err != nil {
		fmt.Printf("Failed to parse meta-data: %v\n", err)
		die()
	}
	if ud, err := initialize.ParseUserData(userdata); err != nil {
		fmt.Printf("Failed to parse user-data: %v\n", err)
		die()
	} else {
		switch t := ud.(type) {
		case *initialize.CloudConfig:
			ccu = t
		case system.Script:
			script = &t
		}
	}

	var cc *initialize.CloudConfig
	if ccm != nil && ccu != nil {
		fmt.Println("Merging cloud-config from meta-data and user-data")
		merged := mergeCloudConfig(*ccm, *ccu)
		cc = &merged
	} else if ccm != nil && ccu == nil {
		fmt.Println("Processing cloud-config from meta-data")
		cc = ccm
	} else if ccm == nil && ccu != nil {
		fmt.Println("Processing cloud-config from user-data")
		cc = ccu
	} else {
		fmt.Println("No cloud-config data to handle.")
	}

	if cc != nil {
		if err = initialize.Apply(*cc, env); err != nil {
			fmt.Printf("Failed to apply cloud-config: %v\n", err)
			die()
		}
	}

	if script != nil {
		if err = runScript(*script, env); err != nil {
			fmt.Printf("Failed to run script: %v\n", err)
			die()
		}
	}
}

// mergeCloudConfig merges certain options from mdcc (a CloudConfig derived from
// meta-data) onto udcc (a CloudConfig derived from user-data), if they are
// not already set on udcc (i.e. user-data always takes precedence)
// NB: This needs to be kept in sync with ParseMetadata so that it tracks all
// elements of a CloudConfig which that function can populate.
func mergeCloudConfig(mdcc, udcc initialize.CloudConfig) (cc initialize.CloudConfig) {
	if mdcc.Hostname != "" {
		if udcc.Hostname != "" {
			fmt.Printf("Warning: user-data hostname (%s) overrides metadata hostname (%s)", udcc.Hostname, mdcc.Hostname)
		} else {
			udcc.Hostname = mdcc.Hostname
		}

	}
	for _, key := range mdcc.SSHAuthorizedKeys {
		udcc.SSHAuthorizedKeys = append(udcc.SSHAuthorizedKeys, key)
	}
	if mdcc.NetworkConfigPath != "" {
		if udcc.NetworkConfigPath != "" {
			fmt.Printf("Warning: user-data NetworkConfigPath %s overrides metadata NetworkConfigPath %s", udcc.NetworkConfigPath, mdcc.NetworkConfigPath)
		} else {
			udcc.NetworkConfigPath = mdcc.NetworkConfigPath
		}
	}
	return udcc
}

// getDatasources creates a slice of possible Datasources for cloudinit based
// on the different source command-line flags.
func getDatasources() []datasource.Datasource {
	dss := make([]datasource.Datasource, 0, 5)
	if sources.file != "" {
		dss = append(dss, datasource.NewLocalFile(sources.file))
	}
	if sources.url != "" {
		dss = append(dss, datasource.NewRemoteFile(sources.url))
	}
	if sources.configDrive != "" {
		dss = append(dss, datasource.NewConfigDrive(sources.configDrive))
	}
	if sources.metadataService {
		dss = append(dss, datasource.NewMetadataService())
	}
	if sources.procCmdLine {
		dss = append(dss, datasource.NewProcCmdline())
	}
	return dss
}

// selectDatasource attempts to choose a valid Datasource to use based on its
// current availability. The first Datasource to report to be available is
// returned. Datasources will be retried if possible if they are not
// immediately available. If all Datasources are permanently unavailable or
// datasourceTimeout is reached before one becomes available, nil is returned.
func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
	ds := make(chan datasource.Datasource)
	stop := make(chan struct{})
	var wg sync.WaitGroup

	for _, s := range sources {
		wg.Add(1)
		go func(s datasource.Datasource) {
			defer wg.Done()

			duration := datasourceInterval
			for {
				fmt.Printf("Checking availability of %q\n", s.Type())
				if s.IsAvailable() {
					ds <- s
					return
				} else if !s.AvailabilityChanges() {
					return
				}
				select {
				case <-stop:
					return
				case <-time.Tick(duration):
					duration = pkg.ExpBackoff(duration, datasourceMaxInterval)
				}
			}
		}(s)
	}

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	var s datasource.Datasource
	select {
	case s = <-ds:
	case <-done:
	case <-time.Tick(datasourceTimeout):
	}

	close(stop)
	return s
}

// TODO(jonboulle): this should probably be refactored and moved into a different module
func runScript(script system.Script, env *initialize.Environment) error {
	err := initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		fmt.Printf("Failed preparing workspace: %v\n", err)
		return err
	}
	path, err := initialize.PersistScriptInWorkspace(script, env.Workspace())
	if err == nil {
		var name string
		name, err = system.ExecuteScript(path)
		initialize.PersistUnitNameInWorkspace(name, env.Workspace())
	}
	return err
}
<commit_msg>coreos-cloudinit: bump to 0.8.5+git<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/pkg"
	"github.com/coreos/coreos-cloudinit/system"
)

const (
	version               = "0.8.5+git"
	datasourceInterval    = 100 * time.Millisecond
	datasourceMaxInterval = 30 * time.Second
	datasourceTimeout     = 5 * time.Minute
)

var (
	printVersion  bool
	ignoreFailure bool
	sources       struct {
		file            string
		configDrive     string
		metadataService bool
		url             string
		procCmdLine     bool
	}
	convertNetconf string
	workspace      string
	sshKeyName     string
)

func init() {
	flag.BoolVar(&printVersion, "version", false, "Print the version and exit")
	flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")
	flag.StringVar(&sources.file, "from-file", "", "Read user-data from provided file")
	flag.StringVar(&sources.configDrive, "from-configdrive", "", "Read data from provided cloud-drive directory")
	flag.BoolVar(&sources.metadataService, "from-metadata-service", false, "Download data from metadata service")
	flag.StringVar(&sources.url, "from-url", "", "Download user-data from provided url")
	flag.BoolVar(&sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", datasource.ProcCmdlineLocation, datasource.ProcCmdlineCloudConfigFlag))
	flag.StringVar(&convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files (requires the -from-configdrive flag)")
	flag.StringVar(&workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")
	flag.StringVar(&sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")
}

func main() {
	flag.Parse()

	die := func() {
		if ignoreFailure {
			os.Exit(0)
		}
		os.Exit(1)
	}

	if printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	if convertNetconf != "" && sources.configDrive == "" {
		fmt.Println("-convert-netconf flag requires -from-configdrive")
		os.Exit(1)
	}

	switch convertNetconf {
	case "":
	case "debian":
	default:
		fmt.Printf("Invalid option to -convert-netconf: '%s'. Supported options: 'debian'\n", convertNetconf)
		os.Exit(1)
	}

	dss := getDatasources()
	if len(dss) == 0 {
		fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-metadata-service, --from-url or --from-proc-cmdline")
		os.Exit(1)
	}

	ds := selectDatasource(dss)
	if ds == nil {
		fmt.Println("No datasources available in time")
		die()
	}

	fmt.Printf("Fetching user-data from datasource of type %q\n", ds.Type())
	userdataBytes, err := ds.FetchUserdata()
	if err != nil {
		fmt.Printf("Failed fetching user-data from datasource: %v\n", err)
		die()
	}

	fmt.Printf("Fetching meta-data from datasource of type %q\n", ds.Type())
	metadataBytes, err := ds.FetchMetadata()
	if err != nil {
		fmt.Printf("Failed fetching meta-data from datasource: %v\n", err)
		die()
	}

	// Extract IPv4 addresses from metadata if possible
	var subs map[string]string
	if len(metadataBytes) > 0 {
		subs, err = initialize.ExtractIPsFromMetadata(metadataBytes)
		if err != nil {
			fmt.Printf("Failed extracting IPs from meta-data: %v\n", err)
			die()
		}
	}

	// Apply environment to user-data
	env := initialize.NewEnvironment("/", ds.ConfigRoot(), workspace, convertNetconf, sshKeyName, subs)
	userdata := env.Apply(string(userdataBytes))

	var ccm, ccu *initialize.CloudConfig
	var script *system.Script
	if ccm, err = initialize.ParseMetaData(string(metadataBytes)); err != nil {
		fmt.Printf("Failed to parse meta-data: %v\n", err)
		die()
	}
	if ud, err := initialize.ParseUserData(userdata); err != nil {
		fmt.Printf("Failed to parse user-data: %v\n", err)
		die()
	} else {
		switch t := ud.(type) {
		case *initialize.CloudConfig:
			ccu = t
		case system.Script:
			script = &t
		}
	}

	var cc *initialize.CloudConfig
	if ccm != nil && ccu != nil {
		fmt.Println("Merging cloud-config from meta-data and user-data")
		merged := mergeCloudConfig(*ccm, *ccu)
		cc = &merged
	} else if ccm != nil && ccu == nil {
		fmt.Println("Processing cloud-config from meta-data")
		cc = ccm
	} else if ccm == nil && ccu != nil {
		fmt.Println("Processing cloud-config from user-data")
		cc = ccu
	} else {
		fmt.Println("No cloud-config data to handle.")
	}

	if cc != nil {
		if err = initialize.Apply(*cc, env); err != nil {
			fmt.Printf("Failed to apply cloud-config: %v\n", err)
			die()
		}
	}

	if script != nil {
		if err = runScript(*script, env); err != nil {
			fmt.Printf("Failed to run script: %v\n", err)
			die()
		}
	}
}

// mergeCloudConfig merges certain options from mdcc (a CloudConfig derived from
// meta-data) onto udcc (a CloudConfig derived from user-data), if they are
// not already set on udcc (i.e. user-data always takes precedence)
// NB: This needs to be kept in sync with ParseMetadata so that it tracks all
// elements of a CloudConfig which that function can populate.
func mergeCloudConfig(mdcc, udcc initialize.CloudConfig) (cc initialize.CloudConfig) {
	if mdcc.Hostname != "" {
		if udcc.Hostname != "" {
			fmt.Printf("Warning: user-data hostname (%s) overrides metadata hostname (%s)", udcc.Hostname, mdcc.Hostname)
		} else {
			udcc.Hostname = mdcc.Hostname
		}

	}
	for _, key := range mdcc.SSHAuthorizedKeys {
		udcc.SSHAuthorizedKeys = append(udcc.SSHAuthorizedKeys, key)
	}
	if mdcc.NetworkConfigPath != "" {
		if udcc.NetworkConfigPath != "" {
			fmt.Printf("Warning: user-data NetworkConfigPath %s overrides metadata NetworkConfigPath %s", udcc.NetworkConfigPath, mdcc.NetworkConfigPath)
		} else {
			udcc.NetworkConfigPath = mdcc.NetworkConfigPath
		}
	}
	return udcc
}

// getDatasources creates a slice of possible Datasources for cloudinit based
// on the different source command-line flags.
func getDatasources() []datasource.Datasource {
	dss := make([]datasource.Datasource, 0, 5)
	if sources.file != "" {
		dss = append(dss, datasource.NewLocalFile(sources.file))
	}
	if sources.url != "" {
		dss = append(dss, datasource.NewRemoteFile(sources.url))
	}
	if sources.configDrive != "" {
		dss = append(dss, datasource.NewConfigDrive(sources.configDrive))
	}
	if sources.metadataService {
		dss = append(dss, datasource.NewMetadataService())
	}
	if sources.procCmdLine {
		dss = append(dss, datasource.NewProcCmdline())
	}
	return dss
}

// selectDatasource attempts to choose a valid Datasource to use based on its
// current availability. The first Datasource to report to be available is
// returned. Datasources will be retried if possible if they are not
// immediately available. If all Datasources are permanently unavailable or
// datasourceTimeout is reached before one becomes available, nil is returned.
func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
	ds := make(chan datasource.Datasource)
	stop := make(chan struct{})
	var wg sync.WaitGroup

	for _, s := range sources {
		wg.Add(1)
		go func(s datasource.Datasource) {
			defer wg.Done()

			duration := datasourceInterval
			for {
				fmt.Printf("Checking availability of %q\n", s.Type())
				if s.IsAvailable() {
					ds <- s
					return
				} else if !s.AvailabilityChanges() {
					return
				}
				select {
				case <-stop:
					return
				case <-time.Tick(duration):
					duration = pkg.ExpBackoff(duration, datasourceMaxInterval)
				}
			}
		}(s)
	}

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	var s datasource.Datasource
	select {
	case s = <-ds:
	case <-done:
	case <-time.Tick(datasourceTimeout):
	}

	close(stop)
	return s
}

// TODO(jonboulle): this should probably be refactored and moved into a different module
func runScript(script system.Script, env *initialize.Environment) error {
	err := initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		fmt.Printf("Failed preparing workspace: %v\n", err)
		return err
	}
	path, err := initialize.PersistScriptInWorkspace(script, env.Workspace())
	if err == nil {
		var name string
		name, err = system.ExecuteScript(path)
		initialize.PersistUnitNameInWorkspace(name, env.Workspace())
	}
	return err
}
<|endoftext|>


<commit_before>package web

import (
	"github.com/anthonynsimon/parrot/datastore"
	"github.com/anthonynsimon/parrot/paths"
	"github.com/pressly/chi"
)

var store datastore.Store

func Register(router *chi.Mux, ds datastore.Store) {
	store = ds
	registerRoutes(router)
}

func registerRoutes(router *chi.Mux) {
	router.Get(paths.PingPath, webHandlerFunc(ping).ServeHTTP)
	router.Post(paths.RegisterPath, webHandlerFunc(newUser).ServeHTTP)

	router.Route(paths.ProjectsPath, func(r chi.Router) {
		r.Get("/:projectID", webHandlerFunc(showProject).ServeHTTP)
		r.Get("/new", webHandlerFunc(newProject).ServeHTTP)

		r.Route("/:projectID"+paths.DocumentsPath, func(r chi.Router) {
			r.Get("/", webHandlerFunc(findDocuments).ServeHTTP)
			r.Get("/:documentID", webHandlerFunc(showDocument).ServeHTTP)
			r.Get("/new", webHandlerFunc(newDocument).ServeHTTP)
		})
	})

	router.Route(paths.UsersPath, func(r chi.Router) {
		r.Get("/:userID", webHandlerFunc(showUser).ServeHTTP)
	})
}
<commit_msg>Add user registration route<commit_after>package web

import (
	"github.com/anthonynsimon/parrot/datastore"
	"github.com/anthonynsimon/parrot/paths"
	"github.com/pressly/chi"
)

var store datastore.Store

func Register(router *chi.Mux, ds datastore.Store) {
	store = ds
	registerRoutes(router)
}

func registerRoutes(router *chi.Mux) {
	router.Get(paths.PingPath, webHandlerFunc(ping).ServeHTTP)
	router.Get(paths.RegisterPath, webHandlerFunc(newUser).ServeHTTP)
	router.Post(paths.RegisterPath, webHandlerFunc(createUser).ServeHTTP)

	router.Route(paths.ProjectsPath, func(r chi.Router) {
		r.Get("/:projectID", webHandlerFunc(showProject).ServeHTTP)
		r.Get("/new", webHandlerFunc(newProject).ServeHTTP)

		r.Route("/:projectID"+paths.DocumentsPath, func(r chi.Router) {
			r.Get("/", webHandlerFunc(findDocuments).ServeHTTP)
			r.Get("/:documentID", webHandlerFunc(showDocument).ServeHTTP)
			r.Get("/new", webHandlerFunc(newDocument).ServeHTTP)
		})
	})

	router.Route(paths.UsersPath, func(r chi.Router) {
		r.Get("/:userID", webHandlerFunc(showUser).ServeHTTP)
	})
}
<|endoftext|>


<commit_before>package web

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/gorilla/mux"
	"github.com/hack4impact/audio-transcription-service/tasks"
	"github.com/hack4impact/audio-transcription-service/transcription"
)

type route struct {
	Name        string
	Method      string
	Pattern     string
	HandlerFunc http.HandlerFunc
}

type transcriptionJobData struct {
	AudioURL       string   `json:"audioURL"`
	EmailAddresses []string `json:"emailAddresses"`
}

var routes = []route{
	route{
		"hello",
		"GET",
		"/hello/{name}",
		helloHandler,
	},
	route{
		"add_job",
		"POST",
		"/add_job",
		initiateTranscriptionJobHandler,
	},
	route{
		"health",
		"GET",
		"/health",
		healthHandler,
	},
	route{
		"job_status",
		"GET",
		"/job_status/{id}",
		jobStatusHandler,
	},
}

func helloHandler(w http.ResponseWriter, r *http.Request) {
	args := mux.Vars(r)
	fmt.Fprintf(w, "Hello %s!", args["name"])
}

// initiateTranscriptionJobHandle takes a POST request containing a json object,
// decodes it into an audioData struct, and returns appropriate message.
func initiateTranscriptionJobHandler(w http.ResponseWriter, r *http.Request) {
	var jsonData transcriptionJobData

	// unmarshal from the response body directly into our struct
	if err := json.NewDecoder(r.Body).Decode(&jsonData); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	id = tasks.DefaultTaskExecuter.QueueTask(transcription.Task(jsonData))

	fmt.Fprintf(w, "Accepted task "+id+"!")
}

func healthHandler(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte("healthy!"))
}

// jobStatusHandler returns the status of a task with given id.
func jobStatusHandler(w http.ResponseWriter, r *http.Request) {
	args := mux.Vars(r)
	id := args["id"]

	status := tasks.DefaultTaskExecuter.GetTaskStatus(id)
	w.Write([]byte(status.String()))
}
<commit_msg>Two args for Task instead of one<commit_after>package web

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/gorilla/mux"
	"github.com/hack4impact/audio-transcription-service/tasks"
	"github.com/hack4impact/audio-transcription-service/transcription"
)

type route struct {
	Name        string
	Method      string
	Pattern     string
	HandlerFunc http.HandlerFunc
}

type transcriptionJobData struct {
	AudioURL       string   `json:"audioURL"`
	EmailAddresses []string `json:"emailAddresses"`
}

var routes = []route{
	route{
		"hello",
		"GET",
		"/hello/{name}",
		helloHandler,
	},
	route{
		"add_job",
		"POST",
		"/add_job",
		initiateTranscriptionJobHandler,
	},
	route{
		"health",
		"GET",
		"/health",
		healthHandler,
	},
	route{
		"job_status",
		"GET",
		"/job_status/{id}",
		jobStatusHandler,
	},
}

func helloHandler(w http.ResponseWriter, r *http.Request) {
	args := mux.Vars(r)
	fmt.Fprintf(w, "Hello %s!", args["name"])
}

// initiateTranscriptionJobHandle takes a POST request containing a json object,
// decodes it into an audioData struct, and returns appropriate message.
func initiateTranscriptionJobHandler(w http.ResponseWriter, r *http.Request) {
	var jsonData transcriptionJobData

	// unmarshal from the response body directly into our struct
	if err := json.NewDecoder(r.Body).Decode(&jsonData); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	id := tasks.DefaultTaskExecuter.QueueTask(transcription.Task(jsonData.AudioURL, jsonData.EmailAddresses))

	fmt.Fprintf(w, "Accepted task "+id+"!")
}

func healthHandler(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte("healthy!"))
}

// jobStatusHandler returns the status of a task with given id.
func jobStatusHandler(w http.ResponseWriter, r *http.Request) {
	args := mux.Vars(r)
	id := args["id"]

	status := tasks.DefaultTaskExecuter.GetTaskStatus(id)
	w.Write([]byte(status.String()))
}
<|endoftext|>


<commit_before>// Package web Cozy Stack API.
//
// Cozy is a personal platform as a service with a focus on data.
package web

import (
	"strings"

	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/web/middlewares"
	"github.com/labstack/echo"
)

func splitHost(host string) (instanceHost string, appSlug string) {
	parts := strings.SplitN(host, ".", 2)
	if len(parts) == 2 {
		return parts[1], parts[0]
	}
	return parts[0], ""
}

// Create returns a new web server that will handle that apps routing given the
// host of the request.
func Create(router *echo.Echo, serveApps echo.HandlerFunc) (*echo.Echo, error) {
	main := echo.New()
	main.Any("/*", func(c echo.Context) error {
		// TODO(optim): minimize the number of instance requests
		if parent, slug := splitHost(c.Request().Host); slug != "" {
			if i, err := instance.Get(parent); err == nil {
				if serveApps != nil {
					c.Set("instance", i)
					c.Set("slug", slug)
					handler := middlewares.LoadSession(serveApps)
					return handler(c)
				}
				return nil
			}
		}

		router.ServeHTTP(c.Response(), c.Request())
		return nil
	})

	return main, nil
}
<commit_msg>Move appsHandler creation outside of the closure<commit_after>// Package web Cozy Stack API.
//
// Cozy is a personal platform as a service with a focus on data.
package web

import (
	"strings"

	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/web/middlewares"
	"github.com/labstack/echo"
)

func splitHost(host string) (instanceHost string, appSlug string) {
	parts := strings.SplitN(host, ".", 2)
	if len(parts) == 2 {
		return parts[1], parts[0]
	}
	return parts[0], ""
}

// Create returns a new web server that will handle that apps routing given the
// host of the request.
func Create(router *echo.Echo, serveApps echo.HandlerFunc) (*echo.Echo, error) {
	appsHandler := middlewares.LoadSession(serveApps)
	main := echo.New()
	main.Any("/*", func(c echo.Context) error {
		// TODO(optim): minimize the number of instance requests
		if parent, slug := splitHost(c.Request().Host); slug != "" {
			if i, err := instance.Get(parent); err == nil {
				if serveApps != nil {
					c.Set("instance", i)
					c.Set("slug", slug)
					return appsHandler(c)
				}
				return nil
			}
		}

		router.ServeHTTP(c.Response(), c.Request())
		return nil
	})

	return main, nil
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2018 IBM, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package k8s

import (
	"fmt"

	"github.com/skydive-project/skydive/filters"
	"github.com/skydive-project/skydive/logging"
	"github.com/skydive-project/skydive/probe"
	"github.com/skydive-project/skydive/topology/graph"
)

const clusterName = "cluster"

type clusterProbe struct {
	graph.DefaultGraphListener
	graph          *graph.Graph
	clusterIndexer *graph.MetadataIndexer
	objectIndexer  *graph.MetadataIndexer
}

func newClusterLinkedObjectIndexer(g *graph.Graph) *graph.MetadataIndexer {
	filter := filters.NewAndFilter(
		filters.NewTermStringFilter("Manager", managerValue),
		filters.NewOrFilter(
			filters.NewTermStringFilter("Type", "namespace"),
			filters.NewTermStringFilter("Type", "node"),
			filters.NewTermStringFilter("Type", "persistentvolume"),
			filters.NewTermStringFilter("Type", "persistentvolumeclaim"),
		),
	)
	m := graph.NewGraphElementFilter(filter)
	return graph.NewMetadataIndexer(g, m)
}

func newClusterIndexer(g *graph.Graph) *graph.MetadataIndexer {
	filter := filters.NewAndFilter(
		filters.NewTermStringFilter("Manager", managerValue),
		filters.NewTermStringFilter("Type", "cluster"),
		filters.NewNotNullFilter("Name"),
	)
	m := graph.NewGraphElementFilter(filter)
	return graph.NewMetadataIndexer(g, m, "Name")
}

func dumpCluster(name string) string {
	return fmt.Sprintf("cluster{'Name': %s}", name)
}

func (p *clusterProbe) newMetadata(name string) graph.Metadata {
	return newMetadata("cluster", "", name, nil)
}

func (p *clusterProbe) linkObject(objNode, clusterNode *graph.Node) {
	addOwnershipLink(p.graph, clusterNode, objNode)
	logging.GetLogger().Debugf("Added link: %s -> %s", dumpGraphNode(clusterNode), dumpGraphNode(objNode))
}

func (p *clusterProbe) addNode(name string) {
	p.graph.Lock()
	defer p.graph.Unlock()

	clusterNode := newNode(p.graph, graph.GenID(), p.newMetadata(name))
	objNodes, _ := p.objectIndexer.Get()
	for _, objNode := range objNodes {
		p.linkObject(objNode, clusterNode)
	}

	logging.GetLogger().Debugf("Added %s", dumpCluster(name))
}

func (p *clusterProbe) delNode(name string) {
	p.graph.Lock()
	defer p.graph.Unlock()

	clusterNodes, _ := p.clusterIndexer.Get(name)
	for _, clusterNode := range clusterNodes {
		p.graph.DelNode(clusterNode)
	}

	logging.GetLogger().Debugf("Deleted %s", dumpCluster(name))
}

func (p *clusterProbe) OnNodeAdded(objNode *graph.Node) {
	logging.GetLogger().Debugf("Got event on adding %s", dumpGraphNode(objNode))
	clusterNodes, _ := p.clusterIndexer.Get(clusterName)
	if len(clusterNodes) > 0 {
		p.linkObject(objNode, clusterNodes[0])
	}
}

func (p *clusterProbe) Start() {
	p.clusterIndexer.Start()
	p.objectIndexer.AddEventListener(p)
	p.objectIndexer.Start()
	p.addNode(clusterName)
}

func (p *clusterProbe) Stop() {
	p.delNode(clusterName)
	p.clusterIndexer.Stop()
	p.objectIndexer.RemoveEventListener(p)
	p.objectIndexer.Stop()
}

func newClusterProbe(g *graph.Graph) probe.Probe {
	p := &clusterProbe{
		graph:          g,
		clusterIndexer: newClusterIndexer(g),
		objectIndexer:  newClusterLinkedObjectIndexer(g),
	}
	return p
}
<commit_msg>k8s: update cluster links also when child is updated<commit_after>/*
 * Copyright (C) 2018 IBM, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package k8s

import (
	"fmt"

	"github.com/skydive-project/skydive/filters"
	"github.com/skydive-project/skydive/logging"
	"github.com/skydive-project/skydive/probe"
	"github.com/skydive-project/skydive/topology/graph"
)

const clusterName = "cluster"

type clusterProbe struct {
	graph.DefaultGraphListener
	graph          *graph.Graph
	clusterIndexer *graph.MetadataIndexer
	objectIndexer  *graph.MetadataIndexer
}

func newClusterLinkedObjectIndexer(g *graph.Graph) *graph.MetadataIndexer {
	filter := filters.NewAndFilter(
		filters.NewTermStringFilter("Manager", managerValue),
		filters.NewOrFilter(
			filters.NewTermStringFilter("Type", "namespace"),
			filters.NewTermStringFilter("Type", "node"),
			filters.NewTermStringFilter("Type", "persistentvolume"),
			filters.NewTermStringFilter("Type", "persistentvolumeclaim"),
		),
	)
	m := graph.NewGraphElementFilter(filter)
	return graph.NewMetadataIndexer(g, m)
}

func newClusterIndexer(g *graph.Graph) *graph.MetadataIndexer {
	filter := filters.NewAndFilter(
		filters.NewTermStringFilter("Manager", managerValue),
		filters.NewTermStringFilter("Type", "cluster"),
		filters.NewNotNullFilter("Name"),
	)
	m := graph.NewGraphElementFilter(filter)
	return graph.NewMetadataIndexer(g, m, "Name")
}

func dumpCluster(name string) string {
	return fmt.Sprintf("cluster{'Name': %s}", name)
}

func (p *clusterProbe) newMetadata(name string) graph.Metadata {
	return newMetadata("cluster", "", name, nil)
}

func (p *clusterProbe) linkObject(objNode, clusterNode *graph.Node) {
	addOwnershipLink(p.graph, clusterNode, objNode)
	logging.GetLogger().Debugf("Added link: %s -> %s", dumpGraphNode(clusterNode), dumpGraphNode(objNode))
}

func (p *clusterProbe) addNode(name string) {
	p.graph.Lock()
	defer p.graph.Unlock()

	clusterNode := newNode(p.graph, graph.GenID(), p.newMetadata(name))
	objNodes, _ := p.objectIndexer.Get()
	for _, objNode := range objNodes {
		p.linkObject(objNode, clusterNode)
	}

	logging.GetLogger().Debugf("Added %s", dumpCluster(name))
}

func (p *clusterProbe) delNode(name string) {
	p.graph.Lock()
	defer p.graph.Unlock()

	clusterNodes, _ := p.clusterIndexer.Get(name)
	for _, clusterNode := range clusterNodes {
		p.graph.DelNode(clusterNode)
	}

	logging.GetLogger().Debugf("Deleted %s", dumpCluster(name))
}

func (p *clusterProbe) OnNodeAdded(objNode *graph.Node) {
	logging.GetLogger().Debugf("Got event on adding %s", dumpGraphNode(objNode))
	clusterNodes, _ := p.clusterIndexer.Get(clusterName)
	if len(clusterNodes) > 0 {
		p.linkObject(objNode, clusterNodes[0])
	}
}

func (p *clusterProbe) OnNodeUpdated(objNode *graph.Node) {
	logging.GetLogger().Debugf("Got event on updating %s", dumpGraphNode(objNode))
	clusterNodes, _ := p.clusterIndexer.Get(clusterName)
	if len(clusterNodes) > 0 {
		p.linkObject(objNode, clusterNodes[0])
	}
}

func (p *clusterProbe) Start() {
	p.clusterIndexer.Start()
	p.objectIndexer.AddEventListener(p)
	p.objectIndexer.Start()
	p.addNode(clusterName)
}

func (p *clusterProbe) Stop() {
	p.delNode(clusterName)
	p.clusterIndexer.Stop()
	p.objectIndexer.RemoveEventListener(p)
	p.objectIndexer.Stop()
}

func newClusterProbe(g *graph.Graph) probe.Probe {
	p := &clusterProbe{
		graph:          g,
		clusterIndexer: newClusterIndexer(g),
		objectIndexer:  newClusterLinkedObjectIndexer(g),
	}
	return p
}
<|endoftext|>


<commit_before>package sftp

// This serves as an example of how to implement the request server handler as
// well as a dummy backend for testing. It implements an in-memory backend that
// works as a very simple filesystem with simple flat key-value lookup system.

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"sync"
	"syscall"
	"time"
)

// InMemHandler returns a Hanlders object with the test handlers.
func InMemHandler() Handlers {
	root := &root{
		files: make(map[string]*memFile),
	}
	root.memFile = newMemFile("/", true)
	return Handlers{root, root, root, root}
}

// Example Handlers
func (fs *root) Fileread(r *Request) (io.ReaderAt, error) {
	if fs.mockErr != nil {
		return nil, fs.mockErr
	}
	_ = r.WithContext(r.Context()) // initialize context for deadlock testing
	fs.filesLock.Lock()
	defer fs.filesLock.Unlock()
	file, err := fs.fetch(r.Filepath)
	if err != nil {
		return nil, err
	}
	if file.symlink != "" {
		file, err = fs.fetch(file.symlink)
		if err != nil {
			return nil, err
		}
	}
	return file.ReaderAt()
}

func (fs *root) Filewrite(r *Request) (io.WriterAt, error) {
	if fs.mockErr != nil {
		return nil, fs.mockErr
	}
	_ = r.WithContext(r.Context()) // initialize context for deadlock testing
	fs.filesLock.Lock()
	defer fs.filesLock.Unlock()
	file, err := fs.fetch(r.Filepath)
	if err == os.ErrNotExist {
		dir, err := fs.fetch(filepath.Dir(r.Filepath))
		if err != nil {
			return nil, err
		}
		if !dir.isdir {
			return nil, os.ErrInvalid
		}
		file = newMemFile(r.Filepath, false)
		fs.files[r.Filepath] = file
	}
	return file.WriterAt()
}

func (fs *root) Filecmd(r *Request) error {
	if fs.mockErr != nil {
		return fs.mockErr
	}
	_ = r.WithContext(r.Context()) // initialize context for deadlock testing
	fs.filesLock.Lock()
	defer fs.filesLock.Unlock()
	switch r.Method {
	case "Setstat":
		return nil
	case "Rename":
		file, err := fs.fetch(r.Filepath)
		if err != nil {
			return err
		}
		if _, ok := fs.files[r.Target]; ok {
			return &os.LinkError{Op: "rename", Old: r.Filepath, New: r.Target,
				Err: fmt.Errorf("dest file exists")}
		}
		file.name = r.Target
		fs.files[r.Target] = file
		delete(fs.files, r.Filepath)
	case "Rmdir", "Remove":
		_, err := fs.fetch(filepath.Dir(r.Filepath))
		if err != nil {
			return err
		}
		delete(fs.files, r.Filepath)
	case "Mkdir":
		_, err := fs.fetch(filepath.Dir(r.Filepath))
		if err != nil {
			return err
		}
		fs.files[r.Filepath] = newMemFile(r.Filepath, true)
	case "Link":
		file, err := fs.fetch(r.Filepath)
		if err != nil {
			return err
		}
		if file.IsDir() {
			return fmt.Errorf("hard link not allowed for directory")
		}
		fs.files[r.Target] = file
	case "Symlink":
		_, err := fs.fetch(r.Filepath)
		if err != nil {
			return err
		}
		link := newMemFile(r.Target, false)
		link.symlink = r.Filepath
		fs.files[r.Target] = link
	}
	return nil
}

type listerat []os.FileInfo

// Modeled after strings.Reader's ReadAt() implementation
func (f listerat) ListAt(ls []os.FileInfo, offset int64) (int, error) {
	var n int
	if offset >= int64(len(f)) {
		return 0, io.EOF
	}
	n = copy(ls, f[offset:])
	if n < len(ls) {
		return n, io.EOF
	}
	return n, nil
}

func (fs *root) Filelist(r *Request) (ListerAt, error) {
	if fs.mockErr != nil {
		return nil, fs.mockErr
	}
	_ = r.WithContext(r.Context()) // initialize context for deadlock testing
	fs.filesLock.Lock()
	defer fs.filesLock.Unlock()

	file, err := fs.fetch(r.Filepath)
	if err != nil {
		return nil, err
	}

	switch r.Method {
	case "List":
		if !file.IsDir() {
			return nil, syscall.ENOTDIR
		}
		orderedNames := []string{}
		for fn := range fs.files {
			if filepath.Dir(fn) == r.Filepath {
				orderedNames = append(orderedNames, fn)
			}
		}
		sort.Strings(orderedNames)
		list := make([]os.FileInfo, len(orderedNames))
		for i, fn := range orderedNames {
			list[i] = fs.files[fn]
		}
		return listerat(list), nil
	case "Stat":
		return listerat([]os.FileInfo{file}), nil
	case "Readlink":
		if file.symlink != "" {
			file, err = fs.fetch(file.symlink)
			if err != nil {
				return nil, err
			}
		}
		return listerat([]os.FileInfo{file}), nil
	}
	return nil, nil
}

// In memory file-system-y thing that the Hanlders live on
type root struct {
	*memFile
	files     map[string]*memFile
	filesLock sync.Mutex
	mockErr   error
}

// Set a mocked error that the next handler call will return.
// Set to nil to reset for no error.
func (fs *root) returnErr(err error) {
	fs.mockErr = err
}

func (fs *root) fetch(path string) (*memFile, error) {
	if path == "/" {
		return fs.memFile, nil
	}
	if file, ok := fs.files[path]; ok {
		return file, nil
	}
	return nil, os.ErrNotExist
}

// Implements os.FileInfo, Reader and Writer interfaces.
// These are the 3 interfaces necessary for the Handlers.
// Implements the optional interface TransferError.
type memFile struct {
	name          string
	modtime       time.Time
	symlink       string
	isdir         bool
	content       []byte
	transferError error
	contentLock   sync.RWMutex
}

// factory to make sure modtime is set
func newMemFile(name string, isdir bool) *memFile {
	return &memFile{
		name:    name,
		modtime: time.Now(),
		isdir:   isdir,
	}
}

// Have memFile fulfill os.FileInfo interface
func (f *memFile) Name() string { return filepath.Base(f.name) }
func (f *memFile) Size() int64  { return int64(len(f.content)) }
func (f *memFile) Mode() os.FileMode {
	ret := os.FileMode(0644)
	if f.isdir {
		ret = os.FileMode(0755) | os.ModeDir
	}
	if f.symlink != "" {
		ret = os.FileMode(0777) | os.ModeSymlink
	}
	return ret
}
func (f *memFile) ModTime() time.Time { return f.modtime }
func (f *memFile) IsDir() bool        { return f.isdir }
func (f *memFile) Sys() interface{} {
	return fakeFileInfoSys()
}

// Read/Write
func (f *memFile) ReaderAt() (io.ReaderAt, error) {
	if f.isdir {
		return nil, os.ErrInvalid
	}
	return bytes.NewReader(f.content), nil
}

func (f *memFile) WriterAt() (io.WriterAt, error) {
	if f.isdir {
		return nil, os.ErrInvalid
	}
	return f, nil
}
func (f *memFile) WriteAt(p []byte, off int64) (int, error) {
	// fmt.Println(string(p), off)
	// mimic write delays, should be optional
	time.Sleep(time.Microsecond * time.Duration(len(p)))
	f.contentLock.Lock()
	defer f.contentLock.Unlock()
	plen := len(p) + int(off)
	if plen >= len(f.content) {
		nc := make([]byte, plen)
		copy(nc, f.content)
		f.content = nc
	}
	copy(f.content[off:], p)
	return len(p), nil
}

func (f *memFile) TransferError(err error) {
	f.transferError = err
}
<commit_msg>When removing or renaming a folder, subfiles/folders of the path was not being updated<commit_after>package sftp

// This serves as an example of how to implement the request server handler as
// well as a dummy backend for testing. It implements an in-memory backend that
// works as a very simple filesystem with simple flat key-value lookup system.

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"syscall"
	"time"
)

// InMemHandler returns a Hanlders object with the test handlers.
func InMemHandler() Handlers {
	root := &root{
		files: make(map[string]*memFile),
	}
	root.memFile = newMemFile("/", true)
	return Handlers{root, root, root, root}
}

// Example Handlers
func (fs *root) Fileread(r *Request) (io.ReaderAt, error) {
	if fs.mockErr != nil {
		return nil, fs.mockErr
	}
	_ = r.WithContext(r.Context()) // initialize context for deadlock testing
	fs.filesLock.Lock()
	defer fs.filesLock.Unlock()
	file, err := fs.fetch(r.Filepath)
	if err != nil {
		return nil, err
	}
	if file.symlink != "" {
		file, err = fs.fetch(file.symlink)
		if err != nil {
			return nil, err
		}
	}
	return file.ReaderAt()
}

func (fs *root) Filewrite(r *Request) (io.WriterAt, error) {
	if fs.mockErr != nil {
		return nil, fs.mockErr
	}
	_ = r.WithContext(r.Context()) // initialize context for deadlock testing
	fs.filesLock.Lock()
	defer fs.filesLock.Unlock()
	file, err := fs.fetch(r.Filepath)
	if err == os.ErrNotExist {
		dir, err := fs.fetch(filepath.Dir(r.Filepath))
		if err != nil {
			return nil, err
		}
		if !dir.isdir {
			return nil, os.ErrInvalid
		}
		file = newMemFile(r.Filepath, false)
		fs.files[r.Filepath] = file
	}
	return file.WriterAt()
}

func (fs *root) Filecmd(r *Request) error {
	if fs.mockErr != nil {
		return fs.mockErr
	}
	_ = r.WithContext(r.Context()) // initialize context for deadlock testing
	fs.filesLock.Lock()
	defer fs.filesLock.Unlock()
	switch r.Method {
	case "Setstat":
		return nil
	case "Rename":
		file, err := fs.fetch(r.Filepath)
		if err != nil {
			return err
		}
		if _, ok := fs.files[r.Target]; ok {
			return &os.LinkError{Op: "rename", Old: r.Filepath, New: r.Target,
				Err: fmt.Errorf("dest file exists")}
		}
		file.name = r.Target
		fs.files[r.Target] = file
		delete(fs.files, r.Filepath)

		if file.IsDir() {
			fmt.Printf("Rename Dir %v, %v, %v\n", r.Target, file.name, r.Filepath)
			for path, file := range fs.files {
				if strings.HasPrefix(path, r.Filepath+"/") {
					fmt.Printf("renaming %v from %v to %v\n", file.name, path, r.Target+path[len(r.Filepath):])
					file.name = r.Target + path[len(r.Filepath):]
					fs.files[r.Target+path[len(r.Filepath):]] = file
					delete(fs.files, path)
				}
			}
		}
	case "Rmdir", "Remove":
		file, err := fs.fetch(filepath.Dir(r.Filepath))
		if err != nil {
			return err
		}

		if file.IsDir() {
			for path := range fs.files {
				if strings.HasPrefix(path, r.Filepath+"/") {
					return &os.PathError{
						Op:   "remove",
						Path: r.Filepath + "/",
						Err:  fmt.Errorf("directory is not empty"),
					}
				}
			}
		}

		delete(fs.files, r.Filepath)
	case "Mkdir":
		_, err := fs.fetch(filepath.Dir(r.Filepath))
		if err != nil {
			return err
		}
		fs.files[r.Filepath] = newMemFile(r.Filepath, true)
	case "Link":
		file, err := fs.fetch(r.Filepath)
		if err != nil {
			return err
		}
		if file.IsDir() {
			return fmt.Errorf("hard link not allowed for directory")
		}
		fs.files[r.Target] = file
	case "Symlink":
		_, err := fs.fetch(r.Filepath)
		if err != nil {
			return err
		}
		link := newMemFile(r.Target, false)
		link.symlink = r.Filepath
		fs.files[r.Target] = link
	}
	return nil
}

type listerat []os.FileInfo

// Modeled after strings.Reader's ReadAt() implementation
func (f listerat) ListAt(ls []os.FileInfo, offset int64) (int, error) {
	var n int
	if offset >= int64(len(f)) {
		return 0, io.EOF
	}
	n = copy(ls, f[offset:])
	if n < len(ls) {
		return n, io.EOF
	}
	return n, nil
}

func (fs *root) Filelist(r *Request) (ListerAt, error) {
	if fs.mockErr != nil {
		return nil, fs.mockErr
	}
	_ = r.WithContext(r.Context()) // initialize context for deadlock testing
	fs.filesLock.Lock()
	defer fs.filesLock.Unlock()

	file, err := fs.fetch(r.Filepath)
	if err != nil {
		return nil, err
	}

	switch r.Method {
	case "List":
		if !file.IsDir() {
			return nil, syscall.ENOTDIR
		}
		orderedNames := []string{}
		for fn := range fs.files {
			if filepath.Dir(fn) == r.Filepath {
				orderedNames = append(orderedNames, fn)
			}
		}
		sort.Strings(orderedNames)
		list := make([]os.FileInfo, len(orderedNames))
		for i, fn := range orderedNames {
			list[i] = fs.files[fn]
		}
		return listerat(list), nil
	case "Stat":
		return listerat([]os.FileInfo{file}), nil
	case "Readlink":
		if file.symlink != "" {
			file, err = fs.fetch(file.symlink)
			if err != nil {
				return nil, err
			}
		}
		return listerat([]os.FileInfo{file}), nil
	}
	return nil, nil
}

// In memory file-system-y thing that the Hanlders live on
type root struct {
	*memFile
	files     map[string]*memFile
	filesLock sync.Mutex
	mockErr   error
}

// Set a mocked error that the next handler call will return.
// Set to nil to reset for no error.
func (fs *root) returnErr(err error) {
	fs.mockErr = err
}

func (fs *root) fetch(path string) (*memFile, error) {
	if path == "/" {
		return fs.memFile, nil
	}
	if file, ok := fs.files[path]; ok {
		return file, nil
	}
	return nil, os.ErrNotExist
}

// Implements os.FileInfo, Reader and Writer interfaces.
// These are the 3 interfaces necessary for the Handlers.
// Implements the optional interface TransferError.
type memFile struct {
	name          string
	modtime       time.Time
	symlink       string
	isdir         bool
	content       []byte
	transferError error
	contentLock   sync.RWMutex
}

// factory to make sure modtime is set
func newMemFile(name string, isdir bool) *memFile {
	return &memFile{
		name:    name,
		modtime: time.Now(),
		isdir:   isdir,
	}
}

// Have memFile fulfill os.FileInfo interface
func (f *memFile) Name() string { return filepath.Base(f.name) }
func (f *memFile) Size() int64  { return int64(len(f.content)) }
func (f *memFile) Mode() os.FileMode {
	ret := os.FileMode(0644)
	if f.isdir {
		ret = os.FileMode(0755) | os.ModeDir
	}
	if f.symlink != "" {
		ret = os.FileMode(0777) | os.ModeSymlink
	}
	return ret
}
func (f *memFile) ModTime() time.Time { return f.modtime }
func (f *memFile) IsDir() bool        { return f.isdir }
func (f *memFile) Sys() interface{} {
	return fakeFileInfoSys()
}

// Read/Write
func (f *memFile) ReaderAt() (io.ReaderAt, error) {
	if f.isdir {
		return nil, os.ErrInvalid
	}
	return bytes.NewReader(f.content), nil
}

func (f *memFile) WriterAt() (io.WriterAt, error) {
	if f.isdir {
		return nil, os.ErrInvalid
	}
	return f, nil
}
func (f *memFile) WriteAt(p []byte, off int64) (int, error) {
	// fmt.Println(string(p), off)
	// mimic write delays, should be optional
	time.Sleep(time.Microsecond * time.Duration(len(p)))
	f.contentLock.Lock()
	defer f.contentLock.Unlock()
	plen := len(p) + int(off)
	if plen >= len(f.content) {
		nc := make([]byte, plen)
		copy(nc, f.content)
		f.content = nc
	}
	copy(f.content[off:], p)
	return len(p), nil
}

func (f *memFile) TransferError(err error) {
	f.transferError = err
}
<|endoftext|>


<commit_before>package request

import (
	"bytes"
	"crypto/md5"
	"crypto/tls"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os/exec"
	"path"
	"regexp"
	"strings"
	"time"

	"golang.org/x/net/context"

	stns_settings "github.com/STNS/STNS/settings"
	"github.com/STNS/STNS/stns"
	"github.com/STNS/libnss_stns/config"
	"github.com/STNS/libnss_stns/logger"
	"github.com/STNS/libnss_stns/settings"
)

type Request struct {
	ApiPath string
	Config  *config.Config
}

func NewRequest(config *config.Config, paths ...string) (*Request, error) {
	logger.Setlog()
	r := Request{}

	r.Config = config
	r.SetPath(paths...)
	return &r, nil
}

func (r *Request) SetPath(paths ...string) {
	r.ApiPath = path.Clean(strings.Join(paths, "/"))
}

// only use wrapper command
func (r *Request) GetRawData() ([]byte, error) {

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	rch := make(chan []byte, len(r.Config.ApiEndPoint))
	ech := make(chan error, len(r.Config.ApiEndPoint))

	for _, endPoint := range r.Config.ApiEndPoint {
		go func() {
			url := strings.TrimRight(endPoint, "/") + "/" + strings.TrimLeft(r.ApiPath, "/")
			req, err := http.NewRequest("GET", url, nil)
			if err != nil {
				ech <- err
				return
			}

			if r.Config.User != "" && r.Config.Password != "" {
				req.SetBasicAuth(r.Config.User, r.Config.Password)
			}

			r.httpDo(
				ctx,
				req,
				func(res *http.Response, err error) {
					if err != nil {
						ech <- err
						return
					}
					defer res.Body.Close()
					body, err := ioutil.ReadAll(res.Body)

					switch res.StatusCode {
					case http.StatusOK:
						reg := regexp.MustCompile(`/v2[/]?$`)
						switch {
						// version1
						case !reg.MatchString(endPoint):
							buffer, err := r.migrateV2Format(body)
							if err != nil {
								ech <- err
								return
							}
							rch <- buffer
							return
						default:
							rch <- body
							return
						}
					case http.StatusNotFound:
						ech <- fmt.Errorf("resource notfound: %s", url)
						return

					case http.StatusUnauthorized:
						ech <- fmt.Errorf("authenticate error: %s", url)
						return
					default:
						ech <- fmt.Errorf("error: %s", url)
						return
					}
				},
			)
		}()
	}

	var cnt int
	for {
		select {
		case r := <-rch:
			return r, nil
		case e := <-ech:
			cnt++
			if cnt == len(r.Config.ApiEndPoint) {
				return nil, e
			}
		}
	}
}

func (r *Request) httpDo(
	ctx context.Context,
	req *http.Request,
	f func(*http.Response, error),
) {

	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: !r.Config.SslVerify},
		Dial: (&net.Dialer{
			Timeout:   time.Duration(r.Config.RequestTimeOut) * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
	}

	client := &http.Client{Transport: tr}

	go func() { f(client.Do(req)) }()
	select {
	case <-ctx.Done():
		tr.CancelRequest(req)
		return
	}
}

func (r *Request) migrateV2Format(body []byte) ([]byte, error) {
	var attr stns.Attributes
	err := json.Unmarshal(body, &attr)

	if err != nil {
		return nil, err
	}

	if attr == nil {
		return nil, errors.New(settings.V2_FORMAT_ERROR)
	}

	mig := stns.ResponseFormat{
		&stns.MetaData{
			1.0,
			false,
			0,
			"sha256",
			stns_settings.SUCCESS,
		},
		&attr,
	}

	j, err := json.Marshal(mig)
	if err != nil {
		return nil, err
	}

	return j, nil
}

func (r *Request) GetByWrapperCmd() (stns.ResponseFormat, error) {
	var stdout bytes.Buffer
	var stderr bytes.Buffer

	cmd := exec.Command(r.Config.WrapperCommand, r.ApiPath)
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()

	if err != nil {
		return stns.ResponseFormat{}, err
	}

	if len(stderr.Bytes()) > 0 {
		return stns.ResponseFormat{}, fmt.Errorf("command error:%s", stderr.String())
	}

	var res stns.ResponseFormat
	err = json.Unmarshal(stdout.Bytes(), &res)
	if err != nil {
		return stns.ResponseFormat{}, err
	}
	return res, nil
}

func (r *Request) GetMD5Hash(text string) string {
	hasher := md5.New()
	hasher.Write([]byte(text))
	return hex.EncodeToString(hasher.Sum(nil))
}
<commit_msg>fix bug<commit_after>package request

import (
	"bytes"
	"crypto/md5"
	"crypto/tls"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os/exec"
	"path"
	"regexp"
	"strings"
	"time"

	"golang.org/x/net/context"

	stns_settings "github.com/STNS/STNS/settings"
	"github.com/STNS/STNS/stns"
	"github.com/STNS/libnss_stns/config"
	"github.com/STNS/libnss_stns/logger"
	"github.com/STNS/libnss_stns/settings"
)

type Request struct {
	ApiPath string
	Config  *config.Config
}

func NewRequest(config *config.Config, paths ...string) (*Request, error) {
	logger.Setlog()
	r := Request{}

	r.Config = config
	r.SetPath(paths...)
	return &r, nil
}

func (r *Request) SetPath(paths ...string) {
	r.ApiPath = path.Clean(strings.Join(paths, "/"))
}

// only use wrapper command
func (r *Request) GetRawData() ([]byte, error) {

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	rch := make(chan []byte, len(r.Config.ApiEndPoint))
	ech := make(chan error, len(r.Config.ApiEndPoint))

	for _, e := range r.Config.ApiEndPoint {
		go func(endPoint string) {
			url := strings.TrimRight(endPoint, "/") + "/" + strings.TrimLeft(r.ApiPath, "/")
			req, err := http.NewRequest("GET", url, nil)
			if err != nil {
				ech <- err
				return
			}

			if r.Config.User != "" && r.Config.Password != "" {
				req.SetBasicAuth(r.Config.User, r.Config.Password)
			}

			r.httpDo(
				ctx,
				req,
				func(res *http.Response, err error) {
					if err != nil {
						ech <- err
						return
					}
					defer res.Body.Close()
					body, err := ioutil.ReadAll(res.Body)

					switch res.StatusCode {
					case http.StatusOK:
						reg := regexp.MustCompile(`/v2[/]?$`)
						switch {
						// version1
						case !reg.MatchString(e):
							buffer, err := r.migrateV2Format(body)
							if err != nil {
								ech <- err
								return
							}
							rch <- buffer
							return
						default:
							rch <- body
							return
						}
					case http.StatusNotFound:
						ech <- fmt.Errorf("resource notfound: %s", url)
						return

					case http.StatusUnauthorized:
						ech <- fmt.Errorf("authenticate error: %s", url)
						return
					default:
						ech <- fmt.Errorf("error: %s", url)
						return
					}
				},
			)
		}(e)
	}

	var cnt int
	for {
		select {
		case r := <-rch:
			return r, nil
		case e := <-ech:
			cnt++
			if cnt == len(r.Config.ApiEndPoint) {
				return nil, e
			}
		}
	}
}

func (r *Request) httpDo(
	ctx context.Context,
	req *http.Request,
	f func(*http.Response, error),
) {

	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: !r.Config.SslVerify},
		Dial: (&net.Dialer{
			Timeout:   time.Duration(r.Config.RequestTimeOut) * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
	}

	client := &http.Client{Transport: tr}

	go func() { f(client.Do(req)) }()
	select {
	case <-ctx.Done():
		tr.CancelRequest(req)
		return
	}
}

func (r *Request) migrateV2Format(body []byte) ([]byte, error) {
	var attr stns.Attributes
	err := json.Unmarshal(body, &attr)

	if err != nil {
		return nil, err
	}

	if attr == nil {
		return nil, errors.New(settings.V2_FORMAT_ERROR)
	}

	mig := stns.ResponseFormat{
		&stns.MetaData{
			1.0,
			false,
			0,
			"sha256",
			stns_settings.SUCCESS,
		},
		&attr,
	}

	j, err := json.Marshal(mig)
	if err != nil {
		return nil, err
	}

	return j, nil
}

func (r *Request) GetByWrapperCmd() (stns.ResponseFormat, error) {
	var stdout bytes.Buffer
	var stderr bytes.Buffer

	cmd := exec.Command(r.Config.WrapperCommand, r.ApiPath)
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()

	if err != nil {
		return stns.ResponseFormat{}, err
	}

	if len(stderr.Bytes()) > 0 {
		return stns.ResponseFormat{}, fmt.Errorf("command error:%s", stderr.String())
	}

	var res stns.ResponseFormat
	err = json.Unmarshal(stdout.Bytes(), &res)
	if err != nil {
		return stns.ResponseFormat{}, err
	}
	return res, nil
}

func (r *Request) GetMD5Hash(text string) string {
	hasher := md5.New()
	hasher.Write([]byte(text))
	return hex.EncodeToString(hasher.Sum(nil))
}
<|endoftext|>


<commit_before>package rclient

import (
	"encoding/json"
	"fmt"
	"github.com/zpatrick/go-series"
	"net/http"
)

// A ResponseReader attempts to read a *http.Response into v.
type ResponseReader func(resp *http.Response, v interface{}) error

// ReadJSONResponse attempts to marshal the response body into v
// if and only if the response StatusCode is in the 200 range.
// Otherwise, an error is thrown.
// It assumes the response body is in JSON format.
func ReadJSONResponse(resp *http.Response, v interface{}) error {
	defer resp.Body.Close()

	switch {
	case !series.Ints(200, 299).Contains(resp.StatusCode):
		return fmt.Errorf("Invalid status code: %d", resp.StatusCode)
	case v == nil:
		return nil
	default:
		return json.NewDecoder(resp.Body).Decode(v)
	}
}
<commit_msg>remove go-series<commit_after>package rclient

import (
	"encoding/json"
	"fmt"
	"net/http"
)

// A ResponseReader attempts to read a *http.Response into v.
type ResponseReader func(resp *http.Response, v interface{}) error

// ReadJSONResponse attempts to marshal the response body into v
// if and only if the response StatusCode is in the 200 range.
// Otherwise, an error is thrown.
// It assumes the response body is in JSON format.
func ReadJSONResponse(resp *http.Response, v interface{}) error {
	defer resp.Body.Close()

	switch {
	case resp.StatusCode < 200, resp.StatusCode > 299:
		return fmt.Errorf("Invalid status code: %d", resp.StatusCode)
	case v == nil:
		return nil
	default:
		return json.NewDecoder(resp.Body).Decode(v)
	}
}
<|endoftext|>


<commit_before>package retail_calendar

import (
	"time"
)

func MonthRange(date time.Time) (startTime time.Time, endTime time.Time) {
	fiscalYear := date.Year()
	if date.Month() == 1 {
		fiscalYear = fiscalYear - 1
	}
	endOfJanuary := time.Date(fiscalYear, 1, 31, 0, 0, 0, 0, time.UTC)
	daysUntilSunday := 7 - endOfJanuary.Weekday()
	startOfFiscalYear := endOfJanuary.AddDate(0, 0, int(daysUntilSunday))
	startTime = startOfFiscalYear
	endTime = startOfFiscalYear

	month := 1
	for month < 13 {
		startTime = endTime
		endTime = startTime.AddDate(0, 0, calcWeeksInMonth(month)*7)
		if date.Before(endTime) {
			return startTime, endTime
		}
		month++
	}
	return startTime, endTime
}

// calculate number of weeks in 454 month
func calcWeeksInMonth(month int) int {
	if month == 2 || month == 5 || month == 8 || month == 11 {
		return 5
	} else {
		return 4
	}
}
<commit_msg>for comments..<commit_after>package retail_calendar

import (
	"time"
)

func MonthRange(date time.Time) (startTime time.Time, endTime time.Time) {
	fiscalYear := date.Year()
	if date.Month() == 1 {
		fiscalYear = fiscalYear - 1
	}
	endOfJanuary := time.Date(fiscalYear, 1, 31, 0, 0, 0, 0, time.UTC)
	daysUntilSunday := 7 - endOfJanuary.Weekday()
	startOfFiscalYear := endOfJanuary.AddDate(0, 0, int(daysUntilSunday))
	startTime = startOfFiscalYear
	endTime = startOfFiscalYear

	month := 1
	for month < 13 {
		startTime = endTime
		endTime = startTime.AddDate(0, 0, calcWeeksInMonth(month)*7)
		if date.Before(endTime) {
			return startTime, endTime
		}
		month++
	}
	return startTime, endTime
}

// calculate number of weeks in 454 month
func calcWeeksInMonth(month int) int {
	if month == 2 || month == 5 || month == 8 || month == 11 {
		return 5
	} else {
		return 4
	}
}

<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"log"
	"net/http"
	"os"
	"sync"
	"time"

	_ "github.com/go-sql-driver/mysql"
	"github.com/nbari/violetear"
)

func main() {
	DSN := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s",
		os.Getenv("MYSQL_USER"),
		os.Getenv("MYSQL_PASS"),
		os.Getenv("MYSQL_HOST"),
		os.Getenv("MYSQL_PORT"),
		os.Getenv("MYSQL_DATABASE"))
	pool, err := sql.Open("mysql", DSN)
	if err != nil {
		log.Fatalf("mysql: Error on initializing database connection: %s", err.Error())
	}

	// sql pool options
	pool.SetConnMaxLifetime(time.Minute)
	pool.SetMaxIdleConns(30)
	pool.SetMaxOpenConns(50)

	err = pool.Ping()
	if err != nil {
		log.Fatalf("mysql: could not connect to the database: %s", err.Error())
	}

	router := violetear.New()
	router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		flusher, ok := w.(http.Flusher)
		if !ok {
			http.Error(w, "webserver doesn't support hijacking", http.StatusInternalServerError)
			return
		}

		w.Header().Set("Content-Type", "text/event-stream")
		w.Header().Set("Cache-Control", "no-cache")
		w.Header().Set("Connection", "keep-alive")
		fmt.Fprintf(w, "Will run this query: SELECT NOW() UNION SELECT SLEEP(3) LIMIT 1\n\n")
		flusher.Flush()

		var wg sync.WaitGroup
		out := make(chan string)
		for i := 0; i < 30; i++ {
			wg.Add(1)
			go func(w *sync.WaitGroup) {
				var now string
				pool.QueryRow("SELECT NOW() UNION SELECT SLEEP(3) LIMIT 1").Scan(&now)
				out <- now
				w.Done()
			}(&wg)
		}

		go func() {
			for o := range out {
				fmt.Fprintf(w, "now: %s\n", o)
				flusher.Flush()
			}
		}()

		wg.Wait()
		close(out)

		s := pool.Stats()
		fmt.Fprintf(w, fmt.Sprintf("OpenConnections: %+v\n", s.OpenConnections))
		flusher.Flush()
	})

	srv := &http.Server{
		Addr:    ":8080",
		Handler: router,
	}

	log.Fatal(srv.ListenAndServe())

}
<commit_msg>sync Thu Jun  6 09:14:09 CEST 2019<commit_after>package main

import (
	"database/sql"
	"fmt"
	"log"
	"net/http"
	"os"
	"sync"
	"time"

	_ "github.com/go-sql-driver/mysql"
	"github.com/nbari/violetear"
)

func main() {
	DSN := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s",
		os.Getenv("MYSQL_USER"),
		os.Getenv("MYSQL_PASS"),
		os.Getenv("MYSQL_HOST"),
		os.Getenv("MYSQL_PORT"),
		os.Getenv("MYSQL_DATABASE"))
	pool, err := sql.Open("mysql", DSN)
	if err != nil {
		log.Fatalf("mysql: Error on initializing database connection: %s", err.Error())
	}

	// sql pool options
	pool.SetConnMaxLifetime(time.Minute)
	pool.SetMaxIdleConns(30)
	pool.SetMaxOpenConns(50)

	err = pool.Ping()
	if err != nil {
		log.Fatalf("mysql: could not connect to the database: %s", err.Error())
	}

	router := violetear.New()
	router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		flusher, ok := w.(http.Flusher)
		if !ok {
			http.Error(w, "webserver doesn't support hijacking", http.StatusInternalServerError)
			return
		}

		w.Header().Set("Content-Type", "text/event-stream")
		w.Header().Set("Cache-Control", "no-cache")
		w.Header().Set("Connection", "keep-alive")
		fmt.Fprintf(w, "Will run this query: SELECT NOW() UNION SELECT SLEEP(3) LIMIT 1\n\n")
		flusher.Flush()

		var wg sync.WaitGroup
		out := make(chan string)
		for i := 0; i < 30; i++ {
			wg.Add(1)
			go func(w *sync.WaitGroup, j int) {
				var now string
				if j%3 == 0 {
					pool.QueryRow("SELECT \"after 3 seconds\" UNION SELECT SLEEP(3) LIMIT 1").Scan(&now)
				} else {
					pool.QueryRow("SELECT NOW()").Scan(&now)
				}
				out <- now
				w.Done()
			}(&wg, i)
		}

		go func() {
			for o := range out {
				fmt.Fprintf(w, "now: %s\n", o)
				flusher.Flush()
			}
		}()

		wg.Wait()
		time.Sleep(time.Second)
		close(out)

		s := pool.Stats()
		fmt.Fprintf(w, fmt.Sprintf("OpenConnections: %+v\n", s.OpenConnections))
		flusher.Flush()
	})

	srv := &http.Server{
		Addr:    ":8080",
		Handler: router,
	}

	log.Fatal(srv.ListenAndServe())

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"os"
	"path"
	"strings"

	"github.com/emirpasic/gods/trees/binaryheap"
	"gopkg.in/src-d/go-git.v4"
	. "gopkg.in/src-d/go-git.v4/_examples"
	"gopkg.in/src-d/go-git.v4/plumbing"
	"gopkg.in/src-d/go-git.v4/plumbing/cache"
	commitgraph_fmt "gopkg.in/src-d/go-git.v4/plumbing/format/commitgraph"
	"gopkg.in/src-d/go-git.v4/plumbing/object"
	"gopkg.in/src-d/go-git.v4/plumbing/object/commitgraph"
	"gopkg.in/src-d/go-git.v4/storage/filesystem"

	"gopkg.in/src-d/go-billy.v4"
	"gopkg.in/src-d/go-billy.v4/osfs"
)

// Example how to resolve a revision into its commit counterpart
func main() {
	CheckArgs("<path>", "<revision>", "<tree path>")

	path := os.Args[1]
	revision := os.Args[2]
	treePath := os.Args[3]

	// We instantiate a new repository targeting the given path (the .git folder)
	fs := osfs.New(path)
	s := filesystem.NewStorageWithOptions(fs, cache.NewObjectLRUDefault(), filesystem.Options{KeepDescriptors: true})
	r, err := git.Open(s, fs)
	CheckIfError(err)

	// Resolve revision into a sha1 commit, only some revisions are resolved
	// look at the doc to get more details
	Info("git rev-parse %s", revision)

	h, err := r.ResolveRevision(plumbing.Revision(revision))
	CheckIfError(err)

	commit, err := r.CommitObject(*h)
	CheckIfError(err)

	tree, err := commit.Tree()
	CheckIfError(err)
	if treePath != "" {
		tree, err = tree.Tree(treePath)
		CheckIfError(err)
	}

	var paths []string
	for _, entry := range tree.Entries {
		paths = append(paths, entry.Name)
	}

	commitNodeIndex, file := getCommitNodeIndex(r, fs)
	if file != nil {
		defer file.Close()
	}

	commitNode, err := commitNodeIndex.Get(*h)
	CheckIfError(err)

	revs, err := getLastCommitForPaths(commitNode, treePath, paths)
	CheckIfError(err)
	for path, rev := range revs {
		// Print one line per file (name hash message)
		hash := rev.Hash.String()
		line := strings.Split(rev.Message, "\n")
		fmt.Println(path, hash[:7], line[0])
	}

	s.Close()
}

func getCommitNodeIndex(r *git.Repository, fs billy.Filesystem) (commitgraph.CommitNodeIndex, io.ReadCloser) {
	file, err := fs.Open(path.Join("objects", "info", "commit-graph"))
	if err == nil {
		index, err := commitgraph_fmt.OpenFileIndex(file)
		if err == nil {
			return commitgraph.NewGraphCommitNodeIndex(index, r.Storer), file
		}
		file.Close()
	}

	return commitgraph.NewObjectCommitNodeIndex(r.Storer), nil
}

type commitAndPaths struct {
	commit commitgraph.CommitNode
	// Paths that are still on the branch represented by commit
	paths []string
	// Set of hashes for the paths
	hashes map[string]plumbing.Hash
}

func getCommitTree(c commitgraph.CommitNode, treePath string) (*object.Tree, error) {
	tree, err := c.Tree()
	if err != nil {
		return nil, err
	}

	// Optimize deep traversals by focusing only on the specific tree
	if treePath != "" {
		tree, err = tree.Tree(treePath)
		if err != nil {
			return nil, err
		}
	}

	return tree, nil
}

func getFullPath(treePath, path string) string {
	if treePath != "" {
		if path != "" {
			return treePath + "/" + path
		}
		return treePath
	}
	return path
}

func getFileHashes(c commitgraph.CommitNode, treePath string, paths []string) (map[string]plumbing.Hash, error) {
	tree, err := getCommitTree(c, treePath)
	if err == object.ErrDirectoryNotFound {
		// The whole tree didn't exist, so return empty map
		return make(map[string]plumbing.Hash), nil
	}
	if err != nil {
		return nil, err
	}

	hashes := make(map[string]plumbing.Hash)
	for _, path := range paths {
		if path != "" {
			entry, err := tree.FindEntry(path)
			if err == nil {
				hashes[path] = entry.Hash
			}
		} else {
			hashes[path] = tree.Hash
		}
	}

	return hashes, nil
}

func getLastCommitForPaths(c commitgraph.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) {
	// We do a tree traversal with nodes sorted by commit time
	heap := binaryheap.NewWith(func(a, b interface{}) int {
		if a.(*commitAndPaths).commit.CommitTime().Before(b.(*commitAndPaths).commit.CommitTime()) {
			return 1
		}
		return -1
	})

	resultNodes := make(map[string]commitgraph.CommitNode)
	initialHashes, err := getFileHashes(c, treePath, paths)
	if err != nil {
		return nil, err
	}

	// Start search from the root commit and with full set of paths
	heap.Push(&commitAndPaths{c, paths, initialHashes})

	for {
		cIn, ok := heap.Pop()
		if !ok {
			break
		}
		current := cIn.(*commitAndPaths)

		// Load the parent commits for the one we are currently examining
		numParents := current.commit.NumParents()
		var parents []commitgraph.CommitNode
		for i := 0; i < numParents; i++ {
			parent, err := current.commit.ParentNode(i)
			if err != nil {
				break
			}
			parents = append(parents, parent)
		}

		// Examine the current commit and set of interesting paths
		pathUnchanged := make([]bool, len(current.paths))
		parentHashes := make([]map[string]plumbing.Hash, len(parents))
		for j, parent := range parents {
			parentHashes[j], err = getFileHashes(parent, treePath, current.paths)
			if err != nil {
				break
			}

			for i, path := range current.paths {
				if parentHashes[j][path] == current.hashes[path] {
					pathUnchanged[i] = true
				}
			}
		}

		var remainingPaths []string
		for i, path := range current.paths {
			// The results could already contain some newer change for the same path,
			// so don't override that and bail out on the file early.
			if resultNodes[path] == nil {
				if pathUnchanged[i] {
					// The path existed with the same hash in at least one parent so it could
					// not have been changed in this commit directly.
					remainingPaths = append(remainingPaths, path)
				} else {
					// There are few possible cases how can we get here:
					// - The path didn't exist in any parent, so it must have been created by
					//   this commit.
					// - The path did exist in the parent commit, but the hash of the file has
					//   changed.
					// - We are looking at a merge commit and the hash of the file doesn't
					//   match any of the hashes being merged. This is more common for directories,
					//   but it can also happen if a file is changed through conflict resolution.
					resultNodes[path] = current.commit
				}
			}
		}

		if len(remainingPaths) > 0 {
			// Add the parent nodes along with remaining paths to the heap for further
			// processing.
			for j, parent := range parents {
				// Combine remainingPath with paths available on the parent branch
				// and make union of them
				remainingPathsForParent := make([]string, 0, len(remainingPaths))
				newRemainingPaths := make([]string, 0, len(remainingPaths))
				for _, path := range remainingPaths {
					if parentHashes[j][path] == current.hashes[path] {
						remainingPathsForParent = append(remainingPathsForParent, path)
					} else {
						newRemainingPaths = append(newRemainingPaths, path)
					}
				}

				if remainingPathsForParent != nil {
					heap.Push(&commitAndPaths{parent, remainingPathsForParent, parentHashes[j]})
				}

				if len(newRemainingPaths) == 0 {
					break
				} else {
					remainingPaths = newRemainingPaths
				}
			}
		}
	}

	// Post-processing
	result := make(map[string]*object.Commit)
	for path, commitNode := range resultNodes {
		var err error
		result[path], err = commitNode.Commit()
		if err != nil {
			return nil, err
		}
	}

	return result, nil
}
<commit_msg>Allow non-.git path for ls example<commit_after>package main

import (
	"fmt"
	"io"
	"os"
	"path"
	"strings"

	"github.com/emirpasic/gods/trees/binaryheap"
	"gopkg.in/src-d/go-git.v4"
	. "gopkg.in/src-d/go-git.v4/_examples"
	"gopkg.in/src-d/go-git.v4/plumbing"
	"gopkg.in/src-d/go-git.v4/plumbing/cache"
	commitgraph_fmt "gopkg.in/src-d/go-git.v4/plumbing/format/commitgraph"
	"gopkg.in/src-d/go-git.v4/plumbing/object"
	"gopkg.in/src-d/go-git.v4/plumbing/object/commitgraph"
	"gopkg.in/src-d/go-git.v4/storage/filesystem"

	"gopkg.in/src-d/go-billy.v4"
	"gopkg.in/src-d/go-billy.v4/osfs"
)

// Example how to resolve a revision into its commit counterpart
func main() {
	CheckArgs("<path>", "<revision>", "<tree path>")

	path := os.Args[1]
	revision := os.Args[2]
	treePath := os.Args[3]

	// We instantiate a new repository targeting the given path (the .git folder)
	fs := osfs.New(path)
	if _, err := fs.Stat(git.GitDirName); err == nil {
		fs, err = fs.Chroot(git.GitDirName)
		CheckIfError(err)
	}

	s := filesystem.NewStorageWithOptions(fs, cache.NewObjectLRUDefault(), filesystem.Options{KeepDescriptors: true})
	r, err := git.Open(s, fs)
	CheckIfError(err)
	defer s.Close()

	// Resolve revision into a sha1 commit, only some revisions are resolved
	// look at the doc to get more details
	Info("git rev-parse %s", revision)

	h, err := r.ResolveRevision(plumbing.Revision(revision))
	CheckIfError(err)

	commit, err := r.CommitObject(*h)
	CheckIfError(err)

	tree, err := commit.Tree()
	CheckIfError(err)
	if treePath != "" {
		tree, err = tree.Tree(treePath)
		CheckIfError(err)
	}

	var paths []string
	for _, entry := range tree.Entries {
		paths = append(paths, entry.Name)
	}

	commitNodeIndex, file := getCommitNodeIndex(r, fs)
	if file != nil {
		defer file.Close()
	}

	commitNode, err := commitNodeIndex.Get(*h)
	CheckIfError(err)

	revs, err := getLastCommitForPaths(commitNode, treePath, paths)
	CheckIfError(err)
	for path, rev := range revs {
		// Print one line per file (name hash message)
		hash := rev.Hash.String()
		line := strings.Split(rev.Message, "\n")
		fmt.Println(path, hash[:7], line[0])
	}
}

func getCommitNodeIndex(r *git.Repository, fs billy.Filesystem) (commitgraph.CommitNodeIndex, io.ReadCloser) {
	file, err := fs.Open(path.Join("objects", "info", "commit-graph"))
	if err == nil {
		index, err := commitgraph_fmt.OpenFileIndex(file)
		if err == nil {
			return commitgraph.NewGraphCommitNodeIndex(index, r.Storer), file
		}
		file.Close()
	}

	return commitgraph.NewObjectCommitNodeIndex(r.Storer), nil
}

type commitAndPaths struct {
	commit commitgraph.CommitNode
	// Paths that are still on the branch represented by commit
	paths []string
	// Set of hashes for the paths
	hashes map[string]plumbing.Hash
}

func getCommitTree(c commitgraph.CommitNode, treePath string) (*object.Tree, error) {
	tree, err := c.Tree()
	if err != nil {
		return nil, err
	}

	// Optimize deep traversals by focusing only on the specific tree
	if treePath != "" {
		tree, err = tree.Tree(treePath)
		if err != nil {
			return nil, err
		}
	}

	return tree, nil
}

func getFullPath(treePath, path string) string {
	if treePath != "" {
		if path != "" {
			return treePath + "/" + path
		}
		return treePath
	}
	return path
}

func getFileHashes(c commitgraph.CommitNode, treePath string, paths []string) (map[string]plumbing.Hash, error) {
	tree, err := getCommitTree(c, treePath)
	if err == object.ErrDirectoryNotFound {
		// The whole tree didn't exist, so return empty map
		return make(map[string]plumbing.Hash), nil
	}
	if err != nil {
		return nil, err
	}

	hashes := make(map[string]plumbing.Hash)
	for _, path := range paths {
		if path != "" {
			entry, err := tree.FindEntry(path)
			if err == nil {
				hashes[path] = entry.Hash
			}
		} else {
			hashes[path] = tree.Hash
		}
	}

	return hashes, nil
}

func getLastCommitForPaths(c commitgraph.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) {
	// We do a tree traversal with nodes sorted by commit time
	heap := binaryheap.NewWith(func(a, b interface{}) int {
		if a.(*commitAndPaths).commit.CommitTime().Before(b.(*commitAndPaths).commit.CommitTime()) {
			return 1
		}
		return -1
	})

	resultNodes := make(map[string]commitgraph.CommitNode)
	initialHashes, err := getFileHashes(c, treePath, paths)
	if err != nil {
		return nil, err
	}

	// Start search from the root commit and with full set of paths
	heap.Push(&commitAndPaths{c, paths, initialHashes})

	for {
		cIn, ok := heap.Pop()
		if !ok {
			break
		}
		current := cIn.(*commitAndPaths)

		// Load the parent commits for the one we are currently examining
		numParents := current.commit.NumParents()
		var parents []commitgraph.CommitNode
		for i := 0; i < numParents; i++ {
			parent, err := current.commit.ParentNode(i)
			if err != nil {
				break
			}
			parents = append(parents, parent)
		}

		// Examine the current commit and set of interesting paths
		pathUnchanged := make([]bool, len(current.paths))
		parentHashes := make([]map[string]plumbing.Hash, len(parents))
		for j, parent := range parents {
			parentHashes[j], err = getFileHashes(parent, treePath, current.paths)
			if err != nil {
				break
			}

			for i, path := range current.paths {
				if parentHashes[j][path] == current.hashes[path] {
					pathUnchanged[i] = true
				}
			}
		}

		var remainingPaths []string
		for i, path := range current.paths {
			// The results could already contain some newer change for the same path,
			// so don't override that and bail out on the file early.
			if resultNodes[path] == nil {
				if pathUnchanged[i] {
					// The path existed with the same hash in at least one parent so it could
					// not have been changed in this commit directly.
					remainingPaths = append(remainingPaths, path)
				} else {
					// There are few possible cases how can we get here:
					// - The path didn't exist in any parent, so it must have been created by
					//   this commit.
					// - The path did exist in the parent commit, but the hash of the file has
					//   changed.
					// - We are looking at a merge commit and the hash of the file doesn't
					//   match any of the hashes being merged. This is more common for directories,
					//   but it can also happen if a file is changed through conflict resolution.
					resultNodes[path] = current.commit
				}
			}
		}

		if len(remainingPaths) > 0 {
			// Add the parent nodes along with remaining paths to the heap for further
			// processing.
			for j, parent := range parents {
				// Combine remainingPath with paths available on the parent branch
				// and make union of them
				remainingPathsForParent := make([]string, 0, len(remainingPaths))
				newRemainingPaths := make([]string, 0, len(remainingPaths))
				for _, path := range remainingPaths {
					if parentHashes[j][path] == current.hashes[path] {
						remainingPathsForParent = append(remainingPathsForParent, path)
					} else {
						newRemainingPaths = append(newRemainingPaths, path)
					}
				}

				if remainingPathsForParent != nil {
					heap.Push(&commitAndPaths{parent, remainingPathsForParent, parentHashes[j]})
				}

				if len(newRemainingPaths) == 0 {
					break
				} else {
					remainingPaths = newRemainingPaths
				}
			}
		}
	}

	// Post-processing
	result := make(map[string]*object.Commit)
	for path, commitNode := range resultNodes {
		var err error
		result[path], err = commitNode.Commit()
		if err != nil {
			return nil, err
		}
	}

	return result, nil
}
<|endoftext|>


<commit_before><commit_msg>adjust min change<commit_after><|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e_node

import (
	"strings"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/kubernetes/pkg/kubelet/cm"
	"k8s.io/kubernetes/test/e2e/framework"
	imageutils "k8s.io/kubernetes/test/utils/image"

	"github.com/golang/glog"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// getResourceList returns a ResourceList with the
// specified cpu and memory resource values
func getResourceList(cpu, memory string) v1.ResourceList {
	res := v1.ResourceList{}
	if cpu != "" {
		res[v1.ResourceCPU] = resource.MustParse(cpu)
	}
	if memory != "" {
		res[v1.ResourceMemory] = resource.MustParse(memory)
	}
	return res
}

// getResourceRequirements returns a ResourceRequirements object
func getResourceRequirements(requests, limits v1.ResourceList) v1.ResourceRequirements {
	res := v1.ResourceRequirements{}
	res.Requests = requests
	res.Limits = limits
	return res
}

const (
	// Kubelet internal cgroup name for burstable tier
	burstableCgroup = "burstable"
	// Kubelet internal cgroup name for besteffort tier
	bestEffortCgroup = "besteffort"
)

// makePodToVerifyCgroups returns a pod that verifies the existence of the specified cgroups.
func makePodToVerifyCgroups(cgroupNames []string) *v1.Pod {
	// convert the names to their literal cgroupfs forms...
	cgroupFsNames := []string{}
	rootCgroupName := cm.NewCgroupName(cm.RootCgroupName, defaultNodeAllocatableCgroup)
	for _, baseName := range cgroupNames {
		// Add top level cgroup used to enforce node allocatable.
		cgroupComponents := strings.Split(baseName, "/")
		cgroupName := cm.NewCgroupName(rootCgroupName, cgroupComponents...)
		cgroupFsNames = append(cgroupFsNames, toCgroupFsName(cgroupName))
	}
	glog.Infof("expecting %v cgroups to be found", cgroupFsNames)
	// build the pod command to either verify cgroups exist
	command := ""
	for _, cgroupFsName := range cgroupFsNames {
		localCommand := "if [ ! -d /tmp/memory/" + cgroupFsName + " ] || [ ! -d /tmp/cpu/" + cgroupFsName + " ]; then exit 1; fi; "
		command += localCommand
	}

	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: "pod" + string(uuid.NewUUID()),
		},
		Spec: v1.PodSpec{
			RestartPolicy: v1.RestartPolicyNever,
			Containers: []v1.Container{
				{
					Image:   busyboxImage,
					Name:    "container" + string(uuid.NewUUID()),
					Command: []string{"sh", "-c", command},
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      "sysfscgroup",
							MountPath: "/tmp",
						},
					},
				},
			},
			Volumes: []v1.Volume{
				{
					Name: "sysfscgroup",
					VolumeSource: v1.VolumeSource{
						HostPath: &v1.HostPathVolumeSource{Path: "/sys/fs/cgroup"},
					},
				},
			},
		},
	}
	return pod
}

// makePodToVerifyCgroupRemoved verfies the specified cgroup does not exist.
func makePodToVerifyCgroupRemoved(baseName string) *v1.Pod {
	components := strings.Split(baseName, "/")
	cgroupName := cm.NewCgroupName(cm.RootCgroupName, components...)
	cgroupFsName := toCgroupFsName(cgroupName)
	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: "pod" + string(uuid.NewUUID()),
		},
		Spec: v1.PodSpec{
			RestartPolicy: v1.RestartPolicyOnFailure,
			Containers: []v1.Container{
				{
					Image:   busyboxImage,
					Name:    "container" + string(uuid.NewUUID()),
					Command: []string{"sh", "-c", "for i in `seq 1 10`; do if [ ! -d /tmp/memory/" + cgroupFsName + " ] && [ ! -d /tmp/cpu/" + cgroupFsName + " ]; then exit 0; else sleep 10; fi; done; exit 1"},
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      "sysfscgroup",
							MountPath: "/tmp",
						},
					},
				},
			},
			Volumes: []v1.Volume{
				{
					Name: "sysfscgroup",
					VolumeSource: v1.VolumeSource{
						HostPath: &v1.HostPathVolumeSource{Path: "/sys/fs/cgroup"},
					},
				},
			},
		},
	}
	return pod
}

var _ = framework.KubeDescribe("Kubelet Cgroup Manager", func() {
	f := framework.NewDefaultFramework("kubelet-cgroup-manager")
	Describe("QOS containers", func() {
		Context("On enabling QOS cgroup hierarchy", func() {
			It("Top level QoS containers should have been created", func() {
				if !framework.TestContext.KubeletConfig.CgroupsPerQOS {
					return
				}
				cgroupsToVerify := []string{burstableCgroup, bestEffortCgroup}
				pod := makePodToVerifyCgroups(cgroupsToVerify)
				f.PodClient().Create(pod)
				err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
				Expect(err).NotTo(HaveOccurred())
			})
		})
	})

	Describe("Pod containers [NodeConformance]", func() {
		Context("On scheduling a Guaranteed Pod", func() {
			It("Pod containers should have been created under the cgroup-root", func() {
				if !framework.TestContext.KubeletConfig.CgroupsPerQOS {
					return
				}
				var (
					guaranteedPod *v1.Pod
					podUID        string
				)
				By("Creating a Guaranteed pod in Namespace", func() {
					guaranteedPod = f.PodClient().Create(&v1.Pod{
						ObjectMeta: metav1.ObjectMeta{
							Name:      "pod" + string(uuid.NewUUID()),
							Namespace: f.Namespace.Name,
						},
						Spec: v1.PodSpec{
							Containers: []v1.Container{
								{
									Image:     imageutils.GetPauseImageName(),
									Name:      "container" + string(uuid.NewUUID()),
									Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("100m", "100Mi")),
								},
							},
						},
					})
					podUID = string(guaranteedPod.UID)
				})
				By("Checking if the pod cgroup was created", func() {
					cgroupsToVerify := []string{"pod" + podUID}
					pod := makePodToVerifyCgroups(cgroupsToVerify)
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
				By("Checking if the pod cgroup was deleted", func() {
					gp := int64(1)
					Expect(f.PodClient().Delete(guaranteedPod.Name, &metav1.DeleteOptions{GracePeriodSeconds: &gp})).NotTo(HaveOccurred())
					pod := makePodToVerifyCgroupRemoved("pod" + podUID)
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
			})
		})
		Context("On scheduling a BestEffort Pod", func() {
			It("Pod containers should have been created under the BestEffort cgroup", func() {
				if !framework.TestContext.KubeletConfig.CgroupsPerQOS {
					return
				}
				var (
					podUID        string
					bestEffortPod *v1.Pod
				)
				By("Creating a BestEffort pod in Namespace", func() {
					bestEffortPod = f.PodClient().Create(&v1.Pod{
						ObjectMeta: metav1.ObjectMeta{
							Name:      "pod" + string(uuid.NewUUID()),
							Namespace: f.Namespace.Name,
						},
						Spec: v1.PodSpec{
							Containers: []v1.Container{
								{
									Image:     imageutils.GetPauseImageName(),
									Name:      "container" + string(uuid.NewUUID()),
									Resources: getResourceRequirements(getResourceList("", ""), getResourceList("", "")),
								},
							},
						},
					})
					podUID = string(bestEffortPod.UID)
				})
				By("Checking if the pod cgroup was created", func() {
					cgroupsToVerify := []string{"besteffort/pod" + podUID}
					pod := makePodToVerifyCgroups(cgroupsToVerify)
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
				By("Checking if the pod cgroup was deleted", func() {
					gp := int64(1)
					Expect(f.PodClient().Delete(bestEffortPod.Name, &metav1.DeleteOptions{GracePeriodSeconds: &gp})).NotTo(HaveOccurred())
					pod := makePodToVerifyCgroupRemoved("besteffort/pod" + podUID)
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
			})
		})
		Context("On scheduling a Burstable Pod", func() {
			It("Pod containers should have been created under the Burstable cgroup", func() {
				if !framework.TestContext.KubeletConfig.CgroupsPerQOS {
					return
				}
				var (
					podUID       string
					burstablePod *v1.Pod
				)
				By("Creating a Burstable pod in Namespace", func() {
					burstablePod = f.PodClient().Create(&v1.Pod{
						ObjectMeta: metav1.ObjectMeta{
							Name:      "pod" + string(uuid.NewUUID()),
							Namespace: f.Namespace.Name,
						},
						Spec: v1.PodSpec{
							Containers: []v1.Container{
								{
									Image:     imageutils.GetPauseImageName(),
									Name:      "container" + string(uuid.NewUUID()),
									Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("200m", "200Mi")),
								},
							},
						},
					})
					podUID = string(burstablePod.UID)
				})
				By("Checking if the pod cgroup was created", func() {
					cgroupsToVerify := []string{"burstable/pod" + podUID}
					pod := makePodToVerifyCgroups(cgroupsToVerify)
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
				By("Checking if the pod cgroup was deleted", func() {
					gp := int64(1)
					Expect(f.PodClient().Delete(burstablePod.Name, &metav1.DeleteOptions{GracePeriodSeconds: &gp})).NotTo(HaveOccurred())
					pod := makePodToVerifyCgroupRemoved("burstable/pod" + podUID)
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
			})
		})
	})
})
<commit_msg>e2e node: mark pod cgroup test as [NodeConformance]<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e_node

import (
	"strings"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/kubernetes/pkg/kubelet/cm"
	"k8s.io/kubernetes/test/e2e/framework"
	imageutils "k8s.io/kubernetes/test/utils/image"

	"github.com/golang/glog"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// getResourceList returns a ResourceList with the
// specified cpu and memory resource values
func getResourceList(cpu, memory string) v1.ResourceList {
	res := v1.ResourceList{}
	if cpu != "" {
		res[v1.ResourceCPU] = resource.MustParse(cpu)
	}
	if memory != "" {
		res[v1.ResourceMemory] = resource.MustParse(memory)
	}
	return res
}

// getResourceRequirements returns a ResourceRequirements object
func getResourceRequirements(requests, limits v1.ResourceList) v1.ResourceRequirements {
	res := v1.ResourceRequirements{}
	res.Requests = requests
	res.Limits = limits
	return res
}

const (
	// Kubelet internal cgroup name for burstable tier
	burstableCgroup = "burstable"
	// Kubelet internal cgroup name for besteffort tier
	bestEffortCgroup = "besteffort"
)

// makePodToVerifyCgroups returns a pod that verifies the existence of the specified cgroups.
func makePodToVerifyCgroups(cgroupNames []string) *v1.Pod {
	// convert the names to their literal cgroupfs forms...
	cgroupFsNames := []string{}
	rootCgroupName := cm.NewCgroupName(cm.RootCgroupName, defaultNodeAllocatableCgroup)
	for _, baseName := range cgroupNames {
		// Add top level cgroup used to enforce node allocatable.
		cgroupComponents := strings.Split(baseName, "/")
		cgroupName := cm.NewCgroupName(rootCgroupName, cgroupComponents...)
		cgroupFsNames = append(cgroupFsNames, toCgroupFsName(cgroupName))
	}
	glog.Infof("expecting %v cgroups to be found", cgroupFsNames)
	// build the pod command to either verify cgroups exist
	command := ""
	for _, cgroupFsName := range cgroupFsNames {
		localCommand := "if [ ! -d /tmp/memory/" + cgroupFsName + " ] || [ ! -d /tmp/cpu/" + cgroupFsName + " ]; then exit 1; fi; "
		command += localCommand
	}

	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: "pod" + string(uuid.NewUUID()),
		},
		Spec: v1.PodSpec{
			RestartPolicy: v1.RestartPolicyNever,
			Containers: []v1.Container{
				{
					Image:   busyboxImage,
					Name:    "container" + string(uuid.NewUUID()),
					Command: []string{"sh", "-c", command},
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      "sysfscgroup",
							MountPath: "/tmp",
						},
					},
				},
			},
			Volumes: []v1.Volume{
				{
					Name: "sysfscgroup",
					VolumeSource: v1.VolumeSource{
						HostPath: &v1.HostPathVolumeSource{Path: "/sys/fs/cgroup"},
					},
				},
			},
		},
	}
	return pod
}

// makePodToVerifyCgroupRemoved verfies the specified cgroup does not exist.
func makePodToVerifyCgroupRemoved(baseName string) *v1.Pod {
	components := strings.Split(baseName, "/")
	cgroupName := cm.NewCgroupName(cm.RootCgroupName, components...)
	cgroupFsName := toCgroupFsName(cgroupName)
	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: "pod" + string(uuid.NewUUID()),
		},
		Spec: v1.PodSpec{
			RestartPolicy: v1.RestartPolicyOnFailure,
			Containers: []v1.Container{
				{
					Image:   busyboxImage,
					Name:    "container" + string(uuid.NewUUID()),
					Command: []string{"sh", "-c", "for i in `seq 1 10`; do if [ ! -d /tmp/memory/" + cgroupFsName + " ] && [ ! -d /tmp/cpu/" + cgroupFsName + " ]; then exit 0; else sleep 10; fi; done; exit 1"},
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      "sysfscgroup",
							MountPath: "/tmp",
						},
					},
				},
			},
			Volumes: []v1.Volume{
				{
					Name: "sysfscgroup",
					VolumeSource: v1.VolumeSource{
						HostPath: &v1.HostPathVolumeSource{Path: "/sys/fs/cgroup"},
					},
				},
			},
		},
	}
	return pod
}

var _ = framework.KubeDescribe("Kubelet Cgroup Manager", func() {
	f := framework.NewDefaultFramework("kubelet-cgroup-manager")
	Describe("QOS containers", func() {
		Context("On enabling QOS cgroup hierarchy", func() {
			It("Top level QoS containers should have been created [NodeConformance]", func() {
				if !framework.TestContext.KubeletConfig.CgroupsPerQOS {
					return
				}
				cgroupsToVerify := []string{burstableCgroup, bestEffortCgroup}
				pod := makePodToVerifyCgroups(cgroupsToVerify)
				f.PodClient().Create(pod)
				err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
				Expect(err).NotTo(HaveOccurred())
			})
		})
	})

	Describe("Pod containers [NodeConformance]", func() {
		Context("On scheduling a Guaranteed Pod", func() {
			It("Pod containers should have been created under the cgroup-root", func() {
				if !framework.TestContext.KubeletConfig.CgroupsPerQOS {
					return
				}
				var (
					guaranteedPod *v1.Pod
					podUID        string
				)
				By("Creating a Guaranteed pod in Namespace", func() {
					guaranteedPod = f.PodClient().Create(&v1.Pod{
						ObjectMeta: metav1.ObjectMeta{
							Name:      "pod" + string(uuid.NewUUID()),
							Namespace: f.Namespace.Name,
						},
						Spec: v1.PodSpec{
							Containers: []v1.Container{
								{
									Image:     imageutils.GetPauseImageName(),
									Name:      "container" + string(uuid.NewUUID()),
									Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("100m", "100Mi")),
								},
							},
						},
					})
					podUID = string(guaranteedPod.UID)
				})
				By("Checking if the pod cgroup was created", func() {
					cgroupsToVerify := []string{"pod" + podUID}
					pod := makePodToVerifyCgroups(cgroupsToVerify)
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
				By("Checking if the pod cgroup was deleted", func() {
					gp := int64(1)
					Expect(f.PodClient().Delete(guaranteedPod.Name, &metav1.DeleteOptions{GracePeriodSeconds: &gp})).NotTo(HaveOccurred())
					pod := makePodToVerifyCgroupRemoved("pod" + podUID)
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
			})
		})
		Context("On scheduling a BestEffort Pod", func() {
			It("Pod containers should have been created under the BestEffort cgroup", func() {
				if !framework.TestContext.KubeletConfig.CgroupsPerQOS {
					return
				}
				var (
					podUID        string
					bestEffortPod *v1.Pod
				)
				By("Creating a BestEffort pod in Namespace", func() {
					bestEffortPod = f.PodClient().Create(&v1.Pod{
						ObjectMeta: metav1.ObjectMeta{
							Name:      "pod" + string(uuid.NewUUID()),
							Namespace: f.Namespace.Name,
						},
						Spec: v1.PodSpec{
							Containers: []v1.Container{
								{
									Image:     imageutils.GetPauseImageName(),
									Name:      "container" + string(uuid.NewUUID()),
									Resources: getResourceRequirements(getResourceList("", ""), getResourceList("", "")),
								},
							},
						},
					})
					podUID = string(bestEffortPod.UID)
				})
				By("Checking if the pod cgroup was created", func() {
					cgroupsToVerify := []string{"besteffort/pod" + podUID}
					pod := makePodToVerifyCgroups(cgroupsToVerify)
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
				By("Checking if the pod cgroup was deleted", func() {
					gp := int64(1)
					Expect(f.PodClient().Delete(bestEffortPod.Name, &metav1.DeleteOptions{GracePeriodSeconds: &gp})).NotTo(HaveOccurred())
					pod := makePodToVerifyCgroupRemoved("besteffort/pod" + podUID)
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
			})
		})
		Context("On scheduling a Burstable Pod", func() {
			It("Pod containers should have been created under the Burstable cgroup", func() {
				if !framework.TestContext.KubeletConfig.CgroupsPerQOS {
					return
				}
				var (
					podUID       string
					burstablePod *v1.Pod
				)
				By("Creating a Burstable pod in Namespace", func() {
					burstablePod = f.PodClient().Create(&v1.Pod{
						ObjectMeta: metav1.ObjectMeta{
							Name:      "pod" + string(uuid.NewUUID()),
							Namespace: f.Namespace.Name,
						},
						Spec: v1.PodSpec{
							Containers: []v1.Container{
								{
									Image:     imageutils.GetPauseImageName(),
									Name:      "container" + string(uuid.NewUUID()),
									Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("200m", "200Mi")),
								},
							},
						},
					})
					podUID = string(burstablePod.UID)
				})
				By("Checking if the pod cgroup was created", func() {
					cgroupsToVerify := []string{"burstable/pod" + podUID}
					pod := makePodToVerifyCgroups(cgroupsToVerify)
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
				By("Checking if the pod cgroup was deleted", func() {
					gp := int64(1)
					Expect(f.PodClient().Delete(burstablePod.Name, &metav1.DeleteOptions{GracePeriodSeconds: &gp})).NotTo(HaveOccurred())
					pod := makePodToVerifyCgroupRemoved("burstable/pod" + podUID)
					f.PodClient().Create(pod)
					err := framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)
					Expect(err).NotTo(HaveOccurred())
				})
			})
		})
	})
})
<|endoftext|>