<commit_before>/*
Ion is a small web framework for people in a hurry.
Ion provides a fast trie based router (julienschmidt/httprouter),
a integrated middleware support (justinas/alice), automatic
context support (via gorilla/context) and some handful helpers.

A short example:

	package main

	import (
		"fmt"
		"github.com/estebarb/ion"
		"github.com/gorilla/context"
		"github.com/julienschmidt/httprouter"
		"net/http"
	)

	func hello(w http.ResponseWriter, r *http.Request) {
		val := context.Get(r, ion.Urlargs).(httprouter.Params)
		if val != nil {
			fmt.Fprintf(w, "Hello, %v!", val.ByName("name"))
		} else {
			fmt.Fprint(w, "Hello world!")
		}
	}

	func main() {
		r := ion.NewRouter()
		r.GetFunc("/", hello)
		r.GetFunc("/:name", hello)
		http.ListenAndServe(":8080", r)
	}

At this point the framework is highly experimental, so please don't
use it in production for now...
*/
package ion

import (
	"github.com/gorilla/context"
	"github.com/julienschmidt/httprouter"
	"github.com/justinas/alice"
	"html/template"
	"net/http"
)

// This is the router used by Ion. It contains the high performance
// trie based httprouter, the middleware manager alice and adapter
// functions that make trivial to use the Go http.Handler
type Router struct {
	*httprouter.Router
	Middleware alice.Chain
}

/*
Ion adds the path arguments by httprouter to
context, so they can be retrieved using:

	// asuming a path like /:name
	// r is a *http.Request
	params := context.Get(r, ion.Urlargs)
	name := params.ByName("name")
*/
const Urlargs = "ion_urlargs"

/*
Returns a new router, with no middleware.
*/
func NewRouter() *Router {
	return &Router{httprouter.New(), alice.New()}
}

/*
Returns a new router, configured with the middlewares
provided.
*/
func NewRouterDefaults(middleware ...alice.Constructor) *Router {
	r := &Router{
		httprouter.New(),
		alice.New(middleware...),
	}
	return r
}

/*
wrapHandler transforms a http.Handler handler to a httprouter.Handle
*/
func wrapHandler(h http.Handler) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
		context.Set(r, Urlargs, ps)
		h.ServeHTTP(w, r)
	}
}

// Registers a new request handler (http.Handler) for the given path and method.
// It also executes the current Middleware in the settings.
func (r *Router) MethodHandle(method, path string, handle http.Handler) {
	r.Handle(method, path, wrapHandler(r.Middleware.Then(handle)))
}

// Registers a new request handler (http.HandlerFunc) for the given path and method.
// It also executes the current Middleware in the settings.
func (r *Router) MethodHandleFunc(method, path string, handle http.HandlerFunc) {
	r.Handle(method, path, wrapHandler(r.Middleware.ThenFunc(handle)))
}

// Shortcut for router.MethodHandle("DELETE", path, handler)
func (r *Router) DELETE(path string, handler http.Handler) {
	r.MethodHandle("DELETE", path, handler)
}

// Shortcut for router.MethodHandle("GET", path, handler)
func (r *Router) GET(path string, handler http.Handler) {
	r.MethodHandle("GET", path, handler)
}

// Shortcut for router.MethodHandle("POST", path, handler)
func (r *Router) POST(path string, handler http.Handler) {
	r.MethodHandle("POST", path, handler)
}

// Shortcut for router.MethodHandle("PATCH", path, handler)
func (r *Router) PATCH(path string, handler http.Handler) {
	r.MethodHandle("PATCH", path, handler)
}

// Shortcut for router.MethodHandle("PUT", path, handler)
func (r *Router) PUT(path string, handler http.Handler) {
	r.MethodHandle("PUT", path, handler)
}

// Shortcut for router.MethodHandleFunc("DELETE", path, handler)
func (r *Router) DeleteFunc(path string, handler http.HandlerFunc) {
	r.MethodHandleFunc("DELETE", path, handler)
}

// Shortcut for router.MethodHandleFunc("GET", path, handler)
func (r *Router) GetFunc(path string, handler http.HandlerFunc) {
	r.MethodHandleFunc("GET", path, handler)
}

// Shortcut for router.MethodHandleFunc("POST", path, handler)
func (r *Router) PostFunc(path string, handler http.HandlerFunc) {
	r.MethodHandleFunc("POST", path, handler)
}

// Shortcut for router.MethodHandleFunc("PATCH", path, handler)
func (r *Router) PatchFunc(path string, handler http.HandlerFunc) {
	r.MethodHandleFunc("PATCH", path, handler)
}

// Shortcut for router.MethodHandleFunc("PUT", path, handler)
func (r *Router) PutFunc(path string, handler http.HandlerFunc) {
	r.MethodHandleFunc("PUT", path, handler)
}

// Returns a handler that can render the given templates. The templates
// receives as parameters the context associated to the current
// request.
func RenderTemplate(t *template.Template) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		args := context.Get(r, Urlargs).(httprouter.Params)
		context.Set(r, Urlargs, paramsToMap(args))
		ctx := context.GetAll(r)
		t.Execute(w, ctx)
	}
}

func paramsToMap(p httprouter.Params) map[string]string {
	ret := make(map[string]string)
	for _, v := range p {
		ret[v.Key] = v.Value
	}
	return ret
}
<commit_msg>Added REST helper<commit_after>/*
Ion is a small web framework for people in a hurry.
Ion provides a fast trie based router (julienschmidt/httprouter),
a integrated middleware support (justinas/alice), automatic
context support (via gorilla/context) and some handful helpers.

A short example:

	package main

	import (
		"fmt"
		"github.com/estebarb/ion"
		"github.com/gorilla/context"
		"github.com/julienschmidt/httprouter"
		"net/http"
	)

	func hello(w http.ResponseWriter, r *http.Request) {
		val := context.Get(r, ion.Urlargs).(httprouter.Params)
		if val != nil {
			fmt.Fprintf(w, "Hello, %v!", val.ByName("name"))
		} else {
			fmt.Fprint(w, "Hello world!")
		}
	}

	func main() {
		r := ion.NewRouter()
		r.GetFunc("/", hello)
		r.GetFunc("/:name", hello)
		http.ListenAndServe(":8080", r)
	}

At this point the framework is highly experimental, so please don't
use it in production for now...
*/
package ion

import (
	"github.com/gorilla/context"
	"github.com/julienschmidt/httprouter"
	"github.com/justinas/alice"
	"html/template"
	"net/http"
)

// This is the router used by Ion. It contains the high performance
// trie based httprouter, the middleware manager alice and adapter
// functions that make trivial to use the Go http.Handler
type Router struct {
	*httprouter.Router
	Middleware alice.Chain
}

/*
Ion adds the path arguments by httprouter to
context, so they can be retrieved using:

	// asuming a path like /:name
	// r is a *http.Request
	params := context.Get(r, ion.Urlargs)
	name := params.ByName("name")
*/
const Urlargs = "ion_urlargs"

/*
Returns a new router, with no middleware.
*/
func NewRouter() *Router {
	return &Router{httprouter.New(), alice.New()}
}

/*
Returns a new router, configured with the middlewares
provided.
*/
func NewRouterDefaults(middleware ...alice.Constructor) *Router {
	r := &Router{
		httprouter.New(),
		alice.New(middleware...),
	}
	return r
}

/*
wrapHandler transforms a http.Handler handler to a httprouter.Handle
*/
func wrapHandler(h http.Handler) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
		context.Set(r, Urlargs, ps)
		h.ServeHTTP(w, r)
	}
}

// Registers a new request handler (http.Handler) for the given path and method.
// It also executes the current Middleware in the settings.
func (r *Router) MethodHandle(method, path string, handle http.Handler) {
	r.Handle(method, path, wrapHandler(r.Middleware.Then(handle)))
}

// Registers a new request handler (http.HandlerFunc) for the given path and method.
// It also executes the current Middleware in the settings.
func (r *Router) MethodHandleFunc(method, path string, handle http.HandlerFunc) {
	r.Handle(method, path, wrapHandler(r.Middleware.ThenFunc(handle)))
}

// Shortcut for router.MethodHandle("DELETE", path, handler)
func (r *Router) DELETE(path string, handler http.Handler) {
	r.MethodHandle("DELETE", path, handler)
}

// Shortcut for router.MethodHandle("GET", path, handler)
func (r *Router) GET(path string, handler http.Handler) {
	r.MethodHandle("GET", path, handler)
}

// Shortcut for router.MethodHandle("POST", path, handler)
func (r *Router) POST(path string, handler http.Handler) {
	r.MethodHandle("POST", path, handler)
}

// Shortcut for router.MethodHandle("PATCH", path, handler)
func (r *Router) PATCH(path string, handler http.Handler) {
	r.MethodHandle("PATCH", path, handler)
}

// Shortcut for router.MethodHandle("PUT", path, handler)
func (r *Router) PUT(path string, handler http.Handler) {
	r.MethodHandle("PUT", path, handler)
}

// Shortcut for router.MethodHandleFunc("DELETE", path, handler)
func (r *Router) DeleteFunc(path string, handler http.HandlerFunc) {
	r.MethodHandleFunc("DELETE", path, handler)
}

// Shortcut for router.MethodHandleFunc("GET", path, handler)
func (r *Router) GetFunc(path string, handler http.HandlerFunc) {
	r.MethodHandleFunc("GET", path, handler)
}

// Shortcut for router.MethodHandleFunc("POST", path, handler)
func (r *Router) PostFunc(path string, handler http.HandlerFunc) {
	r.MethodHandleFunc("POST", path, handler)
}

// Shortcut for router.MethodHandleFunc("PATCH", path, handler)
func (r *Router) PatchFunc(path string, handler http.HandlerFunc) {
	r.MethodHandleFunc("PATCH", path, handler)
}

// Shortcut for router.MethodHandleFunc("PUT", path, handler)
func (r *Router) PutFunc(path string, handler http.HandlerFunc) {
	r.MethodHandleFunc("PUT", path, handler)
}

// Returns a handler that can render the given templates. The templates
// receives as parameters the context associated to the current
// request.
func RenderTemplate(t *template.Template) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		args := context.Get(r, Urlargs).(httprouter.Params)
		context.Set(r, Urlargs, paramsToMap(args))
		ctx := context.GetAll(r)
		t.Execute(w, ctx)
	}
}

// Converts the httprouter.Params array to a map, that can
// be consumed easily from templates.
func paramsToMap(p httprouter.Params) map[string]string {
	ret := make(map[string]string)
	for _, v := range p {
		ret[v.Key] = v.Value
	}
	return ret
}

// This interface works with RegisterREST to provide a shortcut
// to register an RESTful endpoint.

type RESTendpoint interface{
	LIST(w http.ResponseWriter, r *http.Request)
	POST(w http.ResponseWriter, r *http.Request)
	PUT(w http.ResponseWriter, r *http.Request)
	GET(w http.ResponseWriter, r *http.Request)
	DELETE(w http.ResponseWriter, r *http.Request)
}

// Register a RESTendpoint in a router.
// It will register the following routes:
// - GET  path		(list function)
// - POST path		(post function)
// - GET  path/:id	(get function)
// - PUT  path/:id	(put function)
// - DELETE  path/:id	(delete function)
// The path MUST include the trailing slash.
func (r *Router) RegisterREST(path string, handler RESTendpoint){
	r.GetFunc(path, handler.GET)
	r.PostFunc(path, handler.POST)
	r.GetFunc(path+":id", handler.GET)
	r.PutFunc(path+":id", handler.PUT)
	r.DeleteFunc(path+":id", handler.DELETE)
}
<|endoftext|>


<commit_before>//
// handlers
//
// @author darryl.west <darryl.west@raincitysoftware.com>
// @created 2017-07-01 12:57:59
//

package geozipdb

import (
	"fmt"
	"github.com/julienschmidt/httprouter"
	"net/http"
	"strconv"
	"strings"
)

// Keytype - keys are strings
type Keytype string

// Ziptype - zipcodes are strings
type Ziptype string

// Coord - lat/lng
type Coord struct {
	Lat float64
	Lng float64
}

// ZipcodeCoord - zipcode and coord lat/lng
type ZipcodeCoord struct {
	Zipcode Ziptype
	Coord
}

// Service - the primary service struct
type Service struct {
	config *Config
}

var keyMap = make(map[Keytype][]*ZipcodeCoord)
var zipMap = make(map[Ziptype]*Coord)
var initialized = false

// CreateKey - create a key from lat/lng
func (svc Service) CreateKey(lat, lng float64) Keytype {
	llat := int(lat * 10)
	llng := int(lng * 10)

	return Keytype(fmt.Sprintf("%d:%d", llat, llng))
}

// Initialize - initialize the data
func (svc Service) Initialize() {
	log.Info("initialize the database...")
	lines := strings.Split(geodata, "\n")

	log.Info("processing %d data rows...\n", len(lines))
	for i := 0; i < len(lines); i++ {
		fields := strings.Split(lines[i], ",")
		zipcode := Ziptype(fields[0])

		lat, _ := strconv.ParseFloat(fields[1], 64)
		lng, _ := strconv.ParseFloat(fields[2], 64)

		key := svc.CreateKey(lat, lng)

		coord := Coord{lat, lng}
		zipMap[zipcode] = &coord

		zcoord := ZipcodeCoord{zipcode, coord}
		keyMap[key] = append(keyMap[key], &zcoord)
	}

	log.Info("processed %d rows...\n", len(lines))
}

// CoordFromZip - return the coordinate of this zipcode
func (svc Service) CoordFromZip(code Ziptype) (*Coord, bool) {
	v, ok := zipMap[code]
	return v, ok
}

// ZipListFromCoord - return a list of zip codes that are near the coordinates
func (svc Service) ZipListFromCoord(coord *Coord) ([]*ZipcodeCoord, bool) {
	key := svc.CreateKey(coord.Lat, coord.Lng)
	v, ok := keyMap[key]

	return v, ok
}

// NewService - create the service based on config
func NewService(config *Config) *Service {
	svc := new(Service)

	svc.config = config

	return svc
}

func (svc Service) errorHandler(w http.ResponseWriter, r *http.Request, msg string) {
	log.Warn("%s", msg)
	w.WriteHeader(http.StatusNotFound)
	fmt.Fprintf(w, "%s\n\r", msg)
}

// return the zip list for a given coordinate lat/lng
func (svc Service) ziplistHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	p := ps.ByName("coord")
	log.Info("find zip list for coord %s\n", p)

	fields := strings.Split(p, ",")
	if len(fields) != 2 {
		svc.errorHandler(w, r, fmt.Sprintf("coordinates lat/lng not formatted correctly from %s", p))
		return
	}

	lat, err := strconv.ParseFloat(fields[0], 64)
	if err != nil {
		svc.errorHandler(w, r, fmt.Sprintf("could not parse lat from %s", p))
		return
	}

	lng, err := strconv.ParseFloat(fields[1], 64)
	if err != nil {
		svc.errorHandler(w, r, fmt.Sprintf("could not parse lng from %s", p))
		return
	}

	log.Info("find list from coords %f,%f", lat, lng)

	coord := Coord{lat, lng}

	if list, ok := svc.ZipListFromCoord(&coord); ok == true && len(list) > 0 {
        zips := make([]string, len(list))
        for i := 0; i < len(list); i++ {
            zips[i] = string(list[i].Zipcode)
        }

        str := strings.Join(zips, ",")
        log.Info("zip list for coords %s is %s", p, str)

		fmt.Fprintf(w, "%s\n\r", str);
	} else {
		svc.errorHandler(w, r, fmt.Sprintf("could not find zipcodes for %s", p))
		return
	}

}

// return the coordinates for a given zip
func (svc Service) coordHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	zipcode := ps.ByName("zip")
	log.Debug("find coord for zip %s\n", zipcode)

	if coord, ok := svc.CoordFromZip(Ziptype(zipcode)); ok {
		str := fmt.Sprintf("%f,%f", coord.Lat, coord.Lng)
		log.Info("found %s for zip %s", str, zipcode)
		fmt.Fprintf(w, "%s\n\r", str)
	} else {
		svc.errorHandler(w, r, fmt.Sprintf("could not find coordinates for zip %s", zipcode))
		return
	}
}

// Start - initialize the data and start the listener service
func (svc Service) Start() {
	if initialized == false {
		svc.Initialize()
	}

	cfg := svc.config

	router := httprouter.New()

	rname := fmt.Sprintf("%s/coord/:zip", cfg.PrimaryRoute)
	router.GET(rname, svc.coordHandler)
	log.Info("added route %s\n", rname)

	rname = fmt.Sprintf("%s/ziplist/:coord", cfg.PrimaryRoute)
	router.GET(rname, svc.ziplistHandler)
	log.Info("added route %s\n", rname)

	port := svc.config.Port
	host := fmt.Sprintf(":%d", port)
	log.Info("listening on port %d\n", port)

	err := http.ListenAndServe(host, router)
	if err != nil {
		panic(err)
	}
}
<commit_msg>format service<commit_after>//
// handlers
//
// @author darryl.west <darryl.west@raincitysoftware.com>
// @created 2017-07-01 12:57:59
//

package geozipdb

import (
	"fmt"
	"github.com/julienschmidt/httprouter"
	"net/http"
	"strconv"
	"strings"
)

// Keytype - keys are strings
type Keytype string

// Ziptype - zipcodes are strings
type Ziptype string

// Coord - lat/lng
type Coord struct {
	Lat float64
	Lng float64
}

// ZipcodeCoord - zipcode and coord lat/lng
type ZipcodeCoord struct {
	Zipcode Ziptype
	Coord
}

// Service - the primary service struct
type Service struct {
	config *Config
}

var keyMap = make(map[Keytype][]*ZipcodeCoord)
var zipMap = make(map[Ziptype]*Coord)
var initialized = false

// CreateKey - create a key from lat/lng
func (svc Service) CreateKey(lat, lng float64) Keytype {
	llat := int(lat * 10)
	llng := int(lng * 10)

	return Keytype(fmt.Sprintf("%d:%d", llat, llng))
}

// Initialize - initialize the data
func (svc Service) Initialize() {
	log.Info("initialize the database...")
	lines := strings.Split(geodata, "\n")

	log.Info("processing %d data rows...\n", len(lines))
	for i := 0; i < len(lines); i++ {
		fields := strings.Split(lines[i], ",")
		zipcode := Ziptype(fields[0])

		lat, _ := strconv.ParseFloat(fields[1], 64)
		lng, _ := strconv.ParseFloat(fields[2], 64)

		key := svc.CreateKey(lat, lng)

		coord := Coord{lat, lng}
		zipMap[zipcode] = &coord

		zcoord := ZipcodeCoord{zipcode, coord}
		keyMap[key] = append(keyMap[key], &zcoord)
	}

	log.Info("processed %d rows...\n", len(lines))
}

// CoordFromZip - return the coordinate of this zipcode
func (svc Service) CoordFromZip(code Ziptype) (*Coord, bool) {
	v, ok := zipMap[code]
	return v, ok
}

// ZipListFromCoord - return a list of zip codes that are near the coordinates
func (svc Service) ZipListFromCoord(coord *Coord) ([]*ZipcodeCoord, bool) {
	key := svc.CreateKey(coord.Lat, coord.Lng)
	v, ok := keyMap[key]

	return v, ok
}

// NewService - create the service based on config
func NewService(config *Config) *Service {
	svc := new(Service)

	svc.config = config

	return svc
}

func (svc Service) errorHandler(w http.ResponseWriter, r *http.Request, msg string) {
	log.Warn("%s", msg)
	w.WriteHeader(http.StatusNotFound)
	fmt.Fprintf(w, "%s\n\r", msg)
}

// return the zip list for a given coordinate lat/lng
func (svc Service) ziplistHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	p := ps.ByName("coord")
	log.Info("find zip list for coord %s\n", p)

	fields := strings.Split(p, ",")
	if len(fields) != 2 {
		svc.errorHandler(w, r, fmt.Sprintf("coordinates lat/lng not formatted correctly from %s", p))
		return
	}

	lat, err := strconv.ParseFloat(fields[0], 64)
	if err != nil {
		svc.errorHandler(w, r, fmt.Sprintf("could not parse lat from %s", p))
		return
	}

	lng, err := strconv.ParseFloat(fields[1], 64)
	if err != nil {
		svc.errorHandler(w, r, fmt.Sprintf("could not parse lng from %s", p))
		return
	}

	log.Info("find list from coords %f,%f", lat, lng)

	coord := Coord{lat, lng}

	if list, ok := svc.ZipListFromCoord(&coord); ok == true && len(list) > 0 {
		zips := make([]string, len(list))
		for i := 0; i < len(list); i++ {
			zips[i] = string(list[i].Zipcode)
		}

		str := strings.Join(zips, ",")
		log.Info("zip list for coords %s is %s", p, str)

		fmt.Fprintf(w, "%s\n\r", str)
	} else {
		svc.errorHandler(w, r, fmt.Sprintf("could not find zipcodes for %s", p))
		return
	}

}

// return the coordinates for a given zip
func (svc Service) coordHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	zipcode := ps.ByName("zip")
	log.Debug("find coord for zip %s\n", zipcode)

	if coord, ok := svc.CoordFromZip(Ziptype(zipcode)); ok {
		str := fmt.Sprintf("%f,%f", coord.Lat, coord.Lng)
		log.Info("found %s for zip %s", str, zipcode)
		fmt.Fprintf(w, "%s\n\r", str)
	} else {
		svc.errorHandler(w, r, fmt.Sprintf("could not find coordinates for zip %s", zipcode))
		return
	}
}

// Start - initialize the data and start the listener service
func (svc Service) Start() {
	if initialized == false {
		svc.Initialize()
	}

	cfg := svc.config

	router := httprouter.New()

	rname := fmt.Sprintf("%s/coord/:zip", cfg.PrimaryRoute)
	router.GET(rname, svc.coordHandler)
	log.Info("added route %s\n", rname)

	rname = fmt.Sprintf("%s/ziplist/:coord", cfg.PrimaryRoute)
	router.GET(rname, svc.ziplistHandler)
	log.Info("added route %s\n", rname)

	port := svc.config.Port
	host := fmt.Sprintf(":%d", port)
	log.Info("listening on port %d\n", port)

	err := http.ListenAndServe(host, router)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package ec2

import (
	"encoding/xml"
)

//RunInstances to store all attribute to create EC2 instance.
type RunInstances struct {
	ImageID               string
	MinCount              int
	MaxCount              int
	KeyName               string
	InstanceType          string
	SecurityGroups        []SecurityGroup
	KernelID              string
	RamdiskID             string
	UserData              []byte
	AvailZone             string
	PlacementGroupName    string
	Monitoring            bool
	SubnetID              string
	DisableAPITermination bool
	ShutdownBehavior      string
	PrivateIPAddress      string
	BlockDeviceMappings   []BlockDeviceMapping
	NetworkInterfaces     []RunNetworkInterface
}

//SecurityGroup struct.
type SecurityGroup struct {
	ID   string `xml:"groupID"`
	Name string `xml:"groupName"`
}

//BlockDeviceMapping struct to attach device
type BlockDeviceMapping struct {
	DeviceName          string `xml:"deviceName"`
	VirtualName         string `xml:"virtualName"`
	SnapshotID          string `xml:"ebs>snapshotID"`
	VolumeType          string `xml:"ebs>volumeType"`
	VolumeSize          int64  `xml:"ebs>volumeSize"`
	DeleteOnTermination bool   `xml:"ebs>deleteOnTermination"`
	IOPS                int64  `xml:"ebs>iops"`
}

//RunNetworkInterface struct for Ec2.
type RunNetworkInterface struct {
	ID                     string
	DeviceIndex             int
	SubnetID               string
	Description             string
	PrivateIPs              []PrivateIP
	SecurityGroupIds        []string
	DeleteOnTermination     bool
	SecondaryPrivateIPCount int
}

//PrivateIP to assign PrivateIP.
type PrivateIP struct {
	Address   string `xml:"privateIpAddress"`
	DNSName   string `xml:"privateDnsName"`
	IsPrimary bool   `xml:"primary"`
}

//RunInstancesResp response.
type RunInstancesResp struct {
	RequestID      string          `xml:"RequestID"`
	ReservationID string          `xml:"ReservationID"`
	OwnerID        string          `xml:"OwnerID"`
	SecurityGroups []SecurityGroup `xml:"groupSet>item"`
	Instances      []Instance      `xml:"instancesSet>item"`
}

//Instance struct represents running instance.
type Instance struct {
	InstanceID         string             `xml:"instanceID"`
	InstanceType       string             `xml:"instanceType"`
	ImageID            string             `xml:"imageId"`
	PrivateDNSName     string             `xml:"privateDnsName"`
	DNSName            string             `xml:"dnsName"`
	IPAddress          string             `xml:"ipAddress"`
	PrivateIPAddress   string             `xml:"privateIpAddress"`
	SubnetID           string             `xml:"subnetId"`
	VPCID              string             `xml:"vpcID"`
	SourceDestCheck    bool               `xml:"sourceDestCheck"`
	KeyName            string             `xml:"keyName"`
	AMILaunchIndex     int                `xml:"amiLaunchIndex"`
	Hypervisor         string             `xml:"hypervisor"`
	VirtType           string             `xml:"virtualizationType"`
	Monitoring         string             `xml:"monitoring>state"`
	AvailZone          string             `xml:"placement>availabilityZone"`
	PlacementGroupName string             `xml:"placement>groupName"`
	State              InstanceState      `xml:"instanceState"`
	Tags               []Tag              `xml:"tagSet>item"`
	SecurityGroups     []SecurityGroup    `xml:"groupSet>item"`
	NetworkInterfaces  []NetworkInterface `xml:"networkInterfaceSet>item"`
}

//InstanceStateChange stuct represents instance state change.
type InstanceStateChange struct {
	InstanceID    string        `xml:"instanceId"`
	CurrentState  InstanceState `xml:"currentState"`
	PreviousState InstanceState `xml:"previousState"`
}

//SimpleResp stuct represents SimpleResp.
type SimpleResp struct {
	XMLName   xml.Name
	RequestID string `xml:"requestId"`
}

//TerminateInstance struct represents TerminateInstance response.
type TerminateInstancesResp struct {
	RequestID    string                `xml:"requestId"`
	StateChanges []InstanceStateChange `xml:"instancesSet>item"`
}

//InstanceState struct represents InstanceState.
type InstanceState struct {
	Code int    `xml:"code"`
	Name string `xml:"name"`
}

//NetworkInterface reperents running instance NetworkInterface.
type NetworkInterface struct {
	Id               string                     `xml:"networkInterfaceId"`
	SubnetId         string                     `xml:"subnetId"`
	VPCId            string                     `xml:"vpcId"`
	AvailZone        string                     `xml:"availabilityZone"`
	Description      string                     `xml:"description"`
	OwnerId          string                     `xml:"ownerId"`
	RequesterId      string                     `xml:"requesterId"`
	RequesterManaged bool                       `xml:"requesterManaged"`
	Status           string                     `xml:"status"`
	MACAddress       string                     `xml:"macAddress"`
	PrivateIPAddress string                     `xml:"privateIpAddress"`
	PrivateDNSName   string                     `xml:"privateDnsName"`
	SourceDestCheck  bool                       `xml:"sourceDestCheck"`
	Groups           []SecurityGroup            `xml:"groupSet>item"`
	Attachment       NetworkInterfaceAttachment `xml:"attachment"`
	Tags             []Tag                      `xml:"tagSet>item"`
	PrivateIPs       []PrivateIP                `xml:"privateIpAddressesSet>item"`
}

//NetworkInterfaceAttachment represents running instance
type NetworkInterfaceAttachment struct {
	Id                  string `xml:"attachmentId"`
	InstanceId          string `xml:"instanceId"`
	InstanceOwnerId     string `xml:"instanceOwnerId"`
	DeviceIndex         int    `xml:"deviceIndex"`
	Status              string `xml:"status"`
	AttachTime          string `xml:"attachTime"`
	DeleteOnTermination bool   `xml:"deleteOnTermination"`
}

//Tag reperent tag assgin to instance
type Tag struct {
	Key   string `xml:"key"`
	Value string `xml:"value"`
}

//StartInstanceResp response.
type StartInstanceResp struct {
	RequestId    string                `xml:"requestId"`
	StateChanges []InstanceStateChange `xml:"instancesSet>item"`
}

//StopInstanceResp response.
type StopInstanceResp struct {
	RequestId    string                `xml:"requestId"`
	StateChanges []InstanceStateChange `xml:"instancesSet>item"`
}
<commit_msg>golint<commit_after>package ec2

import (
	"encoding/xml"
)

//RunInstances to store all attribute to create EC2 instance.
type RunInstances struct {
	ImageID               string
	MinCount              int
	MaxCount              int
	KeyName               string
	InstanceType          string
	SecurityGroups        []SecurityGroup
	KernelID              string
	RamdiskID             string
	UserData              []byte
	AvailZone             string
	PlacementGroupName    string
	Monitoring            bool
	SubnetID              string
	DisableAPITermination bool
	ShutdownBehavior      string
	PrivateIPAddress      string
	BlockDeviceMappings   []BlockDeviceMapping
	NetworkInterfaces     []RunNetworkInterface
}

//SecurityGroup struct.
type SecurityGroup struct {
	ID   string `xml:"groupID"`
	Name string `xml:"groupName"`
}

//BlockDeviceMapping struct to attach device
type BlockDeviceMapping struct {
	DeviceName          string `xml:"deviceName"`
	VirtualName         string `xml:"virtualName"`
	SnapshotID          string `xml:"ebs>snapshotID"`
	VolumeType          string `xml:"ebs>volumeType"`
	VolumeSize          int64  `xml:"ebs>volumeSize"`
	DeleteOnTermination bool   `xml:"ebs>deleteOnTermination"`
	IOPS                int64  `xml:"ebs>iops"`
}

//RunNetworkInterface struct for Ec2.
type RunNetworkInterface struct {
	ID                     string
	DeviceIndex             int
	SubnetID               string
	Description             string
	PrivateIPs              []PrivateIP
	SecurityGroupIds        []string
	DeleteOnTermination     bool
	SecondaryPrivateIPCount int
}

//PrivateIP to assign PrivateIP.
type PrivateIP struct {
	Address   string `xml:"privateIpAddress"`
	DNSName   string `xml:"privateDnsName"`
	IsPrimary bool   `xml:"primary"`
}

//RunInstancesResp response.
type RunInstancesResp struct {
	RequestID      string          `xml:"RequestID"`
	ReservationID string          `xml:"ReservationID"`
	OwnerID        string          `xml:"OwnerID"`
	SecurityGroups []SecurityGroup `xml:"groupSet>item"`
	Instances      []Instance      `xml:"instancesSet>item"`
}

//Instance struct represents running instance.
type Instance struct {
	InstanceID         string             `xml:"instanceID"`
	InstanceType       string             `xml:"instanceType"`
	ImageID            string             `xml:"imageId"`
	PrivateDNSName     string             `xml:"privateDnsName"`
	DNSName            string             `xml:"dnsName"`
	IPAddress          string             `xml:"ipAddress"`
	PrivateIPAddress   string             `xml:"privateIpAddress"`
	SubnetID           string             `xml:"subnetId"`
	VPCID              string             `xml:"vpcID"`
	SourceDestCheck    bool               `xml:"sourceDestCheck"`
	KeyName            string             `xml:"keyName"`
	AMILaunchIndex     int                `xml:"amiLaunchIndex"`
	Hypervisor         string             `xml:"hypervisor"`
	VirtType           string             `xml:"virtualizationType"`
	Monitoring         string             `xml:"monitoring>state"`
	AvailZone          string             `xml:"placement>availabilityZone"`
	PlacementGroupName string             `xml:"placement>groupName"`
	State              InstanceState      `xml:"instanceState"`
	Tags               []Tag              `xml:"tagSet>item"`
	SecurityGroups     []SecurityGroup    `xml:"groupSet>item"`
	NetworkInterfaces  []NetworkInterface `xml:"networkInterfaceSet>item"`
}

//InstanceStateChange stuct represents instance state change.
type InstanceStateChange struct {
	InstanceID    string        `xml:"instanceId"`
	CurrentState  InstanceState `xml:"currentState"`
	PreviousState InstanceState `xml:"previousState"`
}

//SimpleResp stuct represents SimpleResp.
type SimpleResp struct {
	XMLName   xml.Name
	RequestID string `xml:"requestId"`
}

//TerminateInstance struct represents TerminateInstance response.
type TerminateInstancesResp struct {
	RequestID    string                `xml:"requestID"`
	StateChanges []InstanceStateChange `xml:"instancesSet>item"`
}

//InstanceState struct represents InstanceState.
type InstanceState struct {
	Code int    `xml:"code"`
	Name string `xml:"name"`
}

//NetworkInterface reperents running instance NetworkInterface.
type NetworkInterface struct {
	ID               string                     `xml:"networkInterfaceID"`
	SubnetId         string                     `xml:"subnetId"`
	VPCId            string                     `xml:"vpcId"`
	AvailZone        string                     `xml:"availabilityZone"`
	Description      string                     `xml:"description"`
	OwnerID          string                     `xml:"OwnerID"`
	RequesterID      string                     `xml:"requesterID"`
	RequesterManaged bool                       `xml:"requesterManaged"`
	Status           string                     `xml:"status"`
	MACAddress       string                     `xml:"macAddress"`
	PrivateIPAddress string                     `xml:"privateIpAddress"`
	PrivateDNSName   string                     `xml:"privateDnsName"`
	SourceDestCheck  bool                       `xml:"sourceDestCheck"`
	Groups           []SecurityGroup            `xml:"groupSet>item"`
	Attachment       NetworkInterfaceAttachment `xml:"attachment"`
	Tags             []Tag                      `xml:"tagSet>item"`
	PrivateIPs       []PrivateIP                `xml:"privateIpAddressesSet>item"`
}

//NetworkInterfaceAttachment represents running instance
type NetworkInterfaceAttachment struct {
	ID                  string `xml:"attachmentIDs"`
	InstanceID          string `xml:"instanceID"`
	InstanceOwnerId     string `xml:"instanceOwnerId"`
	DeviceIndex         int    `xml:"deviceIndex"`
	Status              string `xml:"status"`
	AttachTime          string `xml:"attachTime"`
	DeleteOnTermination bool   `xml:"deleteOnTermination"`
}

//Tag reperent tag assgin to instance
type Tag struct {
	Key   string `xml:"key"`
	Value string `xml:"value"`
}

//StartInstanceResp response.
type StartInstanceResp struct {
	RequestID    string                `xml:"requestID"`
	StateChanges []InstanceStateChange `xml:"instancesSet>item"`
}

//StopInstanceResp response.
type StopInstanceResp struct {
	RequestID    string                `xml:"requestID"`
	StateChanges []InstanceStateChange `xml:"instancesSet>item"`
}
<|endoftext|>


<commit_before>package facebook

import (
	"testing"
)

// Tests

func TestPage(t *testing.T) {
	id := "19292868552"
	t.Logf("Fetching facebook page %s\n", id)
	_, err := FetchPage(id)
	if err != nil {
		t.Errorf("Error: %s\n", err.String())
	}
}

func TestPageIntrospect(t *testing.T) {
	id := "19292868552"
	t.Logf("Fetching and introspecting facebook page %s\n", id)
	_, err := FetchPageIntrospect(id)
	if err != nil {
		t.Errorf("Error: %s\n", err.String())
	}
}

func TestUser(t *testing.T) {
	name := "btaylor"
	t.Logf("Fetching facebook user %s\n", name)
	_, err := FetchUser(name)
	if err != nil {
		t.Errorf("Error: %s\n", err.String())
	}
}

func TestUserIntrospect(t *testing.T) {
	id := "btaylor"
	t.Logf("Fetching and introspecting facebook user %s\n", id)
	_, err := FetchUserIntrospect(id)
	if err != nil {
		t.Errorf("Error: %s\n", err.String())
	}
}

// Benchmarks

func BenchmarkPage(b *testing.B) {
	for i := 0; i < b.N; i++ {
		FetchPage("19292868552")
	}
}

func BenchmarkPageIntrospect(b *testing.B) {
	for i := 0; i < b.N; i++ {
		FetchPageIntrospect("19292868552")
	}
}

func BenchmarkUser(b *testing.B) {
	for i := 0; i < b.N; i++ {
		FetchUser("btaylor")
	}
}

func BenchmarkUserIntrospect(b *testing.B) {
	for i := 0; i < b.N; i++ {
		FetchUserIntrospect("btaylor")
	}
}
<commit_msg>Rewrite Tests for multiple value tests.<commit_after>package facebook

import (
	"testing"
)

// Tests

type PageTest struct {
	ID   string
	Name string
}

type UserTest struct {
	Name string
}

var PageTests = []PageTest{
	PageTest{"19292868552", "Facebook Platform"},
	PageTest{"40796308305", "Coca-Cola"},
}

var UserTests = []UserTest{
	UserTest{"btaylor"},
}

func TestPage(t *testing.T) {
	for _, v := range PageTests {
		t.Logf("Fetching facebook page %s\n", v.ID)
		p, err := FetchPage(v.ID)
		if err != nil {
			t.Errorf("Error: %s\n", err.String())
		}
		if p.Name != v.Name {
			t.Errorf("Error: %s expected %s \n", p.Name, v.Name)
		}

		t.Logf("Fetching and introspecting facebook page %s\n", v.ID)
		p, err = FetchPageIntrospect(v.ID)
		if err != nil {
			t.Errorf("Error: %s\n", err.String())
		}
		if p.Name != v.Name {
			t.Errorf("Error: %s expected %s \n", p.Name, v.Name)
		}
	}
}

func TestUser(t *testing.T) {
	for _, v := range UserTests {
		t.Logf("Fetching facebook user %s\n", v.Name)
		_, err := FetchUser(v.Name)
		if err != nil {
			t.Errorf("Error: %s\n", err.String())
		}

		t.Logf("Fetching and introspecting facebook user %s\n", v.Name)
		_, err = FetchUserIntrospect(v.Name)
		if err != nil {
			t.Errorf("Error: %s\n", err.String())
		}
	}
}

// Benchmarks

func BenchmarkPage(b *testing.B) {
	for i := 0; i < b.N; i++ {
		FetchPage("19292868552")
	}
}

func BenchmarkPageIntrospect(b *testing.B) {
	for i := 0; i < b.N; i++ {
		FetchPageIntrospect("19292868552")
	}
}

func BenchmarkUser(b *testing.B) {
	for i := 0; i < b.N; i++ {
		FetchUser("btaylor")
	}
}

func BenchmarkUserIntrospect(b *testing.B) {
	for i := 0; i < b.N; i++ {
		FetchUserIntrospect("btaylor")
	}
}
<|endoftext|>


<commit_before>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gateway

import (
	"context"
	"fmt"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	svc "sigs.k8s.io/service-apis/apis/v1alpha1"

	"istio.io/pkg/ledger"

	controller2 "istio.io/istio/pilot/pkg/serviceregistry/kube/controller"
	"istio.io/istio/pkg/config/schema/collection"
	"istio.io/istio/pkg/config/schema/collections"
	"istio.io/istio/pkg/config/schema/gvk"
	"istio.io/istio/pkg/config/schema/resource"

	"istio.io/istio/pilot/pkg/model"
)

var (
	errUnsupportedOp   = fmt.Errorf("unsupported operation: the gateway config store is a read-only view")
	errUnsupportedType = fmt.Errorf("unsupported type: this operation only supports gateway & virtual service resource type")
	_                  = svc.HTTPRoute{}
	_                  = svc.GatewayClass{}
)

type controller struct {
	client kubernetes.Interface
	cache  model.ConfigStoreCache
	domain string
}

func NewController(client kubernetes.Interface, c model.ConfigStoreCache, options controller2.Options) model.ConfigStoreCache {
	return &controller{client, c, options.DomainSuffix}
}

func (c *controller) GetLedger() ledger.Ledger {
	return c.cache.GetLedger()
}

func (c *controller) SetLedger(l ledger.Ledger) error {
	return c.cache.SetLedger(l)
}

func (c *controller) Schemas() collection.Schemas {
	return collection.SchemasFor(
		collections.IstioNetworkingV1Alpha3Virtualservices,
		collections.IstioNetworkingV1Alpha3Gateways,
	)
}

func (c controller) Get(typ resource.GroupVersionKind, name, namespace string) *model.Config {
	panic("get is not supported")
}

func (c controller) List(typ resource.GroupVersionKind, namespace string) ([]model.Config, error) {
	if typ != gvk.Gateway && typ != gvk.VirtualService {
		return nil, errUnsupportedType
	}

	gatewayClass, err := c.cache.List(collections.K8SServiceApisV1Alpha1Gatewayclasses.Resource().GroupVersionKind(), namespace)
	if err != nil {
		return nil, fmt.Errorf("failed to list type GatewayClass: %v", err)
	}
	gateway, err := c.cache.List(collections.K8SServiceApisV1Alpha1Gateways.Resource().GroupVersionKind(), namespace)
	if err != nil {
		return nil, fmt.Errorf("failed to list type Gateway: %v", err)
	}
	httpRoute, err := c.cache.List(collections.K8SServiceApisV1Alpha1Httproutes.Resource().GroupVersionKind(), namespace)
	if err != nil {
		return nil, fmt.Errorf("failed to list type HTTPRoute: %v", err)
	}
	tcpRoute, err := c.cache.List(collections.K8SServiceApisV1Alpha1Tcproutes.Resource().GroupVersionKind(), namespace)
	if err != nil {
		return nil, fmt.Errorf("failed to list type TcpRoute: %v", err)
	}
	trafficSplit, err := c.cache.List(collections.K8SServiceApisV1Alpha1Trafficsplits.Resource().GroupVersionKind(), namespace)
	if err != nil {
		return nil, fmt.Errorf("failed to list type TrafficSplit: %v", err)
	}

	nsl, err := c.client.CoreV1().Namespaces().List(context.TODO(), metav1.ListOptions{})
	if err != nil {
		return nil, fmt.Errorf("failed to list type Namespaces: %v", err)
	}
	namespaces := map[string]*corev1.Namespace{}
	for _, ns := range nsl.Items {
		namespaces[ns.Name] = &ns
	}
	input := &KubernetesResources{
		GatewayClass: gatewayClass,
		Gateway:      gateway,
		HTTPRoute:    httpRoute,
		TCPRoute:     tcpRoute,
		TrafficSplit: trafficSplit,
		Namespaces:   namespaces,
		Domain:       c.domain,
	}
	output := convertResources(input)

	switch typ {
	case gvk.Gateway:
		return output.Gateway, nil
	case gvk.VirtualService:
		return output.VirtualService, nil
	}
	return nil, errUnsupportedOp
}

func (c controller) Create(config model.Config) (revision string, err error) {
	return "", errUnsupportedOp
}

func (c controller) Update(config model.Config) (newRevision string, err error) {
	return "", errUnsupportedOp
}

func (c controller) Delete(typ resource.GroupVersionKind, name, namespace string) error {
	return errUnsupportedOp
}

func (c controller) Version() string {
	return c.cache.Version()
}

func (c controller) GetResourceAtVersion(version string, key string) (resourceVersion string, err error) {
	return c.cache.GetResourceAtVersion(version, key)
}

func (c controller) RegisterEventHandler(typ resource.GroupVersionKind, handler func(model.Config, model.Config, model.Event)) {
	c.cache.RegisterEventHandler(typ, func(prev, cur model.Config, event model.Event) {
		handler(prev, cur, event)
	})
}

func (c controller) Run(stop <-chan struct{}) {
}

func (c controller) HasSynced() bool {
	return c.cache.HasSynced()
}
<commit_msg>[pilot/gateway] controller: fix range iterator issue (#25559)<commit_after>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gateway

import (
	"context"
	"fmt"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	svc "sigs.k8s.io/service-apis/apis/v1alpha1"

	"istio.io/pkg/ledger"

	controller2 "istio.io/istio/pilot/pkg/serviceregistry/kube/controller"
	"istio.io/istio/pkg/config/schema/collection"
	"istio.io/istio/pkg/config/schema/collections"
	"istio.io/istio/pkg/config/schema/gvk"
	"istio.io/istio/pkg/config/schema/resource"

	"istio.io/istio/pilot/pkg/model"
)

var (
	errUnsupportedOp   = fmt.Errorf("unsupported operation: the gateway config store is a read-only view")
	errUnsupportedType = fmt.Errorf("unsupported type: this operation only supports gateway & virtual service resource type")
	_                  = svc.HTTPRoute{}
	_                  = svc.GatewayClass{}
)

type controller struct {
	client kubernetes.Interface
	cache  model.ConfigStoreCache
	domain string
}

func NewController(client kubernetes.Interface, c model.ConfigStoreCache, options controller2.Options) model.ConfigStoreCache {
	return &controller{client, c, options.DomainSuffix}
}

func (c *controller) GetLedger() ledger.Ledger {
	return c.cache.GetLedger()
}

func (c *controller) SetLedger(l ledger.Ledger) error {
	return c.cache.SetLedger(l)
}

func (c *controller) Schemas() collection.Schemas {
	return collection.SchemasFor(
		collections.IstioNetworkingV1Alpha3Virtualservices,
		collections.IstioNetworkingV1Alpha3Gateways,
	)
}

func (c controller) Get(typ resource.GroupVersionKind, name, namespace string) *model.Config {
	panic("get is not supported")
}

func (c controller) List(typ resource.GroupVersionKind, namespace string) ([]model.Config, error) {
	if typ != gvk.Gateway && typ != gvk.VirtualService {
		return nil, errUnsupportedType
	}

	gatewayClass, err := c.cache.List(collections.K8SServiceApisV1Alpha1Gatewayclasses.Resource().GroupVersionKind(), namespace)
	if err != nil {
		return nil, fmt.Errorf("failed to list type GatewayClass: %v", err)
	}
	gateway, err := c.cache.List(collections.K8SServiceApisV1Alpha1Gateways.Resource().GroupVersionKind(), namespace)
	if err != nil {
		return nil, fmt.Errorf("failed to list type Gateway: %v", err)
	}
	httpRoute, err := c.cache.List(collections.K8SServiceApisV1Alpha1Httproutes.Resource().GroupVersionKind(), namespace)
	if err != nil {
		return nil, fmt.Errorf("failed to list type HTTPRoute: %v", err)
	}
	tcpRoute, err := c.cache.List(collections.K8SServiceApisV1Alpha1Tcproutes.Resource().GroupVersionKind(), namespace)
	if err != nil {
		return nil, fmt.Errorf("failed to list type TcpRoute: %v", err)
	}
	trafficSplit, err := c.cache.List(collections.K8SServiceApisV1Alpha1Trafficsplits.Resource().GroupVersionKind(), namespace)
	if err != nil {
		return nil, fmt.Errorf("failed to list type TrafficSplit: %v", err)
	}

	nsl, err := c.client.CoreV1().Namespaces().List(context.TODO(), metav1.ListOptions{})
	if err != nil {
		return nil, fmt.Errorf("failed to list type Namespaces: %v", err)
	}
	namespaces := map[string]*corev1.Namespace{}
	for i, ns := range nsl.Items {
		namespaces[ns.Name] = &nsl.Items[i]
	}
	input := &KubernetesResources{
		GatewayClass: gatewayClass,
		Gateway:      gateway,
		HTTPRoute:    httpRoute,
		TCPRoute:     tcpRoute,
		TrafficSplit: trafficSplit,
		Namespaces:   namespaces,
		Domain:       c.domain,
	}
	output := convertResources(input)

	switch typ {
	case gvk.Gateway:
		return output.Gateway, nil
	case gvk.VirtualService:
		return output.VirtualService, nil
	}
	return nil, errUnsupportedOp
}

func (c controller) Create(config model.Config) (revision string, err error) {
	return "", errUnsupportedOp
}

func (c controller) Update(config model.Config) (newRevision string, err error) {
	return "", errUnsupportedOp
}

func (c controller) Delete(typ resource.GroupVersionKind, name, namespace string) error {
	return errUnsupportedOp
}

func (c controller) Version() string {
	return c.cache.Version()
}

func (c controller) GetResourceAtVersion(version string, key string) (resourceVersion string, err error) {
	return c.cache.GetResourceAtVersion(version, key)
}

func (c controller) RegisterEventHandler(typ resource.GroupVersionKind, handler func(model.Config, model.Config, model.Event)) {
	c.cache.RegisterEventHandler(typ, func(prev, cur model.Config, event model.Event) {
		handler(prev, cur, event)
	})
}

func (c controller) Run(stop <-chan struct{}) {
}

func (c controller) HasSynced() bool {
	return c.cache.HasSynced()
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vfsclientset

import (
	"bytes"
	"fmt"
	"github.com/golang/glog"
	kops "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/apis/kops/v1alpha1"
	"k8s.io/kops/util/pkg/vfs"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/apis/meta/v1"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/runtime/schema"
	"os"
	"reflect"
	"sort"
	"time"
)

var StoreVersion = v1alpha1.SchemeGroupVersion

type commonVFS struct {
	kind               string
	basePath           vfs.Path
	decoder            runtime.Decoder
	encoder            runtime.Encoder
	defaultReadVersion *schema.GroupVersionKind
}

func (c *commonVFS) init(kind string, basePath vfs.Path, storeVersion runtime.GroupVersioner) {
	yaml, ok := runtime.SerializerInfoForMediaType(api.Codecs.SupportedMediaTypes(), "application/yaml")
	if !ok {
		glog.Fatalf("no YAML serializer registered")
	}
	c.encoder = api.Codecs.EncoderForVersion(yaml.Serializer, storeVersion)
	c.decoder = api.Codecs.DecoderToVersion(yaml.Serializer, kops.SchemeGroupVersion)

	c.kind = kind
	c.basePath = basePath
}

func (c *commonVFS) get(name string) (runtime.Object, error) {
	o, err := c.readConfig(c.basePath.Join(name))
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, fmt.Errorf("error reading %s %q: %v", c.kind, name, err)
	}
	return o, nil
}

func (c *commonVFS) list(items interface{}, options api.ListOptions) (interface{}, error) {
	return c.readAll(items)
}

func (c *commonVFS) create(i runtime.Object) error {
	objectMeta, err := api.ObjectMetaFor(i)
	if err != nil {
		return err
	}

	err = i.(kops.ApiType).Validate()
	if err != nil {
		return err
	}

	if objectMeta.CreationTimestamp.IsZero() {
		objectMeta.CreationTimestamp = v1.NewTime(time.Now().UTC())
	}

	err = c.writeConfig(c.basePath.Join(objectMeta.Name), i, vfs.WriteOptionCreate)
	if err != nil {
		if os.IsExist(err) {
			return err
		}
		return fmt.Errorf("error writing %s: %v", c.kind, err)
	}

	return nil
}

func (c *commonVFS) serialize(o runtime.Object) ([]byte, error) {
	var b bytes.Buffer
	err := c.encoder.Encode(o, &b)
	if err != nil {
		return nil, fmt.Errorf("error encoding object: %v", err)
	}

	return b.Bytes(), nil
}

func (c *commonVFS) readConfig(configPath vfs.Path) (runtime.Object, error) {
	data, err := configPath.ReadFile()
	if err != nil {
		if os.IsNotExist(err) {
			return nil, err
		}
		return nil, fmt.Errorf("error reading %s: %v", configPath, err)
	}

	object, _, err := c.decoder.Decode(data, c.defaultReadVersion, nil)
	if err != nil {
		return nil, fmt.Errorf("error parsing %s: %v", configPath, err)
	}
	return object, nil
}

func (c *commonVFS) writeConfig(configPath vfs.Path, o runtime.Object, writeOptions ...vfs.WriteOption) error {
	data, err := c.serialize(o)
	if err != nil {
		return fmt.Errorf("error marshalling object: %v", err)
	}

	create := false
	for _, writeOption := range writeOptions {
		switch writeOption {
		case vfs.WriteOptionCreate:
			create = true
		case vfs.WriteOptionOnlyIfExists:
			_, err = configPath.ReadFile()
			if err != nil {
				if os.IsNotExist(err) {
					return fmt.Errorf("cannot update configuration file %s: does not exist", configPath)
				}
				return fmt.Errorf("error checking if configuration file %s exists already: %v", configPath, err)
			}
		default:
			return fmt.Errorf("unknown write option: %q", writeOption)
		}
	}

	if create {
		err = configPath.CreateFile(data)
	} else {
		err = configPath.WriteFile(data)
	}
	if err != nil {
		if create && os.IsExist(err) {
			glog.Warningf("failed to create file as already exists: %v", configPath)
			return err
		}
		return fmt.Errorf("error writing configuration file %s: %v", configPath, err)
	}
	return nil
}

func (c *commonVFS) update(i runtime.Object) error {
	objectMeta, err := api.ObjectMetaFor(i)
	if err != nil {
		return err
	}

	err = i.(kops.ApiType).Validate()
	if err != nil {
		return err
	}

	if objectMeta.CreationTimestamp.IsZero() {
		objectMeta.CreationTimestamp = v1.NewTime(time.Now().UTC())
	}

	err = c.writeConfig(c.basePath.Join(objectMeta.Name), i, vfs.WriteOptionOnlyIfExists)
	if err != nil {
		return fmt.Errorf("error writing %s: %v", c.kind, err)
	}

	return nil
}

func (c *commonVFS) delete(name string, options *api.DeleteOptions) error {
	p := c.basePath.Join(name)
	err := p.Remove()
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return fmt.Errorf("error deleting %s configuration %q: %v", c.kind, name, err)
	}
	return nil
}

func (c *commonVFS) listNames() ([]string, error) {
	keys, err := listChildNames(c.basePath)
	if err != nil {
		return nil, fmt.Errorf("error listing %s in state store: %v", c.kind, err)
	}

	// Seems to be an assumption in k8s APIs that items are always returned sorted
	sort.Strings(keys)

	return keys, nil
}

func (c *commonVFS) readAll(items interface{}) (interface{}, error) {
	sliceValue := reflect.ValueOf(items)
	sliceType := reflect.TypeOf(items)
	if sliceType.Kind() != reflect.Slice {
		return nil, fmt.Errorf("expected slice, got %T", items)
	}

	names, err := c.listNames()
	if err != nil {
		return nil, err
	}

	for _, name := range names {
		o, err := c.get(name)
		if err != nil {
			return nil, err
		}

		if o == nil {
			return nil, fmt.Errorf("%s was listed, but then not found %q", c.kind, name)
		}

		sliceValue = reflect.Append(sliceValue, reflect.ValueOf(o).Elem())
	}

	return sliceValue.Interface(), nil
}
<commit_msg>Store in v1alpha2 format<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vfsclientset

import (
	"bytes"
	"fmt"
	"github.com/golang/glog"
	kops "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/apis/kops/v1alpha2"
	"k8s.io/kops/util/pkg/vfs"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/apis/meta/v1"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/runtime/schema"
	"os"
	"reflect"
	"sort"
	"time"
)

var StoreVersion = v1alpha2.SchemeGroupVersion

type commonVFS struct {
	kind               string
	basePath           vfs.Path
	decoder            runtime.Decoder
	encoder            runtime.Encoder
	defaultReadVersion *schema.GroupVersionKind
}

func (c *commonVFS) init(kind string, basePath vfs.Path, storeVersion runtime.GroupVersioner) {
	yaml, ok := runtime.SerializerInfoForMediaType(api.Codecs.SupportedMediaTypes(), "application/yaml")
	if !ok {
		glog.Fatalf("no YAML serializer registered")
	}
	c.encoder = api.Codecs.EncoderForVersion(yaml.Serializer, storeVersion)
	c.decoder = api.Codecs.DecoderToVersion(yaml.Serializer, kops.SchemeGroupVersion)

	c.kind = kind
	c.basePath = basePath
}

func (c *commonVFS) get(name string) (runtime.Object, error) {
	o, err := c.readConfig(c.basePath.Join(name))
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, fmt.Errorf("error reading %s %q: %v", c.kind, name, err)
	}
	return o, nil
}

func (c *commonVFS) list(items interface{}, options api.ListOptions) (interface{}, error) {
	return c.readAll(items)
}

func (c *commonVFS) create(i runtime.Object) error {
	objectMeta, err := api.ObjectMetaFor(i)
	if err != nil {
		return err
	}

	err = i.(kops.ApiType).Validate()
	if err != nil {
		return err
	}

	if objectMeta.CreationTimestamp.IsZero() {
		objectMeta.CreationTimestamp = v1.NewTime(time.Now().UTC())
	}

	err = c.writeConfig(c.basePath.Join(objectMeta.Name), i, vfs.WriteOptionCreate)
	if err != nil {
		if os.IsExist(err) {
			return err
		}
		return fmt.Errorf("error writing %s: %v", c.kind, err)
	}

	return nil
}

func (c *commonVFS) serialize(o runtime.Object) ([]byte, error) {
	var b bytes.Buffer
	err := c.encoder.Encode(o, &b)
	if err != nil {
		return nil, fmt.Errorf("error encoding object: %v", err)
	}

	return b.Bytes(), nil
}

func (c *commonVFS) readConfig(configPath vfs.Path) (runtime.Object, error) {
	data, err := configPath.ReadFile()
	if err != nil {
		if os.IsNotExist(err) {
			return nil, err
		}
		return nil, fmt.Errorf("error reading %s: %v", configPath, err)
	}

	object, _, err := c.decoder.Decode(data, c.defaultReadVersion, nil)
	if err != nil {
		return nil, fmt.Errorf("error parsing %s: %v", configPath, err)
	}
	return object, nil
}

func (c *commonVFS) writeConfig(configPath vfs.Path, o runtime.Object, writeOptions ...vfs.WriteOption) error {
	data, err := c.serialize(o)
	if err != nil {
		return fmt.Errorf("error marshalling object: %v", err)
	}

	create := false
	for _, writeOption := range writeOptions {
		switch writeOption {
		case vfs.WriteOptionCreate:
			create = true
		case vfs.WriteOptionOnlyIfExists:
			_, err = configPath.ReadFile()
			if err != nil {
				if os.IsNotExist(err) {
					return fmt.Errorf("cannot update configuration file %s: does not exist", configPath)
				}
				return fmt.Errorf("error checking if configuration file %s exists already: %v", configPath, err)
			}
		default:
			return fmt.Errorf("unknown write option: %q", writeOption)
		}
	}

	if create {
		err = configPath.CreateFile(data)
	} else {
		err = configPath.WriteFile(data)
	}
	if err != nil {
		if create && os.IsExist(err) {
			glog.Warningf("failed to create file as already exists: %v", configPath)
			return err
		}
		return fmt.Errorf("error writing configuration file %s: %v", configPath, err)
	}
	return nil
}

func (c *commonVFS) update(i runtime.Object) error {
	objectMeta, err := api.ObjectMetaFor(i)
	if err != nil {
		return err
	}

	err = i.(kops.ApiType).Validate()
	if err != nil {
		return err
	}

	if objectMeta.CreationTimestamp.IsZero() {
		objectMeta.CreationTimestamp = v1.NewTime(time.Now().UTC())
	}

	err = c.writeConfig(c.basePath.Join(objectMeta.Name), i, vfs.WriteOptionOnlyIfExists)
	if err != nil {
		return fmt.Errorf("error writing %s: %v", c.kind, err)
	}

	return nil
}

func (c *commonVFS) delete(name string, options *api.DeleteOptions) error {
	p := c.basePath.Join(name)
	err := p.Remove()
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return fmt.Errorf("error deleting %s configuration %q: %v", c.kind, name, err)
	}
	return nil
}

func (c *commonVFS) listNames() ([]string, error) {
	keys, err := listChildNames(c.basePath)
	if err != nil {
		return nil, fmt.Errorf("error listing %s in state store: %v", c.kind, err)
	}

	// Seems to be an assumption in k8s APIs that items are always returned sorted
	sort.Strings(keys)

	return keys, nil
}

func (c *commonVFS) readAll(items interface{}) (interface{}, error) {
	sliceValue := reflect.ValueOf(items)
	sliceType := reflect.TypeOf(items)
	if sliceType.Kind() != reflect.Slice {
		return nil, fmt.Errorf("expected slice, got %T", items)
	}

	names, err := c.listNames()
	if err != nil {
		return nil, err
	}

	for _, name := range names {
		o, err := c.get(name)
		if err != nil {
			return nil, err
		}

		if o == nil {
			return nil, fmt.Errorf("%s was listed, but then not found %q", c.kind, name)
		}

		sliceValue = reflect.Append(sliceValue, reflect.ValueOf(o).Elem())
	}

	return sliceValue.Interface(), nil
}
<|endoftext|>


<commit_before>package admission

import (
	"io"

	"github.com/golang/glog"

	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apiserver/pkg/admission"
	genericapiserver "k8s.io/apiserver/pkg/server"
	kubeapiserver "k8s.io/kubernetes/cmd/kube-apiserver/app/options"

	// Admission control plug-ins used by OpenShift
	authorizationrestrictusers "github.com/openshift/origin/pkg/authorization/admission/restrictusers"
	buildjenkinsbootstrapper "github.com/openshift/origin/pkg/build/admission/jenkinsbootstrapper"
	buildsecretinjector "github.com/openshift/origin/pkg/build/admission/secretinjector"
	buildstrategyrestrictions "github.com/openshift/origin/pkg/build/admission/strategyrestrictions"
	imageadmission "github.com/openshift/origin/pkg/image/admission"
	imagepolicy "github.com/openshift/origin/pkg/image/admission/imagepolicy"
	imagequalify "github.com/openshift/origin/pkg/image/admission/imagequalify"
	ingressadmission "github.com/openshift/origin/pkg/ingress/admission"
	projectlifecycle "github.com/openshift/origin/pkg/project/admission/lifecycle"
	projectnodeenv "github.com/openshift/origin/pkg/project/admission/nodeenv"
	projectrequestlimit "github.com/openshift/origin/pkg/project/admission/requestlimit"
	quotaclusterresourceoverride "github.com/openshift/origin/pkg/quota/admission/clusterresourceoverride"
	quotaclusterresourcequota "github.com/openshift/origin/pkg/quota/admission/clusterresourcequota"
	quotarunonceduration "github.com/openshift/origin/pkg/quota/admission/runonceduration"
	schedulerpodnodeconstraints "github.com/openshift/origin/pkg/scheduler/admission/podnodeconstraints"
	securityadmission "github.com/openshift/origin/pkg/security/admission"
	serviceadmit "github.com/openshift/origin/pkg/service/admission"

	"k8s.io/kubernetes/plugin/pkg/admission/noderestriction"
	expandpvcadmission "k8s.io/kubernetes/plugin/pkg/admission/persistentvolume/resize"
	storageclassdefaultadmission "k8s.io/kubernetes/plugin/pkg/admission/storageclass/setdefault"

	imagepolicyapi "github.com/openshift/origin/pkg/image/admission/imagepolicy/api"
	imagequalifyapi "github.com/openshift/origin/pkg/image/admission/imagequalify/api"
	overrideapi "github.com/openshift/origin/pkg/quota/admission/clusterresourceoverride/api"
	"k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle"

	configlatest "github.com/openshift/origin/pkg/cmd/server/api/latest"
)

// TODO register this per apiserver or at least per process
var OriginAdmissionPlugins = admission.NewPlugins()

func init() {
	RegisterAllAdmissionPlugins(OriginAdmissionPlugins)
}

// RegisterAllAdmissionPlugins registers all admission plugins
func RegisterAllAdmissionPlugins(plugins *admission.Plugins) {
	kubeapiserver.RegisterAllAdmissionPlugins(plugins)
	genericapiserver.RegisterAllAdmissionPlugins(plugins)
	registerOpenshiftAdmissionPlugins(plugins)
}

func registerOpenshiftAdmissionPlugins(plugins *admission.Plugins) {
	authorizationrestrictusers.Register(plugins)
	buildjenkinsbootstrapper.Register(plugins)
	buildsecretinjector.Register(plugins)
	buildstrategyrestrictions.Register(plugins)
	imageadmission.Register(plugins)
	imagepolicy.Register(plugins)
	imagequalify.Register(plugins)
	ingressadmission.Register(plugins)
	projectlifecycle.Register(plugins)
	projectnodeenv.Register(plugins)
	projectrequestlimit.Register(plugins)
	quotaclusterresourceoverride.Register(plugins)
	quotaclusterresourcequota.Register(plugins)
	quotarunonceduration.Register(plugins)
	schedulerpodnodeconstraints.Register(plugins)
	securityadmission.Register(plugins)
	securityadmission.RegisterSCCExecRestrictions(plugins)
	serviceadmit.RegisterExternalIP(plugins)
	serviceadmit.RegisterRestrictedEndpoints(plugins)
}

var (
	DefaultOnPlugins = sets.NewString(
		"OriginNamespaceLifecycle",
		"openshift.io/JenkinsBootstrapper",
		"openshift.io/BuildConfigSecretInjector",
		"BuildByStrategy",
		storageclassdefaultadmission.PluginName,
		imageadmission.PluginName,
		lifecycle.PluginName,
		"OriginPodNodeEnvironment",
		"PodNodeSelector",
		serviceadmit.ExternalIPPluginName,
		serviceadmit.RestrictedEndpointsPluginName,
		"LimitRanger",
		"ServiceAccount",
		noderestriction.PluginName,
		securityadmission.PluginName,
		"SCCExecRestrictions",
		"PersistentVolumeLabel",
		"DefaultStorageClass",
		"OwnerReferencesPermissionEnforcement",
		"ResourceQuota",
		"openshift.io/ClusterResourceQuota",
		"openshift.io/IngressAdmission",
	)

	// DefaultOffPlugins includes plugins which require explicit configuration to run
	// if you wire them incorrectly, they may prevent the server from starting
	DefaultOffPlugins = sets.NewString(
		"ProjectRequestLimit",
		"RunOnceDuration",
		"PodNodeConstraints",
		overrideapi.PluginName,
		imagepolicyapi.PluginName,
		imagequalifyapi.PluginName,
		"AlwaysPullImages",
		"ImagePolicyWebhook",
		"openshift.io/RestrictSubjectBindings",
		"LimitPodHardAntiAffinityTopology",
		"DefaultTolerationSeconds",
		"PodPreset", // default to off while PodPreset is alpha
		"EventRateLimit",
		"PodSecurityPolicy",
		"Priority",
		"Initializers",
		"ValidatingAdmissionWebhook",
		"MutatingAdmissionWebhook",
		"PodTolerationRestriction",
		"ExtendedResourceToleration",
		"PVCProtection",
		expandpvcadmission.PluginName,

		// these should usually be off.
		"AlwaysAdmit",
		"AlwaysDeny",
		"DenyEscalatingExec",
		"DenyExecOnPrivileged",
		"InitialResources",
		"NamespaceAutoProvision",
		"NamespaceExists",
		"SecurityContextDeny",
	)
)

func init() {
	admission.PluginEnabledFn = IsAdmissionPluginActivated
}

func IsAdmissionPluginActivated(name string, config io.Reader) bool {
	// only intercept if we have an explicit enable or disable.  If the check fails in any way,
	// assume that the config was a different type and let the actual admission plugin check it
	if DefaultOnPlugins.Has(name) {
		if enabled, err := configlatest.IsAdmissionPluginActivated(config, true); err == nil && !enabled {
			glog.V(2).Infof("Admission plugin %v is disabled.  It will not be started.", name)
			return false
		}
	} else if DefaultOffPlugins.Has(name) {
		if enabled, err := configlatest.IsAdmissionPluginActivated(config, false); err == nil && !enabled {
			glog.V(2).Infof("Admission plugin %v is not enabled.  It will not be started.", name)
			return false
		}
	}

	return true
}
<commit_msg>enable pod toleration checks by default<commit_after>package admission

import (
	"io"

	"github.com/golang/glog"

	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apiserver/pkg/admission"
	genericapiserver "k8s.io/apiserver/pkg/server"
	kubeapiserver "k8s.io/kubernetes/cmd/kube-apiserver/app/options"

	// Admission control plug-ins used by OpenShift
	authorizationrestrictusers "github.com/openshift/origin/pkg/authorization/admission/restrictusers"
	buildjenkinsbootstrapper "github.com/openshift/origin/pkg/build/admission/jenkinsbootstrapper"
	buildsecretinjector "github.com/openshift/origin/pkg/build/admission/secretinjector"
	buildstrategyrestrictions "github.com/openshift/origin/pkg/build/admission/strategyrestrictions"
	imageadmission "github.com/openshift/origin/pkg/image/admission"
	imagepolicy "github.com/openshift/origin/pkg/image/admission/imagepolicy"
	imagequalify "github.com/openshift/origin/pkg/image/admission/imagequalify"
	ingressadmission "github.com/openshift/origin/pkg/ingress/admission"
	projectlifecycle "github.com/openshift/origin/pkg/project/admission/lifecycle"
	projectnodeenv "github.com/openshift/origin/pkg/project/admission/nodeenv"
	projectrequestlimit "github.com/openshift/origin/pkg/project/admission/requestlimit"
	quotaclusterresourceoverride "github.com/openshift/origin/pkg/quota/admission/clusterresourceoverride"
	quotaclusterresourcequota "github.com/openshift/origin/pkg/quota/admission/clusterresourcequota"
	quotarunonceduration "github.com/openshift/origin/pkg/quota/admission/runonceduration"
	schedulerpodnodeconstraints "github.com/openshift/origin/pkg/scheduler/admission/podnodeconstraints"
	securityadmission "github.com/openshift/origin/pkg/security/admission"
	serviceadmit "github.com/openshift/origin/pkg/service/admission"

	"k8s.io/kubernetes/plugin/pkg/admission/noderestriction"
	expandpvcadmission "k8s.io/kubernetes/plugin/pkg/admission/persistentvolume/resize"
	storageclassdefaultadmission "k8s.io/kubernetes/plugin/pkg/admission/storageclass/setdefault"

	imagepolicyapi "github.com/openshift/origin/pkg/image/admission/imagepolicy/api"
	imagequalifyapi "github.com/openshift/origin/pkg/image/admission/imagequalify/api"
	overrideapi "github.com/openshift/origin/pkg/quota/admission/clusterresourceoverride/api"
	"k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle"

	configlatest "github.com/openshift/origin/pkg/cmd/server/api/latest"
)

// TODO register this per apiserver or at least per process
var OriginAdmissionPlugins = admission.NewPlugins()

func init() {
	RegisterAllAdmissionPlugins(OriginAdmissionPlugins)
}

// RegisterAllAdmissionPlugins registers all admission plugins
func RegisterAllAdmissionPlugins(plugins *admission.Plugins) {
	kubeapiserver.RegisterAllAdmissionPlugins(plugins)
	genericapiserver.RegisterAllAdmissionPlugins(plugins)
	registerOpenshiftAdmissionPlugins(plugins)
}

func registerOpenshiftAdmissionPlugins(plugins *admission.Plugins) {
	authorizationrestrictusers.Register(plugins)
	buildjenkinsbootstrapper.Register(plugins)
	buildsecretinjector.Register(plugins)
	buildstrategyrestrictions.Register(plugins)
	imageadmission.Register(plugins)
	imagepolicy.Register(plugins)
	imagequalify.Register(plugins)
	ingressadmission.Register(plugins)
	projectlifecycle.Register(plugins)
	projectnodeenv.Register(plugins)
	projectrequestlimit.Register(plugins)
	quotaclusterresourceoverride.Register(plugins)
	quotaclusterresourcequota.Register(plugins)
	quotarunonceduration.Register(plugins)
	schedulerpodnodeconstraints.Register(plugins)
	securityadmission.Register(plugins)
	securityadmission.RegisterSCCExecRestrictions(plugins)
	serviceadmit.RegisterExternalIP(plugins)
	serviceadmit.RegisterRestrictedEndpoints(plugins)
}

var (
	DefaultOnPlugins = sets.NewString(
		"OriginNamespaceLifecycle",
		"openshift.io/JenkinsBootstrapper",
		"openshift.io/BuildConfigSecretInjector",
		"BuildByStrategy",
		storageclassdefaultadmission.PluginName,
		imageadmission.PluginName,
		lifecycle.PluginName,
		"OriginPodNodeEnvironment",
		"PodNodeSelector",
		serviceadmit.ExternalIPPluginName,
		serviceadmit.RestrictedEndpointsPluginName,
		"LimitRanger",
		"ServiceAccount",
		noderestriction.PluginName,
		securityadmission.PluginName,
		"SCCExecRestrictions",
		"PersistentVolumeLabel",
		"DefaultStorageClass",
		"OwnerReferencesPermissionEnforcement",
		"PodTolerationRestriction",
		"ResourceQuota",
		"openshift.io/ClusterResourceQuota",
		"openshift.io/IngressAdmission",
	)

	// DefaultOffPlugins includes plugins which require explicit configuration to run
	// if you wire them incorrectly, they may prevent the server from starting
	DefaultOffPlugins = sets.NewString(
		"ProjectRequestLimit",
		"RunOnceDuration",
		"PodNodeConstraints",
		overrideapi.PluginName,
		imagepolicyapi.PluginName,
		imagequalifyapi.PluginName,
		"AlwaysPullImages",
		"ImagePolicyWebhook",
		"openshift.io/RestrictSubjectBindings",
		"LimitPodHardAntiAffinityTopology",
		"DefaultTolerationSeconds",
		"PodPreset", // default to off while PodPreset is alpha
		"EventRateLimit",
		"PodSecurityPolicy",
		"Priority",
		"Initializers",
		"ValidatingAdmissionWebhook",
		"MutatingAdmissionWebhook",
		"ExtendedResourceToleration",
		"PVCProtection",
		expandpvcadmission.PluginName,

		// these should usually be off.
		"AlwaysAdmit",
		"AlwaysDeny",
		"DenyEscalatingExec",
		"DenyExecOnPrivileged",
		"InitialResources",
		"NamespaceAutoProvision",
		"NamespaceExists",
		"SecurityContextDeny",
	)
)

func init() {
	admission.PluginEnabledFn = IsAdmissionPluginActivated
}

func IsAdmissionPluginActivated(name string, config io.Reader) bool {
	// only intercept if we have an explicit enable or disable.  If the check fails in any way,
	// assume that the config was a different type and let the actual admission plugin check it
	if DefaultOnPlugins.Has(name) {
		if enabled, err := configlatest.IsAdmissionPluginActivated(config, true); err == nil && !enabled {
			glog.V(2).Infof("Admission plugin %v is disabled.  It will not be started.", name)
			return false
		}
	} else if DefaultOffPlugins.Has(name) {
		if enabled, err := configlatest.IsAdmissionPluginActivated(config, false); err == nil && !enabled {
			glog.V(2).Infof("Admission plugin %v is not enabled.  It will not be started.", name)
			return false
		}
	}

	return true
}
<|endoftext|>


<commit_before>// Copyright (c) 2019 The OpenSDS Authors All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
This module implements a entry into the OpenSDS metrics controller service.

*/

package metrics

import (
	"encoding/json"
	log "github.com/golang/glog"
	"github.com/opensds/opensds/pkg/dock/client"
	"github.com/opensds/opensds/pkg/model"
	pb "github.com/opensds/opensds/pkg/model/proto"
	"io/ioutil"
	"net/http"
	"strconv"
)

// Controller is an interface for exposing some operations of metric controllers.
type Controller interface {
	GetLatestMetrics(opt *pb.GetMetricsOpts) ([]*model.MetricSpec, error)
	GetInstantMetrics(opt *pb.GetMetricsOpts) ([]*model.MetricSpec, error)
	GetRangeMetrics(opt *pb.GetMetricsOpts) ([]*model.MetricSpec, error)
	SetDock(dockInfo *model.DockSpec)
}

// NewController method creates a controller structure and expose its pointer.
func NewController() Controller {
	return &controller{
		Client: client.NewClient(),
	}
}

type controller struct {
	client.Client
	DockInfo *model.DockSpec
}

// latest+instant metrics structs begin
type InstantMetricReponseFromPrometheus struct {
	Status string `json:"status"`
	Data   Data   `json:"data"`
}
type Metric struct {
	Name       string `json:"__name__"`
	Device     string `json:"device"`
	InstanceID string `json:"instanceID"`
	Job        string `json:"job"`
}
type Result struct {
	Metric Metric        `json:"metric"`
	Value  []interface{} `json:"value"`
}
type Data struct {
	ResultType string   `json:"resultType"`
	Result     []Result `json:"result"`
}

// latest+instant metrics structs end

// latest+range metrics structs begin
type RangeMetricReponseFromPrometheus struct {
	Status string    `json:"status"`
	Data   RangeData `json:"data"`
}
type RangeMetric struct {
	Name     string `json:"__name__"`
	Device   string `json:"device"`
	Instance string `json:"instance"`
	Job      string `json:"job"`
}
type RangeResult struct {
	Metric RangeMetric     `json:"metric"`
	Values [][]interface{} `json:"values"`
}
type RangeData struct {
	ResultType string        `json:"resultType"`
	Result     []RangeResult `json:"result"`
}

// latest+range metrics structs end

func (c *controller) GetLatestMetrics(opt *pb.GetMetricsOpts) ([]*model.MetricSpec, error) {

	// make a call to Prometheus, convert the response to our format, return
	response, err := http.Get("http://localhost:9090/api/v1/query?query=" + opt.MetricName)
	if err != nil {
		log.Infof("The HTTP query request failed with error %s\n", err)
	} else {
		data, _ := ioutil.ReadAll(response.Body)
		log.Infof("response data is %s", string(data))
		// unmarshal the JSON response into a struct (generated using the JSON, using this https://mholt.github.io/json-to-go/
		var fv InstantMetricReponseFromPrometheus
		err0 := json.Unmarshal(data, &fv)
		if err0 != nil {
			log.Infof("unmarshell operation failed ", err0)
		}
		var metrics []*model.MetricSpec
		// now convert to our repsonse struct, so we can marshal it and send out the JSON
		for _, res := range fv.Data.Result {

			metricValues := make([]*model.Metric, 0)
			metricValue := &model.Metric{}
			for _, v := range res.Value {

				switch v.(type) {
				case string:
					metricValue.Value, err = strconv.ParseFloat(v.(string), 64)
				case float64:
					secs := int64(v.(float64))
					metricValue.Timestamp = secs
				default:
					log.Info(v, "is of a type I don't know how to handle")
				}

			}
			metricValues = append(metricValues, metricValue)
			metric := &model.MetricSpec{}
			metric.InstanceID = res.Metric.InstanceID
			metric.Name = res.Metric.Name
			metric.InstanceName = res.Metric.Device
			metric.MetricValues = metricValues
			metrics = append(metrics, metric)
		}

		bArr, _ := json.Marshal(metrics)
		log.Infof("metrics response json is %s", string(bArr))

		if err != nil {
			log.Error(err)
		}
		return metrics, err

	}
	//no response
	return nil, err
}

func (c *controller) GetInstantMetrics(opt *pb.GetMetricsOpts) ([]*model.MetricSpec, error) {

	// make a call to Prometheus, convert the response to our format, return
	response, err := http.Get("http://localhost:9090/api/v1/query?query=" + opt.MetricName + "&time=" + opt.StartTime)
	if err != nil {
		log.Infof("The HTTP query request failed with error %s\n", err)
	} else {
		data, _ := ioutil.ReadAll(response.Body)
		log.Infof("response data is %s", string(data))
		// unmarshal the JSON response into a struct (generated using the JSON, using this https://mholt.github.io/json-to-go/
		var fv InstantMetricReponseFromPrometheus
		err0 := json.Unmarshal(data, &fv)
		if err0 != nil {
			log.Infof("unmarshell operation failed ", err0)
		}
		var metrics []*model.MetricSpec
		// now convert to our repsonse struct, so we can marshal it and send out the JSON
		for _, res := range fv.Data.Result {

			metricValues := make([]*model.Metric, 0)
			metricValue := &model.Metric{}
			for _, v := range res.Value {

				switch v.(type) {
				case string:
					metricValue.Value, err = strconv.ParseFloat(v.(string), 64)
				case float64:
					secs := int64(v.(float64))
					metricValue.Timestamp = secs
				default:
					log.Info(v, "is of a type I don't know how to handle")
				}

			}
			metricValues = append(metricValues, metricValue)
			metric := &model.MetricSpec{}
			metric.InstanceID = res.Metric.InstanceID
			metric.Name = res.Metric.Name
			metric.InstanceName = res.Metric.Device
			metric.MetricValues = metricValues
			metrics = append(metrics, metric)
		}

		bArr, _ := json.Marshal(metrics)
		log.Infof("metrics response json is %s", string(bArr))

		if err != nil {
			log.Error(err)
		}
		return metrics, err

	}
	//no response
	return nil, err
}

func (c *controller) GetRangeMetrics(opt *pb.GetMetricsOpts) ([]*model.MetricSpec, error) {

	//var metrics []model.MetricSpec
	// make a call to Prometheus, convert the response to our format, return
	response, err := http.Get("http://localhost:9090/api/v1/query_range?query=" + opt.MetricName + "&start=" + opt.StartTime + "&end=" + opt.EndTime + "&step=30")
	if err != nil {
		log.Infof("The HTTP query request failed with error %s\n", err)
	} else {
		data, _ := ioutil.ReadAll(response.Body)
		log.Info(string(data))

		// unmarshal the JSON response into a struct (generated using the JSON, using this https://mholt.github.io/json-to-go/
		var fv RangeMetricReponseFromPrometheus
		err0 := json.Unmarshal(data, &fv)
		if err0 != nil {
			log.Infof("unmarshell operation failed ", err0)
		}
		var metrics []*model.MetricSpec
		// now convert to our repsonse struct, so we can marshal it and send out the JSON
		for _, res := range fv.Data.Result {

			metricValues := make([]*model.Metric, 0)
			metricValue := &model.Metric{}
			for j := 0; j < len(res.Values); j++ {
				for _, v := range res.Values[j] {
					switch v.(type) {
					case string:
						metricValue.Value, _ = strconv.ParseFloat(v.(string), 64)
					case float64:
						secs := int64(v.(float64))
						metricValue.Timestamp = secs
					default:
						log.Infof("%s is of a type I don't know how to handle", v)
					}

				}
				metricValues = append(metricValues, metricValue)
				metric := &model.MetricSpec{}
				metric.InstanceID = res.Metric.Instance
				metric.Name = res.Metric.Name
				metric.InstanceName = res.Metric.Device
				metric.MetricValues = metricValues
				metrics = append(metrics, metric)
			}
		}

		bArr, _ := json.Marshal(metrics)
		log.Infof("metrics response json is %s", string(bArr))

		if err != nil {
			log.Error(err)
		}
		return metrics, err

	}
	//no response
	return nil, err
}

func (c *controller) SetDock(dockInfo *model.DockSpec) {
	c.DockInfo = dockInfo
}
<commit_msg>correcting formatting error<commit_after>// Copyright (c) 2019 The OpenSDS Authors All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
This module implements a entry into the OpenSDS metrics controller service.

*/

package metrics

import (
	"encoding/json"
	log "github.com/golang/glog"
	"github.com/opensds/opensds/pkg/dock/client"
	"github.com/opensds/opensds/pkg/model"
	pb "github.com/opensds/opensds/pkg/model/proto"
	"io/ioutil"
	"net/http"
	"strconv"
)

// Controller is an interface for exposing some operations of metric controllers.
type Controller interface {
	GetLatestMetrics(opt *pb.GetMetricsOpts) ([]*model.MetricSpec, error)
	GetInstantMetrics(opt *pb.GetMetricsOpts) ([]*model.MetricSpec, error)
	GetRangeMetrics(opt *pb.GetMetricsOpts) ([]*model.MetricSpec, error)
	SetDock(dockInfo *model.DockSpec)
}

// NewController method creates a controller structure and expose its pointer.
func NewController() Controller {
	return &controller{
		Client: client.NewClient(),
	}
}

type controller struct {
	client.Client
	DockInfo *model.DockSpec
}

// latest+instant metrics structs begin
type InstantMetricReponseFromPrometheus struct {
	Status string `json:"status"`
	Data   Data   `json:"data"`
}
type Metric struct {
	Name       string `json:"__name__"`
	Device     string `json:"device"`
	InstanceID string `json:"instanceID"`
	Job        string `json:"job"`
}
type Result struct {
	Metric Metric        `json:"metric"`
	Value  []interface{} `json:"value"`
}
type Data struct {
	ResultType string   `json:"resultType"`
	Result     []Result `json:"result"`
}

// latest+instant metrics structs end

// latest+range metrics structs begin
type RangeMetricReponseFromPrometheus struct {
	Status string    `json:"status"`
	Data   RangeData `json:"data"`
}
type RangeMetric struct {
	Name     string `json:"__name__"`
	Device   string `json:"device"`
	Instance string `json:"instance"`
	Job      string `json:"job"`
}
type RangeResult struct {
	Metric RangeMetric     `json:"metric"`
	Values [][]interface{} `json:"values"`
}
type RangeData struct {
	ResultType string        `json:"resultType"`
	Result     []RangeResult `json:"result"`
}

// latest+range metrics structs end

func (c *controller) GetLatestMetrics(opt *pb.GetMetricsOpts) ([]*model.MetricSpec, error) {

	// make a call to Prometheus, convert the response to our format, return
	response, err := http.Get("http://localhost:9090/api/v1/query?query=" + opt.MetricName)
	if err != nil {
		log.Infof("The HTTP query request failed with error %s\n", err)
	} else {
		data, _ := ioutil.ReadAll(response.Body)
		log.Infof("response data is %s", string(data))
		// unmarshal the JSON response into a struct (generated using the JSON, using this https://mholt.github.io/json-to-go/
		var fv InstantMetricReponseFromPrometheus
		err0 := json.Unmarshal(data, &fv)
		if err0 != nil {
			log.Infof("unmarshell operation failed %s\n", err0)
		}
		var metrics []*model.MetricSpec
		// now convert to our repsonse struct, so we can marshal it and send out the JSON
		for _, res := range fv.Data.Result {

			metricValues := make([]*model.Metric, 0)
			metricValue := &model.Metric{}
			for _, v := range res.Value {

				switch v.(type) {
				case string:
					metricValue.Value, err = strconv.ParseFloat(v.(string), 64)
				case float64:
					secs := int64(v.(float64))
					metricValue.Timestamp = secs
				default:
					log.Info(v, "is of a type I don't know how to handle")
				}

			}
			metricValues = append(metricValues, metricValue)
			metric := &model.MetricSpec{}
			metric.InstanceID = res.Metric.InstanceID
			metric.Name = res.Metric.Name
			metric.InstanceName = res.Metric.Device
			metric.MetricValues = metricValues
			metrics = append(metrics, metric)
		}

		bArr, _ := json.Marshal(metrics)
		log.Infof("metrics response json is %s", string(bArr))

		if err != nil {
			log.Error(err)
		}
		return metrics, err

	}
	//no response
	return nil, err
}

func (c *controller) GetInstantMetrics(opt *pb.GetMetricsOpts) ([]*model.MetricSpec, error) {

	// make a call to Prometheus, convert the response to our format, return
	response, err := http.Get("http://localhost:9090/api/v1/query?query=" + opt.MetricName + "&time=" + opt.StartTime)
	if err != nil {
		log.Infof("The HTTP query request failed with error %s\n", err)
	} else {
		data, _ := ioutil.ReadAll(response.Body)
		log.Infof("response data is %s", string(data))
		// unmarshal the JSON response into a struct (generated using the JSON, using this https://mholt.github.io/json-to-go/
		var fv InstantMetricReponseFromPrometheus
		err0 := json.Unmarshal(data, &fv)
		if err0 != nil {
			log.Infof("unmarshell operation failed %s\n", err0)
		}
		var metrics []*model.MetricSpec
		// now convert to our repsonse struct, so we can marshal it and send out the JSON
		for _, res := range fv.Data.Result {

			metricValues := make([]*model.Metric, 0)
			metricValue := &model.Metric{}
			for _, v := range res.Value {

				switch v.(type) {
				case string:
					metricValue.Value, err = strconv.ParseFloat(v.(string), 64)
				case float64:
					secs := int64(v.(float64))
					metricValue.Timestamp = secs
				default:
					log.Info(v, "is of a type I don't know how to handle")
				}

			}
			metricValues = append(metricValues, metricValue)
			metric := &model.MetricSpec{}
			metric.InstanceID = res.Metric.InstanceID
			metric.Name = res.Metric.Name
			metric.InstanceName = res.Metric.Device
			metric.MetricValues = metricValues
			metrics = append(metrics, metric)
		}

		bArr, _ := json.Marshal(metrics)
		log.Infof("metrics response json is %s", string(bArr))

		if err != nil {
			log.Error(err)
		}
		return metrics, err

	}
	//no response
	return nil, err
}

func (c *controller) GetRangeMetrics(opt *pb.GetMetricsOpts) ([]*model.MetricSpec, error) {

	//var metrics []model.MetricSpec
	// make a call to Prometheus, convert the response to our format, return
	response, err := http.Get("http://localhost:9090/api/v1/query_range?query=" + opt.MetricName + "&start=" + opt.StartTime + "&end=" + opt.EndTime + "&step=30")
	if err != nil {
		log.Infof("The HTTP query request failed with error %s\n", err)
	} else {
		data, _ := ioutil.ReadAll(response.Body)
		log.Info(string(data))

		// unmarshal the JSON response into a struct (generated using the JSON, using this https://mholt.github.io/json-to-go/
		var fv RangeMetricReponseFromPrometheus
		err0 := json.Unmarshal(data, &fv)
		if err0 != nil {
			log.Infof("unmarshell operation failed %s\n", err0)
		}
		var metrics []*model.MetricSpec
		// now convert to our repsonse struct, so we can marshal it and send out the JSON
		for _, res := range fv.Data.Result {

			metricValues := make([]*model.Metric, 0)
			metricValue := &model.Metric{}
			for j := 0; j < len(res.Values); j++ {
				for _, v := range res.Values[j] {
					switch v.(type) {
					case string:
						metricValue.Value, _ = strconv.ParseFloat(v.(string), 64)
					case float64:
						secs := int64(v.(float64))
						metricValue.Timestamp = secs
					default:
						log.Infof("%s is of a type I don't know how to handle", v)
					}

				}
				metricValues = append(metricValues, metricValue)
				metric := &model.MetricSpec{}
				metric.InstanceID = res.Metric.Instance
				metric.Name = res.Metric.Name
				metric.InstanceName = res.Metric.Device
				metric.MetricValues = metricValues
				metrics = append(metrics, metric)
			}
		}

		bArr, _ := json.Marshal(metrics)
		log.Infof("metrics response json is %s", string(bArr))

		if err != nil {
			log.Error(err)
		}
		return metrics, err

	}
	//no response
	return nil, err
}

func (c *controller) SetDock(dockInfo *model.DockSpec) {
	c.DockInfo = dockInfo
}
<|endoftext|>


<commit_before>package queryrange

import (
	"context"
	"time"

	"github.com/cortexproject/cortex/pkg/querier/queryrange"
	"github.com/grafana/loki/pkg/logproto"
	"github.com/opentracing/opentracing-go"
	otlog "github.com/opentracing/opentracing-go/log"
	"github.com/weaveworks/common/user"
)

// SplitByIntervalMiddleware creates a new Middleware that splits log requests by a given interval.
func SplitByIntervalMiddleware(interval time.Duration, limits queryrange.Limits, merger queryrange.Merger) queryrange.Middleware {
	return queryrange.MiddlewareFunc(func(next queryrange.Handler) queryrange.Handler {
		return &splitByInterval{
			next:     next,
			limits:   limits,
			merger:   merger,
			interval: interval,
		}
	})
}

type lokiResult struct {
	req  queryrange.Request
	resp chan queryrange.Response
	err  chan error
}

type splitByInterval struct {
	next     queryrange.Handler
	limits   queryrange.Limits
	merger   queryrange.Merger
	interval time.Duration
}

func (h *splitByInterval) Feed(ctx context.Context, input []*lokiResult) chan *lokiResult {
	ch := make(chan *lokiResult)

	go func() {
		defer close(ch)
		for _, d := range input {
			select {
			case <-ctx.Done():
				return
			case ch <- d:
				continue
			}
		}
	}()

	return ch
}

func (h *splitByInterval) Process(
	ctx context.Context,
	parallelism int,
	threshold int64,
	input []*lokiResult,
) (responses []queryrange.Response, err error) {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	ch := h.Feed(ctx, input)

	// don't spawn unnecessary goroutines
	var p int = parallelism
	if len(input) < parallelism {
		p = len(input)
	}

	for i := 0; i < p; i++ {
		go h.loop(ctx, ch)
	}

	for _, x := range input {
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		case err := <-x.err:
			return nil, err
		case resp := <-x.resp:

			responses = append(responses, resp)

			// see if we can exit early if a limit has been reached
			threshold -= resp.(*LokiResponse).Count()
			if threshold <= 0 {
				return responses, nil
			}
		}

	}

	return responses, nil
}

func (h *splitByInterval) loop(ctx context.Context, ch <-chan *lokiResult) {

	for data := range ch {

		sp, ctx := opentracing.StartSpanFromContext(ctx, "interval")
		queryrange.LogToSpan(ctx, data.req)

		resp, err := h.next.Do(ctx, data.req)
		if err != nil {
			data.err <- err
		} else {
			data.resp <- resp
		}
		sp.Finish()
	}
}

func (h *splitByInterval) Do(ctx context.Context, r queryrange.Request) (queryrange.Response, error) {
	lokiRequest := r.(*LokiRequest)

	userid, err := user.ExtractOrgID(ctx)
	if err != nil {
		return nil, err
	}

	intervals := splitByTime(lokiRequest, h.interval)

	if sp := opentracing.SpanFromContext(ctx); sp != nil {
		sp.LogFields(otlog.Int("n_intervals", len(intervals)))

	}

	if lokiRequest.Direction == logproto.BACKWARD {
		for i, j := 0, len(intervals)-1; i < j; i, j = i+1, j-1 {
			intervals[i], intervals[j] = intervals[j], intervals[i]
		}
	}

	input := make([]*lokiResult, 0, len(intervals))
	for _, interval := range intervals {
		input = append(input, &lokiResult{
			req:  interval,
			resp: make(chan queryrange.Response, 1),
			err:  make(chan error, 1),
		})
	}

	resps, err := h.Process(ctx, h.limits.MaxQueryParallelism(userid), int64(lokiRequest.Limit), input)
	if err != nil {
		return nil, err
	}

	return h.merger.MergeResponse(resps...)
}

func splitByTime(r *LokiRequest, interval time.Duration) []queryrange.Request {
	var reqs []queryrange.Request
	for start := r.StartTs; start.Before(r.EndTs); start = start.Add(interval) {
		end := start.Add(interval)
		if end.After(r.EndTs) {
			end = r.EndTs
		}
		reqs = append(reqs, &LokiRequest{
			Query:     r.Query,
			Limit:     r.Limit,
			Step:      r.Step,
			Direction: r.Direction,
			Path:      r.Path,
			StartTs:   start,
			EndTs:     end,
		})
	}
	return reqs
}
<commit_msg>refactors splitby to not require buffered channels (#1569)<commit_after>package queryrange

import (
	"context"
	"time"

	"github.com/cortexproject/cortex/pkg/querier/queryrange"
	"github.com/grafana/loki/pkg/logproto"
	"github.com/opentracing/opentracing-go"
	otlog "github.com/opentracing/opentracing-go/log"
	"github.com/weaveworks/common/user"
)

// SplitByIntervalMiddleware creates a new Middleware that splits log requests by a given interval.
func SplitByIntervalMiddleware(interval time.Duration, limits queryrange.Limits, merger queryrange.Merger) queryrange.Middleware {
	return queryrange.MiddlewareFunc(func(next queryrange.Handler) queryrange.Handler {
		return &splitByInterval{
			next:     next,
			limits:   limits,
			merger:   merger,
			interval: interval,
		}
	})
}

type lokiResult struct {
	req queryrange.Request
	ch  chan *packedResp
}

type packedResp struct {
	resp queryrange.Response
	err  error
}

type splitByInterval struct {
	next     queryrange.Handler
	limits   queryrange.Limits
	merger   queryrange.Merger
	interval time.Duration
}

func (h *splitByInterval) Feed(ctx context.Context, input []*lokiResult) chan *lokiResult {
	ch := make(chan *lokiResult)

	go func() {
		defer close(ch)
		for _, d := range input {
			select {
			case <-ctx.Done():
				return
			case ch <- d:
				continue
			}
		}
	}()

	return ch
}

func (h *splitByInterval) Process(
	ctx context.Context,
	parallelism int,
	threshold int64,
	input []*lokiResult,
) (responses []queryrange.Response, err error) {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	ch := h.Feed(ctx, input)

	// don't spawn unnecessary goroutines
	var p int = parallelism
	if len(input) < parallelism {
		p = len(input)
	}

	for i := 0; i < p; i++ {
		go h.loop(ctx, ch)
	}

	for _, x := range input {
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		case data := <-x.ch:
			if data.err != nil {
				return nil, err
			}

			responses = append(responses, data.resp)

			// see if we can exit early if a limit has been reached
			threshold -= data.resp.(*LokiResponse).Count()
			if threshold <= 0 {
				return responses, nil
			}
		}

	}

	return responses, nil
}

func (h *splitByInterval) loop(ctx context.Context, ch <-chan *lokiResult) {

	for data := range ch {

		sp, ctx := opentracing.StartSpanFromContext(ctx, "interval")
		queryrange.LogToSpan(ctx, data.req)

		resp, err := h.next.Do(ctx, data.req)

		select {
		case <-ctx.Done():
			sp.Finish()
			return
		case data.ch <- &packedResp{resp, err}:
			sp.Finish()
		}
	}
}

func (h *splitByInterval) Do(ctx context.Context, r queryrange.Request) (queryrange.Response, error) {
	lokiRequest := r.(*LokiRequest)

	userid, err := user.ExtractOrgID(ctx)
	if err != nil {
		return nil, err
	}

	intervals := splitByTime(lokiRequest, h.interval)

	if sp := opentracing.SpanFromContext(ctx); sp != nil {
		sp.LogFields(otlog.Int("n_intervals", len(intervals)))

	}

	if lokiRequest.Direction == logproto.BACKWARD {
		for i, j := 0, len(intervals)-1; i < j; i, j = i+1, j-1 {
			intervals[i], intervals[j] = intervals[j], intervals[i]
		}
	}

	input := make([]*lokiResult, 0, len(intervals))
	for _, interval := range intervals {
		input = append(input, &lokiResult{
			req: interval,
			ch:  make(chan *packedResp),
		})
	}

	resps, err := h.Process(ctx, h.limits.MaxQueryParallelism(userid), int64(lokiRequest.Limit), input)
	if err != nil {
		return nil, err
	}

	return h.merger.MergeResponse(resps...)
}

func splitByTime(r *LokiRequest, interval time.Duration) []queryrange.Request {
	var reqs []queryrange.Request
	for start := r.StartTs; start.Before(r.EndTs); start = start.Add(interval) {
		end := start.Add(interval)
		if end.After(r.EndTs) {
			end = r.EndTs
		}
		reqs = append(reqs, &LokiRequest{
			Query:     r.Query,
			Limit:     r.Limit,
			Step:      r.Step,
			Direction: r.Direction,
			Path:      r.Path,
			StartTs:   start,
			EndTs:     end,
		})
	}
	return reqs
}
<|endoftext|>