<commit_before>package alicloud

import (
	"fmt"
	"github.com/denverdino/aliyungo/slb"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"log"
	"testing"
)

func TestAccAlicloudSlbAttachment_basic(t *testing.T) {
	var slb slb.LoadBalancerType

	testCheckAttr := func() resource.TestCheckFunc {
		return func(*terraform.State) error {
			log.Printf("testCheckAttr slb BackendServers is: %s", slb.BackendServers)
			return nil
		}
	}

	resource.Test(t, resource.TestCase{
		PreCheck: func() {
			testAccPreCheck(t)
		},

		// module name
		IDRefreshName: "alicloud_slb_attachment.foo",
		Providers:     testAccProviders,
		CheckDestroy:  testAccCheckSlbDestroy,
		Steps: []resource.TestStep{
			//test internet_charge_type is paybybandwidth
			resource.TestStep{
				Config: testAccSlbAttachment,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckSlbExists("alicloud_slb_attachment.foo", &slb),
					testCheckAttr(),
					testAccCheckAttachment("alicloud_instance.foo", &slb),
				),
			},
		},
	})
}

func testAccCheckAttachment(n string, slb *slb.LoadBalancerType) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No ECS ID is set")
		}

		ecsInstanceId := rs.Primary.ID

		backendServers := slb.BackendServers.BackendServer

		if len(backendServers) == 0 {
			return fmt.Errorf("no SLB backendServer: %#v", backendServers)
		}

		log.Printf("slb bacnendservers: %#v", backendServers)

		backendServersInstanceId := backendServers[0].ServerId

		if ecsInstanceId != backendServersInstanceId {
			return fmt.Errorf("SLB attachment check invalid: ECS instance %s is not equal SLB backendServer %s",
				ecsInstanceId, backendServersInstanceId)
		}
		return nil
	}
}

const testAccSlbAttachment = `
resource "alicloud_security_group" "foo" {
	name = "tf_test_foo"
	description = "foo"
}

resource "alicloud_instance" "foo" {
	# cn-beijing
	availability_zone = "cn-beijing-b"
	image_id = "ubuntu_140405_64_40G_cloudinit_20161115.vhd"

	# series II
	instance_type = "ecs.n1.medium"
	internet_charge_type = "PayByBandwidth"
	internet_max_bandwidth_out = "5"
	system_disk_category = "cloud_efficiency"

	security_groups = ["${alicloud_security_group.foo.id}"]
	instance_name = "test_foo"
}

resource "alicloud_slb" "foo" {
	name = "tf_test_slb_bind"
	internet_charge_type = "paybybandwidth"
	bandwidth = "5"
	internet = "true"
}

resource "alicloud_slb_attachment" "foo" {
	slb_id = "${alicloud_slb.foo.id}"
	instances = ["${alicloud_instance.foo.id}"]
}

`
<commit_msg>fix print log format bug<commit_after>package alicloud

import (
	"fmt"
	"github.com/denverdino/aliyungo/slb"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"log"
	"testing"
)

func TestAccAlicloudSlbAttachment_basic(t *testing.T) {
	var slb slb.LoadBalancerType

	testCheckAttr := func() resource.TestCheckFunc {
		return func(*terraform.State) error {
			log.Printf("testCheckAttr slb BackendServers is: %#v", slb.BackendServers)
			return nil
		}
	}

	resource.Test(t, resource.TestCase{
		PreCheck: func() {
			testAccPreCheck(t)
		},

		// module name
		IDRefreshName: "alicloud_slb_attachment.foo",
		Providers:     testAccProviders,
		CheckDestroy:  testAccCheckSlbDestroy,
		Steps: []resource.TestStep{
			//test internet_charge_type is paybybandwidth
			resource.TestStep{
				Config: testAccSlbAttachment,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckSlbExists("alicloud_slb_attachment.foo", &slb),
					testCheckAttr(),
					testAccCheckAttachment("alicloud_instance.foo", &slb),
				),
			},
		},
	})
}

func testAccCheckAttachment(n string, slb *slb.LoadBalancerType) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No ECS ID is set")
		}

		ecsInstanceId := rs.Primary.ID

		backendServers := slb.BackendServers.BackendServer

		if len(backendServers) == 0 {
			return fmt.Errorf("no SLB backendServer: %#v", backendServers)
		}

		log.Printf("slb bacnendservers: %#v", backendServers)

		backendServersInstanceId := backendServers[0].ServerId

		if ecsInstanceId != backendServersInstanceId {
			return fmt.Errorf("SLB attachment check invalid: ECS instance %s is not equal SLB backendServer %s",
				ecsInstanceId, backendServersInstanceId)
		}
		return nil
	}
}

const testAccSlbAttachment = `
resource "alicloud_security_group" "foo" {
	name = "tf_test_foo"
	description = "foo"
}

resource "alicloud_instance" "foo" {
	# cn-beijing
	availability_zone = "cn-beijing-b"
	image_id = "ubuntu_140405_64_40G_cloudinit_20161115.vhd"

	# series II
	instance_type = "ecs.n1.medium"
	internet_charge_type = "PayByBandwidth"
	internet_max_bandwidth_out = "5"
	system_disk_category = "cloud_efficiency"

	security_groups = ["${alicloud_security_group.foo.id}"]
	instance_name = "test_foo"
}

resource "alicloud_slb" "foo" {
	name = "tf_test_slb_bind"
	internet_charge_type = "paybybandwidth"
	bandwidth = "5"
	internet = "true"
}

resource "alicloud_slb_attachment" "foo" {
	slb_id = "${alicloud_slb.foo.id}"
	instances = ["${alicloud_instance.foo.id}"]
}

`
<|endoftext|>


<commit_before>package http

import (
	"encoding/json"
	"io"
)

// BasciAuther is the interface a system must support to return basic auth information.
type BasicAuther interface {
	BasicAuth() (string, string, bool)
}

// Credential represents authentication and authorization configuration for a single user.
type Credential struct {
	Username string   `json:"username,omitempty"`
	Password string   `json:"password,omitempty"`
	Perms    []string `json:"perms",omitempty"`
}

// CredentialStore stores authentication and authorization information for all users.
type CredentialsStore struct {
	store map[string]string
	perms map[string]map[string]bool
}

// NewCredentialsStore returns a new instance of a CredentialStore.
func NewCredentialsStore() *CredentialsStore {
	return &CredentialsStore{
		store: make(map[string]string),
		perms: make(map[string]map[string]bool),
	}
}

// Load loods credentials information from a reader.
func (c *CredentialsStore) Load(r io.Reader) error {
	dec := json.NewDecoder(r)
	// Read open bracket
	_, err := dec.Token()
	if err != nil {
		return err
	}

	var cred Credential
	for dec.More() {
		err := dec.Decode(&cred)
		if err != nil {
			return err
		}
		c.store[cred.Username] = cred.Password
		c.perms[cred.Username] = make(map[string]bool, len(cred.Perms))
		for _, p := range cred.Perms {
			c.perms[cred.Username][p] = true
		}
	}

	// Read closing bracket
	_, err = dec.Token()
	if err != nil {
		return err
	}

	return nil
}

// Check returns true if the password is correct for the given username.
func (c *CredentialsStore) Check(username, password string) bool {
	pw, ok := c.store[username]
	return ok && password == pw
}

// CheckRequest returns true if b contains a valid username and password.
func (c *CredentialsStore) CheckRequest(b BasicAuther) bool {
	username, password, ok := b.BasicAuth()
	if !ok || !c.Check(username, password) {
		return false
	}
	return true
}

// HasPerm returns true if username has the given perm. It does not
// perform any password checking.
func (c *CredentialsStore) HasPerm(username string, perm string) bool {
	m, ok := c.perms[username]
	if !ok {
		return false
	}

	if _, ok := m[perm]; !ok {
		return false
	}
	return true
}

// HasPermRequest returns true if username returned by b has the givem perm.
// It does not perform any password checking.
func (c *CredentialsStore) HasPermRequest(b BasicAuther, perm string) bool {
	username, _, ok := b.BasicAuth()
	if !ok {
		return false
	}
	return c.HasPerm(username, perm)
}
<commit_msg>'go vet' fixes<commit_after>package http

import (
	"encoding/json"
	"io"
)

// BasciAuther is the interface a system must support to return basic auth information.
type BasicAuther interface {
	BasicAuth() (string, string, bool)
}

// Credential represents authentication and authorization configuration for a single user.
type Credential struct {
	Username string   `json:"username,omitempty"`
	Password string   `json:"password,omitempty"`
	Perms    []string `json:"perms,omitempty"`
}

// CredentialStore stores authentication and authorization information for all users.
type CredentialsStore struct {
	store map[string]string
	perms map[string]map[string]bool
}

// NewCredentialsStore returns a new instance of a CredentialStore.
func NewCredentialsStore() *CredentialsStore {
	return &CredentialsStore{
		store: make(map[string]string),
		perms: make(map[string]map[string]bool),
	}
}

// Load loods credentials information from a reader.
func (c *CredentialsStore) Load(r io.Reader) error {
	dec := json.NewDecoder(r)
	// Read open bracket
	_, err := dec.Token()
	if err != nil {
		return err
	}

	var cred Credential
	for dec.More() {
		err := dec.Decode(&cred)
		if err != nil {
			return err
		}
		c.store[cred.Username] = cred.Password
		c.perms[cred.Username] = make(map[string]bool, len(cred.Perms))
		for _, p := range cred.Perms {
			c.perms[cred.Username][p] = true
		}
	}

	// Read closing bracket
	_, err = dec.Token()
	if err != nil {
		return err
	}

	return nil
}

// Check returns true if the password is correct for the given username.
func (c *CredentialsStore) Check(username, password string) bool {
	pw, ok := c.store[username]
	return ok && password == pw
}

// CheckRequest returns true if b contains a valid username and password.
func (c *CredentialsStore) CheckRequest(b BasicAuther) bool {
	username, password, ok := b.BasicAuth()
	if !ok || !c.Check(username, password) {
		return false
	}
	return true
}

// HasPerm returns true if username has the given perm. It does not
// perform any password checking.
func (c *CredentialsStore) HasPerm(username string, perm string) bool {
	m, ok := c.perms[username]
	if !ok {
		return false
	}

	if _, ok := m[perm]; !ok {
		return false
	}
	return true
}

// HasPermRequest returns true if username returned by b has the givem perm.
// It does not perform any password checking.
func (c *CredentialsStore) HasPermRequest(b BasicAuther, perm string) bool {
	username, _, ok := b.BasicAuth()
	if !ok {
		return false
	}
	return c.HasPerm(username, perm)
}
<|endoftext|>


<commit_before>package httptoo

import (
	"bufio"
	"encoding/gob"
	"io"
	"net"
	"net/http"
	"net/url"
	"sync"
)

func deepCopy(dst, src interface{}) error {
	r, w := io.Pipe()
	e := gob.NewEncoder(w)
	d := gob.NewDecoder(r)
	var decErr, encErr error
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		decErr = d.Decode(dst)
		r.Close()
	}()
	encErr = e.Encode(src)
	// Always returns nil.
	w.CloseWithError(encErr)
	wg.Wait()
	if encErr != nil {
		return encErr
	}
	return decErr
}

// Takes a request, and alters its destination fields, for proxying.
func RedirectedRequest(r *http.Request, newUrl string) (ret *http.Request, err error) {
	u, err := url.Parse(newUrl)
	if err != nil {
		return
	}
	ret = new(http.Request)
	*ret = *r
	ret.Header = nil
	err = deepCopy(&ret.Header, r.Header)
	if err != nil {
		return
	}
	ret.URL = u
	ret.RequestURI = ""
	return
}

func ForwardResponse(w http.ResponseWriter, r *http.Response) {
	for h, vs := range r.Header {
		for _, v := range vs {
			w.Header().Add(h, v)
		}
	}
	w.WriteHeader(r.StatusCode)
	// Errors frequently occur writing the body when the client hangs up.
	io.Copy(w, r.Body)
	r.Body.Close()
}

func SetOriginRequestForwardingHeaders(o, f *http.Request) {
	xff := o.Header.Get("X-Forwarded-For")
	hop, _, _ := net.SplitHostPort(f.RemoteAddr)
	if xff == "" {
		xff = hop
	} else {
		xff += "," + hop
	}
	o.Header.Set("X-Forwarded-For", xff)
	o.Header.Set("X-Forwarded-Proto", OriginatingProtocol(f))
}

// w is for the client response. r is the request to send to the origin
// (already "forwarded"). originUrl is where to send the request.
func ReverseProxyUpgrade(w http.ResponseWriter, r *http.Request, originUrl string) (err error) {
	u, err := url.Parse(originUrl)
	if err != nil {
		return
	}
	oc, err := net.Dial("tcp", u.Host)
	if err != nil {
		return
	}
	defer oc.Close()
	err = r.Write(oc)
	if err != nil {
		return
	}
	originConnReadBuffer := bufio.NewReader(oc)
	originResp, err := http.ReadResponse(originConnReadBuffer, r)
	if err != nil {
		return
	}
	if originResp.StatusCode != 101 {
		ForwardResponse(w, originResp)
		return
	}
	cc, _, err := w.(http.Hijacker).Hijack()
	if err != nil {
		return
	}
	defer cc.Close()
	originResp.Write(cc)
	go io.Copy(oc, cc)
	// Let the origin connection control when this routine returns, as we
	// should trust it more.
	io.Copy(cc, originConnReadBuffer)
	return
}

func ReverseProxy(w http.ResponseWriter, r *http.Request, originUrl string, client *http.Client) (err error) {
	originRequest, err := RedirectedRequest(r, originUrl)
	if err != nil {
		return
	}
	SetOriginRequestForwardingHeaders(originRequest, r)
	if r.Header.Get("Connection") == "Upgrade" {
		return ReverseProxyUpgrade(w, originRequest, originUrl)
	}
	originResp, err := client.Do(originRequest)
	if err != nil {
		return
	}
	ForwardResponse(w, originResp)
	return
}
<commit_msg>ReverseProxy using the Transport directly<commit_after>package httptoo

import (
	"bufio"
	"encoding/gob"
	"io"
	"net"
	"net/http"
	"net/url"
	"sync"
)

func deepCopy(dst, src interface{}) error {
	r, w := io.Pipe()
	e := gob.NewEncoder(w)
	d := gob.NewDecoder(r)
	var decErr, encErr error
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		decErr = d.Decode(dst)
		r.Close()
	}()
	encErr = e.Encode(src)
	// Always returns nil.
	w.CloseWithError(encErr)
	wg.Wait()
	if encErr != nil {
		return encErr
	}
	return decErr
}

// Takes a request, and alters its destination fields, for proxying.
func RedirectedRequest(r *http.Request, newUrl string) (ret *http.Request, err error) {
	u, err := url.Parse(newUrl)
	if err != nil {
		return
	}
	ret = new(http.Request)
	*ret = *r
	ret.Header = nil
	err = deepCopy(&ret.Header, r.Header)
	if err != nil {
		return
	}
	ret.URL = u
	ret.RequestURI = ""
	return
}

func ForwardResponse(w http.ResponseWriter, r *http.Response) {
	for h, vs := range r.Header {
		for _, v := range vs {
			w.Header().Add(h, v)
		}
	}
	w.WriteHeader(r.StatusCode)
	// Errors frequently occur writing the body when the client hangs up.
	io.Copy(w, r.Body)
	r.Body.Close()
}

func SetOriginRequestForwardingHeaders(o, f *http.Request) {
	xff := o.Header.Get("X-Forwarded-For")
	hop, _, _ := net.SplitHostPort(f.RemoteAddr)
	if xff == "" {
		xff = hop
	} else {
		xff += "," + hop
	}
	o.Header.Set("X-Forwarded-For", xff)
	o.Header.Set("X-Forwarded-Proto", OriginatingProtocol(f))
}

// w is for the client response. r is the request to send to the origin
// (already "forwarded"). originUrl is where to send the request.
func ReverseProxyUpgrade(w http.ResponseWriter, r *http.Request, originUrl string) (err error) {
	u, err := url.Parse(originUrl)
	if err != nil {
		return
	}
	oc, err := net.Dial("tcp", u.Host)
	if err != nil {
		return
	}
	defer oc.Close()
	err = r.Write(oc)
	if err != nil {
		return
	}
	originConnReadBuffer := bufio.NewReader(oc)
	originResp, err := http.ReadResponse(originConnReadBuffer, r)
	if err != nil {
		return
	}
	if originResp.StatusCode != 101 {
		ForwardResponse(w, originResp)
		return
	}
	cc, _, err := w.(http.Hijacker).Hijack()
	if err != nil {
		return
	}
	defer cc.Close()
	originResp.Write(cc)
	go io.Copy(oc, cc)
	// Let the origin connection control when this routine returns, as we
	// should trust it more.
	io.Copy(cc, originConnReadBuffer)
	return
}

func ReverseProxy(w http.ResponseWriter, r *http.Request, originUrl string, client *http.Client) (err error) {
	originRequest, err := RedirectedRequest(r, originUrl)
	if err != nil {
		return
	}
	SetOriginRequestForwardingHeaders(originRequest, r)
	if r.Header.Get("Connection") == "Upgrade" {
		return ReverseProxyUpgrade(w, originRequest, originUrl)
	}
	rt := client.Transport
	if rt == nil {
		rt = http.DefaultTransport
	}
	originResp, err := rt.RoundTrip(originRequest)
	if err != nil {
		return
	}
	ForwardResponse(w, originResp)
	return
}
<|endoftext|>


<commit_before>package carbonserver

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	_ "net/http/pprof"
	"strings"
	"sync/atomic"
	"time"

	"go.uber.org/zap"

	"github.com/go-graphite/carbonzipper/zipper/httpHeaders"
	protov2 "github.com/go-graphite/protocol/carbonapi_v2_pb"
	protov3 "github.com/go-graphite/protocol/carbonapi_v3_pb"
	pickle "github.com/lomik/og-rek"
)

type findResponse struct {
	data        []byte
	contentType string
	files       int
}

func (listener *CarbonserverListener) findHandler(wr http.ResponseWriter, req *http.Request) {
	// URL: /metrics/find/?local=1&format=pickle&query=the.metric.path.with.glob

	t0 := time.Now()
	ctx := req.Context()

	atomic.AddUint64(&listener.metrics.FindRequests, 1)

	format := req.FormValue("format")
	query := req.Form["query"]

	var response *findResponse

	logger := TraceContextToZap(ctx, listener.logger.With(
		zap.String("handler", "find"),
		zap.String("url", req.URL.RequestURI()),
		zap.String("peer", req.RemoteAddr),
	))

	accessLogger := TraceContextToZap(ctx, listener.accessLogger.With(
		zap.String("handler", "find"),
		zap.String("url", req.URL.RequestURI()),
		zap.String("peer", req.RemoteAddr),
	))

	accepts := req.Header["Accept"]
	for _, accept := range accepts {
		if accept == httpHeaders.ContentTypeCarbonAPIv3PB {
			format = "carbonapi_v3_pb"
			break
		}
	}

	if format == "" {
		format = "json"
	}

	formatCode, ok := knownFormats[format]
	if !ok {
		atomic.AddUint64(&listener.metrics.FindErrors, 1)
		accessLogger.Error("find failed",
			zap.Duration("runtime_seconds", time.Since(t0)),
			zap.String("reason", "unsupported format"),
			zap.Int("http_code", http.StatusBadRequest),
		)
		http.Error(wr, "Bad request (unsupported format)",
			http.StatusBadRequest)
		return
	}

	if formatCode == protoV3Format {
		body, err := ioutil.ReadAll(req.Body)
		if err != nil {
			accessLogger.Error("find failed",
				zap.Duration("runtime_seconds", time.Since(t0)),
				zap.String("reason", err.Error()),
				zap.Int("http_code", http.StatusBadRequest),
			)
			http.Error(wr, "Bad request (unsupported format)",
				http.StatusBadRequest)
		}

		var pv3Request protov3.MultiGlobRequest
		pv3Request.Unmarshal(body)

		fmt.Printf("\n\n%+v\n\n", pv3Request)

		query = pv3Request.Metrics
	}

	logger = logger.With(
		zap.Strings("query", query),
		zap.String("format", format),
	)

	accessLogger = accessLogger.With(
		zap.Strings("query", query),
		zap.String("format", format),
	)

	if len(query) == 0 {
		atomic.AddUint64(&listener.metrics.FindErrors, 1)
		accessLogger.Error("find failed",
			zap.Duration("runtime_seconds", time.Since(t0)),
			zap.String("reason", "empty query"),
			zap.Int("http_code", http.StatusBadRequest),
		)
		http.Error(wr, "Bad request (no query)", http.StatusBadRequest)
		return
	}

	var err error
	fromCache := false
	if listener.findCacheEnabled {
		key := strings.Join(query, ",") + "&" + format
		size := uint64(100 * 1024 * 1024)
		item := listener.findCache.getQueryItem(key, size, 300)
		res, ok := item.FetchOrLock()
		if !ok {
			logger.Debug("find cache miss")
			atomic.AddUint64(&listener.metrics.FindCacheMiss, 1)
			response, err = listener.findMetrics(logger, t0, formatCode, query)
			if err != nil {
				item.StoreAbort()
			} else {
				item.StoreAndUnlock(response)
			}
		} else if res != nil {
			logger.Debug("query cache hit")
			atomic.AddUint64(&listener.metrics.FindCacheHit, 1)
			response = res.(*findResponse)
			fromCache = true
		}
	} else {
		response, err = listener.findMetrics(logger, t0, formatCode, query)
	}

	if response == nil {
		accessLogger.Error("find failed",
			zap.Duration("runtime_seconds", time.Since(t0)),
			zap.String("reason", "internal error while processing request"),
			zap.Error(err),
			zap.Int("http_code", http.StatusInternalServerError),
		)
		http.Error(wr, fmt.Sprintf("Internal error while processing request (%v)", err),
			http.StatusInternalServerError)
		return
	}

	wr.Header().Set("Content-Type", response.contentType)
	wr.Write(response.data)

	if response.files == 0 {
		// to get an idea how often we search for nothing
		atomic.AddUint64(&listener.metrics.FindZero, 1)
	}

	accessLogger.Info("find success",
		zap.Duration("runtime_seconds", time.Since(t0)),
		zap.Int("Files", response.files),
		zap.Bool("find_cache_enabled", listener.findCacheEnabled),
		zap.Bool("from_cache", fromCache),
		zap.Int("http_code", http.StatusOK),
	)
	return
}

var findMetricsNoMetricsError = fmt.Errorf("no metrics available for this query")

type findError struct {
	name string
	err  error
}

type globs struct {
	Name  string
	Files []string
	Leafs []bool
}

func (listener *CarbonserverListener) findMetrics(logger *zap.Logger, t0 time.Time, format responseFormat, names []string) (*findResponse, error) {
	var result findResponse
	var expandedGlobs []globs
	var errors []findError
	var err error
	metricsCount := uint64(0)
	for _, name := range names {

		glob := globs{
			Name: name,
		}
		glob.Files, glob.Leafs, err = listener.expandGlobs(name)
		if err != nil {
			errors = append(errors, findError{name: name, err: err})
			continue
		}

		expandedGlobs = append(expandedGlobs, glob)
	}

	if len(errors) > 0 {
		atomic.AddUint64(&listener.metrics.FindErrors, uint64(len(errors)))

		if len(errors) == len(names) {
			logger.Error("find failed",
				zap.Duration("runtime_seconds", time.Since(t0)),
				zap.String("reason", "can't expand globs"),
				zap.Any("errors", errors),
			)
			return nil, fmt.Errorf("find failed, can't expand globs")
		} else {
			logger.Warn("find partly failed",
				zap.Duration("runtime_seconds", time.Since(t0)),
				zap.String("reason", "can't expand globs for some metrics"),
				zap.Any("errors", errors),
			)
		}
	}

	atomic.AddUint64(&listener.metrics.MetricsFound, metricsCount)
	logger.Debug("expandGlobs result",
		zap.String("action", "expandGlobs"),
		zap.Strings("metrics", names),
		zap.String("format", format.String()),
		zap.Any("result", expandedGlobs),
	)

	switch format {
	case protoV3Format, jsonFormat:
		var err error
		multiResponse := protov3.MultiGlobResponse{}
		for _, glob := range expandedGlobs {
			result.files += len(glob.Files)
			response := protov3.GlobResponse{
				Name:    glob.Name,
				Matches: make([]protov3.GlobMatch, 0),
			}

			for i, p := range glob.Files {
				if glob.Leafs[i] {
					metricsCount++
				}
				response.Matches = append(response.Matches, protov3.GlobMatch{Path: p, IsLeaf: glob.Leafs[i]})
			}
			multiResponse.Metrics = append(multiResponse.Metrics, response)
		}

		logger.Debug("will send out response",
			zap.Any("response", multiResponse),
		)

		switch format {
		case jsonFormat:
			result.contentType = httpHeaders.ContentTypeJSON
			result.data, err = json.Marshal(multiResponse)
		case protoV3Format:
			result.contentType = httpHeaders.ContentTypeCarbonAPIv3PB
			result.data, err = multiResponse.Marshal()
		}

		if err != nil {
			atomic.AddUint64(&listener.metrics.FindErrors, 1)

			logger.Error("find failed",
				zap.Duration("runtime_seconds", time.Since(t0)),
				zap.String("reason", "response encode failed"),
				zap.Error(err),
			)
			return nil, err
		} else {
			if len(multiResponse.Metrics) == 0 {
				err = findMetricsNoMetricsError
			}
		}
		return &result, err
	case protoV2Format:
		result.contentType = httpHeaders.ContentTypeProtobuf
		var err error
		response := protov2.GlobResponse{
			Name:    names[0],
			Matches: make([]protov2.GlobMatch, 0),
		}

		result.files += len(expandedGlobs[0].Files)
		for i, p := range expandedGlobs[0].Files {
			if expandedGlobs[0].Leafs[i] {
				metricsCount++
			}
			response.Matches = append(response.Matches, protov2.GlobMatch{Path: p, IsLeaf: expandedGlobs[0].Leafs[i]})
		}
		result.data, err = response.Marshal()
		result.contentType = httpHeaders.ContentTypeProtobuf

		if err != nil {
			atomic.AddUint64(&listener.metrics.FindErrors, 1)

			logger.Error("find failed",
				zap.Duration("runtime_seconds", time.Since(t0)),
				zap.String("reason", "response encode failed"),
				zap.Error(err),
			)
			return nil, err
		} else {
			if len(response.Matches) == 0 {
				err = findMetricsNoMetricsError
			}
		}
		return &result, err
	case pickleFormat:
		// [{'metric_path': 'metric', 'intervals': [(x,y)], 'isLeaf': True},]
		var metrics []map[string]interface{}
		var m map[string]interface{}
		files := 0

		glob := expandedGlobs[0]
		files += len(glob.Files)
		for i, p := range glob.Files {
			if glob.Leafs[i] {
				metricsCount++
			}
			m = make(map[string]interface{})
			m["path"] = p
			m["is_leaf"] = glob.Leafs[i]

			metrics = append(metrics, m)
		}
		var buf bytes.Buffer
		pEnc := pickle.NewEncoder(&buf)
		pEnc.Encode(metrics)
		return &findResponse{buf.Bytes(), httpHeaders.ContentTypePickle, files}, nil
	}
	return nil, nil
}
<commit_msg>Reply 404 in /find<commit_after>package carbonserver

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	_ "net/http/pprof"
	"strings"
	"sync/atomic"
	"time"

	"go.uber.org/zap"

	"github.com/go-graphite/carbonzipper/zipper/httpHeaders"
	protov2 "github.com/go-graphite/protocol/carbonapi_v2_pb"
	protov3 "github.com/go-graphite/protocol/carbonapi_v3_pb"
	pickle "github.com/lomik/og-rek"
)

type findResponse struct {
	data        []byte
	contentType string
	files       int
}

func (listener *CarbonserverListener) findHandler(wr http.ResponseWriter, req *http.Request) {
	// URL: /metrics/find/?local=1&format=pickle&query=the.metric.path.with.glob

	t0 := time.Now()
	ctx := req.Context()

	atomic.AddUint64(&listener.metrics.FindRequests, 1)

	format := req.FormValue("format")
	query := req.Form["query"]

	var response *findResponse

	logger := TraceContextToZap(ctx, listener.logger.With(
		zap.String("handler", "find"),
		zap.String("url", req.URL.RequestURI()),
		zap.String("peer", req.RemoteAddr),
	))

	accessLogger := TraceContextToZap(ctx, listener.accessLogger.With(
		zap.String("handler", "find"),
		zap.String("url", req.URL.RequestURI()),
		zap.String("peer", req.RemoteAddr),
	))

	accepts := req.Header["Accept"]
	for _, accept := range accepts {
		if accept == httpHeaders.ContentTypeCarbonAPIv3PB {
			format = "carbonapi_v3_pb"
			break
		}
	}

	if format == "" {
		format = "json"
	}

	formatCode, ok := knownFormats[format]
	if !ok {
		atomic.AddUint64(&listener.metrics.FindErrors, 1)
		accessLogger.Error("find failed",
			zap.Duration("runtime_seconds", time.Since(t0)),
			zap.String("reason", "unsupported format"),
			zap.Int("http_code", http.StatusBadRequest),
		)
		http.Error(wr, "Bad request (unsupported format)",
			http.StatusBadRequest)
		return
	}

	if formatCode == protoV3Format {
		body, err := ioutil.ReadAll(req.Body)
		if err != nil {
			accessLogger.Error("find failed",
				zap.Duration("runtime_seconds", time.Since(t0)),
				zap.String("reason", err.Error()),
				zap.Int("http_code", http.StatusBadRequest),
			)
			http.Error(wr, "Bad request (unsupported format)",
				http.StatusBadRequest)
		}

		var pv3Request protov3.MultiGlobRequest
		pv3Request.Unmarshal(body)

		fmt.Printf("\n\n%+v\n\n", pv3Request)

		query = pv3Request.Metrics
	}

	logger = logger.With(
		zap.Strings("query", query),
		zap.String("format", format),
	)

	accessLogger = accessLogger.With(
		zap.Strings("query", query),
		zap.String("format", format),
	)

	if len(query) == 0 {
		atomic.AddUint64(&listener.metrics.FindErrors, 1)
		accessLogger.Error("find failed",
			zap.Duration("runtime_seconds", time.Since(t0)),
			zap.String("reason", "empty query"),
			zap.Int("http_code", http.StatusBadRequest),
		)
		http.Error(wr, "Bad request (no query)", http.StatusBadRequest)
		return
	}

	var err error
	fromCache := false
	if listener.findCacheEnabled {
		key := strings.Join(query, ",") + "&" + format
		size := uint64(100 * 1024 * 1024)
		item := listener.findCache.getQueryItem(key, size, 300)
		res, ok := item.FetchOrLock()
		if !ok {
			logger.Debug("find cache miss")
			atomic.AddUint64(&listener.metrics.FindCacheMiss, 1)
			response, err = listener.findMetrics(logger, t0, formatCode, query)
			if err != nil {
				item.StoreAbort()
			} else {
				item.StoreAndUnlock(response)
			}
		} else if res != nil {
			logger.Debug("query cache hit")
			atomic.AddUint64(&listener.metrics.FindCacheHit, 1)
			response = res.(*findResponse)
			fromCache = true
		}
	} else {
		response, err = listener.findMetrics(logger, t0, formatCode, query)
	}

	if err != nil || response == nil {
		var code int
		var reason string
		if _, ok := err.(errorNotFound); ok {
			reason = "Not Found"
			code = http.StatusNotFound
		} else {
			reason = "Internal error while processing request"
			code = http.StatusInternalServerError
		}

		accessLogger.Error("find failed",
			zap.Duration("runtime_seconds", time.Since(t0)),
			zap.String("reason", reason),
			zap.Error(err),
			zap.Int("http_code", code),
		)
		http.Error(wr, fmt.Sprintf("%s (%v)", reason, err), code)

		return
	}

	wr.Header().Set("Content-Type", response.contentType)
	wr.Write(response.data)

	if response.files == 0 {
		// to get an idea how often we search for nothing
		atomic.AddUint64(&listener.metrics.FindZero, 1)
	}

	accessLogger.Info("find success",
		zap.Duration("runtime_seconds", time.Since(t0)),
		zap.Int("Files", response.files),
		zap.Bool("find_cache_enabled", listener.findCacheEnabled),
		zap.Bool("from_cache", fromCache),
		zap.Int("http_code", http.StatusOK),
	)
	return
}

type errorNotFound struct{}

func (err errorNotFound) Error() string {
	return "Not Found"
}

type findError struct {
	name string
	err  error
}

type globs struct {
	Name  string
	Files []string
	Leafs []bool
}

func (listener *CarbonserverListener) findMetrics(logger *zap.Logger, t0 time.Time, format responseFormat, names []string) (*findResponse, error) {
	var result findResponse
	var expandedGlobs []globs
	var errors []findError
	var err error
	metricsCount := uint64(0)
	for _, name := range names {

		glob := globs{
			Name: name,
		}
		glob.Files, glob.Leafs, err = listener.expandGlobs(name)
		if err != nil {
			errors = append(errors, findError{name: name, err: err})
			continue
		}

		expandedGlobs = append(expandedGlobs, glob)
	}

	if len(errors) > 0 {
		atomic.AddUint64(&listener.metrics.FindErrors, uint64(len(errors)))

		if len(errors) == len(names) {
			logger.Error("find failed",
				zap.Duration("runtime_seconds", time.Since(t0)),
				zap.String("reason", "can't expand globs"),
				zap.Any("errors", errors),
			)
			return nil, fmt.Errorf("find failed, can't expand globs")
		} else {
			logger.Warn("find partly failed",
				zap.Duration("runtime_seconds", time.Since(t0)),
				zap.String("reason", "can't expand globs for some metrics"),
				zap.Any("errors", errors),
			)
		}
	}

	atomic.AddUint64(&listener.metrics.MetricsFound, metricsCount)
	logger.Debug("expandGlobs result",
		zap.String("action", "expandGlobs"),
		zap.Strings("metrics", names),
		zap.String("format", format.String()),
		zap.Any("result", expandedGlobs),
	)

	switch format {
	case protoV3Format, jsonFormat:
		var err error
		multiResponse := protov3.MultiGlobResponse{}
		for _, glob := range expandedGlobs {
			result.files += len(glob.Files)
			response := protov3.GlobResponse{
				Name:    glob.Name,
				Matches: make([]protov3.GlobMatch, 0),
			}

			for i, p := range glob.Files {
				if glob.Leafs[i] {
					metricsCount++
				}
				response.Matches = append(response.Matches, protov3.GlobMatch{Path: p, IsLeaf: glob.Leafs[i]})
			}
			multiResponse.Metrics = append(multiResponse.Metrics, response)
		}

		logger.Debug("will send out response",
			zap.Any("response", multiResponse),
		)

		switch format {
		case jsonFormat:
			result.contentType = httpHeaders.ContentTypeJSON
			result.data, err = json.Marshal(multiResponse)
		case protoV3Format:
			result.contentType = httpHeaders.ContentTypeCarbonAPIv3PB
			result.data, err = multiResponse.Marshal()
		}

		if err != nil {
			atomic.AddUint64(&listener.metrics.FindErrors, 1)

			logger.Error("find failed",
				zap.Duration("runtime_seconds", time.Since(t0)),
				zap.String("reason", "response encode failed"),
				zap.Error(err),
			)
			return nil, err
		}

		if len(multiResponse.Metrics) == 0 {
			return nil, errorNotFound{}
		}

		return &result, err

	case protoV2Format:
		result.contentType = httpHeaders.ContentTypeProtobuf
		var err error
		response := protov2.GlobResponse{
			Name:    names[0],
			Matches: make([]protov2.GlobMatch, 0),
		}

		result.files += len(expandedGlobs[0].Files)
		for i, p := range expandedGlobs[0].Files {
			if expandedGlobs[0].Leafs[i] {
				metricsCount++
			}
			response.Matches = append(response.Matches, protov2.GlobMatch{Path: p, IsLeaf: expandedGlobs[0].Leafs[i]})
		}
		result.data, err = response.Marshal()
		result.contentType = httpHeaders.ContentTypeProtobuf

		if err != nil {
			atomic.AddUint64(&listener.metrics.FindErrors, 1)

			logger.Error("find failed",
				zap.Duration("runtime_seconds", time.Since(t0)),
				zap.String("reason", "response encode failed"),
				zap.Error(err),
			)
			return nil, err
		}

		if len(response.Matches) == 0 {
			return nil, errorNotFound{}
		}

		return &result, err

	case pickleFormat:
		// [{'metric_path': 'metric', 'intervals': [(x,y)], 'isLeaf': True},]
		var metrics []map[string]interface{}
		var m map[string]interface{}
		files := 0

		glob := expandedGlobs[0]
		files += len(glob.Files)
		for i, p := range glob.Files {
			if glob.Leafs[i] {
				metricsCount++
			}
			m = make(map[string]interface{})
			m["path"] = p
			m["is_leaf"] = glob.Leafs[i]

			metrics = append(metrics, m)
		}
		var buf bytes.Buffer
		pEnc := pickle.NewEncoder(&buf)
		pEnc.Encode(metrics)
		return &findResponse{buf.Bytes(), httpHeaders.ContentTypePickle, files}, nil
	}
	return nil, nil
}
<|endoftext|>


<commit_before>package carbonserver

import (
	"errors"
	"math"
	"path/filepath"
	"strings"
	"unsafe"

	"github.com/lomik/zapwriter"
	"go.uber.org/zap"
)

const (
	gstateSplit  = 128
	gstateMatch  = 129
	gstateRanges = 130
)

var endGstate = &gstate{}

type gmatcher struct {
	exact   bool
	root    *gstate
	expr    string
	dstates []*gdstate
	dsindex int
}

func (g *gmatcher) dstate() *gdstate { return g.dstates[len(g.dstates)-1] }
func (g *gmatcher) push(s *gdstate)  { g.dstates = append(g.dstates, s) }
func (g *gmatcher) pop() {
	if len(g.dstates) <= 1 {
		return
	}
	g.dstates = g.dstates[:len(g.dstates)-1]
}

type gstate struct {
	// TODO: make c compact
	c    [131]bool
	next []*gstate
}

func (g *gstate) String() string {
	var toStr func(g *gstate) string

	ref := map[*gstate]string{}
	toStr = func(g *gstate) string {
		if r, ok := ref[g]; ok {
			return r
		}

		if g == endGstate {
			ref[g] = "end"
			return "end"
		}
		var b []byte
		for c, t := range g.c {
			if t {
				if isAlphanumeric(byte(c)) {
					b = append(b, byte(c))
				} else {
					b = append(b, '.')
				}
			}
		}
		var r = string(b)
		ref[g] = r + "(...)"
		for _, n := range g.next {
			r += "(" + toStr(n) + ")"
		}

		ref[g] = r
		return r
	}

	return toStr(g)
}

type gdstate struct {
	next    [131]*gdstate
	gstates []*gstate

	cacheHit int
}

func (g gdstate) String() string {
	var r []string
	for _, s := range g.gstates {
		r = append(r, s.String())
	}
	return strings.Join(r, ",")
}

func (g *gdstate) step(c byte) *gdstate {
	if g.next[c] != nil {
		g.cacheHit++
		return g.next[c]
	}

	var ng gdstate
	for _, s := range g.gstates {
		if s.c[c] || s.c['*'] {
			for _, ns := range s.next {
				ng.gstates = g.add(ng.gstates, ns)
			}
		}
	}
	g.next[c] = &ng
	return &ng
}

func (g *gdstate) add(list []*gstate, s *gstate) []*gstate {
	if s.c[gstateSplit] {
		for _, ns := range s.next {
			list = g.add(list, ns)
		}

		return list
	}

	list = append(list, s)
	return list
}

func (g *gdstate) matched() bool {
	for _, s := range g.gstates {
		if s == endGstate {
			return true
		}
	}
	return false
}

// TODO:
//     add range value validation
//     add ^ in range match
//     support {} cases
func newGlobState(expr string) (*gmatcher, error) {
	var m = gmatcher{root: &gstate{}, exact: true, expr: expr}
	var cur = m.root
	// var inAlternate bool
	for i := 0; i < len(expr); i++ {
		c := expr[i]
		switch c {
		// case '{':
		// case '}':
		case '[':
			m.exact = false
			s := &gstate{}
			i++
			negative := expr[i] == '^'
			if negative {
				i++
			}
			for i < len(expr) && expr[i] != ']' {
				if expr[i] == '-' {
					if i+1 >= len(expr) {
						return nil, errors.New("glob: missing closing range")
					}

					for j := expr[i-1] + 1; j <= expr[i+1]; j++ {
						s.c[j] = true
					}

					i += 2
					continue
				}

				s.c[expr[i]] = true
				i++
			}
			if expr[i] != ']' {
				return nil, errors.New("glob: missing ]")
			}

			if negative {
				// TODO: revisit
				for i := 33; i <= 126; i++ {
					s.c[i] = !s.c[i]
				}
			}

			cur.next = append(cur.next, s)
			cur = s
		case '*':
			m.exact = false
			// de-dup multi stars: *** -> *
			for ; i+1 < len(expr) && expr[i+1] == '*'; i++ {
			}

			s := &gstate{}
			s.c['*'] = true

			var split, star gstate
			star.c['*'] = true
			split.c[gstateSplit] = true
			split.next = append(split.next, &star)
			cur.next = append(cur.next, &split)
			star.next = append(star.next, &split)

			cur = &split
		default:
			s := &gstate{}
			s.c[c] = true
			cur.next = append(cur.next, s)
			cur = s
		}
	}
	cur.next = append(cur.next, endGstate)

	var droot gdstate
	for _, s := range m.root.next {
		droot.gstates = droot.add(droot.gstates, s)
	}
	m.dstates = append(m.dstates, &droot)

	return &m, nil
}

func isAlphanumeric(c byte) bool {
	return 33 <= c && c <= 127
}

type trieIndex struct {
	root      *trieNode
	depth     int
	fileExt   string
	fileCount int
}

// TODO: use compact c (string/[]byte rather than byte)
type trieNode struct {
	c byte
	// node      string
	parent    *trieNode
	childrens []*trieNode
	isFile    bool
	isNodeEnd bool
}

var fileNode = &trieNode{}
var dirNode = &trieNode{}

func newTrie(fileExt string) *trieIndex {
	return &trieIndex{
		root:    &trieNode{},
		fileExt: fileExt,
	}
}

// abc.def.ghi
// abc.def2.ghi
func (ti *trieIndex) insert(path string) {
	path = filepath.Clean(path)
	if path[0] == '/' {
		path = path[1:]
	}
	if path == "" || path == "." {
		return
	}

	cur := ti.root
	if len(path) > ti.depth {
		ti.depth = len(path)
	}

	isFile := strings.HasSuffix(path, ti.fileExt)
	if isFile {
		path = path[:len(path)-len(ti.fileExt)]
	}

outer:
	for i := 0; i < len(path); i++ {
		c := path[i]
		for _, child := range cur.childrens {
			if child.c == byte(c) {
				cur = child
				continue outer
			}
		}

		tnode := &trieNode{c: c, parent: cur}
		cur.childrens = append(cur.childrens, tnode)
		cur = tnode
		cur.parent.isNodeEnd = c == '/'
	}

	if !isFile {
		return
	}

	cur.isFile = true
	cur.isNodeEnd = true
	cur.childrens = append(cur.childrens, fileNode)
	ti.fileCount++
}

// depth first search
func (ti *trieIndex) search(pattern string, limit int) (files []string, isFile []bool, err error) {
	var matchers []*gmatcher
	var exact bool
	for _, node := range strings.Split(pattern, "/") {
		if node == "" {
			return nil, nil, errors.New("empty node in query")
		}
		gs, err := newGlobState(node)
		if err != nil {
			return nil, nil, err
		}
		exact = exact && gs.exact
		matchers = append(matchers, gs)
	}

	var cur = ti.root
	var nindex = make([]int, ti.depth+1)
	var ncindex int
	var mindex int
	var curm = matchers[0]
	var ndstate *gdstate

	for {
		if nindex[ncindex] >= len(cur.childrens) {
			curm.pop()
			goto parent
		}

		cur = cur.childrens[nindex[ncindex]]
		ncindex++

		// log.Printf("cur.fullPath(., ti.depth) = %+v\n", cur.fullPath('.', ti.depth))
		// log.Printf("curm.dstate = %+v\n", curm.dstate())

		if cur.c == '/' {
			if mindex+1 >= len(matchers) || !curm.dstate().matched() {
				goto parent
			}

			mindex++
			curm = matchers[mindex]
			curm.dstates = curm.dstates[:1]

			continue
		}

		ndstate = curm.dstate().step(cur.c)
		if len(ndstate.gstates) == 0 {
			goto parent
		}

		curm.push(ndstate)

		if mindex+1 < len(matchers) {
			continue
		}

		if !cur.isNodeEnd && !cur.isFile {
			continue
		}

		if !curm.dstate().matched() {
			curm.pop()
			goto parent
		}

		files = append(files, cur.fullPath('.', ti.depth))
		isFile = append(isFile, cur.isFile)
		if len(files) >= limit || exact {
			break
		}
		curm.pop()
		goto parent

	parent:
		// use exact for fast exit
		nindex[ncindex] = 0
		ncindex--
		if ncindex < 0 {
			break
		}
		nindex[ncindex]++
		cur = cur.parent

		if cur.c == '/' && nindex[ncindex] >= len(cur.childrens) {
			mindex--
			curm = matchers[mindex]

			goto parent
		}

		continue
	}

	return files, isFile, nil
}

func (tn *trieNode) fullPath(sep byte, depth int) string {
	var r = make([]byte, depth)
	var index = depth - 1
	var cur = tn
	for cur.parent != nil {
		if cur.c == '/' {
			r[index] = sep
		} else {
			r[index] = cur.c
		}
		index--
		cur = cur.parent
	}
	r = r[index+1:]

	return *(*string)(unsafe.Pointer(&r))
}

func (ti *trieIndex) allFiles(sep byte) []string {
	var files = make([]string, 0, ti.fileCount)
	var childIndex = make([]int, ti.depth+1)
	var curLevel int
	var cur = ti.root
	for {
		if len(cur.childrens) == 0 {
			files = append(files, cur.fullPath(sep, ti.depth))
			goto parent
		}

		if childIndex[curLevel] == len(cur.childrens) {
			goto parent
		}

		cur = cur.childrens[childIndex[curLevel]]
		curLevel++
		continue

	parent:
		childIndex[curLevel] = 0
		curLevel--
		if curLevel < 0 {
			break
		}
		childIndex[curLevel]++
		cur = cur.parent
		continue
	}
	return files
}

func (listener *CarbonserverListener) expandGlobsTrie(query string) ([]string, []bool, error) {
	var useGlob bool
	logger := zapwriter.Logger("carbonserver")

	// TODO: Find out why we have set 'useGlob' if 'star == -1'
	if star := strings.IndexByte(query, '*'); strings.IndexByte(query, '[') == -1 && strings.IndexByte(query, '?') == -1 && (star == -1 || star == len(query)-1) {
		useGlob = true
	}
	logger = logger.With(zap.Bool("use_glob", useGlob))

	/* things to glob:
	 * - carbon.relays  -> carbon.relays
	 * - carbon.re      -> carbon.relays, carbon.rewhatever
	 * - carbon.[rz]    -> carbon.relays, carbon.zipper
	 * - carbon.{re,zi} -> carbon.relays, carbon.zipper
	 * - match is either dir or .wsp file
	 * unfortunately, filepath.Glob doesn't handle the curly brace
	 * expansion for us */

	query = strings.Replace(query, ".", "/", -1)

	var globs []string
	globs = append(globs, query)
	// TODO(dgryski): move this loop into its own function + add tests
	for {
		bracematch := false
		var newglobs []string
		for _, glob := range globs {
			lbrace := strings.Index(glob, "{")
			rbrace := -1
			if lbrace > -1 {
				rbrace = strings.Index(glob[lbrace:], "}")
				if rbrace > -1 {
					rbrace += lbrace
				}
			}

			if lbrace > -1 && rbrace > -1 {
				bracematch = true
				expansion := glob[lbrace+1 : rbrace]
				parts := strings.Split(expansion, ",")
				for _, sub := range parts {
					if len(newglobs) > listener.maxGlobs {
						if listener.failOnMaxGlobs {
							return nil, nil, errMaxGlobsExhausted
						}
						break
					}
					newglobs = append(newglobs, glob[:lbrace]+sub+glob[rbrace+1:])
				}
			} else {
				if len(newglobs) > listener.maxGlobs {
					if listener.failOnMaxGlobs {
						return nil, nil, errMaxGlobsExhausted
					}
					break
				}
				newglobs = append(newglobs, glob)
			}
		}
		globs = newglobs
		if !bracematch {
			break
		}
	}

	fidx := listener.CurrentFileIndex()

	var files []string
	var leafs []bool
	for _, g := range globs {
		f, l, err := fidx.trieIdx.search(g, math.MaxInt64)
		if err != nil {
			panic(err)
		}
		files = append(files, f...)
		leafs = append(leafs, l...)
	}
	return files, leafs, nil
}
<commit_msg>trie: drop isFile & isNodeEnd from trieNode<commit_after>package carbonserver

import (
	"errors"
	"math"
	"path/filepath"
	"strings"
	"unsafe"

	"github.com/lomik/zapwriter"
	"go.uber.org/zap"
)

const (
	gstateSplit  = 128
	gstateMatch  = 129
	gstateRanges = 130
)

var endGstate = &gstate{}

type gmatcher struct {
	exact   bool
	root    *gstate
	expr    string
	dstates []*gdstate
	dsindex int
}

func (g *gmatcher) dstate() *gdstate { return g.dstates[len(g.dstates)-1] }
func (g *gmatcher) push(s *gdstate)  { g.dstates = append(g.dstates, s) }
func (g *gmatcher) pop() {
	if len(g.dstates) <= 1 {
		return
	}
	g.dstates = g.dstates[:len(g.dstates)-1]
}

type gstate struct {
	// TODO: make c compact
	c    [131]bool
	next []*gstate
}

func (g *gstate) String() string {
	var toStr func(g *gstate) string

	ref := map[*gstate]string{}
	toStr = func(g *gstate) string {
		if r, ok := ref[g]; ok {
			return r
		}

		if g == endGstate {
			ref[g] = "end"
			return "end"
		}
		var b []byte
		for c, t := range g.c {
			if t {
				if isAlphanumeric(byte(c)) {
					b = append(b, byte(c))
				} else {
					b = append(b, '.')
				}
			}
		}
		var r = string(b)
		ref[g] = r + "(...)"
		for _, n := range g.next {
			r += "(" + toStr(n) + ")"
		}

		ref[g] = r
		return r
	}

	return toStr(g)
}

type gdstate struct {
	next    [131]*gdstate
	gstates []*gstate

	cacheHit int
}

func (g gdstate) String() string {
	var r []string
	for _, s := range g.gstates {
		r = append(r, s.String())
	}
	return strings.Join(r, ",")
}

func (g *gdstate) step(c byte) *gdstate {
	if g.next[c] != nil {
		g.cacheHit++
		return g.next[c]
	}

	var ng gdstate
	for _, s := range g.gstates {
		if s.c[c] || s.c['*'] {
			for _, ns := range s.next {
				ng.gstates = g.add(ng.gstates, ns)
			}
		}
	}
	g.next[c] = &ng
	return &ng
}

func (g *gdstate) add(list []*gstate, s *gstate) []*gstate {
	if s.c[gstateSplit] {
		for _, ns := range s.next {
			list = g.add(list, ns)
		}

		return list
	}

	list = append(list, s)
	return list
}

func (g *gdstate) matched() bool {
	for _, s := range g.gstates {
		if s == endGstate {
			return true
		}
	}
	return false
}

// TODO:
//     add range value validation
//     add ^ in range match
//     support {} cases
func newGlobState(expr string) (*gmatcher, error) {
	var m = gmatcher{root: &gstate{}, exact: true, expr: expr}
	var cur = m.root
	// var inAlternate bool
	for i := 0; i < len(expr); i++ {
		c := expr[i]
		switch c {
		// case '{':
		// case '}':
		case '[':
			m.exact = false
			s := &gstate{}
			i++
			negative := expr[i] == '^'
			if negative {
				i++
			}
			for i < len(expr) && expr[i] != ']' {
				if expr[i] == '-' {
					if i+1 >= len(expr) {
						return nil, errors.New("glob: missing closing range")
					}

					for j := expr[i-1] + 1; j <= expr[i+1]; j++ {
						s.c[j] = true
					}

					i += 2
					continue
				}

				s.c[expr[i]] = true
				i++
			}
			if expr[i] != ']' {
				return nil, errors.New("glob: missing ]")
			}

			if negative {
				// TODO: revisit
				for i := 33; i <= 126; i++ {
					s.c[i] = !s.c[i]
				}
			}

			cur.next = append(cur.next, s)
			cur = s
		case '*':
			m.exact = false
			// de-dup multi stars: *** -> *
			for ; i+1 < len(expr) && expr[i+1] == '*'; i++ {
			}

			s := &gstate{}
			s.c['*'] = true

			var split, star gstate
			star.c['*'] = true
			split.c[gstateSplit] = true
			split.next = append(split.next, &star)
			cur.next = append(cur.next, &split)
			star.next = append(star.next, &split)

			cur = &split
		default:
			s := &gstate{}
			s.c[c] = true
			cur.next = append(cur.next, s)
			cur = s
		}
	}
	cur.next = append(cur.next, endGstate)

	var droot gdstate
	for _, s := range m.root.next {
		droot.gstates = droot.add(droot.gstates, s)
	}
	m.dstates = append(m.dstates, &droot)

	return &m, nil
}

func isAlphanumeric(c byte) bool {
	return 33 <= c && c <= 127
}

type trieIndex struct {
	root      *trieNode
	depth     int
	fileExt   string
	fileCount int
}

// TODO: use compact c (string/[]byte rather than byte)
type trieNode struct {
	c         byte
	parent    *trieNode
	childrens []*trieNode
}

var fileNode = &trieNode{}

func newTrie(fileExt string) *trieIndex {
	return &trieIndex{
		root:    &trieNode{},
		fileExt: fileExt,
	}
}

// abc.def.ghi
// abc.def2.ghi
func (ti *trieIndex) insert(path string) {
	path = filepath.Clean(path)
	if path[0] == '/' {
		path = path[1:]
	}
	if path == "" || path == "." {
		return
	}

	cur := ti.root
	if len(path) > ti.depth {
		ti.depth = len(path)
	}

	isFile := strings.HasSuffix(path, ti.fileExt)
	if isFile {
		path = path[:len(path)-len(ti.fileExt)]
	}

outer:
	for i := 0; i < len(path); i++ {
		c := path[i]
		for _, child := range cur.childrens {
			if child.c == byte(c) {
				cur = child
				continue outer
			}
		}

		tnode := &trieNode{c: c, parent: cur}
		cur.childrens = append(cur.childrens, tnode)
		cur = tnode
	}

	if !isFile {
		if cur.c != '/' {
			cur.childrens = append(cur.childrens, &trieNode{c: '/', parent: cur})
		}
		return
	}

	cur.childrens = append(cur.childrens, fileNode)
	ti.fileCount++
}

// depth first search
func (ti *trieIndex) search(pattern string, limit int) (files []string, isFile []bool, err error) {
	var matchers []*gmatcher
	var exact bool
	for _, node := range strings.Split(pattern, "/") {
		if node == "" {
			return nil, nil, errors.New("empty node in query")
		}
		gs, err := newGlobState(node)
		if err != nil {
			return nil, nil, err
		}
		exact = exact && gs.exact
		matchers = append(matchers, gs)
	}

	var cur = ti.root
	var nindex = make([]int, ti.depth+1)
	var ncindex int
	var mindex int
	var curm = matchers[0]
	var ndstate *gdstate

	for {
		if nindex[ncindex] >= len(cur.childrens) {
			curm.pop()
			goto parent
		}

		cur = cur.childrens[nindex[ncindex]]
		ncindex++

		// log.Printf("cur.fullPath(., ti.depth) = %+v\n", cur.fullPath('.', ti.depth))
		// log.Printf("curm.dstate = %+v\n", curm.dstate())

		if cur.c == '/' {
			if mindex+1 >= len(matchers) || !curm.dstate().matched() {
				goto parent
			}

			mindex++
			curm = matchers[mindex]
			curm.dstates = curm.dstates[:1]

			continue
		}

		ndstate = curm.dstate().step(cur.c)
		if len(ndstate.gstates) == 0 {
			goto parent
		}

		curm.push(ndstate)

		if mindex+1 < len(matchers) {
			continue
		}

		if !(cur.childrens[0].c == '/' || cur.childrens[0] == fileNode) {
			continue
		}

		if !curm.dstate().matched() {
			curm.pop()
			goto parent
		}

		files = append(files, cur.fullPath('.', ti.depth))
		isFile = append(isFile, cur.childrens[0] == fileNode)
		if len(files) >= limit || exact {
			break
		}
		curm.pop()
		goto parent

	parent:
		// use exact for fast exit
		nindex[ncindex] = 0
		ncindex--
		if ncindex < 0 {
			break
		}
		nindex[ncindex]++
		cur = cur.parent

		if cur.c == '/' && nindex[ncindex] >= len(cur.childrens) {
			mindex--
			curm = matchers[mindex]

			goto parent
		}

		continue
	}

	return files, isFile, nil
}

func (tn *trieNode) fullPath(sep byte, depth int) string {
	var r = make([]byte, depth)
	var index = depth - 1
	var cur = tn
	for cur.parent != nil {
		if cur.c == '/' {
			r[index] = sep
		} else {
			r[index] = cur.c
		}
		index--
		cur = cur.parent
	}
	r = r[index+1:]

	return *(*string)(unsafe.Pointer(&r))
}

func (ti *trieIndex) allFiles(sep byte) []string {
	var files = make([]string, 0, ti.fileCount)
	var childIndex = make([]int, ti.depth+1)
	var curLevel int
	var cur = ti.root
	for {
		if len(cur.childrens) == 0 {
			files = append(files, cur.fullPath(sep, ti.depth))
			goto parent
		}

		if childIndex[curLevel] == len(cur.childrens) {
			goto parent
		}

		cur = cur.childrens[childIndex[curLevel]]
		curLevel++
		continue

	parent:
		childIndex[curLevel] = 0
		curLevel--
		if curLevel < 0 {
			break
		}
		childIndex[curLevel]++
		cur = cur.parent
		continue
	}
	return files
}

func (listener *CarbonserverListener) expandGlobsTrie(query string) ([]string, []bool, error) {
	var useGlob bool
	logger := zapwriter.Logger("carbonserver")

	// TODO: Find out why we have set 'useGlob' if 'star == -1'
	if star := strings.IndexByte(query, '*'); strings.IndexByte(query, '[') == -1 && strings.IndexByte(query, '?') == -1 && (star == -1 || star == len(query)-1) {
		useGlob = true
	}
	logger = logger.With(zap.Bool("use_glob", useGlob))

	/* things to glob:
	 * - carbon.relays  -> carbon.relays
	 * - carbon.re      -> carbon.relays, carbon.rewhatever
	 * - carbon.[rz]    -> carbon.relays, carbon.zipper
	 * - carbon.{re,zi} -> carbon.relays, carbon.zipper
	 * - match is either dir or .wsp file
	 * unfortunately, filepath.Glob doesn't handle the curly brace
	 * expansion for us */

	query = strings.Replace(query, ".", "/", -1)

	var globs []string
	globs = append(globs, query)
	// TODO(dgryski): move this loop into its own function + add tests
	for {
		bracematch := false
		var newglobs []string
		for _, glob := range globs {
			lbrace := strings.Index(glob, "{")
			rbrace := -1
			if lbrace > -1 {
				rbrace = strings.Index(glob[lbrace:], "}")
				if rbrace > -1 {
					rbrace += lbrace
				}
			}

			if lbrace > -1 && rbrace > -1 {
				bracematch = true
				expansion := glob[lbrace+1 : rbrace]
				parts := strings.Split(expansion, ",")
				for _, sub := range parts {
					if len(newglobs) > listener.maxGlobs {
						if listener.failOnMaxGlobs {
							return nil, nil, errMaxGlobsExhausted
						}
						break
					}
					newglobs = append(newglobs, glob[:lbrace]+sub+glob[rbrace+1:])
				}
			} else {
				if len(newglobs) > listener.maxGlobs {
					if listener.failOnMaxGlobs {
						return nil, nil, errMaxGlobsExhausted
					}
					break
				}
				newglobs = append(newglobs, glob)
			}
		}
		globs = newglobs
		if !bracematch {
			break
		}
	}

	fidx := listener.CurrentFileIndex()

	var files []string
	var leafs []bool
	for _, g := range globs {
		f, l, err := fidx.trieIdx.search(g, math.MaxInt64)
		if err != nil {
			panic(err)
		}
		files = append(files, f...)
		leafs = append(leafs, l...)
	}
	return files, leafs, nil
}
<|endoftext|>


<commit_before>package dhcpd

import (
	"errors"
	"net"
	"strings"
	"time"

	"github.com/Cloud-Foundations/Dominator/lib/log"
	libnet "github.com/Cloud-Foundations/Dominator/lib/net"
	"github.com/Cloud-Foundations/Dominator/lib/net/util"
	proto "github.com/Cloud-Foundations/Dominator/proto/hypervisor"
	dhcp "github.com/krolaw/dhcp4"
	"golang.org/x/net/ipv4"
)

const sysClassNet = "/sys/class/net"
const leaseTime = time.Hour * 48

type serveIfConn struct {
	ifIndices map[int]struct{}
	conn      *ipv4.PacketConn
	cm        *ipv4.ControlMessage
}

func newServer(interfaceNames []string, logger log.DebugLogger) (
	*DhcpServer, error) {
	dhcpServer := &DhcpServer{
		logger:          logger,
		ackChannels:     make(map[string]chan struct{}),
		ipAddrToMacAddr: make(map[string]string),
		leases:          make(map[string]leaseType),
		requestChannels: make(map[string]chan net.IP),
	}
	if myIP, err := util.GetMyIP(); err != nil {
		return nil, err
	} else {
		dhcpServer.myIP = myIP
	}
	if len(interfaceNames) < 1 {
		logger.Debugln(0, "Starting DHCP server on all broadcast interfaces")
		interfaces, _, err := libnet.ListBroadcastInterfaces(
			libnet.InterfaceTypeEtherNet|libnet.InterfaceTypeBridge, logger)
		if err != nil {
			return nil, err
		} else {
			for _, iface := range interfaces {
				interfaceNames = append(interfaceNames, iface.Name)
			}
		}
	} else {
		logger.Debugln(0, "Starting DHCP server on interfaces: "+
			strings.Join(interfaceNames, ","))
	}
	serveConn := &serveIfConn{
		ifIndices: make(map[int]struct{}, len(interfaceNames)),
	}
	for _, interfaceName := range interfaceNames {
		if iface, err := net.InterfaceByName(interfaceName); err != nil {
			return nil, err
		} else {
			serveConn.ifIndices[iface.Index] = struct{}{}
		}
	}
	listener, err := net.ListenPacket("udp4", ":67")
	if err != nil {
		return nil, err
	}
	pktConn := ipv4.NewPacketConn(listener)
	if err := pktConn.SetControlMessage(ipv4.FlagInterface, true); err != nil {
		listener.Close()
		return nil, err
	}
	serveConn.conn = pktConn
	go func() {
		if err := dhcp.Serve(serveConn, dhcpServer); err != nil {
			logger.Println(err)
		}
	}()
	return dhcpServer, nil
}

func (s *DhcpServer) acknowledgeLease(ipAddr net.IP) {
	ipStr := ipAddr.String()
	s.mutex.Lock()
	ackChan, ok := s.ackChannels[ipStr]
	delete(s.ackChannels, ipStr)
	s.mutex.Unlock()
	if ok {
		ackChan <- struct{}{}
		close(ackChan)
	}
}

func (s *DhcpServer) addLease(address proto.Address, doNetboot bool,
	hostname string, subnet *proto.Subnet) error {
	address.Shrink()
	if len(address.IpAddress) < 1 {
		return errors.New("no IP address")
	}
	ipAddr := address.IpAddress.String()
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if subnet == nil {
		if subnet = s.findMatchingSubnet(address.IpAddress); subnet == nil {
			return errors.New("no subnet found for " + ipAddr)
		}
	}
	if doNetboot {
		if len(s.networkBootImage) < 1 {
			return errors.New("no Network Boot Image name configured")
		}
		if _, ok := s.leases[address.MacAddress]; ok {
			return errors.New("already have lease for: " + address.MacAddress)
		}
	}
	s.ipAddrToMacAddr[ipAddr] = address.MacAddress
	s.leases[address.MacAddress] = leaseType{
		address, hostname, doNetboot, subnet}
	return nil
}

func (s *DhcpServer) addSubnet(subnet proto.Subnet) {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	s.subnets = append(s.subnets, subnet)
}

func (s *DhcpServer) findLease(macAddr string) (*leaseType, *proto.Subnet) {
	s.mutex.RLock()
	defer s.mutex.RUnlock()
	if lease, ok := s.leases[macAddr]; !ok {
		return nil, nil
	} else if lease.subnet != nil {
		return &lease, lease.subnet
	} else {
		return &lease, s.findMatchingSubnet(lease.IpAddress)
	}
}

// This must be called with the lock held.
func (s *DhcpServer) findMatchingSubnet(ipAddr net.IP) *proto.Subnet {
	for _, subnet := range s.subnets {
		subnetMask := net.IPMask(subnet.IpMask)
		subnetAddr := subnet.IpGateway.Mask(subnetMask)
		if ipAddr.Mask(subnetMask).Equal(subnetAddr) {
			return &subnet
		}
	}
	return nil
}

func (s *DhcpServer) makeAcknowledgmentChannel(ipAddr net.IP) <-chan struct{} {
	ipStr := ipAddr.String()
	newChan := make(chan struct{}, 1)
	s.mutex.Lock()
	oldChan, ok := s.ackChannels[ipStr]
	s.ackChannels[ipStr] = newChan
	s.mutex.Unlock()
	if ok {
		close(oldChan)
	}
	return newChan
}

func (s *DhcpServer) makeOptions(subnet *proto.Subnet,
	lease *leaseType) dhcp.Options {
	dnsServers := make([]byte, 0)
	for _, dnsServer := range subnet.DomainNameServers {
		dnsServers = append(dnsServers, dnsServer...)
	}
	leaseOptions := dhcp.Options{
		dhcp.OptionSubnetMask:       subnet.IpMask,
		dhcp.OptionRouter:           subnet.IpGateway,
		dhcp.OptionDomainNameServer: dnsServers,
	}
	if subnet.DomainName != "" {
		leaseOptions[dhcp.OptionDomainName] = []byte(subnet.DomainName)
	}
	if lease.Hostname != "" {
		leaseOptions[dhcp.OptionHostName] = []byte(lease.Hostname)
	}
	if lease.doNetboot {
		leaseOptions[dhcp.OptionBootFileName] = s.networkBootImage
	}
	return leaseOptions
}

func (s *DhcpServer) makeRequestChannel(macAddr string) <-chan net.IP {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if oldChan, ok := s.requestChannels[macAddr]; ok {
		return oldChan
	}
	newChan := make(chan net.IP, 16)
	s.requestChannels[macAddr] = newChan
	return newChan
}

func (s *DhcpServer) notifyRequest(address proto.Address) {
	s.mutex.RLock()
	requestChan, ok := s.requestChannels[address.MacAddress]
	s.mutex.RUnlock()
	if ok {
		select {
		case requestChan <- address.IpAddress:
		default:
		}
	}
}

func (s *DhcpServer) removeLease(ipAddr net.IP) {
	if len(ipAddr) < 1 {
		return
	}
	ipStr := ipAddr.String()
	s.mutex.Lock()
	delete(s.leases, s.ipAddrToMacAddr[ipStr])
	delete(s.ipAddrToMacAddr, ipStr)
	ackChan, ok := s.ackChannels[ipStr]
	delete(s.ackChannels, ipStr)
	s.mutex.Unlock()
	if ok {
		close(ackChan)
	}
}

func (s *DhcpServer) removeSubnet(subnetId string) {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	subnets := make([]proto.Subnet, 0, len(s.subnets)-1)
	for _, subnet := range s.subnets {
		if subnet.Id != subnetId {
			subnets = append(subnets, subnet)
		}
	}
	s.subnets = subnets
}

func (s *DhcpServer) ServeDHCP(req dhcp.Packet, msgType dhcp.MessageType,
	options dhcp.Options) dhcp.Packet {
	switch msgType {
	case dhcp.Discover:
		macAddr := req.CHAddr().String()
		s.logger.Debugf(1, "DHCP Discover from: %s\n", macAddr)
		lease, subnet := s.findLease(macAddr)
		if lease == nil {
			return nil
		}
		if subnet == nil {
			s.logger.Printf("No subnet found for %s\n", lease.IpAddress)
			return nil
		}
		s.logger.Debugf(0, "DHCP Offer: %s for: %s, server: %s\n",
			lease.IpAddress, macAddr, s.myIP)
		leaseOptions := s.makeOptions(subnet, lease)
		packet := dhcp.ReplyPacket(req, dhcp.Offer, s.myIP, lease.IpAddress,
			leaseTime,
			leaseOptions.SelectOrderOrAll(
				options[dhcp.OptionParameterRequestList]))
		packet.SetSIAddr(s.myIP)
		return packet
	case dhcp.Request:
		reqIP := net.IP(options[dhcp.OptionRequestedIPAddress])
		if reqIP == nil {
			s.logger.Debugln(0, "DHCP Request did not request an IP")
			reqIP = net.IP(req.CIAddr())
		}
		reqIP = util.ShrinkIP(reqIP)
		macAddr := req.CHAddr().String()
		s.notifyRequest(proto.Address{reqIP, macAddr})
		server, ok := options[dhcp.OptionServerIdentifier]
		if ok {
			serverIP := net.IP(server)
			if !serverIP.IsUnspecified() && !serverIP.Equal(s.myIP) {
				s.logger.Debugf(0,
					"DHCP Request for: %s from: %s to: %s is not me: %s\n",
					reqIP, macAddr, serverIP, s.myIP)
				return nil // Message not for this DHCP server.
			}
		}
		s.logger.Debugf(0, "DHCP Request for: %s from: %s\n", reqIP, macAddr)
		lease, subnet := s.findLease(macAddr)
		if lease == nil {
			s.logger.Printf("No lease found for %s\n", macAddr)
			return nil
		}
		if subnet == nil {
			s.logger.Printf("No subnet found for %s\n", lease.IpAddress)
			return nil
		}
		if reqIP.Equal(lease.IpAddress) {
			leaseOptions := s.makeOptions(subnet, lease)
			s.logger.Debugf(0, "DHCP ACK for: %s to: %s\n", reqIP, macAddr)
			s.acknowledgeLease(lease.IpAddress)
			packet := dhcp.ReplyPacket(req, dhcp.ACK, s.myIP, reqIP, leaseTime,
				leaseOptions.SelectOrderOrAll(
					options[dhcp.OptionParameterRequestList]))
			packet.SetSIAddr(s.myIP)
			return packet
		} else {
			s.logger.Debugf(0, "DHCP NAK for: %s to: %s\n", reqIP, macAddr)
			return dhcp.ReplyPacket(req, dhcp.NAK, s.myIP, nil, 0, nil)
		}
	default:
		s.logger.Debugf(0, "Unsupported message type: %s\n", msgType)
	}
	return nil
}

func (s *serveIfConn) ReadFrom(b []byte) (n int, addr net.Addr, err error) {
	for {
		n, s.cm, addr, err = s.conn.ReadFrom(b)
		if err != nil || s.cm == nil {
			break
		}
		if _, ok := s.ifIndices[s.cm.IfIndex]; ok {
			break
		}
	}
	return
}

func (s *serveIfConn) WriteTo(b []byte, addr net.Addr) (n int, err error) {
	s.cm.Src = nil
	return s.conn.WriteTo(b, s.cm, addr)
}
<commit_msg>hypervisor: process DHCP packets from VLAN interfaces.<commit_after>package dhcpd

import (
	"errors"
	"net"
	"strings"
	"time"

	"github.com/Cloud-Foundations/Dominator/lib/log"
	"github.com/Cloud-Foundations/Dominator/lib/log/prefixlogger"
	libnet "github.com/Cloud-Foundations/Dominator/lib/net"
	"github.com/Cloud-Foundations/Dominator/lib/net/util"
	proto "github.com/Cloud-Foundations/Dominator/proto/hypervisor"
	dhcp "github.com/krolaw/dhcp4"
	"golang.org/x/net/ipv4"
)

const sysClassNet = "/sys/class/net"
const leaseTime = time.Hour * 48

type serveIfConn struct {
	ifIndices map[int]struct{}
	conn      *ipv4.PacketConn
	cm        *ipv4.ControlMessage
}

func newServer(interfaceNames []string, logger log.DebugLogger) (
	*DhcpServer, error) {
	dhcpServer := &DhcpServer{
		logger:          logger,
		ackChannels:     make(map[string]chan struct{}),
		ipAddrToMacAddr: make(map[string]string),
		leases:          make(map[string]leaseType),
		requestChannels: make(map[string]chan net.IP),
	}
	if myIP, err := util.GetMyIP(); err != nil {
		return nil, err
	} else {
		dhcpServer.myIP = myIP
	}
	if len(interfaceNames) < 1 {
		logger.Debugln(0, "Starting DHCP server on all broadcast interfaces")
		interfaces, _, err := libnet.ListBroadcastInterfaces(
			libnet.InterfaceTypeEtherNet|
				libnet.InterfaceTypeBridge|
				libnet.InterfaceTypeVlan,
			prefixlogger.New("dhcpd: ", logger))
		if err != nil {
			return nil, err
		} else {
			for _, iface := range interfaces {
				interfaceNames = append(interfaceNames, iface.Name)
			}
		}
	} else {
		logger.Debugln(0, "Starting DHCP server on interfaces: "+
			strings.Join(interfaceNames, ","))
	}
	serveConn := &serveIfConn{
		ifIndices: make(map[int]struct{}, len(interfaceNames)),
	}
	for _, interfaceName := range interfaceNames {
		if iface, err := net.InterfaceByName(interfaceName); err != nil {
			return nil, err
		} else {
			serveConn.ifIndices[iface.Index] = struct{}{}
		}
	}
	listener, err := net.ListenPacket("udp4", ":67")
	if err != nil {
		return nil, err
	}
	pktConn := ipv4.NewPacketConn(listener)
	if err := pktConn.SetControlMessage(ipv4.FlagInterface, true); err != nil {
		listener.Close()
		return nil, err
	}
	serveConn.conn = pktConn
	go func() {
		if err := dhcp.Serve(serveConn, dhcpServer); err != nil {
			logger.Println(err)
		}
	}()
	return dhcpServer, nil
}

func (s *DhcpServer) acknowledgeLease(ipAddr net.IP) {
	ipStr := ipAddr.String()
	s.mutex.Lock()
	ackChan, ok := s.ackChannels[ipStr]
	delete(s.ackChannels, ipStr)
	s.mutex.Unlock()
	if ok {
		ackChan <- struct{}{}
		close(ackChan)
	}
}

func (s *DhcpServer) addLease(address proto.Address, doNetboot bool,
	hostname string, subnet *proto.Subnet) error {
	address.Shrink()
	if len(address.IpAddress) < 1 {
		return errors.New("no IP address")
	}
	ipAddr := address.IpAddress.String()
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if subnet == nil {
		if subnet = s.findMatchingSubnet(address.IpAddress); subnet == nil {
			return errors.New("no subnet found for " + ipAddr)
		}
	}
	if doNetboot {
		if len(s.networkBootImage) < 1 {
			return errors.New("no Network Boot Image name configured")
		}
		if _, ok := s.leases[address.MacAddress]; ok {
			return errors.New("already have lease for: " + address.MacAddress)
		}
	}
	s.ipAddrToMacAddr[ipAddr] = address.MacAddress
	s.leases[address.MacAddress] = leaseType{
		address, hostname, doNetboot, subnet}
	return nil
}

func (s *DhcpServer) addSubnet(subnet proto.Subnet) {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	s.subnets = append(s.subnets, subnet)
}

func (s *DhcpServer) findLease(macAddr string) (*leaseType, *proto.Subnet) {
	s.mutex.RLock()
	defer s.mutex.RUnlock()
	if lease, ok := s.leases[macAddr]; !ok {
		return nil, nil
	} else if lease.subnet != nil {
		return &lease, lease.subnet
	} else {
		return &lease, s.findMatchingSubnet(lease.IpAddress)
	}
}

// This must be called with the lock held.
func (s *DhcpServer) findMatchingSubnet(ipAddr net.IP) *proto.Subnet {
	for _, subnet := range s.subnets {
		subnetMask := net.IPMask(subnet.IpMask)
		subnetAddr := subnet.IpGateway.Mask(subnetMask)
		if ipAddr.Mask(subnetMask).Equal(subnetAddr) {
			return &subnet
		}
	}
	return nil
}

func (s *DhcpServer) makeAcknowledgmentChannel(ipAddr net.IP) <-chan struct{} {
	ipStr := ipAddr.String()
	newChan := make(chan struct{}, 1)
	s.mutex.Lock()
	oldChan, ok := s.ackChannels[ipStr]
	s.ackChannels[ipStr] = newChan
	s.mutex.Unlock()
	if ok {
		close(oldChan)
	}
	return newChan
}

func (s *DhcpServer) makeOptions(subnet *proto.Subnet,
	lease *leaseType) dhcp.Options {
	dnsServers := make([]byte, 0)
	for _, dnsServer := range subnet.DomainNameServers {
		dnsServers = append(dnsServers, dnsServer...)
	}
	leaseOptions := dhcp.Options{
		dhcp.OptionSubnetMask:       subnet.IpMask,
		dhcp.OptionRouter:           subnet.IpGateway,
		dhcp.OptionDomainNameServer: dnsServers,
	}
	if subnet.DomainName != "" {
		leaseOptions[dhcp.OptionDomainName] = []byte(subnet.DomainName)
	}
	if lease.Hostname != "" {
		leaseOptions[dhcp.OptionHostName] = []byte(lease.Hostname)
	}
	if lease.doNetboot {
		leaseOptions[dhcp.OptionBootFileName] = s.networkBootImage
	}
	return leaseOptions
}

func (s *DhcpServer) makeRequestChannel(macAddr string) <-chan net.IP {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if oldChan, ok := s.requestChannels[macAddr]; ok {
		return oldChan
	}
	newChan := make(chan net.IP, 16)
	s.requestChannels[macAddr] = newChan
	return newChan
}

func (s *DhcpServer) notifyRequest(address proto.Address) {
	s.mutex.RLock()
	requestChan, ok := s.requestChannels[address.MacAddress]
	s.mutex.RUnlock()
	if ok {
		select {
		case requestChan <- address.IpAddress:
		default:
		}
	}
}

func (s *DhcpServer) removeLease(ipAddr net.IP) {
	if len(ipAddr) < 1 {
		return
	}
	ipStr := ipAddr.String()
	s.mutex.Lock()
	delete(s.leases, s.ipAddrToMacAddr[ipStr])
	delete(s.ipAddrToMacAddr, ipStr)
	ackChan, ok := s.ackChannels[ipStr]
	delete(s.ackChannels, ipStr)
	s.mutex.Unlock()
	if ok {
		close(ackChan)
	}
}

func (s *DhcpServer) removeSubnet(subnetId string) {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	subnets := make([]proto.Subnet, 0, len(s.subnets)-1)
	for _, subnet := range s.subnets {
		if subnet.Id != subnetId {
			subnets = append(subnets, subnet)
		}
	}
	s.subnets = subnets
}

func (s *DhcpServer) ServeDHCP(req dhcp.Packet, msgType dhcp.MessageType,
	options dhcp.Options) dhcp.Packet {
	switch msgType {
	case dhcp.Discover:
		macAddr := req.CHAddr().String()
		s.logger.Debugf(1, "DHCP Discover from: %s\n", macAddr)
		lease, subnet := s.findLease(macAddr)
		if lease == nil {
			return nil
		}
		if subnet == nil {
			s.logger.Printf("No subnet found for %s\n", lease.IpAddress)
			return nil
		}
		s.logger.Debugf(0, "DHCP Offer: %s for: %s, server: %s\n",
			lease.IpAddress, macAddr, s.myIP)
		leaseOptions := s.makeOptions(subnet, lease)
		packet := dhcp.ReplyPacket(req, dhcp.Offer, s.myIP, lease.IpAddress,
			leaseTime,
			leaseOptions.SelectOrderOrAll(
				options[dhcp.OptionParameterRequestList]))
		packet.SetSIAddr(s.myIP)
		return packet
	case dhcp.Request:
		reqIP := net.IP(options[dhcp.OptionRequestedIPAddress])
		if reqIP == nil {
			s.logger.Debugln(0, "DHCP Request did not request an IP")
			reqIP = net.IP(req.CIAddr())
		}
		reqIP = util.ShrinkIP(reqIP)
		macAddr := req.CHAddr().String()
		s.notifyRequest(proto.Address{reqIP, macAddr})
		server, ok := options[dhcp.OptionServerIdentifier]
		if ok {
			serverIP := net.IP(server)
			if !serverIP.IsUnspecified() && !serverIP.Equal(s.myIP) {
				s.logger.Debugf(0,
					"DHCP Request for: %s from: %s to: %s is not me: %s\n",
					reqIP, macAddr, serverIP, s.myIP)
				return nil // Message not for this DHCP server.
			}
		}
		s.logger.Debugf(0, "DHCP Request for: %s from: %s\n", reqIP, macAddr)
		lease, subnet := s.findLease(macAddr)
		if lease == nil {
			s.logger.Printf("No lease found for %s\n", macAddr)
			return nil
		}
		if subnet == nil {
			s.logger.Printf("No subnet found for %s\n", lease.IpAddress)
			return nil
		}
		if reqIP.Equal(lease.IpAddress) {
			leaseOptions := s.makeOptions(subnet, lease)
			s.logger.Debugf(0, "DHCP ACK for: %s to: %s\n", reqIP, macAddr)
			s.acknowledgeLease(lease.IpAddress)
			packet := dhcp.ReplyPacket(req, dhcp.ACK, s.myIP, reqIP, leaseTime,
				leaseOptions.SelectOrderOrAll(
					options[dhcp.OptionParameterRequestList]))
			packet.SetSIAddr(s.myIP)
			return packet
		} else {
			s.logger.Debugf(0, "DHCP NAK for: %s to: %s\n", reqIP, macAddr)
			return dhcp.ReplyPacket(req, dhcp.NAK, s.myIP, nil, 0, nil)
		}
	default:
		s.logger.Debugf(0, "Unsupported message type: %s\n", msgType)
	}
	return nil
}

func (s *serveIfConn) ReadFrom(b []byte) (n int, addr net.Addr, err error) {
	for {
		n, s.cm, addr, err = s.conn.ReadFrom(b)
		if err != nil || s.cm == nil {
			break
		}
		if _, ok := s.ifIndices[s.cm.IfIndex]; ok {
			break
		}
	}
	return
}

func (s *serveIfConn) WriteTo(b []byte, addr net.Addr) (n int, err error) {
	s.cm.Src = nil
	return s.conn.WriteTo(b, s.cm, addr)
}
<|endoftext|>


<commit_before>package awsrules

import (
	"log"

	"github.com/hashicorp/hcl2/hcl"
	"github.com/wata727/tflint/tflint"
)

// AwsS3BucketInvalidACLRule checks the pattern is valid
type AwsS3BucketInvalidACLRule struct {
	resourceType  string
	attributeName string
	enum          []string
}

// NewAwsS3BucketInvalidACLRule returns new rule with default attributes
func NewAwsS3BucketInvalidACLRule() *AwsS3BucketInvalidACLRule {
	return &AwsS3BucketInvalidACLRule{
		resourceType:  "aws_s3_bucket",
		attributeName: "acl",
		enum: []string{
			"private",
			"public-read",
			"public-read-write",
			"authenticated-read",
			"log-delivery-write",
		},
	}
}

// Name returns the rule name
func (r *AwsS3BucketInvalidACLRule) Name() string {
	return "aws_s3_bucket_invalid_acl"
}

// Enabled returns whether the rule is enabled by default
func (r *AwsS3BucketInvalidACLRule) Enabled() bool {
	return true
}

// Severity returns the rule severity
func (r *AwsS3BucketInvalidACLRule) Severity() string {
	return tflint.ERROR
}

// Link returns the rule reference link
func (r *AwsS3BucketInvalidACLRule) Link() string {
	return ""
}

// Check checks the pattern is valid
func (r *AwsS3BucketInvalidACLRule) Check(runner *tflint.Runner) error {
	log.Printf("[TRACE] Check `%s` rule for `%s` runner", r.Name(), runner.TFConfigPath())

	return runner.WalkResourceAttributes(r.resourceType, r.attributeName, func(attribute *hcl.Attribute) error {
		var val string
		err := runner.EvaluateExpr(attribute.Expr, &val)

		return runner.EnsureNoError(err, func() error {
			found := false
			for _, item := range r.enum {
				if item == val {
					found = true
				}
			}
			if !found {
				runner.EmitIssue(
					r,
					`acl is not a valid value`,
					attribute.Expr.Range(),
				)
			}
			return nil
		})
	})
}
<commit_msg>Add aws-exec-read ACL as a valid value<commit_after>package awsrules

import (
	"log"

	"github.com/hashicorp/hcl2/hcl"
	"github.com/wata727/tflint/tflint"
)

// AwsS3BucketInvalidACLRule checks the pattern is valid
type AwsS3BucketInvalidACLRule struct {
	resourceType  string
	attributeName string
	enum          []string
}

// NewAwsS3BucketInvalidACLRule returns new rule with default attributes
func NewAwsS3BucketInvalidACLRule() *AwsS3BucketInvalidACLRule {
	return &AwsS3BucketInvalidACLRule{
		resourceType:  "aws_s3_bucket",
		attributeName: "acl",
		enum: []string{
			"private",
			"public-read",
			"public-read-write",
			"aws-exec-read",
			"authenticated-read",
			"log-delivery-write",
		},
	}
}

// Name returns the rule name
func (r *AwsS3BucketInvalidACLRule) Name() string {
	return "aws_s3_bucket_invalid_acl"
}

// Enabled returns whether the rule is enabled by default
func (r *AwsS3BucketInvalidACLRule) Enabled() bool {
	return true
}

// Severity returns the rule severity
func (r *AwsS3BucketInvalidACLRule) Severity() string {
	return tflint.ERROR
}

// Link returns the rule reference link
func (r *AwsS3BucketInvalidACLRule) Link() string {
	return ""
}

// Check checks the pattern is valid
func (r *AwsS3BucketInvalidACLRule) Check(runner *tflint.Runner) error {
	log.Printf("[TRACE] Check `%s` rule for `%s` runner", r.Name(), runner.TFConfigPath())

	return runner.WalkResourceAttributes(r.resourceType, r.attributeName, func(attribute *hcl.Attribute) error {
		var val string
		err := runner.EvaluateExpr(attribute.Expr, &val)

		return runner.EnsureNoError(err, func() error {
			found := false
			for _, item := range r.enum {
				if item == val {
					found = true
				}
			}
			if !found {
				runner.EmitIssue(
					r,
					`acl is not a valid value`,
					attribute.Expr.Range(),
				)
			}
			return nil
		})
	})
}
<|endoftext|>