<commit_before>package perms // import "a4.io/blobstash/pkg/perms"

import (
	"bytes"
	"fmt"
	"html/template"
	"strings"

	"a4.io/blobstash/pkg/config"
	"github.com/zpatrick/rbac"
)

type ActionType string
type ObjectType string
type ServiceName string

// Actions
const (
	Read     ActionType = "read"
	Stat     ActionType = "stat"
	Write    ActionType = "write"
	List     ActionType = "list"
	Snapshot ActionType = "snapshot"
	Search   ActionType = "search"
	GC       ActionType = "gc"
	Destroy  ActionType = "destroy"
)

// Object types
const (
	Blob      ObjectType = "blob"
	KVEntry   ObjectType = "kv"
	FS        ObjectType = "fs"
	Node      ObjectType = "node"
	GitRepo   ObjectType = "git-repo"
	GitNs     ObjectType = "git-ns"
	Namespace ObjectType = "namespace"
)

// Services
const (
	BlobStore ServiceName = "blobstore"
	KvStore   ServiceName = "kvstore"
	DocStore  ServiceName = "docstore"
	Filetree  ServiceName = "filetree"
	GitServer ServiceName = "gitserver"
	Stash     ServiceName = "stash"
)

// Action formats an action `<action_type>:<object_type>`
func Action(action ActionType, objectType ObjectType) string {
	return fmt.Sprintf("action:%s:%s", action, objectType)
}

func ResourceWithID(service ServiceName, objectType ObjectType, objectID string) string {
	return fmt.Sprintf("resource:%s:%s:%s", service, objectType, objectID)
}

func Resource(service ServiceName, objectType ObjectType) string {
	return fmt.Sprintf("resource:%s:%s:NA", service, objectType)
}

func init() {
	SetupRole(&config.Role{
		Name:  "admin",
		Perms: []*config.Perm{&config.Perm{Action: "action:*", Resource: "resource:*"}},
	})
	SetupRole(&config.Role{
		Template:     "backup",
		Managed:      true,
		ArgsRequired: []string{"name"},
		Perms: []*config.Perm{
			&config.Perm{
				Action:   Action(Stat, Blob),
				Resource: ResourceWithID(BlobStore, Blob, "*"),
			},
			&config.Perm{
				Action:   Action(Write, Blob),
				Resource: ResourceWithID(BlobStore, Blob, "*"),
			},
			&config.Perm{
				Action:   Action(Write, KVEntry),
				Resource: ResourceWithID(KvStore, KVEntry, "_filetree:fs:{{.name}}"),
			},
			&config.Perm{
				Action:   Action(GC, Namespace),
				Resource: ResourceWithID(Stash, Namespace, "{{.name}}"),
			},
			&config.Perm{
				Action:   Action(Snapshot, FS),
				Resource: ResourceWithID(Filetree, FS, "{{.name}}"),
			},
		},
	})
	SetupRole(&config.Role{
		Template:     "git-ro",
		Managed:      true,
		ArgsRequired: []string{"ns", "repo"},
		Perms: []*config.Perm{
			&config.Perm{
				Action:   Action(Read, GitRepo),
				Resource: ResourceWithID(GitServer, GitRepo, "{{.ns}}/{{.repo}}"),
			},
		},
	})
	SetupRole(&config.Role{
		Template:     "git",
		Managed:      true,
		ArgsRequired: []string{"ns", "repo"},
		Perms: []*config.Perm{
			&config.Perm{
				Action:   Action(Read, GitRepo),
				Resource: ResourceWithID(GitServer, GitRepo, "{{.ns}}/{{.repo}}"),
			},
			&config.Perm{
				Action:   Action(Write, GitRepo),
				Resource: ResourceWithID(GitServer, GitRepo, "{{.ns}}/{{.repo}}"),
			},
		},
	})
	SetupRole(&config.Role{
		Name: "git-admin",
		Perms: []*config.Perm{
			&config.Perm{
				Action:   Action(Read, GitRepo),
				Resource: ResourceWithID(GitServer, GitRepo, "*/*"),
			},
			&config.Perm{
				Action:   Action(Write, GitRepo),
				Resource: ResourceWithID(GitServer, GitRepo, "*/*"),
			},
			&config.Perm{
				Action:   Action(List, GitNs),
				Resource: ResourceWithID(GitServer, GitNs, "*"),
			},
		},
	})
}

var roles = map[string]rbac.Role{}
var managedRoles = map[string]*config.Role{}

func newManagedRole(r *config.Role) error {
	for _, k := range r.ArgsRequired {
		if _, ok := r.Args[k]; !ok {
			return fmt.Errorf("missing %s arg for role %s", k, r.Name)
		}
	}
	var buf bytes.Buffer
	mperms := []*config.Perm{}
	for _, p := range r.Perms {
		t := template.Must(template.New("resource").Parse(p.Resource))
		if err := t.Execute(&buf, r.Args); err != nil {
			return err
		}
		mperms = append(mperms, &config.Perm{
			Action:   p.Action,
			Resource: buf.String(),
		})
		buf.Reset()
	}
	SetupRole(&config.Role{
		Name:  r.Name,
		Perms: mperms,
	})
	return nil
}

func SetupRole(r *config.Role) error {
	if r.Template != "" && r.Managed {
		managedRoles[r.Template] = r
		return nil
	}
	if mrole, ok := managedRoles[r.Template]; ok {
		mrole.Args = r.Args
		mrole.Name = r.Name
		defer func(cr *config.Role) {
			cr.Args = nil
			cr.Name = ""
		}(r)
		return newManagedRole(mrole)
	}

	if _, used := roles[r.Name]; used {
		return fmt.Errorf("%q is already used", r.Name)
	}
	perms := rbac.Permissions{}
	for _, p := range r.Perms {
		if !strings.HasPrefix(p.Action, "action:") {
			return fmt.Errorf("invalid action %q", p.Action)
		}
		if !strings.HasPrefix(p.Resource, "resource:") {
			return fmt.Errorf("invalid resource %q", p.Resource)
		}
		perms = append(perms, rbac.NewGlobPermission(p.Action, p.Resource))
	}

	role := rbac.Role{
		RoleID:      r.Name,
		Permissions: perms,
	}
	roles[r.Name] = role
	return nil
}

func GetRole(k string) (rbac.Role, error) {
	r, ok := roles[k]
	if !ok {
		return rbac.Role{}, fmt.Errorf("role %q not found", k)
	}
	return r, nil
}

func GetRoles(keys []string) (rbac.Roles, error) {
	res := rbac.Roles{}
	for _, k := range keys {
		role, err := GetRole(k)
		if err != nil {
			return nil, err
		}
		res = append(res, role)
	}
	return res, nil
}

func Setup(conf *config.Config) error {
	for _, role := range conf.Roles {
		if err := SetupRole(role); err != nil {
			panic(err)
		}
	}
	return nil
}
<commit_msg>perms: add missing perms for the docstore<commit_after>package perms // import "a4.io/blobstash/pkg/perms"

import (
	"bytes"
	"fmt"
	"html/template"
	"strings"

	"a4.io/blobstash/pkg/config"
	"github.com/zpatrick/rbac"
)

type ActionType string
type ObjectType string
type ServiceName string

// Actions
const (
	Read     ActionType = "read"
	Stat     ActionType = "stat"
	Write    ActionType = "write"
	List     ActionType = "list"
	Snapshot ActionType = "snapshot"
	Search   ActionType = "search"
	GC       ActionType = "gc"
	Destroy  ActionType = "destroy"
)

// Object types
const (
	Blob         ObjectType = "blob"
	KVEntry      ObjectType = "kv"
	FS           ObjectType = "fs"
	Node         ObjectType = "node"
	GitRepo      ObjectType = "git-repo"
	GitNs        ObjectType = "git-ns"
	Namespace    ObjectType = "namespace"
	JSONDocument ObjectType = "json-doc"
)

// Services
const (
	BlobStore ServiceName = "blobstore"
	KvStore   ServiceName = "kvstore"
	DocStore  ServiceName = "docstore"
	Filetree  ServiceName = "filetree"
	GitServer ServiceName = "gitserver"
	Stash     ServiceName = "stash"
)

// Action formats an action `<action_type>:<object_type>`
func Action(action ActionType, objectType ObjectType) string {
	return fmt.Sprintf("action:%s:%s", action, objectType)
}

func ResourceWithID(service ServiceName, objectType ObjectType, objectID string) string {
	return fmt.Sprintf("resource:%s:%s:%s", service, objectType, objectID)
}

func Resource(service ServiceName, objectType ObjectType) string {
	return fmt.Sprintf("resource:%s:%s:NA", service, objectType)
}

func init() {
	SetupRole(&config.Role{
		Name:  "admin",
		Perms: []*config.Perm{&config.Perm{Action: "action:*", Resource: "resource:*"}},
	})
	SetupRole(&config.Role{
		Template:     "backup",
		Managed:      true,
		ArgsRequired: []string{"name"},
		Perms: []*config.Perm{
			&config.Perm{
				Action:   Action(Stat, Blob),
				Resource: ResourceWithID(BlobStore, Blob, "*"),
			},
			&config.Perm{
				Action:   Action(Write, Blob),
				Resource: ResourceWithID(BlobStore, Blob, "*"),
			},
			&config.Perm{
				Action:   Action(Write, KVEntry),
				Resource: ResourceWithID(KvStore, KVEntry, "_filetree:fs:{{.name}}"),
			},
			&config.Perm{
				Action:   Action(GC, Namespace),
				Resource: ResourceWithID(Stash, Namespace, "{{.name}}"),
			},
			&config.Perm{
				Action:   Action(Snapshot, FS),
				Resource: ResourceWithID(Filetree, FS, "{{.name}}"),
			},
		},
	})
	SetupRole(&config.Role{
		Template:     "git-ro",
		Managed:      true,
		ArgsRequired: []string{"ns", "repo"},
		Perms: []*config.Perm{
			&config.Perm{
				Action:   Action(Read, GitRepo),
				Resource: ResourceWithID(GitServer, GitRepo, "{{.ns}}/{{.repo}}"),
			},
		},
	})
	SetupRole(&config.Role{
		Template:     "git",
		Managed:      true,
		ArgsRequired: []string{"ns", "repo"},
		Perms: []*config.Perm{
			&config.Perm{
				Action:   Action(Read, GitRepo),
				Resource: ResourceWithID(GitServer, GitRepo, "{{.ns}}/{{.repo}}"),
			},
			&config.Perm{
				Action:   Action(Write, GitRepo),
				Resource: ResourceWithID(GitServer, GitRepo, "{{.ns}}/{{.repo}}"),
			},
		},
	})
	SetupRole(&config.Role{
		Name: "git-admin",
		Perms: []*config.Perm{
			&config.Perm{
				Action:   Action(Read, GitRepo),
				Resource: ResourceWithID(GitServer, GitRepo, "*/*"),
			},
			&config.Perm{
				Action:   Action(Write, GitRepo),
				Resource: ResourceWithID(GitServer, GitRepo, "*/*"),
			},
			&config.Perm{
				Action:   Action(List, GitNs),
				Resource: ResourceWithID(GitServer, GitNs, "*"),
			},
		},
	})
}

var roles = map[string]rbac.Role{}
var managedRoles = map[string]*config.Role{}

func newManagedRole(r *config.Role) error {
	for _, k := range r.ArgsRequired {
		if _, ok := r.Args[k]; !ok {
			return fmt.Errorf("missing %s arg for role %s", k, r.Name)
		}
	}
	var buf bytes.Buffer
	mperms := []*config.Perm{}
	for _, p := range r.Perms {
		t := template.Must(template.New("resource").Parse(p.Resource))
		if err := t.Execute(&buf, r.Args); err != nil {
			return err
		}
		mperms = append(mperms, &config.Perm{
			Action:   p.Action,
			Resource: buf.String(),
		})
		buf.Reset()
	}
	SetupRole(&config.Role{
		Name:  r.Name,
		Perms: mperms,
	})
	return nil
}

func SetupRole(r *config.Role) error {
	if r.Template != "" && r.Managed {
		managedRoles[r.Template] = r
		return nil
	}
	if mrole, ok := managedRoles[r.Template]; ok {
		mrole.Args = r.Args
		mrole.Name = r.Name
		defer func(cr *config.Role) {
			cr.Args = nil
			cr.Name = ""
		}(r)
		return newManagedRole(mrole)
	}

	if _, used := roles[r.Name]; used {
		return fmt.Errorf("%q is already used", r.Name)
	}
	perms := rbac.Permissions{}
	for _, p := range r.Perms {
		if !strings.HasPrefix(p.Action, "action:") {
			return fmt.Errorf("invalid action %q", p.Action)
		}
		if !strings.HasPrefix(p.Resource, "resource:") {
			return fmt.Errorf("invalid resource %q", p.Resource)
		}
		perms = append(perms, rbac.NewGlobPermission(p.Action, p.Resource))
	}

	role := rbac.Role{
		RoleID:      r.Name,
		Permissions: perms,
	}
	roles[r.Name] = role
	return nil
}

func GetRole(k string) (rbac.Role, error) {
	r, ok := roles[k]
	if !ok {
		return rbac.Role{}, fmt.Errorf("role %q not found", k)
	}
	return r, nil
}

func GetRoles(keys []string) (rbac.Roles, error) {
	res := rbac.Roles{}
	for _, k := range keys {
		role, err := GetRole(k)
		if err != nil {
			return nil, err
		}
		res = append(res, role)
	}
	return res, nil
}

func Setup(conf *config.Config) error {
	for _, role := range conf.Roles {
		if err := SetupRole(role); err != nil {
			panic(err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package proxy

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/cilium/cilium/pkg/completion"
	"github.com/cilium/cilium/pkg/flowdebug"
	identityPkg "github.com/cilium/cilium/pkg/identity"
	"github.com/cilium/cilium/pkg/kafka"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/node"
	"github.com/cilium/cilium/pkg/policy"
	"github.com/cilium/cilium/pkg/proxy/accesslog"

	"github.com/optiopay/kafka/proto"
	"github.com/sirupsen/logrus"
)

const (
	fieldID = "id"
)

// kafkaRedirect implements the Redirect interface for an l7 proxy
type kafkaRedirect struct {
	redirect *Redirect
	conf     kafkaConfiguration
	rules    policy.L7DataMap
	socket   *proxySocket
}

type destLookupFunc func(remoteAddr string, dport uint16) (uint32, string, error)

type kafkaConfiguration struct {
	noMarker      bool
	lookupNewDest destLookupFunc
}

// createKafkaRedirect creates a redirect to the kafka proxy. The redirect structure passed
// in is safe to access for reading and writing.
func createKafkaRedirect(r *Redirect, conf kafkaConfiguration) (RedirectImplementation, error) {
	redir := &kafkaRedirect{
		redirect: r,
		conf:     conf,
	}

	if redir.conf.lookupNewDest == nil {
		redir.conf.lookupNewDest = lookupNewDest
	}

	marker := 0
	if !conf.noMarker {
		markIdentity := int(0)
		// As ingress proxy, all replies to incoming requests must have the
		// identity of the endpoint we are proxying for
		if r.ingress {
			markIdentity = int(r.source.GetIdentity())
		}

		marker = GetMagicMark(r.ingress, markIdentity)
	}

	// Listen needs to be in the synchronous part of this function to ensure that
	// the proxy port is never refusing connections.
	socket, err := listenSocket(fmt.Sprintf(":%d", r.ProxyPort), marker)
	if err != nil {
		return nil, err
	}

	redir.socket = socket

	go func() {
		for {
			pair, err := socket.Accept(true)
			select {
			case <-socket.closing:
				// Don't report errors while the socket is being closed
				return
			default:
			}

			if err != nil {
				log.WithField(logfields.Port, r.ProxyPort).WithError(err).Error("Unable to accept connection on port")
				continue
			}

			go redir.handleRequestConnection(pair)
		}
	}()

	return redir, nil
}

func (k *kafkaRedirect) canAccess(req *kafka.RequestMessage, numIdentity identityPkg.NumericIdentity) bool {
	var identity *identityPkg.Identity

	if numIdentity != 0 {
		identity = identityPkg.LookupIdentityByID(numIdentity)
		if identity == nil {
			log.WithFields(logrus.Fields{
				logfields.Request:  req.String(),
				logfields.Identity: numIdentity,
			}).Warn("Unable to resolve identity to labels")
		}
	}

	k.redirect.mutex.RLock()
	rules := k.redirect.rules.GetRelevantRules(identity)
	k.redirect.mutex.RUnlock()

	if rules.Kafka == nil {
		flowdebug.Log(log.WithField(logfields.Request, req.String()),
			"No Kafka rules loaded, rejecting")
		return false
	}

	b, err := json.Marshal(rules.Kafka)
	if err != nil {
		flowdebug.Log(log.WithError(err).WithField(logfields.Request, req.String()),
			"Error marshalling kafka rules to apply")
		return false
	} else {
		flowdebug.Log(log.WithFields(logrus.Fields{
			logfields.Request: req.String(),
			"rule":            string(b),
		}), "Applying rule")
	}

	return req.MatchesRule(rules.Kafka)
}

// kafkaLogRecord wraps an accesslog.LogRecord so that we can define methods with a receiver
type kafkaLogRecord struct {
	accesslog.LogRecord
	redirect *Redirect
	req      *kafka.RequestMessage
}

func (k *kafkaRedirect) newKafkaLogRecord(req *kafka.RequestMessage) kafkaLogRecord {
	record := kafkaLogRecord{
		LogRecord: req.GetLogRecord(),
		req:       req,
		redirect:  k.redirect,
	}

	if k.redirect.ingress {
		record.ObservationPoint = accesslog.Ingress
	} else {
		record.ObservationPoint = accesslog.Egress
	}

	return record
}

func (l *kafkaLogRecord) fillInfo(r *Redirect, srcIPPort, dstIPPort string, srcIdentity uint32) {
	fillInfo(r, &l.LogRecord, srcIPPort, dstIPPort, srcIdentity)
}

// log Kafka log records
func (l *kafkaLogRecord) log(typ accesslog.FlowType, verdict accesslog.FlowVerdict, code int, info string) {
	l.Type = typ
	l.Verdict = verdict
	l.Kafka.ErrorCode = code
	l.Info = info
	l.Timestamp = time.Now().UTC().Format(time.RFC3339Nano)

	l.LogRecord.NodeAddressInfo = accesslog.NodeAddressInfo{
		IPv4: node.GetExternalIPv4().String(),
		IPv6: node.GetIPv6().String(),
	}

	flowdebug.Log(log.WithFields(logrus.Fields{
		accesslog.FieldType:               l.Type,
		accesslog.FieldVerdict:            l.Verdict,
		accesslog.FieldCode:               l.Kafka.ErrorCode,
		accesslog.FieldKafkaAPIKey:        l.Kafka.APIKey,
		accesslog.FieldKafkaAPIVersion:    l.Kafka.APIVersion,
		accesslog.FieldKafkaCorrelationID: l.Kafka.CorrelationID,
		accesslog.FieldMessage:            l.Info,
	}), "Logging Kafka L7 flow record")

	l.redirect.updateAccounting(l.Type, l.Verdict)

	// Log multiple entries for multiple Kafka topics in a single request.
	topics := l.req.GetTopics()
	for i := 0; i < len(topics); i++ {
		l.Kafka.Topic.Topic = topics[i]
		l.Log()
	}
}

func (k *kafkaRedirect) handleRequest(pair *connectionPair, req *kafka.RequestMessage) {
	scopedLog := log.WithField(fieldID, pair.String())
	flowdebug.Log(scopedLog.WithField(logfields.Request, req.String()), "Handling Kafka request")

	record := k.newKafkaLogRecord(req)

	addr := pair.Rx.conn.RemoteAddr()
	if addr == nil {
		info := fmt.Sprint("RemoteAddr() is nil")
		scopedLog.Warn(info)
		record.log(accesslog.TypeRequest, accesslog.VerdictError, kafka.ErrInvalidMessage, info)
		return
	}

	// retrieve identity of source together with original destination IP
	// and destination port
	srcIdentity, dstIPPort, err := k.conf.lookupNewDest(addr.String(), k.redirect.ProxyPort)
	if err != nil {
		scopedLog.WithField("source",
			addr.String()).WithError(err).Error("Unable lookup original destination")
		record.log(accesslog.TypeRequest, accesslog.VerdictError, kafka.ErrInvalidMessage,
			fmt.Sprintf("Unable lookup original destination: %s", err))
		return
	}

	record.fillInfo(k.redirect, addr.String(), dstIPPort, srcIdentity)

	if !k.canAccess(req, identityPkg.NumericIdentity(srcIdentity)) {
		flowdebug.Log(scopedLog, "Kafka request is denied by policy")

		record.log(accesslog.TypeRequest, accesslog.VerdictDenied,
			kafka.ErrTopicAuthorizationFailed, fmt.Sprint("Kafka request is denied by policy"))

		resp, err := req.CreateResponse(proto.ErrTopicAuthorizationFailed)
		if err != nil {
			record.log(accesslog.TypeRequest, accesslog.VerdictError,
				kafka.ErrInvalidMessage, fmt.Sprintf("Unable to create response: %s", err))
			scopedLog.WithError(err).Error("Unable to create Kafka response")
			return
		}

		pair.Rx.Enqueue(resp.GetRaw())
		return
	}

	if pair.Tx.Closed() {
		marker := 0
		if !k.conf.noMarker {
			marker = GetMagicMark(k.redirect.ingress, int(srcIdentity))
		}

		flowdebug.Log(scopedLog.WithFields(logrus.Fields{
			"marker":      marker,
			"destination": dstIPPort,
		}), "Dialing original destination")

		txConn, err := ciliumDialer(marker, addr.Network(), dstIPPort)
		if err != nil {
			scopedLog.WithError(err).WithFields(logrus.Fields{
				"origNetwork": addr.Network(),
				"origDest":    dstIPPort,
			}).Error("Unable to dial original destination")

			record.log(accesslog.TypeRequest, accesslog.VerdictError,
				kafka.ErrNetwork, fmt.Sprintf("Unable to dial original destination: %s", err))

			return
		}

		pair.Tx.SetConnection(txConn)

		// Start go routine to handle responses and pass in a copy of
		// the request record as template for all responses
		go k.handleResponseConnection(pair, record)
	}

	flowdebug.Log(scopedLog, "Forwarding Kafka request")
	// log valid request
	record.log(accesslog.TypeRequest, accesslog.VerdictForwarded, kafka.ErrNone, "")

	// Write the entire raw request onto the outgoing connection
	pair.Tx.Enqueue(req.GetRaw())
}

type kafkaReqMessageHander func(pair *connectionPair, req *kafka.RequestMessage)
type kafkaRespMessageHander func(pair *connectionPair, req *kafka.ResponseMessage)

func handleRequests(done <-chan struct{}, pair *connectionPair, c *proxyConnection,
	record *kafkaLogRecord, handler kafkaReqMessageHander) {
	defer c.Close()
	scopedLog := log.WithField(fieldID, pair.String())
	for {
		req, err := kafka.ReadRequest(c.conn)

		// Ignore any error if the listen socket has been closed, i.e. the
		// port redirect has been removed.
		select {
		case <-done:
			scopedLog.Debug("Redirect removed; closing Kafka request connection")
			return
		default:
		}

		if err != nil {
			if record != nil {
				record.log(accesslog.TypeRequest, accesslog.VerdictError,
					kafka.ErrInvalidMessage, fmt.Sprintf("Unable to parse Kafka request: %s", err))
			}
			scopedLog.WithError(err).Error("Unable to parse Kafka request; closing Kafka request connection")
			return
		}

		handler(pair, req)
	}
}

func (k *kafkaRedirect) handleResponses(done <-chan struct{}, pair *connectionPair, c *proxyConnection, record kafkaLogRecord, handler kafkaRespMessageHander) {
	defer c.Close()
	scopedLog := log.WithField(fieldID, pair.String())
	for {
		rsp, err := kafka.ReadResponse(c.conn)

		// Ignore any error if the listen socket has been closed, i.e. the
		// port redirect has been removed.
		select {
		case <-done:
			scopedLog.Debug("Redirect removed; closing Kafka response connection")
			return
		default:
		}

		if err != nil {
			record.log(accesslog.TypeResponse, accesslog.VerdictError,
				kafka.ErrInvalidMessage,
				fmt.Sprintf("Unable to parse Kafka response: %s", err))
			scopedLog.WithError(err).Error("Unable to parse Kafka response; closing Kafka response connection")
			return
		}

		record.log(accesslog.TypeResponse, accesslog.VerdictForwarded, kafka.ErrNone, "")

		handler(pair, rsp)
	}
}

func (k *kafkaRedirect) handleRequestConnection(pair *connectionPair) {
	flowdebug.Log(log.WithFields(logrus.Fields{
		"from": pair.Rx,
		"to":   pair.Tx,
	}), "Proxying request Kafka connection")

	handleRequests(k.socket.closing, pair, pair.Rx, nil, k.handleRequest)
}

func (k *kafkaRedirect) handleResponseConnection(pair *connectionPair, record kafkaLogRecord) {
	flowdebug.Log(log.WithFields(logrus.Fields{
		"from": pair.Tx,
		"to":   pair.Rx,
	}), "Proxying response Kafka connection")

	k.handleResponses(k.socket.closing, pair, pair.Tx, record,
		func(pair *connectionPair, rsp *kafka.ResponseMessage) {
			pair.Rx.Enqueue(rsp.GetRaw())
		})
}

// UpdateRules replaces old l7 rules of a redirect with new ones.
func (k *kafkaRedirect) UpdateRules(wg *completion.WaitGroup) error {
	return nil
}

// Close the redirect.
func (k *kafkaRedirect) Close(wg *completion.WaitGroup) {
	k.socket.Close()
}

func init() {
	if err := proto.ConfigureParser(proto.ParserConfig{
		SimplifiedMessageSetParsing: false,
	}); err != nil {
		log.WithError(err).Fatal("Unable to configure kafka parser")
	}
}
<commit_msg>kafka: Remove proxymap entries on closing of receive connection<commit_after>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package proxy

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/cilium/cilium/pkg/completion"
	"github.com/cilium/cilium/pkg/flowdebug"
	identityPkg "github.com/cilium/cilium/pkg/identity"
	"github.com/cilium/cilium/pkg/kafka"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/node"
	"github.com/cilium/cilium/pkg/policy"
	"github.com/cilium/cilium/pkg/proxy/accesslog"

	"github.com/optiopay/kafka/proto"
	"github.com/sirupsen/logrus"
)

const (
	fieldID = "id"
)

// kafkaRedirect implements the Redirect interface for an l7 proxy
type kafkaRedirect struct {
	redirect *Redirect
	conf     kafkaConfiguration
	rules    policy.L7DataMap
	socket   *proxySocket
}

type destLookupFunc func(remoteAddr string, dport uint16) (uint32, string, error)

type kafkaConfiguration struct {
	noMarker      bool
	lookupNewDest destLookupFunc
}

// createKafkaRedirect creates a redirect to the kafka proxy. The redirect structure passed
// in is safe to access for reading and writing.
func createKafkaRedirect(r *Redirect, conf kafkaConfiguration) (RedirectImplementation, error) {
	redir := &kafkaRedirect{
		redirect: r,
		conf:     conf,
	}

	if redir.conf.lookupNewDest == nil {
		redir.conf.lookupNewDest = lookupNewDest
	}

	marker := 0
	if !conf.noMarker {
		markIdentity := int(0)
		// As ingress proxy, all replies to incoming requests must have the
		// identity of the endpoint we are proxying for
		if r.ingress {
			markIdentity = int(r.source.GetIdentity())
		}

		marker = GetMagicMark(r.ingress, markIdentity)
	}

	// Listen needs to be in the synchronous part of this function to ensure that
	// the proxy port is never refusing connections.
	socket, err := listenSocket(fmt.Sprintf(":%d", r.ProxyPort), marker)
	if err != nil {
		return nil, err
	}

	redir.socket = socket

	go func() {
		for {
			pair, err := socket.Accept(true)
			select {
			case <-socket.closing:
				// Don't report errors while the socket is being closed
				return
			default:
			}

			if err != nil {
				log.WithField(logfields.Port, r.ProxyPort).WithError(err).Error("Unable to accept connection on port")
				continue
			}

			go redir.handleRequestConnection(pair)
		}
	}()

	return redir, nil
}

func (k *kafkaRedirect) canAccess(req *kafka.RequestMessage, numIdentity identityPkg.NumericIdentity) bool {
	var identity *identityPkg.Identity

	if numIdentity != 0 {
		identity = identityPkg.LookupIdentityByID(numIdentity)
		if identity == nil {
			log.WithFields(logrus.Fields{
				logfields.Request:  req.String(),
				logfields.Identity: numIdentity,
			}).Warn("Unable to resolve identity to labels")
		}
	}

	k.redirect.mutex.RLock()
	rules := k.redirect.rules.GetRelevantRules(identity)
	k.redirect.mutex.RUnlock()

	if rules.Kafka == nil {
		flowdebug.Log(log.WithField(logfields.Request, req.String()),
			"No Kafka rules loaded, rejecting")
		return false
	}

	b, err := json.Marshal(rules.Kafka)
	if err != nil {
		flowdebug.Log(log.WithError(err).WithField(logfields.Request, req.String()),
			"Error marshalling kafka rules to apply")
		return false
	} else {
		flowdebug.Log(log.WithFields(logrus.Fields{
			logfields.Request: req.String(),
			"rule":            string(b),
		}), "Applying rule")
	}

	return req.MatchesRule(rules.Kafka)
}

// kafkaLogRecord wraps an accesslog.LogRecord so that we can define methods with a receiver
type kafkaLogRecord struct {
	accesslog.LogRecord
	redirect *Redirect
	req      *kafka.RequestMessage
}

func (k *kafkaRedirect) newKafkaLogRecord(req *kafka.RequestMessage) kafkaLogRecord {
	record := kafkaLogRecord{
		LogRecord: req.GetLogRecord(),
		req:       req,
		redirect:  k.redirect,
	}

	if k.redirect.ingress {
		record.ObservationPoint = accesslog.Ingress
	} else {
		record.ObservationPoint = accesslog.Egress
	}

	return record
}

func (l *kafkaLogRecord) fillInfo(r *Redirect, srcIPPort, dstIPPort string, srcIdentity uint32) {
	fillInfo(r, &l.LogRecord, srcIPPort, dstIPPort, srcIdentity)
}

// log Kafka log records
func (l *kafkaLogRecord) log(typ accesslog.FlowType, verdict accesslog.FlowVerdict, code int, info string) {
	l.Type = typ
	l.Verdict = verdict
	l.Kafka.ErrorCode = code
	l.Info = info
	l.Timestamp = time.Now().UTC().Format(time.RFC3339Nano)

	l.LogRecord.NodeAddressInfo = accesslog.NodeAddressInfo{
		IPv4: node.GetExternalIPv4().String(),
		IPv6: node.GetIPv6().String(),
	}

	flowdebug.Log(log.WithFields(logrus.Fields{
		accesslog.FieldType:               l.Type,
		accesslog.FieldVerdict:            l.Verdict,
		accesslog.FieldCode:               l.Kafka.ErrorCode,
		accesslog.FieldKafkaAPIKey:        l.Kafka.APIKey,
		accesslog.FieldKafkaAPIVersion:    l.Kafka.APIVersion,
		accesslog.FieldKafkaCorrelationID: l.Kafka.CorrelationID,
		accesslog.FieldMessage:            l.Info,
	}), "Logging Kafka L7 flow record")

	l.redirect.updateAccounting(l.Type, l.Verdict)

	// Log multiple entries for multiple Kafka topics in a single request.
	topics := l.req.GetTopics()
	for i := 0; i < len(topics); i++ {
		l.Kafka.Topic.Topic = topics[i]
		l.Log()
	}
}

func (k *kafkaRedirect) handleRequest(pair *connectionPair, req *kafka.RequestMessage) {
	scopedLog := log.WithField(fieldID, pair.String())
	flowdebug.Log(scopedLog.WithField(logfields.Request, req.String()), "Handling Kafka request")

	record := k.newKafkaLogRecord(req)

	addr := pair.Rx.conn.RemoteAddr()
	if addr == nil {
		info := fmt.Sprint("RemoteAddr() is nil")
		scopedLog.Warn(info)
		record.log(accesslog.TypeRequest, accesslog.VerdictError, kafka.ErrInvalidMessage, info)
		return
	}

	// retrieve identity of source together with original destination IP
	// and destination port
	srcIdentity, dstIPPort, err := k.conf.lookupNewDest(addr.String(), k.redirect.ProxyPort)
	if err != nil {
		scopedLog.WithField("source",
			addr.String()).WithError(err).Error("Unable lookup original destination")
		record.log(accesslog.TypeRequest, accesslog.VerdictError, kafka.ErrInvalidMessage,
			fmt.Sprintf("Unable lookup original destination: %s", err))
		return
	}

	record.fillInfo(k.redirect, addr.String(), dstIPPort, srcIdentity)

	if !k.canAccess(req, identityPkg.NumericIdentity(srcIdentity)) {
		flowdebug.Log(scopedLog, "Kafka request is denied by policy")

		record.log(accesslog.TypeRequest, accesslog.VerdictDenied,
			kafka.ErrTopicAuthorizationFailed, fmt.Sprint("Kafka request is denied by policy"))

		resp, err := req.CreateResponse(proto.ErrTopicAuthorizationFailed)
		if err != nil {
			record.log(accesslog.TypeRequest, accesslog.VerdictError,
				kafka.ErrInvalidMessage, fmt.Sprintf("Unable to create response: %s", err))
			scopedLog.WithError(err).Error("Unable to create Kafka response")
			return
		}

		pair.Rx.Enqueue(resp.GetRaw())
		return
	}

	if pair.Tx.Closed() {
		marker := 0
		if !k.conf.noMarker {
			marker = GetMagicMark(k.redirect.ingress, int(srcIdentity))
		}

		flowdebug.Log(scopedLog.WithFields(logrus.Fields{
			"marker":      marker,
			"destination": dstIPPort,
		}), "Dialing original destination")

		txConn, err := ciliumDialer(marker, addr.Network(), dstIPPort)
		if err != nil {
			scopedLog.WithError(err).WithFields(logrus.Fields{
				"origNetwork": addr.Network(),
				"origDest":    dstIPPort,
			}).Error("Unable to dial original destination")

			record.log(accesslog.TypeRequest, accesslog.VerdictError,
				kafka.ErrNetwork, fmt.Sprintf("Unable to dial original destination: %s", err))

			return
		}

		pair.Tx.SetConnection(txConn)

		// Start go routine to handle responses and pass in a copy of
		// the request record as template for all responses
		go k.handleResponseConnection(pair, record)
	}

	flowdebug.Log(scopedLog, "Forwarding Kafka request")
	// log valid request
	record.log(accesslog.TypeRequest, accesslog.VerdictForwarded, kafka.ErrNone, "")

	// Write the entire raw request onto the outgoing connection
	pair.Tx.Enqueue(req.GetRaw())
}

type kafkaReqMessageHander func(pair *connectionPair, req *kafka.RequestMessage)
type kafkaRespMessageHander func(pair *connectionPair, req *kafka.ResponseMessage)

func handleRequests(done <-chan struct{}, pair *connectionPair, c *proxyConnection,
	record *kafkaLogRecord, handler kafkaReqMessageHander) {
	defer c.Close()
	scopedLog := log.WithField(fieldID, pair.String())
	for {
		req, err := kafka.ReadRequest(c.conn)

		// Ignore any error if the listen socket has been closed, i.e. the
		// port redirect has been removed.
		select {
		case <-done:
			scopedLog.Debug("Redirect removed; closing Kafka request connection")
			return
		default:
		}

		if err != nil {
			if record != nil {
				record.log(accesslog.TypeRequest, accesslog.VerdictError,
					kafka.ErrInvalidMessage, fmt.Sprintf("Unable to parse Kafka request: %s", err))
			}
			scopedLog.WithError(err).Error("Unable to parse Kafka request; closing Kafka request connection")
			return
		}

		handler(pair, req)
	}
}

func (k *kafkaRedirect) handleResponses(done <-chan struct{}, pair *connectionPair, c *proxyConnection, record kafkaLogRecord, handler kafkaRespMessageHander) {
	defer c.Close()
	scopedLog := log.WithField(fieldID, pair.String())
	for {
		rsp, err := kafka.ReadResponse(c.conn)

		// Ignore any error if the listen socket has been closed, i.e. the
		// port redirect has been removed.
		select {
		case <-done:
			scopedLog.Debug("Redirect removed; closing Kafka response connection")
			return
		default:
		}

		if err != nil {
			record.log(accesslog.TypeResponse, accesslog.VerdictError,
				kafka.ErrInvalidMessage,
				fmt.Sprintf("Unable to parse Kafka response: %s", err))
			scopedLog.WithError(err).Error("Unable to parse Kafka response; closing Kafka response connection")
			return
		}

		record.log(accesslog.TypeResponse, accesslog.VerdictForwarded, kafka.ErrNone, "")

		handler(pair, rsp)
	}
}

func (k *kafkaRedirect) handleRequestConnection(pair *connectionPair) {
	flowdebug.Log(log.WithFields(logrus.Fields{
		"from": pair.Rx,
		"to":   pair.Tx,
	}), "Proxying request Kafka connection")

	handleRequests(k.socket.closing, pair, pair.Rx, nil, k.handleRequest)

	// The proxymap contains an entry with metadata for the receive side of the
	// connection, remove it after the connection has been closed.
	if pair.Rx != nil {
		// We are running in our own go routine here so we can just
		// block this go routine until after the connection is
		// guaranteed to have been closed
		time.Sleep(proxyConnectionCloseTimeout + time.Second)

		if err := k.redirect.removeProxyMapEntryOnClose(pair.Rx.conn); err != nil {
			log.WithError(err).Warning("Unable to remove proxymap entry after closing connection")
		}
	}
}

func (k *kafkaRedirect) handleResponseConnection(pair *connectionPair, record kafkaLogRecord) {
	flowdebug.Log(log.WithFields(logrus.Fields{
		"from": pair.Tx,
		"to":   pair.Rx,
	}), "Proxying response Kafka connection")

	k.handleResponses(k.socket.closing, pair, pair.Tx, record,
		func(pair *connectionPair, rsp *kafka.ResponseMessage) {
			pair.Rx.Enqueue(rsp.GetRaw())
		})
}

// UpdateRules replaces old l7 rules of a redirect with new ones.
func (k *kafkaRedirect) UpdateRules(wg *completion.WaitGroup) error {
	return nil
}

// Close the redirect.
func (k *kafkaRedirect) Close(wg *completion.WaitGroup) {
	k.socket.Close()
}

func init() {
	if err := proto.ConfigureParser(proto.ParserConfig{
		SimplifiedMessageSetParsing: false,
	}); err != nil {
		log.WithError(err).Fatal("Unable to configure kafka parser")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rawhttp

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"k8s.io/cli-runtime/pkg/genericclioptions"
	"k8s.io/client-go/rest"
)

// RawPost uses the REST client to POST content
func RawPost(restClient *rest.RESTClient, streams genericclioptions.IOStreams, url, filename string) error {
	return raw(restClient, streams, url, filename, "POST")
}

// RawPut uses the REST client to PUT content
func RawPut(restClient *rest.RESTClient, streams genericclioptions.IOStreams, url, filename string) error {
	return raw(restClient, streams, url, filename, "PUT")
}

// RawGet uses the REST client to GET content
func RawGet(restClient *rest.RESTClient, streams genericclioptions.IOStreams, url string) error {
	return raw(restClient, streams, url, "", "GET")
}

// RawDelete uses the REST client to DELETE content
func RawDelete(restClient *rest.RESTClient, streams genericclioptions.IOStreams, url, filename string) error {
	return raw(restClient, streams, url, filename, "DELETE")
}

// raw makes a simple HTTP request to the provided path on the server using the default credentials.
func raw(restClient *rest.RESTClient, streams genericclioptions.IOStreams, url, filename, requestType string) error {
	var data io.ReadCloser
	switch {
	case len(filename) == 0:
		data = ioutil.NopCloser(bytes.NewBuffer([]byte{}))

	case filename == "-":
		data = ioutil.NopCloser(streams.In)

	default:
		var err error
		data, err = os.Open(filename)
		if err != nil {
			return err
		}
	}

	var request *rest.Request
	switch requestType {
	case "GET":
		request = restClient.Get().RequestURI(url)
	case "PUT":
		request = restClient.Put().RequestURI(url).Body(data)
	case "POST":
		request = restClient.Post().RequestURI(url).Body(data)
	case "DELETE":
		request = restClient.Delete().RequestURI(url).Body(data)

	default:
		return fmt.Errorf("unknown requestType: %q", requestType)
	}

	stream, err := request.Stream(context.TODO())
	if err != nil {
		return err
	}
	defer stream.Close()

	_, err = io.Copy(streams.Out, stream)
	if err != nil && err != io.EOF {
		return err
	}
	return nil
}
<commit_msg>Close the used file<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rawhttp

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"os"

	"k8s.io/cli-runtime/pkg/genericclioptions"
	"k8s.io/client-go/rest"
)

// RawPost uses the REST client to POST content
func RawPost(restClient *rest.RESTClient, streams genericclioptions.IOStreams, url, filename string) error {
	return raw(restClient, streams, url, filename, "POST")
}

// RawPut uses the REST client to PUT content
func RawPut(restClient *rest.RESTClient, streams genericclioptions.IOStreams, url, filename string) error {
	return raw(restClient, streams, url, filename, "PUT")
}

// RawGet uses the REST client to GET content
func RawGet(restClient *rest.RESTClient, streams genericclioptions.IOStreams, url string) error {
	return raw(restClient, streams, url, "", "GET")
}

// RawDelete uses the REST client to DELETE content
func RawDelete(restClient *rest.RESTClient, streams genericclioptions.IOStreams, url, filename string) error {
	return raw(restClient, streams, url, filename, "DELETE")
}

// raw makes a simple HTTP request to the provided path on the server using the default credentials.
func raw(restClient *rest.RESTClient, streams genericclioptions.IOStreams, url, filename, requestType string) error {
	var data io.Reader
	switch {
	case len(filename) == 0:
		data = bytes.NewBuffer([]byte{})

	case filename == "-":
		data = streams.In

	default:
		f, err := os.Open(filename)
		if err != nil {
			return err
		}
		defer f.Close()
		data = f
	}

	var request *rest.Request
	switch requestType {
	case "GET":
		request = restClient.Get().RequestURI(url)
	case "PUT":
		request = restClient.Put().RequestURI(url).Body(data)
	case "POST":
		request = restClient.Post().RequestURI(url).Body(data)
	case "DELETE":
		request = restClient.Delete().RequestURI(url).Body(data)

	default:
		return fmt.Errorf("unknown requestType: %q", requestType)
	}

	stream, err := request.Stream(context.TODO())
	if err != nil {
		return err
	}
	defer stream.Close()

	_, err = io.Copy(streams.Out, stream)
	if err != nil && err != io.EOF {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package gc // import "a4.io/blobstash/pkg/stash/gc"

import (
	"context"
	"fmt"

	"github.com/vmihailenco/msgpack"
	"github.com/yuin/gopher-lua"

	"a4.io/blobstash/pkg/apps/luautil"
	_ "a4.io/blobstash/pkg/blob"
	"a4.io/blobstash/pkg/stash"
	"a4.io/blobstash/pkg/stash/store"
)

// XXX(tsileo): take store interface, and exec store Lua script that can
// read a blob, or a key/version(/iterate keys?) and
// can get the blob for a kv(key, version) and "mark" blob for GC

type GarbageCollector struct {
	dataContext store.DataContext
	stash       *stash.Stash
	L           *lua.LState
	refs        []string
}

func New(s *stash.Stash, dc store.DataContext) *GarbageCollector {
	L := lua.NewState()
	res := &GarbageCollector{
		L:           L,
		dataContext: dc,
		refs:        []string{},
		stash:       s,
	}

	// mark(<blob hash>) is the lowest-level func, it "mark"s a blob to be copied to the root blobstore
	mark := func(L *lua.LState) int {
		ref := L.ToString(1)
		res.refs = append(res.refs, ref)
		return 0
	}
	L.SetGlobal("mark", L.NewFunction(mark))
	L.PreloadModule("json", loadJSON)
	L.PreloadModule("msgpack", loadMsgpack)
	bs, err := newBlobstore(L, dc)
	if err != nil {
		panic(err)
	}
	kvs, err := newKvstore(L, dc)
	if err != nil {
		panic(err)
	}
	rootTable := L.CreateTable(0, 2)
	rootTable.RawSetH(lua.LString("blobstore"), bs)
	rootTable.RawSetH(lua.LString("kvstore"), kvs)
	L.SetGlobal("blobstash", rootTable)
	if err := L.DoString(`
local msgpack = require('msgpack')
function mark_kv (key, version)
  local h = blobstash.kvstore:get_meta_blob(key, version)
  if h ~= nil then
    mark(h)
    local _, ref = blobstash.kvstore:get(key, version)
    if ref ~= '' then
      mark(ref)
    end
  end
end
_G.mark_kv = mark_kv
function mark_filetree_node (ref)
  local data = blobstash.blobstore:get(ref)
  local node = msgpack.decode(data)
  if node.t == 'dir' then
    for _, childRef in ipairs(node.r) do
      mark_filetree_node(childRef)
    end
  else
    for _, contentRef in ipairs(node.r) do
      mark(contentRef[2])
    end
  end
end
_G.mark_filetree_node = mark_filetree_node
`); err != nil {
		panic(err)
	}
	// FIXME(tsileo): do like in the docstore, export code _G.mark_kv(key, version), _G.mark_fs_ref(ref)...
	// and the option to load custom GC script from the filesystem like stored queries
	return res
}

func (gc *GarbageCollector) GC(ctx context.Context, script string) error {
	if err := gc.L.DoString(script); err != nil {
		return err
	}
	fmt.Printf("refs=%+v\n", gc.refs)
	// for _, ref := range gc.refs {
	// 	// FIXME(tsileo): stat before get/put

	// 	data, err := gc.dataContext.BlobStore().Get(ctx, ref)
	// 	if err != nil {
	// 		return err
	// 	}

	// 	if err := gc.stash.Root().BlobStore().Put(ctx, &blob.Blob{Hash: ref, Data: data}); err != nil {
	// 		return err
	// 	}
	// }
	return nil
	// return gc.dataContext.Destroy()
}

func loadMsgpack(L *lua.LState) int {
	// register functions to the table
	mod := L.SetFuncs(L.NewTable(), map[string]lua.LGFunction{
		"decode": msgpackDecode,
		"encode": msgpackEncode,
	})
	// returns the module
	L.Push(mod)
	return 1
}

func msgpackEncode(L *lua.LState) int {
	data := L.CheckAny(1)
	if data == nil {
		L.Push(lua.LNil)
		return 1
	}
	txt, err := msgpack.Marshal(data)
	if err != nil {
		panic(err)
	}
	L.Push(lua.LString(string(txt)))
	return 1
}

// TODO(tsileo): a note about empty list vs empty object
func msgpackDecode(L *lua.LState) int {
	data := L.ToString(1)
	out := map[string]interface{}{}
	if err := msgpack.Unmarshal([]byte(data), &out); err != nil {
		panic(err)
	}
	L.Push(luautil.InterfaceToLValue(L, out))
	return 1
}

func loadJSON(L *lua.LState) int {
	// register functions to the table
	mod := L.SetFuncs(L.NewTable(), map[string]lua.LGFunction{
		"decode": jsonDecode,
		"encode": jsonEncode,
	})
	// returns the module
	L.Push(mod)
	return 1
}

func jsonEncode(L *lua.LState) int {
	data := L.CheckAny(1)
	if data == nil {
		L.Push(lua.LNil)
		return 1
	}
	L.Push(lua.LString(string(luautil.ToJSON(data))))
	return 1
}

// TODO(tsileo): a note about empty list vs empty object
func jsonDecode(L *lua.LState) int {
	data := L.ToString(1)
	L.Push(luautil.FromJSON(L, []byte(data)))
	return 1
}

type blobstore struct {
	dc store.DataContext
}

func newBlobstore(L *lua.LState, dc store.DataContext) (*lua.LUserData, error) {
	bs := &blobstore{dc}
	mt := L.NewTypeMetatable("blobstore")
	L.SetField(mt, "__index", L.SetFuncs(L.NewTable(), map[string]lua.LGFunction{
		"get":  blobstoreGet,
		"stat": blobstoreStat,
	}))
	ud := L.NewUserData()
	ud.Value = bs
	L.SetMetatable(ud, L.GetTypeMetatable("blobstore"))
	return ud, nil
}

func checkBlobstore(L *lua.LState) *blobstore {
	ud := L.CheckUserData(1)
	if v, ok := ud.Value.(*blobstore); ok {
		return v
	}
	L.ArgError(1, "blobstore expected")
	return nil
}

func blobstoreStat(L *lua.LState) int {
	bs := checkBlobstore(L)
	if bs == nil {
		return 1
	}
	data, err := bs.dc.BlobStoreProxy().Stat(context.TODO(), L.ToString(2))
	if err != nil {
		L.Push(lua.LNil)
		return 1
		// TODO(tsileo): handle not found
	}
	if data == true {
		L.Push(lua.LTrue)
	} else {
		L.Push(lua.LFalse)
	}
	return 1
}

func blobstoreGet(L *lua.LState) int {
	bs := checkBlobstore(L)
	if bs == nil {
		return 1
	}
	data, err := bs.dc.BlobStoreProxy().Get(context.TODO(), L.ToString(2))
	if err != nil {
		fmt.Printf("failed to fetch %s: %v\n", L.ToString(2), err)
		L.Push(lua.LNil)
		return 1
		// TODO(tsileo): handle not found
	}
	L.Push(lua.LString(data))
	return 1
	// L.Push(buildBody(L, request.body))
	// return 1
}

type kvstore struct {
	dc store.DataContext
}

func newKvstore(L *lua.LState, dc store.DataContext) (*lua.LUserData, error) {
	kvs := &kvstore{dc}
	mt := L.NewTypeMetatable("kvstore")
	L.SetField(mt, "__index", L.SetFuncs(L.NewTable(), map[string]lua.LGFunction{
		"get_meta_blob": kvstoreGetMetaBlob,
		"get":           kvstoreGet,
	}))
	ud := L.NewUserData()
	ud.Value = kvs
	L.SetMetatable(ud, L.GetTypeMetatable("kvstore"))
	return ud, nil
}

func checkKvstore(L *lua.LState) *kvstore {
	ud := L.CheckUserData(1)
	if v, ok := ud.Value.(*kvstore); ok {
		return v
	}
	L.ArgError(1, "kvstore expected")
	return nil
}

func kvstoreGet(L *lua.LState) int {
	kv := checkKvstore(L)
	if kv == nil {
		return 1
	}
	fkv, err := kv.dc.KvStoreProxy().Get(context.TODO(), L.ToString(2), L.ToInt(3))
	if err != nil {
		panic(err)
	}
	L.Push(lua.LString(fkv.Data))
	L.Push(lua.LString(fkv.HexHash()))
	return 2
}

func kvstoreGetMetaBlob(L *lua.LState) int {
	kv := checkKvstore(L)
	if kv == nil {
		return 1
	}
	data, err := kv.dc.KvStoreProxy().GetMetaBlob(context.TODO(), L.ToString(2), L.ToInt(3))
	if err != nil {
		L.Push(lua.LNil)
		return 1
		// TODO(tsileo): handle not found
	}
	L.Push(lua.LString(data))
	return 1
}
<commit_msg>stash/gc: remove debug<commit_after>package gc // import "a4.io/blobstash/pkg/stash/gc"

import (
	"context"
	"fmt"

	"github.com/vmihailenco/msgpack"
	"github.com/yuin/gopher-lua"

	"a4.io/blobstash/pkg/apps/luautil"
	_ "a4.io/blobstash/pkg/blob"
	"a4.io/blobstash/pkg/stash"
	"a4.io/blobstash/pkg/stash/store"
)

// XXX(tsileo): take store interface, and exec store Lua script that can
// read a blob, or a key/version(/iterate keys?) and
// can get the blob for a kv(key, version) and "mark" blob for GC

type GarbageCollector struct {
	dataContext store.DataContext
	stash       *stash.Stash
	L           *lua.LState
	refs        []string
}

func New(s *stash.Stash, dc store.DataContext) *GarbageCollector {
	L := lua.NewState()
	res := &GarbageCollector{
		L:           L,
		dataContext: dc,
		refs:        []string{},
		stash:       s,
	}

	// mark(<blob hash>) is the lowest-level func, it "mark"s a blob to be copied to the root blobstore
	mark := func(L *lua.LState) int {
		ref := L.ToString(1)
		res.refs = append(res.refs, ref)
		return 0
	}
	L.SetGlobal("mark", L.NewFunction(mark))
	L.PreloadModule("json", loadJSON)
	L.PreloadModule("msgpack", loadMsgpack)
	bs, err := newBlobstore(L, dc)
	if err != nil {
		panic(err)
	}
	kvs, err := newKvstore(L, dc)
	if err != nil {
		panic(err)
	}
	rootTable := L.CreateTable(0, 2)
	rootTable.RawSetH(lua.LString("blobstore"), bs)
	rootTable.RawSetH(lua.LString("kvstore"), kvs)
	L.SetGlobal("blobstash", rootTable)
	if err := L.DoString(`
local msgpack = require('msgpack')
function mark_kv (key, version)
  local h = blobstash.kvstore:get_meta_blob(key, version)
  if h ~= nil then
    mark(h)
    local _, ref = blobstash.kvstore:get(key, version)
    if ref ~= '' then
      mark(ref)
    end
  end
end
_G.mark_kv = mark_kv
function mark_filetree_node (ref)
  local data = blobstash.blobstore:get(ref)
  local node = msgpack.decode(data)
  if node.t == 'dir' then
    for _, childRef in ipairs(node.r) do
      mark_filetree_node(childRef)
    end
  else
    for _, contentRef in ipairs(node.r) do
      mark(contentRef[2])
    end
  end
end
_G.mark_filetree_node = mark_filetree_node
`); err != nil {
		panic(err)
	}
	// FIXME(tsileo): do like in the docstore, export code _G.mark_kv(key, version), _G.mark_fs_ref(ref)...
	// and the option to load custom GC script from the filesystem like stored queries
	return res
}

func (gc *GarbageCollector) GC(ctx context.Context, script string) error {
	if err := gc.L.DoString(script); err != nil {
		return err
	}
	fmt.Printf("refs=%+v\n", gc.refs)
	for _, ref := range gc.refs {
		// FIXME(tsileo): stat before get/put

		data, err := gc.dataContext.BlobStore().Get(ctx, ref)
		if err != nil {
			return err
		}

		if err := gc.stash.Root().BlobStore().Put(ctx, &blob.Blob{Hash: ref, Data: data}); err != nil {
			return err
		}
	}
	return nil
	// return gc.dataContext.Destroy()
}

func loadMsgpack(L *lua.LState) int {
	// register functions to the table
	mod := L.SetFuncs(L.NewTable(), map[string]lua.LGFunction{
		"decode": msgpackDecode,
		"encode": msgpackEncode,
	})
	// returns the module
	L.Push(mod)
	return 1
}

func msgpackEncode(L *lua.LState) int {
	data := L.CheckAny(1)
	if data == nil {
		L.Push(lua.LNil)
		return 1
	}
	txt, err := msgpack.Marshal(data)
	if err != nil {
		panic(err)
	}
	L.Push(lua.LString(string(txt)))
	return 1
}

// TODO(tsileo): a note about empty list vs empty object
func msgpackDecode(L *lua.LState) int {
	data := L.ToString(1)
	out := map[string]interface{}{}
	if err := msgpack.Unmarshal([]byte(data), &out); err != nil {
		panic(err)
	}
	L.Push(luautil.InterfaceToLValue(L, out))
	return 1
}

func loadJSON(L *lua.LState) int {
	// register functions to the table
	mod := L.SetFuncs(L.NewTable(), map[string]lua.LGFunction{
		"decode": jsonDecode,
		"encode": jsonEncode,
	})
	// returns the module
	L.Push(mod)
	return 1
}

func jsonEncode(L *lua.LState) int {
	data := L.CheckAny(1)
	if data == nil {
		L.Push(lua.LNil)
		return 1
	}
	L.Push(lua.LString(string(luautil.ToJSON(data))))
	return 1
}

// TODO(tsileo): a note about empty list vs empty object
func jsonDecode(L *lua.LState) int {
	data := L.ToString(1)
	L.Push(luautil.FromJSON(L, []byte(data)))
	return 1
}

type blobstore struct {
	dc store.DataContext
}

func newBlobstore(L *lua.LState, dc store.DataContext) (*lua.LUserData, error) {
	bs := &blobstore{dc}
	mt := L.NewTypeMetatable("blobstore")
	L.SetField(mt, "__index", L.SetFuncs(L.NewTable(), map[string]lua.LGFunction{
		"get":  blobstoreGet,
		"stat": blobstoreStat,
	}))
	ud := L.NewUserData()
	ud.Value = bs
	L.SetMetatable(ud, L.GetTypeMetatable("blobstore"))
	return ud, nil
}

func checkBlobstore(L *lua.LState) *blobstore {
	ud := L.CheckUserData(1)
	if v, ok := ud.Value.(*blobstore); ok {
		return v
	}
	L.ArgError(1, "blobstore expected")
	return nil
}

func blobstoreStat(L *lua.LState) int {
	bs := checkBlobstore(L)
	if bs == nil {
		return 1
	}
	data, err := bs.dc.BlobStoreProxy().Stat(context.TODO(), L.ToString(2))
	if err != nil {
		L.Push(lua.LNil)
		return 1
		// TODO(tsileo): handle not found
	}
	if data == true {
		L.Push(lua.LTrue)
	} else {
		L.Push(lua.LFalse)
	}
	return 1
}

func blobstoreGet(L *lua.LState) int {
	bs := checkBlobstore(L)
	if bs == nil {
		return 1
	}
	data, err := bs.dc.BlobStoreProxy().Get(context.TODO(), L.ToString(2))
	if err != nil {
		fmt.Printf("failed to fetch %s: %v\n", L.ToString(2), err)
		L.Push(lua.LNil)
		return 1
		// TODO(tsileo): handle not found
	}
	L.Push(lua.LString(data))
	return 1
	// L.Push(buildBody(L, request.body))
	// return 1
}

type kvstore struct {
	dc store.DataContext
}

func newKvstore(L *lua.LState, dc store.DataContext) (*lua.LUserData, error) {
	kvs := &kvstore{dc}
	mt := L.NewTypeMetatable("kvstore")
	L.SetField(mt, "__index", L.SetFuncs(L.NewTable(), map[string]lua.LGFunction{
		"get_meta_blob": kvstoreGetMetaBlob,
		"get":           kvstoreGet,
	}))
	ud := L.NewUserData()
	ud.Value = kvs
	L.SetMetatable(ud, L.GetTypeMetatable("kvstore"))
	return ud, nil
}

func checkKvstore(L *lua.LState) *kvstore {
	ud := L.CheckUserData(1)
	if v, ok := ud.Value.(*kvstore); ok {
		return v
	}
	L.ArgError(1, "kvstore expected")
	return nil
}

func kvstoreGet(L *lua.LState) int {
	kv := checkKvstore(L)
	if kv == nil {
		return 1
	}
	fkv, err := kv.dc.KvStoreProxy().Get(context.TODO(), L.ToString(2), L.ToInt(3))
	if err != nil {
		panic(err)
	}
	L.Push(lua.LString(fkv.Data))
	L.Push(lua.LString(fkv.HexHash()))
	return 2
}

func kvstoreGetMetaBlob(L *lua.LState) int {
	kv := checkKvstore(L)
	if kv == nil {
		return 1
	}
	data, err := kv.dc.KvStoreProxy().GetMetaBlob(context.TODO(), L.ToString(2), L.ToInt(3))
	if err != nil {
		L.Push(lua.LNil)
		return 1
		// TODO(tsileo): handle not found
	}
	L.Push(lua.LString(data))
	return 1
}
<|endoftext|>


<commit_before>// +build linux

package store

import (
	"bytes"
	"crypto/x509"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/adamdecaf/cert-manage/pkg/certutil"
	"github.com/adamdecaf/cert-manage/pkg/file"
	"github.com/adamdecaf/cert-manage/pkg/whitelist"
)

type cadir struct {
	// directory for new/custom certificates
	add string

	// base dir for all ca certs
	dir string

	// the filepath containing all certs (optional)
	all string

	// where to save a backup of all certs
	backup string
}

func (ca *cadir) empty() bool {
	if ca == nil {
		return false
	}
	path, err := filepath.Abs(ca.all)
	return err != nil || !file.Exists(path)
}

var (
	// From Go's source, src/crypto/x509/root_linux.go
	cadirs = []cadir{
		// Debian/Ubuntu/Gentoo/etc..
		{
			add:    "/usr/local/share/ca-certificates",
			dir:    "/usr/share/ca-certificates",
			all:    "/etc/ssl/certs/ca-certificates.crt",
			backup: "/usr/share/ca-certificates.backup",
		},
	}
)

type linuxStore struct {
	ca cadir
}

func platform() Store {
	var ca cadir
	// find the cadir, if it exists
	for _, ca = range cadirs {
		if ca.empty() {
			break
		}
	}

	return linuxStore{
		ca: ca,
	}
}

func (s linuxStore) Add(certs []*x509.Certificate) error {
	if s.ca.empty() {
		return errors.New("unable to find certificate directory")
	}

	// install each certificate
	for i := range certs {
		fp := certutil.GetHexSHA256Fingerprint(*certs[i])
		path := filepath.Join(s.ca.add, fmt.Sprintf("%s.crt", fp))

		err := certutil.ToFile(path, certs[i:i+1])
		if err != nil {
			return err
		}
	}

	if len(certs) > 0 {
		return bundleCerts()
	}
	return nil
}

// Backup takes a snapshot of the current set of CA certificates and
// saves them to another location. It will overwrite any previous backup.
func (s linuxStore) Backup() error {
	return file.MirrorDir(s.ca.dir, s.ca.backup)
}

func (s linuxStore) GetInfo() *Info {
	// TODO(adam): What does this mean on linux? OS name/version? Kernel version?
	// How about the bigger unix world?
	return &Info{}
}

func (s linuxStore) List() ([]*x509.Certificate, error) {
	if s.ca.empty() {
		return nil, nil
	}

	bytes, err := ioutil.ReadFile(s.ca.all)
	if err != nil {
		return nil, err
	}

	certs, err := certutil.ParsePEM(bytes)
	if err != nil {
		return nil, err
	}

	return certs, nil
}

// Remove walks through the installed CA certificates on a linux based
// machine and deactivates those that are not to be trusted.
//
// Steps
// 1. Walk through the dir (/etc/ssl/certs/) and chmod 000 the certs we aren't trusting
// 2. Run `update-ca-certificates` to re-create the ca-certificates.crt file
func (s linuxStore) Remove(wh whitelist.Whitelist) error {
	// Check each CA cert file and optionally disable
	walk := func(path string, info os.FileInfo, err error) error {
		// Ignore SkipDir and directories
		if (err != nil && err != filepath.SkipDir) || info.IsDir() {
			return nil
		}

		// read the cert(s) contained at the file and only keep those
		// that aren't removable
		read, err := certutil.FromFile(path)
		if err != nil {
			return err
		}
		for i := 0; i < len(read); i++ {
			// Remove the cert if we don't match
			if !wh.Matches(read[i]) {
				read = append(read[:i], read[i+1:]...)
				if len(read) == 0 {
					break
				}
			}
		}

		// otherwise, write kept certs from `read` back
		err = certutil.ToFile(path, read)
		if err != nil {
			return err
		}

		return nil
	}

	// Walk the fs and deactivate each cert
	err := filepath.Walk(s.ca.dir, walk)
	if err != nil {
		return err
	}

	return bundleCerts()
}

func (s linuxStore) Restore(where string) error {
	if !file.Exists(s.ca.backup) {
		return errors.New("No backup directory exists")
	}
	// Remove the current dir
	if file.Exists(s.ca.dir) {
		err := os.RemoveAll(s.ca.dir)
		if err != nil && !os.IsNotExist(err) {
			return err
		}
	}

	// Restore
	err := file.MirrorDir(s.ca.backup, s.ca.dir)
	if err != nil {
		return err
	}
	return bundleCerts()
}

// Update the certs trust system-wide
func bundleCerts() error {
	var out bytes.Buffer
	// TOOD(adam): Check for sudo/su
	cmd := exec.Command("/usr/sbin/update-ca-certificates")
	cmd.Stdout = &out

	if debug {
		fmt.Println("store/linux: updated CA certificates")
	}

	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("error updating trust status: err=%v, out=%s", err, out.String())
	}
	return nil
}
<commit_msg>store/linux: use uname for name and version<commit_after>// +build linux

package store

import (
	"bytes"
	"crypto/x509"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/adamdecaf/cert-manage/pkg/certutil"
	"github.com/adamdecaf/cert-manage/pkg/file"
	"github.com/adamdecaf/cert-manage/pkg/whitelist"
)

type cadir struct {
	// directory for new/custom certificates
	add string

	// base dir for all ca certs
	dir string

	// the filepath containing all certs (optional)
	all string

	// where to save a backup of all certs
	backup string
}

func (ca *cadir) empty() bool {
	if ca == nil {
		return false
	}
	path, err := filepath.Abs(ca.all)
	return err != nil || !file.Exists(path)
}

var (
	// From Go's source, src/crypto/x509/root_linux.go
	cadirs = []cadir{
		// Debian/Ubuntu/Gentoo/etc..
		{
			add:    "/usr/local/share/ca-certificates",
			dir:    "/usr/share/ca-certificates",
			all:    "/etc/ssl/certs/ca-certificates.crt",
			backup: "/usr/share/ca-certificates.backup",
		},
	}
)

type linuxStore struct {
	ca cadir
}

func platform() Store {
	var ca cadir
	// find the cadir, if it exists
	for _, ca = range cadirs {
		if ca.empty() {
			break
		}
	}

	return linuxStore{
		ca: ca,
	}
}

func (s linuxStore) Add(certs []*x509.Certificate) error {
	if s.ca.empty() {
		return errors.New("unable to find certificate directory")
	}

	// install each certificate
	for i := range certs {
		fp := certutil.GetHexSHA256Fingerprint(*certs[i])
		path := filepath.Join(s.ca.add, fmt.Sprintf("%s.crt", fp))

		err := certutil.ToFile(path, certs[i:i+1])
		if err != nil {
			return err
		}
	}

	if len(certs) > 0 {
		return bundleCerts()
	}
	return nil
}

// Backup takes a snapshot of the current set of CA certificates and
// saves them to another location. It will overwrite any previous backup.
func (s linuxStore) Backup() error {
	return file.MirrorDir(s.ca.dir, s.ca.backup)
}

func (s linuxStore) uname(args ...string) string {
	out, err := exec.Command("uname", args...).CombinedOutput()
	if err != nil {
		return ""
	}
	return strings.TrimSpace(string(out))
}

func (s linuxStore) GetInfo() *Info {
	return &Info{
		Name:    s.uname("-o"), // GNU/Linux,
		Version: s.uname("-r"), // 4.9.60-linuxkit-aufs
	}
}

func (s linuxStore) List() ([]*x509.Certificate, error) {
	if s.ca.empty() {
		return nil, nil
	}

	bytes, err := ioutil.ReadFile(s.ca.all)
	if err != nil {
		return nil, err
	}

	certs, err := certutil.ParsePEM(bytes)
	if err != nil {
		return nil, err
	}

	return certs, nil
}

// Remove walks through the installed CA certificates on a linux based
// machine and deactivates those that are not to be trusted.
//
// Steps
// 1. Walk through the dir (/etc/ssl/certs/) and chmod 000 the certs we aren't trusting
// 2. Run `update-ca-certificates` to re-create the ca-certificates.crt file
func (s linuxStore) Remove(wh whitelist.Whitelist) error {
	// Check each CA cert file and optionally disable
	walk := func(path string, info os.FileInfo, err error) error {
		// Ignore SkipDir and directories
		if (err != nil && err != filepath.SkipDir) || info.IsDir() {
			return nil
		}

		// read the cert(s) contained at the file and only keep those
		// that aren't removable
		read, err := certutil.FromFile(path)
		if err != nil {
			return err
		}
		for i := 0; i < len(read); i++ {
			// Remove the cert if we don't match
			if !wh.Matches(read[i]) {
				read = append(read[:i], read[i+1:]...)
				if len(read) == 0 {
					break
				}
			}
		}

		// otherwise, write kept certs from `read` back
		err = certutil.ToFile(path, read)
		if err != nil {
			return err
		}

		return nil
	}

	// Walk the fs and deactivate each cert
	err := filepath.Walk(s.ca.dir, walk)
	if err != nil {
		return err
	}

	return bundleCerts()
}

func (s linuxStore) Restore(where string) error {
	if !file.Exists(s.ca.backup) {
		return errors.New("No backup directory exists")
	}
	// Remove the current dir
	if file.Exists(s.ca.dir) {
		err := os.RemoveAll(s.ca.dir)
		if err != nil && !os.IsNotExist(err) {
			return err
		}
	}

	// Restore
	err := file.MirrorDir(s.ca.backup, s.ca.dir)
	if err != nil {
		return err
	}
	return bundleCerts()
}

// Update the certs trust system-wide
func bundleCerts() error {
	var out bytes.Buffer
	// TOOD(adam): Check for sudo/su
	cmd := exec.Command("/usr/sbin/update-ca-certificates")
	cmd.Stdout = &out

	if debug {
		fmt.Println("store/linux: updated CA certificates")
	}

	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("error updating trust status: err=%v, out=%s", err, out.String())
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/kubernetes-incubator/apiserver-builder/pkg/builders"
	"github.com/kubernetes-incubator/apiserver-builder/pkg/cmd/server"
	openapi "k8s.io/apimachinery/pkg/openapi"
	genericapiserver "k8s.io/apiserver/pkg/server"
	genericoptions "k8s.io/apiserver/pkg/server/options"
	"k8s.io/client-go/rest"
)

type TestEnvironment struct {
	StopServer     chan struct{}
	ServerOuput    *io.PipeWriter
	ApiserverPort  int
	BearerToken    string
	EtcdClientPort int
	EtcdPeerPort   int
	EtcdPath       string
	EtcdCmd        *exec.Cmd
	Done           bool
}

func NewTestEnvironment() *TestEnvironment {
	return &TestEnvironment{
		EtcdPath: "/registry/test.kubernetes.io",
	}
}

func (te *TestEnvironment) getPort() int {
	l, _ := net.Listen("tcp", ":0")
	defer l.Close()
	println(l.Addr().String())
	pieces := strings.Split(l.Addr().String(), ":")
	i, err := strconv.Atoi(pieces[len(pieces)-1])
	if err != nil {
		panic(err)
	}
	return i
}

// Stop stops a running server
func (te *TestEnvironment) Stop() {
	te.Done = true
	te.StopServer <- struct{}{}
	te.EtcdCmd.Process.Kill()
}

// Start starts a local Kubernetes server and updates te.ApiserverPort with the port it is listening on
func (te *TestEnvironment) Start(
	apis []*builders.APIGroupBuilder, openapidefs openapi.GetOpenAPIDefinitions) *rest.Config {

	te.EtcdClientPort = te.getPort()
	te.EtcdPeerPort = te.getPort()
	te.ApiserverPort = te.getPort()

	etcdready := make(chan string)
	go te.startEtcd(etcdready)

	// Wait for etcd to start
	// TODO: Poll the /health address to wait for etcd to become healthy
	time.Sleep(time.Second * 1)

	apiserverready := make(chan *rest.Config)
	go te.startApiserver(apiserverready, apis, openapidefs)

	// Wait for everything to be ready
	loopback := <-apiserverready
	<-etcdready
	return loopback
}

func (te *TestEnvironment) startApiserver(
	ready chan *rest.Config, apis []*builders.APIGroupBuilder, openapidefs openapi.GetOpenAPIDefinitions) {
	te.StopServer = make(chan struct{})
	_, te.ServerOuput = io.Pipe()
	server.GetOpenApiDefinition = openapidefs
	cmd, options := server.NewCommandStartServer(
		te.EtcdPath,
		te.ServerOuput, te.ServerOuput, apis, te.StopServer, "API", "v0")

	options.RecommendedOptions.SecureServing.BindPort = te.ApiserverPort
	options.RunDelegatedAuth = false
	options.RecommendedOptions.Etcd.StorageConfig.ServerList = []string{
		fmt.Sprintf("http://localhost:%d", te.EtcdClientPort),
	}
	options.RecommendedOptions.SecureServing.ServerCert = genericoptions.GeneratableKeyCert{}

	// Notify once the apiserver is ready to serve traffic
	options.PostStartHooks = []server.PostStartHook{
		{
			func(context genericapiserver.PostStartHookContext) error {
				// Let the test know the server is ready
				ready <- context.LoopbackClientConfig
				return nil
			},
			"apiserver-ready",
		},
	}

	if err := cmd.Execute(); err != nil {
		panic(err)
	}
}

// startEtcd starts a new etcd process using a random temp data directory and random free port
func (te *TestEnvironment) startEtcd(ready chan string) {
	dirname, err := ioutil.TempDir("/tmp", "apiserver-test")
	if err != nil {
		panic(err)
	}

	clientAddr := fmt.Sprintf("http://localhost:%d", te.EtcdClientPort)
	peerAddr := fmt.Sprintf("http://localhost:%d", te.EtcdPeerPort)
	cmd := exec.Command(
		"etcd",
		"--data-dir", dirname,
		"--listen-client-urls", clientAddr,
		"--listen-peer-urls", peerAddr,
		"--advertise-client-urls", clientAddr,
	)
	te.EtcdCmd = cmd
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		panic(err)
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		panic(err)
	}

	err = cmd.Start()
	if err != nil {
		panic(err)
	}

	go te.waitForEtcdReady(ready, stdout)
	go te.waitForEtcdReady(ready, stderr)

	err = cmd.Wait()
	if err != nil && !te.Done {
		panic(err)
	}
}

// waitForEtcdReady notify's read once the etcd instances is ready to receive traffic
func (te *TestEnvironment) waitForEtcdReady(ready chan string, reader io.Reader) {
	started := regexp.MustCompile("serving insecure client requests on (.+), this is strongly discouraged!")
	buffered := bufio.NewReader(reader)
	for {
		l, _, err := buffered.ReadLine()
		if err != nil {
			time.Sleep(time.Second * 5)
		}
		line := string(l)
		if started.MatchString(line) {
			addr := started.FindStringSubmatch(line)[1]
			// etcd is ready
			ready <- addr
			return
		}
	}
}
<commit_msg>Update test to create apiserver cert in temp dir<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/kubernetes-incubator/apiserver-builder/pkg/builders"
	"github.com/kubernetes-incubator/apiserver-builder/pkg/cmd/server"
	openapi "k8s.io/apimachinery/pkg/openapi"
	genericapiserver "k8s.io/apiserver/pkg/server"
	genericoptions "k8s.io/apiserver/pkg/server/options"
	"k8s.io/client-go/rest"
)

type TestEnvironment struct {
	StopServer     chan struct{}
	ServerOuput    *io.PipeWriter
	ApiserverPort  int
	BearerToken    string
	EtcdClientPort int
	EtcdPeerPort   int
	EtcdPath       string
	EtcdCmd        *exec.Cmd
	Done           bool
}

func NewTestEnvironment() *TestEnvironment {
	return &TestEnvironment{
		EtcdPath: "/registry/test.kubernetes.io",
	}
}

func (te *TestEnvironment) getPort() int {
	l, _ := net.Listen("tcp", ":0")
	defer l.Close()
	println(l.Addr().String())
	pieces := strings.Split(l.Addr().String(), ":")
	i, err := strconv.Atoi(pieces[len(pieces)-1])
	if err != nil {
		panic(err)
	}
	return i
}

// Stop stops a running server
func (te *TestEnvironment) Stop() {
	te.Done = true
	te.StopServer <- struct{}{}
	te.EtcdCmd.Process.Kill()
}

// Start starts a local Kubernetes server and updates te.ApiserverPort with the port it is listening on
func (te *TestEnvironment) Start(
	apis []*builders.APIGroupBuilder, openapidefs openapi.GetOpenAPIDefinitions) *rest.Config {

	te.EtcdClientPort = te.getPort()
	te.EtcdPeerPort = te.getPort()
	te.ApiserverPort = te.getPort()

	etcdready := make(chan string)
	go te.startEtcd(etcdready)

	// Wait for etcd to start
	// TODO: Poll the /health address to wait for etcd to become healthy
	time.Sleep(time.Second * 1)

	apiserverready := make(chan *rest.Config)
	go te.startApiserver(apiserverready, apis, openapidefs)

	// Wait for everything to be ready
	loopback := <-apiserverready
	<-etcdready
	return loopback
}

func (te *TestEnvironment) startApiserver(
	ready chan *rest.Config, apis []*builders.APIGroupBuilder, openapidefs openapi.GetOpenAPIDefinitions) {
	te.StopServer = make(chan struct{})
	_, te.ServerOuput = io.Pipe()
	server.GetOpenApiDefinition = openapidefs
	cmd, options := server.NewCommandStartServer(
		te.EtcdPath,
		te.ServerOuput, te.ServerOuput, apis, te.StopServer, "API", "v0")

	options.RecommendedOptions.SecureServing.BindPort = te.ApiserverPort
	options.RunDelegatedAuth = false
	options.RecommendedOptions.Etcd.StorageConfig.ServerList = []string{
		fmt.Sprintf("http://localhost:%d", te.EtcdClientPort),
	}
	tmpdir, err := ioutil.TempDir("", "apiserver-test")
	if err != nil {
		panic(fmt.Sprintf("Could not create temp dir for testing: %v", err))
	}
	options.RecommendedOptions.SecureServing.ServerCert = genericoptions.GeneratableKeyCert{
		CertDirectory: tmpdir,
	}

	// Notify once the apiserver is ready to serve traffic
	options.PostStartHooks = []server.PostStartHook{
		{
			func(context genericapiserver.PostStartHookContext) error {
				// Let the test know the server is ready
				ready <- context.LoopbackClientConfig
				return nil
			},
			"apiserver-ready",
		},
	}

	if err := cmd.Execute(); err != nil {
		panic(err)
	}
}

// startEtcd starts a new etcd process using a random temp data directory and random free port
func (te *TestEnvironment) startEtcd(ready chan string) {
	dirname, err := ioutil.TempDir("/tmp", "apiserver-test")
	if err != nil {
		panic(err)
	}

	clientAddr := fmt.Sprintf("http://localhost:%d", te.EtcdClientPort)
	peerAddr := fmt.Sprintf("http://localhost:%d", te.EtcdPeerPort)
	cmd := exec.Command(
		"etcd",
		"--data-dir", dirname,
		"--listen-client-urls", clientAddr,
		"--listen-peer-urls", peerAddr,
		"--advertise-client-urls", clientAddr,
	)
	te.EtcdCmd = cmd
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		panic(err)
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		panic(err)
	}

	err = cmd.Start()
	if err != nil {
		panic(err)
	}

	go te.waitForEtcdReady(ready, stdout)
	go te.waitForEtcdReady(ready, stderr)

	err = cmd.Wait()
	if err != nil && !te.Done {
		panic(err)
	}
}

// waitForEtcdReady notify's read once the etcd instances is ready to receive traffic
func (te *TestEnvironment) waitForEtcdReady(ready chan string, reader io.Reader) {
	started := regexp.MustCompile("serving insecure client requests on (.+), this is strongly discouraged!")
	buffered := bufio.NewReader(reader)
	for {
		l, _, err := buffered.ReadLine()
		if err != nil {
			time.Sleep(time.Second * 5)
		}
		line := string(l)
		if started.MatchString(line) {
			addr := started.FindStringSubmatch(line)[1]
			// etcd is ready
			ready <- addr
			return
		}
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2017 Sam Kumar, Michael Andersen, and the University
 * of California, Berkeley.
 *
 * This file is part of Mr. Plotter (the Multi-Resolution Plotter).
 *
 * Mr. Plotter is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published
 * by the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Mr. Plotter is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with Mr. Plotter.  If not, see <http://www.gnu.org/licenses/>.
 */

package csvquery

import (
	"context"
	"encoding/csv"
	"errors"
	"fmt"
	"math"
	"time"

	btrdb "gopkg.in/btrdb.v4"
)

const (
	// AlignedWindowsQuery specifies that an Aligned Windows query (i.e.
	// Statistical Values query) should be made.
	AlignedWindowsQuery = iota

	// WindowsQuery specifies that a Windows query should be made.
	WindowsQuery

	// RawQuery specifies that a Raw Values query should be made.
	RawQuery
)

// CSVQuery stores the parameters for a CSV query.
type CSVQuery struct {
	// QueryType should be one of AlignedWindowsQuery, WindowsQuery, or
	// RawQuery. It specifies what data should be in the CSV file for each
	// stream.
	QueryType int

	// StartTime is the start time for the query, in nanoseconds.
	StartTime int64

	// EndTime is the end time for the query, in nanoseconds.
	EndTime int64

	// WindowSize specifies the size of the window, in nanoseconds, for a
	// Windows query. It is ignored for Aligned Windows queries and Raw Values
	// queries.
	WindowSize uint64

	// Depth specifies the point width exponent for AlignedWindows queries, and
	// the maximum depth for Windows queries. It is ignored for Raw Values
	// queries.
	Depth uint8

	// Streams is a slice of streams to query.
	Streams []*btrdb.Stream

	// Versions is a slice of version numbers to query for each stream.
	// Defaults to 0 (most recent version) for all streams if nil.
	Versions []uint64

	// Labels contains the name to use for each stream in the output CSV file.
	Labels []string
}

type streamquery struct {
	rawc chan btrdb.RawPoint
	stac chan btrdb.StatPoint
	verc chan uint64
	errc chan error
}

// MakeCSVQuery performs a CSV query, and outputs the result using the provided
// CSV writer.
func MakeCSVQuery(ctx context.Context, b *btrdb.BTrDB, q *CSVQuery, w *csv.Writer) error {
	var numstreams = len(q.Streams)
	if numstreams != len(q.Labels) {
		return fmt.Errorf("Got %d streams but %d labels", len(q.Streams), len(q.Labels))
	}

	var versions = q.Versions
	if versions == nil {
		versions = make([]uint64, numstreams, numstreams)
	}

	/* State for each stream. */
	var sq = make([]streamquery, numstreams, numstreams)

	switch q.QueryType {
	case AlignedWindowsQuery:
		for i, s := range q.Streams {
			sq[i].stac, sq[i].verc, sq[i].errc = s.AlignedWindows(ctx, q.StartTime, q.EndTime, q.Depth, versions[i])
		}
		return createStatisticalCSV(sq, q, w)
	case WindowsQuery:
		for i, s := range q.Streams {
			sq[i].stac, sq[i].verc, sq[i].errc = s.Windows(ctx, q.StartTime, q.EndTime, q.WindowSize, q.Depth, versions[i])
		}
		return createStatisticalCSV(sq, q, w)
	case RawQuery:
		for i, s := range q.Streams {
			sq[i].rawc, sq[i].verc, sq[i].errc = s.RawValues(ctx, q.StartTime, q.EndTime, versions[i])
		}
	default:
		return errors.New("Invalid query type")
	}

	return nil
}

type statbufentry struct {
	pt   btrdb.StatPoint
	open bool
}

func createStatisticalCSV(sq []streamquery, q *CSVQuery, w *csv.Writer) error {
	var numstreams = len(sq)
	var numcols = 2 + (numstreams << 2)

	// Buffer for the row of the CSV that we are writing
	var row = make([]string, numcols, numcols)

	// Write the header row
	row[0] = "Timestamp (ns)"
	row[1] = "Date/Time"
	for i, label := range q.Labels {
		offset := 2 + (i << 2)
		row[offset+0] = fmt.Sprintf("%s (Min)", label)
		row[offset+1] = fmt.Sprintf("%s (Mean)", label)
		row[offset+2] = fmt.Sprintf("%s (Max)", label)
		row[offset+3] = fmt.Sprintf("%s (Count)", label)
	}

	var err = w.Write(row)
	if err != nil {
		return err
	}

	var buf = make([]statbufentry, numstreams, numstreams)
	var numopen = numstreams
	for i := range buf {
		buf[i].pt, buf[i].open = <-sq[i].stac
		if !buf[i].open {
			numopen--
			if err = <-sq[i].errc; err != nil {
				return err
			}
		}
	}

	for {
		// Compute the time of the next row
		var earliest int64 = math.MaxInt64
		for i := range buf {
			if buf[i].open && buf[i].pt.Time < earliest {
				earliest = buf[i].pt.Time
			}
		}

		// Compute the next row
		row[0] = fmt.Sprintf("%d", earliest)
		row[1] = time.Unix(0, earliest).Format(time.RFC3339Nano)
		for i := range buf {
			offset := 2 + (i << 2)
			if !buf[i].open {
				continue
			} else if buf[i].pt.Time == earliest {
				row[offset+0] = fmt.Sprintf("%f", buf[i].pt.Min)
				row[offset+1] = fmt.Sprintf("%f", buf[i].pt.Mean)
				row[offset+2] = fmt.Sprintf("%f", buf[i].pt.Max)
				row[offset+3] = fmt.Sprintf("%d", buf[i].pt.Count)

				// We consumed this point, so fetch the next point
				buf[i].pt, buf[i].open = <-sq[i].stac
				if !buf[i].open {
					numopen--
					if err = <-sq[i].errc; err != nil {
						return err
					}
				}
			} else {
				row[offset+0] = ""
				row[offset+1] = ""
				row[offset+2] = ""
				row[offset+3] = ""
			}
		}

		if numopen == 0 {
			break
		}

		// Emit the row
		err = w.Write(row)
		if err != nil {
			return nil
		}
	}

	return nil
}
<commit_msg>Get Raw Values CSV generation working<commit_after>/*
 * Copyright (C) 2017 Sam Kumar, Michael Andersen, and the University
 * of California, Berkeley.
 *
 * This file is part of Mr. Plotter (the Multi-Resolution Plotter).
 *
 * Mr. Plotter is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published
 * by the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Mr. Plotter is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with Mr. Plotter.  If not, see <http://www.gnu.org/licenses/>.
 */

package csvquery

import (
	"context"
	"encoding/csv"
	"errors"
	"fmt"
	"math"
	"time"

	btrdb "gopkg.in/btrdb.v4"
)

const (
	// AlignedWindowsQuery specifies that an Aligned Windows query (i.e.
	// Statistical Values query) should be made.
	AlignedWindowsQuery = iota

	// WindowsQuery specifies that a Windows query should be made.
	WindowsQuery

	// RawQuery specifies that a Raw Values query should be made.
	RawQuery
)

// CSVQuery stores the parameters for a CSV query.
type CSVQuery struct {
	// QueryType should be one of AlignedWindowsQuery, WindowsQuery, or
	// RawQuery. It specifies what data should be in the CSV file for each
	// stream.
	QueryType int

	// StartTime is the start time for the query, in nanoseconds.
	StartTime int64

	// EndTime is the end time for the query, in nanoseconds.
	EndTime int64

	// WindowSize specifies the size of the window, in nanoseconds, for a
	// Windows query. It is ignored for Aligned Windows queries and Raw Values
	// queries.
	WindowSize uint64

	// Depth specifies the point width exponent for AlignedWindows queries, and
	// the maximum depth for Windows queries. It is ignored for Raw Values
	// queries.
	Depth uint8

	// Streams is a slice of streams to query.
	Streams []*btrdb.Stream

	// Versions is a slice of version numbers to query for each stream.
	// Defaults to 0 (most recent version) for all streams if nil.
	Versions []uint64

	// Labels contains the name to use for each stream in the output CSV file.
	Labels []string
}

// streambuffer is a buffer for an array of pending requests, allowing the user
// to individually manipulate the response channels in a type-independent way.
type streambuffer interface {
	getTime(i int) int64
	isOpen(i int) bool
	readPoint(i int) (bool, error)
	writePoint(i int, row []string)
	writeEmptyPoint(i int, row []string)
	getHeaderRow(labels []string) []string
}

type stabufentry struct {
	stac chan btrdb.StatPoint
	verc chan uint64
	errc chan error
	pt   btrdb.StatPoint
	open bool
}

type stabuffer []stabufentry

func (sb stabuffer) getTime(i int) int64 {
	return sb[i].pt.Time
}

func (sb stabuffer) isOpen(i int) bool {
	return sb[i].open
}

func (sb stabuffer) readPoint(i int) (bool, error) {
	sb[i].pt, sb[i].open = <-sb[i].stac
	if !sb[i].open {
		err := <-sb[i].errc
		return sb[i].open, err
	}
	return sb[i].open, nil
}

func (sb stabuffer) writePoint(i int, row []string) {
	offset := 2 + (i << 2)
	row[offset+0] = fmt.Sprintf("%f", sb[i].pt.Min)
	row[offset+1] = fmt.Sprintf("%f", sb[i].pt.Mean)
	row[offset+2] = fmt.Sprintf("%f", sb[i].pt.Max)
	row[offset+3] = fmt.Sprintf("%d", sb[i].pt.Count)
}

func (sb stabuffer) writeEmptyPoint(i int, row []string) {
	offset := 2 + (i << 2)
	row[offset+0] = ""
	row[offset+1] = ""
	row[offset+2] = ""
	row[offset+3] = ""
}

func (sb stabuffer) getHeaderRow(labels []string) []string {
	numcols := 2 + (len(sb) << 2)
	row := make([]string, numcols, numcols)
	row[0] = "Timestamp (ns)"
	row[1] = "Date/Time"
	for i, label := range labels {
		offset := 2 + (i << 2)
		row[offset+0] = fmt.Sprintf("%s (Min)", label)
		row[offset+1] = fmt.Sprintf("%s (Mean)", label)
		row[offset+2] = fmt.Sprintf("%s (Max)", label)
		row[offset+3] = fmt.Sprintf("%s (Count)", label)
	}
	return row
}

type rawbufentry struct {
	rawc chan btrdb.RawPoint
	verc chan uint64
	errc chan error
	pt   btrdb.RawPoint
	open bool
}

type rawbuffer []rawbufentry

func (rb rawbuffer) getTime(i int) int64 {
	return rb[i].pt.Time
}

func (rb rawbuffer) isOpen(i int) bool {
	return rb[i].open
}

func (rb rawbuffer) readPoint(i int) (bool, error) {
	rb[i].pt, rb[i].open = <-rb[i].rawc
	if !rb[i].open {
		err := <-rb[i].errc
		return rb[i].open, err
	}
	return rb[i].open, nil
}

func (rb rawbuffer) writePoint(i int, row []string) {
	offset := 2 + i
	row[offset] = fmt.Sprintf("%f", rb[i].pt.Value)
}

func (rb rawbuffer) writeEmptyPoint(i int, row []string) {
	offset := 2 + i
	row[offset] = ""
}

func (rb rawbuffer) getHeaderRow(labels []string) []string {
	numcols := 2 + len(rb)
	row := make([]string, numcols, numcols)
	row[0] = "Timestamp (ns)"
	row[1] = "Date/Time"
	for i, label := range labels {
		offset := 2 + i
		row[offset+0] = label
	}
	return row
}

// MakeCSVQuery performs a CSV query, and outputs the result using the provided
// CSV writer.
func MakeCSVQuery(ctx context.Context, b *btrdb.BTrDB, q *CSVQuery, w *csv.Writer) error {
	var numstreams = len(q.Streams)
	if numstreams != len(q.Labels) {
		return fmt.Errorf("Got %d streams but %d labels", len(q.Streams), len(q.Labels))
	}

	var versions = q.Versions
	if versions == nil {
		versions = make([]uint64, numstreams, numstreams)
	}

	switch q.QueryType {
	case AlignedWindowsQuery:
		var sq stabuffer = make([]stabufentry, numstreams, numstreams)
		for i, s := range q.Streams {
			sq[i].stac, sq[i].verc, sq[i].errc = s.AlignedWindows(ctx, q.StartTime, q.EndTime, q.Depth, versions[i])
		}
		return createCSV(sq, q, w, true)
	case WindowsQuery:
		var sq stabuffer = make([]stabufentry, numstreams, numstreams)
		for i, s := range q.Streams {
			sq[i].stac, sq[i].verc, sq[i].errc = s.Windows(ctx, q.StartTime, q.EndTime, q.WindowSize, q.Depth, versions[i])
		}
		return createCSV(sq, q, w, true)
	case RawQuery:
		var sq rawbuffer = make([]rawbufentry, numstreams, numstreams)
		for i, s := range q.Streams {
			sq[i].rawc, sq[i].verc, sq[i].errc = s.RawValues(ctx, q.StartTime, q.EndTime, versions[i])
		}
		return createCSV(sq, q, w, false)
	default:
		return errors.New("Invalid query type")
	}
}

func createCSV(buf streambuffer, q *CSVQuery, w *csv.Writer, statistical bool) error {
	// Buffer for the row of the CSV that we are writing
	var row = buf.getHeaderRow(q.Labels)

	// Write the header row
	var err = w.Write(row)
	if err != nil {
		return err
	}

	var open bool
	var numopen = len(q.Streams)
	for i := range q.Streams {
		open, err = buf.readPoint(i)
		if !open {
			numopen--
			if err != nil {
				return err
			}
		}
	}

	for {
		// Compute the time of the next row
		var earliest int64 = math.MaxInt64
		for i := range q.Streams {
			if buf.isOpen(i) && buf.getTime(i) < earliest {
				earliest = buf.getTime(i)
			}
		}

		// Compute the next row
		row[0] = fmt.Sprintf("%d", earliest)
		row[1] = time.Unix(0, earliest).Format(time.RFC3339Nano)
		for i := range q.Streams {
			if !buf.isOpen(i) {
				continue
			} else if buf.getTime(i) == earliest {
				buf.writePoint(i, row)

				// We consumed this point, so fetch the next point
				open, err = buf.readPoint(i)
				if !open {
					numopen--
					if err != nil {
						return err
					}
				}
			} else {
				buf.writeEmptyPoint(i, row)
			}
		}

		if numopen == 0 {
			break
		}

		// Emit the row
		err = w.Write(row)
		if err != nil {
			return nil
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package kvnetfilter

import (
	"github.com/42wim/registrator-work/bridge"
	consulapi "github.com/hashicorp/consul/api"
	"log"
	"net/url"
	"strconv"
	"strings"
	"time"
)

func init() {
	bridge.Register(new(Factory), "kvnetfilter")
}

type Factory struct{}

func (f *Factory) New(uri *url.URL) bridge.RegistryAdapter {
	// init consul
	config := consulapi.DefaultConfig()
	if uri.Host != "" {
		config.Address = uri.Host
	}
	client, err := consulapi.NewClient(config)
	if err != nil {
		log.Fatal("consulkv: ", uri.Scheme)
	}

	params := strings.Split(uri.Path, "/")
	if len(params) != 5 {
		log.Fatal("no correct scheme", len(params), params)
	}

	kvpath := params[1]
	aclpath := params[2]
	// init netfilter
	chain := params[3]
	set := params[4]

	FirewalldInit()
	if firewalldRunning {
		OnReloaded(func() { iptablesInit(chain, set) })
	}
	ipsetInit(set)
	iptablesInit(chain, set)

	return &NetfilterAdapter{Chain: chain, Set: set, client: client, path: kvpath, aclpath: aclpath}

}

type NetfilterAdapter struct {
	Chain   string
	Set     string
	client  *consulapi.Client
	path    string
	aclpath string
}

func (r *NetfilterAdapter) Ping() error {
	return nil
}

func (r *NetfilterAdapter) Register(service *bridge.Service) error {
	if strings.Contains(service.IP, ":") {
		err := r.kvRegister(service)
		if err != nil {
			return err
		}
		var srcRanges []string
		// traverse every tag
		for _, tag := range service.Tags {
			srcRanges = append(srcRanges, r.kvFindACL(service.Name+"/"+tag+"/")...)
		}
		// service too
		srcRanges = append(srcRanges, r.kvFindACL(service.Name+"/_all/")...)

		if len(srcRanges) > 0 {
			log.Println("would allow ", srcRanges)
			for _, src := range srcRanges {
				res := strings.Split(src, "#")
				if len(res) != 2 {
					log.Println("ERROR incorrect value: ", src)
					continue
				}
				srcip := res[0]
				ts, _ := strconv.Atoi(res[1])
				// exclude ourself and stale info
				if int(time.Now().Unix())-ts < service.TTL && service.IP != srcip {
					ipsetSrcDst("add", r.Set, srcip, service.IP, service.Origin.PortType, strconv.Itoa(service.Port), strconv.Itoa(service.TTL))
				} else {
					log.Println("stale service found, not adding", srcip, service.TTL, ts, time.Now().Unix())
				}
			}
		}
	}
	return nil
}

func (r *NetfilterAdapter) Deregister(service *bridge.Service) error {
	if strings.Contains(service.IP, ":") {
		var srcRanges []string
		// traverse every tag
		for _, tag := range service.Tags {
			srcRanges = append(srcRanges, r.kvFindACL(service.Name+"/"+tag+"/")...)
		}
		// service too
		srcRanges = append(srcRanges, r.kvFindACL(service.Name+"/")...)

		if len(srcRanges) > 0 {
			log.Println("would allow ", srcRanges)
			for _, src := range srcRanges {
				res := strings.Split(src, "#")
				if len(res) != 2 {
					log.Println("ERROR incorrect value: ", src)
					continue
				}
				srcip := res[0]
				ipsetSrcDst("del", r.Set, srcip, service.IP, service.Origin.PortType, strconv.Itoa(service.Port), "")
			}
		}
		// deregister after netfilter removal
		err := r.kvDeregister(service)
		if err != nil {
			return err
		}
	}
	return nil
}

func (r *NetfilterAdapter) Refresh(service *bridge.Service) error {
	return r.Register(service)
}

func (r *NetfilterAdapter) kvRegister(service *bridge.Service) error {
	path := r.path + "/" + service.Name + "/" + service.ID
	_, err := r.client.KV().Put(&consulapi.KVPair{Key: path, Value: []byte(service.IP + "#" + strconv.Itoa(int(time.Now().Unix())))}, nil)
	if err != nil {
		log.Println("consulkv: failed to register service:", err)
	}
	for _, tag := range service.Tags {
		path = r.path + "/" + service.Name + "/" + tag + "/" + service.ID
		_, err := r.client.KV().Put(&consulapi.KVPair{Key: path, Value: []byte(service.IP + "#" + strconv.Itoa(int(time.Now().Unix())))}, nil)
		if err != nil {
			log.Println("consulkv: failed to register service:", err)
		}
	}
	return err
}

func (r *NetfilterAdapter) kvDeregister(service *bridge.Service) error {
	if !strings.Contains(service.IP, ":") {
		return nil
	}
	path := r.path + "/" + service.Name + "/" + service.ID
	_, err := r.client.KV().Delete(path, nil)
	if err != nil {
		log.Println("consulkv: failed to deregister service:", err)
	}
	for _, tag := range service.Tags {
		path = r.path + "/" + service.Name + "/" + tag + "/" + service.ID
		_, err := r.client.KV().Delete(path, nil)
		if err != nil {
			log.Println("consulkv: failed to deregister service:", err)
		}
	}
	return err
}

func (r *NetfilterAdapter) kvFindACL(key string) []string {
	var acls []string
	url := "/" + r.aclpath + "/" + key
	log.Println("looking for ACL in ", url)
	kps, _, _ := r.client.KV().List(url, nil)
	for _, kp := range kps {
		if len(kp.Value) > 0 {
			log.Println("keys to search ", string(kp.Value))
			// if ipv6 address, add
			if strings.Contains(string(kp.Value), ":") {
				acls = append(acls, string(kp.Value))
				continue
			}
			rkps, _, _ := r.client.KV().List(string(kp.Value), nil)
			for _, rkp := range rkps {
				log.Print("found acl: ", string(rkp.Value))
				acls = append(acls, string(rkp.Value))
			}
		}
	}
	return acls
}

func (r *NetfilterAdapter) Services() ([]*bridge.Service, error) {
	return []*bridge.Service{}, nil
}
<commit_msg>Look for /_fallback/ ACL when no ACL's found<commit_after>package kvnetfilter

import (
	"github.com/42wim/registrator-work/bridge"
	consulapi "github.com/hashicorp/consul/api"
	"log"
	"net/url"
	"strconv"
	"strings"
	"time"
)

func init() {
	bridge.Register(new(Factory), "kvnetfilter")
}

type Factory struct{}

func (f *Factory) New(uri *url.URL) bridge.RegistryAdapter {
	// init consul
	config := consulapi.DefaultConfig()
	if uri.Host != "" {
		config.Address = uri.Host
	}
	client, err := consulapi.NewClient(config)
	if err != nil {
		log.Fatal("consulkv: ", uri.Scheme)
	}

	params := strings.Split(uri.Path, "/")
	if len(params) != 5 {
		log.Fatal("no correct scheme", len(params), params)
	}

	kvpath := params[1]
	aclpath := params[2]
	// init netfilter
	chain := params[3]
	set := params[4]

	FirewalldInit()
	if firewalldRunning {
		OnReloaded(func() { iptablesInit(chain, set) })
	}
	ipsetInit(set)
	iptablesInit(chain, set)

	return &NetfilterAdapter{Chain: chain, Set: set, client: client, path: kvpath, aclpath: aclpath}

}

type NetfilterAdapter struct {
	Chain   string
	Set     string
	client  *consulapi.Client
	path    string
	aclpath string
}

func (r *NetfilterAdapter) Ping() error {
	return nil
}

func (r *NetfilterAdapter) Register(service *bridge.Service) error {
	if strings.Contains(service.IP, ":") {
		err := r.kvRegister(service)
		if err != nil {
			return err
		}
		var srcRanges []string
		// traverse every tag
		for _, tag := range service.Tags {
			srcRanges = append(srcRanges, r.kvFindACL(service.Name+"/"+tag+"/")...)
		}
		// service too
		srcRanges = append(srcRanges, r.kvFindACL(service.Name+"/_all/")...)

		// no results, use fallback
		if len(srcRanges) == 0 {
			srcRanges = append(srcRanges, r.kvFindACL("/_fallback/")...)
		}

		if len(srcRanges) > 0 {
			log.Println("would allow ", srcRanges)
			for _, src := range srcRanges {
				res := strings.Split(src, "#")
				if len(res) != 2 {
					log.Println("ERROR incorrect value: ", src)
					continue
				}
				srcip := res[0]
				ts, _ := strconv.Atoi(res[1])
				// exclude ourself and stale info
				if int(time.Now().Unix())-ts < service.TTL && service.IP != srcip {
					ipsetSrcDst("add", r.Set, srcip, service.IP, service.Origin.PortType, strconv.Itoa(service.Port), strconv.Itoa(service.TTL))
				} else {
					log.Println("stale service found, not adding", srcip, service.TTL, ts, time.Now().Unix())
				}
			}
		}
	}
	return nil
}

func (r *NetfilterAdapter) Deregister(service *bridge.Service) error {
	if strings.Contains(service.IP, ":") {
		var srcRanges []string
		// traverse every tag
		for _, tag := range service.Tags {
			srcRanges = append(srcRanges, r.kvFindACL(service.Name+"/"+tag+"/")...)
		}
		// service too
		srcRanges = append(srcRanges, r.kvFindACL(service.Name+"/_all/")...)

		// no results, use fallback
		if len(srcRanges) == 0 {
			srcRanges = append(srcRanges, r.kvFindACL("/_fallback/")...)
		}

		if len(srcRanges) > 0 {
			log.Println("would allow ", srcRanges)
			for _, src := range srcRanges {
				res := strings.Split(src, "#")
				if len(res) != 2 {
					log.Println("ERROR incorrect value: ", src)
					continue
				}
				srcip := res[0]
				ipsetSrcDst("del", r.Set, srcip, service.IP, service.Origin.PortType, strconv.Itoa(service.Port), "")
			}
		}
		// deregister after netfilter removal
		err := r.kvDeregister(service)
		if err != nil {
			return err
		}
	}
	return nil
}

func (r *NetfilterAdapter) Refresh(service *bridge.Service) error {
	return r.Register(service)
}

func (r *NetfilterAdapter) kvRegister(service *bridge.Service) error {
	path := r.path + "/" + service.Name + "/" + service.ID
	_, err := r.client.KV().Put(&consulapi.KVPair{Key: path, Value: []byte(service.IP + "#" + strconv.Itoa(int(time.Now().Unix())))}, nil)
	if err != nil {
		log.Println("consulkv: failed to register service:", err)
	}
	for _, tag := range service.Tags {
		path = r.path + "/" + service.Name + "/" + tag + "/" + service.ID
		_, err := r.client.KV().Put(&consulapi.KVPair{Key: path, Value: []byte(service.IP + "#" + strconv.Itoa(int(time.Now().Unix())))}, nil)
		if err != nil {
			log.Println("consulkv: failed to register service:", err)
		}
	}
	return err
}

func (r *NetfilterAdapter) kvDeregister(service *bridge.Service) error {
	if !strings.Contains(service.IP, ":") {
		return nil
	}
	path := r.path + "/" + service.Name + "/" + service.ID
	_, err := r.client.KV().Delete(path, nil)
	if err != nil {
		log.Println("consulkv: failed to deregister service:", err)
	}
	for _, tag := range service.Tags {
		path = r.path + "/" + service.Name + "/" + tag + "/" + service.ID
		_, err := r.client.KV().Delete(path, nil)
		if err != nil {
			log.Println("consulkv: failed to deregister service:", err)
		}
	}
	return err
}

func (r *NetfilterAdapter) kvFindACL(key string) []string {
	var acls []string
	url := "/" + r.aclpath + "/" + key
	log.Println("looking for ACL in ", url)
	kps, _, _ := r.client.KV().List(url, nil)
	for _, kp := range kps {
		if len(kp.Value) > 0 {
			log.Println("keys to search ", string(kp.Value))
			// if ipv6 address, add
			if strings.Contains(string(kp.Value), ":") {
				acls = append(acls, string(kp.Value))
				continue
			}
			rkps, _, _ := r.client.KV().List(string(kp.Value), nil)
			for _, rkp := range rkps {
				log.Print("found acl: ", string(rkp.Value))
				acls = append(acls, string(rkp.Value))
			}
		}
	}
	return acls
}

func (r *NetfilterAdapter) Services() ([]*bridge.Service, error) {
	return []*bridge.Service{}, nil
}
<|endoftext|>


<commit_before>package rollbar

import (
	"encoding/json"
)

type DeployService struct {
	C *Client
}

type DeployResponse struct {
	Err    int           `json:"err"`
	Result DeploysResult `json:"result"`
}

type DeploysResult struct {
	Page    int      `json:"page"`
	Deploys []Deploy `json:"deploys"`
}

type Deploy struct {
	Id         int   `json:"id"`
	ProjectId  int   `json:"project_id"`
	StartTime  int64 `json:"start_time"`
	FinishTime int64 `json:"finish_time"`
}

func (d *DeployService) All() (*DeployResponse, error) {
	var response = &DeployResponse{}

	var body, err = d.C.Request("GET", "deploys")
	if err != nil {
		return response, err
	}

	defer body.Close()

	err = json.NewDecoder(body).Decode(&response)
	if err != nil {
		return response, err
	}

	return response, nil
}
<commit_msg>rollbar: update library<commit_after>package rollbar

import (
	"encoding/json"
)

type DeployService struct {
	C *Client
}

type DeployResponse struct {
	Err    int           `json:"err"`
	Result DeploysResult `json:"result"`
}

type DeploysResult struct {
	Page    int      `json:"page"`
	Deploys []Deploy `json:"deploys"`
}

type Deploy struct {
	Id         int    `json:"id"`
	ProjectId  int    `json:"project_id"`
	StartTime  int64  `json:"start_time"`
	FinishTime int64  `json:"finish_time"`
	Comment    string `json:comment`
}

func (d *DeployService) All() (*DeployResponse, error) {
	var response = &DeployResponse{}

	var body, err = d.C.Request("GET", "deploys")
	if err != nil {
		return response, err
	}

	defer body.Close()

	err = json.NewDecoder(body).Decode(&response)
	if err != nil {
		return response, err
	}

	return response, nil
}
<|endoftext|>