<commit_before>package smokescreen

import (
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/armon/go-proxyproto"
	"github.com/elazarl/goproxy"
	"github.com/sirupsen/logrus"
	"github.com/stripe/go-einhorn/einhorn"
)

const (
	IpOK IpType = iota
	IpOKBlacklistExempted
	IpDenyNotGlobalUnicast
	IpDenyBlacklist

	denyMsgTmpl = "egress proxying denied to host '%s' because %s. " +
		"If you didn't intend for your request to be proxied, you may want a 'no_proxy' environment variable."
)

type IpType int

type aclDecision struct {
	reason, role, project string
	allow                 bool
}

type ctxUserData struct {
	start    time.Time
	decision *aclDecision
}

type denyError error

func (t IpType) String() string {
	switch t {
	case IpOK:
		return "IpOK"
	case IpOKBlacklistExempted:
		return "IpOKBlacklistExempted"
	case IpDenyNotGlobalUnicast:
		return "IpDenyNotGlobalUnicast"
	case IpDenyBlacklist:
		return "IpDenyBlacklist"
	default:
		panic(fmt.Sprintf("unknown ip type %d", t))
	}
}

const errorHeader = "X-Smokescreen-Error"
const roleHeader = "X-Smokescreen-Role"

func ipIsInSetOfNetworks(nets []net.IPNet, ip net.IP) bool {
	for _, network := range nets {
		if network.Contains(ip) {
			return true
		}
	}
	return false
}

func classifyIP(config *Config, ip net.IP) IpType {
	if !(ip.IsGlobalUnicast() || (config.AllowProxyToLoopback && ip.IsLoopback())) {
		return IpDenyNotGlobalUnicast
	}

	blacklisted := ipIsInSetOfNetworks(config.CidrBlacklist, ip)
	whitelisted := ipIsInSetOfNetworks(config.CidrBlacklistExemptions, ip)

	if !blacklisted {
		return IpOK
	} else if whitelisted {
		return IpOKBlacklistExempted
	}

	return IpDenyBlacklist
}

func safeResolve(config *Config, network, addr string) (*net.TCPAddr, error) {
	config.StatsdClient.Count("resolver.attempts_total", 1, []string{}, 0.3)
	resolved, err := net.ResolveTCPAddr(network, addr)
	if err != nil {
		config.StatsdClient.Count("resolver.errors_total", 1, []string{}, 0.3)
		return nil, err
	}

	switch classifyIP(config, resolved.IP) {
	case IpOK:
		return resolved, nil
	case IpOKBlacklistExempted:
		config.StatsdClient.Count("resolver.private_blacklist_exempted_total", 1, []string{}, 0.3)
		return resolved, nil
	case IpDenyNotGlobalUnicast:
		config.StatsdClient.Count("resolver.illegal_total", 1, []string{}, 0.3)
		return nil, denyError(fmt.Errorf("resolves to private address %s", resolved.IP))
	case IpDenyBlacklist:
		config.StatsdClient.Count("resolver.illegal_total", 1, []string{}, 0.3)
		return nil, denyError(fmt.Errorf("resolves to blacklisted address %s", resolved.IP))
	default:
		panic("unknown IP type")
	}
}

func dial(config *Config, network, addr string) (net.Conn, error) {
	resolved, err := safeResolve(config, network, addr)
	if err != nil {
		return nil, err
	}

	return net.DialTimeout(network, resolved.String(), config.ConnectTimeout)
}

func rejectResponse(req *http.Request, config *Config, err error) *http.Response {
	var msg string
	switch err.(type) {
	case denyError:
		msg = fmt.Sprintf(denyMsgTmpl, err.Error(), req.Host)
	default:
		msg = "an unexpected error occurred."
	}

	if config.AdditionalErrorMessageOnDeny != "" {
		msg += " "
		msg += config.AdditionalErrorMessageOnDeny
	}

	resp := goproxy.NewResponse(req,
		goproxy.ContentTypeText,
		http.StatusServiceUnavailable,
		msg+"\n")
	resp.ProtoMajor = req.ProtoMajor
	resp.ProtoMinor = req.ProtoMinor
	resp.Header.Set(errorHeader, msg)
	return resp
}

func BuildProxy(config *Config) *goproxy.ProxyHttpServer {
	proxy := goproxy.NewProxyHttpServer()
	proxy.Verbose = false
	proxy.Tr.Dial = func(network, addr string) (net.Conn, error) {
		return dial(config, network, addr)
	}

	// Handle traditional HTTP proxy
	proxy.OnRequest().DoFunc(func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
		config.Log.WithFields(
			logrus.Fields{
				"remote": ctx.Req.RemoteAddr,
				"host":   ctx.Req.Host,
				"url":    ctx.Req.RequestURI,
			}).Debug("received HTTP proxy request")
		userData := ctxUserData{time.Now(), nil}
		ctx.UserData = &userData

		var err error
		userData.decision, err = checkIfRequestShouldBeProxied(config, ctx.Req, ctx.Req.Host)
		req.Header.Del(roleHeader)
		if err != nil {
			return req, rejectResponse(req, config, err)
		}
		if !userData.decision.allow {
			return req, rejectResponse(req, config, denyError(errors.New(userData.decision.reason)))
		}

		return req, nil
	})

	proxy.OnRequest().HandleConnectFunc(func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) {
		resolved, err := handleConnect(config, ctx)
		if err != nil {
			ctx.Resp = rejectResponse(ctx.Req, config, err)
			return goproxy.RejectConnect, ""
		}
		return goproxy.OkConnect, resolved.String()
	})

	proxy.OnResponse().DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
		if resp != nil {
			resp.Header.Del(errorHeader)
		}

		if resp == nil && ctx.Error != nil {
			return rejectResponse(ctx.Req, config, ctx.Error)
		}

		// In case of an error, this function is called a second time to filter the
		// response we generate so this logger will be called once.
		logHTTP(config, ctx)
		return resp
	})
	return proxy
}

func logProxy(
	config *Config,
	ctx *goproxy.ProxyCtx,
	proxyType string,
	toAddress *net.TCPAddr,
	decision *aclDecision,
	start time.Time,
	err error,
) {
	var contentLength int64
	if ctx.Resp != nil {
		contentLength = ctx.Resp.ContentLength
	}

	hostname := ctx.Req.Host
	fromHost, fromPort, _ := net.SplitHostPort(ctx.Req.RemoteAddr)

	allow := err == nil

	fields := logrus.Fields{
		"proxy_type":     proxyType,
		"src_host":       fromHost,
		"src_port":       fromPort,
		"host":           hostname,
		"start_time":     start.Unix(),
		"end_time":       time.Now().Unix(),
		"content_length": contentLength,
	}

	if _, ok := err.(denyError); !ok && err != nil {
		fields["error"] = err
	}

	if toAddress != nil {
		fields["dest_ip"] = toAddress.IP.String()
		fields["dest_port"] = toAddress.Port
	}

	if decision != nil {
		fields["role"] = decision.role
		fields["project"] = decision.project
		fields["decision_reason"] = decision.reason
		if !decision.allow {
			allow = false
		}
	}
	fields["allow"] = allow

	entry := config.Log.WithFields(fields)
	var logMethod func(...interface{})
	if _, ok := fields["error"]; ok {
		logMethod = entry.Error
	} else if allow {
		logMethod = entry.Info
	} else {
		logMethod = entry.Warn
	}
	logMethod("proxy_response")
}

func logHTTP(config *Config, ctx *goproxy.ProxyCtx) {
	var toAddr *net.TCPAddr
	if ctx.RoundTrip != nil {
		toAddr = ctx.RoundTrip.TCPAddr
	}

	userData := ctx.UserData.(*ctxUserData)

	logProxy(config, ctx, "http", toAddr, userData.decision, userData.start, ctx.Error)
}

func handleConnect(config *Config, ctx *goproxy.ProxyCtx) (*net.TCPAddr, error) {
	config.Log.WithFields(
		logrus.Fields{
			"remote": ctx.Req.RemoteAddr,
			"host":   ctx.Req.Host,
		}).Debug("received CONNECT proxy request")
	start := time.Now()

	// Check if requesting role is allowed to talk to remote
	var resolved *net.TCPAddr
	var err error
	var decision *aclDecision
	defer func() {
		logProxy(config, ctx, "connect", resolved, decision, start, err)
	}()

	decision, err = checkIfRequestShouldBeProxied(config, ctx.Req, ctx.Req.Host)
	if err != nil {
		return nil, err
	}
	if !decision.allow {
		return nil, denyError(errors.New(decision.reason))
	}

	resolved, err = safeResolve(config, "tcp", ctx.Req.Host)
	if err != nil {
		return nil, err
	}

	return resolved, nil
}

func findListener(ip string, defaultPort int) (net.Listener, error) {
	if einhorn.IsWorker() {
		listener, err := einhorn.GetListener(0)
		if err != nil {
			return nil, err
		}

		err = einhorn.Ack()

		return listener, err
	} else {
		return net.Listen("tcp", fmt.Sprintf("%s:%d", ip, defaultPort))
	}
}

func StartWithConfig(config *Config, quit <-chan interface{}) {
	config.Log.Println("starting")
	proxy := BuildProxy(config)

	listener, err := findListener(config.Ip, config.Port)
	if err != nil {
		config.Log.Fatal("can't find listener", err)
	}

	if config.SupportProxyProtocol {
		listener = &proxyproto.Listener{Listener: listener}
	}

	var handler http.Handler = proxy
	if config.MaintenanceFile != "" {
		handler = &HealthcheckMiddleware{
			App:             handler,
			MaintenanceFile: config.MaintenanceFile,
		}
	}

	// TLS support

	if config.TlsConfig != nil {
		listener = tls.NewListener(listener, config.TlsConfig)
	}

	server := http.Server{
		Handler: handler,
	}

	runServer(config, &server, listener, quit)
	return
}

func runServer(config *Config, server *http.Server, listener net.Listener, quit <-chan interface{}) {
	// Runs the server and shuts it down when it receives a signal.
	//
	// Why aren't we using goji's graceful shutdown library? Great question!
	//
	// There are several things we might want to do when shutting down gracefully:
	// 1. close the listening socket (so that we don't accept *new* connections)
	// 2. close *existing* keepalive connections once they become idle
	//
	// It is impossible to close existing keepalive connections, because goproxy
	// hijacks the socket and doesn't tell us when they become idle. So all we
	// can do is close the listening socket when we receive a signal, not accept
	// new connections, and then exit the program after a timeout.

	semiGraceful := true
	kill := make(chan os.Signal, 1)
	signal.Notify(kill, syscall.SIGUSR2, syscall.SIGTERM, syscall.SIGHUP)
	go func() {
		select {
		case <-kill:
			config.Log.Print("quitting semi-gracefully")

		case <-quit:
			config.Log.Print("quitting now")
			semiGraceful = false
		}
		listener.Close()
	}()
	err := server.Serve(listener)
	if !strings.HasSuffix(err.Error(), "use of closed network connection") {
		config.Log.Fatal(err)
	}

	if semiGraceful {
		// the program has exited normally, wait 60s in an attempt to shutdown
		// semi-gracefully
		config.Log.WithField("delay", config.ExitTimeout).Info("Waiting before shutting down")
		time.Sleep(config.ExitTimeout)
	}
}

func checkIfRequestShouldBeProxied(config *Config, req *http.Request, outboundHost string) (*aclDecision, error) {
	decision := &aclDecision{}

	if config.EgressAcl == nil {
		decision.allow = true
		decision.reason = "no egress ACL configured"
		return decision, nil
	}

	role, roleErr := config.RoleFromRequest(req)
	if roleErr != nil {
		// A missing role is OK at this point since we may have a default
		if _, ok := roleErr.(MissingRoleError); !ok {
			return nil, roleErr
		}
	}
	decision.role = role

	decision.project, _ = config.EgressAcl.Project(role)

	submatch := config.hostExtractExpr.FindStringSubmatch(outboundHost)

	result, err := config.EgressAcl.Decide(role, submatch[1])
	if err != nil {
		if rerr, ok := err.(UnknownRoleError); ok {
			var msg string
			if roleErr != nil {
				msg = fmt.Sprintf("unable to extract a role from your request and no default is provided (%s)", err.Error())
			} else {
				msg = fmt.Sprintf("you passed an unknown role '%s'", rerr.Role)
			}
			decision.reason = msg
			return decision, nil
		}
		return nil, err
	}

	switch result {
	case EgressAclDecisionDeny:
		decision.reason = "role is not allowed to access this host"
		return decision, nil

	case EgressAclDecisionAllowAndReport:
		decision.reason = "role is not allowed to access this host but report_only is true"
		decision.allow = true
		return decision, nil

	case EgressAclDecisionAllow:
		// Well, everything is going as expected.
		decision.allow = true
		decision.reason = "your role is allowed to access this host"
		return decision, nil
	}
	panic(fmt.Sprintf("unknown acl decision %v", result))
}
<commit_msg>Use errors instead of panics<commit_after>package smokescreen

import (
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/armon/go-proxyproto"
	"github.com/elazarl/goproxy"
	"github.com/sirupsen/logrus"
	"github.com/stripe/go-einhorn/einhorn"
)

const (
	IpOK IpType = iota
	IpOKBlacklistExempted
	IpDenyNotGlobalUnicast
	IpDenyBlacklist

	denyMsgTmpl = "egress proxying denied to host '%s' because %s. " +
		"If you didn't intend for your request to be proxied, you may want a 'no_proxy' environment variable."
)

type IpType int

type aclDecision struct {
	reason, role, project string
	allow                 bool
}

type ctxUserData struct {
	start    time.Time
	decision *aclDecision
}

type denyError error

func (t IpType) String() string {
	switch t {
	case IpOK:
		return "IpOK"
	case IpOKBlacklistExempted:
		return "IpOKBlacklistExempted"
	case IpDenyNotGlobalUnicast:
		return "IpDenyNotGlobalUnicast"
	case IpDenyBlacklist:
		return "IpDenyBlacklist"
	default:
		panic(fmt.Sprintf("unknown ip type %d", t))
	}
}

const errorHeader = "X-Smokescreen-Error"
const roleHeader = "X-Smokescreen-Role"

func ipIsInSetOfNetworks(nets []net.IPNet, ip net.IP) bool {
	for _, network := range nets {
		if network.Contains(ip) {
			return true
		}
	}
	return false
}

func classifyIP(config *Config, ip net.IP) IpType {
	if !(ip.IsGlobalUnicast() || (config.AllowProxyToLoopback && ip.IsLoopback())) {
		return IpDenyNotGlobalUnicast
	}

	blacklisted := ipIsInSetOfNetworks(config.CidrBlacklist, ip)
	whitelisted := ipIsInSetOfNetworks(config.CidrBlacklistExemptions, ip)

	if !blacklisted {
		return IpOK
	} else if whitelisted {
		return IpOKBlacklistExempted
	}

	return IpDenyBlacklist
}

func safeResolve(config *Config, network, addr string) (*net.TCPAddr, error) {
	config.StatsdClient.Count("resolver.attempts_total", 1, []string{}, 0.3)
	resolved, err := net.ResolveTCPAddr(network, addr)
	if err != nil {
		config.StatsdClient.Count("resolver.errors_total", 1, []string{}, 0.3)
		return nil, err
	}

	classification := classifyIP(config, resolved.IP)
	switch classification {
	case IpOK:
		return resolved, nil
	case IpOKBlacklistExempted:
		config.StatsdClient.Count("resolver.private_blacklist_exempted_total", 1, []string{}, 0.3)
		return resolved, nil
	case IpDenyNotGlobalUnicast:
		config.StatsdClient.Count("resolver.illegal_total", 1, []string{}, 0.3)
		return nil, denyError(fmt.Errorf("resolves to private address %s", resolved.IP))
	case IpDenyBlacklist:
		config.StatsdClient.Count("resolver.illegal_total", 1, []string{}, 0.3)
		return nil, denyError(fmt.Errorf("resolves to blacklisted address %s", resolved.IP))
	default:
		return nil, fmt.Errorf("unknown IP type %v", classification)
	}
}

func dial(config *Config, network, addr string) (net.Conn, error) {
	resolved, err := safeResolve(config, network, addr)
	if err != nil {
		return nil, err
	}

	return net.DialTimeout(network, resolved.String(), config.ConnectTimeout)
}

func rejectResponse(req *http.Request, config *Config, err error) *http.Response {
	var msg string
	switch err.(type) {
	case denyError:
		msg = fmt.Sprintf(denyMsgTmpl, err.Error(), req.Host)
	default:
		msg = "an unexpected error occurred."
	}

	if config.AdditionalErrorMessageOnDeny != "" {
		msg += " "
		msg += config.AdditionalErrorMessageOnDeny
	}

	resp := goproxy.NewResponse(req,
		goproxy.ContentTypeText,
		http.StatusServiceUnavailable,
		msg+"\n")
	resp.ProtoMajor = req.ProtoMajor
	resp.ProtoMinor = req.ProtoMinor
	resp.Header.Set(errorHeader, msg)
	return resp
}

func BuildProxy(config *Config) *goproxy.ProxyHttpServer {
	proxy := goproxy.NewProxyHttpServer()
	proxy.Verbose = false
	proxy.Tr.Dial = func(network, addr string) (net.Conn, error) {
		return dial(config, network, addr)
	}

	// Handle traditional HTTP proxy
	proxy.OnRequest().DoFunc(func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
		config.Log.WithFields(
			logrus.Fields{
				"remote": ctx.Req.RemoteAddr,
				"host":   ctx.Req.Host,
				"url":    ctx.Req.RequestURI,
			}).Debug("received HTTP proxy request")
		userData := ctxUserData{time.Now(), nil}
		ctx.UserData = &userData

		var err error
		userData.decision, err = checkIfRequestShouldBeProxied(config, ctx.Req, ctx.Req.Host)
		req.Header.Del(roleHeader)
		if err != nil {
			return req, rejectResponse(req, config, err)
		}
		if !userData.decision.allow {
			return req, rejectResponse(req, config, denyError(errors.New(userData.decision.reason)))
		}

		return req, nil
	})

	proxy.OnRequest().HandleConnectFunc(func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) {
		resolved, err := handleConnect(config, ctx)
		if err != nil {
			ctx.Resp = rejectResponse(ctx.Req, config, err)
			return goproxy.RejectConnect, ""
		}
		return goproxy.OkConnect, resolved.String()
	})

	proxy.OnResponse().DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
		if resp != nil {
			resp.Header.Del(errorHeader)
		}

		if resp == nil && ctx.Error != nil {
			return rejectResponse(ctx.Req, config, ctx.Error)
		}

		// In case of an error, this function is called a second time to filter the
		// response we generate so this logger will be called once.
		logHTTP(config, ctx)
		return resp
	})
	return proxy
}

func logProxy(
	config *Config,
	ctx *goproxy.ProxyCtx,
	proxyType string,
	toAddress *net.TCPAddr,
	decision *aclDecision,
	start time.Time,
	err error,
) {
	var contentLength int64
	if ctx.Resp != nil {
		contentLength = ctx.Resp.ContentLength
	}

	hostname := ctx.Req.Host
	fromHost, fromPort, _ := net.SplitHostPort(ctx.Req.RemoteAddr)

	allow := err == nil

	fields := logrus.Fields{
		"proxy_type":     proxyType,
		"src_host":       fromHost,
		"src_port":       fromPort,
		"host":           hostname,
		"start_time":     start.Unix(),
		"end_time":       time.Now().Unix(),
		"content_length": contentLength,
	}

	if _, ok := err.(denyError); !ok && err != nil {
		fields["error"] = err
	}

	if toAddress != nil {
		fields["dest_ip"] = toAddress.IP.String()
		fields["dest_port"] = toAddress.Port
	}

	if decision != nil {
		fields["role"] = decision.role
		fields["project"] = decision.project
		fields["decision_reason"] = decision.reason
		if !decision.allow {
			allow = false
		}
	}
	fields["allow"] = allow

	entry := config.Log.WithFields(fields)
	var logMethod func(...interface{})
	if _, ok := fields["error"]; ok {
		logMethod = entry.Error
	} else if allow {
		logMethod = entry.Info
	} else {
		logMethod = entry.Warn
	}
	logMethod("proxy_response")
}

func logHTTP(config *Config, ctx *goproxy.ProxyCtx) {
	var toAddr *net.TCPAddr
	if ctx.RoundTrip != nil {
		toAddr = ctx.RoundTrip.TCPAddr
	}

	userData := ctx.UserData.(*ctxUserData)

	logProxy(config, ctx, "http", toAddr, userData.decision, userData.start, ctx.Error)
}

func handleConnect(config *Config, ctx *goproxy.ProxyCtx) (*net.TCPAddr, error) {
	config.Log.WithFields(
		logrus.Fields{
			"remote": ctx.Req.RemoteAddr,
			"host":   ctx.Req.Host,
		}).Debug("received CONNECT proxy request")
	start := time.Now()

	// Check if requesting role is allowed to talk to remote
	var resolved *net.TCPAddr
	var err error
	var decision *aclDecision
	defer func() {
		logProxy(config, ctx, "connect", resolved, decision, start, err)
	}()

	decision, err = checkIfRequestShouldBeProxied(config, ctx.Req, ctx.Req.Host)
	if err != nil {
		return nil, err
	}
	if !decision.allow {
		return nil, denyError(errors.New(decision.reason))
	}

	resolved, err = safeResolve(config, "tcp", ctx.Req.Host)
	if err != nil {
		return nil, err
	}

	return resolved, nil
}

func findListener(ip string, defaultPort int) (net.Listener, error) {
	if einhorn.IsWorker() {
		listener, err := einhorn.GetListener(0)
		if err != nil {
			return nil, err
		}

		err = einhorn.Ack()

		return listener, err
	} else {
		return net.Listen("tcp", fmt.Sprintf("%s:%d", ip, defaultPort))
	}
}

func StartWithConfig(config *Config, quit <-chan interface{}) {
	config.Log.Println("starting")
	proxy := BuildProxy(config)

	listener, err := findListener(config.Ip, config.Port)
	if err != nil {
		config.Log.Fatal("can't find listener", err)
	}

	if config.SupportProxyProtocol {
		listener = &proxyproto.Listener{Listener: listener}
	}

	var handler http.Handler = proxy
	if config.MaintenanceFile != "" {
		handler = &HealthcheckMiddleware{
			App:             handler,
			MaintenanceFile: config.MaintenanceFile,
		}
	}

	// TLS support

	if config.TlsConfig != nil {
		listener = tls.NewListener(listener, config.TlsConfig)
	}

	server := http.Server{
		Handler: handler,
	}

	runServer(config, &server, listener, quit)
	return
}

func runServer(config *Config, server *http.Server, listener net.Listener, quit <-chan interface{}) {
	// Runs the server and shuts it down when it receives a signal.
	//
	// Why aren't we using goji's graceful shutdown library? Great question!
	//
	// There are several things we might want to do when shutting down gracefully:
	// 1. close the listening socket (so that we don't accept *new* connections)
	// 2. close *existing* keepalive connections once they become idle
	//
	// It is impossible to close existing keepalive connections, because goproxy
	// hijacks the socket and doesn't tell us when they become idle. So all we
	// can do is close the listening socket when we receive a signal, not accept
	// new connections, and then exit the program after a timeout.

	semiGraceful := true
	kill := make(chan os.Signal, 1)
	signal.Notify(kill, syscall.SIGUSR2, syscall.SIGTERM, syscall.SIGHUP)
	go func() {
		select {
		case <-kill:
			config.Log.Print("quitting semi-gracefully")

		case <-quit:
			config.Log.Print("quitting now")
			semiGraceful = false
		}
		listener.Close()
	}()
	err := server.Serve(listener)
	if !strings.HasSuffix(err.Error(), "use of closed network connection") {
		config.Log.Fatal(err)
	}

	if semiGraceful {
		// the program has exited normally, wait 60s in an attempt to shutdown
		// semi-gracefully
		config.Log.WithField("delay", config.ExitTimeout).Info("Waiting before shutting down")
		time.Sleep(config.ExitTimeout)
	}
}

func checkIfRequestShouldBeProxied(config *Config, req *http.Request, outboundHost string) (*aclDecision, error) {
	decision := &aclDecision{}

	if config.EgressAcl == nil {
		decision.allow = true
		decision.reason = "no egress ACL configured"
		return decision, nil
	}

	role, roleErr := config.RoleFromRequest(req)
	if roleErr != nil {
		// A missing role is OK at this point since we may have a default
		if _, ok := roleErr.(MissingRoleError); !ok {
			return nil, roleErr
		}
	}
	decision.role = role

	decision.project, _ = config.EgressAcl.Project(role)

	submatch := config.hostExtractExpr.FindStringSubmatch(outboundHost)

	result, err := config.EgressAcl.Decide(role, submatch[1])
	if err != nil {
		if rerr, ok := err.(UnknownRoleError); ok {
			var msg string
			if roleErr != nil {
				msg = fmt.Sprintf("unable to extract a role from your request and no default is provided (%s)", err.Error())
			} else {
				msg = fmt.Sprintf("you passed an unknown role '%s'", rerr.Role)
			}
			decision.reason = msg
			return decision, nil
		}
		return nil, err
	}

	switch result {
	case EgressAclDecisionDeny:
		decision.reason = "role is not allowed to access this host"
		return decision, nil

	case EgressAclDecisionAllowAndReport:
		decision.reason = "role is not allowed to access this host but report_only is true"
		decision.allow = true
		return decision, nil

	case EgressAclDecisionAllow:
		// Well, everything is going as expected.
		decision.allow = true
		decision.reason = "your role is allowed to access this host"
		return decision, nil
	default:
		return nil, fmt.Errorf("unknown acl decision %v", result)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gaemiddleware

import (
	"context"
	"fmt"

	mc "go.chromium.org/gae/service/memcache"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/server/portal"
	"go.chromium.org/luci/server/settings"
)

// settingsKey is key for global GAE settings (described by gaeSettings struct)
// in the settings store. See go.chromium.org/luci/server/settings.
const settingsKey = "gae"

// gaeSettings contain global Appengine related tweaks. They are stored in app
// settings store (based on the datastore, see appengine/gaesettings module)
// under settingsKey key.
type gaeSettings struct {
	// LoggingLevel is logging level to set the default logger to.
	//
	// Log entries below this level will be completely ignored. They won't even
	// reach GAE logging service. Default is logging.Debug (all logs hit logging
	// service).
	LoggingLevel logging.Level `json:"logging_level"`

	// DisableDSCache is true to disable dscache (the memcache layer on top of
	// the datastore).
	DisableDSCache portal.YesOrNo `json:"disable_dscache"`

	// SimulateMemcacheOutage is true to make every memcache call fail.
	//
	// Useful to verify apps can survive a memcache outage.
	SimulateMemcacheOutage portal.YesOrNo `json:"simulate_memcache_outage"`
}

// fetchCachedSettings fetches gaeSettings from the settings store or panics.
//
// Uses in-process global cache to avoid hitting datastore often. The cache
// expiration time is 1 min (see gaesettings.expirationTime), meaning
// the instance will refetch settings once a minute (blocking only one unlucky
// request to do so).
//
// Panics only if there's no cached value (i.e. it is the first call to this
// function in this process ever) and datastore operation fails. It is a good
// idea to implement /_ah/warmup to warm this up.
func fetchCachedSettings(c context.Context) gaeSettings {
	s := gaeSettings{}
	switch err := settings.Get(c, settingsKey, &s); {
	case err == nil:
		return s
	case err == settings.ErrNoSettings:
		// Defaults.
		return gaeSettings{LoggingLevel: logging.Debug}
	default:
		panic(fmt.Errorf("could not fetch GAE settings - %s", err))
	}
}

////////////////////////////////////////////////////////////////////////////////
// UI for GAE settings.

type settingsPage struct {
	portal.BasePage
}

func (settingsPage) Title(c context.Context) (string, error) {
	return "Appengine related settings", nil
}

func (settingsPage) Fields(c context.Context) ([]portal.Field, error) {
	return []portal.Field{
		{
			ID:    "LoggingLevel",
			Title: "Minimal logging level",
			Type:  portal.FieldChoice,
			ChoiceVariants: []string{
				"debug",
				"info",
				"warning",
				"error",
			},
			Validator: func(v string) error {
				var l logging.Level
				return l.Set(v)
			},
			Help: `Log entries below this level will be completely ignored.
They won't even reach GAE logging service.`,
		},
		portal.YesOrNoField(portal.Field{
			ID:    "DisableDSCache",
			Title: "Disable datastore cache",
			Help: `Usually caching is a good thing and it can be left enabled. You may
want to disable it if memcache is having issues that prevent entity writes to
succeed. See
dscache documentation for more information. Toggling this on and off has
consequences: memcache is completely flushed. Do not toy with this
setting.`,
		}),
		portal.YesOrNoField(portal.Field{
			ID:    "SimulateMemcacheOutage",
			Title: "Simulate memcache outage",
			Help: `Intended for development only. Do not use in production
applications. When Yes, all memcache calls will fail, as if the memcache
service is unavailable. This is useful to test how application behaves when a
real memcache outage happens.`,
		}),
	}, nil
}

func (settingsPage) ReadSettings(c context.Context) (map[string]string, error) {
	s := gaeSettings{}
	err := settings.GetUncached(c, settingsKey, &s)
	if err != nil && err != settings.ErrNoSettings {
		return nil, err
	}
	return map[string]string{
		"LoggingLevel":           s.LoggingLevel.String(),
		"DisableDSCache":         s.DisableDSCache.String(),
		"SimulateMemcacheOutage": s.SimulateMemcacheOutage.String(),
	}, nil
}

func (settingsPage) WriteSettings(c context.Context, values map[string]string, who, why string) error {
	modified := gaeSettings{}
	if err := modified.LoggingLevel.Set(values["LoggingLevel"]); err != nil {
		return err
	}
	if err := modified.DisableDSCache.Set(values["DisableDSCache"]); err != nil {
		return err
	}
	if err := modified.SimulateMemcacheOutage.Set(values["SimulateMemcacheOutage"]); err != nil {
		return err
	}

	// When switching dscache back on, wipe memcache.
	existing := gaeSettings{}
	err := settings.GetUncached(c, settingsKey, &existing)
	if err != nil && err != settings.ErrNoSettings {
		return err
	}
	if existing.DisableDSCache && !modified.DisableDSCache {
		logging.Warningf(c, "DSCache was reenabled, flushing memcache")
		if err := mc.Flush(c); err != nil {
			return fmt.Errorf("failed to flush memcache after reenabling dscache - %s", err)
		}
	}

	return settings.SetIfChanged(c, settingsKey, &modified, who, why)
}

func init() {
	portal.RegisterPage(settingsKey, settingsPage{})
}
<commit_msg>Report value of DSCacheDisabled to tsmon<commit_after>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gaemiddleware

import (
	"context"
	"fmt"

	mc "go.chromium.org/gae/service/memcache"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/tsmon"
	"go.chromium.org/luci/common/tsmon/metric"
	"go.chromium.org/luci/server/portal"
	"go.chromium.org/luci/server/settings"
)

// settingsKey is key for global GAE settings (described by gaeSettings struct)
// in the settings store. See go.chromium.org/luci/server/settings.
const settingsKey = "gae"

// gaeSettings contain global Appengine related tweaks. They are stored in app
// settings store (based on the datastore, see appengine/gaesettings module)
// under settingsKey key.
type gaeSettings struct {
	// LoggingLevel is logging level to set the default logger to.
	//
	// Log entries below this level will be completely ignored. They won't even
	// reach GAE logging service. Default is logging.Debug (all logs hit logging
	// service).
	LoggingLevel logging.Level `json:"logging_level"`

	// DisableDSCache is true to disable dscache (the memcache layer on top of
	// the datastore).
	DisableDSCache portal.YesOrNo `json:"disable_dscache"`

	// SimulateMemcacheOutage is true to make every memcache call fail.
	//
	// Useful to verify apps can survive a memcache outage.
	SimulateMemcacheOutage portal.YesOrNo `json:"simulate_memcache_outage"`
}

// fetchCachedSettings fetches gaeSettings from the settings store or panics.
//
// Uses in-process global cache to avoid hitting datastore often. The cache
// expiration time is 1 min (see gaesettings.expirationTime), meaning
// the instance will refetch settings once a minute (blocking only one unlucky
// request to do so).
//
// Panics only if there's no cached value (i.e. it is the first call to this
// function in this process ever) and datastore operation fails. It is a good
// idea to implement /_ah/warmup to warm this up.
func fetchCachedSettings(c context.Context) gaeSettings {
	s := gaeSettings{}
	switch err := settings.Get(c, settingsKey, &s); {
	case err == nil:
		return s
	case err == settings.ErrNoSettings:
		// Defaults.
		return gaeSettings{LoggingLevel: logging.Debug}
	default:
		panic(fmt.Errorf("could not fetch GAE settings - %s", err))
	}
}

// dsCacheDisabled is a metric for reporting the value of DSCacheDisabled in
// gaeSettings.
var dsCacheDisabled = metric.NewBool(
	"appengine/settings/dscache_disabled",
	"Whether or not dscache is disabled in the admin portal.",
	nil,
)

// reportDSCacheDisabled reports the value of DSCacheDisabled in settings to
// tsmon.
func reportDSCacheDisabled(c context.Context) {
	dsCacheDisabled.Set(c, bool(fetchCachedSettings(c).DisableDSCache))
}

////////////////////////////////////////////////////////////////////////////////
// UI for GAE settings.

type settingsPage struct {
	portal.BasePage
}

func (settingsPage) Title(c context.Context) (string, error) {
	return "Appengine related settings", nil
}

func (settingsPage) Fields(c context.Context) ([]portal.Field, error) {
	return []portal.Field{
		{
			ID:    "LoggingLevel",
			Title: "Minimal logging level",
			Type:  portal.FieldChoice,
			ChoiceVariants: []string{
				"debug",
				"info",
				"warning",
				"error",
			},
			Validator: func(v string) error {
				var l logging.Level
				return l.Set(v)
			},
			Help: `Log entries below this level will be completely ignored.
They won't even reach GAE logging service.`,
		},
		portal.YesOrNoField(portal.Field{
			ID:    "DisableDSCache",
			Title: "Disable datastore cache",
			Help: `Usually caching is a good thing and it can be left enabled. You may
want to disable it if memcache is having issues that prevent entity writes to
succeed. See
dscache documentation for more information. Toggling this on and off has
consequences: memcache is completely flushed. Do not toy with this
setting.`,
		}),
		portal.YesOrNoField(portal.Field{
			ID:    "SimulateMemcacheOutage",
			Title: "Simulate memcache outage",
			Help: `Intended for development only. Do not use in production
applications. When Yes, all memcache calls will fail, as if the memcache
service is unavailable. This is useful to test how application behaves when a
real memcache outage happens.`,
		}),
	}, nil
}

func (settingsPage) ReadSettings(c context.Context) (map[string]string, error) {
	s := gaeSettings{}
	err := settings.GetUncached(c, settingsKey, &s)
	if err != nil && err != settings.ErrNoSettings {
		return nil, err
	}
	return map[string]string{
		"LoggingLevel":           s.LoggingLevel.String(),
		"DisableDSCache":         s.DisableDSCache.String(),
		"SimulateMemcacheOutage": s.SimulateMemcacheOutage.String(),
	}, nil
}

func (settingsPage) WriteSettings(c context.Context, values map[string]string, who, why string) error {
	modified := gaeSettings{}
	if err := modified.LoggingLevel.Set(values["LoggingLevel"]); err != nil {
		return err
	}
	if err := modified.DisableDSCache.Set(values["DisableDSCache"]); err != nil {
		return err
	}
	if err := modified.SimulateMemcacheOutage.Set(values["SimulateMemcacheOutage"]); err != nil {
		return err
	}

	// When switching dscache back on, wipe memcache.
	existing := gaeSettings{}
	err := settings.GetUncached(c, settingsKey, &existing)
	if err != nil && err != settings.ErrNoSettings {
		return err
	}
	if existing.DisableDSCache && !modified.DisableDSCache {
		logging.Warningf(c, "DSCache was reenabled, flushing memcache")
		if err := mc.Flush(c); err != nil {
			return fmt.Errorf("failed to flush memcache after reenabling dscache - %s", err)
		}
	}

	return settings.SetIfChanged(c, settingsKey, &modified, who, why)
}

func init() {
	portal.RegisterPage(settingsKey, settingsPage{})
	tsmon.RegisterGlobalCallback(reportDSCacheDisabled, dsCacheDisabled)
}
<|endoftext|>


<commit_before>// Copyright 2015 Ka-Hing Cheung
// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import (
	"os"
	"strings"
	"time"

	"github.com/codegangsta/cli"
)

// Set up custom help text for goofys; in particular the usage section.
func init() {
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} {{if .Flags}}[global options]{{end}} bucket[:prefix] mountpoint
   {{if .Version}}
VERSION:
   {{.Version}}
   {{end}}{{if len .Authors}}
AUTHOR(S):
   {{range .Authors}}{{ . }}{{end}}
   {{end}}{{if .Commands}}
COMMANDS:
   {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
   {{end}}{{end}}{{if .Flags}}
GLOBAL OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}{{if .Copyright }}
COPYRIGHT:
   {{.Copyright}}
   {{end}}
`
}

func NewApp() (app *cli.App) {
	uid, gid := MyUserAndGroup()

	app = &cli.App{
		Name:     "goofys",
		Version:  "0.0.8",
		Usage:    "Mount an S3 bucket locally",
		HideHelp: true,
		Writer:   os.Stderr,
		Flags: []cli.Flag{

			cli.BoolFlag{
				Name:  "help, h",
				Usage: "Print this help text and exit successfully.",
			},

			/////////////////////////
			// File system
			/////////////////////////

			cli.StringSliceFlag{
				Name:  "o",
				Usage: "Additional system-specific mount options. Be careful!",
			},

			cli.IntFlag{
				Name:  "dir-mode",
				Value: 0755,
				Usage: "Permission bits for directories. (default: 0755)",
			},

			cli.IntFlag{
				Name:  "file-mode",
				Value: 0644,
				Usage: "Permission bits for files. (default: 0644)",
			},

			cli.IntFlag{
				Name:  "uid",
				Value: uid,
				Usage: "UID owner of all inodes.",
			},

			cli.IntFlag{
				Name:  "gid",
				Value: gid,
				Usage: "GID owner of all inodes.",
			},

			/////////////////////////
			// S3
			/////////////////////////

			cli.StringFlag{
				Name:  "endpoint",
				Value: "",
				Usage: "The non-AWS endpoint to connect to." +
					" Possible values: http://127.0.0.1:8081/",
			},

			cli.StringFlag{
				Name:  "region",
				Value: "us-east-1",
				Usage: "The region to connect to." +
					" Possible values: us-east-1, us-west-1, us-west-2, eu-west-1, " +
					"eu-central-1, ap-southeast-1, ap-southeast-2, ap-northeast-1, " +
					"sa-east-1, cn-north-1",
			},

			cli.StringFlag{
				Name:  "storage-class",
				Value: "STANDARD",
				Usage: "The type of storage to use when writing objects." +
					" Possible values: REDUCED_REDUNDANCY, STANDARD, STANDARD_IA.",
			},

			cli.BoolFlag{
				Name: "use-path-request",
				Usage: "Use a path-style request instead of virtual host-style." +
					" (deprecated, always on)",
			},

			cli.StringFlag{
				Name:  "profile",
				Usage: "Use a named profile from $HOME/.aws/credentials instead of \"default\"",
			},

			cli.BoolFlag{
				Name:  "use-content-type",
				Usage: "Set Content-Type according to file extension and /etc/mime.types (default: off)",
			},

			/// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
			/// See http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html
			cli.BoolFlag{
				Name:  "sse",
				Usage: "Enable basic server-side encryption at rest (SSE-S3)in S3 for all writes (default: off)",
			},

			cli.StringFlag{
				Name:  "sse-kms",
				Usage: "Enable KMS encryption (SSE-KMS) for all writes using this particular KMS `key-id`. Leave blank to Use the account's CMK - customer master key (default: off)",
				Value: "",
			},

			/////////////////////////
			// Tuning
			/////////////////////////

			cli.DurationFlag{
				Name:  "stat-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache StatObject results and inode attributes.",
			},

			cli.DurationFlag{
				Name:  "type-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache name -> file/dir mappings in directory " +
					"inodes.",
			},

			/////////////////////////
			// Debugging
			/////////////////////////

			cli.BoolFlag{
				Name:  "debug_fuse",
				Usage: "Enable fuse-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "debug_s3",
				Usage: "Enable S3-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "f",
				Usage: "Run goofys in foreground.",
			},
		},
	}

	return
}

type FlagStorage struct {
	// File system
	MountOptions map[string]string
	DirMode      os.FileMode
	FileMode     os.FileMode
	Uid          uint32
	Gid          uint32

	// S3
	Endpoint       string
	Region         string
	RegionSet      bool
	StorageClass   string
	UsePathRequest bool
	Profile        string
	UseContentType bool
	UseSSE         bool
	UseKMS         bool
	KMSKeyID       string

	// Tuning
	StatCacheTTL time.Duration
	TypeCacheTTL time.Duration

	// Debugging
	DebugFuse  bool
	DebugS3    bool
	Foreground bool
}

func parseOptions(m map[string]string, s string) {
	// NOTE(jacobsa): The man pages don't define how escaping works, and as far
	// as I can tell there is no way to properly escape or quote a comma in the
	// options list for an fstab entry. So put our fingers in our ears and hope
	// that nobody needs a comma.
	for _, p := range strings.Split(s, ",") {
		var name string
		var value string

		// Split on the first equals sign.
		if equalsIndex := strings.IndexByte(p, '='); equalsIndex != -1 {
			name = p[:equalsIndex]
			value = p[equalsIndex+1:]
		} else {
			name = p
		}

		m[name] = value
	}

	return
}

// Add the flags accepted by run to the supplied flag set, returning the
// variables into which the flags will parse.
func PopulateFlags(c *cli.Context) (flags *FlagStorage) {
	flags = &FlagStorage{
		// File system
		MountOptions: make(map[string]string),
		DirMode:      os.FileMode(c.Int("dir-mode")),
		FileMode:     os.FileMode(c.Int("file-mode")),
		Uid:          uint32(c.Int("uid")),
		Gid:          uint32(c.Int("gid")),

		// Tuning,
		StatCacheTTL: c.Duration("stat-cache-ttl"),
		TypeCacheTTL: c.Duration("type-cache-ttl"),

		// S3
		Endpoint:       c.String("endpoint"),
		Region:         c.String("region"),
		RegionSet:      c.IsSet("region"),
		StorageClass:   c.String("storage-class"),
		UsePathRequest: c.Bool("use-path-request"),
		Profile:        c.String("profile"),
		UseContentType: c.Bool("use-content-type"),
		UseSSE:         c.Bool("sse"),
		UseKMS:         c.IsSet("sse-kms"),
		KMSKeyID:       c.String("sse-kms"),

		// Debugging,
		DebugFuse:  c.Bool("debug_fuse"),
		DebugS3:    c.Bool("debug_s3"),
		Foreground: c.Bool("f"),
	}

	// KMS implies SSE
	if flags.UseKMS {
		flags.UseSSE = true
	}

	// Handle the repeated "-o" flag.
	for _, o := range c.StringSlice("o") {
		parseOptions(flags.MountOptions, o)
	}
	return
}

func MassageMountFlags(args []string) (ret []string) {
	if len(args) == 5 && args[3] == "-o" {
		// looks like it's coming from fstab!
		mountOptions := ""
		ret = append(ret, args[0])

		for _, p := range strings.Split(args[4], ",") {
			if strings.HasPrefix(p, "-") {
				ret = append(ret, p)
			} else {
				mountOptions += p
				mountOptions += ","
			}
		}

		if len(mountOptions) != 0 {
			// remove trailing ,
			mountOptions = mountOptions[:len(mountOptions)-1]
			ret = append(ret, "-o")
			ret = append(ret, mountOptions)
		}

		ret = append(ret, args[1])
		ret = append(ret, args[2])
	} else {
		return args
	}

	return
}
<commit_msg>group aws specific options<commit_after>// Copyright 2015 Ka-Hing Cheung
// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import (
	"io"
	"os"
	"strings"
	"text/tabwriter"
	"text/template"
	"time"

	"github.com/codegangsta/cli"
)

var awsFlags map[string]bool

// Set up custom help text for goofys; in particular the usage section.
func onlyAws(flags []cli.Flag, aws bool) (ret []cli.Flag) {
	for _, f := range flags {
		if awsFlags[f.GetName()] == aws {
			ret = append(ret, f)
		}
	}
	return
}

func init() {
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} {{if .Flags}}[global options]{{end}} bucket[:prefix] mountpoint
   {{if .Version}}
VERSION:
   {{.Version}}
   {{end}}{{if len .Authors}}
AUTHOR(S):
   {{range .Authors}}{{ . }}{{end}}
   {{end}}{{if .Commands}}
COMMANDS:
   {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
   {{end}}{{end}}{{if .Flags}}
GLOBAL OPTIONS:
   {{range onlyAws .Flags false}}{{.}}
   {{end}}
AWS S3 OPTIONS:
   {{range onlyAws .Flags true}}{{.}}
   {{end}}{{end}}{{if .Copyright }}
COPYRIGHT:
   {{.Copyright}}
   {{end}}
`
}

func NewApp() (app *cli.App) {
	uid, gid := MyUserAndGroup()

	app = &cli.App{
		Name:     "goofys",
		Version:  "0.0.8",
		Usage:    "Mount an S3 bucket locally",
		HideHelp: true,
		Writer:   os.Stderr,
		Flags: []cli.Flag{

			cli.BoolFlag{
				Name:  "help, h",
				Usage: "Print this help text and exit successfully.",
			},

			/////////////////////////
			// File system
			/////////////////////////

			cli.StringSliceFlag{
				Name:  "o",
				Usage: "Additional system-specific mount options. Be careful!",
			},

			cli.IntFlag{
				Name:  "dir-mode",
				Value: 0755,
				Usage: "Permission bits for directories. (default: 0755)",
			},

			cli.IntFlag{
				Name:  "file-mode",
				Value: 0644,
				Usage: "Permission bits for files. (default: 0644)",
			},

			cli.IntFlag{
				Name:  "uid",
				Value: uid,
				Usage: "UID owner of all inodes.",
			},

			cli.IntFlag{
				Name:  "gid",
				Value: gid,
				Usage: "GID owner of all inodes.",
			},

			/////////////////////////
			// S3
			/////////////////////////

			cli.StringFlag{
				Name:  "endpoint",
				Value: "",
				Usage: "The non-AWS endpoint to connect to." +
					" Possible values: http://127.0.0.1:8081/",
			},

			cli.StringFlag{
				Name:  "region",
				Value: "us-east-1",
				Usage: "The region to connect to. Usually this is auto-detected." +
					" Possible values: us-east-1, us-west-1, us-west-2, eu-west-1, " +
					"eu-central-1, ap-southeast-1, ap-southeast-2, ap-northeast-1, " +
					"sa-east-1, cn-north-1",
			},

			cli.StringFlag{
				Name:  "storage-class",
				Value: "STANDARD",
				Usage: "The type of storage to use when writing objects." +
					" Possible values: REDUCED_REDUNDANCY, STANDARD, STANDARD_IA.",
			},

			cli.BoolFlag{
				Name: "use-path-request",
				Usage: "Use a path-style request instead of virtual host-style." +
					" (deprecated, always on)",
				Hidden: true,
			},

			cli.StringFlag{
				Name:  "profile",
				Usage: "Use a named profile from $HOME/.aws/credentials instead of \"default\"",
			},

			cli.BoolFlag{
				Name:  "use-content-type",
				Usage: "Set Content-Type according to file extension and /etc/mime.types (default: off)",
			},

			/// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
			/// See http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html
			cli.BoolFlag{
				Name:  "sse",
				Usage: "Enable basic server-side encryption at rest (SSE-S3)in S3 for all writes (default: off)",
			},

			cli.StringFlag{
				Name:  "sse-kms",
				Usage: "Enable KMS encryption (SSE-KMS) for all writes using this particular KMS `key-id`. Leave blank to Use the account's CMK - customer master key (default: off)",
				Value: "",
			},

			/////////////////////////
			// Tuning
			/////////////////////////

			cli.DurationFlag{
				Name:  "stat-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache StatObject results and inode attributes.",
			},

			cli.DurationFlag{
				Name:  "type-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache name -> file/dir mappings in directory " +
					"inodes.",
			},

			/////////////////////////
			// Debugging
			/////////////////////////

			cli.BoolFlag{
				Name:  "debug_fuse",
				Usage: "Enable fuse-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "debug_s3",
				Usage: "Enable S3-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "f",
				Usage: "Run goofys in foreground.",
			},
		},
	}

	var funcMap = template.FuncMap{
		"onlyAws": onlyAws,
		"join":    strings.Join,
	}

	awsFlags = map[string]bool{"region": true, "sse": true, "sse-kms": true, "storage-class": true}

	cli.HelpPrinter = func(w io.Writer, templ string, data interface{}) {
		w = tabwriter.NewWriter(w, 1, 8, 2, ' ', 0)
		var tmplGet = template.Must(template.New("help").Funcs(funcMap).Parse(templ))
		tmplGet.Execute(w, app)
	}

	return
}

type FlagStorage struct {
	// File system
	MountOptions map[string]string
	DirMode      os.FileMode
	FileMode     os.FileMode
	Uid          uint32
	Gid          uint32

	// S3
	Endpoint       string
	Region         string
	RegionSet      bool
	StorageClass   string
	UsePathRequest bool
	Profile        string
	UseContentType bool
	UseSSE         bool
	UseKMS         bool
	KMSKeyID       string

	// Tuning
	StatCacheTTL time.Duration
	TypeCacheTTL time.Duration

	// Debugging
	DebugFuse  bool
	DebugS3    bool
	Foreground bool
}

func parseOptions(m map[string]string, s string) {
	// NOTE(jacobsa): The man pages don't define how escaping works, and as far
	// as I can tell there is no way to properly escape or quote a comma in the
	// options list for an fstab entry. So put our fingers in our ears and hope
	// that nobody needs a comma.
	for _, p := range strings.Split(s, ",") {
		var name string
		var value string

		// Split on the first equals sign.
		if equalsIndex := strings.IndexByte(p, '='); equalsIndex != -1 {
			name = p[:equalsIndex]
			value = p[equalsIndex+1:]
		} else {
			name = p
		}

		m[name] = value
	}

	return
}

// Add the flags accepted by run to the supplied flag set, returning the
// variables into which the flags will parse.
func PopulateFlags(c *cli.Context) (flags *FlagStorage) {
	flags = &FlagStorage{
		// File system
		MountOptions: make(map[string]string),
		DirMode:      os.FileMode(c.Int("dir-mode")),
		FileMode:     os.FileMode(c.Int("file-mode")),
		Uid:          uint32(c.Int("uid")),
		Gid:          uint32(c.Int("gid")),

		// Tuning,
		StatCacheTTL: c.Duration("stat-cache-ttl"),
		TypeCacheTTL: c.Duration("type-cache-ttl"),

		// S3
		Endpoint:       c.String("endpoint"),
		Region:         c.String("region"),
		RegionSet:      c.IsSet("region"),
		StorageClass:   c.String("storage-class"),
		UsePathRequest: c.Bool("use-path-request"),
		Profile:        c.String("profile"),
		UseContentType: c.Bool("use-content-type"),
		UseSSE:         c.Bool("sse"),
		UseKMS:         c.IsSet("sse-kms"),
		KMSKeyID:       c.String("sse-kms"),

		// Debugging,
		DebugFuse:  c.Bool("debug_fuse"),
		DebugS3:    c.Bool("debug_s3"),
		Foreground: c.Bool("f"),
	}

	// KMS implies SSE
	if flags.UseKMS {
		flags.UseSSE = true
	}

	// Handle the repeated "-o" flag.
	for _, o := range c.StringSlice("o") {
		parseOptions(flags.MountOptions, o)
	}
	return
}

func MassageMountFlags(args []string) (ret []string) {
	if len(args) == 5 && args[3] == "-o" {
		// looks like it's coming from fstab!
		mountOptions := ""
		ret = append(ret, args[0])

		for _, p := range strings.Split(args[4], ",") {
			if strings.HasPrefix(p, "-") {
				ret = append(ret, p)
			} else {
				mountOptions += p
				mountOptions += ","
			}
		}

		if len(mountOptions) != 0 {
			// remove trailing ,
			mountOptions = mountOptions[:len(mountOptions)-1]
			ret = append(ret, "-o")
			ret = append(ret, mountOptions)
		}

		ret = append(ret, args[1])
		ret = append(ret, args[2])
	} else {
		return args
	}

	return
}
<|endoftext|>


<commit_before>/*
 * Copyright 2016 Frank Wessels <fwessels@xs4all.nl>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package kv

import (
	"encoding/hex"
	"fmt"
	"github.com/s3git/s3git-go/internal/config"
	"github.com/bmatsuo/lmdb-go/lmdb"
	"os"
	"path"
)

// TODO: Use new transaction style for lmdb

var env *lmdb.Env

// KV databases containing root level digests for different object types
// When a particular key is present, the value is as follows:
//   - when empty, underlying chunk(s) are not cached locally
//   - when set, it is the concatenation of the leaf level digests for all nodes
//     (and thus necessarily needs to correspond when BLAKE2'd to its key)
//
// If you know the type of the key, you can fetch it directly for the corresponding database
// If you do not know the type, you need to search all stores

var dbiLevel1Blobs lmdb.DBI
var dbiLevel1Commits lmdb.DBI
var dbiLevel1Prefixes lmdb.DBI
var dbiLevel1Trees lmdb.DBI

// KV database containing overview of added/removed blobs in stage
var dbiStage lmdb.DBI

// KV database that marks commits objects as being a parent commit
var dbiLevel1CommitsIsParent lmdb.DBI

func OpenDatabase() error {

	mdbDir := path.Join(config.Config.BasePath, config.S3GIT_DIR, "mdb")
	err := os.MkdirAll(mdbDir, os.ModePerm)
	if err != nil {
		return err
	}

	env, _ = lmdb.NewEnv()
	// TODO: Figure out proper size for lmdb
	// TODO: Windows: max size is capped at 32
	env.SetMapSize(1 << 36) // max file size
	env.SetMaxDBs(10)       // up to 10 named databases
	env.Open(mdbDir, 0, 0664)

	err = env.Update(func(txn *lmdb.Txn) (err error) {

		// overview of blobs in stage
		dbiStage, err = txn.OpenDBI("stage", lmdb.Create)
		if err != nil {
			return err
		}

		// Level 1 databases
		dbiLevel1Blobs, err = txn.OpenDBI("l1blobs", lmdb.Create)
		if err != nil {
			return err
		}
		dbiLevel1Commits, err = txn.OpenDBI("l1commits", lmdb.Create)
		if err != nil {
			return err
		}
		dbiLevel1Prefixes, err = txn.OpenDBI("l1prefixes", lmdb.Create)
		if err != nil {
			return err
		}
		dbiLevel1Trees, err = txn.OpenDBI("l1trees", lmdb.Create)
		if err != nil {
			return err
		}

		// list of top most commits
		dbiLevel1CommitsIsParent, _ = txn.OpenDBI("l1commitsisparent", lmdb.Create)

		return nil
	})
	if err != nil {
		// ...
	}

	// From https://godoc.org/github.com/bmatsuo/lmdb-go/lmdb
	//   A database is referenced by an opaque handle known as its DBI which
	//   must be opened inside a transaction with the OpenDBI or OpenRoot methods.
	//   DBIs may be closed but it is not required. Typically, applications acquire
	//   handles for all their databases immediately after opening an environment
	//   and retain them for the lifetime of the process.

	return nil
}

func AddToStage(key string) error {

	hx, _ := hex.DecodeString(key)

	txn, _ := env.BeginTxn(nil, 0)
	txn.Put(dbiStage, hx, nil, 0)
	txn.Commit()

	return nil
}

func ClearStage() error {

	list, err := listMdb(&dbiStage, "")
	if err != nil {
		return err
	}

	txn, _ := env.BeginTxn(nil, 0)
	for k := range list {
		txn.Del(dbiStage, k, nil)
	}
	txn.Commit()

	return nil
}

func ListStage() (<-chan []byte, error) {

	return listMdb(&dbiStage, "")
}

func MarkCommitAsParent(key string) error {

	hx, _ := hex.DecodeString(key)

	txn, _ := env.BeginTxn(nil, 0)
	txn.Put(dbiLevel1CommitsIsParent, hx, nil, 0)
	txn.Commit()

	return nil
}

func CommitIsParent(key []byte) (bool, error) {

	txn, _ := env.BeginTxn(nil, lmdb.Readonly)
	defer txn.Abort()

	_, err := txn.Get(dbiLevel1CommitsIsParent, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return false, err
	} else if !lmdb.IsNotFound(err) {
		return true, err
	}

	return false, lmdb.NotFound
}

func ListTopMostCommits() (<-chan []byte, error) {

	list, err := ListLevel1Commits()
	if err != nil {
		return nil, err
	}

	result := make(chan []byte)

	go func() {
		// make sure we always close the channel
		defer close(result)

		for l := range list {
			isParent, err := CommitIsParent(l)
			if err != nil && !lmdb.IsNotFound(err) {
				return
			}

			if !isParent { // In case this commit is not a parent --> output it as a top most commit
				result <- l
			}
		}
	}()

	return result, nil
}

func ListLevel1Commits() (<-chan []byte, error) {

	return listMdb(&dbiLevel1Commits, "")
}

func ListLevel1Prefixes() (<-chan []byte, error) {

	return listMdb(&dbiLevel1Prefixes, "")
}

func ListLevel1Trees() (<-chan []byte, error) {

	return listMdb(&dbiLevel1Trees, "")
}

func ListLevel1Blobs(query string) (<-chan []byte, error) {

	return listMdb(&dbiLevel1Blobs, query)
}

func GetLevel1BlobsStats() (uint64, error) {

	txn, _ := env.BeginTxn(nil, lmdb.Readonly)
	defer txn.Abort()
	stats, err := txn.Stat(dbiLevel1Blobs)
	if err != nil {
		return 0, err
	}

	return stats.Entries, nil
}

const BLOB = "blob"
const COMMIT = "commit"
const PREFIX = "prefix"
const TREE = "tree"

func getDbForObjectType(objType string) *lmdb.DBI {

	var dbi *lmdb.DBI
	switch objType {
	case BLOB:
		dbi = &dbiLevel1Blobs
	case COMMIT:
		dbi = &dbiLevel1Commits
	case PREFIX:
		dbi = &dbiLevel1Prefixes
	case TREE:
		dbi = &dbiLevel1Trees
	default:
		panic(fmt.Sprintf("Bad type: %s", objType))
	}
	return dbi
}

func AddToLevel1(key, value []byte, objType string) error {

	dbi := getDbForObjectType(objType)
	txn, _ := env.BeginTxn(nil, 0)
	txn.Put(*dbi, key, value, 0)
	txn.Commit()

	return nil
}

func AddMultiToLevel1(keys, values [][]byte, objType string) error {

	dbi := getDbForObjectType(objType)
	txn, _ := env.BeginTxn(nil, 0)
	for index, key := range keys {
		txn.Put(*dbi, key, values[index], 0)
	}
	txn.Commit()

	return nil
}

// Get object of any type, return value and type
func GetLevel1(key []byte) ([]byte, string, error) {

	txn, _ := env.BeginTxn(nil, lmdb.Readonly)
	defer txn.Abort()

	val, err := txn.Get(dbiLevel1Blobs, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return nil, "", err
	} else if !lmdb.IsNotFound(err) {
		return val, BLOB, err
	}

	val, err = txn.Get(dbiLevel1Commits, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return nil, "", err
	} else if !lmdb.IsNotFound(err) {
		return val, COMMIT, err
	}

	val, err = txn.Get(dbiLevel1Prefixes, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return nil, "", err
	} else if !lmdb.IsNotFound(err) {
		return val, PREFIX, err
	}

	val, err = txn.Get(dbiLevel1Trees, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return nil, "", err
	} else if !lmdb.IsNotFound(err) {
		return val, TREE, err
	}

	return nil, "", lmdb.NotFound
}

func listMdb(dbi *lmdb.DBI, query string) (<-chan []byte, error) {

	result := make(chan []byte)

	go func() {
		// make sure we always close the channel
		defer close(result)

		// scan the database
		txn, _ := env.BeginTxn(nil, lmdb.Readonly)
		defer txn.Abort()
		cursor, _ := txn.OpenCursor(*dbi)
		defer cursor.Close()

		setRangeUponStart := len(query) > 0
		var queryKey []byte
		if setRangeUponStart {
			q := query
			if len(q)%2 == 1 {
				q = q + "0"
			}
			queryKey, _ = hex.DecodeString(q)
		}

		for {

			var bkey []byte
			if setRangeUponStart {
				var err error
				bkey, _, err = cursor.Get(queryKey, nil, lmdb.SetRange)
				if lmdb.IsNotFound(err) {
					break
				}
				if err != nil {
					// TODO: Log error
					return
				}

				setRangeUponStart = false
			} else {
				var err error
				bkey, _, err = cursor.Get(nil, nil, lmdb.Next)
				if lmdb.IsNotFound(err) {
					break
				}
				if err != nil {
					// TODO: Log error
					return
				}
			}

			// break early if start of key is not longer
			if hex.EncodeToString(bkey)[:len(query)] != query {
				break
			}
			result <- bkey
		}

	}()

	return result, nil
}
<commit_msg>Output error to stderr<commit_after>/*
 * Copyright 2016 Frank Wessels <fwessels@xs4all.nl>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package kv

import (
	"encoding/hex"
	"fmt"
	"github.com/s3git/s3git-go/internal/config"
	"github.com/bmatsuo/lmdb-go/lmdb"
	"os"
	"path"
)

// TODO: Use new transaction style for lmdb

var env *lmdb.Env

// KV databases containing root level digests for different object types
// When a particular key is present, the value is as follows:
//   - when empty, underlying chunk(s) are not cached locally
//   - when set, it is the concatenation of the leaf level digests for all nodes
//     (and thus necessarily needs to correspond when BLAKE2'd to its key)
//
// If you know the type of the key, you can fetch it directly for the corresponding database
// If you do not know the type, you need to search all stores

var dbiLevel1Blobs lmdb.DBI
var dbiLevel1Commits lmdb.DBI
var dbiLevel1Prefixes lmdb.DBI
var dbiLevel1Trees lmdb.DBI

// KV database containing overview of added/removed blobs in stage
var dbiStage lmdb.DBI

// KV database that marks commits objects as being a parent commit
var dbiLevel1CommitsIsParent lmdb.DBI

func OpenDatabase() error {

	mdbDir := path.Join(config.Config.BasePath, config.S3GIT_DIR, "mdb")
	err := os.MkdirAll(mdbDir, os.ModePerm)
	if err != nil {
		return err
	}

	env, _ = lmdb.NewEnv()
	// TODO: Figure out proper size for lmdb
	// TODO: Windows: max size is capped at 32
	env.SetMapSize(1 << 36) // max file size
	env.SetMaxDBs(10)       // up to 10 named databases
	env.Open(mdbDir, 0, 0664)

	err = env.Update(func(txn *lmdb.Txn) (err error) {

		// overview of blobs in stage
		dbiStage, err = txn.OpenDBI("stage", lmdb.Create)
		if err != nil {
			return err
		}

		// Level 1 databases
		dbiLevel1Blobs, err = txn.OpenDBI("l1blobs", lmdb.Create)
		if err != nil {
			return err
		}
		dbiLevel1Commits, err = txn.OpenDBI("l1commits", lmdb.Create)
		if err != nil {
			return err
		}
		dbiLevel1Prefixes, err = txn.OpenDBI("l1prefixes", lmdb.Create)
		if err != nil {
			return err
		}
		dbiLevel1Trees, err = txn.OpenDBI("l1trees", lmdb.Create)
		if err != nil {
			return err
		}

		// list of top most commits
		dbiLevel1CommitsIsParent, _ = txn.OpenDBI("l1commitsisparent", lmdb.Create)

		return nil
	})
	if err != nil {
		// ...
	}

	// From https://godoc.org/github.com/bmatsuo/lmdb-go/lmdb
	//   A database is referenced by an opaque handle known as its DBI which
	//   must be opened inside a transaction with the OpenDBI or OpenRoot methods.
	//   DBIs may be closed but it is not required. Typically, applications acquire
	//   handles for all their databases immediately after opening an environment
	//   and retain them for the lifetime of the process.

	return nil
}

func AddToStage(key string) error {

	hx, _ := hex.DecodeString(key)

	txn, _ := env.BeginTxn(nil, 0)
	txn.Put(dbiStage, hx, nil, 0)
	txn.Commit()

	return nil
}

func ClearStage() error {

	list, err := listMdb(&dbiStage, "")
	if err != nil {
		return err
	}

	txn, _ := env.BeginTxn(nil, 0)
	for k := range list {
		txn.Del(dbiStage, k, nil)
	}
	txn.Commit()

	return nil
}

func ListStage() (<-chan []byte, error) {

	return listMdb(&dbiStage, "")
}

func MarkCommitAsParent(key string) error {

	hx, _ := hex.DecodeString(key)

	txn, _ := env.BeginTxn(nil, 0)
	txn.Put(dbiLevel1CommitsIsParent, hx, nil, 0)
	txn.Commit()

	return nil
}

func CommitIsParent(key []byte) (bool, error) {

	txn, _ := env.BeginTxn(nil, lmdb.Readonly)
	defer txn.Abort()

	_, err := txn.Get(dbiLevel1CommitsIsParent, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return false, err
	} else if !lmdb.IsNotFound(err) {
		return true, err
	}

	return false, lmdb.NotFound
}

func ListTopMostCommits() (<-chan []byte, error) {

	list, err := ListLevel1Commits()
	if err != nil {
		return nil, err
	}

	result := make(chan []byte)

	go func() {
		// make sure we always close the channel
		defer close(result)

		for l := range list {
			isParent, err := CommitIsParent(l)
			if err != nil && !lmdb.IsNotFound(err) {
				return
			}

			if !isParent { // In case this commit is not a parent --> output it as a top most commit
				result <- l
			}
		}
	}()

	return result, nil
}

func ListLevel1Commits() (<-chan []byte, error) {

	return listMdb(&dbiLevel1Commits, "")
}

func ListLevel1Prefixes() (<-chan []byte, error) {

	return listMdb(&dbiLevel1Prefixes, "")
}

func ListLevel1Trees() (<-chan []byte, error) {

	return listMdb(&dbiLevel1Trees, "")
}

func ListLevel1Blobs(query string) (<-chan []byte, error) {

	return listMdb(&dbiLevel1Blobs, query)
}

func GetLevel1BlobsStats() (uint64, error) {

	txn, _ := env.BeginTxn(nil, lmdb.Readonly)
	defer txn.Abort()
	stats, err := txn.Stat(dbiLevel1Blobs)
	if err != nil {
		return 0, err
	}

	return stats.Entries, nil
}

const BLOB = "blob"
const COMMIT = "commit"
const PREFIX = "prefix"
const TREE = "tree"

func getDbForObjectType(objType string) *lmdb.DBI {

	var dbi *lmdb.DBI
	switch objType {
	case BLOB:
		dbi = &dbiLevel1Blobs
	case COMMIT:
		dbi = &dbiLevel1Commits
	case PREFIX:
		dbi = &dbiLevel1Prefixes
	case TREE:
		dbi = &dbiLevel1Trees
	default:
		panic(fmt.Sprintf("Bad type: %s", objType))
	}
	return dbi
}

func AddToLevel1(key, value []byte, objType string) error {

	dbi := getDbForObjectType(objType)
	txn, _ := env.BeginTxn(nil, 0)
	txn.Put(*dbi, key, value, 0)
	txn.Commit()

	return nil
}

func AddMultiToLevel1(keys, values [][]byte, objType string) error {

	dbi := getDbForObjectType(objType)
	txn, _ := env.BeginTxn(nil, 0)
	for index, key := range keys {
		txn.Put(*dbi, key, values[index], 0)
	}
	txn.Commit()

	return nil
}

// Get object of any type, return value and type
func GetLevel1(key []byte) ([]byte, string, error) {

	txn, _ := env.BeginTxn(nil, lmdb.Readonly)
	defer txn.Abort()

	val, err := txn.Get(dbiLevel1Blobs, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return nil, "", err
	} else if !lmdb.IsNotFound(err) {
		return val, BLOB, err
	}

	val, err = txn.Get(dbiLevel1Commits, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return nil, "", err
	} else if !lmdb.IsNotFound(err) {
		return val, COMMIT, err
	}

	val, err = txn.Get(dbiLevel1Prefixes, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return nil, "", err
	} else if !lmdb.IsNotFound(err) {
		return val, PREFIX, err
	}

	val, err = txn.Get(dbiLevel1Trees, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return nil, "", err
	} else if !lmdb.IsNotFound(err) {
		return val, TREE, err
	}

	return nil, "", lmdb.NotFound
}

func listMdb(dbi *lmdb.DBI, query string) (<-chan []byte, error) {

	result := make(chan []byte)

	go func() {
		// make sure we always close the channel
		defer close(result)

		// scan the database
		txn, _ := env.BeginTxn(nil, lmdb.Readonly)
		defer txn.Abort()
		cursor, _ := txn.OpenCursor(*dbi)
		defer cursor.Close()

		setRangeUponStart := len(query) > 0
		var queryKey []byte
		if setRangeUponStart {
			q := query
			if len(q)%2 == 1 {
				q = q + "0"
			}
			queryKey, _ = hex.DecodeString(q)
		}

		for {

			var bkey []byte
			if setRangeUponStart {
				var err error
				bkey, _, err = cursor.Get(queryKey, nil, lmdb.SetRange)
				if lmdb.IsNotFound(err) {
					break
				}
				if err != nil {
					fmt.Fprint(os.Stderr, err)
					return
				}

				setRangeUponStart = false
			} else {
				var err error
				bkey, _, err = cursor.Get(nil, nil, lmdb.Next)
				if lmdb.IsNotFound(err) {
					break
				}
				if err != nil {
					fmt.Fprint(os.Stderr, err)
					return
				}
			}

			// break early if start of key is not longer
			if hex.EncodeToString(bkey)[:len(query)] != query {
				break
			}
			result <- bkey
		}

	}()

	return result, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import "time"

// DateTime is the public API representation of a point in time.
type DateTime struct {
	// Text is the time specified as a string. The time is displayed in
	// the corresponding TimeZone.
	Text string `json:"text"`

	// TimeZone is the name of the time zone in the IANA Time Zone Database. For
	// example, "America/New_York" or "Australia/Sydney".
	TimeZone string `json:"time_zone"`

	// Value is the number of milliseconds since midnight 01 January, 1970 UTC.
	Value int64 `json:"value"`
}

// Time returns the time.Time for this DateTime.
func (dt *DateTime) Time() time.Time {
	if dt == nil {
		return time.Time{}
	}

	loc, err := time.LoadLocation(dt.TimeZone)
	t := time.Unix(0, dt.Value*1e6)
	if err == nil && loc != nil {
		t = t.In(loc)
	}
	return t
}

// NewDateTime builds a DateTime from the given time.Time. This will be nil
// if time.Time is the zero time.
func NewDateTime(t time.Time) *DateTime {
	if t.IsZero() {
		return nil
	}

	loc := t.Location()
	return &DateTime{
		Text:     t.Format(time.RFC1123), // TODO(samthor): better format
		TimeZone: loc.String(),
		Value:    t.UnixNano() / int64(time.Millisecond),
	}
}

// Duration is the public API representation of a duration.
type Duration struct {
	// Value indicates the duration, in seconds.
	Value int64 `json:"value"`

	// Text contains a human-readable representation of the duration.
	Text string `json:"text"`
}

// Duration returns the time.Duration for this internal Duration.
func (d *Duration) Duration() time.Duration {
	if d == nil {
		return time.Duration(0)
	}
	return time.Duration(d.Value) * time.Second
}

// NewDuration builds an internal Duration from the passed time.Duration.
func NewDuration(d time.Duration) *Duration {
	if d == 0 {
		return &Duration{}
	}
	return &Duration{
		Value: int64(d / time.Second),
		Text:  d.String(),
	}
}

// Location is the Roads API+ representation of a location on the Earth. It
// differs only in the encoded names, which are the longer forms of 'latitude'
// and 'longitude'.
type Location struct {
	Latitude  float64 `json:"latitude"`
	Longitude float64 `json:"longitude"`
}
<commit_msg>Correct erroneous time.Time values returned by the library.   Per: https://developers.google.com/maps/documentation/directions/intro#TransitDetails   Time values are in seconds, not milliseconds.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import "time"

// DateTime is the public API representation of a point in time.
type DateTime struct {
	// Text is the time specified as a string. The time is displayed in
	// the corresponding TimeZone.
	Text string `json:"text"`

	// TimeZone is the name of the time zone in the IANA Time Zone Database. For
	// example, "America/New_York" or "Australia/Sydney".
	TimeZone string `json:"time_zone"`

	// Value is the number of seconds since midnight 01 January, 1970 UTC.
	Value int64 `json:"value"`
}

// Time returns the time.Time for this DateTime.
func (dt *DateTime) Time() time.Time {
	if dt == nil {
		return time.Time{}
	}

	loc, err := time.LoadLocation(dt.TimeZone)
	t := time.Unix(dt.Value, 0)
	if err == nil && loc != nil {
		t = t.In(loc)
	}
	return t
}

// NewDateTime builds a DateTime from the given time.Time. This will be nil
// if time.Time is the zero time.
func NewDateTime(t time.Time) *DateTime {
	if t.IsZero() {
		return nil
	}

	loc := t.Location()
	return &DateTime{
		Text:     t.Format(time.RFC1123), // TODO(samthor): better format
		TimeZone: loc.String(),
		Value:    t.UnixNano() / int64(time.Second),
	}
}

// Duration is the public API representation of a duration.
type Duration struct {
	// Value indicates the duration, in seconds.
	Value int64 `json:"value"`

	// Text contains a human-readable representation of the duration.
	Text string `json:"text"`
}

// Duration returns the time.Duration for this internal Duration.
func (d *Duration) Duration() time.Duration {
	if d == nil {
		return time.Duration(0)
	}
	return time.Duration(d.Value) * time.Second
}

// NewDuration builds an internal Duration from the passed time.Duration.
func NewDuration(d time.Duration) *Duration {
	if d == 0 {
		return &Duration{}
	}
	return &Duration{
		Value: int64(d / time.Second),
		Text:  d.String(),
	}
}

// Location is the Roads API+ representation of a location on the Earth. It
// differs only in the encoded names, which are the longer forms of 'latitude'
// and 'longitude'.
type Location struct {
	Latitude  float64 `json:"latitude"`
	Longitude float64 `json:"longitude"`
}
<|endoftext|>


<commit_before>// Package log implements a wrapper around the Go standard library's
// logging package. Clients should set the current log level; only
// messages below that level will actually be logged. For example, if
// Level is set to LevelWarning, only log messages at the Warning,
// Error, and Critical levels will be logged.
package log

import (
	"flag"
	"fmt"
	"log"
	"os"
	"sync"
)

// The following constants represent logging levels in increasing levels of seriousness.
const (
	// LevelDebug is the log level for Debug statements.
	LevelDebug = iota
	// LevelInfo is the log level for Info statements.
	LevelInfo
	// LevelWarning is the log level for Warning statements.
	LevelWarning
	// LevelError is the log level for Error statements.
	LevelError
	// LevelCritical is the log level for Critical statements.
	LevelCritical
	// LevelFatal is the log level for Fatal statements.
	LevelFatal
)

var levelPrefix = [...]string{
	LevelDebug:    "DEBUG",
	LevelInfo:     "INFO",
	LevelWarning:  "WARNING",
	LevelError:    "ERROR",
	LevelCritical: "CRITICAL",
	LevelFatal:    "FATAL",
}

// Level stores the current logging level.
var Level = LevelInfo

// SyslogWriter specifies the necessary methods for an alternate output
// destination passed in via SetLogger.
//
// SyslogWriter is satisfied by *syslog.Writer.
type SyslogWriter interface {
	Debug(string) error
	Info(string) error
	Warning(string) error
	Err(string) error
	Crit(string) error
	Emerg(string) error
}

// syslogWriter stores the SetLogger() parameter.
var syslogWriter SyslogWriter

// SetLogger sets the output used for output by this package.
// A *syslog.Writer is a good choice for the logger parameter.
// Call with a nil parameter to revert to default behavior.
func SetLogger(logger SyslogWriter) {
	syslogWriter = logger
}

// once ensures that the loglevel flag isn't redefined
// if multiple packages import "github.com/cloudflare/cfssl/log"
var once sync.Once

func init() {
	once.Do(func() { flag.IntVar(&Level, "loglevel", LevelInfo, "Log level (0 = DEBUG, 5 = FATAL)") })
}

func print(l int, msg string) {
	if l >= Level {
		if syslogWriter != nil {
			var err error
			switch l {
			case LevelDebug:
				err = syslogWriter.Debug(msg)
			case LevelInfo:
				err = syslogWriter.Info(msg)
			case LevelWarning:
				err = syslogWriter.Warning(msg)
			case LevelError:
				err = syslogWriter.Err(msg)
			case LevelCritical:
				err = syslogWriter.Crit(msg)
			case LevelFatal:
				err = syslogWriter.Emerg(msg)
			}
			if err != nil {
				log.Printf("Unable to write syslog: %v for msg: %s\n", err, msg)
			}
		} else {
			log.Printf("[%s] %s", levelPrefix[l], msg)
		}
	}
}

func outputf(l int, format string, v []interface{}) {
	print(l, fmt.Sprintf(format, v...))
}

func output(l int, v []interface{}) {
	print(l, fmt.Sprint(v...))
}

// Fatalf logs a formatted message at the "fatal" level and then exits. The
// arguments are handled in the same manner as fmt.Printf.
func Fatalf(format string, v ...interface{}) {
	outputf(LevelFatal, format, v)
	os.Exit(1)
}

// Fatal logs its arguments at the "fatal" level and then exits.
func Fatal(v ...interface{}) {
	output(LevelFatal, v)
	os.Exit(1)
}

// Criticalf logs a formatted message at the "critical" level. The
// arguments are handled in the same manner as fmt.Printf.
func Criticalf(format string, v ...interface{}) {
	outputf(LevelCritical, format, v)
}

// Critical logs its arguments at the "critical" level.
func Critical(v ...interface{}) {
	output(LevelCritical, v)
}

// Errorf logs a formatted message at the "error" level. The arguments
// are handled in the same manner as fmt.Printf.
func Errorf(format string, v ...interface{}) {
	outputf(LevelError, format, v)
}

// Error logs its arguments at the "error" level.
func Error(v ...interface{}) {
	output(LevelError, v)
}

// Warningf logs a formatted message at the "warning" level. The
// arguments are handled in the same manner as fmt.Printf.
func Warningf(format string, v ...interface{}) {
	outputf(LevelWarning, format, v)
}

// Warning logs its arguments at the "warning" level.
func Warning(v ...interface{}) {
	output(LevelWarning, v)
}

// Infof logs a formatted message at the "info" level. The arguments
// are handled in the same manner as fmt.Printf.
func Infof(format string, v ...interface{}) {
	outputf(LevelInfo, format, v)
}

// Info logs its arguments at the "info" level.
func Info(v ...interface{}) {
	output(LevelInfo, v)
}

// Debugf logs a formatted message at the "debug" level. The arguments
// are handled in the same manner as fmt.Printf.
func Debugf(format string, v ...interface{}) {
	outputf(LevelDebug, format, v)
}

// Debug logs its arguments at the "debug" level.
func Debug(v ...interface{}) {
	output(LevelDebug, v)
}
<commit_msg>Better fix of loglevel redefinition bug<commit_after>// Package log implements a wrapper around the Go standard library's
// logging package. Clients should set the current log level; only
// messages below that level will actually be logged. For example, if
// Level is set to LevelWarning, only log messages at the Warning,
// Error, and Critical levels will be logged.
package log

import (
	"flag"
	"fmt"
	"log"
	"os"
)

// The following constants represent logging levels in increasing levels of seriousness.
const (
	// LevelDebug is the log level for Debug statements.
	LevelDebug = iota
	// LevelInfo is the log level for Info statements.
	LevelInfo
	// LevelWarning is the log level for Warning statements.
	LevelWarning
	// LevelError is the log level for Error statements.
	LevelError
	// LevelCritical is the log level for Critical statements.
	LevelCritical
	// LevelFatal is the log level for Fatal statements.
	LevelFatal
)

var levelPrefix = [...]string{
	LevelDebug:    "DEBUG",
	LevelInfo:     "INFO",
	LevelWarning:  "WARNING",
	LevelError:    "ERROR",
	LevelCritical: "CRITICAL",
	LevelFatal:    "FATAL",
}

// Level stores the current logging level.
var Level = LevelInfo

// SyslogWriter specifies the necessary methods for an alternate output
// destination passed in via SetLogger.
//
// SyslogWriter is satisfied by *syslog.Writer.
type SyslogWriter interface {
	Debug(string) error
	Info(string) error
	Warning(string) error
	Err(string) error
	Crit(string) error
	Emerg(string) error
}

// syslogWriter stores the SetLogger() parameter.
var syslogWriter SyslogWriter

// SetLogger sets the output used for output by this package.
// A *syslog.Writer is a good choice for the logger parameter.
// Call with a nil parameter to revert to default behavior.
func SetLogger(logger SyslogWriter) {
	syslogWriter = logger
}

func init() {
	// Only define loglevel flag once.
	if flag.Lookup("loglevel") == nil {
		flag.IntVar(&Level, "loglevel", LevelInfo, "Log level (0 = DEBUG, 5 = FATAL)")
	}
}

func print(l int, msg string) {
	if l >= Level {
		if syslogWriter != nil {
			var err error
			switch l {
			case LevelDebug:
				err = syslogWriter.Debug(msg)
			case LevelInfo:
				err = syslogWriter.Info(msg)
			case LevelWarning:
				err = syslogWriter.Warning(msg)
			case LevelError:
				err = syslogWriter.Err(msg)
			case LevelCritical:
				err = syslogWriter.Crit(msg)
			case LevelFatal:
				err = syslogWriter.Emerg(msg)
			}
			if err != nil {
				log.Printf("Unable to write syslog: %v for msg: %s\n", err, msg)
			}
		} else {
			log.Printf("[%s] %s", levelPrefix[l], msg)
		}
	}
}

func outputf(l int, format string, v []interface{}) {
	print(l, fmt.Sprintf(format, v...))
}

func output(l int, v []interface{}) {
	print(l, fmt.Sprint(v...))
}

// Fatalf logs a formatted message at the "fatal" level and then exits. The
// arguments are handled in the same manner as fmt.Printf.
func Fatalf(format string, v ...interface{}) {
	outputf(LevelFatal, format, v)
	os.Exit(1)
}

// Fatal logs its arguments at the "fatal" level and then exits.
func Fatal(v ...interface{}) {
	output(LevelFatal, v)
	os.Exit(1)
}

// Criticalf logs a formatted message at the "critical" level. The
// arguments are handled in the same manner as fmt.Printf.
func Criticalf(format string, v ...interface{}) {
	outputf(LevelCritical, format, v)
}

// Critical logs its arguments at the "critical" level.
func Critical(v ...interface{}) {
	output(LevelCritical, v)
}

// Errorf logs a formatted message at the "error" level. The arguments
// are handled in the same manner as fmt.Printf.
func Errorf(format string, v ...interface{}) {
	outputf(LevelError, format, v)
}

// Error logs its arguments at the "error" level.
func Error(v ...interface{}) {
	output(LevelError, v)
}

// Warningf logs a formatted message at the "warning" level. The
// arguments are handled in the same manner as fmt.Printf.
func Warningf(format string, v ...interface{}) {
	outputf(LevelWarning, format, v)
}

// Warning logs its arguments at the "warning" level.
func Warning(v ...interface{}) {
	output(LevelWarning, v)
}

// Infof logs a formatted message at the "info" level. The arguments
// are handled in the same manner as fmt.Printf.
func Infof(format string, v ...interface{}) {
	outputf(LevelInfo, format, v)
}

// Info logs its arguments at the "info" level.
func Info(v ...interface{}) {
	output(LevelInfo, v)
}

// Debugf logs a formatted message at the "debug" level. The arguments
// are handled in the same manner as fmt.Printf.
func Debugf(format string, v ...interface{}) {
	outputf(LevelDebug, format, v)
}

// Debug logs its arguments at the "debug" level.
func Debug(v ...interface{}) {
	output(LevelDebug, v)
}
<|endoftext|>


<commit_before>///bin/true; exec /usr/bin/env go run "$0" "$@"

package main

import (
	"flag"
	"fmt"
	"github.com/omakoto/mlib"
	"io/ioutil"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	// Patterns to extract PIDs from logs.
	pidPatterns = []*regexp.Regexp{
		// brief
		regexp.MustCompile(`^[A-Z]/.+?\(\s*(\d+)`),
		// process or thread
		regexp.MustCompile(`^[A-Z]\(\s*(\d+)`),
		// threadtime
		regexp.MustCompile(`^\d{2}-\d{2}\s+[\d\:\.]+\s+(\d+)`),
		// time
		regexp.MustCompile(`^\d{2}-\d{2} [\d\:\.]+ [A-Z]\/.*?\(\s*(\d+)`),
		// regexp.MustCompile(`^.*?\(\s*(\d+)`),
		// long
		regexp.MustCompile(`^\[\s*\d{2}-\d{2}\s+[\d\:\.]+\s+(\d+)`),
	}

	// Patterns to extract PIDs for process deaths logs.
	diePatterns = []*regexp.Regexp{
		regexp.MustCompile(`ActivityManager.*?Process .*?\(pid (\d+)\) has died`),
		regexp.MustCompile(`ActivityManager.*?Killing (\d+)`),
		// Any more?
	}

	whiteSpaces = regexp.MustCompile(`\s+`)

	// Flags
	width = flag.Int("w", 40, "formatting width")
	// autoflush = flag.Bool("f", false, "autoflush") // Stdout seems like always flushing

	// Output line format
	outFormat string

	// Process info cache.
	procecces = make(map[int]processInfo)

	cacheExpiration = time.Minute * 10
)

const (
	PRE_INITIALIZED                       = "<pre-initialized>"
	MAX_PRE_INITIALIZED_RETRY             = 5
	MAX_PRE_INITIALIZED_RETRY_INTERVAL_MS = 200
)

type processInfo struct {
	name       string
	expiration time.Time
}

func getProcessNameFromAdbRaw(pid int) string {
	cmd := exec.Command("adb", "shell", fmt.Sprintf("cat /proc/%d/cmdline 2>/dev/null", pid))
	stdout, err := cmd.StdoutPipe()
	mlib.Check(err)

	err = cmd.Start()
	mlib.Check(err)
	defer cmd.Wait()

	cmdline, err := ioutil.ReadAll(stdout)
	mlib.Check(err)

	procname := string(cmdline)
	procname = strings.TrimRight(procname, "\000")
	procname = strings.Replace(procname, "\000", " ", -1)
	return procname
}

// Same as getProcessNameFromAdbRaw, but retries when getting PRE_INITIALIZED
func getProcessNameFromAdb(pid int) string {
	var pname string = "(unknown)"

	mlib.Debug("Getting process name for %d\n", pid)

	for i := 0; i <= MAX_PRE_INITIALIZED_RETRY; i++ {
		rawName := getProcessNameFromAdbRaw(pid)
		if rawName == PRE_INITIALIZED {
			mlib.Debug("%s detected\n", PRE_INITIALIZED)
			time.Sleep(MAX_PRE_INITIALIZED_RETRY_INTERVAL_MS * time.Millisecond)
			continue
		}
		if rawName != "" {
			pname = rawName
		}
		break
	}
	return fmt.Sprintf("%s %d", pname, pid)
}

func getProcessNameWithCache(pid int) string {
	now := time.Now()
	pinfo, ok := procecces[pid]
	if !ok || now.After(pinfo.expiration) {
		name := getProcessNameFromAdb(pid)
		procecces[pid] = processInfo{name: name, expiration: now.Add(cacheExpiration)}
	}
	return pinfo.name
}

// Run for each line
func processLine(line string) {
	var pid = 0
	var processName = ""

	// Find the pid from the line and get the process name
	for _, re := range pidPatterns {
		s := re.FindStringSubmatch(line)
		if s != nil {
			pid, _ = strconv.Atoi(s[1])
			// mlib.Debug("pid=%d\n", pid)
			processName = getProcessNameWithCache(pid)
			break
		}
	}

	// Any process died?
	for _, re := range diePatterns {
		s := re.FindStringSubmatch(line)
		if s != nil {
			diedPid, _ := strconv.Atoi(s[1])
			mlib.Debug("Process %d died\n", diedPid)
			delete(procecces, diedPid)
			break
		}
	}

	fmt.Printf(outFormat, processName, line)
}

func main() {
	flag.Parse()

	outFormat = fmt.Sprintf("[%%-%ds] %%s", *width)

	for line := range mlib.ReadFilesFromArgs() {
		processLine(line)
	}
}
<commit_msg>Simplify<commit_after>///bin/true; exec /usr/bin/env go run "$0" "$@"

package main

import (
	"flag"
	"fmt"
	"github.com/omakoto/mlib"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	// Patterns to extract PIDs from logs.
	pidPatterns = []*regexp.Regexp{
		// brief
		regexp.MustCompile(`^[A-Z]/.+?\(\s*(\d+)`),
		// process or thread
		regexp.MustCompile(`^[A-Z]\(\s*(\d+)`),
		// threadtime
		regexp.MustCompile(`^\d{2}-\d{2}\s+[\d\:\.]+\s+(\d+)`),
		// time
		regexp.MustCompile(`^\d{2}-\d{2} [\d\:\.]+ [A-Z]\/.*?\(\s*(\d+)`),
		// regexp.MustCompile(`^.*?\(\s*(\d+)`),
		// long
		regexp.MustCompile(`^\[\s*\d{2}-\d{2}\s+[\d\:\.]+\s+(\d+)`),
	}

	// Patterns to extract PIDs for process deaths logs.
	diePatterns = []*regexp.Regexp{
		regexp.MustCompile(`ActivityManager.*?Process .*?\(pid (\d+)\) has died`),
		regexp.MustCompile(`ActivityManager.*?Killing (\d+)`),
		// Any more?
	}

	whiteSpaces = regexp.MustCompile(`\s+`)

	// Flags
	width = flag.Int("w", 40, "formatting width")
	// autoflush = flag.Bool("f", false, "autoflush") // Stdout seems like always flushing

	// Output line format
	outFormat string

	// Process info cache.
	procecces = make(map[int]processInfo)

	cacheExpiration = time.Minute * 10
)

const (
	PRE_INITIALIZED                       = "<pre-initialized>"
	MAX_PRE_INITIALIZED_RETRY             = 5
	MAX_PRE_INITIALIZED_RETRY_INTERVAL_MS = 200
)

type processInfo struct {
	name       string
	expiration time.Time
}

func getProcessNameFromAdbRaw(pid int) string {
	cmd := exec.Command("adb", "shell", fmt.Sprintf("cat /proc/%d/cmdline 2>/dev/null", pid))
	out, err := cmd.Output()
	mlib.Check(err)

	procname := string(out)
	procname = strings.TrimRight(procname, "\000")
	procname = strings.Replace(procname, "\000", " ", -1)
	return procname
}

// Same as getProcessNameFromAdbRaw, but retries when getting PRE_INITIALIZED
func getProcessNameFromAdb(pid int) string {
	var pname string = "(unknown)"

	mlib.Debug("Getting process name for %d\n", pid)

	for i := 0; i <= MAX_PRE_INITIALIZED_RETRY; i++ {
		rawName := getProcessNameFromAdbRaw(pid)
		if rawName == PRE_INITIALIZED {
			mlib.Debug("%s detected\n", PRE_INITIALIZED)
			time.Sleep(MAX_PRE_INITIALIZED_RETRY_INTERVAL_MS * time.Millisecond)
			continue
		}
		if rawName != "" {
			pname = rawName
		}
		break
	}
	return fmt.Sprintf("%s %d", pname, pid)
}

func getProcessNameWithCache(pid int) string {
	now := time.Now()
	pinfo, ok := procecces[pid]
	if !ok || now.After(pinfo.expiration) {
		name := getProcessNameFromAdb(pid)
		procecces[pid] = processInfo{name: name, expiration: now.Add(cacheExpiration)}
	}
	return pinfo.name
}

// Run for each line
func processLine(line string) {
	var pid = 0
	var processName = ""

	// Find the pid from the line and get the process name
	for _, re := range pidPatterns {
		s := re.FindStringSubmatch(line)
		if s != nil {
			pid, _ = strconv.Atoi(s[1])
			// mlib.Debug("pid=%d\n", pid)
			processName = getProcessNameWithCache(pid)
			break
		}
	}

	// Any process died?
	for _, re := range diePatterns {
		s := re.FindStringSubmatch(line)
		if s != nil {
			diedPid, _ := strconv.Atoi(s[1])
			mlib.Debug("Process %d died\n", diedPid)
			delete(procecces, diedPid)
			break
		}
	}

	fmt.Printf(outFormat, processName, line)
}

func main() {
	flag.Parse()

	outFormat = fmt.Sprintf("[%%-%ds] %%s", *width)

	for line := range mlib.ReadFilesFromArgs() {
		processLine(line)
	}
}
<|endoftext|>