<commit_before>package tests

import (
	"testing"

	"github.com/polydawn/go-errcat"

	"go.polydawn.net/go-timeless-api"
	"go.polydawn.net/go-timeless-api/repeatr"
	. "go.polydawn.net/repeatr/testutil"
)

var (
	// Base formula full of sensible defaults and ready to run:
	baseFormula = api.Formula{
		Inputs: map[api.AbsPath]api.WareID{
			"/": api.WareID{"tar", "6q7G4hWr283FpTa5Lf8heVqw9t97b5VoMU6AGszuBYAz9EzQdeHVFAou7c4W9vFcQ6"},
		},
		Action: api.FormulaAction{
			Exec: []string{"/bin/echo", "hello world"},
		},
	}
	baseFormulaCtx = api.FormulaContext{
		FetchUrls: map[api.AbsPath][]api.WarehouseAddr{
			"/": []api.WarehouseAddr{
				"file://../../../fixtures/busybash.tgz",
			},
		},
	}
)

func CheckHelloWorldTxt(t *testing.T, runTool repeatr.RunFunc) {
	t.Run("hello-world formula should work", func(t *testing.T) {
		frm, frmCtx := baseFormula.Clone(), baseFormulaCtx
		rr, txt := shouldRun(t, runTool, frm, frmCtx)
		t.Run("exit code should be success", func(t *testing.T) {
			WantEqual(t, rr.ExitCode, 0)
		})
		t.Run("txt should be echo'd string", func(t *testing.T) {
			WantEqual(t, txt, "hello world\n")
		})
	})
}

func CheckRoundtripRootfs(t *testing.T, runTool repeatr.RunFunc) {
	t.Run("output unmodified root fileset should work", func(t *testing.T) {
		frm, frmCtx := baseFormula.Clone(), baseFormulaCtx
		frm.Outputs = map[api.AbsPath]api.OutputSpec{
			"/": {PackType: "tar", Filters: api.Filter_NoMutation},
		}
		rr, _ := shouldRun(t, runTool, frm, frmCtx)
		t.Run("output ware from '/' should be familiar!", func(t *testing.T) {
			WantEqual(t, map[api.AbsPath]api.WareID{
				"/": baseFormula.Inputs["/"],
			}, rr.Results)
		})
	})
}

func CheckReportingExitCodes(t *testing.T, runTool repeatr.RunFunc) {
	t.Run("non-zero exits should report cleanly", func(t *testing.T) {
		frm, frmCtx := baseFormula.Clone(), baseFormulaCtx
		frm.Action = api.FormulaAction{
			Exec: []string{"/bin/bash", "-c", "exit 14"},
		}
		rr, _ := shouldRun(t, runTool, frm, frmCtx)
		WantEqual(t, rr.ExitCode, 14)
	})
}

func CheckErrorFromUnfetchableWares(t *testing.T, runTool repeatr.RunFunc) {
	t.Run("an unfetchable input should error", func(t *testing.T) {
		frm, frmCtx := baseFormula.Clone(), baseFormulaCtx
		// Add a ware (the hash doesn't matter much), and yet no fetch URLs.
		frm.Inputs["/unfetchable"] = api.WareID{"tar", "asdfasdfasdf"}
		rr, txt, err := run(t, runTool, frm, frmCtx)
		WantEqual(t, errcat.Category(err), repeatr.ErrWarehouseUnavailable)
		WantEqual(t, rr.ExitCode, -1)
		WantEqual(t, txt, "")
	})
}

func CheckDefaultUid(t *testing.T, runTool repeatr.RunFunc) {
	t.Run("the default uid should be non-zero", func(t *testing.T) {
		frm, frmCtx := baseFormula.Clone(), baseFormulaCtx
		frm.Action = api.FormulaAction{
			Exec: []string{"/bin/bash", "-c", "echo $UID"}, // bash sets the UID env.
		}
		rr, txt := shouldRun(t, runTool, frm, frmCtx)
		WantEqual(t, rr.ExitCode, 0)
		WantEqual(t, txt, "1000\n")
	})
}
<commit_msg>executor: snipefix test with invert got/want text.<commit_after>package tests

import (
	"testing"

	"github.com/polydawn/go-errcat"

	"go.polydawn.net/go-timeless-api"
	"go.polydawn.net/go-timeless-api/repeatr"
	. "go.polydawn.net/repeatr/testutil"
)

var (
	// Base formula full of sensible defaults and ready to run:
	baseFormula = api.Formula{
		Inputs: map[api.AbsPath]api.WareID{
			"/": api.WareID{"tar", "6q7G4hWr283FpTa5Lf8heVqw9t97b5VoMU6AGszuBYAz9EzQdeHVFAou7c4W9vFcQ6"},
		},
		Action: api.FormulaAction{
			Exec: []string{"/bin/echo", "hello world"},
		},
	}
	baseFormulaCtx = api.FormulaContext{
		FetchUrls: map[api.AbsPath][]api.WarehouseAddr{
			"/": []api.WarehouseAddr{
				"file://../../../fixtures/busybash.tgz",
			},
		},
	}
)

func CheckHelloWorldTxt(t *testing.T, runTool repeatr.RunFunc) {
	t.Run("hello-world formula should work", func(t *testing.T) {
		frm, frmCtx := baseFormula.Clone(), baseFormulaCtx
		rr, txt := shouldRun(t, runTool, frm, frmCtx)
		t.Run("exit code should be success", func(t *testing.T) {
			WantEqual(t, rr.ExitCode, 0)
		})
		t.Run("txt should be echo'd string", func(t *testing.T) {
			WantEqual(t, txt, "hello world\n")
		})
	})
}

func CheckRoundtripRootfs(t *testing.T, runTool repeatr.RunFunc) {
	t.Run("output unmodified root fileset should work", func(t *testing.T) {
		frm, frmCtx := baseFormula.Clone(), baseFormulaCtx
		frm.Outputs = map[api.AbsPath]api.OutputSpec{
			"/": {PackType: "tar", Filters: api.Filter_NoMutation},
		}
		rr, _ := shouldRun(t, runTool, frm, frmCtx)
		t.Run("output ware from '/' should be familiar!", func(t *testing.T) {
			WantEqual(t, rr.Results,
				map[api.AbsPath]api.WareID{
					"/": baseFormula.Inputs["/"],
				})
		})
	})
}

func CheckReportingExitCodes(t *testing.T, runTool repeatr.RunFunc) {
	t.Run("non-zero exits should report cleanly", func(t *testing.T) {
		frm, frmCtx := baseFormula.Clone(), baseFormulaCtx
		frm.Action = api.FormulaAction{
			Exec: []string{"/bin/bash", "-c", "exit 14"},
		}
		rr, _ := shouldRun(t, runTool, frm, frmCtx)
		WantEqual(t, rr.ExitCode, 14)
	})
}

func CheckErrorFromUnfetchableWares(t *testing.T, runTool repeatr.RunFunc) {
	t.Run("an unfetchable input should error", func(t *testing.T) {
		frm, frmCtx := baseFormula.Clone(), baseFormulaCtx
		// Add a ware (the hash doesn't matter much), and yet no fetch URLs.
		frm.Inputs["/unfetchable"] = api.WareID{"tar", "asdfasdfasdf"}
		rr, txt, err := run(t, runTool, frm, frmCtx)
		WantEqual(t, errcat.Category(err), repeatr.ErrWarehouseUnavailable)
		WantEqual(t, rr.ExitCode, -1)
		WantEqual(t, txt, "")
	})
}

func CheckDefaultUid(t *testing.T, runTool repeatr.RunFunc) {
	t.Run("the default uid should be non-zero", func(t *testing.T) {
		frm, frmCtx := baseFormula.Clone(), baseFormulaCtx
		frm.Action = api.FormulaAction{
			Exec: []string{"/bin/bash", "-c", "echo $UID"}, // bash sets the UID env.
		}
		rr, txt := shouldRun(t, runTool, frm, frmCtx)
		WantEqual(t, rr.ExitCode, 0)
		WantEqual(t, txt, "1000\n")
	})
}
<|endoftext|>


<commit_before>// The command package provides the implementation the command protocol for the sphere-client process.
// The implementation delegates execution to a helper script. The helper script is responsible for
// validating that the command is a supported command and for generating a json output.
package command

import (
	"encoding/json"
	"fmt"
	"github.com/ninjasphere/go-ninja/api"
	"os"
)

type Configuration struct {
	Conn       *ninja.Connection
	Device     ninja.Device
	HelperPath string
}

// Format of a request to execute a command
type Request struct {
	Command string   `json:"command,omitempty"`
	Args    []string `json:"args,omitempty"`
}

// Format of a response from a command.
type Response struct {
	Data   interface{} `json:"data,omitempty"`   // present if the command generates json structured output
	Output *[]string   `json:"output,omitempty"` // present if the command does not generate json structured output
	Error  *[]string   `json:"error,omitempty"`  // any output written to the commands stderr
	Status *int        `json:"status,omitempty"` // the exit status of the command.
}

// The command service.
type CommandChannel struct {
	config Configuration
	sender func(event string, payload ...interface{}) error
}

// Exports the command service into the bus
func ExportCommandChannel(config *Configuration) error {
	if config == nil {
		return fmt.Errorf("illegal argument: config == nil")
	}

	channel := &CommandChannel{
		config: *config,
	}
	return config.Conn.ExportChannel(config.Device, channel, "command")
}

func (cs *CommandChannel) GetProtocol() string {
	return "http://schema.ninjablocks.com/protocol/command"
}

func (cs *CommandChannel) SetEventHandler(sender func(event string, payload ...interface{}) error) {
	cs.sender = sender
}

// Support for the exec method.
func (cs *CommandChannel) Exec(request *Request) (*Response, error) {
	if fromChild, toParent, err := os.Pipe(); err != nil {
		return nil, err
	} else if fromParent, toChild, err := os.Pipe(); err != nil {
		return nil, err
	} else {
		procAttr := &os.ProcAttr{
			Files: []*os.File{
				fromParent,
				toParent,
				os.Stderr,
			},
		}
		args := make([]string, len(request.Args)+3)
		args[0] = cs.config.HelperPath
		args[1] = "exec"
		args[2] = request.Command
		copy(args[3:], request.Args)
		if _, err := os.StartProcess(cs.config.HelperPath, args, procAttr); err != nil {
			return nil, err
		}

		fromParent.Close()
		toParent.Close()

		toChild.Close()

		response := &Response{}
		err := json.NewDecoder(fromChild).Decode(response)
		return response, err
	}

}
<commit_msg>Add async support.<commit_after>// The command package provides the implementation the command protocol for the sphere-client process.
// The implementation delegates execution to a helper script. The helper script is responsible for
// validating that the command is a supported command and for generating a json output.
package command

import (
	"encoding/json"
	"fmt"
	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/logger"
	"os"
)

type Configuration struct {
	Conn       *ninja.Connection
	Device     ninja.Device
	HelperPath string
}

// Format of a request to execute a command
type Request struct {
	Command string   `json:"command,omitempty"`
	Args    []string `json:"args,omitempty"`
	Async   bool     `json:"async,omitempty"`
}

// Format of a response from a command.
type Response struct {
	Data   interface{} `json:"data,omitempty"`   // present if the command generates json structured output
	Output *[]string   `json:"output,omitempty"` // present if the command does not generate json structured output
	Error  *[]string   `json:"error,omitempty"`  // any output written to the commands stderr
	Status *int        `json:"status,omitempty"` // the exit status of the command.
	Pid    *int        `json:"pid,omitempty"`    // the PID of the started or completed command.
}

// The command service.
type CommandChannel struct {
	config Configuration
	sender func(event string, payload ...interface{}) error
}

var log = logger.GetLogger("sphere-client.command")

// Exports the command service into the bus
func ExportCommandChannel(config *Configuration) error {
	if config == nil {
		return fmt.Errorf("illegal argument: config == nil")
	}

	channel := &CommandChannel{
		config: *config,
	}
	return config.Conn.ExportChannel(config.Device, channel, "command")
}

func (cs *CommandChannel) GetProtocol() string {
	return "http://schema.ninjablocks.com/protocol/command"
}

func (cs *CommandChannel) SetEventHandler(sender func(event string, payload ...interface{}) error) {
	cs.sender = sender
}

// Support for the exec method.
func (cs *CommandChannel) Exec(request *Request) (*Response, error) {
	if fromChild, toParent, err := os.Pipe(); err != nil {
		return nil, err
	} else if fromParent, toChild, err := os.Pipe(); err != nil {
		return nil, err
	} else {
		procAttr := &os.ProcAttr{
			Files: []*os.File{
				fromParent,
				toParent,
				os.Stderr,
			},
		}
		args := make([]string, len(request.Args)+3)
		args[0] = cs.config.HelperPath
		args[1] = "exec"
		args[2] = request.Command
		copy(args[3:], request.Args)
		if proc, err := os.StartProcess(cs.config.HelperPath, args, procAttr); err != nil {
			return nil, err
		} else {

			fromParent.Close()
			toParent.Close()

			toChild.Close()

			response := &Response{
				Pid: &proc.Pid,
			}

			executor := func(response *Response) error {

				err := json.NewDecoder(fromChild).Decode(response)
				if err != nil {
					log.Errorf("failed to parse response from command: %v", err)
				}

				cs.sender("completed", response)
				fromChild.Close()
				return err
			}

			if request.Async {
				asyncResponse := &Response{
					Pid: &proc.Pid,
				}
				go executor(asyncResponse)
				return response, nil
			}

			return response, executor(response)
		}
	}

}
<|endoftext|>


<commit_before>package command

import (
	"bytes"
	"compress/gzip"
	"expvar"
	"fmt"
	"io/ioutil"

	"github.com/golang/protobuf/proto"
)

const (
	defaultBatchThreshold = 5
	defaultSizeThreshold  = 150
)

// Requester is the interface objects must support to be marshaled
// successfully.
type Requester interface {
	proto.Message
	GetRequest() *Request
}

// RequestMarshaler marshals Request objects, potentially performing
// gzip compression.
type RequestMarshaler struct {
	BatchThreshold   int
	SizeThreshold    int
	ForceCompression bool
}

const (
	numRequests             = "num_requests"
	numCompressedRequests   = "num_compressed_requests"
	numUncompressedRequests = "num_uncompressed_requests"
	numCompressedBytes      = "num_compressed_bytes"
	numPrecompressedBytes   = "num_precompressed_bytes"
	numUncompressedBytes    = "num_uncompressed_bytes"
	numCompressionMisses    = "num_compression_misses"
)

// stats captures stats for the Proto marshaler.
var stats *expvar.Map

func init() {
	stats = expvar.NewMap("proto")
	stats.Add(numRequests, 0)
	stats.Add(numCompressedRequests, 0)
	stats.Add(numUncompressedRequests, 0)
	stats.Add(numCompressedBytes, 0)
	stats.Add(numUncompressedBytes, 0)
	stats.Add(numCompressionMisses, 0)
	stats.Add(numPrecompressedBytes, 0)
}

// NewRequestMarshaler returns an initialized RequestMarshaler.
func NewRequestMarshaler() *RequestMarshaler {
	return &RequestMarshaler{
		BatchThreshold: defaultBatchThreshold,
		SizeThreshold:  defaultSizeThreshold,
	}
}

// Marshal marshals a Requester object, returning a byte slice, a bool
// indicating whether the contents are compressed, or an error.
func (m *RequestMarshaler) Marshal(r Requester) ([]byte, bool, error) {
	stats.Add(numRequests, 0)
	compress := false

	gzw, err := gzip.NewWriterLevel(nil, gzip.BestCompression)
	if err != nil {
		panic(fmt.Sprintf("failed to create GZIP writer: %s", err.Error()))
	}

	stmts := r.GetRequest().GetStatements()
	if len(stmts) >= m.BatchThreshold {
		compress = true
	} else {
		for i := range stmts {
			if len(stmts[i].Sql) >= m.SizeThreshold {
				compress = true
				break
			}
		}
	}

	b, err := proto.Marshal(r)
	if err != nil {
		return nil, false, err
	}
	ubz := len(b)
	stats.Add(numPrecompressedBytes, int64(ubz))

	if compress {
		// Let's try compression.
		var buf bytes.Buffer
		gzw.Reset(&buf)
		if _, err := gzw.Write(b); err != nil {
			return nil, false, fmt.Errorf("gzip Write: %s", err)
		}
		if err := gzw.Close(); err != nil {
			return nil, false, fmt.Errorf("gzip Close: %s", err)
		}

		// Is compression better?
		if ubz > len(buf.Bytes()) || m.ForceCompression {
			// Yes! Let's keep it.
			b = buf.Bytes()
			stats.Add(numCompressedRequests, 1)
			stats.Add(numCompressedBytes, int64(len(b)))
		} else {
			// No. :-( Dump it.
			compress = false
			stats.Add(numCompressionMisses, 1)
		}
	} else {
		stats.Add(numUncompressedRequests, 1)
		stats.Add(numUncompressedBytes, int64(len(b)))
	}

	return b, compress, nil
}

// Stats returns status and diagnostic information about
// the RequestMarshaler.
func (m *RequestMarshaler) Stats() map[string]interface{} {
	return map[string]interface{}{
		"compression_size":  m.SizeThreshold,
		"compression_batch": m.BatchThreshold,
		"force_compression": m.ForceCompression,
	}
}

// Marshal marshals a Command.
func Marshal(c *Command) ([]byte, error) {
	return proto.Marshal(c)
}

// Unmarshal unmarshals a Command
func Unmarshal(b []byte, c *Command) error {
	return proto.Unmarshal(b, c)
}

// MarshalMetadataSet marshals a MetadataSet command
func MarshalMetadataSet(c *MetadataSet) ([]byte, error) {
	return proto.Marshal(c)
}

// UnMarshalMetadataSet unmarshals a MetadataSet command
func UnMarshalMetadataSet(b []byte, c *MetadataSet) error {
	return proto.Unmarshal(b, c)
}

// MarshalMetadataDelete marshals a MetadataDelete command
func MarshalMetadataDelete(c *MetadataDelete) ([]byte, error) {
	return proto.Marshal(c)
}

// UnMarshalMetadataDelete unmarshals a MetadataDelete command
func UnMarshalMetadataDelete(b []byte, c *MetadataDelete) error {
	return proto.Unmarshal(b, c)
}

// MarshalNoop marshals a Noop command
func MarshalNoop(c *Noop) ([]byte, error) {
	return proto.Marshal(c)
}

// UnmarshalNoop unmarshals a Noop command
func UnmarshalNoop(b []byte, c *Noop) error {
	return proto.Unmarshal(b, c)
}

// UnmarshalSubCommand unmarshalls a sub command m. It assumes that
// m is the correct type.
func UnmarshalSubCommand(c *Command, m proto.Message) error {
	b := c.SubCommand
	if c.Compressed {
		gz, err := gzip.NewReader(bytes.NewReader(b))
		if err != nil {
			fmt.Errorf("unmarshal sub gzip NewReader: %s", err)
		}

		ub, err := ioutil.ReadAll(gz)
		if err != nil {
			fmt.Errorf("unmarshal sub gzip ReadAll: %s", err)
		}

		if err := gz.Close(); err != nil {
			fmt.Errorf("unmarshal sub gzip Close: %s", err)
		}
		b = ub
	}

	if err := proto.Unmarshal(b, m); err != nil {
		return fmt.Errorf("proto unmarshal: %s", err)
	}
	return nil
}
<commit_msg>Simpler creation of GZIP writer<commit_after>package command

import (
	"bytes"
	"compress/gzip"
	"expvar"
	"fmt"
	"io/ioutil"

	"github.com/golang/protobuf/proto"
)

const (
	defaultBatchThreshold = 5
	defaultSizeThreshold  = 150
)

// Requester is the interface objects must support to be marshaled
// successfully.
type Requester interface {
	proto.Message
	GetRequest() *Request
}

// RequestMarshaler marshals Request objects, potentially performing
// gzip compression.
type RequestMarshaler struct {
	BatchThreshold   int
	SizeThreshold    int
	ForceCompression bool
}

const (
	numRequests             = "num_requests"
	numCompressedRequests   = "num_compressed_requests"
	numUncompressedRequests = "num_uncompressed_requests"
	numCompressedBytes      = "num_compressed_bytes"
	numPrecompressedBytes   = "num_precompressed_bytes"
	numUncompressedBytes    = "num_uncompressed_bytes"
	numCompressionMisses    = "num_compression_misses"
)

// stats captures stats for the Proto marshaler.
var stats *expvar.Map

func init() {
	stats = expvar.NewMap("proto")
	stats.Add(numRequests, 0)
	stats.Add(numCompressedRequests, 0)
	stats.Add(numUncompressedRequests, 0)
	stats.Add(numCompressedBytes, 0)
	stats.Add(numUncompressedBytes, 0)
	stats.Add(numCompressionMisses, 0)
	stats.Add(numPrecompressedBytes, 0)
}

// NewRequestMarshaler returns an initialized RequestMarshaler.
func NewRequestMarshaler() *RequestMarshaler {
	return &RequestMarshaler{
		BatchThreshold: defaultBatchThreshold,
		SizeThreshold:  defaultSizeThreshold,
	}
}

// Marshal marshals a Requester object, returning a byte slice, a bool
// indicating whether the contents are compressed, or an error.
func (m *RequestMarshaler) Marshal(r Requester) ([]byte, bool, error) {
	stats.Add(numRequests, 0)
	compress := false

	stmts := r.GetRequest().GetStatements()
	if len(stmts) >= m.BatchThreshold {
		compress = true
	} else {
		for i := range stmts {
			if len(stmts[i].Sql) >= m.SizeThreshold {
				compress = true
				break
			}
		}
	}

	b, err := proto.Marshal(r)
	if err != nil {
		return nil, false, err
	}
	ubz := len(b)
	stats.Add(numPrecompressedBytes, int64(ubz))

	if compress {
		// Let's try compression.
		var buf bytes.Buffer
		gzw, err := gzip.NewWriterLevel(&buf, gzip.BestCompression)
		if err != nil {
			return nil, false, fmt.Errorf("gzip new writer: %s", err)
		}

		if _, err := gzw.Write(b); err != nil {
			return nil, false, fmt.Errorf("gzip Write: %s", err)
		}
		if err := gzw.Close(); err != nil {
			return nil, false, fmt.Errorf("gzip Close: %s", err)
		}

		// Is compression better?
		if ubz > len(buf.Bytes()) || m.ForceCompression {
			// Yes! Let's keep it.
			b = buf.Bytes()
			stats.Add(numCompressedRequests, 1)
			stats.Add(numCompressedBytes, int64(len(b)))
		} else {
			// No. :-( Dump it.
			compress = false
			stats.Add(numCompressionMisses, 1)
		}
	} else {
		stats.Add(numUncompressedRequests, 1)
		stats.Add(numUncompressedBytes, int64(len(b)))
	}

	return b, compress, nil
}

// Stats returns status and diagnostic information about
// the RequestMarshaler.
func (m *RequestMarshaler) Stats() map[string]interface{} {
	return map[string]interface{}{
		"compression_size":  m.SizeThreshold,
		"compression_batch": m.BatchThreshold,
		"force_compression": m.ForceCompression,
	}
}

// Marshal marshals a Command.
func Marshal(c *Command) ([]byte, error) {
	return proto.Marshal(c)
}

// Unmarshal unmarshals a Command
func Unmarshal(b []byte, c *Command) error {
	return proto.Unmarshal(b, c)
}

// MarshalMetadataSet marshals a MetadataSet command
func MarshalMetadataSet(c *MetadataSet) ([]byte, error) {
	return proto.Marshal(c)
}

// UnMarshalMetadataSet unmarshals a MetadataSet command
func UnMarshalMetadataSet(b []byte, c *MetadataSet) error {
	return proto.Unmarshal(b, c)
}

// MarshalMetadataDelete marshals a MetadataDelete command
func MarshalMetadataDelete(c *MetadataDelete) ([]byte, error) {
	return proto.Marshal(c)
}

// UnMarshalMetadataDelete unmarshals a MetadataDelete command
func UnMarshalMetadataDelete(b []byte, c *MetadataDelete) error {
	return proto.Unmarshal(b, c)
}

// MarshalNoop marshals a Noop command
func MarshalNoop(c *Noop) ([]byte, error) {
	return proto.Marshal(c)
}

// UnmarshalNoop unmarshals a Noop command
func UnmarshalNoop(b []byte, c *Noop) error {
	return proto.Unmarshal(b, c)
}

// UnmarshalSubCommand unmarshalls a sub command m. It assumes that
// m is the correct type.
func UnmarshalSubCommand(c *Command, m proto.Message) error {
	b := c.SubCommand
	if c.Compressed {
		gz, err := gzip.NewReader(bytes.NewReader(b))
		if err != nil {
			fmt.Errorf("unmarshal sub gzip NewReader: %s", err)
		}

		ub, err := ioutil.ReadAll(gz)
		if err != nil {
			fmt.Errorf("unmarshal sub gzip ReadAll: %s", err)
		}

		if err := gz.Close(); err != nil {
			fmt.Errorf("unmarshal sub gzip Close: %s", err)
		}
		b = ub
	}

	if err := proto.Unmarshal(b, m); err != nil {
		return fmt.Errorf("proto unmarshal: %s", err)
	}
	return nil
}
<|endoftext|>


<commit_before>package es

import (
	"fmt"
)

type Facet struct {
	Type           string      `json:"_type,omitempty"`
	Missing        int         `json:"missing,omitempty"`
	Total          int         `json:"total,omitempty"`
	Other          int         `json:"other,omitempty"`
	Terms          *FacetTerms `json:"terms,omitempty"`
	*Entries       `json:"entries,omitempty"`
	*DateHistogram `json:"date_histogram,omitempty"`
	Statistical    *StatisticalFacet
}

type FacetTerms struct {
	Field   string        `json:"field,omitempty"`
	Size    int           `json:"size,omitempty"`
	Exclude []interface{} `json:"exclude,omitempty"`
}

type StatisticalFacet struct {
	Type         string  `json:"_type,omitempty"`
	Count        int64   `json:"count,omitempty"`
	Total        int64   `json:"total,omitempty"`
	Min          float64 `json:"min,omitempty"`
	Max          float64 `json:"max,omitempty"`
	Mean         float64 `json:"mean,omitempty"`
	SumOfSquares float64 `json:"sum_of_squares,omitempty"`
	Variance     float64 `json:"variance,omitempty"`
	StdDeviation float64 `json:"std_deviation,omitempty"`
}

func interfaceToInt64(i interface{}) int64 {
	switch v := i.(type) {
	case int:
		return int64(v)
	case int32:
		return int64(v)
	case int64:
		return v
	default:
		panic(fmt.Sprintf("unable to convert %T %v to float64", v, v))
	}
}

func interfaceToFloat64(i interface{}) float64 {
	switch v := i.(type) {
	case float64:
		return v
	case int:
		return float64(v)
	case int32:
		return float64(v)
	case int64:
		return float64(v)
	default:
		panic(fmt.Sprintf("unable to convert %T %v to float64", v, v))
	}
}

func (sf *StatisticalFacet) Load(m map[string]interface{}) error {
	sf.Type, _ = m["_type"].(string)
	sf.Count = interfaceToInt64(m["count"])
	sf.Total = interfaceToInt64(m["total"])
	sf.Min = interfaceToFloat64(m["min"])
	sf.Max = interfaceToFloat64(m["max"])
	sf.Mean = interfaceToFloat64(m["mean"])
	sf.SumOfSquares = interfaceToFloat64(m["sum_of_squares"])
	sf.Variance = interfaceToFloat64(m["variance"])
	sf.StdDeviation = interfaceToFloat64(m["std_deviation"])
	return nil
}

type TermsFacet struct {
	Type    string             `json:"_type,omitempty"`
	Missing int64              `json:"missing,omitempty"`
	Total   int64              `json:"total,omitempty"`
	Other   int64              `json:"other,omitempty"`
	Terms   FacetResponseTerms `json:"terms,omitempty"`
}

type FacetResponseTerms []*FacetResponseTerm

type FacetResponseTerm struct {
	Term  interface{}
	Count int64
}

func (tf *TermsFacet) Load(m map[string]interface{}) error {
	tf.Type, _ = m["_type"].(string)
	tf.Missing = interfaceToInt64(m["missing"])
	tf.Total = interfaceToInt64(m["total"])
	tf.Other = interfaceToInt64(m["other"])
	terms, ok := m["terms"].([]map[string]interface{})
	if ok {
		tf.Terms = make(FacetResponseTerms, 0, len(terms))
		for _, term := range terms {
			t := &FacetResponseTerm{
				Count: interfaceToInt64(term["count"]),
				Term:  term["term"],
			}
			tf.Terms = append(tf.Terms, t)
		}
	}
	return nil
}

type DateHistogramFacet struct {
	Type    string                     `json:"term,omitempty"`
	Entries []*DateHistogramFacetEntry `json:"entries,omitempty"`
}

type DateHistogramFacetEntry struct {
	Time       int64 `json:"time,omitempty"`
	Count      int64 `json:"time,omitempty"`
	Min        float64
	Max        float64
	Total      int64
	TotalCount int64
	Mean       float64
}

func (dhf *DateHistogramFacet) Load(m map[string]interface{}) error {
	dhf.Type, _ = m["_type"].(string)
	entries, ok := m["entries"].([]map[string]interface{})
	if ok {
		dhf.Entries = make([]*DateHistogramFacetEntry, 0, len(entries))
		for _, e := range entries {
			entry := &DateHistogramFacetEntry{
				Time:       interfaceToInt64(e["time"]),
				Count:      interfaceToInt64(e["count"]),
				Min:        interfaceToFloat64(e["min"]),
				Max:        interfaceToFloat64(e["max"]),
				Total:      interfaceToInt64(e["total"]),
				TotalCount: interfaceToInt64(e["total_count"]),
				Mean:       interfaceToFloat64(e["mean"]),
			}
			dhf.Entries = append(dhf.Entries, entry)
		}
	}
	return nil
}
<commit_msg>omit empty statistical facets<commit_after>package es

import (
	"fmt"
)

type Facet struct {
	Type           string      `json:"_type,omitempty"`
	Missing        int         `json:"missing,omitempty"`
	Total          int         `json:"total,omitempty"`
	Other          int         `json:"other,omitempty"`
	Terms          *FacetTerms `json:"terms,omitempty"`
	*Entries       `json:"entries,omitempty"`
	*DateHistogram `json:"date_histogram,omitempty"`
	Statistical    *StatisticalFacet `json:"statistical,omitempty"`
}

type FacetTerms struct {
	Field   string        `json:"field,omitempty"`
	Size    int           `json:"size,omitempty"`
	Exclude []interface{} `json:"exclude,omitempty"`
}

type StatisticalFacet struct {
	Type         string  `json:"_type,omitempty"`
	Count        int64   `json:"count,omitempty"`
	Total        int64   `json:"total,omitempty"`
	Min          float64 `json:"min,omitempty"`
	Max          float64 `json:"max,omitempty"`
	Mean         float64 `json:"mean,omitempty"`
	SumOfSquares float64 `json:"sum_of_squares,omitempty"`
	Variance     float64 `json:"variance,omitempty"`
	StdDeviation float64 `json:"std_deviation,omitempty"`
}

func interfaceToInt64(i interface{}) int64 {
	switch v := i.(type) {
	case int:
		return int64(v)
	case int32:
		return int64(v)
	case int64:
		return v
	default:
		panic(fmt.Sprintf("unable to convert %T %v to float64", v, v))
	}
}

func interfaceToFloat64(i interface{}) float64 {
	switch v := i.(type) {
	case float64:
		return v
	case int:
		return float64(v)
	case int32:
		return float64(v)
	case int64:
		return float64(v)
	default:
		panic(fmt.Sprintf("unable to convert %T %v to float64", v, v))
	}
}

func (sf *StatisticalFacet) Load(m map[string]interface{}) error {
	sf.Type, _ = m["_type"].(string)
	sf.Count = interfaceToInt64(m["count"])
	sf.Total = interfaceToInt64(m["total"])
	sf.Min = interfaceToFloat64(m["min"])
	sf.Max = interfaceToFloat64(m["max"])
	sf.Mean = interfaceToFloat64(m["mean"])
	sf.SumOfSquares = interfaceToFloat64(m["sum_of_squares"])
	sf.Variance = interfaceToFloat64(m["variance"])
	sf.StdDeviation = interfaceToFloat64(m["std_deviation"])
	return nil
}

type TermsFacet struct {
	Type    string             `json:"_type,omitempty"`
	Missing int64              `json:"missing,omitempty"`
	Total   int64              `json:"total,omitempty"`
	Other   int64              `json:"other,omitempty"`
	Terms   FacetResponseTerms `json:"terms,omitempty"`
}

type FacetResponseTerms []*FacetResponseTerm

type FacetResponseTerm struct {
	Term  interface{}
	Count int64
}

func (tf *TermsFacet) Load(m map[string]interface{}) error {
	tf.Type, _ = m["_type"].(string)
	tf.Missing = interfaceToInt64(m["missing"])
	tf.Total = interfaceToInt64(m["total"])
	tf.Other = interfaceToInt64(m["other"])
	terms, ok := m["terms"].([]map[string]interface{})
	if ok {
		tf.Terms = make(FacetResponseTerms, 0, len(terms))
		for _, term := range terms {
			t := &FacetResponseTerm{
				Count: interfaceToInt64(term["count"]),
				Term:  term["term"],
			}
			tf.Terms = append(tf.Terms, t)
		}
	}
	return nil
}

type DateHistogramFacet struct {
	Type    string                     `json:"term,omitempty"`
	Entries []*DateHistogramFacetEntry `json:"entries,omitempty"`
}

type DateHistogramFacetEntry struct {
	Time       int64 `json:"time,omitempty"`
	Count      int64 `json:"time,omitempty"`
	Min        float64
	Max        float64
	Total      int64
	TotalCount int64
	Mean       float64
}

func (dhf *DateHistogramFacet) Load(m map[string]interface{}) error {
	dhf.Type, _ = m["_type"].(string)
	entries, ok := m["entries"].([]map[string]interface{})
	if ok {
		dhf.Entries = make([]*DateHistogramFacetEntry, 0, len(entries))
		for _, e := range entries {
			entry := &DateHistogramFacetEntry{
				Time:       interfaceToInt64(e["time"]),
				Count:      interfaceToInt64(e["count"]),
				Min:        interfaceToFloat64(e["min"]),
				Max:        interfaceToFloat64(e["max"]),
				Total:      interfaceToInt64(e["total"]),
				TotalCount: interfaceToInt64(e["total_count"]),
				Mean:       interfaceToFloat64(e["mean"]),
			}
			dhf.Entries = append(dhf.Entries, entry)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Package ipfix decodes IPFIX packets
//: ----------------------------------------------------------------------------
//: Copyright (C) 2017 Verizon.  All Rights Reserved.
//: All Rights Reserved
//:
//: file:    marshal.go
//: details: TODO
//: author:  Mehrdad Arshad Rad
//: date:    02/01/2017
//:
//: Licensed under the Apache License, Version 2.0 (the "License");
//: you may not use this file except in compliance with the License.
//: You may obtain a copy of the License at
//:
//:     http://www.apache.org/licenses/LICENSE-2.0
//:
//: Unless required by applicable law or agreed to in writing, software
//: distributed under the License is distributed on an "AS IS" BASIS,
//: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//: See the License for the specific language governing permissions and
//: limitations under the License.
//: ----------------------------------------------------------------------------
package ipfix

import (
	"bytes"
	"errors"
	"fmt"
	"net"
	"strconv"
)

var errUknownMarshalDataType = errors.New("unknown data type to marshal")

// JSONMarshal encodes IPFIX message
func (m *Message) JSONMarshal(b *bytes.Buffer) ([]byte, error) {
	b.WriteString("{")

	// encode agent id
	m.encodeAgent(b)

	// encode header
	m.encodeHeader(b)

	// encode data sets
	if err := m.encodeDataSet(b); err != nil {
		return nil, err
	}

	b.WriteString("}")

	return b.Bytes(), nil
}

func (m *Message) encodeDataSet(b *bytes.Buffer) error {
	var length, dsLength int

	b.WriteString("\"DataSets\":")
	dsLength = len(m.DataSets)

	b.WriteString("[")
	for i := range m.DataSets {
		length = len(m.DataSets[i])

		b.WriteString("[")
		for j := range m.DataSets[i] {
			b.WriteString("{\"ID\":")
			b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].ID), 10))
			b.WriteString(",\"Value\":")

			switch m.DataSets[i][j].Value.(type) {
			case uint:
				b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].Value.(uint)), 10))
			case uint8:
				b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].Value.(uint8)), 10))
			case uint16:
				b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].Value.(uint16)), 10))
			case uint32:
				b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].Value.(uint32)), 10))
			case uint64:
				b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].Value.(uint64)), 10))
			case int:
				b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].Value.(int)), 10))
			case int8:
				b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].Value.(int8)), 10))
			case int16:
				b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].Value.(int16)), 10))
			case int32:
				b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].Value.(int32)), 10))
			case int64:
				b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].Value.(int64)), 10))
			case float32:
				b.WriteString(strconv.FormatFloat(float64(m.DataSets[i][j].Value.(float32)), 'E', -1, 32))
			case float64:
				b.WriteString(strconv.FormatFloat(m.DataSets[i][j].Value.(float64), 'E', -1, 64))
			case string:
				b.WriteString("\"")
				b.WriteString(m.DataSets[i][j].Value.(string))
				b.WriteString("\"")
			case net.IP:
				b.WriteString("\"")
				b.WriteString(m.DataSets[i][j].Value.(net.IP).String())
				b.WriteString("\"")
			case net.HardwareAddr:
				b.WriteString("\"")
				b.WriteString(m.DataSets[i][j].Value.(net.HardwareAddr).String())
				b.WriteString("\"")
			case []uint8:
				b.WriteString("\"")
				b.WriteString(fmt.Sprintf("0x%x", m.DataSets[i][j].Value.([]uint8)))
				b.WriteString("\"")
			default:
				return errUknownMarshalDataType
			}
			if j < length-1 {
				b.WriteString("},")
			} else {
				b.WriteString("}")
			}
		}
		if i < dsLength-1 {
			b.WriteString("],")
		} else {
			b.WriteString("]")
		}
	}
	b.WriteString("]")

	return nil
}

func (m *Message) encodeHeader(b *bytes.Buffer) {
	b.WriteString("\"Header\":{\"Version\":")
	b.WriteString(strconv.FormatInt(int64(m.Header.Version), 10))
	b.WriteString(",\"Length\":")
	b.WriteString(strconv.FormatInt(int64(m.Header.Length), 10))
	b.WriteString(",\"ExportTime\":")
	b.WriteString(strconv.FormatInt(int64(m.Header.ExportTime), 10))
	b.WriteString(",\"SequenceNo\":")
	b.WriteString(strconv.FormatInt(int64(m.Header.SequenceNo), 10))
	b.WriteString(",\"DomainID\":")
	b.WriteString(strconv.FormatInt(int64(m.Header.DomainID), 10))
	b.WriteString("},")
}

func (m *Message) encodeAgent(b *bytes.Buffer) {
	b.WriteString("\"AgentID\":\"")
	b.WriteString(m.AgentID)
	b.WriteString("\",")
}
<commit_msg>optimize<commit_after>// Package ipfix decodes IPFIX packets
//: ----------------------------------------------------------------------------
//: Copyright (C) 2017 Verizon.  All Rights Reserved.
//: All Rights Reserved
//:
//: file:    marshal.go
//: details: TODO
//: author:  Mehrdad Arshad Rad
//: date:    02/01/2017
//:
//: Licensed under the Apache License, Version 2.0 (the "License");
//: you may not use this file except in compliance with the License.
//: You may obtain a copy of the License at
//:
//:     http://www.apache.org/licenses/LICENSE-2.0
//:
//: Unless required by applicable law or agreed to in writing, software
//: distributed under the License is distributed on an "AS IS" BASIS,
//: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//: See the License for the specific language governing permissions and
//: limitations under the License.
//: ----------------------------------------------------------------------------
package ipfix

import (
	"bytes"
	"encoding/hex"
	"errors"
	"net"
	"strconv"
)

var errUknownMarshalDataType = errors.New("unknown data type to marshal")

// JSONMarshal encodes IPFIX message
func (m *Message) JSONMarshal(b *bytes.Buffer) ([]byte, error) {
	b.WriteString("{")

	// encode agent id
	m.encodeAgent(b)

	// encode header
	m.encodeHeader(b)

	// encode data sets
	if err := m.encodeDataSet(b); err != nil {
		return nil, err
	}

	b.WriteString("}")

	return b.Bytes(), nil
}

func (m *Message) encodeDataSet(b *bytes.Buffer) error {
	var length, dsLength int

	b.WriteString("\"DataSets\":")
	dsLength = len(m.DataSets)

	b.WriteByte('[')
	for i := range m.DataSets {
		length = len(m.DataSets[i])

		b.WriteByte('[')
		for j := range m.DataSets[i] {
			b.WriteString("{\"ID\":")
			b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].ID), 10))
			b.WriteString(",\"Value\":")

			switch m.DataSets[i][j].Value.(type) {
			case uint:
				b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].Value.(uint)), 10))
			case uint8:
				b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].Value.(uint8)), 10))
			case uint16:
				b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].Value.(uint16)), 10))
			case uint32:
				b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].Value.(uint32)), 10))
			case uint64:
				b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].Value.(uint64)), 10))
			case int:
				b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].Value.(int)), 10))
			case int8:
				b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].Value.(int8)), 10))
			case int16:
				b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].Value.(int16)), 10))
			case int32:
				b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].Value.(int32)), 10))
			case int64:
				b.WriteString(strconv.FormatInt(int64(m.DataSets[i][j].Value.(int64)), 10))
			case float32:
				b.WriteString(strconv.FormatFloat(float64(m.DataSets[i][j].Value.(float32)), 'E', -1, 32))
			case float64:
				b.WriteString(strconv.FormatFloat(m.DataSets[i][j].Value.(float64), 'E', -1, 64))
			case string:
				b.WriteByte('"')
				b.WriteString(m.DataSets[i][j].Value.(string))
				b.WriteByte('"')
			case net.IP:
				b.WriteByte('"')
				b.WriteString(m.DataSets[i][j].Value.(net.IP).String())
				b.WriteByte('"')
			case net.HardwareAddr:
				b.WriteByte('"')
				b.WriteString(m.DataSets[i][j].Value.(net.HardwareAddr).String())
				b.WriteByte('"')
			case []uint8:
				b.WriteByte('"')
				b.WriteString("0x" + hex.EncodeToString(m.DataSets[i][j].Value.([]uint8)))
				b.WriteByte('"')
			default:
				return errUknownMarshalDataType
			}
			if j < length-1 {
				b.WriteString("},")
			} else {
				b.WriteByte('}')
			}
		}
		if i < dsLength-1 {
			b.WriteString("],")
		} else {
			b.WriteByte(']')
		}
	}
	b.WriteByte(']')

	return nil
}

func (m *Message) encodeHeader(b *bytes.Buffer) {
	b.WriteString("\"Header\":{\"Version\":")
	b.WriteString(strconv.FormatInt(int64(m.Header.Version), 10))
	b.WriteString(",\"Length\":")
	b.WriteString(strconv.FormatInt(int64(m.Header.Length), 10))
	b.WriteString(",\"ExportTime\":")
	b.WriteString(strconv.FormatInt(int64(m.Header.ExportTime), 10))
	b.WriteString(",\"SequenceNo\":")
	b.WriteString(strconv.FormatInt(int64(m.Header.SequenceNo), 10))
	b.WriteString(",\"DomainID\":")
	b.WriteString(strconv.FormatInt(int64(m.Header.DomainID), 10))
	b.WriteString("},")
}

func (m *Message) encodeAgent(b *bytes.Buffer) {
	b.WriteString("\"AgentID\":\"")
	b.WriteString(m.AgentID)
	b.WriteString("\",")
}
<|endoftext|>


<commit_before>//Package redis provides a cache implementation of onecache using redis as the backend
package redis

import (
	"time"

	"github.com/go-redis/redis"
)

//Default prefix to prevent collision with other key stored in redis
const PREFIX = "onecache:"

type RedisStore struct {
	client *redis.Client
	prefix string
}

//Returns a new instance of the RedisStore
//If prefix is an empty string, the default cache prefix is used
func NewRedisStore(opts *redis.Options, prefix string) *RedisStore {

	var p string

	if prefix == "" {
		p = PREFIX
	} else {
		p = prefix
	}

	return &RedisStore{redis.NewClient(opts), p}
}

func (r *RedisStore) Set(key string, data interface{}, expires time.Duration) error {
	return r.client.Set(r.key(key), data, expires).Err()
}

func (r *RedisStore) Get(key string) (interface{}, error) {

	val, err := r.client.Get(r.key(key)).Result()

	if err != nil {
		return "", err
	}

	return val, nil
}

func (r *RedisStore) key(k string) string {
	return r.prefix + k
}
<commit_msg>Implemented Delete method for redisstore<commit_after>//Package redis provides a cache implementation of onecache using redis as the backend
package redis

import (
	"time"

	"github.com/go-redis/redis"
)

//Default prefix to prevent collision with other key stored in redis
const PREFIX = "onecache:"

type RedisStore struct {
	client *redis.Client
	prefix string
}

//Returns a new instance of the RedisStore
//If prefix is an empty string, the default cache prefix is used
func NewRedisStore(opts *redis.Options, prefix string) *RedisStore {

	var p string

	if prefix == "" {
		p = PREFIX
	} else {
		p = prefix
	}

	return &RedisStore{redis.NewClient(opts), p}
}

func (r *RedisStore) Set(key string, data interface{}, expires time.Duration) error {
	return r.client.Set(r.key(key), data, expires).Err()
}

func (r *RedisStore) Get(key string) (interface{}, error) {

	val, err := r.client.Get(r.key(key)).Result()

	if err != nil {
		return "", err
	}

	return val, nil
}

func (r *RedisStore) Delete(key string) error {
	return r.client.Del(r.key(key)).Err()
}

func (r *RedisStore) key(k string) string {
	return r.prefix + k
}
<|endoftext|>


<commit_before>// 6 july 2014

package ui

import (
	"runtime"
	"sync"
	"unsafe"
)

// Go initializes and runs package ui.
// It returns a non-nil error if initialization fails.
// Otherwise, it will run the event loop and not return until Stop is called.
// Due to platform-specific issues, it must be called from the main OS thread; in general, do not call Go() from anywhere except main() (including any goroutines).
func Go() error {
	runtime.LockOSThread()
	if err := uiinit(); err != nil {
		return err
	}
	go uiissueloop()
	uimsgloop()
	return nil
}

// To ensure that Do() and Stop() only do things after Go() has been called, this channel accepts the requests to issue. The issuing is done by uiissueloop() below.
// Notice that this is a pointer ot a function. See Do() below for details.
var issuer = make(chan *func())

// Do performs f on the main loop, as if it were an event handler.
// It waits for f to execute before returning.
// Do cannot be called within event handlers or within Do itself.
func Do(f func()) {
	done := make(chan struct{})
	defer close(done)
	// THIS MUST BE A POINTER.
	// Previously, the pointer was constructed within issue().
	// This meant that if the Do() was stalled, the garbage collector came in and reused the pointer value too soon!
	call := func() {
		f()
		done <- struct{}{}
	}
	issuer <- &call
	<-done
}

// Stop informs package ui that it should stop.
// Stop then returns immediately.
// Some time after this request is received, Go() will return without performing any final cleanup.
// Stop will not have an effect until any event handlers return.
func Stop() {
	// can't send this directly across issuer
	go func() {
		Do(uistop)
	}()
}

func uiissueloop() {
	for f := range issuer {
		issue(f)
	}
}

type event struct {
	// All events internally return bool; those that don't will be wrapped around to return a dummy value.
	do		func() bool
	lock		sync.Mutex
}

func newEvent() *event {
	return &event{
		do:	func() bool {
			return false
		},
	}
}

func (e *event) set(f func()) {
	e.lock.Lock()
	defer e.lock.Unlock()

	if f == nil {
		f = func() {}
	}
	e.do = func() bool {
		f()
		return false
	}
}

func (e *event) setbool(f func() bool) {
	e.lock.Lock()
	defer e.lock.Unlock()

	if f == nil {
		f = func() bool {
			return false
		}
	}
	e.do = f
}

// This is the common code for running an event.
// It runs on the main thread without a message pump; it provides its own.
func (e *event) fire() bool {
	e.lock.Lock()
	defer e.lock.Unlock()

	return e.do()
}

// Common code for performing a requested action (ui.Do() or ui.Stop()).
// This should run on the main thread.
// Implementations of issue() should call this.
func perform(fp unsafe.Pointer) {
	f := (*func())(fp)
	(*f)()
}
<commit_msg>Added foreign events. Untested for now.<commit_after>// 6 july 2014

package ui

import (
	"runtime"
	"sync"
	"unsafe"
	"reflect"
)

// Go initializes and runs package ui.
// It returns a non-nil error if initialization fails.
// Otherwise, it will run the event loop and not return until Stop is called.
// Due to platform-specific issues, it must be called from the main OS thread; in general, do not call Go() from anywhere except main() (including any goroutines).
func Go() error {
	runtime.LockOSThread()
	if err := uiinit(); err != nil {
		return err
	}
	go uiissueloop()
	uimsgloop()
	return nil
}

// To ensure that Do() and Stop() only do things after Go() has been called, this channel accepts the requests to issue. The issuing is done by uiissueloop() below.
// Notice that this is a pointer ot a function. See Do() below for details.
var issuer = make(chan *func())

// Do performs f on the main loop, as if it were an event handler.
// It waits for f to execute before returning.
// Do cannot be called within event handlers or within Do itself.
func Do(f func()) {
	done := make(chan struct{})
	defer close(done)
	// THIS MUST BE A POINTER.
	// Previously, the pointer was constructed within issue().
	// This meant that if the Do() was stalled, the garbage collector came in and reused the pointer value too soon!
	call := func() {
		f()
		done <- struct{}{}
	}
	issuer <- &call
	<-done
}

// Stop informs package ui that it should stop.
// Stop then returns immediately.
// Some time after this request is received, Go() will return without performing any final cleanup.
// Stop will not have an effect until any event handlers return.
func Stop() {
	// can't send this directly across issuer
	go func() {
		Do(uistop)
	}()
}

func uiissueloop() {
	for f := range issuer {
		issue(f)
	}
}

type event struct {
	// All events internally return bool; those that don't will be wrapped around to return a dummy value.
	do		func() bool
	lock		sync.Mutex
}

func newEvent() *event {
	return &event{
		do:	func() bool {
			return false
		},
	}
}

func (e *event) set(f func()) {
	e.lock.Lock()
	defer e.lock.Unlock()

	if f == nil {
		f = func() {}
	}
	e.do = func() bool {
		f()
		return false
	}
}

func (e *event) setbool(f func() bool) {
	e.lock.Lock()
	defer e.lock.Unlock()

	if f == nil {
		f = func() bool {
			return false
		}
	}
	e.do = f
}

// This is the common code for running an event.
// It runs on the main thread without a message pump; it provides its own.
func (e *event) fire() bool {
	e.lock.Lock()
	defer e.lock.Unlock()

	return e.do()
}

// Common code for performing a requested action (ui.Do() or ui.Stop()).
// This should run on the main thread.
// Implementations of issue() should call this.
func perform(fp unsafe.Pointer) {
	f := (*func())(fp)
	(*f)()
}

// ForeignEvent wraps a channel in such a way that it can be used safely with package ui.
type ForeignEvent struct {
	c	reflect.Value
	e	*event
	d	interface{}
}

// NewForeignEvent creates a new ForeignEvent with the specified channel.
// It panics if the argument is not a receivable channel.
// The returned ForeignEvent assumes ownership of the channel.
// Each time a value is received on the channel, the returned function is invoked on the main thread.
func NewForeignEvent(channel interface{}, handler func(data interface{})) *ForeignEvent {
	c := reflect.ValueOf(channel)
	t := c.Type()
	if t.Kind() != reflect.Chan || (t.ChanDir() & reflect.RecvDir) == 0 {
		panic("non-channel or non-receivable channel passed to NewForeignEvent()")
	}
	fe := &ForeignEvent{
		c:		c,
		e:		newEvent(),
	}
	fe.e.set(func() {
		handler(fe.d)
	})
	go fe.do()
	return fe
}

func (fe *ForeignEvent) do() {
	wait := make(chan struct{})
	defer close(wait)
	for {
		v, ok := fe.c.Recv()
		if !ok {
			break
		}
		fe.d = v.Interface()
		Do(func() {
			fe.e.fire()
			wait <- struct{}{}
		})
		<-wait
	}
}

// Stop ceases all future invocations of the handler passed to NewForeignEvent() on fe; the values read from the channel are merely discarded.
func (fe *ForeignEvent) Stop() {
	fe.e.set(nil)
}
<|endoftext|>


<commit_before>package main

import (
	_ "net/http/pprof"

	"fmt"
	"os"
	"os/signal"

	"github.com/siggy/bbox/beatboxer/render"
	"github.com/siggy/bbox/beatboxer/render/web"

	"github.com/siggy/bbox/bbox"
	"github.com/siggy/bbox/beatboxer"
	"github.com/siggy/bbox/beatboxer/programs/ceottk"
	"github.com/siggy/bbox/beatboxer/programs/drums"
)

func main() {
	// log.SetLevel(log.DebugLevel)
	// file, err := os.OpenFile("beatboxer_noleds.log", os.O_CREATE|os.O_WRONLY, 0666)
	// if err == nil {
	// 	log.SetOutput(file)
	// } else {
	// 	log.Info("Failed to log to file, using default stderr")
	// }
	// defer file.Close()

	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt, os.Kill)

	harness := beatboxer.InitHarness(
		[]render.Renderer{web.InitWeb()},
		// []render.Renderer{},
		bbox.KeyMapsPC,
	)

	harness.Register(&drums.DrumMachine{})
	harness.Register(&ceottk.Ceottk{})

	go harness.Run()

	for {
		select {
		case <-sig:
			fmt.Printf("Received OS signal, exiting")
			return
		}
	}
}
<commit_msg>disable pprof<commit_after>package main

import (
	// _ "net/http/pprof"

	"fmt"
	"os"
	"os/signal"

	"github.com/siggy/bbox/beatboxer/render"
	"github.com/siggy/bbox/beatboxer/render/web"

	"github.com/siggy/bbox/bbox"
	"github.com/siggy/bbox/beatboxer"
	"github.com/siggy/bbox/beatboxer/programs/ceottk"
	"github.com/siggy/bbox/beatboxer/programs/drums"
)

func main() {
	// log.SetLevel(log.DebugLevel)
	// file, err := os.OpenFile("beatboxer_noleds.log", os.O_CREATE|os.O_WRONLY, 0666)
	// if err == nil {
	// 	log.SetOutput(file)
	// } else {
	// 	log.Info("Failed to log to file, using default stderr")
	// }
	// defer file.Close()

	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt, os.Kill)

	harness := beatboxer.InitHarness(
		[]render.Renderer{web.InitWeb()},
		// []render.Renderer{},
		bbox.KeyMapsPC,
	)

	harness.Register(&drums.DrumMachine{})
	harness.Register(&ceottk.Ceottk{})

	go harness.Run()

	for {
		select {
		case <-sig:
			fmt.Printf("Received OS signal, exiting")
			return
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"time"

	"golang.org/x/crypto/ssh"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc"
	"github.com/concourse/baggageclaim/baggageclaimcmd"
	"github.com/concourse/bin/bindata"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/restart"
	"github.com/tedsuo/ifrit/sigmon"
)

type WorkerCommand struct {
	Name string   `long:"name" description:"The name to set for the worker during registration. If not specified, the hostname will be used."`
	Tags []string `long:"tag" description:"A tag to set during registration. Can be specified multiple times."`

	TeamName string `long:"team" description:"The name of the team that this worker will be assigned to."`

	HTTPProxy  URLFlag  `long:"http-proxy"  env:"http_proxy"                  description:"HTTP proxy endpoint to use for containers."`
	HTTPSProxy URLFlag  `long:"https-proxy" env:"https_proxy"                 description:"HTTPS proxy endpoint to use for containers."`
	NoProxy    []string `long:"no-proxy"    env:"no_proxy"    env-delim:","   description:"Blacklist of addresses to skip the proxy when reaching."`

	WorkDir DirFlag `long:"work-dir" required:"true" description:"Directory in which to place container data."`

	BindIP   IPFlag `long:"bind-ip"   default:"127.0.0.1" description:"IP address on which to listen for the Garden server."`
	BindPort uint16 `long:"bind-port" default:"7777"      description:"Port on which to listen for the Garden server."`

	PeerIP IPFlag `long:"peer-ip" description:"IP used to reach this worker from the ATC nodes. If omitted, the worker will be forwarded through the SSH connection to the TSA."`

	Garden GardenBackend `group:"Garden Configuration" namespace:"garden"`

	Baggageclaim baggageclaimcmd.BaggageclaimCommand `group:"Baggageclaim Configuration" namespace:"baggageclaim"`

	TSA BeaconConfig `group:"TSA Configuration" namespace:"tsa"`

	Metrics struct {
		YellerAPIKey      string `long:"yeller-api-key"     description:"Yeller API key. If specified, all errors logged will be emitted."`
		YellerEnvironment string `long:"yeller-environment" description:"Environment to tag on all Yeller events emitted."`
	} `group:"Metrics & Diagnostics"`
}

func (cmd *WorkerCommand) Execute(args []string) error {
	logger := lager.NewLogger("worker")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.INFO))

	hasAssets, err := cmd.setup(logger.Session("setup"))
	if err != nil {
		return err
	}

	worker, gardenRunner, err := cmd.gardenRunner(logger.Session("garden"), hasAssets)
	if err != nil {
		return err
	}

	worker.Version = WorkerVersion

	baggageclaimRunner, err := cmd.baggageclaimRunner(logger.Session("baggageclaim"), hasAssets)
	if err != nil {
		return err
	}

	members := grouper.Members{
		{
			Name:   "garden",
			Runner: gardenRunner,
		},
		{
			Name:   "baggageclaim",
			Runner: baggageclaimRunner,
		},
	}

	if cmd.TSA.WorkerPrivateKey != "" {
		members = append(members, grouper.Member{
			Name:   "beacon",
			Runner: cmd.beaconRunner(logger.Session("beacon"), worker),
		})
	}

	runner := sigmon.New(grouper.NewParallel(os.Interrupt, members))

	return <-ifrit.Invoke(runner).Wait()
}

func (cmd *WorkerCommand) assetPath(paths ...string) string {
	return filepath.Join(append([]string{cmd.WorkDir.Path(), Version, "assets"}, paths...)...)
}

func (cmd *WorkerCommand) setup(logger lager.Logger) (bool, error) {
	okMarker := cmd.assetPath("ok")

	_, err := os.Stat(okMarker)
	if err == nil {
		logger.Info("already-done")
		return true, nil
	}

	logger.Info("unpacking")

	err = bindata.RestoreAssets(filepath.Split(cmd.assetPath()))
	if err != nil {
		logger.Error("failed-to-unpack", err)
		return false, err
	}

	_, err = os.Stat(cmd.assetPath())
	if os.IsNotExist(err) {
		logger.Info("no-assets")
		return false, nil
	}

	ok, err := os.Create(okMarker)
	if err != nil {
		logger.Error("failed-to-create-ok-marker", err)
		return false, err
	}

	err = ok.Close()
	if err != nil {
		logger.Error("failed-to-close-ok-marker", err)
		return false, err
	}

	logger.Info("done")

	return true, nil
}

func (cmd *WorkerCommand) workerName() (string, error) {
	if cmd.Name != "" {
		return cmd.Name, nil
	}

	return os.Hostname()
}

func (cmd *WorkerCommand) baggageclaimRunner(logger lager.Logger, hasAssets bool) (ifrit.Runner, error) {
	volumesDir := filepath.Join(cmd.WorkDir.Path(), "volumes")

	err := os.MkdirAll(volumesDir, 0755)
	if err != nil {
		return nil, err
	}

	cmd.Baggageclaim.Metrics = cmd.Metrics
	cmd.Baggageclaim.VolumesDir = baggageclaimcmd.DirFlag(volumesDir)

	if hasAssets {
		cmd.Baggageclaim.MkfsBin = cmd.assetPath("btrfs", "mkfs.btrfs")
		cmd.Baggageclaim.BtrfsBin = cmd.assetPath("btrfs", "btrfs")
	}

	return cmd.Baggageclaim.Runner(nil)
}

func (cmd *WorkerCommand) beaconRunner(logger lager.Logger, worker atc.Worker) ifrit.Runner {
	beacon := Beacon{
		Logger: logger,
		Config: cmd.TSA,
	}

	var beaconRunner ifrit.RunFunc
	if cmd.PeerIP != nil {
		worker.GardenAddr = fmt.Sprintf("%s:%d", cmd.PeerIP.IP(), cmd.BindPort)
		worker.BaggageclaimURL = fmt.Sprintf("http://%s:%d", cmd.PeerIP.IP(), cmd.Baggageclaim.BindPort)
		beaconRunner = beacon.Register
	} else {
		worker.GardenAddr = fmt.Sprintf("%s:%d", cmd.BindIP.IP(), cmd.BindPort)
		worker.BaggageclaimURL = fmt.Sprintf("http://%s:%d", cmd.Baggageclaim.BindIP.IP(), cmd.Baggageclaim.BindPort)
		beaconRunner = beacon.Forward
	}

	beacon.Worker = worker

	return restart.Restarter{
		Runner: beaconRunner,
		Load: func(prevRunner ifrit.Runner, prevErr error) ifrit.Runner {
			if prevErr == nil {
				return nil
			}

			if _, ok := prevErr.(*ssh.ExitError); !ok {
				logger.Error("restarting", prevErr)
				time.Sleep(5 * time.Second)
				return beaconRunner
			}

			return nil
		},
	}
}
<commit_msg>auto-configure overlays dir<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"time"

	"golang.org/x/crypto/ssh"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc"
	"github.com/concourse/baggageclaim/baggageclaimcmd"
	"github.com/concourse/bin/bindata"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/restart"
	"github.com/tedsuo/ifrit/sigmon"
)

type WorkerCommand struct {
	Name string   `long:"name" description:"The name to set for the worker during registration. If not specified, the hostname will be used."`
	Tags []string `long:"tag" description:"A tag to set during registration. Can be specified multiple times."`

	TeamName string `long:"team" description:"The name of the team that this worker will be assigned to."`

	HTTPProxy  URLFlag  `long:"http-proxy"  env:"http_proxy"                  description:"HTTP proxy endpoint to use for containers."`
	HTTPSProxy URLFlag  `long:"https-proxy" env:"https_proxy"                 description:"HTTPS proxy endpoint to use for containers."`
	NoProxy    []string `long:"no-proxy"    env:"no_proxy"    env-delim:","   description:"Blacklist of addresses to skip the proxy when reaching."`

	WorkDir DirFlag `long:"work-dir" required:"true" description:"Directory in which to place container data."`

	BindIP   IPFlag `long:"bind-ip"   default:"127.0.0.1" description:"IP address on which to listen for the Garden server."`
	BindPort uint16 `long:"bind-port" default:"7777"      description:"Port on which to listen for the Garden server."`

	PeerIP IPFlag `long:"peer-ip" description:"IP used to reach this worker from the ATC nodes. If omitted, the worker will be forwarded through the SSH connection to the TSA."`

	Garden GardenBackend `group:"Garden Configuration" namespace:"garden"`

	Baggageclaim baggageclaimcmd.BaggageclaimCommand `group:"Baggageclaim Configuration" namespace:"baggageclaim"`

	TSA BeaconConfig `group:"TSA Configuration" namespace:"tsa"`

	Metrics struct {
		YellerAPIKey      string `long:"yeller-api-key"     description:"Yeller API key. If specified, all errors logged will be emitted."`
		YellerEnvironment string `long:"yeller-environment" description:"Environment to tag on all Yeller events emitted."`
	} `group:"Metrics & Diagnostics"`
}

func (cmd *WorkerCommand) Execute(args []string) error {
	logger := lager.NewLogger("worker")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.INFO))

	hasAssets, err := cmd.setup(logger.Session("setup"))
	if err != nil {
		return err
	}

	worker, gardenRunner, err := cmd.gardenRunner(logger.Session("garden"), hasAssets)
	if err != nil {
		return err
	}

	worker.Version = WorkerVersion

	baggageclaimRunner, err := cmd.baggageclaimRunner(logger.Session("baggageclaim"), hasAssets)
	if err != nil {
		return err
	}

	members := grouper.Members{
		{
			Name:   "garden",
			Runner: gardenRunner,
		},
		{
			Name:   "baggageclaim",
			Runner: baggageclaimRunner,
		},
	}

	if cmd.TSA.WorkerPrivateKey != "" {
		members = append(members, grouper.Member{
			Name:   "beacon",
			Runner: cmd.beaconRunner(logger.Session("beacon"), worker),
		})
	}

	runner := sigmon.New(grouper.NewParallel(os.Interrupt, members))

	return <-ifrit.Invoke(runner).Wait()
}

func (cmd *WorkerCommand) assetPath(paths ...string) string {
	return filepath.Join(append([]string{cmd.WorkDir.Path(), Version, "assets"}, paths...)...)
}

func (cmd *WorkerCommand) setup(logger lager.Logger) (bool, error) {
	okMarker := cmd.assetPath("ok")

	_, err := os.Stat(okMarker)
	if err == nil {
		logger.Info("already-done")
		return true, nil
	}

	logger.Info("unpacking")

	err = bindata.RestoreAssets(filepath.Split(cmd.assetPath()))
	if err != nil {
		logger.Error("failed-to-unpack", err)
		return false, err
	}

	_, err = os.Stat(cmd.assetPath())
	if os.IsNotExist(err) {
		logger.Info("no-assets")
		return false, nil
	}

	ok, err := os.Create(okMarker)
	if err != nil {
		logger.Error("failed-to-create-ok-marker", err)
		return false, err
	}

	err = ok.Close()
	if err != nil {
		logger.Error("failed-to-close-ok-marker", err)
		return false, err
	}

	logger.Info("done")

	return true, nil
}

func (cmd *WorkerCommand) workerName() (string, error) {
	if cmd.Name != "" {
		return cmd.Name, nil
	}

	return os.Hostname()
}

func (cmd *WorkerCommand) baggageclaimRunner(logger lager.Logger, hasAssets bool) (ifrit.Runner, error) {
	volumesDir := filepath.Join(cmd.WorkDir.Path(), "volumes")

	err := os.MkdirAll(volumesDir, 0755)
	if err != nil {
		return nil, err
	}

	cmd.Baggageclaim.Metrics = cmd.Metrics
	cmd.Baggageclaim.VolumesDir = baggageclaimcmd.DirFlag(volumesDir)

	cmd.Baggageclaim.OverlaysDir = filepath.Join(cmd.WorkDir.Path(), "overlays")

	if hasAssets {
		cmd.Baggageclaim.MkfsBin = cmd.assetPath("btrfs", "mkfs.btrfs")
		cmd.Baggageclaim.BtrfsBin = cmd.assetPath("btrfs", "btrfs")
	}

	return cmd.Baggageclaim.Runner(nil)
}

func (cmd *WorkerCommand) beaconRunner(logger lager.Logger, worker atc.Worker) ifrit.Runner {
	beacon := Beacon{
		Logger: logger,
		Config: cmd.TSA,
	}

	var beaconRunner ifrit.RunFunc
	if cmd.PeerIP != nil {
		worker.GardenAddr = fmt.Sprintf("%s:%d", cmd.PeerIP.IP(), cmd.BindPort)
		worker.BaggageclaimURL = fmt.Sprintf("http://%s:%d", cmd.PeerIP.IP(), cmd.Baggageclaim.BindPort)
		beaconRunner = beacon.Register
	} else {
		worker.GardenAddr = fmt.Sprintf("%s:%d", cmd.BindIP.IP(), cmd.BindPort)
		worker.BaggageclaimURL = fmt.Sprintf("http://%s:%d", cmd.Baggageclaim.BindIP.IP(), cmd.Baggageclaim.BindPort)
		beaconRunner = beacon.Forward
	}

	beacon.Worker = worker

	return restart.Restarter{
		Runner: beaconRunner,
		Load: func(prevRunner ifrit.Runner, prevErr error) ifrit.Runner {
			if prevErr == nil {
				return nil
			}

			if _, ok := prevErr.(*ssh.ExitError); !ok {
				logger.Error("restarting", prevErr)
				time.Sleep(5 * time.Second)
				return beaconRunner
			}

			return nil
		},
	}
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"bufio"
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync/atomic"
	"unicode/utf8"

	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"

	"github.com/Debian/debiman/internal/archive"
	"github.com/Debian/debiman/internal/manpage"
	"github.com/Debian/debiman/internal/recode"

	"pault.ag/go/debian/deb"
	"pault.ag/go/debian/version"
)

// canSkip returns true if the package is present in the same (or a
// newer) version on disk already.
func canSkip(p pkgEntry, vPath string) bool {
	v, err := ioutil.ReadFile(vPath)
	if err != nil {
		return false
	}

	vCurrent, err := version.Parse(string(v))
	if err != nil {
		log.Printf("Warning: could not parse current package version from %q: %v", vPath, err)
		return false
	}

	return version.Compare(vCurrent, p.version) >= 0
}

// findClosestFile returns a manpage struct for name, if name exists in the same suite.
// TODO(stapelberg): resolve multiple matches: consider dependencies of src
func findClosestFile(logger *log.Logger, p pkgEntry, src, name string, contentByPath map[string][]*contentEntry) string {
	logger.Printf("findClosestFile(src=%q, name=%q)", src, name)
	c, ok := contentByPath[strings.TrimPrefix(name, "/usr/share/man/")]
	if !ok {
		return ""
	}

	// Ensure we only consider choices within the same suite.
	filtered := make([]*contentEntry, 0, len(c))
	for _, e := range c {
		if e.suite != p.suite {
			continue
		}
		filtered = append(filtered, e)
	}
	c = filtered

	// We still have more than one choice. In case the candidate is in
	// the same package as the source link, we take it.
	if len(c) > 1 {
		var last *contentEntry
		cnt := 0
		for _, e := range c {
			if e.binarypkg != p.binarypkg {
				continue
			}
			last = e
			if cnt++; cnt > 1 {
				break
			}
		}
		if cnt == 1 {
			c = []*contentEntry{last}
		}
	}
	if len(c) == 1 {
		m, err := manpage.FromManPath(strings.TrimPrefix(name, "/usr/share/man/"), &manpage.PkgMeta{
			Binarypkg: c[0].binarypkg,
			Suite:     c[0].suite,
		})
		logger.Printf("parsing %q as man: %v", name, err)
		if err == nil {
			return m.ServingPath() + ".gz"
		}
	}
	return ""
}

func findFile(logger *log.Logger, src, name string, contentByPath map[string][]*contentEntry) (string, string, bool) {
	// TODO: where is searchPath defined canonically?
	// TODO(later): why is "/"+ in front of src necessary?
	searchPath := []string{
		"/" + filepath.Dir(src), // “.”
		// To prefer manpages in the same language, add “..”, e.g.:
		// /usr/share/man/fr/man7/bash-builtins.7 references
		// man1/bash.1, which should be taken from
		// /usr/share/man/fr/man1/bash.1 instead of
		// /usr/share/man/man1/bash.1.
		"/" + filepath.Dir(src) + "/..",
		"/usr/local/man",
		"/usr/share/man",
	}
	logger.Printf("searching reference so=%q", name)
	for _, search := range searchPath {
		var check string
		if filepath.IsAbs(name) {
			check = filepath.Clean(name)
		} else {
			check = filepath.Join(search, name)
		}
		// Some references include the .gz suffix, some don’t.
		if !strings.HasSuffix(check, ".gz") {
			check = check + ".gz"
		}

		c, ok := contentByPath[strings.TrimPrefix(check, "/usr/share/man/")]
		if !ok {
			log.Printf("%q does not exist", check)
			continue
		}

		m, err := manpage.FromManPath(strings.TrimPrefix(check, "/usr/share/man/"), &manpage.PkgMeta{
			Binarypkg: c[0].binarypkg,
			Suite:     c[0].suite,
		})
		logger.Printf("parsing %q as man: %v", check, err)
		if err == nil {
			return m.ServingPath() + ".gz", "", true
		}

		// TODO: we currently use the first manpage we find. this is non-deterministic, so sort.
		// TODO(later): try to resolve this reference intelligently, i.e. consider installability to narrow down the list of candidates. add a testcase with all cases that we have in all Debian suites currently
		return c[0].suite + "/" + c[0].binarypkg + "/aux" + check, check, true
	}
	return name, "", false
}

func soElim(logger *log.Logger, src string, r io.Reader, w io.Writer, contentByPath map[string][]*contentEntry) ([]string, error) {
	var refs []string
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := scanner.Text()
		if !strings.HasPrefix(line, ".so ") {
			fmt.Fprintln(w, line)
			continue
		}
		so := strings.TrimSpace(line[len(".so "):])

		resolved, ref, ok := findFile(logger, src, so, contentByPath)
		if !ok {
			// Omitting .so lines which cannot be found is consistent
			// with what man(1) and other online man viewers do.
			logger.Printf("WARNING: could not find .so referenced file %q, omitting the .so line", so)
			continue
		}

		fmt.Fprintf(w, ".so %s\n", resolved)
		if ref != "" {
			refs = append(refs, ref)
		}
	}
	return refs, scanner.Err()
}

func writeManpage(logger *log.Logger, src, dest string, r io.Reader, m *manpage.Meta, contentByPath map[string][]*contentEntry) ([]string, error) {
	var refs []string
	content, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}
	if !utf8.Valid(content) {
		content, err = ioutil.ReadAll(recode.Reader(bytes.NewReader(content), m.Language))
		if err != nil {
			return nil, err
		}
	}
	err = writeAtomically(dest, true, func(w io.Writer) error {
		var err error
		refs, err = soElim(logger, src, bytes.NewReader(content), w, contentByPath)
		return err
	})
	return refs, err
}

func downloadPkg(ar *archive.Getter, p pkgEntry, gv globalView) error {
	vPath := filepath.Join(*servingDir, p.suite, p.binarypkg, "VERSION")

	if !*forceReextract && canSkip(p, vPath) {
		return nil
	}

	logger := log.New(os.Stderr, p.suite+"/"+p.binarypkg+": ", log.LstdFlags)

	tmp, err := ar.Get(p.filename, p.sha256)
	if err != nil {
		return err
	}

	if _, err := tmp.Seek(0, os.SEEK_SET); err != nil {
		return err
	}

	allRefs := make(map[string]bool)

	d, err := deb.Load(tmp, p.filename)
	if err != nil {
		return err
	}
	for {
		header, err := d.Data.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		if header.Typeflag != tar.TypeReg &&
			header.Typeflag != tar.TypeRegA &&
			header.Typeflag != tar.TypeSymlink &&
			header.Typeflag != tar.TypeLink {
			continue
		}
		if header.FileInfo().IsDir() {
			continue
		}
		if !strings.HasPrefix(header.Name, "./usr/share/man/") {
			continue
		}

		destdir := filepath.Join(*servingDir, p.suite, p.binarypkg)
		if err := os.MkdirAll(destdir, 0755); err != nil {
			return err
		}

		// TODO: return m?
		m, err := manpage.FromManPath(strings.TrimPrefix(header.Name, "./usr/share/man/"), &manpage.PkgMeta{
			Binarypkg: p.binarypkg,
			Suite:     p.suite,
		})

		if err != nil {
			logger.Printf("WARNING: file name %q (underneath /usr/share/man) cannot be parsed: %v", header.Name, err)
			continue
		}

		destPath := filepath.Join(*servingDir, m.ServingPath()+".gz")
		if header.Typeflag == tar.TypeLink {
			d, err := manpage.FromManPath(strings.TrimPrefix(header.Linkname, "./usr/share/man/"), &manpage.PkgMeta{
				Binarypkg: p.binarypkg,
				Suite:     p.suite,
			})
			if err != nil {
				logger.Printf("WARNING: hard link name %q (underneath /usr/share/man) cannot be parsed: %v", header.Linkname, err)
				continue
			}
			if err := os.Link(filepath.Join(*servingDir, d.ServingPath()+".gz"), m.ServingPath()+".gz"); err != nil {
				if os.IsExist(err) {
					continue
				}
				return err
			}
			continue
		}
		if header.Typeflag == tar.TypeSymlink {
			// filepath.Join calls filepath.Abs
			resolved := filepath.Join(filepath.Dir(strings.TrimPrefix(header.Name, ".")), header.Linkname)
			if !strings.HasSuffix(resolved, ".gz") {
				resolved = resolved + ".gz"
			}

			destsp := findClosestFile(logger, p, header.Name, resolved, gv.contentByPath)
			if destsp == "" {
				// Try to extract the resolved file as non-manpage
				// file. If the resolved file does not live in this
				// package, this will result in a dangling symlink.
				allRefs[resolved] = true
				destsp = filepath.Join(filepath.Dir(m.ServingPath()), "aux", resolved)
				logger.Printf("WARNING: possibly dangling symlink %q -> %q", header.Name, header.Linkname)
			}

			// TODO(stapelberg): add a unit test for this entire function
			// TODO(stapelberg): ganeti has an interesting twist: their manpages live outside of usr/share/man, and they only have symlinks. in this case, we should extract the file to aux/ and then mangle the symlink dest. problem: manpages actually are in a separate package (ganeti-2.15) and use an absolute symlink (/etc/ganeti/share), which is not shipped with the package.
			rel, err := filepath.Rel(filepath.Dir(m.ServingPath()), destsp)
			if err != nil {
				logger.Printf("WARNING: %v", err)
				continue
			}
			if err := os.Symlink(rel, destPath); err != nil {
				if os.IsExist(err) {
					continue
				}
				return err
			}
			if err := maybeSetLinkMtime(destPath, header.ModTime); err != nil {
				return err
			}

			continue
		}

		r, err := gzip.NewReader(d.Data)
		if err != nil {
			return err
		}
		refs, err := writeManpage(logger, header.Name, destPath, r, m, gv.contentByPath)
		if err != nil {
			return err
		}
		if err := os.Chtimes(destPath, header.ModTime, header.ModTime); err != nil {
			return err
		}
		if err := r.Close(); err != nil {
			return err
		}

		for _, r := range refs {
			allRefs[r] = true
		}
	}

	// Extract all non-manpage files which were referenced via .so
	// statements, if any.
	if len(allRefs) > 0 {
		if _, err := tmp.Seek(0, os.SEEK_SET); err != nil {
			return err
		}

		d, err = deb.Load(tmp, p.filename)
		if err != nil {
			return err
		}
		for {
			header, err := d.Data.Next()
			if err == io.EOF {
				break
			}
			if err != nil {
				return err
			}

			if header.Typeflag != tar.TypeReg &&
				header.Typeflag != tar.TypeRegA &&
				header.Typeflag != tar.TypeSymlink {
				continue
			}

			if header.FileInfo().IsDir() {
				continue
			}

			if !allRefs[strings.TrimPrefix(header.Name, ".")] {
				continue
			}

			destPath := filepath.Join(*servingDir, p.suite, p.binarypkg, "aux", header.Name)
			logger.Printf("extracting referenced non-manpage file %q to %q", header.Name, destPath)
			if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
				return err
			}
			if err := writeAtomically(destPath, false, func(w io.Writer) error {
				_, err := io.Copy(w, d.Data)
				return err
			}); err != nil {
				return err
			}
		}
	}

	if err := ioutil.WriteFile(vPath, []byte(p.version.String()), 0644); err != nil {
		if os.IsNotExist(err) {
			// If the directory does not exist, we did not extract any
			// manpages. Since Contents files are not precise (they
			// might lag behind), this can happen occasionally.
			return nil
		}
		return fmt.Errorf("Writing version file %q: %v", err)
	}

	atomic.AddUint64(&gv.stats.PackagesExtracted, 1)

	return nil
}

func parallelDownload(ar *archive.Getter, gv globalView) error {
	eg, ctx := errgroup.WithContext(context.Background())
	downloadChan := make(chan pkgEntry)
	// TODO: flag for parallelism level
	for i := 0; i < 10; i++ {
		eg.Go(func() error {
			for p := range downloadChan {
				if err := downloadPkg(ar, p, gv); err != nil {
					return err
				}
			}
			return nil
		})
	}
	for _, p := range gv.pkgs {
		select {
		case downloadChan <- *p:
		case <-ctx.Done():
			break
		}
	}
	close(downloadChan)
	return eg.Wait()
}
<commit_msg>download: resolve symlink ambiguities deterministically<commit_after>package main

import (
	"archive/tar"
	"bufio"
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync/atomic"
	"unicode/utf8"

	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"

	"github.com/Debian/debiman/internal/archive"
	"github.com/Debian/debiman/internal/manpage"
	"github.com/Debian/debiman/internal/recode"

	"pault.ag/go/debian/deb"
	"pault.ag/go/debian/version"
)

// canSkip returns true if the package is present in the same (or a
// newer) version on disk already.
func canSkip(p pkgEntry, vPath string) bool {
	v, err := ioutil.ReadFile(vPath)
	if err != nil {
		return false
	}

	vCurrent, err := version.Parse(string(v))
	if err != nil {
		log.Printf("Warning: could not parse current package version from %q: %v", vPath, err)
		return false
	}

	return version.Compare(vCurrent, p.version) >= 0
}

type contentByBinarypkg []*contentEntry

func (p contentByBinarypkg) Len() int           { return len(p) }
func (p contentByBinarypkg) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p contentByBinarypkg) Less(i, j int) bool { return p[i].binarypkg < p[j].binarypkg }

// findClosestFile returns a manpage struct for name, if name exists in the same suite.
// TODO(stapelberg): resolve multiple matches: consider dependencies of src
func findClosestFile(logger *log.Logger, p pkgEntry, src, name string, contentByPath map[string][]*contentEntry) string {
	logger.Printf("findClosestFile(src=%q, name=%q)", src, name)
	c, ok := contentByPath[strings.TrimPrefix(name, "/usr/share/man/")]
	if !ok {
		return ""
	}

	// Ensure we only consider choices within the same suite.
	filtered := make([]*contentEntry, 0, len(c))
	for _, e := range c {
		if e.suite != p.suite {
			continue
		}
		filtered = append(filtered, e)
	}
	c = filtered

	// We still have more than one choice. In case the candidate is in
	// the same package as the source link, we take it.
	if len(c) > 1 {
		var last *contentEntry
		cnt := 0
		for _, e := range c {
			if e.binarypkg != p.binarypkg {
				continue
			}
			last = e
			if cnt++; cnt > 1 {
				break
			}
		}
		if cnt == 1 {
			c = []*contentEntry{last}
		}
	}
	if len(c) > 1 {
		// We can’t make a 100% correct choice, but we can at least
		// make a deterministic choice. The user will see the
		// conflicting packages in the navigation panel to ultimately
		// resolve the situation, if necessary.
		sort.Sort(contentByBinarypkg(c))
	}

	m, err := manpage.FromManPath(strings.TrimPrefix(name, "/usr/share/man/"), &manpage.PkgMeta{
		Binarypkg: c[0].binarypkg,
		Suite:     c[0].suite,
	})
	logger.Printf("parsing %q as man: %v", name, err)
	if err == nil {
		return m.ServingPath() + ".gz"
	}

	return ""
}

func findFile(logger *log.Logger, src, name string, contentByPath map[string][]*contentEntry) (string, string, bool) {
	// TODO: where is searchPath defined canonically?
	// TODO(later): why is "/"+ in front of src necessary?
	searchPath := []string{
		"/" + filepath.Dir(src), // “.”
		// To prefer manpages in the same language, add “..”, e.g.:
		// /usr/share/man/fr/man7/bash-builtins.7 references
		// man1/bash.1, which should be taken from
		// /usr/share/man/fr/man1/bash.1 instead of
		// /usr/share/man/man1/bash.1.
		"/" + filepath.Dir(src) + "/..",
		"/usr/local/man",
		"/usr/share/man",
	}
	logger.Printf("searching reference so=%q", name)
	for _, search := range searchPath {
		var check string
		if filepath.IsAbs(name) {
			check = filepath.Clean(name)
		} else {
			check = filepath.Join(search, name)
		}
		// Some references include the .gz suffix, some don’t.
		if !strings.HasSuffix(check, ".gz") {
			check = check + ".gz"
		}

		c, ok := contentByPath[strings.TrimPrefix(check, "/usr/share/man/")]
		if !ok {
			log.Printf("%q does not exist", check)
			continue
		}

		m, err := manpage.FromManPath(strings.TrimPrefix(check, "/usr/share/man/"), &manpage.PkgMeta{
			Binarypkg: c[0].binarypkg,
			Suite:     c[0].suite,
		})
		logger.Printf("parsing %q as man: %v", check, err)
		if err == nil {
			return m.ServingPath() + ".gz", "", true
		}

		// TODO: we currently use the first manpage we find. this is non-deterministic, so sort.
		// TODO(later): try to resolve this reference intelligently, i.e. consider installability to narrow down the list of candidates. add a testcase with all cases that we have in all Debian suites currently
		return c[0].suite + "/" + c[0].binarypkg + "/aux" + check, check, true
	}
	return name, "", false
}

func soElim(logger *log.Logger, src string, r io.Reader, w io.Writer, contentByPath map[string][]*contentEntry) ([]string, error) {
	var refs []string
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := scanner.Text()
		if !strings.HasPrefix(line, ".so ") {
			fmt.Fprintln(w, line)
			continue
		}
		so := strings.TrimSpace(line[len(".so "):])

		resolved, ref, ok := findFile(logger, src, so, contentByPath)
		if !ok {
			// Omitting .so lines which cannot be found is consistent
			// with what man(1) and other online man viewers do.
			logger.Printf("WARNING: could not find .so referenced file %q, omitting the .so line", so)
			continue
		}

		fmt.Fprintf(w, ".so %s\n", resolved)
		if ref != "" {
			refs = append(refs, ref)
		}
	}
	return refs, scanner.Err()
}

func writeManpage(logger *log.Logger, src, dest string, r io.Reader, m *manpage.Meta, contentByPath map[string][]*contentEntry) ([]string, error) {
	var refs []string
	content, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}
	if !utf8.Valid(content) {
		content, err = ioutil.ReadAll(recode.Reader(bytes.NewReader(content), m.Language))
		if err != nil {
			return nil, err
		}
	}
	err = writeAtomically(dest, true, func(w io.Writer) error {
		var err error
		refs, err = soElim(logger, src, bytes.NewReader(content), w, contentByPath)
		return err
	})
	return refs, err
}

func downloadPkg(ar *archive.Getter, p pkgEntry, gv globalView) error {
	vPath := filepath.Join(*servingDir, p.suite, p.binarypkg, "VERSION")

	if !*forceReextract && canSkip(p, vPath) {
		return nil
	}

	logger := log.New(os.Stderr, p.suite+"/"+p.binarypkg+": ", log.LstdFlags)

	tmp, err := ar.Get(p.filename, p.sha256)
	if err != nil {
		return err
	}

	if _, err := tmp.Seek(0, os.SEEK_SET); err != nil {
		return err
	}

	allRefs := make(map[string]bool)

	d, err := deb.Load(tmp, p.filename)
	if err != nil {
		return err
	}
	for {
		header, err := d.Data.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		if header.Typeflag != tar.TypeReg &&
			header.Typeflag != tar.TypeRegA &&
			header.Typeflag != tar.TypeSymlink &&
			header.Typeflag != tar.TypeLink {
			continue
		}
		if header.FileInfo().IsDir() {
			continue
		}
		if !strings.HasPrefix(header.Name, "./usr/share/man/") {
			continue
		}

		destdir := filepath.Join(*servingDir, p.suite, p.binarypkg)
		if err := os.MkdirAll(destdir, 0755); err != nil {
			return err
		}

		// TODO: return m?
		m, err := manpage.FromManPath(strings.TrimPrefix(header.Name, "./usr/share/man/"), &manpage.PkgMeta{
			Binarypkg: p.binarypkg,
			Suite:     p.suite,
		})

		if err != nil {
			logger.Printf("WARNING: file name %q (underneath /usr/share/man) cannot be parsed: %v", header.Name, err)
			continue
		}

		destPath := filepath.Join(*servingDir, m.ServingPath()+".gz")
		if header.Typeflag == tar.TypeLink {
			d, err := manpage.FromManPath(strings.TrimPrefix(header.Linkname, "./usr/share/man/"), &manpage.PkgMeta{
				Binarypkg: p.binarypkg,
				Suite:     p.suite,
			})
			if err != nil {
				logger.Printf("WARNING: hard link name %q (underneath /usr/share/man) cannot be parsed: %v", header.Linkname, err)
				continue
			}
			if err := os.Link(filepath.Join(*servingDir, d.ServingPath()+".gz"), m.ServingPath()+".gz"); err != nil {
				if os.IsExist(err) {
					continue
				}
				return err
			}
			continue
		}
		if header.Typeflag == tar.TypeSymlink {
			// filepath.Join calls filepath.Abs
			resolved := filepath.Join(filepath.Dir(strings.TrimPrefix(header.Name, ".")), header.Linkname)
			if !strings.HasSuffix(resolved, ".gz") {
				resolved = resolved + ".gz"
			}

			destsp := findClosestFile(logger, p, header.Name, resolved, gv.contentByPath)
			if destsp == "" {
				// Try to extract the resolved file as non-manpage
				// file. If the resolved file does not live in this
				// package, this will result in a dangling symlink.
				allRefs[resolved] = true
				destsp = filepath.Join(filepath.Dir(m.ServingPath()), "aux", resolved)
				logger.Printf("WARNING: possibly dangling symlink %q -> %q", header.Name, header.Linkname)
			}

			// TODO(stapelberg): add a unit test for this entire function
			// TODO(stapelberg): ganeti has an interesting twist: their manpages live outside of usr/share/man, and they only have symlinks. in this case, we should extract the file to aux/ and then mangle the symlink dest. problem: manpages actually are in a separate package (ganeti-2.15) and use an absolute symlink (/etc/ganeti/share), which is not shipped with the package.
			rel, err := filepath.Rel(filepath.Dir(m.ServingPath()), destsp)
			if err != nil {
				logger.Printf("WARNING: %v", err)
				continue
			}
			if err := os.Symlink(rel, destPath); err != nil {
				if os.IsExist(err) {
					continue
				}
				return err
			}
			if err := maybeSetLinkMtime(destPath, header.ModTime); err != nil {
				return err
			}

			continue
		}

		r, err := gzip.NewReader(d.Data)
		if err != nil {
			return err
		}
		refs, err := writeManpage(logger, header.Name, destPath, r, m, gv.contentByPath)
		if err != nil {
			return err
		}
		if err := os.Chtimes(destPath, header.ModTime, header.ModTime); err != nil {
			return err
		}
		if err := r.Close(); err != nil {
			return err
		}

		for _, r := range refs {
			allRefs[r] = true
		}
	}

	// Extract all non-manpage files which were referenced via .so
	// statements, if any.
	if len(allRefs) > 0 {
		if _, err := tmp.Seek(0, os.SEEK_SET); err != nil {
			return err
		}

		d, err = deb.Load(tmp, p.filename)
		if err != nil {
			return err
		}
		for {
			header, err := d.Data.Next()
			if err == io.EOF {
				break
			}
			if err != nil {
				return err
			}

			if header.Typeflag != tar.TypeReg &&
				header.Typeflag != tar.TypeRegA &&
				header.Typeflag != tar.TypeSymlink {
				continue
			}

			if header.FileInfo().IsDir() {
				continue
			}

			if !allRefs[strings.TrimPrefix(header.Name, ".")] {
				continue
			}

			destPath := filepath.Join(*servingDir, p.suite, p.binarypkg, "aux", header.Name)
			logger.Printf("extracting referenced non-manpage file %q to %q", header.Name, destPath)
			if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
				return err
			}
			if err := writeAtomically(destPath, false, func(w io.Writer) error {
				_, err := io.Copy(w, d.Data)
				return err
			}); err != nil {
				return err
			}
		}
	}

	if err := ioutil.WriteFile(vPath, []byte(p.version.String()), 0644); err != nil {
		if os.IsNotExist(err) {
			// If the directory does not exist, we did not extract any
			// manpages. Since Contents files are not precise (they
			// might lag behind), this can happen occasionally.
			return nil
		}
		return fmt.Errorf("Writing version file %q: %v", err)
	}

	atomic.AddUint64(&gv.stats.PackagesExtracted, 1)

	return nil
}

func parallelDownload(ar *archive.Getter, gv globalView) error {
	eg, ctx := errgroup.WithContext(context.Background())
	downloadChan := make(chan pkgEntry)
	// TODO: flag for parallelism level
	for i := 0; i < 10; i++ {
		eg.Go(func() error {
			for p := range downloadChan {
				if err := downloadPkg(ar, p, gv); err != nil {
					return err
				}
			}
			return nil
		})
	}
	for _, p := range gv.pkgs {
		select {
		case downloadChan <- *p:
		case <-ctx.Done():
			break
		}
	}
	close(downloadChan)
	return eg.Wait()
}
<|endoftext|>


<commit_before>// Copyright (c) 2012-2014 Jeremy Latt
// Copyright (c) 2016 Daniel Oaks <daniel@danieloaks.net>
// released under the MIT license

package utils

import (
	"fmt"
	"net"
	"regexp"
	"strings"
)

var (
	// subnet mask for an ipv6 /128:
	mask128             = net.CIDRMask(128, 128)
	IPv4LoopbackAddress = net.ParseIP("127.0.0.1").To16()

	validHostnameLabelRegexp = regexp.MustCompile(`^[0-9A-Za-z.\-]+$`)
)

// AddrToIP returns the IP address for a net.Addr; unix domain sockets are treated as IPv4 loopback
func AddrToIP(addr net.Addr) net.IP {
	if tcpaddr, ok := addr.(*net.TCPAddr); ok {
		return tcpaddr.IP.To16()
	} else if _, ok := addr.(*net.UnixAddr); ok {
		return IPv4LoopbackAddress
	} else {
		return nil
	}
}

// IPStringToHostname converts a string representation of an IP address to an IRC-ready hostname
func IPStringToHostname(ipStr string) string {
	if 0 < len(ipStr) && ipStr[0] == ':' {
		// fix for IPv6 hostnames (so they don't start with a colon), same as all other IRCds
		ipStr = "0" + ipStr
	}
	return ipStr
}

// IsHostname returns whether we consider `name` a valid hostname.
func IsHostname(name string) bool {
	name = strings.TrimSuffix(name, ".")
	if len(name) < 1 || len(name) > 253 {
		return false
	}

	// ensure each part of hostname is valid
	for _, part := range strings.Split(name, ".") {
		if len(part) < 1 || len(part) > 63 || strings.HasPrefix(part, "-") || strings.HasSuffix(part, "-") {
			return false
		}
		if !validHostnameLabelRegexp.MatchString(part) {
			return false
		}
	}

	return true
}

// IsServerName returns whether we consider `name` a valid IRC server name.
func IsServerName(name string) bool {
	// IRC server names specifically require a period
	return IsHostname(name) && strings.IndexByte(name, '.') != -1
}

// Convenience to test whether `ip` is contained in any of `nets`.
func IPInNets(ip net.IP, nets []net.IPNet) bool {
	for _, network := range nets {
		if network.Contains(ip) {
			return true
		}
	}
	return false
}

// NormalizeIPToNet represents an address (v4 or v6) as the v6 /128 CIDR
// containing only it.
func NormalizeIPToNet(addr net.IP) (network net.IPNet) {
	// represent ipv4 addresses as ipv6 addresses, using the 4-in-6 prefix
	// (actually this should be a no-op for any address returned by ParseIP)
	addr = addr.To16()
	// the network corresponding to this address is now an ipv6 /128:
	return net.IPNet{
		IP:   addr,
		Mask: mask128,
	}
}

// NormalizeNet normalizes an IPNet to a v6 CIDR, using the 4-in-6 prefix.
// (this is like IP.To16(), but for IPNet instead of IP)
func NormalizeNet(network net.IPNet) (result net.IPNet) {
	if len(network.IP) == 16 {
		return network
	}
	ones, _ := network.Mask.Size()
	return net.IPNet{
		IP: network.IP.To16(),
		// include the 96 bits of the 4-in-6 prefix
		Mask: net.CIDRMask(96+ones, 128),
	}
}

// Given a network, produce a human-readable string
// (i.e., CIDR if it's actually a network, IPv6 address if it's a v6 /128,
// dotted quad if it's a v4 /32).
func NetToNormalizedString(network net.IPNet) string {
	ones, bits := network.Mask.Size()
	if ones == bits && ones == len(network.IP)*8 {
		// either a /32 or a /128, output the address:
		return network.IP.String()
	}
	return network.String()
}

// Parse a human-readable description (an address or CIDR, either v4 or v6)
// into a normalized v6 net.IPNet.
func NormalizedNetFromString(str string) (result net.IPNet, err error) {
	_, network, err := net.ParseCIDR(str)
	if err == nil {
		return NormalizeNet(*network), nil
	}
	ip := net.ParseIP(str)
	if ip == nil {
		err = &net.AddrError{
			Err:  "Couldn't interpret as either CIDR or address",
			Addr: str,
		}
		return
	}
	return NormalizeIPToNet(ip), nil
}

// Parse a list of IPs and nets as they would appear in one of our config
// files, e.g., proxy-allowed-from or a throttling exemption list.
func ParseNetList(netList []string) (nets []net.IPNet, err error) {
	var network net.IPNet
	for _, netStr := range netList {
		if netStr == "localhost" {
			ipv4Loopback, _ := NormalizedNetFromString("127.0.0.0/8")
			ipv6Loopback, _ := NormalizedNetFromString("::1/128")
			nets = append(nets, ipv4Loopback)
			nets = append(nets, ipv6Loopback)
			continue
		}
		network, err = NormalizedNetFromString(netStr)
		if err != nil {
			return
		} else {
			nets = append(nets, network)
		}
	}
	return
}

// Process the X-Forwarded-For header, validating against a list of trusted IPs.
// Returns the address that the request was forwarded for, or nil if no trustworthy
// data was available.
func HandleXForwardedFor(remoteAddr string, xForwardedFor string, whitelist []net.IPNet) (result net.IP) {
	// http.Request.RemoteAddr "has no defined format". with TCP it's typically "127.0.0.1:23784",
	// with unix domain it's typically "@"
	var remoteIP net.IP
	host, _, err := net.SplitHostPort(remoteAddr)
	if err != nil {
		remoteIP = IPv4LoopbackAddress
	} else {
		remoteIP = net.ParseIP(host)
	}

	if remoteIP == nil || !IPInNets(remoteIP, whitelist) {
		return remoteIP
	}

	// walk backwards through the X-Forwarded-For chain looking for an IP
	// that is *not* trusted. that means it was added by one of our trusted
	// forwarders (either remoteIP or something ahead of it in the chain)
	// and we can trust it:
	result = remoteIP
	forwardedIPs := strings.Split(xForwardedFor, ",")
	for i := len(forwardedIPs) - 1; i >= 0; i-- {
		proxiedIP := net.ParseIP(strings.TrimSpace(forwardedIPs[i]))
		if proxiedIP == nil {
			return
		} else if !IPInNets(proxiedIP, whitelist) {
			return proxiedIP
		} else {
			result = proxiedIP
		}
	}

	// no valid untrusted IPs were found in the chain;
	// return either the last valid and trusted IP (which must be the origin),
	// or nil:
	return
}

func DescribeConn(conn net.Conn) string {
	// XXX for unix domain sockets, this is not informative enough for an operator
	// to determine who holds the other side of the connection. there seems to be
	// no way to get either the correct file descriptor of the connection, or the
	// udiag_ino from `man 7 sock_diag`. maybe there's something else we can do?
	return fmt.Sprintf("%s <-> %s", conn.LocalAddr().String(), conn.RemoteAddr().String())
}
<commit_msg>Output unix socket credentials where applicable<commit_after>// Copyright (c) 2012-2014 Jeremy Latt
// Copyright (c) 2016 Daniel Oaks <daniel@danieloaks.net>
// released under the MIT license

package utils

import (
	"fmt"
	"net"
	"regexp"
	"strings"
	"syscall"
)

var (
	// subnet mask for an ipv6 /128:
	mask128             = net.CIDRMask(128, 128)
	IPv4LoopbackAddress = net.ParseIP("127.0.0.1").To16()

	validHostnameLabelRegexp = regexp.MustCompile(`^[0-9A-Za-z.\-]+$`)
)

// AddrToIP returns the IP address for a net.Addr; unix domain sockets are treated as IPv4 loopback
func AddrToIP(addr net.Addr) net.IP {
	if tcpaddr, ok := addr.(*net.TCPAddr); ok {
		return tcpaddr.IP.To16()
	} else if _, ok := addr.(*net.UnixAddr); ok {
		return IPv4LoopbackAddress
	} else {
		return nil
	}
}

// IPStringToHostname converts a string representation of an IP address to an IRC-ready hostname
func IPStringToHostname(ipStr string) string {
	if 0 < len(ipStr) && ipStr[0] == ':' {
		// fix for IPv6 hostnames (so they don't start with a colon), same as all other IRCds
		ipStr = "0" + ipStr
	}
	return ipStr
}

// IsHostname returns whether we consider `name` a valid hostname.
func IsHostname(name string) bool {
	name = strings.TrimSuffix(name, ".")
	if len(name) < 1 || len(name) > 253 {
		return false
	}

	// ensure each part of hostname is valid
	for _, part := range strings.Split(name, ".") {
		if len(part) < 1 || len(part) > 63 || strings.HasPrefix(part, "-") || strings.HasSuffix(part, "-") {
			return false
		}
		if !validHostnameLabelRegexp.MatchString(part) {
			return false
		}
	}

	return true
}

// IsServerName returns whether we consider `name` a valid IRC server name.
func IsServerName(name string) bool {
	// IRC server names specifically require a period
	return IsHostname(name) && strings.IndexByte(name, '.') != -1
}

// Convenience to test whether `ip` is contained in any of `nets`.
func IPInNets(ip net.IP, nets []net.IPNet) bool {
	for _, network := range nets {
		if network.Contains(ip) {
			return true
		}
	}
	return false
}

// NormalizeIPToNet represents an address (v4 or v6) as the v6 /128 CIDR
// containing only it.
func NormalizeIPToNet(addr net.IP) (network net.IPNet) {
	// represent ipv4 addresses as ipv6 addresses, using the 4-in-6 prefix
	// (actually this should be a no-op for any address returned by ParseIP)
	addr = addr.To16()
	// the network corresponding to this address is now an ipv6 /128:
	return net.IPNet{
		IP:   addr,
		Mask: mask128,
	}
}

// NormalizeNet normalizes an IPNet to a v6 CIDR, using the 4-in-6 prefix.
// (this is like IP.To16(), but for IPNet instead of IP)
func NormalizeNet(network net.IPNet) (result net.IPNet) {
	if len(network.IP) == 16 {
		return network
	}
	ones, _ := network.Mask.Size()
	return net.IPNet{
		IP: network.IP.To16(),
		// include the 96 bits of the 4-in-6 prefix
		Mask: net.CIDRMask(96+ones, 128),
	}
}

// Given a network, produce a human-readable string
// (i.e., CIDR if it's actually a network, IPv6 address if it's a v6 /128,
// dotted quad if it's a v4 /32).
func NetToNormalizedString(network net.IPNet) string {
	ones, bits := network.Mask.Size()
	if ones == bits && ones == len(network.IP)*8 {
		// either a /32 or a /128, output the address:
		return network.IP.String()
	}
	return network.String()
}

// Parse a human-readable description (an address or CIDR, either v4 or v6)
// into a normalized v6 net.IPNet.
func NormalizedNetFromString(str string) (result net.IPNet, err error) {
	_, network, err := net.ParseCIDR(str)
	if err == nil {
		return NormalizeNet(*network), nil
	}
	ip := net.ParseIP(str)
	if ip == nil {
		err = &net.AddrError{
			Err:  "Couldn't interpret as either CIDR or address",
			Addr: str,
		}
		return
	}
	return NormalizeIPToNet(ip), nil
}

// Parse a list of IPs and nets as they would appear in one of our config
// files, e.g., proxy-allowed-from or a throttling exemption list.
func ParseNetList(netList []string) (nets []net.IPNet, err error) {
	var network net.IPNet
	for _, netStr := range netList {
		if netStr == "localhost" {
			ipv4Loopback, _ := NormalizedNetFromString("127.0.0.0/8")
			ipv6Loopback, _ := NormalizedNetFromString("::1/128")
			nets = append(nets, ipv4Loopback)
			nets = append(nets, ipv6Loopback)
			continue
		}
		network, err = NormalizedNetFromString(netStr)
		if err != nil {
			return
		} else {
			nets = append(nets, network)
		}
	}
	return
}

// Process the X-Forwarded-For header, validating against a list of trusted IPs.
// Returns the address that the request was forwarded for, or nil if no trustworthy
// data was available.
func HandleXForwardedFor(remoteAddr string, xForwardedFor string, whitelist []net.IPNet) (result net.IP) {
	// http.Request.RemoteAddr "has no defined format". with TCP it's typically "127.0.0.1:23784",
	// with unix domain it's typically "@"
	var remoteIP net.IP
	host, _, err := net.SplitHostPort(remoteAddr)
	if err != nil {
		remoteIP = IPv4LoopbackAddress
	} else {
		remoteIP = net.ParseIP(host)
	}

	if remoteIP == nil || !IPInNets(remoteIP, whitelist) {
		return remoteIP
	}

	// walk backwards through the X-Forwarded-For chain looking for an IP
	// that is *not* trusted. that means it was added by one of our trusted
	// forwarders (either remoteIP or something ahead of it in the chain)
	// and we can trust it:
	result = remoteIP
	forwardedIPs := strings.Split(xForwardedFor, ",")
	for i := len(forwardedIPs) - 1; i >= 0; i-- {
		proxiedIP := net.ParseIP(strings.TrimSpace(forwardedIPs[i]))
		if proxiedIP == nil {
			return
		} else if !IPInNets(proxiedIP, whitelist) {
			return proxiedIP
		} else {
			result = proxiedIP
		}
	}

	// no valid untrusted IPs were found in the chain;
	// return either the last valid and trusted IP (which must be the origin),
	// or nil:
	return
}

// Output a description of a connection that can identify it to other systems
// administration tools.
func DescribeConn(c net.Conn) (description string) {
	description = "<error>"
	switch conn := c.(type) {
	case *net.UnixConn:
		f, err := conn.File()
		if err != nil {
			return
		}
		defer f.Close()
		ucred, err := syscall.GetsockoptUcred(int(f.Fd()), syscall.SOL_SOCKET, syscall.SO_PEERCRED)
		if err != nil {
			return
		}
		return fmt.Sprintf("%s <-> %s [pid=%d, uid=%d]", conn.LocalAddr().String(), conn.RemoteAddr().String(), ucred.Pid, ucred.Uid)
	default:
		// *net.TCPConn or *tls.Conn
		return fmt.Sprintf("%s <-> %s", conn.LocalAddr().String(), conn.RemoteAddr().String())
	}
}
<|endoftext|>