<commit_before>package exp

import (
	"bufio"
	"crypto/sha256"
	"fmt"
	"github.com/codegangsta/cli"
	"io"
	"io/ioutil"
	"os"
	"pfi/sensorbee/sensorbee/bql"
	"pfi/sensorbee/sensorbee/bql/parser"
	"pfi/sensorbee/sensorbee/bql/udf"
	"pfi/sensorbee/sensorbee/core"
	"pfi/sensorbee/sensorbee/data"
	"time"
)

func setUpRun() cli.Command {
	cmd := cli.Command{
		Name:   "run",
		Usage:  "run BQL statements with cache",
		Action: runRun,
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:  "remove-cache",
				Usage: "remove outdated cache files after execution is done",
			},
			cli.BoolFlag{
				Name:  "ignore-cache",
				Usage: "run all statements again by ignoring existing cache files",
			},
		},
	}

	// TODO: flags
	// --ignore-cache: run all statements again.
	// --from <hash>: invalidate cache of the statement having the hash and run it again.
	// --cache <name> (default .sensorbee-exp-cache)
	// --leave-intermediate-cache: create a cache file not only on the last node but also for intermediate nodes
	// --limit <num>: the maximum number of tuples generated from the source.
	return cmd
}

// TODO: write a note that all sources mustn't be rewindable and must be paused.
// They have to be stopped after they emit all tuples.

// TODO: support save/load of UDSs
//       UDSs need to be saved on each step

// TODO: add subcommand which deletes all caches except the latest ones.
// TODO: support annotation like "-- @something" in BQL statements (only at the beginning of each line)

// TODO: reduce number of files to be generated

const (
	cacheInfoFilename = ".sensorbee-exp-cache" // TODO: this should be an option
)

func runRun(c *cli.Context) {
	err := func() error {
		if a := c.Args(); len(a) != 1 {
			cli.ShowSubcommandHelp(c)
			os.Exit(1)
		}
		filename := c.Args()[0]

		body, err := ioutil.ReadFile(filename)
		if err != nil {
			return fmt.Errorf("cannot read BQL statements from %v: %v", filename, err)
		}

		cache, err := LoadCacheFromFile(cacheInfoFilename)
		if err != nil {
			return fmt.Errorf("cannot load cache information:%v", err)
		}

		stmts, err := Parse(string(body))
		if err != nil {
			return fmt.Errorf("cannot parse BQL statements:\n%v", err)
		}

		ith := -1
		if !c.Bool("ignore-cache") {
			ith = findCache(cache, stmts)
		}
		cache.TruncateAfter(ith)
		ith++
		for i := ith; i < len(stmts.Stmts); i++ {
			fmt.Println("Run:", stmts.Stmts[i].String())
			if err := execute(cache, stmts, i); err != nil {
				return err
			}
			if err := SaveCacheToFile(cache, cacheInfoFilename); err != nil {
				return err
			}
		}
		if c.Bool("remove-cache") {
			if err := removeCacheFiles(cache.Outdated); err != nil {
				return err
			}
			cache.Outdated = nil
			if err := SaveCacheToFile(cache, cacheInfoFilename); err != nil {
				return err
			}
		}
		fmt.Println("Done")
		return nil
	}()
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %v\n", err)
		os.Exit(1)
	}
}

func findCache(cache *Cache, stmts *Statements) int {
	if len(stmts.Stmts) == 0 {
		return -1
	}
	prevHash := cache.Seed
	for i, stmt := range stmts.Stmts {
		hash := hashStatement(prevHash, stmt)
		if !hasCache(cache, i, hash, stmt) {
			return i - 1
		}
		prevHash = hash
	}
	return len(stmts.Stmts) - 1
}

func hasCache(cache *Cache, ith int, hash string, stmt *Statement) bool {
	ent := cache.Get(ith)
	if ent == nil {
		return false
	}
	if ent.Hash != hash {
		return false
	}

	// Because STREAMs and INSERT INTOs may change the state of UDSs, all
	// states needs to be saved.
	if !(stmt.IsStream() || stmt.IsInsertStatement()) {
		// The statement doesn't have intermediate results.
		return true
	}

	// TODO: check cache information of all UDSs created so far

	if stmt.IsInsertStatement() {
		return true
	}

	_, err := os.Stat(CacheFilename(ent))
	if err != nil {
		return false
	}
	return true
}

func execute(cache *Cache, stmts *Statements, ith int) error {
	prevHash := ""
	if ith < 1 {
		prevHash = cache.Seed
	} else {
		prevHash = cache.Get(ith - 1).Hash
	}

	conf := &core.ContextConfig{}
	conf.Flags.DroppedTupleLog.Set(true)
	conf.Flags.DroppedTupleSummarization.Set(true)
	t := core.NewDefaultTopology(core.NewContext(conf), "sensorbee_exp")
	defer t.Stop()
	tb, err := bql.NewTopologyBuilder(t)
	if err != nil {
		return fmt.Errorf("cannot create a topology builder: %v", err)
	}

	curStmt := stmts.Stmts[ith]
	pausedSrcs, err := setUpTopology(tb, cache, stmts, ith)
	if err != nil {
		return err
	}

	resumeAll := func() error {
		for _, name := range pausedSrcs {
			s, err := tb.Topology().Source(name)
			if err != nil {
				return fmt.Errorf("cannot find the source %v: %v", name, err)
			}
			if _, ok := s.Source().(core.RewindableSource); ok {
				return fmt.Errorf(`rewindable source "%v" isn't supported`, name)
			}
			if err := s.Resume(); err != nil {
				return fmt.Errorf("cannot resume the source %v: %v", name, err)
			}
		}
		return nil
	}
	if n, err := tb.AddStmt(curStmt.Stmt); err != nil {
		return fmt.Errorf(`cannot add the statement "%v": %v`, curStmt, err)
	} else if n != nil && n.Type() == core.NTSource {
		pausedSrcs = append(pausedSrcs, n.Name())
	}

	hash := hashStatement(prevHash, curStmt)
	ent := &CacheEntry{
		NodeName:  curStmt.NodeName(),
		Hash:      hash,
		Stmt:      curStmt.String(),
		Timestamp: time.Now(),
	}
	if err := executeTarget(t, ent, curStmt, resumeAll); err != nil {
		return err
	}

	// Because state can be changed through various statements, they have to
	// be saved on each state execution.
	if err := saveStates(t, ent); err != nil {
		return err
	}
	cache.Add(ent)
	return nil
}

func setUpTopology(tb *bql.TopologyBuilder, cache *Cache, stmts *Statements, ith int) ([]string, error) {
	curStmt := stmts.Stmts[ith]
	inputs, err := curStmt.Input()
	if err != nil {
		return nil, fmt.Errorf(`cannot get input information of the statement "%v": %v`, curStmt, err)
	}
	latestEnt := cache.Get(ith - 1) // this can be nil

	if len(inputs) > 1 {
		// TODO: reduce number of steps that execute goes back due to JOIN.
		// Executing from the nearest common ancestor might work fine.
		// Because INSERT INTO or other statements may change UDSs, it isn't
		// a simple problem.

		// TODO: add option to perform JOIN using cache without thinking processing lag.

		var pausedSrcs []string
		for i := 0; i < ith; i++ {
			stmt := stmts.Stmts[i]
			if stmt.IsInsertStatement() {
				// Although INSERT INTO might change UDSs, UDSs are cached on
				// each execution of a statement. Therefore, they don't have to
				// be run again. However, it means a UDF which modify UDSs can
				// result in a confusing behavior due to duplicated evaluation.
				continue
			}

			if stmt.IsCreateStateStatement() {
				if err := setUpState(tb, latestEnt, stmt); err != nil {
					return nil, err
				}
				continue
			}

			if n, err := addStmt(tb, stmt); err != nil {
				return nil, err
			} else if n != nil && n.Type() == core.NTSource {
				pausedSrcs = append(pausedSrcs, n.Name())
			}
		}
		return pausedSrcs, nil
	}

	for i := 0; i < ith; i++ {
		stmt := stmts.Stmts[i]
		if stmt.IsDataSourceNodeQuery() || stmt.IsInsertStatement() {
			continue
		}

		if stmt.IsCreateStateStatement() {
			if err := setUpState(tb, latestEnt, stmt); err != nil {
				return nil, err
			}
			continue
		}

		if _, err := addStmt(tb, stmt); err != nil {
			return nil, err
		}
	}

	// len(inputs) <= 1 due to the condition above.
	if len(inputs) == 0 {
		return nil, nil
	}

	ent := cache.Node(inputs[0])
	if ent == nil {
		return nil, fmt.Errorf("The node %v doesn't have cache.\nPlease run all without cache again.", inputs[0])
	}
	s, err := newJSONLSource(CacheFilename(ent))
	if err != nil {
		return nil, fmt.Errorf("The node %v doesn't have cache.\nPlease run all without cache again.", ent.NodeName)
	}
	if _, err := tb.Topology().AddSource(ent.NodeName, s, &core.SourceConfig{
		PausedOnStartup: true,
	}); err != nil {
		return nil, fmt.Errorf("canoot add a source for a cache file: %v", err)
	}
	return []string{ent.NodeName}, nil
}

func addStmt(tb *bql.TopologyBuilder, stmt *Statement) (core.Node, error) {
	n, err := tb.AddStmt(stmt.Stmt)
	if err != nil {
		return nil, fmt.Errorf(`cannot add a statement "%v": %v`, stmt, err)
	}
	return n, nil
}

// setUpState tries to load a saved state from the latest cache.
func setUpState(tb *bql.TopologyBuilder, latestEnt *CacheEntry, stmt *Statement) error {
	if latestEnt == nil { // the first entry
		_, err := addStmt(tb, stmt)
		return err
	}

	name := string(stmt.Stmt.(parser.CreateStateStmt).Name)
	typeName := string(stmt.Stmt.(parser.CreateStateStmt).Type)
	for _, n := range latestEnt.States {
		if name != n {
			continue
		}

		c, err := tb.UDSCreators.Lookup(typeName)
		if err != nil {
			return fmt.Errorf("cannot find the creator of the UDS type '%v': %v", typeName, err)
		}
		l, ok := c.(udf.UDSLoader)
		if !ok {
			break
		}

		err = func() error {
			fn := StateCacheFilename(latestEnt, name)
			f, err := os.Open(fn)
			if err != nil {
				return fmt.Errorf("cannot open the cache file of the UDS '%v': %v", err)
			}
			defer f.Close()
			r := bufio.NewReader(f)
			s, err := l.LoadState(tb.Topology().Context(), r, data.Map{}) // TODO: support loading parameters
			if err != nil {
				return fmt.Errorf("cannot load the saved UDS '%v' from the cache: %v", name, err)
			}
			return tb.Topology().Context().SharedStates.Add(name, typeName, s)
		}()
		return err
	}
	_, err := addStmt(tb, stmt)
	return err
}

func executeTarget(t core.Topology, ent *CacheEntry, stmt *Statement, resumeAll func() error) error {
	switch stmt.Stmt.(type) {
	case parser.CreateSourceStmt, parser.CreateStreamAsSelectStmt, parser.CreateStreamAsSelectUnionStmt:
		sink, err := newJSONLSink(CacheFilename(ent))
		if err != nil {
			return fmt.Errorf("cannot create a cache file %v: %v", CacheFilename(ent), err)
		}
		sn, err := t.AddSink("sensorbee_exp_jsonl_sink", sink, nil)
		if err != nil {
			return fmt.Errorf("cannot add a sink for a cache file: %v", err)
		}
		if err := sn.Input(ent.NodeName, nil); err != nil {
			return fmt.Errorf("cannot add an input from %v to a sink for a cache file: %v", ent.NodeName, err)
		}
		sn.StopOnDisconnect()
		if err := resumeAll(); err != nil {
			return err
		}
		sn.State().Wait(core.TSStopped)

	case parser.InsertIntoFromStmt, parser.InsertIntoSelectStmt:
		if err := resumeAll(); err != nil {
			return err
		}
		s, err := t.Sink(ent.NodeName)
		if err != nil {
			return fmt.Errorf("cannot find the sink %v: %v", ent.NodeName, err)
		}
		s.StopOnDisconnect()
		s.State().Wait(core.TSStopped)
	}
	return nil
}

func saveStates(t core.Topology, ent *CacheEntry) error {
	states, err := t.Context().SharedStates.List()
	if err != nil {
		return fmt.Errorf("cannot list created UDSs: %v", err)
	}
	for name, state := range states {
		s, ok := state.(core.SavableSharedState)
		if !ok {
			continue
		}

		err := func() error {
			fn := StateCacheFilename(ent, name)
			f, err := os.OpenFile(fn, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
			if err != nil {
				return fmt.Errorf("cannot create a cache file for a UDS '%v': %v", fn, err)
			}
			defer f.Close()

			w := bufio.NewWriter(f)
			if err := s.Save(t.Context(), w, data.Map{}); err != nil {
				return err
			}
			return w.Flush()
		}()
		if err != nil {
			return err
		}
		ent.States = append(ent.States, name)
	}
	return nil
}

func hashStatement(prev string, stmt *Statement) string {
	// TODO: compute hash taking dependencies into account so that cache can
	// tolerate reordering of statements. Be careful of some statements like
	// UPDATE which may have hidden effect on later execution.
	h := sha256.New()
	io.WriteString(h, prev)
	io.WriteString(h, stmt.String())
	return fmt.Sprintf("%x", h.Sum(nil))
}
<commit_msg>change error chack position<commit_after>package exp

import (
	"bufio"
	"crypto/sha256"
	"fmt"
	"github.com/codegangsta/cli"
	"io"
	"io/ioutil"
	"os"
	"pfi/sensorbee/sensorbee/bql"
	"pfi/sensorbee/sensorbee/bql/parser"
	"pfi/sensorbee/sensorbee/bql/udf"
	"pfi/sensorbee/sensorbee/core"
	"pfi/sensorbee/sensorbee/data"
	"time"
)

func setUpRun() cli.Command {
	cmd := cli.Command{
		Name:   "run",
		Usage:  "run BQL statements with cache",
		Action: runRun,
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:  "remove-cache",
				Usage: "remove outdated cache files after execution is done",
			},
			cli.BoolFlag{
				Name:  "ignore-cache",
				Usage: "run all statements again by ignoring existing cache files",
			},
		},
	}

	// TODO: flags
	// --ignore-cache: run all statements again.
	// --from <hash>: invalidate cache of the statement having the hash and run it again.
	// --cache <name> (default .sensorbee-exp-cache)
	// --leave-intermediate-cache: create a cache file not only on the last node but also for intermediate nodes
	// --limit <num>: the maximum number of tuples generated from the source.
	return cmd
}

// TODO: write a note that all sources mustn't be rewindable and must be paused.
// They have to be stopped after they emit all tuples.

// TODO: support save/load of UDSs
//       UDSs need to be saved on each step

// TODO: add subcommand which deletes all caches except the latest ones.
// TODO: support annotation like "-- @something" in BQL statements (only at the beginning of each line)

// TODO: reduce number of files to be generated

const (
	cacheInfoFilename = ".sensorbee-exp-cache" // TODO: this should be an option
)

func runRun(c *cli.Context) {
	err := func() error {
		if a := c.Args(); len(a) != 1 {
			cli.ShowSubcommandHelp(c)
			os.Exit(1)
		}
		filename := c.Args()[0]

		body, err := ioutil.ReadFile(filename)
		if err != nil {
			return fmt.Errorf("cannot read BQL statements from %v: %v", filename, err)
		}

		cache, err := LoadCacheFromFile(cacheInfoFilename)
		if err != nil {
			return fmt.Errorf("cannot load cache information:%v", err)
		}

		stmts, err := Parse(string(body))
		if err != nil {
			return fmt.Errorf("cannot parse BQL statements:\n%v", err)
		}

		ith := -1
		if !c.Bool("ignore-cache") {
			ith = findCache(cache, stmts)
		}
		cache.TruncateAfter(ith)
		ith++
		for i := ith; i < len(stmts.Stmts); i++ {
			fmt.Println("Run:", stmts.Stmts[i].String())
			if err := execute(cache, stmts, i); err != nil {
				return err
			}
			if err := SaveCacheToFile(cache, cacheInfoFilename); err != nil {
				return err
			}
		}
		if c.Bool("remove-cache") {
			if err := removeCacheFiles(cache.Outdated); err != nil {
				return err
			}
			cache.Outdated = nil
			if err := SaveCacheToFile(cache, cacheInfoFilename); err != nil {
				return err
			}
		}
		fmt.Println("Done")
		return nil
	}()
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %v\n", err)
		os.Exit(1)
	}
}

func findCache(cache *Cache, stmts *Statements) int {
	if len(stmts.Stmts) == 0 {
		return -1
	}
	prevHash := cache.Seed
	for i, stmt := range stmts.Stmts {
		hash := hashStatement(prevHash, stmt)
		if !hasCache(cache, i, hash, stmt) {
			return i - 1
		}
		prevHash = hash
	}
	return len(stmts.Stmts) - 1
}

func hasCache(cache *Cache, ith int, hash string, stmt *Statement) bool {
	ent := cache.Get(ith)
	if ent == nil {
		return false
	}
	if ent.Hash != hash {
		return false
	}

	// Because STREAMs and INSERT INTOs may change the state of UDSs, all
	// states needs to be saved.
	if !(stmt.IsStream() || stmt.IsInsertStatement()) {
		// The statement doesn't have intermediate results.
		return true
	}

	// TODO: check cache information of all UDSs created so far

	if stmt.IsInsertStatement() {
		return true
	}

	_, err := os.Stat(CacheFilename(ent))
	if err != nil {
		return false
	}
	return true
}

func execute(cache *Cache, stmts *Statements, ith int) error {
	prevHash := ""
	if ith < 1 {
		prevHash = cache.Seed
	} else {
		prevHash = cache.Get(ith - 1).Hash
	}

	conf := &core.ContextConfig{}
	conf.Flags.DroppedTupleLog.Set(true)
	conf.Flags.DroppedTupleSummarization.Set(true)
	t := core.NewDefaultTopology(core.NewContext(conf), "sensorbee_exp")
	defer t.Stop()
	tb, err := bql.NewTopologyBuilder(t)
	if err != nil {
		return fmt.Errorf("cannot create a topology builder: %v", err)
	}

	curStmt := stmts.Stmts[ith]
	pausedSrcs, err := setUpTopology(tb, cache, stmts, ith)
	if err != nil {
		return err
	}

	resumeAll := func() error {
		for _, name := range pausedSrcs {
			s, err := tb.Topology().Source(name)
			if err != nil {
				return fmt.Errorf("cannot find the source %v: %v", name, err)
			}
			if err := s.Resume(); err != nil {
				return fmt.Errorf("cannot resume the source %v: %v", name, err)
			}
		}
		return nil
	}
	if n, err := tb.AddStmt(curStmt.Stmt); err != nil {
		return fmt.Errorf(`cannot add the statement "%v": %v`, curStmt, err)
	} else if n != nil && n.Type() == core.NTSource {
		sn, _ := n.(core.SourceNode)
		if _, ok := sn.Source().(core.RewindableSource); ok {
			return fmt.Errorf(`rewindable source "%v" isn't supported`, n.Name())
		}
		pausedSrcs = append(pausedSrcs, n.Name())
	}

	hash := hashStatement(prevHash, curStmt)
	ent := &CacheEntry{
		NodeName:  curStmt.NodeName(),
		Hash:      hash,
		Stmt:      curStmt.String(),
		Timestamp: time.Now(),
	}
	if err := executeTarget(t, ent, curStmt, resumeAll); err != nil {
		return err
	}

	// Because state can be changed through various statements, they have to
	// be saved on each state execution.
	if err := saveStates(t, ent); err != nil {
		return err
	}
	cache.Add(ent)
	return nil
}

func setUpTopology(tb *bql.TopologyBuilder, cache *Cache, stmts *Statements, ith int) ([]string, error) {
	curStmt := stmts.Stmts[ith]
	inputs, err := curStmt.Input()
	if err != nil {
		return nil, fmt.Errorf(`cannot get input information of the statement "%v": %v`, curStmt, err)
	}
	latestEnt := cache.Get(ith - 1) // this can be nil

	if len(inputs) > 1 {
		// TODO: reduce number of steps that execute goes back due to JOIN.
		// Executing from the nearest common ancestor might work fine.
		// Because INSERT INTO or other statements may change UDSs, it isn't
		// a simple problem.

		// TODO: add option to perform JOIN using cache without thinking processing lag.

		var pausedSrcs []string
		for i := 0; i < ith; i++ {
			stmt := stmts.Stmts[i]
			if stmt.IsInsertStatement() {
				// Although INSERT INTO might change UDSs, UDSs are cached on
				// each execution of a statement. Therefore, they don't have to
				// be run again. However, it means a UDF which modify UDSs can
				// result in a confusing behavior due to duplicated evaluation.
				continue
			}

			if stmt.IsCreateStateStatement() {
				if err := setUpState(tb, latestEnt, stmt); err != nil {
					return nil, err
				}
				continue
			}

			if n, err := addStmt(tb, stmt); err != nil {
				return nil, err
			} else if n != nil && n.Type() == core.NTSource {
				pausedSrcs = append(pausedSrcs, n.Name())
			}
		}
		return pausedSrcs, nil
	}

	for i := 0; i < ith; i++ {
		stmt := stmts.Stmts[i]
		if stmt.IsDataSourceNodeQuery() || stmt.IsInsertStatement() {
			continue
		}

		if stmt.IsCreateStateStatement() {
			if err := setUpState(tb, latestEnt, stmt); err != nil {
				return nil, err
			}
			continue
		}

		if _, err := addStmt(tb, stmt); err != nil {
			return nil, err
		}
	}

	// len(inputs) <= 1 due to the condition above.
	if len(inputs) == 0 {
		return nil, nil
	}

	ent := cache.Node(inputs[0])
	if ent == nil {
		return nil, fmt.Errorf("The node %v doesn't have cache.\nPlease run all without cache again.", inputs[0])
	}
	s, err := newJSONLSource(CacheFilename(ent))
	if err != nil {
		return nil, fmt.Errorf("The node %v doesn't have cache.\nPlease run all without cache again.", ent.NodeName)
	}
	if _, err := tb.Topology().AddSource(ent.NodeName, s, &core.SourceConfig{
		PausedOnStartup: true,
	}); err != nil {
		return nil, fmt.Errorf("canoot add a source for a cache file: %v", err)
	}
	return []string{ent.NodeName}, nil
}

func addStmt(tb *bql.TopologyBuilder, stmt *Statement) (core.Node, error) {
	n, err := tb.AddStmt(stmt.Stmt)
	if err != nil {
		return nil, fmt.Errorf(`cannot add a statement "%v": %v`, stmt, err)
	}
	return n, nil
}

// setUpState tries to load a saved state from the latest cache.
func setUpState(tb *bql.TopologyBuilder, latestEnt *CacheEntry, stmt *Statement) error {
	if latestEnt == nil { // the first entry
		_, err := addStmt(tb, stmt)
		return err
	}

	name := string(stmt.Stmt.(parser.CreateStateStmt).Name)
	typeName := string(stmt.Stmt.(parser.CreateStateStmt).Type)
	for _, n := range latestEnt.States {
		if name != n {
			continue
		}

		c, err := tb.UDSCreators.Lookup(typeName)
		if err != nil {
			return fmt.Errorf("cannot find the creator of the UDS type '%v': %v", typeName, err)
		}
		l, ok := c.(udf.UDSLoader)
		if !ok {
			break
		}

		err = func() error {
			fn := StateCacheFilename(latestEnt, name)
			f, err := os.Open(fn)
			if err != nil {
				return fmt.Errorf("cannot open the cache file of the UDS '%v': %v", err)
			}
			defer f.Close()
			r := bufio.NewReader(f)
			s, err := l.LoadState(tb.Topology().Context(), r, data.Map{}) // TODO: support loading parameters
			if err != nil {
				return fmt.Errorf("cannot load the saved UDS '%v' from the cache: %v", name, err)
			}
			return tb.Topology().Context().SharedStates.Add(name, typeName, s)
		}()
		return err
	}
	_, err := addStmt(tb, stmt)
	return err
}

func executeTarget(t core.Topology, ent *CacheEntry, stmt *Statement, resumeAll func() error) error {
	switch stmt.Stmt.(type) {
	case parser.CreateSourceStmt, parser.CreateStreamAsSelectStmt, parser.CreateStreamAsSelectUnionStmt:
		sink, err := newJSONLSink(CacheFilename(ent))
		if err != nil {
			return fmt.Errorf("cannot create a cache file %v: %v", CacheFilename(ent), err)
		}
		sn, err := t.AddSink("sensorbee_exp_jsonl_sink", sink, nil)
		if err != nil {
			return fmt.Errorf("cannot add a sink for a cache file: %v", err)
		}
		if err := sn.Input(ent.NodeName, nil); err != nil {
			return fmt.Errorf("cannot add an input from %v to a sink for a cache file: %v", ent.NodeName, err)
		}
		sn.StopOnDisconnect()
		if err := resumeAll(); err != nil {
			return err
		}
		sn.State().Wait(core.TSStopped)

	case parser.InsertIntoFromStmt, parser.InsertIntoSelectStmt:
		if err := resumeAll(); err != nil {
			return err
		}
		s, err := t.Sink(ent.NodeName)
		if err != nil {
			return fmt.Errorf("cannot find the sink %v: %v", ent.NodeName, err)
		}
		s.StopOnDisconnect()
		s.State().Wait(core.TSStopped)
	}
	return nil
}

func saveStates(t core.Topology, ent *CacheEntry) error {
	states, err := t.Context().SharedStates.List()
	if err != nil {
		return fmt.Errorf("cannot list created UDSs: %v", err)
	}
	for name, state := range states {
		s, ok := state.(core.SavableSharedState)
		if !ok {
			continue
		}

		err := func() error {
			fn := StateCacheFilename(ent, name)
			f, err := os.OpenFile(fn, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
			if err != nil {
				return fmt.Errorf("cannot create a cache file for a UDS '%v': %v", fn, err)
			}
			defer f.Close()

			w := bufio.NewWriter(f)
			if err := s.Save(t.Context(), w, data.Map{}); err != nil {
				return err
			}
			return w.Flush()
		}()
		if err != nil {
			return err
		}
		ent.States = append(ent.States, name)
	}
	return nil
}

func hashStatement(prev string, stmt *Statement) string {
	// TODO: compute hash taking dependencies into account so that cache can
	// tolerate reordering of statements. Be careful of some statements like
	// UPDATE which may have hidden effect on later execution.
	h := sha256.New()
	io.WriteString(h, prev)
	io.WriteString(h, stmt.String())
	return fmt.Sprintf("%x", h.Sum(nil))
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/gob"
	"github.com/Symantec/Dominator/lib/mdb"
	"github.com/Symantec/Dominator/lib/srpc"
	"github.com/Symantec/Dominator/proto/mdbserver"
	"io"
	"log"
	"sync"
)

type rpcType struct {
	currentMdb *mdb.Mdb
	logger     *log.Logger
	rwMutex    sync.RWMutex
	// Protected by lock.
	updateChannels map[*srpc.Conn]chan<- mdbserver.MdbUpdate
}

func startRpcd(logger *log.Logger) *rpcType {
	rpcObj := &rpcType{
		currentMdb:     &mdb.Mdb{},
		logger:         logger,
		updateChannels: make(map[*srpc.Conn]chan<- mdbserver.MdbUpdate),
	}
	srpc.RegisterName("MdbServer", rpcObj)
	return rpcObj
}

func (t *rpcType) GetMdbUpdates(conn *srpc.Conn) error {
	encoder := gob.NewEncoder(conn)
	updateChannel := make(chan mdbserver.MdbUpdate, 1)
	t.rwMutex.Lock()
	t.updateChannels[conn] = updateChannel
	t.rwMutex.Unlock()
	mdbUpdate := mdbserver.MdbUpdate{MachinesToAdd: t.currentMdb.Machines}
	if err := encoder.Encode(mdbUpdate); err != nil {
		return nil
	}
	if err := conn.Flush(); err != nil {
		return nil
	}
	closeChannel := getCloseNotifier(conn)
	for {
		var err error
		select {
		case mdbUpdate := <-updateChannel:
			if err = encoder.Encode(mdbUpdate); err != nil {
				break
			}
			if err = conn.Flush(); err != nil {
				break
			}
		case err = <-closeChannel:
			break
		}
		if err != nil {
			t.rwMutex.Lock()
			delete(t.updateChannels, conn)
			t.rwMutex.Unlock()
			if err != io.EOF {
				t.logger.Println(err)
				return err
			} else {
				return nil
			}
		}
	}
}

func (t *rpcType) pushUpdateToAll(old, new *mdb.Mdb) {
	t.currentMdb = new
	t.rwMutex.RLock()
	numConnections := len(t.updateChannels)
	t.rwMutex.RUnlock()
	if numConnections < 1 {
		return
	}
	mdbUpdate := mdbserver.MdbUpdate{}
	if old == nil {
		old = &mdb.Mdb{}
	}
	oldMachines := make(map[string]mdb.Machine, len(old.Machines))
	for _, machine := range old.Machines {
		oldMachines[machine.Hostname] = machine
	}
	for _, newMachine := range new.Machines {
		if oldMachine, ok := oldMachines[newMachine.Hostname]; ok {
			if newMachine != oldMachine {
				mdbUpdate.MachinesToUpdate = append(mdbUpdate.MachinesToUpdate,
					newMachine)
			}
		} else {
			mdbUpdate.MachinesToAdd = append(mdbUpdate.MachinesToAdd,
				newMachine)
		}
	}
	for _, machine := range new.Machines {
		delete(oldMachines, machine.Hostname)
	}
	for name := range oldMachines {
		mdbUpdate.MachinesToDelete = append(mdbUpdate.MachinesToDelete, name)
	}
	t.rwMutex.RLock()
	defer t.rwMutex.RUnlock()
	for _, channel := range t.updateChannels {
		channel <- mdbUpdate
	}
}

func getCloseNotifier(conn *srpc.Conn) <-chan error {
	closeChannel := make(chan error)
	go func() {
		for {
			buf := make([]byte, 1)
			if _, err := conn.Read(buf); err != nil {
				closeChannel <- err
				return
			}
		}
	}()
	return closeChannel
}
<commit_msg>Fix potential mdbd deadlock and do not send initial empty MDB to clients.<commit_after>package main

import (
	"encoding/gob"
	"github.com/Symantec/Dominator/lib/mdb"
	"github.com/Symantec/Dominator/lib/srpc"
	"github.com/Symantec/Dominator/proto/mdbserver"
	"io"
	"log"
	"sync"
)

type rpcType struct {
	currentMdb *mdb.Mdb
	logger     *log.Logger
	rwMutex    sync.RWMutex
	// Protected by lock.
	updateChannels map[*srpc.Conn]chan<- mdbserver.MdbUpdate
}

func startRpcd(logger *log.Logger) *rpcType {
	rpcObj := &rpcType{
		logger:         logger,
		updateChannels: make(map[*srpc.Conn]chan<- mdbserver.MdbUpdate),
	}
	srpc.RegisterName("MdbServer", rpcObj)
	return rpcObj
}

func (t *rpcType) GetMdbUpdates(conn *srpc.Conn) error {
	encoder := gob.NewEncoder(conn)
	updateChannel := make(chan mdbserver.MdbUpdate, 1)
	t.rwMutex.Lock()
	t.updateChannels[conn] = updateChannel
	t.rwMutex.Unlock()
	defer func() {
		close(updateChannel)
		t.rwMutex.Lock()
		delete(t.updateChannels, conn)
		t.rwMutex.Unlock()
	}()
	if t.currentMdb != nil {
		mdbUpdate := mdbserver.MdbUpdate{MachinesToAdd: t.currentMdb.Machines}
		if err := encoder.Encode(mdbUpdate); err != nil {
			return err
		}
		if err := conn.Flush(); err != nil {
			return err
		}
	}
	closeChannel := getCloseNotifier(conn)
	for {
		var err error
		select {
		case mdbUpdate := <-updateChannel:
			if err = encoder.Encode(mdbUpdate); err != nil {
				break
			}
			if err = conn.Flush(); err != nil {
				break
			}
		case err = <-closeChannel:
			break
		}
		if err != nil {
			if err != io.EOF {
				t.logger.Println(err)
				return err
			} else {
				return nil
			}
		}
	}
}

func (t *rpcType) pushUpdateToAll(old, new *mdb.Mdb) {
	t.currentMdb = new
	updateChannels := t.getUpdateChannels()
	if len(updateChannels) < 1 {
		return
	}
	mdbUpdate := mdbserver.MdbUpdate{}
	if old == nil {
		old = &mdb.Mdb{}
	}
	oldMachines := make(map[string]mdb.Machine, len(old.Machines))
	for _, machine := range old.Machines {
		oldMachines[machine.Hostname] = machine
	}
	for _, newMachine := range new.Machines {
		if oldMachine, ok := oldMachines[newMachine.Hostname]; ok {
			if newMachine != oldMachine {
				mdbUpdate.MachinesToUpdate = append(mdbUpdate.MachinesToUpdate,
					newMachine)
			}
		} else {
			mdbUpdate.MachinesToAdd = append(mdbUpdate.MachinesToAdd,
				newMachine)
		}
	}
	for _, machine := range new.Machines {
		delete(oldMachines, machine.Hostname)
	}
	for name := range oldMachines {
		mdbUpdate.MachinesToDelete = append(mdbUpdate.MachinesToDelete, name)
	}
	if isEmptyUpdate(mdbUpdate) {
		t.logger.Println("Ignoring empty update")
		return
	}
	for _, channel := range updateChannels {
		sendUpdate(channel, mdbUpdate)
	}
}

func (t *rpcType) getUpdateChannels() []chan<- mdbserver.MdbUpdate {
	t.rwMutex.RLock()
	defer t.rwMutex.RUnlock()
	channels := make([]chan<- mdbserver.MdbUpdate, 0, len(t.updateChannels))
	for _, channel := range t.updateChannels {
		channels = append(channels, channel)
	}
	return channels
}

func isEmptyUpdate(mdbUpdate mdbserver.MdbUpdate) bool {
	if len(mdbUpdate.MachinesToAdd) > 0 {
		return false
	}
	if len(mdbUpdate.MachinesToUpdate) > 0 {
		return false
	}
	if len(mdbUpdate.MachinesToDelete) > 0 {
		return false
	}
	return true
}

func sendUpdate(channel chan<- mdbserver.MdbUpdate,
	mdbUpdate mdbserver.MdbUpdate) {
	defer func() { recover() }()
	channel <- mdbUpdate
}

func getCloseNotifier(conn *srpc.Conn) <-chan error {
	closeChannel := make(chan error)
	go func() {
		for {
			buf := make([]byte, 1)
			if _, err := conn.Read(buf); err != nil {
				closeChannel <- err
				return
			}
		}
	}()
	return closeChannel
}
<|endoftext|>


<commit_before>// mdfmt formats Markdown content containing frontmatter.
//
// Based on https://github.com/moorereason/mdfmt
// Released under the MIT License.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/scanner"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/spf13/hugo/parser"
)

var (
	// Main operation modes.
	list   = flag.Bool("l", false, "list files whose formatting differs from mdfmt's")
	write  = flag.Bool("w", false, "write result to (source) file instead of stdout")
	doDiff = flag.Bool("d", false, "display diffs instead of rewriting files")

	exitCode = 0
)

func report(err error) {
	scanner.PrintError(os.Stderr, err)
	exitCode = 2
}

func usage() {
	fmt.Fprintf(os.Stderr, "usage: mdfmt [flags] [path ...]\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func isMarkdownFile(f os.FileInfo) bool {
	name := f.Name()
	ext := filepath.Ext(name)
	return !f.IsDir() && !strings.HasPrefix(name, ".") && (ext == "md" || ext == "markdown")
}

func run(command []string, content []byte) ([]byte, error) {
	cmd := exec.Command(command[0], command[1:]...)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}
	go func() {
		defer stdin.Close()
		stdin.Write(content)
	}()
	out, err := cmd.CombinedOutput()
	if err != nil {
		fmt.Fprintln(os.Stderr, out)
		return nil, err
	}
	return bytes.TrimSpace(out), nil
}

func pandocProcess(content []byte) ([]byte, error) {
	out, err := run([]string{
		"pandoc", "-t", "html",
	}, content)
	if err != nil {
		return nil, err
	}
	out = bytes.Replace(out, []byte(""), []byte("☢"), -1)
	out = bytes.Replace(out, []byte(""), []byte("☢"), -1)
	out, err = run([]string{
		"pandoc", "-f", "html", "-t", "markdown", "--reference-links",
	}, out)
	if err != nil {
		return nil, err
	}
	out = bytes.Replace(out, []byte("☢"), []byte("_"), -1)
	return out, nil
}

func processFile(filename string, in io.ReadSeeker, out io.Writer, stdin bool) error {
	if in == nil {
		f, err := os.Open(filename)
		if err != nil {
			return err
		}
		defer f.Close()
		in = f
	}

	// slurp in the whole file for comparison later
	src, err := ioutil.ReadAll(in)
	if err != nil {
		return err
	}
	in.Seek(0, 0)

	// parse the file with hugo/parser to extract front matter
	page, err := parser.ReadFrom(in)
	if err != nil {
		return err
	}

	md, err := pandocProcess(page.Content())
	if err != nil {
		return err
	}

	// If we have front matter, insert a newline to separate the front matter
	// from the markdown content.
	sep := ""
	if len(page.FrontMatter()) > 0 {
		sep = "\n"
	}

	res := make([]byte, len(page.FrontMatter())+len(sep)+len(md)+1)
	copy(res, append(append(append(page.FrontMatter(), []byte(sep)...), md...), '\n'))

	if !bytes.Equal(src, res) {
		// formatting has changed
		if *list {
			fmt.Fprintln(out, filename)
		}
		if *write {
			err = ioutil.WriteFile(filename, res, 0)
			if err != nil {
				return err
			}
		}
		if *doDiff {
			data, err := diff(src, res)
			if err != nil {
				return fmt.Errorf("computing diff: %s", err)
			}
			fmt.Printf("diff %s mdfmt/%s\n", filename, filename)
			out.Write(data)
		}
	}

	if !*list && !*write && !*doDiff {
		_, err = out.Write(res)
	}

	return err
}

func visitFile(path string, f os.FileInfo, err error) error {
	if err == nil && isMarkdownFile(f) {
		err = processFile(path, nil, os.Stdout, false)
	}
	if err != nil {
		report(err)
	}
	return nil
}

func walkDir(path string) {
	filepath.Walk(path, visitFile)
}

func main() {
	// call mdfmtMain in a separate function
	// so that it can use defer and have them
	// run before the exit.
	mdfmtMain()
	os.Exit(exitCode)
}

func mdfmtMain() {
	flag.Usage = usage
	flag.Parse()

	if flag.NArg() == 0 {
		if err := processFile("<standard input>", os.Stdin, os.Stdout, true); err != nil {
			report(err)
		}
		return
	}

	for i := 0; i < flag.NArg(); i++ {
		path := flag.Arg(i)
		switch dir, err := os.Stat(path); {
		case err != nil:
			report(err)
		case dir.IsDir():
			walkDir(path)
		default:
			if err := processFile(path, nil, os.Stdout, false); err != nil {
				report(err)
			}
		}
	}
}

func diff(b1, b2 []byte) (data []byte, err error) {
	f1, err := ioutil.TempFile("", "mdfmt")
	if err != nil {
		return
	}
	defer os.Remove(f1.Name())
	defer f1.Close()

	f2, err := ioutil.TempFile("", "mdfmt")
	if err != nil {
		return
	}
	defer os.Remove(f2.Name())
	defer f2.Close()

	f1.Write(b1)
	f2.Write(b2)

	data, err = exec.Command("diff", "-u", f1.Name(), f2.Name()).CombinedOutput()
	if len(data) > 0 {
		// diff exits with a non-zero status when the files don't match.
		// Ignore that failure as long as we get output.
		err = nil
	}
	return
}
<commit_msg>mdfmt: add new option -i<commit_after>// mdfmt formats Markdown content containing frontmatter.
//
// Based on https://github.com/moorereason/mdfmt
// Released under the MIT License.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/scanner"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/spf13/hugo/parser"
)

var (
	// Main operation modes.
	list        = flag.Bool("l", false, "list files whose formatting differs from mdfmt's")
	write       = flag.Bool("w", false, "write result to (source) file instead of stdout")
	doDiff      = flag.Bool("d", false, "display diffs instead of rewriting files")
	inlineLinks = flag.Bool("i", false, "use inline links, rather than reference-style links")

	exitCode = 0
)

func report(err error) {
	scanner.PrintError(os.Stderr, err)
	exitCode = 2
}

func usage() {
	fmt.Fprintf(os.Stderr, "usage: mdfmt [flags] [path ...]\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func isMarkdownFile(f os.FileInfo) bool {
	name := f.Name()
	ext := filepath.Ext(name)
	return !f.IsDir() && !strings.HasPrefix(name, ".") && (ext == "md" || ext == "markdown")
}

func run(command []string, content []byte) ([]byte, error) {
	cmd := exec.Command(command[0], command[1:]...)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}
	go func() {
		defer stdin.Close()
		stdin.Write(content)
	}()
	out, err := cmd.CombinedOutput()
	if err != nil {
		fmt.Fprintln(os.Stderr, out)
		return nil, err
	}
	return bytes.TrimSpace(out), nil
}

func pandocProcess(content []byte) ([]byte, error) {
	out, err := run([]string{
		"pandoc", "-t", "html",
	}, content)
	if err != nil {
		return nil, err
	}
	out = bytes.Replace(out, []byte(""), []byte("☢"), -1)
	out = bytes.Replace(out, []byte(""), []byte("☢"), -1)
	args := []string{
		"pandoc", "-f", "html", "-t", "markdown",
	}
	if !*inlineLinks {
		args = append(args, "--reference-links")
	}
	out, err = run(args, out)
	if err != nil {
		return nil, err
	}
	out = bytes.Replace(out, []byte("☢"), []byte("_"), -1)
	return out, nil
}

func processFile(filename string, in io.ReadSeeker, out io.Writer, stdin bool) error {
	if in == nil {
		f, err := os.Open(filename)
		if err != nil {
			return err
		}
		defer f.Close()
		in = f
	}

	// slurp in the whole file for comparison later
	src, err := ioutil.ReadAll(in)
	if err != nil {
		return err
	}
	in.Seek(0, 0)

	// parse the file with hugo/parser to extract front matter
	page, err := parser.ReadFrom(in)
	if err != nil {
		return err
	}

	md, err := pandocProcess(page.Content())
	if err != nil {
		return err
	}

	// If we have front matter, insert a newline to separate the front matter
	// from the markdown content.
	sep := ""
	if len(page.FrontMatter()) > 0 {
		sep = "\n"
	}

	res := make([]byte, len(page.FrontMatter())+len(sep)+len(md)+1)
	copy(res, append(append(append(page.FrontMatter(), []byte(sep)...), md...), '\n'))

	if !bytes.Equal(src, res) {
		// formatting has changed
		if *list {
			fmt.Fprintln(out, filename)
		}
		if *write {
			err = ioutil.WriteFile(filename, res, 0)
			if err != nil {
				return err
			}
		}
		if *doDiff {
			data, err := diff(src, res)
			if err != nil {
				return fmt.Errorf("computing diff: %s", err)
			}
			fmt.Printf("diff %s mdfmt/%s\n", filename, filename)
			out.Write(data)
		}
	}

	if !*list && !*write && !*doDiff {
		_, err = out.Write(res)
	}

	return err
}

func visitFile(path string, f os.FileInfo, err error) error {
	if err == nil && isMarkdownFile(f) {
		err = processFile(path, nil, os.Stdout, false)
	}
	if err != nil {
		report(err)
	}
	return nil
}

func walkDir(path string) {
	filepath.Walk(path, visitFile)
}

func main() {
	// call mdfmtMain in a separate function
	// so that it can use defer and have them
	// run before the exit.
	mdfmtMain()
	os.Exit(exitCode)
}

func mdfmtMain() {
	flag.Usage = usage
	flag.Parse()

	if flag.NArg() == 0 {
		if err := processFile("<standard input>", os.Stdin, os.Stdout, true); err != nil {
			report(err)
		}
		return
	}

	for i := 0; i < flag.NArg(); i++ {
		path := flag.Arg(i)
		switch dir, err := os.Stat(path); {
		case err != nil:
			report(err)
		case dir.IsDir():
			walkDir(path)
		default:
			if err := processFile(path, nil, os.Stdout, false); err != nil {
				report(err)
			}
		}
	}
}

func diff(b1, b2 []byte) (data []byte, err error) {
	f1, err := ioutil.TempFile("", "mdfmt")
	if err != nil {
		return
	}
	defer os.Remove(f1.Name())
	defer f1.Close()

	f2, err := ioutil.TempFile("", "mdfmt")
	if err != nil {
		return
	}
	defer os.Remove(f2.Name())
	defer f2.Close()

	f1.Write(b1)
	f2.Write(b2)

	data, err = exec.Command("diff", "-u", f1.Name(), f2.Name()).CombinedOutput()
	if len(data) > 0 {
		// diff exits with a non-zero status when the files don't match.
		// Ignore that failure as long as we get output.
		err = nil
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"image"
	"image/draw"
	"math"
	"runtime"
	"sync"
	"time"

	kingpin "gopkg.in/alecthomas/kingpin.v2"

	"github.com/fogleman/fauxgl"
	"github.com/fogleman/gg"
	"github.com/fogleman/slicer"
)

var (
	quiet     = kingpin.Flag("quiet", "Run in silent mode.").Short('q').Bool()
	size      = kingpin.Flag("size", "Set the slice thickness.").Short('s').Float()
	number    = kingpin.Flag("number", "Set the number of slices.").Short('n').Int()
	directory = kingpin.Flag("directory", "Set the output directory.").Short('d').ExistingDir()
	files     = kingpin.Arg("files", "Mesh files to slice.").Required().ExistingFiles()
)

func main() {
	kingpin.Parse()
	if *size == 0 && *number == 0 {
		kingpin.Fatalf("must specify slice thickness or count, try --help")
	}
	for _, filename := range *files {
		process(filename)
	}
}

func timed(name string) func() {
	if *quiet {
		return func() {}
	}
	fmt.Printf("%s... ", name)
	start := time.Now()
	return func() {
		fmt.Println(time.Since(start))
	}
}

func log(text string) {
	if !*quiet {
		fmt.Println(text)
	}
}

func process(infile string) {
	var done func()

	log(fmt.Sprintf("input: %s", infile))

	// load mesh
	done = timed("loading mesh")
	mesh, err := fauxgl.LoadMesh(infile)
	if err != nil {
		panic(err)
	}
	box := mesh.BoundingBox()
	done()

	// determine slice thickness
	var step float64
	if *size != 0 {
		step = *size
	}
	if *number != 0 {
		step = box.Size().Z / float64(*number)
	}

	// slice mesh
	done = timed("slicing mesh")
	layers := slicer.SliceMesh(mesh, step)
	done()

	// determine output filename
	// dir, name := filepath.Split(infile)
	// if *directory != "" {
	// 	dir = *directory
	// }
	// outfile, _ := filepath.Abs(filepath.Join(dir, name) + ".svg")

	// write output
	// done = timed("creating svg")
	// svg := createSVG(box, layers)
	// ioutil.WriteFile(outfile, []byte(svg), 0644)
	// done()

	// log(fmt.Sprintf("output: %s", outfile))
	// log("")

	render(box, layers)
}

func createSVG(box fauxgl.Box, layers []slicer.Layer) string {
	const S = 1600
	const P = 32

	center := box.Center()
	size := box.Size()
	sx := (S - P*2) / size.X
	sy := (S - P*2) / size.Y
	scale := math.Min(sx, sy)
	width := 0.25 / scale
	transform := fmt.Sprintf(
		"translate(%d %d) scale(%g) translate(%g %g)",
		S/2, S/2, scale, -center.X, -center.Y)

	var buf bytes.Buffer
	buf.WriteString(fmt.Sprintf("<svg version=\"1.1\" width=\"%d\" height=\"%d\" xmlns=\"http://www.w3.org/2000/svg\">\n", S, S))
	buf.WriteString(fmt.Sprintf("<g transform=\"%s\">\n", transform))
	for _, layer := range layers {
		buf.WriteString(fmt.Sprintf("<path slicer-z=\"%g\" stroke=\"#000000\" stroke-width=\"%g\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"0.01\" d=\"%s\"></path>\n", layer.Z, width, layer.SVG()))
	}
	buf.WriteString("</g>\n")
	buf.WriteString("</svg>\n")
	return buf.String()
}

type job struct {
	i     int
	layer slicer.Layer
	box   fauxgl.Box
}

func render(box fauxgl.Box, layers []slicer.Layer) {
	wn := runtime.NumCPU()
	ch := make(chan job, len(layers))
	var wg sync.WaitGroup
	for wi := 0; wi < wn; wi++ {
		wg.Add(1)
		go worker(ch, &wg)
	}
	for i, l := range layers {
		ch <- job{i, l, box}
	}
	close(ch)
	wg.Wait()
}

func worker(ch chan job, wg *sync.WaitGroup) {
	const Scale = 25
	const W = Scale * 300
	const H = Scale * 200
	// const P = 50
	for j := range ch {
		i := j.i
		layer := j.layer
		box := j.box
		center := box.Center()
		// size := box.Size()
		// sx := (S - P*2) / size.X
		// sy := (S - P*2) / size.Y
		// scale := math.Min(sx, sy)
		dc := gg.NewContext(W, H)
		dc.InvertY()
		dc.SetRGB(1, 1, 1)
		dc.Clear()
		dc.Translate(W/2, H/2)
		dc.Scale(Scale, Scale)
		dc.Translate(-center.X, -center.Y)
		dc.SetFillRuleWinding()
		for _, path := range layer.Paths {
			dc.NewSubPath()
			for _, point := range path {
				dc.LineTo(point.X, point.Y)
			}
			dc.ClosePath()
		}
		dc.SetRGB(0, 0, 0)
		dc.Fill()

		// dc.SetRGB(0.6, 0.6, 0.6)
		// dc.FillPreserve()
		// dc.SetRGB(0, 0, 0)
		// dc.SetLineWidth(3)
		// dc.Stroke()
		// dc.SavePNG(fmt.Sprintf("out/%04d.png", i))

		src := dc.Image()
		dst := image.NewGray(src.Bounds())
		draw.Draw(dst, src.Bounds(), src, image.ZP, draw.Src)

		path := fmt.Sprintf("out/%04d.png", i)
		gg.SavePNG(path, dst)

		fmt.Println(path)
	}
	wg.Done()
}
<commit_msg>png slicer<commit_after>package main

import (
	"fmt"
	"image"
	"image/png"
	"os"
	"path/filepath"
	"runtime"
	"sync"
	"time"

	kingpin "gopkg.in/alecthomas/kingpin.v2"

	"github.com/fogleman/fauxgl"
	"github.com/fogleman/gg"
	"github.com/fogleman/slicer"
)

var (
	quiet     = kingpin.Flag("quiet", "Run in silent mode.").Short('q').Bool()
	size      = kingpin.Flag("size", "Set the slice thickness.").Short('s').Float()
	number    = kingpin.Flag("number", "Set the number of slices.").Short('n').Int()
	directory = kingpin.Flag("directory", "Set the output directory.").Short('d').ExistingDir()
	width     = kingpin.Flag("width", "Set the raster width in model units.").Required().Short('w').Float()
	height    = kingpin.Flag("height", "Set the raster height in model units.").Required().Short('h').Float()
	scale     = kingpin.Flag("scale", "Set the raster scale.").Short('x').Required().Float()
	files     = kingpin.Arg("files", "Mesh files to slice.").Required().ExistingFiles()
)

func main() {
	kingpin.Parse()
	if *size == 0 && *number == 0 {
		kingpin.Fatalf("must specify slice thickness or count, try --help")
	}
	for _, filename := range *files {
		process(filename)
	}
}

func timed(name string) func() {
	if *quiet {
		return func() {}
	}
	fmt.Printf("%s... ", name)
	start := time.Now()
	return func() {
		fmt.Println(time.Since(start))
	}
}

func log(text string) {
	if !*quiet {
		fmt.Println(text)
	}
}

func process(infile string) {
	var done func()

	log(fmt.Sprintf("input: %s", infile))

	// load mesh
	done = timed("loading mesh")
	mesh, err := fauxgl.LoadMesh(infile)
	if err != nil {
		panic(err)
	}
	box := mesh.BoundingBox()
	done()

	// determine slice thickness
	var step float64
	if *size != 0 {
		step = *size
	}
	if *number != 0 {
		step = box.Size().Z / float64(*number)
	}

	// slice mesh
	done = timed("slicing mesh")
	layers := slicer.SliceMesh(mesh, step)
	done()

	// render pngs
	render(box, layers)
}

type job struct {
	i     int
	layer slicer.Layer
	box   fauxgl.Box
}

func render(box fauxgl.Box, layers []slicer.Layer) {
	wn := runtime.NumCPU()
	ch := make(chan job, len(layers))
	var wg sync.WaitGroup
	for wi := 0; wi < wn; wi++ {
		wg.Add(1)
		go worker(ch, &wg)
	}
	for i, l := range layers {
		ch <- job{i, l, box}
	}
	close(ch)
	wg.Wait()
}

func fastRGBAToGray(src *image.RGBA) *image.Gray {
	dst := image.NewGray(src.Bounds())
	w := src.Bounds().Size().X
	h := src.Bounds().Size().Y
	for y := 0; y < h; y++ {
		i := src.PixOffset(0, y)
		j := dst.PixOffset(0, y)
		for x := 0; x < w; x++ {
			dst.Pix[j] = src.Pix[i]
			i += 4
			j++
		}
	}
	return dst
}

func savePNG(path string, im image.Image) error {
	file, err := os.Create(path)
	if err != nil {
		return err
	}
	defer file.Close()
	encoder := png.Encoder{
		CompressionLevel: png.BestSpeed,
	}
	return encoder.Encode(file, im)
}

func worker(ch chan job, wg *sync.WaitGroup) {
	s := *scale
	w := *width * s
	h := *height * s
	dir := "."
	if *directory != "" {
		dir = *directory
	}
	for j := range ch {
		i := j.i
		layer := j.layer
		box := j.box
		center := box.Center()
		dc := gg.NewContext(int(w), int(h))
		dc.InvertY()
		dc.SetRGB(1, 1, 1)
		dc.Clear()
		dc.Translate(w/2, h/2)
		dc.Scale(s, s)
		dc.Translate(-center.X, -center.Y)
		dc.SetFillRuleWinding()
		for _, path := range layer.Paths {
			dc.NewSubPath()
			for _, point := range path {
				dc.LineTo(point.X, point.Y)
			}
			dc.ClosePath()
		}
		dc.SetRGB(0, 0, 0)
		dc.Fill()
		src := dc.Image()
		dst := fastRGBAToGray(src.(*image.RGBA))
		path, _ := filepath.Abs(filepath.Join(dir, fmt.Sprintf("%04d.png", i)))
		savePNG(path, dst)
		fmt.Println(path)
	}
	wg.Done()
}
<|endoftext|>


<commit_before>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"tchaik.com/index"
	"tchaik.com/index/checklist"
	"tchaik.com/index/favourite"
	"tchaik.com/index/history"
)

type Meta struct {
	history    history.Store
	favourites favourite.Store
	checklist  checklist.Store
}

func (m *Meta) annotateFavourites(p index.Path, g group) group {
	if len(g.Tracks) > 0 || len(g.Groups) > 0 {
		g.Favourite = m.favourites.Get(p)
	}
	return g
}

func (m *Meta) annotateChecklist(p index.Path, g group) group {
	if len(g.Tracks) > 0 || len(g.Groups) > 0 {
		g.Checklist = m.checklist.Get(p)
	}
	return g
}
<commit_msg>Added godoc to Meta.<commit_after>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"tchaik.com/index"
	"tchaik.com/index/checklist"
	"tchaik.com/index/favourite"
	"tchaik.com/index/history"
)

// Meta is a container for extra metadata which wraps the central media library.
// TODO: Refactor meta to be user-specific.
type Meta struct {
	history    history.Store
	favourites favourite.Store
	checklist  checklist.Store
}

func (m *Meta) annotateFavourites(p index.Path, g group) group {
	if len(g.Tracks) > 0 || len(g.Groups) > 0 {
		g.Favourite = m.favourites.Get(p)
	}
	return g
}

func (m *Meta) annotateChecklist(p index.Path, g group) group {
	if len(g.Tracks) > 0 || len(g.Groups) > 0 {
		g.Checklist = m.checklist.Get(p)
	}
	return g
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"github.com/drosseau/degob"
)

var (
	outFile     = flag.String("ofile", "", "Output file (defaults to stdout)")
	inFile      = flag.String("ifile", "", "Input file (defaults to stdin)")
	truncateOut = flag.Bool("trunc", false, "Truncate output file")
	base64d     = flag.Bool("b64", false, "base64 input")
	base64urld  = flag.Bool("b64url", false, "base64url input")
	noComments  = flag.Bool("nc", false, "don't print additional comments")
	noTypes     = flag.Bool("nt", false, "don't print type information")
	json        = flag.Bool("json", false, "show value as json")
)

func errorf(s string, v ...interface{}) {
	_, _ = fmt.Fprintf(os.Stderr, s, v...)
	os.Exit(1)
}

func getWriter() io.WriteCloser {
	if *outFile != "" {
		opts := os.O_WRONLY | os.O_CREATE
		if *truncateOut {
			opts |= os.O_TRUNC
		} else {
			opts |= os.O_APPEND
		}
		f, err := os.OpenFile(*outFile, opts, 0644)
		if err != nil {
			errorf("failed to open `%s` for writing: %v\n", *outFile, err)
		}
		return f
	}
	return os.Stdout
}

func getReader() io.ReadCloser {
	if *inFile != "" {
		f, err := os.Open(*outFile)
		if err != nil {
			errorf("failed to open `%s` for reading: %v\n", *inFile, err)
		}
		return f
	}
	return ioutil.NopCloser(os.Stdin)
}

type writer struct {
	w   io.Writer
	err error
}

func (w writer) Write(b []byte) (int, error) {
	if w.err != nil {
		errorf("error writing output: %v\n", w.err)
	}
	var n int
	n, w.err = w.w.Write(b)
	return n, w.err
}

func (w writer) writeComment(s string, v ...interface{}) {
	if *noComments {
		return
	}
	if w.err != nil {
		errorf("error writing output: %v\n", w.err)
	}
	_, w.err = fmt.Fprintf(w.w, s, v...)
}

func main() {
	flag.Parse()
	out := getWriter()
	defer out.Close()
	in := getReader()
	defer in.Close()

	if *base64d {
		in = ioutil.NopCloser(base64.NewDecoder(base64.StdEncoding, in))
	} else if *base64urld {
		in = ioutil.NopCloser(base64.NewDecoder(base64.URLEncoding, in))
	}

	w := writer{w: out}

	dec := degob.NewDecoder(in)
	gobs, err := dec.Decode()
	if err != nil {
		errorf("failed to decode gob: %s\n", err)
	}
	for i, g := range gobs {
		w.writeComment("// Decoded gob %d\n\n//Types\n", i+1)
		if !*noTypes {
			err = g.WriteTypes(w)
			if err != nil {
				errorf("error writing types: %v\n", err)
			}
		}
		w.writeComment("// Values:")
		if *json {
			err = g.WriteValue(w, degob.JSON)
		} else {
			err = g.WriteValue(w, degob.SingleLine)
		}
		if err != nil {
			errorf("error writing values: %v\n", err)
		}
		w.writeComment("\n// End gob %d\n\n", i+1)
	}
}
<commit_msg>pkg flag on command will allow you to output a package header<commit_after>package main

import (
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"github.com/drosseau/degob"
)

var (
	outFile     = flag.String("ofile", "", "Output file (defaults to stdout)")
	inFile      = flag.String("ifile", "", "Input file (defaults to stdin)")
	truncateOut = flag.Bool("trunc", false, "Truncate output file")
	base64d     = flag.Bool("b64", false, "base64 input")
	base64urld  = flag.Bool("b64url", false, "base64url input")
	noComments  = flag.Bool("nc", false, "don't print additional comments")
	noTypes     = flag.Bool("nt", false, "don't print type information")
	json        = flag.Bool("json", false, "show value as json")
	pkgName     = flag.String("pkg", "", "include a package definition in the output with the given name")
)

func errorf(s string, v ...interface{}) {
	_, _ = fmt.Fprintf(os.Stderr, s, v...)
	os.Exit(1)
}

func getWriter() io.WriteCloser {
	if *outFile != "" {
		opts := os.O_WRONLY | os.O_CREATE
		if *truncateOut {
			opts |= os.O_TRUNC
		} else {
			opts |= os.O_APPEND
		}
		f, err := os.OpenFile(*outFile, opts, 0644)
		if err != nil {
			errorf("failed to open `%s` for writing: %v\n", *outFile, err)
		}
		return f
	}
	return os.Stdout
}

func getReader() io.ReadCloser {
	if *inFile != "" {
		f, err := os.Open(*outFile)
		if err != nil {
			errorf("failed to open `%s` for reading: %v\n", *inFile, err)
		}
		return f
	}
	return ioutil.NopCloser(os.Stdin)
}

type writer struct {
	w   io.Writer
	err error
}

func (w writer) Write(b []byte) (int, error) {
	if w.err != nil {
		errorf("error writing output: %v\n", w.err)
	}
	var n int
	n, w.err = w.w.Write(b)
	return n, w.err
}

func (w writer) writeComment(s string, v ...interface{}) {
	if *noComments {
		return
	}
	w.writeStr(s, v...)
}

func (w writer) writeStr(s string, v ...interface{}) {
	if w.err != nil {
		errorf("error writing output: %v\n")
	}
	_, w.err = fmt.Fprintf(w.w, s, v...)
}

func main() {
	flag.Parse()
	out := getWriter()
	defer out.Close()
	in := getReader()
	defer in.Close()

	if *base64d {
		in = ioutil.NopCloser(base64.NewDecoder(base64.StdEncoding, in))
	} else if *base64urld {
		in = ioutil.NopCloser(base64.NewDecoder(base64.URLEncoding, in))
	}

	w := writer{w: out}

	dec := degob.NewDecoder(in)
	gobs, err := dec.Decode()
	if err != nil {
		errorf("failed to decode gob: %s\n", err)
	}
	if *pkgName != "" {
		w.writeStr("package %s\n\n", *pkgName)
	}
	for i, g := range gobs {
		w.writeComment("// Decoded gob %d\n\n//Types\n", i+1)
		if !*noTypes {
			err = g.WriteTypes(w)
			if err != nil {
				errorf("error writing types: %v\n", err)
			}
		}
		w.writeComment("// Values:")
		if *json {
			err = g.WriteValue(w, degob.JSON)
		} else {
			err = g.WriteValue(w, degob.SingleLine)
		}
		if err != nil {
			errorf("error writing values: %v\n", err)
		}
		w.writeComment("\n// End gob %d\n\n", i+1)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package audit

import (
	"bytes"
	"fmt"
	"net/http"
	"time"

	"github.com/golang/glog"
	"github.com/pborman/uuid"

	"reflect"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	utilnet "k8s.io/apimachinery/pkg/util/net"
	auditinternal "k8s.io/apiserver/pkg/apis/audit"
	"k8s.io/apiserver/pkg/authentication/user"
	"k8s.io/apiserver/pkg/authorization/authorizer"
)

func NewEventFromRequest(req *http.Request, level auditinternal.Level, attribs authorizer.Attributes) (*auditinternal.Event, error) {
	ev := &auditinternal.Event{
		RequestReceivedTimestamp: metav1.NewMicroTime(time.Now()),
		Verb:       attribs.GetVerb(),
		RequestURI: req.URL.RequestURI(),
	}

	ev.Level = level

	// prefer the id from the headers. If not available, create a new one.
	// TODO(audit): do we want to forbid the header for non-front-proxy users?
	ids := req.Header.Get(auditinternal.HeaderAuditID)
	if ids != "" {
		ev.AuditID = types.UID(ids)
	} else {
		ev.AuditID = types.UID(uuid.NewRandom().String())
	}

	ips := utilnet.SourceIPs(req)
	ev.SourceIPs = make([]string, len(ips))
	for i := range ips {
		ev.SourceIPs[i] = ips[i].String()
	}

	if user := attribs.GetUser(); user != nil {
		ev.User.Username = user.GetName()
		ev.User.Extra = map[string]auditinternal.ExtraValue{}
		for k, v := range user.GetExtra() {
			ev.User.Extra[k] = auditinternal.ExtraValue(v)
		}
		ev.User.Groups = user.GetGroups()
		ev.User.UID = user.GetUID()
	}

	if attribs.IsResourceRequest() {
		ev.ObjectRef = &auditinternal.ObjectReference{
			Namespace:   attribs.GetNamespace(),
			Name:        attribs.GetName(),
			Resource:    attribs.GetResource(),
			Subresource: attribs.GetSubresource(),
			APIGroup:    attribs.GetAPIGroup(),
			APIVersion:  attribs.GetAPIVersion(),
		}
	}

	return ev, nil
}

// LogImpersonatedUser fills in the impersonated user attributes into an audit event.
func LogImpersonatedUser(ae *auditinternal.Event, user user.Info) {
	if ae == nil || ae.Level.Less(auditinternal.LevelMetadata) {
		return
	}
	ae.ImpersonatedUser = &auditinternal.UserInfo{
		Username: user.GetName(),
	}
	ae.ImpersonatedUser.Groups = user.GetGroups()
	ae.ImpersonatedUser.UID = user.GetUID()
	ae.ImpersonatedUser.Extra = map[string]auditinternal.ExtraValue{}
	for k, v := range user.GetExtra() {
		ae.ImpersonatedUser.Extra[k] = auditinternal.ExtraValue(v)
	}
}

// LogRequestObject fills in the request object into an audit event. The passed runtime.Object
// will be converted to the given gv.
func LogRequestObject(ae *auditinternal.Event, obj runtime.Object, gvr schema.GroupVersionResource, subresource string, s runtime.NegotiatedSerializer) {
	if ae == nil || ae.Level.Less(auditinternal.LevelMetadata) {
		return
	}

	// complete ObjectRef
	if ae.ObjectRef == nil {
		ae.ObjectRef = &auditinternal.ObjectReference{}
	}
	if acc, ok := obj.(metav1.ObjectMetaAccessor); ok {
		meta := acc.GetObjectMeta()
		if len(ae.ObjectRef.Namespace) == 0 {
			ae.ObjectRef.Namespace = meta.GetNamespace()
		}
		if len(ae.ObjectRef.Name) == 0 {
			ae.ObjectRef.Name = meta.GetName()
		}
		if len(ae.ObjectRef.UID) == 0 {
			ae.ObjectRef.UID = meta.GetUID()
		}
		if len(ae.ObjectRef.ResourceVersion) == 0 {
			ae.ObjectRef.ResourceVersion = meta.GetResourceVersion()
		}
	}
	// TODO: ObjectRef should include the API group.
	if len(ae.ObjectRef.APIVersion) == 0 {
		ae.ObjectRef.APIGroup = gvr.Group
		ae.ObjectRef.APIVersion = gvr.Version
	}
	if len(ae.ObjectRef.Resource) == 0 {
		ae.ObjectRef.Resource = gvr.Resource
	}
	if len(ae.ObjectRef.Subresource) == 0 {
		ae.ObjectRef.Subresource = subresource
	}

	if ae.Level.Less(auditinternal.LevelRequest) {
		return
	}

	// TODO(audit): hook into the serializer to avoid double conversion
	var err error
	ae.RequestObject, err = encodeObject(obj, gvr.GroupVersion(), s)
	if err != nil {
		// TODO(audit): add error slice to audit event struct
		glog.Warningf("Auditing failed of %v request: %v", reflect.TypeOf(obj).Name(), err)
		return
	}
}

// LogRquestPatch fills in the given patch as the request object into an audit event.
func LogRequestPatch(ae *auditinternal.Event, patch []byte) {
	if ae == nil || ae.Level.Less(auditinternal.LevelRequest) {
		return
	}

	ae.RequestObject = &runtime.Unknown{
		Raw:         patch,
		ContentType: runtime.ContentTypeJSON,
	}
}

// LogResponseObject fills in the response object into an audit event. The passed runtime.Object
// will be converted to the given gv.
func LogResponseObject(ae *auditinternal.Event, obj runtime.Object, gv schema.GroupVersion, s runtime.NegotiatedSerializer) {
	if ae == nil || ae.Level.Less(auditinternal.LevelMetadata) {
		return
	}
	if status, ok := obj.(*metav1.Status); ok {
		ae.ResponseStatus = status
	}

	if ae.Level.Less(auditinternal.LevelRequestResponse) {
		return
	}
	// TODO(audit): hook into the serializer to avoid double conversion
	var err error
	ae.ResponseObject, err = encodeObject(obj, gv, s)
	if err != nil {
		glog.Warningf("Audit failed for %q response: %v", reflect.TypeOf(obj).Name(), err)
	}
}

func encodeObject(obj runtime.Object, gv schema.GroupVersion, serializer runtime.NegotiatedSerializer) (*runtime.Unknown, error) {
	supported := serializer.SupportedMediaTypes()
	for i := range supported {
		if supported[i].MediaType == "application/json" {
			enc := serializer.EncoderForVersion(supported[i].Serializer, gv)
			var buf bytes.Buffer
			if err := enc.Encode(obj, &buf); err != nil {
				return nil, fmt.Errorf("encoding failed: %v", err)
			}

			return &runtime.Unknown{
				Raw:         buf.Bytes(),
				ContentType: runtime.ContentTypeJSON,
			}, nil
		}
	}
	return nil, fmt.Errorf("no json encoder found")
}

// LogAnnotation fills in the Annotations according to the key value pair.
func LogAnnotation(ae *auditinternal.Event, key, value string) {
	if ae == nil || ae.Level.Less(auditinternal.LevelMetadata) {
		return
	}
	if ae.Annotations == nil {
		ae.Annotations = make(map[string]string)
	}
	if v, ok := ae.Annotations[key]; ok && v != value {
		glog.Warningf("Failed to set annotations[%q] to %q for audit:%q, it has already been set to %q", key, value, ae.AuditID, ae.Annotations[key])
		return
	}
	ae.Annotations[key] = value
}

// LogAnnotations fills in the Annotations according to the annotations map.
func LogAnnotations(ae *auditinternal.Event, annotations map[string]string) {
	if ae == nil || ae.Level.Less(auditinternal.LevelMetadata) {
		return
	}
	for key, value := range annotations {
		LogAnnotation(ae, key, value)
	}
}
<commit_msg>fix typo and remove inaccurate TODO<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package audit

import (
	"bytes"
	"fmt"
	"net/http"
	"time"

	"github.com/golang/glog"
	"github.com/pborman/uuid"

	"reflect"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	utilnet "k8s.io/apimachinery/pkg/util/net"
	auditinternal "k8s.io/apiserver/pkg/apis/audit"
	"k8s.io/apiserver/pkg/authentication/user"
	"k8s.io/apiserver/pkg/authorization/authorizer"
)

func NewEventFromRequest(req *http.Request, level auditinternal.Level, attribs authorizer.Attributes) (*auditinternal.Event, error) {
	ev := &auditinternal.Event{
		RequestReceivedTimestamp: metav1.NewMicroTime(time.Now()),
		Verb:       attribs.GetVerb(),
		RequestURI: req.URL.RequestURI(),
	}

	ev.Level = level

	// prefer the id from the headers. If not available, create a new one.
	// TODO(audit): do we want to forbid the header for non-front-proxy users?
	ids := req.Header.Get(auditinternal.HeaderAuditID)
	if ids != "" {
		ev.AuditID = types.UID(ids)
	} else {
		ev.AuditID = types.UID(uuid.NewRandom().String())
	}

	ips := utilnet.SourceIPs(req)
	ev.SourceIPs = make([]string, len(ips))
	for i := range ips {
		ev.SourceIPs[i] = ips[i].String()
	}

	if user := attribs.GetUser(); user != nil {
		ev.User.Username = user.GetName()
		ev.User.Extra = map[string]auditinternal.ExtraValue{}
		for k, v := range user.GetExtra() {
			ev.User.Extra[k] = auditinternal.ExtraValue(v)
		}
		ev.User.Groups = user.GetGroups()
		ev.User.UID = user.GetUID()
	}

	if attribs.IsResourceRequest() {
		ev.ObjectRef = &auditinternal.ObjectReference{
			Namespace:   attribs.GetNamespace(),
			Name:        attribs.GetName(),
			Resource:    attribs.GetResource(),
			Subresource: attribs.GetSubresource(),
			APIGroup:    attribs.GetAPIGroup(),
			APIVersion:  attribs.GetAPIVersion(),
		}
	}

	return ev, nil
}

// LogImpersonatedUser fills in the impersonated user attributes into an audit event.
func LogImpersonatedUser(ae *auditinternal.Event, user user.Info) {
	if ae == nil || ae.Level.Less(auditinternal.LevelMetadata) {
		return
	}
	ae.ImpersonatedUser = &auditinternal.UserInfo{
		Username: user.GetName(),
	}
	ae.ImpersonatedUser.Groups = user.GetGroups()
	ae.ImpersonatedUser.UID = user.GetUID()
	ae.ImpersonatedUser.Extra = map[string]auditinternal.ExtraValue{}
	for k, v := range user.GetExtra() {
		ae.ImpersonatedUser.Extra[k] = auditinternal.ExtraValue(v)
	}
}

// LogRequestObject fills in the request object into an audit event. The passed runtime.Object
// will be converted to the given gv.
func LogRequestObject(ae *auditinternal.Event, obj runtime.Object, gvr schema.GroupVersionResource, subresource string, s runtime.NegotiatedSerializer) {
	if ae == nil || ae.Level.Less(auditinternal.LevelMetadata) {
		return
	}

	// complete ObjectRef
	if ae.ObjectRef == nil {
		ae.ObjectRef = &auditinternal.ObjectReference{}
	}
	if acc, ok := obj.(metav1.ObjectMetaAccessor); ok {
		meta := acc.GetObjectMeta()
		if len(ae.ObjectRef.Namespace) == 0 {
			ae.ObjectRef.Namespace = meta.GetNamespace()
		}
		if len(ae.ObjectRef.Name) == 0 {
			ae.ObjectRef.Name = meta.GetName()
		}
		if len(ae.ObjectRef.UID) == 0 {
			ae.ObjectRef.UID = meta.GetUID()
		}
		if len(ae.ObjectRef.ResourceVersion) == 0 {
			ae.ObjectRef.ResourceVersion = meta.GetResourceVersion()
		}
	}
	if len(ae.ObjectRef.APIVersion) == 0 {
		ae.ObjectRef.APIGroup = gvr.Group
		ae.ObjectRef.APIVersion = gvr.Version
	}
	if len(ae.ObjectRef.Resource) == 0 {
		ae.ObjectRef.Resource = gvr.Resource
	}
	if len(ae.ObjectRef.Subresource) == 0 {
		ae.ObjectRef.Subresource = subresource
	}

	if ae.Level.Less(auditinternal.LevelRequest) {
		return
	}

	// TODO(audit): hook into the serializer to avoid double conversion
	var err error
	ae.RequestObject, err = encodeObject(obj, gvr.GroupVersion(), s)
	if err != nil {
		// TODO(audit): add error slice to audit event struct
		glog.Warningf("Auditing failed of %v request: %v", reflect.TypeOf(obj).Name(), err)
		return
	}
}

// LogRequestPatch fills in the given patch as the request object into an audit event.
func LogRequestPatch(ae *auditinternal.Event, patch []byte) {
	if ae == nil || ae.Level.Less(auditinternal.LevelRequest) {
		return
	}

	ae.RequestObject = &runtime.Unknown{
		Raw:         patch,
		ContentType: runtime.ContentTypeJSON,
	}
}

// LogResponseObject fills in the response object into an audit event. The passed runtime.Object
// will be converted to the given gv.
func LogResponseObject(ae *auditinternal.Event, obj runtime.Object, gv schema.GroupVersion, s runtime.NegotiatedSerializer) {
	if ae == nil || ae.Level.Less(auditinternal.LevelMetadata) {
		return
	}
	if status, ok := obj.(*metav1.Status); ok {
		ae.ResponseStatus = status
	}

	if ae.Level.Less(auditinternal.LevelRequestResponse) {
		return
	}
	// TODO(audit): hook into the serializer to avoid double conversion
	var err error
	ae.ResponseObject, err = encodeObject(obj, gv, s)
	if err != nil {
		glog.Warningf("Audit failed for %q response: %v", reflect.TypeOf(obj).Name(), err)
	}
}

func encodeObject(obj runtime.Object, gv schema.GroupVersion, serializer runtime.NegotiatedSerializer) (*runtime.Unknown, error) {
	supported := serializer.SupportedMediaTypes()
	for i := range supported {
		if supported[i].MediaType == "application/json" {
			enc := serializer.EncoderForVersion(supported[i].Serializer, gv)
			var buf bytes.Buffer
			if err := enc.Encode(obj, &buf); err != nil {
				return nil, fmt.Errorf("encoding failed: %v", err)
			}

			return &runtime.Unknown{
				Raw:         buf.Bytes(),
				ContentType: runtime.ContentTypeJSON,
			}, nil
		}
	}
	return nil, fmt.Errorf("no json encoder found")
}

// LogAnnotation fills in the Annotations according to the key value pair.
func LogAnnotation(ae *auditinternal.Event, key, value string) {
	if ae == nil || ae.Level.Less(auditinternal.LevelMetadata) {
		return
	}
	if ae.Annotations == nil {
		ae.Annotations = make(map[string]string)
	}
	if v, ok := ae.Annotations[key]; ok && v != value {
		glog.Warningf("Failed to set annotations[%q] to %q for audit:%q, it has already been set to %q", key, value, ae.AuditID, ae.Annotations[key])
		return
	}
	ae.Annotations[key] = value
}

// LogAnnotations fills in the Annotations according to the annotations map.
func LogAnnotations(ae *auditinternal.Event, annotations map[string]string) {
	if ae == nil || ae.Level.Less(auditinternal.LevelMetadata) {
		return
	}
	for key, value := range annotations {
		LogAnnotation(ae, key, value)
	}
}
<|endoftext|>


<commit_before>// Copyright 2022 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bufferv2

import (
	"fmt"
	"io"

	"gvisor.dev/gvisor/pkg/sync"
)

// ReadSize is the default amount that a View's size is increased by when an
// io.Reader has more data than a View can hold during calls to ReadFrom.
const ReadSize = 512

var viewPool = sync.Pool{
	New: func() interface{} {
		return &View{}
	},
}

// View is a window into a shared chunk. Views are held by Buffers in
// viewLists to represent contiguous memory.
//
// A View must be created with NewView, NewViewWithData, or Clone. Owners are
// responsible for maintaining ownership over their views. When Views need to be
// shared or copied, the owner should create a new View with Clone. Clone must
// only ever be called on a owned View, not a borrowed one.
//
// Users are responsible for calling Release when finished with their View so
// that its resources can be returned to the pool.
//
// Users must not write directly to slices returned by AsSlice. Instead, they
// must use Write/WriteAt/CopyIn to modify the underlying View. This preserves
// the safety guarantees of copy-on-write.
//
// +stateify savable
type View struct {
	viewEntry `state:"nosave"`
	read      int
	write     int
	chunk     *chunk
}

// NewView creates a new view with capacity at least as big as cap. It is
// analogous to make([]byte, 0, cap).
func NewView(cap int) *View {
	c := newChunk(cap)
	v := viewPool.Get().(*View)
	*v = View{chunk: c}
	return v
}

// NewViewSize creates a new view with capacity at least as big as size and
// length that is exactly size. It is analogous to make([]byte, size).
func NewViewSize(size int) *View {
	v := NewView(size)
	v.Grow(size)
	return v
}

// NewViewWithData creates a new view and initializes it with data. This
// function should be used with caution to avoid unnecessary []byte allocations.
// When in doubt use NewWithView to maximize chunk reuse in production
// environments.
func NewViewWithData(data []byte) *View {
	c := newChunk(len(data))
	v := viewPool.Get().(*View)
	*v = View{chunk: c}
	v.Write(data)
	return v
}

// Clone creates a shallow clone of v where the underlying chunk is shared.
//
// The caller must own the View to call Clone. It is not safe to call Clone
// on a borrowed or shared View because it can race with other View methods.
func (v *View) Clone() *View {
	if v == nil {
		panic("cannot clone a nil view")
	}
	v.chunk.IncRef()
	newV := viewPool.Get().(*View)
	newV.chunk = v.chunk
	newV.read = v.read
	newV.write = v.write
	return newV
}

// Release releases the chunk held by v and returns v to the pool.
func (v *View) Release() {
	if v == nil {
		panic("cannot release a nil view")
	}
	v.chunk.DecRef()
	*v = View{}
	viewPool.Put(v)
}

// Reset sets the view's read and write indices back to zero.
func (v *View) Reset() {
	if v == nil {
		panic("cannot reset a nil view")
	}
	v.read = 0
	v.write = 0
}

func (v *View) sharesChunk() bool {
	return v.chunk.refCount.Load() > 1
}

// Full indicates the chunk is full.
//
// This indicates there is no capacity left to write.
func (v *View) Full() bool {
	return v == nil || v.write == len(v.chunk.data)
}

// Capacity returns the total size of this view's chunk.
func (v *View) Capacity() int {
	if v == nil {
		return 0
	}
	return len(v.chunk.data)
}

// Size returns the size of data written to the view.
func (v *View) Size() int {
	if v == nil {
		return 0
	}
	return v.write - v.read
}

// TrimFront advances the read index by the given amount.
func (v *View) TrimFront(n int) {
	if v.read+n > v.write {
		panic("cannot trim past the end of a view")
	}
	v.read += n
}

// AsSlice returns a slice of the data written to this view.
func (v *View) AsSlice() []byte {
	if v.Size() == 0 {
		return nil
	}
	return v.chunk.data[v.read:v.write]
}

// AvailableSize returns the number of bytes available for writing.
func (v *View) AvailableSize() int {
	if v == nil {
		return 0
	}
	return len(v.chunk.data) - v.write
}

// Read reads v's data into p.
//
// Implements the io.Reader interface.
func (v *View) Read(p []byte) (int, error) {
	if len(p) == 0 {
		return 0, nil
	}
	if v.Size() == 0 {
		return 0, io.EOF
	}
	n := copy(p, v.AsSlice())
	v.TrimFront(n)
	return n, nil
}

// ReadByte implements the io.ByteReader interface.
func (v *View) ReadByte() (byte, error) {
	if v.Size() == 0 {
		return 0, io.EOF
	}
	b := v.AsSlice()[0]
	v.read++
	return b, nil
}

// WriteTo writes data to w until the view is empty or an error occurs. The
// return value n is the number of bytes written.
//
// WriteTo implements the io.WriterTo interface.
func (v *View) WriteTo(w io.Writer) (n int64, err error) {
	if v.Size() > 0 {
		sz := v.Size()
		m, e := w.Write(v.AsSlice())
		v.TrimFront(m)
		n = int64(m)
		if e != nil {
			return n, e
		}
		if m != sz {
			return n, io.ErrShortWrite
		}
	}
	return n, nil
}

// ReadAt reads data to the p starting at offset.
//
// Implements the io.ReaderAt interface.
func (v *View) ReadAt(p []byte, off int) (int, error) {
	if off < 0 || off > v.Size() {
		return 0, fmt.Errorf("ReadAt(): offset out of bounds: want 0 < off < %d, got off=%d", v.Size(), off)
	}
	n := copy(p, v.AsSlice()[off:])
	return n, nil
}

// Write writes data to the view's chunk starting at the v.write index. If the
// view's chunk has a reference count greater than 1, the chunk is copied first
// and then written to.
//
// Implements the io.Writer interface.
func (v *View) Write(p []byte) (int, error) {
	if v == nil {
		panic("cannot write to a nil view")
	}
	if v.AvailableSize() < len(p) {
		v.growCap(len(p) - v.AvailableSize())
	} else if v.sharesChunk() {
		defer v.chunk.DecRef()
		v.chunk = v.chunk.Clone()
	}
	n := copy(v.chunk.data[v.write:], p)
	v.write += n
	if n < len(p) {
		return n, io.ErrShortWrite
	}
	return n, nil
}

// ReadFrom reads data from r until EOF and appends it to the buffer, growing
// the buffer as needed. The return value n is the number of bytes read. Any
// error except io.EOF encountered during the read is also returned.
//
// ReadFrom implements the io.ReaderFrom interface.
func (v *View) ReadFrom(r io.Reader) (n int64, err error) {
	if v == nil {
		panic("cannot write to a nil view")
	}
	if v.sharesChunk() {
		defer v.chunk.DecRef()
		v.chunk = v.chunk.Clone()
	}
	for {
		// Check for EOF to avoid an unnnecesary allocation.
		if _, e := r.Read(nil); e == io.EOF {
			return n, nil
		}
		if v.AvailableSize() == 0 {
			v.growCap(ReadSize)
		}
		m, e := r.Read(v.availableSlice())
		v.write += m
		n += int64(m)

		if e == io.EOF {
			return n, nil
		}
		if e != nil {
			return n, e
		}
	}
}

// WriteAt writes data to the views's chunk starting at start. If the
// view's chunk has a reference count greater than 1, the chunk is copied first
// and then written to.
//
// Implements the io.WriterAt interface.
func (v *View) WriteAt(p []byte, off int) (int, error) {
	if v == nil {
		panic("cannot write to a nil view")
	}
	if off < 0 || off > v.Size() {
		return 0, fmt.Errorf("write offset out of bounds: want 0 < off < %d, got off=%d", v.Size(), off)
	}
	if v.sharesChunk() {
		defer v.chunk.DecRef()
		v.chunk = v.chunk.Clone()
	}
	n := copy(v.AsSlice()[off:], p)
	if n < len(p) {
		return n, io.ErrShortWrite
	}
	return n, nil
}

// Grow increases the size of the view. If the new size is greater than the
// view's current capacity, Grow will reallocate the view with an increased
// capacity.
func (v *View) Grow(n int) {
	if v == nil {
		panic("cannot grow a nil view")
	}
	if v.write+n > v.Capacity() {
		v.growCap(n)
	}
	v.write += n
}

// growCap increases the capacity of the view by at least n.
func (v *View) growCap(n int) {
	if v == nil {
		panic("cannot grow a nil view")
	}
	defer v.chunk.DecRef()
	old := v.AsSlice()
	v.chunk = newChunk(v.Capacity() + n)
	copy(v.chunk.data, old)
	v.read = 0
	v.write = len(old)
}

// CapLength caps the length of the view's read slice to n. If n > v.Size(),
// the function is a no-op.
func (v *View) CapLength(n int) {
	if v == nil {
		panic("cannot resize a nil view")
	}
	if n < 0 {
		panic("n must be >= 0")
	}
	if n > v.Size() {
		n = v.Size()
	}
	v.write = v.read + n
}

func (v *View) availableSlice() []byte {
	if v.sharesChunk() {
		defer v.chunk.DecRef()
		c := v.chunk.Clone()
		v.chunk = c
	}
	return v.chunk.data[v.write:]
}
<commit_msg>Add a method that returns an owned copy of the data in a View.<commit_after>// Copyright 2022 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bufferv2

import (
	"fmt"
	"io"

	"gvisor.dev/gvisor/pkg/sync"
)

// ReadSize is the default amount that a View's size is increased by when an
// io.Reader has more data than a View can hold during calls to ReadFrom.
const ReadSize = 512

var viewPool = sync.Pool{
	New: func() interface{} {
		return &View{}
	},
}

// View is a window into a shared chunk. Views are held by Buffers in
// viewLists to represent contiguous memory.
//
// A View must be created with NewView, NewViewWithData, or Clone. Owners are
// responsible for maintaining ownership over their views. When Views need to be
// shared or copied, the owner should create a new View with Clone. Clone must
// only ever be called on a owned View, not a borrowed one.
//
// Users are responsible for calling Release when finished with their View so
// that its resources can be returned to the pool.
//
// Users must not write directly to slices returned by AsSlice. Instead, they
// must use Write/WriteAt/CopyIn to modify the underlying View. This preserves
// the safety guarantees of copy-on-write.
//
// +stateify savable
type View struct {
	viewEntry `state:"nosave"`
	read      int
	write     int
	chunk     *chunk
}

// NewView creates a new view with capacity at least as big as cap. It is
// analogous to make([]byte, 0, cap).
func NewView(cap int) *View {
	c := newChunk(cap)
	v := viewPool.Get().(*View)
	*v = View{chunk: c}
	return v
}

// NewViewSize creates a new view with capacity at least as big as size and
// length that is exactly size. It is analogous to make([]byte, size).
func NewViewSize(size int) *View {
	v := NewView(size)
	v.Grow(size)
	return v
}

// NewViewWithData creates a new view and initializes it with data. This
// function should be used with caution to avoid unnecessary []byte allocations.
// When in doubt use NewWithView to maximize chunk reuse in production
// environments.
func NewViewWithData(data []byte) *View {
	c := newChunk(len(data))
	v := viewPool.Get().(*View)
	*v = View{chunk: c}
	v.Write(data)
	return v
}

// Clone creates a shallow clone of v where the underlying chunk is shared.
//
// The caller must own the View to call Clone. It is not safe to call Clone
// on a borrowed or shared View because it can race with other View methods.
func (v *View) Clone() *View {
	if v == nil {
		panic("cannot clone a nil view")
	}
	v.chunk.IncRef()
	newV := viewPool.Get().(*View)
	newV.chunk = v.chunk
	newV.read = v.read
	newV.write = v.write
	return newV
}

// Release releases the chunk held by v and returns v to the pool.
func (v *View) Release() {
	if v == nil {
		panic("cannot release a nil view")
	}
	v.chunk.DecRef()
	*v = View{}
	viewPool.Put(v)
}

// Reset sets the view's read and write indices back to zero.
func (v *View) Reset() {
	if v == nil {
		panic("cannot reset a nil view")
	}
	v.read = 0
	v.write = 0
}

func (v *View) sharesChunk() bool {
	return v.chunk.refCount.Load() > 1
}

// Full indicates the chunk is full.
//
// This indicates there is no capacity left to write.
func (v *View) Full() bool {
	return v == nil || v.write == len(v.chunk.data)
}

// Capacity returns the total size of this view's chunk.
func (v *View) Capacity() int {
	if v == nil {
		return 0
	}
	return len(v.chunk.data)
}

// Size returns the size of data written to the view.
func (v *View) Size() int {
	if v == nil {
		return 0
	}
	return v.write - v.read
}

// TrimFront advances the read index by the given amount.
func (v *View) TrimFront(n int) {
	if v.read+n > v.write {
		panic("cannot trim past the end of a view")
	}
	v.read += n
}

// AsSlice returns a slice of the data written to this view.
func (v *View) AsSlice() []byte {
	if v.Size() == 0 {
		return nil
	}
	return v.chunk.data[v.read:v.write]
}

// ToSlice returns an owned copy of the data in this view.
func (v *View) ToSlice() []byte {
	if v.Size() == 0 {
		return nil
	}
	s := make([]byte, v.Size())
	copy(s, v.AsSlice())
	return s
}

// AvailableSize returns the number of bytes available for writing.
func (v *View) AvailableSize() int {
	if v == nil {
		return 0
	}
	return len(v.chunk.data) - v.write
}

// Read reads v's data into p.
//
// Implements the io.Reader interface.
func (v *View) Read(p []byte) (int, error) {
	if len(p) == 0 {
		return 0, nil
	}
	if v.Size() == 0 {
		return 0, io.EOF
	}
	n := copy(p, v.AsSlice())
	v.TrimFront(n)
	return n, nil
}

// ReadByte implements the io.ByteReader interface.
func (v *View) ReadByte() (byte, error) {
	if v.Size() == 0 {
		return 0, io.EOF
	}
	b := v.AsSlice()[0]
	v.read++
	return b, nil
}

// WriteTo writes data to w until the view is empty or an error occurs. The
// return value n is the number of bytes written.
//
// WriteTo implements the io.WriterTo interface.
func (v *View) WriteTo(w io.Writer) (n int64, err error) {
	if v.Size() > 0 {
		sz := v.Size()
		m, e := w.Write(v.AsSlice())
		v.TrimFront(m)
		n = int64(m)
		if e != nil {
			return n, e
		}
		if m != sz {
			return n, io.ErrShortWrite
		}
	}
	return n, nil
}

// ReadAt reads data to the p starting at offset.
//
// Implements the io.ReaderAt interface.
func (v *View) ReadAt(p []byte, off int) (int, error) {
	if off < 0 || off > v.Size() {
		return 0, fmt.Errorf("ReadAt(): offset out of bounds: want 0 < off < %d, got off=%d", v.Size(), off)
	}
	n := copy(p, v.AsSlice()[off:])
	return n, nil
}

// Write writes data to the view's chunk starting at the v.write index. If the
// view's chunk has a reference count greater than 1, the chunk is copied first
// and then written to.
//
// Implements the io.Writer interface.
func (v *View) Write(p []byte) (int, error) {
	if v == nil {
		panic("cannot write to a nil view")
	}
	if v.AvailableSize() < len(p) {
		v.growCap(len(p) - v.AvailableSize())
	} else if v.sharesChunk() {
		defer v.chunk.DecRef()
		v.chunk = v.chunk.Clone()
	}
	n := copy(v.chunk.data[v.write:], p)
	v.write += n
	if n < len(p) {
		return n, io.ErrShortWrite
	}
	return n, nil
}

// ReadFrom reads data from r until EOF and appends it to the buffer, growing
// the buffer as needed. The return value n is the number of bytes read. Any
// error except io.EOF encountered during the read is also returned.
//
// ReadFrom implements the io.ReaderFrom interface.
func (v *View) ReadFrom(r io.Reader) (n int64, err error) {
	if v == nil {
		panic("cannot write to a nil view")
	}
	if v.sharesChunk() {
		defer v.chunk.DecRef()
		v.chunk = v.chunk.Clone()
	}
	for {
		// Check for EOF to avoid an unnnecesary allocation.
		if _, e := r.Read(nil); e == io.EOF {
			return n, nil
		}
		if v.AvailableSize() == 0 {
			v.growCap(ReadSize)
		}
		m, e := r.Read(v.availableSlice())
		v.write += m
		n += int64(m)

		if e == io.EOF {
			return n, nil
		}
		if e != nil {
			return n, e
		}
	}
}

// WriteAt writes data to the views's chunk starting at start. If the
// view's chunk has a reference count greater than 1, the chunk is copied first
// and then written to.
//
// Implements the io.WriterAt interface.
func (v *View) WriteAt(p []byte, off int) (int, error) {
	if v == nil {
		panic("cannot write to a nil view")
	}
	if off < 0 || off > v.Size() {
		return 0, fmt.Errorf("write offset out of bounds: want 0 < off < %d, got off=%d", v.Size(), off)
	}
	if v.sharesChunk() {
		defer v.chunk.DecRef()
		v.chunk = v.chunk.Clone()
	}
	n := copy(v.AsSlice()[off:], p)
	if n < len(p) {
		return n, io.ErrShortWrite
	}
	return n, nil
}

// Grow increases the size of the view. If the new size is greater than the
// view's current capacity, Grow will reallocate the view with an increased
// capacity.
func (v *View) Grow(n int) {
	if v == nil {
		panic("cannot grow a nil view")
	}
	if v.write+n > v.Capacity() {
		v.growCap(n)
	}
	v.write += n
}

// growCap increases the capacity of the view by at least n.
func (v *View) growCap(n int) {
	if v == nil {
		panic("cannot grow a nil view")
	}
	defer v.chunk.DecRef()
	old := v.AsSlice()
	v.chunk = newChunk(v.Capacity() + n)
	copy(v.chunk.data, old)
	v.read = 0
	v.write = len(old)
}

// CapLength caps the length of the view's read slice to n. If n > v.Size(),
// the function is a no-op.
func (v *View) CapLength(n int) {
	if v == nil {
		panic("cannot resize a nil view")
	}
	if n < 0 {
		panic("n must be >= 0")
	}
	if n > v.Size() {
		n = v.Size()
	}
	v.write = v.read + n
}

func (v *View) availableSlice() []byte {
	if v.sharesChunk() {
		defer v.chunk.DecRef()
		c := v.chunk.Clone()
		v.chunk = c
	}
	return v.chunk.data[v.write:]
}
<|endoftext|>


<commit_before>// +build !windows

/*
Copyright (c) for portions of rename.go are held by The Go Authors, 2016 and are provided under
the BSD license.

https://github.com/golang/go/blob/master/LICENSE

Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fs

import (
	"os"
	"syscall"

	"github.com/pkg/errors"
)

// renameFallback attempts to determine the appropriate fallback to failed rename
// operation depending on the resulting error.
func renameFallback(err error, src, dst string) error {
	// Rename may fail if src and dst are on different devices; fall back to
	// copy if we detect that case. syscall.EXDEV is the common name for the
	// cross device link error which has varying output text across different
	// operating systems.
	terr, ok := err.(*os.LinkError)
	if !ok {
		return err
	} else if terr.Err != syscall.EXDEV {
		return errors.Wrapf(terr, "link error: cannot rename %s to %s", src, dst)
	}

	return renameByCopy(src, dst)
}
<commit_msg>Fix url for bsd license<commit_after>// +build !windows

/*
Copyright (c) for portions of rename.go are held by The Go Authors, 2016 and are provided under
the BSD license.

https://github.com/golang/dep/blob/master/LICENSE

Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fs

import (
	"os"
	"syscall"

	"github.com/pkg/errors"
)

// renameFallback attempts to determine the appropriate fallback to failed rename
// operation depending on the resulting error.
func renameFallback(err error, src, dst string) error {
	// Rename may fail if src and dst are on different devices; fall back to
	// copy if we detect that case. syscall.EXDEV is the common name for the
	// cross device link error which has varying output text across different
	// operating systems.
	terr, ok := err.(*os.LinkError)
	if !ok {
		return err
	} else if terr.Err != syscall.EXDEV {
		return errors.Wrapf(terr, "link error: cannot rename %s to %s", src, dst)
	}

	return renameByCopy(src, dst)
}
<|endoftext|>


<commit_before>package server

import (
	"encoding/json"
	"fmt"
	"net"
	"net/http"

	fcollector "github.com/dongyiyang/k8sconnection/pkg/flowcollector"
	tcounter "github.com/dongyiyang/k8sconnection/pkg/transactioncounter"

	"github.com/golang/glog"
)

// Server is a http.Handler which exposes kubelet functionality over HTTP.
type Server struct {
	counter       *tcounter.TransactionCounter
	flowCollector *fcollector.FlowCollector
	mux           *http.ServeMux
}

// NewServer initializes and configures a kubelet.Server object to handle HTTP requests.
func NewServer(counter *tcounter.TransactionCounter, flowCollector *fcollector.FlowCollector) Server {
	server := Server{
		counter:       counter,
		flowCollector: flowCollector,
		mux:           http.NewServeMux(),
	}
	server.InstallDefaultHandlers()
	return server
}

// InstallDefaultHandlers registers the default set of supported HTTP request patterns with the mux.
func (s *Server) InstallDefaultHandlers() {
	s.mux.HandleFunc("/", handler)
	s.mux.HandleFunc("/transactions/count", s.getTransactionsCount)
	s.mux.HandleFunc("/transactions", s.getAllTransactionsAndReset)
	s.mux.HandleFunc("/flows", s.getAllFlows)
}

// ServeHTTP responds to HTTP requests on the Kubelet.
func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	s.mux.ServeHTTP(w, req)
}

func (s *Server) getAllTransactionsAndReset(w http.ResponseWriter, r *http.Request) {
	transactions := s.counter.GetAllTransactions()

	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.WriteHeader(http.StatusOK)
	if err := json.NewEncoder(w).Encode(transactions); err != nil {
		panic(err)
	}

	s.resetCounter()
}

func (s *Server) getTransactionsCount(w http.ResponseWriter, r *http.Request) {
	transactions := s.counter.GetAllTransactions()

	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.WriteHeader(http.StatusOK)
	if err := json.NewEncoder(w).Encode(transactions); err != nil {
		panic(err)
	}
}

func (s *Server) getAllFlows(w http.ResponseWriter, r *http.Request) {
	flows := s.flowCollector.GetAllFlows()

	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.WriteHeader(http.StatusOK)
	if err := json.NewEncoder(w).Encode(flows); err != nil {
		panic(err)
	}
}

func (s *Server) resetCounter() {
	s.counter.Reset()
}

func handler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Vmturbo k8sconntrack Service.")
}

// TODO: For now the address and port number is hardcoded. The actual port number need to be discussed.
func ListenAndServeProxyServer(counter *tcounter.TransactionCounter, flowCollector *fcollector.FlowCollector) {
	glog.V(3).Infof("Start VMT Kube-proxy server")
	handler := NewServer(counter, flowCollector)
	s := &http.Server{
		Addr:           net.JoinHostPort("0.0.0.0", "2223"),
		Handler:        &handler,
		MaxHeaderBytes: 1 << 20,
	}
	glog.Fatal(s.ListenAndServe())
}
<commit_msg>Fixed Wrong Port Number<commit_after>package server

import (
	"encoding/json"
	"fmt"
	"net"
	"net/http"

	fcollector "github.com/dongyiyang/k8sconnection/pkg/flowcollector"
	tcounter "github.com/dongyiyang/k8sconnection/pkg/transactioncounter"

	"github.com/golang/glog"
)

// Server is a http.Handler which exposes kubelet functionality over HTTP.
type Server struct {
	counter       *tcounter.TransactionCounter
	flowCollector *fcollector.FlowCollector
	mux           *http.ServeMux
}

// NewServer initializes and configures a kubelet.Server object to handle HTTP requests.
func NewServer(counter *tcounter.TransactionCounter, flowCollector *fcollector.FlowCollector) Server {
	server := Server{
		counter:       counter,
		flowCollector: flowCollector,
		mux:           http.NewServeMux(),
	}
	server.InstallDefaultHandlers()
	return server
}

// InstallDefaultHandlers registers the default set of supported HTTP request patterns with the mux.
func (s *Server) InstallDefaultHandlers() {
	s.mux.HandleFunc("/", handler)
	s.mux.HandleFunc("/transactions/count", s.getTransactionsCount)
	s.mux.HandleFunc("/transactions", s.getAllTransactionsAndReset)
	s.mux.HandleFunc("/flows", s.getAllFlows)
}

// ServeHTTP responds to HTTP requests on the Kubelet.
func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	s.mux.ServeHTTP(w, req)
}

func (s *Server) getAllTransactionsAndReset(w http.ResponseWriter, r *http.Request) {
	transactions := s.counter.GetAllTransactions()

	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.WriteHeader(http.StatusOK)
	if err := json.NewEncoder(w).Encode(transactions); err != nil {
		panic(err)
	}

	s.resetCounter()
}

func (s *Server) getTransactionsCount(w http.ResponseWriter, r *http.Request) {
	transactions := s.counter.GetAllTransactions()

	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.WriteHeader(http.StatusOK)
	if err := json.NewEncoder(w).Encode(transactions); err != nil {
		panic(err)
	}
}

func (s *Server) getAllFlows(w http.ResponseWriter, r *http.Request) {
	flows := s.flowCollector.GetAllFlows()

	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.WriteHeader(http.StatusOK)
	if err := json.NewEncoder(w).Encode(flows); err != nil {
		panic(err)
	}
}

func (s *Server) resetCounter() {
	s.counter.Reset()
}

func handler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Vmturbo k8sconntrack Service.")
}

// TODO: For now the address and port number is hardcoded. The actual port number need to be discussed.
func ListenAndServeProxyServer(counter *tcounter.TransactionCounter, flowCollector *fcollector.FlowCollector) {
	glog.V(3).Infof("Start VMT Kube-proxy server")
	handler := NewServer(counter, flowCollector)
	s := &http.Server{
		Addr:           net.JoinHostPort("0.0.0.0", "2222"),
		Handler:        &handler,
		MaxHeaderBytes: 1 << 20,
	}
	glog.Fatal(s.ListenAndServe())
}
<|endoftext|>