<commit_before>package crawler

import (
	"encoding/json"
	"fmt"
	"github.com/dokterbob/ipfs-search/indexer"
	"github.com/dokterbob/ipfs-search/queue"
	"gopkg.in/ipfs/go-ipfs-api.v1"
	"log"
	"net"
	"net/http"
	"net/url"
	// "path"
	"strings"
	"syscall"
	"time"
)

const (
	// Reconnect time in seconds
	RECONNECT_WAIT    = 2
	TIKA_TIMEOUT      = 120
	METADATA_MAX_SIZE = 50 * 1024 * 1024
)

type CrawlerArgs struct {
	Hash       string
	Name       string
	Size       uint64
	ParentHash string
	ParentName string // This is legacy, should be removed
}

type Crawler struct {
	sh *shell.Shell
	id *indexer.Indexer
	fq *queue.TaskQueue
	hq *queue.TaskQueue
}

func NewCrawler(sh *shell.Shell, id *indexer.Indexer, fq *queue.TaskQueue, hq *queue.TaskQueue) *Crawler {
	return &Crawler{
		sh: sh,
		id: id,
		fq: fq,
		hq: hq,
	}
}

func hashUrl(hash string) string {
	return fmt.Sprintf("/ipfs/%s", hash)
}

// Update references with name, parent_hash and parent_name. Returns true when updated
func update_references(references []indexer.Reference, name string, parent_hash string) ([]indexer.Reference, bool) {
	if parent_hash == "" {
		// No parent hash, don't bother adding reference
		return references, false
	}

	for i := range references {
		if references[i].ParentHash == parent_hash {
			log.Printf("Reference '%s' for %s exists, not updating", name, parent_hash)
			return references, false
		}
	}

	log.Printf("Adding reference '%s' for %s", name, parent_hash)

	references = append(references, indexer.Reference{
		Name:       name,
		ParentHash: parent_hash,
	})

	return references, true
}

// Handle IPFS errors graceously, returns try again bool and original error
func (c Crawler) handleError(err error, hash string) (bool, error) {
	if _, ok := err.(*shell.Error); ok && strings.Contains(err.Error(), "proto") {
		// We're not recovering from protocol errors, so panic

		// Attempt to index panic to prevent re-indexing
		metadata := map[string]interface{}{
			"error": err.Error(),
		}

		c.id.IndexItem("invalid", hash, metadata)

		panic(err)
	}

	if uerr, ok := err.(*url.Error); ok {
		// URL errors

		log.Printf("URL error %v", uerr)

		if uerr.Timeout() {
			// Fail on timeouts
			return false, err
		}

		if uerr.Temporary() {
			// Retry on other temp errors
			return true, nil
		}

		// Somehow, the errors below are not temp errors !?
		switch t := uerr.Err.(type) {
		case *net.OpError:
			if t.Op == "dial" {
				log.Printf("Unknown host %v", t)
				return true, nil

			} else if t.Op == "read" {
				log.Printf("Connection refused %v", t)
				return true, nil
			}

		case syscall.Errno:
			if t == syscall.ECONNREFUSED {
				log.Printf("Connection refused %v", t)
				return true, nil
			}
		}
	}

	return false, err
}

func (c Crawler) index_references(hash string, name string, parent_hash string) ([]indexer.Reference, bool, error) {
	var references []indexer.Reference

	references, item_type, err := c.id.GetReferences(hash)
	if err != nil {
		return nil, false, err
	}

	if references != nil {
		log.Printf("Already indexed '%s'.", hash)

		references, references_updated := update_references(references, name, parent_hash)

		if references_updated {
			log.Printf("Updating references for '%s'.", hash)

			properties := map[string]interface{}{
				"references": references,
			}

			err := c.id.IndexItem(item_type, hash, properties)
			if err != nil {
				return nil, false, err
			}
		} else {
			log.Printf("Not updating references for '%s'", hash)
		}

		return references, true, nil
	} else {
		// Initialize references
		references = []indexer.Reference{
			{
				Name:       name,
				ParentHash: parent_hash,
			}}
	}

	return references, false, nil
}

// Given a particular hash (file or directory), start crawling
func (c Crawler) CrawlHash(hash string, name string, parent_hash string, parent_name string) error {
	references, already_indexed, err := c.index_references(hash, name, parent_hash)

	if err != nil {
		return err
	}

	if already_indexed {
		return nil
	}

	log.Printf("Crawling hash '%s' (%s)", hash, name)

	url := hashUrl(hash)

	var list *shell.UnixLsObject

	try_again := true
	for try_again {
		list, err = c.sh.FileList(url)

		try_again, err = c.handleError(err, hash)

		if try_again {
			log.Printf("Retrying in %d seconds", RECONNECT_WAIT)
			time.Sleep(RECONNECT_WAIT * time.Duration(time.Second))
		}
	}

	if err != nil {
		return err
	}

	switch list.Type {
	case "File":
		// Add to file crawl queue
		args := CrawlerArgs{
			Hash:       hash,
			Name:       name,
			Size:       list.Size,
			ParentHash: parent_hash,
		}

		err = c.fq.AddTask(args)
		if err != nil {
			// failed to send the task
			return err
		}
	case "Directory":
		// Index name and size for directory and directory items
		properties := map[string]interface{}{
			"links":      list.Links,
			"size":       list.Size,
			"references": references,
		}

		err := c.id.IndexItem("directory", hash, properties)
		if err != nil {
			return err
		}

		for _, link := range list.Links {
			args := CrawlerArgs{
				Hash:       link.Hash,
				Name:       link.Name,
				Size:       link.Size,
				ParentHash: hash,
			}

			switch link.Type {
			case "File":
				// Add file to crawl queue
				err = c.fq.AddTask(args)
				if err != nil {
					// failed to send the task
					return err
				}

			case "Directory":
				// Add directory to crawl queue
				c.hq.AddTask(args)
				if err != nil {
					// failed to send the task
					return err
				}
			default:
				log.Printf("Type '%s' skipped for '%s'", list.Type, hash)
			}
		}
	default:
		log.Printf("Type '%s' skipped for '%s'", list.Type, hash)
	}

	log.Printf("Finished hash %s", hash)

	return nil
}

func getMetadata(path string, metadata *map[string]interface{}) error {
	const ipfs_tika_url = "http://localhost:8081"

	client := http.Client{
		Timeout: TIKA_TIMEOUT * time.Duration(time.Second),
	}

	resp, err := client.Get(ipfs_tika_url + path)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return fmt.Errorf("Undesired status '%s' from ipfs-tika.", resp.Status)
	}

	// Parse resulting JSON
	if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil {
		return err
	}

	return err
}

// Crawl a single object, known to be a file
func (c Crawler) CrawlFile(hash string, name string, parent_hash string, parent_name string, size uint64) error {
	if size == 262144 && parent_hash == "" {
		// Assertion error.
		// REMOVE ME!
		log.Printf("Skipping unreferenced partial content for %s", hash)
		return nil
	}

	references, already_indexed, err := c.index_references(hash, name, parent_hash)

	if err != nil {
		return err
	}

	if already_indexed {
		return nil
	}

	log.Printf("Crawling file %s (%s)\n", hash, name)

	metadata := make(map[string]interface{})

	if size > 0 {
		if size > METADATA_MAX_SIZE {
			// Fail hard for really large files, for now
			return fmt.Errorf("%s (%s) too large, not indexing (for now).", hash, name)
		}

		var path string
		if name != "" && parent_hash != "" {
			path = fmt.Sprintf("/ipfs/%s/%s", parent_hash, name)
		} else {
			path = fmt.Sprintf("/ipfs/%s", hash)
		}

		try_again := true
		for try_again {
			err = getMetadata(path, &metadata)

			try_again, err = c.handleError(err, hash)

			if try_again {
				log.Printf("Retrying in %d seconds", RECONNECT_WAIT)
				time.Sleep(RECONNECT_WAIT * time.Duration(time.Second))
			}
		}

		if err != nil {
			return err
		}

		// Check for IPFS links in content
		/*
			for raw_url := range metadata.urls {
				url, err := URL.Parse(raw_url)

				if err != nil {
					return err
				}

				if strings.HasPrefix(url.Path, "/ipfs/") {
					// Found IPFS link!
					args := CrawlerArgs{
						Hash:       link.Hash,
						Name:       link.Name,
						Size:       link.Size,
						ParentHash: hash,
					}

				}
			}
		*/
	}

	metadata["size"] = size
	metadata["references"] = references

	err = c.id.IndexItem("file", hash, metadata)
	if err != nil {
		return err
	}

	log.Printf("Finished file %s", hash)

	return nil
}
<commit_msg>Rework index_references.<commit_after>package crawler

import (
	"encoding/json"
	"fmt"
	"github.com/dokterbob/ipfs-search/indexer"
	"github.com/dokterbob/ipfs-search/queue"
	"gopkg.in/ipfs/go-ipfs-api.v1"
	"log"
	"net"
	"net/http"
	"net/url"
	// "path"
	"strings"
	"syscall"
	"time"
)

const (
	// Reconnect time in seconds
	RECONNECT_WAIT    = 2
	TIKA_TIMEOUT      = 120
	METADATA_MAX_SIZE = 50 * 1024 * 1024
)

type CrawlerArgs struct {
	Hash       string
	Name       string
	Size       uint64
	ParentHash string
	ParentName string // This is legacy, should be removed
}

type Crawler struct {
	sh *shell.Shell
	id *indexer.Indexer
	fq *queue.TaskQueue
	hq *queue.TaskQueue
}

func NewCrawler(sh *shell.Shell, id *indexer.Indexer, fq *queue.TaskQueue, hq *queue.TaskQueue) *Crawler {
	return &Crawler{
		sh: sh,
		id: id,
		fq: fq,
		hq: hq,
	}
}

func hashUrl(hash string) string {
	return fmt.Sprintf("/ipfs/%s", hash)
}

// Update references with name, parent_hash and parent_name. Returns true when updated
func update_references(references []indexer.Reference, name string, parent_hash string) ([]indexer.Reference, bool) {
	if references == nil {
		// Initialize empty references when none have been found
		references = []indexer.Reference{}
	}

	if parent_hash == "" {
		// No parent hash, don't bother adding reference
		return references, false
	}

	for i := range references {
		if references[i].ParentHash == parent_hash {
			log.Printf("Reference '%s' for %s exists, not updating", name, parent_hash)
			return references, false
		}
	}

	log.Printf("Adding reference '%s' for %s", name, parent_hash)

	references = append(references, indexer.Reference{
		Name:       name,
		ParentHash: parent_hash,
	})

	return references, true
}

// Handle IPFS errors graceously, returns try again bool and original error
func (c Crawler) handleError(err error, hash string) (bool, error) {
	if _, ok := err.(*shell.Error); ok && strings.Contains(err.Error(), "proto") {
		// We're not recovering from protocol errors, so panic

		// Attempt to index panic to prevent re-indexing
		metadata := map[string]interface{}{
			"error": err.Error(),
		}

		c.id.IndexItem("invalid", hash, metadata)

		panic(err)
	}

	if uerr, ok := err.(*url.Error); ok {
		// URL errors

		log.Printf("URL error %v", uerr)

		if uerr.Timeout() {
			// Fail on timeouts
			return false, err
		}

		if uerr.Temporary() {
			// Retry on other temp errors
			return true, nil
		}

		// Somehow, the errors below are not temp errors !?
		switch t := uerr.Err.(type) {
		case *net.OpError:
			if t.Op == "dial" {
				log.Printf("Unknown host %v", t)
				return true, nil

			} else if t.Op == "read" {
				log.Printf("Connection refused %v", t)
				return true, nil
			}

		case syscall.Errno:
			if t == syscall.ECONNREFUSED {
				log.Printf("Connection refused %v", t)
				return true, nil
			}
		}
	}

	return false, err
}

func (c Crawler) index_references(hash string, name string, parent_hash string) ([]indexer.Reference, bool, error) {
	var already_indexed bool

	references, item_type, err := c.id.GetReferences(hash)
	if err != nil {
		return nil, false, err
	}

	// TODO: Handle this more explicitly, use and detect NotFound
	if references == nil {
		already_indexed = false
	} else {
		already_indexed = true
	}

	references, references_updated := update_references(references, name, parent_hash)

	if already_indexed {
		if references_updated {
			log.Printf("Found '%s', updating references.", hash)

			properties := map[string]interface{}{
				"references": references,
			}

			err := c.id.IndexItem(item_type, hash, properties)
			if err != nil {
				return nil, false, err
			}
		} else {
			log.Printf("Found '%s', not updating references.", hash)
		}
	}

	return references, already_indexed, nil
}

// Given a particular hash (file or directory), start crawling
func (c Crawler) CrawlHash(hash string, name string, parent_hash string, parent_name string) error {
	references, already_indexed, err := c.index_references(hash, name, parent_hash)

	if err != nil {
		return err
	}

	if already_indexed {
		return nil
	}

	log.Printf("Crawling hash '%s' (%s)", hash, name)

	url := hashUrl(hash)

	var list *shell.UnixLsObject

	try_again := true
	for try_again {
		list, err = c.sh.FileList(url)

		try_again, err = c.handleError(err, hash)

		if try_again {
			log.Printf("Retrying in %d seconds", RECONNECT_WAIT)
			time.Sleep(RECONNECT_WAIT * time.Duration(time.Second))
		}
	}

	if err != nil {
		return err
	}

	switch list.Type {
	case "File":
		// Add to file crawl queue
		args := CrawlerArgs{
			Hash:       hash,
			Name:       name,
			Size:       list.Size,
			ParentHash: parent_hash,
		}

		err = c.fq.AddTask(args)
		if err != nil {
			// failed to send the task
			return err
		}
	case "Directory":
		// Index name and size for directory and directory items
		properties := map[string]interface{}{
			"links":      list.Links,
			"size":       list.Size,
			"references": references,
		}

		err := c.id.IndexItem("directory", hash, properties)
		if err != nil {
			return err
		}

		for _, link := range list.Links {
			args := CrawlerArgs{
				Hash:       link.Hash,
				Name:       link.Name,
				Size:       link.Size,
				ParentHash: hash,
			}

			switch link.Type {
			case "File":
				// Add file to crawl queue
				err = c.fq.AddTask(args)
				if err != nil {
					// failed to send the task
					return err
				}

			case "Directory":
				// Add directory to crawl queue
				c.hq.AddTask(args)
				if err != nil {
					// failed to send the task
					return err
				}
			default:
				log.Printf("Type '%s' skipped for '%s'", list.Type, hash)
			}
		}
	default:
		log.Printf("Type '%s' skipped for '%s'", list.Type, hash)
	}

	log.Printf("Finished hash %s", hash)

	return nil
}

func getMetadata(path string, metadata *map[string]interface{}) error {
	const ipfs_tika_url = "http://localhost:8081"

	client := http.Client{
		Timeout: TIKA_TIMEOUT * time.Duration(time.Second),
	}

	resp, err := client.Get(ipfs_tika_url + path)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return fmt.Errorf("Undesired status '%s' from ipfs-tika.", resp.Status)
	}

	// Parse resulting JSON
	if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil {
		return err
	}

	return err
}

// Crawl a single object, known to be a file
func (c Crawler) CrawlFile(hash string, name string, parent_hash string, parent_name string, size uint64) error {
	if size == 262144 && parent_hash == "" {
		// Assertion error.
		// REMOVE ME!
		log.Printf("Skipping unreferenced partial content for %s", hash)
		return nil
	}

	references, already_indexed, err := c.index_references(hash, name, parent_hash)

	if err != nil {
		return err
	}

	if already_indexed {
		return nil
	}

	log.Printf("Crawling file %s (%s)\n", hash, name)

	metadata := make(map[string]interface{})

	if size > 0 {
		if size > METADATA_MAX_SIZE {
			// Fail hard for really large files, for now
			return fmt.Errorf("%s (%s) too large, not indexing (for now).", hash, name)
		}

		var path string
		if name != "" && parent_hash != "" {
			path = fmt.Sprintf("/ipfs/%s/%s", parent_hash, name)
		} else {
			path = fmt.Sprintf("/ipfs/%s", hash)
		}

		try_again := true
		for try_again {
			err = getMetadata(path, &metadata)

			try_again, err = c.handleError(err, hash)

			if try_again {
				log.Printf("Retrying in %d seconds", RECONNECT_WAIT)
				time.Sleep(RECONNECT_WAIT * time.Duration(time.Second))
			}
		}

		if err != nil {
			return err
		}

		// Check for IPFS links in content
		/*
			for raw_url := range metadata.urls {
				url, err := URL.Parse(raw_url)

				if err != nil {
					return err
				}

				if strings.HasPrefix(url.Path, "/ipfs/") {
					// Found IPFS link!
					args := CrawlerArgs{
						Hash:       link.Hash,
						Name:       link.Name,
						Size:       link.Size,
						ParentHash: hash,
					}

				}
			}
		*/
	}

	metadata["size"] = size
	metadata["references"] = references

	err = c.id.IndexItem("file", hash, metadata)
	if err != nil {
		return err
	}

	log.Printf("Finished file %s", hash)

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/grafana/grafana/pkg/cmd/grafana-cli/commands"
	"github.com/grafana/grafana/pkg/cmd/grafana-cli/log"
	"os"
	"runtime"
)

var version = "master"

func getGrafanaPluginPath() string {
	//TODO: try to get path from os:env GF_PLUGIN_FOLDER

	os := runtime.GOOS
	if os == "windows" {
		return "C:\\opt\\grafana\\plugins"
	} else {
		return "/var/lib/grafana/plugins"
	}
}

func main() {
	SetupLogging()

	app := cli.NewApp()
	app.Name = "Grafana cli"
	app.Author = "raintank"
	app.Email = "https://github.com/grafana/grafana"
	app.Version = version
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "path",
			Usage: "path to the grafana installation",
			Value: getGrafanaPluginPath(),
		},
		cli.StringFlag{
			Name:  "repo",
			Usage: "url to the plugin repository",
			Value: "https://raw.githubusercontent.com/grafana/grafana-plugin-repository/master/repo.json",
		},
		cli.BoolFlag{
			Name:  "debug, d",
			Usage: "enable debug logging",
		},
	}

	app.Commands = commands.Commands
	app.CommandNotFound = cmdNotFound

	if err := app.Run(os.Args); err != nil {
		log.Errorf("%v", err)
	}
}

func SetupLogging() {
	for _, f := range os.Args {
		if f == "-D" || f == "--debug" || f == "-debug" {
			log.SetDebug(true)
		}
	}
}

func cmdNotFound(c *cli.Context, command string) {
	fmt.Printf(
		"%s: '%s' is not a %s command. See '%s --help'.\n",
		c.App.Name,
		command,
		c.App.Name,
		os.Args[0],
	)
	os.Exit(1)
}
<commit_msg>feat(cli): add suppot for plugindir as environment variable<commit_after>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/grafana/grafana/pkg/cmd/grafana-cli/commands"
	"github.com/grafana/grafana/pkg/cmd/grafana-cli/log"
	"os"
	"runtime"
)

var version = "master"

func getGrafanaPluginPath() string {
	if os.Getenv("GF_PLUGIN_DIR") != "" {
		return os.Getenv("GF_PLUGIN_DIR")
	}

	os := runtime.GOOS
	if os == "windows" {
		return "C:\\opt\\grafana\\plugins"
	} else {
		return "/var/lib/grafana/plugins"
	}
}

func main() {
	SetupLogging()

	app := cli.NewApp()
	app.Name = "Grafana cli"
	app.Author = "raintank"
	app.Email = "https://github.com/grafana/grafana"
	app.Version = version
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "path",
			Usage: "path to the grafana installation",
			Value: getGrafanaPluginPath(),
		},
		cli.StringFlag{
			Name:  "repo",
			Usage: "url to the plugin repository",
			Value: "https://raw.githubusercontent.com/grafana/grafana-plugin-repository/master/repo.json",
		},
		cli.BoolFlag{
			Name:  "debug, d",
			Usage: "enable debug logging",
		},
	}

	app.Commands = commands.Commands
	app.CommandNotFound = cmdNotFound

	if err := app.Run(os.Args); err != nil {
		log.Errorf("%v", err)
	}
}

func SetupLogging() {
	for _, f := range os.Args {
		if f == "-D" || f == "--debug" || f == "-debug" {
			log.SetDebug(true)
		}
	}
}

func cmdNotFound(c *cli.Context, command string) {
	fmt.Printf(
		"%s: '%s' is not a %s command. See '%s --help'.\n",
		c.App.Name,
		command,
		c.App.Name,
		os.Args[0],
	)
	os.Exit(1)
}
<|endoftext|>


<commit_before>package email

import (
	"code.google.com/p/go-uuid/uuid"

	"bytes"
	"encoding/json"
	"fmt"
	"mime/multipart"
	"net/mail"
	"net/textproto"
	"os"
	"path"
	"strings"
	"time"
)

// Individual message for sending to a host.
type Email struct {
	Id      string
	Host    string
	From    string
	To      []string
	Message []byte
}

// Add the specified content to a multipart/alternative writer.
func addPart(w *multipart.Writer, contentType string, content []byte) error {
	header := make(textproto.MIMEHeader)
	header.Add("Content-Type", contentType)
	writer, err := w.CreatePart(header)
	if err != nil {
		return err
	}
	_, err = writer.Write(content)
	return err
}

// Create a multipart/alternative body with the specified data. Both the body
// and the separator are returned. Because data is written to a bytes.Buffer,
// it is probably safe to ignore any errors.
func createMultipartBody(text, html string) ([]byte, string) {
	var (
		buff   = &bytes.Buffer{}
		writer = multipart.NewWriter(buff)
	)
	addPart(writer, "text/plain", []byte(text))
	addPart(writer, "text/html", []byte(html))
	writer.Close()
	return buff.Bytes(), writer.Boundary()
}

// Attempt to extract the host from the provided email address.
func hostFromAddress(data string) (string, error) {
	if addr, err := mail.ParseAddress(data); err != nil {
		return "", err
	} else {
		return strings.Split(addr.Address, "@")[1], nil
	}
}

// Group a list of email addresses by their host.
func groupAddressesByHost(addrs []string) (map[string][]string, error) {
	m := make(map[string][]string)
	for _, addr := range addrs {
		if host, err := hostFromAddress(addr); err != nil {
			return nil, err
		} else {
			if m[host] == nil {
				m[host] = make([]string, 0, 1)
			}
			m[host] = append(m[host], addr)
		}
	}
	return m, nil
}

// Create an array of emails from the specified information. An email will be
// generated for each individual host and for each BCC recipient.
func NewEmails(from string, to, cc, bcc []string, subject string, text, html string) ([]*Email, error) {

	// Retrieve the current hostname
	hostname, err := os.Hostname()
	if err != nil {
		return nil, err
	}

	// Create the array of emails that will be returned, the buffer that will
	// be used for generating the message, and each of the individual parts
	// of the message
	var (
		emails         = make([]*Email, 0, 1)
		buff           = &bytes.Buffer{}
		id             = fmt.Sprintf("<%s@%s>", uuid.New(), hostname)
		body, boundary = createMultipartBody(text, html)
		hdrs           = map[string]string{
			"Message-ID":   id,
			"From":         from,
			"To":           strings.Join(to, ","),
			"Subject":      subject,
			"Date":         time.Now().Format(time.RubyDate),
			"MIME-Version": "1.0",
			"Content-Type": fmt.Sprintf("multipart/alternative; boundary=\"%s\"", boundary),
		}
	)

	// If any Cc addresses were provided, add them to the headers
	if len(cc) > 0 {
		hdrs["Cc"] = strings.Join(cc, ",")
	}

	// Write all of the headers followed by the body
	for h := range hdrs {
		buff.Write([]byte(fmt.Sprintf("%s: %s\r\n", h, hdrs[h])))
	}

	// Write the extra CRLF and body
	buff.Write([]byte("\r\n"))
	buff.Write(body)

	// Combine the "To", "Cc", and "Bcc" addresses and group them by hostname
	// in order to make delivery much more efficient
	addrMap, err := groupAddressesByHost(append(append(to, cc...), bcc...))
	for host := range addrMap {
		emails = append(emails, &Email{
			Id:      id,
			Host:    host,
			From:    from,
			To:      addrMap[host],
			Message: buff.Bytes(),
		})
	}

	return emails, nil
}

// Load an email from disk.
func LoadEmail(filename string) (*Email, error) {

	// Attempt to open the file
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	// Decode the JSON from the file
	var e Email
	if err = json.NewDecoder(f).Decode(&e); err != nil {
		return nil, err
	}

	return &e, nil
}

// Save the email to the specified directory.
func (e *Email) Save(directory string) error {

	// Attempt to create the , making sure nobody else can read it
	f, err := os.OpenFile(path.Join(directory, e.Id), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err
	}

	// Encode the data as JSON
	if err = json.NewEncoder(f).Encode(e); err != nil {
		return err
	}

	return nil
}

// Delete the email from the specified directory.
func (e *Email) Delete(directory string) error {
	return os.Remove(path.Join(directory, e.Id))
}
<commit_msg>Adjusted the filename of emails written to disk.<commit_after>package email

import (
	"code.google.com/p/go-uuid/uuid"

	"bytes"
	"encoding/json"
	"fmt"
	"mime/multipart"
	"net/mail"
	"net/textproto"
	"os"
	"path"
	"strings"
	"time"
)

// Individual message for sending to a host.
type Email struct {
	Id      string
	Host    string
	From    string
	To      []string
	Message []byte
}

// Add the specified content to a multipart/alternative writer.
func addPart(w *multipart.Writer, contentType string, content []byte) error {
	header := make(textproto.MIMEHeader)
	header.Add("Content-Type", contentType)
	writer, err := w.CreatePart(header)
	if err != nil {
		return err
	}
	_, err = writer.Write(content)
	return err
}

// Create a multipart/alternative body with the specified data. Both the body
// and the separator are returned. Because data is written to a bytes.Buffer,
// it is probably safe to ignore any errors.
func createMultipartBody(text, html string) ([]byte, string) {
	var (
		buff   = &bytes.Buffer{}
		writer = multipart.NewWriter(buff)
	)
	addPart(writer, "text/plain", []byte(text))
	addPart(writer, "text/html", []byte(html))
	writer.Close()
	return buff.Bytes(), writer.Boundary()
}

// Attempt to extract the host from the provided email address.
func hostFromAddress(data string) (string, error) {
	if addr, err := mail.ParseAddress(data); err != nil {
		return "", err
	} else {
		return strings.Split(addr.Address, "@")[1], nil
	}
}

// Group a list of email addresses by their host.
func groupAddressesByHost(addrs []string) (map[string][]string, error) {
	m := make(map[string][]string)
	for _, addr := range addrs {
		if host, err := hostFromAddress(addr); err != nil {
			return nil, err
		} else {
			if m[host] == nil {
				m[host] = make([]string, 0, 1)
			}
			m[host] = append(m[host], addr)
		}
	}
	return m, nil
}

// Create an array of emails from the specified information. An email will be
// generated for each individual host and for each BCC recipient.
func NewEmails(from string, to, cc, bcc []string, subject string, text, html string) ([]*Email, error) {

	// Retrieve the current hostname
	hostname, err := os.Hostname()
	if err != nil {
		return nil, err
	}

	// Create the array of emails that will be returned, the buffer that will
	// be used for generating the message, and each of the individual parts
	// of the message
	var (
		emails         = make([]*Email, 0, 1)
		buff           = &bytes.Buffer{}
		id             = uuid.New()
		body, boundary = createMultipartBody(text, html)
		hdrs           = map[string]string{
			"Message-ID":   fmt.Sprintf("<%s@%s>", id, hostname),
			"From":         from,
			"To":           strings.Join(to, ","),
			"Subject":      subject,
			"Date":         time.Now().Format(time.RubyDate),
			"MIME-Version": "1.0",
			"Content-Type": fmt.Sprintf("multipart/alternative; boundary=\"%s\"", boundary),
		}
	)

	// If any Cc addresses were provided, add them to the headers
	if len(cc) > 0 {
		hdrs["Cc"] = strings.Join(cc, ",")
	}

	// Write all of the headers followed by the body
	for h := range hdrs {
		buff.Write([]byte(fmt.Sprintf("%s: %s\r\n", h, hdrs[h])))
	}

	// Write the extra CRLF and body
	buff.Write([]byte("\r\n"))
	buff.Write(body)

	// Combine the "To", "Cc", and "Bcc" addresses and group them by hostname
	// in order to make delivery much more efficient
	addrMap, err := groupAddressesByHost(append(append(to, cc...), bcc...))
	for host := range addrMap {
		emails = append(emails, &Email{
			Id:      id,
			Host:    host,
			From:    from,
			To:      addrMap[host],
			Message: buff.Bytes(),
		})
	}

	return emails, nil
}

// Load an email from disk.
func LoadEmail(filename string) (*Email, error) {

	// Attempt to open the file
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	// Decode the JSON from the file
	var e Email
	if err = json.NewDecoder(f).Decode(&e); err != nil {
		return nil, err
	}

	return &e, nil
}

// Save the email to the specified directory.
func (e *Email) Save(directory string) error {

	// Attempt to create the , making sure nobody else can read it
	f, err := os.OpenFile(path.Join(directory, e.Id), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err
	}

	// Encode the data as JSON
	if err = json.NewEncoder(f).Encode(e); err != nil {
		return err
	}

	return nil
}

// Delete the email from the specified directory.
func (e *Email) Delete(directory string) error {
	return os.Remove(path.Join(directory, e.Id))
}
<|endoftext|>


<commit_before>package wats

import (
	"fmt"
	"net/url"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
)

var _ = Describe("Logs from apps hosted by Diego", func() {
	BeforeEach(func() {
		Eventually(pushNora(appName), CF_PUSH_TIMEOUT).Should(Succeed())
		enableDiego(appName)
		Eventually(runCf("start", appName), CF_PUSH_TIMEOUT).Should(Succeed())
	})

	Context("when the app is running", func() {
		BeforeEach(func() {
			Eventually(helpers.CurlingAppRoot(appName)).Should(ContainSubstring("hello i am nora"))
		})

		It("captures stdout logs with the correct tag", func() {
			var message string
			var logs *Session

			By("logging application stdout")
			message = "message-from-stdout"
			helpers.CurlApp(appName, fmt.Sprintf("/print/%s", url.QueryEscape(message)))
			//TODO: make nora output message
			//			Eventually(helpers.CurlApp(appName, fmt.Sprintf("/print/%s", url.QueryEscape(message)))).Should(ContainSubstring(message))

			logs = cf.Cf("logs", appName, "--recent")
			Eventually(logs).Should(Exit(0))
			Expect(logs.Out).To(Say(fmt.Sprintf("\\[APP/0\\]\\s+OUT %s", message)))

		})

		It("captures stderr logs with the correct tag", func() {
			var message string
			var logs *Session

			By("logging application stderr")
			message = "message-from-stderr"
			helpers.CurlApp(appName, fmt.Sprintf("/print_err/%s", url.QueryEscape(message)))

			logs = cf.Cf("logs", appName, "--recent")
			Eventually(logs).Should(Exit(0))
			Expect(logs.Out).To(Say(fmt.Sprintf("\\[APP/0\\]\\s+ERR %s", message)))
		})
	})
})
<commit_msg>Router changed logging for apps<commit_after>package wats

import (
	"fmt"
	"net/url"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
)

var _ = Describe("Logs from apps hosted by Diego", func() {
	BeforeEach(func() {
		Eventually(pushNora(appName), CF_PUSH_TIMEOUT).Should(Succeed())
		enableDiego(appName)
		Eventually(runCf("start", appName), CF_PUSH_TIMEOUT).Should(Succeed())
	})

	Context("when the app is running", func() {
		BeforeEach(func() {
			Eventually(helpers.CurlingAppRoot(appName)).Should(ContainSubstring("hello i am nora"))
		})

		It("captures stdout logs with the correct tag", func() {
			var message string
			var logs *Session

			By("logging application stdout")
			message = "message-from-stdout"
			helpers.CurlApp(appName, fmt.Sprintf("/print/%s", url.QueryEscape(message)))
			//TODO: make nora output message
			//			Eventually(helpers.CurlApp(appName, fmt.Sprintf("/print/%s", url.QueryEscape(message)))).Should(ContainSubstring(message))

			logs = cf.Cf("logs", appName, "--recent")
			Eventually(logs).Should(Exit(0))
			Expect(logs.Out).To(Say(fmt.Sprintf("\\[APP(.*)/0\\]\\s*OUT %s", message)))

		})

		It("captures stderr logs with the correct tag", func() {
			var message string
			var logs *Session

			By("logging application stderr")
			message = "message-from-stderr"
			helpers.CurlApp(appName, fmt.Sprintf("/print_err/%s", url.QueryEscape(message)))

			logs = cf.Cf("logs", appName, "--recent")
			Eventually(logs).Should(Exit(0))
			Expect(logs.Out).To(Say(fmt.Sprintf("\\[APP(.*)/0\\]\\s*ERR %s", message)))
		})
	})
})
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"regexp"
	"strings"
)

const (
	startPrefix = "=== RUN "
	passPrefix = "--- PASS: "
	failPrefix = "--- FAIL: "
)

var endRegexp *regexp.Regexp = regexp.MustCompile(`([^ ]+) \((\d+\.\d+)`)

type Test struct {
	Name, Time, Message string
	Failed bool
}

func parseEnd(prefix, line string) (string, string, error) {
	matches := endRegexp.FindStringSubmatch(line[len(prefix):])

	if len(matches) == 0 {
		return "", "", fmt.Errorf("can't parse %s", line)
	}

	return matches[1], matches[2], nil
}

func parseOutput(rd io.Reader) ([]*Test, error) {
	tests := []*Test{}

	reader := bufio.NewReader(rd)
	var test *Test = nil
	for {
		/* FIXME: Handle isPrefix */
		buf, _, err := reader.ReadLine()

		switch err {
		case io.EOF:
			if test != nil {
				tests = append(tests, test)
			}
			return tests, nil
		case nil:
			;
		default:
			return nil, err
		}

		line := string(buf)
		switch {
		case strings.HasPrefix(line, startPrefix):
			if test != nil {
				tests = append(tests, test)
			}
			test = &Test{Name: line[len(startPrefix):]}
		case strings.HasPrefix(line, failPrefix):
			if test == nil {
				return nil, fmt.Errorf("fail not inside test")
			}
			test.Failed = true
			name, time, err := parseEnd(failPrefix, line)
			if err != nil {
				return nil, err
			}
			if name != test.Name {
				return nil, fmt.Errorf("wrong test end (%s!=%s)", name, test.Name)
			}
			test.Time = time
		case strings.HasPrefix(line, passPrefix):
			if test == nil {
				return nil, fmt.Errorf("pass not inside test")
			}
			test.Failed = false
			name, time, err := parseEnd(passPrefix, line)
			if err != nil {
				return nil, err
			}
			if name != test.Name {
				return nil, fmt.Errorf("wrong test end (%s!=%s)", name, test.Name)
			}
			test.Time = time
		default:
			if test != nil {
				test.Message += line + "\n"
			}
		}
	}

	return tests, nil
}

func getInput(filename string) (io.Reader, error) {
	if filename == "-" || filename == "" {
		return os.Stdin, nil
	}

	return os.Open(filename)
}

func numFailures(tests []*Test) int {
	count := 0
	for _, test := range tests {
		if test.Failed {
			count ++
		}
	}

	return count
}

func writeXML(tests []*Test, out io.Writer) {
	newline := func() { fmt.Fprintln(out) }

	fmt.Fprintf(out, `<?xml version="1.0" encoding="utf-8"?>`)
	newline()
	fmt.Fprintf(out, `<testsuite name="go2xunit" tests="%d" errors="0" failures="%d" skip="0">`,
					 len(tests), numFailures(tests))
	newline()
	for _, test := range(tests) {
		fmt.Fprintf(out, `  <testcase classname="go2xunit" name="%s" time="%s"`,
		                 test.Name, test.Time)
		if !test.Failed {
			fmt.Fprintf(out, " />\n")
			continue
		}
		fmt.Fprintln(out, ">")
		fmt.Fprintf(out,  `    <failure type="go.error" message="error">`)
		newline()
		fmt.Fprintf(out, "<![CDATA[%s]]>\n", test.Message)
		fmt.Fprintln(out, "    </failure>")
		fmt.Fprintln(out, "  </testcase>")
	}
	fmt.Fprintln(out, "</testsuite>")
}

func getOutput(filename string) (io.Writer, error) {
	if filename == "-" || filename == "" {
		return os.Stdout, nil
	}

	return os.Create(filename)
}

func getIO(inputFile, outputFile string) (io.Reader, io.Writer, error) {
	input, err := getInput(inputFile)
	if err != nil {
		return nil, nil, fmt.Errorf("can't open %s for reading: %s", inputFile, err)
	}

	output, err := getOutput(outputFile)
	if err != nil {
		return nil, nil, fmt.Errorf("can't open %s for writing: %s", outputFile, err)
	}

	return input, output, nil
}

func fatal(formt string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, formt, args...)
	fmt.Fprintln(os.Stderr)
	os.Exit(1)
}

func main() {
	inputFile := flag.String("input", "", "input file (default to stdin)")
	outputFile := flag.String("output", "", "output file (default to stdout)")
	//run := flag.Bool("run", false, "run go test yourself")
	flag.Parse()

	if flag.NArg() > 0 {
		fatal("error: %s does not take parameters (did you mean -input?)", os.Args[0])
	}

	/*
	if len(*inputFile) > 0 && *run {
		log.Fatalf("error: can't specify -run and -input at the same time\n")
	}
	*/

	input, output, err := getIO(*inputFile, *outputFile)
	if err != nil {
		fatal("error: %s", err)
	}

	tests, err := parseOutput(input)
	if err != nil {
		fatal("error: %s", err)
	}

	writeXML(tests, output)
}
<commit_msg>No run, -fail, fatal<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"regexp"
	"strings"
)

const (
	startPrefix = "=== RUN "
	passPrefix = "--- PASS: "
	failPrefix = "--- FAIL: "
)

var endRegexp *regexp.Regexp = regexp.MustCompile(`([^ ]+) \((\d+\.\d+)`)

type Test struct {
	Name, Time, Message string
	Failed bool
}

func parseEnd(prefix, line string) (string, string, error) {
	matches := endRegexp.FindStringSubmatch(line[len(prefix):])

	if len(matches) == 0 {
		return "", "", fmt.Errorf("can't parse %s", line)
	}

	return matches[1], matches[2], nil
}

func parseOutput(rd io.Reader) ([]*Test, error) {
	tests := []*Test{}

	reader := bufio.NewReader(rd)
	var test *Test = nil
	for {
		/* FIXME: Handle isPrefix */
		buf, _, err := reader.ReadLine()

		switch err {
		case io.EOF:
			if test != nil {
				tests = append(tests, test)
			}
			return tests, nil
		case nil:
			;
		default:
			return nil, err
		}

		line := string(buf)
		switch {
		case strings.HasPrefix(line, startPrefix):
			if test != nil {
				tests = append(tests, test)
			}
			test = &Test{Name: line[len(startPrefix):]}
		case strings.HasPrefix(line, failPrefix):
			if test == nil {
				return nil, fmt.Errorf("fail not inside test")
			}
			test.Failed = true
			name, time, err := parseEnd(failPrefix, line)
			if err != nil {
				return nil, err
			}
			if name != test.Name {
				return nil, fmt.Errorf("wrong test end (%s!=%s)", name, test.Name)
			}
			test.Time = time
		case strings.HasPrefix(line, passPrefix):
			if test == nil {
				return nil, fmt.Errorf("pass not inside test")
			}
			test.Failed = false
			name, time, err := parseEnd(passPrefix, line)
			if err != nil {
				return nil, err
			}
			if name != test.Name {
				return nil, fmt.Errorf("wrong test end (%s!=%s)", name, test.Name)
			}
			test.Time = time
		default:
			if test != nil {
				test.Message += line + "\n"
			}
		}
	}

	return tests, nil
}

func getInput(filename string) (io.Reader, error) {
	if filename == "-" || filename == "" {
		return os.Stdin, nil
	}

	return os.Open(filename)
}

func numFailures(tests []*Test) int {
	count := 0
	for _, test := range tests {
		if test.Failed {
			count ++
		}
	}

	return count
}

func writeXML(tests []*Test, out io.Writer) {
	newline := func() { fmt.Fprintln(out) }

	fmt.Fprintf(out, `<?xml version="1.0" encoding="utf-8"?>`)
	newline()
	fmt.Fprintf(out, `<testsuite name="go2xunit" tests="%d" errors="0" failures="%d" skip="0">`,
					 len(tests), numFailures(tests))
	newline()
	for _, test := range(tests) {
		fmt.Fprintf(out, `  <testcase classname="go2xunit" name="%s" time="%s"`,
		                 test.Name, test.Time)
		if !test.Failed {
			fmt.Fprintf(out, " />\n")
			continue
		}
		fmt.Fprintln(out, ">")
		fmt.Fprintf(out,  `    <failure type="go.error" message="error">`)
		newline()
		fmt.Fprintf(out, "<![CDATA[%s]]>\n", test.Message)
		fmt.Fprintln(out, "    </failure>")
		fmt.Fprintln(out, "  </testcase>")
	}
	fmt.Fprintln(out, "</testsuite>")
}

func getOutput(filename string) (io.Writer, error) {
	if filename == "-" || filename == "" {
		return os.Stdout, nil
	}

	return os.Create(filename)
}

func getIO(inputFile, outputFile string) (io.Reader, io.Writer, error) {
	input, err := getInput(inputFile)
	if err != nil {
		return nil, nil, fmt.Errorf("can't open %s for reading: %s", inputFile, err)
	}

	output, err := getOutput(outputFile)
	if err != nil {
		return nil, nil, fmt.Errorf("can't open %s for writing: %s", outputFile, err)
	}

	return input, output, nil
}

func fatal(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, fmt.Sprintf("error: %s", format), args...)
	fmt.Fprintln(os.Stderr)
	os.Exit(1)
}

func main() {
	inputFile := flag.String("input", "", "input file (default to stdin)")
	outputFile := flag.String("output", "", "output file (default to stdout)")
	fail := flag.Bool("fail", false, "fail (non zero exit) if any test failed")
	flag.Parse()

	if flag.NArg() > 0 {
		fatal("%s does not take parameters (did you mean -input?)", os.Args[0])
	}

	input, output, err := getIO(*inputFile, *outputFile) //, *run)
	if err != nil {
		fatal("%s", err)
	}

	tests, err := parseOutput(input)
	if err != nil {
		fatal("%s", err)
	}
	if len(tests) == 0 {
		fatal("no tests found")
		os.Exit(1)
	}

	writeXML(tests, output)
	if *fail && numFailures(tests) > 0 {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright 2014 VMware, Inc.  All rights reserved.  Licensed under the Apache v2 License.
 */

package vmwarefusion

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/docker/machine/libmachine/log"
)

var (
	vmrunbin    = setVmwareCmd("vmrun")
	vdiskmanbin = setVmwareCmd("vmware-vdiskmanager")
)

var (
	ErrMachineExist    = errors.New("machine already exists")
	ErrMachineNotExist = errors.New("machine does not exist")
	ErrVMRUNNotFound   = errors.New("VMRUN not found")
)

// detect the vmrun and vmware-vdiskmanager cmds' path if needed
func setVmwareCmd(cmd string) string {
	if path, err := exec.LookPath(cmd); err == nil {
		return path
	}
	return filepath.Join("/Applications/VMware Fusion.app/Contents/Library/", cmd)
}

func vmrun(args ...string) (string, string, error) {
	cmd := exec.Command(vmrunbin, args...)
	if os.Getenv("MACHINE_DEBUG") != "" {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}

	var stdout bytes.Buffer
	var stderr bytes.Buffer
	cmd.Stdout, cmd.Stderr = &stdout, &stderr
	log.Debugf("executing: %v %v", vmrunbin, strings.Join(args, " "))

	err := cmd.Run()
	if err != nil {
		if ee, ok := err.(*exec.Error); ok && ee == exec.ErrNotFound {
			err = ErrVMRUNNotFound
		}
	}

	return stdout.String(), stderr.String(), err
}

// Make a vmdk disk image with the given size (in MB).
func vdiskmanager(dest string, size int) error {
	cmd := exec.Command(vdiskmanbin, "-c", "-t", "0", "-s", fmt.Sprintf("%dMB", size), "-a", "lsilogic", dest)
	if os.Getenv("MACHINE_DEBUG") != "" {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}

	if stdout := cmd.Run(); stdout != nil {
		if ee, ok := stdout.(*exec.Error); ok && ee == exec.ErrNotFound {
			return ErrVMRUNNotFound
		}
	}
	return nil
}
<commit_msg>Explicitly set umask before invoking vmrun in vmwarefusion<commit_after>/*
 * Copyright 2014 VMware, Inc.  All rights reserved.  Licensed under the Apache v2 License.
 */

package vmwarefusion

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/docker/machine/libmachine/log"
)

var (
	vmrunbin    = setVmwareCmd("vmrun")
	vdiskmanbin = setVmwareCmd("vmware-vdiskmanager")
)

var (
	ErrMachineExist    = errors.New("machine already exists")
	ErrMachineNotExist = errors.New("machine does not exist")
	ErrVMRUNNotFound   = errors.New("VMRUN not found")
)

// detect the vmrun and vmware-vdiskmanager cmds' path if needed
func setVmwareCmd(cmd string) string {
	if path, err := exec.LookPath(cmd); err == nil {
		return path
	}
	return filepath.Join("/Applications/VMware Fusion.app/Contents/Library/", cmd)
}

func vmrun(args ...string) (string, string, error) {
	// vmrun with nogui on VMware Fusion through at least 8.0.1 doesn't work right
	// if the umask is set to not allow world-readable permissions
	_ = syscall.Umask(022)
	cmd := exec.Command(vmrunbin, args...)
	if os.Getenv("MACHINE_DEBUG") != "" {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}

	var stdout bytes.Buffer
	var stderr bytes.Buffer
	cmd.Stdout, cmd.Stderr = &stdout, &stderr
	log.Debugf("executing: %v %v", vmrunbin, strings.Join(args, " "))

	err := cmd.Run()
	if err != nil {
		if ee, ok := err.(*exec.Error); ok && ee == exec.ErrNotFound {
			err = ErrVMRUNNotFound
		}
	}

	return stdout.String(), stderr.String(), err
}

// Make a vmdk disk image with the given size (in MB).
func vdiskmanager(dest string, size int) error {
	cmd := exec.Command(vdiskmanbin, "-c", "-t", "0", "-s", fmt.Sprintf("%dMB", size), "-a", "lsilogic", dest)
	if os.Getenv("MACHINE_DEBUG") != "" {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}

	if stdout := cmd.Run(); stdout != nil {
		if ee, ok := stdout.(*exec.Error); ok && ee == exec.ErrNotFound {
			return ErrVMRUNNotFound
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"testing"
	"time"
)

var testArgs = OpsArgs{"testKey", "testName", "testDescription", 99, "month", time.Second * 10, true}

func TestCreateUrl(t *testing.T) {
	var requestParams = make(map[string]string)
	requestParams["apiKey"] = "test"
	var url = createUrl("/v1/test", requestParams)
	var testUrl = "https://api.opsgenie.com/v1/test?apiKey=test"
	if url != testUrl {
		t.Errorf("Url not correct is [%s] but should be [%s]", url, testUrl)
	}
}

func TestAllContentParams(t *testing.T) {
	var all = allContentParams(testArgs)
	if all["apiKey"] != testArgs.apiKey || all["name"] != testArgs.name || all["description"] != testArgs.description || all["interval"] != testArgs.interval || all["intervalUnit"] != testArgs.intervalUnit {
		t.Errorf("OpsArgs [%+v] are not the same as all content params [%s]", testArgs, all)
	}
}
<commit_msg>test mandatory request params<commit_after>package main

import (
	"testing"
	"time"
)

var testArgs = OpsArgs{"testKey", "testName", "testDescription", 99, "month", time.Second * 10, true}

func TestCreateUrl(t *testing.T) {
	var requestParams = make(map[string]string)
	requestParams["apiKey"] = "test"
	var url = createUrl("/v1/test", requestParams)
	var testUrl = "https://api.opsgenie.com/v1/test?apiKey=test"
	if url != testUrl {
		t.Errorf("Url not correct is [%s] but should be [%s]", url, testUrl)
	}
}

func TestAllContentParams(t *testing.T) {
	var all = allContentParams(testArgs)
	if all["apiKey"] != testArgs.apiKey && all["name"] != testArgs.name && all["description"] != testArgs.description && all["interval"] != testArgs.interval && all["intervalUnit"] != testArgs.intervalUnit {
		t.Errorf("OpsArgs [%+v] are not the same as all content params [%s]", testArgs, all)
	}
}

func TestMandatoryRequestParams(t *testing.T) {
	var params = mandatoryRequestParams(testArgs)
	if params["apiKey"] != testArgs.apiKey && params["name"] != testArgs.name {
		t.Errorf("Requested params [%s] are not the same as from OpsArgs [%+v]", params, testArgs)
	}
}
<|endoftext|>


<commit_before>package hive

import (
	"bufio"
	"errors"
	// "fmt"
	"github.com/eaciit/errorlib"
	"io"
	"log"
	"os/exec"
	"strings"
)

const (
	BEE_CLI_STR  = "0: jdbc:hive2:"
	CLOSE_SCRIPT = "!quit"
)

type DuplexTerm struct {
	Writer     *bufio.Writer
	Reader     *bufio.Reader
	Cmd        *exec.Cmd
	CmdStr     string
	Stdin      io.WriteCloser
	Stdout     io.ReadCloser
	FnReceive  FnHiveReceive
	OutputType string
	DateFormat string
}

/*func (d *DuplexTerm) Open() (e error) {
	if d.Stdin, e = d.Cmd.StdinPipe(); e != nil {
		return
	}

	if d.Stdout, e = d.Cmd.StdoutPipe(); e != nil {
		return
	}

	d.Writer = bufio.NewWriter(d.Stdin)
	d.Reader = bufio.NewReader(d.Stdout)

	e = d.Cmd.Start()
	return
}*/

var hr *HiveResult

func (d *DuplexTerm) Open() (e error) {
	if d.CmdStr != "" {
		arg := append([]string{"-c"}, d.CmdStr)
		d.Cmd = exec.Command("sh", arg...)

		if d.Stdin, e = d.Cmd.StdinPipe(); e != nil {
			return
		}

		if d.Stdout, e = d.Cmd.StdoutPipe(); e != nil {
			return
		}

		d.Writer = bufio.NewWriter(d.Stdin)
		d.Reader = bufio.NewReader(d.Stdout)

		if d.FnReceive != nil {
			go func() {
				_, e = d.Wait()
			}()
		}
		e = d.Cmd.Start()
	} else {
		errorlib.Error("", "", "Open", "The Connection Config not Set")
	}

	return
}

func (d *DuplexTerm) Close() {
	result, e := d.SendInput(CLOSE_SCRIPT)

	_ = result
	_ = e

	d.Cmd.Wait()
	d.Stdin.Close()
	d.Stdout.Close()
}

func (d *DuplexTerm) SendInput(input string) (result []string, e error) {
	iwrite, e := d.Writer.WriteString(input + "\n")
	if iwrite == 0 {
		e = errors.New("Writing only 0 byte")
	} else {
		e = d.Writer.Flush()
	}

	if e != nil {
		return
	}

	if d.FnReceive == nil {
		result, e = d.Wait()
	}

	return
}

func (d *DuplexTerm) Wait() (result []string, e error) {
	for {
		peekBefore, _ := d.Reader.Peek(14)
		peekBeforeStr := string(peekBefore)
		// log.Printf("peekBefore: %v\n", peekBefore)
		// log.Printf("peekBeforeStr: %v\n", peekBeforeStr)

		bread, e := d.Reader.ReadString('\n')
		bread = strings.TrimRight(bread, "\n")

		peek, _ := d.Reader.Peek(14)
		peekStr := string(peek)
		// log.Printf("peek: %v\n", peek)
		// log.Printf("peekStr: %v\n", peekStr)

		delimiter := "\t"

		if d.OutputType == CSV {
			delimiter = ","
		}

		if BEE_CLI_STR == peekBeforeStr {
			hr := HiveResult{}
			hr.constructHeader(bread, delimiter)
			log.Printf("model: %v\n", hr)

			for _, val = range hr.Header {
				log.Printf("header: %v\n", val)
			}
		}

		if !strings.Contains(bread, BEE_CLI_STR) {
			//result = append(result, bread)
			if d.FnReceive != nil {
				/*Parse(hr.Header, bread, hr.ResultObj, d.OutputType, d.DateFormat)
				log.Printf("model: %v\n", hr.ResultObj)*/
				d.FnReceive(bread)
			} else {
				result = append(result, bread)
			}
		}

		if d.FnReceive != nil {
			if (e != nil && e.Error() == "EOF") || (strings.Contains(peekStr, CLOSE_SCRIPT)) {
				break
			}
		} else {
			if (e != nil && e.Error() == "EOF") || (BEE_CLI_STR == peekStr) {
				break
			}
		}

	}

	return
}
<commit_msg>bug fixing for exec<commit_after>package hive

import (
	"bufio"
	"errors"
	// "fmt"
	"github.com/eaciit/errorlib"
	"io"
	"log"
	"os/exec"
	"strings"
)

const (
	BEE_CLI_STR  = "0: jdbc:hive2:"
	CLOSE_SCRIPT = "!quit"
)

type DuplexTerm struct {
	Writer     *bufio.Writer
	Reader     *bufio.Reader
	Cmd        *exec.Cmd
	CmdStr     string
	Stdin      io.WriteCloser
	Stdout     io.ReadCloser
	FnReceive  FnHiveReceive
	OutputType string
	DateFormat string
}

/*func (d *DuplexTerm) Open() (e error) {
	if d.Stdin, e = d.Cmd.StdinPipe(); e != nil {
		return
	}

	if d.Stdout, e = d.Cmd.StdoutPipe(); e != nil {
		return
	}

	d.Writer = bufio.NewWriter(d.Stdin)
	d.Reader = bufio.NewReader(d.Stdout)

	e = d.Cmd.Start()
	return
}*/

var hr *HiveResult

func (d *DuplexTerm) Open() (e error) {
	if d.CmdStr != "" {
		arg := append([]string{"-c"}, d.CmdStr)
		d.Cmd = exec.Command("sh", arg...)

		if d.Stdin, e = d.Cmd.StdinPipe(); e != nil {
			return
		}

		if d.Stdout, e = d.Cmd.StdoutPipe(); e != nil {
			return
		}

		d.Writer = bufio.NewWriter(d.Stdin)
		d.Reader = bufio.NewReader(d.Stdout)

		if d.FnReceive != nil {
			go func() {
				_, e = d.Wait()
			}()
		}
		e = d.Cmd.Start()
	} else {
		errorlib.Error("", "", "Open", "The Connection Config not Set")
	}

	return
}

func (d *DuplexTerm) Close() {
	result, e := d.SendInput(CLOSE_SCRIPT)

	_ = result
	_ = e

	d.Cmd.Wait()
	d.Stdin.Close()
	d.Stdout.Close()
}

func (d *DuplexTerm) SendInput(input string) (result []string, e error) {
	iwrite, e := d.Writer.WriteString(input + "\n")
	if iwrite == 0 {
		e = errors.New("Writing only 0 byte")
	} else {
		e = d.Writer.Flush()
	}

	if e != nil {
		return
	}

	if d.FnReceive == nil {
		result, e = d.Wait()
	}

	return
}

func (d *DuplexTerm) Wait() (result []string, e error) {
	for {
		peekBefore, _ := d.Reader.Peek(14)
		peekBeforeStr := string(peekBefore)
		// log.Printf("peekBefore: %v\n", peekBefore)
		// log.Printf("peekBeforeStr: %v\n", peekBeforeStr)

		bread, e := d.Reader.ReadString('\n')
		bread = strings.TrimRight(bread, "\n")

		peek, _ := d.Reader.Peek(14)
		peekStr := string(peek)
		// log.Printf("peek: %v\n", peek)
		// log.Printf("peekStr: %v\n", peekStr)

		delimiter := "\t"

		if d.OutputType == CSV {
			delimiter = ","
		}

		if BEE_CLI_STR == peekBeforeStr {
			hr := HiveResult{}
			hr.constructHeader(bread, delimiter)
			log.Printf("model: %v\n", hr)

			for _, val := range hr.Header {
				log.Printf("header: %v\n", val)
			}
		}

		if !strings.Contains(bread, BEE_CLI_STR) {
			//result = append(result, bread)
			if d.FnReceive != nil {
				/*Parse(hr.Header, bread, hr.ResultObj, d.OutputType, d.DateFormat)
				log.Printf("model: %v\n", hr.ResultObj)*/
				d.FnReceive(bread)
			} else {
				result = append(result, bread)
			}
		}

		if d.FnReceive != nil {
			if (e != nil && e.Error() == "EOF") || (strings.Contains(peekStr, CLOSE_SCRIPT)) {
				break
			}
		} else {
			if (e != nil && e.Error() == "EOF") || (BEE_CLI_STR == peekStr) {
				break
			}
		}

	}

	return
}
<|endoftext|>


<commit_before>package component

import (
	"reflect"
	"testing"

	"rsprd.com/spread/pkg/image"

	"k8s.io/kubernetes/pkg/api"
	"rsprd.com/spread/pkg/deploy"
)

func TestNewImage(t *testing.T) {
	imageName := "gcr.io/google_containers/cassandra:v7"
	simple, err := image.FromString(imageName)
	if err != nil {
		t.Fatalf("Could not create image.Image: %v", err)
	}

	secret := api.Secret{
		ObjectMeta: api.ObjectMeta{
			Name:      "sosecret",
			Namespace: "default",
		},
		Type: api.SecretTypeOpaque,
		Data: map[string][]byte{"test": []byte("secret")},
	}

	image, err := NewImage(simple, "test", &secret)
	if err != nil {
		t.Fatalf("Could not create Image component: %v", err)
	}

	expectedPod := api.Pod{
		Spec: api.PodSpec{
			Containers: []api.Container{
				api.Container{
					Image: imageName,
				},
			},
		},
	}

	expected := deploy.NewDeployment()
	expected.Add(&secret)
	expected.Add(&expectedPod)

	actual := image.Deployment()

	if !reflect.DeepEqual(expected, actual) {
		t.Errorf("Expected: %#v, saw: %#v", expected, actual)
	}
}
<commit_msg>added more image tests<commit_after>package component

import (
	"reflect"
	"testing"

	"rsprd.com/spread/pkg/image"

	"github.com/stretchr/testify/assert"
	"k8s.io/kubernetes/pkg/api"
	"rsprd.com/spread/pkg/deploy"
)

func TestNewImage(t *testing.T) {
	imageName := "gcr.io/google_containers/cassandra:v7"
	simple := newImage(t, imageName)

	secret := api.Secret{
		ObjectMeta: api.ObjectMeta{
			Name:      "sosecret",
			Namespace: "default",
		},
		Type: api.SecretTypeOpaque,
		Data: map[string][]byte{"test": []byte("secret")},
	}

	image, err := NewImage(simple, "test", &secret)
	if err != nil {
		t.Fatalf("Could not create Image component: %v", err)
	}

	// check images
	images := image.Images()
	assert.Len(t, images, 1, "supposed to be single image")
	assert.EqualValues(t, simple, images[0], "should return image it was created with")

	expectedPod := api.Pod{
		Spec: api.PodSpec{
			Containers: []api.Container{
				api.Container{
					Image: imageName,
				},
			},
		},
	}

	expected := deploy.NewDeployment()
	expected.Add(&secret)
	expected.Add(&expectedPod)

	actual := image.Deployment()

	if !reflect.DeepEqual(expected, actual) {
		t.Errorf("Expected: %#v, saw: %#v", expected, actual)
	}
}

func TestNilImage(t *testing.T) {
	var image *image.Image
	_, err := NewImage(image, "")
	if err == nil {
		t.Errorf("Nil should return an error.")
	}
}

func TestImageType(t *testing.T) {
	image := newImage(t, "ghost:latest")

	component, err := NewImage(image, "")
	if err != nil {
		t.Fatalf("Could not create Image component: %v", err)
	}

	assert.Equal(t, ComponentImage, component.Type(), "incorrect type")
}

func newImage(t *testing.T, imageName string) *image.Image {
	simple, err := image.FromString(imageName)
	if err != nil {
		t.Fatalf("Could not create image.Image: %v", err)
	}
	return simple
}
<|endoftext|>


<commit_before>// Copyright 2018 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package shareable

import (
	"image"
	"image/color"
	"runtime"
	"sync"

	"github.com/hajimehoshi/ebiten/internal/affine"
	"github.com/hajimehoshi/ebiten/internal/graphics"
	"github.com/hajimehoshi/ebiten/internal/packing"
	"github.com/hajimehoshi/ebiten/internal/restorable"
)

type backend struct {
	restorable *restorable.Image

	// If page is nil, the backend is not shared.
	page *packing.Page
}

func (b *backend) TryAlloc(width, height int) (*packing.Node, bool) {
	// If the region is allocated without any extention, it's fine.
	if n := b.page.Alloc(width, height); n != nil {
		return n, true
	}

	// Simulate the extending the page and calculate the appropriate page size.
	page := b.page.Clone()
	nExtended := 0
	for {
		if !page.Extend() {
			// The page can't be extended any more. Return as failure.
			return nil, false
		}
		nExtended++
		if n := page.Alloc(width, height); n != nil {
			break
		}
	}

	for i := 0; i < nExtended; i++ {
		b.page.Extend()
	}
	s := b.page.Size()
	newImg := restorable.NewImage(s, s, false)
	oldImg := b.restorable
	w, h := oldImg.Size()
	vs := graphics.QuadVertices(w, h, 0, 0, w, h, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1)
	is := graphics.QuadIndices()
	newImg.DrawImage(oldImg, vs, is, nil, graphics.CompositeModeCopy, graphics.FilterNearest)
	oldImg.Dispose()
	b.restorable = newImg

	n := b.page.Alloc(width, height)
	if n == nil {
		panic("not reached")
	}
	return n, true
}

var (
	// backendsM is a mutex for critical sections of the backend and packing.Node objects.
	backendsM sync.Mutex

	// theBackends is a set of actually shared images.
	theBackends = []*backend{}
)

type Image struct {
	width    int
	height   int
	disposed bool

	backend *backend

	node          *packing.Node
	countForShare int

	neverShared bool
}

func (i *Image) moveTo(dst *Image) {
	dst.dispose(false)
	*dst = *i

	// i is no longer available but Dispose must not be called
	// since i and dst have the same values like node.
	runtime.SetFinalizer(i, nil)
}

func (i *Image) isShared() bool {
	return i.node != nil
}

func (i *Image) IsSharedForTesting() bool {
	backendsM.Lock()
	defer backendsM.Unlock()
	return i.isShared()
}

func (i *Image) ensureNotShared() {
	if i.backend == nil {
		i.allocate(false)
		return
	}

	if !i.isShared() {
		return
	}

	x, y, w, h := i.region()
	newImg := restorable.NewImage(w, h, false)
	vw, vh := i.backend.restorable.Size()
	vs := graphics.QuadVertices(vw, vh, x, y, x+w, y+h, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1)
	is := graphics.QuadIndices()
	newImg.DrawImage(i.backend.restorable, vs, is, nil, graphics.CompositeModeCopy, graphics.FilterNearest)

	i.dispose(false)
	i.backend = &backend{
		restorable: newImg,
	}
}

func (i *Image) forceShared() {
	if i.backend == nil {
		i.allocate(true)
		return
	}

	if i.isShared() {
		return
	}

	if !i.shareable() {
		panic("not reached")
	}

	newI := NewImage(i.width, i.height)
	pixels := make([]byte, 4*i.width*i.height)
	for y := 0; y < i.height; y++ {
		for x := 0; x < i.width; x++ {
			c := i.at(x, y)
			pixels[4*(x+i.width*y)] = c.R
			pixels[4*(x+i.width*y)+1] = c.G
			pixels[4*(x+i.width*y)+2] = c.B
			pixels[4*(x+i.width*y)+3] = c.A
		}
	}
	newI.replacePixels(pixels)
	newI.moveTo(i)
	i.countForShare = 0
}

func (i *Image) region() (x, y, width, height int) {
	if i.backend == nil {
		panic("not reached")
	}
	if !i.isShared() {
		w, h := i.backend.restorable.Size()
		return 0, 0, w, h
	}
	return i.node.Region()
}

func (i *Image) Size() (width, height int) {
	return i.width, i.height
}

func (i *Image) QuadVertices(sx0, sy0, sx1, sy1 int, a, b, c, d, tx, ty float32, cr, cg, cb, ca float32) []float32 {
	if i.backend == nil {
		i.allocate(true)
	}
	ox, oy, _, _ := i.region()
	w, h := i.backend.restorable.SizePowerOf2()
	return graphics.QuadVertices(w, h, sx0+ox, sy0+oy, sx1+ox, sy1+oy, a, b, c, d, tx, ty, cr, cg, cb, ca)
}

func (i *Image) PutVertex(dest []float32, dx, dy, sx, sy float32, cr, cg, cb, ca float32) {
	if i.backend == nil {
		i.allocate(true)
	}
	ox, oy, _, _ := i.region()
	w, h := i.backend.restorable.SizePowerOf2()
	graphics.PutVertex(dest, w, h, dx, dy, sx+float32(ox), sy+float32(oy), cr, cg, cb, ca)
}

const MaxCountForShare = 10

func (i *Image) DrawImage(img *Image, vertices []float32, indices []uint16, colorm *affine.ColorM, mode graphics.CompositeMode, filter graphics.Filter) {
	backendsM.Lock()
	defer backendsM.Unlock()

	if img.disposed {
		panic("shareable: the drawing source image must not be disposed")
	}
	if i.disposed {
		panic("shareable: the drawing target image must not be disposed")
	}
	if img.backend == nil {
		img.allocate(true)
	}

	i.ensureNotShared()

	// Compare i and img after ensuring i is not shared, or
	// i and img might share the same texture even though i != img.
	if i.backend.restorable == img.backend.restorable {
		panic("shareable: Image.DrawImage: img must be different from the receiver")
	}

	i.backend.restorable.DrawImage(img.backend.restorable, vertices, indices, colorm, mode, filter)

	i.countForShare = 0

	// TODO: Reusing shared images is temporarily suspended for performance. See #661.
	//
	// if !img.isShared() && img.shareable() {
	//	img.countForShare++
	//	if img.countForShare >= MaxCountForShare {
	//		img.forceShared()
	//		img.countForShare = 0
	//	}
	// }
}

func (i *Image) ReplacePixels(p []byte) {
	backendsM.Lock()
	defer backendsM.Unlock()
	i.replacePixels(p)
}

func (i *Image) replacePixels(p []byte) {
	if i.disposed {
		panic("shareable: the image must not be disposed")
	}
	if i.backend == nil {
		if p == nil {
			return
		}
		i.allocate(true)
	}

	x, y, w, h := i.region()
	if p != nil {
		if l := 4 * w * h; len(p) != l {
			panic("not reached")
		}
	}
	i.backend.restorable.ReplacePixels(p, x, y, w, h)
}

func (i *Image) At(x, y int) color.RGBA {
	backendsM.Lock()
	defer backendsM.Unlock()
	return i.at(x, y)
}

func (i *Image) at(x, y int) color.RGBA {
	if i.backend == nil {
		return color.RGBA{}
	}

	ox, oy, w, h := i.region()
	if x < 0 || y < 0 || x >= w || y >= h {
		return color.RGBA{}
	}

	return i.backend.restorable.At(x+ox, y+oy)
}

func (i *Image) Dispose() {
	backendsM.Lock()
	defer backendsM.Unlock()
	i.dispose(true)
}

func (i *Image) dispose(markDisposed bool) {
	defer func() {
		if markDisposed {
			i.disposed = true
		}
		i.backend = nil
		i.node = nil
		if markDisposed {
			runtime.SetFinalizer(i, nil)
		}
	}()

	if i.disposed {
		return
	}

	if i.backend == nil {
		// Not allocated yet.
		return
	}

	if !i.isShared() {
		i.backend.restorable.Dispose()
		return
	}

	i.backend.page.Free(i.node)
	if !i.backend.page.IsEmpty() {
		// As this part can be reused, this should be cleared explicitly.
		x0, y0, x1, y1 := i.region()
		i.backend.restorable.ReplacePixels(nil, x0, y0, x1, y1)
		return
	}

	i.backend.restorable.Dispose()
	index := -1
	for idx, sh := range theBackends {
		if sh == i.backend {
			index = idx
			break
		}
	}
	if index == -1 {
		panic("not reached")
	}
	theBackends = append(theBackends[:index], theBackends[index+1:]...)
}

func (i *Image) IsVolatile() bool {
	backendsM.Lock()
	defer backendsM.Unlock()
	if i.backend == nil {
		// Not allocated yet. Only non-volatile images can do lazy allocation so far.
		return false
	}
	return i.backend.restorable.IsVolatile()
}

func (i *Image) IsInvalidated() (bool, error) {
	backendsM.Lock()
	defer backendsM.Unlock()
	v, err := i.backend.restorable.IsInvalidated()
	return v, err
}

func NewImage(width, height int) *Image {
	// Actual allocation is done lazily.
	return &Image{
		width:  width,
		height: height,
	}
}

func (i *Image) shareable() bool {
	if i.neverShared {
		return false
	}
	return i.width <= maxSize && i.height <= maxSize
}

func (i *Image) allocate(shareable bool) {
	if i.backend != nil {
		panic("not reached")
	}

	if !shareable || !i.shareable() {
		i.backend = &backend{
			restorable: restorable.NewImage(i.width, i.height, false),
		}
		runtime.SetFinalizer(i, (*Image).Dispose)
		return
	}

	for _, b := range theBackends {
		if n, ok := b.TryAlloc(i.width, i.height); ok {
			i.backend = b
			i.node = n
			runtime.SetFinalizer(i, (*Image).Dispose)
			return
		}
	}
	size := initSize
	for i.width > size || i.height > size {
		if size == maxSize {
			panic("not reached")
		}
		size *= 2
	}

	b := &backend{
		restorable: restorable.NewImage(size, size, false),
		page:       packing.NewPage(size, maxSize),
	}
	theBackends = append(theBackends, b)

	n := b.page.Alloc(i.width, i.height)
	if n == nil {
		panic("not reached")
	}
	i.backend = b
	i.node = n
	runtime.SetFinalizer(i, (*Image).Dispose)
	return
}

func NewVolatileImage(width, height int) *Image {
	backendsM.Lock()
	defer backendsM.Unlock()

	r := restorable.NewImage(width, height, true)
	i := &Image{
		width:  width,
		height: height,
		backend: &backend{
			restorable: r,
		},
		neverShared: true,
	}
	runtime.SetFinalizer(i, (*Image).Dispose)
	return i
}

func NewScreenFramebufferImage(width, height int) *Image {
	backendsM.Lock()
	defer backendsM.Unlock()

	r := restorable.NewScreenFramebufferImage(width, height)
	i := &Image{
		width:  width,
		height: height,
		backend: &backend{
			restorable: r,
		},
		neverShared: true,
	}
	runtime.SetFinalizer(i, (*Image).Dispose)
	return i
}

func InitializeGraphicsDriverState() error {
	backendsM.Lock()
	defer backendsM.Unlock()
	return restorable.InitializeGraphicsDriverState()
}

func ResolveStaleImages() {
	backendsM.Lock()
	defer backendsM.Unlock()
	restorable.ResolveStaleImages()
}

func IsRestoringEnabled() bool {
	// As IsRestoringEnabled is an immutable state, no need to lock here.
	return restorable.IsRestoringEnabled()
}

func Restore() error {
	backendsM.Lock()
	defer backendsM.Unlock()
	return restorable.Restore()
}

func Images() []image.Image {
	backendsM.Lock()
	defer backendsM.Unlock()
	return restorable.Images()
}

func Error() error {
	backendsM.Lock()
	defer backendsM.Unlock()
	return restorable.Error()
}
<commit_msg>shareable: Fix wrong variable names<commit_after>// Copyright 2018 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package shareable

import (
	"image"
	"image/color"
	"runtime"
	"sync"

	"github.com/hajimehoshi/ebiten/internal/affine"
	"github.com/hajimehoshi/ebiten/internal/graphics"
	"github.com/hajimehoshi/ebiten/internal/packing"
	"github.com/hajimehoshi/ebiten/internal/restorable"
)

type backend struct {
	restorable *restorable.Image

	// If page is nil, the backend is not shared.
	page *packing.Page
}

func (b *backend) TryAlloc(width, height int) (*packing.Node, bool) {
	// If the region is allocated without any extention, it's fine.
	if n := b.page.Alloc(width, height); n != nil {
		return n, true
	}

	// Simulate the extending the page and calculate the appropriate page size.
	page := b.page.Clone()
	nExtended := 0
	for {
		if !page.Extend() {
			// The page can't be extended any more. Return as failure.
			return nil, false
		}
		nExtended++
		if n := page.Alloc(width, height); n != nil {
			break
		}
	}

	for i := 0; i < nExtended; i++ {
		b.page.Extend()
	}
	s := b.page.Size()
	newImg := restorable.NewImage(s, s, false)
	oldImg := b.restorable
	w, h := oldImg.Size()
	vs := graphics.QuadVertices(w, h, 0, 0, w, h, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1)
	is := graphics.QuadIndices()
	newImg.DrawImage(oldImg, vs, is, nil, graphics.CompositeModeCopy, graphics.FilterNearest)
	oldImg.Dispose()
	b.restorable = newImg

	n := b.page.Alloc(width, height)
	if n == nil {
		panic("not reached")
	}
	return n, true
}

var (
	// backendsM is a mutex for critical sections of the backend and packing.Node objects.
	backendsM sync.Mutex

	// theBackends is a set of actually shared images.
	theBackends = []*backend{}
)

type Image struct {
	width    int
	height   int
	disposed bool

	backend *backend

	node          *packing.Node
	countForShare int

	neverShared bool
}

func (i *Image) moveTo(dst *Image) {
	dst.dispose(false)
	*dst = *i

	// i is no longer available but Dispose must not be called
	// since i and dst have the same values like node.
	runtime.SetFinalizer(i, nil)
}

func (i *Image) isShared() bool {
	return i.node != nil
}

func (i *Image) IsSharedForTesting() bool {
	backendsM.Lock()
	defer backendsM.Unlock()
	return i.isShared()
}

func (i *Image) ensureNotShared() {
	if i.backend == nil {
		i.allocate(false)
		return
	}

	if !i.isShared() {
		return
	}

	x, y, w, h := i.region()
	newImg := restorable.NewImage(w, h, false)
	vw, vh := i.backend.restorable.Size()
	vs := graphics.QuadVertices(vw, vh, x, y, x+w, y+h, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1)
	is := graphics.QuadIndices()
	newImg.DrawImage(i.backend.restorable, vs, is, nil, graphics.CompositeModeCopy, graphics.FilterNearest)

	i.dispose(false)
	i.backend = &backend{
		restorable: newImg,
	}
}

func (i *Image) forceShared() {
	if i.backend == nil {
		i.allocate(true)
		return
	}

	if i.isShared() {
		return
	}

	if !i.shareable() {
		panic("not reached")
	}

	newI := NewImage(i.width, i.height)
	pixels := make([]byte, 4*i.width*i.height)
	for y := 0; y < i.height; y++ {
		for x := 0; x < i.width; x++ {
			c := i.at(x, y)
			pixels[4*(x+i.width*y)] = c.R
			pixels[4*(x+i.width*y)+1] = c.G
			pixels[4*(x+i.width*y)+2] = c.B
			pixels[4*(x+i.width*y)+3] = c.A
		}
	}
	newI.replacePixels(pixels)
	newI.moveTo(i)
	i.countForShare = 0
}

func (i *Image) region() (x, y, width, height int) {
	if i.backend == nil {
		panic("not reached")
	}
	if !i.isShared() {
		w, h := i.backend.restorable.Size()
		return 0, 0, w, h
	}
	return i.node.Region()
}

func (i *Image) Size() (width, height int) {
	return i.width, i.height
}

func (i *Image) QuadVertices(sx0, sy0, sx1, sy1 int, a, b, c, d, tx, ty float32, cr, cg, cb, ca float32) []float32 {
	if i.backend == nil {
		i.allocate(true)
	}
	ox, oy, _, _ := i.region()
	w, h := i.backend.restorable.SizePowerOf2()
	return graphics.QuadVertices(w, h, sx0+ox, sy0+oy, sx1+ox, sy1+oy, a, b, c, d, tx, ty, cr, cg, cb, ca)
}

func (i *Image) PutVertex(dest []float32, dx, dy, sx, sy float32, cr, cg, cb, ca float32) {
	if i.backend == nil {
		i.allocate(true)
	}
	ox, oy, _, _ := i.region()
	w, h := i.backend.restorable.SizePowerOf2()
	graphics.PutVertex(dest, w, h, dx, dy, sx+float32(ox), sy+float32(oy), cr, cg, cb, ca)
}

const MaxCountForShare = 10

func (i *Image) DrawImage(img *Image, vertices []float32, indices []uint16, colorm *affine.ColorM, mode graphics.CompositeMode, filter graphics.Filter) {
	backendsM.Lock()
	defer backendsM.Unlock()

	if img.disposed {
		panic("shareable: the drawing source image must not be disposed")
	}
	if i.disposed {
		panic("shareable: the drawing target image must not be disposed")
	}
	if img.backend == nil {
		img.allocate(true)
	}

	i.ensureNotShared()

	// Compare i and img after ensuring i is not shared, or
	// i and img might share the same texture even though i != img.
	if i.backend.restorable == img.backend.restorable {
		panic("shareable: Image.DrawImage: img must be different from the receiver")
	}

	i.backend.restorable.DrawImage(img.backend.restorable, vertices, indices, colorm, mode, filter)

	i.countForShare = 0

	// TODO: Reusing shared images is temporarily suspended for performance. See #661.
	//
	// if !img.isShared() && img.shareable() {
	//	img.countForShare++
	//	if img.countForShare >= MaxCountForShare {
	//		img.forceShared()
	//		img.countForShare = 0
	//	}
	// }
}

func (i *Image) ReplacePixels(p []byte) {
	backendsM.Lock()
	defer backendsM.Unlock()
	i.replacePixels(p)
}

func (i *Image) replacePixels(p []byte) {
	if i.disposed {
		panic("shareable: the image must not be disposed")
	}
	if i.backend == nil {
		if p == nil {
			return
		}
		i.allocate(true)
	}

	x, y, w, h := i.region()
	if p != nil {
		if l := 4 * w * h; len(p) != l {
			panic("not reached")
		}
	}
	i.backend.restorable.ReplacePixels(p, x, y, w, h)
}

func (i *Image) At(x, y int) color.RGBA {
	backendsM.Lock()
	defer backendsM.Unlock()
	return i.at(x, y)
}

func (i *Image) at(x, y int) color.RGBA {
	if i.backend == nil {
		return color.RGBA{}
	}

	ox, oy, w, h := i.region()
	if x < 0 || y < 0 || x >= w || y >= h {
		return color.RGBA{}
	}

	return i.backend.restorable.At(x+ox, y+oy)
}

func (i *Image) Dispose() {
	backendsM.Lock()
	defer backendsM.Unlock()
	i.dispose(true)
}

func (i *Image) dispose(markDisposed bool) {
	defer func() {
		if markDisposed {
			i.disposed = true
		}
		i.backend = nil
		i.node = nil
		if markDisposed {
			runtime.SetFinalizer(i, nil)
		}
	}()

	if i.disposed {
		return
	}

	if i.backend == nil {
		// Not allocated yet.
		return
	}

	if !i.isShared() {
		i.backend.restorable.Dispose()
		return
	}

	i.backend.page.Free(i.node)
	if !i.backend.page.IsEmpty() {
		// As this part can be reused, this should be cleared explicitly.
		x, y, w, h := i.region()
		i.backend.restorable.ReplacePixels(nil, x, y, w, h)
		return
	}

	i.backend.restorable.Dispose()
	index := -1
	for idx, sh := range theBackends {
		if sh == i.backend {
			index = idx
			break
		}
	}
	if index == -1 {
		panic("not reached")
	}
	theBackends = append(theBackends[:index], theBackends[index+1:]...)
}

func (i *Image) IsVolatile() bool {
	backendsM.Lock()
	defer backendsM.Unlock()
	if i.backend == nil {
		// Not allocated yet. Only non-volatile images can do lazy allocation so far.
		return false
	}
	return i.backend.restorable.IsVolatile()
}

func (i *Image) IsInvalidated() (bool, error) {
	backendsM.Lock()
	defer backendsM.Unlock()
	v, err := i.backend.restorable.IsInvalidated()
	return v, err
}

func NewImage(width, height int) *Image {
	// Actual allocation is done lazily.
	return &Image{
		width:  width,
		height: height,
	}
}

func (i *Image) shareable() bool {
	if i.neverShared {
		return false
	}
	return i.width <= maxSize && i.height <= maxSize
}

func (i *Image) allocate(shareable bool) {
	if i.backend != nil {
		panic("not reached")
	}

	if !shareable || !i.shareable() {
		i.backend = &backend{
			restorable: restorable.NewImage(i.width, i.height, false),
		}
		runtime.SetFinalizer(i, (*Image).Dispose)
		return
	}

	for _, b := range theBackends {
		if n, ok := b.TryAlloc(i.width, i.height); ok {
			i.backend = b
			i.node = n
			runtime.SetFinalizer(i, (*Image).Dispose)
			return
		}
	}
	size := initSize
	for i.width > size || i.height > size {
		if size == maxSize {
			panic("not reached")
		}
		size *= 2
	}

	b := &backend{
		restorable: restorable.NewImage(size, size, false),
		page:       packing.NewPage(size, maxSize),
	}
	theBackends = append(theBackends, b)

	n := b.page.Alloc(i.width, i.height)
	if n == nil {
		panic("not reached")
	}
	i.backend = b
	i.node = n
	runtime.SetFinalizer(i, (*Image).Dispose)
	return
}

func NewVolatileImage(width, height int) *Image {
	backendsM.Lock()
	defer backendsM.Unlock()

	r := restorable.NewImage(width, height, true)
	i := &Image{
		width:  width,
		height: height,
		backend: &backend{
			restorable: r,
		},
		neverShared: true,
	}
	runtime.SetFinalizer(i, (*Image).Dispose)
	return i
}

func NewScreenFramebufferImage(width, height int) *Image {
	backendsM.Lock()
	defer backendsM.Unlock()

	r := restorable.NewScreenFramebufferImage(width, height)
	i := &Image{
		width:  width,
		height: height,
		backend: &backend{
			restorable: r,
		},
		neverShared: true,
	}
	runtime.SetFinalizer(i, (*Image).Dispose)
	return i
}

func InitializeGraphicsDriverState() error {
	backendsM.Lock()
	defer backendsM.Unlock()
	return restorable.InitializeGraphicsDriverState()
}

func ResolveStaleImages() {
	backendsM.Lock()
	defer backendsM.Unlock()
	restorable.ResolveStaleImages()
}

func IsRestoringEnabled() bool {
	// As IsRestoringEnabled is an immutable state, no need to lock here.
	return restorable.IsRestoringEnabled()
}

func Restore() error {
	backendsM.Lock()
	defer backendsM.Unlock()
	return restorable.Restore()
}

func Images() []image.Image {
	backendsM.Lock()
	defer backendsM.Unlock()
	return restorable.Images()
}

func Error() error {
	backendsM.Lock()
	defer backendsM.Unlock()
	return restorable.Error()
}
<|endoftext|>


<commit_before>package cephfs

/*
#cgo LDFLAGS: -lcephfs
#cgo CPPFLAGS: -D_FILE_OFFSET_BITS=64
#include <stdlib.h>
#include <cephfs/libcephfs.h>
*/
import "C"

import (
	"fmt"
	"unsafe"

	"github.com/ceph/go-ceph/errutil"
)

type cephError int

func (e cephError) Error() string {
	errno, s := errutil.FormatErrno(int(e))
	if s == "" {
		return fmt.Sprintf("cephfs: ret=%d", errno)
	}
	return fmt.Sprintf("cephfs: ret=%d, %s", errno, s)
}

// MountInfo exports ceph's ceph_mount_info from libcephfs.cc
type MountInfo struct {
	mount *C.struct_ceph_mount_info
}

// CreateMount creates a mount handle for interacting with Ceph.
func CreateMount() (*MountInfo, error) {
	mount := &MountInfo{}
	ret := C.ceph_create(&mount.mount, nil)
	if ret != 0 {
		return nil, cephError(ret)
	}
	return mount, nil
}

// ReadDefaultConfigFile loads the ceph configuration from the specified config file.
func (mount *MountInfo) ReadDefaultConfigFile() error {
	ret := C.ceph_conf_read_file(mount.mount, nil)
	if ret != 0 {
		return cephError(ret)
	}
	return nil
}

// Mount mounts the mount handle.
func (mount *MountInfo) Mount() error {
	ret := C.ceph_mount(mount.mount, nil)
	if ret != 0 {
		return cephError(ret)
	}
	return nil
}

// Unmount unmounts the mount handle.
func (mount *MountInfo) Unmount() error {
	ret := C.ceph_unmount(mount.mount)
	if ret != 0 {
		return cephError(ret)
	}
	return nil
}

// Release destroys the mount handle.
func (mount *MountInfo) Release() error {
	ret := C.ceph_release(mount.mount)
	if ret != 0 {
		return cephError(ret)
	}
	return nil
}

// SyncFs synchronizes all filesystem data to persistent media.
func (mount *MountInfo) SyncFs() error {
	ret := C.ceph_sync_fs(mount.mount)
	if ret != 0 {
		return cephError(ret)
	}
	return nil
}

// CurrentDir gets the current working directory.
func (mount *MountInfo) CurrentDir() string {
	cDir := C.ceph_getcwd(mount.mount)
	return C.GoString(cDir)
}

// ChangeDir changes the current working directory.
func (mount *MountInfo) ChangeDir(path string) error {
	cPath := C.CString(path)
	defer C.free(unsafe.Pointer(cPath))

	ret := C.ceph_chdir(mount.mount, cPath)
	if ret != 0 {
		return cephError(ret)
	}
	return nil
}

// MakeDir creates a directory.
func (mount *MountInfo) MakeDir(path string, mode uint32) error {
	cPath := C.CString(path)
	defer C.free(unsafe.Pointer(cPath))

	ret := C.ceph_mkdir(mount.mount, cPath, C.mode_t(mode))
	if ret != 0 {
		return cephError(ret)
	}
	return nil
}

// RemoveDir removes a directory.
func (mount *MountInfo) RemoveDir(path string) error {
	cPath := C.CString(path)
	defer C.free(unsafe.Pointer(cPath))

	ret := C.ceph_rmdir(mount.mount, cPath)
	if ret != 0 {
		return cephError(ret)
	}
	return nil
}

// Chmod changes the mode bits (permissions) of a file/directory.
func (mount *MountInfo) Chmod(path string, mode uint32) error {
	cPath := C.CString(path)
	defer C.free(unsafe.Pointer(cPath))

	ret := C.ceph_chmod(mount.mount, cPath, C.mode_t(mode))
	if ret != 0 {
		return cephError(ret)
	}
	return nil
}

// Chown changes the ownership of a file/directory.
func (mount *MountInfo) Chown(path string, user uint32, group uint32) error {
	cPath := C.CString(path)
	defer C.free(unsafe.Pointer(cPath))

	ret := C.ceph_chown(mount.mount, cPath, C.int(user), C.int(group))
	if ret != 0 {
		return cephError(ret)
	}
	return nil
}

// IsMounted checks mount status.
func (mount *MountInfo) IsMounted() bool {
	ret := C.ceph_is_mounted(mount.mount)
	return ret == 1
}
<commit_msg>cephfs: convert cephfs error type to exported type<commit_after>package cephfs

/*
#cgo LDFLAGS: -lcephfs
#cgo CPPFLAGS: -D_FILE_OFFSET_BITS=64
#include <stdlib.h>
#include <cephfs/libcephfs.h>
*/
import "C"

import (
	"fmt"
	"unsafe"

	"github.com/ceph/go-ceph/errutil"
)

type CephFSError int

func (e CephFSError) Error() string {
	errno, s := errutil.FormatErrno(int(e))
	if s == "" {
		return fmt.Sprintf("cephfs: ret=%d", errno)
	}
	return fmt.Sprintf("cephfs: ret=%d, %s", errno, s)
}

func getError(e C.int) error {
	if e == 0 {
		return nil
	}
	return CephFSError(e)
}

// MountInfo exports ceph's ceph_mount_info from libcephfs.cc
type MountInfo struct {
	mount *C.struct_ceph_mount_info
}

// CreateMount creates a mount handle for interacting with Ceph.
func CreateMount() (*MountInfo, error) {
	mount := &MountInfo{}
	ret := C.ceph_create(&mount.mount, nil)
	if ret != 0 {
		return nil, getError(ret)
	}
	return mount, nil
}

// ReadDefaultConfigFile loads the ceph configuration from the specified config file.
func (mount *MountInfo) ReadDefaultConfigFile() error {
	ret := C.ceph_conf_read_file(mount.mount, nil)
	return getError(ret)
}

// Mount mounts the mount handle.
func (mount *MountInfo) Mount() error {
	ret := C.ceph_mount(mount.mount, nil)
	return getError(ret)
}

// Unmount unmounts the mount handle.
func (mount *MountInfo) Unmount() error {
	ret := C.ceph_unmount(mount.mount)
	return getError(ret)
}

// Release destroys the mount handle.
func (mount *MountInfo) Release() error {
	ret := C.ceph_release(mount.mount)
	return getError(ret)
}

// SyncFs synchronizes all filesystem data to persistent media.
func (mount *MountInfo) SyncFs() error {
	ret := C.ceph_sync_fs(mount.mount)
	return getError(ret)
}

// CurrentDir gets the current working directory.
func (mount *MountInfo) CurrentDir() string {
	cDir := C.ceph_getcwd(mount.mount)
	return C.GoString(cDir)
}

// ChangeDir changes the current working directory.
func (mount *MountInfo) ChangeDir(path string) error {
	cPath := C.CString(path)
	defer C.free(unsafe.Pointer(cPath))

	ret := C.ceph_chdir(mount.mount, cPath)
	return getError(ret)
}

// MakeDir creates a directory.
func (mount *MountInfo) MakeDir(path string, mode uint32) error {
	cPath := C.CString(path)
	defer C.free(unsafe.Pointer(cPath))

	ret := C.ceph_mkdir(mount.mount, cPath, C.mode_t(mode))
	return getError(ret)
}

// RemoveDir removes a directory.
func (mount *MountInfo) RemoveDir(path string) error {
	cPath := C.CString(path)
	defer C.free(unsafe.Pointer(cPath))

	ret := C.ceph_rmdir(mount.mount, cPath)
	return getError(ret)
}

// Chmod changes the mode bits (permissions) of a file/directory.
func (mount *MountInfo) Chmod(path string, mode uint32) error {
	cPath := C.CString(path)
	defer C.free(unsafe.Pointer(cPath))

	ret := C.ceph_chmod(mount.mount, cPath, C.mode_t(mode))
	return getError(ret)
}

// Chown changes the ownership of a file/directory.
func (mount *MountInfo) Chown(path string, user uint32, group uint32) error {
	cPath := C.CString(path)
	defer C.free(unsafe.Pointer(cPath))

	ret := C.ceph_chown(mount.mount, cPath, C.int(user), C.int(group))
	return getError(ret)
}

// IsMounted checks mount status.
func (mount *MountInfo) IsMounted() bool {
	ret := C.ceph_is_mounted(mount.mount)
	return ret == 1
}
<|endoftext|>


<commit_before>package godjango

import (
	"reflect"
	"fmt"
)

var _regModels map[string]reflect.Type

func RegisterModel(model interface{}){
	fmt.Println("%v",reflect.TypeOf(model))
}
<commit_msg>added tests<commit_after>package godjango

import _ "github.com/kittuov/go-django/admin"<|endoftext|>


<commit_before>package soaap

import (
	"encoding/gob"
	"errors"
	"fmt"
	"io"
	"math"
	"os"
	"strings"
)

type CallGraph struct {
	nodes  map[string]GraphNode
	roots  strset
	leaves strset
	calls  map[Call]int
}

//
// Create a new, empty CallGraph with enough capacity to hold some calls.
//
func NewCallGraph() CallGraph {
	return CallGraph{
		make(map[string]GraphNode),
		make(strset),
		make(strset),
		make(map[Call]int),
	}
}

//
// Load a CallGraph from a binary-encoded file.
//
func LoadGraph(f *os.File, report func(string)) (CallGraph, error) {
	var graph CallGraph
	err := gob.NewDecoder(f).Decode(&graph)

	return graph, err
}

func (cg *CallGraph) AddCall(caller string, callee string) {
	cg.calls[Call{caller, callee}] += 1

	cg.roots.Remove(callee)
	cg.leaves.Remove(caller)
}

func (cg *CallGraph) AddNode(node GraphNode) {
	name := node.Name

	cg.nodes[name] = node
	cg.roots.Add(name)
	cg.leaves.Add(name)
}

//
// Save a CallGraph to an os.File using a binary encoding.
//
func (cg *CallGraph) Save(f *os.File) error {
	return gob.NewEncoder(f).Encode(cg)
}

//
// Simplify a CallGraph by collapsing call chains and dropping any
// unreferenced calls.
//
func (cg *CallGraph) Simplify() {
}

func (cg *CallGraph) Union(g CallGraph) error {
	for id, node := range g.nodes {
		// If we already have a GraphNode with this identifier,
		// merge the two descriptions and tag sets.
		if n, have := cg.nodes[id]; have {
			if n.Name != node.Name {
				return errors.New(fmt.Sprintf(
					"Nodes in CallGraph union have"+
						" same identifier ('%s') but"+
						" different names ('%s' vs '%s')",
					id, n.Name, node.Name))
			}

			if n.Description != node.Description {
				node.Description =
					n.Description + "\\n" + node.Description
			}

			for tag := range n.Tags {
				node.Tags[tag] = true
			}

			cg.nodes[id] = node
		} else {
			cg.AddNode(node)
		}
	}

	for call, count := range g.calls {
		cg.AddCall(call.Caller, call.Callee)
		cg.calls[call] += (count - 1)
	}

	return nil
}

func (cg CallGraph) WriteDot(out io.Writer) {
	fmt.Fprintln(out, `digraph {

	node [ fontname = "Inconsolata" ];
	edge [ fontname = "Avenir" ];

	labeljust = "l";
	labelloc = "b";
	rankdir = "BT";

`)

	for _, n := range cg.nodes {
		fmt.Fprintf(out, "	%s\n", n.Dot())
	}

	for c, count := range cg.calls {
		fmt.Fprintf(out, "	%s\n", c.Dot(cg, count))
	}

	fmt.Fprintf(out, "}\n")
}

//
// A node in a call graph.
//
// This is derived from a call site or other program location, but can have
// an arbitrary name and description appropriate to a particular analysis.
//
type GraphNode struct {
	Name        string
	Description string
	Location    SourceLocation

	// A vulnerability (current or previous) is known at this location.
	CVE []CVE

	// The name of this node's sandbox (or the empty string if unsandboxed).
	Sandbox string

	// The name of the sandbox(es) that own the data being accessed.
	Owners []string

	Tags map[string]bool
}

func newGraphNode(name string) GraphNode {
	var node GraphNode
	node.Name = name

	return node
}

//
// Construct a GraphViz Dot description of a GraphNode.
//
// This applies SOAAP-specific styling depending on a node's tags.
//
func (n GraphNode) Dot() string {
	attrs := map[string]interface{}{
		"label": n.Description,
		"style": "filled",
	}

	if len(n.CVE) > 0 {
		attrs["label"] = fmt.Sprintf("%s\\n%s", n.CVE, n.Description)
	}

	switch true {
	case len(n.CVE) > 0 && n.Sandbox != "":
		// A vulnerability has been mitigated through sandboxing!
		attrs["fillcolor"] = "#ffff66cc"
		attrs["shape"] = "octagon"

	case len(n.CVE) > 0:
		// A vulnerability exists/existed outside a sandbox.
		attrs["fillcolor"] = "#ff9999cc"
		attrs["shape"] = "doubleoctagon"

	case len(n.Owners) > 0:
		// Sandbox-private data was accessed outside the sandbox.
		attrs["fillcolor"] = "#ff99cccc"
		attrs["shape"] = "invhouse"

	case n.Sandbox != "":
		attrs["fillcolor"] = "#99ff9999"
		attrs["style"] = "dashed,filled"

	default:
		attrs["fillcolor"] = "#cccccccc"
	}

	return fmt.Sprintf("\"%s\" %s;", n.Name, dotAttrs(attrs))
}

func (n GraphNode) HasTag(tag string) bool {
	_, present := n.Tags[tag]
	return present
}

type Call struct {
	// Identifier of the caller.
	Caller string

	// Identifier of the callee.
	Callee string
}

// Output GraphViz for a Call.
func (c Call) Dot(graph CallGraph, weight int) string {
	caller := graph.nodes[c.Caller]
	callee := graph.nodes[c.Callee]

	attrs := map[string]interface{}{
		"label":    callee.Location.String(),
		"penwidth": weight,
		"weight":   weight,
	}

	return fmt.Sprintf("\"%s\" -> \"%s\" %s;\n",
		caller.Name, callee.Name, dotAttrs(attrs))
}

//
// A function that extracts a CallGraph from SOAAP Results.
//
type graphFn func(results Results, progress func(string)) CallGraph

var graphExtractors map[string]graphFn = map[string]graphFn{
	"privaccess": PrivAccessGraph,
	"vuln":       VulnGraph,
}

func GraphAnalyses() []string {
	keys := make([]string, len(graphExtractors))

	i := 0
	for k, _ := range graphExtractors {
		keys[i] = k
		i++
	}

	return keys
}

type nodeMaker func(CallSite) GraphNode

//
// Construct a callgraph from SOAAP's vulnerability analysis.
//
func VulnGraph(results Results, progress func(string)) CallGraph {
	graph := NewCallGraph()

	for _, v := range results.Vulnerabilities {
		trace := results.Traces[v.Trace]

		fn := func(cs CallSite) GraphNode {
			node := newGraphNode(cs.String() + v.Sandbox)
			node.Name = cs.String() + v.Sandbox
			node.Description = cs.Function
			if v.Sandbox != "" {
				node.Description += "\\n<<" + v.Sandbox + ">>"
			}

			node.Location = cs.Location
			node.Sandbox = v.Sandbox

			return node
		}

		top := fn(v.CallSite)
		top.CVE = v.CVE
		graph.AddNode(top)

		graph.Union(trace.graph(top, results.Traces, fn))
	}

	return graph
}

//
// Construct a callgraph of sandbox-private data accesses outside of sandboxes.
//
func PrivAccessGraph(results Results, progress func(string)) CallGraph {
	graph := NewCallGraph()
	accesses := results.PrivateAccess
	total := len(accesses)
	chunk := int(math.Ceil(math.Pow(10, math.Log10(float64(total)/500))))

	go progress(fmt.Sprintf("Processing %d private accesses", total))

	count := 0
	for _, a := range accesses {
		trace := results.Traces[a.Trace]

		fn := func(cs CallSite) GraphNode {
			sandboxes := strings.Join(a.Sandboxes, ",")

			node := newGraphNode(cs.String() + sandboxes)
			node.Description = cs.Function
			if sandboxes != "" {
				node.Description += "\\n<<" + sandboxes + ">>"
			}

			node.Location = cs.Location

			return node
		}

		top := fn(a.CallSite)
		top.Owners = a.Sandboxes
		graph.AddNode(top)

		graph.Union(trace.graph(top, results.Traces, fn))

		count++
		if count%chunk == 0 {
			go progress(
				fmt.Sprintf("Processed %d/%d accesses",
					count, total))
		}
	}

	return graph
}

//
// Graph a single CallTrace, using a nodeMaker function to convert
// CallSite instances into graph nodes with identifiers, tags, etc.,
// appropriate to the analysis we're performing.
//
func (t CallTrace) graph(top GraphNode, traces []CallTrace, nm nodeMaker) CallGraph {
	graph := NewCallGraph()
	graph.AddNode(top)
	callee := top.Name

	t.Foreach(traces, func(cs CallSite) {
		node := nm(cs)
		graph.AddNode(node)

		caller := node.Name
		graph.AddCall(caller, callee)
		callee = caller
	})

	return graph
}

//
// Format a map as a GraphViz attribute list.
//
func dotAttrs(attrs map[string]interface{}) string {
	fields := make([]string, len(attrs))

	i := 0
	for k, v := range attrs {
		switch v.(type) {
		case string:
			v = fmt.Sprintf("\"%s\"", v)
		}

		fields[i] = fmt.Sprintf("\"%s\" = %v", k, v)
		i++
	}

	return fmt.Sprintf("[ %s ]", strings.Join(fields, ", "))
}
<commit_msg>Keep track of each node's callees and callers.<commit_after>package soaap

import (
	"encoding/gob"
	"errors"
	"fmt"
	"io"
	"math"
	"os"
	"strings"
)

type CallGraph struct {
	nodes  map[string]GraphNode
	roots  strset
	leaves strset
	calls  map[Call]int
}

//
// Create a new, empty CallGraph with enough capacity to hold some calls.
//
func NewCallGraph() CallGraph {
	return CallGraph{
		make(map[string]GraphNode),
		make(strset),
		make(strset),
		make(map[Call]int),
	}
}

//
// Load a CallGraph from a binary-encoded file.
//
func LoadGraph(f *os.File, report func(string)) (CallGraph, error) {
	var graph CallGraph
	err := gob.NewDecoder(f).Decode(&graph)

	return graph, err
}

func (cg *CallGraph) AddCall(caller string, callee string) {
	cg.calls[Call{caller, callee}] += 1

	// This idiom (copy, modify, update) is terribly tedious, but
	// Go maps always return copies rather than references.
	c := cg.nodes[caller]
	c.Callees.Add(callee)
	cg.nodes[caller] = c

	c = cg.nodes[callee]
	c.Callers.Add(caller)
	cg.nodes[callee] = c

	cg.roots.Remove(callee)
	cg.leaves.Remove(caller)
}

func (cg *CallGraph) AddNode(node GraphNode) {
	name := node.Name

	cg.nodes[name] = node
	cg.roots.Add(name)
	cg.leaves.Add(name)
}

//
// Save a CallGraph to an os.File using a binary encoding.
//
func (cg *CallGraph) Save(f *os.File) error {
	return gob.NewEncoder(f).Encode(cg)
}

//
// Simplify a CallGraph by collapsing call chains and dropping any
// unreferenced calls.
//
func (cg *CallGraph) Simplify() {
}

func (cg *CallGraph) Union(g CallGraph) error {
	for id, node := range g.nodes {
		// If we already have a GraphNode with this identifier,
		// merge the two descriptions and tag sets.
		if n, have := cg.nodes[id]; have {
			if n.Name != node.Name {
				return errors.New(fmt.Sprintf(
					"Nodes in CallGraph union have"+
						" same identifier ('%s') but"+
						" different names ('%s' vs '%s')",
					id, n.Name, node.Name))
			}

			if n.Description != node.Description {
				node.Description =
					n.Description + "\\n" + node.Description
			}

			for tag := range n.Tags {
				node.Tags[tag] = true
			}

			cg.nodes[id] = node
		} else {
			cg.AddNode(node)
		}
	}

	for call, count := range g.calls {
		cg.AddCall(call.Caller, call.Callee)
		cg.calls[call] += (count - 1)
	}

	return nil
}

func (cg CallGraph) WriteDot(out io.Writer) {
	fmt.Fprintln(out, `digraph {

	node [ fontname = "Inconsolata" ];
	edge [ fontname = "Avenir" ];

	labeljust = "l";
	labelloc = "b";
	rankdir = "BT";

`)

	for _, n := range cg.nodes {
		fmt.Fprintf(out, "	%s\n", n.Dot())
	}

	for c, count := range cg.calls {
		fmt.Fprintf(out, "	%s\n", c.Dot(cg, count))
	}

	fmt.Fprintf(out, "}\n")
}

//
// A node in a call graph.
//
// This is derived from a call site or other program location, but can have
// an arbitrary name and description appropriate to a particular analysis.
//
type GraphNode struct {
	Name        string
	Description string
	Location    SourceLocation

	// A vulnerability (current or previous) is known at this location.
	CVE []CVE

	// The name of this node's sandbox (or the empty string if unsandboxed).
	Sandbox string

	// The name of the sandbox(es) that own the data being accessed.
	Owners []string

	Callees strset
	Callers strset

	Tags map[string]bool
}

func newGraphNode(name string) GraphNode {
	var node GraphNode
	node.Name = name
	node.Callees = make(strset)
	node.Callers = make(strset)

	return node
}

//
// Construct a GraphViz Dot description of a GraphNode.
//
// This applies SOAAP-specific styling depending on a node's tags.
//
func (n GraphNode) Dot() string {
	attrs := map[string]interface{}{
		"label": n.Description,
		"style": "filled",
	}

	if len(n.CVE) > 0 {
		attrs["label"] = fmt.Sprintf("%s\\n%s", n.CVE, n.Description)
	}

	switch true {
	case len(n.CVE) > 0 && n.Sandbox != "":
		// A vulnerability has been mitigated through sandboxing!
		attrs["fillcolor"] = "#ffff66cc"
		attrs["shape"] = "octagon"

	case len(n.CVE) > 0:
		// A vulnerability exists/existed outside a sandbox.
		attrs["fillcolor"] = "#ff9999cc"
		attrs["shape"] = "doubleoctagon"

	case len(n.Owners) > 0:
		// Sandbox-private data was accessed outside the sandbox.
		attrs["fillcolor"] = "#ff99cccc"
		attrs["shape"] = "invhouse"

	case n.Sandbox != "":
		attrs["fillcolor"] = "#99ff9999"
		attrs["style"] = "dashed,filled"

	default:
		attrs["fillcolor"] = "#cccccccc"
	}

	return fmt.Sprintf("\"%s\" %s;", n.Name, dotAttrs(attrs))
}

func (n GraphNode) HasTag(tag string) bool {
	_, present := n.Tags[tag]
	return present
}

type Call struct {
	// Identifier of the caller.
	Caller string

	// Identifier of the callee.
	Callee string
}

// Output GraphViz for a Call.
func (c Call) Dot(graph CallGraph, weight int) string {
	caller := graph.nodes[c.Caller]
	callee := graph.nodes[c.Callee]

	attrs := map[string]interface{}{
		"label":    callee.Location.String(),
		"penwidth": weight,
		"weight":   weight,
	}

	return fmt.Sprintf("\"%s\" -> \"%s\" %s;\n",
		caller.Name, callee.Name, dotAttrs(attrs))
}

//
// A function that extracts a CallGraph from SOAAP Results.
//
type graphFn func(results Results, progress func(string)) CallGraph

var graphExtractors map[string]graphFn = map[string]graphFn{
	"privaccess": PrivAccessGraph,
	"vuln":       VulnGraph,
}

func GraphAnalyses() []string {
	keys := make([]string, len(graphExtractors))

	i := 0
	for k, _ := range graphExtractors {
		keys[i] = k
		i++
	}

	return keys
}

type nodeMaker func(CallSite) GraphNode

//
// Construct a callgraph from SOAAP's vulnerability analysis.
//
func VulnGraph(results Results, progress func(string)) CallGraph {
	graph := NewCallGraph()

	for _, v := range results.Vulnerabilities {
		trace := results.Traces[v.Trace]

		fn := func(cs CallSite) GraphNode {
			node := newGraphNode(cs.String() + v.Sandbox)
			node.Name = cs.String() + v.Sandbox
			node.Description = cs.Function
			if v.Sandbox != "" {
				node.Description += "\\n<<" + v.Sandbox + ">>"
			}

			node.Location = cs.Location
			node.Sandbox = v.Sandbox

			return node
		}

		top := fn(v.CallSite)
		top.CVE = v.CVE
		graph.AddNode(top)

		graph.Union(trace.graph(top, results.Traces, fn))
	}

	return graph
}

//
// Construct a callgraph of sandbox-private data accesses outside of sandboxes.
//
func PrivAccessGraph(results Results, progress func(string)) CallGraph {
	graph := NewCallGraph()
	accesses := results.PrivateAccess
	total := len(accesses)
	chunk := int(math.Ceil(math.Pow(10, math.Log10(float64(total)/500))))

	go progress(fmt.Sprintf("Processing %d private accesses", total))

	count := 0
	for _, a := range accesses {
		trace := results.Traces[a.Trace]

		fn := func(cs CallSite) GraphNode {
			sandboxes := strings.Join(a.Sandboxes, ",")

			node := newGraphNode(cs.String() + sandboxes)
			node.Description = cs.Function
			if sandboxes != "" {
				node.Description += "\\n<<" + sandboxes + ">>"
			}

			node.Location = cs.Location

			return node
		}

		top := fn(a.CallSite)
		top.Owners = a.Sandboxes
		graph.AddNode(top)

		graph.Union(trace.graph(top, results.Traces, fn))

		count++
		if count%chunk == 0 {
			go progress(
				fmt.Sprintf("Processed %d/%d accesses",
					count, total))
		}
	}

	return graph
}

//
// Graph a single CallTrace, using a nodeMaker function to convert
// CallSite instances into graph nodes with identifiers, tags, etc.,
// appropriate to the analysis we're performing.
//
func (t CallTrace) graph(top GraphNode, traces []CallTrace, nm nodeMaker) CallGraph {
	graph := NewCallGraph()
	graph.AddNode(top)
	callee := top.Name

	t.Foreach(traces, func(cs CallSite) {
		node := nm(cs)
		graph.AddNode(node)

		caller := node.Name
		graph.AddCall(caller, callee)
		callee = caller
	})

	return graph
}

//
// Format a map as a GraphViz attribute list.
//
func dotAttrs(attrs map[string]interface{}) string {
	fields := make([]string, len(attrs))

	i := 0
	for k, v := range attrs {
		switch v.(type) {
		case string:
			v = fmt.Sprintf("\"%s\"", v)
		}

		fields[i] = fmt.Sprintf("\"%s\" = %v", k, v)
		i++
	}

	return fmt.Sprintf("[ %s ]", strings.Join(fields, ", "))
}
<|endoftext|>