<commit_before>package acomm

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/cerana/cerana/pkg/logrusx"
)

const (
	statusStarted = iota
	statusStopping
	statusStopped
)

// Tracker keeps track of requests waiting on a response.
type Tracker struct {
	status           int
	responseListener *UnixListener
	httpStreamURL    *url.URL
	externalProxyURL *url.URL
	defaultTimeout   time.Duration
	requestsLock     sync.Mutex // Protects requests
	requests         map[string]*Request
	dsLock           sync.Mutex // Protects dataStreams
	dataStreams      map[string]*UnixListener
	waitgroup        sync.WaitGroup
}

// NewTracker creates and initializes a new Tracker. If a socketPath is not
// provided, the response socket will be created in a temporary directory.
func NewTracker(socketPath string, httpStreamURL, externalProxyURL *url.URL, defaultTimeout time.Duration) (*Tracker, error) {
	if socketPath == "" {
		var err error
		socketPath, err = generateTempSocketPath("", "acommTrackerResponses-")
		if err != nil {
			return nil, err
		}
	}

	if defaultTimeout <= 0 {
		defaultTimeout = time.Minute
	}

	return &Tracker{
		status:           statusStopped,
		responseListener: NewUnixListener(socketPath, 0),
		httpStreamURL:    httpStreamURL,
		externalProxyURL: externalProxyURL,
		dataStreams:      make(map[string]*UnixListener),
		defaultTimeout:   defaultTimeout,
	}, nil
}

func generateTempSocketPath(dir, prefix string) (string, error) {
	// Use TempFile to allocate a uniquely named file in either the specified
	// dir or the default temp dir. It is then removed so that the unix socket
	// can be created with that name.
	// TODO: Decide on permissions
	if dir != "" {
		if err := os.MkdirAll(dir, os.ModePerm); err != nil {
			log.WithFields(log.Fields{
				"directory": dir,
				"perm":      os.ModePerm,
				"error":     err,
			}).Error("failed to create directory for socket")
			return "", err
		}
	}

	f, err := ioutil.TempFile(dir, prefix)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Error("failed to create temp file for response socket")
		return "", err
	}
	_ = f.Close()
	_ = os.Remove(f.Name())

	return fmt.Sprintf("%s.sock", f.Name()), nil
}

// NumRequests returns the number of tracked requests
func (t *Tracker) NumRequests() int {
	t.requestsLock.Lock()
	defer t.requestsLock.Unlock()

	return len(t.requests)
}

// Addr returns the string representation of the Tracker's response listener socket.
func (t *Tracker) Addr() string {
	return t.responseListener.Addr()
}

// URL returns the URL of the Tracker's response listener socket.
func (t *Tracker) URL() *url.URL {
	return t.responseListener.URL()
}

// Start activates the tracker. This allows tracking of requests as well as
// listening for and handling responses.
func (t *Tracker) Start() error {
	if t.status == statusStarted {
		return nil
	}
	if t.status == statusStopping {
		return errors.New("can't start tracker while stopping")
	}

	t.requests = make(map[string]*Request)

	// start the proxy response listener
	if err := t.responseListener.Start(); err != nil {
		return err
	}

	go t.listenForResponses()

	t.status = statusStarted

	return nil
}

// listenForResponse continually accepts new responses on the listener.
func (t *Tracker) listenForResponses() {
	for {
		conn := t.responseListener.NextConn()
		if conn == nil {
			return
		}

		go t.handleConn(conn)
	}
}

// handleConn handles the response connection and parses the data.
func (t *Tracker) handleConn(conn net.Conn) {
	defer t.responseListener.DoneConn(conn)

	resp := &Response{}
	if err := UnmarshalConnData(conn, resp); err != nil {
		return
	}

	_ = SendConnData(conn, &Response{})

	go t.HandleResponse(resp)
}

// HandleResponse associates a response with a request and either forwards the
// response or calls the request's handler.
func (t *Tracker) HandleResponse(resp *Response) {
	req := t.retrieveRequest(resp.ID)
	if req == nil {
		err := errors.New("response does not have tracked request")
		log.WithFields(log.Fields{
			"error":    err,
			"response": resp,
		}).Error(err)
		return
	}
	defer t.waitgroup.Done()

	// Stop the request timeout. The result doesn't matter.
	if req.timeout != nil {
		_ = req.timeout.Stop()
	}

	// If there are handlers, this is the final destination, so handle the
	// response. Otherwise, forward the response along.
	// Known issue: If this is the final destination and there are
	// no handlers, there will be an extra redirects back here. Since the
	// request has already been removed from the tracker, it will only happen
	// once.
	if !req.proxied {
		req.HandleResponse(resp)
		return
	}

	if resp.StreamURL != nil {
		streamURL, err := t.ProxyStreamHTTPURL(resp.StreamURL) // Replace the StreamURL with a proxy stream url
		if err != nil {
			streamURL = nil
		}
		resp.StreamURL = streamURL
	}

	// Forward the response along
	_ = req.Respond(resp)
	return
}

// Stop deactivates the tracker. It blocks until all active connections or tracked requests to finish.
func (t *Tracker) Stop() {
	// Nothing to do if it's not listening.
	if t.responseListener == nil {
		return
	}

	// Prevent new requests from being tracked
	t.status = statusStopping

	// Handle any requests that are expected
	t.waitgroup.Wait()

	// Stop listening for responses
	t.responseListener.Stop(0)

	// Stop any data streamers
	var dsWG sync.WaitGroup
	t.dsLock.Lock()
	for _, ds := range t.dataStreams {
		dsWG.Add(1)
		go func(ds *UnixListener) {
			defer dsWG.Done()
			ds.Stop(0)
		}(ds)
	}
	t.dsLock.Unlock()
	dsWG.Wait()

	t.status = statusStopped
	return
}

// TrackRequest tracks a request. This does not need to be called after using
// ProxyUnix.
func (t *Tracker) TrackRequest(req *Request, timeout time.Duration) error {
	t.requestsLock.Lock()
	defer t.requestsLock.Unlock()

	if t.status == statusStarted {
		if _, ok := t.requests[req.ID]; ok {
			err := errors.New("request id already traacked")
			log.WithFields(log.Fields{
				"request": req,
				"error":   err,
			}).Error(err)
			return err
		}
		t.waitgroup.Add(1)
		t.requests[req.ID] = req

		t.setRequestTimeout(req, timeout)
		return nil
	}

	err := errors.New("failed to track request in unstarted tracker")
	log.WithFields(log.Fields{
		"request":       req,
		"trackerStatus": t.status,
		"error":         err,
	}).Error(err)
	return err
}

// RemoveRequest should be used to remove a tracked request. Use in cases such
// as sending failures, where there is no hope of a response being received.
func (t *Tracker) RemoveRequest(req *Request) bool {
	if r := t.retrieveRequest(req.ID); r != nil {
		if r.timeout != nil {
			_ = r.timeout.Stop()
		}
		t.waitgroup.Done()
		return true
	}
	return false
}

// retrieveRequest returns a tracked Request based on ID and stops tracking it.
func (t *Tracker) retrieveRequest(id string) *Request {
	t.requestsLock.Lock()
	defer t.requestsLock.Unlock()

	if req, ok := t.requests[id]; ok {
		delete(t.requests, id)
		return req
	}

	return nil
}

func (t *Tracker) setRequestTimeout(req *Request, timeout time.Duration) {
	// Fallback to default timeout
	if timeout <= 0 {
		timeout = t.defaultTimeout
	}

	resp, err := NewResponse(req, nil, nil, errors.New("response timeout"))
	if err != nil {
		return
	}

	req.timeout = time.AfterFunc(timeout, func() {
		t.HandleResponse(resp)
	})
	return
}

// ProxyUnix proxies requests that have response hooks and stream urls of
// non-unix sockets. If the response hook and stream url are already unix
// sockets, it returns the original request. If the response hook is not, it
// tracks the original request and returns a new request with a unix socket
// response hook. If the stream url is not, it pipes the original stream
// through a new unix socket and updates the stream url. The purpose of this is
// so that there can be a single entry and exit point for external
// communication, while local services can reply directly to each other.
func (t *Tracker) ProxyUnix(req *Request, timeout time.Duration) (*Request, error) {
	if t.responseListener == nil {
		err := errors.New("request tracker's response listener not active")
		log.WithFields(log.Fields{
			"error": err,
		}).Error(err)
		return nil, err
	}

	if req.StreamURL != nil && req.StreamURL.Scheme != "unix" {
		// proxy the stream
		r, w := io.Pipe()

		go func(src *url.URL) {
			defer logrusx.LogReturnedErr(w.Close, nil, "failed to close proxy stream writer")
			if err := Stream(w, src); err != nil {
				log.WithFields(log.Fields{
					"error":     err,
					"streamURL": src,
				}).Error("failed to stream")
			}
		}(req.StreamURL)

		addr, err := t.NewStreamUnix("", r)
		if err != nil {
			return nil, err
		}

		req.StreamURL = addr
	}

	unixReq := req
	if req.ResponseHook.Scheme != "unix" {
		// proxy the request
		unixReq = &Request{
			ID:           req.ID,
			Task:         req.Task,
			ResponseHook: t.responseListener.URL(),
			StreamURL:    req.StreamURL,
			Args:         req.Args,
			// Success and ErrorHandler are unnecessary here and intentionally
			// omitted.
		}
		if err := t.TrackRequest(req, timeout); err != nil {
			return nil, err
		}
		req.proxied = true
	}

	return unixReq, nil
}

// ProxyExternal proxies a request intended for an external destination
func (t *Tracker) ProxyExternal(req *Request, timeout time.Duration) (*Request, error) {
	if t.externalProxyURL == nil {
		err := errors.New("tracker missing external proxy url")
		log.WithFields(log.Fields{
			"error": err,
		}).Error(err)
		return nil, err
	}
	if t.responseListener == nil {
		err := errors.New("request tracker's response listener not active")
		log.WithFields(log.Fields{
			"error": err,
		}).Error(err)
		return nil, err
	}

	externalReq := &Request{
		ID:           req.ID,
		Task:         req.Task,
		ResponseHook: t.externalProxyURL,
		StreamURL:    req.StreamURL,
		Args:         req.Args,
	}
	if err := t.TrackRequest(req, timeout); err != nil {
		return nil, err
	}
	req.proxied = true

	return externalReq, nil
}

// ProxyExternalHandler is an HTTP HandlerFunc for proxying an external request.
func (t *Tracker) ProxyExternalHandler(w http.ResponseWriter, r *http.Request) {
	resp := &Response{}
	body, err := ioutil.ReadAll(r.Body)
	if err == nil {
		err = json.Unmarshal(body, resp)
	}

	ack := &Response{
		Error: err,
	}
	ackJSON, err := json.Marshal(ack)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
			"ack":   ack,
		}).Error("failed to marshal ack")
		return
	}
	if _, err := w.Write(ackJSON); err != nil {
		log.WithFields(log.Fields{
			"error": err,
			"ack":   ack,
		}).Error("failed to ack response")
		return
	}

	if ack.Error != nil {
		return
	}
	t.HandleResponse(resp)
}
<commit_msg>Proxy StreamURL for external requests<commit_after>package acomm

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/cerana/cerana/pkg/logrusx"
)

const (
	statusStarted = iota
	statusStopping
	statusStopped
)

// Tracker keeps track of requests waiting on a response.
type Tracker struct {
	status           int
	responseListener *UnixListener
	httpStreamURL    *url.URL
	externalProxyURL *url.URL
	defaultTimeout   time.Duration
	requestsLock     sync.Mutex // Protects requests
	requests         map[string]*Request
	dsLock           sync.Mutex // Protects dataStreams
	dataStreams      map[string]*UnixListener
	waitgroup        sync.WaitGroup
}

// NewTracker creates and initializes a new Tracker. If a socketPath is not
// provided, the response socket will be created in a temporary directory.
func NewTracker(socketPath string, httpStreamURL, externalProxyURL *url.URL, defaultTimeout time.Duration) (*Tracker, error) {
	if socketPath == "" {
		var err error
		socketPath, err = generateTempSocketPath("", "acommTrackerResponses-")
		if err != nil {
			return nil, err
		}
	}

	if defaultTimeout <= 0 {
		defaultTimeout = time.Minute
	}

	return &Tracker{
		status:           statusStopped,
		responseListener: NewUnixListener(socketPath, 0),
		httpStreamURL:    httpStreamURL,
		externalProxyURL: externalProxyURL,
		dataStreams:      make(map[string]*UnixListener),
		defaultTimeout:   defaultTimeout,
	}, nil
}

func generateTempSocketPath(dir, prefix string) (string, error) {
	// Use TempFile to allocate a uniquely named file in either the specified
	// dir or the default temp dir. It is then removed so that the unix socket
	// can be created with that name.
	// TODO: Decide on permissions
	if dir != "" {
		if err := os.MkdirAll(dir, os.ModePerm); err != nil {
			log.WithFields(log.Fields{
				"directory": dir,
				"perm":      os.ModePerm,
				"error":     err,
			}).Error("failed to create directory for socket")
			return "", err
		}
	}

	f, err := ioutil.TempFile(dir, prefix)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Error("failed to create temp file for response socket")
		return "", err
	}
	_ = f.Close()
	_ = os.Remove(f.Name())

	return fmt.Sprintf("%s.sock", f.Name()), nil
}

// NumRequests returns the number of tracked requests
func (t *Tracker) NumRequests() int {
	t.requestsLock.Lock()
	defer t.requestsLock.Unlock()

	return len(t.requests)
}

// Addr returns the string representation of the Tracker's response listener socket.
func (t *Tracker) Addr() string {
	return t.responseListener.Addr()
}

// URL returns the URL of the Tracker's response listener socket.
func (t *Tracker) URL() *url.URL {
	return t.responseListener.URL()
}

// Start activates the tracker. This allows tracking of requests as well as
// listening for and handling responses.
func (t *Tracker) Start() error {
	if t.status == statusStarted {
		return nil
	}
	if t.status == statusStopping {
		return errors.New("can't start tracker while stopping")
	}

	t.requests = make(map[string]*Request)

	// start the proxy response listener
	if err := t.responseListener.Start(); err != nil {
		return err
	}

	go t.listenForResponses()

	t.status = statusStarted

	return nil
}

// listenForResponse continually accepts new responses on the listener.
func (t *Tracker) listenForResponses() {
	for {
		conn := t.responseListener.NextConn()
		if conn == nil {
			return
		}

		go t.handleConn(conn)
	}
}

// handleConn handles the response connection and parses the data.
func (t *Tracker) handleConn(conn net.Conn) {
	defer t.responseListener.DoneConn(conn)

	resp := &Response{}
	if err := UnmarshalConnData(conn, resp); err != nil {
		return
	}

	_ = SendConnData(conn, &Response{})

	go t.HandleResponse(resp)
}

// HandleResponse associates a response with a request and either forwards the
// response or calls the request's handler.
func (t *Tracker) HandleResponse(resp *Response) {
	req := t.retrieveRequest(resp.ID)
	if req == nil {
		err := errors.New("response does not have tracked request")
		log.WithFields(log.Fields{
			"error":    err,
			"response": resp,
		}).Error(err)
		return
	}
	defer t.waitgroup.Done()

	// Stop the request timeout. The result doesn't matter.
	if req.timeout != nil {
		_ = req.timeout.Stop()
	}

	// If there are handlers, this is the final destination, so handle the
	// response. Otherwise, forward the response along.
	// Known issue: If this is the final destination and there are
	// no handlers, there will be an extra redirects back here. Since the
	// request has already been removed from the tracker, it will only happen
	// once.
	if !req.proxied {
		req.HandleResponse(resp)
		return
	}

	if resp.StreamURL != nil {
		streamURL, err := t.ProxyStreamHTTPURL(resp.StreamURL) // Replace the StreamURL with a proxy stream url
		if err != nil {
			streamURL = nil
		}
		resp.StreamURL = streamURL
	}

	// Forward the response along
	_ = req.Respond(resp)
	return
}

// Stop deactivates the tracker. It blocks until all active connections or tracked requests to finish.
func (t *Tracker) Stop() {
	// Nothing to do if it's not listening.
	if t.responseListener == nil {
		return
	}

	// Prevent new requests from being tracked
	t.status = statusStopping

	// Handle any requests that are expected
	t.waitgroup.Wait()

	// Stop listening for responses
	t.responseListener.Stop(0)

	// Stop any data streamers
	var dsWG sync.WaitGroup
	t.dsLock.Lock()
	for _, ds := range t.dataStreams {
		dsWG.Add(1)
		go func(ds *UnixListener) {
			defer dsWG.Done()
			ds.Stop(0)
		}(ds)
	}
	t.dsLock.Unlock()
	dsWG.Wait()

	t.status = statusStopped
	return
}

// TrackRequest tracks a request. This does not need to be called after using
// ProxyUnix.
func (t *Tracker) TrackRequest(req *Request, timeout time.Duration) error {
	t.requestsLock.Lock()
	defer t.requestsLock.Unlock()

	if t.status == statusStarted {
		if _, ok := t.requests[req.ID]; ok {
			err := errors.New("request id already traacked")
			log.WithFields(log.Fields{
				"request": req,
				"error":   err,
			}).Error(err)
			return err
		}
		t.waitgroup.Add(1)
		t.requests[req.ID] = req

		t.setRequestTimeout(req, timeout)
		return nil
	}

	err := errors.New("failed to track request in unstarted tracker")
	log.WithFields(log.Fields{
		"request":       req,
		"trackerStatus": t.status,
		"error":         err,
	}).Error(err)
	return err
}

// RemoveRequest should be used to remove a tracked request. Use in cases such
// as sending failures, where there is no hope of a response being received.
func (t *Tracker) RemoveRequest(req *Request) bool {
	if r := t.retrieveRequest(req.ID); r != nil {
		if r.timeout != nil {
			_ = r.timeout.Stop()
		}
		t.waitgroup.Done()
		return true
	}
	return false
}

// retrieveRequest returns a tracked Request based on ID and stops tracking it.
func (t *Tracker) retrieveRequest(id string) *Request {
	t.requestsLock.Lock()
	defer t.requestsLock.Unlock()

	if req, ok := t.requests[id]; ok {
		delete(t.requests, id)
		return req
	}

	return nil
}

func (t *Tracker) setRequestTimeout(req *Request, timeout time.Duration) {
	// Fallback to default timeout
	if timeout <= 0 {
		timeout = t.defaultTimeout
	}

	resp, err := NewResponse(req, nil, nil, errors.New("response timeout"))
	if err != nil {
		return
	}

	req.timeout = time.AfterFunc(timeout, func() {
		t.HandleResponse(resp)
	})
	return
}

// ProxyUnix proxies requests that have response hooks and stream urls of
// non-unix sockets. If the response hook and stream url are already unix
// sockets, it returns the original request. If the response hook is not, it
// tracks the original request and returns a new request with a unix socket
// response hook. If the stream url is not, it pipes the original stream
// through a new unix socket and updates the stream url. The purpose of this is
// so that there can be a single entry and exit point for external
// communication, while local services can reply directly to each other.
func (t *Tracker) ProxyUnix(req *Request, timeout time.Duration) (*Request, error) {
	if t.responseListener == nil {
		err := errors.New("request tracker's response listener not active")
		log.WithFields(log.Fields{
			"error": err,
		}).Error(err)
		return nil, err
	}

	if req.StreamURL != nil && req.StreamURL.Scheme != "unix" {
		// proxy the stream
		r, w := io.Pipe()

		go func(src *url.URL) {
			defer logrusx.LogReturnedErr(w.Close, nil, "failed to close proxy stream writer")
			if err := Stream(w, src); err != nil {
				log.WithFields(log.Fields{
					"error":     err,
					"streamURL": src,
				}).Error("failed to stream")
			}
		}(req.StreamURL)

		addr, err := t.NewStreamUnix("", r)
		if err != nil {
			return nil, err
		}

		req.StreamURL = addr
	}

	unixReq := req
	if req.ResponseHook.Scheme != "unix" {
		// proxy the request
		unixReq = &Request{
			ID:           req.ID,
			Task:         req.Task,
			ResponseHook: t.responseListener.URL(),
			StreamURL:    req.StreamURL,
			Args:         req.Args,
			// Success and ErrorHandler are unnecessary here and intentionally
			// omitted.
		}
		if err := t.TrackRequest(req, timeout); err != nil {
			return nil, err
		}
		req.proxied = true
	}

	return unixReq, nil
}

// ProxyExternal proxies a request intended for an external destination
func (t *Tracker) ProxyExternal(req *Request, timeout time.Duration) (*Request, error) {
	if t.externalProxyURL == nil {
		err := errors.New("tracker missing external proxy url")
		log.WithFields(log.Fields{
			"error": err,
		}).Error(err)
		return nil, err
	}
	if t.responseListener == nil {
		err := errors.New("request tracker's response listener not active")
		log.WithFields(log.Fields{
			"error": err,
		}).Error(err)
		return nil, err
	}

	externalReq := &Request{
		ID:           req.ID,
		Task:         req.Task,
		ResponseHook: t.externalProxyURL,
		Args:         req.Args,
	}
	if req.StreamURL != nil {
		streamURL, err := t.ProxyStreamHTTPURL(req.StreamURL) // Replace the StreamURL with a proxy stream url
		if err != nil {
			return nil, err
		}
		externalReq.StreamURL = streamURL
	}

	if err := t.TrackRequest(req, timeout); err != nil {
		return nil, err
	}
	req.proxied = true

	return externalReq, nil
}

// ProxyExternalHandler is an HTTP HandlerFunc for proxying an external request.
func (t *Tracker) ProxyExternalHandler(w http.ResponseWriter, r *http.Request) {
	resp := &Response{}
	body, err := ioutil.ReadAll(r.Body)
	if err == nil {
		err = json.Unmarshal(body, resp)
	}

	ack := &Response{
		Error: err,
	}
	ackJSON, err := json.Marshal(ack)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
			"ack":   ack,
		}).Error("failed to marshal ack")
		return
	}
	if _, err := w.Write(ackJSON); err != nil {
		log.WithFields(log.Fields{
			"error": err,
			"ack":   ack,
		}).Error("failed to ack response")
		return
	}

	if ack.Error != nil {
		return
	}
	t.HandleResponse(resp)
}
<|endoftext|>


<commit_before>package action

import (
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"github.com/Masterminds/glide/cfg"
	"github.com/Masterminds/glide/msg"
	gpath "github.com/Masterminds/glide/path"
	"github.com/Masterminds/glide/util"
)

// EnsureConfig loads and returns a config file.
//
// Any error will cause an immediate exit, with an error printed to Stderr.
func EnsureConfig() *cfg.Config {
	yamlpath, err := gpath.Glide()
	if err != nil {
		msg.ExitCode(2)
		msg.Die("Failed to find %s file in directory tree: %s", gpath.GlideFile, err)
	}

	yml, err := ioutil.ReadFile(yamlpath)
	if err != nil {
		msg.ExitCode(2)
		msg.Die("Failed to load %s: %s", yamlpath, err)
	}
	conf, err := cfg.ConfigFromYaml(yml)
	if err != nil {
		msg.ExitCode(3)
		msg.Die("Failed to parse %s: %s", yamlpath, err)
	}

	b := filepath.Dir(yamlpath)
	buildContext, err := util.GetBuildContext()
	if err != nil {
		msg.Die("Failed to build an import context while ensuring config: %s", err)
	}
	cwd, err := os.Getwd()
	if err != nil {
		msg.Err("Unable to get the current working directory")
	} else {
		// Determining a package name requires a relative path
		b, err = filepath.Rel(b, cwd)
		if err == nil {
			name := buildContext.PackageName(b)
			if name != conf.Name {
				msg.Warn("The name listed in the config file (%s) does not match the current location (%s)", conf.Name, name)
			}
		} else {
			msg.Warn("Problem finding the config file path (%s) relative to the current directory (%s): %s", b, cwd, err)
		}
	}

	return conf
}

// EnsureCacheDir ensures the existence of the cache directory
func EnsureCacheDir() {
	msg.Warn("ensure.go: ensureCacheDir is not implemented.")
}

// EnsureGoVendor ensures that the Go version is correct.
func EnsureGoVendor() {
	// 6l was removed in 1.5, when vendoring was introduced.
	cmd := exec.Command(goExecutable(), "tool", "6l")
	if _, err := cmd.CombinedOutput(); err == nil {
		msg.Warn("You must install the Go 1.5 or greater toolchain to work with Glide.\n")
		os.Exit(1)
	}

	// Check if this is go15, which requires GO15VENDOREXPERIMENT
	// Any release after go15 does not require that env var.
	cmd = exec.Command(goExecutable(), "version")
	if out, err := cmd.CombinedOutput(); err != nil {
		msg.Err("Error getting version: %s.\n", err)
		os.Exit(1)
	} else if strings.HasPrefix(string(out), "go version 1.5") {
		// This works with 1.5 and 1.6.
		cmd = exec.Command(goExecutable(), "env", "GO15VENDOREXPERIMENT")
		if out, err := cmd.CombinedOutput(); err != nil {
			msg.Err("Error looking for $GOVENDOREXPERIMENT: %s.\n", err)
			os.Exit(1)
		} else if strings.TrimSpace(string(out)) != "1" {
			msg.Err("To use Glide, you must set GO15VENDOREXPERIMENT=1")
			os.Exit(1)
		}
	}

	// In the case where vendoring is explicitly disabled, balk.
	if os.Getenv("GO15VENDOREXPERIMENT") == "0" {
		msg.Err("To use Glide, you must set GO15VENDOREXPERIMENT=1")
		os.Exit(1)
	}

	// Verify the setup isn't for the old version of glide. That is, this is
	// no longer assuming the _vendor directory as the GOPATH. Inform of
	// the change.
	if _, err := os.Stat("_vendor/"); err == nil {
		msg.Warn(`Your setup appears to be for the previous version of Glide.
Previously, vendor packages were stored in _vendor/src/ and
_vendor was set as your GOPATH. As of Go 1.5 the go tools
recognize the vendor directory as a location for these
files. Glide has embraced this. Please remove the _vendor
directory or move the _vendor/src/ directory to vendor/.` + "\n")
		os.Exit(1)
	}
}

// EnsureVendorDir ensures that a vendor/ directory is present in the cwd.
func EnsureVendorDir() {
	fi, err := os.Stat(gpath.VendorDir)
	if err != nil {
		msg.Debug("Creating %s", gpath.VendorDir)
		if err := os.MkdirAll(gpath.VendorDir, os.ModeDir|0755); err != nil {
			msg.Die("Could not create %s: %s", gpath.VendorDir, err)
		}
	} else if !fi.IsDir() {
		msg.Die("Vendor is not a directory")
	}
}

// EnsureGopath fails if GOPATH is not set, or if $GOPATH/src is missing.
//
// Otherwise it returns the value of GOPATH.
func EnsureGopath() string {
	gps := gpath.Gopaths()
	if len(gps) == 0 {
		msg.Die("$GOPATH is not set.")
	}

	for _, gp := range gps {
		_, err := os.Stat(path.Join(gp, "src"))
		if err != nil {
			msg.Warn("%s", err)
			continue
		}
		return gp
	}

	msg.Err("Could not find any of %s/src.\n", strings.Join(gps, "/src, "))
	msg.Info("As of Glide 0.5/Go 1.5, this is required.\n")
	msg.Die("Wihtout src, cannot continue.")
	return ""
}

// goExecutable checks for a set environment variable of GLIDE_GO_EXECUTABLE
// for the go executable name. The Google App Engine SDK ships with a python
// wrapper called goapp
//
// Example usage: GLIDE_GO_EXECUTABLE=goapp glide install
func goExecutable() string {
	goExecutable := os.Getenv("GLIDE_GO_EXECUTABLE")
	if len(goExecutable) <= 0 {
		goExecutable = "go"
	}

	return goExecutable
}
<commit_msg>Fixed typo in action/ensure.go<commit_after>package action

import (
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"github.com/Masterminds/glide/cfg"
	"github.com/Masterminds/glide/msg"
	gpath "github.com/Masterminds/glide/path"
	"github.com/Masterminds/glide/util"
)

// EnsureConfig loads and returns a config file.
//
// Any error will cause an immediate exit, with an error printed to Stderr.
func EnsureConfig() *cfg.Config {
	yamlpath, err := gpath.Glide()
	if err != nil {
		msg.ExitCode(2)
		msg.Die("Failed to find %s file in directory tree: %s", gpath.GlideFile, err)
	}

	yml, err := ioutil.ReadFile(yamlpath)
	if err != nil {
		msg.ExitCode(2)
		msg.Die("Failed to load %s: %s", yamlpath, err)
	}
	conf, err := cfg.ConfigFromYaml(yml)
	if err != nil {
		msg.ExitCode(3)
		msg.Die("Failed to parse %s: %s", yamlpath, err)
	}

	b := filepath.Dir(yamlpath)
	buildContext, err := util.GetBuildContext()
	if err != nil {
		msg.Die("Failed to build an import context while ensuring config: %s", err)
	}
	cwd, err := os.Getwd()
	if err != nil {
		msg.Err("Unable to get the current working directory")
	} else {
		// Determining a package name requires a relative path
		b, err = filepath.Rel(b, cwd)
		if err == nil {
			name := buildContext.PackageName(b)
			if name != conf.Name {
				msg.Warn("The name listed in the config file (%s) does not match the current location (%s)", conf.Name, name)
			}
		} else {
			msg.Warn("Problem finding the config file path (%s) relative to the current directory (%s): %s", b, cwd, err)
		}
	}

	return conf
}

// EnsureCacheDir ensures the existence of the cache directory
func EnsureCacheDir() {
	msg.Warn("ensure.go: ensureCacheDir is not implemented.")
}

// EnsureGoVendor ensures that the Go version is correct.
func EnsureGoVendor() {
	// 6l was removed in 1.5, when vendoring was introduced.
	cmd := exec.Command(goExecutable(), "tool", "6l")
	if _, err := cmd.CombinedOutput(); err == nil {
		msg.Warn("You must install the Go 1.5 or greater toolchain to work with Glide.\n")
		os.Exit(1)
	}

	// Check if this is go15, which requires GO15VENDOREXPERIMENT
	// Any release after go15 does not require that env var.
	cmd = exec.Command(goExecutable(), "version")
	if out, err := cmd.CombinedOutput(); err != nil {
		msg.Err("Error getting version: %s.\n", err)
		os.Exit(1)
	} else if strings.HasPrefix(string(out), "go version 1.5") {
		// This works with 1.5 and 1.6.
		cmd = exec.Command(goExecutable(), "env", "GO15VENDOREXPERIMENT")
		if out, err := cmd.CombinedOutput(); err != nil {
			msg.Err("Error looking for $GOVENDOREXPERIMENT: %s.\n", err)
			os.Exit(1)
		} else if strings.TrimSpace(string(out)) != "1" {
			msg.Err("To use Glide, you must set GO15VENDOREXPERIMENT=1")
			os.Exit(1)
		}
	}

	// In the case where vendoring is explicitly disabled, balk.
	if os.Getenv("GO15VENDOREXPERIMENT") == "0" {
		msg.Err("To use Glide, you must set GO15VENDOREXPERIMENT=1")
		os.Exit(1)
	}

	// Verify the setup isn't for the old version of glide. That is, this is
	// no longer assuming the _vendor directory as the GOPATH. Inform of
	// the change.
	if _, err := os.Stat("_vendor/"); err == nil {
		msg.Warn(`Your setup appears to be for the previous version of Glide.
Previously, vendor packages were stored in _vendor/src/ and
_vendor was set as your GOPATH. As of Go 1.5 the go tools
recognize the vendor directory as a location for these
files. Glide has embraced this. Please remove the _vendor
directory or move the _vendor/src/ directory to vendor/.` + "\n")
		os.Exit(1)
	}
}

// EnsureVendorDir ensures that a vendor/ directory is present in the cwd.
func EnsureVendorDir() {
	fi, err := os.Stat(gpath.VendorDir)
	if err != nil {
		msg.Debug("Creating %s", gpath.VendorDir)
		if err := os.MkdirAll(gpath.VendorDir, os.ModeDir|0755); err != nil {
			msg.Die("Could not create %s: %s", gpath.VendorDir, err)
		}
	} else if !fi.IsDir() {
		msg.Die("Vendor is not a directory")
	}
}

// EnsureGopath fails if GOPATH is not set, or if $GOPATH/src is missing.
//
// Otherwise it returns the value of GOPATH.
func EnsureGopath() string {
	gps := gpath.Gopaths()
	if len(gps) == 0 {
		msg.Die("$GOPATH is not set.")
	}

	for _, gp := range gps {
		_, err := os.Stat(path.Join(gp, "src"))
		if err != nil {
			msg.Warn("%s", err)
			continue
		}
		return gp
	}

	msg.Err("Could not find any of %s/src.\n", strings.Join(gps, "/src, "))
	msg.Info("As of Glide 0.5/Go 1.5, this is required.\n")
	msg.Die("Without src, cannot continue.")
	return ""
}

// goExecutable checks for a set environment variable of GLIDE_GO_EXECUTABLE
// for the go executable name. The Google App Engine SDK ships with a python
// wrapper called goapp
//
// Example usage: GLIDE_GO_EXECUTABLE=goapp glide install
func goExecutable() string {
	goExecutable := os.Getenv("GLIDE_GO_EXECUTABLE")
	if len(goExecutable) <= 0 {
		goExecutable = "go"
	}

	return goExecutable
}
<|endoftext|>


<commit_before>package actions

import (
	"fmt"
	"os"
	"path/filepath"

	"code.google.com/p/go-uuid/uuid"
	"github.com/arschles/gci/config"
	"github.com/arschles/gci/dockutil"
	"github.com/arschles/gci/log"
	"github.com/codegangsta/cli"
)

func Build(c *cli.Context) {
	config := config.ReadOrDie(c.String(FlagConfigFile))
	paths := pathsOrDie()
	projName := filepath.Base(paths.cwd)
	binary := config.Build.GetOutputBinary(projName)

	dockerClient := dockutil.ClientOrDie()

	name := fmt.Sprintf("gci-build-%s-%s", projName, uuid.New())
	cmd := []string{"go", "build", "-o", binary}
	createContainerOpts, hostConfig := dockutil.CreateAndStartContainerOpts(name, cmd, paths.gopath, paths.pkg)
	container, err := dockerClient.CreateContainer(createContainerOpts)
	if err != nil {
		log.Err("creating container [%s]", err)
		os.Exit(1)
	}

	log.Msg(dockutil.CmdStr(createContainerOpts, hostConfig))

	if err := dockerClient.StartContainer(container.ID, &hostConfig); err != nil {
		log.Err("starting container [%s]", err)
		os.Exit(1)
	}

	attachOpts := dockutil.AttachToContainerOpts(container.ID, os.Stdout, os.Stderr)
	attachErrCh, waitErrCh, waitCodeCh := dockutil.AttachAndWait(dockerClient, container.ID, attachOpts)

	select {
	case err := <-attachErrCh:
		log.Err("Attaching to the build container [%s]", err)
		os.Exit(1)
	case err := <-waitErrCh:
		log.Err("Waiting for the build container to finish [%s]", err)
		os.Exit(1)
	case code := <-waitCodeCh:
		if code != 0 {
			log.Err("Build exited %d", code)
			os.Exit(code)
		} else {
			log.Info("Success")
		}
	}
}
<commit_msg>avoiding name collision with imported package<commit_after>package actions

import (
	"fmt"
	"os"
	"path/filepath"

	"code.google.com/p/go-uuid/uuid"
	"github.com/arschles/gci/config"
	"github.com/arschles/gci/dockutil"
	"github.com/arschles/gci/log"
	"github.com/codegangsta/cli"
)

func Build(c *cli.Context) {
	cfg := config.ReadOrDie(c.String(FlagConfigFile))
	paths := pathsOrDie()
	projName := filepath.Base(paths.cwd)
	binary := cfg.Build.GetOutputBinary(projName)

	dockerClient := dockutil.ClientOrDie()

	name := fmt.Sprintf("gci-build-%s-%s", projName, uuid.New())
	cmd := []string{"go", "build", "-o", binary}
	createContainerOpts, hostConfig := dockutil.CreateAndStartContainerOpts(name, cmd, paths.gopath, paths.pkg)
	container, err := dockerClient.CreateContainer(createContainerOpts)
	if err != nil {
		log.Err("creating container [%s]", err)
		os.Exit(1)
	}

	log.Msg(dockutil.CmdStr(createContainerOpts, hostConfig))

	if err := dockerClient.StartContainer(container.ID, &hostConfig); err != nil {
		log.Err("starting container [%s]", err)
		os.Exit(1)
	}

	attachOpts := dockutil.AttachToContainerOpts(container.ID, os.Stdout, os.Stderr)
	attachErrCh, waitErrCh, waitCodeCh := dockutil.AttachAndWait(dockerClient, container.ID, attachOpts)

	select {
	case err := <-attachErrCh:
		log.Err("Attaching to the build container [%s]", err)
		os.Exit(1)
	case err := <-waitErrCh:
		log.Err("Waiting for the build container to finish [%s]", err)
		os.Exit(1)
	case code := <-waitCodeCh:
		if code != 0 {
			log.Err("Build exited %d", code)
			os.Exit(code)
		} else {
			log.Info("Success")
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2022 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"errors"
	"fmt"
	"net/http"
	"time"

	"github.com/google/syzkaller/dashboard/dashapi"
	"github.com/google/syzkaller/pkg/asset"
	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"
	"google.golang.org/appengine/v2"
	db "google.golang.org/appengine/v2/datastore"
	"google.golang.org/appengine/v2/log"
)

// TODO: decide if we want to save job-related assets.

func appendBuildAssets(c context.Context, ns, buildID string, assets []Asset) (*Build, error) {
	var retBuild *Build
	tx := func(c context.Context) error {
		build, err := loadBuild(c, ns, buildID)
		if err != nil {
			return err
		}
		retBuild = build
		appendedOk := false
		var appendErr error
		for _, newAsset := range assets {
			appendErr = build.AppendAsset(newAsset)
			if appendErr == nil {
				appendedOk = true
			}
		}
		// It took quite a number of resources to upload the files, so we return success
		// even if we managed to save at least one of the new assets.
		if !appendedOk {
			return fmt.Errorf("failed to append all assets, last error %w", appendErr)
		}
		if _, err := db.Put(c, buildKey(c, ns, buildID), build); err != nil {
			return fmt.Errorf("failed to put build: %w", err)
		}
		log.Infof(c, "updated build: %#v", build)
		return nil
	}
	if err := db.RunInTransaction(c, tx, &db.TransactionOptions{}); err != nil {
		return nil, err
	}
	return retBuild, nil
}

var ErrAssetDuplicated = errors.New("an asset of this type is already present")

func (build *Build) AppendAsset(addAsset Asset) error {
	typeInfo := asset.GetTypeDescription(addAsset.Type)
	if typeInfo == nil {
		return fmt.Errorf("unknown asset type")
	}
	if !typeInfo.AllowMultiple {
		for _, obj := range build.Assets {
			if obj.Type == addAsset.Type {
				return ErrAssetDuplicated
			}
		}
	}
	build.Assets = append(build.Assets, addAsset)
	return nil
}

func queryNeededAssets(c context.Context) (*dashapi.NeededAssetsResp, error) {
	// So far only build assets.
	var builds []*Build
	_, err := db.NewQuery("Build").
		Filter("Assets.DownloadURL>", "").
		Project("Assets.DownloadURL").
		GetAll(c, &builds)
	if err != nil {
		return nil, fmt.Errorf("failed to query builds: %w", err)
	}
	log.Infof(c, "queried %v builds with assets", len(builds))
	resp := &dashapi.NeededAssetsResp{}
	for _, build := range builds {
		for _, asset := range build.Assets {
			resp.DownloadURLs = append(resp.DownloadURLs, asset.DownloadURL)
		}
	}
	return resp, nil
}

func handleDeprecateAssets(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	for ns := range config.Namespaces {
		err := deprecateNamespaceAssets(c, ns)
		if err != nil {
			log.Errorf(c, "deprecateNamespaceAssets failed for ns=%v: %v", ns, err)
		}
	}
}

func deprecateNamespaceAssets(c context.Context, ns string) error {
	ad := assetDeprecator{
		ns: ns,
		c:  c,
	}
	const buildBatchSize = 16
	err := ad.batchProcessBuilds(buildBatchSize)
	if err != nil {
		return fmt.Errorf("build batch processing failed: %w", err)
	}
	return nil
}

type assetDeprecator struct {
	ns           string
	c            context.Context
	bugsQueried  bool
	relevantBugs map[string]bool
}

const keepAssetsForClosedBugs = time.Hour * 24 * 30

func (ad *assetDeprecator) queryBugs() error {
	if ad.bugsQueried {
		return nil
	}
	var openBugKeys []*db.Key
	var closedBugKeys []*db.Key
	g, _ := errgroup.WithContext(context.Background())
	g.Go(func() error {
		// Query open bugs.
		var err error
		openBugKeys, err = db.NewQuery("Bug").
			Filter("Namespace=", ad.ns).
			Filter("Status=", BugStatusOpen).
			KeysOnly().
			GetAll(ad.c, nil)
		if err != nil {
			return fmt.Errorf("failed to fetch open builds: %w", err)
		}
		return nil
	})
	g.Go(func() error {
		// Query recently closed bugs.
		var err error
		closedBugKeys, err = db.NewQuery("Bug").
			Filter("Namespace=", ad.ns).
			Filter("Closed>", timeNow(ad.c).Add(-keepAssetsForClosedBugs)).
			KeysOnly().
			GetAll(ad.c, nil)
		if err != nil {
			return fmt.Errorf("failed to fetch closed builds: %w", err)
		}
		return nil
	})
	err := g.Wait()
	if err != nil {
		return fmt.Errorf("failed to query bugs: %w", err)
	}
	ad.relevantBugs = map[string]bool{}
	for _, key := range append(append([]*db.Key{}, openBugKeys...), closedBugKeys...) {
		ad.relevantBugs[key.String()] = true
	}
	return nil
}

func (ad *assetDeprecator) buildArchivePolicy(build *Build, asset *Asset) (bool, error) {
	// If the asset is reasonably new, we always keep it.
	const alwaysKeepPeriod = time.Hour * 24 * 14
	if asset.CreateDate.After(timeNow(ad.c).Add(-alwaysKeepPeriod)) {
		return true, nil
	}
	// Query builds to see whether there's a newer same-type asset on the same week.
	var builds []*Build
	_, err := db.NewQuery("Build").
		Filter("Namespace=", ad.ns).
		Filter("Manager=", build.Manager).
		Filter("Assets.Type=", asset.Type).
		Filter("Assets.CreateDate>", asset.CreateDate).
		Limit(1).
		Order("Assets.CreateDate").
		GetAll(ad.c, &builds)
	if err != nil {
		return false, fmt.Errorf("failed to query newer assets: %w", err)
	}
	log.Infof(ad.c, "running archive policy for %s, date %s; queried %d builds",
		asset.DownloadURL, asset.CreateDate, len(builds))
	sameWeek := false
	if len(builds) > 0 {
		origY, origW := asset.CreateDate.ISOWeek()
		for _, nextAsset := range builds[0].Assets {
			if nextAsset.Type != asset.Type {
				continue
			}
			if nextAsset.CreateDate.Before(asset.CreateDate) ||
				nextAsset.CreateDate.Equal(asset.CreateDate) {
				continue
			}
			nextY, nextW := nextAsset.CreateDate.ISOWeek()
			if origY == nextY && origW == nextW {
				log.Infof(ad.c, "found a newer asset: %s, date %s",
					nextAsset.DownloadURL, nextAsset.CreateDate)
				sameWeek = true
				break
			}
		}
	}
	return !sameWeek, nil
}

func (ad *assetDeprecator) buildBugStatusPolicy(build *Build) (bool, error) {
	if err := ad.queryBugs(); err != nil {
		return false, fmt.Errorf("failed to query bugs: %w", err)
	}
	keys, err := db.NewQuery("Crash").
		Filter("BuildID=", build.ID).
		KeysOnly().
		GetAll(ad.c, nil)
	if err != nil {
		return false, fmt.Errorf("failed to query crashes: %w", err)
	}
	for _, key := range keys {
		bugKey := key.Parent()
		if _, ok := ad.relevantBugs[bugKey.String()]; ok {
			// At least one crash is related to an opened/recently closed bug.
			return true, nil
		}
	}
	return false, nil
}

func (ad *assetDeprecator) needThisBuildAsset(build *Build, buildAsset *Asset) (bool, error) {
	if buildAsset.Type == dashapi.HTMLCoverageReport {
		// We want to keep coverage reports forever, not just
		// while there are any open bugs. But we don't want to
		// keep all coverage reports, just a share of them.
		return ad.buildArchivePolicy(build, buildAsset)
	}
	if build.Type == BuildNormal {
		// A build-related asset, keep it only while there are open bugs with crashes
		// related to this build.
		return ad.buildBugStatusPolicy(build)
	}
	// TODO: fix this once this is no longer the case.
	return false, fmt.Errorf("job-related assets are not supported yet")
}

func (ad *assetDeprecator) updateBuild(buildID string, urlsToDelete []string) error {
	toDelete := map[string]bool{}
	for _, url := range urlsToDelete {
		toDelete[url] = true
	}
	tx := func(c context.Context) error {
		build, err := loadBuild(ad.c, ad.ns, buildID)
		if build == nil || err != nil {
			// Assume the DB has been updated in the meanwhile.
			return nil
		}
		newAssets := []Asset{}
		for _, asset := range build.Assets {
			if _, ok := toDelete[asset.DownloadURL]; !ok {
				newAssets = append(newAssets, asset)
			}
		}
		build.Assets = newAssets
		build.AssetsLastCheck = timeNow(ad.c)
		if _, err := db.Put(ad.c, buildKey(ad.c, ad.ns, buildID), build); err != nil {
			return fmt.Errorf("failed to save build: %w", err)
		}
		return nil
	}
	if err := db.RunInTransaction(ad.c, tx, nil); err != nil {
		return fmt.Errorf("failed to update build: %w", err)
	}
	return nil
}

func (ad *assetDeprecator) batchProcessBuilds(count int) error {
	// We cannot query only the Build with non-empty Assets array and yet sort
	// by AssetsLastCheck. The datastore returns "The first sort property must
	// be the same as the property to which the inequality filter is applied.
	// In your query the first sort property is AssetsLastCheck but the inequality
	// filter is on Assets.DownloadURL.
	// So we have to omit Filter("Assets.DownloadURL>", ""). here.
	var builds []*Build
	_, err := db.NewQuery("Build").
		Filter("Namespace=", ad.ns).
		Order("AssetsLastCheck").
		Limit(count).
		GetAll(ad.c, &builds)
	if err != nil {
		return fmt.Errorf("failed to fetch builds: %w", err)
	}
	for _, build := range builds {
		toDelete := []string{}
		for _, asset := range build.Assets {
			needed, err := ad.needThisBuildAsset(build, &asset)
			if err != nil {
				return fmt.Errorf("failed to test asset: %w", err)
			} else if !needed {
				toDelete = append(toDelete, asset.DownloadURL)
			}
		}
		err := ad.updateBuild(build.ID, toDelete)
		if err != nil {
			return err
		}
	}
	return nil
}

func queryLatestManagerAssets(c context.Context, ns string, assetType dashapi.AssetType,
	period time.Duration) (map[string]Asset, error) {
	var builds []*Build
	startTime := timeNow(c).Add(-period)
	_, err := db.NewQuery("Build").
		Filter("Namespace=", ns).
		Filter("Assets.Type=", assetType).
		Filter("Assets.CreateDate>", startTime).
		Order("Assets.CreateDate").
		GetAll(c, &builds)
	if err != nil {
		return nil, err
	}
	ret := map[string]Asset{}
	for _, build := range builds {
		for _, asset := range build.Assets {
			if asset.Type != assetType {
				continue
			}
			ret[build.Manager] = asset
		}
	}
	return ret, nil
}
<commit_msg>dashboard: enable build error asset deprecation<commit_after>// Copyright 2022 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"errors"
	"fmt"
	"net/http"
	"time"

	"github.com/google/syzkaller/dashboard/dashapi"
	"github.com/google/syzkaller/pkg/asset"
	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"
	"google.golang.org/appengine/v2"
	db "google.golang.org/appengine/v2/datastore"
	"google.golang.org/appengine/v2/log"
)

// TODO: decide if we want to save job-related assets.

func appendBuildAssets(c context.Context, ns, buildID string, assets []Asset) (*Build, error) {
	var retBuild *Build
	tx := func(c context.Context) error {
		build, err := loadBuild(c, ns, buildID)
		if err != nil {
			return err
		}
		retBuild = build
		appendedOk := false
		var appendErr error
		for _, newAsset := range assets {
			appendErr = build.AppendAsset(newAsset)
			if appendErr == nil {
				appendedOk = true
			}
		}
		// It took quite a number of resources to upload the files, so we return success
		// even if we managed to save at least one of the new assets.
		if !appendedOk {
			return fmt.Errorf("failed to append all assets, last error %w", appendErr)
		}
		if _, err := db.Put(c, buildKey(c, ns, buildID), build); err != nil {
			return fmt.Errorf("failed to put build: %w", err)
		}
		log.Infof(c, "updated build: %#v", build)
		return nil
	}
	if err := db.RunInTransaction(c, tx, &db.TransactionOptions{}); err != nil {
		return nil, err
	}
	return retBuild, nil
}

var ErrAssetDuplicated = errors.New("an asset of this type is already present")

func (build *Build) AppendAsset(addAsset Asset) error {
	typeInfo := asset.GetTypeDescription(addAsset.Type)
	if typeInfo == nil {
		return fmt.Errorf("unknown asset type")
	}
	if !typeInfo.AllowMultiple {
		for _, obj := range build.Assets {
			if obj.Type == addAsset.Type {
				return ErrAssetDuplicated
			}
		}
	}
	build.Assets = append(build.Assets, addAsset)
	return nil
}

func queryNeededAssets(c context.Context) (*dashapi.NeededAssetsResp, error) {
	// So far only build assets.
	var builds []*Build
	_, err := db.NewQuery("Build").
		Filter("Assets.DownloadURL>", "").
		Project("Assets.DownloadURL").
		GetAll(c, &builds)
	if err != nil {
		return nil, fmt.Errorf("failed to query builds: %w", err)
	}
	log.Infof(c, "queried %v builds with assets", len(builds))
	resp := &dashapi.NeededAssetsResp{}
	for _, build := range builds {
		for _, asset := range build.Assets {
			resp.DownloadURLs = append(resp.DownloadURLs, asset.DownloadURL)
		}
	}
	return resp, nil
}

func handleDeprecateAssets(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	for ns := range config.Namespaces {
		err := deprecateNamespaceAssets(c, ns)
		if err != nil {
			log.Errorf(c, "deprecateNamespaceAssets failed for ns=%v: %v", ns, err)
		}
	}
}

func deprecateNamespaceAssets(c context.Context, ns string) error {
	ad := assetDeprecator{
		ns: ns,
		c:  c,
	}
	const buildBatchSize = 16
	err := ad.batchProcessBuilds(buildBatchSize)
	if err != nil {
		return fmt.Errorf("build batch processing failed: %w", err)
	}
	return nil
}

type assetDeprecator struct {
	ns           string
	c            context.Context
	bugsQueried  bool
	relevantBugs map[string]bool
}

const keepAssetsForClosedBugs = time.Hour * 24 * 30

func (ad *assetDeprecator) queryBugs() error {
	if ad.bugsQueried {
		return nil
	}
	var openBugKeys []*db.Key
	var closedBugKeys []*db.Key
	g, _ := errgroup.WithContext(context.Background())
	g.Go(func() error {
		// Query open bugs.
		var err error
		openBugKeys, err = db.NewQuery("Bug").
			Filter("Namespace=", ad.ns).
			Filter("Status=", BugStatusOpen).
			KeysOnly().
			GetAll(ad.c, nil)
		if err != nil {
			return fmt.Errorf("failed to fetch open builds: %w", err)
		}
		return nil
	})
	g.Go(func() error {
		// Query recently closed bugs.
		var err error
		closedBugKeys, err = db.NewQuery("Bug").
			Filter("Namespace=", ad.ns).
			Filter("Closed>", timeNow(ad.c).Add(-keepAssetsForClosedBugs)).
			KeysOnly().
			GetAll(ad.c, nil)
		if err != nil {
			return fmt.Errorf("failed to fetch closed builds: %w", err)
		}
		return nil
	})
	err := g.Wait()
	if err != nil {
		return fmt.Errorf("failed to query bugs: %w", err)
	}
	ad.relevantBugs = map[string]bool{}
	for _, key := range append(append([]*db.Key{}, openBugKeys...), closedBugKeys...) {
		ad.relevantBugs[key.String()] = true
	}
	return nil
}

func (ad *assetDeprecator) buildArchivePolicy(build *Build, asset *Asset) (bool, error) {
	// If the asset is reasonably new, we always keep it.
	const alwaysKeepPeriod = time.Hour * 24 * 14
	if asset.CreateDate.After(timeNow(ad.c).Add(-alwaysKeepPeriod)) {
		return true, nil
	}
	// Query builds to see whether there's a newer same-type asset on the same week.
	var builds []*Build
	_, err := db.NewQuery("Build").
		Filter("Namespace=", ad.ns).
		Filter("Manager=", build.Manager).
		Filter("Assets.Type=", asset.Type).
		Filter("Assets.CreateDate>", asset.CreateDate).
		Limit(1).
		Order("Assets.CreateDate").
		GetAll(ad.c, &builds)
	if err != nil {
		return false, fmt.Errorf("failed to query newer assets: %w", err)
	}
	log.Infof(ad.c, "running archive policy for %s, date %s; queried %d builds",
		asset.DownloadURL, asset.CreateDate, len(builds))
	sameWeek := false
	if len(builds) > 0 {
		origY, origW := asset.CreateDate.ISOWeek()
		for _, nextAsset := range builds[0].Assets {
			if nextAsset.Type != asset.Type {
				continue
			}
			if nextAsset.CreateDate.Before(asset.CreateDate) ||
				nextAsset.CreateDate.Equal(asset.CreateDate) {
				continue
			}
			nextY, nextW := nextAsset.CreateDate.ISOWeek()
			if origY == nextY && origW == nextW {
				log.Infof(ad.c, "found a newer asset: %s, date %s",
					nextAsset.DownloadURL, nextAsset.CreateDate)
				sameWeek = true
				break
			}
		}
	}
	return !sameWeek, nil
}

func (ad *assetDeprecator) buildBugStatusPolicy(build *Build) (bool, error) {
	if err := ad.queryBugs(); err != nil {
		return false, fmt.Errorf("failed to query bugs: %w", err)
	}
	keys, err := db.NewQuery("Crash").
		Filter("BuildID=", build.ID).
		KeysOnly().
		GetAll(ad.c, nil)
	if err != nil {
		return false, fmt.Errorf("failed to query crashes: %w", err)
	}
	for _, key := range keys {
		bugKey := key.Parent()
		if _, ok := ad.relevantBugs[bugKey.String()]; ok {
			// At least one crash is related to an opened/recently closed bug.
			return true, nil
		}
	}
	return false, nil
}

func (ad *assetDeprecator) needThisBuildAsset(build *Build, buildAsset *Asset) (bool, error) {
	if buildAsset.Type == dashapi.HTMLCoverageReport {
		// We want to keep coverage reports forever, not just
		// while there are any open bugs. But we don't want to
		// keep all coverage reports, just a share of them.
		return ad.buildArchivePolicy(build, buildAsset)
	}
	if build.Type == BuildNormal || build.Type == BuildFailed {
		// A build-related asset, keep it only while there are open bugs with crashes
		// related to this build.
		return ad.buildBugStatusPolicy(build)
	}
	// TODO: fix this once this is no longer the case.
	return false, fmt.Errorf("job-related assets are not supported yet")
}

func (ad *assetDeprecator) updateBuild(buildID string, urlsToDelete []string) error {
	toDelete := map[string]bool{}
	for _, url := range urlsToDelete {
		toDelete[url] = true
	}
	tx := func(c context.Context) error {
		build, err := loadBuild(ad.c, ad.ns, buildID)
		if build == nil || err != nil {
			// Assume the DB has been updated in the meanwhile.
			return nil
		}
		newAssets := []Asset{}
		for _, asset := range build.Assets {
			if _, ok := toDelete[asset.DownloadURL]; !ok {
				newAssets = append(newAssets, asset)
			}
		}
		build.Assets = newAssets
		build.AssetsLastCheck = timeNow(ad.c)
		if _, err := db.Put(ad.c, buildKey(ad.c, ad.ns, buildID), build); err != nil {
			return fmt.Errorf("failed to save build: %w", err)
		}
		return nil
	}
	if err := db.RunInTransaction(ad.c, tx, nil); err != nil {
		return fmt.Errorf("failed to update build: %w", err)
	}
	return nil
}

func (ad *assetDeprecator) batchProcessBuilds(count int) error {
	// We cannot query only the Build with non-empty Assets array and yet sort
	// by AssetsLastCheck. The datastore returns "The first sort property must
	// be the same as the property to which the inequality filter is applied.
	// In your query the first sort property is AssetsLastCheck but the inequality
	// filter is on Assets.DownloadURL.
	// So we have to omit Filter("Assets.DownloadURL>", ""). here.
	var builds []*Build
	_, err := db.NewQuery("Build").
		Filter("Namespace=", ad.ns).
		Order("AssetsLastCheck").
		Limit(count).
		GetAll(ad.c, &builds)
	if err != nil {
		return fmt.Errorf("failed to fetch builds: %w", err)
	}
	for _, build := range builds {
		toDelete := []string{}
		for _, asset := range build.Assets {
			needed, err := ad.needThisBuildAsset(build, &asset)
			if err != nil {
				return fmt.Errorf("failed to test asset: %w", err)
			} else if !needed {
				toDelete = append(toDelete, asset.DownloadURL)
			}
		}
		err := ad.updateBuild(build.ID, toDelete)
		if err != nil {
			return err
		}
	}
	return nil
}

func queryLatestManagerAssets(c context.Context, ns string, assetType dashapi.AssetType,
	period time.Duration) (map[string]Asset, error) {
	var builds []*Build
	startTime := timeNow(c).Add(-period)
	_, err := db.NewQuery("Build").
		Filter("Namespace=", ns).
		Filter("Assets.Type=", assetType).
		Filter("Assets.CreateDate>", startTime).
		Order("Assets.CreateDate").
		GetAll(c, &builds)
	if err != nil {
		return nil, err
	}
	ret := map[string]Asset{}
	for _, build := range builds {
		for _, asset := range build.Assets {
			if asset.Type != assetType {
				continue
			}
			ret[build.Manager] = asset
		}
	}
	return ret, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2016, Nick Patavalis (npat@efault.net).
// All rights reserved.
// Use of this source code is governed by a BSD-style license that can
// be found in the LICENSE.txt file.

// Package task provides types and functions for managing tasks. Task
// is a process performed by a goroutine, or a group of related
// goroutines, that can be Kill'ed (stopped) and Wait'ed-for. In
// addition, a helper type is provided for controlling the life-cycle
// of servers (starting, stopping, waiting for, and restarting them).
package task

import (
	"context"
	"errors"
	"sync"
)

var (
	ErrNotStarted = errors.New("Server not started")
)

// Task is a process performed by a goroutine, or a group of related
// goroutines, that can be Kill'ed (stopped) and Wait'ed-for. Starting
// the task is beyond the scope of this interface. Specific
// implementations (like the ones included in this package), provide
// methods to start the respective tasks. All Task interface methods
// can be safely called concurently from multiple goroutines.
type Task interface {
	// Kill requests that the task terminates as soon as
	// possible. Kill returns immediately and does not wait for
	// the task to terminate. It is ok to call Kill multiple
	// times. After the first, subsequent calls do nothing.
	Kill() Task
	// Wait waits for the task to terminate, and returns its exit
	// status. It is ok to call Wait multiple times. If called
	// after the task has terminated, it returns its exit status
	// immediately.
	Wait() error
	// WaitChan returns a channel that will be closed when the
	// task terminates. WaitChan does not wait for the task to
	// terminate. Receiving from the returned channel blocks until
	// it does. Once the receive from the returned channel
	// succeeds, Wait can be called to retrieve the task's exit
	// status.
	WaitChan() <-chan struct{}
}

// StartFunc is an entry-point function that can be run as a task. The
// function must take a context.Context as an argument, which may be
// canceled to request the task's termination.
type StartFunc func(context.Context) error

// Single is used to start a signle goroutine as a task (a goroutine
// that can be killed and waited-for). All Single methods can be
// called concurently.
type Single struct {
	cancel func()
	end    chan struct{}
	err    error
}

// Go starts a task using the given function as entry point.
func Go(f StartFunc) *Single {
	return GoWithContext(context.Background(), f)
}

// GoWithContext is similar to Go, but uses ctx as the parent of
// the context that will be used for the task's cancelation.
func GoWithContext(ctx context.Context, f StartFunc) *Single {
	s := &Single{}
	ctx, s.cancel = context.WithCancel(ctx)
	s.end = make(chan struct{})
	go func() {
		s.err = f(ctx)
		s.cancel()
		close(s.end)
	}()
	return s
}

// Kill requests task s to quit. Kill returns imediatelly (does not
// wait for the task to terminate).
func (s *Single) Kill() Task {
	s.cancel()
	return s
}

// Wait waits for task s to terminate and returns its exit status
// (i.e. the return value of its entry-point StartFunc).
func (s *Single) Wait() error {
	<-s.end
	return s.err
}

// WaitChan returns a channel that will be closed when the task
// terminates. After the receive from the returned channel suceeds,
// Wait can be called to retrieve the task's exit status. Usefull for
// select statements.
func (s *Single) WaitChan() <-chan struct{} {
	return s.end
}

// Grp is used to start a set (a group) of goroutines as a single
// task. All goroutines share the same cancelation context and thus
// can be killed and waited-for collectively.
type Grp struct {
	sync.Mutex
	ctx         context.Context
	cancel      func()
	wg          sync.WaitGroup
	killOnError bool
	end         chan struct{}
	err         error
}

// NewGrp creates and returns a new group. Grp implements the Task
// interface. Initially the group is empty (no running
// goroutines). Grp.Go must be subsequently called to start goroutines
// in the group.
func NewGrp() *Grp {
	g := &Grp{}
	g.ctx, g.cancel = context.WithCancel(context.Background())
	return g
}

// NewGrpWithContext is similar to NewGrp, but uses ctx as the parent
// of the context that will be used for the task's cancelation.
func NewGrpWithContext(ctx context.Context) *Grp {
	g := &Grp{}
	g.ctx, g.cancel = context.WithCancel(ctx)
	return g
}

// KillOnError enables the kill-on-error behavior for the group (by
// default disabled). If enabled, the task is Kill'ed if one of it's
// goroutines returns a non-nil error. Usually KillOnError is called
// before starting the group's goroutines.
func (g *Grp) KillOnError() *Grp {
	g.Lock()
	g.killOnError = true
	g.Unlock()
	return g
}

// Go starts a goroutine in the group using the StartFunc f as an
// entry point. Go can be called multiple times to start multiple
// goroutines.
func (g *Grp) Go(f StartFunc) *Grp {
	g.wg.Add(1)
	go func() {
		defer g.wg.Done()
		if err := f(g.ctx); err != nil {
			g.Lock()
			if g.err == nil {
				// keep first non-nil error
				g.err = err
			}
			if g.killOnError {
				g.cancel()
			}
			g.Unlock()
		}
	}()
	return g
}

// Kill requests task g to quit (signals all its goroutines to
// terminate). Kill returns imediatelly (does not wait for the task to
// terminate).
func (g *Grp) Kill() Task {
	g.cancel()
	return g
}

// Wait waits for task g to terminate and returns its exit-status. The
// task terminates when all it's goroutines exit. As exit status of
// the task is considered the first non-nil value returned by the
// StartFunc of one of it's goroutines.
func (g *Grp) Wait() error {
	g.wg.Wait()
	g.cancel()
	return g.err
}

// WaitChan returns a channel that will be closed when task g
// terminates.
func (g *Grp) WaitChan() <-chan struct{} {
	g.Lock()
	defer g.Unlock()
	if g.end == nil {
		g.end = make(chan struct{})
		go func() {
			g.Wait()
			close(g.end)
		}()
	}
	return g.end
}

// SrvCtl is a server controller. It is a helper type that povides
// methods for starting, stopping, waiting, and re-starting
// server-instances in a convenient, race-free manner. A pointer to
// SrvCtl can be embedded (either anonymoysly or not) in the
// controlled server type. SrvCtl is initialized (via NewSrvCtl) by
// two functions (most likely method-values): One is spawned as the
// sever-task (goroutine) when Start is called; the other (which may
// be nil) is called immediately before. It is safe to call all SrvCtl
// methods concurently. See example for details.
//
type SrvCtl struct {
	m       sync.Mutex
	t       *Single
	fnStart StartFunc // func(context.Context) error
	fnPre   func()
}

// NewSrvCtl returns (a pointer to) an initialized server
// controller. The returned pointer can be embedded (either
// anonymously, or as a named field) in the controlled server type /
// structure. It is intialized with two functions (most likely
// method-values): fnStart is the function that will be spawned as the
// server task (goroutine). fnPre is a function to be called before
// spawning the server goroutine. fnPre may be nil.
func NewSrvCtl(fnStart StartFunc, fnPre func()) *SrvCtl {
	return &SrvCtl{fnStart: fnStart, fnPre: fnPre}
}

// task is a helper that reads and returns a pointer to the
// task-structure atomically
func (sc *SrvCtl) task() *Single {
	sc.m.Lock()
	t := sc.t
	sc.m.Unlock()
	return t
}

// Start starts the controlled server instance as a task (i.e in its
// own goroutine). If already running, it stops it, waits for it to
// terminate, and re-starts it.
func (sc *SrvCtl) Start() Task {
	t := sc.task()
	if t != nil {
		t.Kill().Wait()
	}
	sc.m.Lock()
	defer sc.m.Unlock()
	if sc.t != t {
		// Concurrent re-start.
		// Another racer won.
		return sc
	}
	if sc.fnPre != nil {
		sc.fnPre()
	}
	sc.t = Go(sc.fnStart)
	return sc
}

// Kill requests the termination of the controlled server instance. It
// does not wait for the server to terminate.
func (sc *SrvCtl) Kill() Task {
	t := sc.task()
	if t == nil {
		return sc
	}
	t.Kill()
	return sc
}

// Wait waits for the controlled server to terminate and returns it's
// exit code (the return-value of the fnStart function, see
// NewSrvCtl). If the managed server has never been started, returns
// ErrNotStarted.
func (sc *SrvCtl) Wait() error {
	t := sc.task()
	if t == nil {
		return ErrNotStarted
	}
	return t.Wait()
}

// WaitChan returns a channel from which the managed server's exit
// code (the server's Serve return value) can be received. WaitChan
// does not wait for the server to terminate. Reading from the channel
// blocks the reader until it does. If the server has *never* been
// started, a receive from the returned channel will yield
// ErrNotStarted.
func (sc *SrvCtl) WaitChan() <-chan struct{} {
	t := sc.task()
	if t == nil {
		cerr := make(chan struct{})
		close(cerr)
		return cerr
	}
	return t.WaitChan()
}
<commit_msg>Comments<commit_after>// Copyright (c) 2016, Nick Patavalis (npat@efault.net).
// All rights reserved.
// Use of this source code is governed by a BSD-style license that can
// be found in the LICENSE.txt file.

// Package task provides types and functions for managing tasks. Task
// is a process performed by a goroutine, or a group of related
// goroutines, that can be Kill'ed (stopped) and Wait'ed-for. In
// addition, a helper type is provided for controlling the life-cycle
// of servers (starting, stopping, waiting for, and restarting them).
package task

import (
	"context"
	"errors"
	"sync"
)

var (
	ErrNotStarted = errors.New("Server not started")
)

// Task is a process performed by a goroutine, or a group of related
// goroutines, that can be Kill'ed (stopped) and Wait'ed-for. Starting
// the task is beyond the scope of this interface. Specific
// implementations (like the ones included in this package), provide
// methods to start the respective tasks. All Task interface methods
// can be safely called concurently from multiple goroutines.
type Task interface {
	// Kill requests that the task terminates as soon as
	// possible. Kill returns immediately and does not wait for
	// the task to terminate. It is ok to call Kill multiple
	// times. After the first, subsequent calls do nothing.
	Kill() Task
	// Wait waits for the task to terminate, and returns its exit
	// status. It is ok to call Wait multiple times. If called
	// after the task has terminated, it returns its exit status
	// immediately.
	Wait() error
	// WaitChan returns a channel that will be closed when the
	// task terminates. WaitChan does not wait for the task to
	// terminate. Receiving from the returned channel blocks until
	// it does. Once the receive from the returned channel
	// succeeds, Wait can be called to retrieve the task's exit
	// status.
	WaitChan() <-chan struct{}
}

// StartFunc is an entry-point function that can be run as a task. The
// function must take a context.Context as an argument, which may be
// canceled to request the task's termination.
type StartFunc func(context.Context) error

// Single is used to start a signle goroutine as a task (a goroutine
// that can be killed and waited-for). All Single methods can be
// called concurently.
type Single struct {
	cancel func()
	end    chan struct{}
	err    error
}

// Go starts a task using the given function as entry point.
func Go(f StartFunc) *Single {
	return GoWithContext(context.Background(), f)
}

// GoWithContext is similar to Go, but uses ctx as the parent of
// the context that will be used for the task's cancelation.
func GoWithContext(ctx context.Context, f StartFunc) *Single {
	s := &Single{}
	ctx, s.cancel = context.WithCancel(ctx)
	s.end = make(chan struct{})
	go func() {
		s.err = f(ctx)
		s.cancel()
		close(s.end)
	}()
	return s
}

// Kill requests task s to quit. Kill returns imediatelly (does not
// wait for the task to terminate).
func (s *Single) Kill() Task {
	s.cancel()
	return s
}

// Wait waits for task s to terminate and returns its exit status
// (i.e. the return value of its entry-point StartFunc).
func (s *Single) Wait() error {
	<-s.end
	return s.err
}

// WaitChan returns a channel that will be closed when the task
// terminates. After the receive from the returned channel suceeds,
// Wait can be called to retrieve the task's exit status. Usefull for
// select statements.
func (s *Single) WaitChan() <-chan struct{} {
	return s.end
}

// Grp is used to start a set (a group) of goroutines as a single
// task. All goroutines share the same cancelation context and thus
// can be killed and waited-for collectively.
type Grp struct {
	sync.Mutex
	ctx         context.Context
	cancel      func()
	wg          sync.WaitGroup
	killOnError bool
	end         chan struct{}
	err         error
}

// NewGrp creates and returns a new group. Grp implements the Task
// interface. Initially the group is empty (no running
// goroutines). Grp.Go must be subsequently called to start goroutines
// in the group.
func NewGrp() *Grp {
	g := &Grp{}
	g.ctx, g.cancel = context.WithCancel(context.Background())
	return g
}

// NewGrpWithContext is similar to NewGrp, but uses ctx as the parent
// of the context that will be used for the task's cancelation.
func NewGrpWithContext(ctx context.Context) *Grp {
	g := &Grp{}
	g.ctx, g.cancel = context.WithCancel(ctx)
	return g
}

// KillOnError enables the kill-on-error behavior for the group (by
// default disabled). If enabled, the task is Kill'ed if one of it's
// goroutines returns a non-nil error. Usually KillOnError is called
// before starting the group's goroutines.
func (g *Grp) KillOnError() *Grp {
	g.Lock()
	g.killOnError = true
	g.Unlock()
	return g
}

// Go starts a goroutine in the group using the StartFunc f as an
// entry point. Go can be called multiple times to start multiple
// goroutines.
func (g *Grp) Go(f StartFunc) *Grp {
	g.wg.Add(1)
	go func() {
		defer g.wg.Done()
		if err := f(g.ctx); err != nil {
			g.Lock()
			if g.err == nil {
				// keep first non-nil error
				g.err = err
			}
			if g.killOnError {
				g.cancel()
			}
			g.Unlock()
		}
	}()
	return g
}

// Kill requests task g to quit (signals all its goroutines to
// terminate). Kill returns imediatelly (does not wait for the task to
// terminate).
func (g *Grp) Kill() Task {
	g.cancel()
	return g
}

// Wait waits for task g to terminate and returns its exit-status. The
// task terminates when all it's goroutines exit. As exit status of
// the task is considered the first non-nil value returned by the
// StartFunc of one of it's goroutines.
func (g *Grp) Wait() error {
	g.wg.Wait()
	g.cancel()
	return g.err
}

// WaitChan returns a channel that will be closed when task g
// terminates.
func (g *Grp) WaitChan() <-chan struct{} {
	g.Lock()
	defer g.Unlock()
	if g.end == nil {
		g.end = make(chan struct{})
		go func() {
			g.Wait()
			close(g.end)
		}()
	}
	return g.end
}

// SrvCtl is a server controller. It is a helper type that povides
// methods for starting, stopping, waiting, and re-starting
// server-instances in a convenient, race-free manner. A pointer to
// SrvCtl can be embedded (either anonymoysly or not) in the
// controlled server type. SrvCtl is initialized (via NewSrvCtl) by
// two functions (most likely method-values): One is spawned as the
// sever-task (goroutine) when Start is called; the other (which may
// be nil) is called immediately before. It is safe to call all SrvCtl
// methods concurently. See example for details.
//
type SrvCtl struct {
	m       sync.Mutex
	t       *Single
	fnStart StartFunc // func(context.Context) error
	fnPre   func()
}

// NewSrvCtl returns (a pointer to) an initialized server
// controller. The returned pointer can be embedded (either
// anonymously, or as a named field) in the controlled server type /
// structure. It is intialized with two functions (most likely
// method-values): fnStart is the function that will be spawned as the
// server task (goroutine). fnPre is a function to be called before
// spawning the server goroutine. fnPre may be nil.
func NewSrvCtl(fnStart StartFunc, fnPre func()) *SrvCtl {
	return &SrvCtl{fnStart: fnStart, fnPre: fnPre}
}

// task is a helper that reads and returns a pointer to the
// task-structure atomically
func (sc *SrvCtl) task() *Single {
	sc.m.Lock()
	t := sc.t
	sc.m.Unlock()
	return t
}

// Start starts the controlled server instance as a task (i.e in its
// own goroutine). If already running, it stops it, waits for it to
// terminate, and re-starts it.
func (sc *SrvCtl) Start() Task {
	t := sc.task()
	if t != nil {
		t.Kill().Wait()
	}
	sc.m.Lock()
	defer sc.m.Unlock()
	if sc.t != t {
		// Concurrent re-start.
		// Another racer won.
		return sc
	}
	if sc.fnPre != nil {
		sc.fnPre()
	}
	sc.t = Go(sc.fnStart)
	return sc
}

// Kill requests the termination of the controlled server instance. It
// does not wait for the server to terminate.
func (sc *SrvCtl) Kill() Task {
	t := sc.task()
	if t == nil {
		return sc
	}
	t.Kill()
	return sc
}

// Wait waits for the controlled server to terminate and returns it's
// exit code (the return-value of the fnStart function, see
// NewSrvCtl). If the managed server has never been started, returns
// ErrNotStarted.
func (sc *SrvCtl) Wait() error {
	t := sc.task()
	if t == nil {
		return ErrNotStarted
	}
	return t.Wait()
}

// WaitChan returns a channel that will be closed when the
// controlled-server terminates. After the receive from the returned
// channel suceeds, Wait can be called to retrieve the server's exit
// status. Usefull for select statements.
func (sc *SrvCtl) WaitChan() <-chan struct{} {
	t := sc.task()
	if t == nil {
		cerr := make(chan struct{})
		close(cerr)
		return cerr
	}
	return t.WaitChan()
}
<|endoftext|>


<commit_before>package eval

import (
	"os"
	"sync"
)

const (
	stdoutChanSize = 32
	stderrChanSize = 32
)

type evalerPorts struct {
	ports      [3]*Port
	relayeWait *sync.WaitGroup
}

func newEvalerPorts(stdin, stdout, stderr *os.File, prefix *string) evalerPorts {
	stdoutChan := make(chan Value, stdoutChanSize)
	stderrChan := make(chan Value, stderrChanSize)

	var relayerWait sync.WaitGroup
	relayerWait.Add(2)
	go relayChanToFile(stdoutChan, stdout, prefix, &relayerWait)
	go relayChanToFile(stderrChan, stderr, prefix, &relayerWait)

	return evalerPorts{
		[3]*Port{
			&Port{File: stdin, Chan: ClosedChan},
			&Port{File: stdout, Chan: stdoutChan, CloseChan: true},
			&Port{File: stderr, Chan: stderrChan, CloseChan: true},
		},
		&relayerWait,
	}
}

func relayChanToFile(ch <-chan Value, file *os.File, prefix *string, w *sync.WaitGroup) {
	for v := range ch {
		file.WriteString(*prefix)
		file.WriteString(v.Repr(initIndent))
		file.WriteString("\n")
	}
	w.Done()
}

func (ep *evalerPorts) close() {
	ep.ports[1].Close()
	ep.ports[2].Close()
	ep.relayeWait.Wait()
}
<commit_msg>gofmt -s<commit_after>package eval

import (
	"os"
	"sync"
)

const (
	stdoutChanSize = 32
	stderrChanSize = 32
)

type evalerPorts struct {
	ports      [3]*Port
	relayeWait *sync.WaitGroup
}

func newEvalerPorts(stdin, stdout, stderr *os.File, prefix *string) evalerPorts {
	stdoutChan := make(chan Value, stdoutChanSize)
	stderrChan := make(chan Value, stderrChanSize)

	var relayerWait sync.WaitGroup
	relayerWait.Add(2)
	go relayChanToFile(stdoutChan, stdout, prefix, &relayerWait)
	go relayChanToFile(stderrChan, stderr, prefix, &relayerWait)

	return evalerPorts{
		[3]*Port{
			{File: stdin, Chan: ClosedChan},
			{File: stdout, Chan: stdoutChan, CloseChan: true},
			{File: stderr, Chan: stderrChan, CloseChan: true},
		},
		&relayerWait,
	}
}

func relayChanToFile(ch <-chan Value, file *os.File, prefix *string, w *sync.WaitGroup) {
	for v := range ch {
		file.WriteString(*prefix)
		file.WriteString(v.Repr(initIndent))
		file.WriteString("\n")
	}
	w.Done()
}

func (ep *evalerPorts) close() {
	ep.ports[1].Close()
	ep.ports[2].Close()
	ep.relayeWait.Wait()
}
<|endoftext|>


<commit_before>package evaluation

import "fmt"

func Err(ctx *Context, msg, tag string, fmts ...interface{}) Object {
	msg = fmt.Sprintf(msg, fmts...)

	e := &Instance{
		Base: ctx.Get("Error"),
	}

	if e.Base == nil {
		panic("Since the prelude isn't loaded, errors cannot be thrown!")
	}

	e.Data = map[string]Object{
		"tag": &String{Value: tag},
		"msg": &String{Value: msg},
	}

	return e
}

func IsErr(o Object) bool {
	if instance, ok := o.(*Instance); ok {
		return instance.Base.(*Class).Name == "Error"
	}

	return false
}
<commit_msg>Add comments to error functions<commit_after>package evaluation

import "fmt"

// Err returns an error object with the message and tag provided
func Err(ctx *Context, msg, tag string, fmts ...interface{}) Object {
	msg = fmt.Sprintf(msg, fmts...)

	e := &Instance{
		Base: ctx.Get("Error"),
	}

	if e.Base == nil {
		panic("Since the prelude isn't loaded, errors cannot be thrown!")
	}

	e.Data = map[string]Object{
		"tag": &String{Value: tag},
		"msg": &String{Value: msg},
	}

	return e
}

// IsErr checks if an object is an instance of Error
func IsErr(o Object) bool {
	if instance, ok := o.(*Instance); ok {
		return instance.Base.(*Class).Name == "Error"
	}

	return false
}
<|endoftext|>


<commit_before>package aeutil

import (
	"net/http"
	"appengine"
)

var (
	// To be set by an external environment, if it wants control over context creation
	ContextProvider func(*http.Request) appengine.Context
)

func init() {
	ContextProvider = func(r *http.Request) appengine.Context {
		return appengine.NewContext(r)
	}
}

func UncurryWithContext(handler func(http.ResponseWriter, *http.Request, appengine.Context)) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		handler(w, r, ContextProvider(r))
	}
}
<commit_msg>more GAE work<commit_after>package aeutil

import (
	"net/http"
	"appengine"
	"appengine/aetest"
)

var (
	// To be set by an external environment, if it wants control over context creation
	ContextProvider func(*http.Request) appengine.Context
)

func init() {
	ContextProvider = func(r *http.Request) appengine.Context {
		return appengine.NewContext(r)
	}
}

func UncurryWithContext(handler func(http.ResponseWriter, *http.Request, appengine.Context)) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		handler(w, r, ContextProvider(r))
	}
}

/**
 * The Context returned from this function is registered with the current ContextProvider,
 * so calls to UncurryWithContext will resolve to this context.
 */
func CreateAndRegisterTestContext() (context aetest.Context, err error) {
	context, err = aetest.NewContext(nil)

	// TODO change this function to maintain and consult a map, so we can
	// make multiple requests in parallel / in a recursive fashion
	ContextProvider = func(*http.Request) appengine.Context { return context }

	return
}
<|endoftext|>


<commit_before>package faker

import "github.com/raitucarp/faker/locales"

type Gender int

const (
	Male Gender = iota
	Female
	RandomGender
)

var gender Gender

type Faker struct {
	Name     Name
	Address  Address
	Commerce Commerce
	Company  Company
	Date     Date
	Finance  Finance
	Hacker   Hacker
	Image    Image
	Internet Internet
	Lorem    Lorem

	// temp data handling
	data interface{}
}

type Definitions map[string]interface{}

var data interface{}

func (f *Faker) generate() {
	err := SetLocale(locales.EN)
	if err != nil {
		panic(err)
	}
}

func New() (f Faker) {
	f.generate()

	return
}

/*
func SetGender(g Gender) {
	gender = g
}*/
<commit_msg>add phone field<commit_after>package faker

import "github.com/raitucarp/faker/locales"

type Gender int

const (
	Male Gender = iota
	Female
	RandomGender
)

var gender Gender

type Faker struct {
	Name     Name
	Address  Address
	Commerce Commerce
	Company  Company
	Date     Date
	Finance  Finance
	Hacker   Hacker
	Image    Image
	Internet Internet
	Lorem    Lorem
	Phone    Phone
	// temp data handling
	data interface{}
}

type Definitions map[string]interface{}

var data interface{}

func (f *Faker) generate() {
	err := SetLocale(locales.EN)
	if err != nil {
		panic(err)
	}
}

func New() (f Faker) {
	f.generate()

	return
}

/*
func SetGender(g Gender) {
	gender = g
}*/
<|endoftext|>


<commit_before>package goinsta

import (
	"encoding/json"
	"fmt"
)

// Feed is the object for all feed endpoints.
type Feed struct {
	inst *Instagram
}

// newFeed creates new Feed structure
func newFeed(inst *Instagram) *Feed {
	return &Feed{
		inst: inst,
	}
}

// Feed search by locationID
func (feed *Feed) LocationID(locationID int64) (*FeedLocation, error) {
	insta := feed.inst
	body, err := insta.sendRequest(
		&reqOptions{
			Endpoint: fmt.Sprintf(urlFeedLocationID, locationID),
			Query: map[string]string{
				"rank_token":     insta.rankToken,
				"ranked_content": "true",
			},
		},
	)
	if err != nil {
		return nil, err
	}

	res := &FeedLocation{}
	err = json.Unmarshal(body, res)
	return res, err
}

// FeedLocation is the struct that fits the structure returned by instagram on LocationID search.
type FeedLocation struct {
	RankedItems         []Item   `json:"ranked_items"`
	Items               []Item   `json:"items"`
	NumResults          int      `json:"num_results"`
	NextID              string   `json:"next_max_id"`
	MoreAvailable       bool     `json:"more_available"`
	AutoLoadMoreEnabled bool     `json:"auto_load_more_enabled"`
	MediaCount          int      `json:"media_count"`
	Location            Location `json:"location"`
	Status              string   `json:"status"`
}

// Tags search by Tag in user Feed
//
// (sorry for returning FeedTag. See #FeedTag)
func (feed *Feed) Tags(tag string) (*FeedTag, error) {
	insta := feed.inst
	body, err := insta.sendRequest(
		&reqOptions{
			Endpoint: fmt.Sprintf(urlFeedTag, tag),
			Query: map[string]string{
				"rank_token":     insta.rankToken,
				"ranked_content": "true",
			},
		},
	)
	if err != nil {
		return nil, err
	}
	res := &FeedTag{}
	err = json.Unmarshal(body, res)
	if err != nil {
		return nil, err
	}
	for i := range res.RankedItems {
		res.RankedItems[i].media = &FeedMedia{
			inst:   insta,
			NextID: res.RankedItems[i].ID,
		}
	}
	return res, nil
}

// FeedTag is the struct that fits the structure returned by instagram on TagSearch.
type FeedTag struct {
	RankedItems         []Item     `json:"ranked_items"`
	Images              []Item     `json:"items"`
	NumResults          int        `json:"num_results"`
	NextID              string     `json:"next_max_id"`
	MoreAvailable       bool       `json:"more_available"`
	AutoLoadMoreEnabled bool       `json:"auto_load_more_enabled"`
	Story               StoryMedia `json:"story"`
	Status              string     `json:"status"`
}
<commit_msg>Assign missing value to Image's media attribute (#252)<commit_after>package goinsta

import (
	"encoding/json"
	"fmt"
)

// Feed is the object for all feed endpoints.
type Feed struct {
	inst *Instagram
}

// newFeed creates new Feed structure
func newFeed(inst *Instagram) *Feed {
	return &Feed{
		inst: inst,
	}
}

// Feed search by locationID
func (feed *Feed) LocationID(locationID int64) (*FeedLocation, error) {
	insta := feed.inst
	body, err := insta.sendRequest(
		&reqOptions{
			Endpoint: fmt.Sprintf(urlFeedLocationID, locationID),
			Query: map[string]string{
				"rank_token":     insta.rankToken,
				"ranked_content": "true",
			},
		},
	)
	if err != nil {
		return nil, err
	}

	res := &FeedLocation{}
	err = json.Unmarshal(body, res)
	return res, err
}

// FeedLocation is the struct that fits the structure returned by instagram on LocationID search.
type FeedLocation struct {
	RankedItems         []Item   `json:"ranked_items"`
	Items               []Item   `json:"items"`
	NumResults          int      `json:"num_results"`
	NextID              string   `json:"next_max_id"`
	MoreAvailable       bool     `json:"more_available"`
	AutoLoadMoreEnabled bool     `json:"auto_load_more_enabled"`
	MediaCount          int      `json:"media_count"`
	Location            Location `json:"location"`
	Status              string   `json:"status"`
}

// Tags search by Tag in user Feed
//
// (sorry for returning FeedTag. See #FeedTag)
func (feed *Feed) Tags(tag string) (*FeedTag, error) {
	insta := feed.inst
	body, err := insta.sendRequest(
		&reqOptions{
			Endpoint: fmt.Sprintf(urlFeedTag, tag),
			Query: map[string]string{
				"rank_token":     insta.rankToken,
				"ranked_content": "true",
			},
		},
	)
	if err != nil {
		return nil, err
	}
	res := &FeedTag{}
	err = json.Unmarshal(body, res)
	if err != nil {
		return nil, err
	}
	for i := range res.RankedItems {
		res.RankedItems[i].media = &FeedMedia{
			inst:   insta,
			NextID: res.RankedItems[i].ID,
		}
	}

	for i := range res.Images {
		res.Images[i].media = &FeedMedia{
			inst:   insta,
			NextID: res.Images[i].ID,
		}
	}

	return res, nil
}

// FeedTag is the struct that fits the structure returned by instagram on TagSearch.
type FeedTag struct {
	RankedItems         []Item     `json:"ranked_items"`
	Images              []Item     `json:"items"`
	NumResults          int        `json:"num_results"`
	NextID              string     `json:"next_max_id"`
	MoreAvailable       bool       `json:"more_available"`
	AutoLoadMoreEnabled bool       `json:"auto_load_more_enabled"`
	Story               StoryMedia `json:"story"`
	Status              string     `json:"status"`
}
<|endoftext|>


<commit_before>// Feeds.go contain HTTP routes for rendering RSS and Atom feeds.
package main

import (
	"log"
	"net/http"
	"strings"
	"time"

	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/feeds"
	"github.com/jinzhu/gorm"
	_ "github.com/lib/pq"
	"github.com/martini-contrib/render"
	_ "github.com/mattn/go-sqlite3"
)

// ReadFeed renders RSS or Atom feed of latest published posts.
// It determines the feed type with strings.Split(r.URL.Path[1:], "/")[1].
func ReadFeed(w http.ResponseWriter, res render.Render, db *gorm.DB, r *http.Request) {

	w.Header().Set("Content-Type", "application/xml")

	urlhost := urlHost()

	feed := &feeds.Feed{
		Title:       Settings.Name,
		Link:        &feeds.Link{Href: urlhost},
		Description: Settings.Description,
	}

	var post Post
	posts, err := post.GetAll(db)
	if err != nil {
		log.Println(err)
		res.JSON(500, map[string]interface{}{"error": "Internal server error"})
		return
	}

	for _, post := range posts {

		var user User
		user.ID = post.Author
		user, err := user.Get(db)
		if err != nil {
			log.Println(err)
			res.JSON(500, map[string]interface{}{"error": "Internal server error"})
			return
		}

		// The email in &feeds.Author is not actually exported, as it is left out by user.Get().
		// However, the package panics if too few values are exported, so that will do.

		item := &feeds.Item{
			Title:       post.Title,
			Link:        &feeds.Link{Href: urlhost + post.Slug},
			Description: post.Excerpt,
			Author:      &feeds.Author{user.Name, user.Email},
			Created:     time.Unix(post.Date, 0),
		}
		feed.Items = append(feed.Items, item)

	}

	// Default to RSS feed.
	result, err := feed.ToRss()
	if err != nil {
		log.Println(err)
		res.JSON(500, map[string]interface{}{"error": "Internal server error"})
		return
	}

	format := strings.Split(r.URL.Path[1:], "/")[1]
	if format == "atom" {
		result, err = feed.ToAtom()
		if err != nil {
			log.Println(err)
			res.JSON(500, map[string]interface{}{"error": "Internal server error"})
			return
		}
	}

	w.Write([]byte(result))
}
<commit_msg>updated feeds.go to prevent exposure of unpublished items<commit_after>// Feeds.go contain HTTP routes for rendering RSS and Atom feeds.
package main

import (
	"log"
	"net/http"
	"strings"
	"time"

	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/feeds"
	"github.com/jinzhu/gorm"
	_ "github.com/lib/pq"
	"github.com/martini-contrib/render"
	_ "github.com/mattn/go-sqlite3"
)

// ReadFeed renders RSS or Atom feed of latest published posts.
// It determines the feed type with strings.Split(r.URL.Path[1:], "/")[1].
func ReadFeed(w http.ResponseWriter, res render.Render, db *gorm.DB, r *http.Request) {

	w.Header().Set("Content-Type", "application/xml")

	urlhost := urlHost()

	feed := &feeds.Feed{
		Title:       Settings.Name,
		Link:        &feeds.Link{Href: urlhost},
		Description: Settings.Description,
	}

	var post Post
	posts, err := post.GetAll(db)
	if err != nil {
		log.Println(err)
		res.JSON(500, map[string]interface{}{"error": "Internal server error"})
		return
	}

	for _, post := range posts {

		var user User
		user.ID = post.Author
		user, err := user.Get(db)
		if err != nil {
			log.Println(err)
			res.JSON(500, map[string]interface{}{"error": "Internal server error"})
			return
		}

		// Don't expose unpublished items to the feeds
		if !post.Published {
			continue
		}

		// The email in &feeds.Author is not actually exported, as it is left out by user.Get().
		// However, the package panics if too few values are exported, so that will do.
		item := &feeds.Item{
			Title:       post.Title,
			Link:        &feeds.Link{Href: urlhost + post.Slug},
			Description: post.Excerpt,
			Author:      &feeds.Author{user.Name, user.Email},
			Created:     time.Unix(post.Date, 0),
		}
		feed.Items = append(feed.Items, item)

	}

	// Default to RSS feed.
	result, err := feed.ToRss()
	if err != nil {
		log.Println(err)
		res.JSON(500, map[string]interface{}{"error": "Internal server error"})
		return
	}

	format := strings.Split(r.URL.Path[1:], "/")[1]
	if format == "atom" {
		result, err = feed.ToAtom()
		if err != nil {
			log.Println(err)
			res.JSON(500, map[string]interface{}{"error": "Internal server error"})
			return
		}
	}

	w.Write([]byte(result))
}
<|endoftext|>


<commit_before>package api

import (
	"io"
	"log"
	"mime"
	"net/http"
	"path"
	"strconv"
	"strings"
	"sync/atomic"
	"time"

	"github.com/mdlayher/wavepipe/data"
)

// httpStream provides a common method to transfer a file stream using a HTTP response writer
func httpStream(song *data.Song, mimeType string, fileSize int64, stream io.ReadCloser, httpRes http.ResponseWriter) error {
	// Total bytes transferred
	var total int64

	// Track the stream's progress via log
	stopProgressChan := make(chan struct{})
	go func() {
		// Track start time
		startTime := time.Now()

		// Print progress every 5 seconds
		progress := time.NewTicker(5 * time.Second)

		// Calculate total file size
		totalSize := float64(fileSize) / 1024 / 1024
		for {
			select {
			// Print progress
			case <-progress.C:
				// Capture current progress
				currTotal := atomic.LoadInt64(&total)
				current := float64(currTotal) / 1024 / 1024

				// Capture current transfer rate
				rate := float64(float64((currTotal*8)/1024/1024) / float64(time.Now().Sub(startTime).Seconds()))

				// If size available, we can print percentage and file sizes
				if fileSize > 0 {
					// Capture current percentage
					percent := int64(float64(float64(currTotal)/float64(fileSize)) * 100)

					log.Printf("[%d] [%03d%%] %02.3f / %02.3f MB [%02.3f Mbps]", song.ID, percent, current, totalSize, rate)
					break
				}

				// Else, print the current transfer size and rate
				log.Printf("[%d] sent: %02.3f MB [%02.3f Mbps]", song.ID, current, rate)
			// Stop printing
			case <-stopProgressChan:
				return
			}
		}
	}()

	// Stop progress on return
	defer func() {
		close(stopProgressChan)
	}()

	// Buffer to store and transfer file bytes
	buf := make([]byte, 8192)

	// Indicate when stream is complete
	streamComplete := false

	// Set necessary output HTTP headers

	// Set Content-Length if set
	// NOTE: HTTP standards specify that this must be an exact length, so we cannot estimate it for
	// transcodes unless the entire file is transcoded and then sent
	if fileSize > 0 {
		httpRes.Header().Set("Content-Length", strconv.FormatInt(fileSize, 10))
	}

	// Override Content-Type if set
	contentType := mime.TypeByExtension(path.Ext(song.FileName))
	if mimeType != "" {
		contentType = mimeType
	}
	httpRes.Header().Set("Content-Type", contentType)

	// Get song modify time in RFC1123 format, replace UTC with GMT
	lastMod := strings.Replace(time.Unix(song.LastModified, 0).UTC().Format(time.RFC1123), "UTC", "GMT", 1)

	// Set Last-Modified using filesystem modify time
	httpRes.Header().Set("Last-Modified", lastMod)

	// Begin transferring the data stream
	for {
		// Read in a buffer from the file
		n, err := stream.Read(buf)
		if err != nil && err != io.EOF {
			return err
		} else if err == io.EOF {
			// Halt streaming after next write
			streamComplete = true
		}

		// Count bytes
		atomic.AddInt64(&total, int64(n))

		// Write bytes over HTTP
		if _, err := httpRes.Write(buf[:n]); err != nil && err != io.EOF {
			return err
		}

		// If stream is complete, break loop
		if streamComplete {
			break
		}
	}

	// No errors
	return nil
}
<commit_msg>Close connection on stream<commit_after>package api

import (
	"io"
	"log"
	"mime"
	"net/http"
	"path"
	"strconv"
	"strings"
	"sync/atomic"
	"time"

	"github.com/mdlayher/wavepipe/data"
)

// httpStream provides a common method to transfer a file stream using a HTTP response writer
func httpStream(song *data.Song, mimeType string, fileSize int64, stream io.ReadCloser, httpRes http.ResponseWriter) error {
	// Total bytes transferred
	var total int64

	// Track the stream's progress via log
	stopProgressChan := make(chan struct{})
	go func() {
		// Track start time
		startTime := time.Now()

		// Print progress every 5 seconds
		progress := time.NewTicker(5 * time.Second)

		// Calculate total file size
		totalSize := float64(fileSize) / 1024 / 1024
		for {
			select {
			// Print progress
			case <-progress.C:
				// Capture current progress
				currTotal := atomic.LoadInt64(&total)
				current := float64(currTotal) / 1024 / 1024

				// Capture current transfer rate
				rate := float64(float64((currTotal*8)/1024/1024) / float64(time.Now().Sub(startTime).Seconds()))

				// If size available, we can print percentage and file sizes
				if fileSize > 0 {
					// Capture current percentage
					percent := int64(float64(float64(currTotal)/float64(fileSize)) * 100)

					log.Printf("[%d] [%03d%%] %02.3f / %02.3f MB [%02.3f Mbps]", song.ID, percent, current, totalSize, rate)
					break
				}

				// Else, print the current transfer size and rate
				log.Printf("[%d] sent: %02.3f MB [%02.3f Mbps]", song.ID, current, rate)
			// Stop printing
			case <-stopProgressChan:
				return
			}
		}
	}()

	// Stop progress on return
	defer func() {
		close(stopProgressChan)
	}()

	// Buffer to store and transfer file bytes
	buf := make([]byte, 8192)

	// Indicate when stream is complete
	streamComplete := false

	// Set necessary output HTTP headers

	// Set Content-Length if set
	// NOTE: HTTP standards specify that this must be an exact length, so we cannot estimate it for
	// transcodes unless the entire file is transcoded and then sent
	if fileSize > 0 {
		httpRes.Header().Set("Content-Length", strconv.FormatInt(fileSize, 10))
	}

	// Override Content-Type if set
	contentType := mime.TypeByExtension(path.Ext(song.FileName))
	if mimeType != "" {
		contentType = mimeType
	}
	httpRes.Header().Set("Content-Type", contentType)

	// Get song modify time in RFC1123 format, replace UTC with GMT
	lastMod := strings.Replace(time.Unix(song.LastModified, 0).UTC().Format(time.RFC1123), "UTC", "GMT", 1)

	// Set Last-Modified using filesystem modify time
	httpRes.Header().Set("Last-Modified", lastMod)

	// Specify connection close on send
	httpRes.Header().Set("Connection", "close")

	// Begin transferring the data stream
	for {
		// Read in a buffer from the file
		n, err := stream.Read(buf)
		if err != nil && err != io.EOF {
			return err
		} else if err == io.EOF {
			// Halt streaming after next write
			streamComplete = true
		}

		// Count bytes
		atomic.AddInt64(&total, int64(n))

		// Write bytes over HTTP
		if _, err := httpRes.Write(buf[:n]); err != nil && err != io.EOF {
			return err
		}

		// If stream is complete, break loop
		if streamComplete {
			break
		}
	}

	// No errors
	return nil
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"testing"
	"reflect"
)

func TestActivate(t *testing.T) {
	d := NewDenv("test")
	cases := []struct {
		in string
		want *Denv
	}{
		{"", nil},
		{"test", d},
		{"notexist", nil},
	}
	for _, c := range cases {
		fmt.Printf("Testing Activate(%s)\n", c.in)
		got, err := Activate(c.in)
		if c.want == nil && got != nil && err != nil {
			t.Errorf("Activate(%q) returned an error, but not nil Denv")
		} else if err != nil && c.want != nil {
			t.Errorf("Wanted something, but threw an error %s", err)
		} else if !reflect.DeepEqual(c.want, got) {
			t.Errorf("Activate(%q) wanted %s, got %s", c.in, c.want.ToString(), got.ToString())
		}
	}
}
<commit_msg>init local_test<commit_after>package api

import (
	"fmt"
	"testing"
	"reflect"
)

func TestActivate(t *testing.T) {
	d := NewDenv("test")
	cases := []struct {
		in string
		want *Denv
	}{
		{"", nil},
		{"test", d},
		{"notexist", nil},
	}
	for _, c := range cases {
		fmt.Printf("Testing Activate(%s)\n", c.in)
		got, err := Activate(c.in)
		if c.want == nil && got != nil && err != nil {
			t.Errorf("Activate(%q) returned an error, but not nil Denv")
		} else if err != nil && c.want != nil {
			t.Errorf("Wanted something, but threw an error %s", err)
		} else if !reflect.DeepEqual(c.want, got) {
			t.Errorf("Activate(%q) wanted %s, got %s", c.in, c.want.ToString(), got.ToString())
		}
	}
	denv, err := Activate("test")
	if err != nil {
		t.Errorf("Error creating Denv, %s", err)
	}
	if Info.Current != denv {
		t.Errorf("Activate(test) did not properly assign Info.Current, %s", Info.ToString())
	}
}

func TestDeactivate(t *testing.T) {
	NewDenv("test") // Forces creation if not there
	active, err := Activate("test")
	if err != nil {
		t.Errorf("Could not Activate(test), %s", err)
	}
	deactive := Deactivate()
	if active != deactive {
		t.Errorf("Deactivate() reaturned different denv, %p, %p", &active, &deactive)
	}
	if Info.Current != nil {
		t.Errorf("Deactivate() did not clear Info.Current, %s", Info.ToString())
	}
}

func TestList(t *testing.T) {
	// Maybe implement a set
	denvs := map[*Denv]bool {
		NewDenv("d1"): true,
		NewDenv("d2"): true,
	}
	ls := List()
	set := make(map[string]int)
	for d, _ := range ls {
		set[d.Path] += 1
	}
	for d, _ := range denvs {
		if count, found := set[d.Path]; !found {
			t.Errorf("Denvs not subset of List()\nDenvs %s\nList() %s", denvs, ls)
		} else if count < 1 {
			t.Errorf("Denvs not subset of List()\nDenvs %s\nList() %s", denvs, ls)
		} else {
			set[d.Path] = count - 1
		}
	}
}

func TestWhich(t *testing.T) {
	NewDenv("test")
	d, _ := Activate("test")
	if Which() != d {
		t.Errorf("Which() did not return active denv, %s, %s", d, Which())
	}
	d = Deactivate()
	if Which() != nil {
		t.Errorf("Which() did not return nil on deactivate, %s, %s", d, Which())
	}

}
<|endoftext|>