<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package local

import (
	"fmt"
	"net"
	"os"
	"os/user"
	"syscall"

	"github.com/juju/loggo"

	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/instance"
	"launchpad.net/juju-core/juju/osenv"
	"launchpad.net/juju-core/provider"
	"launchpad.net/juju-core/utils"
	"launchpad.net/juju-core/version"
)

var logger = loggo.GetLogger("juju.provider.local")

var _ environs.EnvironProvider = (*environProvider)(nil)

type environProvider struct{}

var providerInstance = &environProvider{}

func init() {
	environs.RegisterProvider(provider.Local, providerInstance)
}

var userCurrent = user.Current

// Open implements environs.EnvironProvider.Open.
func (environProvider) Open(cfg *config.Config) (environs.Environ, error) {
	logger.Infof("opening environment %q", cfg.Name())
	if _, ok := cfg.AgentVersion(); !ok {
		newCfg, err := cfg.Apply(map[string]interface{}{
			"agent-version": version.Current.Number.String(),
		})
		if err != nil {
			return nil, err
		}
		cfg = newCfg
	}
	// Set the "namespace" attribute. We do this here, and not in Prepare,
	// for backwards compatibility: older versions did not store the namespace
	// in config.
	if namespace, _ := cfg.UnknownAttrs()["namespace"].(string); namespace == "" {
		username := os.Getenv("USER")
		if username == "" {
			u, err := userCurrent()
			if err != nil {
				return nil, fmt.Errorf("failed to determine username for namespace: %v", err)
			}
			username = u.Username
		}
		var err error
		namespace = fmt.Sprintf("%s-%s", username, cfg.Name())
		cfg, err = cfg.Apply(map[string]interface{}{"namespace": namespace})
		if err != nil {
			return nil, fmt.Errorf("failed to create namespace: %v", err)
		}
	}
	// Do the initial validation on the config.
	localConfig, err := providerInstance.newConfig(cfg)
	if err != nil {
		return nil, err
	}
	if err := VerifyPrerequisites(localConfig.container()); err != nil {
		logger.Errorf("failed verification of local provider prerequisites: %v", err)
		return nil, err
	}
	environ := &localEnviron{name: cfg.Name()}
	if err := environ.SetConfig(cfg); err != nil {
		logger.Errorf("failure setting config: %v", err)
		return nil, err
	}
	return environ, nil
}

var detectAptProxies = utils.DetectAptProxies

// Prepare implements environs.EnvironProvider.Prepare.
func (p environProvider) Prepare(ctx environs.BootstrapContext, cfg *config.Config) (environs.Environ, error) {
	// The user must not set bootstrap-ip; this is determined by the provider,
	// and its presence used to determine whether the environment has yet been
	// bootstrapped.
	if _, ok := cfg.UnknownAttrs()["bootstrap-ip"]; ok {
		return nil, fmt.Errorf("bootstrap-ip must not be specified")
	}
	err := checkLocalPort(cfg.StatePort(), "state port")
	if err != nil {
		return nil, err
	}
	err = checkLocalPort(cfg.APIPort(), "API port")
	if err != nil {
		return nil, err
	}
	// If the user has specified no values for any of the three normal
	// proxies, then look in the environment and set them.
	attrs := map[string]interface{}{
		// We must not proxy SSH through the API server in a
		// local provider environment. Besides not being useful,
		// it may not work; there is no requirement for sshd to
		// be available on machine-0.
		"proxy-ssh": false,
	}
	setIfNotBlank := func(key, value string) {
		if value != "" {
			attrs[key] = value
		}
	}
	logger.Tracef("Look for proxies?")
	if cfg.HttpProxy() == "" &&
		cfg.HttpsProxy() == "" &&
		cfg.FtpProxy() == "" &&
		cfg.NoProxy() == "" {
		proxy := osenv.DetectProxies()
		logger.Tracef("Proxies detected %#v", proxy)
		setIfNotBlank("http-proxy", proxy.Http)
		setIfNotBlank("https-proxy", proxy.Https)
		setIfNotBlank("ftp-proxy", proxy.Ftp)
		setIfNotBlank("no-proxy", proxy.NoProxy)
	}
	if cfg.AptHttpProxy() == "" &&
		cfg.AptHttpsProxy() == "" &&
		cfg.AptFtpProxy() == "" {
		proxy, err := detectAptProxies()
		if err != nil {
			return nil, err
		}
		setIfNotBlank("apt-http-proxy", proxy.Http)
		setIfNotBlank("apt-https-proxy", proxy.Https)
		setIfNotBlank("apt-ftp-proxy", proxy.Ftp)
	}
	if len(attrs) > 0 {
		cfg, err = cfg.Apply(attrs)
		if err != nil {
			return nil, err
		}
	}

	return p.Open(cfg)
}

// checkLocalPort checks that the passed port is not used so far.
var checkLocalPort = func(port int, description string) error {
	logger.Infof("checking %s", description)
	// Try to connect the port on localhost.
	address := fmt.Sprintf("localhost:%d", port)
	// TODO(mue) Add a timeout?
	conn, err := net.Dial("tcp", address)
	if err != nil {
		if nerr, ok := err.(*net.OpError); ok {
			if nerr.Err == syscall.ECONNREFUSED {
				// No connection, so everything is fine.
				return nil
			}
		}
		return err
	}
	// Connected, so port is in use.
	err = conn.Close()
	if err != nil {
		return err
	}
	return fmt.Errorf("cannot use %d as %s, already in use", port, description)
}

// Validate implements environs.EnvironProvider.Validate.
func (provider environProvider) Validate(cfg, old *config.Config) (valid *config.Config, err error) {
	// Check for valid changes for the base config values.
	if err := config.Validate(cfg, old); err != nil {
		return nil, err
	}
	validated, err := cfg.ValidateUnknownAttrs(configFields, configDefaults)
	if err != nil {
		logger.Errorf("failed to validate unknown attrs: %v", err)
		return nil, err
	}
	localConfig := newEnvironConfig(cfg, validated)
	// Before potentially creating directories, make sure that the
	// root directory has not changed.
	containerType := localConfig.container()
	if old != nil {
		oldLocalConfig, err := provider.newConfig(old)
		if err != nil {
			return nil, fmt.Errorf("old config is not a valid local config: %v", old)
		}
		if containerType != oldLocalConfig.container() {
			return nil, fmt.Errorf("cannot change container from %q to %q",
				oldLocalConfig.container(), containerType)
		}
		if localConfig.rootDir() != oldLocalConfig.rootDir() {
			return nil, fmt.Errorf("cannot change root-dir from %q to %q",
				oldLocalConfig.rootDir(),
				localConfig.rootDir())
		}
		if localConfig.networkBridge() != oldLocalConfig.networkBridge() {
			return nil, fmt.Errorf("cannot change network-bridge from %q to %q",
				oldLocalConfig.rootDir(),
				localConfig.rootDir())
		}
		if localConfig.storagePort() != oldLocalConfig.storagePort() {
			return nil, fmt.Errorf("cannot change storage-port from %v to %v",
				oldLocalConfig.storagePort(),
				localConfig.storagePort())
		}
	}
	// Currently only supported containers are "lxc" and "kvm".
	if containerType != instance.LXC && containerType != instance.KVM {
		return nil, fmt.Errorf("unsupported container type: %q", containerType)
	}
	dir, err := utils.NormalizePath(localConfig.rootDir())
	if err != nil {
		return nil, err
	}
	if dir == "." {
		dir = osenv.JujuHomePath(cfg.Name())
	}
	// Always assign the normalized path.
	localConfig.attrs["root-dir"] = dir

	if containerType != instance.KVM {
		fastOptionAvailable := useFastLXC(containerType)
		if _, found := localConfig.attrs["lxc-clone"]; !found {
			localConfig.attrs["lxc-clone"] = fastOptionAvailable
		}
	}

	// Apply the coerced unknown values back into the config.
	return cfg.Apply(localConfig.attrs)
}

// BoilerplateConfig implements environs.EnvironProvider.BoilerplateConfig.
func (environProvider) BoilerplateConfig() string {
	return `
# https://juju.ubuntu.com/docs/config-local.html
local:
    type: local

    # root-dir holds the directory that is used for the storage files and
    # database. The default location is $JUJU_HOME/<env-name>.
    # $JUJU_HOME defaults to ~/.juju. Override if needed.
    #
    # root-dir: ~/.juju/local

    # storage-port holds the port where the local provider starts the
    # HTTP file server. Override the value if you have multiple local
    # providers, or if the default port is used by another program.
    #
    # storage-port: 8040

    # network-bridge holds the name of the LXC network bridge to use.
    # Override if the default LXC network bridge is different.
    #
    #
    # network-bridge: lxcbr0

    # The default series to deploy the state-server and charms on.
    #
    # default-series: precise

`[1:]
}

// SecretAttrs implements environs.EnvironProvider.SecretAttrs.
func (environProvider) SecretAttrs(cfg *config.Config) (map[string]string, error) {
	// don't have any secret attrs
	return nil, nil
}

func (p environProvider) newConfig(cfg *config.Config) (*environConfig, error) {
	valid, err := p.Validate(cfg, nil)
	if err != nil {
		return nil, err
	}
	return newEnvironConfig(valid, valid.UnknownAttrs()), nil
}
<commit_msg>[r=wallyworld],[bug=1309805] Added comment to boilerplate environments.yaml stating that the default-series option needs to be uncommented and set to the desired series.<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package local

import (
	"fmt"
	"net"
	"os"
	"os/user"
	"syscall"

	"github.com/juju/loggo"

	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/instance"
	"launchpad.net/juju-core/juju/osenv"
	"launchpad.net/juju-core/provider"
	"launchpad.net/juju-core/utils"
	"launchpad.net/juju-core/version"
)

var logger = loggo.GetLogger("juju.provider.local")

var _ environs.EnvironProvider = (*environProvider)(nil)

type environProvider struct{}

var providerInstance = &environProvider{}

func init() {
	environs.RegisterProvider(provider.Local, providerInstance)
}

var userCurrent = user.Current

// Open implements environs.EnvironProvider.Open.
func (environProvider) Open(cfg *config.Config) (environs.Environ, error) {
	logger.Infof("opening environment %q", cfg.Name())
	if _, ok := cfg.AgentVersion(); !ok {
		newCfg, err := cfg.Apply(map[string]interface{}{
			"agent-version": version.Current.Number.String(),
		})
		if err != nil {
			return nil, err
		}
		cfg = newCfg
	}
	// Set the "namespace" attribute. We do this here, and not in Prepare,
	// for backwards compatibility: older versions did not store the namespace
	// in config.
	if namespace, _ := cfg.UnknownAttrs()["namespace"].(string); namespace == "" {
		username := os.Getenv("USER")
		if username == "" {
			u, err := userCurrent()
			if err != nil {
				return nil, fmt.Errorf("failed to determine username for namespace: %v", err)
			}
			username = u.Username
		}
		var err error
		namespace = fmt.Sprintf("%s-%s", username, cfg.Name())
		cfg, err = cfg.Apply(map[string]interface{}{"namespace": namespace})
		if err != nil {
			return nil, fmt.Errorf("failed to create namespace: %v", err)
		}
	}
	// Do the initial validation on the config.
	localConfig, err := providerInstance.newConfig(cfg)
	if err != nil {
		return nil, err
	}
	if err := VerifyPrerequisites(localConfig.container()); err != nil {
		logger.Errorf("failed verification of local provider prerequisites: %v", err)
		return nil, err
	}
	environ := &localEnviron{name: cfg.Name()}
	if err := environ.SetConfig(cfg); err != nil {
		logger.Errorf("failure setting config: %v", err)
		return nil, err
	}
	return environ, nil
}

var detectAptProxies = utils.DetectAptProxies

// Prepare implements environs.EnvironProvider.Prepare.
func (p environProvider) Prepare(ctx environs.BootstrapContext, cfg *config.Config) (environs.Environ, error) {
	// The user must not set bootstrap-ip; this is determined by the provider,
	// and its presence used to determine whether the environment has yet been
	// bootstrapped.
	if _, ok := cfg.UnknownAttrs()["bootstrap-ip"]; ok {
		return nil, fmt.Errorf("bootstrap-ip must not be specified")
	}
	err := checkLocalPort(cfg.StatePort(), "state port")
	if err != nil {
		return nil, err
	}
	err = checkLocalPort(cfg.APIPort(), "API port")
	if err != nil {
		return nil, err
	}
	// If the user has specified no values for any of the three normal
	// proxies, then look in the environment and set them.
	attrs := map[string]interface{}{
		// We must not proxy SSH through the API server in a
		// local provider environment. Besides not being useful,
		// it may not work; there is no requirement for sshd to
		// be available on machine-0.
		"proxy-ssh": false,
	}
	setIfNotBlank := func(key, value string) {
		if value != "" {
			attrs[key] = value
		}
	}
	logger.Tracef("Look for proxies?")
	if cfg.HttpProxy() == "" &&
		cfg.HttpsProxy() == "" &&
		cfg.FtpProxy() == "" &&
		cfg.NoProxy() == "" {
		proxy := osenv.DetectProxies()
		logger.Tracef("Proxies detected %#v", proxy)
		setIfNotBlank("http-proxy", proxy.Http)
		setIfNotBlank("https-proxy", proxy.Https)
		setIfNotBlank("ftp-proxy", proxy.Ftp)
		setIfNotBlank("no-proxy", proxy.NoProxy)
	}
	if cfg.AptHttpProxy() == "" &&
		cfg.AptHttpsProxy() == "" &&
		cfg.AptFtpProxy() == "" {
		proxy, err := detectAptProxies()
		if err != nil {
			return nil, err
		}
		setIfNotBlank("apt-http-proxy", proxy.Http)
		setIfNotBlank("apt-https-proxy", proxy.Https)
		setIfNotBlank("apt-ftp-proxy", proxy.Ftp)
	}
	if len(attrs) > 0 {
		cfg, err = cfg.Apply(attrs)
		if err != nil {
			return nil, err
		}
	}

	return p.Open(cfg)
}

// checkLocalPort checks that the passed port is not used so far.
var checkLocalPort = func(port int, description string) error {
	logger.Infof("checking %s", description)
	// Try to connect the port on localhost.
	address := fmt.Sprintf("localhost:%d", port)
	// TODO(mue) Add a timeout?
	conn, err := net.Dial("tcp", address)
	if err != nil {
		if nerr, ok := err.(*net.OpError); ok {
			if nerr.Err == syscall.ECONNREFUSED {
				// No connection, so everything is fine.
				return nil
			}
		}
		return err
	}
	// Connected, so port is in use.
	err = conn.Close()
	if err != nil {
		return err
	}
	return fmt.Errorf("cannot use %d as %s, already in use", port, description)
}

// Validate implements environs.EnvironProvider.Validate.
func (provider environProvider) Validate(cfg, old *config.Config) (valid *config.Config, err error) {
	// Check for valid changes for the base config values.
	if err := config.Validate(cfg, old); err != nil {
		return nil, err
	}
	validated, err := cfg.ValidateUnknownAttrs(configFields, configDefaults)
	if err != nil {
		logger.Errorf("failed to validate unknown attrs: %v", err)
		return nil, err
	}
	localConfig := newEnvironConfig(cfg, validated)
	// Before potentially creating directories, make sure that the
	// root directory has not changed.
	containerType := localConfig.container()
	if old != nil {
		oldLocalConfig, err := provider.newConfig(old)
		if err != nil {
			return nil, fmt.Errorf("old config is not a valid local config: %v", old)
		}
		if containerType != oldLocalConfig.container() {
			return nil, fmt.Errorf("cannot change container from %q to %q",
				oldLocalConfig.container(), containerType)
		}
		if localConfig.rootDir() != oldLocalConfig.rootDir() {
			return nil, fmt.Errorf("cannot change root-dir from %q to %q",
				oldLocalConfig.rootDir(),
				localConfig.rootDir())
		}
		if localConfig.networkBridge() != oldLocalConfig.networkBridge() {
			return nil, fmt.Errorf("cannot change network-bridge from %q to %q",
				oldLocalConfig.rootDir(),
				localConfig.rootDir())
		}
		if localConfig.storagePort() != oldLocalConfig.storagePort() {
			return nil, fmt.Errorf("cannot change storage-port from %v to %v",
				oldLocalConfig.storagePort(),
				localConfig.storagePort())
		}
	}
	// Currently only supported containers are "lxc" and "kvm".
	if containerType != instance.LXC && containerType != instance.KVM {
		return nil, fmt.Errorf("unsupported container type: %q", containerType)
	}
	dir, err := utils.NormalizePath(localConfig.rootDir())
	if err != nil {
		return nil, err
	}
	if dir == "." {
		dir = osenv.JujuHomePath(cfg.Name())
	}
	// Always assign the normalized path.
	localConfig.attrs["root-dir"] = dir

	if containerType != instance.KVM {
		fastOptionAvailable := useFastLXC(containerType)
		if _, found := localConfig.attrs["lxc-clone"]; !found {
			localConfig.attrs["lxc-clone"] = fastOptionAvailable
		}
	}

	// Apply the coerced unknown values back into the config.
	return cfg.Apply(localConfig.attrs)
}

// BoilerplateConfig implements environs.EnvironProvider.BoilerplateConfig.
func (environProvider) BoilerplateConfig() string {
	return `
# https://juju.ubuntu.com/docs/config-local.html
local:
    type: local

    # root-dir holds the directory that is used for the storage files and
    # database. The default location is $JUJU_HOME/<env-name>.
    # $JUJU_HOME defaults to ~/.juju. Override if needed.
    #
    # root-dir: ~/.juju/local

    # storage-port holds the port where the local provider starts the
    # HTTP file server. Override the value if you have multiple local
    # providers, or if the default port is used by another program.
    #
    # storage-port: 8040

    # network-bridge holds the name of the LXC network bridge to use.
    # Override if the default LXC network bridge is different.
    #
    #
    # network-bridge: lxcbr0

    # The default series to deploy the state-server and charms on.
    # Make sure to uncomment the following option and set the value to
    # precise or trusty as desired.
    #
    # default-series: precise

`[1:]
}

// SecretAttrs implements environs.EnvironProvider.SecretAttrs.
func (environProvider) SecretAttrs(cfg *config.Config) (map[string]string, error) {
	// don't have any secret attrs
	return nil, nil
}

func (p environProvider) newConfig(cfg *config.Config) (*environConfig, error) {
	valid, err := p.Validate(cfg, nil)
	if err != nil {
		return nil, err
	}
	return newEnvironConfig(valid, valid.UnknownAttrs()), nil
}
<|endoftext|>


<commit_before>package pomf

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"

	"github.com/dchest/uniuri"
	_ "github.com/go-sql-driver/mysql"
)

const (
	LENGTH         = 6
	PORT           = ":8080"
	UPDIRECTORY    = "/tmp/"
	GRILLDIRECTORY = "pomf/build/img/"
	POMFDIRECTORY  = "pomf/build"
	UPADDRESS      = "http://localhost"
	dbUSERNAME     = ""
	dbNAME         = ""
	dbPASSWORD     = ""
	DATABASE       = dbUSERNAME + ":" + dbPASSWORD + "@/" + dbNAME + "?charset=utf8"
	MAXSIZE        = 10 * 1024 * 1024
)

type Result struct {
	URL  string `json:"url"`
	Name string `json:"name"`
	Hash string `json:"hash"`
	Size int64  `json:"size"`
}

type Response struct {
	Success     bool     `json:"success"`
	ErrorCode   int      `json:"errorcode,omitempty"`
	Description string   `json:"description,omitempty"`
	Files       []Result `json:"files,omitempty"`
}

func generateName() (string, error) {
	name := uniuri.NewLen(LENGTH)
	db, err := sql.Open("mysql", DATABASE)
	if err != nil {
		return "", err
	}
	var id string
	err = db.QueryRow("select id from files where id=?", name).Scan(&id)
	if err != sql.ErrNoRows {
		generateName()
	}
	db.Close()

	return name, nil
}
func respond(w http.ResponseWriter, output string, resp Response) {
	if resp.ErrorCode != 0 {
		resp.Files = []Result{}
		resp.Success = false
	} else {
		resp.Success = true
	}

	switch output {
	case "json":

		w.Header().Set("Content-Type", "application/json")
		err := json.NewEncoder(w).Encode(resp)

		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	case "xml":
		x, err := xml.MarshalIndent(resp, "", "  ")
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		w.Header().Set("Content-Type", "application/xml")
		w.Write(x)

	case "html":
		w.Header().Set("Content-Type", "text/html")
		for _, file := range resp.Files {
			io.WriteString(w, ""+file.URL+"
")
		}

	case "gyazo", "text":
		w.Header().Set("Content-Type", "plain/text")
		for _, file := range resp.Files {
			io.WriteString(w, file.URL+"\n")
		}

	case "csv":
		w.Header().Set("Content-Type", "text/csv")
		io.WriteString(w, "name, url, hash, size\n")
		for _, file := range resp.Files {
			io.WriteString(w, file.Name+","+file.URL+","+file.Hash+","+strconv.FormatInt(file.Size, 10)+"\n")
		}

	default:
		w.Header().Set("Content-Type", "application/json")
		err := json.NewEncoder(w).Encode(resp)

		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

	}

}

func grillHandler(w http.ResponseWriter, r *http.Request) {
	kawaii, err := ioutil.ReadDir(GRILLDIRECTORY)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
	http.Redirect(w, r, "/img/"+kawaii[rand.Intn(len(kawaii))].Name(), http.StatusFound)
}

func uploadHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	output := r.FormValue("output")
	reader, err := r.MultipartReader()
	resp := Response{Files: []Result{}}
	if err != nil {
		resp.ErrorCode = http.StatusInternalServerError
		resp.Description = err.Error()
		respond(w, output, resp)
		return
	}

	db, err := sql.Open("mysql", DATABASE)
	if err != nil {
		resp.ErrorCode = http.StatusInternalServerError
		resp.Description = err.Error()
		return
	}

	for {
		part, err := reader.NextPart()
		if err == io.EOF {
			break
		}

		if part.FileName() == "" {
			continue
		}

		s, err := generateName()
		if err != nil {
			resp.ErrorCode = http.StatusInternalServerError
			resp.Description = err.Error()
			break
		}
		extName := filepath.Ext(part.FileName())
		filename := s + extName
		dst, err := os.Create(UPDIRECTORY + filename)

		if err != nil {
			resp.ErrorCode = http.StatusInternalServerError
			resp.Description = err.Error()
			respond(w, output, resp)
			break
		}
		defer dst.Close()

		h := sha1.New()
		t := io.TeeReader(part, h)
		_, err = io.Copy(dst, t)

		if err != nil {
			resp.ErrorCode = http.StatusInternalServerError
			resp.Description = err.Error()
			respond(w, output, resp)
			break
		}
		hash := h.Sum(nil)
		sha1 := base64.URLEncoding.EncodeToString(hash)
		stat, _ := dst.Stat()
		size := stat.Size()
		if size > MAXSIZE {
			resp.ErrorCode = http.StatusRequestEntityTooLarge
			resp.Description = err.Error()
			break
		}

		originalname := part.FileName()
		err = db.QueryRow("select originalname, filename, size from files where hash=?", sha1).Scan(&originalname, &filename, &size)
		res := Result{
			URL:  UPADDRESS + "/" + filename,
			Name: originalname,
			Hash: sha1,
			Size: size,
		}
		if err == sql.ErrNoRows {
			query, err := db.Prepare("INSERT into files(hash, originalname, filename, size, date) values(?, ?, ?, ?, ?)")
			if err != nil {
				resp.ErrorCode = http.StatusInternalServerError
				resp.Description = err.Error()
				break
			}
			_, err = query.Exec(res.Hash, res.Name, filename, res.Size, time.Now().Format("2016-01-02"))
			if err != nil {
				resp.ErrorCode = http.StatusInternalServerError
				resp.Description = err.Error()
				break
			}
		}
		resp.Files = append(resp.Files, res)
	}
	respond(w, output, resp)
}

func main() {
	http.HandleFunc("/upload.php", uploadHandler)
	http.HandleFunc("/grill.php", grillHandler)
	http.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(POMFDIRECTORY))))
	err := http.ListenAndServe(PORT, nil)
	if err != nil {
		panic(err)
	}

}
<commit_msg>Start writing documentation<commit_after>// Package pomf provides a simple file hosting pomf compatible web application
package pomf

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"

	"github.com/dchest/uniuri"
	_ "github.com/go-sql-driver/mysql"
)

const (
	// Length is used to specify filename length
	LENGTH = 6
	// port is the port pomf will listen on
	PORT = ":8080"
	// UPDIRECTORY specifies what directory pomf will upload to
	UPDIRECTORY = "/tmp/"
	// Kawaii grill directory
	GRILLDIRECTORY = "pomf/build/img/"
	// Static pomf directory
	POMFDIRECTORY = "pomf/build"
	// Domain to serve static files from
	UPADDRESS = "http://localhost"
	// Database username
	dbUSERNAME = ""
	// database name
	dbNAME = ""
	// database password
	dbPASSWORD = ""
	// DATABASE connection constant
	DATABASE = dbUSERNAME + ":" + dbPASSWORD + "@/" + dbNAME + "?charset=utf8"
	// MAXSIZE in bytes
	MAXSIZE = 10 * 1024 * 1024
)

// Result information struct
type Result struct {
	URL  string `json:"url"`
	Name string `json:"name"`
	Hash string `json:"hash"`
	Size int64  `json:"size"`
}

// Pomf compatible Response struct
type Response struct {
	Success     bool     `json:"success"`
	ErrorCode   int      `json:"errorcode,omitempty"`
	Description string   `json:"description,omitempty"`
	Files       []Result `json:"files,omitempty"`
}

// generateName returns a random string that isn't in the database
func generateName() (string, error) {
	name := uniuri.NewLen(LENGTH)
	db, err := sql.Open("mysql", DATABASE)
	if err != nil {
		return "", err
	}
	var id string
	err = db.QueryRow("select id from files where id=?", name).Scan(&id)
	if err != sql.ErrNoRows {
		generateName()
	}
	db.Close()

	return name, nil
}

// respond
func respond(w http.ResponseWriter, output string, resp Response) {
	if resp.ErrorCode != 0 {
		resp.Files = []Result{}
		resp.Success = false
	} else {
		resp.Success = true
	}

	switch output {
	case "json":

		w.Header().Set("Content-Type", "application/json")
		err := json.NewEncoder(w).Encode(resp)

		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	case "xml":
		x, err := xml.MarshalIndent(resp, "", "  ")
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		w.Header().Set("Content-Type", "application/xml")
		w.Write(x)

	case "html":
		w.Header().Set("Content-Type", "text/html")
		for _, file := range resp.Files {
			io.WriteString(w, ""+file.URL+"
")
		}

	case "gyazo", "text":
		w.Header().Set("Content-Type", "plain/text")
		for _, file := range resp.Files {
			io.WriteString(w, file.URL+"\n")
		}

	case "csv":
		w.Header().Set("Content-Type", "text/csv")
		io.WriteString(w, "name, url, hash, size\n")
		for _, file := range resp.Files {
			io.WriteString(w, file.Name+","+file.URL+","+file.Hash+","+strconv.FormatInt(file.Size, 10)+"\n")
		}

	default:
		w.Header().Set("Content-Type", "application/json")
		err := json.NewEncoder(w).Encode(resp)

		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

	}

}

func grillHandler(w http.ResponseWriter, r *http.Request) {
	kawaii, err := ioutil.ReadDir(GRILLDIRECTORY)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
	http.Redirect(w, r, "/img/"+kawaii[rand.Intn(len(kawaii))].Name(), http.StatusFound)
}

func uploadHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	output := r.FormValue("output")
	reader, err := r.MultipartReader()
	resp := Response{Files: []Result{}}
	if err != nil {
		resp.ErrorCode = http.StatusInternalServerError
		resp.Description = err.Error()
		respond(w, output, resp)
		return
	}

	db, err := sql.Open("mysql", DATABASE)
	if err != nil {
		resp.ErrorCode = http.StatusInternalServerError
		resp.Description = err.Error()
		return
	}

	for {
		part, err := reader.NextPart()
		if err == io.EOF {
			break
		}

		if part.FileName() == "" {
			continue
		}

		s, err := generateName()
		if err != nil {
			resp.ErrorCode = http.StatusInternalServerError
			resp.Description = err.Error()
			break
		}
		extName := filepath.Ext(part.FileName())
		filename := s + extName
		dst, err := os.Create(UPDIRECTORY + filename)

		if err != nil {
			resp.ErrorCode = http.StatusInternalServerError
			resp.Description = err.Error()
			respond(w, output, resp)
			break
		}
		defer dst.Close()

		h := sha1.New()
		t := io.TeeReader(part, h)
		_, err = io.Copy(dst, t)

		if err != nil {
			resp.ErrorCode = http.StatusInternalServerError
			resp.Description = err.Error()
			respond(w, output, resp)
			break
		}
		hash := h.Sum(nil)
		sha1 := base64.URLEncoding.EncodeToString(hash)
		stat, _ := dst.Stat()
		size := stat.Size()
		if size > MAXSIZE {
			resp.ErrorCode = http.StatusRequestEntityTooLarge
			resp.Description = err.Error()
			break
		}

		originalname := part.FileName()
		err = db.QueryRow("select originalname, filename, size from files where hash=?", sha1).Scan(&originalname, &filename, &size)
		res := Result{
			URL:  UPADDRESS + "/" + filename,
			Name: originalname,
			Hash: sha1,
			Size: size,
		}
		if err == sql.ErrNoRows {
			query, err := db.Prepare("INSERT into files(hash, originalname, filename, size, date) values(?, ?, ?, ?, ?)")
			if err != nil {
				resp.ErrorCode = http.StatusInternalServerError
				resp.Description = err.Error()
				break
			}
			_, err = query.Exec(res.Hash, res.Name, filename, res.Size, time.Now().Format("2016-01-02"))
			if err != nil {
				resp.ErrorCode = http.StatusInternalServerError
				resp.Description = err.Error()
				break
			}
		}
		resp.Files = append(resp.Files, res)
	}
	respond(w, output, resp)
}

func main() {
	http.HandleFunc("/upload.php", uploadHandler)
	http.HandleFunc("/grill.php", grillHandler)
	http.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(POMFDIRECTORY))))
	err := http.ListenAndServe(PORT, nil)
	if err != nil {
		panic(err)
	}

}
<|endoftext|>


<commit_before>// A generic resource pool for databases etc
package pool

import (
	"errors"
	"io"
	"sync/atomic"
	"time"

	"github.com/bountylabs/api_common/errutil"
)

var (
	TimeoutError              = errors.New("Pool Timeout")
	PoolClosedError           = errors.New("Pool is closed")
	NewConnectionLimitedError = errors.New("Exceeded rate limit for creating new pool connections")
)

type Limiter interface {
	Allow() bool
}

// ResourceOpener opens a resource
type ResourceOpener interface {
	Open() (Resource, error)
}

type Resource interface {
	io.Closer
	// Good returns true when the resource is in good state
	Good() bool
}

// Ticket abstract type that represents the right to acquire a resource
type Ticket struct{}

type PooledResource interface {
	// Release releases the resource back to the pool for reuse
	Release() error
	// Destroy destroys the resource. It is no longer usable.
	Destroy() error
	// Resource returns the underlying Resource
	Resource() Resource
}

type PoolMetrics interface {
	ReportResources(stats ResourcePoolStat)
	ReportWait(wt time.Duration)       //how long did we wait for a connection
	ReportBorrowTime(wt time.Duration) //how long was the connection used for
	ReportNew(time time.Duration)      //how long did it take for a new connection to be opened
}

type ResourcePoolStat struct {
	AvailableNow  uint32
	ResourcesOpen uint32
	Cap           uint32
	InUse         uint32
}

// ResourcePool manages a pool of resources for reuse.
type ResourcePool struct {
	metrics PoolMetrics //metrics interface to track how the pool performs

	// To get a resource, get a ticket first and then get the resource from
	// either the reserve or open a new one.
	// To release a resource, first put the resource back to the reserve
	// and then put the ticket back.
	// The order is important to make sure we never create >cap(tickets)
	// number of resources.
	reserve chan Resource // idle resources, ready for use
	tickets chan *Ticket  // ticket to own resources

	opener               ResourceOpener
	closed               chan struct{}
	nOpenResources       uint32
	newConnectionLimiter Limiter
}

// NewPool creates a new pool of Clients.
func NewPool(maxReserve, maxOpen uint32, opener ResourceOpener, m PoolMetrics, newConnectionLimiter Limiter) *ResourcePool {
	if maxOpen < maxReserve {
		panic("maxOpen must be > maxReserve")
	}
	tickets := make(chan *Ticket, maxOpen)
	for i := uint32(0); i < maxOpen; i++ {
		tickets <- &Ticket{}
	}
	return &ResourcePool{
		metrics:              m,
		reserve:              make(chan Resource, maxReserve),
		tickets:              tickets,
		opener:               opener,
		closed:               make(chan struct{}),
		newConnectionLimiter: newConnectionLimiter,
	}
}

func (p *ResourcePool) Get() (PooledResource, error) {
	return p.GetWithTimeout(365 * 24 * time.Hour) // 1 year is forever
}

type pooledResource struct {
	ticket *Ticket
	served time.Time
	p      *ResourcePool
	res    Resource
}

func (pr *pooledResource) Release() error {
	defer pr.p.reportResources()
	pr.p.reportBorrowTime(time.Now().Sub(pr.served))
	return pr.p.release(pr)
}

func (pr *pooledResource) Destroy() error {
	defer pr.p.reportResources()
	pr.p.reportBorrowTime(time.Now().Sub(pr.served))
	return pr.p.destroy(pr)
}

func (pr *pooledResource) Resource() Resource {
	return pr.res
}

func (p *ResourcePool) releaseTicket(ticket *Ticket) {
	if ticket == nil {
		panic("BUG: can not release a nil ticket")
	}
	select {
	case p.tickets <- ticket:
	default:
		// releasing ticket should never block.
		panic("BUG: releaseTicket is called when ticket is not issued.")
	}
}

// WarmUp allocates resources until we have approximately allocated
// cap(p.reserve) resources.  We may under or over allocate resources (the code
// is subject to races) if someone is getting a resource while we are warming
// up.
func (p *ResourcePool) WarmUp() (count int, err error) {

	//loop until our open resources equals our reserve size
	for int(p.GetNOpenResources()) < cap(p.reserve) {

		var ticket *Ticket
		select {
		case ticket = <-p.tickets:
		case <-p.closed:
			return count, nil
		default: //all tickets are handed out
			return count, nil
		}

		if pooledResource, err := p.open(ticket, true); err != nil {
			return count, err
		} else {
			count++
			pooledResource.Release()
			continue
		}
	}

	return count, nil
}

func (p *ResourcePool) GetWithTimeout(timeout time.Duration) (pr PooledResource, err error) {

	// order is important: first ticket then reserve
	defer p.reportResources()
	defer func(start time.Time) {
		p.reportWait(time.Now().Sub(start))
	}(time.Now())

	timer := time.NewTimer(timeout)

	var ticket *Ticket
	select {
	case ticket = <-p.tickets:
	case <-timer.C:
		return nil, TimeoutError
	case <-p.closed:
		timer.Stop()
		return nil, PoolClosedError
	}
	timer.Stop()
	if p.isClosed() {
		// release ticket on close
		p.releaseTicket(ticket)
		return nil, PoolClosedError
	}

	if pooledResource := p.getFromReserve(ticket); pooledResource != nil {
		return pooledResource, nil
	}

	return p.open(ticket, false)

}

// getFromReserve attempts to get a pooledResource from the reserve pool, requires a ticket!
func (p *ResourcePool) getFromReserve(ticket *Ticket) *pooledResource {

	if ticket == nil {
		panic("BUG: can not acquire a reserved resource without a ticket")
	}

	for {
		select {
		case r := <-p.reserve:
			if r.Good() {
				return &pooledResource{ticket: ticket, served: time.Now(), p: p, res: r}
			} else {
				p.closeResource(r)
			}
			continue
		default:
			return nil
		}
	}
}

// open, opens a new resource, requires a ticket!
func (p *ResourcePool) open(ticket *Ticket, fromWarmup bool) (*pooledResource, error) {

	if ticket == nil {
		panic("BUG: can not open a resource without a ticket")
	}

	if !fromWarmup && p.newConnectionLimiter != nil {
		if !p.newConnectionLimiter.Allow() {
			return nil, NewConnectionLimitedError
		}
	}

	start := time.Now()
	r, err := p.opener.Open()
	p.reportNew(time.Now().Sub(start))

	if err != nil {
		// release ticket on error
		p.releaseTicket(ticket)
		return nil, err
	}
	p.incrOpen()
	return &pooledResource{ticket: ticket, served: time.Now(), p: p, res: r}, nil
}

func (p *ResourcePool) closeResource(res Resource) error {
	p.decrOpen()
	return res.Close()
}

func (p *ResourcePool) release(pr *pooledResource) error {
	var err error
	// order is important: first reserve then ticket
	res := pr.Resource()
	select {
	case p.reserve <- res:
	default:
		// reserve is full
		err = p.closeResource(res)
	}
	p.releaseTicket(pr.ticket)
	pr.ticket = nil

	if p.isClosed() {
		p.drainReserve()
	}
	return err
}

func (p *ResourcePool) destroy(pr *pooledResource) error {
	p.releaseTicket(pr.ticket)
	pr.ticket = nil
	return p.closeResource(pr.Resource())
}

// Close closes the pool. Resources in use are not affected.
func (p *ResourcePool) Close() error {
	close(p.closed)
	return p.drainReserve()
}

func (p *ResourcePool) isClosed() bool {
	select {
	case <-p.closed:
		return true
	default:
		return false
	}
}

func (p *ResourcePool) drainReserve() error {
	out := []error{}
	for {
		select {
		case r := <-p.reserve:
			if err := p.closeResource(r); err != nil {
				out = append(out, err)
			}
		default:
			return errutil.ErrSlice(out)
		}
	}
}

/**
Metrics
**/

func (p *ResourcePool) reportNew(time time.Duration) {
	if p.metrics != nil {
		p.metrics.ReportNew(time)
	}
}

func (p *ResourcePool) reportWait(wt time.Duration) {
	if p.metrics != nil {
		p.metrics.ReportWait(wt)
	}
}

func (p *ResourcePool) reportBorrowTime(wt time.Duration) {
	if p.metrics != nil {
		p.metrics.ReportBorrowTime(wt)
	}
}

func (p *ResourcePool) reportResources() {
	if p.metrics != nil {
		stats := p.Stats()
		p.metrics.ReportResources(stats)
	}
}

func (p *ResourcePool) incrOpen() {
	atomic.AddUint32(&p.nOpenResources, 1)
}

func (p *ResourcePool) decrOpen() {
	atomic.AddUint32(&p.nOpenResources, ^uint32(0))
}

func (p *ResourcePool) GetNOpenResources() uint32 {
	return atomic.LoadUint32(&p.nOpenResources)
}

// Stats returns an inconsistent (approximate) view of the pool's metrics. EG:
// we may say there are more in use connections than are open.
func (p *ResourcePool) Stats() ResourcePoolStat {
	tot := uint32(cap(p.tickets))
	n := uint32(len(p.tickets))
	inuse := tot - n
	available := uint32(len(p.reserve))

	return ResourcePoolStat{
		AvailableNow:  available,
		ResourcesOpen: p.GetNOpenResources(),
		Cap:           tot,
		InUse:         inuse,
	}
}
<commit_msg>Report when rate limited (#7)<commit_after>// A generic resource pool for databases etc
package pool

import (
	"errors"
	"io"
	"sync/atomic"
	"time"

	"github.com/bountylabs/api_common/errutil"
)

var (
	TimeoutError              = errors.New("Pool Timeout")
	PoolClosedError           = errors.New("Pool is closed")
	NewConnectionLimitedError = errors.New("Exceeded rate limit for creating new pool connections")
)

type Limiter interface {
	Allow() bool
}

// ResourceOpener opens a resource
type ResourceOpener interface {
	Open() (Resource, error)
}

type Resource interface {
	io.Closer
	// Good returns true when the resource is in good state
	Good() bool
}

// Ticket abstract type that represents the right to acquire a resource
type Ticket struct{}

type PooledResource interface {
	// Release releases the resource back to the pool for reuse
	Release() error
	// Destroy destroys the resource. It is no longer usable.
	Destroy() error
	// Resource returns the underlying Resource
	Resource() Resource
}

type PoolMetrics interface {
	ReportResources(stats ResourcePoolStat)
	ReportWait(wt time.Duration)       //how long did we wait for a connection
	ReportBorrowTime(wt time.Duration) //how long was the connection used for
	ReportNew(time time.Duration)      //how long did it take for a new connection to be opened
	ReportNewConnectionRateLimited()   // increment each time we don't create a new connection because we are rate limited
}

type ResourcePoolStat struct {
	AvailableNow  uint32
	ResourcesOpen uint32
	Cap           uint32
	InUse         uint32
}

// ResourcePool manages a pool of resources for reuse.
type ResourcePool struct {
	metrics PoolMetrics //metrics interface to track how the pool performs

	// To get a resource, get a ticket first and then get the resource from
	// either the reserve or open a new one.
	// To release a resource, first put the resource back to the reserve
	// and then put the ticket back.
	// The order is important to make sure we never create >cap(tickets)
	// number of resources.
	reserve chan Resource // idle resources, ready for use
	tickets chan *Ticket  // ticket to own resources

	opener               ResourceOpener
	closed               chan struct{}
	nOpenResources       uint32
	newConnectionLimiter Limiter
}

// NewPool creates a new pool of Clients.
func NewPool(maxReserve, maxOpen uint32, opener ResourceOpener, m PoolMetrics, newConnectionLimiter Limiter) *ResourcePool {
	if maxOpen < maxReserve {
		panic("maxOpen must be > maxReserve")
	}
	tickets := make(chan *Ticket, maxOpen)
	for i := uint32(0); i < maxOpen; i++ {
		tickets <- &Ticket{}
	}
	return &ResourcePool{
		metrics:              m,
		reserve:              make(chan Resource, maxReserve),
		tickets:              tickets,
		opener:               opener,
		closed:               make(chan struct{}),
		newConnectionLimiter: newConnectionLimiter,
	}
}

func (p *ResourcePool) Get() (PooledResource, error) {
	return p.GetWithTimeout(365 * 24 * time.Hour) // 1 year is forever
}

type pooledResource struct {
	ticket *Ticket
	served time.Time
	p      *ResourcePool
	res    Resource
}

func (pr *pooledResource) Release() error {
	defer pr.p.reportResources()
	pr.p.reportBorrowTime(time.Now().Sub(pr.served))
	return pr.p.release(pr)
}

func (pr *pooledResource) Destroy() error {
	defer pr.p.reportResources()
	pr.p.reportBorrowTime(time.Now().Sub(pr.served))
	return pr.p.destroy(pr)
}

func (pr *pooledResource) Resource() Resource {
	return pr.res
}

func (p *ResourcePool) releaseTicket(ticket *Ticket) {
	if ticket == nil {
		panic("BUG: can not release a nil ticket")
	}
	select {
	case p.tickets <- ticket:
	default:
		// releasing ticket should never block.
		panic("BUG: releaseTicket is called when ticket is not issued.")
	}
}

// WarmUp allocates resources until we have approximately allocated
// cap(p.reserve) resources.  We may under or over allocate resources (the code
// is subject to races) if someone is getting a resource while we are warming
// up.
func (p *ResourcePool) WarmUp() (count int, err error) {

	//loop until our open resources equals our reserve size
	for int(p.GetNOpenResources()) < cap(p.reserve) {

		var ticket *Ticket
		select {
		case ticket = <-p.tickets:
		case <-p.closed:
			return count, nil
		default: //all tickets are handed out
			return count, nil
		}

		if pooledResource, err := p.open(ticket, true); err != nil {
			return count, err
		} else {
			count++
			pooledResource.Release()
			continue
		}
	}

	return count, nil
}

func (p *ResourcePool) GetWithTimeout(timeout time.Duration) (pr PooledResource, err error) {

	// order is important: first ticket then reserve
	defer p.reportResources()
	defer func(start time.Time) {
		p.reportWait(time.Now().Sub(start))
	}(time.Now())

	timer := time.NewTimer(timeout)

	var ticket *Ticket
	select {
	case ticket = <-p.tickets:
	case <-timer.C:
		return nil, TimeoutError
	case <-p.closed:
		timer.Stop()
		return nil, PoolClosedError
	}
	timer.Stop()
	if p.isClosed() {
		// release ticket on close
		p.releaseTicket(ticket)
		return nil, PoolClosedError
	}

	if pooledResource := p.getFromReserve(ticket); pooledResource != nil {
		return pooledResource, nil
	}

	return p.open(ticket, false)

}

// getFromReserve attempts to get a pooledResource from the reserve pool, requires a ticket!
func (p *ResourcePool) getFromReserve(ticket *Ticket) *pooledResource {

	if ticket == nil {
		panic("BUG: can not acquire a reserved resource without a ticket")
	}

	for {
		select {
		case r := <-p.reserve:
			if r.Good() {
				return &pooledResource{ticket: ticket, served: time.Now(), p: p, res: r}
			} else {
				p.closeResource(r)
			}
			continue
		default:
			return nil
		}
	}
}

// open, opens a new resource, requires a ticket!
func (p *ResourcePool) open(ticket *Ticket, fromWarmup bool) (*pooledResource, error) {

	if ticket == nil {
		panic("BUG: can not open a resource without a ticket")
	}

	if !fromWarmup && p.newConnectionLimiter != nil {
		if !p.newConnectionLimiter.Allow() {
			p.metrics.ReportNewConnectionRateLimited()
			return nil, NewConnectionLimitedError
		}
	}

	start := time.Now()
	r, err := p.opener.Open()
	p.reportNew(time.Now().Sub(start))

	if err != nil {
		// release ticket on error
		p.releaseTicket(ticket)
		return nil, err
	}
	p.incrOpen()
	return &pooledResource{ticket: ticket, served: time.Now(), p: p, res: r}, nil
}

func (p *ResourcePool) closeResource(res Resource) error {
	p.decrOpen()
	return res.Close()
}

func (p *ResourcePool) release(pr *pooledResource) error {
	var err error
	// order is important: first reserve then ticket
	res := pr.Resource()
	select {
	case p.reserve <- res:
	default:
		// reserve is full
		err = p.closeResource(res)
	}
	p.releaseTicket(pr.ticket)
	pr.ticket = nil

	if p.isClosed() {
		p.drainReserve()
	}
	return err
}

func (p *ResourcePool) destroy(pr *pooledResource) error {
	p.releaseTicket(pr.ticket)
	pr.ticket = nil
	return p.closeResource(pr.Resource())
}

// Close closes the pool. Resources in use are not affected.
func (p *ResourcePool) Close() error {
	close(p.closed)
	return p.drainReserve()
}

func (p *ResourcePool) isClosed() bool {
	select {
	case <-p.closed:
		return true
	default:
		return false
	}
}

func (p *ResourcePool) drainReserve() error {
	out := []error{}
	for {
		select {
		case r := <-p.reserve:
			if err := p.closeResource(r); err != nil {
				out = append(out, err)
			}
		default:
			return errutil.ErrSlice(out)
		}
	}
}

/**
Metrics
**/

func (p *ResourcePool) reportNew(time time.Duration) {
	if p.metrics != nil {
		p.metrics.ReportNew(time)
	}
}

func (p *ResourcePool) reportWait(wt time.Duration) {
	if p.metrics != nil {
		p.metrics.ReportWait(wt)
	}
}

func (p *ResourcePool) reportBorrowTime(wt time.Duration) {
	if p.metrics != nil {
		p.metrics.ReportBorrowTime(wt)
	}
}

func (p *ResourcePool) reportResources() {
	if p.metrics != nil {
		stats := p.Stats()
		p.metrics.ReportResources(stats)
	}
}

func (p *ResourcePool) incrOpen() {
	atomic.AddUint32(&p.nOpenResources, 1)
}

func (p *ResourcePool) decrOpen() {
	atomic.AddUint32(&p.nOpenResources, ^uint32(0))
}

func (p *ResourcePool) GetNOpenResources() uint32 {
	return atomic.LoadUint32(&p.nOpenResources)
}

// Stats returns an inconsistent (approximate) view of the pool's metrics. EG:
// we may say there are more in use connections than are open.
func (p *ResourcePool) Stats() ResourcePoolStat {
	tot := uint32(cap(p.tickets))
	n := uint32(len(p.tickets))
	inuse := tot - n
	available := uint32(len(p.reserve))

	return ResourcePoolStat{
		AvailableNow:  available,
		ResourcesOpen: p.GetNOpenResources(),
		Cap:           tot,
		InUse:         inuse,
	}
}
<|endoftext|>


<commit_before>package rethinkdb

import (
	"errors"
	"fmt"
	"net"
	"sync"

	"golang.org/x/net/context"
	"gopkg.in/fatih/pool.v2"
)

var (
	errPoolClosed = errors.New("rethinkdb: pool is closed")
)

// A Pool is used to store a pool of connections to a single RethinkDB server
type Pool struct {
	host Host
	opts *ConnectOpts

	pool pool.Pool

	mu     sync.RWMutex // protects following fields
	closed bool
}

// NewPool creates a new connection pool for the given host
func NewPool(host Host, opts *ConnectOpts) (*Pool, error) {
	initialCap := opts.InitialCap
	if initialCap <= 0 {
		// Fallback to MaxIdle if InitialCap is zero, this should be removed
		// when MaxIdle is removed
		initialCap = opts.MaxIdle
	}

	maxOpen := opts.MaxOpen
	if maxOpen <= 0 {
		maxOpen = 2
	}

	p, err := pool.NewChannelPool(initialCap, maxOpen, func() (net.Conn, error) {
		conn, err := NewConnection(host.String(), opts)
		if err != nil {
			return nil, err
		}

		return conn, err
	})
	if err != nil {
		return nil, err
	}

	return &Pool{
		pool: p,
		host: host,
		opts: opts,
	}, nil
}

// Ping verifies a connection to the database is still alive,
// establishing a connection if necessary.
func (p *Pool) Ping() error {
	_, pc, err := p.conn()
	if err != nil {
		return err
	}
	return pc.Close()
}

// Close closes the database, releasing any open resources.
//
// It is rare to Close a Pool, as the Pool handle is meant to be
// long-lived and shared between many goroutines.
func (p *Pool) Close() error {
	p.mu.RLock()
	defer p.mu.RUnlock()
	if p.closed {
		return nil
	}

	p.pool.Close()

	return nil
}

func (p *Pool) conn() (*Connection, *pool.PoolConn, error) {
	p.mu.RLock()
	defer p.mu.RUnlock()

	if p.closed {
		return nil, nil, errPoolClosed
	}

	nc, err := p.pool.Get()
	if err != nil {
		return nil, nil, err
	}

	pc, ok := nc.(*pool.PoolConn)
	if !ok {
		// This should never happen!
		return nil, nil, fmt.Errorf("Invalid connection in pool")
	}

	conn, ok := pc.Conn.(*Connection)
	if !ok {
		// This should never happen!
		return nil, nil, fmt.Errorf("Invalid connection in pool")
	}

	return conn, pc, nil
}

// SetInitialPoolCap sets the initial capacity of the connection pool.
//
// Deprecated: This value should only be set when connecting
func (p *Pool) SetInitialPoolCap(n int) {
	return
}

// SetMaxIdleConns sets the maximum number of connections in the idle
// connection pool.
//
// Deprecated: This value should only be set when connecting
func (p *Pool) SetMaxIdleConns(n int) {
	return
}

// SetMaxOpenConns sets the maximum number of open connections to the database.
//
// Deprecated: This value should only be set when connecting
func (p *Pool) SetMaxOpenConns(n int) {
	return
}

// Query execution functions

// Exec executes a query without waiting for any response.
func (p *Pool) Exec(ctx context.Context, q Query) error {
	c, pc, err := p.conn()
	if err != nil {
		return err
	}
	defer pc.Close()

	_, _, err = c.Query(ctx, q)

	if c.isBad() {
		pc.MarkUnusable()
	}

	return err
}

// Query executes a query and waits for the response
func (p *Pool) Query(ctx context.Context, q Query) (*Cursor, error) {
	c, pc, err := p.conn()
	if err != nil {
		return nil, err
	}

	_, cursor, err := c.Query(ctx, q)

	if err == nil {
		cursor.releaseConn = releaseConn(c, pc)
	} else {
		if c.isBad() {
			pc.MarkUnusable()
		}
		pc.Close()
	}

	return cursor, err
}

// Server returns the server name and server UUID being used by a connection.
func (p *Pool) Server() (ServerResponse, error) {
	var response ServerResponse

	c, pc, err := p.conn()
	if err != nil {
		return response, err
	}
	defer pc.Close()

	response, err = c.Server()

	if c.isBad() {
		pc.MarkUnusable()
	}

	return response, err
}

func releaseConn(c *Connection, pc *pool.PoolConn) func() error {
	return func() error {
		if c.isBad() {
			pc.MarkUnusable()
		}

		return pc.Close()
	}
}
<commit_msg>simple connection reusage round robin<commit_after>package rethinkdb

import (
	"errors"
	"sync"
	"sync/atomic"

	"golang.org/x/net/context"
	"gopkg.in/fatih/pool.v2"
)

var (
	errPoolClosed = errors.New("rethinkdb: pool is closed")
)

// A Pool is used to store a pool of connections to a single RethinkDB server
type Pool struct {
	host Host
	opts *ConnectOpts

	conns   []*Connection
	pointer int32

	mu     sync.RWMutex // protects following fields
	closed bool
}

// NewPool creates a new connection pool for the given host
func NewPool(host Host, opts *ConnectOpts) (*Pool, error) {
	initialCap := opts.InitialCap
	if initialCap <= 0 {
		// Fallback to MaxIdle if InitialCap is zero, this should be removed
		// when MaxIdle is removed
		initialCap = opts.MaxIdle
	}

	maxOpen := opts.MaxOpen
	if maxOpen <= 0 {
		maxOpen = 1
	}

	conns := make([]*Connection, maxOpen)
	var err error
	for i := range conns {
		conns[i], err = NewConnection(host.String(), opts)
		if err != nil {
			return nil, err
		}
	}

	return &Pool{
		conns:   conns,
		pointer: -1,
		host:    host,
		opts:    opts,
	}, nil
}

// Ping verifies a connection to the database is still alive,
// establishing a connection if necessary.
func (p *Pool) Ping() error {
	_, _, err := p.conn()
	return err
}

// Close closes the database, releasing any open resources.
//
// It is rare to Close a Pool, as the Pool handle is meant to be
// long-lived and shared between many goroutines.
func (p *Pool) Close() error {
	p.mu.RLock()
	defer p.mu.RUnlock()
	if p.closed {
		return nil
	}

	for _, c := range p.conns {
		err := c.Close()
		if err != nil {
			return err
		}
	}

	return nil
}

func (p *Pool) conn() (*Connection, *pool.PoolConn, error) {
	p.mu.RLock()

	if p.closed {
		p.mu.RUnlock()
		return nil, nil, errPoolClosed
	}
	p.mu.RUnlock()

	pos := atomic.AddInt32(&p.pointer, 1)
	if pos == int32(len(p.conns)) {
		atomic.StoreInt32(&p.pointer, 0)
	}
	pos = pos % int32(len(p.conns))

	return p.conns[pos], nil, nil
}

// SetInitialPoolCap sets the initial capacity of the connection pool.
//
// Deprecated: This value should only be set when connecting
func (p *Pool) SetInitialPoolCap(n int) {
	return
}

// SetMaxIdleConns sets the maximum number of connections in the idle
// connection pool.
//
// Deprecated: This value should only be set when connecting
func (p *Pool) SetMaxIdleConns(n int) {
	return
}

// SetMaxOpenConns sets the maximum number of open connections to the database.
//
// Deprecated: This value should only be set when connecting
func (p *Pool) SetMaxOpenConns(n int) {
	return
}

// Query execution functions

// Exec executes a query without waiting for any response.
func (p *Pool) Exec(ctx context.Context, q Query) error {
	c, _, err := p.conn()
	if err != nil {
		return err
	}

	_, _, err = c.Query(ctx, q)
	return err
}

// Query executes a query and waits for the response
func (p *Pool) Query(ctx context.Context, q Query) (*Cursor, error) {
	c, _, err := p.conn()
	if err != nil {
		return nil, err
	}

	_, cursor, err := c.Query(ctx, q)
	return cursor, err
}

// Server returns the server name and server UUID being used by a connection.
func (p *Pool) Server() (ServerResponse, error) {
	var response ServerResponse

	c, _, err := p.conn()
	if err != nil {
		return response, err
	}

	response, err = c.Server()
	return response, err
}
<|endoftext|>


<commit_before>package main

import (
	."code.google.com/p/gordon-go/gui"
	"code.google.com/p/go.exp/go/types"
)

type port struct {
	*ViewBase
	out bool
	node node
	obj *types.Var
	valView *typeView
	conns []*connection
	focused bool
	connsChanged func()
}

const portSize = 11

func newInput(n node, v *types.Var) *port { return newPort(false, n, v) }
func newOutput(n node, v *types.Var) *port { return newPort(true, n, v) }
func newPort(out bool, n node, v *types.Var) *port {
	p := &port{out:out, node:n, obj:v}
	p.ViewBase = NewView(p)
	p.valView = newValueView(v)
	p.valView.Hide()
	p.connsChanged = func(){}
	p.AddChild(p.valView)
	p.Resize(portSize, portSize)
	p.Pan(Pt(portSize, portSize).Div(-2))
	p.setType(*p.valView.typ)
	return p
}

func (p *port) setType(t types.Type) {
	p.valView.setType(t)
	if p.out {
		p.valView.Move(Pt(12, -p.valView.Height() / 2))
	} else {
		p.valView.Move(Pt(-p.valView.Width() - 12, -p.valView.Height() / 2))
	}
}

func (p port) canConnect(q *port) bool {
	pSeq, qSeq := p.obj.Type == seqType, q.obj.Type == seqType
	if pSeq && !qSeq || !pSeq && qSeq { return false }
	return p.out != q.out
}
func (p *port) connect(c *connection) {
	p.conns = append(p.conns, c)
}
func (p *port) disconnect(c *connection) {
	for i, c2 := range p.conns {
		if c2 == c {
			p.conns = append(p.conns[:i], p.conns[i+1:]...)
			return
		}
	}
}

func (p *port) TookKeyboardFocus() { p.focused = true; p.Repaint(); p.valView.Show() }
func (p *port) LostKeyboardFocus() { p.focused = false; p.Repaint(); p.valView.Hide() }

func (p *port) KeyPressed(event KeyEvent) {
	if p.out && event.Key == KeyLeft || !p.out && event.Key == KeyRight {
		p.node.TakeKeyboardFocus()
		return
	}
	if p.obj.Type != seqType &&
	  (p.out && event.Key == KeyRight && len(p.conns) > 0 ||
	   !p.out && event.Key == KeyLeft && len(p.conns) > 0) {
		p.conns[len(p.conns) - 1].TakeKeyboardFocus()
		return
	}
	if p.obj.Type == seqType && event.Key == KeyDown && len(p.conns) > 0 {
		p.conns[len(p.conns) - 1].TakeKeyboardFocus()
		return
	}

	switch event.Key {
	case KeyEnter:
		c := newConnection()
		if p.out {
			c.setSrc(p)
		} else {
			c.setDst(p)
		}
		c.startEditing()
	case KeyLeft, KeyRight, KeyUp, KeyDown:
		p.node.block().outermost().focusNearestView(p, event.Key)
	case KeyBackspace, KeyDelete:
		if p.out && event.Key == KeyBackspace || !p.out && event.Key == KeyDelete {
			if n, ok := p.node.(interface{removePort(*port)}); ok {
				n.removePort(p)
			} else {
				p.node.TakeKeyboardFocus()
			}
		} else if len(p.conns) > 0 {
			p.conns[len(p.conns) - 1].TakeKeyboardFocus()
		}
	case KeyEscape:
		p.node.TakeKeyboardFocus()
	default:
		p.ViewBase.KeyPressed(event)
	}
}

func (p *port) MousePressed(button int, pt Point) {
	p.TakeKeyboardFocus()
	c := newConnection()
	if p.out {
		c.setSrc(p)
		c.dstHandle.SetMouseFocus(c.dstHandle, button)
	} else {
		c.setDst(p)
		c.srcHandle.SetMouseFocus(c.srcHandle, button)
	}
	c.startEditing()
}
func (p port) MouseDragged(button int, pt Point) {}
func (p port) MouseReleased(button int, pt Point) {}

func (p port) Paint() {
	if p.focused {
		SetColor(Color{.3, .3, .7, .5})
		SetPointSize(portSize)
		DrawPoint(ZP)
	}
}
<commit_msg>Flux:  Disallow cycle-forming connections.<commit_after>package main

import (
	."code.google.com/p/gordon-go/gui"
	"code.google.com/p/go.exp/go/types"
)

type port struct {
	*ViewBase
	out bool
	node node
	obj *types.Var
	valView *typeView
	conns []*connection
	focused bool
	connsChanged func()
}

const portSize = 11

func newInput(n node, v *types.Var) *port { return newPort(false, n, v) }
func newOutput(n node, v *types.Var) *port { return newPort(true, n, v) }
func newPort(out bool, n node, v *types.Var) *port {
	p := &port{out:out, node:n, obj:v}
	p.ViewBase = NewView(p)
	p.valView = newValueView(v)
	p.valView.Hide()
	p.connsChanged = func(){}
	p.AddChild(p.valView)
	p.Resize(portSize, portSize)
	p.Pan(Pt(portSize, portSize).Div(-2))
	p.setType(*p.valView.typ)
	return p
}

func (p *port) setType(t types.Type) {
	p.valView.setType(t)
	if p.out {
		p.valView.Move(Pt(12, -p.valView.Height() / 2))
	} else {
		p.valView.Move(Pt(-p.valView.Width() - 12, -p.valView.Height() / 2))
	}
}

func (p port) canConnect(q *port) bool {
	pSeq, qSeq := p.obj.Type == seqType, q.obj.Type == seqType
	if pSeq && !qSeq || !pSeq && qSeq {
		return false
	}
	if p.out == q.out {
		return false
	}

	src, dst := q.node, p.node
	if p.out {
		src, dst = dst, src
	}
	return !precedes(dst, src)
}

func precedes(start, end node) bool {
loop:
	for n1 := start; n1.block() != nil; n1 = n1.block().node {
		for n2 := end; n2.block() != nil; n2 = n2.block().node {
			if n1.block() == n2.block() {
				start, end = n1, n2
				break loop
			}
		}
	}

	if start == end {
		return true
	}

	for _, c := range start.outConns() {
		if c.dst == nil {
			continue
		}
		if precedes(c.dst.node, end) {
			return true
		}
	}

	return false
}

func (p *port) connect(c *connection) {
	p.conns = append(p.conns, c)
}

func (p *port) disconnect(c *connection) {
	for i, c2 := range p.conns {
		if c2 == c {
			p.conns = append(p.conns[:i], p.conns[i+1:]...)
			return
		}
	}
}

func (p *port) TookKeyboardFocus() { p.focused = true; p.Repaint(); p.valView.Show() }
func (p *port) LostKeyboardFocus() { p.focused = false; p.Repaint(); p.valView.Hide() }

func (p *port) KeyPressed(event KeyEvent) {
	if p.out && event.Key == KeyLeft || !p.out && event.Key == KeyRight {
		p.node.TakeKeyboardFocus()
		return
	}
	if p.obj.Type != seqType &&
	  (p.out && event.Key == KeyRight && len(p.conns) > 0 ||
	   !p.out && event.Key == KeyLeft && len(p.conns) > 0) {
		p.conns[len(p.conns) - 1].TakeKeyboardFocus()
		return
	}
	if p.obj.Type == seqType && event.Key == KeyDown && len(p.conns) > 0 {
		p.conns[len(p.conns) - 1].TakeKeyboardFocus()
		return
	}

	switch event.Key {
	case KeyEnter:
		c := newConnection()
		if p.out {
			c.setSrc(p)
		} else {
			c.setDst(p)
		}
		c.startEditing()
	case KeyLeft, KeyRight, KeyUp, KeyDown:
		p.node.block().outermost().focusNearestView(p, event.Key)
	case KeyBackspace, KeyDelete:
		if p.out && event.Key == KeyBackspace || !p.out && event.Key == KeyDelete {
			if n, ok := p.node.(interface{removePort(*port)}); ok {
				n.removePort(p)
			} else {
				p.node.TakeKeyboardFocus()
			}
		} else if len(p.conns) > 0 {
			p.conns[len(p.conns) - 1].TakeKeyboardFocus()
		}
	case KeyEscape:
		p.node.TakeKeyboardFocus()
	default:
		p.ViewBase.KeyPressed(event)
	}
}

func (p *port) MousePressed(button int, pt Point) {
	p.TakeKeyboardFocus()
	c := newConnection()
	if p.out {
		c.setSrc(p)
		c.dstHandle.SetMouseFocus(c.dstHandle, button)
	} else {
		c.setDst(p)
		c.srcHandle.SetMouseFocus(c.srcHandle, button)
	}
	c.startEditing()
}
func (p port) MouseDragged(button int, pt Point) {}
func (p port) MouseReleased(button int, pt Point) {}

func (p port) Paint() {
	if p.focused {
		SetColor(Color{.3, .3, .7, .5})
		SetPointSize(portSize)
		DrawPoint(ZP)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
	"internal/testenv"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"testing"
)

var lldbPath string

func checkLldbPython(t *testing.T) {
	cmd := exec.Command("lldb", "-P")
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Skipf("skipping due to issue running lldb: %v\n%s", err, out)
	}
	lldbPath = strings.TrimSpace(string(out))

	cmd = exec.Command("/usr/bin/python2.7", "-c", "import sys;sys.path.append(sys.argv[1]);import lldb; print('go lldb python support')", lldbPath)
	out, err = cmd.CombinedOutput()

	if err != nil {
		t.Skipf("skipping due to issue running python: %v\n%s", err, out)
	}
	if string(out) != "go lldb python support\n" {
		t.Skipf("skipping due to lack of python lldb support: %s", out)
	}

	if runtime.GOOS == "darwin" {
		// Try to see if we have debugging permissions.
		cmd = exec.Command("/usr/sbin/DevToolsSecurity", "-status")
		out, err = cmd.CombinedOutput()
		if err != nil {
			t.Skipf("DevToolsSecurity failed: %v", err)
		} else if !strings.Contains(string(out), "enabled") {
			t.Skip(string(out))
		}
		cmd = exec.Command("/usr/bin/groups")
		out, err = cmd.CombinedOutput()
		if err != nil {
			t.Skipf("groups failed: %v", err)
		} else if !strings.Contains(string(out), "_developer") {
			t.Skip("Not in _developer group")
		}
	}
}

const lldbHelloSource = `
package main
import "fmt"
func main() {
	mapvar := make(map[string]string,5)
	mapvar["abc"] = "def"
	mapvar["ghi"] = "jkl"
	intvar := 42
	ptrvar := &intvar
	fmt.Println("hi") // line 10
	_ = ptrvar
}
`

const lldbScriptSource = `
import sys
sys.path.append(sys.argv[1])
import lldb
import os

TIMEOUT_SECS = 5

debugger = lldb.SBDebugger.Create()
debugger.SetAsync(True)
target = debugger.CreateTargetWithFileAndArch("a.exe", None)
if target:
  print "Created target"
  main_bp = target.BreakpointCreateByLocation("main.go", 10)
  if main_bp.GetNumLocations() != 0:
    print "Created breakpoint"
  else:
    # This happens if lldb can't read the program's DWARF. See https://golang.org/issue/25925.
    print "SKIP: no matching locations for breakpoint"
    exit(1)
  process = target.LaunchSimple(None, None, os.getcwd())
  if process:
    print "Process launched"
    listener = debugger.GetListener()
    process.broadcaster.AddListener(listener, lldb.SBProcess.eBroadcastBitStateChanged)
    while True:
      event = lldb.SBEvent()
      if listener.WaitForEvent(TIMEOUT_SECS, event):
        if lldb.SBProcess.GetRestartedFromEvent(event):
          continue
        state = process.GetState()
        if state in [lldb.eStateUnloaded, lldb.eStateLaunching, lldb.eStateRunning]:
          continue
      else:
        print "SKIP: Timeout launching"
      break
    if state == lldb.eStateStopped:
      for t in process.threads:
        if t.GetStopReason() == lldb.eStopReasonBreakpoint:
          print "Hit breakpoint"
          frame = t.GetFrameAtIndex(0)
          if frame:
            if frame.line_entry:
              print "Stopped at %s:%d" % (frame.line_entry.file.basename, frame.line_entry.line)
            if frame.function:
              print "Stopped in %s" % (frame.function.name,)
            var = frame.FindVariable('intvar')
            if var:
              print "intvar = %s" % (var.GetValue(),)
            else:
              print "no intvar"
    else:
      print "Process state", state
    process.Destroy()
else:
  print "Failed to create target a.exe"

lldb.SBDebugger.Destroy(debugger)
sys.exit()
`

const expectedLldbOutput = `Created target
Created breakpoint
Process launched
Hit breakpoint
Stopped at main.go:10
Stopped in main.main
intvar = 42
`

func TestLldbPython(t *testing.T) {
	testenv.MustHaveGoBuild(t)
	if final := os.Getenv("GOROOT_FINAL"); final != "" && runtime.GOROOT() != final {
		t.Skip("gdb test can fail with GOROOT_FINAL pending")
	}

	checkLldbPython(t)

	dir, err := ioutil.TempDir("", "go-build")
	if err != nil {
		t.Fatalf("failed to create temp directory: %v", err)
	}
	defer os.RemoveAll(dir)

	src := filepath.Join(dir, "main.go")
	err = ioutil.WriteFile(src, []byte(lldbHelloSource), 0644)
	if err != nil {
		t.Fatalf("failed to create file: %v", err)
	}

	cmd := exec.Command(testenv.GoToolPath(t), "build", "-gcflags=all=-N -l", "-o", "a.exe")
	cmd.Dir = dir
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("building source %v\n%s", err, out)
	}

	src = filepath.Join(dir, "script.py")
	err = ioutil.WriteFile(src, []byte(lldbScriptSource), 0755)
	if err != nil {
		t.Fatalf("failed to create script: %v", err)
	}

	cmd = exec.Command("/usr/bin/python2.7", "script.py", lldbPath)
	cmd.Dir = dir
	got, _ := cmd.CombinedOutput()

	if string(got) != expectedLldbOutput {
		skipReason := regexp.MustCompile("SKIP: .*\n").Find(got)
		if skipReason != nil {
			t.Skip(string(skipReason))
		}
		t.Fatalf("Unexpected lldb output:\n%s", got)
	}
}
<commit_msg>runtime: disable compressed DWARF for lldb test<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
	"internal/testenv"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"testing"
)

var lldbPath string

func checkLldbPython(t *testing.T) {
	cmd := exec.Command("lldb", "-P")
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Skipf("skipping due to issue running lldb: %v\n%s", err, out)
	}
	lldbPath = strings.TrimSpace(string(out))

	cmd = exec.Command("/usr/bin/python2.7", "-c", "import sys;sys.path.append(sys.argv[1]);import lldb; print('go lldb python support')", lldbPath)
	out, err = cmd.CombinedOutput()

	if err != nil {
		t.Skipf("skipping due to issue running python: %v\n%s", err, out)
	}
	if string(out) != "go lldb python support\n" {
		t.Skipf("skipping due to lack of python lldb support: %s", out)
	}

	if runtime.GOOS == "darwin" {
		// Try to see if we have debugging permissions.
		cmd = exec.Command("/usr/sbin/DevToolsSecurity", "-status")
		out, err = cmd.CombinedOutput()
		if err != nil {
			t.Skipf("DevToolsSecurity failed: %v", err)
		} else if !strings.Contains(string(out), "enabled") {
			t.Skip(string(out))
		}
		cmd = exec.Command("/usr/bin/groups")
		out, err = cmd.CombinedOutput()
		if err != nil {
			t.Skipf("groups failed: %v", err)
		} else if !strings.Contains(string(out), "_developer") {
			t.Skip("Not in _developer group")
		}
	}
}

const lldbHelloSource = `
package main
import "fmt"
func main() {
	mapvar := make(map[string]string,5)
	mapvar["abc"] = "def"
	mapvar["ghi"] = "jkl"
	intvar := 42
	ptrvar := &intvar
	fmt.Println("hi") // line 10
	_ = ptrvar
}
`

const lldbScriptSource = `
import sys
sys.path.append(sys.argv[1])
import lldb
import os

TIMEOUT_SECS = 5

debugger = lldb.SBDebugger.Create()
debugger.SetAsync(True)
target = debugger.CreateTargetWithFileAndArch("a.exe", None)
if target:
  print "Created target"
  main_bp = target.BreakpointCreateByLocation("main.go", 10)
  if main_bp.GetNumLocations() != 0:
    print "Created breakpoint"
  else:
    # This happens if lldb can't read the program's DWARF. See https://golang.org/issue/25925.
    print "SKIP: no matching locations for breakpoint"
    exit(1)
  process = target.LaunchSimple(None, None, os.getcwd())
  if process:
    print "Process launched"
    listener = debugger.GetListener()
    process.broadcaster.AddListener(listener, lldb.SBProcess.eBroadcastBitStateChanged)
    while True:
      event = lldb.SBEvent()
      if listener.WaitForEvent(TIMEOUT_SECS, event):
        if lldb.SBProcess.GetRestartedFromEvent(event):
          continue
        state = process.GetState()
        if state in [lldb.eStateUnloaded, lldb.eStateLaunching, lldb.eStateRunning]:
          continue
      else:
        print "SKIP: Timeout launching"
      break
    if state == lldb.eStateStopped:
      for t in process.threads:
        if t.GetStopReason() == lldb.eStopReasonBreakpoint:
          print "Hit breakpoint"
          frame = t.GetFrameAtIndex(0)
          if frame:
            if frame.line_entry:
              print "Stopped at %s:%d" % (frame.line_entry.file.basename, frame.line_entry.line)
            if frame.function:
              print "Stopped in %s" % (frame.function.name,)
            var = frame.FindVariable('intvar')
            if var:
              print "intvar = %s" % (var.GetValue(),)
            else:
              print "no intvar"
    else:
      print "Process state", state
    process.Destroy()
else:
  print "Failed to create target a.exe"

lldb.SBDebugger.Destroy(debugger)
sys.exit()
`

const expectedLldbOutput = `Created target
Created breakpoint
Process launched
Hit breakpoint
Stopped at main.go:10
Stopped in main.main
intvar = 42
`

func TestLldbPython(t *testing.T) {
	testenv.MustHaveGoBuild(t)
	if final := os.Getenv("GOROOT_FINAL"); final != "" && runtime.GOROOT() != final {
		t.Skip("gdb test can fail with GOROOT_FINAL pending")
	}

	checkLldbPython(t)

	dir, err := ioutil.TempDir("", "go-build")
	if err != nil {
		t.Fatalf("failed to create temp directory: %v", err)
	}
	defer os.RemoveAll(dir)

	src := filepath.Join(dir, "main.go")
	err = ioutil.WriteFile(src, []byte(lldbHelloSource), 0644)
	if err != nil {
		t.Fatalf("failed to create file: %v", err)
	}

	// As of 2018-07-17, lldb doesn't support compressed DWARF, so
	// disable it for this test.
	cmd := exec.Command(testenv.GoToolPath(t), "build", "-gcflags=all=-N -l", "-ldflags=-compressdwarf=false", "-o", "a.exe")
	cmd.Dir = dir
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("building source %v\n%s", err, out)
	}

	src = filepath.Join(dir, "script.py")
	err = ioutil.WriteFile(src, []byte(lldbScriptSource), 0755)
	if err != nil {
		t.Fatalf("failed to create script: %v", err)
	}

	cmd = exec.Command("/usr/bin/python2.7", "script.py", lldbPath)
	cmd.Dir = dir
	got, _ := cmd.CombinedOutput()

	if string(got) != expectedLldbOutput {
		skipReason := regexp.MustCompile("SKIP: .*\n").Find(got)
		if skipReason != nil {
			t.Skip(string(skipReason))
		}
		t.Fatalf("Unexpected lldb output:\n%s", got)
	}
}
<|endoftext|>


<commit_before>package s3git

import (
	"errors"
	"fmt"

	"github.com/s3git/s3git-go/internal/backend"
	"github.com/s3git/s3git-go/internal/cas"
	"github.com/s3git/s3git-go/internal/core"
	"github.com/s3git/s3git-go/internal/kv"
	"github.com/s3git/s3git-go/internal/util"

	"encoding/hex"
	"github.com/dustin/go-humanize"
	"sync"
)

// Perform a push to the back end for the repository
func (repo Repository) Push(hydrated bool, progress func(maxTicks int64)) error {

	list, err := kv.ListLevel1Prefixes()
	if err != nil {
		return err
	}

	return push(list, hydrated, progress)
}

// Push any new commit objects including all added objects to the back end store
func push(prefixChan <-chan []byte, hydrated bool, progress func(maxTicks int64)) error {

	client, err := backend.GetDefaultClient()
	if err != nil {
		return err
	}

	// Get map of prefixes already in store
	prefixesInBackend, err := listPrefixes(client)
	if err != nil {
		return err
	}

	prefixesToPush := []string{}

	for prefixByte := range prefixChan {

		prefix := hex.EncodeToString(prefixByte)

		_, verified := prefixesInBackend[prefix]

		// We can safely skip in case a prefix object is verified (pushed as last object)
		if !verified {

			prefixesToPush = append(prefixesToPush, prefix)
		}
	}

	if len(prefixesToPush) == 0 {
		return nil
	}

	progress(int64(len(prefixesToPush)))

	for _, prefix := range prefixesToPush {

		// Get prefix object
		po, err := core.GetPrefixObject(prefix)
		if err != nil {
			return err
		}

		// Get commit object
		co, err := core.GetCommitObject(po.S3gitFollowMe)
		if err != nil {
			return err
		}

		// Get tree object
		to, err := core.GetTreeObject(co.S3gitTree)
		if err != nil {
			return err
		}

		// first push all added blobs in this commit ...
		err = pushBlobRange(to.S3gitAdded, nil, client)
		if err != nil {
			return err
		}

		// then push tree object
		_, err = pushBlob(co.S3gitTree, nil, client)
		if err != nil {
			return err
		}

		// then push commit object
		_, err = pushBlob(po.S3gitFollowMe, nil, client)
		if err != nil {
			return err
		}

		// ...  finally push prefix object itself
		// (if something goes in chain above, the prefix object will be missing so
		//  will be (attempted to) uploaded again during the next push)
		_, err = pushBlob(prefix, nil, client)
		if err != nil {
			return err
		}

		progress(int64(len(prefixesToPush)))
	}

	return nil
}

// Push a blob to the back end store
func pushBlob(hash string, size *uint64, client backend.Backend) (newlyUploaded bool, err error) {

	startOfLine := ""
	if size != nil {
		startOfLine = fmt.Sprintf("Uploading %s (%s)", util.FriendlyHash(hash), humanize.Bytes(*size))
	} else {
		startOfLine = fmt.Sprintf("Uploading %s", util.FriendlyHash(hash))
	}

	// TODO: Consider whether we want to verify again...
	if false {
		verified, err := client.VerifyHash(hash)
		if err != nil {
			fmt.Println(startOfLine, "verification failed", err)
			return false, err
		}

		if verified { // Resource already in back-end
			fmt.Println(startOfLine, "already in store")

			return false, nil
		}
	}

	// TODO: for back ends storing whole files: consider multipart upload?

	// TODO: implement back ends that store chunks (not whole files)
	// TODO: for back ends storing chunks: consider uploading chunks in parallel
	cr := cas.MakeReader(hash)
	if cr == nil {
		panic(errors.New("Failed to create cas reader"))
	}

	err = client.UploadWithReader(hash, cr)
	if err != nil {
		fmt.Println(startOfLine, "failed to upload to store", err)
		return false, err
	}

	// Move blob from .stage to .cache directory upon successful upload
	err = cas.MoveBlobToCache(hash)
	if err != nil {
		fmt.Println(startOfLine, "failed to move underlying blobs to cache", err)
		return false, err
	}

	fmt.Println(startOfLine, "successfully uploaded to store")

	return true, nil
}

func minu64(x, y uint64) uint64 {
	if x < y {
		return x
	}
	return y
}

// Push a range of blobs to the back end store in parallel
//
// See https://github.com/adonovan/gopl.io/blob/master/ch8/thumbnail/thumbnail_test.go
//
func pushBlobRange(hashes []string, size *uint64, client backend.Backend) error {

	var wg sync.WaitGroup
	var msgs = make(chan string)
	var results = make(chan error)

	for i := 0; i < min(len(hashes), 100); i++ {
		wg.Add(1)

		go func() {
			defer wg.Done()
			for hash := range msgs {
				_, err := pushBlob(hash, size, client)
				results <- err
			}
		}()
	}

	go func() {
		for _, hash := range hashes {
			msgs <- hash
		}
		close(msgs)
	}()

	go func() {
		wg.Wait()
		close(results)
	}()

	var err error
	for e := range results {
		if e != nil {
			err = e
		}
	}

	return err
}

// List prefixes at back end store, doing 16 lists in parallel
func listPrefixes(client backend.Backend) (map[string]bool, error) {

	var wg sync.WaitGroup
	var results = make(chan []string)

	for i := 0x0; i <= 0xf; i++ {
		wg.Add(1)

		go func(i int) {
			defer wg.Done()
			result := make([]string, 0, 1000)

			// TODO: Hard coded prefix of 7 zero's
			client.List(fmt.Sprintf("0000000%x", i), func(key string) {
				result = append(result, key)

				// TODO: WE NEED TO wg.Done() HERE WHEN LAST KEY HAS BEEN RECEIVED
				//       -- SEE /Users/frankw/golang/src/github.com/fwessels/listperf/listperf.go
				// IMPORTANT: WE WILL BE MISSING OBJECTS HERE
			})

			results <- result
		}(i)
	}

	go func() {
		wg.Wait()
		close(results)
	}()

	prefixHash := make(map[string]bool)
	for result := range results {
		for _, r := range result {
			prefixHash[r] = true
		}
	}

	return prefixHash, nil
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}
<commit_msg>Disable outputting of pushBlob<commit_after>package s3git

import (
	"errors"
	"fmt"

	"github.com/s3git/s3git-go/internal/backend"
	"github.com/s3git/s3git-go/internal/cas"
	"github.com/s3git/s3git-go/internal/core"
	"github.com/s3git/s3git-go/internal/kv"
	"github.com/s3git/s3git-go/internal/util"

	"encoding/hex"
	"github.com/dustin/go-humanize"
	"sync"
)

// Perform a push to the back end for the repository
func (repo Repository) Push(hydrated bool, progress func(maxTicks int64)) error {

	list, err := kv.ListLevel1Prefixes()
	if err != nil {
		return err
	}

	return push(list, hydrated, progress)
}

// Push any new commit objects including all added objects to the back end store
func push(prefixChan <-chan []byte, hydrated bool, progress func(maxTicks int64)) error {

	client, err := backend.GetDefaultClient()
	if err != nil {
		return err
	}

	// Get map of prefixes already in store
	prefixesInBackend, err := listPrefixes(client)
	if err != nil {
		return err
	}

	prefixesToPush := []string{}

	for prefixByte := range prefixChan {

		prefix := hex.EncodeToString(prefixByte)

		_, verified := prefixesInBackend[prefix]

		// We can safely skip in case a prefix object is verified (pushed as last object)
		if !verified {

			prefixesToPush = append(prefixesToPush, prefix)
		}
	}

	if len(prefixesToPush) == 0 {
		return nil
	}

	progress(int64(len(prefixesToPush)))

	for _, prefix := range prefixesToPush {

		// Get prefix object
		po, err := core.GetPrefixObject(prefix)
		if err != nil {
			return err
		}

		// Get commit object
		co, err := core.GetCommitObject(po.S3gitFollowMe)
		if err != nil {
			return err
		}

		// Get tree object
		to, err := core.GetTreeObject(co.S3gitTree)
		if err != nil {
			return err
		}

		// first push all added blobs in this commit ...
		err = pushBlobRange(to.S3gitAdded, nil, client)
		if err != nil {
			return err
		}

		// then push tree object
		_, err = pushBlob(co.S3gitTree, nil, client)
		if err != nil {
			return err
		}

		// then push commit object
		_, err = pushBlob(po.S3gitFollowMe, nil, client)
		if err != nil {
			return err
		}

		// ...  finally push prefix object itself
		// (if something goes in chain above, the prefix object will be missing so
		//  will be (attempted to) uploaded again during the next push)
		_, err = pushBlob(prefix, nil, client)
		if err != nil {
			return err
		}

		progress(int64(len(prefixesToPush)))
	}

	return nil
}

// Push a blob to the back end store
func pushBlob(hash string, size *uint64, client backend.Backend) (newlyUploaded bool, err error) {

	startOfLine := ""
	if size != nil {
		startOfLine = fmt.Sprintf("Uploading %s (%s)", util.FriendlyHash(hash), humanize.Bytes(*size))
	} else {
		startOfLine = fmt.Sprintf("Uploading %s", util.FriendlyHash(hash))
	}

	// TODO: Consider whether we want to verify again...
	if false {
		verified, err := client.VerifyHash(hash)
		if err != nil {
			fmt.Println(startOfLine, "verification failed", err)
			return false, err
		}

		if verified { // Resource already in back-end
			fmt.Println(startOfLine, "already in store")

			return false, nil
		}
	}

	// TODO: for back ends storing whole files: consider multipart upload?

	// TODO: implement back ends that store chunks (not whole files)
	// TODO: for back ends storing chunks: consider uploading chunks in parallel
	cr := cas.MakeReader(hash)
	if cr == nil {
		panic(errors.New("Failed to create cas reader"))
	}

	err = client.UploadWithReader(hash, cr)
	if err != nil {
		fmt.Println(startOfLine, "failed to upload to store", err)
		return false, err
	}

	// Move blob from .stage to .cache directory upon successful upload
	err = cas.MoveBlobToCache(hash)
	if err != nil {
		fmt.Println(startOfLine, "failed to move underlying blobs to cache", err)
		return false, err
	}

	fmt.Println(startOfLine, "successfully uploaded to store")
	//fmt.Println(startOfLine, "successfully uploaded to store")

	return true, nil
}

func minu64(x, y uint64) uint64 {
	if x < y {
		return x
	}
	return y
}

// Push a range of blobs to the back end store in parallel
//
// See https://github.com/adonovan/gopl.io/blob/master/ch8/thumbnail/thumbnail_test.go
//
func pushBlobRange(hashes []string, size *uint64, client backend.Backend) error {

	var wg sync.WaitGroup
	var msgs = make(chan string)
	var results = make(chan error)

	for i := 0; i < min(len(hashes), 100); i++ {
		wg.Add(1)

		go func() {
			defer wg.Done()
			for hash := range msgs {
				_, err := pushBlob(hash, size, client)
				results <- err
			}
		}()
	}

	go func() {
		for _, hash := range hashes {
			msgs <- hash
		}
		close(msgs)
	}()

	go func() {
		wg.Wait()
		close(results)
	}()

	var err error
	for e := range results {
		if e != nil {
			err = e
		}
	}

	return err
}

// List prefixes at back end store, doing 16 lists in parallel
func listPrefixes(client backend.Backend) (map[string]bool, error) {

	var wg sync.WaitGroup
	var results = make(chan []string)

	for i := 0x0; i <= 0xf; i++ {
		wg.Add(1)

		go func(i int) {
			defer wg.Done()
			result := make([]string, 0, 1000)

			// TODO: Hard coded prefix of 7 zero's
			client.List(fmt.Sprintf("0000000%x", i), func(key string) {
				result = append(result, key)

				// TODO: WE NEED TO wg.Done() HERE WHEN LAST KEY HAS BEEN RECEIVED
				//       -- SEE /Users/frankw/golang/src/github.com/fwessels/listperf/listperf.go
				// IMPORTANT: WE WILL BE MISSING OBJECTS HERE
			})

			results <- result
		}(i)
	}

	go func() {
		wg.Wait()
		close(results)
	}()

	prefixHash := make(map[string]bool)
	for result := range results {
		for _, r := range result {
			prefixHash[r] = true
		}
	}

	return prefixHash, nil
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}
<|endoftext|>


<commit_before>/*
 * Copyright 2016 Frank Wessels <fwessels@xs4all.nl>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package s3git

import (
	"errors"
	"fmt"

	"github.com/s3git/s3git-go/internal/backend"
	"github.com/s3git/s3git-go/internal/cas"
	"github.com/s3git/s3git-go/internal/core"
	"github.com/s3git/s3git-go/internal/kv"

	"bytes"
	"encoding/hex"
	"sync"
)

// Perform a push to the back end for the repository
func (repo Repository) Push(hydrated bool, progress func(maxTicks int64)) error {

	list, err := kv.ListLevel1Prefixes()
	if err != nil {
		return err
	}

	return push(list, hydrated, progress)
}

// Push any new commit objects including all added objects to the back end store
func push(prefixChan <-chan []byte, hydrated bool, progress func(maxTicks int64)) error {

	client, err := backend.GetDefaultClient()
	if err != nil {
		return err
	}

	// Get map of prefixes already in store
	prefixesInBackend, err := listPrefixes(client)
	if err != nil {
		return err
	}

	prefixesToPush := []string{}

	for prefixByte := range prefixChan {

		prefix := hex.EncodeToString(prefixByte)

		_, verified := prefixesInBackend[prefix]

		// We can safely skip in case a prefix object is verified (pushed as last object)
		if !verified {

			prefixesToPush = append(prefixesToPush, prefix)
		}
	}

	if len(prefixesToPush) == 0 {
		return nil
	}

	progress(int64(len(prefixesToPush)))

	for _, prefix := range prefixesToPush {

		// Get prefix object
		po, err := core.GetPrefixObject(prefix)
		if err != nil {
			return err
		}

		// Get commit object
		co, err := core.GetCommitObject(po.S3gitFollowMe)
		if err != nil {
			return err
		}

		// Check if there is a tree object
		if co.S3gitTree != "" {
			// Get tree object
			to, err := core.GetTreeObject(co.S3gitTree)
			if err != nil {
				return err
			}

			// first push all added blobs in this commit ...
			err = pushBlobRange(to.S3gitAdded, nil, hydrated, client)
			if err != nil {
				return err
			}

			// then push tree object
			_, err = pushBlob(co.S3gitTree, nil, client)
			if err != nil {
				return err
			}
		}

		// Check if there is a snapshot object
		if co.S3gitSnapshot != "" {

			err = pushSnapshotWithChildren(co.S3gitSnapshot, client)
			if err != nil {
				return err
			}
		}

		// then push commit object
		_, err = pushBlob(po.S3gitFollowMe, nil, client)
		if err != nil {
			return err
		}

		// ...  finally push prefix object itself
		// (if something goes in chain above, the prefix object will be missing so
		//  will be (attempted to) uploaded again during the next push)
		_, err = pushBlob(prefix, nil, client)
		if err != nil {
			return err
		}

		progress(int64(len(prefixesToPush)))
	}

	return nil
}

func pushSnapshotWithChildren(hash string, client backend.Backend) error {

	// Get snapshot object
	so, err := core.GetSnapshotObject(hash)
	if err != nil {
		return err
	}

	for _, entry := range so.S3gitEntries {
		if entry.IsDirectory() {
			err = pushSnapshotWithChildren(entry.Blob, client)
		}
	}

	// Push snapshot object
	_, err = pushBlob(hash, nil, client)
	if err != nil {
		return err
	}

	return nil
}
// Push a blob to the back end store
func pushBlob(hash string, size *uint64, client backend.Backend) (newlyUploaded bool, err error) {

	// TODO: Consider whether we want to verify again...
	if false {
		verified, err := client.VerifyHash(hash)
		if err != nil {
			return false, err
		} else if verified { // Resource already in back-end
			return false, nil
		}
	}

	// TODO: [perf] for back ends storing whole files: consider multipart upload?
	cr := cas.MakeReader(hash)
	if cr == nil {
		panic(errors.New("Failed to create cas reader"))
	}

	err = client.UploadWithReader(hash, cr)
	if err != nil {
		return false, err
	}

	// Move blob from .stage to .cache directory upon successful upload
	err = cas.MoveBlobToCache(hash)
	if err != nil {
		return false, err
	}

	return true, nil
}

// Push a blob to the back end store in deduplicated format
func PushBlobDeduped(hash string, size *uint64, client backend.Backend) (newlyUploaded bool, err error) {

	// TODO: [perf] for back ends storing chunks: upload chunks in parallel

	hx, err := hex.DecodeString(hash)
	if err != nil {
		return false, err
	}

	// Get hashes for leaves
	leafHashes, _, err := kv.GetLevel1(hx)
	if err != nil {
		return false, err
	} else if len(leafHashes) == 0 {
		return false, errors.New(fmt.Sprintf("Unable to push an empty blob: %s", hash))
	}

	// Iterate over the leaves and push up to remote
	for i := 0; i < len(leafHashes); i += cas.KeySize {

		// TODO: verify whether leaf blob is already in back end, and skip if so
		err := cas.PushLeafBlob(hex.EncodeToString(leafHashes[i:i+cas.KeySize]), client)
		if err != nil {
			return false, err
		}
	}

	// Finally upload root hash
	b := bytes.NewBuffer(leafHashes)
	err = client.UploadWithReader(hash, b)
	if err != nil {
		return false, err
	}

	// Move blob from .stage to .cache directory upon successful upload
	err = cas.MoveBlobToCache(hash)
	if err != nil {
		return false, err
	}

	return true, nil
}

// Push a range of blobs to the back end store in parallel
//
// See https://github.com/adonovan/gopl.io/blob/master/ch8/thumbnail/thumbnail_test.go
//
func pushBlobRange(hashes []string, size *uint64, hydrated bool, client backend.Backend) error {

	var wg sync.WaitGroup
	var msgs = make(chan string)
	var results = make(chan error)

	for i := 0; i < min(len(hashes), 100); i++ {
		wg.Add(1)

		go func() {
			defer wg.Done()
			for hash := range msgs {

				pushHydratedToRemote := hydrated
				if !checkIfLeavesAreEqualSize(hash) {
					pushHydratedToRemote = false // Cannot push hydrated to remote back end when eg rolling hash is used (as we do not know where the boundaries are)
				}
				var err error
				if pushHydratedToRemote {
					_, err = pushBlob(hash, size, client)
				} else {
					_, err = PushBlobDeduped(hash, size, client)
				}
				results <- err
			}
		}()
	}

	go func() {
		for _, hash := range hashes {
			msgs <- hash
		}
		close(msgs)
	}()

	go func() {
		wg.Wait()
		close(results)
	}()

	var err error
	for e := range results {
		if e != nil {
			err = e
		}
	}

	return err
}

func checkIfLeavesAreEqualSize(hash string) bool {
	// TODO: Implement: iterate over all leaves, check whether (except for last node) all sizes are equal
	return true
}

// List prefixes at back end store, doing 16 lists in parallel
func listPrefixes(client backend.Backend) (map[string]bool, error) {

	var wg sync.WaitGroup
	var results = make(chan []string)

	for i := 0x0; i <= 0xf; i++ {
		wg.Add(1)

		go func(i int) {
			defer wg.Done()
			result := make([]string, 0, 1000)

			client.List(fmt.Sprintf("%s%x", core.Prefix(), i), func(key string) {
				result = append(result, key)

				// TODO: WE NEED TO wg.Done() HERE WHEN LAST KEY HAS BEEN RECEIVED
				//       -- SEE /Users/frankw/golang/src/github.com/fwessels/listperf/listperf.go
				// IMPORTANT: WE WILL BE MISSING OBJECTS HERE
			})

			results <- result
		}(i)
	}

	go func() {
		wg.Wait()
		close(results)
	}()

	prefixHash := make(map[string]bool)
	for result := range results {
		for _, r := range result {
			prefixHash[r] = true
		}
	}

	return prefixHash, nil
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}
<commit_msg>Add TODO to push snapshots objects in parallel<commit_after>/*
 * Copyright 2016 Frank Wessels <fwessels@xs4all.nl>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package s3git

import (
	"errors"
	"fmt"

	"github.com/s3git/s3git-go/internal/backend"
	"github.com/s3git/s3git-go/internal/cas"
	"github.com/s3git/s3git-go/internal/core"
	"github.com/s3git/s3git-go/internal/kv"

	"bytes"
	"encoding/hex"
	"sync"
)

// Perform a push to the back end for the repository
func (repo Repository) Push(hydrated bool, progress func(maxTicks int64)) error {

	list, err := kv.ListLevel1Prefixes()
	if err != nil {
		return err
	}

	return push(list, hydrated, progress)
}

// Push any new commit objects including all added objects to the back end store
func push(prefixChan <-chan []byte, hydrated bool, progress func(maxTicks int64)) error {

	client, err := backend.GetDefaultClient()
	if err != nil {
		return err
	}

	// Get map of prefixes already in store
	prefixesInBackend, err := listPrefixes(client)
	if err != nil {
		return err
	}

	prefixesToPush := []string{}

	for prefixByte := range prefixChan {

		prefix := hex.EncodeToString(prefixByte)

		_, verified := prefixesInBackend[prefix]

		// We can safely skip in case a prefix object is verified (pushed as last object)
		if !verified {

			prefixesToPush = append(prefixesToPush, prefix)
		}
	}

	if len(prefixesToPush) == 0 {
		return nil
	}

	progress(int64(len(prefixesToPush)))

	for _, prefix := range prefixesToPush {

		// Get prefix object
		po, err := core.GetPrefixObject(prefix)
		if err != nil {
			return err
		}

		// Get commit object
		co, err := core.GetCommitObject(po.S3gitFollowMe)
		if err != nil {
			return err
		}

		// Check if there is a tree object
		if co.S3gitTree != "" {
			// Get tree object
			to, err := core.GetTreeObject(co.S3gitTree)
			if err != nil {
				return err
			}

			// first push all added blobs in this commit ...
			err = pushBlobRange(to.S3gitAdded, nil, hydrated, client)
			if err != nil {
				return err
			}

			// then push tree object
			_, err = pushBlob(co.S3gitTree, nil, client)
			if err != nil {
				return err
			}
		}

		// Check if there is a snapshot object
		if co.S3gitSnapshot != "" {

			err = pushSnapshotWithChildren(co.S3gitSnapshot, client)
			if err != nil {
				return err
			}
		}

		// then push commit object
		_, err = pushBlob(po.S3gitFollowMe, nil, client)
		if err != nil {
			return err
		}

		// ...  finally push prefix object itself
		// (if something goes in chain above, the prefix object will be missing so
		//  will be (attempted to) uploaded again during the next push)
		_, err = pushBlob(prefix, nil, client)
		if err != nil {
			return err
		}

		progress(int64(len(prefixesToPush)))
	}

	return nil
}

func pushSnapshotWithChildren(hash string, client backend.Backend) error {

	// TODO: [perf] Push snapshot objects in parallel

	// Get snapshot object
	so, err := core.GetSnapshotObject(hash)
	if err != nil {
		return err
	}

	for _, entry := range so.S3gitEntries {
		if entry.IsDirectory() {
			err = pushSnapshotWithChildren(entry.Blob, client)
		}
	}

	// Push snapshot object
	_, err = pushBlob(hash, nil, client)
	if err != nil {
		return err
	}

	return nil
}
// Push a blob to the back end store
func pushBlob(hash string, size *uint64, client backend.Backend) (newlyUploaded bool, err error) {

	// TODO: Consider whether we want to verify again...
	if false {
		verified, err := client.VerifyHash(hash)
		if err != nil {
			return false, err
		} else if verified { // Resource already in back-end
			return false, nil
		}
	}

	// TODO: [perf] for back ends storing whole files: consider multipart upload?
	cr := cas.MakeReader(hash)
	if cr == nil {
		panic(errors.New("Failed to create cas reader"))
	}

	err = client.UploadWithReader(hash, cr)
	if err != nil {
		return false, err
	}

	// Move blob from .stage to .cache directory upon successful upload
	err = cas.MoveBlobToCache(hash)
	if err != nil {
		return false, err
	}

	return true, nil
}

// Push a blob to the back end store in deduplicated format
func PushBlobDeduped(hash string, size *uint64, client backend.Backend) (newlyUploaded bool, err error) {

	// TODO: [perf] for back ends storing chunks: upload chunks in parallel

	hx, err := hex.DecodeString(hash)
	if err != nil {
		return false, err
	}

	// Get hashes for leaves
	leafHashes, _, err := kv.GetLevel1(hx)
	if err != nil {
		return false, err
	} else if len(leafHashes) == 0 {
		return false, errors.New(fmt.Sprintf("Unable to push an empty blob: %s", hash))
	}

	// Iterate over the leaves and push up to remote
	for i := 0; i < len(leafHashes); i += cas.KeySize {

		// TODO: verify whether leaf blob is already in back end, and skip if so
		err := cas.PushLeafBlob(hex.EncodeToString(leafHashes[i:i+cas.KeySize]), client)
		if err != nil {
			return false, err
		}
	}

	// Finally upload root hash
	b := bytes.NewBuffer(leafHashes)
	err = client.UploadWithReader(hash, b)
	if err != nil {
		return false, err
	}

	// Move blob from .stage to .cache directory upon successful upload
	err = cas.MoveBlobToCache(hash)
	if err != nil {
		return false, err
	}

	return true, nil
}

// Push a range of blobs to the back end store in parallel
//
// See https://github.com/adonovan/gopl.io/blob/master/ch8/thumbnail/thumbnail_test.go
//
func pushBlobRange(hashes []string, size *uint64, hydrated bool, client backend.Backend) error {

	var wg sync.WaitGroup
	var msgs = make(chan string)
	var results = make(chan error)

	for i := 0; i < min(len(hashes), 100); i++ {
		wg.Add(1)

		go func() {
			defer wg.Done()
			for hash := range msgs {

				pushHydratedToRemote := hydrated
				if !checkIfLeavesAreEqualSize(hash) {
					pushHydratedToRemote = false // Cannot push hydrated to remote back end when eg rolling hash is used (as we do not know where the boundaries are)
				}
				var err error
				if pushHydratedToRemote {
					_, err = pushBlob(hash, size, client)
				} else {
					_, err = PushBlobDeduped(hash, size, client)
				}
				results <- err
			}
		}()
	}

	go func() {
		for _, hash := range hashes {
			msgs <- hash
		}
		close(msgs)
	}()

	go func() {
		wg.Wait()
		close(results)
	}()

	var err error
	for e := range results {
		if e != nil {
			err = e
		}
	}

	return err
}

func checkIfLeavesAreEqualSize(hash string) bool {
	// TODO: Implement: iterate over all leaves, check whether (except for last node) all sizes are equal
	return true
}

// List prefixes at back end store, doing 16 lists in parallel
func listPrefixes(client backend.Backend) (map[string]bool, error) {

	var wg sync.WaitGroup
	var results = make(chan []string)

	for i := 0x0; i <= 0xf; i++ {
		wg.Add(1)

		go func(i int) {
			defer wg.Done()
			result := make([]string, 0, 1000)

			client.List(fmt.Sprintf("%s%x", core.Prefix(), i), func(key string) {
				result = append(result, key)

				// TODO: WE NEED TO wg.Done() HERE WHEN LAST KEY HAS BEEN RECEIVED
				//       -- SEE /Users/frankw/golang/src/github.com/fwessels/listperf/listperf.go
				// IMPORTANT: WE WILL BE MISSING OBJECTS HERE
			})

			results <- result
		}(i)
	}

	go func() {
		wg.Wait()
		close(results)
	}()

	prefixHash := make(map[string]bool)
	for result := range results {
		for _, r := range result {
			prefixHash[r] = true
		}
	}

	return prefixHash, nil
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}
<|endoftext|>