<commit_before>package abstract_sql

import (
	"context"
	"database/sql"
	"encoding/base64"
	"fmt"
	"strings"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {

	dirStr, dirHash, name := genDirAndName(key)

	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlInsert, dirHash, name, dirStr, value)
	if err != nil {
		if !strings.Contains(strings.ToLower(err.Error()), "duplicate") {
			return fmt.Errorf("kv insert: %s", err)
		}
	}

	// now the insert failed possibly due to duplication constraints
	glog.V(1).Infof("kv insert falls back to update: %s", err)

	res, err = store.getTxOrDB(ctx).ExecContext(ctx, store.SqlUpdate, value, dirHash, name, dirStr)
	if err != nil {
		return fmt.Errorf("kv upsert: %s", err)
	}

	_, err = res.RowsAffected()
	if err != nil {
		return fmt.Errorf("kv upsert no rows affected: %s", err)
	}
	return nil

}

func (store *AbstractSqlStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) {

	dirStr, dirHash, name := genDirAndName(key)
	row := store.getTxOrDB(ctx).QueryRowContext(ctx, store.SqlFind, dirHash, name, dirStr)

	err = row.Scan(&value)

	if err == sql.ErrNoRows {
		return nil, filer.ErrKvNotFound
	}

	if err != nil {
		return nil, fmt.Errorf("kv get: %v", err)
	}

	return
}

func (store *AbstractSqlStore) KvDelete(ctx context.Context, key []byte) (err error) {

	dirStr, dirHash, name := genDirAndName(key)

	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlDelete, dirHash, name, dirStr)
	if err != nil {
		return fmt.Errorf("kv delete: %s", err)
	}

	_, err = res.RowsAffected()
	if err != nil {
		return fmt.Errorf("kv delete no rows affected: %s", err)
	}

	return nil

}

func genDirAndName(key []byte) (dirStr string, dirHash int64, name string) {
	for len(key) < 8 {
		key = append(key, 0)
	}

	dirHash = int64(util.BytesToUint64(key[:8]))
	dirStr = base64.StdEncoding.EncodeToString(key[:8])
	name = base64.StdEncoding.EncodeToString(key[8:])

	return
}
<commit_msg>sql put kv: avoid unnecessary update<commit_after>package abstract_sql

import (
	"context"
	"database/sql"
	"encoding/base64"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
	"strings"
)

func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {

	dirStr, dirHash, name := genDirAndName(key)

	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlInsert, dirHash, name, dirStr, value)
	if err == nil {
		return
	}

	if !strings.Contains(strings.ToLower(err.Error()), "duplicate") {
		return fmt.Errorf("kv insert: %s", err)
	}

	// now the insert failed possibly due to duplication constraints
	glog.V(1).Infof("kv insert falls back to update: %s", err)

	res, err = store.getTxOrDB(ctx).ExecContext(ctx, store.SqlUpdate, value, dirHash, name, dirStr)
	if err != nil {
		return fmt.Errorf("kv upsert: %s", err)
	}

	_, err = res.RowsAffected()
	if err != nil {
		return fmt.Errorf("kv upsert no rows affected: %s", err)
	}
	return nil

}

func (store *AbstractSqlStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) {

	dirStr, dirHash, name := genDirAndName(key)
	row := store.getTxOrDB(ctx).QueryRowContext(ctx, store.SqlFind, dirHash, name, dirStr)

	err = row.Scan(&value)

	if err == sql.ErrNoRows {
		return nil, filer.ErrKvNotFound
	}

	if err != nil {
		return nil, fmt.Errorf("kv get: %v", err)
	}

	return
}

func (store *AbstractSqlStore) KvDelete(ctx context.Context, key []byte) (err error) {

	dirStr, dirHash, name := genDirAndName(key)

	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlDelete, dirHash, name, dirStr)
	if err != nil {
		return fmt.Errorf("kv delete: %s", err)
	}

	_, err = res.RowsAffected()
	if err != nil {
		return fmt.Errorf("kv delete no rows affected: %s", err)
	}

	return nil

}

func genDirAndName(key []byte) (dirStr string, dirHash int64, name string) {
	for len(key) < 8 {
		key = append(key, 0)
	}

	dirHash = int64(util.BytesToUint64(key[:8]))
	dirStr = base64.StdEncoding.EncodeToString(key[:8])
	name = base64.StdEncoding.EncodeToString(key[8:])

	return
}
<|endoftext|>


<commit_before>package controller

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	ct "github.com/flynn/flynn-controller/types"
	"github.com/flynn/flynn-controller/utils"
	"github.com/flynn/go-discoverd"
	"github.com/flynn/go-discoverd/dialer"
	"github.com/flynn/go-flynn/pinned"
	"github.com/flynn/rpcplus"
	"github.com/flynn/strowger/types"
)

func NewClient(uri, key string) (*Client, error) {
	if uri == "" {
		uri = "discoverd+http://flynn-controller"
	}
	u, err := url.Parse(uri)
	if err != nil {
		return nil, err
	}
	c := &Client{
		url:  uri,
		addr: u.Host,
		http: http.DefaultClient,
		key:  key,
	}
	if u.Scheme == "discoverd+http" {
		if err := discoverd.Connect(""); err != nil {
			return nil, err
		}
		dialer := dialer.New(discoverd.DefaultClient, nil)
		c.dial = dialer.Dial
		c.dialClose = dialer
		c.http = &http.Client{Transport: &http.Transport{Dial: c.dial}}
		u.Scheme = "http"
		c.url = u.String()
	}
	return c, nil
}

func NewClientWithPin(uri, key string, pin []byte) (*Client, error) {
	u, err := url.Parse(uri)
	if err != nil {
		return nil, err
	}
	c := &Client{
		dial: (&pinned.Config{Pin: pin}).Dial,
		key:  key,
	}
	if _, port, _ := net.SplitHostPort(u.Host); port == "" {
		u.Host += ":443"
	}
	c.addr = u.Host
	u.Scheme = "http"
	c.url = u.String()
	c.http = &http.Client{Transport: &http.Transport{Dial: c.dial}}
	return c, nil
}

type Client struct {
	url  string
	key  string
	addr string
	http *http.Client

	dial      rpcplus.DialFunc
	dialClose io.Closer
}

func (c *Client) Close() error {
	if c.dialClose != nil {
		c.dialClose.Close()
	}
	return nil
}

var ErrNotFound = errors.New("controller: not found")

func toJSON(v interface{}) (io.Reader, error) {
	data, err := json.Marshal(v)
	return bytes.NewBuffer(data), err
}

func (c *Client) rawReq(method, path string, contentType string, in, out interface{}) (*http.Response, error) {
	var payload io.Reader
	switch v := in.(type) {
	case io.Reader:
		payload = v
	case nil:
	default:
		var err error
		payload, err = toJSON(in)
		if err != nil {
			return nil, err
		}
	}

	req, err := http.NewRequest(method, c.url+path, payload)
	if err != nil {
		return nil, err
	}
	if contentType == "" {
		contentType = "application/json"
	}
	req.Header.Set("Content-Type", contentType)
	req.SetBasicAuth("", c.key)
	res, err := c.http.Do(req)
	if err != nil {
		return nil, err
	}
	if res.StatusCode == 404 {
		res.Body.Close()
		return res, ErrNotFound
	}
	if res.StatusCode != 200 {
		res.Body.Close()
		return res, &url.Error{
			Op:  req.Method,
			URL: req.URL.String(),
			Err: fmt.Errorf("controller: unexpected status %d", res.StatusCode),
		}
	}
	if out != nil {
		defer res.Body.Close()
		return res, json.NewDecoder(res.Body).Decode(out)
	}
	return res, nil
}

func (c *Client) send(method, path string, in, out interface{}) error {
	_, err := c.rawReq(method, path, "", in, out)
	return err
}

func (c *Client) put(path string, in, out interface{}) error {
	return c.send("PUT", path, in, out)
}

func (c *Client) post(path string, in, out interface{}) error {
	return c.send("POST", path, in, out)
}

func (c *Client) get(path string, out interface{}) error {
	_, err := c.rawReq("GET", path, "", nil, out)
	return err
}

func (c *Client) delete(path string) error {
	res, err := c.rawReq("DELETE", path, "", nil, nil)
	if err == nil {
		res.Body.Close()
	}
	return err
}

func (c *Client) StreamFormations(since *time.Time) (<-chan *ct.ExpandedFormation, *error) {
	if since == nil {
		s := time.Unix(0, 0)
		since = &s
	}
	dial := c.dial
	if dial == nil {
		dial = net.Dial
	}
	ch := make(chan *ct.ExpandedFormation)
	conn, err := dial("tcp", c.addr)
	if err != nil {
		close(ch)
		return ch, &err
	}
	header := make(http.Header)
	header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(":"+c.key)))
	client, err := rpcplus.NewHTTPClient(conn, rpcplus.DefaultRPCPath, header)
	if err != nil {
		close(ch)
		return ch, &err
	}
	return ch, &client.StreamGo("Controller.StreamFormations", since, ch).Error
}

func (c *Client) CreateArtifact(artifact *ct.Artifact) error {
	return c.post("/artifacts", artifact, artifact)
}

func (c *Client) CreateRelease(release *ct.Release) error {
	return c.post("/releases", release, release)
}

func (c *Client) CreateApp(app *ct.App) error {
	return c.post("/apps", app, app)
}

func (c *Client) CreateProvider(provider *ct.Provider) error {
	return c.post("/providers", provider, provider)
}

func (c *Client) ProvisionResource(req *ct.ResourceReq) (*ct.Resource, error) {
	if req.ProviderID == "" {
		return nil, errors.New("controller: missing provider id")
	}
	res := &ct.Resource{}
	err := c.post(fmt.Sprintf("/providers/%s/resources", req.ProviderID), req, res)
	return res, err
}

func (c *Client) PutResource(resource *ct.Resource) error {
	if resource.ID == "" || resource.ProviderID == "" {
		return errors.New("controller: missing id and/or provider id")
	}
	return c.put(fmt.Sprintf("/providers/%s/resources/%s", resource.ProviderID, resource.ID), resource, resource)
}

func (c *Client) PutFormation(formation *ct.Formation) error {
	if formation.AppID == "" || formation.ReleaseID == "" {
		return errors.New("controller: missing app id and/or release id")
	}
	return c.put(fmt.Sprintf("/apps/%s/formations/%s", formation.AppID, formation.ReleaseID), formation, formation)
}

func (c *Client) SetAppRelease(appID, releaseID string) error {
	return c.put(fmt.Sprintf("/apps/%s/release", appID), &ct.Release{ID: releaseID}, nil)
}

func (c *Client) GetAppRelease(appID string) (*ct.Release, error) {
	release := &ct.Release{}
	return release, c.get(fmt.Sprintf("/apps/%s/release", appID), release)
}

func (c *Client) CreateRoute(appID string, route *strowger.Route) error {
	return c.post(fmt.Sprintf("/apps/%s/routes", appID), route, route)
}

func (c *Client) GetFormation(appID, releaseID string) (*ct.Formation, error) {
	formation := &ct.Formation{}
	return formation, c.get(fmt.Sprintf("/apps/%s/formations/%s", appID, releaseID), formation)
}

func (c *Client) GetRelease(releaseID string) (*ct.Release, error) {
	release := &ct.Release{}
	return release, c.get(fmt.Sprintf("/releases/%s", releaseID), release)
}

func (c *Client) GetArtifact(artifactID string) (*ct.Artifact, error) {
	artifact := &ct.Artifact{}
	return artifact, c.get(fmt.Sprintf("/artifacts/%s", artifactID), artifact)
}

func (c *Client) GetApp(appID string) (*ct.App, error) {
	app := &ct.App{}
	return app, c.get(fmt.Sprintf("/apps/%s", appID), app)
}

func (c *Client) GetJobLog(appID, jobID string) (io.ReadCloser, error) {
	res, err := c.rawReq("GET", fmt.Sprintf("/apps/%s/jobs/%s/log", appID, jobID), "", nil, nil)
	if err != nil {
		return nil, err
	}
	return res.Body, nil
}

func (c *Client) RunJobAttached(appID string, job *ct.NewJob) (utils.ReadWriteCloser, error) {
	data, err := toJSON(job)
	if err != nil {
		return nil, err
	}
	req, err := http.NewRequest("POST", fmt.Sprintf("%s/apps/%s/jobs", c.url, appID), data)
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Accept", "application/vnd.flynn.attach")
	req.SetBasicAuth("", c.key)
	var dial rpcplus.DialFunc
	if c.dial != nil {
		dial = c.dial
	}
	res, rwc, err := utils.HijackRequest(req, dial)
	if err != nil {
		res.Body.Close()
		return nil, err
	}
	return rwc, nil
}

func (c *Client) RunJobDetached(appID string, req *ct.NewJob) (*ct.Job, error) {
	job := &ct.Job{}
	return job, c.post(fmt.Sprintf("/apps/%s/jobs", appID), req, job)
}

func (c *Client) JobList(appID string) ([]*ct.Job, error) {
	var jobs []*ct.Job
	return jobs, c.get(fmt.Sprintf("/apps/%s/jobs", appID), &jobs)
}

func (c *Client) KeyList() ([]*ct.Key, error) {
	var keys []*ct.Key
	return keys, c.get("/keys", &keys)
}

func (c *Client) CreateKey(pubKey string) (*ct.Key, error) {
	key := &ct.Key{}
	return key, c.post("/keys", &ct.Key{Key: pubKey}, key)
}

func (c *Client) DeleteKey(id string) error {
	return c.delete("/keys/" + strings.Replace(id, ":", "", -1))
}

func (c *Client) ProviderList() ([]*ct.Provider, error) {
	var providers []*ct.Provider
	return providers, c.get("/providers", &providers)
}
<commit_msg>controller: Add client method to get route list from controller<commit_after>package controller

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	ct "github.com/flynn/flynn-controller/types"
	"github.com/flynn/flynn-controller/utils"
	"github.com/flynn/go-discoverd"
	"github.com/flynn/go-discoverd/dialer"
	"github.com/flynn/go-flynn/pinned"
	"github.com/flynn/rpcplus"
	"github.com/flynn/strowger/types"
)

func NewClient(uri, key string) (*Client, error) {
	if uri == "" {
		uri = "discoverd+http://flynn-controller"
	}
	u, err := url.Parse(uri)
	if err != nil {
		return nil, err
	}
	c := &Client{
		url:  uri,
		addr: u.Host,
		http: http.DefaultClient,
		key:  key,
	}
	if u.Scheme == "discoverd+http" {
		if err := discoverd.Connect(""); err != nil {
			return nil, err
		}
		dialer := dialer.New(discoverd.DefaultClient, nil)
		c.dial = dialer.Dial
		c.dialClose = dialer
		c.http = &http.Client{Transport: &http.Transport{Dial: c.dial}}
		u.Scheme = "http"
		c.url = u.String()
	}
	return c, nil
}

func NewClientWithPin(uri, key string, pin []byte) (*Client, error) {
	u, err := url.Parse(uri)
	if err != nil {
		return nil, err
	}
	c := &Client{
		dial: (&pinned.Config{Pin: pin}).Dial,
		key:  key,
	}
	if _, port, _ := net.SplitHostPort(u.Host); port == "" {
		u.Host += ":443"
	}
	c.addr = u.Host
	u.Scheme = "http"
	c.url = u.String()
	c.http = &http.Client{Transport: &http.Transport{Dial: c.dial}}
	return c, nil
}

type Client struct {
	url  string
	key  string
	addr string
	http *http.Client

	dial      rpcplus.DialFunc
	dialClose io.Closer
}

func (c *Client) Close() error {
	if c.dialClose != nil {
		c.dialClose.Close()
	}
	return nil
}

var ErrNotFound = errors.New("controller: not found")

func toJSON(v interface{}) (io.Reader, error) {
	data, err := json.Marshal(v)
	return bytes.NewBuffer(data), err
}

func (c *Client) rawReq(method, path string, contentType string, in, out interface{}) (*http.Response, error) {
	var payload io.Reader
	switch v := in.(type) {
	case io.Reader:
		payload = v
	case nil:
	default:
		var err error
		payload, err = toJSON(in)
		if err != nil {
			return nil, err
		}
	}

	req, err := http.NewRequest(method, c.url+path, payload)
	if err != nil {
		return nil, err
	}
	if contentType == "" {
		contentType = "application/json"
	}
	req.Header.Set("Content-Type", contentType)
	req.SetBasicAuth("", c.key)
	res, err := c.http.Do(req)
	if err != nil {
		return nil, err
	}
	if res.StatusCode == 404 {
		res.Body.Close()
		return res, ErrNotFound
	}
	if res.StatusCode != 200 {
		res.Body.Close()
		return res, &url.Error{
			Op:  req.Method,
			URL: req.URL.String(),
			Err: fmt.Errorf("controller: unexpected status %d", res.StatusCode),
		}
	}
	if out != nil {
		defer res.Body.Close()
		return res, json.NewDecoder(res.Body).Decode(out)
	}
	return res, nil
}

func (c *Client) send(method, path string, in, out interface{}) error {
	_, err := c.rawReq(method, path, "", in, out)
	return err
}

func (c *Client) put(path string, in, out interface{}) error {
	return c.send("PUT", path, in, out)
}

func (c *Client) post(path string, in, out interface{}) error {
	return c.send("POST", path, in, out)
}

func (c *Client) get(path string, out interface{}) error {
	_, err := c.rawReq("GET", path, "", nil, out)
	return err
}

func (c *Client) delete(path string) error {
	res, err := c.rawReq("DELETE", path, "", nil, nil)
	if err == nil {
		res.Body.Close()
	}
	return err
}

func (c *Client) StreamFormations(since *time.Time) (<-chan *ct.ExpandedFormation, *error) {
	if since == nil {
		s := time.Unix(0, 0)
		since = &s
	}
	dial := c.dial
	if dial == nil {
		dial = net.Dial
	}
	ch := make(chan *ct.ExpandedFormation)
	conn, err := dial("tcp", c.addr)
	if err != nil {
		close(ch)
		return ch, &err
	}
	header := make(http.Header)
	header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(":"+c.key)))
	client, err := rpcplus.NewHTTPClient(conn, rpcplus.DefaultRPCPath, header)
	if err != nil {
		close(ch)
		return ch, &err
	}
	return ch, &client.StreamGo("Controller.StreamFormations", since, ch).Error
}

func (c *Client) CreateArtifact(artifact *ct.Artifact) error {
	return c.post("/artifacts", artifact, artifact)
}

func (c *Client) CreateRelease(release *ct.Release) error {
	return c.post("/releases", release, release)
}

func (c *Client) CreateApp(app *ct.App) error {
	return c.post("/apps", app, app)
}

func (c *Client) CreateProvider(provider *ct.Provider) error {
	return c.post("/providers", provider, provider)
}

func (c *Client) ProvisionResource(req *ct.ResourceReq) (*ct.Resource, error) {
	if req.ProviderID == "" {
		return nil, errors.New("controller: missing provider id")
	}
	res := &ct.Resource{}
	err := c.post(fmt.Sprintf("/providers/%s/resources", req.ProviderID), req, res)
	return res, err
}

func (c *Client) PutResource(resource *ct.Resource) error {
	if resource.ID == "" || resource.ProviderID == "" {
		return errors.New("controller: missing id and/or provider id")
	}
	return c.put(fmt.Sprintf("/providers/%s/resources/%s", resource.ProviderID, resource.ID), resource, resource)
}

func (c *Client) PutFormation(formation *ct.Formation) error {
	if formation.AppID == "" || formation.ReleaseID == "" {
		return errors.New("controller: missing app id and/or release id")
	}
	return c.put(fmt.Sprintf("/apps/%s/formations/%s", formation.AppID, formation.ReleaseID), formation, formation)
}

func (c *Client) SetAppRelease(appID, releaseID string) error {
	return c.put(fmt.Sprintf("/apps/%s/release", appID), &ct.Release{ID: releaseID}, nil)
}

func (c *Client) GetAppRelease(appID string) (*ct.Release, error) {
	release := &ct.Release{}
	return release, c.get(fmt.Sprintf("/apps/%s/release", appID), release)
}

func (c *Client) RouteList(appID string) ([]*strowger.Route, error) {
	var routes []*strowger.Route
	return routes, c.get(fmt.Sprintf("/apps/%s/routes", appID), &routes)
}

func (c *Client) CreateRoute(appID string, route *strowger.Route) error {
	return c.post(fmt.Sprintf("/apps/%s/routes", appID), route, route)
}

func (c *Client) GetFormation(appID, releaseID string) (*ct.Formation, error) {
	formation := &ct.Formation{}
	return formation, c.get(fmt.Sprintf("/apps/%s/formations/%s", appID, releaseID), formation)
}

func (c *Client) GetRelease(releaseID string) (*ct.Release, error) {
	release := &ct.Release{}
	return release, c.get(fmt.Sprintf("/releases/%s", releaseID), release)
}

func (c *Client) GetArtifact(artifactID string) (*ct.Artifact, error) {
	artifact := &ct.Artifact{}
	return artifact, c.get(fmt.Sprintf("/artifacts/%s", artifactID), artifact)
}

func (c *Client) GetApp(appID string) (*ct.App, error) {
	app := &ct.App{}
	return app, c.get(fmt.Sprintf("/apps/%s", appID), app)
}

func (c *Client) GetJobLog(appID, jobID string) (io.ReadCloser, error) {
	res, err := c.rawReq("GET", fmt.Sprintf("/apps/%s/jobs/%s/log", appID, jobID), "", nil, nil)
	if err != nil {
		return nil, err
	}
	return res.Body, nil
}

func (c *Client) RunJobAttached(appID string, job *ct.NewJob) (utils.ReadWriteCloser, error) {
	data, err := toJSON(job)
	if err != nil {
		return nil, err
	}
	req, err := http.NewRequest("POST", fmt.Sprintf("%s/apps/%s/jobs", c.url, appID), data)
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Accept", "application/vnd.flynn.attach")
	req.SetBasicAuth("", c.key)
	var dial rpcplus.DialFunc
	if c.dial != nil {
		dial = c.dial
	}
	res, rwc, err := utils.HijackRequest(req, dial)
	if err != nil {
		res.Body.Close()
		return nil, err
	}
	return rwc, nil
}

func (c *Client) RunJobDetached(appID string, req *ct.NewJob) (*ct.Job, error) {
	job := &ct.Job{}
	return job, c.post(fmt.Sprintf("/apps/%s/jobs", appID), req, job)
}

func (c *Client) JobList(appID string) ([]*ct.Job, error) {
	var jobs []*ct.Job
	return jobs, c.get(fmt.Sprintf("/apps/%s/jobs", appID), &jobs)
}

func (c *Client) KeyList() ([]*ct.Key, error) {
	var keys []*ct.Key
	return keys, c.get("/keys", &keys)
}

func (c *Client) CreateKey(pubKey string) (*ct.Key, error) {
	key := &ct.Key{}
	return key, c.post("/keys", &ct.Key{Key: pubKey}, key)
}

func (c *Client) DeleteKey(id string) error {
	return c.delete("/keys/" + strings.Replace(id, ":", "", -1))
}

func (c *Client) ProviderList() ([]*ct.Provider, error) {
	var providers []*ct.Provider
	return providers, c.get("/providers", &providers)
}
<|endoftext|>


<commit_before>package client

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"time"

	"github.com/flynn/go-tuf/data"
	"github.com/flynn/go-tuf/keys"
	"github.com/flynn/go-tuf/signed"
	"github.com/flynn/go-tuf/util"
)

var (
	ErrNotFound       = errors.New("tuf: file not found")
	ErrLatest         = errors.New("tuf: the current version is the latest")
	ErrWrongSize      = errors.New("tuf: unexpected file size")
	ErrNoRootKeys     = errors.New("tuf: no root keys found in local meta store")
	ErrChecksumFailed = errors.New("tuf: checksum failed")
)

// LocalStore is local storage for downloaded top-level metadata.
type LocalStore interface {
	// GetMeta returns top-level metadata from local storage. The keys are
	// in the form `ROLE.json`, with ROLE being a valid top-level role.
	GetMeta() (map[string]json.RawMessage, error)

	// SetMeta persists the given top-level metadata in local storage, the
	// name taking the same format as the keys returned by GetMeta.
	SetMeta(name string, meta json.RawMessage) error
}

// RemoteStore downloads top-level metadata and target files from a remote
// repository.
type RemoteStore interface {
	// Get downloads the given file from remote storage.
	//
	// `file` should be relative to the root of remote repository (e.g.
	// "root.json" or "targets/path/to/target/file.txt")
	Get(file string, size int64) (io.ReadCloser, error)
}

// Client provides methods for fetching updates from a remote repository and
// downloading remote target files.
type Client struct {
	local  LocalStore
	remote RemoteStore

	// The following four fields represent decoded metatdata from
	// local storage
	root      *data.Root
	targets   *data.Targets
	snapshot  *data.Snapshot
	timestamp *data.Timestamp

	// localMeta is the raw metadata from local storage and is used to
	// check whether remote metadata is present locally
	localMeta map[string]json.RawMessage

	// db is a key DB used for verifying metadata
	db *keys.DB
}

func NewClient(local LocalStore, remote RemoteStore) *Client {
	return &Client{
		local:  local,
		remote: remote,
	}
}

// Init initializes a local repository.
//
// The latest root.json is fetched from remote storage, verified using rootKeys
// and threshold, and then saved in local storage. It is expected that rootKeys
// were securely distributed with the software being updated.
func (c *Client) Init(rootKeys []*data.Key, threshold int) error {
	rootJSON, err := c.downloadMeta("root.json", nil)
	if err != nil {
		return err
	}

	c.db = keys.NewDB()
	rootKeyIDs := make([]string, len(rootKeys))
	for i, key := range rootKeys {
		id := key.ID()
		rootKeyIDs[i] = id
		if err := c.db.AddKey(id, key); err != nil {
			return err
		}
	}
	role := &data.Role{Threshold: threshold, KeyIDs: rootKeyIDs}
	if err := c.db.AddRole("root", role); err != nil {
		return err
	}

	if err := c.verifyRoot(rootJSON); err != nil {
		return err
	}

	return c.local.SetMeta("root.json", rootJSON)
}

// Update downloads and verifies remote metadata and returns updated targets.
//
// It performs the update part of "The client application" workflow from
// section 5.1 of the TUF spec:
//
// https://github.com/theupdateframework/tuf/blob/v0.9.9/docs/tuf-spec.txt#L714
func (c *Client) Update() (data.Files, error) {
	// Always start the update using local metadata
	if err := c.getLocalMeta(); err != nil {
		return nil, err
	}

	// TODO: If we get an invalid signature downloading timestamp.json
	//       or snapshot.json, download root.json and start again.

	// Get timestamp.json, extract snapshot.json file meta and save the
	// timestamp.json locally
	timestampJSON, err := c.downloadMeta("timestamp.json", nil)
	if err != nil {
		return nil, err
	}
	snapshotMeta, err := c.decodeTimestamp(timestampJSON)
	if err != nil {
		return nil, err
	}
	if err := c.local.SetMeta("timestamp.json", timestampJSON); err != nil {
		return nil, err
	}

	// Return ErrLatest if we already have the latest snapshot.json
	if c.hasMeta("snapshot.json", snapshotMeta) {
		return nil, ErrLatest
	}

	// Get snapshot.json, then extract root.json and targets.json file meta.
	//
	// The snapshot.json is only saved locally after checking root.json and
	// targets.json so that it will be re-downloaded on subsequent updates
	// if this update fails.
	snapshotJSON, err := c.downloadMeta("snapshot.json", &snapshotMeta)
	if err != nil {
		return nil, err
	}
	rootMeta, targetsMeta, err := c.decodeSnapshot(snapshotJSON)
	if err != nil {
		return nil, err
	}

	// If we don't have the root.json, download it, save it in local
	// storage and restart the update
	if !c.hasMeta("root.json", rootMeta) {
		rootJSON, err := c.downloadMeta("root.json", &rootMeta)
		if err != nil {
			return nil, err
		}
		if err := c.verifyRoot(rootJSON); err != nil {
			return nil, err
		}
		if err := c.local.SetMeta("root.json", rootJSON); err != nil {
			return nil, err
		}
		return c.Update()
	}

	// If we don't have the targets.json, download it, determine updated
	// targets and save targets.json in local storage
	var updatedTargets data.Files
	if !c.hasMeta("targets.json", targetsMeta) {
		targetsJSON, err := c.downloadMeta("targets.json", &targetsMeta)
		if err != nil {
			return nil, err
		}
		updatedTargets, err = c.decodeTargets(targetsJSON)
		if err != nil {
			return nil, err
		}
		if err := c.local.SetMeta("targets.json", targetsJSON); err != nil {
			return nil, err
		}
	}

	// Save the snapshot.json now it has been processed successfully
	if err := c.local.SetMeta("snapshot.json", snapshotJSON); err != nil {
		return nil, err
	}

	return updatedTargets, nil
}

// getLocalMeta decodes and verifies metadata from local storage.
//
// The verification of local files is purely for consistency, if an attacker
// has compromised the local storage, there is no guarantee it can be trusted.
func (c *Client) getLocalMeta() error {
	meta, err := c.local.GetMeta()
	if err != nil {
		return err
	}

	if rootJSON, ok := meta["root.json"]; ok {
		// unmarshal root.json without verifying as we need the root
		// keys first
		s := &data.Signed{}
		if err := json.Unmarshal(rootJSON, s); err != nil {
			return err
		}
		root := &data.Root{}
		if err := json.Unmarshal(s.Signed, root); err != nil {
			return err
		}
		db := keys.NewDB()
		for id, k := range root.Keys {
			if err := db.AddKey(id, k); err != nil {
				return err
			}
		}
		for name, role := range root.Roles {
			if err := db.AddRole(name, role); err != nil {
				return err
			}
		}
		if err := signed.Verify(s, "root", 0, db); err != nil {
			return err
		}
		c.root = root
		c.db = db
	} else {
		return ErrNoRootKeys
	}

	if snapshotJSON, ok := meta["snapshot.json"]; ok {
		snapshot := &data.Snapshot{}
		if err := signed.Unmarshal(snapshotJSON, snapshot, "snapshot", 0, c.db); err != nil {
			return err
		}
		c.snapshot = snapshot
	}

	if targetsJSON, ok := meta["targets.json"]; ok {
		targets := &data.Targets{}
		if err := signed.Unmarshal(targetsJSON, targets, "targets", 0, c.db); err != nil {
			return err
		}
		c.targets = targets
	}

	if timestampJSON, ok := meta["timestamp.json"]; ok {
		timestamp := &data.Timestamp{}
		if err := signed.Unmarshal(timestampJSON, timestamp, "timestamp", 0, c.db); err != nil {
			return err
		}
		c.timestamp = timestamp
	}

	c.localMeta = meta
	return nil
}

// maxMetaSize is the maximum number of bytes that will be downloaded when
// getting remote metadata without knowing it's length.
const maxMetaSize = 50 * 1024

// downloadMeta downloads top-level metadata from remote storage and verifies
// it using the given file metadata.
func (c *Client) downloadMeta(name string, m *data.FileMeta) ([]byte, error) {
	var size int64
	if m != nil {
		size = m.Length
	}
	r, err := c.remote.Get(name, size)
	if err != nil {
		if err == ErrNotFound {
			return nil, ErrMissingRemoteMetadata{name}
		}
		return nil, err
	}
	defer r.Close()

	// if m is nil (e.g. when downloading timestamp.json, which has unknown
	// size), just read the stream (up to a sane maximum) and return it
	if m == nil {
		b, err := ioutil.ReadAll(io.LimitReader(r, maxMetaSize))
		if err != nil {
			return nil, err
		}
		return b, nil
	}

	// read exactly m.Length bytes from the stream
	buf := make([]byte, m.Length)
	if _, err := io.ReadFull(r, buf); err != nil {
		return nil, err
	}

	// verify buf matches given metadata
	meta, err := util.GenerateFileMeta(bytes.NewReader(buf))
	if err != nil {
		return nil, err
	}
	if err := util.FileMetaEqual(meta, *m); err != nil {
		return nil, ErrDownloadFailed{name, err}
	}
	return buf, nil
}

// verifyRoot verifies root metadata.
func (c *Client) verifyRoot(b json.RawMessage) error {
	var minVer int
	if c.root != nil {
		minVer = c.root.Version
	}
	s := &data.Signed{}
	if err := json.Unmarshal(b, s); err != nil {
		return err
	}
	return signed.Verify(s, "root", minVer, c.db)
}

// decodeSnapshot decodes and verifies snapshot metadata, and returns the new
// root and targets file meta.
func (c *Client) decodeSnapshot(b json.RawMessage) (data.FileMeta, data.FileMeta, error) {
	var minVer int
	if c.snapshot != nil {
		minVer = c.snapshot.Version
	}
	snapshot := &data.Snapshot{}
	if err := signed.Unmarshal(b, snapshot, "snapshot", minVer, c.db); err != nil {
		return data.FileMeta{}, data.FileMeta{}, err
	}
	return snapshot.Meta["root.json"], snapshot.Meta["targets.json"], nil
}

// decodeTargets decodes and verifies targets metadata, and returns updated
// targets.
func (c *Client) decodeTargets(b json.RawMessage) (data.Files, error) {
	var minVer int
	var currTargets data.Files
	if c.targets != nil {
		minVer = c.targets.Version
		currTargets = c.targets.Targets
	}
	targets := &data.Targets{}
	if err := signed.Unmarshal(b, targets, "targets", minVer, c.db); err != nil {
		return nil, err
	}
	updatedTargets := make(data.Files)
	for path, meta := range targets.Targets {
		if curr, ok := currTargets[path]; ok {
			if err := util.FileMetaEqual(curr, meta); err == nil {
				continue
			}
		}
		updatedTargets[path] = meta
	}
	return updatedTargets, nil
}

// decodeTimestamp decodes and verifies timestamp metadata, and returns the
// new snapshot file meta.
func (c *Client) decodeTimestamp(b json.RawMessage) (data.FileMeta, error) {
	var minVer int
	if c.timestamp != nil {
		minVer = c.timestamp.Version
	}
	timestamp := &data.Timestamp{}
	if err := signed.Unmarshal(b, timestamp, "timestamp", minVer, c.db); err != nil {
		return data.FileMeta{}, err
	}
	return timestamp.Meta["snapshot.json"], nil
}

// hasMeta checks whether local metadata has the given file meta
func (c *Client) hasMeta(name string, m data.FileMeta) bool {
	b, ok := c.localMeta[name]
	if !ok {
		return false
	}
	meta, err := util.GenerateFileMeta(bytes.NewReader(b))
	if err != nil {
		return false
	}
	err = util.FileMetaEqual(meta, m)
	return err == nil
}

func (c *Client) Expires() time.Time {
	return time.Time{}
}

func (c *Client) Version() int {
	return 0
}

func (c *Client) Files() data.Files {
	return nil
}

type Destination interface {
	io.Writer
	Size() (int, error)
	Delete() error
}

func (c *Client) Download(name string, dest Destination) error {
	/*
		The software update system instructs TUF to download a specific target file.

		TUF downloads and verifies the file and then makes the file available to the
		software update system.
	*/
	return nil
}
<commit_msg>Only store local versions and targets in the Client struct<commit_after>package client

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"time"

	"github.com/flynn/go-tuf/data"
	"github.com/flynn/go-tuf/keys"
	"github.com/flynn/go-tuf/signed"
	"github.com/flynn/go-tuf/util"
)

var (
	ErrNotFound       = errors.New("tuf: file not found")
	ErrLatest         = errors.New("tuf: the current version is the latest")
	ErrWrongSize      = errors.New("tuf: unexpected file size")
	ErrNoRootKeys     = errors.New("tuf: no root keys found in local meta store")
	ErrChecksumFailed = errors.New("tuf: checksum failed")
)

// LocalStore is local storage for downloaded top-level metadata.
type LocalStore interface {
	// GetMeta returns top-level metadata from local storage. The keys are
	// in the form `ROLE.json`, with ROLE being a valid top-level role.
	GetMeta() (map[string]json.RawMessage, error)

	// SetMeta persists the given top-level metadata in local storage, the
	// name taking the same format as the keys returned by GetMeta.
	SetMeta(name string, meta json.RawMessage) error
}

// RemoteStore downloads top-level metadata and target files from a remote
// repository.
type RemoteStore interface {
	// Get downloads the given file from remote storage.
	//
	// `file` should be relative to the root of remote repository (e.g.
	// "root.json" or "targets/path/to/target/file.txt")
	Get(file string, size int64) (io.ReadCloser, error)
}

// Client provides methods for fetching updates from a remote repository and
// downloading remote target files.
type Client struct {
	local  LocalStore
	remote RemoteStore

	// The following four fields represent the versions of metatdata from
	// local storage
	localRootVer      int
	localTargetsVer   int
	localSnapshotVer  int
	localTimestampVer int

	// localTargets is the list of targets from local storage
	localTargets data.Files

	// localMeta is the raw metadata from local storage and is used to
	// check whether remote metadata is present locally
	localMeta map[string]json.RawMessage

	// db is a key DB used for verifying metadata
	db *keys.DB
}

func NewClient(local LocalStore, remote RemoteStore) *Client {
	return &Client{
		local:  local,
		remote: remote,
	}
}

// Init initializes a local repository.
//
// The latest root.json is fetched from remote storage, verified using rootKeys
// and threshold, and then saved in local storage. It is expected that rootKeys
// were securely distributed with the software being updated.
func (c *Client) Init(rootKeys []*data.Key, threshold int) error {
	rootJSON, err := c.downloadMeta("root.json", nil)
	if err != nil {
		return err
	}

	c.db = keys.NewDB()
	rootKeyIDs := make([]string, len(rootKeys))
	for i, key := range rootKeys {
		id := key.ID()
		rootKeyIDs[i] = id
		if err := c.db.AddKey(id, key); err != nil {
			return err
		}
	}
	role := &data.Role{Threshold: threshold, KeyIDs: rootKeyIDs}
	if err := c.db.AddRole("root", role); err != nil {
		return err
	}

	if err := c.verifyRoot(rootJSON); err != nil {
		return err
	}

	return c.local.SetMeta("root.json", rootJSON)
}

// Update downloads and verifies remote metadata and returns updated targets.
//
// It performs the update part of "The client application" workflow from
// section 5.1 of the TUF spec:
//
// https://github.com/theupdateframework/tuf/blob/v0.9.9/docs/tuf-spec.txt#L714
func (c *Client) Update() (data.Files, error) {
	// Always start the update using local metadata
	if err := c.getLocalMeta(); err != nil {
		return nil, err
	}

	// TODO: If we get an invalid signature downloading timestamp.json
	//       or snapshot.json, download root.json and start again.

	// Get timestamp.json, extract snapshot.json file meta and save the
	// timestamp.json locally
	timestampJSON, err := c.downloadMeta("timestamp.json", nil)
	if err != nil {
		return nil, err
	}
	snapshotMeta, err := c.decodeTimestamp(timestampJSON)
	if err != nil {
		return nil, err
	}
	if err := c.local.SetMeta("timestamp.json", timestampJSON); err != nil {
		return nil, err
	}

	// Return ErrLatest if we already have the latest snapshot.json
	if c.hasMeta("snapshot.json", snapshotMeta) {
		return nil, ErrLatest
	}

	// Get snapshot.json, then extract root.json and targets.json file meta.
	//
	// The snapshot.json is only saved locally after checking root.json and
	// targets.json so that it will be re-downloaded on subsequent updates
	// if this update fails.
	snapshotJSON, err := c.downloadMeta("snapshot.json", &snapshotMeta)
	if err != nil {
		return nil, err
	}
	rootMeta, targetsMeta, err := c.decodeSnapshot(snapshotJSON)
	if err != nil {
		return nil, err
	}

	// If we don't have the root.json, download it, save it in local
	// storage and restart the update
	if !c.hasMeta("root.json", rootMeta) {
		rootJSON, err := c.downloadMeta("root.json", &rootMeta)
		if err != nil {
			return nil, err
		}
		if err := c.verifyRoot(rootJSON); err != nil {
			return nil, err
		}
		if err := c.local.SetMeta("root.json", rootJSON); err != nil {
			return nil, err
		}
		return c.Update()
	}

	// If we don't have the targets.json, download it, determine updated
	// targets and save targets.json in local storage
	var updatedTargets data.Files
	if !c.hasMeta("targets.json", targetsMeta) {
		targetsJSON, err := c.downloadMeta("targets.json", &targetsMeta)
		if err != nil {
			return nil, err
		}
		updatedTargets, err = c.decodeTargets(targetsJSON)
		if err != nil {
			return nil, err
		}
		if err := c.local.SetMeta("targets.json", targetsJSON); err != nil {
			return nil, err
		}
	}

	// Save the snapshot.json now it has been processed successfully
	if err := c.local.SetMeta("snapshot.json", snapshotJSON); err != nil {
		return nil, err
	}

	return updatedTargets, nil
}

// getLocalMeta decodes and verifies metadata from local storage.
//
// The verification of local files is purely for consistency, if an attacker
// has compromised the local storage, there is no guarantee it can be trusted.
func (c *Client) getLocalMeta() error {
	meta, err := c.local.GetMeta()
	if err != nil {
		return err
	}

	if rootJSON, ok := meta["root.json"]; ok {
		// unmarshal root.json without verifying as we need the root
		// keys first
		s := &data.Signed{}
		if err := json.Unmarshal(rootJSON, s); err != nil {
			return err
		}
		root := &data.Root{}
		if err := json.Unmarshal(s.Signed, root); err != nil {
			return err
		}
		db := keys.NewDB()
		for id, k := range root.Keys {
			if err := db.AddKey(id, k); err != nil {
				return err
			}
		}
		for name, role := range root.Roles {
			if err := db.AddRole(name, role); err != nil {
				return err
			}
		}
		if err := signed.Verify(s, "root", 0, db); err != nil {
			return err
		}
		c.localRootVer = root.Version
		c.db = db
	} else {
		return ErrNoRootKeys
	}

	if snapshotJSON, ok := meta["snapshot.json"]; ok {
		snapshot := &data.Snapshot{}
		if err := signed.Unmarshal(snapshotJSON, snapshot, "snapshot", 0, c.db); err != nil {
			return err
		}
		c.localSnapshotVer = snapshot.Version
	}

	if targetsJSON, ok := meta["targets.json"]; ok {
		targets := &data.Targets{}
		if err := signed.Unmarshal(targetsJSON, targets, "targets", 0, c.db); err != nil {
			return err
		}
		c.localTargetsVer = targets.Version
		c.localTargets = targets.Targets
	}

	if timestampJSON, ok := meta["timestamp.json"]; ok {
		timestamp := &data.Timestamp{}
		if err := signed.Unmarshal(timestampJSON, timestamp, "timestamp", 0, c.db); err != nil {
			return err
		}
		c.localTimestampVer = timestamp.Version
	}

	c.localMeta = meta
	return nil
}

// maxMetaSize is the maximum number of bytes that will be downloaded when
// getting remote metadata without knowing it's length.
const maxMetaSize = 50 * 1024

// downloadMeta downloads top-level metadata from remote storage and verifies
// it using the given file metadata.
func (c *Client) downloadMeta(name string, m *data.FileMeta) ([]byte, error) {
	var size int64
	if m != nil {
		size = m.Length
	}
	r, err := c.remote.Get(name, size)
	if err != nil {
		if err == ErrNotFound {
			return nil, ErrMissingRemoteMetadata{name}
		}
		return nil, err
	}
	defer r.Close()

	// if m is nil (e.g. when downloading timestamp.json, which has unknown
	// size), just read the stream (up to a sane maximum) and return it
	if m == nil {
		b, err := ioutil.ReadAll(io.LimitReader(r, maxMetaSize))
		if err != nil {
			return nil, err
		}
		return b, nil
	}

	// read exactly m.Length bytes from the stream
	buf := make([]byte, m.Length)
	if _, err := io.ReadFull(r, buf); err != nil {
		return nil, err
	}

	// verify buf matches given metadata
	meta, err := util.GenerateFileMeta(bytes.NewReader(buf))
	if err != nil {
		return nil, err
	}
	if err := util.FileMetaEqual(meta, *m); err != nil {
		return nil, ErrDownloadFailed{name, err}
	}
	return buf, nil
}

// verifyRoot verifies root metadata.
func (c *Client) verifyRoot(b json.RawMessage) error {
	s := &data.Signed{}
	if err := json.Unmarshal(b, s); err != nil {
		return err
	}
	return signed.Verify(s, "root", c.localRootVer, c.db)
}

// decodeSnapshot decodes and verifies snapshot metadata, and returns the new
// root and targets file meta.
func (c *Client) decodeSnapshot(b json.RawMessage) (data.FileMeta, data.FileMeta, error) {
	snapshot := &data.Snapshot{}
	if err := signed.Unmarshal(b, snapshot, "snapshot", c.localSnapshotVer, c.db); err != nil {
		return data.FileMeta{}, data.FileMeta{}, err
	}
	return snapshot.Meta["root.json"], snapshot.Meta["targets.json"], nil
}

// decodeTargets decodes and verifies targets metadata, and returns updated
// targets.
func (c *Client) decodeTargets(b json.RawMessage) (data.Files, error) {
	targets := &data.Targets{}
	if err := signed.Unmarshal(b, targets, "targets", c.localTargetsVer, c.db); err != nil {
		return nil, err
	}
	updatedTargets := make(data.Files)
	for path, meta := range targets.Targets {
		if local, ok := c.localTargets[path]; ok {
			if err := util.FileMetaEqual(local, meta); err == nil {
				continue
			}
		}
		updatedTargets[path] = meta
	}
	return updatedTargets, nil
}

// decodeTimestamp decodes and verifies timestamp metadata, and returns the
// new snapshot file meta.
func (c *Client) decodeTimestamp(b json.RawMessage) (data.FileMeta, error) {
	timestamp := &data.Timestamp{}
	if err := signed.Unmarshal(b, timestamp, "timestamp", c.localTimestampVer, c.db); err != nil {
		return data.FileMeta{}, err
	}
	return timestamp.Meta["snapshot.json"], nil
}

// hasMeta checks whether local metadata has the given file meta
func (c *Client) hasMeta(name string, m data.FileMeta) bool {
	b, ok := c.localMeta[name]
	if !ok {
		return false
	}
	meta, err := util.GenerateFileMeta(bytes.NewReader(b))
	if err != nil {
		return false
	}
	err = util.FileMetaEqual(meta, m)
	return err == nil
}

func (c *Client) Expires() time.Time {
	return time.Time{}
}

func (c *Client) Version() int {
	return 0
}

func (c *Client) Files() data.Files {
	return nil
}

type Destination interface {
	io.Writer
	Size() (int, error)
	Delete() error
}

func (c *Client) Download(name string, dest Destination) error {
	/*
		The software update system instructs TUF to download a specific target file.

		TUF downloads and verifies the file and then makes the file available to the
		software update system.
	*/
	return nil
}
<|endoftext|>


<commit_before>package brats_test

import (
	"strings"

	"github.com/cloudfoundry/libbuildpack/bratshelper"
	"github.com/cloudfoundry/libbuildpack/cutlass"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Nodejs buildpack", func() {
	bratshelper.UnbuiltBuildpack("node", CopyBrats)
	bratshelper.DeployingAnAppWithAnUpdatedVersionOfTheSameBuildpack(CopyBrats)
	bratshelper.StagingWithBuildpackThatSetsEOL("node", CopyBrats)
	bratshelper.StagingWithCustomBuildpackWithCredentialsInDependencies(CopyBrats)
	bratshelper.DeployAppWithExecutableProfileScript("node", CopyBrats)
	bratshelper.DeployAnAppWithSensitiveEnvironmentVariables(CopyBrats)
	bratshelper.ForAllSupportedVersions("node", CopyBrats, func(nodeVersion string, app *cutlass.App) {
		PushApp(app)

		By("runs a simple webserver", func() {
			Expect(app.GetBody("/")).To(ContainSubstring("Hello World!"))
		})
		By("supports bcrypt", func() {
			Expect(app.GetBody("/bcrypt")).To(ContainSubstring("Hello Bcrypt!"))
		})
		By("supports bson-ext", func() {
			if strings.HasPrefix(nodeVersion, "12") {
				// TODO: Bson-ext doesn't work with NodeJS 12 yet. When it does work (this fails), this can be removed.
				Expect(app.GetBody("/bson-ext")).To(ContainSubstring("502 Bad Gateway: Registered endpoint failed to handle the request."))
			} else {
				Expect(app.GetBody("/bson-ext")).To(ContainSubstring("Hello Bson-ext!"))
			}
		})
		By("installs the correct version", func() {
			Expect(app.Stdout.String()).To(ContainSubstring("Installing node " + nodeVersion))
		})
	})
})
<commit_msg>Remove bson-ext exclusion from brats<commit_after>package brats_test

import (
	"github.com/cloudfoundry/libbuildpack/bratshelper"
	"github.com/cloudfoundry/libbuildpack/cutlass"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Nodejs buildpack", func() {
	bratshelper.UnbuiltBuildpack("node", CopyBrats)
	bratshelper.DeployingAnAppWithAnUpdatedVersionOfTheSameBuildpack(CopyBrats)
	bratshelper.StagingWithBuildpackThatSetsEOL("node", CopyBrats)
	bratshelper.StagingWithCustomBuildpackWithCredentialsInDependencies(CopyBrats)
	bratshelper.DeployAppWithExecutableProfileScript("node", CopyBrats)
	bratshelper.DeployAnAppWithSensitiveEnvironmentVariables(CopyBrats)
	bratshelper.ForAllSupportedVersions("node", CopyBrats, func(nodeVersion string, app *cutlass.App) {
		PushApp(app)

		By("runs a simple webserver", func() {
			Expect(app.GetBody("/")).To(ContainSubstring("Hello World!"))
		})
		By("supports bcrypt", func() {
			Expect(app.GetBody("/bcrypt")).To(ContainSubstring("Hello Bcrypt!"))
		})
		By("supports bson-ext", func() {
			Expect(app.GetBody("/bson-ext")).To(ContainSubstring("Hello Bson-ext!"))
		})
		By("installs the correct version", func() {
			Expect(app.Stdout.String()).To(ContainSubstring("Installing node " + nodeVersion))
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2016 Koichi Shiraishi. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package commands

import (
	"fmt"
	"go/build"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"time"

	"nvim-go/config"
	"nvim-go/nvim"
	"nvim-go/nvim/profile"
	"nvim-go/nvim/quickfix"

	"github.com/neovim-go/vim"
	"github.com/pkg/errors"
	"golang.org/x/tools/refactor/rename"
)

const pkgRename = "GoRename"

type cmdRenameEval struct {
	Cwd        string `msgpack:",array"`
	File       string
	RenameFrom string
}

func (c *Commands) cmdRename(args []string, bang bool, eval *cmdRenameEval) {
	go c.Rename(args, bang, eval)
}

// Rename rename the current cursor word use golang.org/x/tools/refactor/rename.
func (c *Commands) Rename(args []string, bang bool, eval *cmdRenameEval) error {
	defer profile.Start(time.Now(), "GoRename")
	dir := filepath.Dir(eval.File)
	defer c.ctxt.SetContext(dir)()

	var (
		b vim.Buffer
		w vim.Window
	)
	if c.p == nil {
		c.p = c.v.NewPipeline()
	}
	c.p.CurrentBuffer(&b)
	c.p.CurrentWindow(&w)
	if err := c.p.Wait(); err != nil {
		err = errors.Wrap(err, pkgRename)
		return nvim.ErrorWrap(c.v, err)
	}

	offset, err := nvim.ByteOffset(c.v, b, w)
	if err != nil {
		err = errors.Wrap(err, pkgRename)
		return nvim.ErrorWrap(c.v, err)
	}
	pos := fmt.Sprintf("%s:#%d", eval.File, offset)

	var renameTo string
	if len(args) > 0 {
		renameTo = args[0]
	} else {
		askMessage := fmt.Sprintf("%s: Rename '%s' to: ", pkgRename, eval.RenameFrom)
		var toResult interface{}
		if config.RenamePrefill {
			err := c.v.Call("input", &toResult, askMessage, eval.RenameFrom)
			if err != nil {
				return nvim.EchohlErr(c.v, pkgRename, "Keyboard interrupt")
			}
		} else {
			err := c.v.Call("input", &toResult, askMessage)
			if err != nil {
				return nvim.EchohlErr(c.v, pkgRename, "Keyboard interrupt")
			}
		}
		if toResult.(string) == "" {
			return nvim.EchohlErr(c.v, pkgRename, "Not enough arguments for rename destination name")
		}
		renameTo = fmt.Sprintf("%s", toResult)
	}

	c.v.Command(fmt.Sprintf("echo '%s: Renaming ' | echohl Identifier | echon '%s' | echohl None | echon ' to ' | echohl Identifier | echon '%s' | echohl None | echon ' ...'", pkgRename, eval.RenameFrom, renameTo))

	if bang {
		rename.Force = true
	}

	// TODO(zchee): More elegant way
	saveStdout, saveStderr := os.Stdout, os.Stderr
	os.Stderr = os.Stdout
	read, write, _ := os.Pipe()
	os.Stdout, os.Stderr = write, write
	defer func() {
		os.Stderr = saveStdout
		os.Stderr = saveStderr
	}()

	if err := rename.Main(&build.Default, pos, "", renameTo); err != nil {
		write.Close()
		er, _ := ioutil.ReadAll(read)
		log.Printf("er: %+v\n", string(er))
		go func() {
			loclist, _ := quickfix.ParseError(er, eval.Cwd, &c.ctxt.Build)
			quickfix.SetLoclist(c.v, loclist)
			quickfix.OpenLoclist(c.v, w, loclist, true)
		}()

		err = errors.Wrap(err, pkgRename)
		return nvim.ErrorWrap(c.v, err)
	}

	write.Close()
	out, _ := ioutil.ReadAll(read)
	defer nvim.EchoSuccess(c.v, pkgRename, fmt.Sprintf("%s", out))

	// TODO(zchee): 'edit' command is ugly.
	// Should create tempfile and use SetBufferLines.
	return c.v.Command("silent edit")
}
<commit_msg>cmds/rename: fix save stdio to more better way and add error handling<commit_after>// Copyright 2016 Koichi Shiraishi. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package commands

import (
	"fmt"
	"go/build"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"time"

	"nvim-go/config"
	"nvim-go/nvim"
	"nvim-go/nvim/profile"
	"nvim-go/nvim/quickfix"

	"github.com/neovim-go/vim"
	"github.com/pkg/errors"
	"golang.org/x/tools/refactor/rename"
)

const pkgRename = "GoRename"

type cmdRenameEval struct {
	Cwd        string `msgpack:",array"`
	File       string
	RenameFrom string
}

func (c *Commands) cmdRename(args []string, bang bool, eval *cmdRenameEval) {
	go c.Rename(args, bang, eval)
}

// Rename rename the current cursor word use golang.org/x/tools/refactor/rename.
func (c *Commands) Rename(args []string, bang bool, eval *cmdRenameEval) error {
	defer profile.Start(time.Now(), "GoRename")
	dir := filepath.Dir(eval.File)
	defer c.ctxt.SetContext(dir)()

	var (
		b vim.Buffer
		w vim.Window
	)
	if c.p == nil {
		c.p = c.v.NewPipeline()
	}
	c.p.CurrentBuffer(&b)
	c.p.CurrentWindow(&w)
	if err := c.p.Wait(); err != nil {
		err = errors.Wrap(err, pkgRename)
		return nvim.ErrorWrap(c.v, err)
	}

	offset, err := nvim.ByteOffset(c.v, b, w)
	if err != nil {
		err = errors.Wrap(err, pkgRename)
		return nvim.ErrorWrap(c.v, err)
	}
	pos := fmt.Sprintf("%s:#%d", eval.File, offset)

	var renameTo string
	if len(args) > 0 {
		renameTo = args[0]
	} else {
		askMessage := fmt.Sprintf("%s: Rename '%s' to: ", pkgRename, eval.RenameFrom)
		var toResult interface{}
		if config.RenamePrefill {
			err := c.v.Call("input", &toResult, askMessage, eval.RenameFrom)
			if err != nil {
				return nvim.EchohlErr(c.v, pkgRename, "Keyboard interrupt")
			}
		} else {
			err := c.v.Call("input", &toResult, askMessage)
			if err != nil {
				return nvim.EchohlErr(c.v, pkgRename, "Keyboard interrupt")
			}
		}
		if toResult.(string) == "" {
			return nvim.EchohlErr(c.v, pkgRename, "Not enough arguments for rename destination name")
		}
		renameTo = fmt.Sprintf("%s", toResult)
	}

	c.v.Command(fmt.Sprintf("echo '%s: Renaming ' | echohl Identifier | echon '%s' | echohl None | echon ' to ' | echohl Identifier | echon '%s' | echohl None | echon ' ...'", pkgRename, eval.RenameFrom, renameTo))

	if bang {
		rename.Force = true
	}

	// TODO(zchee): More elegant way
	// save original stdout and stderr
	saveStdout, saveStderr := os.Stdout, os.Stderr
	read, write, _ := os.Pipe()
	// migrate stderr and stdout
	os.Stderr = os.Stdout
	os.Stderr = write
	defer func() {
		os.Stderr = saveStdout
		os.Stderr = saveStderr
	}()

	if err := rename.Main(&build.Default, pos, "", renameTo); err != nil {
		write.Close()
		renameErr, err := ioutil.ReadAll(read)
		if err != nil {
			err = errors.Wrap(err, pkgRename)
			return nvim.ErrorWrap(c.v, err)
		}

		log.Printf("er: %+v\n", string(renameErr))
		go func() {
			loclist, _ := quickfix.ParseError(renameErr, eval.Cwd, &c.ctxt.Build)
			quickfix.SetLoclist(c.v, loclist)
			quickfix.OpenLoclist(c.v, w, loclist, true)
		}()

		err = errors.Wrap(err, pkgRename)
		return nvim.ErrorWrap(c.v, err)
	}

	write.Close()
	out, _ := ioutil.ReadAll(read)
	defer nvim.EchoSuccess(c.v, pkgRename, fmt.Sprintf("%s", out))

	// TODO(zchee): 'edit' command is ugly.
	// Should create tempfile and use SetBufferLines.
	return c.v.Command("silent edit")
}
<|endoftext|>


<commit_before>package commands

import (
	"bytes"
	"fmt"
	"go/build"
	"io"
	"nvim-go/gb"
	"nvim-go/nvim"
	"os"
	"strings"

	"golang.org/x/tools/refactor/rename"

	"github.com/garyburd/neovim-go/vim"
	"github.com/garyburd/neovim-go/vim/plugin"
)

func init() {
	plugin.HandleCommand("Gorename",
		&plugin.CommandOptions{
			NArgs: "?", Eval: "[expand('%:p:h'), expand('%:p'), line2byte(line('.'))+(col('.')-2)]"},
		cmdRename)
}

var (
	renamePrefill  = "go#rename#prefill"
	vRenamePrefill interface{}
)

type onRenameEval struct {
	Dir    string `msgpack:",array"`
	File   string
	Offset int
}

func cmdRename(v *vim.Vim, args []string, eval *onRenameEval) error {
	go Rename(v, args, eval)
	return nil
}

// Rename rename the current cursor word use golang.org/x/tools/refactor/rename.
func Rename(v *vim.Vim, args []string, eval *onRenameEval) error {
	defer gb.WithGoBuildForPath(eval.Dir)()

	v.Var(renamePrefill, &vRenamePrefill)
	from, err := v.CommandOutput(fmt.Sprintf("silent! echo expand('<cword>')"))
	if err != nil {
		nvim.Echomsg(v, "%s", err)
	}

	var (
		b vim.Buffer
		w vim.Window
	)
	p := v.NewPipeline()
	p.CurrentBuffer(&b)
	p.CurrentWindow(&w)
	if err := p.Wait(); err != nil {
		return err
	}

	offset := fmt.Sprintf("%s:#%d", eval.File, eval.Offset)
	fmt.Printf(offset)

	askMessage := fmt.Sprintf("%s: Rename '%s' to: ", "nvim-go", from[1:])
	var to interface{}
	if vRenamePrefill.(int64) == int64(1) {
		p.Call("input", &to, askMessage, from[1:])
		if err := p.Wait(); err != nil {
			return nvim.Echomsg(v, "%s", err)
		}
	} else {
		p.Call("input", &to, askMessage)
		if err := p.Wait(); err != nil {
			return nvim.Echomsg(v, "%s", err)
		}
	}

	// out, err := ParseStdout(fmt.Sprintln(eval.Offset))
	// if err != nil {
	// 	return nvim.Echomsg(v, "%s", err)
	// }

	if err := rename.Main(&build.Default, offset, "", fmt.Sprint(to)); err != nil {
		if err != rename.ConflictError {
			nvim.Echomsg(v, "%s", err)
		}
	}
	p.Command("edit!")
	// nvim.Echohl(v, nvim.PackageName, "Function", out)

	return p.Wait()
}

func ParseStdout(trim string) (string, error) {
	stdout := os.Stdout
	r, w, err := os.Pipe()
	if err != nil {
		return "", err
	}
	os.Stdout = w
	outC := make(chan string)

	var buf bytes.Buffer
	_, err = io.Copy(&buf, r)
	r.Close()
	if err != nil {
		return "", err
	}
	outC <- buf.String()

	w.Close()
	os.Stdout = stdout
	out := <-outC

	ntext := strings.SplitN(out, "#", 2)
	return strings.Trim(ntext[1], trim), nil
}
<commit_msg>Remove unused code<commit_after>package commands

import (
	"fmt"
	"go/build"
	"nvim-go/gb"
	"nvim-go/nvim"

	"golang.org/x/tools/refactor/rename"

	"github.com/garyburd/neovim-go/vim"
	"github.com/garyburd/neovim-go/vim/plugin"
)

func init() {
	plugin.HandleCommand("Gorename",
		&plugin.CommandOptions{
			NArgs: "?", Eval: "[expand('%:p:h'), expand('%:p'), line2byte(line('.'))+(col('.')-2)]"},
		cmdRename)
}

var (
	renamePrefill  = "go#rename#prefill"
	vRenamePrefill interface{}
)

type onRenameEval struct {
	Dir    string `msgpack:",array"`
	File   string
	Offset int
}

func cmdRename(v *vim.Vim, args []string, eval *onRenameEval) error {
	go Rename(v, args, eval)
	return nil
}

// Rename rename the current cursor word use golang.org/x/tools/refactor/rename.
func Rename(v *vim.Vim, args []string, eval *onRenameEval) error {
	defer gb.WithGoBuildForPath(eval.Dir)()

	v.Var(renamePrefill, &vRenamePrefill)
	from, err := v.CommandOutput(fmt.Sprintf("silent! echo expand('<cword>')"))
	if err != nil {
		nvim.Echomsg(v, "%s", err)
	}

	var (
		b vim.Buffer
		w vim.Window
	)
	p := v.NewPipeline()
	p.CurrentBuffer(&b)
	p.CurrentWindow(&w)
	if err := p.Wait(); err != nil {
		return err
	}

	offset := fmt.Sprintf("%s:#%d", eval.File, eval.Offset)
	fmt.Printf(offset)

	askMessage := fmt.Sprintf("%s: Rename '%s' to: ", "nvim-go", from[1:])
	var to interface{}
	if vRenamePrefill.(int64) == int64(1) {
		p.Call("input", &to, askMessage, from[1:])
		if err := p.Wait(); err != nil {
			return nvim.Echomsg(v, "%s", err)
		}
	} else {
		p.Call("input", &to, askMessage)
		if err := p.Wait(); err != nil {
			return nvim.Echomsg(v, "%s", err)
		}
	}

	if err := rename.Main(&build.Default, offset, "", fmt.Sprint(to)); err != nil {
		if err != rename.ConflictError {
			nvim.Echomsg(v, "%s", err)
		}
	}
	p.Command("edit!")

	return p.Wait()
}
<|endoftext|>


<commit_before>package end2end_test

import (
	"fmt"
	"math/rand"
	"sync"
	"time"

	"golang.org/x/net/context"

	"github.com/apoydence/talaria/pb"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("End2end", func() {
	Context("Data has been written", func() {
		var (
			fileInfo *pb.File
		)

		var writeTo = func(name string, data []byte, writer pb.Talaria_WriteClient) {
			packet := &pb.WriteDataPacket{
				FileName: name,
				Message:  data,
			}
			Expect(writer.Send(packet)).To(Succeed())
		}

		var fetchReaderWithIndex = func(name string, index uint64, client pb.TalariaClient) (chan []byte, chan uint64) {
			c := make(chan []byte, 100)
			idx := make(chan uint64, 100)

			fileInfo = &pb.File{
				FileName:   name,
				StartIndex: index,
			}

			reader, err := client.Read(context.Background(), fileInfo)
			Expect(err).ToNot(HaveOccurred())

			go func() {
				for {
					packet, err := reader.Recv()
					if err != nil {
						return
					}
					c <- packet.Message
					idx <- packet.Index
				}
			}()
			return c, idx
		}

		var fetchReaderLastIndex = func(name string, client pb.TalariaClient) (chan []byte, chan uint64) {
			c := make(chan []byte, 100)
			idx := make(chan uint64, 100)

			fileInfo = &pb.File{
				FileName:     name,
				StartIndex:   1,
				StartFromEnd: true,
			}

			reader, err := client.Read(context.Background(), fileInfo)
			Expect(err).ToNot(HaveOccurred())

			go func() {
				for {
					packet, err := reader.Recv()
					if err != nil {
						return
					}
					c <- packet.Message
					idx <- packet.Index
				}
			}()
			return c, idx
		}

		var writeSlowly = func(count int, fileInfo *pb.File, writer pb.Talaria_WriteClient) *sync.WaitGroup {
			var wg sync.WaitGroup
			wg.Add(1)
			go func() {
				defer wg.Done()
				for i := 0; i < count; i++ {
					time.Sleep(time.Millisecond)
					writeTo(fileInfo.FileName, []byte(fmt.Sprintf("some-data-%d", i)), writer)
				}
			}()
			return &wg
		}

		BeforeEach(func() {
			fileInfo = &pb.File{
				FileName: createFileName(),
			}
		})

		Context("file has been created", func() {
			BeforeEach(func() {
				talariaClient.Create(context.Background(), fileInfo)
			})

			Context("start tailing from beginning", func() {
				It("writes data to a subscriber", func() {
					writer, err := talariaClient.Write(context.Background())
					Expect(err).ToNot(HaveOccurred())
					writeTo(fileInfo.FileName, []byte("some-data-1"), writer)
					writeTo(fileInfo.FileName, []byte("some-data-2"), writer)

					data, indexes := fetchReaderWithIndex(fileInfo.FileName, 0, talariaClient)
					Eventually(data).Should(Receive(Equal([]byte("some-data-1"))))
					Eventually(indexes).Should(Receive(BeEquivalentTo(0)))
					Eventually(data).Should(Receive(Equal([]byte("some-data-2"))))
					Eventually(indexes).Should(Receive(BeEquivalentTo(1)))
				})

				It("tails via Read()", func() {
					data, _ := fetchReaderWithIndex(fileInfo.FileName, 0, talariaClient)
					writer, err := talariaClient.Write(context.Background())
					Expect(err).ToNot(HaveOccurred())

					wg := writeSlowly(10, fileInfo, writer)
					defer wg.Wait()

					for i := 0; i < 10; i++ {
						expectedData := []byte(fmt.Sprintf("some-data-%d", i))
						Eventually(data).Should(Receive(Equal(expectedData)))
					}
				})
			})

			Context("tail from middle", func() {
				It("reads from the given index", func() {
					writer, err := talariaClient.Write(context.Background())
					Expect(err).ToNot(HaveOccurred())
					writeTo(fileInfo.FileName, []byte("some-data-1"), writer)
					writeTo(fileInfo.FileName, []byte("some-data-2"), writer)
					writeTo(fileInfo.FileName, []byte("some-data-3"), writer)

					data, indexes := fetchReaderWithIndex(fileInfo.FileName, 1, talariaClient)

					var idx uint64
					Eventually(indexes).Should(Receive(&idx))
					Expect(idx).To(BeEquivalentTo(1))
					Expect(data).To(Receive(Equal([]byte("some-data-2"))))
				})
			})

			Context("tail from end", func() {
				It("reads from the given index", func() {
					writer, err := talariaClient.Write(context.Background())
					Expect(err).ToNot(HaveOccurred())
					writeTo(fileInfo.FileName, []byte("some-data-1"), writer)
					writeTo(fileInfo.FileName, []byte("some-data-2"), writer)
					writeTo(fileInfo.FileName, []byte("some-data-3"), writer)

					data, indexes := fetchReaderLastIndex(fileInfo.FileName, talariaClient)

					var idx uint64
					Eventually(indexes).Should(Receive(&idx))
					Expect(idx).To(BeEquivalentTo(2))
					Expect(data).To(Receive(Equal([]byte("some-data-3"))))
				})
			})
		})

		Context("file has not been created", func() {
			It("returns an error", func() {
				writer, err := talariaClient.Write(context.Background())
				Expect(err).ToNot(HaveOccurred())

				_, err = writer.CloseAndRecv()
				Expect(err).To(HaveOccurred())
			})
		})
	})
})

func createFileName() string {
	return fmt.Sprintf("some-file-%d", rand.Int63())
}
<commit_msg>Fix flaky end2end tetst<commit_after>package end2end_test

import (
	"fmt"
	"math/rand"
	"sync"
	"time"

	"golang.org/x/net/context"

	"github.com/apoydence/talaria/pb"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("End2end", func() {
	Context("Data has been written", func() {
		var (
			fileInfo *pb.File
		)

		var writeTo = func(name string, data []byte, writer pb.Talaria_WriteClient) {
			packet := &pb.WriteDataPacket{
				FileName: name,
				Message:  data,
			}
			Expect(writer.Send(packet)).To(Succeed())
		}

		var fetchReaderWithIndex = func(name string, index uint64, client pb.TalariaClient) (chan []byte, chan uint64) {
			c := make(chan []byte, 100)
			idx := make(chan uint64, 100)

			fileInfo = &pb.File{
				FileName:   name,
				StartIndex: index,
			}

			reader, err := client.Read(context.Background(), fileInfo)
			Expect(err).ToNot(HaveOccurred())

			go func() {
				for {
					packet, err := reader.Recv()
					if err != nil {
						return
					}
					c <- packet.Message
					idx <- packet.Index
				}
			}()
			return c, idx
		}

		var fetchReaderLastIndex = func(name string, client pb.TalariaClient) (chan []byte, chan uint64) {
			c := make(chan []byte, 100)
			idx := make(chan uint64, 100)

			fileInfo = &pb.File{
				FileName:     name,
				StartIndex:   1,
				StartFromEnd: true,
			}

			reader, err := client.Read(context.Background(), fileInfo)
			Expect(err).ToNot(HaveOccurred())

			go func() {
				for {
					packet, err := reader.Recv()
					if err != nil {
						return
					}
					c <- packet.Message
					idx <- packet.Index
				}
			}()
			return c, idx
		}

		var writeSlowly = func(count int, fileInfo *pb.File, writer pb.Talaria_WriteClient) *sync.WaitGroup {
			var wg sync.WaitGroup
			wg.Add(1)
			go func() {
				defer wg.Done()
				for i := 0; i < count; i++ {
					time.Sleep(time.Millisecond)
					writeTo(fileInfo.FileName, []byte(fmt.Sprintf("some-data-%d", i)), writer)
				}
			}()
			return &wg
		}

		BeforeEach(func() {
			fileInfo = &pb.File{
				FileName: createFileName(),
			}
		})

		Context("file has been created", func() {
			BeforeEach(func() {
				talariaClient.Create(context.Background(), fileInfo)
			})

			Context("start tailing from beginning", func() {
				It("writes data to a subscriber", func() {
					writer, err := talariaClient.Write(context.Background())
					Expect(err).ToNot(HaveOccurred())
					writeTo(fileInfo.FileName, []byte("some-data-1"), writer)
					writeTo(fileInfo.FileName, []byte("some-data-2"), writer)

					data, indexes := fetchReaderWithIndex(fileInfo.FileName, 0, talariaClient)
					Eventually(data).Should(Receive(Equal([]byte("some-data-1"))))
					Eventually(indexes).Should(Receive(BeEquivalentTo(0)))
					Eventually(data).Should(Receive(Equal([]byte("some-data-2"))))
					Eventually(indexes).Should(Receive(BeEquivalentTo(1)))
				})

				It("tails via Read()", func() {
					data, _ := fetchReaderWithIndex(fileInfo.FileName, 0, talariaClient)
					writer, err := talariaClient.Write(context.Background())
					Expect(err).ToNot(HaveOccurred())

					wg := writeSlowly(10, fileInfo, writer)
					defer wg.Wait()

					for i := 0; i < 10; i++ {
						expectedData := []byte(fmt.Sprintf("some-data-%d", i))
						Eventually(data).Should(Receive(Equal(expectedData)))
					}
				})
			})

			Context("tail from middle", func() {
				It("reads from the given index", func() {
					writer, err := talariaClient.Write(context.Background())
					Expect(err).ToNot(HaveOccurred())
					writeTo(fileInfo.FileName, []byte("some-data-1"), writer)
					writeTo(fileInfo.FileName, []byte("some-data-2"), writer)
					writeTo(fileInfo.FileName, []byte("some-data-3"), writer)

					data, indexes := fetchReaderWithIndex(fileInfo.FileName, 1, talariaClient)

					var idx uint64
					Eventually(indexes).Should(Receive(&idx))
					Expect(idx).To(BeEquivalentTo(1))
					Expect(data).To(Receive(Equal([]byte("some-data-2"))))
				})
			})

			Context("tail from end", func() {
				var waitForData = func() {
					data, _ := fetchReaderWithIndex(fileInfo.FileName, 0, talariaClient)
					Eventually(data).Should(HaveLen(3))
				}

				It("reads from the given index", func() {
					writer, err := talariaClient.Write(context.Background())
					Expect(err).ToNot(HaveOccurred())
					writeTo(fileInfo.FileName, []byte("some-data-1"), writer)
					writeTo(fileInfo.FileName, []byte("some-data-2"), writer)
					writeTo(fileInfo.FileName, []byte("some-data-3"), writer)
					waitForData()

					data, indexes := fetchReaderLastIndex(fileInfo.FileName, talariaClient)

					var idx uint64
					Eventually(indexes).Should(Receive(&idx))
					Expect(idx).To(BeEquivalentTo(2))
					Expect(data).To(Receive(Equal([]byte("some-data-3"))))
				})
			})
		})

		Context("file has not been created", func() {
			It("returns an error", func() {
				writer, err := talariaClient.Write(context.Background())
				Expect(err).ToNot(HaveOccurred())

				_, err = writer.CloseAndRecv()
				Expect(err).To(HaveOccurred())
			})
		})
	})
})

func createFileName() string {
	return fmt.Sprintf("some-file-%d", rand.Int63())
}
<|endoftext|>


<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package save

import (
	"testing"

	"golang.org/x/net/context"

	"github.com/jacobsa/comeback/internal/blob"
	"github.com/jacobsa/comeback/internal/blob/mock"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

func TestBlobStore(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type VisitorTest struct {
	ctx       context.Context
	blobStore blob.Store

	node fsNode
}

func init() { RegisterTestSuite(&VisitorTest{}) }

func (t *VisitorTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx
	t.blobStore = mock_blob.NewMockStore(ti.MockController, "blobStore")
}

func (t *VisitorTest) call() (err error) {
	visitor := newVisitor(t.blobStore, make(chan *fsNode, 1))
	err = visitor.Visit(t.ctx, &t.node)
	return
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *VisitorTest) ScoresAlreadyPresent_Empty() {
	scores := []blob.Score{}
	t.node.Scores = scores

	err := t.call()
	AssertEq(nil, err)
	ExpectThat(t.node.Scores, DeepEquals(scores))
}

func (t *VisitorTest) ScoresAlreadyPresent_NonEmpty() {
	scores := []blob.Score{blob.ComputeScore([]byte("taco"))}
	t.node.Scores = scores

	err := t.call()
	AssertEq(nil, err)
	ExpectThat(t.node.Scores, DeepEquals(scores))
}

func (t *VisitorTest) Symlink() {
	AssertTrue(false, "TODO")
}

func (t *VisitorTest) Directory() {
	AssertTrue(false, "TODO")
}

func (t *VisitorTest) File_Empty() {
	AssertTrue(false, "TODO")
}

func (t *VisitorTest) File_LastChunkIsFull() {
	AssertTrue(false, "TODO")
}

func (t *VisitorTest) File_LastChunkIsPartial() {
	AssertTrue(false, "TODO")
}
<commit_msg>VisitorTest.Symlink<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package save

import (
	"io/ioutil"
	"os"
	"path"
	"testing"

	"golang.org/x/net/context"

	"github.com/jacobsa/comeback/internal/blob"
	"github.com/jacobsa/comeback/internal/blob/mock"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

func TestBlobStore(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type VisitorTest struct {
	ctx       context.Context
	blobStore blob.Store

	node fsNode

	// A temporary directory removed at the end of the test.
	dir string
}

func init() { RegisterTestSuite(&VisitorTest{}) }

var _ SetUpInterface = &VisitorTest{}
var _ TearDownInterface = &VisitorTest{}

func (t *VisitorTest) SetUp(ti *TestInfo) {
	var err error

	t.ctx = ti.Ctx
	t.blobStore = mock_blob.NewMockStore(ti.MockController, "blobStore")

	// Set up the directory.
	t.dir, err = ioutil.TempDir("", "score_map_test")
	AssertEq(nil, err)
}

func (t *VisitorTest) TearDown() {
	var err error

	err = os.RemoveAll(t.dir)
	AssertEq(nil, err)
}

func (t *VisitorTest) call() (err error) {
	visitor := newVisitor(t.blobStore, make(chan *fsNode, 1))
	err = visitor.Visit(t.ctx, &t.node)
	return
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *VisitorTest) ScoresAlreadyPresent_Empty() {
	scores := []blob.Score{}
	t.node.Scores = scores

	err := t.call()
	AssertEq(nil, err)
	ExpectThat(t.node.Scores, DeepEquals(scores))
}

func (t *VisitorTest) ScoresAlreadyPresent_NonEmpty() {
	scores := []blob.Score{blob.ComputeScore([]byte("taco"))}
	t.node.Scores = scores

	err := t.call()
	AssertEq(nil, err)
	ExpectThat(t.node.Scores, DeepEquals(scores))
}

func (t *VisitorTest) Symlink() {
	var err error

	// Node setup
	p := path.Join(t.dir, "foo")

	err = os.Symlink("blah", p)
	AssertEq(nil, err)

	t.node.Info, err = os.Lstat(p)
	AssertEq(nil, err)
	AssertEq(os.ModeSymlink, t.node.Info.Mode()&os.ModeType)

	// Call
	err = t.call()
	AssertEq(nil, err)

	ExpectEq(nil, t.node.Scores)
}

func (t *VisitorTest) Directory() {
	AssertTrue(false, "TODO")
}

func (t *VisitorTest) File_Empty() {
	AssertTrue(false, "TODO")
}

func (t *VisitorTest) File_LastChunkIsFull() {
	AssertTrue(false, "TODO")
}

func (t *VisitorTest) File_LastChunkIsPartial() {
	AssertTrue(false, "TODO")
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package setters

import (
	"fmt"
	"os"
	"os/exec"
	"strings"

	"sigs.k8s.io/kustomize/kyaml/kio"
	"sigs.k8s.io/kustomize/kyaml/setters"
)

func PerformSetters(path string) error {
	rw := &kio.LocalPackageReadWriter{
		PackagePath:           path,
		KeepReaderAnnotations: false,
		IncludeSubpackages:    true,
	}

	// auto-fill setters from the environment
	var fltrs []kio.Filter
	for i := range os.Environ() {
		e := os.Environ()[i]
		if !strings.HasPrefix(e, "KPT_SET_") {
			continue
		}
		parts := strings.SplitN(e, "=", 2)
		if len(parts) < 2 {
			continue
		}
		k, v := strings.TrimPrefix(parts[0], "KPT_SET_"), parts[1]
		k = strings.ToLower(k)
		fltrs = append(fltrs, &setters.PerformSetters{Name: k, Value: v, SetBy: "kpt"})
	}

	// auto-fill setters from gcloud
	gcloudConfig := []string{"compute.region", "compute.zone", "core.project"}
	for _, c := range gcloudConfig {
		gcloudCmd := exec.Command("gcloud",
			"config", "list", "--format", fmt.Sprintf("value(%s)", c))
		b, err := gcloudCmd.Output()
		if err != nil {
			// don't fail if gcloud fails -- it may not be installed or have this config property
			continue
		}
		v := strings.TrimSpace(string(b))
		if v == "" {
			// don't replace values that aren't set - stick with the defaults as defined in the manifest
			continue
		}
		fltrs = append(fltrs, &setters.PerformSetters{Name: fmt.Sprintf("gcloud.%s", c), Value: v, SetBy: "kpt"})
	}

	if len(fltrs) == 0 {
		return nil
	}

	return kio.Pipeline{Inputs: []kio.Reader{rw}, Filters: fltrs, Outputs: []kio.Writer{rw}}.
		Execute()
}
<commit_msg>add an auto setter for gcloud.project.projectNumber<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package setters

import (
	"fmt"
	"os"
	"os/exec"
	"strings"

	"sigs.k8s.io/kustomize/kyaml/kio"
	"sigs.k8s.io/kustomize/kyaml/setters"
)

func PerformSetters(path string) error {
	rw := &kio.LocalPackageReadWriter{
		PackagePath:           path,
		KeepReaderAnnotations: false,
		IncludeSubpackages:    true,
	}

	// auto-fill setters from the environment
	var fltrs []kio.Filter
	for i := range os.Environ() {
		e := os.Environ()[i]
		if !strings.HasPrefix(e, "KPT_SET_") {
			continue
		}
		parts := strings.SplitN(e, "=", 2)
		if len(parts) < 2 {
			continue
		}
		k, v := strings.TrimPrefix(parts[0], "KPT_SET_"), parts[1]
		k = strings.ToLower(k)
		fltrs = append(fltrs, &setters.PerformSetters{Name: k, Value: v, SetBy: "kpt"})
	}

	// auto-fill setters from gcloud
	gcloudConfig := []string{"compute.region", "compute.zone", "core.project"}
	projectID := ""
	for _, c := range gcloudConfig {
		gcloudCmd := exec.Command("gcloud",
			"config", "list", "--format", fmt.Sprintf("value(%s)", c))
		b, err := gcloudCmd.Output()
		if err != nil {
			// don't fail if gcloud fails -- it may not be installed or have this config property
			continue
		}
		v := strings.TrimSpace(string(b))
		if v == "" {
			// don't replace values that aren't set - stick with the defaults as defined in the manifest
			continue
		}
		if c == "core.project" {
			projectID = v
		}
		fltrs = append(fltrs, &setters.PerformSetters{Name: fmt.Sprintf("gcloud.%s", c), Value: v, SetBy: "kpt"})
	}

	if projectID != "" {
		projectNumber, err := GetProjectNumberFromProjectID(projectID)
		if err == nil && projectNumber != "" {
			fltrs = append(fltrs, &setters.PerformSetters{Name: "gcloud.project.projectNumber", Value: projectNumber, SetBy: "kpt"})
		}
	}

	if len(fltrs) == 0 {
		return nil
	}

	return kio.Pipeline{Inputs: []kio.Reader{rw}, Filters: fltrs, Outputs: []kio.Writer{rw}}.
		Execute()
}

func GetProjectNumberFromProjectID(projectID string) (string, error) {
	gcloudCmd := exec.Command("gcloud",
		"projects", "describe", projectID, "--format", "value(projectNumber)")
	b, err := gcloudCmd.Output()
	if err != nil {
		fmt.Printf("err: %v\n", err)
		return "", err
	}
	fmt.Println(strings.TrimSpace(string(b)))
	return strings.TrimSpace(string(b)), nil
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package lzw implements the Lempel-Ziv-Welch compressed data format,
// described in T. A. Welch, ``A Technique for High-Performance Data
// Compression'', Computer, 17(6) (June 1984), pp 8-19.
//
// In particular, it implements LZW as used by the GIF, TIFF and PDF file
// formats, which means variable-width codes up to 12 bits and the first
// two non-literal codes are a clear code and an EOF code.
package lzw

// TODO(nigeltao): check that TIFF and PDF use LZW in the same way as GIF,
// modulo LSB/MSB packing order.

import (
	"bufio"
	"fmt"
	"io"
	"os"
)

// Order specifies the bit ordering in an LZW data stream.
type Order int

const (
	// LSB means Least Significant Bits first, as used in the GIF file format.
	LSB Order = iota
	// MSB means Most Significant Bits first, as used in the TIFF and PDF
	// file formats.
	MSB
)

// decoder is the state from which the readXxx method converts a byte
// stream into a code stream.
type decoder struct {
	r     io.ByteReader
	bits  uint32
	nBits uint
	width uint
}

// readLSB returns the next code for "Least Significant Bits first" data.
func (d *decoder) readLSB() (uint16, os.Error) {
	for d.nBits < d.width {
		x, err := d.r.ReadByte()
		if err != nil {
			return 0, err
		}
		d.bits |= uint32(x) << d.nBits
		d.nBits += 8
	}
	code := uint16(d.bits & (1<<d.width - 1))
	d.bits >>= d.width
	d.nBits -= d.width
	return code, nil
}

// readMSB returns the next code for "Most Significant Bits first" data.
func (d *decoder) readMSB() (uint16, os.Error) {
	for d.nBits < d.width {
		x, err := d.r.ReadByte()
		if err != nil {
			return 0, err
		}
		d.bits |= uint32(x) << (24 - d.nBits)
		d.nBits += 8
	}
	code := uint16(d.bits >> (32 - d.width))
	d.bits <<= d.width
	d.nBits -= d.width
	return code, nil
}

// decode decompresses bytes from r and writes them to pw.
// read specifies how to decode bytes into codes.
// litWidth is the width in bits of literal codes.
func decode(r io.Reader, read func(*decoder) (uint16, os.Error), litWidth int, pw *io.PipeWriter) {
	br, ok := r.(io.ByteReader)
	if !ok {
		br = bufio.NewReader(r)
	}
	pw.CloseWithError(decode1(pw, br, read, uint(litWidth)))
}

func decode1(pw *io.PipeWriter, r io.ByteReader, read func(*decoder) (uint16, os.Error), litWidth uint) os.Error {
	const (
		maxWidth    = 12
		invalidCode = 0xffff
	)
	d := decoder{r, 0, 0, 1 + litWidth}
	w := bufio.NewWriter(pw)
	// The first 1<<litWidth codes are literal codes.
	// The next two codes mean clear and EOF.
	// Other valid codes are in the range [lo, hi] where lo := clear + 2,
	// with the upper bound incrementing on each code seen.
	clear := uint16(1) << litWidth
	eof, hi := clear+1, clear+1
	// overflow is the code at which hi overflows the code width.
	overflow := uint16(1) << d.width
	var (
		// Each code c in [lo, hi] expands to two or more bytes. For c != hi:
		//   suffix[c] is the last of these bytes.
		//   prefix[c] is the code for all but the last byte.
		//   This code can either be a literal code or another code in [lo, c).
		// The c == hi case is a special case.
		suffix [1 << maxWidth]uint8
		prefix [1 << maxWidth]uint16
		// buf is a scratch buffer for reconstituting the bytes that a code expands to.
		// Code suffixes are written right-to-left from the end of the buffer.
		buf [1 << maxWidth]byte
	)

	// Loop over the code stream, converting codes into decompressed bytes.
	last := uint16(invalidCode)
	for {
		code, err := read(&d)
		if err != nil {
			if err == os.EOF {
				err = io.ErrUnexpectedEOF
			}
			return err
		}
		switch {
		case code < clear:
			// We have a literal code.
			if err := w.WriteByte(uint8(code)); err != nil {
				return err
			}
			if last != invalidCode {
				// Save what the hi code expands to.
				suffix[hi] = uint8(code)
				prefix[hi] = last
			}
		case code == clear:
			d.width = 1 + litWidth
			hi = eof
			overflow = 1 << d.width
			last = invalidCode
			continue
		case code == eof:
			return w.Flush()
		case code <= hi:
			c, i := code, len(buf)-1
			if code == hi {
				// code == hi is a special case which expands to the last expansion
				// followed by the head of the last expansion. To find the head, we walk
				// the prefix chain until we find a literal code.
				c = last
				for c >= clear {
					c = prefix[c]
				}
				buf[i] = uint8(c)
				i--
				c = last
			}
			// Copy the suffix chain into buf and then write that to w.
			for c >= clear {
				buf[i] = suffix[c]
				i--
				c = prefix[c]
			}
			buf[i] = uint8(c)
			if _, err := w.Write(buf[i:]); err != nil {
				return err
			}
			// Save what the hi code expands to.
			suffix[hi] = uint8(c)
			prefix[hi] = last
		default:
			return os.NewError("lzw: invalid code")
		}
		last, hi = code, hi+1
		if hi == overflow {
			if d.width == maxWidth {
				return os.NewError("lzw: missing clear code")
			}
			d.width++
			overflow <<= 1
		}
	}
	panic("unreachable")
}

// NewReader creates a new io.ReadCloser that satisfies reads by decompressing
// the data read from r.
// It is the caller's responsibility to call Close on the ReadCloser when
// finished reading.
// The number of bits to use for literal codes, litWidth, must be in the
// range [2,8] and is typically 8.
func NewReader(r io.Reader, order Order, litWidth int) io.ReadCloser {
	pr, pw := io.Pipe()
	var read func(*decoder) (uint16, os.Error)
	switch order {
	case LSB:
		read = (*decoder).readLSB
	case MSB:
		read = (*decoder).readMSB
	default:
		pw.CloseWithError(os.NewError("lzw: unknown order"))
		return pr
	}
	if litWidth < 2 || 8 < litWidth {
		pw.CloseWithError(fmt.Errorf("lzw: litWidth %d out of range", litWidth))
		return pr
	}
	go decode(r, read, litWidth, pw)
	return pr
}
<commit_msg>compress/lzw: silently drop implied codes that are too large, instead of returning an error.<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package lzw implements the Lempel-Ziv-Welch compressed data format,
// described in T. A. Welch, ``A Technique for High-Performance Data
// Compression'', Computer, 17(6) (June 1984), pp 8-19.
//
// In particular, it implements LZW as used by the GIF, TIFF and PDF file
// formats, which means variable-width codes up to 12 bits and the first
// two non-literal codes are a clear code and an EOF code.
package lzw

// TODO(nigeltao): check that TIFF and PDF use LZW in the same way as GIF,
// modulo LSB/MSB packing order.

import (
	"bufio"
	"fmt"
	"io"
	"os"
)

// Order specifies the bit ordering in an LZW data stream.
type Order int

const (
	// LSB means Least Significant Bits first, as used in the GIF file format.
	LSB Order = iota
	// MSB means Most Significant Bits first, as used in the TIFF and PDF
	// file formats.
	MSB
)

// decoder is the state from which the readXxx method converts a byte
// stream into a code stream.
type decoder struct {
	r     io.ByteReader
	bits  uint32
	nBits uint
	width uint
}

// readLSB returns the next code for "Least Significant Bits first" data.
func (d *decoder) readLSB() (uint16, os.Error) {
	for d.nBits < d.width {
		x, err := d.r.ReadByte()
		if err != nil {
			return 0, err
		}
		d.bits |= uint32(x) << d.nBits
		d.nBits += 8
	}
	code := uint16(d.bits & (1<<d.width - 1))
	d.bits >>= d.width
	d.nBits -= d.width
	return code, nil
}

// readMSB returns the next code for "Most Significant Bits first" data.
func (d *decoder) readMSB() (uint16, os.Error) {
	for d.nBits < d.width {
		x, err := d.r.ReadByte()
		if err != nil {
			return 0, err
		}
		d.bits |= uint32(x) << (24 - d.nBits)
		d.nBits += 8
	}
	code := uint16(d.bits >> (32 - d.width))
	d.bits <<= d.width
	d.nBits -= d.width
	return code, nil
}

// decode decompresses bytes from r and writes them to pw.
// read specifies how to decode bytes into codes.
// litWidth is the width in bits of literal codes.
func decode(r io.Reader, read func(*decoder) (uint16, os.Error), litWidth int, pw *io.PipeWriter) {
	br, ok := r.(io.ByteReader)
	if !ok {
		br = bufio.NewReader(r)
	}
	pw.CloseWithError(decode1(pw, br, read, uint(litWidth)))
}

func decode1(pw *io.PipeWriter, r io.ByteReader, read func(*decoder) (uint16, os.Error), litWidth uint) os.Error {
	const (
		maxWidth    = 12
		invalidCode = 0xffff
	)
	d := decoder{r, 0, 0, 1 + litWidth}
	w := bufio.NewWriter(pw)
	// The first 1<<litWidth codes are literal codes.
	// The next two codes mean clear and EOF.
	// Other valid codes are in the range [lo, hi] where lo := clear + 2,
	// with the upper bound incrementing on each code seen.
	clear := uint16(1) << litWidth
	eof, hi := clear+1, clear+1
	// overflow is the code at which hi overflows the code width.
	overflow := uint16(1) << d.width
	var (
		// Each code c in [lo, hi] expands to two or more bytes. For c != hi:
		//   suffix[c] is the last of these bytes.
		//   prefix[c] is the code for all but the last byte.
		//   This code can either be a literal code or another code in [lo, c).
		// The c == hi case is a special case.
		suffix [1 << maxWidth]uint8
		prefix [1 << maxWidth]uint16
		// buf is a scratch buffer for reconstituting the bytes that a code expands to.
		// Code suffixes are written right-to-left from the end of the buffer.
		buf [1 << maxWidth]byte
	)

	// Loop over the code stream, converting codes into decompressed bytes.
	last := uint16(invalidCode)
	for {
		code, err := read(&d)
		if err != nil {
			if err == os.EOF {
				err = io.ErrUnexpectedEOF
			}
			return err
		}
		switch {
		case code < clear:
			// We have a literal code.
			if err := w.WriteByte(uint8(code)); err != nil {
				return err
			}
			if last != invalidCode {
				// Save what the hi code expands to.
				suffix[hi] = uint8(code)
				prefix[hi] = last
			}
		case code == clear:
			d.width = 1 + litWidth
			hi = eof
			overflow = 1 << d.width
			last = invalidCode
			continue
		case code == eof:
			return w.Flush()
		case code <= hi:
			c, i := code, len(buf)-1
			if code == hi {
				// code == hi is a special case which expands to the last expansion
				// followed by the head of the last expansion. To find the head, we walk
				// the prefix chain until we find a literal code.
				c = last
				for c >= clear {
					c = prefix[c]
				}
				buf[i] = uint8(c)
				i--
				c = last
			}
			// Copy the suffix chain into buf and then write that to w.
			for c >= clear {
				buf[i] = suffix[c]
				i--
				c = prefix[c]
			}
			buf[i] = uint8(c)
			if _, err := w.Write(buf[i:]); err != nil {
				return err
			}
			if last != invalidCode {
				// Save what the hi code expands to.
				suffix[hi] = uint8(c)
				prefix[hi] = last
			}
		default:
			return os.NewError("lzw: invalid code")
		}
		last, hi = code, hi+1
		if hi >= overflow {
			if d.width == maxWidth {
				last = invalidCode
				continue
			}
			d.width++
			overflow <<= 1
		}
	}
	panic("unreachable")
}

// NewReader creates a new io.ReadCloser that satisfies reads by decompressing
// the data read from r.
// It is the caller's responsibility to call Close on the ReadCloser when
// finished reading.
// The number of bits to use for literal codes, litWidth, must be in the
// range [2,8] and is typically 8.
func NewReader(r io.Reader, order Order, litWidth int) io.ReadCloser {
	pr, pw := io.Pipe()
	var read func(*decoder) (uint16, os.Error)
	switch order {
	case LSB:
		read = (*decoder).readLSB
	case MSB:
		read = (*decoder).readMSB
	default:
		pw.CloseWithError(os.NewError("lzw: unknown order"))
		return pr
	}
	if litWidth < 2 || 8 < litWidth {
		pw.CloseWithError(fmt.Errorf("lzw: litWidth %d out of range", litWidth))
		return pr
	}
	go decode(r, read, litWidth, pw)
	return pr
}
<|endoftext|>


<commit_before>// Tools to convert a TomlTree to different representations
package toml

import (
	"fmt"
	"strconv"
	"strings"
	"time"
)

// encodes a string to a TOML-compliant string value
func encodeTomlString(value string) string {
	result := ""
	for _, rr := range value {
		intRr := uint16(rr)
		switch rr {
		case '\b':
			result += "\\b"
		case '\t':
			result += "\\t"
		case '\n':
			result += "\\n"
		case '\f':
			result += "\\f"
		case '\r':
			result += "\\r"
		case '"':
			result += "\\\""
		case '\\':
			result += "\\\\"
		default:
			if intRr < 0x001F {
				result += fmt.Sprintf("\\u%0.4X", intRr)
			} else {
				result += string(rr)
			}
		}
	}
	return result
}

// Value print support function for ToString()
// Outputs the TOML compliant string representation of a value
func toTomlValue(item interface{}, indent int) string {
	tab := strings.Repeat(" ", indent)
	switch value := item.(type) {
	case int64:
		return tab + strconv.FormatInt(value, 10)
	case float64:
		return tab + strconv.FormatFloat(value, 'f', -1, 64)
	case string:
		return tab + "\"" + encodeTomlString(value) + "\""
	case bool:
		if value {
			return "true"
		}
		return "false"
	case time.Time:
		return tab + value.Format(time.RFC3339)
	case []interface{}:
		result := tab + "[\n"
		for _, item := range value {
			result += toTomlValue(item, indent+2) + ",\n"
		}
		return result + tab + "]"
	default:
		panic(fmt.Sprintf("unsupported value type: %v", value))
	}
}

// Recursive support function for ToString()
// Outputs a tree, using the provided keyspace to prefix group names
func (t *TomlTree) toToml(indent, keyspace string) string {
	result := ""
	for k, v := range t.values {
		// figure out the keyspace
		combinedKey := k
		if keyspace != "" {
			combinedKey = keyspace + "." + combinedKey
		}
		// output based on type
		switch node := v.(type) {
		case []*TomlTree:
			for _, item := range node {
				if len(item.Keys()) > 0 {
					result += fmt.Sprintf("\n%s[[%s]]\n", indent, combinedKey)
				}
				result += item.toToml(indent+"  ", combinedKey)
			}
		case *TomlTree:
			if len(node.Keys()) > 0 {
				result += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
			}
			result += node.toToml(indent+"  ", combinedKey)
		case map[string]interface{}:
			sub := TreeFromMap(node)

			if len(sub.Keys()) > 0 {
				result += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
			}
			result += sub.toToml(indent+"  ", combinedKey)
		case *tomlValue:
			result += fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(node.value, 0))
		default:
			result += fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(v, 0))
		}
	}
	return result
}

// ToString generates a human-readable representation of the current tree.
// Output spans multiple lines, and is suitable for ingest by a TOML parser
func (t *TomlTree) ToString() string {
	return t.toToml("", "")
}

// ToMap recursively generates a representation of the current tree using map[string]interface{}.
func (t *TomlTree) ToMap() map[string]interface{} {
	result := map[string]interface{}{}

	for k, v := range t.values {
		switch node := v.(type) {
		case []*TomlTree:
			result[k] = make([]interface{}, 0)
			for _, item := range node {
				result[k] = item.ToMap()
			}
		case *TomlTree:
			result[k] = node.ToMap()
		case map[string]interface{}:
			sub := TreeFromMap(node)
			result[k] = sub.ToMap()
		case *tomlValue:
			result[k] = node.value
		}
	}

	return result
}
<commit_msg>Implement fmt.Stringer and alias ToString (#73)<commit_after>// Tools to convert a TomlTree to different representations
package toml

import (
	"fmt"
	"strconv"
	"strings"
	"time"
)

// encodes a string to a TOML-compliant string value
func encodeTomlString(value string) string {
	result := ""
	for _, rr := range value {
		intRr := uint16(rr)
		switch rr {
		case '\b':
			result += "\\b"
		case '\t':
			result += "\\t"
		case '\n':
			result += "\\n"
		case '\f':
			result += "\\f"
		case '\r':
			result += "\\r"
		case '"':
			result += "\\\""
		case '\\':
			result += "\\\\"
		default:
			if intRr < 0x001F {
				result += fmt.Sprintf("\\u%0.4X", intRr)
			} else {
				result += string(rr)
			}
		}
	}
	return result
}

// Value print support function for ToString()
// Outputs the TOML compliant string representation of a value
func toTomlValue(item interface{}, indent int) string {
	tab := strings.Repeat(" ", indent)
	switch value := item.(type) {
	case int64:
		return tab + strconv.FormatInt(value, 10)
	case float64:
		return tab + strconv.FormatFloat(value, 'f', -1, 64)
	case string:
		return tab + "\"" + encodeTomlString(value) + "\""
	case bool:
		if value {
			return "true"
		}
		return "false"
	case time.Time:
		return tab + value.Format(time.RFC3339)
	case []interface{}:
		result := tab + "[\n"
		for _, item := range value {
			result += toTomlValue(item, indent+2) + ",\n"
		}
		return result + tab + "]"
	default:
		panic(fmt.Sprintf("unsupported value type: %v", value))
	}
}

// Recursive support function for ToString()
// Outputs a tree, using the provided keyspace to prefix group names
func (t *TomlTree) toToml(indent, keyspace string) string {
	result := ""
	for k, v := range t.values {
		// figure out the keyspace
		combinedKey := k
		if keyspace != "" {
			combinedKey = keyspace + "." + combinedKey
		}
		// output based on type
		switch node := v.(type) {
		case []*TomlTree:
			for _, item := range node {
				if len(item.Keys()) > 0 {
					result += fmt.Sprintf("\n%s[[%s]]\n", indent, combinedKey)
				}
				result += item.toToml(indent+"  ", combinedKey)
			}
		case *TomlTree:
			if len(node.Keys()) > 0 {
				result += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
			}
			result += node.toToml(indent+"  ", combinedKey)
		case map[string]interface{}:
			sub := TreeFromMap(node)

			if len(sub.Keys()) > 0 {
				result += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
			}
			result += sub.toToml(indent+"  ", combinedKey)
		case *tomlValue:
			result += fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(node.value, 0))
		default:
			result += fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(v, 0))
		}
	}
	return result
}

// ToString is an alias for String
func (t *TomlTree) ToString() string {
	return t.String()
}

// ToString generates a human-readable representation of the current tree.
// Output spans multiple lines, and is suitable for ingest by a TOML parser
func (t *TomlTree) String() string {
	return t.toToml("", "")
}

// ToMap recursively generates a representation of the current tree using map[string]interface{}.
func (t *TomlTree) ToMap() map[string]interface{} {
	result := map[string]interface{}{}

	for k, v := range t.values {
		switch node := v.(type) {
		case []*TomlTree:
			result[k] = make([]interface{}, 0)
			for _, item := range node {
				result[k] = item.ToMap()
			}
		case *TomlTree:
			result[k] = node.ToMap()
		case map[string]interface{}:
			sub := TreeFromMap(node)
			result[k] = sub.ToMap()
		case *tomlValue:
			result[k] = node.value
		}
	}

	return result
}
<|endoftext|>