<commit_before>package remote

import (
	"encoding/json"
	"errors"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"mime"
	"net/http"
	"net/url"
	"path"
	"regexp"
	"runtime"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/echo"
)

var (
	// ErrNotFoundRemote is used when no request is defined for a doctype
	ErrNotFoundRemote = errors.New("the doctype has no request defined")
	// ErrInvalidRequest is used when we can't use the request defined by the
	// developer
	ErrInvalidRequest = errors.New("the request is not valid")
	// ErrRequestFailed is used when the connexion to the remote website can't
	// be established
	ErrRequestFailed = errors.New("can't connect to the remote host")
	// ErrInvalidVariables is used when the variables can't be extracted from
	// the request
	ErrInvalidVariables = errors.New("the variables are not valid")
	// ErrMissingVar is used when trying to use a variable that has not been defined
	ErrMissingVar = errors.New("a variable is used in the template, but no value was given")
	// ErrInvalidContentType is used when the response has a content-type that
	// we deny for security reasons
	ErrInvalidContentType = errors.New("the content-type for the response is not authorized")
)

const rawURL = "https://raw.githubusercontent.com/cozy/cozy-doctypes/master/%s/request"

// Doctype is used to describe a doctype, its request for a remote doctype for example
type Doctype struct {
	DocID   string `json:"_id,omitempty"`
	DocRev  string `json:"_rev,omitempty"`
	Request string `json:"request"`
}

// ID is used to implement the couchdb.Doc interface
func (d *Doctype) ID() string { return d.DocID }

// Rev is used to implement the couchdb.Doc interface
func (d *Doctype) Rev() string { return d.DocRev }

// SetID is used to implement the couchdb.Doc interface
func (d *Doctype) SetID(id string) { d.DocID = id }

// SetRev is used to implement the couchdb.Doc interface
func (d *Doctype) SetRev(rev string) { d.DocRev = rev }

// DocType implements couchdb.Doc
func (d *Doctype) DocType() string { return consts.Doctypes }

// Clone implements couchdb.Doc
func (d *Doctype) Clone() couchdb.Doc { cloned := *d; return &cloned }

// Request is used to log in couchdb a call to a remote website
type Request struct {
	DocID         string            `json:"_id,omitempty"`
	DocRev        string            `json:"_rev,omitempty"`
	RemoteDoctype string            `json:"doctype"`
	Verb          string            `json:"verb"`
	URL           string            `json:"url"`
	ResponseCode  int               `json:"response_code"`
	ContentType   string            `json:"content_type"`
	Variables     map[string]string `json:"variables"`
	CreatedAt     time.Time         `json:"created_at"`
}

// ID is used to implement the couchdb.Doc interface
func (r *Request) ID() string { return r.DocID }

// Rev is used to implement the couchdb.Doc interface
func (r *Request) Rev() string { return r.DocRev }

// SetID is used to implement the couchdb.Doc interface
func (r *Request) SetID(id string) { r.DocID = id }

// SetRev is used to implement the couchdb.Doc interface
func (r *Request) SetRev(rev string) { r.DocRev = rev }

// DocType implements couchdb.Doc
func (r *Request) DocType() string { return consts.RemoteRequests }

// Clone implements couchdb.Doc
func (r *Request) Clone() couchdb.Doc { cloned := *r; return &cloned }

// Remote is the struct used to call a remote website for a doctype
type Remote struct {
	Verb    string
	URL     *url.URL
	Headers map[string]string
	Body    string
}

var log = logger.WithNamespace("remote")

// ParseRawRequest takes a string and parse it as a remote struct.
// First line is verb and URL.
// Then, we have the headers.
// And for a POST, we have a blank line, and then the body.
func ParseRawRequest(doctype, raw string) (*Remote, error) {
	lines := strings.Split(raw, "\n")
	parts := strings.SplitN(lines[0], " ", 2)
	if len(parts) != 2 {
		log.Infof("%s cannot be used as a remote doctype", doctype)
		return nil, ErrInvalidRequest
	}
	var remote Remote
	remote.Verb = parts[0]
	if remote.Verb != echo.GET && remote.Verb != echo.POST {
		log.Infof("Invalid verb for remote doctype %s: %s", doctype, remote.Verb)
		return nil, ErrInvalidRequest
	}
	u, err := url.Parse(parts[1])
	if err != nil {
		log.Infof("Invalid URL for remote doctype %s: %s", doctype, parts[1])
		return nil, ErrInvalidRequest
	}
	if u.Scheme != "https" && u.Scheme != "http" {
		log.Infof("Invalid scheme for remote doctype %s: %s", doctype, u.Scheme)
		return nil, ErrInvalidRequest
	}
	remote.URL = u
	remote.Headers = make(map[string]string)
	for i, line := range lines[1:] {
		if line == "" {
			if remote.Verb == echo.GET {
				continue
			}
			remote.Body = strings.Join(lines[i+2:], "\n")
			break
		}
		parts = strings.SplitN(line, ":", 2)
		if len(parts) != 2 {
			log.Infof("Invalid header for remote doctype %s: %s", doctype, line)
			return nil, ErrInvalidRequest
		}
		remote.Headers[parts[0]] = strings.TrimSpace(parts[1])
	}
	return &remote, nil
}

// Find finds the request defined for the given doctype
func Find(ins *instance.Instance, doctype string) (*Remote, error) {
	var raw string

	if config.GetConfig().Doctypes == "" {
		dt := Doctype{
			DocID: consts.Doctypes + "/" + doctype,
		}
		err := couchdb.GetDoc(ins, consts.Doctypes, dt.DocID, &dt)
		if err != nil {
			u := fmt.Sprintf(rawURL, doctype)
			res, err := http.Get(u)
			log.Debugf("Fetch remote doctype from %s\n", doctype)
			if err != nil {
				log.Infof("Request not found for remote doctype %s: %s", doctype, err)
				return nil, ErrNotFoundRemote
			}
			defer res.Body.Close()
			b, err := ioutil.ReadAll(res.Body)
			if err != nil {
				log.Infof("Request not found for remote doctype %s: %s", doctype, err)
				return nil, ErrNotFoundRemote
			}
			dt.Request = string(b)
			err = couchdb.CreateNamedDocWithDB(ins, &dt)
			if err != nil {
				log.Infof("Cannot save remote doctype %s: %s", doctype, err)
			}
		}
		raw = dt.Request
	} else {
		filename := path.Join(config.GetConfig().Doctypes, doctype, "request")
		bytes, err := ioutil.ReadFile(filename)
		if err != nil {
			return nil, ErrNotFoundRemote
		}
		raw = string(bytes)
	}

	return ParseRawRequest(doctype, raw)
}

// extractVariables extracts the variables:
// - from the query string for a GET
// - from the body formatted as JSON for a POST
func extractVariables(verb string, in *http.Request) (map[string]string, error) {
	vars := make(map[string]string)
	if verb == echo.GET {
		for k, v := range in.URL.Query() {
			vars[k] = v[0]
		}
	} else {
		err := json.NewDecoder(in.Body).Decode(&vars)
		if err != nil {
			return nil, err
		}
	}
	return vars, nil
}

var injectionRegexp = regexp.MustCompile(`{{[0-9A-Za-z_ ]+}}`)

func injectVar(src string, vars map[string]string, defautFunc string) (string, error) {
	var err error
	result := injectionRegexp.ReplaceAllStringFunc(src, func(m string) string {
		m = strings.TrimSpace(m[2 : len(m)-2])

		var funname string
		var varname string
		if defautFunc == "" {
			ms := strings.SplitN(m, " ", 2)
			if len(ms) == 1 {
				varname = ms[0]
			} else {
				funname = ms[0]
				varname = ms[1]
			}
		} else {
			varname = m
			funname = defautFunc
		}

		val, ok := vars[varname]
		if !ok {
			err = ErrMissingVar
			return ""
		}

		switch funname {
		case "":
			return val
		case "query":
			return url.QueryEscape(val)
		case "path":
			return url.PathEscape(val)
		case "header":
			return strings.Replace(val, "\n", "\\n", -1)
		case "json":
			var b []byte
			b, err = json.Marshal(val)
			if err != nil {
				return ""
			}
			return string(b[1 : len(b)-1])
		case "html":
			return html.EscapeString(val)
		default:
			err = fmt.Errorf("remote: unknown template function %s", funname)
			return ""
		}
	})
	return result, err
}

// injectVariables replaces {{variable}} by its value in some fields of the
// remote struct
func injectVariables(remote *Remote, vars map[string]string) error {
	var err error
	if strings.Contains(remote.URL.Path, "{{") {
		remote.URL.Path, err = injectVar(remote.URL.Path, vars, "path")
		if err != nil {
			return err
		}
	}
	if strings.Contains(remote.URL.RawQuery, "{{") {
		remote.URL.RawQuery, err = injectVar(remote.URL.RawQuery, vars, "query")
		if err != nil {
			return err
		}
	}
	for k, v := range remote.Headers {
		if strings.Contains(v, "{{") {
			remote.Headers[k], err = injectVar(v, vars, "header")
			if err != nil {
				return err
			}
		}
	}
	if strings.Contains(remote.Body, "{{") {
		remote.Body, err = injectVar(remote.Body, vars, "")
	}
	return err
}

// ProxyTo calls the external website and proxy the reponse
func (remote *Remote) ProxyTo(doctype string, ins *instance.Instance, rw http.ResponseWriter, in *http.Request) error {
	vars, err := extractVariables(remote.Verb, in)
	if err != nil {
		log.Infof("Error on extracting variables: %s", err)
		return ErrInvalidVariables
	}
	if err = injectVariables(remote, vars); err != nil {
		return err
	}

	// Sanitize the remote URL
	if strings.Contains(remote.URL.Host, ":") {
		log.Infof("Invalid host for remote doctype %s: %s", doctype, remote.URL.Host)
		return ErrInvalidRequest
	}
	remote.URL.User = nil
	remote.URL.Fragment = ""

	req, err := http.NewRequest(remote.Verb, remote.URL.String(), nil)
	if err != nil {
		return ErrInvalidRequest
	}

	req.Header.Set("User-Agent", "cozy-stack "+config.Version+" ("+runtime.Version()+")")
	for k, v := range remote.Headers {
		req.Header.Set(k, v)
	}

	client := http.Client{Timeout: 20 * time.Second}
	res, err := client.Do(req)
	if err != nil {
		log.Infof("Error on request %s: %s", remote.URL.String(), err)
		return ErrRequestFailed
	}
	defer res.Body.Close()

	ctype, _, err := mime.ParseMediaType(res.Header.Get("Content-Type"))
	if err != nil {
		log.Infof("request %s has an invalid content-type", remote.URL.String())
		return ErrInvalidContentType
	}
	if ctype != "application/json" &&
		ctype != "text/xml" &&
		ctype != "application/xml" &&
		ctype != "application/sparql-results+json" {
		class := strings.SplitN(ctype, "/", 2)[0]
		if class != "image" && class != "audio" && class != "video" {
			log.Infof("request %s has a content-type that is not allowed: %s",
				remote.URL.String(), ctype)
			return ErrInvalidContentType
		}
	}

	logged := &Request{
		RemoteDoctype: doctype,
		Verb:          remote.Verb,
		URL:           remote.URL.String(),
		ResponseCode:  res.StatusCode,
		ContentType:   ctype,
		Variables:     vars,
		CreatedAt:     time.Now(),
	}
	err = couchdb.CreateDoc(ins, logged)
	if err != nil {
		log.Errorf("Can't save remote request: %s", err)
	}
	log.Debugf("Remote request: %#v\n", logged)

	rw.WriteHeader(res.StatusCode)
	copyHeader(rw.Header(), res.Header)
	_, err = io.Copy(rw, res.Body)
	if err != nil {
		log.Infof("Error on copying response from %s: %s", remote.URL.String(), err)
	}
	return nil
}

func copyHeader(dst, src http.Header) {
	for k, vv := range src {
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

var (
	_ couchdb.Doc = (*Doctype)(nil)
	_ couchdb.Doc = (*Request)(nil)
)
<commit_msg>Update remote doctypes<commit_after>package remote

import (
	"encoding/json"
	"errors"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"mime"
	"net/http"
	"net/url"
	"path"
	"regexp"
	"runtime"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/echo"
)

var (
	// ErrNotFoundRemote is used when no request is defined for a doctype
	ErrNotFoundRemote = errors.New("the doctype has no request defined")
	// ErrInvalidRequest is used when we can't use the request defined by the
	// developer
	ErrInvalidRequest = errors.New("the request is not valid")
	// ErrRequestFailed is used when the connexion to the remote website can't
	// be established
	ErrRequestFailed = errors.New("can't connect to the remote host")
	// ErrInvalidVariables is used when the variables can't be extracted from
	// the request
	ErrInvalidVariables = errors.New("the variables are not valid")
	// ErrMissingVar is used when trying to use a variable that has not been defined
	ErrMissingVar = errors.New("a variable is used in the template, but no value was given")
	// ErrInvalidContentType is used when the response has a content-type that
	// we deny for security reasons
	ErrInvalidContentType = errors.New("the content-type for the response is not authorized")
)

const rawURL = "https://raw.githubusercontent.com/cozy/cozy-doctypes/master/%s/request"

// Doctype is used to describe a doctype, its request for a remote doctype for example
type Doctype struct {
	DocID     string    `json:"_id,omitempty"`
	DocRev    string    `json:"_rev,omitempty"`
	Request   string    `json:"request"`
	UpdatedAt time.Time `json:"updated_at"`
}

// ID is used to implement the couchdb.Doc interface
func (d *Doctype) ID() string { return d.DocID }

// Rev is used to implement the couchdb.Doc interface
func (d *Doctype) Rev() string { return d.DocRev }

// SetID is used to implement the couchdb.Doc interface
func (d *Doctype) SetID(id string) { d.DocID = id }

// SetRev is used to implement the couchdb.Doc interface
func (d *Doctype) SetRev(rev string) { d.DocRev = rev }

// DocType implements couchdb.Doc
func (d *Doctype) DocType() string { return consts.Doctypes }

// Clone implements couchdb.Doc
func (d *Doctype) Clone() couchdb.Doc { cloned := *d; return &cloned }

// Request is used to log in couchdb a call to a remote website
type Request struct {
	DocID         string            `json:"_id,omitempty"`
	DocRev        string            `json:"_rev,omitempty"`
	RemoteDoctype string            `json:"doctype"`
	Verb          string            `json:"verb"`
	URL           string            `json:"url"`
	ResponseCode  int               `json:"response_code"`
	ContentType   string            `json:"content_type"`
	Variables     map[string]string `json:"variables"`
	CreatedAt     time.Time         `json:"created_at"`
}

// ID is used to implement the couchdb.Doc interface
func (r *Request) ID() string { return r.DocID }

// Rev is used to implement the couchdb.Doc interface
func (r *Request) Rev() string { return r.DocRev }

// SetID is used to implement the couchdb.Doc interface
func (r *Request) SetID(id string) { r.DocID = id }

// SetRev is used to implement the couchdb.Doc interface
func (r *Request) SetRev(rev string) { r.DocRev = rev }

// DocType implements couchdb.Doc
func (r *Request) DocType() string { return consts.RemoteRequests }

// Clone implements couchdb.Doc
func (r *Request) Clone() couchdb.Doc { cloned := *r; return &cloned }

// Remote is the struct used to call a remote website for a doctype
type Remote struct {
	Verb    string
	URL     *url.URL
	Headers map[string]string
	Body    string
}

var log = logger.WithNamespace("remote")

// ParseRawRequest takes a string and parse it as a remote struct.
// First line is verb and URL.
// Then, we have the headers.
// And for a POST, we have a blank line, and then the body.
func ParseRawRequest(doctype, raw string) (*Remote, error) {
	lines := strings.Split(raw, "\n")
	parts := strings.SplitN(lines[0], " ", 2)
	if len(parts) != 2 {
		log.Infof("%s cannot be used as a remote doctype", doctype)
		return nil, ErrInvalidRequest
	}
	var remote Remote
	remote.Verb = parts[0]
	if remote.Verb != echo.GET && remote.Verb != echo.POST {
		log.Infof("Invalid verb for remote doctype %s: %s", doctype, remote.Verb)
		return nil, ErrInvalidRequest
	}
	u, err := url.Parse(parts[1])
	if err != nil {
		log.Infof("Invalid URL for remote doctype %s: %s", doctype, parts[1])
		return nil, ErrInvalidRequest
	}
	if u.Scheme != "https" && u.Scheme != "http" {
		log.Infof("Invalid scheme for remote doctype %s: %s", doctype, u.Scheme)
		return nil, ErrInvalidRequest
	}
	remote.URL = u
	remote.Headers = make(map[string]string)
	for i, line := range lines[1:] {
		if line == "" {
			if remote.Verb == echo.GET {
				continue
			}
			remote.Body = strings.Join(lines[i+2:], "\n")
			break
		}
		parts = strings.SplitN(line, ":", 2)
		if len(parts) != 2 {
			log.Infof("Invalid header for remote doctype %s: %s", doctype, line)
			return nil, ErrInvalidRequest
		}
		remote.Headers[parts[0]] = strings.TrimSpace(parts[1])
	}
	return &remote, nil
}

// Find finds the request defined for the given doctype
func Find(ins *instance.Instance, doctype string) (*Remote, error) {
	var raw string

	if config.GetConfig().Doctypes == "" {
		dt := Doctype{
			DocID: consts.Doctypes + "/" + doctype,
		}
		err := couchdb.GetDoc(ins, consts.Doctypes, dt.DocID, &dt)
		if err != nil || dt.UpdatedAt.Add(24*time.Hour).Before(time.Now()) {
			rev := dt.Rev()
			u := fmt.Sprintf(rawURL, doctype)
			res, err := http.Get(u)
			log.Debugf("Fetch remote doctype from %s\n", doctype)
			if err != nil {
				log.Infof("Request not found for remote doctype %s: %s", doctype, err)
				return nil, ErrNotFoundRemote
			}
			defer res.Body.Close()
			b, err := ioutil.ReadAll(res.Body)
			if err != nil {
				log.Infof("Request not found for remote doctype %s: %s", doctype, err)
				return nil, ErrNotFoundRemote
			}
			dt.Request = string(b)
			dt.UpdatedAt = time.Now()
			if rev == "" {
				err = couchdb.CreateNamedDocWithDB(ins, &dt)
			} else {
				dt.SetRev(rev)
				err = couchdb.UpdateDoc(ins, &dt)
			}
			if err != nil {
				log.Infof("Cannot save remote doctype %s: %s", doctype, err)
			}
		}
		raw = dt.Request
	} else {
		filename := path.Join(config.GetConfig().Doctypes, doctype, "request")
		bytes, err := ioutil.ReadFile(filename)
		if err != nil {
			return nil, ErrNotFoundRemote
		}
		raw = string(bytes)
	}

	return ParseRawRequest(doctype, raw)
}

// extractVariables extracts the variables:
// - from the query string for a GET
// - from the body formatted as JSON for a POST
func extractVariables(verb string, in *http.Request) (map[string]string, error) {
	vars := make(map[string]string)
	if verb == echo.GET {
		for k, v := range in.URL.Query() {
			vars[k] = v[0]
		}
	} else {
		err := json.NewDecoder(in.Body).Decode(&vars)
		if err != nil {
			return nil, err
		}
	}
	return vars, nil
}

var injectionRegexp = regexp.MustCompile(`{{[0-9A-Za-z_ ]+}}`)

func injectVar(src string, vars map[string]string, defautFunc string) (string, error) {
	var err error
	result := injectionRegexp.ReplaceAllStringFunc(src, func(m string) string {
		m = strings.TrimSpace(m[2 : len(m)-2])

		var funname string
		var varname string
		if defautFunc == "" {
			ms := strings.SplitN(m, " ", 2)
			if len(ms) == 1 {
				varname = ms[0]
			} else {
				funname = ms[0]
				varname = ms[1]
			}
		} else {
			varname = m
			funname = defautFunc
		}

		val, ok := vars[varname]
		if !ok {
			err = ErrMissingVar
			return ""
		}

		switch funname {
		case "":
			return val
		case "query":
			return url.QueryEscape(val)
		case "path":
			return url.PathEscape(val)
		case "header":
			return strings.Replace(val, "\n", "\\n", -1)
		case "json":
			var b []byte
			b, err = json.Marshal(val)
			if err != nil {
				return ""
			}
			return string(b[1 : len(b)-1])
		case "html":
			return html.EscapeString(val)
		default:
			err = fmt.Errorf("remote: unknown template function %s", funname)
			return ""
		}
	})
	return result, err
}

// injectVariables replaces {{variable}} by its value in some fields of the
// remote struct
func injectVariables(remote *Remote, vars map[string]string) error {
	var err error
	if strings.Contains(remote.URL.Path, "{{") {
		remote.URL.Path, err = injectVar(remote.URL.Path, vars, "path")
		if err != nil {
			return err
		}
	}
	if strings.Contains(remote.URL.RawQuery, "{{") {
		remote.URL.RawQuery, err = injectVar(remote.URL.RawQuery, vars, "query")
		if err != nil {
			return err
		}
	}
	for k, v := range remote.Headers {
		if strings.Contains(v, "{{") {
			remote.Headers[k], err = injectVar(v, vars, "header")
			if err != nil {
				return err
			}
		}
	}
	if strings.Contains(remote.Body, "{{") {
		remote.Body, err = injectVar(remote.Body, vars, "")
	}
	return err
}

// ProxyTo calls the external website and proxy the reponse
func (remote *Remote) ProxyTo(doctype string, ins *instance.Instance, rw http.ResponseWriter, in *http.Request) error {
	vars, err := extractVariables(remote.Verb, in)
	if err != nil {
		log.Infof("Error on extracting variables: %s", err)
		return ErrInvalidVariables
	}
	if err = injectVariables(remote, vars); err != nil {
		return err
	}

	// Sanitize the remote URL
	if strings.Contains(remote.URL.Host, ":") {
		log.Infof("Invalid host for remote doctype %s: %s", doctype, remote.URL.Host)
		return ErrInvalidRequest
	}
	remote.URL.User = nil
	remote.URL.Fragment = ""

	req, err := http.NewRequest(remote.Verb, remote.URL.String(), nil)
	if err != nil {
		return ErrInvalidRequest
	}

	req.Header.Set("User-Agent", "cozy-stack "+config.Version+" ("+runtime.Version()+")")
	for k, v := range remote.Headers {
		req.Header.Set(k, v)
	}

	client := http.Client{Timeout: 20 * time.Second}
	res, err := client.Do(req)
	if err != nil {
		log.Infof("Error on request %s: %s", remote.URL.String(), err)
		return ErrRequestFailed
	}
	defer res.Body.Close()

	ctype, _, err := mime.ParseMediaType(res.Header.Get("Content-Type"))
	if err != nil {
		log.Infof("request %s has an invalid content-type", remote.URL.String())
		return ErrInvalidContentType
	}
	if ctype != "application/json" &&
		ctype != "text/xml" &&
		ctype != "application/xml" &&
		ctype != "application/sparql-results+json" {
		class := strings.SplitN(ctype, "/", 2)[0]
		if class != "image" && class != "audio" && class != "video" {
			log.Infof("request %s has a content-type that is not allowed: %s",
				remote.URL.String(), ctype)
			return ErrInvalidContentType
		}
	}

	logged := &Request{
		RemoteDoctype: doctype,
		Verb:          remote.Verb,
		URL:           remote.URL.String(),
		ResponseCode:  res.StatusCode,
		ContentType:   ctype,
		Variables:     vars,
		CreatedAt:     time.Now(),
	}
	err = couchdb.CreateDoc(ins, logged)
	if err != nil {
		log.Errorf("Can't save remote request: %s", err)
	}
	log.Debugf("Remote request: %#v\n", logged)

	rw.WriteHeader(res.StatusCode)
	copyHeader(rw.Header(), res.Header)
	_, err = io.Copy(rw, res.Body)
	if err != nil {
		log.Infof("Error on copying response from %s: %s", remote.URL.String(), err)
	}
	return nil
}

func copyHeader(dst, src http.Header) {
	for k, vv := range src {
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

var (
	_ couchdb.Doc = (*Doctype)(nil)
	_ couchdb.Doc = (*Request)(nil)
)
<|endoftext|>


<commit_before>package runner

import (
	"bufio"
	"encoding/json"
	"io"
	"strconv"
)

type RunConfig struct {
	Language string `json:"language"`
	Code     string `json:"code"`
}

type Response struct {
	Stream string `json:"stream"`
	Chunk  string `json:"chunk"`
}

type Runner struct {
	Client *Client

	Input  chan []byte
	Output chan []byte
	Errs   chan error
}

func (r *Runner) Read() {
	var (
		containerID string
		stop        chan bool
		err         error
	)
	for input := range r.Input {
		var config RunConfig
		if err := json.Unmarshal(input, &config); err != nil {
			r.handleError(err)
			continue
		}
		if containerID != "" {
			go r.stop(containerID, stop)
		}
		containerID, err = r.Client.Prepare(config.Language, config.Code)
		if err != nil {
			r.handleError(err)
			continue
		}
		stop = make(chan bool, 3)
		go r.execute(containerID, stop)
	}
	r.stop(containerID, stop)
	close(r.Output)
	close(r.Errs)
}

func (r *Runner) execute(containerID string, stop chan bool) {
	defer func() {
		if err := r.Client.Clean(containerID); err != nil {
			r.handleError(err)
		}
	}()
	status, err := r.Client.Execute(containerID, func(stdout, stderr io.Reader) {
		go r.broadcast("stdout", stdout, stop)
		r.broadcast("stderr", stderr, stop)
	})
	if err != nil {
		r.handleError(err)
		return
	}
	select {
	case <-stop:
	default:
		r.write("status", strconv.Itoa(status))
	}
}

func (r *Runner) stop(containerID string, stop chan bool) {
	for i := 0; i < 3; i++ {
		stop <- true
	}
	r.Client.Interrupt(containerID)
}

func (r *Runner) write(stream, chunk string) {
	response, err := json.Marshal(Response{Stream: stream, Chunk: chunk})
	if err != nil {
		r.handleError(err)
		return
	}
	r.Output <- response
}

func (r *Runner) broadcast(stream string, output io.Reader, stop chan bool) {
	var (
		reader = bufio.NewReader(output)
		buffer = make([]byte, 1024)
	)
	for {
		n, err := reader.Read(buffer)
		if err != nil {
			return
		}
		select {
		case <-stop:
			return
		default:
			if n > 0 {
				r.write(stream, string(buffer[0:n]))
			}
		}
	}
}

func (r *Runner) handleError(err error) {
	r.Errs <- err
	if err == ErrorProgramTooLarge || err == ErrorLanguageNotSpecified {
		r.write("error", err.Error())
	}
}
<commit_msg>add runner error generic message<commit_after>package runner

import (
	"bufio"
	"encoding/json"
	"io"
	"strconv"
)

type RunConfig struct {
	Language string `json:"language"`
	Code     string `json:"code"`
}

type Response struct {
	Stream string `json:"stream"`
	Chunk  string `json:"chunk"`
}

type Runner struct {
	Client *Client

	Input  chan []byte
	Output chan []byte
	Errs   chan error
}

func (r *Runner) Read() {
	var (
		containerID string
		stop        chan bool
		err         error
	)
	for input := range r.Input {
		var config RunConfig
		if err := json.Unmarshal(input, &config); err != nil {
			r.notifyError(err)
			continue
		}
		if containerID != "" {
			go r.stop(containerID, stop)
		}
		containerID, err = r.Client.Prepare(config.Language, config.Code)
		if err != nil {
			r.notifyError(err)
			continue
		}
		stop = make(chan bool, 3)
		go r.execute(containerID, stop)
	}
	r.stop(containerID, stop)
	close(r.Output)
	close(r.Errs)
}

func (r *Runner) execute(containerID string, stop chan bool) {
	defer func() {
		if err := r.Client.Clean(containerID); err != nil {
			r.Errs <- err
		}
	}()
	status, err := r.Client.Execute(containerID, func(stdout, stderr io.Reader) {
		go r.broadcast("stdout", stdout, stop)
		r.broadcast("stderr", stderr, stop)
	})
	if err != nil {
		r.notifyError(err)
		return
	}
	select {
	case <-stop:
	default:
		r.write("status", strconv.Itoa(status))
	}
}

func (r *Runner) stop(containerID string, stop chan bool) {
	for i := 0; i < 3; i++ {
		stop <- true
	}
	r.Client.Interrupt(containerID)
}

func (r *Runner) write(stream, chunk string) {
	response, err := json.Marshal(Response{Stream: stream, Chunk: chunk})
	if err != nil {
		r.Errs <- err
		return
	}
	r.Output <- response
}

func (r *Runner) broadcast(stream string, output io.Reader, stop chan bool) {
	var (
		reader = bufio.NewReader(output)
		buffer = make([]byte, 1024)
	)
	for {
		n, err := reader.Read(buffer)
		if err != nil {
			return
		}
		select {
		case <-stop:
			return
		default:
			if n > 0 {
				r.write(stream, string(buffer[0:n]))
			}
		}
	}
}

func (r *Runner) notifyError(err error) {
	r.Errs <- err
	if err == ErrorProgramTooLarge || err == ErrorLanguageNotSpecified {
		r.write("error", err.Error())
	} else {
		r.write("error", "An error occured.")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package search

import (
	"bytes"
	"fmt"
	"strings"
	"time"

	"camlistore.org/pkg/blobref"
)

type Result struct {
	BlobRef     *blobref.BlobRef
	Signer      *blobref.BlobRef // may be nil
	LastModTime int64            // seconds since epoch
}

// Results exists mostly for debugging, to provide a String method on
// a slice of Result.
type Results []*Result

func (s Results) String() string {
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "[%d search results: ", len(s))
	for _, r := range s {
		fmt.Fprintf(&buf, "{BlobRef: %s, Signer: %s, LastModTime: %d}",
			r.BlobRef, r.Signer, r.LastModTime)
	}
	buf.WriteString("]")
	return buf.String()
}

// TODO: move this to schema or something?
type Claim struct {
	BlobRef, Signer, Permanode *blobref.BlobRef

	Date time.Time
	Type string // "set-attribute", "add-attribute", etc

	// If an attribute modification
	Attr, Value string
}

func (c *Claim) String() string {
	return fmt.Sprintf(
		"search.Claim{BlobRef: %s, Signer: %s, Permanode: %s, Date: %s, Type: %s, Attr: %s, Value: %s}",
		c.BlobRef, c.Signer, c.Permanode, c.Date, c.Type, c.Attr, c.Value)
}

type ClaimList []*Claim

func (cl ClaimList) Len() int {
	return len(cl)
}

func (cl ClaimList) Less(i, j int) bool {
	// TODO: memoize Seconds in unexported Claim field
	return cl[i].Date.Unix() < cl[j].Date.Unix()
}

func (cl ClaimList) Swap(i, j int) {
	cl[i], cl[j] = cl[j], cl[i]
}

func (cl ClaimList) String() string {
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "[%d claims: ", len(cl))
	for _, r := range cl {
		buf.WriteString(r.String())
	}
	buf.WriteString("]")
	return buf.String()
}

type FileInfo struct {
	Size     int64  `json:"size"`
	FileName string `json:"fileName"`
	MimeType string `json:"mimeType"`
}

func (fi *FileInfo) IsImage() bool {
	return strings.HasPrefix(fi.MimeType, "image/")
}

type Path struct {
	Claim, Base, Target *blobref.BlobRef
	ClaimDate           string
	Suffix              string
}

func (p *Path) String() string {
	return fmt.Sprintf("Path{Claim: %v, %v; Base: %v + Suffix %q => Target %v}",
		p.Claim, p.ClaimDate, p.Base, p.Suffix, p.Target)
}

type PermanodeByAttrRequest struct {
	Signer *blobref.BlobRef

	// Attribute to search. currently supported: "tag", "title"
	// If FuzzyMatch is set, this can be blank to search all
	// attributes.
	Attribute string

	// The attribute value to find exactly (or roughly, if
	// FuzzyMatch is set)
	Query string

	FuzzyMatch bool // by default, an exact match is required
	MaxResults int  // optional max results
}

type Index interface {
	// dest must be closed, even when returning an error.
	// limit is <= 0 for default.  smallest possible default is 0
	GetRecentPermanodes(dest chan *Result,
		owner *blobref.BlobRef,
		limit int) error

	// SearchPermanodes finds permanodes matching the provided
	// request and sends unique permanode blobrefs to dest.
	// In particular, if request.FuzzyMatch is true, a fulltext
	// search is performed (if supported by the attribute(s))
	// instead of an exact match search.
	// Additionally, if request.Attribute is blank, all attributes
	// are searched (as fulltext), otherwise the search is
	// restricted  to the named attribute.
	//
	// dest is always closed, regardless of the error return value.
	SearchPermanodesWithAttr(dest chan<- *blobref.BlobRef,
		request *PermanodeByAttrRequest) error

	GetOwnerClaims(permaNode, owner *blobref.BlobRef) (ClaimList, error)

	// os.ErrNotExist should be returned if the blob isn't known
	GetBlobMimeType(blob *blobref.BlobRef) (mime string, size int64, err error)

	// ExistingFileSchemas returns 0 or more blobrefs of "bytes"
	// (TODO(bradfitz): or file?) schema blobs that represent the
	// bytes of a file given in bytesRef.  The file schema blobs
	// returned are not guaranteed to reference chunks that still
	// exist on the blobservers, though.  It's purely a hint for
	// clients to avoid uploads if possible.  Before re-using any
	// returned blobref they should be checked.
	//
	// Use case: a user drag & drops a large file onto their
	// browser to upload.  (imagine that "large" means anything
	// larger than a blobserver's max blob size) JavaScript can
	// first SHA-1 the large file locally, then send the
	// wholeFileRef to this call and see if they'd previously
	// uploaded the same file in the past.  If so, the upload
	// can be avoided if at least one of the returned schemaRefs
	// can be validated (with a validating HEAD request) to still
	// all exist on the blob server.
	ExistingFileSchemas(wholeFileRef *blobref.BlobRef) (schemaRefs []*blobref.BlobRef, err error)

	// Should return os.ErrNotExist if not found.
	GetFileInfo(fileRef *blobref.BlobRef) (*FileInfo, error)

	// Given an owner key, a camliType 'claim', 'attribute' name,
	// and specific 'value', find the most recent permanode that has
	// a corresponding 'set-attribute' claim attached.
	// Returns os.ErrNotExist if none is found.
	// Only attributes white-listed by IsIndexedAttribute are valid.
	PermanodeOfSignerAttrValue(signer *blobref.BlobRef, attr, val string) (*blobref.BlobRef, error)

	// PathsOfSignerTarget queries the index about "camliPath:"
	// URL-dispatch attributes.
	//
	// It returns a list of all the path claims that have been signed
	// by the provided signer and point at the given target.
	//
	// This is used when editing a permanode, to figure work up
	// the name resolution tree backwards ultimately to a
	// camliRoot permanode (which should know its base URL), and
	// then the complete URL(s) of a target can be found.
	PathsOfSignerTarget(signer, target *blobref.BlobRef) ([]*Path, error)

	// All Path claims for (signer, base, suffix)
	PathsLookup(signer, base *blobref.BlobRef, suffix string) ([]*Path, error)

	// Most recent Path claim for (signer, base, suffix) as of
	// provided time 'at', or most recent if 'at' is nil.
	PathLookup(signer, base *blobref.BlobRef, suffix string, at time.Time) (*Path, error)
}

// TODO(bradfitz): rename this? This is really about signer-attr-value
// (PermanodeOfSignerAttrValue), and not about indexed attributes in general.
func IsIndexedAttribute(attr string) bool {
	switch attr {
	case "camliRoot", "tag", "title":
		return true
	}
	return false
}

func IsFulltextAttribute(attr string) bool {
	switch attr {
	case "tag", "title":
		return true
	}
	return false
}
<commit_msg>add TODO<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package search

import (
	"bytes"
	"fmt"
	"strings"
	"time"

	"camlistore.org/pkg/blobref"
)

type Result struct {
	BlobRef     *blobref.BlobRef
	Signer      *blobref.BlobRef // may be nil
	LastModTime int64            // seconds since epoch
}

// Results exists mostly for debugging, to provide a String method on
// a slice of Result.
type Results []*Result

func (s Results) String() string {
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "[%d search results: ", len(s))
	for _, r := range s {
		fmt.Fprintf(&buf, "{BlobRef: %s, Signer: %s, LastModTime: %d}",
			r.BlobRef, r.Signer, r.LastModTime)
	}
	buf.WriteString("]")
	return buf.String()
}

// TODO: move this to schema or something?
type Claim struct {
	BlobRef, Signer, Permanode *blobref.BlobRef

	Date time.Time
	Type string // "set-attribute", "add-attribute", etc

	// If an attribute modification
	Attr, Value string
}

func (c *Claim) String() string {
	return fmt.Sprintf(
		"search.Claim{BlobRef: %s, Signer: %s, Permanode: %s, Date: %s, Type: %s, Attr: %s, Value: %s}",
		c.BlobRef, c.Signer, c.Permanode, c.Date, c.Type, c.Attr, c.Value)
}

type ClaimList []*Claim

func (cl ClaimList) Len() int {
	return len(cl)
}

func (cl ClaimList) Less(i, j int) bool {
	// TODO: memoize Seconds in unexported Claim field
	return cl[i].Date.Unix() < cl[j].Date.Unix()
}

func (cl ClaimList) Swap(i, j int) {
	cl[i], cl[j] = cl[j], cl[i]
}

func (cl ClaimList) String() string {
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "[%d claims: ", len(cl))
	for _, r := range cl {
		buf.WriteString(r.String())
	}
	buf.WriteString("]")
	return buf.String()
}

type FileInfo struct {
	Size     int64  `json:"size"`
	FileName string `json:"fileName"`
	MimeType string `json:"mimeType"`
}

func (fi *FileInfo) IsImage() bool {
	return strings.HasPrefix(fi.MimeType, "image/")
}

type Path struct {
	Claim, Base, Target *blobref.BlobRef
	ClaimDate           string
	Suffix              string
}

func (p *Path) String() string {
	return fmt.Sprintf("Path{Claim: %v, %v; Base: %v + Suffix %q => Target %v}",
		p.Claim, p.ClaimDate, p.Base, p.Suffix, p.Target)
}

type PermanodeByAttrRequest struct {
	Signer *blobref.BlobRef

	// Attribute to search. currently supported: "tag", "title"
	// If FuzzyMatch is set, this can be blank to search all
	// attributes.
	Attribute string

	// The attribute value to find exactly (or roughly, if
	// FuzzyMatch is set)
	Query string

	FuzzyMatch bool // by default, an exact match is required
	MaxResults int  // optional max results
}

type Index interface {
	// dest must be closed, even when returning an error.
	// limit is <= 0 for default.  smallest possible default is 0
	GetRecentPermanodes(dest chan *Result,
		owner *blobref.BlobRef,
		limit int) error

	// SearchPermanodes finds permanodes matching the provided
	// request and sends unique permanode blobrefs to dest.
	// In particular, if request.FuzzyMatch is true, a fulltext
	// search is performed (if supported by the attribute(s))
	// instead of an exact match search.
	// Additionally, if request.Attribute is blank, all attributes
	// are searched (as fulltext), otherwise the search is
	// restricted  to the named attribute.
	//
	// dest is always closed, regardless of the error return value.
	SearchPermanodesWithAttr(dest chan<- *blobref.BlobRef,
		request *PermanodeByAttrRequest) error

	GetOwnerClaims(permaNode, owner *blobref.BlobRef) (ClaimList, error)

	// os.ErrNotExist should be returned if the blob isn't known
	GetBlobMimeType(blob *blobref.BlobRef) (mime string, size int64, err error)

	// ExistingFileSchemas returns 0 or more blobrefs of "bytes"
	// (TODO(bradfitz): or file?) schema blobs that represent the
	// bytes of a file given in bytesRef.  The file schema blobs
	// returned are not guaranteed to reference chunks that still
	// exist on the blobservers, though.  It's purely a hint for
	// clients to avoid uploads if possible.  Before re-using any
	// returned blobref they should be checked.
	//
	// Use case: a user drag & drops a large file onto their
	// browser to upload.  (imagine that "large" means anything
	// larger than a blobserver's max blob size) JavaScript can
	// first SHA-1 the large file locally, then send the
	// wholeFileRef to this call and see if they'd previously
	// uploaded the same file in the past.  If so, the upload
	// can be avoided if at least one of the returned schemaRefs
	// can be validated (with a validating HEAD request) to still
	// all exist on the blob server.
	ExistingFileSchemas(wholeFileRef *blobref.BlobRef) (schemaRefs []*blobref.BlobRef, err error)

	// Should return os.ErrNotExist if not found.
	GetFileInfo(fileRef *blobref.BlobRef) (*FileInfo, error)

	// Given an owner key, a camliType 'claim', 'attribute' name,
	// and specific 'value', find the most recent permanode that has
	// a corresponding 'set-attribute' claim attached.
	// Returns os.ErrNotExist if none is found.
	// TODO(bradfitz): ErrNotExist here is a weird error message ("file" not found). change.
	// Only attributes white-listed by IsIndexedAttribute are valid.
	PermanodeOfSignerAttrValue(signer *blobref.BlobRef, attr, val string) (*blobref.BlobRef, error)

	// PathsOfSignerTarget queries the index about "camliPath:"
	// URL-dispatch attributes.
	//
	// It returns a list of all the path claims that have been signed
	// by the provided signer and point at the given target.
	//
	// This is used when editing a permanode, to figure work up
	// the name resolution tree backwards ultimately to a
	// camliRoot permanode (which should know its base URL), and
	// then the complete URL(s) of a target can be found.
	PathsOfSignerTarget(signer, target *blobref.BlobRef) ([]*Path, error)

	// All Path claims for (signer, base, suffix)
	PathsLookup(signer, base *blobref.BlobRef, suffix string) ([]*Path, error)

	// Most recent Path claim for (signer, base, suffix) as of
	// provided time 'at', or most recent if 'at' is nil.
	PathLookup(signer, base *blobref.BlobRef, suffix string, at time.Time) (*Path, error)
}

// TODO(bradfitz): rename this? This is really about signer-attr-value
// (PermanodeOfSignerAttrValue), and not about indexed attributes in general.
func IsIndexedAttribute(attr string) bool {
	switch attr {
	case "camliRoot", "tag", "title":
		return true
	}
	return false
}

func IsFulltextAttribute(attr string) bool {
	switch attr {
	case "tag", "title":
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package snapshot

import (
	"math"
	"time"

	crdv1 "github.com/kubernetes-incubator/external-storage/snapshot/pkg/apis/crd/v1"
	"github.com/libopenstorage/stork/pkg/rule"
	"github.com/portworx/sched-ops/k8s"
	"github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/wait"
)

const (
	defaultCmdExecutorImage              = "openstorage/cmdexecutor:0.1"
	cmdExecutorImageOverrideKey          = "stork.libopenstorage.org/cmdexecutor-image"
	storkServiceAccount                  = "stork-account"
	podsWithRunningCommandsKeyDeprecated = "stork/pods-with-running-cmds"
	podsWithRunningCommandsKey           = "stork.libopenstorage.org/pods-with-running-cmds"

	// constants
	perPodCommandExecTimeout = 900 // 15 minutes

	execPodCmdRetryInterval = 5 * time.Second
	execPodCmdRetryFactor   = 1
	execPodStepLow          = 12
	execPodStepMed          = 36
	execPodStepsHigh        = math.MaxInt32

	storkRuleAnnotationPrefixDeprecated = "stork.rule"
	storkRuleAnnotationPrefix           = "stork.libopenstorage.org"
	preSnapRuleAnnotationKey            = storkRuleAnnotationPrefix + "/pre-snapshot-rule"
	postSnapRuleAnnotationKey           = storkRuleAnnotationPrefix + "/post-snapshot-rule"
	preSnapRuleAnnotationKeyDeprecated  = storkRuleAnnotationPrefixDeprecated + "/pre-snapshot"
	postSnapRuleAnnotationKeyDeprecated = storkRuleAnnotationPrefixDeprecated + "/post-snapshot"
)

var ruleAnnotationKeyTypes = map[string]rule.Type{
	preSnapRuleAnnotationKey:            rule.PreExecRule,
	postSnapRuleAnnotationKey:           rule.PostExecRule,
	preSnapRuleAnnotationKeyDeprecated:  rule.PreExecRule,
	postSnapRuleAnnotationKeyDeprecated: rule.PostExecRule,
}

// pod is a simple type to encapsulate a pod's uid and namespace
type pod struct {
	uid       string
	namespace string
}

type podErrorResponse struct {
	pod v1.Pod
	err error
}

// commandTask tracks pods where commands for a taskID might still be running
type commandTask struct {
	taskID string
	pods   []pod
}

var execCmdBackoff = wait.Backoff{
	Duration: execPodCmdRetryInterval,
	Factor:   execPodCmdRetryFactor,
	Steps:    execPodStepsHigh,
}

var snapAPICallBackoff = wait.Backoff{
	Duration: 2 * time.Second,
	Factor:   1.5,
	Steps:    20,
}

// validateSnapRules validates the rules if they are present in the given snapshot's annotations
func validateSnapRules(snap *crdv1.VolumeSnapshot) error {
	if snap.Metadata.Annotations != nil {
		ruleAnnotations := []string{
			preSnapRuleAnnotationKey,
			postSnapRuleAnnotationKey,
			preSnapRuleAnnotationKeyDeprecated,
			preSnapRuleAnnotationKeyDeprecated,
		}
		for _, annotation := range ruleAnnotations {
			ruleName, present := snap.Metadata.Annotations[annotation]
			if present && len(ruleName) > 0 {
				r, err := k8s.Instance().GetRule(ruleName, snap.Metadata.Namespace)
				if err != nil {
					return err
				}
				err = rule.ValidateRule(r, ruleAnnotationKeyTypes[annotation])
				if err != nil {
					return err
				}
			}
		}
	}

	return nil
}

// ExecutePreSnapRule executes the pre snapshot rule. pvcs is a list of PVCs that are associated
// with the snapshot. It returns a channel which the caller can trigger to delete the termination of background commands
func ExecutePreSnapRule(snap *crdv1.VolumeSnapshot, pvcs []v1.PersistentVolumeClaim) (chan bool, error) {
	if snap.Metadata.Annotations != nil {
		ruleName, present := snap.Metadata.Annotations[preSnapRuleAnnotationKey]
		if !present {
			ruleName, present = snap.Metadata.Annotations[preSnapRuleAnnotationKeyDeprecated]
			if !present {
				return nil, nil
			}
		}
		r, err := k8s.Instance().GetRule(ruleName, snap.Metadata.Namespace)
		if err != nil {
			return nil, err
		}
		return rule.ExecuteRule(r, rule.PreExecRule, snap, pvcs)
	}
	return nil, nil
}

// ExecutePostSnapRule executes the post snapshot rule for the given snapshot. pvcs is a list of PVCs
// that are associated with the snapshot
func ExecutePostSnapRule(pvcs []v1.PersistentVolumeClaim, snap *crdv1.VolumeSnapshot) error {
	if snap.Metadata.Annotations != nil {
		ruleName, present := snap.Metadata.Annotations[postSnapRuleAnnotationKey]
		if !present {
			ruleName, present = snap.Metadata.Annotations[postSnapRuleAnnotationKeyDeprecated]
			if !present {
				return nil
			}
		}
		r, err := k8s.Instance().GetRule(ruleName, snap.Metadata.Namespace)
		if err != nil {
			return err
		}
		_, err = rule.ExecuteRule(r, rule.PostExecRule, snap, pvcs)
		return err
	}
	return nil
}

// performRuleRecovery terminates potential background commands running pods for
// the given snapshot
func performRuleRecovery() error {
	allSnaps, err := k8s.Instance().ListSnapshots(v1.NamespaceAll)
	if err != nil {
		logrus.Errorf("Failed to list all snapshots due to: %v. Will retry.", err)
		return err
	}

	if allSnaps == nil {
		return nil
	}

	var lastError error
	for _, snap := range allSnaps.Items {
		err := rule.PerformRuleRecovery(&snap)
		if err != nil {
			lastError = err
		}
	}
	return lastError
}
<commit_msg>Removed unused variables<commit_after>package snapshot

import (
	crdv1 "github.com/kubernetes-incubator/external-storage/snapshot/pkg/apis/crd/v1"
	"github.com/libopenstorage/stork/pkg/rule"
	"github.com/portworx/sched-ops/k8s"
	"github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
)

const (
	storkRuleAnnotationPrefixDeprecated = "stork.rule"
	storkRuleAnnotationPrefix           = "stork.libopenstorage.org"
	preSnapRuleAnnotationKey            = storkRuleAnnotationPrefix + "/pre-snapshot-rule"
	postSnapRuleAnnotationKey           = storkRuleAnnotationPrefix + "/post-snapshot-rule"
	preSnapRuleAnnotationKeyDeprecated  = storkRuleAnnotationPrefixDeprecated + "/pre-snapshot"
	postSnapRuleAnnotationKeyDeprecated = storkRuleAnnotationPrefixDeprecated + "/post-snapshot"
)

var ruleAnnotationKeyTypes = map[string]rule.Type{
	preSnapRuleAnnotationKey:            rule.PreExecRule,
	postSnapRuleAnnotationKey:           rule.PostExecRule,
	preSnapRuleAnnotationKeyDeprecated:  rule.PreExecRule,
	postSnapRuleAnnotationKeyDeprecated: rule.PostExecRule,
}

// validateSnapRules validates the rules if they are present in the given snapshot's annotations
func validateSnapRules(snap *crdv1.VolumeSnapshot) error {
	if snap.Metadata.Annotations != nil {
		ruleAnnotations := []string{
			preSnapRuleAnnotationKey,
			postSnapRuleAnnotationKey,
			preSnapRuleAnnotationKeyDeprecated,
			preSnapRuleAnnotationKeyDeprecated,
		}
		for _, annotation := range ruleAnnotations {
			ruleName, present := snap.Metadata.Annotations[annotation]
			if present && len(ruleName) > 0 {
				r, err := k8s.Instance().GetRule(ruleName, snap.Metadata.Namespace)
				if err != nil {
					return err
				}
				err = rule.ValidateRule(r, ruleAnnotationKeyTypes[annotation])
				if err != nil {
					return err
				}
			}
		}
	}

	return nil
}

// ExecutePreSnapRule executes the pre snapshot rule. pvcs is a list of PVCs that are associated
// with the snapshot. It returns a channel which the caller can trigger to delete the termination of background commands
func ExecutePreSnapRule(snap *crdv1.VolumeSnapshot, pvcs []v1.PersistentVolumeClaim) (chan bool, error) {
	if snap.Metadata.Annotations != nil {
		ruleName, present := snap.Metadata.Annotations[preSnapRuleAnnotationKey]
		if !present {
			ruleName, present = snap.Metadata.Annotations[preSnapRuleAnnotationKeyDeprecated]
			if !present {
				return nil, nil
			}
		}
		r, err := k8s.Instance().GetRule(ruleName, snap.Metadata.Namespace)
		if err != nil {
			return nil, err
		}
		return rule.ExecuteRule(r, rule.PreExecRule, snap, pvcs)
	}
	return nil, nil
}

// ExecutePostSnapRule executes the post snapshot rule for the given snapshot. pvcs is a list of PVCs
// that are associated with the snapshot
func ExecutePostSnapRule(pvcs []v1.PersistentVolumeClaim, snap *crdv1.VolumeSnapshot) error {
	if snap.Metadata.Annotations != nil {
		ruleName, present := snap.Metadata.Annotations[postSnapRuleAnnotationKey]
		if !present {
			ruleName, present = snap.Metadata.Annotations[postSnapRuleAnnotationKeyDeprecated]
			if !present {
				return nil
			}
		}
		r, err := k8s.Instance().GetRule(ruleName, snap.Metadata.Namespace)
		if err != nil {
			return err
		}
		_, err = rule.ExecuteRule(r, rule.PostExecRule, snap, pvcs)
		return err
	}
	return nil
}

// performRuleRecovery terminates potential background commands running pods for
// the given snapshot
func performRuleRecovery() error {
	allSnaps, err := k8s.Instance().ListSnapshots(v1.NamespaceAll)
	if err != nil {
		logrus.Errorf("Failed to list all snapshots due to: %v. Will retry.", err)
		return err
	}

	if allSnaps == nil {
		return nil
	}

	var lastError error
	for _, snap := range allSnaps.Items {
		err := rule.PerformRuleRecovery(&snap)
		if err != nil {
			lastError = err
		}
	}
	return lastError
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package test

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"v.io/x/devtools/internal/collect"
	"v.io/x/devtools/internal/test"
	"v.io/x/devtools/internal/tool"
	"v.io/x/devtools/internal/xunit"
)

const (
	numAttempts = 3
)

// vanadiumBootstrap runs a test of Vanadium bootstrapping.
func vanadiumBootstrap(ctx *tool.Context, testName string, _ ...Opt) (_ *test.Result, e error) {
	// Initialize the test.
	cleanup, err := initTest(ctx, testName, nil)
	if err != nil {
		return nil, internalTestError{err, "Init"}
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	// Create a new temporary V23_ROOT.
	oldRoot := os.Getenv("V23_ROOT")
	defer collect.Error(func() error { return os.Setenv("V23_ROOT", oldRoot) }, &e)
	tmpDir, err := ctx.Run().TempDir("", "")
	if err != nil {
		return nil, internalTestError{err, "TempDir"}
	}
	defer collect.Error(func() error { return ctx.Run().RemoveAll(tmpDir) }, &e)

	root := filepath.Join(tmpDir, "root")
	if err := os.Setenv("V23_ROOT", root); err != nil {
		return nil, internalTestError{err, "Setenv"}
	}

	// Run the setup script.
	var out bytes.Buffer
	opts := ctx.Run().Opts()
	opts.Stdout = io.MultiWriter(opts.Stdout, &out)
	opts.Stderr = io.MultiWriter(opts.Stderr, &out)
	opts.Env["PATH"] = strings.Replace(os.Getenv("PATH"), filepath.Join(oldRoot, "devtools", "bin"), "", -1)
	for i := 1; i <= numAttempts; i++ {
		if i > 1 {
			fmt.Fprintf(ctx.Stdout(), "Attempt %d/%d:\n", i, numAttempts)
		}
		if err = ctx.Run().CommandWithOpts(opts, filepath.Join(oldRoot, "scripts", "bootstrap")); err == nil {
			break
		}
	}
	if err != nil {
		// Create xUnit report.
		if err := xunit.CreateFailureReport(ctx, testName, "VanadiumGo", "bootstrap", "Vanadium bootstrapping failed", out.String()); err != nil {
			return nil, err
		}
		return &test.Result{Status: test.Failed}, nil
	}
	return &test.Result{Status: test.Passed}, nil
}
<commit_msg>TBR devtools/v23: the bootstrap target assumes that the PATH element containing the v23 binary is $V23_ROOT/devtools/bin. This is not the case when presubmit tests are executed as of go/vcl/9919. This CL provides a fix for this.<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package test

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"v.io/x/devtools/internal/collect"
	"v.io/x/devtools/internal/test"
	"v.io/x/devtools/internal/tool"
	"v.io/x/devtools/internal/xunit"
)

const (
	numAttempts = 3
)

// vanadiumBootstrap runs a test of Vanadium bootstrapping.
func vanadiumBootstrap(ctx *tool.Context, testName string, _ ...Opt) (_ *test.Result, e error) {
	// Initialize the test.
	cleanup, err := initTest(ctx, testName, nil)
	if err != nil {
		return nil, internalTestError{err, "Init"}
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	// Create a new temporary V23_ROOT.
	oldRoot := os.Getenv("V23_ROOT")
	defer collect.Error(func() error { return os.Setenv("V23_ROOT", oldRoot) }, &e)
	tmpDir, err := ctx.Run().TempDir("", "")
	if err != nil {
		return nil, internalTestError{err, "TempDir"}
	}
	defer collect.Error(func() error { return ctx.Run().RemoveAll(tmpDir) }, &e)

	root := filepath.Join(tmpDir, "root")
	if err := os.Setenv("V23_ROOT", root); err != nil {
		return nil, internalTestError{err, "Setenv"}
	}

	// Run the setup script.
	var out bytes.Buffer
	opts := ctx.Run().Opts()
	opts.Stdout = io.MultiWriter(opts.Stdout, &out)
	opts.Stderr = io.MultiWriter(opts.Stderr, &out)
	// Find the PATH element containing the "v23" binary and remove it.
	v23Path, err := exec.LookPath("v23")
	if err != nil {
		return nil, internalTestError{err, "LookPath"}
	}
	opts.Env["PATH"] = strings.Replace(os.Getenv("PATH"), filepath.Dir(v23Path), "", -1)
	for i := 1; i <= numAttempts; i++ {
		if i > 1 {
			fmt.Fprintf(ctx.Stdout(), "Attempt %d/%d:\n", i, numAttempts)
		}
		if err = ctx.Run().CommandWithOpts(opts, filepath.Join(oldRoot, "scripts", "bootstrap")); err == nil {
			break
		}
	}
	if err != nil {
		// Create xUnit report.
		if err := xunit.CreateFailureReport(ctx, testName, "VanadiumGo", "bootstrap", "Vanadium bootstrapping failed", out.String()); err != nil {
			return nil, err
		}
		return &test.Result{Status: test.Failed}, nil
	}
	return &test.Result{Status: test.Passed}, nil
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hydraproxy

import (
	"bytes"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"

	"github.com/gorilla/mux" /* copybara-comment */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/apis/hydraapi" /* copybara-comment: hydraapi */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/storage" /* copybara-comment: storage */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/test/fakehydra" /* copybara-comment: fakehydra */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/test/httptestclient" /* copybara-comment: httptestclient */

	tpb "github.com/GoogleCloudPlatform/healthcare-federated-access-services/proto/store/tokens" /* copybara-comment: go_proto */
)

func TestOAuthToken_code(t *testing.T) {
	s, f := setupOAuthTokenTest(t)

	wantCode := "code"
	sendExchangeToken(s, "code", wantCode, "")

	wantURL := "https://example.com/oauth2/token"
	if f.ExchangeTokenReqURL != wantURL {
		t.Errorf("ExchangeTokenReqURL in dest server = %s want %s", f.ExchangeTokenReqURL, wantURL)
	}

	if f.ExchangeTokenReq.Get("code") != wantCode {
		t.Errorf("ExchangeTokenReq[code] = %s want %s", f.ExchangeTokenReq.Get("code"), wantCode)
	}
}

func TestOAuthToken_refresh(t *testing.T) {
	s, f := setupOAuthTokenTest(t)

	wantRefreshToken := "reftok"
	sub := "sub"

	f.IntrospectionResp = &hydraapi.Introspection{
		Subject: sub,
		Extra:   map[string]interface{}{"tid": "token-id"},
	}

	sendExchangeToken(s, "refresh_token", "", wantRefreshToken)

	if f.IntrospectionReqToken != wantRefreshToken {
		t.Errorf("IntrospectionReqToken = %s want %s", f.IntrospectionReqToken, wantRefreshToken)
	}

	wantURL := "https://example.com/oauth2/token"
	if f.ExchangeTokenReqURL != wantURL {
		t.Errorf("ExchangeTokenReqURL = %s want %s", f.ExchangeTokenReqURL, wantURL)
	}

	if f.ExchangeTokenReq.Get("refresh_token") != wantRefreshToken {
		t.Errorf("ExchangeTokenReq[refresh_token] = %s want %s", f.ExchangeTokenReq.Get("refresh_token"), wantRefreshToken)
	}
}

func TestOAuthToken_refresh_deleted(t *testing.T) {
	s, f := setupOAuthTokenTest(t)

	sub := "sub"
	tokenID := "token-id"
	f.IntrospectionResp = &hydraapi.Introspection{
		Subject: sub,
		Extra:   map[string]interface{}{"tid": tokenID},
	}

	pending := &tpb.PendingDeleteToken{}
	s.store.Write(storage.PendingDeleteTokenDatatype, storage.DefaultRealm, sub, tokenID, storage.LatestRev, pending, nil)

	wantRefreshToken := "reftok"
	resp := sendExchangeToken(s, "refresh_token", "", wantRefreshToken)

	if resp.StatusCode != http.StatusUnauthorized {
		t.Errorf("StatusCode = %d want %d", resp.StatusCode, http.StatusUnauthorized)
	}

	if f.ExchangeTokenReq != nil {
		t.Errorf("ExchangeTokenReq = %v want nil", f.ExchangeTokenReq)
	}

	if f.RevokeTokenReq != wantRefreshToken {
		t.Errorf("RevokeTokenReq = %s want %s", f.RevokeTokenReq, wantRefreshToken)
	}

	if err := s.store.Read(storage.PendingDeleteTokenDatatype, storage.DefaultRealm, sub, tokenID, storage.LatestRev, pending); !storage.ErrNotFound(err) {
		t.Errorf("PendingDeleteToken should delete got value=%v err=%v", pending, err)
	}
}

func TestOAuthToken_refresh_error(t *testing.T) {
	s, f := setupOAuthTokenTest(t)

	tests := []struct {
		name          string
		introspectReq *hydraapi.Introspection
		introspectErr *hydraapi.GenericError
		wantStatus    int
	}{
		{
			name:          "no tid",
			introspectReq: &hydraapi.Introspection{},
			wantStatus:    http.StatusInternalServerError,
		},
		{
			name: "tid not string",
			introspectReq: &hydraapi.Introspection{
				Extra: map[string]interface{}{"tid": 1},
			},
			wantStatus: http.StatusInternalServerError,
		},
		{
			name: "introspect err",
			introspectErr: &hydraapi.GenericError{
				Code: http.StatusUnauthorized,
			},
			// TODO: should convert hydra err to status err.
			wantStatus: http.StatusInternalServerError,
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			f.Clear()
			f.IntrospectionResp = tc.introspectReq
			f.IntrospectionErr = tc.introspectErr

			resp := sendExchangeToken(s, "refresh_token", "code", "tok")

			if resp.StatusCode != tc.wantStatus {
				t.Errorf("StatusCode = %d want %d", resp.StatusCode, tc.wantStatus)
			}

			if f.ExchangeTokenReq != nil {
				t.Errorf("ExchangeTokenReq = %v want nil", f.ExchangeTokenReq)
			}
		})
	}
}

func sendExchangeToken(s *Service, grantType, code, refreshToken string) *http.Response {
	target := "https://example.com/oauth2/token"
	q := url.Values{}
	q.Set("grant_type", grantType)
	if len(code) > 0 {
		q.Set("code", code)
	}
	if len(refreshToken) > 0 {
		q.Set("refresh_token", refreshToken)
	}

	r := httptest.NewRequest(http.MethodPost, target, bytes.NewBufferString(q.Encode()))
	r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	w := httptest.NewRecorder()

	s.HydraOAuthToken(w, r)

	return w.Result()
}

func setupOAuthTokenTest(t *testing.T) (*Service, *fakehydra.Server) {
	t.Helper()

	store := storage.NewMemoryStorage("ic-min", "testdata/config")

	router := mux.NewRouter()
	h := fakehydra.New(router)
	client := httptestclient.New(router)

	s, err := New(client, "http://hydra-admin.example.com", "http://hydra-pub-internal.example.com", store)
	if err != nil {
		t.Fatalf("New service failed: %v", err)
	}
	s.hydraPublicURLProxy.Transport = client.Transport

	return s, h
}
<commit_msg>Add test cover: hydra revoke token response error<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hydraproxy

import (
	"bytes"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"

	"github.com/gorilla/mux" /* copybara-comment */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/apis/hydraapi" /* copybara-comment: hydraapi */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/storage" /* copybara-comment: storage */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/test/fakehydra" /* copybara-comment: fakehydra */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/test/httptestclient" /* copybara-comment: httptestclient */

	tpb "github.com/GoogleCloudPlatform/healthcare-federated-access-services/proto/store/tokens" /* copybara-comment: go_proto */
)

func TestOAuthToken_code(t *testing.T) {
	s, f := setupOAuthTokenTest(t)

	wantCode := "code"
	sendExchangeToken(s, "code", wantCode, "")

	wantURL := "https://example.com/oauth2/token"
	if f.ExchangeTokenReqURL != wantURL {
		t.Errorf("ExchangeTokenReqURL in dest server = %s, want %s", f.ExchangeTokenReqURL, wantURL)
	}

	if f.ExchangeTokenReq.Get("code") != wantCode {
		t.Errorf("ExchangeTokenReq[code] = %s, want %s", f.ExchangeTokenReq.Get("code"), wantCode)
	}
}

func TestOAuthToken_refresh(t *testing.T) {
	s, f := setupOAuthTokenTest(t)

	wantRefreshToken := "reftok"
	sub := "sub"

	f.IntrospectionResp = &hydraapi.Introspection{
		Subject: sub,
		Extra:   map[string]interface{}{"tid": "token-id"},
	}

	sendExchangeToken(s, "refresh_token", "", wantRefreshToken)

	if f.IntrospectionReqToken != wantRefreshToken {
		t.Errorf("IntrospectionReqToken = %s, want %s", f.IntrospectionReqToken, wantRefreshToken)
	}

	wantURL := "https://example.com/oauth2/token"
	if f.ExchangeTokenReqURL != wantURL {
		t.Errorf("ExchangeTokenReqURL = %s, want %s", f.ExchangeTokenReqURL, wantURL)
	}

	if f.ExchangeTokenReq.Get("refresh_token") != wantRefreshToken {
		t.Errorf("ExchangeTokenReq[refresh_token] = %s, want %s", f.ExchangeTokenReq.Get("refresh_token"), wantRefreshToken)
	}
}

func TestOAuthToken_refresh_deleted(t *testing.T) {
	s, f := setupOAuthTokenTest(t)

	sub := "sub"
	tokenID := "token-id"
	f.IntrospectionResp = &hydraapi.Introspection{
		Subject: sub,
		Extra:   map[string]interface{}{"tid": tokenID},
	}

	pending := &tpb.PendingDeleteToken{}
	s.store.Write(storage.PendingDeleteTokenDatatype, storage.DefaultRealm, sub, tokenID, storage.LatestRev, pending, nil)

	wantRefreshToken := "reftok"
	resp := sendExchangeToken(s, "refresh_token", "", wantRefreshToken)

	if resp.StatusCode != http.StatusUnauthorized {
		t.Errorf("StatusCode = %d, want %d", resp.StatusCode, http.StatusUnauthorized)
	}

	if f.ExchangeTokenReq != nil {
		t.Errorf("ExchangeTokenReq = %v, want nil", f.ExchangeTokenReq)
	}

	if f.RevokeTokenReq != wantRefreshToken {
		t.Errorf("RevokeTokenReq = %s, want %s", f.RevokeTokenReq, wantRefreshToken)
	}

	if err := s.store.Read(storage.PendingDeleteTokenDatatype, storage.DefaultRealm, sub, tokenID, storage.LatestRev, pending); !storage.ErrNotFound(err) {
		t.Errorf("PendingDeleteToken should delete got value=%v err=%v", pending, err)
	}
}

func TestOAuthToken_refresh_error(t *testing.T) {
	s, f := setupOAuthTokenTest(t)

	tests := []struct {
		name          string
		introspectReq *hydraapi.Introspection
		introspectErr *hydraapi.GenericError
		wantStatus    int
	}{
		{
			name:          "no tid",
			introspectReq: &hydraapi.Introspection{},
			wantStatus:    http.StatusInternalServerError,
		},
		{
			name: "tid not string",
			introspectReq: &hydraapi.Introspection{
				Extra: map[string]interface{}{"tid": 1},
			},
			wantStatus: http.StatusInternalServerError,
		},
		{
			name: "introspect err",
			introspectErr: &hydraapi.GenericError{
				Code: http.StatusUnauthorized,
			},
			// TODO: should convert hydra err to status err.
			wantStatus: http.StatusInternalServerError,
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			f.Clear()
			f.IntrospectionResp = tc.introspectReq
			f.IntrospectionErr = tc.introspectErr

			resp := sendExchangeToken(s, "refresh_token", "code", "tok")

			if resp.StatusCode != tc.wantStatus {
				t.Errorf("StatusCode = %d, want %d", resp.StatusCode, tc.wantStatus)
			}

			if f.ExchangeTokenReq != nil {
				t.Errorf("ExchangeTokenReq = %v, want nil", f.ExchangeTokenReq)
			}
		})
	}
}

func TestOAuthToken_refresh_deleted_err(t *testing.T) {
	s, f := setupOAuthTokenTest(t)

	sub := "sub"
	tokenID := "token-id"
	f.IntrospectionResp = &hydraapi.Introspection{
		Subject: sub,
		Extra:   map[string]interface{}{"tid": tokenID},
	}
	f.RevokeTokenErr = &hydraapi.GenericError{
		Code: http.StatusUnauthorized,
	}

	pending := &tpb.PendingDeleteToken{}
	s.store.Write(storage.PendingDeleteTokenDatatype, storage.DefaultRealm, sub, tokenID, storage.LatestRev, pending, nil)

	resp := sendExchangeToken(s, "refresh_token", "", "tok")

	if resp.StatusCode != http.StatusInternalServerError {
		t.Errorf("StatusCode = %d, want %d", resp.StatusCode, http.StatusInternalServerError)
	}

	if f.ExchangeTokenReq != nil {
		t.Errorf("ExchangeTokenReq = %v, want nil", f.ExchangeTokenReq)
	}

	if err := s.store.Read(storage.PendingDeleteTokenDatatype, storage.DefaultRealm, sub, tokenID, storage.LatestRev, pending); err != nil {
		t.Errorf("PendingDeleteToken should not delete")
	}
}

func sendExchangeToken(s *Service, grantType, code, refreshToken string) *http.Response {
	target := "https://example.com/oauth2/token"
	q := url.Values{}
	q.Set("grant_type", grantType)
	if len(code) > 0 {
		q.Set("code", code)
	}
	if len(refreshToken) > 0 {
		q.Set("refresh_token", refreshToken)
	}

	r := httptest.NewRequest(http.MethodPost, target, bytes.NewBufferString(q.Encode()))
	r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	w := httptest.NewRecorder()

	s.HydraOAuthToken(w, r)

	return w.Result()
}

func setupOAuthTokenTest(t *testing.T) (*Service, *fakehydra.Server) {
	t.Helper()

	store := storage.NewMemoryStorage("ic-min", "testdata/config")

	router := mux.NewRouter()
	h := fakehydra.New(router)
	client := httptestclient.New(router)

	s, err := New(client, "http://hydra-admin.example.com", "http://hydra-pub-internal.example.com", store)
	if err != nil {
		t.Fatalf("New service failed: %v", err)
	}
	s.hydraPublicURLProxy.Transport = client.Transport

	return s, h
}
<|endoftext|>


<commit_before>package accumulate

const testVersion = 1

func Accumulate([]string, func(string) string) []string
<commit_msg>Solution for exercise Accumulate<commit_after>// Package accumulate contains the implementation of the Exercism's go exercise 'accumulate'.
package accumulate

// testVersion is the current version of the test
const testVersion = 1

// Accumulate applies the converter to each element of the input slice and return the mapped slice
func Accumulate(input []string, converter func(string) string) []string {
	// if a nil or empty slice then return it
	if input == nil || len(input) == 0 {
		return input
	}

	// otherwise iterate over the elements of the slice and apply the converter function to each
	var mappedInput []string
	for _, element := range input {
		mappedInput = append(mappedInput, converter(element))
	}

	// return the mapped input
	return mappedInput
}
<|endoftext|>


<commit_before>package loadbalancer

import (
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/xtracdev/xavi/config"
	"os"
	"strings"
)

//PreferLocalLoadBalancer is a load balancer that looks to route traffic locally before sending
//traffic remote. Local is defined as servers that have the same hostname as that the load
//balancer is deployed on. The load balancer keeps two server pools - a local pool and a remote
//pool. Local traffic is used and distributed via a round robin load balancer when a local server
//is available, other wise remote servers are used.
type PreferLocalLoadBalancer struct {
	BackendName   string
	LocalServers  LoadBalancer
	RemoteServers LoadBalancer
}

//PreferLocalLoadBalancerFactory is used to instantiate PreferLocalLoadBalancer instances.
type PreferLocalLoadBalancerFactory struct{}

func splitHostFromAddress(server string) string {
	return strings.Split(server, ".")[0]
}

func sameServer(hostname string, servername string) bool {
	lcHost := strings.ToLower(hostname)
	lcServer := strings.ToLower(servername)

	lendiff := len(lcHost) - len(lcServer)

	if lendiff == 0 {
		return lcHost == lcServer
	} else if lendiff > 0 {
		return strings.Index(lcHost, lcServer) == 0 && lcHost[len(lcServer)] == '.'
	} else {
		return strings.Index(lcServer, lcHost) == 0 && lcServer[len(lcHost)] == '.'
	}
}

//isLocal is a predicate that indicates in the given server is local or not, based on matching the
//hostname returned by os.Hostname. Note that we consider localhost as a special case, treating it as
//local.
func isLocal(server string) (bool, error) {

	if server == "localhost" {
		return true, nil
	}

	//For Virtual App Handle environments, os.Hostname will not give the virtual host
	//name, which is what we need to treat as the hostname. In these environments, we
	//treat the value of the APPHANDLE environment variable as the hostname
	appHandle := os.Getenv("APPHANDLE")
	if appHandle != "" {
		return sameServer(appHandle, server), nil
	}

	host, err := os.Hostname()
	if err != nil {
		return false, err
	}

	return sameServer(host, server), nil
}

//partitionServers partitions a slice of servers into a slice of local servers and a slice of
//remote servers
func partitionServers(servers []config.ServerConfig) ([]config.ServerConfig, []config.ServerConfig, error) {
	var localServers, remoteServers []config.ServerConfig
	for _, s := range servers {
		local, err := isLocal(s.Address)
		if err != nil {
			return nil, nil, err
		}

		if local {
			localServers = append(localServers, s)
		} else {
			remoteServers = append(remoteServers, s)
		}
	}

	return localServers, remoteServers, nil
}

//NewLoadBalancer creates an instance of PreferLocalLoadBalancer
func (pl *PreferLocalLoadBalancerFactory) NewLoadBalancer(backendName, caCertPath string, servers []config.ServerConfig) (LoadBalancer, error) {

	if backendName == "" {
		return nil, fmt.Errorf("Expected non-empty backend name")
	}

	if len(servers) == 0 {
		return nil, fmt.Errorf("Expected at least one server in servers argument")
	}

	log.Info("Creating prefer-local load balancer for backend ", backendName, " with ", len(servers), " servers")

	var preferLocalLB PreferLocalLoadBalancer
	var roundRobinFactory LoadBalancerFactory = new(RoundRobinLoadBalancerFactory)

	localServers, remoteServers, err := partitionServers(servers)
	if err != nil {
		return nil, err
	}

	preferLocalLB.BackendName = backendName

	if len(localServers) > 0 {
		localLB, err := roundRobinFactory.NewLoadBalancer(backendName, caCertPath, localServers)
		if err != nil {
			return nil, err
		}

		preferLocalLB.LocalServers = localLB
	} else {
		log.Warn("No local servers specified in prefer-local configuration")
	}

	if len(remoteServers) > 0 {
		remoteLB, err := roundRobinFactory.NewLoadBalancer(backendName, caCertPath, remoteServers)
		if err != nil {
			return nil, err
		}

		preferLocalLB.RemoteServers = remoteLB
	} else {
		log.Warn("No remote servers specified in prefer-local configuration")
	}

	return &preferLocalLB, nil
}

//getConnectAddress returns a connection address from the given load balancer via a call
//to the load balancer's GetConnectAddress method if it is not nil, otherwise returning
//an error
func getConnectAddress(poolname string, lb LoadBalancer) (string, error) {
	if lb == nil {
		return "", fmt.Errorf("No servers in %s pool configuration", poolname)
	}

	return lb.GetConnectAddress()
}

//markEndpointDown marks the endpoint down for the given load balancer via a call
//to the load balancer's MarkEndpointDown method if it is not nil, otherwise returning
//an error
func markEndpointDown(poolname string, endpoint string, lb LoadBalancer) error {
	if lb == nil {
		return fmt.Errorf("No servers in %s pool configuration", poolname)
	}

	return lb.MarkEndpointDown(endpoint)
}

//markEndpointUp marks the endpoint down for the given load balancer via a call
//to the load balancer's MarkEndpointUp method if it is not nil, otherwise returning
//an error
func markEndpointUp(poolname string, endpoint string, lb LoadBalancer) error {
	if lb == nil {
		return fmt.Errorf("No servers in %s pool configuration", poolname)
	}

	return lb.MarkEndpointUp(endpoint)
}

//GetConnectAddress returns the connect address for the PreferLocalLoadBalancer instance
func (pl *PreferLocalLoadBalancer) GetConnectAddress() (string, error) {
	address, err := getConnectAddress("local server", pl.LocalServers)
	switch err {
	case nil:
		return address, err
	default:
		log.Warn(fmt.Sprintf("No local address found: %s. Will look for remote address.", err.Error()))
		return getConnectAddress("remote server", pl.RemoteServers)
	}
}

//MarkEndpointDown marks the given endpoint down for the PreferLocalLoadBalancer instance
func (pl *PreferLocalLoadBalancer) MarkEndpointDown(endpoint string) error {
	err := markEndpointDown("local server", endpoint, pl.LocalServers)
	switch err {
	case nil:
		return nil
	default:
		return markEndpointDown("remote server", endpoint, pl.RemoteServers)
	}
}

//MarkEndpointUp marks the given endpoint down for the PreferLocalLoadBalancer instance
func (pl *PreferLocalLoadBalancer) MarkEndpointUp(endpoint string) error {
	err := markEndpointUp("local server", endpoint, pl.LocalServers)
	switch err {
	case nil:
		return nil
	default:
		return markEndpointUp("remote server", endpoint, pl.RemoteServers)
	}
}

//GetEndpoints returns the endpoints associated with the load balancer, partitioning
//the set of endpoints into healthy and unhealthy endpoints
func (pl *PreferLocalLoadBalancer) GetEndpoints() ([]string, []string) {
	var healthy, unhealthy []string

	if pl.LocalServers != nil {
		lh, luh := pl.LocalServers.GetEndpoints()
		fmt.Printf("%v\n", lh)
		fmt.Printf("%v\n", luh)
		healthy = append(healthy, lh...)
		unhealthy = append(unhealthy, luh...)
	}

	if pl.RemoteServers != nil {
		rh, ruh := pl.RemoteServers.GetEndpoints()
		healthy = append(healthy, rh...)
		unhealthy = append(unhealthy, ruh...)
	}

	return healthy, unhealthy
}
<commit_msg>Removed extraneous logging<commit_after>package loadbalancer

import (
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/xtracdev/xavi/config"
	"os"
	"strings"
)

//PreferLocalLoadBalancer is a load balancer that looks to route traffic locally before sending
//traffic remote. Local is defined as servers that have the same hostname as that the load
//balancer is deployed on. The load balancer keeps two server pools - a local pool and a remote
//pool. Local traffic is used and distributed via a round robin load balancer when a local server
//is available, other wise remote servers are used.
type PreferLocalLoadBalancer struct {
	BackendName   string
	LocalServers  LoadBalancer
	RemoteServers LoadBalancer
}

//PreferLocalLoadBalancerFactory is used to instantiate PreferLocalLoadBalancer instances.
type PreferLocalLoadBalancerFactory struct{}

func splitHostFromAddress(server string) string {
	return strings.Split(server, ".")[0]
}

func sameServer(hostname string, servername string) bool {
	lcHost := strings.ToLower(hostname)
	lcServer := strings.ToLower(servername)

	lendiff := len(lcHost) - len(lcServer)

	if lendiff == 0 {
		return lcHost == lcServer
	} else if lendiff > 0 {
		return strings.Index(lcHost, lcServer) == 0 && lcHost[len(lcServer)] == '.'
	} else {
		return strings.Index(lcServer, lcHost) == 0 && lcServer[len(lcHost)] == '.'
	}
}

//isLocal is a predicate that indicates in the given server is local or not, based on matching the
//hostname returned by os.Hostname. Note that we consider localhost as a special case, treating it as
//local.
func isLocal(server string) (bool, error) {

	if server == "localhost" {
		return true, nil
	}

	//For Virtual App Handle environments, os.Hostname will not give the virtual host
	//name, which is what we need to treat as the hostname. In these environments, we
	//treat the value of the APPHANDLE environment variable as the hostname
	appHandle := os.Getenv("APPHANDLE")
	if appHandle != "" {
		return sameServer(appHandle, server), nil
	}

	host, err := os.Hostname()
	if err != nil {
		return false, err
	}

	return sameServer(host, server), nil
}

//partitionServers partitions a slice of servers into a slice of local servers and a slice of
//remote servers
func partitionServers(servers []config.ServerConfig) ([]config.ServerConfig, []config.ServerConfig, error) {
	var localServers, remoteServers []config.ServerConfig
	for _, s := range servers {
		local, err := isLocal(s.Address)
		if err != nil {
			return nil, nil, err
		}

		if local {
			localServers = append(localServers, s)
		} else {
			remoteServers = append(remoteServers, s)
		}
	}

	return localServers, remoteServers, nil
}

//NewLoadBalancer creates an instance of PreferLocalLoadBalancer
func (pl *PreferLocalLoadBalancerFactory) NewLoadBalancer(backendName, caCertPath string, servers []config.ServerConfig) (LoadBalancer, error) {

	if backendName == "" {
		return nil, fmt.Errorf("Expected non-empty backend name")
	}

	if len(servers) == 0 {
		return nil, fmt.Errorf("Expected at least one server in servers argument")
	}

	log.Info("Creating prefer-local load balancer for backend ", backendName, " with ", len(servers), " servers")

	var preferLocalLB PreferLocalLoadBalancer
	var roundRobinFactory LoadBalancerFactory = new(RoundRobinLoadBalancerFactory)

	localServers, remoteServers, err := partitionServers(servers)
	if err != nil {
		return nil, err
	}

	preferLocalLB.BackendName = backendName

	if len(localServers) > 0 {
		localLB, err := roundRobinFactory.NewLoadBalancer(backendName, caCertPath, localServers)
		if err != nil {
			return nil, err
		}

		preferLocalLB.LocalServers = localLB
	} else {
		log.Warn("No local servers specified in prefer-local configuration")
	}

	if len(remoteServers) > 0 {
		remoteLB, err := roundRobinFactory.NewLoadBalancer(backendName, caCertPath, remoteServers)
		if err != nil {
			return nil, err
		}

		preferLocalLB.RemoteServers = remoteLB
	} else {
		log.Warn("No remote servers specified in prefer-local configuration")
	}

	return &preferLocalLB, nil
}

//getConnectAddress returns a connection address from the given load balancer via a call
//to the load balancer's GetConnectAddress method if it is not nil, otherwise returning
//an error
func getConnectAddress(poolname string, lb LoadBalancer) (string, error) {
	if lb == nil {
		return "", fmt.Errorf("No servers in %s pool configuration", poolname)
	}

	return lb.GetConnectAddress()
}

//markEndpointDown marks the endpoint down for the given load balancer via a call
//to the load balancer's MarkEndpointDown method if it is not nil, otherwise returning
//an error
func markEndpointDown(poolname string, endpoint string, lb LoadBalancer) error {
	if lb == nil {
		return fmt.Errorf("No servers in %s pool configuration", poolname)
	}

	return lb.MarkEndpointDown(endpoint)
}

//markEndpointUp marks the endpoint down for the given load balancer via a call
//to the load balancer's MarkEndpointUp method if it is not nil, otherwise returning
//an error
func markEndpointUp(poolname string, endpoint string, lb LoadBalancer) error {
	if lb == nil {
		return fmt.Errorf("No servers in %s pool configuration", poolname)
	}

	return lb.MarkEndpointUp(endpoint)
}

//GetConnectAddress returns the connect address for the PreferLocalLoadBalancer instance
func (pl *PreferLocalLoadBalancer) GetConnectAddress() (string, error) {
	address, err := getConnectAddress("local server", pl.LocalServers)
	switch err {
	case nil:
		return address, err
	default:
		log.Warn(fmt.Sprintf("No local address found: %s. Will look for remote address.", err.Error()))
		return getConnectAddress("remote server", pl.RemoteServers)
	}
}

//MarkEndpointDown marks the given endpoint down for the PreferLocalLoadBalancer instance
func (pl *PreferLocalLoadBalancer) MarkEndpointDown(endpoint string) error {
	err := markEndpointDown("local server", endpoint, pl.LocalServers)
	switch err {
	case nil:
		return nil
	default:
		return markEndpointDown("remote server", endpoint, pl.RemoteServers)
	}
}

//MarkEndpointUp marks the given endpoint down for the PreferLocalLoadBalancer instance
func (pl *PreferLocalLoadBalancer) MarkEndpointUp(endpoint string) error {
	err := markEndpointUp("local server", endpoint, pl.LocalServers)
	switch err {
	case nil:
		return nil
	default:
		return markEndpointUp("remote server", endpoint, pl.RemoteServers)
	}
}

//GetEndpoints returns the endpoints associated with the load balancer, partitioning
//the set of endpoints into healthy and unhealthy endpoints
func (pl *PreferLocalLoadBalancer) GetEndpoints() ([]string, []string) {
	var healthy, unhealthy []string

	if pl.LocalServers != nil {
		lh, luh := pl.LocalServers.GetEndpoints()
		healthy = append(healthy, lh...)
		unhealthy = append(unhealthy, luh...)
	}

	if pl.RemoteServers != nil {
		rh, ruh := pl.RemoteServers.GetEndpoints()
		healthy = append(healthy, rh...)
		unhealthy = append(unhealthy, ruh...)
	}

	return healthy, unhealthy
}
<|endoftext|>


<commit_before>package network

import (
	"context"
	"fmt"
	"io"
	"time"

	bsmsg "github.com/ipfs/go-ipfs/exchange/bitswap/message"

	inet "gx/ipfs/QmNa31VPzC561NWwRsJLE7nGYZYuuD2QfpK2b1q9BK54J1/go-libp2p-net"
	cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid"
	routing "gx/ipfs/QmPR2JzfKd9poHx9XBhzoFeBBC31ZM3W5iUPKJZWyaoZZm/go-libp2p-routing"
	pstore "gx/ipfs/QmPgDWmTmuzvP7QE5zwo1TmjbJme9pmZHNujB2453jkCTr/go-libp2p-peerstore"
	logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
	ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr"
	peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer"
	ggio "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/io"
	host "gx/ipfs/Qmc1XhrFEiSeBNn3mpfg6gEuYCt5im2gYmNVmncsvmpeAk/go-libp2p-host"
)

var log = logging.Logger("bitswap_network")

var sendMessageTimeout = time.Minute * 10

// NewFromIpfsHost returns a BitSwapNetwork supported by underlying IPFS host
func NewFromIpfsHost(host host.Host, r routing.ContentRouting) BitSwapNetwork {
	bitswapNetwork := impl{
		host:    host,
		routing: r,
	}
	host.SetStreamHandler(ProtocolBitswap, bitswapNetwork.handleNewStream)
	host.SetStreamHandler(ProtocolBitswapOne, bitswapNetwork.handleNewStream)
	host.SetStreamHandler(ProtocolBitswapNoVers, bitswapNetwork.handleNewStream)
	host.Network().Notify((*netNotifiee)(&bitswapNetwork))
	// TODO: StopNotify.

	return &bitswapNetwork
}

// impl transforms the ipfs network interface, which sends and receives
// NetMessage objects, into the bitswap network interface.
type impl struct {
	host    host.Host
	routing routing.ContentRouting

	// inbound messages from the network are forwarded to the receiver
	receiver Receiver
}

type streamMessageSender struct {
	s inet.Stream
}

func (s *streamMessageSender) Close() error {
	return s.s.Close()
}

func (s *streamMessageSender) Reset() error {
	return s.s.Reset()
}

func (s *streamMessageSender) SendMsg(ctx context.Context, msg bsmsg.BitSwapMessage) error {
	return msgToStream(ctx, s.s, msg)
}

func msgToStream(ctx context.Context, s inet.Stream, msg bsmsg.BitSwapMessage) error {
	deadline := time.Now().Add(sendMessageTimeout)
	if dl, ok := ctx.Deadline(); ok {
		deadline = dl
	}

	if err := s.SetWriteDeadline(deadline); err != nil {
		log.Warningf("error setting deadline: %s", err)
	}

	switch s.Protocol() {
	case ProtocolBitswap:
		if err := msg.ToNetV1(s); err != nil {
			log.Debugf("error: %s", err)
			return err
		}
	case ProtocolBitswapOne, ProtocolBitswapNoVers:
		if err := msg.ToNetV0(s); err != nil {
			log.Debugf("error: %s", err)
			return err
		}
	default:
		return fmt.Errorf("unrecognized protocol on remote: %s", s.Protocol())
	}

	if err := s.SetWriteDeadline(time.Time{}); err != nil {
		log.Warningf("error resetting deadline: %s", err)
	}
	return nil
}

func (bsnet *impl) NewMessageSender(ctx context.Context, p peer.ID) (MessageSender, error) {
	s, err := bsnet.newStreamToPeer(ctx, p)
	if err != nil {
		return nil, err
	}

	return &streamMessageSender{s: s}, nil
}

func (bsnet *impl) newStreamToPeer(ctx context.Context, p peer.ID) (inet.Stream, error) {

	// first, make sure we're connected.
	// if this fails, we cannot connect to given peer.
	//TODO(jbenet) move this into host.NewStream?
	if err := bsnet.host.Connect(ctx, pstore.PeerInfo{ID: p}); err != nil {
		return nil, err
	}

	return bsnet.host.NewStream(ctx, p, ProtocolBitswap, ProtocolBitswapOne, ProtocolBitswapNoVers)
}

func (bsnet *impl) SendMessage(
	ctx context.Context,
	p peer.ID,
	outgoing bsmsg.BitSwapMessage) error {

	s, err := bsnet.newStreamToPeer(ctx, p)
	if err != nil {
		return err
	}

	err = msgToStream(ctx, s, outgoing)
	if err != nil {
		s.Reset()
	} else {
		s.Close()
	}
	return err
}

func (bsnet *impl) SetDelegate(r Receiver) {
	bsnet.receiver = r
}

func (bsnet *impl) ConnectTo(ctx context.Context, p peer.ID) error {
	return bsnet.host.Connect(ctx, pstore.PeerInfo{ID: p})
}

// FindProvidersAsync returns a channel of providers for the given key
func (bsnet *impl) FindProvidersAsync(ctx context.Context, k *cid.Cid, max int) <-chan peer.ID {

	// Since routing queries are expensive, give bitswap the peers to which we
	// have open connections. Note that this may cause issues if bitswap starts
	// precisely tracking which peers provide certain keys. This optimization
	// would be misleading. In the long run, this may not be the most
	// appropriate place for this optimization, but it won't cause any harm in
	// the short term.
	connectedPeers := bsnet.host.Network().Peers()
	out := make(chan peer.ID, len(connectedPeers)) // just enough buffer for these connectedPeers
	for _, id := range connectedPeers {
		if id == bsnet.host.ID() {
			continue // ignore self as provider
		}
		out <- id
	}

	go func() {
		defer close(out)
		providers := bsnet.routing.FindProvidersAsync(ctx, k, max)
		for info := range providers {
			if info.ID == bsnet.host.ID() {
				continue // ignore self as provider
			}
			bsnet.host.Peerstore().AddAddrs(info.ID, info.Addrs, pstore.TempAddrTTL)
			select {
			case <-ctx.Done():
				return
			case out <- info.ID:
			}
		}
	}()
	return out
}

// Provide provides the key to the network
func (bsnet *impl) Provide(ctx context.Context, k *cid.Cid) error {
	return bsnet.routing.Provide(ctx, k, true)
}

// handleNewStream receives a new stream from the network.
func (bsnet *impl) handleNewStream(s inet.Stream) {
	defer s.Close()

	if bsnet.receiver == nil {
		s.Reset()
		return
	}

	reader := ggio.NewDelimitedReader(s, inet.MessageSizeMax)
	for {
		received, err := bsmsg.FromPBReader(reader)
		if err != nil {
			if err != io.EOF {
				s.Reset()
				go bsnet.receiver.ReceiveError(err)
				log.Debugf("bitswap net handleNewStream from %s error: %s", s.Conn().RemotePeer(), err)
			}
			return
		}

		p := s.Conn().RemotePeer()
		ctx := context.Background()
		log.Debugf("bitswap net handleNewStream from %s", s.Conn().RemotePeer())
		bsnet.receiver.ReceiveMessage(ctx, p, received)
	}
}

type netNotifiee impl

func (nn *netNotifiee) impl() *impl {
	return (*impl)(nn)
}

func (nn *netNotifiee) Connected(n inet.Network, v inet.Conn) {
	nn.impl().receiver.PeerConnected(v.RemotePeer())
}

func (nn *netNotifiee) Disconnected(n inet.Network, v inet.Conn) {
	nn.impl().receiver.PeerDisconnected(v.RemotePeer())
}

func (nn *netNotifiee) OpenedStream(n inet.Network, v inet.Stream) {}
func (nn *netNotifiee) ClosedStream(n inet.Network, v inet.Stream) {}
func (nn *netNotifiee) Listen(n inet.Network, a ma.Multiaddr)      {}
func (nn *netNotifiee) ListenClose(n inet.Network, a ma.Multiaddr) {}
<commit_msg>NewStream now creates a connection if necessary<commit_after>package network

import (
	"context"
	"fmt"
	"io"
	"time"

	bsmsg "github.com/ipfs/go-ipfs/exchange/bitswap/message"

	inet "gx/ipfs/QmNa31VPzC561NWwRsJLE7nGYZYuuD2QfpK2b1q9BK54J1/go-libp2p-net"
	cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid"
	routing "gx/ipfs/QmPR2JzfKd9poHx9XBhzoFeBBC31ZM3W5iUPKJZWyaoZZm/go-libp2p-routing"
	pstore "gx/ipfs/QmPgDWmTmuzvP7QE5zwo1TmjbJme9pmZHNujB2453jkCTr/go-libp2p-peerstore"
	logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
	ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr"
	peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer"
	ggio "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/io"
	host "gx/ipfs/Qmc1XhrFEiSeBNn3mpfg6gEuYCt5im2gYmNVmncsvmpeAk/go-libp2p-host"
)

var log = logging.Logger("bitswap_network")

var sendMessageTimeout = time.Minute * 10

// NewFromIpfsHost returns a BitSwapNetwork supported by underlying IPFS host
func NewFromIpfsHost(host host.Host, r routing.ContentRouting) BitSwapNetwork {
	bitswapNetwork := impl{
		host:    host,
		routing: r,
	}
	host.SetStreamHandler(ProtocolBitswap, bitswapNetwork.handleNewStream)
	host.SetStreamHandler(ProtocolBitswapOne, bitswapNetwork.handleNewStream)
	host.SetStreamHandler(ProtocolBitswapNoVers, bitswapNetwork.handleNewStream)
	host.Network().Notify((*netNotifiee)(&bitswapNetwork))
	// TODO: StopNotify.

	return &bitswapNetwork
}

// impl transforms the ipfs network interface, which sends and receives
// NetMessage objects, into the bitswap network interface.
type impl struct {
	host    host.Host
	routing routing.ContentRouting

	// inbound messages from the network are forwarded to the receiver
	receiver Receiver
}

type streamMessageSender struct {
	s inet.Stream
}

func (s *streamMessageSender) Close() error {
	return s.s.Close()
}

func (s *streamMessageSender) Reset() error {
	return s.s.Reset()
}

func (s *streamMessageSender) SendMsg(ctx context.Context, msg bsmsg.BitSwapMessage) error {
	return msgToStream(ctx, s.s, msg)
}

func msgToStream(ctx context.Context, s inet.Stream, msg bsmsg.BitSwapMessage) error {
	deadline := time.Now().Add(sendMessageTimeout)
	if dl, ok := ctx.Deadline(); ok {
		deadline = dl
	}

	if err := s.SetWriteDeadline(deadline); err != nil {
		log.Warningf("error setting deadline: %s", err)
	}

	switch s.Protocol() {
	case ProtocolBitswap:
		if err := msg.ToNetV1(s); err != nil {
			log.Debugf("error: %s", err)
			return err
		}
	case ProtocolBitswapOne, ProtocolBitswapNoVers:
		if err := msg.ToNetV0(s); err != nil {
			log.Debugf("error: %s", err)
			return err
		}
	default:
		return fmt.Errorf("unrecognized protocol on remote: %s", s.Protocol())
	}

	if err := s.SetWriteDeadline(time.Time{}); err != nil {
		log.Warningf("error resetting deadline: %s", err)
	}
	return nil
}

func (bsnet *impl) NewMessageSender(ctx context.Context, p peer.ID) (MessageSender, error) {
	s, err := bsnet.newStreamToPeer(ctx, p)
	if err != nil {
		return nil, err
	}

	return &streamMessageSender{s: s}, nil
}

func (bsnet *impl) newStreamToPeer(ctx context.Context, p peer.ID) (inet.Stream, error) {
	return bsnet.host.NewStream(ctx, p, ProtocolBitswap, ProtocolBitswapOne, ProtocolBitswapNoVers)
}

func (bsnet *impl) SendMessage(
	ctx context.Context,
	p peer.ID,
	outgoing bsmsg.BitSwapMessage) error {

	s, err := bsnet.newStreamToPeer(ctx, p)
	if err != nil {
		return err
	}

	err = msgToStream(ctx, s, outgoing)
	if err != nil {
		s.Reset()
	} else {
		s.Close()
	}
	return err
}

func (bsnet *impl) SetDelegate(r Receiver) {
	bsnet.receiver = r
}

func (bsnet *impl) ConnectTo(ctx context.Context, p peer.ID) error {
	return bsnet.host.Connect(ctx, pstore.PeerInfo{ID: p})
}

// FindProvidersAsync returns a channel of providers for the given key
func (bsnet *impl) FindProvidersAsync(ctx context.Context, k *cid.Cid, max int) <-chan peer.ID {

	// Since routing queries are expensive, give bitswap the peers to which we
	// have open connections. Note that this may cause issues if bitswap starts
	// precisely tracking which peers provide certain keys. This optimization
	// would be misleading. In the long run, this may not be the most
	// appropriate place for this optimization, but it won't cause any harm in
	// the short term.
	connectedPeers := bsnet.host.Network().Peers()
	out := make(chan peer.ID, len(connectedPeers)) // just enough buffer for these connectedPeers
	for _, id := range connectedPeers {
		if id == bsnet.host.ID() {
			continue // ignore self as provider
		}
		out <- id
	}

	go func() {
		defer close(out)
		providers := bsnet.routing.FindProvidersAsync(ctx, k, max)
		for info := range providers {
			if info.ID == bsnet.host.ID() {
				continue // ignore self as provider
			}
			bsnet.host.Peerstore().AddAddrs(info.ID, info.Addrs, pstore.TempAddrTTL)
			select {
			case <-ctx.Done():
				return
			case out <- info.ID:
			}
		}
	}()
	return out
}

// Provide provides the key to the network
func (bsnet *impl) Provide(ctx context.Context, k *cid.Cid) error {
	return bsnet.routing.Provide(ctx, k, true)
}

// handleNewStream receives a new stream from the network.
func (bsnet *impl) handleNewStream(s inet.Stream) {
	defer s.Close()

	if bsnet.receiver == nil {
		s.Reset()
		return
	}

	reader := ggio.NewDelimitedReader(s, inet.MessageSizeMax)
	for {
		received, err := bsmsg.FromPBReader(reader)
		if err != nil {
			if err != io.EOF {
				s.Reset()
				go bsnet.receiver.ReceiveError(err)
				log.Debugf("bitswap net handleNewStream from %s error: %s", s.Conn().RemotePeer(), err)
			}
			return
		}

		p := s.Conn().RemotePeer()
		ctx := context.Background()
		log.Debugf("bitswap net handleNewStream from %s", s.Conn().RemotePeer())
		bsnet.receiver.ReceiveMessage(ctx, p, received)
	}
}

type netNotifiee impl

func (nn *netNotifiee) impl() *impl {
	return (*impl)(nn)
}

func (nn *netNotifiee) Connected(n inet.Network, v inet.Conn) {
	nn.impl().receiver.PeerConnected(v.RemotePeer())
}

func (nn *netNotifiee) Disconnected(n inet.Network, v inet.Conn) {
	nn.impl().receiver.PeerDisconnected(v.RemotePeer())
}

func (nn *netNotifiee) OpenedStream(n inet.Network, v inet.Stream) {}
func (nn *netNotifiee) ClosedStream(n inet.Network, v inet.Stream) {}
func (nn *netNotifiee) Listen(n inet.Network, a ma.Multiaddr)      {}
func (nn *netNotifiee) ListenClose(n inet.Network, a ma.Multiaddr) {}
<|endoftext|>


<commit_before>package translator

// func getDeclarations(unit *cc.TranslationUnit) []cc.Declaration {

// }
<commit_msg>AST walking prototype, getting type declarations for simple cases works already.<commit_after>package translator

import "github.com/cznic/c/internal/cc"

func (t *Translator) walkAST(unit *cc.TranslationUnit) ([]CTypeDecl, error) {
	var declarations []CTypeDecl

	next, decl, err := walkUnit(unit)
	if err != nil {
		return nil, err
	} else if decl != nil {
		declarations = append(declarations, *decl)
	}

	for next != nil {
		next, decl, err = walkUnit(next)
		if err != nil {
			return nil, err
		} else if decl != nil {
			declarations = append(declarations, *decl)
		}
	}
	return declarations, nil
}

func walkUnit(unit *cc.TranslationUnit) (next *cc.TranslationUnit, decl *CTypeDecl, err error) {
	if unit == nil {
		return
	}
	if unit.ExternalDeclaration != nil {
		decl, err = walkDeclaration(unit.ExternalDeclaration.Declaration)
	}
	next = unit.TranslationUnit
	return
}

func walkDeclaration(decl *cc.Declaration) (*CTypeDecl, error) {
	if decl == nil {
		return nil, nil
	}

	spec := &CTypeSpec{}
	next := collectDeclarationSpec(decl.DeclarationSpecifiers, spec)
	for next != nil {
		next = collectDeclarationSpec(next, spec)
	}

	ctdecl := &CTypeDecl{
		Pos:  decl.Token.Pos(),
		Spec: spec,
	}

	// TODO
	return ctdecl, nil
}

func collectDeclarationSpec(declSpec *cc.DeclarationSpecifiers, spec *CTypeSpec) (next *cc.DeclarationSpecifiers) {
	if declSpec == nil {
		return nil
	}
	switch declSpec.Case {
	case 0: // StorageClassSpecifier DeclarationSpecifiersOpt
		next = declSpec.DeclarationSpecifiersOpt.DeclarationSpecifiers
	case 2: // TypeQualifier DeclarationSpecifiersOpt
		spec.Const = (declSpec.TypeQualifier.Case == 0)
		next = declSpec.DeclarationSpecifiersOpt.DeclarationSpecifiers
	case 1: // TypeSpecifier DeclarationSpecifiersOpt
		collectTypeSpec(declSpec.TypeSpecifier, spec)
		if declSpec.DeclarationSpecifiersOpt != nil {
			next = declSpec.DeclarationSpecifiersOpt.DeclarationSpecifiers
		}
	}
	return
}

func collectTypeSpec(typeSpec *cc.TypeSpecifier, spec *CTypeSpec) {
	if typeSpec == nil {
		return
	}
	switch typeSpec.Case {
	case 0:
		spec.Base = "void"
	case 1:
		spec.Base = "char"
	case 2:
		spec.Short = true
	case 3:
		spec.Base = "int"
	case 4:
		if spec.Long {
			spec.Base = "long"
		} else {
			spec.Long = true
		}
	case 5:
		spec.Base = "float"
	case 6:
		spec.Base = "double"
	// case 7:
	case 8:
		spec.Unsigned = true
	case 9:
		spec.Base = "_Bool"
	case 10:
		spec.Base = "_Complex"
	case 11:
		spec.Struct = true
	// case 12: TODO enums
	case 13:
		spec.Base = string(typeSpec.Token.S())
	}
}
<|endoftext|>