<commit_before>package lfs

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/cheggaaa/pb"
	"github.com/rubyist/tracerx"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
)

const (
	mediaType = "application/vnd.git-lfs+json; charset-utf-8"
)

var (
	lfsMediaTypeRE             = regexp.MustCompile(`\Aapplication/vnd\.git\-lfs\+json(;|\z)`)
	mediaMediaTypeRE           = regexp.MustCompile(`\Aapplication/json(;|\z)`)
	objectRelationDoesNotExist = errors.New("relation does not exist")
	hiddenHeaders              = map[string]bool{
		"Authorization": true,
	}

	// 401 and 403 print the same default error message
	defaultErrors = map[int]string{
		400: "Client error: %s",
		401: "Authorization error: %s\nCheck that you have proper access to the repository",
		404: "Repository or object not found: %s\nCheck that it exists and that you have proper access to it",
		500: "Server error: %s",
	}
)

type objectResource struct {
	Oid   string                   `json:"oid,omitempty"`
	Size  int64                    `json:"size,omitempty"`
	Links map[string]*linkRelation `json:"_links,omitempty"`
}

func (o *objectResource) NewRequest(relation, method string) (*http.Request, Creds, error) {
	rel, ok := o.Rel(relation)
	if !ok {
		return nil, nil, objectRelationDoesNotExist
	}

	req, creds, err := newClientRequest(method, rel.Href)
	if err != nil {
		return nil, nil, err
	}

	for h, v := range rel.Header {
		req.Header.Set(h, v)
	}

	return req, creds, nil
}

func (o *objectResource) Rel(name string) (*linkRelation, bool) {
	if o.Links == nil {
		return nil, false
	}

	rel, ok := o.Links[name]
	return rel, ok
}

type linkRelation struct {
	Href   string            `json:"href"`
	Header map[string]string `json:"header,omitempty"`
}

type ClientError struct {
	Message          string `json:"message"`
	DocumentationUrl string `json:"documentation_url,omitempty"`
	RequestId        string `json:"request_id,omitempty"`
}

func (e *ClientError) Error() string {
	msg := e.Message
	if len(e.DocumentationUrl) > 0 {
		msg += "\nDocs: " + e.DocumentationUrl
	}
	if len(e.RequestId) > 0 {
		msg += "\nRequest ID: " + e.RequestId
	}
	return msg
}

func Download(oid string) (io.ReadCloser, int64, *WrappedError) {
	req, creds, err := newApiRequest("GET", oid)
	if err != nil {
		return nil, 0, Error(err)
	}

	res, obj, wErr := doApiRequest(req, creds)
	if wErr != nil {
		return nil, 0, wErr
	}

	req, creds, err = obj.NewRequest("download", "GET")
	if err != nil {
		return nil, 0, Error(err)
	}

	res, wErr = doHttpRequest(req, creds)
	if wErr != nil {
		return nil, 0, wErr
	}

	return res.Body, res.ContentLength, nil
}

func Upload(oidPath, filename string, cb CopyCallback) *WrappedError {
	oid := filepath.Base(oidPath)
	file, err := os.Open(oidPath)
	if err != nil {
		return Error(err)
	}
	defer file.Close()

	stat, err := file.Stat()
	if err != nil {
		return Error(err)
	}

	reqObj := &objectResource{
		Oid:  oid,
		Size: stat.Size(),
	}

	by, err := json.Marshal(reqObj)
	if err != nil {
		return Error(err)
	}

	req, creds, err := newApiRequest("POST", "")
	if err != nil {
		return Error(err)
	}

	req.Header.Set("Content-Type", mediaType)
	req.Header.Set("Content-Length", strconv.Itoa(len(by)))
	req.ContentLength = int64(len(by))
	req.Body = ioutil.NopCloser(bytes.NewReader(by))

	tracerx.Printf("api: uploading %s (%s)", filename, oid)
	res, obj, wErr := doApiRequest(req, creds)
	if wErr != nil {
		return wErr
	}

	if res.StatusCode == 200 {
		return nil
	}

	req, creds, err = obj.NewRequest("upload", "PUT")
	if err != nil {
		return Error(err)
	}

	if len(req.Header.Get("Content-Type")) == 0 {
		req.Header.Set("Content-Type", "application/octet-stream")
	}
	req.Header.Set("Content-Length", strconv.FormatInt(reqObj.Size, 10))
	req.ContentLength = reqObj.Size

	reader := &CallbackReader{
		C:         cb,
		TotalSize: reqObj.Size,
		Reader:    file,
	}

	bar := pb.New64(reqObj.Size)
	bar.SetUnits(pb.U_BYTES)
	bar.Start()

	req.Body = ioutil.NopCloser(bar.NewProxyReader(reader))

	res, wErr = doHttpRequest(req, creds)
	if wErr != nil {
		return wErr
	}

	if res.StatusCode > 299 {
		return Errorf(nil, "Invalid status for %s %s: %d", req.Method, req.URL, res.StatusCode)
	}

	req, creds, err = obj.NewRequest("verify", "POST")
	if err == objectRelationDoesNotExist {
		return nil
	} else if err != nil {
		return Error(err)
	}

	req.Header.Set("Content-Type", mediaType)
	req.Header.Set("Content-Length", strconv.Itoa(len(by)))
	req.ContentLength = int64(len(by))
	req.Body = ioutil.NopCloser(bytes.NewReader(by))
	_, wErr = doHttpRequest(req, creds)

	return wErr
}

func doHttpRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError) {
	res, err := DoHTTP(Config, req)

	var wErr *WrappedError

	if err != nil {
		wErr = Errorf(err, "Error for %s %s", res.Request.Method, res.Request.URL)
	} else {
		if creds != nil {
			saveCredentials(creds, res)
		}

		wErr = handleResponse(res)
	}

	if wErr != nil {
		if res != nil {
			setErrorResponseContext(wErr, res)
		} else {
			setErrorRequestContext(wErr, req)
		}
	}

	return res, wErr
}

func doApiRequest(req *http.Request, creds Creds) (*http.Response, *objectResource, *WrappedError) {
	res, wErr := doHttpRequest(req, creds)
	if wErr != nil {
		return res, nil, wErr
	}

	obj := &objectResource{}
	wErr = decodeApiResponse(res, obj)

	if wErr != nil {
		setErrorResponseContext(wErr, res)
	}

	return res, obj, wErr
}

func handleResponse(res *http.Response) *WrappedError {
	if res.StatusCode < 400 {
		return nil
	}

	defer func() {
		io.Copy(ioutil.Discard, res.Body)
		res.Body.Close()
	}()

	cliErr := &ClientError{}
	wErr := decodeApiResponse(res, cliErr)
	if wErr == nil {
		if len(cliErr.Message) == 0 {
			wErr = defaultError(res)
		} else {
			wErr = Error(cliErr)
		}
	}

	wErr.Panic = res.StatusCode > 499 && res.StatusCode != 501 && res.StatusCode != 509
	return wErr
}

func decodeApiResponse(res *http.Response, obj interface{}) *WrappedError {
	ctype := res.Header.Get("Content-Type")
	if !(lfsMediaTypeRE.MatchString(ctype) || mediaMediaTypeRE.MatchString(ctype)) {
		return nil
	}

	err := json.NewDecoder(res.Body).Decode(obj)
	if err != nil {
		return Errorf(err, "Unable to parse HTTP response for %s %s", res.Request.Method, res.Request.URL)
	}

	return nil
}

func defaultError(res *http.Response) *WrappedError {
	var msgFmt string

	if f, ok := defaultErrors[res.StatusCode]; ok {
		msgFmt = f
	} else if res.StatusCode < 500 {
		msgFmt = defaultErrors[400] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
	} else {
		msgFmt = defaultErrors[500] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
	}

	return Error(fmt.Errorf(msgFmt, res.Request.URL))
}

func saveCredentials(creds Creds, res *http.Response) {
	if creds == nil {
		return
	}

	if res.StatusCode < 300 {
		execCreds(creds, "approve")
	} else if res.StatusCode == 401 {
		execCreds(creds, "reject")
	}
}

func newApiRequest(method, oid string) (*http.Request, Creds, error) {
	u, err := Config.ObjectUrl(oid)
	if err != nil {
		return nil, nil, err
	}

	req, creds, err := newClientRequest(method, u.String())
	if err == nil {
		req.Header.Set("Accept", mediaType)
	}
	return req, creds, err
}

func newClientRequest(method, rawurl string) (*http.Request, Creds, error) {
	req, err := http.NewRequest(method, rawurl, nil)
	if err != nil {
		return req, nil, err
	}

	req.Header.Set("User-Agent", UserAgent)
	creds, err := getCreds(req)
	return req, creds, err
}

func getCreds(req *http.Request) (Creds, error) {
	if len(req.Header.Get("Authorization")) > 0 {
		return nil, nil
	}

	apiUrl, err := Config.ObjectUrl("")
	if err != nil {
		return nil, err
	}

	if req.URL.Scheme == apiUrl.Scheme &&
		req.URL.Host == apiUrl.Host {
		creds, err := credentials(req.URL)
		if err != nil {
			return nil, err
		}

		token := fmt.Sprintf("%s:%s", creds["username"], creds["password"])
		auth := "Basic " + base64.URLEncoding.EncodeToString([]byte(token))
		req.Header.Set("Authorization", auth)
		return creds, nil
	}

	return nil, nil
}

func setErrorRequestContext(err *WrappedError, req *http.Request) {
	err.Set("Endpoint", Config.Endpoint())
	err.Set("URL", fmt.Sprintf("%s %s", req.Method, req.URL.String()))
	setErrorHeaderContext(err, "Response", req.Header)
}

func setErrorResponseContext(err *WrappedError, res *http.Response) {
	err.Set("Status", res.Status)
	setErrorHeaderContext(err, "Request", res.Header)
	setErrorRequestContext(err, res.Request)
}

func setErrorHeaderContext(err *WrappedError, prefix string, head http.Header) {
	for key, _ := range head {
		contextKey := fmt.Sprintf("%s:%s", prefix, key)
		if _, skip := hiddenHeaders[key]; skip {
			err.Set(contextKey, "--")
		} else {
			err.Set(contextKey, head.Get(key))
		}
	}
}

func init() {
	defaultErrors[403] = defaultErrors[401]
}
<commit_msg>ンンンンン ンンンン<commit_after>package lfs

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/cheggaaa/pb"
	"github.com/rubyist/tracerx"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
)

const (
	mediaType = "application/vnd.git-lfs+json; charset-utf-8"
)

var (
	lfsMediaTypeRE             = regexp.MustCompile(`\Aapplication/vnd\.git\-lfs\+json(;|\z)`)
	mediaMediaTypeRE           = regexp.MustCompile(`\Aapplication/json(;|\z)`)
	objectRelationDoesNotExist = errors.New("relation does not exist")
	hiddenHeaders              = map[string]bool{
		"Authorization": true,
	}

	// 401 and 403 print the same default error message
	defaultErrors = map[int]string{
		400: "Client error: %s",
		401: "Authorization error: %s\nCheck that you have proper access to the repository",
		404: "Repository or object not found: %s\nCheck that it exists and that you have proper access to it",
		500: "Server error: %s",
	}
)

type objectResource struct {
	Oid   string                   `json:"oid,omitempty"`
	Size  int64                    `json:"size,omitempty"`
	Links map[string]*linkRelation `json:"_links,omitempty"`
}

func (o *objectResource) NewRequest(relation, method string) (*http.Request, Creds, error) {
	rel, ok := o.Rel(relation)
	if !ok {
		return nil, nil, objectRelationDoesNotExist
	}

	req, creds, err := newClientRequest(method, rel.Href)
	if err != nil {
		return nil, nil, err
	}

	for h, v := range rel.Header {
		req.Header.Set(h, v)
	}

	return req, creds, nil
}

func (o *objectResource) Rel(name string) (*linkRelation, bool) {
	if o.Links == nil {
		return nil, false
	}

	rel, ok := o.Links[name]
	return rel, ok
}

type linkRelation struct {
	Href   string            `json:"href"`
	Header map[string]string `json:"header,omitempty"`
}

type ClientError struct {
	Message          string `json:"message"`
	DocumentationUrl string `json:"documentation_url,omitempty"`
	RequestId        string `json:"request_id,omitempty"`
}

func (e *ClientError) Error() string {
	msg := e.Message
	if len(e.DocumentationUrl) > 0 {
		msg += "\nDocs: " + e.DocumentationUrl
	}
	if len(e.RequestId) > 0 {
		msg += "\nRequest ID: " + e.RequestId
	}
	return msg
}

func Download(oid string) (io.ReadCloser, int64, *WrappedError) {
	req, creds, err := newApiRequest("GET", oid)
	if err != nil {
		return nil, 0, Error(err)
	}

	res, obj, wErr := doApiRequest(req, creds)
	if wErr != nil {
		return nil, 0, wErr
	}

	req, creds, err = obj.NewRequest("download", "GET")
	if err != nil {
		return nil, 0, Error(err)
	}

	res, wErr = doHttpRequest(req, creds)
	if wErr != nil {
		return nil, 0, wErr
	}

	return res.Body, res.ContentLength, nil
}

func Upload(oidPath, filename string, cb CopyCallback) *WrappedError {
	oid := filepath.Base(oidPath)
	file, err := os.Open(oidPath)
	if err != nil {
		return Error(err)
	}
	defer file.Close()

	stat, err := file.Stat()
	if err != nil {
		return Error(err)
	}

	reqObj := &objectResource{
		Oid:  oid,
		Size: stat.Size(),
	}

	by, err := json.Marshal(reqObj)
	if err != nil {
		return Error(err)
	}

	req, creds, err := newApiRequest("POST", "")
	if err != nil {
		return Error(err)
	}

	req.Header.Set("Content-Type", mediaType)
	req.Header.Set("Content-Length", strconv.Itoa(len(by)))
	req.ContentLength = int64(len(by))
	req.Body = ioutil.NopCloser(bytes.NewReader(by))

	tracerx.Printf("api: uploading %s (%s)", filename, oid)
	res, obj, wErr := doApiRequest(req, creds)
	if wErr != nil {
		return wErr
	}

	if res.StatusCode == 200 {
		return nil
	}

	req, creds, err = obj.NewRequest("upload", "PUT")
	if err != nil {
		return Error(err)
	}

	if len(req.Header.Get("Content-Type")) == 0 {
		req.Header.Set("Content-Type", "application/octet-stream")
	}
	req.Header.Set("Content-Length", strconv.FormatInt(reqObj.Size, 10))
	req.ContentLength = reqObj.Size

	reader := &CallbackReader{
		C:         cb,
		TotalSize: reqObj.Size,
		Reader:    file,
	}

	bar := pb.New64(reqObj.Size)
	bar.SetUnits(pb.U_BYTES)
	bar.Start()

	req.Body = ioutil.NopCloser(bar.NewProxyReader(reader))

	res, wErr = doHttpRequest(req, creds)
	if wErr != nil {
		return wErr
	}

	if res.StatusCode > 299 {
		return Errorf(nil, "Invalid status for %s %s: %d", req.Method, req.URL, res.StatusCode)
	}

	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	req, creds, err = obj.NewRequest("verify", "POST")
	if err == objectRelationDoesNotExist {
		return nil
	} else if err != nil {
		return Error(err)
	}

	req.Header.Set("Content-Type", mediaType)
	req.Header.Set("Content-Length", strconv.Itoa(len(by)))
	req.ContentLength = int64(len(by))
	req.Body = ioutil.NopCloser(bytes.NewReader(by))
	res, wErr = doHttpRequest(req, creds)

	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	return wErr
}

func doHttpRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError) {
	res, err := DoHTTP(Config, req)

	var wErr *WrappedError

	if err != nil {
		wErr = Errorf(err, "Error for %s %s", res.Request.Method, res.Request.URL)
	} else {
		if creds != nil {
			saveCredentials(creds, res)
		}

		wErr = handleResponse(res)
	}

	if wErr != nil {
		if res != nil {
			setErrorResponseContext(wErr, res)
		} else {
			setErrorRequestContext(wErr, req)
		}
	}

	return res, wErr
}

func doApiRequest(req *http.Request, creds Creds) (*http.Response, *objectResource, *WrappedError) {
	res, wErr := doHttpRequest(req, creds)
	if wErr != nil {
		return res, nil, wErr
	}

	obj := &objectResource{}
	wErr = decodeApiResponse(res, obj)

	if wErr != nil {
		setErrorResponseContext(wErr, res)
	}

	return res, obj, wErr
}

func handleResponse(res *http.Response) *WrappedError {
	if res.StatusCode < 400 {
		return nil
	}

	defer func() {
		io.Copy(ioutil.Discard, res.Body)
		res.Body.Close()
	}()

	cliErr := &ClientError{}
	wErr := decodeApiResponse(res, cliErr)
	if wErr == nil {
		if len(cliErr.Message) == 0 {
			wErr = defaultError(res)
		} else {
			wErr = Error(cliErr)
		}
	}

	wErr.Panic = res.StatusCode > 499 && res.StatusCode != 501 && res.StatusCode != 509
	return wErr
}

func decodeApiResponse(res *http.Response, obj interface{}) *WrappedError {
	ctype := res.Header.Get("Content-Type")
	if !(lfsMediaTypeRE.MatchString(ctype) || mediaMediaTypeRE.MatchString(ctype)) {
		return nil
	}

	err := json.NewDecoder(res.Body).Decode(obj)
	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	if err != nil {
		return Errorf(err, "Unable to parse HTTP response for %s %s", res.Request.Method, res.Request.URL)
	}

	return nil
}

func defaultError(res *http.Response) *WrappedError {
	var msgFmt string

	if f, ok := defaultErrors[res.StatusCode]; ok {
		msgFmt = f
	} else if res.StatusCode < 500 {
		msgFmt = defaultErrors[400] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
	} else {
		msgFmt = defaultErrors[500] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
	}

	return Error(fmt.Errorf(msgFmt, res.Request.URL))
}

func saveCredentials(creds Creds, res *http.Response) {
	if creds == nil {
		return
	}

	if res.StatusCode < 300 {
		execCreds(creds, "approve")
	} else if res.StatusCode == 401 {
		execCreds(creds, "reject")
	}
}

func newApiRequest(method, oid string) (*http.Request, Creds, error) {
	u, err := Config.ObjectUrl(oid)
	if err != nil {
		return nil, nil, err
	}

	req, creds, err := newClientRequest(method, u.String())
	if err == nil {
		req.Header.Set("Accept", mediaType)
	}
	return req, creds, err
}

func newClientRequest(method, rawurl string) (*http.Request, Creds, error) {
	req, err := http.NewRequest(method, rawurl, nil)
	if err != nil {
		return req, nil, err
	}

	req.Header.Set("User-Agent", UserAgent)
	creds, err := getCreds(req)
	return req, creds, err
}

func getCreds(req *http.Request) (Creds, error) {
	if len(req.Header.Get("Authorization")) > 0 {
		return nil, nil
	}

	apiUrl, err := Config.ObjectUrl("")
	if err != nil {
		return nil, err
	}

	if req.URL.Scheme == apiUrl.Scheme &&
		req.URL.Host == apiUrl.Host {
		creds, err := credentials(req.URL)
		if err != nil {
			return nil, err
		}

		token := fmt.Sprintf("%s:%s", creds["username"], creds["password"])
		auth := "Basic " + base64.URLEncoding.EncodeToString([]byte(token))
		req.Header.Set("Authorization", auth)
		return creds, nil
	}

	return nil, nil
}

func setErrorRequestContext(err *WrappedError, req *http.Request) {
	err.Set("Endpoint", Config.Endpoint())
	err.Set("URL", fmt.Sprintf("%s %s", req.Method, req.URL.String()))
	setErrorHeaderContext(err, "Response", req.Header)
}

func setErrorResponseContext(err *WrappedError, res *http.Response) {
	err.Set("Status", res.Status)
	setErrorHeaderContext(err, "Request", res.Header)
	setErrorRequestContext(err, res.Request)
}

func setErrorHeaderContext(err *WrappedError, prefix string, head http.Header) {
	for key, _ := range head {
		contextKey := fmt.Sprintf("%s:%s", prefix, key)
		if _, skip := hiddenHeaders[key]; skip {
			err.Set(contextKey, "--")
		} else {
			err.Set(contextKey, head.Get(key))
		}
	}
}

func init() {
	defaultErrors[403] = defaultErrors[401]
}
<|endoftext|>


<commit_before>package lfs

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/cheggaaa/pb"
	"github.com/rubyist/tracerx"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
)

const (
	mediaType = "application/vnd.git-lfs+json; charset-utf-8"
)

var (
	lfsMediaTypeRE             = regexp.MustCompile(`\Aapplication/vnd\.git\-lfs\+json(;|\z)`)
	mediaMediaTypeRE           = regexp.MustCompile(`\Aapplication/json(;|\z)`)
	objectRelationDoesNotExist = errors.New("relation does not exist")
	hiddenHeaders              = map[string]bool{
		"Authorization": true,
	}

	// 401 and 403 print the same default error message
	defaultErrors = map[int]string{
		400: "Client error: %s",
		401: "Authorization error: %s\nCheck that you have proper access to the repository",
		404: "Repository or object not found: %s\nCheck that it exists and that you have proper access to it",
		500: "Server error: %s",
	}
)

type objectResource struct {
	Oid   string                   `json:"oid,omitempty"`
	Size  int64                    `json:"size,omitempty"`
	Links map[string]*linkRelation `json:"_links,omitempty"`
}

func (o *objectResource) NewRequest(relation, method string) (*http.Request, Creds, error) {
	rel, ok := o.Rel(relation)
	if !ok {
		return nil, nil, objectRelationDoesNotExist
	}

	req, creds, err := newClientRequest(method, rel.Href)
	if err != nil {
		return nil, nil, err
	}

	for h, v := range rel.Header {
		req.Header.Set(h, v)
	}

	return req, creds, nil
}

func (o *objectResource) Rel(name string) (*linkRelation, bool) {
	if o.Links == nil {
		return nil, false
	}

	rel, ok := o.Links[name]
	return rel, ok
}

type linkRelation struct {
	Href   string            `json:"href"`
	Header map[string]string `json:"header,omitempty"`
}

type ClientError struct {
	Message          string `json:"message"`
	DocumentationUrl string `json:"documentation_url,omitempty"`
	RequestId        string `json:"request_id,omitempty"`
}

func (e *ClientError) Error() string {
	msg := e.Message
	if len(e.DocumentationUrl) > 0 {
		msg += "\nDocs: " + e.DocumentationUrl
	}
	if len(e.RequestId) > 0 {
		msg += "\nRequest ID: " + e.RequestId
	}
	return msg
}

func Download(oid string) (io.ReadCloser, int64, *WrappedError) {
	req, creds, err := newApiRequest("GET", oid)
	if err != nil {
		return nil, 0, Error(err)
	}

	res, obj, wErr := doApiRequest(req, creds)
	if wErr != nil {
		return nil, 0, wErr
	}

	req, creds, err = obj.NewRequest("download", "GET")
	if err != nil {
		return nil, 0, Error(err)
	}

	res, wErr = doHttpRequest(req, creds)
	if wErr != nil {
		return nil, 0, wErr
	}

	return res.Body, res.ContentLength, nil
}

func Upload(oidPath, filename string, cb CopyCallback) *WrappedError {
	oid := filepath.Base(oidPath)
	file, err := os.Open(oidPath)
	if err != nil {
		return Error(err)
	}
	defer file.Close()

	stat, err := file.Stat()
	if err != nil {
		return Error(err)
	}

	reqObj := &objectResource{
		Oid:  oid,
		Size: stat.Size(),
	}

	by, err := json.Marshal(reqObj)
	if err != nil {
		return Error(err)
	}

	req, creds, err := newApiRequest("POST", "")
	if err != nil {
		return Error(err)
	}

	req.Header.Set("Content-Type", mediaType)
	req.Header.Set("Content-Length", strconv.Itoa(len(by)))
	req.Body = ioutil.NopCloser(bytes.NewReader(by))

	tracerx.Printf("api: uploading %s (%s)", filename, oid)
	res, obj, wErr := doApiRequest(req, creds)
	if wErr != nil {
		return wErr
	}

	if res.StatusCode == 200 {
		return nil
	}

	req, creds, err = obj.NewRequest("upload", "PUT")
	if err != nil {
		return Error(err)
	}

	if len(req.Header.Get("Content-Type")) == 0 {
		req.Header.Set("Content-Type", "application/octet-stream")
	}
	req.Header.Set("Content-Length", strconv.FormatInt(reqObj.Size, 10))
	req.ContentLength = reqObj.Size

	reader := &CallbackReader{
		C:         cb,
		TotalSize: reqObj.Size,
		Reader:    file,
	}

	bar := pb.New64(reqObj.Size)
	bar.SetUnits(pb.U_BYTES)
	bar.Start()

	req.Body = ioutil.NopCloser(bar.NewProxyReader(reader))

	res, wErr = doHttpRequest(req, creds)
	if wErr != nil {
		return wErr
	}

	if res.StatusCode > 299 {
		return Errorf(nil, "Invalid status for %s %s: %d", req.Method, req.URL, res.StatusCode)
	}

	req, creds, err = obj.NewRequest("verify", "POST")
	if err == objectRelationDoesNotExist {
		return nil
	} else if err != nil {
		return Error(err)
	}

	req.Header.Set("Content-Type", mediaType)
	req.Header.Set("Content-Length", strconv.Itoa(len(by)))
	req.Body = ioutil.NopCloser(bytes.NewReader(by))
	_, wErr = doHttpRequest(req, creds)

	return wErr
}

func doHttpRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError) {
	res, err := DoHTTP(Config, req)

	var wErr *WrappedError

	if err != nil {
		wErr = Errorf(err, "Error for %s %s", res.Request.Method, res.Request.URL)
	} else {
		if creds != nil {
			saveCredentials(creds, res)
		}

		wErr = handleResponse(res)
	}

	if wErr != nil {
		if res != nil {
			setErrorResponseContext(wErr, res)
		} else {
			setErrorRequestContext(wErr, req)
		}
	}

	return res, wErr
}

func doApiRequest(req *http.Request, creds Creds) (*http.Response, *objectResource, *WrappedError) {
	res, wErr := doHttpRequest(req, creds)
	if wErr != nil {
		return res, nil, wErr
	}

	obj := &objectResource{}
	wErr = decodeApiResponse(res, obj)

	if wErr != nil {
		setErrorResponseContext(wErr, res)
	}

	return res, obj, wErr
}

func handleResponse(res *http.Response) *WrappedError {
	if res.StatusCode < 400 {
		return nil
	}

	defer func() {
		io.Copy(ioutil.Discard, res.Body)
		res.Body.Close()
	}()

	cliErr := &ClientError{}
	wErr := decodeApiResponse(res, cliErr)
	if wErr == nil {
		if len(cliErr.Message) == 0 {
			wErr = defaultError(res)
		} else {
			wErr = Error(cliErr)
		}
	}

	wErr.Panic = res.StatusCode > 499 && res.StatusCode != 501 && res.StatusCode != 509
	return wErr
}

func decodeApiResponse(res *http.Response, obj interface{}) *WrappedError {
	ctype := res.Header.Get("Content-Type")
	if !(lfsMediaTypeRE.MatchString(ctype) || mediaMediaTypeRE.MatchString(ctype)) {
		return nil
	}

	err := json.NewDecoder(res.Body).Decode(obj)
	if err != nil {
		return Errorf(err, "Unable to parse HTTP response for %s %s", res.Request.Method, res.Request.URL)
	}

	return nil
}

func defaultError(res *http.Response) *WrappedError {
	var msgFmt string

	if f, ok := defaultErrors[res.StatusCode]; ok {
		msgFmt = f
	} else if res.StatusCode < 500 {
		msgFmt = defaultErrors[400] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
	} else {
		msgFmt = defaultErrors[500] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
	}

	return Error(fmt.Errorf(msgFmt, res.Request.URL))
}

func saveCredentials(creds Creds, res *http.Response) {
	if creds == nil {
		return
	}

	if res.StatusCode < 300 {
		execCreds(creds, "approve")
	} else if res.StatusCode == 401 {
		execCreds(creds, "reject")
	}
}

func newApiRequest(method, oid string) (*http.Request, Creds, error) {
	u, err := Config.ObjectUrl(oid)
	if err != nil {
		return nil, nil, err
	}

	req, creds, err := newClientRequest(method, u.String())
	if err == nil {
		req.Header.Set("Accept", mediaType)
	}
	return req, creds, err
}

func newClientRequest(method, rawurl string) (*http.Request, Creds, error) {
	req, err := http.NewRequest(method, rawurl, nil)
	if err != nil {
		return req, nil, err
	}

	req.Header.Set("User-Agent", UserAgent)
	creds, err := getCreds(req)
	return req, creds, err
}

func getCreds(req *http.Request) (Creds, error) {
	if len(req.Header.Get("Authorization")) > 0 {
		return nil, nil
	}

	apiUrl, err := Config.ObjectUrl("")
	if err != nil {
		return nil, err
	}

	if req.URL.Scheme == apiUrl.Scheme &&
		req.URL.Host == apiUrl.Host {
		creds, err := credentials(req.URL)
		if err != nil {
			return nil, err
		}

		token := fmt.Sprintf("%s:%s", creds["username"], creds["password"])
		auth := "Basic " + base64.URLEncoding.EncodeToString([]byte(token))
		req.Header.Set("Authorization", auth)
		return creds, nil
	}

	return nil, nil
}

func setErrorRequestContext(err *WrappedError, req *http.Request) {
	err.Set("Endpoint", Config.Endpoint())
	err.Set("URL", fmt.Sprintf("%s %s", req.Method, req.URL.String()))
	setErrorHeaderContext(err, "Response", req.Header)
}

func setErrorResponseContext(err *WrappedError, res *http.Response) {
	err.Set("Status", res.Status)
	setErrorHeaderContext(err, "Request", res.Header)
	setErrorRequestContext(err, res.Request)
}

func setErrorHeaderContext(err *WrappedError, prefix string, head http.Header) {
	for key, _ := range head {
		contextKey := fmt.Sprintf("%s:%s", prefix, key)
		if _, skip := hiddenHeaders[key]; skip {
			err.Set(contextKey, "--")
		} else {
			err.Set(contextKey, head.Get(key))
		}
	}
}

func init() {
	defaultErrors[403] = defaultErrors[401]
}
<commit_msg>ラララララ ラー ララララー<commit_after>package lfs

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/cheggaaa/pb"
	"github.com/rubyist/tracerx"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
)

const (
	mediaType = "application/vnd.git-lfs+json; charset-utf-8"
)

var (
	lfsMediaTypeRE             = regexp.MustCompile(`\Aapplication/vnd\.git\-lfs\+json(;|\z)`)
	mediaMediaTypeRE           = regexp.MustCompile(`\Aapplication/json(;|\z)`)
	objectRelationDoesNotExist = errors.New("relation does not exist")
	hiddenHeaders              = map[string]bool{
		"Authorization": true,
	}

	// 401 and 403 print the same default error message
	defaultErrors = map[int]string{
		400: "Client error: %s",
		401: "Authorization error: %s\nCheck that you have proper access to the repository",
		404: "Repository or object not found: %s\nCheck that it exists and that you have proper access to it",
		500: "Server error: %s",
	}
)

type objectResource struct {
	Oid   string                   `json:"oid,omitempty"`
	Size  int64                    `json:"size,omitempty"`
	Links map[string]*linkRelation `json:"_links,omitempty"`
}

func (o *objectResource) NewRequest(relation, method string) (*http.Request, Creds, error) {
	rel, ok := o.Rel(relation)
	if !ok {
		return nil, nil, objectRelationDoesNotExist
	}

	req, creds, err := newClientRequest(method, rel.Href)
	if err != nil {
		return nil, nil, err
	}

	for h, v := range rel.Header {
		req.Header.Set(h, v)
	}

	return req, creds, nil
}

func (o *objectResource) Rel(name string) (*linkRelation, bool) {
	if o.Links == nil {
		return nil, false
	}

	rel, ok := o.Links[name]
	return rel, ok
}

type linkRelation struct {
	Href   string            `json:"href"`
	Header map[string]string `json:"header,omitempty"`
}

type ClientError struct {
	Message          string `json:"message"`
	DocumentationUrl string `json:"documentation_url,omitempty"`
	RequestId        string `json:"request_id,omitempty"`
}

func (e *ClientError) Error() string {
	msg := e.Message
	if len(e.DocumentationUrl) > 0 {
		msg += "\nDocs: " + e.DocumentationUrl
	}
	if len(e.RequestId) > 0 {
		msg += "\nRequest ID: " + e.RequestId
	}
	return msg
}

func Download(oid string) (io.ReadCloser, int64, *WrappedError) {
	req, creds, err := newApiRequest("GET", oid)
	if err != nil {
		return nil, 0, Error(err)
	}

	res, obj, wErr := doApiRequest(req, creds)
	if wErr != nil {
		return nil, 0, wErr
	}

	req, creds, err = obj.NewRequest("download", "GET")
	if err != nil {
		return nil, 0, Error(err)
	}

	res, wErr = doHttpRequest(req, creds)
	if wErr != nil {
		return nil, 0, wErr
	}

	return res.Body, res.ContentLength, nil
}

func Upload(oidPath, filename string, cb CopyCallback) *WrappedError {
	oid := filepath.Base(oidPath)
	file, err := os.Open(oidPath)
	if err != nil {
		return Error(err)
	}
	defer file.Close()

	stat, err := file.Stat()
	if err != nil {
		return Error(err)
	}

	reqObj := &objectResource{
		Oid:  oid,
		Size: stat.Size(),
	}

	by, err := json.Marshal(reqObj)
	if err != nil {
		return Error(err)
	}

	req, creds, err := newApiRequest("POST", "")
	if err != nil {
		return Error(err)
	}

	req.Header.Set("Content-Type", mediaType)
	req.Header.Set("Content-Length", strconv.Itoa(len(by)))
	req.ContentLength = int64(len(by))
	req.Body = ioutil.NopCloser(bytes.NewReader(by))

	tracerx.Printf("api: uploading %s (%s)", filename, oid)
	res, obj, wErr := doApiRequest(req, creds)
	if wErr != nil {
		return wErr
	}

	if res.StatusCode == 200 {
		return nil
	}

	req, creds, err = obj.NewRequest("upload", "PUT")
	if err != nil {
		return Error(err)
	}

	if len(req.Header.Get("Content-Type")) == 0 {
		req.Header.Set("Content-Type", "application/octet-stream")
	}
	req.Header.Set("Content-Length", strconv.FormatInt(reqObj.Size, 10))
	req.ContentLength = reqObj.Size

	reader := &CallbackReader{
		C:         cb,
		TotalSize: reqObj.Size,
		Reader:    file,
	}

	bar := pb.New64(reqObj.Size)
	bar.SetUnits(pb.U_BYTES)
	bar.Start()

	req.Body = ioutil.NopCloser(bar.NewProxyReader(reader))

	res, wErr = doHttpRequest(req, creds)
	if wErr != nil {
		return wErr
	}

	if res.StatusCode > 299 {
		return Errorf(nil, "Invalid status for %s %s: %d", req.Method, req.URL, res.StatusCode)
	}

	req, creds, err = obj.NewRequest("verify", "POST")
	if err == objectRelationDoesNotExist {
		return nil
	} else if err != nil {
		return Error(err)
	}

	req.Header.Set("Content-Type", mediaType)
	req.Header.Set("Content-Length", strconv.Itoa(len(by)))
	req.ContentLength = int64(len(by))
	req.Body = ioutil.NopCloser(bytes.NewReader(by))
	_, wErr = doHttpRequest(req, creds)

	return wErr
}

func doHttpRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError) {
	res, err := DoHTTP(Config, req)

	var wErr *WrappedError

	if err != nil {
		wErr = Errorf(err, "Error for %s %s", res.Request.Method, res.Request.URL)
	} else {
		if creds != nil {
			saveCredentials(creds, res)
		}

		wErr = handleResponse(res)
	}

	if wErr != nil {
		if res != nil {
			setErrorResponseContext(wErr, res)
		} else {
			setErrorRequestContext(wErr, req)
		}
	}

	return res, wErr
}

func doApiRequest(req *http.Request, creds Creds) (*http.Response, *objectResource, *WrappedError) {
	res, wErr := doHttpRequest(req, creds)
	if wErr != nil {
		return res, nil, wErr
	}

	obj := &objectResource{}
	wErr = decodeApiResponse(res, obj)

	if wErr != nil {
		setErrorResponseContext(wErr, res)
	}

	return res, obj, wErr
}

func handleResponse(res *http.Response) *WrappedError {
	if res.StatusCode < 400 {
		return nil
	}

	defer func() {
		io.Copy(ioutil.Discard, res.Body)
		res.Body.Close()
	}()

	cliErr := &ClientError{}
	wErr := decodeApiResponse(res, cliErr)
	if wErr == nil {
		if len(cliErr.Message) == 0 {
			wErr = defaultError(res)
		} else {
			wErr = Error(cliErr)
		}
	}

	wErr.Panic = res.StatusCode > 499 && res.StatusCode != 501 && res.StatusCode != 509
	return wErr
}

func decodeApiResponse(res *http.Response, obj interface{}) *WrappedError {
	ctype := res.Header.Get("Content-Type")
	if !(lfsMediaTypeRE.MatchString(ctype) || mediaMediaTypeRE.MatchString(ctype)) {
		return nil
	}

	err := json.NewDecoder(res.Body).Decode(obj)
	if err != nil {
		return Errorf(err, "Unable to parse HTTP response for %s %s", res.Request.Method, res.Request.URL)
	}

	return nil
}

func defaultError(res *http.Response) *WrappedError {
	var msgFmt string

	if f, ok := defaultErrors[res.StatusCode]; ok {
		msgFmt = f
	} else if res.StatusCode < 500 {
		msgFmt = defaultErrors[400] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
	} else {
		msgFmt = defaultErrors[500] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
	}

	return Error(fmt.Errorf(msgFmt, res.Request.URL))
}

func saveCredentials(creds Creds, res *http.Response) {
	if creds == nil {
		return
	}

	if res.StatusCode < 300 {
		execCreds(creds, "approve")
	} else if res.StatusCode == 401 {
		execCreds(creds, "reject")
	}
}

func newApiRequest(method, oid string) (*http.Request, Creds, error) {
	u, err := Config.ObjectUrl(oid)
	if err != nil {
		return nil, nil, err
	}

	req, creds, err := newClientRequest(method, u.String())
	if err == nil {
		req.Header.Set("Accept", mediaType)
	}
	return req, creds, err
}

func newClientRequest(method, rawurl string) (*http.Request, Creds, error) {
	req, err := http.NewRequest(method, rawurl, nil)
	if err != nil {
		return req, nil, err
	}

	req.Header.Set("User-Agent", UserAgent)
	creds, err := getCreds(req)
	return req, creds, err
}

func getCreds(req *http.Request) (Creds, error) {
	if len(req.Header.Get("Authorization")) > 0 {
		return nil, nil
	}

	apiUrl, err := Config.ObjectUrl("")
	if err != nil {
		return nil, err
	}

	if req.URL.Scheme == apiUrl.Scheme &&
		req.URL.Host == apiUrl.Host {
		creds, err := credentials(req.URL)
		if err != nil {
			return nil, err
		}

		token := fmt.Sprintf("%s:%s", creds["username"], creds["password"])
		auth := "Basic " + base64.URLEncoding.EncodeToString([]byte(token))
		req.Header.Set("Authorization", auth)
		return creds, nil
	}

	return nil, nil
}

func setErrorRequestContext(err *WrappedError, req *http.Request) {
	err.Set("Endpoint", Config.Endpoint())
	err.Set("URL", fmt.Sprintf("%s %s", req.Method, req.URL.String()))
	setErrorHeaderContext(err, "Response", req.Header)
}

func setErrorResponseContext(err *WrappedError, res *http.Response) {
	err.Set("Status", res.Status)
	setErrorHeaderContext(err, "Request", res.Header)
	setErrorRequestContext(err, res.Request)
}

func setErrorHeaderContext(err *WrappedError, prefix string, head http.Header) {
	for key, _ := range head {
		contextKey := fmt.Sprintf("%s:%s", prefix, key)
		if _, skip := hiddenHeaders[key]; skip {
			err.Set(contextKey, "--")
		} else {
			err.Set(contextKey, head.Get(key))
		}
	}
}

func init() {
	defaultErrors[403] = defaultErrors[401]
}
<|endoftext|>


<commit_before>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"fmt"

	"github.com/tsuru/config"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/db/storage"
	"gopkg.in/mgo.v2"
)

type autoScaleRule struct {
	MetadataFilter    string `bson:"_id"`
	Enabled           bool
	MaxContainerCount int
	ScaleDownRatio    float32
	PreventRebalance  bool
	MaxMemoryRatio    float32
}

func autoScaleRuleCollection() (*storage.Collection, error) {
	conn, err := db.Conn()
	if err != nil {
		return nil, err
	}
	name, err := config.GetString("docker:collection")
	if err != nil {
		return nil, err
	}
	return conn.Collection(fmt.Sprintf("%s_auto_scale_rule", name)), nil
}

func legacyAutoScaleRule() *autoScaleRule {
	metadataFilter, _ := config.GetString("docker:auto-scale:metadata-filter")
	maxContainerCount, _ := config.GetInt("docker:auto-scale:max-container-count")
	scaleDownRatio, _ := config.GetFloat("docker:auto-scale:scale-down-ratio")
	preventRebalance, _ := config.GetBool("docker:auto-scale:prevent-rebalance")
	maxUsedMemory, _ := config.GetFloat("docker:scheduler:max-used-memory")
	return &autoScaleRule{
		MaxMemoryRatio:    float32(maxUsedMemory),
		MaxContainerCount: maxContainerCount,
		MetadataFilter:    metadataFilter,
		ScaleDownRatio:    float32(scaleDownRatio),
		PreventRebalance:  preventRebalance,
		Enabled:           true,
	}
}

func autoScaleRuleForMetadata(metadataFilter string) (*autoScaleRule, error) {
	coll, err := autoScaleRuleCollection()
	if err != nil {
		return nil, err
	}
	var rule autoScaleRule
	err = coll.FindId(metadataFilter).One(&rule)
	if err == mgo.ErrNotFound {
		legacyRule := legacyAutoScaleRule()
		if legacyRule.MetadataFilter == metadataFilter {
			rule = *legacyRule
			err = nil
		}
	}
	if err != nil {
		return nil, err
	}
	if rule.ScaleDownRatio == 0.0 {
		rule.ScaleDownRatio = 1.333
	} else if rule.ScaleDownRatio <= 1.0 {
		return nil, fmt.Errorf("invalid rule, scale down ratio needs to be greater than 1.0, got %f", rule.ScaleDownRatio)
	}
	if rule.MaxMemoryRatio == 0.0 {
		maxMemoryRatio, _ := config.GetFloat("docker:scheduler:max-used-memory")
		rule.MaxMemoryRatio = float32(maxMemoryRatio)
	}
	return &rule, nil
}
<commit_msg>provision/docker: fix connection leak in auto scale rule<commit_after>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"fmt"

	"github.com/tsuru/config"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/db/storage"
	"gopkg.in/mgo.v2"
)

type autoScaleRule struct {
	MetadataFilter    string `bson:"_id"`
	Enabled           bool
	MaxContainerCount int
	ScaleDownRatio    float32
	PreventRebalance  bool
	MaxMemoryRatio    float32
}

func autoScaleRuleCollection() (*storage.Collection, error) {
	conn, err := db.Conn()
	if err != nil {
		return nil, err
	}
	name, err := config.GetString("docker:collection")
	if err != nil {
		return nil, err
	}
	return conn.Collection(fmt.Sprintf("%s_auto_scale_rule", name)), nil
}

func legacyAutoScaleRule() *autoScaleRule {
	metadataFilter, _ := config.GetString("docker:auto-scale:metadata-filter")
	maxContainerCount, _ := config.GetInt("docker:auto-scale:max-container-count")
	scaleDownRatio, _ := config.GetFloat("docker:auto-scale:scale-down-ratio")
	preventRebalance, _ := config.GetBool("docker:auto-scale:prevent-rebalance")
	maxUsedMemory, _ := config.GetFloat("docker:scheduler:max-used-memory")
	return &autoScaleRule{
		MaxMemoryRatio:    float32(maxUsedMemory),
		MaxContainerCount: maxContainerCount,
		MetadataFilter:    metadataFilter,
		ScaleDownRatio:    float32(scaleDownRatio),
		PreventRebalance:  preventRebalance,
		Enabled:           true,
	}
}

func autoScaleRuleForMetadata(metadataFilter string) (*autoScaleRule, error) {
	coll, err := autoScaleRuleCollection()
	if err != nil {
		return nil, err
	}
	defer coll.Close()
	var rule autoScaleRule
	err = coll.FindId(metadataFilter).One(&rule)
	if err == mgo.ErrNotFound {
		legacyRule := legacyAutoScaleRule()
		if legacyRule.MetadataFilter == metadataFilter {
			rule = *legacyRule
			err = nil
		}
	}
	if err != nil {
		return nil, err
	}
	if rule.ScaleDownRatio == 0.0 {
		rule.ScaleDownRatio = 1.333
	} else if rule.ScaleDownRatio <= 1.0 {
		return nil, fmt.Errorf("invalid rule, scale down ratio needs to be greater than 1.0, got %f", rule.ScaleDownRatio)
	}
	if rule.MaxMemoryRatio == 0.0 {
		maxMemoryRatio, _ := config.GetFloat("docker:scheduler:max-used-memory")
		rule.MaxMemoryRatio = float32(maxMemoryRatio)
	}
	return &rule, nil
}
<|endoftext|>


<commit_before>package node

import (
	"testing"
	"time"

	"github.com/stretchr/testify/assert"

	"github.com/tendermint/tmlibs/log"

	cfg "github.com/tendermint/tendermint/config"
	"github.com/tendermint/tendermint/types"
)

func TestNodeStartStop(t *testing.T) {
	config := cfg.ResetTestRoot("node_node_test")

	// create & start node
	n, err := DefaultNewNode(config, log.TestingLogger())
	assert.NoError(t, err, "expected no err on DefaultNewNode")
	n.Start()
	t.Logf("Started node %v", n.sw.NodeInfo())

	// wait for the node to produce a block
	blockCh := make(chan struct{})
	types.AddListenerForEvent(n.EventSwitch(), "node_test", types.EventStringNewBlock(), func(types.TMEventData) {
		blockCh <- struct{}{}
	})
	select {
	case <-blockCh:
	case <-time.After(5 * time.Second):
		t.Fatal("timed out waiting for the node to produce a block")
	}

	// stop the node
	go func() {
		n.Stop()
	}()

	select {
	case <-n.Quit:
	case <-time.After(5 * time.Second):
		t.Fatal("timed out waiting for shutdown")
	}
}
<commit_msg>rewrite node test to use new pubsub<commit_after>package node

import (
	"context"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"

	"github.com/tendermint/tmlibs/log"

	cfg "github.com/tendermint/tendermint/config"
	"github.com/tendermint/tendermint/types"
)

func TestNodeStartStop(t *testing.T) {
	config := cfg.ResetTestRoot("node_node_test")

	// create & start node
	n, err := DefaultNewNode(config, log.TestingLogger())
	assert.NoError(t, err, "expected no err on DefaultNewNode")
	n.Start()
	t.Logf("Started node %v", n.sw.NodeInfo())

	// wait for the node to produce a block
	blockCh := make(chan interface{})
	err = n.EventBus().Subscribe(context.Background(), "node_test", types.EventQueryNewBlock, blockCh)
	assert.NoError(t, err)
	select {
	case <-blockCh:
	case <-time.After(5 * time.Second):
		t.Fatal("timed out waiting for the node to produce a block")
	}

	// stop the node
	go func() {
		n.Stop()
	}()

	select {
	case <-n.Quit:
	case <-time.After(5 * time.Second):
		t.Fatal("timed out waiting for shutdown")
	}
}
<|endoftext|>


<commit_before>package gearcmd

import (
	"bufio"
	"bytes"
	"container/ring"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"

	"github.com/Clever/baseworker-go"
	"github.com/Clever/gearcmd/argsparser"
)

// TaskConfig defines the configuration for the task.
type TaskConfig struct {
	FunctionName, FunctionCmd string
	WarningLines              int
	ParseArgs                 bool
}

// Process runs the Gearman job by running the configured task.
// We need to implement the Task interface so we return (byte[], error)
// though the byte[] is always nil.
func (conf TaskConfig) Process(job baseworker.Job) ([]byte, error) {
	// This wraps the actual processing to do some logging
	log.Printf("STARTING %s %s %s", conf.FunctionName, getJobId(job), string(job.Data()))
	err := conf.doProcess(job)
	if err != nil {
		log.Printf("ENDING WITH ERROR %s %s %s %s", conf.FunctionName, getJobId(job), err.Error(), string(job.Data()))
	} else {
		log.Printf("ENDING %s %s %s", conf.FunctionName, getJobId(job), string(job.Data()))
	}
	return nil, err
}

// getJobId returns the jobId from the job handle
func getJobId(job baseworker.Job) string {
	splits := strings.Split(job.Handle(), ":")
	return splits[len(splits)-1]
}

func (conf TaskConfig) doProcess(job baseworker.Job) error {

	defer func() {
		// If we panicked then set the panic message as a warning. Gearman-go will
		// handle marking this job as failed.
		if r := recover(); r != nil {
			err := r.(error)
			job.SendWarning([]byte(err.Error()))
		}
	}()
	var args []string
	var err error
	if conf.ParseArgs {
		args, err = argsparser.ParseArgs(string(job.Data()))
		if err != nil {
			return err
		}
	} else {
		args = []string{string(job.Data())}

	}
	cmd := exec.Command(conf.FunctionCmd, args...)

	// Write the stdout and stderr of the process to both this process' stdout and stderr
	// and also write it to a byte buffer so that we can return it with the Gearman job
	// data as necessary.
	var stderrbuf bytes.Buffer
	cmd.Stderr = io.MultiWriter(os.Stderr, &stderrbuf)
	stdoutReader, stdoutWriter := io.Pipe()
	cmd.Stdout = io.MultiWriter(os.Stdout, stdoutWriter)
	finishedProcessingStdout := make(chan error)
	go func() {
		finishedProcessingStdout <- streamToGearman(stdoutReader, job)
	}()
	// Save the cmdErr. We want to process stdout and stderr before we return it
	cmdErr := cmd.Run()
	stdoutWriter.Close()
	sendStderrWarnings(&stderrbuf, job, conf.WarningLines)
	stdoutErr := <-finishedProcessingStdout
	if cmdErr != nil {
		return cmdErr
	} else if stdoutErr != nil {
		return stdoutErr
	}
	return nil
}

// This function streams the reader to the Gearman job (through job.SendData())
func streamToGearman(reader io.Reader, job baseworker.Job) error {
	buffer := make([]byte, 1024)
	for {
		n, err := reader.Read(buffer)
		// Process the data before processing the error (as per the io.Reader docs)
		if n > 0 {
			job.SendData(buffer[:n])
		}
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}
	}
}

// sendStderrWarnings sends the last X lines in the stderr output and to the job's warnings
// field
func sendStderrWarnings(buffer io.Reader, job baseworker.Job, warningLines int) error {
	scanner := bufio.NewScanner(buffer)
	// Create a circular buffer for the last X lines
	lastStderrLines := ring.New(warningLines)
	for scanner.Scan() {
		lastStderrLines = lastStderrLines.Next()
		lastStderrLines.Value = scanner.Bytes()
	}
	// Walk forward through the buffer to get all the last X entries. Note that we call next first
	// so that we start at the oldest entry.
	for i := 0; i < lastStderrLines.Len(); i++ {
		if lastStderrLines = lastStderrLines.Next(); lastStderrLines.Value != nil {
			job.SendWarning(lastStderrLines.Value.([]byte))
		}
	}
	return scanner.Err()
}
<commit_msg>Use kayvee for logging<commit_after>package gearcmd

import (
	"bufio"
	"bytes"
	"container/ring"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"

	"github.com/Clever/baseworker-go"
	"github.com/Clever/gearcmd/argsparser"
	kayvee "github.com/Clever/kayvee-go"
)

// TaskConfig defines the configuration for the task.
type TaskConfig struct {
	FunctionName, FunctionCmd string
	WarningLines              int
	ParseArgs                 bool
}

// Process runs the Gearman job by running the configured task.
// We need to implement the Task interface so we return (byte[], error)
// though the byte[] is always nil.
func (conf TaskConfig) Process(job baseworker.Job) ([]byte, error) {
	// This wraps the actual processing to do some logging
	log.Printf(kayvee.FormatLog("gearcmd", "info", "START",
		map[string]interface{}{"function_name": conf.FunctionName, "job_id": getJobId(job), "job_data": string(job.Data())}))
	err := conf.doProcess(job)
	if err != nil {
		log.Printf(kayvee.FormatLog("gearcmd", "error", "END_WITH_ERROR",
			map[string]interface{}{"function_name": conf.FunctionName, "job_id": getJobId(job),
				"error_message": err.Error(), "job_data": string(job.Data())}))
	} else {
		log.Printf(kayvee.FormatLog("gearcmd", "info", "END",
			map[string]interface{}{"function_name": conf.FunctionName, "job_id": getJobId(job),
				"job_data": string(job.Data())}))
	}
	return nil, err
}

// getJobId returns the jobId from the job handle
func getJobId(job baseworker.Job) string {
	splits := strings.Split(job.Handle(), ":")
	return splits[len(splits)-1]
}

func (conf TaskConfig) doProcess(job baseworker.Job) error {

	defer func() {
		// If we panicked then set the panic message as a warning. Gearman-go will
		// handle marking this job as failed.
		if r := recover(); r != nil {
			err := r.(error)
			job.SendWarning([]byte(err.Error()))
		}
	}()
	var args []string
	var err error
	if conf.ParseArgs {
		args, err = argsparser.ParseArgs(string(job.Data()))
		if err != nil {
			return err
		}
	} else {
		args = []string{string(job.Data())}

	}
	cmd := exec.Command(conf.FunctionCmd, args...)

	// Write the stdout and stderr of the process to both this process' stdout and stderr
	// and also write it to a byte buffer so that we can return it with the Gearman job
	// data as necessary.
	var stderrbuf bytes.Buffer
	cmd.Stderr = io.MultiWriter(os.Stderr, &stderrbuf)
	stdoutReader, stdoutWriter := io.Pipe()
	cmd.Stdout = io.MultiWriter(os.Stdout, stdoutWriter)
	finishedProcessingStdout := make(chan error)
	go func() {
		finishedProcessingStdout <- streamToGearman(stdoutReader, job)
	}()
	// Save the cmdErr. We want to process stdout and stderr before we return it
	cmdErr := cmd.Run()
	stdoutWriter.Close()
	sendStderrWarnings(&stderrbuf, job, conf.WarningLines)
	stdoutErr := <-finishedProcessingStdout
	if cmdErr != nil {
		return cmdErr
	} else if stdoutErr != nil {
		return stdoutErr
	}
	return nil
}

// This function streams the reader to the Gearman job (through job.SendData())
func streamToGearman(reader io.Reader, job baseworker.Job) error {
	buffer := make([]byte, 1024)
	for {
		n, err := reader.Read(buffer)
		// Process the data before processing the error (as per the io.Reader docs)
		if n > 0 {
			job.SendData(buffer[:n])
		}
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}
	}
}

// sendStderrWarnings sends the last X lines in the stderr output and to the job's warnings
// field
func sendStderrWarnings(buffer io.Reader, job baseworker.Job, warningLines int) error {
	scanner := bufio.NewScanner(buffer)
	// Create a circular buffer for the last X lines
	lastStderrLines := ring.New(warningLines)
	for scanner.Scan() {
		lastStderrLines = lastStderrLines.Next()
		lastStderrLines.Value = scanner.Bytes()
	}
	// Walk forward through the buffer to get all the last X entries. Note that we call next first
	// so that we start at the oldest entry.
	for i := 0; i < lastStderrLines.Len(); i++ {
		if lastStderrLines = lastStderrLines.Next(); lastStderrLines.Value != nil {
			job.SendWarning(lastStderrLines.Value.([]byte))
		}
	}
	return scanner.Err()
}
<|endoftext|>


<commit_before>package geard

import (
	"errors"
	"fmt"
	"strings"
	log "code.google.com/p/log4go"
	"io/ioutil"
	"net/http"
	"sync"
	"time"
)

// Errors introduced by handling requests
var (
	ErrRequestCancelled = errors.New("sending request is cancelled")
)

type RawRequest struct {
	Method       string
	Url          string
	ContentType         string
	Cancel       <-chan bool
}

// NewRawRequest returns a new RawRequest
func NewRawRequest(method, url, contenttype string, cancel <-chan bool) *RawRequest {
	return &RawRequest{
		Method:       method,
		Url:          url,
		ContentType:  contenttype,
		Cancel:       cancel,
	}
}

func (c *Client) Install(name, json string) (*RawResponse, error) {
	url := "http://"+c.Host+":"+c.Port+"/container/"+name
	req := NewRawRequest("PUT", url, "Content-Type: application/json", nil)
	resp, err := c.SendRequest(req, json)
	return resp, err
}

func (c *Client) Start(name string) (*RawResponse, error) {
	url := "http://"+c.Host+":"+c.Port+"/container/"+name+"/started"
	req := NewRawRequest("PUT", url, "Content-Type: application/json", nil)
	resp, err := c.SendRequest(req, "")
	return resp, err
}

func (c *Client) Stop(name string) (*RawResponse, error) {
	url := "http://"+c.Host+":"+c.Port+"/container/"+name+"/stopped"
	req := NewRawRequest("PUT", url, "Content-Type: application/json", nil)
	resp, err := c.SendRequest(req, "")
	return resp, err
}

func (c *Client) Restart(name string) (*RawResponse, error) {
	url := "http://"+c.Host+":"+c.Port+"/container/"+name+"/restart"
	req := NewRawRequest("POST", url, "Content-Type: application/json", nil)
	resp, err := c.SendRequest(req, "")
	return resp, err
}

// SendRequest sends a HTTP request and returns a Response as defined by etcd
func (c *Client) SendRequest(rr *RawRequest, json string) (*RawResponse, error) {

	var req *http.Request
	var resp *http.Response
	var httpPath string
	var err error
	var respBody []byte

	var numReqs = 1
        log.Info("------------entry------")
	checkRetry := c.CheckRetry
	if checkRetry == nil {
		checkRetry = DefaultCheckRetry
	}

	cancelled := make(chan bool, 1)
	reqLock := new(sync.Mutex)

	if rr.Cancel != nil {
		cancelRoutine := make(chan bool)
		defer close(cancelRoutine)

		go func() {
			select {
			case <-rr.Cancel:
				cancelled <- true
				fmt.Println("send.request is cancelled")
			case <-cancelRoutine:
				return
			}

			// Repeat canceling request until this thread is stopped
			// because we have no idea about whether it succeeds.
			for {
				reqLock.Lock()
				c.httpClient.Transport.(*http.Transport).CancelRequest(req)
				reqLock.Unlock()

				select {
				case <-time.After(100 * time.Millisecond):
				case <-cancelRoutine:
					return
				}
			}
		}()
	}

	    log.Info("Connecting to geard deamon: attempt")

        httpPath = rr.Url

		// Return a cURL command if curlChan is set
        c.OpenCURL()
		if c.cURLch != nil {
			command := fmt.Sprintf("curl")
			if rr.Method != "" {
				command += fmt.Sprintf(" -X %s", rr.Method)
			 }
			if rr.Url != "" {
				command += fmt.Sprintf(" %s", rr.Url)
			 }

			if json != "" {
				command += fmt.Sprintf(" -d %s", json)
			 }
            log.Info(command)
			c.sendCURL(command)
		}

		reqLock.Lock()

		if json == "" {
			if req, err = http.NewRequest(rr.Method, httpPath, nil); err != nil {
				return nil, err
			}
		} else {
			body := strings.NewReader(json)
			if req, err = http.NewRequest(rr.Method, httpPath, body); err != nil {
				return nil, err
			}

			req.Header.Set("Content-Type", rr.ContentType)
		}
		reqLock.Unlock()
		resp, err = c.httpClient.Do(req)
		defer func() {
			if resp != nil {
				resp.Body.Close()
			}
		}()

		// If the request was cancelled, return ErrRequestCancelled directly
		select {
		case <-cancelled:
			return nil, ErrRequestCancelled
		default:
		}

		numReqs++

		// network error, change a machine!
		if err != nil {
			log.Error("network error:", err.Error())
			lastResp := http.Response{}
			if checkErr := checkRetry(numReqs, lastResp, err); checkErr != nil {
				return nil, checkErr
			}

			//c.cluster.switchLeader(attempt % len(c.cluster.Machines))
			//continue
		}

		// if there is no error, it should receive response
		log.Error("recv.response.from", httpPath)
                log.Info(resp)
		if validHttpStatusCode[resp.StatusCode] {
			// try to read byte code and break the loop
                        log.Info("--------if entry----------")
                         log.Info(resp.StatusCode)
			respBody, err = ioutil.ReadAll(resp.Body)
			if err == nil {
				log.Error("recv.success.", httpPath)
				//break
			}
			// ReadAll error may be caused due to cancel request
			select {
			case <-cancelled:
				return nil, ErrRequestCancelled
			default:
			}
		}

		// if resp is TemporaryRedirect, set the new leader and retry
		if resp.StatusCode == http.StatusTemporaryRedirect {
			u, err := resp.Location()

			if err != nil {
				log.Error(err)
			} else {
				// Update cluster leader based on redirect location
				// because it should point to the leader address
				//c.cluster.updateLeaderFromURL(u)
				log.Error("recv.response.relocate", u.String())
			}
			resp.Body.Close()
			//continue
		}

		if checkErr := checkRetry(numReqs, *resp,
			errors.New("Unexpected HTTP status code")); checkErr != nil {
			return nil, checkErr
		}
		resp.Body.Close()
	//}

	r := &RawResponse{
		StatusCode: resp.StatusCode,
		Body:       respBody,
		Header:     resp.Header,
	}

	return r, nil
}

// DefaultCheckRetry defines the retrying behaviour for bad HTTP requests
// If we have retried 2 * machine number, stop retrying.
// If status code is InternalServerError, sleep for 200ms.
func DefaultCheckRetry(numReqs int, lastResp http.Response,
	err error) error {

	code := lastResp.StatusCode
	if code == http.StatusInternalServerError {
		time.Sleep(time.Millisecond * 200)

	}

	log.Error("bad response status code", code)
	return nil
}

<commit_msg>small fix for geard launch<commit_after>package geard

import (
	"errors"
	"fmt"
	"strings"
	log "code.google.com/p/log4go"
	"io/ioutil"
	"net/http"
	"sync"
	"time"
)

// Errors introduced by handling requests
var (
	ErrRequestCancelled = errors.New("sending request is cancelled")
)

type RawRequest struct {
	Method       string
	Url          string
	ContentType         string
	Cancel       <-chan bool
}

// NewRawRequest returns a new RawRequest
func NewRawRequest(method, url, contenttype string, cancel <-chan bool) *RawRequest {
	return &RawRequest{
		Method:       method,
		Url:          url,
		ContentType:  contenttype,
		Cancel:       cancel,
	}
}

func (c *Client) Install(name, json string) (*RawResponse, error) {
	url := "http://"+c.Host+":"+c.Port+"/container/"+name
	req := NewRawRequest("PUT", url, "Content-Type: application/json", nil)
	resp, err := c.SendRequest(req, json)
	return resp, err
}

func (c *Client) Start(name string) (*RawResponse, error) {
	url := "http://"+c.Host+":"+c.Port+"/container/"+name+"/started"
	req := NewRawRequest("PUT", url, "Content-Type: application/json", nil)
	resp, err := c.SendRequest(req, "")
	return resp, err
}

func (c *Client) Stop(name string) (*RawResponse, error) {
	url := "http://"+c.Host+":"+c.Port+"/container/"+name+"/stopped"
	req := NewRawRequest("PUT", url, "Content-Type: application/json", nil)
	resp, err := c.SendRequest(req, "")
	return resp, err
}

func (c *Client) Restart(name string) (*RawResponse, error) {
	url := "http://"+c.Host+":"+c.Port+"/container/"+name+"/restart"
	req := NewRawRequest("POST", url, "Content-Type: application/json", nil)
	resp, err := c.SendRequest(req, "")
	return resp, err
}

// SendRequest sends a HTTP request and returns a Response as defined by etcd
func (c *Client) SendRequest(rr *RawRequest, json string) (*RawResponse, error) {

	var req *http.Request
	var resp *http.Response
	var httpPath string
	var err error
	var respBody []byte

	var numReqs = 1
        log.Info("------------entry------")
	checkRetry := c.CheckRetry
	if checkRetry == nil {
		checkRetry = DefaultCheckRetry
	}

	cancelled := make(chan bool, 1)
	reqLock := new(sync.Mutex)

	if rr.Cancel != nil {
		cancelRoutine := make(chan bool)
		defer close(cancelRoutine)

		go func() {
			select {
			case <-rr.Cancel:
				cancelled <- true
				fmt.Println("send.request is cancelled")
			case <-cancelRoutine:
				return
			}

			// Repeat canceling request until this thread is stopped
			// because we have no idea about whether it succeeds.
			for {
				reqLock.Lock()
				c.httpClient.Transport.(*http.Transport).CancelRequest(req)
				reqLock.Unlock()

				select {
				case <-time.After(100 * time.Millisecond):
				case <-cancelRoutine:
					return
				}
			}
		}()
	}

	    log.Info("Connecting to geard deamon: attempt")

        httpPath = rr.Url

		// Return a cURL command if curlChan is set
        c.OpenCURL()
		if c.cURLch != nil {
			command := fmt.Sprintf("curl")
			if rr.Method != "" {
				command += fmt.Sprintf(" -X %s", rr.Method)
			 }
			if rr.Url != "" {
				command += fmt.Sprintf(" %s", rr.Url)
			 }

			if json != "" {
				command += fmt.Sprintf(" -d '%s' ", json)
			 }
            log.Info(command)
			c.sendCURL(command)
		}

		reqLock.Lock()

		if json == "" {
			if req, err = http.NewRequest(rr.Method, httpPath, nil); err != nil {
				return nil, err
			}
		} else {
			body := strings.NewReader(json)
			if req, err = http.NewRequest(rr.Method, httpPath, body); err != nil {
				return nil, err
			}

			req.Header.Set("Content-Type", rr.ContentType)
		}
		reqLock.Unlock()
		resp, err = c.httpClient.Do(req)
		defer func() {
			if resp != nil {
				resp.Body.Close()
			}
		}()

		// If the request was cancelled, return ErrRequestCancelled directly
		select {
		case <-cancelled:
			return nil, ErrRequestCancelled
		default:
		}

		numReqs++

		// network error, change a machine!
		if err != nil {
			log.Error("network error:", err.Error())
			lastResp := http.Response{}
			if checkErr := checkRetry(numReqs, lastResp, err); checkErr != nil {
				return nil, checkErr
			}

			//c.cluster.switchLeader(attempt % len(c.cluster.Machines))
			//continue
		}

		// if there is no error, it should receive response
		log.Error("recv.response.from", httpPath)
                log.Info(resp)
		if validHttpStatusCode[resp.StatusCode] {
			// try to read byte code and break the loop
                        log.Info("--------if entry----------")
                         log.Info(resp.StatusCode)
			respBody, err = ioutil.ReadAll(resp.Body)
			if err == nil {
				log.Error("recv.success.", httpPath)
				//break
			}
			// ReadAll error may be caused due to cancel request
			select {
			case <-cancelled:
				return nil, ErrRequestCancelled
			default:
			}
		}

		// if resp is TemporaryRedirect, set the new leader and retry
		if resp.StatusCode == http.StatusTemporaryRedirect {
			u, err := resp.Location()

			if err != nil {
				log.Error(err)
			} else {
				// Update cluster leader based on redirect location
				// because it should point to the leader address
				//c.cluster.updateLeaderFromURL(u)
				log.Error("recv.response.relocate", u.String())
			}
			resp.Body.Close()
			//continue
		}

		if checkErr := checkRetry(numReqs, *resp,
			errors.New("Unexpected HTTP status code")); checkErr != nil {
			return nil, checkErr
		}
		resp.Body.Close()
	//}

	r := &RawResponse{
		StatusCode: resp.StatusCode,
		Body:       respBody,
		Header:     resp.Header,
	}

	return r, nil
}

// DefaultCheckRetry defines the retrying behaviour for bad HTTP requests
// If we have retried 2 * machine number, stop retrying.
// If status code is InternalServerError, sleep for 200ms.
func DefaultCheckRetry(numReqs int, lastResp http.Response,
	err error) error {

	code := lastResp.StatusCode
	if code == http.StatusInternalServerError {
		time.Sleep(time.Millisecond * 200)

	}

	log.Error("bad response status code", code)
	return nil
}

<|endoftext|>


<commit_before>package imapsrv

import (
	"bufio"
	"strings"
	"testing"
)

func TestQstring(t *testing.T) {

	r := bufio.NewReader(strings.NewReader("quoted string\"\n"))
	l := createLexer(r)
	l.skipSpace()
	tk := l.qstring()

	if tk.value != "quoted string" {
		t.Fail()
	}

}

func TestLiteral(t *testing.T) {

	r := bufio.NewReader(strings.NewReader("0}\n"))
	l := createLexer(r)
	l.skipSpace()
	tk := l.literal()

	if tk.value != "" {
		t.Fail()
	}

}

func TestAstring(t *testing.T) {

	r := bufio.NewReader(strings.NewReader("tHiS_IS#A_VAL!D_ASTRING \n"))
	l := createLexer(r)
	l.skipSpace()
	tk := l.astring()

	if tk.value != "tHiS_IS#A_VAL!D_ASTRING" {
		t.Fail()
	}

}

func TestSkipSpace(t *testing.T) {

	r := bufio.NewReader(strings.NewReader("abc one"))
	l := createLexer(r)

	// lexer instantiates with space at current
	if l.current != byte(' ') {
		t.Fail()
	}

	l.skipSpace()
	// skips past the initialized space
	if l.current != byte('a') {
		t.Fail()
	}

}

func TestConsumeEol(t *testing.T) {

	r := bufio.NewReader(strings.NewReader("abc\none"))
	l := createLexer(r)
	l.consumeEol()

	if l.current != byte('\n') {
		t.Fail()
	}

}

func TestConsume(t *testing.T) {

	r := bufio.NewReader(strings.NewReader("abc\none"))
	l := createLexer(r)
	l.skipSpace()
	l.consume()

	if l.current != byte('b') {
		t.Fail()
	}

}

func TestLexesAstring(t *testing.T) {

	r := bufio.NewReader(strings.NewReader("a0001)\n"))
	l := createLexer(r)
	token := l.next(asAString)

	if token.tokType != stringTokenType {
		t.Fail()
	}

	if token.value != "a0001" {
		t.Fail()
	}

}

func TestLexesQuotedString(t *testing.T) {

	r := bufio.NewReader(strings.NewReader("\"A12312\"\n"))
	l := createLexer(r)
	token := l.next(asAString)

	if token.tokType != stringTokenType {
		t.Fail()
	}

	if token.value != "A12312" {
		t.Fail()
	}

}

func TestLexesLiteral(t *testing.T) {

	r := bufio.NewReader(strings.NewReader("{11}\nFRED FOOBAR {7}\n"))
	l := createLexer(r)
	token := l.next(asAString)

	if token.tokType != stringTokenType {
		t.Fail()
	}

	// the token after {11} should be of length 11
	if 11 != len(token.value) {
		t.Fail()
	}

	if "\nFRED FOOBA" != token.value {
		t.Fail()
	}

}
<commit_msg>Expands lexer tests for astring with closer attention to ABNF in spec<commit_after>package imapsrv

import (
	"bufio"
	"strings"
	"testing"
)

func TestQstring(t *testing.T) {

	r := bufio.NewReader(strings.NewReader("quoted string\"\n"))
	l := createLexer(r)
	l.skipSpace()
	tk := l.qstring()

	if tk.value != "quoted string" {
		t.Fail()
	}

}

func TestLiteral(t *testing.T) {

	r := bufio.NewReader(strings.NewReader("0}\n"))
	l := createLexer(r)
	l.skipSpace()
	tk := l.literal()

	if tk.value != "" {
		t.Fail()
	}

}

// TestAstring checks the lexer will return a valid <astring> per the ABNF rule, or panic on a failing test
//
// Astring = 1*ASTRING-CHAR / string
//     ASTRING-CHAR = ATOM-CHAR / resp-specials
//         ATOM-CHAR = <any CHAR except atom-specials>
//             atom-specials = "(" / ")" / "{" / SP / CTL / list-wildcards / quoted-specials / resp-specials
//                 list-wildcards = "%" / "*"
//                 quoted-specials = DQUOTE / "\"
//                 resp-specials   = "]"
//     string = quoted / literal
//         quoted = DQUOTE *QUOTED-CHAR DQUOTE
//             QUOTED-CHAR = <any TEXT-CHAR except quoted-specials> / "\" quoted-specials
//                 TEXT-CHAR = <any CHAR except CR and LF>
//                 quoted-specials = DQUOTE / "\"
//         literal = "{" number "}" CRLF *CHAR8 ; number represents the number of CHAR8s
//
// SP  = %x20
// CTL = %x00-1F / %x7F ; controls
// DQUOTE = %x22
// CR  = %x0D
// LF  = %x0A
func TestAstring(t *testing.T) {

	// Test cases receive a map of OUTPUT => INPUT
	passing := map[string]string{
		"a":    "a\r\n",     // 1*ASTRING-CHAR - single
		"this": "this\r\n",  // 1*ASTRING-CHAR - many
		"burb": "burb)\r\n", // 1*ASTRING-CHAR - stop at )
		"\"\"": "\"\"\r\n",  // <quoted> with no *QUOTED-CHAR
		"[":    "[\r\n",
		//"{5} abcd": "{5}\r\n abcd\n", // TODO : Should pass under <string> alternative <literal>?
		//"]":        "]\n",            // TODO : Should pass in <ASTRING-CHAR> under the <resp-specials> alternative
	}

	// The failing test case map key is largely irrelevant as they should panic, just included for consistency
	failing := map[string]string{
		" ": " ", // SP
		//"":   "",   // 1*ASTRING-CHAR should have at least one char // TODO : Gets EOF -- should panic?
		"\\": "\\", // <quoted-specials> not allowed in ATOM-CHAR
		//"\"": "\"", // DQUOTE // TODO : Gets EOF -- should panic?
		"%": "%", // <list-wildcard>
		"*": "*", // <list-wildcard>
		")": ")", // <atom-specials> not allowed in ATOM-CHAR
		"(": "(", // <atom-specials> not allowed in ATOM-CHAR
	}

	panicCount := 0

	testAstring := func(in, out string) bool {

		// Catch the panics and increment the panic counter for failures
		defer func() {
			if r := recover(); r != nil {
				// EOFs are easily obscured as they are also a form of panic in the system
				// but do not constitute an 'expected' panic type here
				if r.(parseError).Error() == "EOF" {
					t.Logf("Bad panic on input: %q, output: %q", in, out)
					panic("EOF found in TestAstring - should not be present, correct the test(s)")
				}
				panicCount += 1
			}
		}()

		r := bufio.NewReader(strings.NewReader(in))
		l := createLexer(r)
		l.skipSpace()
		tk := l.astring()

		return tk.value == out

	}

	for o, i := range passing {
		if testAstring(i, o) != true {
			t.Logf("Failed on passing case: input %q, output %q", i, o)
			t.Fail()
		}
	}

	for o, i := range failing {
		if testAstring(i, o) != false {
			// This should not be reached as all failing test cases should trigger a panic
			t.Logf("Failed on failing case: input %q, output %q", i, o)
			t.Fail()
		}
	}

	if panicCount != len(failing) {
		t.Logf("Expected %d panics, found %d", len(failing), panicCount)
		t.Fail()
	}

}

func TestSkipSpace(t *testing.T) {

	r := bufio.NewReader(strings.NewReader("abc one"))
	l := createLexer(r)

	// lexer instantiates with space at current
	if l.current != byte(' ') {
		t.Fail()
	}

	l.skipSpace()
	// skips past the initialized space
	if l.current != byte('a') {
		t.Fail()
	}

}

func TestConsumeEol(t *testing.T) {

	r := bufio.NewReader(strings.NewReader("abc\none"))
	l := createLexer(r)
	l.consumeEol()

	if l.current != byte('\n') {
		t.Fail()
	}

}

func TestConsume(t *testing.T) {

	r := bufio.NewReader(strings.NewReader("abc\none"))
	l := createLexer(r)
	l.skipSpace()
	l.consume()

	if l.current != byte('b') {
		t.Fail()
	}

}

func TestLexesAstring(t *testing.T) {

	r := bufio.NewReader(strings.NewReader("a0001)\n"))
	l := createLexer(r)
	token := l.next(asAString)

	if token.tokType != stringTokenType {
		t.Fail()
	}

	if token.value != "a0001" {
		t.Fail()
	}

}

func TestLexesQuotedString(t *testing.T) {

	r := bufio.NewReader(strings.NewReader("\"A12312\"\n"))
	l := createLexer(r)
	token := l.next(asAString)

	if token.tokType != stringTokenType {
		t.Fail()
	}

	if token.value != "A12312" {
		t.Fail()
	}

}

func TestLexesLiteral(t *testing.T) {

	r := bufio.NewReader(strings.NewReader("{11}\nFRED FOOBAR {7}\n"))
	l := createLexer(r)
	token := l.next(asAString)

	if token.tokType != stringTokenType {
		t.Fail()
	}

	// the token after {11} should be of length 11
	if 11 != len(token.value) {
		t.Fail()
	}

	if "\nFRED FOOBA" != token.value {
		t.Fail()
	}

}
<|endoftext|>


<commit_before>package jail

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"sync"

	"github.com/robertkrimen/otto"
	"github.com/status-im/status-go/geth/common"
	"github.com/status-im/status-go/geth/rpc"
	"github.com/status-im/status-go/static"
)

const (
	web3InstanceCode = `
		var Web3 = require('web3');
		var web3 = new Web3(jeth);
		var Bignumber = require("bignumber.js");
		function bn(val) {
			return new Bignumber(val);
		}
	`
)

var (
	web3Code = string(static.MustAsset("scripts/web3.js"))
	// ErrNoRPCClient is returned when an RPC client is required but it's nil.
	ErrNoRPCClient = errors.New("RPC client is not available")
)

// RPCClientProvider is an interface that provides a way
// to obtain an rpc.Client.
type RPCClientProvider interface {
	RPCClient() *rpc.Client
}

// Jail manages multiple JavaScript execution contexts (JavaScript VMs) called cells.
// Each cell is a separate VM with web3.js set up.
//
// As rpc.Client might not be available during Jail initialization,
// a provider function is used.
type Jail struct {
	rpcClientProvider RPCClientProvider
	baseJS            string
	cellsMx           sync.RWMutex
	cells             map[string]*Cell
}

// New returns a new Jail.
func New(provider RPCClientProvider) *Jail {
	return NewWithBaseJS(provider, "")
}

// NewWithBaseJS returns a new Jail with base JS configured.
func NewWithBaseJS(provider RPCClientProvider, code string) *Jail {
	return &Jail{
		rpcClientProvider: provider,
		baseJS:            code,
		cells:             make(map[string]*Cell),
	}
}

// SetBaseJS sets initial JavaScript code loaded to each new cell.
func (j *Jail) SetBaseJS(js string) {
	j.baseJS = js
}

// Stop stops jail and all assosiacted cells.
func (j *Jail) Stop() {
	j.cellsMx.Lock()
	defer j.cellsMx.Unlock()

	for _, cell := range j.cells {
		cell.Stop() //nolint: errcheck
	}

	// TODO(tiabc): Move this initialisation to a proper place.
	j.cells = make(map[string]*Cell)
}

// createCell creates a new cell if it does not exists.
func (j *Jail) createCell(chatID string) (*Cell, error) {
	j.cellsMx.Lock()
	defer j.cellsMx.Unlock()

	if cell, ok := j.cells[chatID]; ok {
		return cell, fmt.Errorf("cell with id '%s' already exists", chatID)
	}

	cell, err := NewCell(chatID)
	if err != nil {
		return nil, err
	}

	j.cells[chatID] = cell

	return cell, nil
}

// CreateCell creates a new cell. It returns an error
// if a cell with a given ID already exists.
func (j *Jail) CreateCell(chatID string) (common.JailCell, error) {
	return j.createCell(chatID)
}

// initCell initializes a cell with default JavaScript handlers and user code.
func (j *Jail) initCell(cell *Cell) error {
	// Register objects being a bridge between Go and JavaScript.
	if err := registerWeb3Provider(j, cell); err != nil {
		return err
	}

	if err := registerStatusSignals(cell); err != nil {
		return err
	}

	// Run some initial JS code to provide some global objects.
	c := []string{
		j.baseJS,
		web3Code,
		web3InstanceCode,
	}

	_, err := cell.Run(strings.Join(c, ";"))
	return err
}

// CreateAndInitCell creates and initializes a new Cell.
func (j *Jail) createAndInitCell(chatID string, code ...string) (*Cell, error) {
	cell, err := j.createCell(chatID)
	if err != nil {
		return nil, err
	}

	if err := j.initCell(cell); err != nil {
		return nil, err
	}

	// Run custom user code
	for _, js := range code {
		_, err := cell.Run(js)
		if err != nil {
			return nil, err
		}
	}

	return cell, nil
}

// CreateAndInitCell creates and initializes new Cell. Additionally,
// it creates a `catalog` variable in the VM.
// It returns the response as a JSON string.
func (j *Jail) CreateAndInitCell(chatID string, code ...string) string {
	cell, err := j.createAndInitCell(chatID, code...)
	if err != nil {
		return newJailErrorResponse(err)
	}

	return j.makeCatalogVariable(cell)
}

// Parse creates a new jail cell context, with the given chatID as identifier.
// New context executes provided JavaScript code, right after the initialization.
// DEPRECATED
func (j *Jail) Parse(chatID, code string) string {
	cell, err := j.cell(chatID)
	if err != nil {
		// cell does not exist
		cell, err = j.createAndInitCell(chatID, code)
	} else {
		// cell already exist, so just execute the code
		_, err = cell.Run(code)
	}

	if err != nil {
		return newJailErrorResponse(err)
	}

	return j.makeCatalogVariable(cell)
}

// makeCatalogVariable provides `catalog` as a global variable.
// TODO(divan): this can and should be implemented outside of jail,
// on a clojure side. Moving this into separate method to nuke it later
// easier.
func (j *Jail) makeCatalogVariable(cell *Cell) string {
	_, err := cell.Run(`var catalog = JSON.stringify(_status_catalog)`)
	if err != nil {
		return newJailErrorResponse(err)
	}

	value, err := cell.Get("catalog")
	if err != nil {
		return newJailErrorResponse(err)
	}

	return newJailResultResponse(value)
}

func (j *Jail) cell(chatID string) (*Cell, error) {
	j.cellsMx.RLock()
	defer j.cellsMx.RUnlock()

	cell, ok := j.cells[chatID]
	if !ok {
		return nil, fmt.Errorf("cell '%s' not found", chatID)
	}

	return cell, nil
}

// Cell returns a cell by chatID. If it does not exist, error is returned.
// Required by the Backend.
func (j *Jail) Cell(chatID string) (common.JailCell, error) {
	return j.cell(chatID)
}

// Execute allows to run arbitrary JS code within a cell.
func (j *Jail) Execute(chatID, code string) string {
	cell, err := j.cell(chatID)
	if err != nil {
		return newJailErrorResponse(err)
	}

	value, err := cell.Run(code)
	if err != nil {
		return newJailErrorResponse(err)
	}

	return value.String()
}

// Call executes the `call` function within a cell with chatID.
// Returns a string being a valid JS code. In case of a successful result,
// it's {"result": any}. In case of an error: {"error": "some error"}.
//
// Call calls commands from `_status_catalog`.
// commandPath is an array of properties to retrieve a function.
// For instance:
//   `["prop1", "prop2"]` is translated to `_status_catalog["prop1"]["prop2"]`.
func (j *Jail) Call(chatID, commandPath, args string) string {
	cell, err := j.cell(chatID)
	if err != nil {
		return newJailErrorResponse(err)
	}

	value, err := cell.Call("call", nil, commandPath, args)
	if err != nil {
		return newJailErrorResponse(err)
	}

	return newJailResultResponse(value)
}

// RPCClient returns an rpc.Client.
func (j *Jail) RPCClient() *rpc.Client {
	if j.rpcClientProvider == nil {
		return nil
	}

	return j.rpcClientProvider.RPCClient()
}

// sendRPCCall executes a raw JSON-RPC request.
func (j *Jail) sendRPCCall(request string) (interface{}, error) {
	client := j.RPCClient()
	if client == nil {
		return nil, ErrNoRPCClient
	}

	rawResponse := client.CallRaw(request)

	var response interface{}
	if err := json.Unmarshal([]byte(rawResponse), &response); err != nil {
		return nil, fmt.Errorf("failed to unmarshal response: %s", err)
	}

	return response, nil
}

// newJailErrorResponse returns an error.
func newJailErrorResponse(err error) string {
	response := struct {
		Error string `json:"error"`
	}{
		Error: err.Error(),
	}

	rawResponse, err := json.Marshal(response)
	if err != nil {
		return `{"error": "` + err.Error() + `"}`
	}

	return string(rawResponse)
}

// newJailResultResponse returns a string that is a valid JavaScript code.
// Marshaling is not required as result.String() produces a string
// that is a valid JavaScript code.
func newJailResultResponse(result otto.Value) string {
	return `{"result": ` + result.String() + `}`
}
<commit_msg>fix Parse<commit_after>package jail

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"sync"

	"github.com/robertkrimen/otto"
	"github.com/status-im/status-go/geth/common"
	"github.com/status-im/status-go/geth/rpc"
	"github.com/status-im/status-go/static"
)

const (
	web3InstanceCode = `
		var Web3 = require('web3');
		var web3 = new Web3(jeth);
		var Bignumber = require("bignumber.js");
		function bn(val) {
			return new Bignumber(val);
		}
	`
)

var (
	web3Code = string(static.MustAsset("scripts/web3.js"))
	// ErrNoRPCClient is returned when an RPC client is required but it's nil.
	ErrNoRPCClient = errors.New("RPC client is not available")
)

// RPCClientProvider is an interface that provides a way
// to obtain an rpc.Client.
type RPCClientProvider interface {
	RPCClient() *rpc.Client
}

// Jail manages multiple JavaScript execution contexts (JavaScript VMs) called cells.
// Each cell is a separate VM with web3.js set up.
//
// As rpc.Client might not be available during Jail initialization,
// a provider function is used.
type Jail struct {
	rpcClientProvider RPCClientProvider
	baseJS            string
	cellsMx           sync.RWMutex
	cells             map[string]*Cell
}

// New returns a new Jail.
func New(provider RPCClientProvider) *Jail {
	return NewWithBaseJS(provider, "")
}

// NewWithBaseJS returns a new Jail with base JS configured.
func NewWithBaseJS(provider RPCClientProvider, code string) *Jail {
	return &Jail{
		rpcClientProvider: provider,
		baseJS:            code,
		cells:             make(map[string]*Cell),
	}
}

// SetBaseJS sets initial JavaScript code loaded to each new cell.
func (j *Jail) SetBaseJS(js string) {
	j.baseJS = js
}

// Stop stops jail and all assosiacted cells.
func (j *Jail) Stop() {
	j.cellsMx.Lock()
	defer j.cellsMx.Unlock()

	for _, cell := range j.cells {
		cell.Stop() //nolint: errcheck
	}

	// TODO(tiabc): Move this initialisation to a proper place.
	j.cells = make(map[string]*Cell)
}

// createCell creates a new cell if it does not exists.
func (j *Jail) createCell(chatID string) (*Cell, error) {
	j.cellsMx.Lock()
	defer j.cellsMx.Unlock()

	if cell, ok := j.cells[chatID]; ok {
		return cell, fmt.Errorf("cell with id '%s' already exists", chatID)
	}

	cell, err := NewCell(chatID)
	if err != nil {
		return nil, err
	}

	j.cells[chatID] = cell

	return cell, nil
}

// CreateCell creates a new cell. It returns an error
// if a cell with a given ID already exists.
func (j *Jail) CreateCell(chatID string) (common.JailCell, error) {
	return j.createCell(chatID)
}

// initCell initializes a cell with default JavaScript handlers and user code.
func (j *Jail) initCell(cell *Cell) error {
	// Register objects being a bridge between Go and JavaScript.
	if err := registerWeb3Provider(j, cell); err != nil {
		return err
	}

	if err := registerStatusSignals(cell); err != nil {
		return err
	}

	// Run some initial JS code to provide some global objects.
	c := []string{
		j.baseJS,
		web3Code,
		web3InstanceCode,
	}

	_, err := cell.Run(strings.Join(c, ";"))
	return err
}

// CreateAndInitCell creates and initializes a new Cell.
func (j *Jail) createAndInitCell(chatID string, code ...string) (*Cell, error) {
	cell, err := j.createCell(chatID)
	if err != nil {
		return nil, err
	}

	if err := j.initCell(cell); err != nil {
		return nil, err
	}

	// Run custom user code
	for _, js := range code {
		_, err := cell.Run(js)
		if err != nil {
			return nil, err
		}
	}

	return cell, nil
}

// CreateAndInitCell creates and initializes new Cell. Additionally,
// it creates a `catalog` variable in the VM.
// It returns the response as a JSON string.
func (j *Jail) CreateAndInitCell(chatID string, code ...string) string {
	cell, err := j.createAndInitCell(chatID, code...)
	if err != nil {
		return newJailErrorResponse(err)
	}

	return j.makeCatalogVariable(cell)
}

// Parse creates a new jail cell context, with the given chatID as identifier.
// New context executes provided JavaScript code, right after the initialization.
// DEPRECATED
func (j *Jail) Parse(chatID, code string) string {
	cell, err := j.cell(chatID)
	if err != nil {
		// cell does not exist, so create and init it
		cell, err = j.createAndInitCell(chatID, code)
	} else {
		// cell already exists, so just reinit it
		err = j.initCell(cell)
	}

	if err != nil {
		return newJailErrorResponse(err)
	}

	if _, err = cell.Run(code); err != nil {
		return newJailErrorResponse(err)
	}

	return j.makeCatalogVariable(cell)
}

// makeCatalogVariable provides `catalog` as a global variable.
// TODO(divan): this can and should be implemented outside of jail,
// on a clojure side. Moving this into separate method to nuke it later
// easier.
func (j *Jail) makeCatalogVariable(cell *Cell) string {
	_, err := cell.Run(`var catalog = JSON.stringify(_status_catalog)`)
	if err != nil {
		return newJailErrorResponse(err)
	}

	value, err := cell.Get("catalog")
	if err != nil {
		return newJailErrorResponse(err)
	}

	return newJailResultResponse(value)
}

func (j *Jail) cell(chatID string) (*Cell, error) {
	j.cellsMx.RLock()
	defer j.cellsMx.RUnlock()

	cell, ok := j.cells[chatID]
	if !ok {
		return nil, fmt.Errorf("cell '%s' not found", chatID)
	}

	return cell, nil
}

// Cell returns a cell by chatID. If it does not exist, error is returned.
// Required by the Backend.
func (j *Jail) Cell(chatID string) (common.JailCell, error) {
	return j.cell(chatID)
}

// Execute allows to run arbitrary JS code within a cell.
func (j *Jail) Execute(chatID, code string) string {
	cell, err := j.cell(chatID)
	if err != nil {
		return newJailErrorResponse(err)
	}

	value, err := cell.Run(code)
	if err != nil {
		return newJailErrorResponse(err)
	}

	return value.String()
}

// Call executes the `call` function within a cell with chatID.
// Returns a string being a valid JS code. In case of a successful result,
// it's {"result": any}. In case of an error: {"error": "some error"}.
//
// Call calls commands from `_status_catalog`.
// commandPath is an array of properties to retrieve a function.
// For instance:
//   `["prop1", "prop2"]` is translated to `_status_catalog["prop1"]["prop2"]`.
func (j *Jail) Call(chatID, commandPath, args string) string {
	cell, err := j.cell(chatID)
	if err != nil {
		return newJailErrorResponse(err)
	}

	value, err := cell.Call("call", nil, commandPath, args)
	if err != nil {
		return newJailErrorResponse(err)
	}

	return newJailResultResponse(value)
}

// RPCClient returns an rpc.Client.
func (j *Jail) RPCClient() *rpc.Client {
	if j.rpcClientProvider == nil {
		return nil
	}

	return j.rpcClientProvider.RPCClient()
}

// sendRPCCall executes a raw JSON-RPC request.
func (j *Jail) sendRPCCall(request string) (interface{}, error) {
	client := j.RPCClient()
	if client == nil {
		return nil, ErrNoRPCClient
	}

	rawResponse := client.CallRaw(request)

	var response interface{}
	if err := json.Unmarshal([]byte(rawResponse), &response); err != nil {
		return nil, fmt.Errorf("failed to unmarshal response: %s", err)
	}

	return response, nil
}

// newJailErrorResponse returns an error.
func newJailErrorResponse(err error) string {
	response := struct {
		Error string `json:"error"`
	}{
		Error: err.Error(),
	}

	rawResponse, err := json.Marshal(response)
	if err != nil {
		return `{"error": "` + err.Error() + `"}`
	}

	return string(rawResponse)
}

// newJailResultResponse returns a string that is a valid JavaScript code.
// Marshaling is not required as result.String() produces a string
// that is a valid JavaScript code.
func newJailResultResponse(result otto.Value) string {
	return `{"result": ` + result.String() + `}`
}
<|endoftext|>


<commit_before>package lfs

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"

	"github.com/github/git-lfs/git"
)

type Configuration struct {
	CurrentRemote         string
	gitConfig             map[string]string
	remotes               []string
	httpClient            *http.Client
	redirectingHttpClient *http.Client
	isTracingHttp         bool
	loading               sync.Mutex
}

type Endpoint struct {
	Url            string
	SshUserAndHost string
	SshPath        string
}

var (
	Config        = NewConfig()
	httpPrefixRe  = regexp.MustCompile("\\Ahttps?://")
	defaultRemote = "origin"
)

func NewConfig() *Configuration {
	c := &Configuration{
		CurrentRemote: defaultRemote,
		isTracingHttp: len(os.Getenv("GIT_CURL_VERBOSE")) > 0,
	}
	return c
}

func ObjectUrl(endpoint Endpoint, oid string) (*url.URL, error) {
	u, err := url.Parse(endpoint.Url)
	if err != nil {
		return nil, err
	}

	u.Path = path.Join(u.Path, "objects")
	if len(oid) > 0 {
		u.Path = path.Join(u.Path, oid)
	}
	return u, nil
}

func (c *Configuration) Endpoint() Endpoint {
	if url, ok := c.GitConfig("lfs.url"); ok {
		return Endpoint{Url: url}
	}

	if len(c.CurrentRemote) > 0 && c.CurrentRemote != defaultRemote {
		if endpoint := c.RemoteEndpoint(c.CurrentRemote); len(endpoint.Url) > 0 {
			return endpoint
		}
	}

	return c.RemoteEndpoint(defaultRemote)
}

func (c *Configuration) ConcurrentTransfers() int {
	uploads := 3

	if v, ok := c.GitConfig("lfs.concurrenttransfers"); ok {
		n, err := strconv.Atoi(v)
		if err == nil && n > 0 {
			uploads = n
		}
	}

	return uploads
}

func (c *Configuration) BatchTransfer() bool {
	if v, ok := c.GitConfig("lfs.batch"); ok {
		if v == "true" || v == "" {
			return true
		}

		// Any numeric value except 0 is considered true
		if n, err := strconv.Atoi(v); err == nil && n != 0 {
			return true
		}
	}
	return false
}

func (c *Configuration) RemoteEndpoint(remote string) Endpoint {
	if len(remote) == 0 {
		remote = defaultRemote
	}

	if url, ok := c.GitConfig("remote." + remote + ".lfsurl"); ok {
		return Endpoint{Url: url}
	}

	if url, ok := c.GitConfig("remote." + remote + ".url"); ok {
		endpoint := Endpoint{Url: url}

		if !httpPrefixRe.MatchString(url) {
			pieces := strings.SplitN(url, ":", 2)
			hostPieces := strings.SplitN(pieces[0], "@", 2)
			if len(hostPieces) < 2 {
				endpoint.Url = "<unknown>"
				return endpoint
			}

			endpoint.SshUserAndHost = pieces[0]
			endpoint.SshPath = pieces[1]
			endpoint.Url = fmt.Sprintf("https://%s/%s", hostPieces[1], pieces[1])
		}

		if path.Ext(url) == ".git" {
			endpoint.Url += "/info/lfs"
		} else {
			endpoint.Url += ".git/info/lfs"
		}

		return endpoint
	}

	return Endpoint{}
}

func (c *Configuration) Remotes() []string {
	c.loadGitConfig()
	return c.remotes
}

func (c *Configuration) GitConfig(key string) (string, bool) {
	c.loadGitConfig()
	value, ok := c.gitConfig[strings.ToLower(key)]
	return value, ok
}

func (c *Configuration) SetConfig(key, value string) {
	c.loadGitConfig()
	c.gitConfig[key] = value
}

func (c *Configuration) ObjectUrl(oid string) (*url.URL, error) {
	return ObjectUrl(c.Endpoint(), oid)
}

type AltConfig struct {
	Remote map[string]*struct {
		Media string
	}

	Media struct {
		Url string
	}
}

func (c *Configuration) loadGitConfig() {
	c.loading.Lock()
	defer c.loading.Unlock()

	if c.gitConfig != nil {
		return
	}

	uniqRemotes := make(map[string]bool)

	c.gitConfig = make(map[string]string)

	var output string
	listOutput, err := git.Config.List()
	if err != nil {
		panic(fmt.Errorf("Error listing git config: %s", err))
	}

	configFile := filepath.Join(LocalWorkingDir, ".gitconfig")
	fileOutput, err := git.Config.ListFromFile(configFile)
	if err != nil {
		panic(fmt.Errorf("Error listing git config from file: %s", err))
	}

	output = fileOutput + "\n" + listOutput

	lines := strings.Split(output, "\n")
	for _, line := range lines {
		pieces := strings.SplitN(line, "=", 2)
		if len(pieces) < 2 {
			continue
		}
		key := strings.ToLower(pieces[0])
		c.gitConfig[key] = pieces[1]

		keyParts := strings.Split(key, ".")
		if len(keyParts) > 1 && keyParts[0] == "remote" {
			remote := keyParts[1]
			uniqRemotes[remote] = remote == "origin"
		}
	}

	c.remotes = make([]string, 0, len(uniqRemotes))
	for remote, isOrigin := range uniqRemotes {
		if isOrigin {
			continue
		}
		c.remotes = append(c.remotes, remote)
	}
}
<commit_msg>Removed unused struct<commit_after>package lfs

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"

	"github.com/github/git-lfs/git"
)

type Configuration struct {
	CurrentRemote         string
	gitConfig             map[string]string
	remotes               []string
	httpClient            *http.Client
	redirectingHttpClient *http.Client
	isTracingHttp         bool
	loading               sync.Mutex
}

type Endpoint struct {
	Url            string
	SshUserAndHost string
	SshPath        string
}

var (
	Config        = NewConfig()
	httpPrefixRe  = regexp.MustCompile("\\Ahttps?://")
	defaultRemote = "origin"
)

func NewConfig() *Configuration {
	c := &Configuration{
		CurrentRemote: defaultRemote,
		isTracingHttp: len(os.Getenv("GIT_CURL_VERBOSE")) > 0,
	}
	return c
}

func ObjectUrl(endpoint Endpoint, oid string) (*url.URL, error) {
	u, err := url.Parse(endpoint.Url)
	if err != nil {
		return nil, err
	}

	u.Path = path.Join(u.Path, "objects")
	if len(oid) > 0 {
		u.Path = path.Join(u.Path, oid)
	}
	return u, nil
}

func (c *Configuration) Endpoint() Endpoint {
	if url, ok := c.GitConfig("lfs.url"); ok {
		return Endpoint{Url: url}
	}

	if len(c.CurrentRemote) > 0 && c.CurrentRemote != defaultRemote {
		if endpoint := c.RemoteEndpoint(c.CurrentRemote); len(endpoint.Url) > 0 {
			return endpoint
		}
	}

	return c.RemoteEndpoint(defaultRemote)
}

func (c *Configuration) ConcurrentTransfers() int {
	uploads := 3

	if v, ok := c.GitConfig("lfs.concurrenttransfers"); ok {
		n, err := strconv.Atoi(v)
		if err == nil && n > 0 {
			uploads = n
		}
	}

	return uploads
}

func (c *Configuration) BatchTransfer() bool {
	if v, ok := c.GitConfig("lfs.batch"); ok {
		if v == "true" || v == "" {
			return true
		}

		// Any numeric value except 0 is considered true
		if n, err := strconv.Atoi(v); err == nil && n != 0 {
			return true
		}
	}
	return false
}

func (c *Configuration) RemoteEndpoint(remote string) Endpoint {
	if len(remote) == 0 {
		remote = defaultRemote
	}

	if url, ok := c.GitConfig("remote." + remote + ".lfsurl"); ok {
		return Endpoint{Url: url}
	}

	if url, ok := c.GitConfig("remote." + remote + ".url"); ok {
		endpoint := Endpoint{Url: url}

		if !httpPrefixRe.MatchString(url) {
			pieces := strings.SplitN(url, ":", 2)
			hostPieces := strings.SplitN(pieces[0], "@", 2)
			if len(hostPieces) < 2 {
				endpoint.Url = "<unknown>"
				return endpoint
			}

			endpoint.SshUserAndHost = pieces[0]
			endpoint.SshPath = pieces[1]
			endpoint.Url = fmt.Sprintf("https://%s/%s", hostPieces[1], pieces[1])
		}

		if path.Ext(url) == ".git" {
			endpoint.Url += "/info/lfs"
		} else {
			endpoint.Url += ".git/info/lfs"
		}

		return endpoint
	}

	return Endpoint{}
}

func (c *Configuration) Remotes() []string {
	c.loadGitConfig()
	return c.remotes
}

func (c *Configuration) GitConfig(key string) (string, bool) {
	c.loadGitConfig()
	value, ok := c.gitConfig[strings.ToLower(key)]
	return value, ok
}

func (c *Configuration) SetConfig(key, value string) {
	c.loadGitConfig()
	c.gitConfig[key] = value
}

func (c *Configuration) ObjectUrl(oid string) (*url.URL, error) {
	return ObjectUrl(c.Endpoint(), oid)
}

func (c *Configuration) loadGitConfig() {
	c.loading.Lock()
	defer c.loading.Unlock()

	if c.gitConfig != nil {
		return
	}

	uniqRemotes := make(map[string]bool)

	c.gitConfig = make(map[string]string)

	var output string
	listOutput, err := git.Config.List()
	if err != nil {
		panic(fmt.Errorf("Error listing git config: %s", err))
	}

	configFile := filepath.Join(LocalWorkingDir, ".gitconfig")
	fileOutput, err := git.Config.ListFromFile(configFile)
	if err != nil {
		panic(fmt.Errorf("Error listing git config from file: %s", err))
	}

	output = fileOutput + "\n" + listOutput

	lines := strings.Split(output, "\n")
	for _, line := range lines {
		pieces := strings.SplitN(line, "=", 2)
		if len(pieces) < 2 {
			continue
		}
		key := strings.ToLower(pieces[0])
		c.gitConfig[key] = pieces[1]

		keyParts := strings.Split(key, ".")
		if len(keyParts) > 1 && keyParts[0] == "remote" {
			remote := keyParts[1]
			uniqRemotes[remote] = remote == "origin"
		}
	}

	c.remotes = make([]string, 0, len(uniqRemotes))
	for remote, isOrigin := range uniqRemotes {
		if isOrigin {
			continue
		}
		c.remotes = append(c.remotes, remote)
	}
}
<|endoftext|>