<commit_before>package lfs

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strconv"

	"github.com/rubyist/tracerx"
)

const (
	mediaType = "application/vnd.git-lfs+json; charset-utf-8"
)

// The apiEvent* statuses (and the apiEvent channel) are used by
// UploadQueue to know when it is OK to process uploads concurrently.
const (
	apiEventFail = iota
	apiEventSuccess
)

var (
	lfsMediaTypeRE             = regexp.MustCompile(`\Aapplication/vnd\.git\-lfs\+json(;|\z)`)
	jsonMediaTypeRE            = regexp.MustCompile(`\Aapplication/json(;|\z)`)
	objectRelationDoesNotExist = errors.New("relation does not exist")
	hiddenHeaders              = map[string]bool{
		"Authorization": true,
	}

	defaultErrors = map[int]string{
		400: "Client error: %s",
		401: "Authorization error: %s\nCheck that you have proper access to the repository",
		403: "Authorization error: %s\nCheck that you have proper access to the repository",
		404: "Repository or object not found: %s\nCheck that it exists and that you have proper access to it",
		500: "Server error: %s",
	}

	apiEvent = make(chan int)
)

type objectResource struct {
	Oid   string                   `json:"oid,omitempty"`
	Size  int64                    `json:"size,omitempty"`
	Links map[string]*linkRelation `json:"_links,omitempty"`
}

func (o *objectResource) NewRequest(relation, method string) (*http.Request, Creds, error) {
	rel, ok := o.Rel(relation)
	if !ok {
		return nil, nil, objectRelationDoesNotExist
	}

	req, creds, err := newClientRequest(method, rel.Href)
	if err != nil {
		return nil, nil, err
	}

	for h, v := range rel.Header {
		req.Header.Set(h, v)
	}

	return req, creds, nil
}

func (o *objectResource) Rel(name string) (*linkRelation, bool) {
	if o.Links == nil {
		return nil, false
	}

	rel, ok := o.Links[name]
	return rel, ok
}

type linkRelation struct {
	Href   string            `json:"href"`
	Header map[string]string `json:"header,omitempty"`
}

type ClientError struct {
	Message          string `json:"message"`
	DocumentationUrl string `json:"documentation_url,omitempty"`
	RequestId        string `json:"request_id,omitempty"`
}

func (e *ClientError) Error() string {
	msg := e.Message
	if len(e.DocumentationUrl) > 0 {
		msg += "\nDocs: " + e.DocumentationUrl
	}
	if len(e.RequestId) > 0 {
		msg += "\nRequest ID: " + e.RequestId
	}
	return msg
}

func Download(oid string) (io.ReadCloser, int64, *WrappedError) {
	req, creds, err := newApiRequest("GET", oid)
	if err != nil {
		return nil, 0, Error(err)
	}

	res, obj, wErr := doApiRequest(req, creds)
	if wErr != nil {
		return nil, 0, wErr
	}

	req, creds, err = obj.NewRequest("download", "GET")
	if err != nil {
		return nil, 0, Error(err)
	}

	res, wErr = doHttpRequest(req, creds)
	if wErr != nil {
		return nil, 0, wErr
	}

	return res.Body, res.ContentLength, nil
}

type byteCloser struct {
	*bytes.Reader
}

func (b *byteCloser) Close() error {
	return nil
}

func Upload(oidPath, filename string, cb CopyCallback) *WrappedError {
	oid := filepath.Base(oidPath)
	file, err := os.Open(oidPath)
	if err != nil {
		return Error(err)
	}
	defer file.Close()

	stat, err := file.Stat()
	if err != nil {
		return Error(err)
	}

	reqObj := &objectResource{
		Oid:  oid,
		Size: stat.Size(),
	}

	by, err := json.Marshal(reqObj)
	if err != nil {
		return Error(err)
	}

	req, creds, err := newApiRequest("POST", oid)
	if err != nil {
		return Error(err)
	}

	req.Header.Set("Content-Type", mediaType)
	req.Header.Set("Content-Length", strconv.Itoa(len(by)))
	req.ContentLength = int64(len(by))
	req.Body = &byteCloser{bytes.NewReader(by)}

	tracerx.Printf("api: uploading %s (%s)", filename, oid)
	res, obj, wErr := doApiRequest(req, creds)
	if wErr != nil {
		sendApiEvent(apiEventFail)
		return wErr
	}

	sendApiEvent(apiEventSuccess)

	reader := &CallbackReader{
		C:         cb,
		TotalSize: reqObj.Size,
		Reader:    file,
	}

	if res.StatusCode == 200 {
		// Drain the reader to update any progress bars
		io.Copy(ioutil.Discard, reader)
		return nil
	}

	req, creds, err = obj.NewRequest("upload", "PUT")
	if err != nil {
		return Error(err)
	}

	if len(req.Header.Get("Content-Type")) == 0 {
		req.Header.Set("Content-Type", "application/octet-stream")
	}
	req.Header.Set("Content-Length", strconv.FormatInt(reqObj.Size, 10))
	req.ContentLength = reqObj.Size

	req.Body = ioutil.NopCloser(reader)

	res, wErr = doHttpRequest(req, creds)
	if wErr != nil {
		return wErr
	}

	if res.StatusCode > 299 {
		return Errorf(nil, "Invalid status for %s %s: %d", req.Method, req.URL, res.StatusCode)
	}

	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	req, creds, err = obj.NewRequest("verify", "POST")
	if err == objectRelationDoesNotExist {
		return nil
	} else if err != nil {
		return Error(err)
	}

	req.Header.Set("Content-Type", mediaType)
	req.Header.Set("Content-Length", strconv.Itoa(len(by)))
	req.ContentLength = int64(len(by))
	req.Body = ioutil.NopCloser(bytes.NewReader(by))
	res, wErr = doHttpRequest(req, creds)

	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	return wErr
}

func doHttpRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError) {
	res, err := DoHTTP(Config, req)

	var wErr *WrappedError

	if err != nil {
		wErr = Errorf(err, "Error for %s %s", res.Request.Method, res.Request.URL)
	} else {
		if creds != nil {
			saveCredentials(creds, res)
		}

		wErr = handleResponse(res)
	}

	if wErr != nil {
		if res != nil {
			setErrorResponseContext(wErr, res)
		} else {
			setErrorRequestContext(wErr, req)
		}
	}

	return res, wErr
}

func doApiRequestWithRedirects(req *http.Request, creds Creds, via []*http.Request) (*http.Response, *WrappedError) {
	res, wErr := doHttpRequest(req, creds)
	if wErr != nil {
		return nil, wErr
	}

	if res.StatusCode == 307 {
		redirectedReq, redirectedCreds, err := newClientRequest(req.Method, res.Header.Get("Location"))
		if err != nil {
			return nil, Errorf(err, err.Error())
		}

		via = append(via, req)
		seeker, ok := req.Body.(io.Seeker)
		if !ok {
			return nil, Errorf(nil, "Request body needs to be an io.Seeker to handle redirects.")
		}

		if _, err := seeker.Seek(0, 0); err != nil {
			return nil, Error(err)
		}
		redirectedReq.Body = req.Body
		redirectedReq.ContentLength = req.ContentLength

		if err = checkRedirect(redirectedReq, via); err != nil {
			return nil, Errorf(err, err.Error())
		}

		return doApiRequestWithRedirects(redirectedReq, redirectedCreds, via)
	}

	return res, nil
}

func doApiRequest(req *http.Request, creds Creds) (*http.Response, *objectResource, *WrappedError) {
	via := make([]*http.Request, 0, 4)
	res, wErr := doApiRequestWithRedirects(req, creds, via)
	if wErr != nil {
		return nil, nil, wErr
	}

	obj := &objectResource{}
	wErr = decodeApiResponse(res, obj)

	if wErr != nil {
		setErrorResponseContext(wErr, res)
		return nil, nil, wErr
	}

	return res, obj, nil
}

func handleResponse(res *http.Response) *WrappedError {
	if res.StatusCode < 400 {
		return nil
	}

	defer func() {
		io.Copy(ioutil.Discard, res.Body)
		res.Body.Close()
	}()

	cliErr := &ClientError{}
	wErr := decodeApiResponse(res, cliErr)
	if wErr == nil {
		if len(cliErr.Message) == 0 {
			wErr = defaultError(res)
		} else {
			wErr = Error(cliErr)
		}
	}

	wErr.Panic = res.StatusCode > 499 && res.StatusCode != 501 && res.StatusCode != 509
	return wErr
}

func decodeApiResponse(res *http.Response, obj interface{}) *WrappedError {
	ctype := res.Header.Get("Content-Type")
	if !(lfsMediaTypeRE.MatchString(ctype) || jsonMediaTypeRE.MatchString(ctype)) {
		return nil
	}

	err := json.NewDecoder(res.Body).Decode(obj)
	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	if err != nil {
		return Errorf(err, "Unable to parse HTTP response for %s %s", res.Request.Method, res.Request.URL)
	}

	return nil
}

func defaultError(res *http.Response) *WrappedError {
	var msgFmt string

	if f, ok := defaultErrors[res.StatusCode]; ok {
		msgFmt = f
	} else if res.StatusCode < 500 {
		msgFmt = defaultErrors[400] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
	} else {
		msgFmt = defaultErrors[500] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
	}

	return Error(fmt.Errorf(msgFmt, res.Request.URL))
}

func saveCredentials(creds Creds, res *http.Response) {
	if creds == nil {
		return
	}

	if res.StatusCode < 300 {
		execCreds(creds, "approve")
	} else if res.StatusCode == 401 {
		execCreds(creds, "reject")
	}
}

func newApiRequest(method, oid string) (*http.Request, Creds, error) {
	endpoint := Config.Endpoint()
	objectOid := oid
	operation := "download"
	if method == "POST" {
		objectOid = ""
		operation = "upload"
	}

	res, err := sshAuthenticate(endpoint, operation, oid)
	if err != nil {
		tracerx.Printf("ssh: attempted with %s.  Error: %s",
			endpoint.SshUserAndHost, err.Error(),
		)
	}

	if len(res.Href) > 0 {
		endpoint.Url = res.Href
	}

	u, err := ObjectUrl(endpoint, objectOid)
	if err != nil {
		return nil, nil, err
	}

	req, creds, err := newClientRequest(method, u.String())
	if err != nil {
		return nil, nil, err
	}

	req.Header.Set("Accept", mediaType)
	if res.Header != nil {
		for key, value := range res.Header {
			req.Header.Set(key, value)
		}
	}

	return req, creds, nil
}

func newClientRequest(method, rawurl string) (*http.Request, Creds, error) {
	req, err := http.NewRequest(method, rawurl, nil)
	if err != nil {
		return nil, nil, err
	}

	req.Header.Set("User-Agent", UserAgent)
	creds, err := getCreds(req)
	if err != nil {
		return nil, nil, err
	}

	return req, creds, nil
}

func getCreds(req *http.Request) (Creds, error) {
	if len(req.Header.Get("Authorization")) > 0 {
		return nil, nil
	}

	apiUrl, err := Config.ObjectUrl("")
	if err != nil {
		return nil, err
	}

	if req.URL.Scheme == apiUrl.Scheme &&
		req.URL.Host == apiUrl.Host {
		creds, err := credentials(req.URL)
		if err != nil {
			return nil, err
		}

		token := fmt.Sprintf("%s:%s", creds["username"], creds["password"])
		auth := "Basic " + base64.URLEncoding.EncodeToString([]byte(token))
		req.Header.Set("Authorization", auth)
		return creds, nil
	}

	return nil, nil
}

func setErrorResponseContext(err *WrappedError, res *http.Response) {
	err.Set("Status", res.Status)
	setErrorHeaderContext(err, "Request", res.Header)
	setErrorRequestContext(err, res.Request)
}

func setErrorRequestContext(err *WrappedError, req *http.Request) {
	err.Set("Endpoint", Config.Endpoint().Url)
	err.Set("URL", fmt.Sprintf("%s %s", req.Method, req.URL.String()))
	setErrorHeaderContext(err, "Response", req.Header)
}

func setErrorHeaderContext(err *WrappedError, prefix string, head http.Header) {
	for key, _ := range head {
		contextKey := fmt.Sprintf("%s:%s", prefix, key)
		if _, skip := hiddenHeaders[key]; skip {
			err.Set(contextKey, "--")
		} else {
			err.Set(contextKey, head.Get(key))
		}
	}
}

func sendApiEvent(event int) {
	select {
	case apiEvent <- event:
	default:
	}
}
<commit_msg>アー アアアア アーアー<commit_after>package lfs

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strconv"

	"github.com/rubyist/tracerx"
)

const (
	mediaType = "application/vnd.git-lfs+json; charset=utf-8"
)

// The apiEvent* statuses (and the apiEvent channel) are used by
// UploadQueue to know when it is OK to process uploads concurrently.
const (
	apiEventFail = iota
	apiEventSuccess
)

var (
	lfsMediaTypeRE             = regexp.MustCompile(`\Aapplication/vnd\.git\-lfs\+json(;|\z)`)
	jsonMediaTypeRE            = regexp.MustCompile(`\Aapplication/json(;|\z)`)
	objectRelationDoesNotExist = errors.New("relation does not exist")
	hiddenHeaders              = map[string]bool{
		"Authorization": true,
	}

	defaultErrors = map[int]string{
		400: "Client error: %s",
		401: "Authorization error: %s\nCheck that you have proper access to the repository",
		403: "Authorization error: %s\nCheck that you have proper access to the repository",
		404: "Repository or object not found: %s\nCheck that it exists and that you have proper access to it",
		500: "Server error: %s",
	}

	apiEvent = make(chan int)
)

type objectResource struct {
	Oid   string                   `json:"oid,omitempty"`
	Size  int64                    `json:"size,omitempty"`
	Links map[string]*linkRelation `json:"_links,omitempty"`
}

func (o *objectResource) NewRequest(relation, method string) (*http.Request, Creds, error) {
	rel, ok := o.Rel(relation)
	if !ok {
		return nil, nil, objectRelationDoesNotExist
	}

	req, creds, err := newClientRequest(method, rel.Href)
	if err != nil {
		return nil, nil, err
	}

	for h, v := range rel.Header {
		req.Header.Set(h, v)
	}

	return req, creds, nil
}

func (o *objectResource) Rel(name string) (*linkRelation, bool) {
	if o.Links == nil {
		return nil, false
	}

	rel, ok := o.Links[name]
	return rel, ok
}

type linkRelation struct {
	Href   string            `json:"href"`
	Header map[string]string `json:"header,omitempty"`
}

type ClientError struct {
	Message          string `json:"message"`
	DocumentationUrl string `json:"documentation_url,omitempty"`
	RequestId        string `json:"request_id,omitempty"`
}

func (e *ClientError) Error() string {
	msg := e.Message
	if len(e.DocumentationUrl) > 0 {
		msg += "\nDocs: " + e.DocumentationUrl
	}
	if len(e.RequestId) > 0 {
		msg += "\nRequest ID: " + e.RequestId
	}
	return msg
}

func Download(oid string) (io.ReadCloser, int64, *WrappedError) {
	req, creds, err := newApiRequest("GET", oid)
	if err != nil {
		return nil, 0, Error(err)
	}

	res, obj, wErr := doApiRequest(req, creds)
	if wErr != nil {
		return nil, 0, wErr
	}

	req, creds, err = obj.NewRequest("download", "GET")
	if err != nil {
		return nil, 0, Error(err)
	}

	res, wErr = doHttpRequest(req, creds)
	if wErr != nil {
		return nil, 0, wErr
	}

	return res.Body, res.ContentLength, nil
}

type byteCloser struct {
	*bytes.Reader
}

func (b *byteCloser) Close() error {
	return nil
}

func Upload(oidPath, filename string, cb CopyCallback) *WrappedError {
	oid := filepath.Base(oidPath)
	file, err := os.Open(oidPath)
	if err != nil {
		return Error(err)
	}
	defer file.Close()

	stat, err := file.Stat()
	if err != nil {
		return Error(err)
	}

	reqObj := &objectResource{
		Oid:  oid,
		Size: stat.Size(),
	}

	by, err := json.Marshal(reqObj)
	if err != nil {
		return Error(err)
	}

	req, creds, err := newApiRequest("POST", oid)
	if err != nil {
		return Error(err)
	}

	req.Header.Set("Content-Type", mediaType)
	req.Header.Set("Content-Length", strconv.Itoa(len(by)))
	req.ContentLength = int64(len(by))
	req.Body = &byteCloser{bytes.NewReader(by)}

	tracerx.Printf("api: uploading %s (%s)", filename, oid)
	res, obj, wErr := doApiRequest(req, creds)
	if wErr != nil {
		sendApiEvent(apiEventFail)
		return wErr
	}

	sendApiEvent(apiEventSuccess)

	reader := &CallbackReader{
		C:         cb,
		TotalSize: reqObj.Size,
		Reader:    file,
	}

	if res.StatusCode == 200 {
		// Drain the reader to update any progress bars
		io.Copy(ioutil.Discard, reader)
		return nil
	}

	req, creds, err = obj.NewRequest("upload", "PUT")
	if err != nil {
		return Error(err)
	}

	if len(req.Header.Get("Content-Type")) == 0 {
		req.Header.Set("Content-Type", "application/octet-stream")
	}
	req.Header.Set("Content-Length", strconv.FormatInt(reqObj.Size, 10))
	req.ContentLength = reqObj.Size

	req.Body = ioutil.NopCloser(reader)

	res, wErr = doHttpRequest(req, creds)
	if wErr != nil {
		return wErr
	}

	if res.StatusCode > 299 {
		return Errorf(nil, "Invalid status for %s %s: %d", req.Method, req.URL, res.StatusCode)
	}

	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	req, creds, err = obj.NewRequest("verify", "POST")
	if err == objectRelationDoesNotExist {
		return nil
	} else if err != nil {
		return Error(err)
	}

	req.Header.Set("Content-Type", mediaType)
	req.Header.Set("Content-Length", strconv.Itoa(len(by)))
	req.ContentLength = int64(len(by))
	req.Body = ioutil.NopCloser(bytes.NewReader(by))
	res, wErr = doHttpRequest(req, creds)

	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	return wErr
}

func doHttpRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError) {
	res, err := DoHTTP(Config, req)

	var wErr *WrappedError

	if err != nil {
		wErr = Errorf(err, "Error for %s %s", res.Request.Method, res.Request.URL)
	} else {
		if creds != nil {
			saveCredentials(creds, res)
		}

		wErr = handleResponse(res)
	}

	if wErr != nil {
		if res != nil {
			setErrorResponseContext(wErr, res)
		} else {
			setErrorRequestContext(wErr, req)
		}
	}

	return res, wErr
}

func doApiRequestWithRedirects(req *http.Request, creds Creds, via []*http.Request) (*http.Response, *WrappedError) {
	res, wErr := doHttpRequest(req, creds)
	if wErr != nil {
		return nil, wErr
	}

	if res.StatusCode == 307 {
		redirectedReq, redirectedCreds, err := newClientRequest(req.Method, res.Header.Get("Location"))
		if err != nil {
			return nil, Errorf(err, err.Error())
		}

		via = append(via, req)
		seeker, ok := req.Body.(io.Seeker)
		if !ok {
			return nil, Errorf(nil, "Request body needs to be an io.Seeker to handle redirects.")
		}

		if _, err := seeker.Seek(0, 0); err != nil {
			return nil, Error(err)
		}
		redirectedReq.Body = req.Body
		redirectedReq.ContentLength = req.ContentLength

		if err = checkRedirect(redirectedReq, via); err != nil {
			return nil, Errorf(err, err.Error())
		}

		return doApiRequestWithRedirects(redirectedReq, redirectedCreds, via)
	}

	return res, nil
}

func doApiRequest(req *http.Request, creds Creds) (*http.Response, *objectResource, *WrappedError) {
	via := make([]*http.Request, 0, 4)
	res, wErr := doApiRequestWithRedirects(req, creds, via)
	if wErr != nil {
		return nil, nil, wErr
	}

	obj := &objectResource{}
	wErr = decodeApiResponse(res, obj)

	if wErr != nil {
		setErrorResponseContext(wErr, res)
		return nil, nil, wErr
	}

	return res, obj, nil
}

func handleResponse(res *http.Response) *WrappedError {
	if res.StatusCode < 400 {
		return nil
	}

	defer func() {
		io.Copy(ioutil.Discard, res.Body)
		res.Body.Close()
	}()

	cliErr := &ClientError{}
	wErr := decodeApiResponse(res, cliErr)
	if wErr == nil {
		if len(cliErr.Message) == 0 {
			wErr = defaultError(res)
		} else {
			wErr = Error(cliErr)
		}
	}

	wErr.Panic = res.StatusCode > 499 && res.StatusCode != 501 && res.StatusCode != 509
	return wErr
}

func decodeApiResponse(res *http.Response, obj interface{}) *WrappedError {
	ctype := res.Header.Get("Content-Type")
	if !(lfsMediaTypeRE.MatchString(ctype) || jsonMediaTypeRE.MatchString(ctype)) {
		return nil
	}

	err := json.NewDecoder(res.Body).Decode(obj)
	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	if err != nil {
		return Errorf(err, "Unable to parse HTTP response for %s %s", res.Request.Method, res.Request.URL)
	}

	return nil
}

func defaultError(res *http.Response) *WrappedError {
	var msgFmt string

	if f, ok := defaultErrors[res.StatusCode]; ok {
		msgFmt = f
	} else if res.StatusCode < 500 {
		msgFmt = defaultErrors[400] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
	} else {
		msgFmt = defaultErrors[500] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
	}

	return Error(fmt.Errorf(msgFmt, res.Request.URL))
}

func saveCredentials(creds Creds, res *http.Response) {
	if creds == nil {
		return
	}

	if res.StatusCode < 300 {
		execCreds(creds, "approve")
	} else if res.StatusCode == 401 {
		execCreds(creds, "reject")
	}
}

func newApiRequest(method, oid string) (*http.Request, Creds, error) {
	endpoint := Config.Endpoint()
	objectOid := oid
	operation := "download"
	if method == "POST" {
		objectOid = ""
		operation = "upload"
	}

	res, err := sshAuthenticate(endpoint, operation, oid)
	if err != nil {
		tracerx.Printf("ssh: attempted with %s.  Error: %s",
			endpoint.SshUserAndHost, err.Error(),
		)
	}

	if len(res.Href) > 0 {
		endpoint.Url = res.Href
	}

	u, err := ObjectUrl(endpoint, objectOid)
	if err != nil {
		return nil, nil, err
	}

	req, creds, err := newClientRequest(method, u.String())
	if err != nil {
		return nil, nil, err
	}

	req.Header.Set("Accept", mediaType)
	if res.Header != nil {
		for key, value := range res.Header {
			req.Header.Set(key, value)
		}
	}

	return req, creds, nil
}

func newClientRequest(method, rawurl string) (*http.Request, Creds, error) {
	req, err := http.NewRequest(method, rawurl, nil)
	if err != nil {
		return nil, nil, err
	}

	req.Header.Set("User-Agent", UserAgent)
	creds, err := getCreds(req)
	if err != nil {
		return nil, nil, err
	}

	return req, creds, nil
}

func getCreds(req *http.Request) (Creds, error) {
	if len(req.Header.Get("Authorization")) > 0 {
		return nil, nil
	}

	apiUrl, err := Config.ObjectUrl("")
	if err != nil {
		return nil, err
	}

	if req.URL.Scheme == apiUrl.Scheme &&
		req.URL.Host == apiUrl.Host {
		creds, err := credentials(req.URL)
		if err != nil {
			return nil, err
		}

		token := fmt.Sprintf("%s:%s", creds["username"], creds["password"])
		auth := "Basic " + base64.URLEncoding.EncodeToString([]byte(token))
		req.Header.Set("Authorization", auth)
		return creds, nil
	}

	return nil, nil
}

func setErrorResponseContext(err *WrappedError, res *http.Response) {
	err.Set("Status", res.Status)
	setErrorHeaderContext(err, "Request", res.Header)
	setErrorRequestContext(err, res.Request)
}

func setErrorRequestContext(err *WrappedError, req *http.Request) {
	err.Set("Endpoint", Config.Endpoint().Url)
	err.Set("URL", fmt.Sprintf("%s %s", req.Method, req.URL.String()))
	setErrorHeaderContext(err, "Response", req.Header)
}

func setErrorHeaderContext(err *WrappedError, prefix string, head http.Header) {
	for key, _ := range head {
		contextKey := fmt.Sprintf("%s:%s", prefix, key)
		if _, skip := hiddenHeaders[key]; skip {
			err.Set(contextKey, "--")
		} else {
			err.Set(contextKey, head.Get(key))
		}
	}
}

func sendApiEvent(event int) {
	select {
	case apiEvent <- event:
	default:
	}
}
<|endoftext|>


<commit_before>package lfs

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/cheggaaa/pb"
	"github.com/rubyist/tracerx"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
)

const (
	mediaType = "application/vnd.git-lfs+json; charset-utf-8"
)

var (
	lfsMediaTypeRE             = regexp.MustCompile(`\Aapplication/vnd\.git\-lfs\+json(;|\z)`)
	jsonMediaTypeRE            = regexp.MustCompile(`\Aapplication/json(;|\z)`)
	objectRelationDoesNotExist = errors.New("relation does not exist")
	hiddenHeaders              = map[string]bool{
		"Authorization": true,
	}

	// 401 and 403 print the same default error message
	defaultErrors = map[int]string{
		400: "Client error: %s",
		401: "Authorization error: %s\nCheck that you have proper access to the repository",
		404: "Repository or object not found: %s\nCheck that it exists and that you have proper access to it",
		500: "Server error: %s",
	}
)

type objectResource struct {
	Oid   string                   `json:"oid,omitempty"`
	Size  int64                    `json:"size,omitempty"`
	Links map[string]*linkRelation `json:"_links,omitempty"`
}

func (o *objectResource) NewRequest(relation, method string) (*http.Request, Creds, error) {
	rel, ok := o.Rel(relation)
	if !ok {
		return nil, nil, objectRelationDoesNotExist
	}

	req, creds, err := newClientRequest(method, rel.Href)
	if err != nil {
		return nil, nil, err
	}

	for h, v := range rel.Header {
		req.Header.Set(h, v)
	}

	return req, creds, nil
}

func (o *objectResource) Rel(name string) (*linkRelation, bool) {
	if o.Links == nil {
		return nil, false
	}

	rel, ok := o.Links[name]
	return rel, ok
}

type linkRelation struct {
	Href   string            `json:"href"`
	Header map[string]string `json:"header,omitempty"`
}

type ClientError struct {
	Message          string `json:"message"`
	DocumentationUrl string `json:"documentation_url,omitempty"`
	RequestId        string `json:"request_id,omitempty"`
}

func (e *ClientError) Error() string {
	msg := e.Message
	if len(e.DocumentationUrl) > 0 {
		msg += "\nDocs: " + e.DocumentationUrl
	}
	if len(e.RequestId) > 0 {
		msg += "\nRequest ID: " + e.RequestId
	}
	return msg
}

func Download(oid string) (io.ReadCloser, int64, *WrappedError) {
	req, creds, err := newApiRequest("GET", oid)
	if err != nil {
		return nil, 0, Error(err)
	}

	res, obj, wErr := doApiRequest(req, creds)
	if wErr != nil {
		return nil, 0, wErr
	}

	req, creds, err = obj.NewRequest("download", "GET")
	if err != nil {
		return nil, 0, Error(err)
	}

	res, wErr = doHttpRequest(req, creds)
	if wErr != nil {
		return nil, 0, wErr
	}

	return res.Body, res.ContentLength, nil
}

type byteCloser struct {
	*bytes.Reader
}

func (b *byteCloser) Close() error {
	return nil
}

func Upload(oidPath, filename string, cb CopyCallback) *WrappedError {
	oid := filepath.Base(oidPath)
	file, err := os.Open(oidPath)
	if err != nil {
		return Error(err)
	}
	defer file.Close()

	stat, err := file.Stat()
	if err != nil {
		return Error(err)
	}

	reqObj := &objectResource{
		Oid:  oid,
		Size: stat.Size(),
	}

	by, err := json.Marshal(reqObj)
	if err != nil {
		return Error(err)
	}

	req, creds, err := newApiRequest("POST", "")
	if err != nil {
		return Error(err)
	}

	req.Header.Set("Content-Type", mediaType)
	req.Header.Set("Content-Length", strconv.Itoa(len(by)))
	req.ContentLength = int64(len(by))
	req.Body = &byteCloser{bytes.NewReader(by)}

	tracerx.Printf("api: uploading %s (%s)", filename, oid)
	res, obj, wErr := doApiRequest(req, creds)
	if wErr != nil {
		return wErr
	}

	if res.StatusCode == 200 {
		return nil
	}

	req, creds, err = obj.NewRequest("upload", "PUT")
	if err != nil {
		return Error(err)
	}

	if len(req.Header.Get("Content-Type")) == 0 {
		req.Header.Set("Content-Type", "application/octet-stream")
	}
	req.Header.Set("Content-Length", strconv.FormatInt(reqObj.Size, 10))
	req.ContentLength = reqObj.Size

	reader := &CallbackReader{
		C:         cb,
		TotalSize: reqObj.Size,
		Reader:    file,
	}

	bar := pb.New64(reqObj.Size)
	bar.SetUnits(pb.U_BYTES)
	bar.Start()

	req.Body = ioutil.NopCloser(bar.NewProxyReader(reader))

	res, wErr = doHttpRequest(req, creds)
	if wErr != nil {
		return wErr
	}

	if res.StatusCode > 299 {
		return Errorf(nil, "Invalid status for %s %s: %d", req.Method, req.URL, res.StatusCode)
	}

	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	req, creds, err = obj.NewRequest("verify", "POST")
	if err == objectRelationDoesNotExist {
		return nil
	} else if err != nil {
		return Error(err)
	}

	req.Header.Set("Content-Type", mediaType)
	req.Header.Set("Content-Length", strconv.Itoa(len(by)))
	req.ContentLength = int64(len(by))
	req.Body = ioutil.NopCloser(bytes.NewReader(by))
	res, wErr = doHttpRequest(req, creds)

	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	return wErr
}

func doHttpRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError) {
	res, err := DoHTTP(Config, req)

	var wErr *WrappedError

	if err != nil {
		wErr = Errorf(err, "Error for %s %s", res.Request.Method, res.Request.URL)
	} else {
		if creds != nil {
			saveCredentials(creds, res)
		}

		wErr = handleResponse(res)
	}

	if wErr != nil {
		if res != nil {
			setErrorResponseContext(wErr, res)
		} else {
			setErrorRequestContext(wErr, req)
		}
	}

	return res, wErr
}

func doApiRequestWithRedirects(req *http.Request, creds Creds, via []*http.Request) (*http.Response, *WrappedError) {
	fmt.Println("doApiRequest:", req)
	res, wErr := doHttpRequest(req, creds)
	if wErr != nil {
		return res, wErr
	}

	if res.StatusCode == 307 {
		redirectedReq, redirectedCreds, err := newClientRequest(req.Method, res.Header.Get("Location"))
		if err != nil {
			return res, Errorf(err, err.Error())
		}

		via = append(via, req)
		fmt.Println("via:", via)
		if seeker, ok := req.Body.(io.Seeker); ok {
			_, err := seeker.Seek(0, 0)
			if err != nil {
				return res, Error(err)
			}
			redirectedReq.Body = req.Body
			redirectedReq.ContentLength = req.ContentLength
		} else {
			return res, Errorf(nil, "Request body needs to be an io.Seeker to handle redirects.")
		}

		if err = checkRedirect(redirectedReq, via); err != nil {
			return res, Errorf(err, err.Error())
		}

		return doApiRequestWithRedirects(redirectedReq, redirectedCreds, via)
	}

	return res, wErr
}

func doApiRequest(req *http.Request, creds Creds) (*http.Response, *objectResource, *WrappedError) {
	via := make([]*http.Request, 0, 4)
	res, wErr := doApiRequestWithRedirects(req, creds, via)
	if wErr != nil {
		return res, nil, wErr
	}

	obj := &objectResource{}
	wErr = decodeApiResponse(res, obj)

	if wErr != nil {
		setErrorResponseContext(wErr, res)
	}

	return res, obj, wErr
}

func handleResponse(res *http.Response) *WrappedError {
	if res.StatusCode < 400 {
		return nil
	}

	defer func() {
		io.Copy(ioutil.Discard, res.Body)
		res.Body.Close()
	}()

	cliErr := &ClientError{}
	wErr := decodeApiResponse(res, cliErr)
	if wErr == nil {
		if len(cliErr.Message) == 0 {
			wErr = defaultError(res)
		} else {
			wErr = Error(cliErr)
		}
	}

	wErr.Panic = res.StatusCode > 499 && res.StatusCode != 501 && res.StatusCode != 509
	return wErr
}

func decodeApiResponse(res *http.Response, obj interface{}) *WrappedError {
	ctype := res.Header.Get("Content-Type")
	if !(lfsMediaTypeRE.MatchString(ctype) || jsonMediaTypeRE.MatchString(ctype)) {
		return nil
	}

	err := json.NewDecoder(res.Body).Decode(obj)
	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	if err != nil {
		return Errorf(err, "Unable to parse HTTP response for %s %s", res.Request.Method, res.Request.URL)
	}

	return nil
}

func defaultError(res *http.Response) *WrappedError {
	var msgFmt string

	if f, ok := defaultErrors[res.StatusCode]; ok {
		msgFmt = f
	} else if res.StatusCode < 500 {
		msgFmt = defaultErrors[400] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
	} else {
		msgFmt = defaultErrors[500] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
	}

	return Error(fmt.Errorf(msgFmt, res.Request.URL))
}

func saveCredentials(creds Creds, res *http.Response) {
	if creds == nil {
		return
	}

	if res.StatusCode < 300 {
		execCreds(creds, "approve")
	} else if res.StatusCode == 401 {
		execCreds(creds, "reject")
	}
}

func newApiRequest(method, oid string) (*http.Request, Creds, error) {
	u, err := Config.ObjectUrl(oid)
	if err != nil {
		return nil, nil, err
	}

	req, creds, err := newClientRequest(method, u.String())
	if err == nil {
		req.Header.Set("Accept", mediaType)
	}
	return req, creds, err
}

func newClientRequest(method, rawurl string) (*http.Request, Creds, error) {
	req, err := http.NewRequest(method, rawurl, nil)
	if err != nil {
		return req, nil, err
	}

	req.Header.Set("User-Agent", UserAgent)
	creds, err := getCreds(req)
	return req, creds, err
}

func getCreds(req *http.Request) (Creds, error) {
	if len(req.Header.Get("Authorization")) > 0 {
		return nil, nil
	}

	apiUrl, err := Config.ObjectUrl("")
	if err != nil {
		return nil, err
	}

	if req.URL.Scheme == apiUrl.Scheme &&
		req.URL.Host == apiUrl.Host {
		creds, err := credentials(req.URL)
		if err != nil {
			return nil, err
		}

		token := fmt.Sprintf("%s:%s", creds["username"], creds["password"])
		auth := "Basic " + base64.URLEncoding.EncodeToString([]byte(token))
		req.Header.Set("Authorization", auth)
		return creds, nil
	}

	return nil, nil
}

func setErrorRequestContext(err *WrappedError, req *http.Request) {
	err.Set("Endpoint", Config.Endpoint())
	err.Set("URL", fmt.Sprintf("%s %s", req.Method, req.URL.String()))
	setErrorHeaderContext(err, "Response", req.Header)
}

func setErrorResponseContext(err *WrappedError, res *http.Response) {
	err.Set("Status", res.Status)
	setErrorHeaderContext(err, "Request", res.Header)
	setErrorRequestContext(err, res.Request)
}

func setErrorHeaderContext(err *WrappedError, prefix string, head http.Header) {
	for key, _ := range head {
		contextKey := fmt.Sprintf("%s:%s", prefix, key)
		if _, skip := hiddenHeaders[key]; skip {
			err.Set(contextKey, "--")
		} else {
			err.Set(contextKey, head.Get(key))
		}
	}
}

func init() {
	defaultErrors[403] = defaultErrors[401]
}
<commit_msg>remove debugging prints<commit_after>package lfs

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/cheggaaa/pb"
	"github.com/rubyist/tracerx"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
)

const (
	mediaType = "application/vnd.git-lfs+json; charset-utf-8"
)

var (
	lfsMediaTypeRE             = regexp.MustCompile(`\Aapplication/vnd\.git\-lfs\+json(;|\z)`)
	jsonMediaTypeRE            = regexp.MustCompile(`\Aapplication/json(;|\z)`)
	objectRelationDoesNotExist = errors.New("relation does not exist")
	hiddenHeaders              = map[string]bool{
		"Authorization": true,
	}

	// 401 and 403 print the same default error message
	defaultErrors = map[int]string{
		400: "Client error: %s",
		401: "Authorization error: %s\nCheck that you have proper access to the repository",
		404: "Repository or object not found: %s\nCheck that it exists and that you have proper access to it",
		500: "Server error: %s",
	}
)

type objectResource struct {
	Oid   string                   `json:"oid,omitempty"`
	Size  int64                    `json:"size,omitempty"`
	Links map[string]*linkRelation `json:"_links,omitempty"`
}

func (o *objectResource) NewRequest(relation, method string) (*http.Request, Creds, error) {
	rel, ok := o.Rel(relation)
	if !ok {
		return nil, nil, objectRelationDoesNotExist
	}

	req, creds, err := newClientRequest(method, rel.Href)
	if err != nil {
		return nil, nil, err
	}

	for h, v := range rel.Header {
		req.Header.Set(h, v)
	}

	return req, creds, nil
}

func (o *objectResource) Rel(name string) (*linkRelation, bool) {
	if o.Links == nil {
		return nil, false
	}

	rel, ok := o.Links[name]
	return rel, ok
}

type linkRelation struct {
	Href   string            `json:"href"`
	Header map[string]string `json:"header,omitempty"`
}

type ClientError struct {
	Message          string `json:"message"`
	DocumentationUrl string `json:"documentation_url,omitempty"`
	RequestId        string `json:"request_id,omitempty"`
}

func (e *ClientError) Error() string {
	msg := e.Message
	if len(e.DocumentationUrl) > 0 {
		msg += "\nDocs: " + e.DocumentationUrl
	}
	if len(e.RequestId) > 0 {
		msg += "\nRequest ID: " + e.RequestId
	}
	return msg
}

func Download(oid string) (io.ReadCloser, int64, *WrappedError) {
	req, creds, err := newApiRequest("GET", oid)
	if err != nil {
		return nil, 0, Error(err)
	}

	res, obj, wErr := doApiRequest(req, creds)
	if wErr != nil {
		return nil, 0, wErr
	}

	req, creds, err = obj.NewRequest("download", "GET")
	if err != nil {
		return nil, 0, Error(err)
	}

	res, wErr = doHttpRequest(req, creds)
	if wErr != nil {
		return nil, 0, wErr
	}

	return res.Body, res.ContentLength, nil
}

type byteCloser struct {
	*bytes.Reader
}

func (b *byteCloser) Close() error {
	return nil
}

func Upload(oidPath, filename string, cb CopyCallback) *WrappedError {
	oid := filepath.Base(oidPath)
	file, err := os.Open(oidPath)
	if err != nil {
		return Error(err)
	}
	defer file.Close()

	stat, err := file.Stat()
	if err != nil {
		return Error(err)
	}

	reqObj := &objectResource{
		Oid:  oid,
		Size: stat.Size(),
	}

	by, err := json.Marshal(reqObj)
	if err != nil {
		return Error(err)
	}

	req, creds, err := newApiRequest("POST", "")
	if err != nil {
		return Error(err)
	}

	req.Header.Set("Content-Type", mediaType)
	req.Header.Set("Content-Length", strconv.Itoa(len(by)))
	req.ContentLength = int64(len(by))
	req.Body = &byteCloser{bytes.NewReader(by)}

	tracerx.Printf("api: uploading %s (%s)", filename, oid)
	res, obj, wErr := doApiRequest(req, creds)
	if wErr != nil {
		return wErr
	}

	if res.StatusCode == 200 {
		return nil
	}

	req, creds, err = obj.NewRequest("upload", "PUT")
	if err != nil {
		return Error(err)
	}

	if len(req.Header.Get("Content-Type")) == 0 {
		req.Header.Set("Content-Type", "application/octet-stream")
	}
	req.Header.Set("Content-Length", strconv.FormatInt(reqObj.Size, 10))
	req.ContentLength = reqObj.Size

	reader := &CallbackReader{
		C:         cb,
		TotalSize: reqObj.Size,
		Reader:    file,
	}

	bar := pb.New64(reqObj.Size)
	bar.SetUnits(pb.U_BYTES)
	bar.Start()

	req.Body = ioutil.NopCloser(bar.NewProxyReader(reader))

	res, wErr = doHttpRequest(req, creds)
	if wErr != nil {
		return wErr
	}

	if res.StatusCode > 299 {
		return Errorf(nil, "Invalid status for %s %s: %d", req.Method, req.URL, res.StatusCode)
	}

	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	req, creds, err = obj.NewRequest("verify", "POST")
	if err == objectRelationDoesNotExist {
		return nil
	} else if err != nil {
		return Error(err)
	}

	req.Header.Set("Content-Type", mediaType)
	req.Header.Set("Content-Length", strconv.Itoa(len(by)))
	req.ContentLength = int64(len(by))
	req.Body = ioutil.NopCloser(bytes.NewReader(by))
	res, wErr = doHttpRequest(req, creds)

	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	return wErr
}

func doHttpRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError) {
	res, err := DoHTTP(Config, req)

	var wErr *WrappedError

	if err != nil {
		wErr = Errorf(err, "Error for %s %s", res.Request.Method, res.Request.URL)
	} else {
		if creds != nil {
			saveCredentials(creds, res)
		}

		wErr = handleResponse(res)
	}

	if wErr != nil {
		if res != nil {
			setErrorResponseContext(wErr, res)
		} else {
			setErrorRequestContext(wErr, req)
		}
	}

	return res, wErr
}

func doApiRequestWithRedirects(req *http.Request, creds Creds, via []*http.Request) (*http.Response, *WrappedError) {
	res, wErr := doHttpRequest(req, creds)
	if wErr != nil {
		return res, wErr
	}

	if res.StatusCode == 307 {
		redirectedReq, redirectedCreds, err := newClientRequest(req.Method, res.Header.Get("Location"))
		if err != nil {
			return res, Errorf(err, err.Error())
		}

		via = append(via, req)
		if seeker, ok := req.Body.(io.Seeker); ok {
			_, err := seeker.Seek(0, 0)
			if err != nil {
				return res, Error(err)
			}
			redirectedReq.Body = req.Body
			redirectedReq.ContentLength = req.ContentLength
		} else {
			return res, Errorf(nil, "Request body needs to be an io.Seeker to handle redirects.")
		}

		if err = checkRedirect(redirectedReq, via); err != nil {
			return res, Errorf(err, err.Error())
		}

		return doApiRequestWithRedirects(redirectedReq, redirectedCreds, via)
	}

	return res, wErr
}

func doApiRequest(req *http.Request, creds Creds) (*http.Response, *objectResource, *WrappedError) {
	via := make([]*http.Request, 0, 4)
	res, wErr := doApiRequestWithRedirects(req, creds, via)
	if wErr != nil {
		return res, nil, wErr
	}

	obj := &objectResource{}
	wErr = decodeApiResponse(res, obj)

	if wErr != nil {
		setErrorResponseContext(wErr, res)
	}

	return res, obj, wErr
}

func handleResponse(res *http.Response) *WrappedError {
	if res.StatusCode < 400 {
		return nil
	}

	defer func() {
		io.Copy(ioutil.Discard, res.Body)
		res.Body.Close()
	}()

	cliErr := &ClientError{}
	wErr := decodeApiResponse(res, cliErr)
	if wErr == nil {
		if len(cliErr.Message) == 0 {
			wErr = defaultError(res)
		} else {
			wErr = Error(cliErr)
		}
	}

	wErr.Panic = res.StatusCode > 499 && res.StatusCode != 501 && res.StatusCode != 509
	return wErr
}

func decodeApiResponse(res *http.Response, obj interface{}) *WrappedError {
	ctype := res.Header.Get("Content-Type")
	if !(lfsMediaTypeRE.MatchString(ctype) || jsonMediaTypeRE.MatchString(ctype)) {
		return nil
	}

	err := json.NewDecoder(res.Body).Decode(obj)
	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	if err != nil {
		return Errorf(err, "Unable to parse HTTP response for %s %s", res.Request.Method, res.Request.URL)
	}

	return nil
}

func defaultError(res *http.Response) *WrappedError {
	var msgFmt string

	if f, ok := defaultErrors[res.StatusCode]; ok {
		msgFmt = f
	} else if res.StatusCode < 500 {
		msgFmt = defaultErrors[400] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
	} else {
		msgFmt = defaultErrors[500] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
	}

	return Error(fmt.Errorf(msgFmt, res.Request.URL))
}

func saveCredentials(creds Creds, res *http.Response) {
	if creds == nil {
		return
	}

	if res.StatusCode < 300 {
		execCreds(creds, "approve")
	} else if res.StatusCode == 401 {
		execCreds(creds, "reject")
	}
}

func newApiRequest(method, oid string) (*http.Request, Creds, error) {
	u, err := Config.ObjectUrl(oid)
	if err != nil {
		return nil, nil, err
	}

	req, creds, err := newClientRequest(method, u.String())
	if err == nil {
		req.Header.Set("Accept", mediaType)
	}
	return req, creds, err
}

func newClientRequest(method, rawurl string) (*http.Request, Creds, error) {
	req, err := http.NewRequest(method, rawurl, nil)
	if err != nil {
		return req, nil, err
	}

	req.Header.Set("User-Agent", UserAgent)
	creds, err := getCreds(req)
	return req, creds, err
}

func getCreds(req *http.Request) (Creds, error) {
	if len(req.Header.Get("Authorization")) > 0 {
		return nil, nil
	}

	apiUrl, err := Config.ObjectUrl("")
	if err != nil {
		return nil, err
	}

	if req.URL.Scheme == apiUrl.Scheme &&
		req.URL.Host == apiUrl.Host {
		creds, err := credentials(req.URL)
		if err != nil {
			return nil, err
		}

		token := fmt.Sprintf("%s:%s", creds["username"], creds["password"])
		auth := "Basic " + base64.URLEncoding.EncodeToString([]byte(token))
		req.Header.Set("Authorization", auth)
		return creds, nil
	}

	return nil, nil
}

func setErrorRequestContext(err *WrappedError, req *http.Request) {
	err.Set("Endpoint", Config.Endpoint())
	err.Set("URL", fmt.Sprintf("%s %s", req.Method, req.URL.String()))
	setErrorHeaderContext(err, "Response", req.Header)
}

func setErrorResponseContext(err *WrappedError, res *http.Response) {
	err.Set("Status", res.Status)
	setErrorHeaderContext(err, "Request", res.Header)
	setErrorRequestContext(err, res.Request)
}

func setErrorHeaderContext(err *WrappedError, prefix string, head http.Header) {
	for key, _ := range head {
		contextKey := fmt.Sprintf("%s:%s", prefix, key)
		if _, skip := hiddenHeaders[key]; skip {
			err.Set(contextKey, "--")
		} else {
			err.Set(contextKey, head.Get(key))
		}
	}
}

func init() {
	defaultErrors[403] = defaultErrors[401]
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
)

type lhcbConfig struct {
	ConfigBase
}

func newLHCbConfig() *lhcbConfig {
	return &lhcbConfig{
		ConfigBase: ConfigBase{
			siteroot: os.Getenv("MYSITEROOT"),
			repourl:  "http://test-lbrpm.web.cern.ch/test-lbrpm",
		},
	}
}

func (cfg *lhcbConfig) Name() string {
	return "lhcb"
}

func (cfg *lhcbConfig) DefaultSiteroot() string {
	return "/opt/LHCbSoft"
}

// RelocateArgs returns the arguments to be passed to RPM for the repositories
func (cfg *lhcbConfig) RelocateArgs(siteroot string) []string {
	return []string{
		"--relocate", fmt.Sprintf("%s=%s", "/opt/lcg", filepath.Join(siteroot, "lcg", "releases")),
		"--relocate", fmt.Sprintf("%s=%s", "/opt/LHCbSoft", siteroot),
		"--badreloc",
	}
}

// RelocateFile returns the relocated file path
func (cfg *lhcbConfig) RelocateFile(fname, siteroot string) string {
	fname = strings.Replace(fname, "/opt/lcg", filepath.Join(siteroot, "lcg", "releases"), 1)
	fname = strings.Replace(fname, "/opt/LHCbSoft", siteroot, 1)
	return fname
}

func (cfg *lhcbConfig) InitYum(ctx *Context) error {
	var err error
	repourl := cfg.RepoUrl()
	if repourl[len(repourl)-1] == '/' {
		repourl = repourl[:len(repourl)-1]
	}
	repodir := ctx.yumreposd
	err = os.MkdirAll(repodir, 0644)
	if err != nil {
		return err
	}

	// lcg stuff
	{
		repo := filepath.Join(repodir, "lcg.repo")
		f, err := os.Create(repo)
		if err != nil {
			return err
		}
		defer f.Close()

		err = ctx.writeYumRepo(f, map[string]string{
			"name": "lcg",
			"url":  "http://service-spi.web.cern.ch/service-spi/external/rpms/lcg",
		})
		if err != nil {
			return err
		}

		err = f.Sync()
		if err != nil {
			return err
		}

		err = f.Close()
		if err != nil {
			return err
		}
	}

	// lhcb stuff
	{
		repo := filepath.Join(repodir, "lhcb.repo")
		f, err := os.Create(repo)
		if err != nil {
			return err
		}
		defer f.Close()

		err = ctx.writeYumRepo(f, map[string]string{
			"name": "lhcbold",
			"url":  repourl + "/rpm",
		})
		if err != nil {
			return err
		}

		err = ctx.writeYumRepo(f, map[string]string{
			"name": "lhcb",
			"url":  repourl + "/lhcb",
		})
		if err != nil {
			return err
		}

		err = f.Sync()
		if err != nil {
			return err
		}

		err = f.Close()
		if err != nil {
			return err
		}
	}

	return err
}
<commit_msg>lhcbconfig: point to the new rpm repository<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
)

type lhcbConfig struct {
	ConfigBase
}

func newLHCbConfig() *lhcbConfig {
	return &lhcbConfig{
		ConfigBase: ConfigBase{
			siteroot: os.Getenv("MYSITEROOT"),
			repourl:  "http://cern.ch/lhcbproject/dist/rpm",
		},
	}
}

func (cfg *lhcbConfig) Name() string {
	return "lhcb"
}

func (cfg *lhcbConfig) DefaultSiteroot() string {
	return "/opt/LHCbSoft"
}

// RelocateArgs returns the arguments to be passed to RPM for the repositories
func (cfg *lhcbConfig) RelocateArgs(siteroot string) []string {
	return []string{
		"--relocate", fmt.Sprintf("%s=%s", "/opt/lcg", filepath.Join(siteroot, "lcg", "releases")),
		"--relocate", fmt.Sprintf("%s=%s", "/opt/LHCbSoft", siteroot),
		"--badreloc",
	}
}

// RelocateFile returns the relocated file path
func (cfg *lhcbConfig) RelocateFile(fname, siteroot string) string {
	fname = strings.Replace(fname, "/opt/lcg", filepath.Join(siteroot, "lcg", "releases"), 1)
	fname = strings.Replace(fname, "/opt/LHCbSoft", siteroot, 1)
	return fname
}

func (cfg *lhcbConfig) InitYum(ctx *Context) error {
	var err error
	repourl := cfg.RepoUrl()
	if repourl[len(repourl)-1] == '/' {
		repourl = repourl[:len(repourl)-1]
	}
	repodir := ctx.yumreposd
	err = os.MkdirAll(repodir, 0644)
	if err != nil {
		return err
	}

	// lcg stuff
	{
		repo := filepath.Join(repodir, "lcg.repo")
		f, err := os.Create(repo)
		if err != nil {
			return err
		}
		defer f.Close()

		err = ctx.writeYumRepo(f, map[string]string{
			"name": "lcg",
			"url":  "http://service-spi.web.cern.ch/service-spi/external/rpms/lcg",
		})
		if err != nil {
			return err
		}

		err = f.Sync()
		if err != nil {
			return err
		}

		err = f.Close()
		if err != nil {
			return err
		}
	}

	// lhcb stuff
	{
		repo := filepath.Join(repodir, "lhcb.repo")
		f, err := os.Create(repo)
		if err != nil {
			return err
		}
		defer f.Close()

		err = ctx.writeYumRepo(f, map[string]string{
			"name": "lhcb",
			"url":  repourl + "/lhcb",
		})
		if err != nil {
			return err
		}

		err = f.Sync()
		if err != nil {
			return err
		}

		err = f.Close()
		if err != nil {
			return err
		}
	}

	return err
}
<|endoftext|>


<commit_before>package gae

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"runtime"
	"strings"
	"sync"
	"time"

	"appengine"
	"appengine/taskqueue"
	"code.google.com/p/go-uuid/uuid"
	"github.com/go-martini/martini"
	"github.com/knightso/base/gae/bq"
	"github.com/martini-contrib/render"
)

type ExMartini struct {
	*martini.Martini
	martini.Router
}

type MartiniOption struct {
	AdditionalHandlers []martini.Handler
	Log2bq             bool
}

func NewMartini(option MartiniOption) *ExMartini {
	r := martini.NewRouter()
	m := martini.New()

	m.Use(func(c martini.Context, r *http.Request, l *log.Logger) {
		ac := appengine.NewContext(r)
		gaelog := log.New(logWriter{ac}, l.Prefix(), l.Flags())
		c.Map(gaelog)
	})
	m.Use(func(c martini.Context, r *http.Request) {
		ac := appengine.NewContext(r)
		c.Next()
		for l := popLog(); l != ""; l = popLog() {
			if option.Log2bq == false {
				ac.Debugf(l)
				continue
			}

			id := uuid.NewUUID()
			uuidString := id.String()
			now := time.Now()
			record := make(map[string]interface{})
			record["id"] = uuidString
			record["date"] = now
			record["log"] = l
			task := bq.Task{
				LogID:    "DebugLog",
				InsertID: uuidString,
				Time:     now,
				Record:   record,
			}
			payload, err := json.Marshal(task)
			if err != nil {
				ac.Warningf("%s", err.Error())
				continue
			}
			_, err = taskqueue.Add(ac, &taskqueue.Task{
				Payload: payload,
				Method:  "PULL",
				Tag:     "debuglog",
			}, "log2bigquery")
			if err != nil {
				ac.Warningf("%s", err.Error())
				continue
			}

			ac.Debugf(l)
		}
	})
	m.Use(martini.Logger())
	m.Use(martini.Recovery())
	m.Use(render.Renderer())
	for _, h := range option.AdditionalHandlers {
		m.Use(h)
	}
	m.MapTo(r, (*martini.Route)(nil))
	m.Action(r.Handle)

	return &ExMartini{m, r}
}

type logWriter struct {
	ac appengine.Context
}

func (w logWriter) Write(p []byte) (n int, err error) {
	w.ac.Debugf(string(p))
	return len(p), nil
}

var logs []string
var logMutex sync.Mutex

var LOG_ENABLED bool

func init() {
	logs = make([]string, 0)
}

func Logf(s string, a ...interface{}) {
	if !LOG_ENABLED {
		return
	}

	logMutex.Lock()
	defer logMutex.Unlock()

	logs = append(logs, location()+": "+fmt.Sprintf(s, a...))
}

func popLog() string {
	if !LOG_ENABLED {
		return ""
	}

	logMutex.Lock()
	defer logMutex.Unlock()

	if len(logs) == 0 {
		return ""
	}

	s := logs[0]
	logs = logs[1:]

	return s
}

func location() string {
	_, file, line, ok := runtime.Caller(2)
	if ok {
		if index := strings.LastIndex(file, "/"); index >= 0 {
			file = file[index+1:]
		} else if index = strings.LastIndex(file, "\\"); index >= 0 {
			file = file[index+1:]
		}
	} else {
		file = "???"
		line = -1
	}
	return fmt.Sprintf("%s:%d", file, line)
}
<commit_msg>modified to use bq.SendLog<commit_after>package gae

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"runtime"
	"strings"
	"sync"
	"time"

	"appengine"
	"appengine/taskqueue"
	"code.google.com/p/go-uuid/uuid"
	"github.com/go-martini/martini"
	"github.com/knightso/base/gae/bq"
	"github.com/martini-contrib/render"
)

type ExMartini struct {
	*martini.Martini
	martini.Router
}

type MartiniOption struct {
	AdditionalHandlers []martini.Handler
	Log2bq             bool
}

func NewMartini(option MartiniOption) *ExMartini {
	r := martini.NewRouter()
	m := martini.New()

	m.Use(func(c martini.Context, r *http.Request, l *log.Logger) {
		ac := appengine.NewContext(r)
		gaelog := log.New(logWriter{ac}, l.Prefix(), l.Flags())
		c.Map(gaelog)
	})
	m.Use(func(c martini.Context, r *http.Request) {
		ac := appengine.NewContext(r)
		c.Next()
		for l := popLog(); l != ""; l = popLog() {
			if option.Log2bq == false {
				ac.Debugf(l)
				continue
			}

			id := uuid.NewUUID()
			uuidString := id.String()
			now := time.Now()
			record := make(map[string]interface{})
			record["id"] = uuidString
			record["date"] = now
			record["log"] = l

			err := bq.SendLog(ac, "debuglog", uuidString, record)
			if err != nil {
				ac.Warningf("%s", err.Error())
				continue
			}

			ac.Debugf(l)
		}
	})
	m.Use(martini.Logger())
	m.Use(martini.Recovery())
	m.Use(render.Renderer())
	for _, h := range option.AdditionalHandlers {
		m.Use(h)
	}
	m.MapTo(r, (*martini.Route)(nil))
	m.Action(r.Handle)

	return &ExMartini{m, r}
}

type logWriter struct {
	ac appengine.Context
}

func (w logWriter) Write(p []byte) (n int, err error) {
	w.ac.Debugf(string(p))
	return len(p), nil
}

var logs []string
var logMutex sync.Mutex

var LOG_ENABLED bool

func init() {
	logs = make([]string, 0)
}

func Logf(s string, a ...interface{}) {
	if !LOG_ENABLED {
		return
	}

	logMutex.Lock()
	defer logMutex.Unlock()

	logs = append(logs, location()+": "+fmt.Sprintf(s, a...))
}

func popLog() string {
	if !LOG_ENABLED {
		return ""
	}

	logMutex.Lock()
	defer logMutex.Unlock()

	if len(logs) == 0 {
		return ""
	}

	s := logs[0]
	logs = logs[1:]

	return s
}

func location() string {
	_, file, line, ok := runtime.Caller(2)
	if ok {
		if index := strings.LastIndex(file, "/"); index >= 0 {
			file = file[index+1:]
		} else if index = strings.LastIndex(file, "\\"); index >= 0 {
			file = file[index+1:]
		}
	} else {
		file = "???"
		line = -1
	}
	return fmt.Sprintf("%s:%d", file, line)
}
<|endoftext|>


<commit_before>package mirror

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"time"

	"github.com/cybozu-go/aptutil/apt"
	"github.com/cybozu-go/log"
	"github.com/pkg/errors"
	"golang.org/x/net/context"
	"golang.org/x/net/context/ctxhttp"
)

const (
	timestampFormat  = "20060102_150405"
	progressInterval = 5 * time.Minute
)

var (
	validID = regexp.MustCompile(`^[a-z0-9_-]+$`)
)

// Mirror implements mirroring logics.
type Mirror struct {
	id      string
	dir     string
	mc      *MirrConfig
	storage *Storage
	current *Storage

	semaphore chan struct{}
	client    *http.Client
}

// NewMirror constructs a Mirror for given mirror id.
func NewMirror(t time.Time, id string, c *Config) (*Mirror, error) {
	dir := filepath.Clean(c.Dir)
	mc, ok := c.Mirrors[id]
	if !ok {
		return nil, errors.New("no such mirror: " + id)
	}

	// sanity checks
	if !validID.MatchString(id) {
		return nil, errors.New("invalid id: " + id)
	}
	if err := mc.Check(); err != nil {
		return nil, errors.Wrap(err, id)
	}

	var currentStorage *Storage
	curdir, err := filepath.EvalSymlinks(filepath.Join(dir, id))
	switch {
	case os.IsNotExist(err):
	case err != nil:
		return nil, errors.Wrap(err, id)
	default:
		currentStorage, err = NewStorage(filepath.Dir(curdir), id)
		if err != nil {
			return nil, errors.Wrap(err, id)
		}
		err = currentStorage.Load()
		if err != nil {
			return nil, errors.Wrap(err, id)
		}
	}

	d := filepath.Join(dir, "."+id+"."+t.Format(timestampFormat))
	err = os.Mkdir(d, 0755)
	if err != nil {
		return nil, errors.Wrap(err, id)
	}
	storage, err := NewStorage(d, id)
	if err != nil {
		return nil, errors.Wrap(err, id)
	}

	sem := make(chan struct{}, c.MaxConns)
	for i := 0; i < c.MaxConns; i++ {
		sem <- struct{}{}
	}

	transport := &http.Transport{
		Proxy:               http.ProxyFromEnvironment,
		MaxIdleConnsPerHost: c.MaxConns,
	}

	mr := &Mirror{
		id:        id,
		dir:       dir,
		mc:        mc,
		storage:   storage,
		current:   currentStorage,
		semaphore: sem,
		client: &http.Client{
			Transport: transport,
		},
	}
	return mr, nil
}

// Update updates mirrored files.
//
// This method is intended to be called as goroutine.
func (m *Mirror) Update(ctx context.Context, ch chan<- error) {
	log.Info("download Release/InRelease", map[string]interface{}{
		"_id": m.id,
	})
	fiMap, err := m.downloadRelease(ctx)
	if err != nil {
		ch <- errors.Wrap(err, m.id)
		return
	}

	if len(fiMap) == 0 {
		ch <- errors.New(m.id + ": found no Release/InRelease")
		return
	}

	// WORKAROUND: some (dell) repositories have invalid Release
	// that contains wrong checksum for itself.  Ignore them.
	for _, p := range m.mc.ReleaseFiles() {
		delete(fiMap, p)
	}

	// download (or reuse) all indices
	log.Info("download other indices", map[string]interface{}{
		"_id":      m.id,
		"_indices": len(fiMap),
	})
	err = m.downloadFiles(ctx, fiMap, true)
	if err != nil {
		ch <- errors.Wrap(err, m.id)
		return
	}

	// extract file information
	fiMap2 := make(map[string]*apt.FileInfo)
	for p := range fiMap {
		if !m.mc.MatchingIndex(p) || !apt.IsSupported(p) {
			continue
		}
		f, err := m.storage.Open(p)
		switch {
		case err == nil:
		case os.IsNotExist(err):
			continue
		default:
			ch <- errors.Wrap(err, m.id)
			return
		}
		fil, err := apt.ExtractFileInfo(p, f)
		f.Close()
		if err != nil {
			ch <- errors.Wrap(err, m.id)
			return
		}
		for _, fi2 := range fil {
			fi2path := fi2.Path()
			if _, ok := fiMap[fi2path]; ok {
				// already included in Release/InRelease
				continue
			}
			fiMap2[fi2.Path()] = fi2
		}
	}

	// download all files matching the configuration.
	log.Info("download items", map[string]interface{}{
		"_id":    m.id,
		"_items": len(fiMap2),
	})
	err = m.downloadFiles(ctx, fiMap2, false)
	if err != nil {
		ch <- errors.Wrap(err, m.id)
		return
	}

	// all files are downloaded (or reused)
	log.Info("saving meta data", map[string]interface{}{
		"_id": m.id,
	})
	err = m.storage.Save()
	if err != nil {
		ch <- errors.Wrap(err, m.id)
		return
	}

	// replace the symlink atomically
	tname := filepath.Join(m.dir, m.id+".tmp")
	os.Remove(tname)
	err = os.Symlink(filepath.Join(m.storage.Dir(), m.id), tname)
	if err != nil {
		ch <- errors.Wrap(err, m.id)
		return
	}
	DirSync(m.dir)
	err = os.Rename(tname, filepath.Join(m.dir, m.id))
	if err != nil {
		ch <- errors.Wrap(err, m.id)
		return
	}
	DirSync(m.dir)

	log.Info("update succeeded", map[string]interface{}{
		"_id": m.id,
	})
	ch <- nil
}

type dlResult struct {
	status int
	path   string
	fi     *apt.FileInfo
	data   []byte
	err    error
}

// download is a goroutine to download an item.
func (m *Mirror) download(ctx context.Context,
	p string, fi *apt.FileInfo, ch chan<- *dlResult) {

	r := &dlResult{
		path: p,
	}
	defer func() {
		ch <- r
		m.semaphore <- struct{}{}
	}()

	resp, err := ctxhttp.Get(ctx, m.client, m.mc.Resolve(p).String())
	if err != nil {
		r.err = err
		return
	}
	if log.Enabled(log.LvDebug) {
		log.Debug("downloaded", map[string]interface{}{
			"_id":     m.id,
			"_path":   p,
			"_status": resp.StatusCode,
		})
	}

	r.status = resp.StatusCode
	data, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		r.err = err
		return
	}
	if r.status != 200 {
		return
	}

	fi2 := apt.MakeFileInfo(p, data)
	if fi != nil && !fi.Same(fi2) {
		r.err = errors.New("invalid checksum for " + p)
		return
	}
	r.fi = fi2
	r.data = data
}

func (m *Mirror) downloadRelease(ctx context.Context) (map[string]*apt.FileInfo, error) {
	releases := m.mc.ReleaseFiles()
	results := make(chan *dlResult, len(releases))

	for _, p := range releases {
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		case <-m.semaphore:
		}

		go m.download(ctx, p, nil, results)
	}

	fiMap := make(map[string]*apt.FileInfo)
	for i := 0; i < len(releases); i++ {
		r := <-results
		if r.err != nil {
			return nil, errors.Wrap(r.err, "download")
		}
		switch {
		case r.status == http.StatusOK:
			err := m.storage.Store(r.fi, r.data)
			if err != nil {
				return nil, errors.Wrap(err, "storage.Store")
			}
			if apt.IsSupported(r.path) {
				fil, err := apt.ExtractFileInfo(r.path, bytes.NewReader(r.data))
				if err != nil {
					return nil, errors.Wrap(err, "ExtractFileInfo: "+r.path)
				}
				for _, fi := range fil {
					fiMap[fi.Path()] = fi
				}
			}

		case 400 <= r.status && r.status < 500:
			continue

		default:
			return nil, fmt.Errorf("status %d for %s", r.status, r.path)
		}
	}

	return fiMap, nil
}

func (m *Mirror) downloadFiles(ctx context.Context,
	fiMap map[string]*apt.FileInfo, allowMissing bool) error {
	results := make(chan *dlResult, len(fiMap))

	loggedAt := time.Now()

	var reused, downloading, downloaded int
	for p, fi := range fiMap {
		now := time.Now()
		if now.Sub(loggedAt) > progressInterval {
			loggedAt = now
			log.Info("download progress", map[string]interface{}{
				"_id":         m.id,
				"_total":      len(fiMap),
				"_reused":     reused,
				"_downloaded": downloaded,
			})
		}
		if m.current != nil {
			fi2, fullpath := m.current.Lookup(fi)
			if fi2 != nil {
				err := m.storage.StoreLink(fi2, fullpath)
				if err != nil {
					return errors.Wrap(err, "storage.StoreLink")
				}
				reused++
				if log.Enabled(log.LvDebug) {
					log.Debug("reuse item", map[string]interface{}{
						"_id":   m.id,
						"_path": p,
					})
				}
				continue
			}
		}

		// check download results early
		for {
			select {
			case r := <-results:
				downloaded++
				if r.err != nil {
					return errors.Wrap(r.err, "download")
				}
				if allowMissing && r.status == http.StatusNotFound {
					log.Warn("missing file", map[string]interface{}{
						"_id":   m.id,
						"_path": r.path,
					})
					continue
				}
				if r.status != http.StatusOK {
					return fmt.Errorf("status %d for %s", r.status, r.path)
				}
				err := m.storage.Store(r.fi, r.data)
				if err != nil {
					return errors.Wrap(err, "storage.Store")
				}
			default:
				goto DOWNLOAD
			}
		}

	DOWNLOAD:
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-m.semaphore:
		}

		go m.download(ctx, p, fi, results)
		downloading++
	}

	for downloading != downloaded {
		r := <-results
		downloaded++
		if r.err != nil {
			return errors.Wrap(r.err, "download")
		}
		if allowMissing && r.status == http.StatusNotFound {
			log.Warn("missing file", map[string]interface{}{
				"_id":   m.id,
				"_path": r.path,
			})
			continue
		}
		if r.status != http.StatusOK {
			return fmt.Errorf("status %d for %s", r.status, r.path)
		}
		err := m.storage.Store(r.fi, r.data)
		if err != nil {
			return errors.Wrap(err, "storage.Store")
		}
	}

	log.Info("stats", map[string]interface{}{
		"_id":         m.id,
		"_reused":     reused,
		"_downloaded": downloaded,
	})

	return nil
}
<commit_msg>[mirror] retry on HTTP errors.<commit_after>package mirror

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"time"

	"github.com/cybozu-go/aptutil/apt"
	"github.com/cybozu-go/log"
	"github.com/pkg/errors"
	"golang.org/x/net/context"
	"golang.org/x/net/context/ctxhttp"
)

const (
	timestampFormat  = "20060102_150405"
	progressInterval = 5 * time.Minute
	httpRetries      = 3
)

var (
	validID = regexp.MustCompile(`^[a-z0-9_-]+$`)
)

// Mirror implements mirroring logics.
type Mirror struct {
	id      string
	dir     string
	mc      *MirrConfig
	storage *Storage
	current *Storage

	semaphore chan struct{}
	client    *http.Client
}

// NewMirror constructs a Mirror for given mirror id.
func NewMirror(t time.Time, id string, c *Config) (*Mirror, error) {
	dir := filepath.Clean(c.Dir)
	mc, ok := c.Mirrors[id]
	if !ok {
		return nil, errors.New("no such mirror: " + id)
	}

	// sanity checks
	if !validID.MatchString(id) {
		return nil, errors.New("invalid id: " + id)
	}
	if err := mc.Check(); err != nil {
		return nil, errors.Wrap(err, id)
	}

	var currentStorage *Storage
	curdir, err := filepath.EvalSymlinks(filepath.Join(dir, id))
	switch {
	case os.IsNotExist(err):
	case err != nil:
		return nil, errors.Wrap(err, id)
	default:
		currentStorage, err = NewStorage(filepath.Dir(curdir), id)
		if err != nil {
			return nil, errors.Wrap(err, id)
		}
		err = currentStorage.Load()
		if err != nil {
			return nil, errors.Wrap(err, id)
		}
	}

	d := filepath.Join(dir, "."+id+"."+t.Format(timestampFormat))
	err = os.Mkdir(d, 0755)
	if err != nil {
		return nil, errors.Wrap(err, id)
	}
	storage, err := NewStorage(d, id)
	if err != nil {
		return nil, errors.Wrap(err, id)
	}

	sem := make(chan struct{}, c.MaxConns)
	for i := 0; i < c.MaxConns; i++ {
		sem <- struct{}{}
	}

	transport := &http.Transport{
		Proxy:               http.ProxyFromEnvironment,
		MaxIdleConnsPerHost: c.MaxConns,
	}

	mr := &Mirror{
		id:        id,
		dir:       dir,
		mc:        mc,
		storage:   storage,
		current:   currentStorage,
		semaphore: sem,
		client: &http.Client{
			Transport: transport,
		},
	}
	return mr, nil
}

// Update updates mirrored files.
//
// This method is intended to be called as goroutine.
func (m *Mirror) Update(ctx context.Context, ch chan<- error) {
	log.Info("download Release/InRelease", map[string]interface{}{
		"_id": m.id,
	})
	fiMap, err := m.downloadRelease(ctx)
	if err != nil {
		ch <- errors.Wrap(err, m.id)
		return
	}

	if len(fiMap) == 0 {
		ch <- errors.New(m.id + ": found no Release/InRelease")
		return
	}

	// WORKAROUND: some (dell) repositories have invalid Release
	// that contains wrong checksum for itself.  Ignore them.
	for _, p := range m.mc.ReleaseFiles() {
		delete(fiMap, p)
	}

	// download (or reuse) all indices
	log.Info("download other indices", map[string]interface{}{
		"_id":      m.id,
		"_indices": len(fiMap),
	})
	err = m.downloadFiles(ctx, fiMap, true)
	if err != nil {
		ch <- errors.Wrap(err, m.id)
		return
	}

	// extract file information
	fiMap2 := make(map[string]*apt.FileInfo)
	for p := range fiMap {
		if !m.mc.MatchingIndex(p) || !apt.IsSupported(p) {
			continue
		}
		f, err := m.storage.Open(p)
		switch {
		case err == nil:
		case os.IsNotExist(err):
			continue
		default:
			ch <- errors.Wrap(err, m.id)
			return
		}
		fil, err := apt.ExtractFileInfo(p, f)
		f.Close()
		if err != nil {
			ch <- errors.Wrap(err, m.id)
			return
		}
		for _, fi2 := range fil {
			fi2path := fi2.Path()
			if _, ok := fiMap[fi2path]; ok {
				// already included in Release/InRelease
				continue
			}
			fiMap2[fi2.Path()] = fi2
		}
	}

	// download all files matching the configuration.
	log.Info("download items", map[string]interface{}{
		"_id":    m.id,
		"_items": len(fiMap2),
	})
	err = m.downloadFiles(ctx, fiMap2, false)
	if err != nil {
		ch <- errors.Wrap(err, m.id)
		return
	}

	// all files are downloaded (or reused)
	log.Info("saving meta data", map[string]interface{}{
		"_id": m.id,
	})
	err = m.storage.Save()
	if err != nil {
		ch <- errors.Wrap(err, m.id)
		return
	}

	// replace the symlink atomically
	tname := filepath.Join(m.dir, m.id+".tmp")
	os.Remove(tname)
	err = os.Symlink(filepath.Join(m.storage.Dir(), m.id), tname)
	if err != nil {
		ch <- errors.Wrap(err, m.id)
		return
	}
	DirSync(m.dir)
	err = os.Rename(tname, filepath.Join(m.dir, m.id))
	if err != nil {
		ch <- errors.Wrap(err, m.id)
		return
	}
	DirSync(m.dir)

	log.Info("update succeeded", map[string]interface{}{
		"_id": m.id,
	})
	ch <- nil
}

type dlResult struct {
	status int
	path   string
	fi     *apt.FileInfo
	data   []byte
	err    error
}

// download is a goroutine to download an item.
func (m *Mirror) download(ctx context.Context,
	p string, fi *apt.FileInfo, ch chan<- *dlResult) {

	r := &dlResult{
		path: p,
	}
	defer func() {
		ch <- r
		m.semaphore <- struct{}{}
	}()

	var retries int

RETRY:
	if retries > 0 {
		log.Warn("retrying download", map[string]interface{}{
			"_id":   m.id,
			"_path": p,
		})
	}
	resp, err := ctxhttp.Get(ctx, m.client, m.mc.Resolve(p).String())
	if err != nil {
		if retries < httpRetries {
			retries++
			goto RETRY
		}
		r.err = err
		return
	}
	if log.Enabled(log.LvDebug) {
		log.Debug("downloaded", map[string]interface{}{
			"_id":     m.id,
			"_path":   p,
			"_status": resp.StatusCode,
		})
	}

	r.status = resp.StatusCode
	data, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		if retries < httpRetries {
			retries++
			goto RETRY
		}
		r.err = err
		return
	}
	if r.status >= 500 && retries < httpRetries {
		retries++
		goto RETRY
	}
	if r.status != 200 {
		return
	}

	fi2 := apt.MakeFileInfo(p, data)
	if fi != nil && !fi.Same(fi2) {
		r.err = errors.New("invalid checksum for " + p)
		return
	}
	r.fi = fi2
	r.data = data
}

func (m *Mirror) downloadRelease(ctx context.Context) (map[string]*apt.FileInfo, error) {
	releases := m.mc.ReleaseFiles()
	results := make(chan *dlResult, len(releases))

	for _, p := range releases {
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		case <-m.semaphore:
		}

		go m.download(ctx, p, nil, results)
	}

	fiMap := make(map[string]*apt.FileInfo)
	for i := 0; i < len(releases); i++ {
		r := <-results
		if r.err != nil {
			return nil, errors.Wrap(r.err, "download")
		}
		switch {
		case r.status == http.StatusOK:
			err := m.storage.Store(r.fi, r.data)
			if err != nil {
				return nil, errors.Wrap(err, "storage.Store")
			}
			if apt.IsSupported(r.path) {
				fil, err := apt.ExtractFileInfo(r.path, bytes.NewReader(r.data))
				if err != nil {
					return nil, errors.Wrap(err, "ExtractFileInfo: "+r.path)
				}
				for _, fi := range fil {
					fiMap[fi.Path()] = fi
				}
			}

		case 400 <= r.status && r.status < 500:
			continue

		default:
			return nil, fmt.Errorf("status %d for %s", r.status, r.path)
		}
	}

	return fiMap, nil
}

func (m *Mirror) downloadFiles(ctx context.Context,
	fiMap map[string]*apt.FileInfo, allowMissing bool) error {
	results := make(chan *dlResult, len(fiMap))

	loggedAt := time.Now()

	var reused, downloading, downloaded int
	for p, fi := range fiMap {
		now := time.Now()
		if now.Sub(loggedAt) > progressInterval {
			loggedAt = now
			log.Info("download progress", map[string]interface{}{
				"_id":         m.id,
				"_total":      len(fiMap),
				"_reused":     reused,
				"_downloaded": downloaded,
			})
		}
		if m.current != nil {
			fi2, fullpath := m.current.Lookup(fi)
			if fi2 != nil {
				err := m.storage.StoreLink(fi2, fullpath)
				if err != nil {
					return errors.Wrap(err, "storage.StoreLink")
				}
				reused++
				if log.Enabled(log.LvDebug) {
					log.Debug("reuse item", map[string]interface{}{
						"_id":   m.id,
						"_path": p,
					})
				}
				continue
			}
		}

		// check download results early
		for {
			select {
			case r := <-results:
				downloaded++
				if r.err != nil {
					return errors.Wrap(r.err, "download")
				}
				if allowMissing && r.status == http.StatusNotFound {
					log.Warn("missing file", map[string]interface{}{
						"_id":   m.id,
						"_path": r.path,
					})
					continue
				}
				if r.status != http.StatusOK {
					return fmt.Errorf("status %d for %s", r.status, r.path)
				}
				err := m.storage.Store(r.fi, r.data)
				if err != nil {
					return errors.Wrap(err, "storage.Store")
				}
			default:
				goto DOWNLOAD
			}
		}

	DOWNLOAD:
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-m.semaphore:
		}

		go m.download(ctx, p, fi, results)
		downloading++
	}

	for downloading != downloaded {
		r := <-results
		downloaded++
		if r.err != nil {
			return errors.Wrap(r.err, "download")
		}
		if allowMissing && r.status == http.StatusNotFound {
			log.Warn("missing file", map[string]interface{}{
				"_id":   m.id,
				"_path": r.path,
			})
			continue
		}
		if r.status != http.StatusOK {
			return fmt.Errorf("status %d for %s", r.status, r.path)
		}
		err := m.storage.Store(r.fi, r.data)
		if err != nil {
			return errors.Wrap(err, "storage.Store")
		}
	}

	log.Info("stats", map[string]interface{}{
		"_id":         m.id,
		"_reused":     reused,
		"_downloaded": downloaded,
	})

	return nil
}
<|endoftext|>


<commit_before>package plugins

import (
	"path/filepath"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestPluginScans(t *testing.T) {

	Convey("When scaning for plugins", t, func() {
		path, _ := filepath.Abs("../../src/app/plugins")
		err := scan(path)

		So(err, ShouldBeNil)
		So(len(DataSources), ShouldEqual, 1)
	})
}
<commit_msg>Fixed plugins go test<commit_after>package plugins

import (
	"path/filepath"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestPluginScans(t *testing.T) {

	Convey("When scaning for plugins", t, func() {
		path, _ := filepath.Abs("../../src/app/plugins")
		err := scan(path)

		So(err, ShouldBeNil)
		So(len(DataSources), ShouldBeGreaterThan, 1)
	})
}
<|endoftext|>


<commit_before>package core

import (
	"sync"
	"sync/atomic"

	"github.com/coel-lang/coel/src/lib/debug"
)

type thunkState int32

const (
	normal thunkState = iota
	app
	spinLock
)

// Thunk you all!
type Thunk struct {
	result    Value
	function  *Thunk
	args      Arguments
	state     thunkState
	blackHole sync.WaitGroup
	info      *debug.Info
}

// Normal creates a thunk of a normal value as its result.
func Normal(v Value) *Thunk {
	if t, ok := v.(*Thunk); ok {
		return t
	}

	return &Thunk{result: v, state: normal}
}

// App creates a thunk applying a function to arguments.
func App(f *Thunk, args Arguments) *Thunk {
	return AppWithInfo(f, args, debug.NewGoInfo(1))
}

// AppWithInfo is the same as App except that it stores debug information
// in the thunk.
func AppWithInfo(f *Thunk, args Arguments, i *debug.Info) *Thunk {
	t := &Thunk{
		function: f,
		args:     args,
		state:    app,
		info:     i,
	}
	t.blackHole.Add(1)
	return t
}

// PApp is not PPap.
func PApp(f *Thunk, ps ...*Thunk) *Thunk {
	return AppWithInfo(f, NewPositionalArguments(ps...), debug.NewGoInfo(1))
}

// evalAny evaluates a thunk and returns a pure or impure (effect) value.
func (t *Thunk) evalAny() Value {
	if t.lock(normal) {
		for {
			v := t.function.Eval()
			t.function = nil

			if _, ok := v.(ErrorType); ok {
				t.result = v
				break
			}

			f, ok := v.(callable)

			if !ok {
				t.result = NotCallableError(v).Eval()
				break
			}

			t.result = f.call(t.args)
			t.args = Arguments{}

			if _, ok := t.result.(ErrorType); ok {
				break
			}

			child, ok := t.result.(*Thunk)

			if !ok {
				break
			}

			if ok := child.delegateEval(t); !ok {
				t.result = child.evalAny()
				break
			}
		}

		if e, ok := t.result.(ErrorType); ok {
			t.result = e.Chain(t.info)
		}

		t.finalize()
	} else {
		t.blackHole.Wait()
	}

	return t.result
}

func (t *Thunk) lock(s thunkState) bool {
	for {
		switch t.loadState() {
		case normal:
			return false
		case app:
			if t.compareAndSwapState(app, s) {
				return true
			}
		}
	}
}

func (t *Thunk) delegateEval(parent *Thunk) bool {
	if t.lock(spinLock) {
		parent.function, t.function = t.function, identity
		parent.args, t.args = t.args, NewPositionalArguments(parent)
		parent.info = t.info
		t.storeState(app)
		return true
	}

	return false
}

func (t *Thunk) finalize() {
	t.function = nil
	t.args = Arguments{}
	t.storeState(normal)
	t.blackHole.Done()
}

func (t *Thunk) compareAndSwapState(old, new thunkState) bool {
	return atomic.CompareAndSwapInt32((*int32)(&t.state), int32(old), int32(new))
}

func (t *Thunk) loadState() thunkState {
	return thunkState(atomic.LoadInt32((*int32)(&t.state)))
}

func (t *Thunk) storeState(new thunkState) {
	atomic.StoreInt32((*int32)(&t.state), int32(new))
}

// Eval evaluates a pure value.
func (t *Thunk) Eval() Value {
	if _, ok := t.evalAny().(effectType); ok {
		return impureFunctionError().Eval().(ErrorType).Chain(t.info)
	}

	return t.result
}

// EvalEffect evaluates an effect expression.
func (t *Thunk) EvalEffect() Value {
	v := t.evalAny()
	e, ok := v.(effectType)

	if !ok {
		return NotEffectError(v).Eval().(ErrorType).Chain(t.info)
	}

	return e.value.Eval()
}
<commit_msg>Refactor thunk evaluation algorithm<commit_after>package core

import (
	"sync"
	"sync/atomic"

	"github.com/coel-lang/coel/src/lib/debug"
)

type thunkState int32

const (
	normal thunkState = iota
	app
	spinLock
)

// Thunk you all!
type Thunk struct {
	result    Value
	function  *Thunk
	args      Arguments
	state     thunkState
	blackHole sync.WaitGroup
	info      *debug.Info
}

// Normal creates a thunk of a normal value as its result.
func Normal(v Value) *Thunk {
	if t, ok := v.(*Thunk); ok {
		return t
	}

	return &Thunk{result: v, state: normal}
}

// App creates a thunk applying a function to arguments.
func App(f *Thunk, args Arguments) *Thunk {
	return AppWithInfo(f, args, debug.NewGoInfo(1))
}

// AppWithInfo is the same as App except that it stores debug information
// in the thunk.
func AppWithInfo(f *Thunk, args Arguments, i *debug.Info) *Thunk {
	t := &Thunk{
		function: f,
		args:     args,
		state:    app,
		info:     i,
	}
	t.blackHole.Add(1)
	return t
}

// PApp is not PPap.
func PApp(f *Thunk, ps ...*Thunk) *Thunk {
	return AppWithInfo(f, NewPositionalArguments(ps...), debug.NewGoInfo(1))
}

// evalAny evaluates a thunk and returns a pure or impure (effect) value.
func (t *Thunk) evalAny() Value {
	if t.lock(normal) {
		for {
			v := t.function.Eval()
			t.function = nil

			f, ok := v.(callable)

			if !ok {
				t.result = NotCallableError(v).Eval()
				t.args = Arguments{}
				break
			}

			t.result = f.call(t.args)
			t.args = Arguments{}

			child, ok := t.result.(*Thunk)

			if !ok {
				break
			}

			if !child.delegateEval(t) {
				t.result = child.evalAny()
				break
			}
		}

		if e, ok := t.result.(ErrorType); ok {
			t.result = e.Chain(t.info)
		}

		// No need to clean up or finalize t.function, t.args, and t.state here
		// because of invariants.
		t.blackHole.Done()
	} else {
		t.blackHole.Wait()
	}

	return t.result
}

func (t *Thunk) lock(s thunkState) bool {
	for {
		switch t.loadState() {
		case normal:
			return false
		case app:
			if t.compareAndSwapState(app, s) {
				return true
			}
		}
	}
}

func (t *Thunk) delegateEval(parent *Thunk) bool {
	if t.lock(spinLock) {
		parent.function, t.function = t.function, identity
		parent.args, t.args = t.args, NewPositionalArguments(parent)
		parent.info = t.info
		t.storeState(app)
		return true
	}

	return false
}

func (t *Thunk) compareAndSwapState(old, new thunkState) bool {
	return atomic.CompareAndSwapInt32((*int32)(&t.state), int32(old), int32(new))
}

func (t *Thunk) loadState() thunkState {
	return thunkState(atomic.LoadInt32((*int32)(&t.state)))
}

func (t *Thunk) storeState(new thunkState) {
	atomic.StoreInt32((*int32)(&t.state), int32(new))
}

// Eval evaluates a pure value.
func (t *Thunk) Eval() Value {
	if _, ok := t.evalAny().(effectType); ok {
		return impureFunctionError().Eval().(ErrorType).Chain(t.info)
	}

	return t.result
}

// EvalEffect evaluates an effect expression.
func (t *Thunk) EvalEffect() Value {
	v := t.evalAny()
	e, ok := v.(effectType)

	if !ok {
		return NotEffectError(v).Eval().(ErrorType).Chain(t.info)
	}

	return e.value.Eval()
}
<|endoftext|>


<commit_before>// Copyright 2010 The GoGo Authors. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package libgogo

type ObjectDesc struct {
    name string;
    packagename string;
    class uint64;
    objtype *TypeDesc;
    ptrtype byte; //If 0, type objtype, otherwise type *objtype
    next *ObjectDesc;
};

type TypeDesc struct {
    name string;
    packagename string;
    forwarddecl byte; //If 1, type is forward declared
    form uint64;
    len uint64;
    fields *ObjectDesc;
    base *TypeDesc;
    next *TypeDesc;
};

//
// Pseudo constants that specify the descriptor sizes
//
var OBJECT_SIZE uint64 = 64; //8*8 bytes space for an object
var TYPE_SIZE uint64 = 80;  //10*8 bytes space for a type

//
// Classes for objects
//
var CLASS_VAR uint64 = 1;
var CLASS_FIELD uint64 = 2;

//
// Forms for types
//
var FORM_SIMPLE uint64 = 1;
var FORM_STRUCT uint64 = 2;
var FORM_ARRAY uint64 = 3;

//
// Convert the uint64 value (returned from malloc) to a real object address
//
func Uint64ToObjectDescPtr(adr uint64) *ObjectDesc;

//
// Convert the uint64 value (returned from malloc) to a real type object address
//
func Uint64ToTypeDescPtr(adr uint64) *TypeDesc;

//
// Appends an object to the list
//
func AppendObject(object *ObjectDesc, list *ObjectDesc) *ObjectDesc {
    var tmpObj *ObjectDesc = object;
    if list != nil {
        for tmpObj = list; tmpObj.next != nil; tmpObj = tmpObj.next {
        }
        tmpObj.next = object;
        tmpObj = list;
    }
    return tmpObj;

		//Alternate version: PrependObject:
		/*object.next = list;
    return object;*/
}

//
// Appends a type to the list
//
func AppendType(objtype *TypeDesc, list *TypeDesc) *TypeDesc {
    var tmpObjType *TypeDesc = objtype;
    if list != nil {
        for tmpObjType = list; tmpObjType.next != nil; tmpObjType = tmpObjType.next { }
        tmpObjType.next = objtype;
        tmpObjType = list;
    }
    return tmpObjType;

    //Alternate version: PrependType:
    /*objtype.next = list;
    return objtype;*/
}

//
// Add a field in form of an object descriptor to the type descriptor given
//
func AddFields(object *ObjectDesc, objtype *TypeDesc) {
    objtype.form = FORM_STRUCT;
    objtype.fields = AppendObject(object, objtype.fields);
}

func HasField(name string, objtype *TypeDesc) uint64 {
    var tmpObj *ObjectDesc;
    var retVal uint64 = 0;
    tmpObj = GetObject(name, "", objtype.fields);
    if tmpObj != nil {
        retVal = 1;
    }
    return retVal;
}

//
// Sets an object's type
//
func SetObjType(object *ObjectDesc, objtype *TypeDesc) {
    object.objtype = objtype;
}

//
// Marks the type of the given object as pointer
//
func FlagObjectTypeAsPointer(object *ObjectDesc) {
    object.ptrtype = 1;
}

//
// Returns whether an object's type is a pointer
//
func IsPointerType(object *ObjectDesc) byte {
    return object.ptrtype;
}

//
// Returns a type's name
//
func GetTypeName(objtype *TypeDesc) string {
    return objtype.name;
}

//
// Returns an object's name
//
func GetObjectName(obj *ObjectDesc) string {
    return obj.name;
}

func IsForwardDecl(objtype *TypeDesc) byte {
    return objtype.forwarddecl;
}

func UnsetForwardDecl(objtype *TypeDesc) {
    objtype.forwarddecl = 0;
}

func GetTypeSize(objtype *TypeDesc) uint64 {
    var size uint64 = 0;
    var tempobj *ObjectDesc;
    if objtype != nil {
        if objtype.forwarddecl == 0 {
            if objtype.form == FORM_SIMPLE {
                size = objtype.len;
            }
            if objtype.form == FORM_STRUCT {
                for tempobj = objtype.fields; tempobj != nil; tempobj = tempobj.next { //Sum of all fields
                    size = size + GetObjectSize(tempobj); //Add size of each field
                }
            }
            if objtype.form == FORM_ARRAY {
                size = objtype.len * GetTypeSize(objtype.base); //Array length * size of one item
            }
        } else {
            ; //TODO: if type is only forward declared => error!
        }
    } else {
        ; //TODO: if objtype is nil => error!
    }
    return size;
}

func GetObjectSize(obj *ObjectDesc) uint64 {
    var size uint64;
    if obj.ptrtype == 1 { //Pointer only, not the whole type
       size = 8;
    } else { //Actual type
       size = GetTypeSize(obj.objtype);
    }
    return size;
}

func GetObjectOffset(obj *ObjectDesc, list *ObjectDesc) uint64 {
    var offset uint64 = 0;
    var tmp *ObjectDesc;
    for tmp = list; tmp != nil; tmp = tmp.next {
        if tmp == obj {
            break;
        }
        offset = offset + GetObjectSize(tmp);
    }
    if tmp == nil {
        offset = 0; //TODO: Raise error as obj is appearently not within the list
    }
    return offset;
}

//
// Fetches an object with a specific identifier or nil if it is not in the specified list
//
func GetObject(name string, packagename string, list *ObjectDesc) *ObjectDesc {
    var tmpObject *ObjectDesc;
    var retValue *ObjectDesc = nil;
    for tmpObject = list; tmpObject != nil; tmpObject = tmpObject.next {
        if (StringCompare(tmpObject.name,name) == 0) && ((StringLength(tmpObject.packagename) == 0) || (StringCompare(tmpObject.packagename,packagename) == 0)) { //Empty package name indicates internal types
            retValue = tmpObject;
            break;
        }
    }
    return retValue;
}

//
// Fetches a type with a given name or nil if it is not in the specified list
// If includeforward is 0, no forward declared types will be returned; otherwise, forward declared types will also be returned
//
func GetType(name string, packagename string, list *TypeDesc, includeforward byte) *TypeDesc {
    var tmpType *TypeDesc;
    var retValue *TypeDesc = nil;
    for tmpType = list; tmpType != nil; tmpType = tmpType.next {
        if (StringCompare(tmpType.name,name) == 0) && ((StringLength(tmpType.packagename) == 0) || (StringCompare(tmpType.packagename,packagename) == 0)) { //Empty package name indicates internal types
            if (includeforward == 1) || ((includeforward == 0) && (tmpType.forwarddecl == 0)) {
                retValue = tmpType;
                break;
            }
        }
    }
    return retValue;
}

func GetFirstForwardDeclType(list *TypeDesc) *TypeDesc {
    var retValue *TypeDesc;
    for retValue = list; retValue != nil; retValue = retValue.next {
        if retValue.forwarddecl == 1 {
            break;
        }
    }
    return retValue;
}

//
// Creates a new object
//
func NewObject(name string, packagename string, class uint64) *ObjectDesc {
    var adr uint64 = Alloc(OBJECT_SIZE);
    var obj *ObjectDesc = Uint64ToObjectDescPtr(adr);
    obj.name = name; //TODO: Copy string?
    obj.packagename = packagename; //Copy string?
    obj.class = class;
    obj.objtype = nil;
    obj.ptrtype = 0;
    obj.next = nil;
    return obj;
}

//
// Creates a new type
//
func NewType(name string, packagename string, forwarddecl byte, len uint64, basetype *TypeDesc) *TypeDesc {
    var adr uint64 = Alloc(TYPE_SIZE);
    var objtype *TypeDesc = Uint64ToTypeDescPtr(adr);
    objtype.name = name; //TODO: Copy string?
    objtype.forwarddecl = forwarddecl;
    objtype.packagename = packagename; //TODO: Copy string?
    if basetype != nil {
        objtype.form = FORM_ARRAY;
    } else {
        objtype.form = FORM_SIMPLE;
    }
    objtype.len = len;
    objtype.next = nil;
    objtype.fields = nil;
    objtype.base = basetype;
    return objtype;
}

func PrintObjects(list *ObjectDesc) {
    var o *ObjectDesc;
    for o = list; o != nil; o = o.next {
        PrintString("Object ");
        if StringLength(o.packagename) != 0 {
            PrintString(o.packagename);
            PrintChar('.');
        }
        PrintString(o.name);
        PrintString(" (type: ");
        if o.ptrtype != 0 {
            PrintString("pointer to ");
        }
        if o.objtype != nil {
            if o.objtype.base != nil {
                PrintString("array of ");
                PrintString(o.objtype.base.name);
                PrintString(" of length ");
                PrintNumber(o.objtype.len);
                PrintString(", internally named ");
            }
            if StringLength(o.objtype.packagename) != 0 {
                PrintString(o.objtype.packagename);
                PrintChar('.');
            }
            PrintString(o.objtype.name);
        } else {
            PrintString("<unknown>");
        }
        PrintString(", size: ");
        PrintNumber(GetObjectSize(o));
        PrintString(", offset: ");
        PrintNumber(GetObjectOffset(o, list));
        PrintString(")\n");
    }
}

func PrintTypes(list *TypeDesc) {
    var t *TypeDesc;
    var o *ObjectDesc;
    for t = list; t != nil; t = t.next {
        PrintString("Type ");
        if StringLength(t.packagename) != 0 {
            PrintString(t.packagename);
            PrintChar('.');
        }
        PrintString(t.name);
        PrintString(" (size: ");
        PrintNumber(GetTypeSize(t));
        PrintString(")\n");
				for o = t.fields; o != nil; o = o.next {
            PrintString("  ");
            PrintString(o.name);
            PrintString(" (type: ");
            if o.ptrtype != 0 {
                PrintString("pointer to ");
            }
            if o.objtype != nil {
                if o.objtype.base != nil {
                    PrintString("array of ");
                    PrintString(o.objtype.base.name);
                    PrintString(" of length ");
                    PrintNumber(o.objtype.len);
                    PrintString(", internally named ");
                }
                if StringLength(o.objtype.packagename) != 0 {
                    PrintString(o.objtype.packagename);
                    PrintChar('.');
                }
                PrintString(o.objtype.name);
            } else {
                PrintString("<unknown>");
            }
            PrintString(", size: ");
            PrintNumber(GetObjectSize(o));
            PrintString(", offset: ");
            PrintNumber(GetObjectOffset(o, t.fields));
            PrintString(")\n");
        }
    }
}
<commit_msg>libgogo.go: Documentation<commit_after>// Copyright 2010 The GoGo Authors. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package libgogo

type ObjectDesc struct {
    name string;
    packagename string;
    class uint64;
    objtype *TypeDesc;
    ptrtype byte; //If 0, type objtype, otherwise type *objtype
    next *ObjectDesc;
};

type TypeDesc struct {
    name string;
    packagename string;
    forwarddecl byte; //If 1, type is forward declared
    form uint64;
    len uint64;
    fields *ObjectDesc;
    base *TypeDesc;
    next *TypeDesc;
};

//
// Pseudo constants that specify the descriptor sizes
//
var OBJECT_SIZE uint64 = 64; //8*8 bytes space for an object
var TYPE_SIZE uint64 = 80;  //10*8 bytes space for a type

//
// Classes for objects
//
var CLASS_VAR uint64 = 1;
var CLASS_FIELD uint64 = 2;

//
// Forms for types
//
var FORM_SIMPLE uint64 = 1;
var FORM_STRUCT uint64 = 2;
var FORM_ARRAY uint64 = 3;

//
// Convert the uint64 value (returned from malloc) to a real object address
//
func Uint64ToObjectDescPtr(adr uint64) *ObjectDesc;

//
// Convert the uint64 value (returned from malloc) to a real type object address
//
func Uint64ToTypeDescPtr(adr uint64) *TypeDesc;

//
// Appends an object to the list
//
func AppendObject(object *ObjectDesc, list *ObjectDesc) *ObjectDesc {
    var tmpObj *ObjectDesc = object;
    if list != nil {
        for tmpObj = list; tmpObj.next != nil; tmpObj = tmpObj.next {
        }
        tmpObj.next = object;
        tmpObj = list;
    }
    return tmpObj;

		//Alternate version: PrependObject:
		/*object.next = list;
    return object;*/
}

//
// Appends a type to the list
//
func AppendType(objtype *TypeDesc, list *TypeDesc) *TypeDesc {
    var tmpObjType *TypeDesc = objtype;
    if list != nil {
        for tmpObjType = list; tmpObjType.next != nil; tmpObjType = tmpObjType.next { }
        tmpObjType.next = objtype;
        tmpObjType = list;
    }
    return tmpObjType;

    //Alternate version: PrependType:
    /*objtype.next = list;
    return objtype;*/
}

//
// Add a field in form of an object descriptor to the type descriptor given
//
func AddFields(object *ObjectDesc, objtype *TypeDesc) {
    objtype.form = FORM_STRUCT;
    objtype.fields = AppendObject(object, objtype.fields);
}

//
// Returns 0 if the given (struct) type has a field with the given name, or 1 otherwise
//
func HasField(name string, objtype *TypeDesc) uint64 {
    var tmpObj *ObjectDesc;
    var retVal uint64 = 0;
    tmpObj = GetObject(name, "", objtype.fields);
    if tmpObj != nil {
        retVal = 1;
    }
    return retVal;
}

//
// Sets an object's type
//
func SetObjType(object *ObjectDesc, objtype *TypeDesc) {
    object.objtype = objtype;
}

//
// Marks the type of the given object as pointer
//
func FlagObjectTypeAsPointer(object *ObjectDesc) {
    object.ptrtype = 1;
}

//
// Returns whether an object's type is a pointer
//
func IsPointerType(object *ObjectDesc) byte {
    return object.ptrtype;
}

//
// Returns a type's name
//
func GetTypeName(objtype *TypeDesc) string {
    return objtype.name;
}

//
// Returns an object's name
//
func GetObjectName(obj *ObjectDesc) string {
    return obj.name;
}

//
// Returns 0 if the given type if forward declared, or 1 if it is not
//
func IsForwardDecl(objtype *TypeDesc) byte {
    return objtype.forwarddecl;
}

//
// Unsets the forward declaration flag of the given type, making it a "normal" type
//
func UnsetForwardDecl(objtype *TypeDesc) {
    objtype.forwarddecl = 0;
}

//
// Returns the (memory) size of the given type in bytes which is required when hypothetically allocating one variable of this type
// Same semantics as sizeof(objtype)
//
func GetTypeSize(objtype *TypeDesc) uint64 {
    var size uint64 = 0;
    var tempobj *ObjectDesc;
    if objtype != nil {
        if objtype.forwarddecl == 0 {
            if objtype.form == FORM_SIMPLE {
                size = objtype.len;
            }
            if objtype.form == FORM_STRUCT {
                for tempobj = objtype.fields; tempobj != nil; tempobj = tempobj.next { //Sum of all fields
                    size = size + GetObjectSize(tempobj); //Add size of each field
                }
            }
            if objtype.form == FORM_ARRAY {
                size = objtype.len * GetTypeSize(objtype.base); //Array length * size of one item
            }
        } else {
            ; //TODO: if type is only forward declared => error!
        }
    } else {
        ; //TODO: if objtype is nil => error!
    }
    return size;
}

//
// Returns the (memory) size required by the object in bytes, considering whether or not the object is a pointer
// Same semantics as sizeof(obj)
//
func GetObjectSize(obj *ObjectDesc) uint64 {
    var size uint64;
    if obj.ptrtype == 1 { //Pointer only, not the whole type
       size = 8;
    } else { //Actual type
       size = GetTypeSize(obj.objtype);
    }
    return size;
}

//
// Calculates the (memory) offset of a given field of the specified (struct) type in bytes
//
func GetObjectOffset(obj *ObjectDesc, list *ObjectDesc) uint64 {
    var offset uint64 = 0;
    var tmp *ObjectDesc;
    for tmp = list; tmp != nil; tmp = tmp.next {
        if tmp == obj {
            break;
        }
        offset = offset + GetObjectSize(tmp);
    }
    if tmp == nil {
        offset = 0; //TODO: Raise error as obj is appearently not within the list
    }
    return offset;
}

//
// Fetches an object with a specific identifier or nil if it is not in the specified list
//
func GetObject(name string, packagename string, list *ObjectDesc) *ObjectDesc {
    var tmpObject *ObjectDesc;
    var retValue *ObjectDesc = nil;
    for tmpObject = list; tmpObject != nil; tmpObject = tmpObject.next {
        if (StringCompare(tmpObject.name,name) == 0) && ((StringLength(tmpObject.packagename) == 0) || (StringCompare(tmpObject.packagename,packagename) == 0)) { //Empty package name indicates internal types
            retValue = tmpObject;
            break;
        }
    }
    return retValue;
}

//
// Fetches a type with a given name or nil if it is not in the specified list
// If includeforward is 0, no forward declared types will be returned; otherwise, forward declared types will also be returned
//
func GetType(name string, packagename string, list *TypeDesc, includeforward byte) *TypeDesc {
    var tmpType *TypeDesc;
    var retValue *TypeDesc = nil;
    for tmpType = list; tmpType != nil; tmpType = tmpType.next {
        if (StringCompare(tmpType.name,name) == 0) && ((StringLength(tmpType.packagename) == 0) || (StringCompare(tmpType.packagename,packagename) == 0)) { //Empty package name indicates internal types
            if (includeforward == 1) || ((includeforward == 0) && (tmpType.forwarddecl == 0)) {
                retValue = tmpType;
                break;
            }
        }
    }
    return retValue;
}

//
// Returns the first type from the list which is forward declared
//
func GetFirstForwardDeclType(list *TypeDesc) *TypeDesc {
    var retValue *TypeDesc;
    for retValue = list; retValue != nil; retValue = retValue.next {
        if retValue.forwarddecl == 1 {
            break;
        }
    }
    return retValue;
}

//
// Creates a new object
//
func NewObject(name string, packagename string, class uint64) *ObjectDesc {
    var adr uint64 = Alloc(OBJECT_SIZE);
    var obj *ObjectDesc = Uint64ToObjectDescPtr(adr);
    obj.name = name; //TODO: Copy string?
    obj.packagename = packagename; //Copy string?
    obj.class = class;
    obj.objtype = nil;
    obj.ptrtype = 0;
    obj.next = nil;
    return obj;
}

//
// Creates a new type
//
func NewType(name string, packagename string, forwarddecl byte, len uint64, basetype *TypeDesc) *TypeDesc {
    var adr uint64 = Alloc(TYPE_SIZE);
    var objtype *TypeDesc = Uint64ToTypeDescPtr(adr);
    objtype.name = name; //TODO: Copy string?
    objtype.forwarddecl = forwarddecl;
    objtype.packagename = packagename; //TODO: Copy string?
    if basetype != nil {
        objtype.form = FORM_ARRAY;
    } else {
        objtype.form = FORM_SIMPLE;
    }
    objtype.len = len;
    objtype.next = nil;
    objtype.fields = nil;
    objtype.base = basetype;
    return objtype;
}

//
// Prints a formatted output of a given list of objects, including type, size etc.
//
func PrintObjects(list *ObjectDesc) {
    var o *ObjectDesc;
    for o = list; o != nil; o = o.next {
        PrintString("Object ");
        if StringLength(o.packagename) != 0 {
            PrintString(o.packagename);
            PrintChar('.');
        }
        PrintString(o.name);
        PrintString(" (type: ");
        if o.ptrtype != 0 {
            PrintString("pointer to ");
        }
        if o.objtype != nil {
            if o.objtype.base != nil {
                PrintString("array of ");
                PrintString(o.objtype.base.name);
                PrintString(" of length ");
                PrintNumber(o.objtype.len);
                PrintString(", internally named ");
            }
            if StringLength(o.objtype.packagename) != 0 {
                PrintString(o.objtype.packagename);
                PrintChar('.');
            }
            PrintString(o.objtype.name);
        } else {
            PrintString("<unknown>");
        }
        PrintString(", size: ");
        PrintNumber(GetObjectSize(o));
        PrintString(", offset: ");
        PrintNumber(GetObjectOffset(o, list));
        PrintString(")\n");
    }
}

//
// Prints a formatted output of a given list of types, including form, size etc.
//
func PrintTypes(list *TypeDesc) {
    var t *TypeDesc;
    var o *ObjectDesc;
    for t = list; t != nil; t = t.next {
        PrintString("Type ");
        if StringLength(t.packagename) != 0 {
            PrintString(t.packagename);
            PrintChar('.');
        }
        PrintString(t.name);
        PrintString(" (size: ");
        PrintNumber(GetTypeSize(t));
        PrintString(")\n");
				for o = t.fields; o != nil; o = o.next {
            PrintString("  ");
            PrintString(o.name);
            PrintString(" (type: ");
            if o.ptrtype != 0 {
                PrintString("pointer to ");
            }
            if o.objtype != nil {
                if o.objtype.base != nil {
                    PrintString("array of ");
                    PrintString(o.objtype.base.name);
                    PrintString(" of length ");
                    PrintNumber(o.objtype.len);
                    PrintString(", internally named ");
                }
                if StringLength(o.objtype.packagename) != 0 {
                    PrintString(o.objtype.packagename);
                    PrintChar('.');
                }
                PrintString(o.objtype.name);
            } else {
                PrintString("<unknown>");
            }
            PrintString(", size: ");
            PrintNumber(GetObjectSize(o));
            PrintString(", offset: ");
            PrintNumber(GetObjectOffset(o, t.fields));
            PrintString(")\n");
        }
    }
}
<|endoftext|>