<commit_before>package typhon

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/monzo/terrors"
	"google.golang.org/protobuf/encoding/protojson"
	"google.golang.org/protobuf/proto"
)

// A Request is Typhon's wrapper around http.Request, used by both clients and servers.
//
// Note that Typhon makes no guarantees that a Request is safe to access or mutate concurrently. If a single Request
// object is to be used by multiple goroutines concurrently, callers must make sure to properly synchronise accesses.
type Request struct {
	http.Request
	context.Context
	err      error // Any error from request construction; read by ErrorFilter
	hijacker http.Hijacker
	server   *Server
}

// unwrappedContext returns the most "unwrapped" Context possible for that in the request.
// This is useful as it's very often the case that Typhon users will use a parent request
// as a parent for a child request. The context library knows how to unwrap its own
// types to most efficiently perform certain operations (eg. cancellation chaining), but
// it can't do that with Typhon-wrapped contexts.
func (r *Request) unwrappedContext() context.Context {
	switch c := r.Context.(type) {
	case Request:
		return c.unwrappedContext()
	case *Request:
		return c.unwrappedContext()
	default:
		return c
	}
}

// Encode maps to to EncodeAsJSON
// TODO: Remove in the next major release and require encoding to explicitly go through either EncodeAsJSON, EncodeAsProtoJSON or EncodeAsProtobuf
func (r *Request) Encode(v interface{}) {
	r.EncodeAsJSON(v)
}

// EncodeAsJSON serialises the passed object as JSON into the body (and sets appropriate headers).
func (r *Request) EncodeAsJSON(v interface{}) {
	// If we were given an io.ReadCloser or an io.Reader (that is not also a json.Marshaler), use it directly
	switch v := v.(type) {
	case json.Marshaler:
	case io.ReadCloser:
		r.Body = v
		r.ContentLength = -1
		return
	case io.Reader:
		r.Body = ioutil.NopCloser(v)
		r.ContentLength = -1
		return
	}

	if err := json.NewEncoder(r).Encode(v); err != nil {
		r.err = terrors.Wrap(err, nil)
		return
	}
	r.Header.Set("Content-Type", "application/json")
}

// EncodeAsProtoJSON serialises the passed object as ProtoJSON into the body (and sets appropriate headers).
func (r *Request) EncodeAsProtoJSON(m proto.Message) {
	out, err := protojson.Marshal(m)
	if err != nil {
		r.err = terrors.Wrap(err, nil)
		return
	}

	n, err := r.Write(out)
	if err != nil {
		r.err = terrors.Wrap(err, nil)
		return
	}
	r.Header.Set("Content-Type", "application/jsonpb")
	r.ContentLength = int64(n)
}

// EncodeAsProtobuf serialises the passed object as protobuf into the body (and sets appropriate headers).
func (r *Request) EncodeAsProtobuf(m proto.Message) {
	out, err := proto.Marshal(m)
	if err != nil {
		r.err = terrors.Wrap(err, nil)
		return
	}

	n, err := r.Write(out)
	if err != nil {
		r.err = terrors.Wrap(err, nil)
		return
	}
	r.Header.Set("Content-Type", "application/protobuf")
	r.ContentLength = int64(n)
}

// Decode de-serialises the body into the passed object.
func (r Request) Decode(v interface{}) error {
	b, err := r.BodyBytes(true)
	if err != nil {
		return terrors.WrapWithCode(err, nil, terrors.ErrBadRequest)
	}

	switch r.Header.Get("Content-Type") {
	// application/x-protobuf is the "canonical" use, application/protobuf is defined in an expired IETF draft.
	// See: https://datatracker.ietf.org/doc/html/draft-rfernando-protocol-buffers-00#section-3.2
	// See: https://github.com/google/protorpc/blob/eb03145/python/protorpc/protobuf.py#L49-L51
	case "application/octet-stream", "application/x-google-protobuf", "application/protobuf", "application/x-protobuf":
		m, ok := v.(proto.Message)
		if !ok {
			return terrors.InternalService("invalid_type", "could not decode proto message", nil)
		}
		err = proto.Unmarshal(b, m)
	// Proper JSON handling requires the protojson package in Go. application/jsonpb is a suggestion by grpc-gateway:
	// https://github.com/grpc-ecosystem/grpc-gateway/blob/f4371f7/runtime/marshaler_registry.go#L89-L90
	// This is a backward compatibility break for those using google.golang.org/protobuf/proto.Message incorrectly.

	// Older versions of typhon marshal/unmarshal using json, to prevent a regression, we only use protojson if the
	// content-type explicitly declares that this message is protojson
	case "application/jsonpb", "application/protojson":
		m, ok := v.(proto.Message)
		if !ok {
			return terrors.InternalService("invalid_type", "could not decode proto message", nil)
		}
		err = protojson.Unmarshal(b, m)

	default:
		m, ok := v.(proto.Message)
		if !ok {
			return terrors.InternalService("invalid_type", "could not decode proto message", nil)
		}
		err = json.Unmarshal(b, m)
	}

	return terrors.WrapWithCode(err, nil, terrors.ErrBadRequest)
}

// Write writes the passed bytes to the request's body.
func (r *Request) Write(b []byte) (n int, err error) {
	switch rc := r.Body.(type) {
	// In the "normal" case, the response body will be a buffer, to which we can write
	case io.Writer:
		n, err = rc.Write(b)
		if err != nil {
			return n, err
		}
	// If a caller manually sets Response.Body, then we may not be able to write to it. In that case, we need to be
	// cleverer.
	default:
		buf := &bufCloser{}
		if rc != nil {
			if _, err := io.Copy(buf, rc); err != nil {
				// This can be quite bad; we have consumed (and possibly lost) some of the original body
				return 0, err
			}
			// rc will never again be accessible: once it's copied it must be closed
			rc.Close()
		}
		r.Body = buf
		n, err = buf.Write(b)
		if err != nil {
			return n, err
		}
	}

	if r.ContentLength >= 0 {
		r.ContentLength += int64(n)
		// If this write pushed the content length above the chunking threshold,
		// set to -1 (unknown) to trigger chunked encoding
		if r.ContentLength >= chunkThreshold {
			r.ContentLength = -1
		}
	}
	return n, nil
}

// BodyBytes fully reads the request body and returns the bytes read.
//
// If consume is true, this is equivalent to ioutil.ReadAll; if false, the caller will observe the body to be in
// the same state that it was before (ie. any remaining unread body can be read again).
func (r *Request) BodyBytes(consume bool) ([]byte, error) {
	if consume {
		defer r.Body.Close()
		return ioutil.ReadAll(r.Body)
	}

	switch rc := r.Body.(type) {
	case *bufCloser:
		return rc.Bytes(), nil
	default:
		buf := &bufCloser{}
		r.Body = buf
		rdr := io.TeeReader(rc, buf)
		// rc will never again be accessible: once it's copied it must be closed
		defer rc.Close()
		return ioutil.ReadAll(rdr)
	}
}

// Send round-trips the request via the default Client. It does not block, instead returning a ResponseFuture
// representing the asynchronous operation to produce the response. It is equivalent to:
//
//  r.SendVia(Client)
func (r Request) Send() *ResponseFuture {
	return Send(r)
}

// SendVia round-trips the request via the passed Service. It does not block, instead returning a ResponseFuture
// representing the asynchronous operation to produce the response.
func (r Request) SendVia(svc Service) *ResponseFuture {
	return SendVia(r, svc)
}

// Response constructs a new Response to the request, and if non-nil, encodes the given body into it.
func (r Request) Response(body interface{}) Response {
	rsp := NewResponse(r)
	if body != nil {
		rsp.Encode(body)
	}
	return rsp
}

// ResponseWithCode constructs a new Response with the given status code to the request, and if non-nil, encodes the
// given body into it.
func (r Request) ResponseWithCode(body interface{}, statusCode int) Response {
	rsp := NewResponseWithCode(r, statusCode)
	if body != nil {
		rsp.Encode(body)
	}
	return rsp
}

func (r Request) String() string {
	if r.URL == nil {
		return "Request(Unknown)"
	}
	return fmt.Sprintf("Request(%s %s://%s%s)", r.Method, r.URL.Scheme, r.Host, r.URL.Path)
}

// NewRequest constructs a new Request with the given parameters, and if non-nil, encodes the given body into it.
func NewRequest(ctx context.Context, method, url string, body interface{}) Request {
	if ctx == nil {
		ctx = context.Background()
	}
	httpReq, err := http.NewRequest(method, url, nil)
	req := Request{
		Context: ctx,
		err:     err}
	if httpReq != nil {
		httpReq.ContentLength = 0
		httpReq.Body = &bufCloser{}
		req.Request = *httpReq

		// Attach any metadata in the context to the request as headers.
		meta := MetadataFromContext(ctx)
		for k, v := range meta {
			req.Header[strings.ToLower(k)] = v
		}
	}
	if body != nil && err == nil {
		req.EncodeAsJSON(body)
	}
	return req
}
<commit_msg>Remove ProtoJSON support<commit_after>package typhon

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/monzo/terrors"
	"google.golang.org/protobuf/proto"
)

// A Request is Typhon's wrapper around http.Request, used by both clients and servers.
//
// Note that Typhon makes no guarantees that a Request is safe to access or mutate concurrently. If a single Request
// object is to be used by multiple goroutines concurrently, callers must make sure to properly synchronise accesses.
type Request struct {
	http.Request
	context.Context
	err      error // Any error from request construction; read by ErrorFilter
	hijacker http.Hijacker
	server   *Server
}

// unwrappedContext returns the most "unwrapped" Context possible for that in the request.
// This is useful as it's very often the case that Typhon users will use a parent request
// as a parent for a child request. The context library knows how to unwrap its own
// types to most efficiently perform certain operations (eg. cancellation chaining), but
// it can't do that with Typhon-wrapped contexts.
func (r *Request) unwrappedContext() context.Context {
	switch c := r.Context.(type) {
	case Request:
		return c.unwrappedContext()
	case *Request:
		return c.unwrappedContext()
	default:
		return c
	}
}

// Encode maps to to EncodeAsJSON
// TODO: Remove in the next major release and require encoding to explicitly go through either EncodeAsJSON, EncodeAsProtoJSON or EncodeAsProtobuf
func (r *Request) Encode(v interface{}) {
	r.EncodeAsJSON(v)
}

// EncodeAsJSON serialises the passed object as JSON into the body (and sets appropriate headers).
func (r *Request) EncodeAsJSON(v interface{}) {
	// If we were given an io.ReadCloser or an io.Reader (that is not also a json.Marshaler), use it directly
	switch v := v.(type) {
	case json.Marshaler:
	case io.ReadCloser:
		r.Body = v
		r.ContentLength = -1
		return
	case io.Reader:
		r.Body = ioutil.NopCloser(v)
		r.ContentLength = -1
		return
	}

	if err := json.NewEncoder(r).Encode(v); err != nil {
		r.err = terrors.Wrap(err, nil)
		return
	}
	r.Header.Set("Content-Type", "application/json")
}

// EncodeAsProtobuf serialises the passed object as protobuf into the body (and sets appropriate headers).
func (r *Request) EncodeAsProtobuf(m proto.Message) {
	out, err := proto.Marshal(m)
	if err != nil {
		r.err = terrors.Wrap(err, nil)
		return
	}

	n, err := r.Write(out)
	if err != nil {
		r.err = terrors.Wrap(err, nil)
		return
	}
	r.Header.Set("Content-Type", "application/protobuf")
	r.ContentLength = int64(n)
}

// Decode de-serialises the body into the passed object.
func (r Request) Decode(v interface{}) error {
	b, err := r.BodyBytes(true)
	if err != nil {
		return terrors.WrapWithCode(err, nil, terrors.ErrBadRequest)
	}

	switch r.Header.Get("Content-Type") {
	// application/x-protobuf is the "canonical" use, application/protobuf is defined in an expired IETF draft.
	// See: https://datatracker.ietf.org/doc/html/draft-rfernando-protocol-buffers-00#section-3.2
	// See: https://github.com/google/protorpc/blob/eb03145/python/protorpc/protobuf.py#L49-L51
	case "application/octet-stream", "application/x-google-protobuf", "application/protobuf", "application/x-protobuf":
		m, ok := v.(proto.Message)
		if !ok {
			return terrors.InternalService("invalid_type", "could not decode proto message", nil)
		}
		err = proto.Unmarshal(b, m)
	// As older versions of typhon used json, we don't use protojson here as they are mutually exclusive standards with
	// major differences in how they handle some types (such as Enums)
	default:
		m, ok := v.(proto.Message)
		if !ok {
			return terrors.InternalService("invalid_type", "could not decode proto message", nil)
		}
		err = json.Unmarshal(b, m)
	}

	return terrors.WrapWithCode(err, nil, terrors.ErrBadRequest)
}

// Write writes the passed bytes to the request's body.
func (r *Request) Write(b []byte) (n int, err error) {
	switch rc := r.Body.(type) {
	// In the "normal" case, the response body will be a buffer, to which we can write
	case io.Writer:
		n, err = rc.Write(b)
		if err != nil {
			return n, err
		}
	// If a caller manually sets Response.Body, then we may not be able to write to it. In that case, we need to be
	// cleverer.
	default:
		buf := &bufCloser{}
		if rc != nil {
			if _, err := io.Copy(buf, rc); err != nil {
				// This can be quite bad; we have consumed (and possibly lost) some of the original body
				return 0, err
			}
			// rc will never again be accessible: once it's copied it must be closed
			rc.Close()
		}
		r.Body = buf
		n, err = buf.Write(b)
		if err != nil {
			return n, err
		}
	}

	if r.ContentLength >= 0 {
		r.ContentLength += int64(n)
		// If this write pushed the content length above the chunking threshold,
		// set to -1 (unknown) to trigger chunked encoding
		if r.ContentLength >= chunkThreshold {
			r.ContentLength = -1
		}
	}
	return n, nil
}

// BodyBytes fully reads the request body and returns the bytes read.
//
// If consume is true, this is equivalent to ioutil.ReadAll; if false, the caller will observe the body to be in
// the same state that it was before (ie. any remaining unread body can be read again).
func (r *Request) BodyBytes(consume bool) ([]byte, error) {
	if consume {
		defer r.Body.Close()
		return ioutil.ReadAll(r.Body)
	}

	switch rc := r.Body.(type) {
	case *bufCloser:
		return rc.Bytes(), nil
	default:
		buf := &bufCloser{}
		r.Body = buf
		rdr := io.TeeReader(rc, buf)
		// rc will never again be accessible: once it's copied it must be closed
		defer rc.Close()
		return ioutil.ReadAll(rdr)
	}
}

// Send round-trips the request via the default Client. It does not block, instead returning a ResponseFuture
// representing the asynchronous operation to produce the response. It is equivalent to:
//
//  r.SendVia(Client)
func (r Request) Send() *ResponseFuture {
	return Send(r)
}

// SendVia round-trips the request via the passed Service. It does not block, instead returning a ResponseFuture
// representing the asynchronous operation to produce the response.
func (r Request) SendVia(svc Service) *ResponseFuture {
	return SendVia(r, svc)
}

// Response constructs a new Response to the request, and if non-nil, encodes the given body into it.
func (r Request) Response(body interface{}) Response {
	rsp := NewResponse(r)
	if body != nil {
		rsp.Encode(body)
	}
	return rsp
}

// ResponseWithCode constructs a new Response with the given status code to the request, and if non-nil, encodes the
// given body into it.
func (r Request) ResponseWithCode(body interface{}, statusCode int) Response {
	rsp := NewResponseWithCode(r, statusCode)
	if body != nil {
		rsp.Encode(body)
	}
	return rsp
}

func (r Request) String() string {
	if r.URL == nil {
		return "Request(Unknown)"
	}
	return fmt.Sprintf("Request(%s %s://%s%s)", r.Method, r.URL.Scheme, r.Host, r.URL.Path)
}

// NewRequest constructs a new Request with the given parameters, and if non-nil, encodes the given body into it.
func NewRequest(ctx context.Context, method, url string, body interface{}) Request {
	if ctx == nil {
		ctx = context.Background()
	}
	httpReq, err := http.NewRequest(method, url, nil)
	req := Request{
		Context: ctx,
		err:     err}
	if httpReq != nil {
		httpReq.ContentLength = 0
		httpReq.Body = &bufCloser{}
		req.Request = *httpReq

		// Attach any metadata in the context to the request as headers.
		meta := MetadataFromContext(ctx)
		for k, v := range meta {
			req.Header[strings.ToLower(k)] = v
		}
	}
	if body != nil && err == nil {
		req.EncodeAsJSON(body)
	}
	return req
}
<|endoftext|>


<commit_before>package goinsta

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/spf13/cast"
)

type reqOptions struct {
	// Endpoint is the request path of instagram api
	Endpoint string

	// IsPost setted to true will send request with POST method.
	//
	// By default this option is false.
	IsPost bool

	// Query is the parameters of the request
	//
	// This parameters are independents of the request method (POST|GET)
	Query map[string]string
}

func (insta *Instagram) sendSimpleRequest(uri string, a ...interface{}) (body []byte, err error) {
	return insta.sendRequest(
		&reqOptions{
			Endpoint: fmt.Sprintf(uri, a...),
		},
	)
}

func (inst *Instagram) sendRequest(o *reqOptions) (body []byte, err error) {
	method := "GET"
	if o.IsPost {
		method = "POST"
	}

	u, err := url.Parse(goInstaAPIUrl + o.Endpoint)
	if err != nil {
		return nil, err
	}

	vs := url.Values{}
	bf := bytes.NewBuffer([]byte{})

	for k, v := range o.Query {
		vs.Add(k, v)
	}

	if o.IsPost {
		bf.WriteString(vs.Encode())
	} else {
		u.RawQuery += vs.Encode()
	}

	var req *http.Request
	req, err = http.NewRequest(method, u.String(), bf)
	if err != nil {
		return
	}

	req.Header.Set("Connection", "close")
	req.Header.Set("Accept", "*/*")
	req.Header.Set("Content-type", "application/x-www-form-urlencoded; charset=UTF-8")
	req.Header.Set("Cookie2", "$Version=1")
	req.Header.Set("Accept-Language", "en-US")
	req.Header.Set("User-Agent", goInstaUserAgent)

	resp, err := inst.c.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	u, _ = url.Parse(goInstaAPIUrl)
	for _, value := range inst.c.Jar.Cookies(u) {
		if strings.Contains(value.Name, "csrftoken") {
			inst.token = value.Value
		}
	}

	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	switch resp.StatusCode {
	case 200:
	default:
		ierr := instaError{}
		err = json.Unmarshal(body, &ierr)
		if err != nil {
			return nil, fmt.Errorf("Invalid status code: %d", resp.StatusCode)
		}
		return nil, instaToErr(ierr)
	}

	return body, err
}

func (insta *Instagram) prepareData(other ...map[string]interface{}) (string, error) {
	data := map[string]interface{}{
		"_uuid":      insta.uuid,
		"_uid":       insta.Account.ID,
		"_csrftoken": insta.token,
	}
	for i := range other {
		for key, value := range other[i] {
			data[key] = value
		}
	}
	b, err := json.Marshal(data)
	if err == nil {
		return b2s(b), err
	}
	return "", err
}

func (insta *Instagram) prepareDataQuery(other ...map[string]interface{}) map[string]string {
	data := map[string]string{
		"_uuid":      insta.uuid,
		"_csrftoken": insta.token,
	}
	for i := range other {
		for key, value := range other[i] {
			data[key] = cast.ToString(value)
		}
	}
	return data
}
<commit_msg>Fixed error with GET request<commit_after>package goinsta

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/spf13/cast"
)

type reqOptions struct {
	// Endpoint is the request path of instagram api
	Endpoint string

	// IsPost setted to true will send request with POST method.
	//
	// By default this option is false.
	IsPost bool

	// Query is the parameters of the request
	//
	// This parameters are independents of the request method (POST|GET)
	Query map[string]string
}

func (insta *Instagram) sendSimpleRequest(uri string, a ...interface{}) (body []byte, err error) {
	return insta.sendRequest(
		&reqOptions{
			Endpoint: fmt.Sprintf(uri, a...),
		},
	)
}

func (inst *Instagram) sendRequest(o *reqOptions) (body []byte, err error) {
	method := "GET"
	if o.IsPost {
		method = "POST"
	}

	u, err := url.Parse(goInstaAPIUrl + o.Endpoint)
	if err != nil {
		return nil, err
	}

	vs := url.Values{}
	bf := bytes.NewBuffer([]byte{})

	for k, v := range o.Query {
		vs.Add(k, v)
	}
	if !o.IsPost {
		for k, v := range u.Query() {
			vs.Add(k, strings.Join(v, " "))
		}
	}

	if o.IsPost {
		bf.WriteString(vs.Encode())
	} else {
		u.RawQuery = vs.Encode()
	}

	var req *http.Request
	req, err = http.NewRequest(method, u.String(), bf)
	if err != nil {
		return
	}

	req.Header.Set("Connection", "close")
	req.Header.Set("Accept", "*/*")
	req.Header.Set("Content-type", "application/x-www-form-urlencoded; charset=UTF-8")
	req.Header.Set("Cookie2", "$Version=1")
	req.Header.Set("Accept-Language", "en-US")
	req.Header.Set("User-Agent", goInstaUserAgent)

	resp, err := inst.c.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	u, _ = url.Parse(goInstaAPIUrl)
	for _, value := range inst.c.Jar.Cookies(u) {
		if strings.Contains(value.Name, "csrftoken") {
			inst.token = value.Value
		}
	}

	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	switch resp.StatusCode {
	case 200:
	default:
		ierr := instaError{}
		err = json.Unmarshal(body, &ierr)
		if err != nil {
			return nil, fmt.Errorf("Invalid status code: %d", resp.StatusCode)
		}
		return nil, instaToErr(ierr)
	}

	return body, err
}

func (insta *Instagram) prepareData(other ...map[string]interface{}) (string, error) {
	data := map[string]interface{}{
		"_uuid":      insta.uuid,
		"_uid":       insta.Account.ID,
		"_csrftoken": insta.token,
	}
	for i := range other {
		for key, value := range other[i] {
			data[key] = value
		}
	}
	b, err := json.Marshal(data)
	if err == nil {
		return b2s(b), err
	}
	return "", err
}

func (insta *Instagram) prepareDataQuery(other ...map[string]interface{}) map[string]string {
	data := map[string]string{
		"_uuid":      insta.uuid,
		"_csrftoken": insta.token,
	}
	for i := range other {
		for key, value := range other[i] {
			data[key] = cast.ToString(value)
		}
	}
	return data
}
<|endoftext|>


<commit_before>package air

import (
	"bytes"
	"io"
	"mime/multipart"

	"github.com/valyala/fasthttp"
)

// Request for HTTP request.
type Request struct {
	fastCtx *fasthttp.RequestCtx
	Header  *RequestHeader
	URI     *URI
	Logger  Logger
}

// NewRequest returns `Request` instance.
func NewRequest(c *fasthttp.RequestCtx, l Logger) *Request {
	return &Request{
		fastCtx: c,
		URI:     &URI{fastURI: c.URI()},
		Header:  &RequestHeader{fastRequestHeader: &c.Request.Header},
		Logger:  l,
	}
}

// IsTLS returns true if HTTP connection is TLS otherwise false.
func (r *Request) IsTLS() bool {
	return r.fastCtx.IsTLS()
}

// Scheme returns the HTTP protocol scheme, `http` or `https`.
func (r *Request) Scheme() string {
	return string(r.fastCtx.Request.URI().Scheme())
}

// Host returns HTTP request host. Per RFC 2616, this is either the value of
// the `Host` header or the host name given in the URI itself.
func (r *Request) Host() string {
	return string(r.fastCtx.Request.Host())
}

// Referer returns the referring URI, if sent in the request.
func (r *Request) Referer() string {
	return string(r.fastCtx.Request.Header.Referer())
}

// ContentLength returns the size of request's body.
func (r *Request) ContentLength() int64 {
	return int64(r.fastCtx.Request.Header.ContentLength())
}

// UserAgent returns the client's `User-Agent`.
func (r *Request) UserAgent() string {
	return string(r.fastCtx.UserAgent())
}

// RemoteAddr returns the client's network address.
func (r *Request) RemoteAddr() string {
	return r.fastCtx.RemoteAddr().String()
}

// Method returns the request's HTTP function.
func (r *Request) Method() string {
	return string(r.fastCtx.Method())
}

// SetMethod sets the HTTP method of the request.
func (r *Request) SetMethod(method string) {
	r.fastCtx.Request.Header.SetMethodBytes([]byte(method))
}

// RequestURI returns the unmodified `Request-URI` sent by the client.
func (r *Request) RequestURI() string {
	return string(r.fastCtx.Request.RequestURI())
}

// SetURI sets the URI of the request.
func (r *Request) SetURI(uri string) {
	r.fastCtx.Request.Header.SetRequestURI(uri)
}

// Body returns request's body.
func (r *Request) Body() io.Reader {
	return bytes.NewBuffer(r.fastCtx.Request.Body())
}

// Body sets request's body.
func (r *Request) SetBody(reader io.Reader) {
	r.fastCtx.Request.SetBodyStream(reader, 0)
}

// FormValue returns the form field value for the provided name.
func (r *Request) FormValue(name string) string {
	return string(r.fastCtx.FormValue(name))
}

// FormParams returns the form parameters.
func (r *Request) FormParams() (params map[string][]string) {
	params = make(map[string][]string)
	mf, err := r.fastCtx.Request.MultipartForm()

	if err == fasthttp.ErrNoMultipartForm {
		r.fastCtx.PostArgs().VisitAll(func(k, v []byte) {
			key := string(k)
			if _, ok := params[key]; ok {
				params[key] = append(params[key], string(v))
			} else {
				params[string(k)] = []string{string(v)}
			}
		})
	} else if err == nil {
		for k, v := range mf.Value {
			if len(v) > 0 {
				params[k] = v
			}
		}
	}

	return
}

// FormFile returns the multipart form file for the provided name.
func (r *Request) FormFile(name string) (*multipart.FileHeader, error) {
	return r.fastCtx.FormFile(name)
}

// MultipartForm returns the multipart form.
func (r *Request) MultipartForm() (*multipart.Form, error) {
	return r.fastCtx.MultipartForm()
}

// Cookie returns the named cookie provided in the request.
func (r *Request) Cookie(name string) (Cookie, error) {
	c := new(fasthttp.Cookie)
	b := r.fastCtx.Request.Header.Cookie(name)
	if b == nil {
		return Cookie{}, ErrCookieNotFound
	}
	c.SetKey(name)
	c.SetValueBytes(b)
	return Cookie{c}, nil
}

// Cookies returns the HTTP cookies sent with the request.
func (r *Request) Cookies() []Cookie {
	cookies := []Cookie{}
	r.fastCtx.Request.Header.VisitAllCookie(func(name, value []byte) {
		c := new(fasthttp.Cookie)
		c.SetKeyBytes(name)
		c.SetValueBytes(value)
		cookies = append(cookies, Cookie{c})
	})
	return cookies
}

func (r *Request) reset(c *fasthttp.RequestCtx, h *RequestHeader, u *URI) {
	r.fastCtx = c
	r.Header = h
	r.URI = u
}
<commit_msg>refactor: rename method Request#RemoteAddr() to Request#RemoteAddress()<commit_after>package air

import (
	"bytes"
	"io"
	"mime/multipart"

	"github.com/valyala/fasthttp"
)

// Request for HTTP request.
type Request struct {
	fastCtx *fasthttp.RequestCtx
	Header  *RequestHeader
	URI     *URI
	Logger  Logger
}

// NewRequest returns `Request` instance.
func NewRequest(c *fasthttp.RequestCtx, l Logger) *Request {
	return &Request{
		fastCtx: c,
		URI:     &URI{fastURI: c.URI()},
		Header:  &RequestHeader{fastRequestHeader: &c.Request.Header},
		Logger:  l,
	}
}

// IsTLS returns true if HTTP connection is TLS otherwise false.
func (r *Request) IsTLS() bool {
	return r.fastCtx.IsTLS()
}

// Scheme returns the HTTP protocol scheme, `http` or `https`.
func (r *Request) Scheme() string {
	return string(r.fastCtx.Request.URI().Scheme())
}

// Host returns HTTP request host. Per RFC 2616, this is either the value of
// the `Host` header or the host name given in the URI itself.
func (r *Request) Host() string {
	return string(r.fastCtx.Request.Host())
}

// Referer returns the referring URI, if sent in the request.
func (r *Request) Referer() string {
	return string(r.fastCtx.Request.Header.Referer())
}

// ContentLength returns the size of request's body.
func (r *Request) ContentLength() int64 {
	return int64(r.fastCtx.Request.Header.ContentLength())
}

// UserAgent returns the client's `User-Agent`.
func (r *Request) UserAgent() string {
	return string(r.fastCtx.UserAgent())
}

// RemoteAddress returns the client's network address.
func (r *Request) RemoteAddress() string {
	return r.fastCtx.RemoteAddr().String()
}

// Method returns the request's HTTP function.
func (r *Request) Method() string {
	return string(r.fastCtx.Method())
}

// SetMethod sets the HTTP method of the request.
func (r *Request) SetMethod(method string) {
	r.fastCtx.Request.Header.SetMethodBytes([]byte(method))
}

// RequestURI returns the unmodified `Request-URI` sent by the client.
func (r *Request) RequestURI() string {
	return string(r.fastCtx.Request.RequestURI())
}

// SetURI sets the URI of the request.
func (r *Request) SetURI(uri string) {
	r.fastCtx.Request.Header.SetRequestURI(uri)
}

// Body returns request's body.
func (r *Request) Body() io.Reader {
	return bytes.NewBuffer(r.fastCtx.Request.Body())
}

// Body sets request's body.
func (r *Request) SetBody(reader io.Reader) {
	r.fastCtx.Request.SetBodyStream(reader, 0)
}

// FormValue returns the form field value for the provided name.
func (r *Request) FormValue(name string) string {
	return string(r.fastCtx.FormValue(name))
}

// FormParams returns the form parameters.
func (r *Request) FormParams() (params map[string][]string) {
	params = make(map[string][]string)
	mf, err := r.fastCtx.Request.MultipartForm()

	if err == fasthttp.ErrNoMultipartForm {
		r.fastCtx.PostArgs().VisitAll(func(k, v []byte) {
			key := string(k)
			if _, ok := params[key]; ok {
				params[key] = append(params[key], string(v))
			} else {
				params[string(k)] = []string{string(v)}
			}
		})
	} else if err == nil {
		for k, v := range mf.Value {
			if len(v) > 0 {
				params[k] = v
			}
		}
	}

	return
}

// FormFile returns the multipart form file for the provided name.
func (r *Request) FormFile(name string) (*multipart.FileHeader, error) {
	return r.fastCtx.FormFile(name)
}

// MultipartForm returns the multipart form.
func (r *Request) MultipartForm() (*multipart.Form, error) {
	return r.fastCtx.MultipartForm()
}

// Cookie returns the named cookie provided in the request.
func (r *Request) Cookie(name string) (Cookie, error) {
	c := new(fasthttp.Cookie)
	b := r.fastCtx.Request.Header.Cookie(name)
	if b == nil {
		return Cookie{}, ErrCookieNotFound
	}
	c.SetKey(name)
	c.SetValueBytes(b)
	return Cookie{c}, nil
}

// Cookies returns the HTTP cookies sent with the request.
func (r *Request) Cookies() []Cookie {
	cookies := []Cookie{}
	r.fastCtx.Request.Header.VisitAllCookie(func(name, value []byte) {
		c := new(fasthttp.Cookie)
		c.SetKeyBytes(name)
		c.SetValueBytes(value)
		cookies = append(cookies, Cookie{c})
	})
	return cookies
}

func (r *Request) reset(c *fasthttp.RequestCtx, h *RequestHeader, u *URI) {
	r.fastCtx = c
	r.Header = h
	r.URI = u
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"sync"
	"time"

	"github.com/codegangsta/cli"
	"github.com/itsabot/abot/core"
	"github.com/itsabot/abot/core/websocket"
	"github.com/itsabot/abot/shared/log"
	"github.com/labstack/echo"
	_ "github.com/lib/pq"
)

var tmplLayout *template.Template
var ws = websocket.NewAtomicWebSocketSet()

func main() {
	rand.Seed(time.Now().UnixNano())
	log.SetDebug(true)
	app := cli.NewApp()
	app.Name = "abot"
	app.Usage = "digital assistant framework"
	app.Version = "0.0.1"
	app.Commands = []cli.Command{
		{
			Name:    "server",
			Aliases: []string{"s"},
			Usage:   "run server",
			Action: func(c *cli.Context) {
				var err error
				if err = startServer(); err != nil {
					l := log.New("")
					l.SetFlags(0)
					l.Fatalf("could not start server\n%s", err)
				}
			},
		},
		{
			Name:    "plugin",
			Aliases: []string{"p"},
			Usage:   "manage and install plugins from plugins.json",
			Subcommands: []cli.Command{
				{
					Name:  "install",
					Usage: "download and install plugins listed in plugins.json",
					Action: func(c *cli.Context) {
						if err := installPlugins(); err != nil {
							l := log.New("")
							l.SetFlags(0)
							l.Fatalf("could not start server\n%s", err)
						}
					},
				},
			},
		},
		{
			Name:    "console",
			Aliases: []string{"c"},
			Usage:   "communicate with a running abot server",
			Action: func(c *cli.Context) {
				if err := startConsole(c); err != nil {
					l := log.New("")
					l.SetFlags(0)
					l.Fatalf("could not start console\n%s", err)
				}
			},
		},
	}
	app.Action = func(c *cli.Context) {
		cli.ShowAppHelp(c)
	}
	if err := app.Run(os.Args); err != nil {
		log.Fatal(err)
	}
}

// startServer initializes any clients that are needed and boots plugins
func startServer() error {
	var e *echo.Echo
	var err error
	e, err = core.NewServer()
	if err != nil {
		return err
	}
	tmplLayout, err = template.ParseFiles("assets/html/layout.html")
	if err != nil {
		return err
	}
	initRoutes(e)
	e.Run(":" + os.Getenv("PORT"))
	return nil
}

func startConsole(c *cli.Context) error {
	args := c.Args()
	if len(args) == 0 || len(args) >= 3 {
		return errors.New("usage: abot console abot-address user-phone")
	}
	var addr, phone string
	if len(args) == 1 {
		addr = "localhost:" + os.Getenv("PORT")
		phone = args[0]
	} else if len(args) == 2 {
		addr = args[0]
		phone = args[1]
	}
	// Capture ^C interrupt to add a newline
	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt)
	go func() {
		for _ = range sig {
			fmt.Println("")
			os.Exit(0)
		}
	}()
	base := "http://" + addr + "?flexidtype=2&flexid=" + url.QueryEscape(phone) + "&cmd="
	scanner := bufio.NewScanner(os.Stdin)
	// Test connection
	req, err := http.NewRequest("GET", base, nil)
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	if err = resp.Body.Close(); err != nil {
		return err
	}
	fmt.Print("> ")
	for scanner.Scan() {
		cmd := scanner.Text()
		req, err := http.NewRequest("POST", base+url.QueryEscape(cmd), nil)
		client := &http.Client{}
		resp, err := client.Do(req)
		if err != nil {
			return err
		}
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		if err = resp.Body.Close(); err != nil {
			return err
		}
		fmt.Println(string(body))
		fmt.Print("> ")
	}
	if err := scanner.Err(); err != nil {
		return err
	}
	return nil
}

func installPlugins() error {
	l := log.New("")
	l.SetFlags(0)
	// Delete all plugins in the /plugins and /public directories
	err := os.RemoveAll("./plugins")
	if err != nil && err.Error() !=
		"remove ./plugins: no such file or directory" {
		l.Fatal(err)
	}
	err = os.RemoveAll("./public")
	if err != nil && err.Error() !=
		"remove ./public: no such file or directory" {
		l.Fatal(err)
	}
	// Read plugins.json, unmarshal into struct
	contents, err := ioutil.ReadFile("./plugins.json")
	if err != nil {
		l.Fatal(err)
	}
	var plugins pluginJSON
	if err = json.Unmarshal(contents, &plugins); err != nil {
		l.Fatal(err)
	}
	// Remake the /plugins dir for plugin Go code
	if err = os.Mkdir("./plugins", 0775); err != nil {
		l.Fatal(err)
	}
	// Remake the /public dir for assets
	if err = os.Mkdir("./public", 0775); err != nil {
		l.Fatal(err)
	}
	// Fetch plugins
	l.Info("Fetching", len(plugins.Dependencies), "plugins...")
	var wg sync.WaitGroup
	wg.Add(len(plugins.Dependencies))
	rand.Seed(time.Now().UTC().UnixNano())
	for url, version := range plugins.Dependencies {
		go func(url, version string) {
			// Download source as a zip
			var resp *http.Response
			resp, err = http.Get("https://" + url + "/archive/master.zip")
			if err != nil {
				l.Fatal(err)
			}
			defer func() {
				if err = resp.Body.Close(); err != nil {
					l.Fatal(err)
				}
			}()
			if resp.StatusCode != 200 {
				e := fmt.Sprintf("err fetching plugin %s: %d", url,
					resp.StatusCode)
				l.Fatal(errors.New(e))
			}
			fiName := "tmp_" + randSeq(8) + ".zip"
			fpZip := filepath.Join("./plugins", fiName)
			var out *os.File
			out, err = os.Create(fpZip)
			if err != nil {
				l.Fatal(err)
			}
			_, err = io.Copy(out, resp.Body)
			if err != nil {
				_ = out.Close()
				l.Fatal(err)
			}

			// Unzip source to directory
			if err = unzip(fpZip, "./plugins"); err != nil {
				_ = out.Close()
				l.Fatal(err)
			}

			// Close zip file
			if err = out.Close(); err != nil {
				l.Fatal(err)
			}

			// Delete zip file
			if err = os.Remove(fpZip); err != nil {
				l.Fatal(err)
			}

			// Sync to get dependencies
			var outC []byte
			outC, err = exec.
				Command("/bin/sh", "-c", "glock sync $(pwd | sed 's/^.*src\\///')").
				CombinedOutput()
			if err != nil {
				l.Debug(string(outC))
				l.Fatal(err)
			}

			// Anonymously increment the plugin's download count
			// at itsabot.org
			p := struct {
				Path string
			}{Path: url}
			outC, err = json.Marshal(p)
			if err != nil {
				l.Info("failed to build itsabot.org JSON.", err)
				wg.Done()
				return
			}
			var u string
			if len(os.Getenv("ITSABOT_URL")) > 0 {
				u = os.Getenv("ITSABOT_URL") + "/api/plugins.json"
			} else {
				u = "https://www.itsabot.org/api/plugins.json"
			}
			resp, err = http.Post(u, "application/json",
				bytes.NewBuffer(outC))
			if err != nil {
				l.Info("failed to update itsabot.org.", err)
				wg.Done()
				return
			}
			defer func() {
				if err = resp.Body.Close(); err != nil {
					l.Fatal(err)
				}
			}()
			if resp.StatusCode != 200 {
				l.Info("WARN: %d - %s\n", resp.StatusCode,
					resp.Status)
			}
			wg.Done()
		}(url, version)
	}
	wg.Wait()
	l.Info("Installing plugins...")
	outC, err := exec.
		Command("/bin/sh", "-c", "go install ./...").
		CombinedOutput()
	if err != nil {
		l.Debug(string(outC))
		l.Fatal(err)
	}
	l.Info("Success!")
	return nil
}

var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")

type pluginJSON struct {
	Dependencies map[string]string
}

// From https://stackoverflow.com/questions/20357223/easy-way-to-unzip-file-with-golang
func unzip(src, dest string) error {
	r, err := zip.OpenReader(src)
	if err != nil {
		return err
	}
	defer func() {
		if err = r.Close(); err != nil {
			panic(err)
		}
	}()
	if err = os.MkdirAll(dest, 0755); err != nil {
		return err
	}
	for _, f := range r.File {
		err = extractAndWriteFile(dest, f)
		if err != nil {
			return err
		}
	}
	return nil
}

// From https://stackoverflow.com/questions/20357223/easy-way-to-unzip-file-with-golang
func extractAndWriteFile(dest string, f *zip.File) error {
	rc, err := f.Open()
	if err != nil {
		return err
	}
	defer func() {
		if err = rc.Close(); err != nil {
			panic(err)
		}
	}()
	path := filepath.Join(dest, f.Name)
	if f.FileInfo().IsDir() {
		if err = os.MkdirAll(path, f.Mode()); err != nil {
			return err
		}
	} else {
		f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
		if err != nil {
			return err
		}
		defer func() {
			if err = f.Close(); err != nil {
				panic(err)
			}
		}()
		_, err = io.Copy(f, rc)
		if err != nil {
			return err
		}
	}
	return nil
}

// From https://stackoverflow.com/questions/22892120/how-to-generate-a-random-string-of-a-fixed-length-in-golang
func randSeq(n int) string {
	b := make([]rune, n)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}
<commit_msg>Fix log statement<commit_after>package main

import (
	"archive/zip"
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"sync"
	"time"

	"github.com/codegangsta/cli"
	"github.com/itsabot/abot/core"
	"github.com/itsabot/abot/core/websocket"
	"github.com/itsabot/abot/shared/log"
	"github.com/labstack/echo"
	_ "github.com/lib/pq"
)

var tmplLayout *template.Template
var ws = websocket.NewAtomicWebSocketSet()

func main() {
	rand.Seed(time.Now().UnixNano())
	log.SetDebug(true)
	app := cli.NewApp()
	app.Name = "abot"
	app.Usage = "digital assistant framework"
	app.Version = "0.0.1"
	app.Commands = []cli.Command{
		{
			Name:    "server",
			Aliases: []string{"s"},
			Usage:   "run server",
			Action: func(c *cli.Context) {
				var err error
				if err = startServer(); err != nil {
					l := log.New("")
					l.SetFlags(0)
					l.Fatalf("could not start server\n%s", err)
				}
			},
		},
		{
			Name:    "plugin",
			Aliases: []string{"p"},
			Usage:   "manage and install plugins from plugins.json",
			Subcommands: []cli.Command{
				{
					Name:  "install",
					Usage: "download and install plugins listed in plugins.json",
					Action: func(c *cli.Context) {
						if err := installPlugins(); err != nil {
							l := log.New("")
							l.SetFlags(0)
							l.Fatalf("could not install plugins\n%s", err)
						}
					},
				},
			},
		},
		{
			Name:    "console",
			Aliases: []string{"c"},
			Usage:   "communicate with a running abot server",
			Action: func(c *cli.Context) {
				if err := startConsole(c); err != nil {
					l := log.New("")
					l.SetFlags(0)
					l.Fatalf("could not start console\n%s", err)
				}
			},
		},
	}
	app.Action = func(c *cli.Context) {
		cli.ShowAppHelp(c)
	}
	if err := app.Run(os.Args); err != nil {
		log.Fatal(err)
	}
}

// startServer initializes any clients that are needed and boots plugins
func startServer() error {
	var e *echo.Echo
	var err error
	e, err = core.NewServer()
	if err != nil {
		return err
	}
	tmplLayout, err = template.ParseFiles("assets/html/layout.html")
	if err != nil {
		return err
	}
	initRoutes(e)
	e.Run(":" + os.Getenv("PORT"))
	return nil
}

func startConsole(c *cli.Context) error {
	args := c.Args()
	if len(args) == 0 || len(args) >= 3 {
		return errors.New("usage: abot console abot-address user-phone")
	}
	var addr, phone string
	if len(args) == 1 {
		addr = "localhost:" + os.Getenv("PORT")
		phone = args[0]
	} else if len(args) == 2 {
		addr = args[0]
		phone = args[1]
	}
	// Capture ^C interrupt to add a newline
	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt)
	go func() {
		for _ = range sig {
			fmt.Println("")
			os.Exit(0)
		}
	}()
	base := "http://" + addr + "?flexidtype=2&flexid=" + url.QueryEscape(phone) + "&cmd="
	scanner := bufio.NewScanner(os.Stdin)
	// Test connection
	req, err := http.NewRequest("GET", base, nil)
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	if err = resp.Body.Close(); err != nil {
		return err
	}
	fmt.Print("> ")
	for scanner.Scan() {
		cmd := scanner.Text()
		req, err := http.NewRequest("POST", base+url.QueryEscape(cmd), nil)
		client := &http.Client{}
		resp, err := client.Do(req)
		if err != nil {
			return err
		}
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		if err = resp.Body.Close(); err != nil {
			return err
		}
		fmt.Println(string(body))
		fmt.Print("> ")
	}
	if err := scanner.Err(); err != nil {
		return err
	}
	return nil
}

func installPlugins() error {
	l := log.New("")
	l.SetFlags(0)
	// Delete all plugins in the /plugins and /public directories
	err := os.RemoveAll("./plugins")
	if err != nil && err.Error() !=
		"remove ./plugins: no such file or directory" {
		l.Fatal(err)
	}
	err = os.RemoveAll("./public")
	if err != nil && err.Error() !=
		"remove ./public: no such file or directory" {
		l.Fatal(err)
	}
	// Read plugins.json, unmarshal into struct
	contents, err := ioutil.ReadFile("./plugins.json")
	if err != nil {
		l.Fatal(err)
	}
	var plugins pluginJSON
	if err = json.Unmarshal(contents, &plugins); err != nil {
		l.Fatal(err)
	}
	// Remake the /plugins dir for plugin Go code
	if err = os.Mkdir("./plugins", 0775); err != nil {
		l.Fatal(err)
	}
	// Remake the /public dir for assets
	if err = os.Mkdir("./public", 0775); err != nil {
		l.Fatal(err)
	}
	// Fetch plugins
	l.Info("Fetching", len(plugins.Dependencies), "plugins...")
	var wg sync.WaitGroup
	wg.Add(len(plugins.Dependencies))
	rand.Seed(time.Now().UTC().UnixNano())
	for url, version := range plugins.Dependencies {
		go func(url, version string) {
			// Download source as a zip
			var resp *http.Response
			resp, err = http.Get("https://" + url + "/archive/master.zip")
			if err != nil {
				l.Fatal(err)
			}
			defer func() {
				if err = resp.Body.Close(); err != nil {
					l.Fatal(err)
				}
			}()
			if resp.StatusCode != 200 {
				e := fmt.Sprintf("err fetching plugin %s: %d", url,
					resp.StatusCode)
				l.Fatal(errors.New(e))
			}
			fiName := "tmp_" + randSeq(8) + ".zip"
			fpZip := filepath.Join("./plugins", fiName)
			var out *os.File
			out, err = os.Create(fpZip)
			if err != nil {
				l.Fatal(err)
			}
			_, err = io.Copy(out, resp.Body)
			if err != nil {
				_ = out.Close()
				l.Fatal(err)
			}

			// Unzip source to directory
			if err = unzip(fpZip, "./plugins"); err != nil {
				_ = out.Close()
				l.Fatal(err)
			}

			// Close zip file
			if err = out.Close(); err != nil {
				l.Fatal(err)
			}

			// Delete zip file
			if err = os.Remove(fpZip); err != nil {
				l.Fatal(err)
			}

			// Sync to get dependencies
			var outC []byte
			outC, err = exec.
				Command("/bin/sh", "-c", "glock sync $(pwd | sed 's/^.*src\\///')").
				CombinedOutput()
			if err != nil {
				l.Debug(string(outC))
				l.Fatal(err)
			}

			// Anonymously increment the plugin's download count
			// at itsabot.org
			p := struct {
				Path string
			}{Path: url}
			outC, err = json.Marshal(p)
			if err != nil {
				l.Info("failed to build itsabot.org JSON.", err)
				wg.Done()
				return
			}
			var u string
			if len(os.Getenv("ITSABOT_URL")) > 0 {
				u = os.Getenv("ITSABOT_URL") + "/api/plugins.json"
			} else {
				u = "https://www.itsabot.org/api/plugins.json"
			}
			resp, err = http.Post(u, "application/json",
				bytes.NewBuffer(outC))
			if err != nil {
				l.Info("failed to update itsabot.org.", err)
				wg.Done()
				return
			}
			defer func() {
				if err = resp.Body.Close(); err != nil {
					l.Fatal(err)
				}
			}()
			if resp.StatusCode != 200 {
				l.Info("WARN: %d - %s\n", resp.StatusCode,
					resp.Status)
			}
			wg.Done()
		}(url, version)
	}
	wg.Wait()
	l.Info("Installing plugins...")
	outC, err := exec.
		Command("/bin/sh", "-c", "go install ./...").
		CombinedOutput()
	if err != nil {
		l.Debug(string(outC))
		l.Fatal(err)
	}
	l.Info("Success!")
	return nil
}

var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")

type pluginJSON struct {
	Dependencies map[string]string
}

// From https://stackoverflow.com/questions/20357223/easy-way-to-unzip-file-with-golang
func unzip(src, dest string) error {
	r, err := zip.OpenReader(src)
	if err != nil {
		return err
	}
	defer func() {
		if err = r.Close(); err != nil {
			panic(err)
		}
	}()
	if err = os.MkdirAll(dest, 0755); err != nil {
		return err
	}
	for _, f := range r.File {
		err = extractAndWriteFile(dest, f)
		if err != nil {
			return err
		}
	}
	return nil
}

// From https://stackoverflow.com/questions/20357223/easy-way-to-unzip-file-with-golang
func extractAndWriteFile(dest string, f *zip.File) error {
	rc, err := f.Open()
	if err != nil {
		return err
	}
	defer func() {
		if err = rc.Close(); err != nil {
			panic(err)
		}
	}()
	path := filepath.Join(dest, f.Name)
	if f.FileInfo().IsDir() {
		if err = os.MkdirAll(path, f.Mode()); err != nil {
			return err
		}
	} else {
		f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
		if err != nil {
			return err
		}
		defer func() {
			if err = f.Close(); err != nil {
				panic(err)
			}
		}()
		_, err = io.Copy(f, rc)
		if err != nil {
			return err
		}
	}
	return nil
}

// From https://stackoverflow.com/questions/22892120/how-to-generate-a-random-string-of-a-fixed-length-in-golang
func randSeq(n int) string {
	b := make([]rune, n)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}
<|endoftext|>


<commit_before><commit_msg>fixed error handling of redis array-replies<commit_after><|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ratelimit_test

import (
	"testing"

	. "github.com/jacobsa/ogletest"
)

func TestThrottledReader(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type ThrottledReaderTest struct {
}

func init() { RegisterTestSuite(&ThrottledReaderTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ThrottledReaderTest) CallsThrottle() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) ThrottleSaysCancelled() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) CallsWrapped() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) WrappedReturnsError() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) WrappedReturnsEOF() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) WrappedReturnsFullRead() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_CallsAgain() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondFails() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondSuceeds() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) ReadSizeIsAboveThrottleCapacity() {
	AssertTrue(false, "TODO")
}
<commit_msg>ThrottledReaderTest.SetUp<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ratelimit_test

import (
	"io"
	"testing"

	"golang.org/x/net/context"

	"github.com/googlecloudplatform/gcsfuse/ratelimit"
	. "github.com/jacobsa/ogletest"
)

func TestThrottledReader(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// An io.Reader that defers to a function.
type funcReader struct {
	f func([]byte) (int, error)
}

func (fr *funcReader) Read(p []byte) (n int, err error) {
	n, err = fr.f(p)
	return
}

// A throttler that defers to a function.
type funcThrottle struct {
	f func(context.Context, uint64) bool
}

func (ft *funcThrottle) Capacity() (c uint64) {
	return 1024
}

func (ft *funcThrottle) Wait(
	ctx context.Context,
	tokens uint64) (ok bool) {
	ok = ft.f(ctx, tokens)
	return
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type ThrottledReaderTest struct {
	ctx context.Context

	wrapped  funcReader
	throttle funcThrottle

	reader io.Reader
}

var _ SetUpInterface = &ThrottledReaderTest{}

func init() { RegisterTestSuite(&ThrottledReaderTest{}) }

func (t *ThrottledReaderTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx

	// Set up the default throttle function.
	t.throttle.f = func(ctx context.Context, tokens uint64) (ok bool) {
		ok = true
		return
	}

	// Set up the reader.
	t.reader = ratelimit.ThrottledReader(t.ctx, &t.wrapped, &t.throttle)
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ThrottledReaderTest) CallsThrottle() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) ThrottleSaysCancelled() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) CallsWrapped() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) WrappedReturnsError() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) WrappedReturnsEOF() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) WrappedReturnsFullRead() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_CallsAgain() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondFails() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondSuceeds() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) ReadSizeIsAboveThrottleCapacity() {
	AssertTrue(false, "TODO")
}
<|endoftext|>


<commit_before>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

package parse

import (
	"code.google.com/p/go.text/unicode/norm"
	"fmt"
	"github.com/demizer/go-elog"
	"github.com/demizer/go-spew/spew"
	"reflect"
)

var spd = spew.ConfigState{Indent: "\t", DisableMethods: true}

type systemMessageLevel int

const (
	levelInfo systemMessageLevel = iota
	levelWarning
	levelError
	levelSevere
)

var systemMessageLevels = [...]string{
	"INFO",
	"WARNING",
	"ERROR",
	"SEVERE",
}

func (s systemMessageLevel) String() string {
	return systemMessageLevels[s]
}

type parserMessage int

const (
	warningShortUnderline parserMessage = iota
	errorUnexpectedSectionTitle
	errorUnexpectedSectionTitleOrTransition
)

var parserErrors = [...]string{
	"warningShortUnderline",
	"errorUnexpectedSectionTitle",
	"errorUnexpectedSectionTitleOrTransition",
}

func (p parserMessage) String() string {
	return parserErrors[p]
}

func (p parserMessage) Message() (s string) {
	switch p {
	case warningShortUnderline:
		s = "Title underline too short."
	case errorUnexpectedSectionTitle:
		s = "Unexpected section title."
	case errorUnexpectedSectionTitleOrTransition:
		s = "Unexpected section title or transition."
	}
	return
}

func (p parserMessage) Level() (s systemMessageLevel) {
	switch p {
	case warningShortUnderline:
		s = levelWarning
	case errorUnexpectedSectionTitle:
		s = levelSevere
	case errorUnexpectedSectionTitleOrTransition:
		s = levelSevere
	}
	return
}

type sectionLevels []*SectionNode

func (s *sectionLevels) String() string {
	var out string
	for _, sec := range *s {
		out += fmt.Sprintf("level: %d, rune: %q, overline: %t, length: %d\n",
			sec.Level, sec.UnderLine.Rune, sec.OverLine != nil, sec.Length)
	}
	return out
}

// Returns nil if not found
func (s *sectionLevels) FindByRune(adornChar rune) *SectionNode {
	for _, sec := range *s {
		if sec.UnderLine.Rune == adornChar {
			return sec
		}
	}
	return nil
}

// If exists == true, a section node with the same text and underline has been found in
// sectionLevels, sec is the matching SectionNode. If exists == false, then the sec return value is
// the similarly leveled SectionNode. If exists == false and sec == nil, then the SectionNode added
// to sectionLevels is a new Node.
func (s *sectionLevels) Add(section *SectionNode) (exists bool, sec *SectionNode) {
	sec = s.FindByRune(section.UnderLine.Rune)
	if sec != nil {
		if sec.Text == section.Text {
			return true, sec
		} else if sec.Text != section.Text {
			section.Level = sec.Level
		}
	} else {
		section.Level = len(*s) + 1
	}
	exists = false
	*s = append(*s, section)
	return
}

func (s *sectionLevels) Level() int {
	return len(*s)
}

// Parse is the entry point for the reStructuredText parser.
func Parse(name, text string) (t *Tree, errors []error) {
	t = New(name)
	if !norm.NFC.IsNormalString(text) {
		text = norm.NFC.String(text)
	}
	t.text = text
	_, errors = t.Parse(text, t)
	return
}

func New(name string) *Tree {
	return &Tree{
		Name:          name,
		Nodes:         newList(),
		nodeTarget:    newList(),
		sectionLevels: new(sectionLevels),
		indentWidth:   indentWidth,
	}
}

const (
	zed         = 3
	indentWidth = 4 // Default indent width
)

type Tree struct {
	Name             string
	Nodes            *NodeList // The root node list
	nodeTarget       *NodeList // Used by the parser to add nodes to a target NodeList
	Errors           []error
	text             string
	lex              *lexer
	tokenBackupCount int
	peekCount        int
	token            [7]*item
	sectionLevels    *sectionLevels // Encountered section levels
	id               int            // The unique id of the node in the tree
	indentWidth      int
	indentLevel      int
}

// startParse initializes the parser, using the lexer.
func (t *Tree) startParse(lex *lexer) {
	t.lex = lex
}

func (t *Tree) Parse(text string, treeSet *Tree) (tree *Tree, errors []error) {
	log.Debugln("Start")
	t.startParse(lex(t.Name, text))
	t.text = text
	t.parse(treeSet)
	log.Debugln("End")
	return t, t.Errors
}

func (t *Tree) parse(tree *Tree) {
	log.Debugln("Start")

	t.nodeTarget = t.Nodes

	for t.peek(1).Type != itemEOF {
		var n Node

		token := t.next()
		log.Infof("\nParser got token: %#+v\n\n", token)

		switch token.Type {
		case itemSectionAdornment:
			n = t.section(token)
		case itemParagraph:
			n = newParagraph(token, &t.id)
		case itemSpace:
			n = t.indent(token)
			if n == nil {
				continue
			}
		case itemTitle, itemBlankLine:
			// itemTitle is consumed when evaluating itemSectionAdornment
			continue
		}

		t.nodeTarget.append(n)
		switch n.NodeType() {
		case NodeSection, NodeBlockQuote:
			// Set the loop to append items to the NodeList of the new section
			t.nodeTarget = reflect.ValueOf(n).Elem().FieldByName("NodeList").Addr().Interface().(*NodeList)
		}
	}

	log.Debugln("End")
}

func (t *Tree) peekBack(pos int) *item {
	return t.token[zed-pos]
}

func (t *Tree) peek(pos int) *item {
	// log.Debugln("\n", "Pos:", pos)
	// log.Debugf("##### peek() before #####\n")
	// spd.Dump(t.token)
	nItem := t.token[zed]
	for i := 1; i <= pos; i++ {
		if t.token[zed+i] != nil {
			nItem = t.token[zed+i]
			log.Debugf("Using %#+v\n", nItem)
			continue
		} else {
			log.Debugln("Getting next item")
			t.token[zed+i] = t.lex.nextItem()
			nItem = t.token[zed+i]
		}
	}
	// log.Debugf("\n##### peek() aftermath #####\n")
	// spd.Dump(t.token)
	// log.Debugf("Returning: %#+v\n", nItem)
	return nItem
}

func (t *Tree) peekSkip(pos int, iSkip itemElement) *item {
	var nItem *item
outer:
	for i := 1; i <= pos; i++ {
		for {
			nItem = t.peek(i)
			if nItem.Type == iSkip {
				continue
			} else {
				break outer
			}
		}
	}
	return nItem
}

func (t *Tree) next() *item {
	// log.Debugln("t.peekCount:", t.peekCount)
	// skip shifts the pointers left in t.token, pos is the amount to shift
	skip := func(num int) {
		for i := num; i > 0; i-- {
			for x := 0; x < len(t.token)-1; x++ {
				t.token[x] = t.token[x+1]
				t.token[x+1] = nil
			}
		}
	}
	if t.peekCount > 0 {
		skip(t.peekCount)
	} else {
		skip(1)
		t.token[zed] = t.lex.nextItem()
	}
	t.tokenBackupCount, t.peekCount = 0, 0
	// log.Debugf("\n##### next() aftermath #####\n\n")
	// spd.Dump(t.token)
	return t.token[zed]
}

func (t *Tree) section(i *item) Node {
	log.Debugln("Start")
	var overAdorn, title, underAdorn *item
	var sysMessage Node

	peekForward := t.peekSkip(1, itemSpace)
	if peekForward != nil && peekForward.Type == itemTitle {
		log.Debugln("FOUND SECTION WITH OVERLINE")
		overAdorn = i
		t.next()
	loop:
		for {
			switch tTok := t.token[zed]; tTok.Type {
			case itemTitle:
				title = tTok
				t.next()
				cur := t.token[zed]
				if cur != nil && cur.Type == itemSectionAdornment {
					continue
				}
			case itemSectionAdornment:
				underAdorn = tTok
				break loop
			}
		}
	} else {
		if peekBack := t.peekBack(1); peekBack != nil && peekBack.Type == itemSpace {
			// Looking back past the white space
			if t.peekBack(2).Type == itemTitle {
				return t.systemMessage(errorUnexpectedSectionTitle)
			}
			return t.systemMessage(errorUnexpectedSectionTitleOrTransition)
		}
		title = t.peekBack(1)
		underAdorn = i
	}

	sec := newSection(title, overAdorn, underAdorn, indent, &t.id)
	exists, eSec := t.sectionLevels.Add(sec)
	if !exists && eSec != nil {
		// There is a matching level in sectionLevels
		t.nodeTarget = &(*t.sectionLevels)[sec.Level-2].NodeList
	}

	// System messages have to be applied after the section is created in order to preserve
	// a consecutive id number.
	if title.Length != underAdorn.Length {
		sysMessage = t.systemMessage(warningShortUnderline)
		sec.NodeList = append(sec.NodeList, sysMessage)
	}

	log.Debugln("End")
	return sec
}

func (t *Tree) systemMessage(err parserMessage) Node {
	var lbText string
	var lbTextLen int
	var backToken int

	s := newSystemMessage(&item{
		Type: itemSystemMessage,
		Line: t.token[zed].Line,
	},
		err.Level(), &t.id)

	msg := newParagraph(&item{
		Text:   err.Message(),
		Length: len(err.Message()),
	}, &t.id)

	log.Debugln("FOUND", err)

	switch err {
	case warningShortUnderline, errorUnexpectedSectionTitle:
		backToken = zed - 1
		if t.peekBack(1).Type == itemSpace {
			backToken = zed - 2
		}
		lbText = t.token[backToken].Text.(string) + "\n" + t.token[zed].Text.(string)
		lbTextLen = len(lbText) + 1
	case errorUnexpectedSectionTitleOrTransition:
		lbText = t.token[zed].Text.(string)
		lbTextLen = len(lbText)
	}

	lb := newLiteralBlock(&item{
		Type:   itemLiteralBlock,
		Text:   lbText,
		Length: lbTextLen, // Add one to account for the backslash
	}, &t.id)

	s.NodeList = append(s.NodeList, msg, lb)
	return s
}

func (t *Tree) indent(i *item) Node {
	level := i.Length / t.indentWidth
	if t.peekBack(1).Type == itemBlankLine {
		if t.indentLevel == level {
			// Append to the current blockquote NodeList
			return nil
		}
		t.indentLevel = level
		return newBlockQuote(&item{Type: itemBlockquote, Line: i.Line}, level, &t.id)
	}
	return nil
}
<commit_msg>parse.go: Add itemEOF back to Tree.parse() switch<commit_after>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

package parse

import (
	"code.google.com/p/go.text/unicode/norm"
	"fmt"
	"github.com/demizer/go-elog"
	"github.com/demizer/go-spew/spew"
	"reflect"
)

var spd = spew.ConfigState{Indent: "\t", DisableMethods: true}

type systemMessageLevel int

const (
	levelInfo systemMessageLevel = iota
	levelWarning
	levelError
	levelSevere
)

var systemMessageLevels = [...]string{
	"INFO",
	"WARNING",
	"ERROR",
	"SEVERE",
}

func (s systemMessageLevel) String() string {
	return systemMessageLevels[s]
}

type parserMessage int

const (
	warningShortUnderline parserMessage = iota
	errorUnexpectedSectionTitle
	errorUnexpectedSectionTitleOrTransition
)

var parserErrors = [...]string{
	"warningShortUnderline",
	"errorUnexpectedSectionTitle",
	"errorUnexpectedSectionTitleOrTransition",
}

func (p parserMessage) String() string {
	return parserErrors[p]
}

func (p parserMessage) Message() (s string) {
	switch p {
	case warningShortUnderline:
		s = "Title underline too short."
	case errorUnexpectedSectionTitle:
		s = "Unexpected section title."
	case errorUnexpectedSectionTitleOrTransition:
		s = "Unexpected section title or transition."
	}
	return
}

func (p parserMessage) Level() (s systemMessageLevel) {
	switch p {
	case warningShortUnderline:
		s = levelWarning
	case errorUnexpectedSectionTitle:
		s = levelSevere
	case errorUnexpectedSectionTitleOrTransition:
		s = levelSevere
	}
	return
}

type sectionLevels []*SectionNode

func (s *sectionLevels) String() string {
	var out string
	for _, sec := range *s {
		out += fmt.Sprintf("level: %d, rune: %q, overline: %t, length: %d\n",
			sec.Level, sec.UnderLine.Rune, sec.OverLine != nil, sec.Length)
	}
	return out
}

// Returns nil if not found
func (s *sectionLevels) FindByRune(adornChar rune) *SectionNode {
	for _, sec := range *s {
		if sec.UnderLine.Rune == adornChar {
			return sec
		}
	}
	return nil
}

// If exists == true, a section node with the same text and underline has been found in
// sectionLevels, sec is the matching SectionNode. If exists == false, then the sec return value is
// the similarly leveled SectionNode. If exists == false and sec == nil, then the SectionNode added
// to sectionLevels is a new Node.
func (s *sectionLevels) Add(section *SectionNode) (exists bool, sec *SectionNode) {
	sec = s.FindByRune(section.UnderLine.Rune)
	if sec != nil {
		if sec.Text == section.Text {
			return true, sec
		} else if sec.Text != section.Text {
			section.Level = sec.Level
		}
	} else {
		section.Level = len(*s) + 1
	}
	exists = false
	*s = append(*s, section)
	return
}

func (s *sectionLevels) Level() int {
	return len(*s)
}

// Parse is the entry point for the reStructuredText parser.
func Parse(name, text string) (t *Tree, errors []error) {
	t = New(name)
	if !norm.NFC.IsNormalString(text) {
		text = norm.NFC.String(text)
	}
	t.text = text
	_, errors = t.Parse(text, t)
	return
}

func New(name string) *Tree {
	return &Tree{
		Name:          name,
		Nodes:         newList(),
		nodeTarget:    newList(),
		sectionLevels: new(sectionLevels),
		indentWidth:   indentWidth,
	}
}

const (
	zed         = 3
	indentWidth = 4 // Default indent width
)

type Tree struct {
	Name             string
	Nodes            *NodeList // The root node list
	nodeTarget       *NodeList // Used by the parser to add nodes to a target NodeList
	Errors           []error
	text             string
	lex              *lexer
	tokenBackupCount int
	peekCount        int
	token            [7]*item
	sectionLevels    *sectionLevels // Encountered section levels
	id               int            // The unique id of the node in the tree
	indentWidth      int
	indentLevel      int
}

// startParse initializes the parser, using the lexer.
func (t *Tree) startParse(lex *lexer) {
	t.lex = lex
}

func (t *Tree) Parse(text string, treeSet *Tree) (tree *Tree, errors []error) {
	log.Debugln("Start")
	t.startParse(lex(t.Name, text))
	t.text = text
	t.parse(treeSet)
	log.Debugln("End")
	return t, t.Errors
}

func (t *Tree) parse(tree *Tree) {
	log.Debugln("Start")

	t.nodeTarget = t.Nodes

	for t.peek(1).Type != itemEOF {
		var n Node

		token := t.next()
		log.Infof("\nParser got token: %#+v\n\n", token)

		switch token.Type {
		case itemSectionAdornment:
			n = t.section(token)
		case itemParagraph:
			n = newParagraph(token, &t.id)
		case itemSpace:
			n = t.indent(token)
			if n == nil {
				continue
			}
		case itemEOF:
			goto exit
		case itemTitle, itemBlankLine:
			// itemTitle is consumed when evaluating itemSectionAdornment
			continue
		}

		t.nodeTarget.append(n)
		switch n.NodeType() {
		case NodeSection, NodeBlockQuote:
			// Set the loop to append items to the NodeList of the new section
			t.nodeTarget = reflect.ValueOf(n).Elem().FieldByName("NodeList").Addr().Interface().(*NodeList)
		}
	}

exit:
	log.Debugln("End")
}

func (t *Tree) peekBack(pos int) *item {
	return t.token[zed-pos]
}

func (t *Tree) peek(pos int) *item {
	// log.Debugln("\n", "Pos:", pos)
	// log.Debugf("##### peek() before #####\n")
	// spd.Dump(t.token)
	nItem := t.token[zed]
	for i := 1; i <= pos; i++ {
		if t.token[zed+i] != nil {
			nItem = t.token[zed+i]
			log.Debugf("Using %#+v\n", nItem)
			continue
		} else {
			log.Debugln("Getting next item")
			t.token[zed+i] = t.lex.nextItem()
			nItem = t.token[zed+i]
		}
	}
	// log.Debugf("\n##### peek() aftermath #####\n")
	// spd.Dump(t.token)
	// log.Debugf("Returning: %#+v\n", nItem)
	return nItem
}

func (t *Tree) peekSkip(pos int, iSkip itemElement) *item {
	var nItem *item
outer:
	for i := 1; i <= pos; i++ {
		for {
			nItem = t.peek(i)
			if nItem.Type == iSkip {
				continue
			} else {
				break outer
			}
		}
	}
	return nItem
}

func (t *Tree) next() *item {
	// log.Debugln("t.peekCount:", t.peekCount)
	// skip shifts the pointers left in t.token, pos is the amount to shift
	skip := func(num int) {
		for i := num; i > 0; i-- {
			for x := 0; x < len(t.token)-1; x++ {
				t.token[x] = t.token[x+1]
				t.token[x+1] = nil
			}
		}
	}
	if t.peekCount > 0 {
		skip(t.peekCount)
	} else {
		skip(1)
		t.token[zed] = t.lex.nextItem()
	}
	t.tokenBackupCount, t.peekCount = 0, 0
	// log.Debugf("\n##### next() aftermath #####\n\n")
	// spd.Dump(t.token)
	return t.token[zed]
}

func (t *Tree) section(i *item) Node {
	log.Debugln("Start")
	var overAdorn, title, underAdorn *item
	var sysMessage Node

	peekForward := t.peekSkip(1, itemSpace)
	if peekForward != nil && peekForward.Type == itemTitle {
		log.Debugln("FOUND SECTION WITH OVERLINE")
		overAdorn = i
		t.next()
	loop:
		for {
			switch tTok := t.token[zed]; tTok.Type {
			case itemTitle:
				title = tTok
				t.next()
				cur := t.token[zed]
				if cur != nil && cur.Type == itemSectionAdornment {
					continue
				}
			case itemSectionAdornment:
				underAdorn = tTok
				break loop
			}
		}
	} else {
		if peekBack := t.peekBack(1); peekBack != nil && peekBack.Type == itemSpace {
			// Looking back past the white space
			if t.peekBack(2).Type == itemTitle {
				return t.systemMessage(errorUnexpectedSectionTitle)
			}
			return t.systemMessage(errorUnexpectedSectionTitleOrTransition)
		}
		title = t.peekBack(1)
		underAdorn = i
	}

	sec := newSection(title, overAdorn, underAdorn, indent, &t.id)
	exists, eSec := t.sectionLevels.Add(sec)
	if !exists && eSec != nil {
		// There is a matching level in sectionLevels
		t.nodeTarget = &(*t.sectionLevels)[sec.Level-2].NodeList
	}

	// System messages have to be applied after the section is created in order to preserve
	// a consecutive id number.
	if title.Length != underAdorn.Length {
		sysMessage = t.systemMessage(warningShortUnderline)
		sec.NodeList = append(sec.NodeList, sysMessage)
	}

	log.Debugln("End")
	return sec
}

func (t *Tree) systemMessage(err parserMessage) Node {
	var lbText string
	var lbTextLen int
	var backToken int

	s := newSystemMessage(&item{
		Type: itemSystemMessage,
		Line: t.token[zed].Line,
	},
		err.Level(), &t.id)

	msg := newParagraph(&item{
		Text:   err.Message(),
		Length: len(err.Message()),
	}, &t.id)

	log.Debugln("FOUND", err)

	switch err {
	case warningShortUnderline, errorUnexpectedSectionTitle:
		backToken = zed - 1
		if t.peekBack(1).Type == itemSpace {
			backToken = zed - 2
		}
		lbText = t.token[backToken].Text.(string) + "\n" + t.token[zed].Text.(string)
		lbTextLen = len(lbText) + 1
	case errorUnexpectedSectionTitleOrTransition:
		lbText = t.token[zed].Text.(string)
		lbTextLen = len(lbText)
	}

	lb := newLiteralBlock(&item{
		Type:   itemLiteralBlock,
		Text:   lbText,
		Length: lbTextLen, // Add one to account for the backslash
	}, &t.id)

	s.NodeList = append(s.NodeList, msg, lb)
	return s
}

func (t *Tree) indent(i *item) Node {
	level := i.Length / t.indentWidth
	if t.peekBack(1).Type == itemBlankLine {
		if t.indentLevel == level {
			// Append to the current blockquote NodeList
			return nil
		}
		t.indentLevel = level
		return newBlockQuote(&item{Type: itemBlockquote, Line: i.Line}, level, &t.id)
	}
	return nil
}
<|endoftext|>


<commit_before>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

package parse

import (
	"fmt"
	"github.com/demizer/go-elog"
	"github.com/demizer/go-spew/spew"
	"os"
	"reflect"
)

var spd = spew.ConfigState{Indent: "\t", DisableMethods: true}

type systemMessageLevel int

const (
	levelInfo systemMessageLevel = iota
	levelWarning
	levelError
	levelSevere
)

var systemMessageLevels = [...]string{
	"INFO",
	"WARNING",
	"ERROR",
	"SEVERE",
}

func (s systemMessageLevel) String() string {
	return systemMessageLevels[s]
}

type systemMessage struct {
	level  systemMessageLevel
	line   int
	source string
	items  []item
}

type sectionLevels []*SectionNode

func (s *sectionLevels) String() string {
	var out string
	for _, sec := range *s {
		out += fmt.Sprintf("level: %d, rune: %q, overline: %t, length: %d\n",
			sec.Level, sec.UnderLine.Rune, sec.OverLine != nil, sec.Length)
	}
	return out
}

// Returns nil if not found
func (s *sectionLevels) FindByRune(adornChar rune) *SectionNode {
	for _, sec := range *s {
		if sec.UnderLine.Rune == adornChar {
			return sec
		}
	}
	return nil
}

// If exists == true, a section node with the same text and underline has been found in
// sectionLevels, sec is the matching SectionNode. If exists == false, then the sec return value is
// the similarly leveled SectionNode. If exists == false and sec == nil, then the SectionNode added
// to sectionLevels is a new Node.
func (s *sectionLevels) Add(section *SectionNode) (exists bool, sec *SectionNode) {
	sec = s.FindByRune(section.UnderLine.Rune)
	if sec != nil {
		if sec.Text == section.Text {
			return true, sec
		} else if sec.Text != section.Text {
			section.Level = sec.Level
		}
	} else {
		section.Level = len(*s) + 1
	}
	exists = false
	*s = append(*s, section)
	return
}

func (s *sectionLevels) Level() int {
	return len(*s)
}

// Parse is the entry point for the reStructuredText parser.
func Parse(name, text string) (t *Tree, errors []error) {
	t = New(name)
	t.text = text
	_, errors = t.Parse(text, t)
	return
}

func New(name string) *Tree {
	return &Tree{Name: name, Nodes: newList(), nodeTarget: newList(), sectionLevels: new(sectionLevels)}
}

var tokenPos = 2

type Tree struct {
	Name             string
	Nodes            *NodeList // The root node list
	nodeTarget       *NodeList // Used by the parser to add nodes to a target NodeList
	Errors           []error
	text             string
	lex              *lexer
	tokenBackupCount int
	tokenPeekCount   int
	token            [5]*item
	sectionLevels    *sectionLevels // Encountered section levels
	id               int            // The unique id of the node in the tree
}

func (t *Tree) errorf(format string, args ...interface{}) {
	format = fmt.Sprintf("go-rst: %s:%d: %s\n", t.Name, t.lex.lineNumber(), format)
	t.Errors = append(t.Errors, fmt.Errorf(format, args...))
}

func (t *Tree) error(err error) {
	t.errorf("%s\n", err)
}

// startParse initializes the parser, using the lexer.
func (t *Tree) startParse(lex *lexer) {
	t.lex = lex
}

// stopParse terminates parsing.
func (t *Tree) stopParse() {
	t.Nodes = nil
	t.nodeTarget = nil
	t.lex = nil
}

func (t *Tree) Parse(text string, treeSet *Tree) (tree *Tree, errors []error) {
	log.Debugln("Start")
	t.startParse(lex(t.Name, text))
	t.text = text
	t.parse(treeSet)
	log.Debugln("End")
	return t, t.Errors
}

func (t *Tree) parse(tree *Tree) {
	log.Debugln("Start")

	t.nodeTarget = t.Nodes

	for t.peek(1).Type != itemEOF {
		var n Node
		token := t.next()
		log.Infof("Got token: %#+v\n", token)

		switch token.Type {
		case itemTitle: // Section includes overline/underline
			n = t.section(token)
		case itemBlankLine:
			n = newBlankLine(token, &t.id)
		case itemParagraph:
			n = newParagraph(token, &t.id)
		case itemSpace:
			n = newSpace(token, &t.id)
		case itemSectionAdornment:
			// Section adornments should be consumed with itemTitle
			panic("Parser should not find itemSectionAdornment!")
		default:
			t.errorf("%q Not implemented!", token.Type)
			continue
		}

		t.nodeTarget.append(n)
		if n.NodeType() == NodeSection {
			t.nodeTarget =
				reflect.ValueOf(n).Elem().FieldByName("NodeList").Addr().Interface().(*NodeList)
		}
	}

	log.Debugln("End")
}

func (t *Tree) backup() *item {
	t.tokenBackupCount++
	// log.Debugln("t.tokenBackupCount:", t.tokenPeekCount)
	if t.tokenBackupCount > 2 {
		panic("t.backup() can only be used twice consecutively.")
	}
	for i := 4; i > 0; i-- {
		t.token[i] = t.token[i-1]
		t.token[i-1] = nil
	}
	// log.Debugf("\n##### backup() aftermath #####\n\n")
	// spd.Dump(t.token)
	return t.token[tokenPos-t.tokenBackupCount]
}

func (t *Tree) peekBack(pos int) *item {
	if pos > 2 {
		panic("Cannot peek back more than two positions!")
	}
	return t.token[tokenPos-pos]
}

func (t *Tree) peek(pos int) *item {
	// log.Debugln("t.tokenPeekCount:", t.tokenPeekCount)
	if pos > 2 {
		panic("It is only possible to peek ahead two positions!")
	}
	for i := pos; i <= pos; i++ {
		t.tokenPeekCount++
		if t.token[tokenPos+t.tokenPeekCount] == nil {
			t.token[tokenPos+t.tokenPeekCount] = t.lex.nextItem()
		}
	}
	// log.Debugf("\n##### peek() aftermath #####\n\n")
	// spd.Dump(t.token)
	return t.token[tokenPos+t.tokenPeekCount]
}

func (t *Tree) next() *item {
	// shifts the pointers left in t.token, pos is the amount to shift
	shift := func(pos int) {
		for i := pos; i > 0; i-- {
			if t.tokenPeekCount > 0 {
				if t.token[tokenPos+t.tokenPeekCount+1] != nil {
					panic("t.token[t.tokenPeekCount] should be nil!")
				}
				t.token[tokenPos+t.tokenPeekCount+1] = t.lex.nextItem()
				t.tokenPeekCount--
			}
			for x := 0; x < 4; x++ {
				t.token[x] = t.token[x+1]
				t.token[x+1] = nil
			}
		}
	}
	if t.tokenPeekCount > 0 {
		shift(t.tokenPeekCount)
	} else {
		shift(1)
	}
	t.tokenBackupCount, t.tokenPeekCount = 0, 0
	return t.token[tokenPos]
}

func (t *Tree) section(i *item) Node {
	log.Debugln("Start")
	var overAdorn, title, underAdorn *item
	var overline bool

	peekBack := t.peekBack(1)
	if peekBack != nil {
		switch peekBack.Type {
		case itemSectionAdornment:
			overline = true
			overAdorn = peekBack
		case itemSpace:
			// TODO: Handle indented titles here!
			log.Debugln("FOUND ITEMSPACE BEFORE TITLE")
		}
	}

	title = i
	underAdorn = t.next() // Grab the section underline

	// Check adornment for proper syntax
	if title.Length != underAdorn.Length {
		t.errorf("Section under line  not equal to title length!")
	} else if overline && title.Length != overAdorn.Length {
		t.errorf("Section over line not equal to title length!")
	} else if overline && overAdorn.Text != underAdorn.Text {
		t.errorf("Section title over line does not match section title under line.")
	}

	sec := newSection(title, &t.id, overAdorn, underAdorn)
	exists, eSec := t.sectionLevels.Add(sec)
	if exists && eSec != nil {
		t.errorf("SectionNode using Text \"%s\" and Rune '%s' was previously parsed!",
			sec.Text, string(sec.UnderLine.Rune))
	} else if !exists && eSec != nil {
		// There is a matching level in sectionLevels
		t.nodeTarget = &(*t.sectionLevels)[sec.Level - 2].NodeList
	}

	log.Debugln("End")
	return sec
}
<commit_msg>parse.go: Cleanup nextItem() handling in t.next()<commit_after>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

package parse

import (
	"fmt"
	"github.com/demizer/go-elog"
	"github.com/demizer/go-spew/spew"
	"os"
	"reflect"
)

var spd = spew.ConfigState{Indent: "\t", DisableMethods: true}

type systemMessageLevel int

const (
	levelInfo systemMessageLevel = iota
	levelWarning
	levelError
	levelSevere
)

var systemMessageLevels = [...]string{
	"INFO",
	"WARNING",
	"ERROR",
	"SEVERE",
}

func (s systemMessageLevel) String() string {
	return systemMessageLevels[s]
}

type systemMessage struct {
	level  systemMessageLevel
	line   int
	source string
	items  []item
}

type sectionLevels []*SectionNode

func (s *sectionLevels) String() string {
	var out string
	for _, sec := range *s {
		out += fmt.Sprintf("level: %d, rune: %q, overline: %t, length: %d\n",
			sec.Level, sec.UnderLine.Rune, sec.OverLine != nil, sec.Length)
	}
	return out
}

// Returns nil if not found
func (s *sectionLevels) FindByRune(adornChar rune) *SectionNode {
	for _, sec := range *s {
		if sec.UnderLine.Rune == adornChar {
			return sec
		}
	}
	return nil
}

// If exists == true, a section node with the same text and underline has been found in
// sectionLevels, sec is the matching SectionNode. If exists == false, then the sec return value is
// the similarly leveled SectionNode. If exists == false and sec == nil, then the SectionNode added
// to sectionLevels is a new Node.
func (s *sectionLevels) Add(section *SectionNode) (exists bool, sec *SectionNode) {
	sec = s.FindByRune(section.UnderLine.Rune)
	if sec != nil {
		if sec.Text == section.Text {
			return true, sec
		} else if sec.Text != section.Text {
			section.Level = sec.Level
		}
	} else {
		section.Level = len(*s) + 1
	}
	exists = false
	*s = append(*s, section)
	return
}

func (s *sectionLevels) Level() int {
	return len(*s)
}

// Parse is the entry point for the reStructuredText parser.
func Parse(name, text string) (t *Tree, errors []error) {
	t = New(name)
	t.text = text
	_, errors = t.Parse(text, t)
	return
}

func New(name string) *Tree {
	return &Tree{Name: name, Nodes: newList(), nodeTarget: newList(), sectionLevels: new(sectionLevels)}
}

var tokenPos = 2

type Tree struct {
	Name             string
	Nodes            *NodeList // The root node list
	nodeTarget       *NodeList // Used by the parser to add nodes to a target NodeList
	Errors           []error
	text             string
	lex              *lexer
	tokenBackupCount int
	tokenPeekCount   int
	token            [5]*item
	sectionLevels    *sectionLevels // Encountered section levels
	id               int            // The unique id of the node in the tree
}

func (t *Tree) errorf(format string, args ...interface{}) {
	format = fmt.Sprintf("go-rst: %s:%d: %s\n", t.Name, t.lex.lineNumber(), format)
	t.Errors = append(t.Errors, fmt.Errorf(format, args...))
}

func (t *Tree) error(err error) {
	t.errorf("%s\n", err)
}

// startParse initializes the parser, using the lexer.
func (t *Tree) startParse(lex *lexer) {
	t.lex = lex
}

// stopParse terminates parsing.
func (t *Tree) stopParse() {
	t.Nodes = nil
	t.nodeTarget = nil
	t.lex = nil
}

func (t *Tree) Parse(text string, treeSet *Tree) (tree *Tree, errors []error) {
	log.Debugln("Start")
	t.startParse(lex(t.Name, text))
	t.text = text
	t.parse(treeSet)
	log.Debugln("End")
	return t, t.Errors
}

func (t *Tree) parse(tree *Tree) {
	log.Debugln("Start")

	t.nodeTarget = t.Nodes

	for t.peek(1).Type != itemEOF {
		var n Node
		token := t.next()
		log.Infof("Got token: %#+v\n", token)

		switch token.Type {
		case itemTitle: // Section includes overline/underline
			n = t.section(token)
		case itemBlankLine:
			n = newBlankLine(token, &t.id)
		case itemParagraph:
			n = newParagraph(token, &t.id)
		case itemSpace:
			n = newSpace(token, &t.id)
		case itemSectionAdornment:
			// Section adornments should be consumed with itemTitle
			panic("Parser should not find itemSectionAdornment!")
		default:
			t.errorf("%q Not implemented!", token.Type)
			continue
		}

		t.nodeTarget.append(n)
		if n.NodeType() == NodeSection {
			t.nodeTarget =
				reflect.ValueOf(n).Elem().FieldByName("NodeList").Addr().Interface().(*NodeList)
		}
	}

	log.Debugln("End")
}

func (t *Tree) backup() *item {
	t.tokenBackupCount++
	// log.Debugln("t.tokenBackupCount:", t.tokenPeekCount)
	if t.tokenBackupCount > 2 {
		panic("t.backup() can only be used twice consecutively.")
	}
	for i := 4; i > 0; i-- {
		t.token[i] = t.token[i-1]
		t.token[i-1] = nil
	}
	// log.Debugf("\n##### backup() aftermath #####\n\n")
	// spd.Dump(t.token)
	return t.token[tokenPos-t.tokenBackupCount]
}

func (t *Tree) peekBack(pos int) *item {
	if pos > 2 {
		panic("Cannot peek back more than two positions!")
	}
	return t.token[tokenPos-pos]
}

func (t *Tree) peek(pos int) *item {
	// log.Debugln("t.tokenPeekCount:", t.tokenPeekCount)
	if pos > 2 {
		panic("It is only possible to peek ahead two positions!")
	}
	for i := pos; i <= pos; i++ {
		t.tokenPeekCount++
		if t.token[tokenPos+t.tokenPeekCount] == nil {
			t.token[tokenPos+t.tokenPeekCount] = t.lex.nextItem()
		}
	}
	// log.Debugf("\n##### peek() aftermath #####\n\n")
	// spd.Dump(t.token)
	return t.token[tokenPos+t.tokenPeekCount]
}

func (t *Tree) next() *item {
	// log.Debugln("t.tokenPeekCount:", t.tokenPeekCount)
	// shifts the pointers left in t.token, pos is the amount to shift
	shift := func(pos int) {
		for i := pos; i > 0; i-- {
			for x := 0; x < 4; x++ {
				t.token[x] = t.token[x+1]
				t.token[x+1] = nil
			}
		}
	}
	if t.tokenPeekCount > 0 {
		shift(t.tokenPeekCount)
	} else {
		shift(1)
		t.token[tokenPos] = t.lex.nextItem()
	}
	t.tokenBackupCount, t.tokenPeekCount = 0, 0
	// log.Debugf("\n##### next() aftermath #####\n\n")
	// spd.Dump(t.token)
	return t.token[tokenPos]
}

func (t *Tree) section(i *item) Node {
	log.Debugln("Start")
	var overAdorn, title, underAdorn *item
	var overline bool

	peekBack := t.peekBack(1)
	if peekBack != nil {
		switch peekBack.Type {
		case itemSectionAdornment:
			overline = true
			overAdorn = peekBack
		case itemSpace:
			// TODO: Handle indented titles here!
			log.Debugln("FOUND ITEMSPACE BEFORE TITLE")
		}
	}

	title = i
	underAdorn = t.next() // Grab the section underline

	// Check adornment for proper syntax
	if title.Length != underAdorn.Length {
		t.errorf("Section under line  not equal to title length!")
	} else if overline && title.Length != overAdorn.Length {
		t.errorf("Section over line not equal to title length!")
	} else if overline && overAdorn.Text != underAdorn.Text {
		t.errorf("Section title over line does not match section title under line.")
	}

	sec := newSection(title, &t.id, overAdorn, underAdorn)
	exists, eSec := t.sectionLevels.Add(sec)
	if exists && eSec != nil {
		t.errorf("SectionNode using Text \"%s\" and Rune '%s' was previously parsed!",
			sec.Text, string(sec.UnderLine.Rune))
	} else if !exists && eSec != nil {
		// There is a matching level in sectionLevels
		t.nodeTarget = &(*t.sectionLevels)[sec.Level - 2].NodeList
	}

	log.Debugln("End")
	return sec
}
<|endoftext|>


<commit_before>package lru

import (
	"sync"

	"github.com/hashicorp/golang-lru/internal"
)

// ARCCache is a thread-safe fixed size Adaptive Replacement Cache (ARC).
// ARC is an enhancement over the standard LRU cache in that tracks both
// frequency and recency of use. This avoids a burst in access to new
// entries from evicting the frequently used older entries. It adds some
// additional tracking overhead to a standard LRU cache, computationally
// it is roughly 2x the cost, and the extra memory overhead is linear
// with the size of the cache.
type ARCCache struct {
	size int // Size is the total capacity of the cache
	p    int // P is the dynamic preference towards T1 or T2

	t1 *internal.LRU // T1 is the LRU for recently accessed items
	b1 *internal.LRU // B1 is the LRU for evictions from t1

	t2 *internal.LRU // T2 is the LRU for frequently accessed items
	b2 *internal.LRU // B2 is the LRU for evictions from t2

	lock sync.RWMutex
}

// NewARC creates an ARC of the given size
func NewARC(size int) (*ARCCache, error) {
	// Create the sub LRUs
	b1, err := internal.NewLRU(size, nil)
	if err != nil {
		return nil, err
	}
	b2, err := internal.NewLRU(size, nil)
	if err != nil {
		return nil, err
	}
	t1, err := internal.NewLRU(size, nil)
	if err != nil {
		return nil, err
	}
	t2, err := internal.NewLRU(size, nil)
	if err != nil {
		return nil, err
	}

	// Initialize the ARC
	c := &ARCCache{
		size: size,
		p:    0,
		t1:   t1,
		b1:   b1,
		t2:   t2,
		b2:   b2,
	}
	return c, nil
}

// Get looks up a key's value from the cache.
func (c *ARCCache) Get(key interface{}) (interface{}, bool) {
	c.lock.Lock()
	defer c.lock.Unlock()

	// Ff the value is contained in T1 (recent), then
	// promote it to T2 (frequent)
	if val, ok := c.t1.Peek(key); ok {
		c.t1.Remove(key)
		c.t2.Add(key, val)
		return val, ok
	}

	// Check if the value is contained in T2 (frequent)
	if val, ok := c.t2.Get(key); ok {
		return val, ok
	}

	// No hit
	return nil, false
}

// Add adds a value to the cache.
func (c *ARCCache) Add(key, value interface{}) {
	c.lock.Lock()
	defer c.lock.Unlock()

	// Check if the value is contained in T1 (recent), and potentially
	// promote it to frequent T2
	if c.t1.Contains(key) {
		c.t1.Remove(key)
		c.t2.Add(key, value)
		return
	}

	// Check if the value is already in T2 (frequent) and update it
	if c.t2.Contains(key) {
		c.t2.Add(key, value)
		return
	}

	// Check if this value was recently evicted as part of the
	// recently used list
	if c.b1.Contains(key) {
		// T1 set is too small, increase P appropriately
		delta := 1
		b1Len := c.b1.Len()
		b2Len := c.b2.Len()
		if b2Len > b1Len {
			delta = b2Len / b1Len
		}
		if c.p+delta >= c.size {
			c.p = c.size
		} else {
			c.p += delta
		}

		// Potentially need to make room in the cache
		if c.t1.Len()+c.t2.Len() >= c.size {
			c.replace(false)
		}

		// Remove from B1
		c.b1.Remove(key)

		// Add the key to the frequently used list
		c.t2.Add(key, value)
		return
	}

	// Check if this value was recently evicted as part of the
	// frequently used list
	if c.b2.Contains(key) {
		// T2 set is too small, decrease P appropriately
		delta := 1
		b1Len := c.b1.Len()
		b2Len := c.b2.Len()
		if b1Len > b2Len {
			delta = b1Len / b2Len
		}
		if delta >= c.p {
			c.p = 0
		} else {
			c.p -= delta
		}

		// Potentially need to make room in the cache
		if c.t1.Len()+c.t2.Len() >= c.size {
			c.replace(true)
		}

		// Remove from B2
		c.b2.Remove(key)

		// Add the key to the frequntly used list
		c.t2.Add(key, value)
		return
	}

	// Potentially need to make room in the cache
	if c.t1.Len()+c.t2.Len() >= c.size {
		c.replace(false)
	}

	// Keep the size of the ghost buffers trim
	if c.b1.Len() > c.size-c.p {
		c.b1.RemoveOldest()
	}
	if c.b2.Len() > c.p {
		c.b2.RemoveOldest()
	}

	// Add to the recently seen list
	c.t1.Add(key, value)
	return
}

// replace is used to adaptively evict from either T1 or T2
// based on the current learned value of P
func (c *ARCCache) replace(b2ContainsKey bool) {
	t1Len := c.t1.Len()
	if t1Len > 0 && (t1Len > c.p || (t1Len == c.p && b2ContainsKey)) {
		k, _, ok := c.t1.RemoveOldest()
		if ok {
			c.b1.Add(k, nil)
		}
	} else {
		k, _, ok := c.t2.RemoveOldest()
		if ok {
			c.b2.Add(k, nil)
		}
	}
}

// Len returns the number of cached entries
func (c *ARCCache) Len() int {
	c.lock.RLock()
	defer c.lock.RUnlock()
	return c.t1.Len() + c.t2.Len()
}

// Keys returns all the cached keys
func (c *ARCCache) Keys() []interface{} {
	c.lock.RLock()
	defer c.lock.RUnlock()
	k1 := c.t1.Keys()
	k2 := c.t2.Keys()
	return append(k1, k2...)
}

// Remove is used to purge a key from the cache
func (c *ARCCache) Remove(key interface{}) {
	c.lock.Lock()
	defer c.lock.Unlock()
	if c.t1.Remove(key) {
		return
	}
	if c.t2.Remove(key) {
		return
	}
	if c.b1.Remove(key) {
		return
	}
	if c.b2.Remove(key) {
		return
	}
}

// Purge is used to clear the cache
func (c *ARCCache) Purge() {
	c.lock.Lock()
	defer c.lock.Unlock()
	c.t1.Purge()
	c.t2.Purge()
	c.b1.Purge()
	c.b2.Purge()
}

// Contains is used to check if the cache contains a key
// without updating recency or frequency.
func (c *ARCCache) Contains(key interface{}) bool {
	c.lock.RLock()
	defer c.lock.RUnlock()
	return c.t1.Contains(key) || c.t2.Contains(key)
}

// Peek is used to inspect the cache value of a key
// without updating recency or frequency.
func (c *ARCCache) Peek(key interface{}) (interface{}, bool) {
	c.lock.RLock()
	defer c.lock.RUnlock()
	if val, ok := c.t1.Peek(key); ok {
		return val, ok
	}
	return c.t2.Peek(key)
}
<commit_msg>Document IBM patent<commit_after>package lru

import (
	"sync"

	"github.com/hashicorp/golang-lru/internal"
)

// ARCCache is a thread-safe fixed size Adaptive Replacement Cache (ARC).
// ARC is an enhancement over the standard LRU cache in that tracks both
// frequency and recency of use. This avoids a burst in access to new
// entries from evicting the frequently used older entries. It adds some
// additional tracking overhead to a standard LRU cache, computationally
// it is roughly 2x the cost, and the extra memory overhead is linear
// with the size of the cache. ARC has been patented by IBM, but is
// similar to the TwoQueueCache (2Q) which requires setting parameters.
type ARCCache struct {
	size int // Size is the total capacity of the cache
	p    int // P is the dynamic preference towards T1 or T2

	t1 *internal.LRU // T1 is the LRU for recently accessed items
	b1 *internal.LRU // B1 is the LRU for evictions from t1

	t2 *internal.LRU // T2 is the LRU for frequently accessed items
	b2 *internal.LRU // B2 is the LRU for evictions from t2

	lock sync.RWMutex
}

// NewARC creates an ARC of the given size
func NewARC(size int) (*ARCCache, error) {
	// Create the sub LRUs
	b1, err := internal.NewLRU(size, nil)
	if err != nil {
		return nil, err
	}
	b2, err := internal.NewLRU(size, nil)
	if err != nil {
		return nil, err
	}
	t1, err := internal.NewLRU(size, nil)
	if err != nil {
		return nil, err
	}
	t2, err := internal.NewLRU(size, nil)
	if err != nil {
		return nil, err
	}

	// Initialize the ARC
	c := &ARCCache{
		size: size,
		p:    0,
		t1:   t1,
		b1:   b1,
		t2:   t2,
		b2:   b2,
	}
	return c, nil
}

// Get looks up a key's value from the cache.
func (c *ARCCache) Get(key interface{}) (interface{}, bool) {
	c.lock.Lock()
	defer c.lock.Unlock()

	// Ff the value is contained in T1 (recent), then
	// promote it to T2 (frequent)
	if val, ok := c.t1.Peek(key); ok {
		c.t1.Remove(key)
		c.t2.Add(key, val)
		return val, ok
	}

	// Check if the value is contained in T2 (frequent)
	if val, ok := c.t2.Get(key); ok {
		return val, ok
	}

	// No hit
	return nil, false
}

// Add adds a value to the cache.
func (c *ARCCache) Add(key, value interface{}) {
	c.lock.Lock()
	defer c.lock.Unlock()

	// Check if the value is contained in T1 (recent), and potentially
	// promote it to frequent T2
	if c.t1.Contains(key) {
		c.t1.Remove(key)
		c.t2.Add(key, value)
		return
	}

	// Check if the value is already in T2 (frequent) and update it
	if c.t2.Contains(key) {
		c.t2.Add(key, value)
		return
	}

	// Check if this value was recently evicted as part of the
	// recently used list
	if c.b1.Contains(key) {
		// T1 set is too small, increase P appropriately
		delta := 1
		b1Len := c.b1.Len()
		b2Len := c.b2.Len()
		if b2Len > b1Len {
			delta = b2Len / b1Len
		}
		if c.p+delta >= c.size {
			c.p = c.size
		} else {
			c.p += delta
		}

		// Potentially need to make room in the cache
		if c.t1.Len()+c.t2.Len() >= c.size {
			c.replace(false)
		}

		// Remove from B1
		c.b1.Remove(key)

		// Add the key to the frequently used list
		c.t2.Add(key, value)
		return
	}

	// Check if this value was recently evicted as part of the
	// frequently used list
	if c.b2.Contains(key) {
		// T2 set is too small, decrease P appropriately
		delta := 1
		b1Len := c.b1.Len()
		b2Len := c.b2.Len()
		if b1Len > b2Len {
			delta = b1Len / b2Len
		}
		if delta >= c.p {
			c.p = 0
		} else {
			c.p -= delta
		}

		// Potentially need to make room in the cache
		if c.t1.Len()+c.t2.Len() >= c.size {
			c.replace(true)
		}

		// Remove from B2
		c.b2.Remove(key)

		// Add the key to the frequntly used list
		c.t2.Add(key, value)
		return
	}

	// Potentially need to make room in the cache
	if c.t1.Len()+c.t2.Len() >= c.size {
		c.replace(false)
	}

	// Keep the size of the ghost buffers trim
	if c.b1.Len() > c.size-c.p {
		c.b1.RemoveOldest()
	}
	if c.b2.Len() > c.p {
		c.b2.RemoveOldest()
	}

	// Add to the recently seen list
	c.t1.Add(key, value)
	return
}

// replace is used to adaptively evict from either T1 or T2
// based on the current learned value of P
func (c *ARCCache) replace(b2ContainsKey bool) {
	t1Len := c.t1.Len()
	if t1Len > 0 && (t1Len > c.p || (t1Len == c.p && b2ContainsKey)) {
		k, _, ok := c.t1.RemoveOldest()
		if ok {
			c.b1.Add(k, nil)
		}
	} else {
		k, _, ok := c.t2.RemoveOldest()
		if ok {
			c.b2.Add(k, nil)
		}
	}
}

// Len returns the number of cached entries
func (c *ARCCache) Len() int {
	c.lock.RLock()
	defer c.lock.RUnlock()
	return c.t1.Len() + c.t2.Len()
}

// Keys returns all the cached keys
func (c *ARCCache) Keys() []interface{} {
	c.lock.RLock()
	defer c.lock.RUnlock()
	k1 := c.t1.Keys()
	k2 := c.t2.Keys()
	return append(k1, k2...)
}

// Remove is used to purge a key from the cache
func (c *ARCCache) Remove(key interface{}) {
	c.lock.Lock()
	defer c.lock.Unlock()
	if c.t1.Remove(key) {
		return
	}
	if c.t2.Remove(key) {
		return
	}
	if c.b1.Remove(key) {
		return
	}
	if c.b2.Remove(key) {
		return
	}
}

// Purge is used to clear the cache
func (c *ARCCache) Purge() {
	c.lock.Lock()
	defer c.lock.Unlock()
	c.t1.Purge()
	c.t2.Purge()
	c.b1.Purge()
	c.b2.Purge()
}

// Contains is used to check if the cache contains a key
// without updating recency or frequency.
func (c *ARCCache) Contains(key interface{}) bool {
	c.lock.RLock()
	defer c.lock.RUnlock()
	return c.t1.Contains(key) || c.t2.Contains(key)
}

// Peek is used to inspect the cache value of a key
// without updating recency or frequency.
func (c *ARCCache) Peek(key interface{}) (interface{}, bool) {
	c.lock.RLock()
	defer c.lock.RUnlock()
	if val, ok := c.t1.Peek(key); ok {
		return val, ok
	}
	return c.t2.Peek(key)
}
<|endoftext|>