<commit_before>package main

import (
	"bytes"
	b64 "encoding/base64"
	"github.com/gorilla/context"
	"github.com/pmylund/go-cache"
	"io"
	"net/http"
	"runtime/pprof"
	"strconv"
	"strings"
	"time"
)

// ContextKey is a key type to avoid collisions
type ContextKey int

// Enums for keys to be stored in a session context - this is how gorilla expects
// these to be implemented and is lifted pretty much from docs
const (
	SessionData       = 0
	AuthHeaderValue   = 1
	VersionData       = 2
	VersionKeyContext = 3
	OrgSessionContext = 4
	ContextData       = 5
	RetainHost        = 6
)

var SessionCache *cache.Cache = cache.New(10*time.Second, 5*time.Second)
var ExpiryCache *cache.Cache = cache.New(600*time.Second, 5*time.Second)

type ReturningHttpHandler interface {
	ServeHTTP(http.ResponseWriter, *http.Request) *http.Response
	ServeHTTPForCache(http.ResponseWriter, *http.Request) *http.Response
	CopyResponse(io.Writer, io.Reader)
	New(interface{}, *APISpec) (TykResponseHandler, error)
}

// TykMiddleware wraps up the ApiSpec and Proxy objects to be included in a
// middleware handler, this can probably be handled better.
type TykMiddleware struct {
	Spec  *APISpec
	Proxy ReturningHttpHandler
}

func SetUpSessionCache() *cache.Cache {
	sessionLength := 10
	evictionTime := 5
	if config.LocalSessionCache.CachedSessionTimeout > 0 {
		sessionLength = config.LocalSessionCache.CachedSessionTimeout
	}
	if config.LocalSessionCache.CacheSessionEviction > 0 {
		evictionTime = config.LocalSessionCache.CacheSessionEviction
	}

	return cache.New(time.Duration(sessionLength)*time.Second, time.Duration(evictionTime)*time.Second)
}

func (t TykMiddleware) GetOrgSession(key string) (SessionState, bool) {
	// Try and get the session from the session store
	var thisSession SessionState
	var found bool

	thisSession, found = t.Spec.OrgSessionManager.GetSessionDetail(key)
	if found {
		// If exists, assume it has been authorized and pass on
		if config.EnforceOrgDataAge {
			// We cache org expiry data
			log.Debug("Setting data expiry: ", thisSession.OrgID)
			go t.SetOrgExpiry(thisSession.OrgID, thisSession.DataExpires)
		}
		return thisSession, true
	}

	return thisSession, found
}

func (t TykMiddleware) SetOrgExpiry(orgid string, expiry int64) {
	ExpiryCache.Set(orgid, expiry, cache.DefaultExpiration)
}

func (t TykMiddleware) GetOrgSessionExpiry(orgid string) int64 {
	log.Debug("Checking: ", orgid)
	cachedVal, found := ExpiryCache.Get(orgid)
	if !found {
		log.Debug("no cached entry found, returning 7 days")
		return 604800
	}

	return cachedVal.(int64)
}

// ApplyPolicyIfExists will check if a policy is loaded, if it is, it will overwrite the session state to use the policy values
func (t TykMiddleware) ApplyPolicyIfExists(key string, thisSession *SessionState) {
	if thisSession.ApplyPolicyID != "" {
		log.Debug("Session has policy, checking")
		policy, ok := Policies[thisSession.ApplyPolicyID]
		if ok {
			// Check ownership, policy org owner must be the same as API,
			// otherwise youcould overwrite a session key with a policy from a different org!
			if policy.OrgID != t.Spec.APIDefinition.OrgID {
				log.Error("Attempting to apply policy from different organisation to key, skipping")
				return
			}

			log.Debug("Found policy, applying")

			if policy.Partitions.Quota || policy.Partitions.RateLimit || policy.Partitions.Acl {
				// This is a partitioned policy, only apply what is active
				log.Debug("Applying partitioned policy")

				if policy.Partitions.Quota {
					// Quotas
					log.Debug("Applying partition: Quota")
					thisSession.QuotaMax = policy.QuotaMax
					thisSession.QuotaRenewalRate = policy.QuotaRenewalRate
				}

				if policy.Partitions.RateLimit {
					// Rate limting
					log.Debug("Applying partition: Rate Limit")
					thisSession.Allowance = policy.Rate // This is a legacy thing, merely to make sure output is consistent. Needs to be purged
					thisSession.Rate = policy.Rate
					thisSession.Per = policy.Per
					if policy.LastUpdated != "" {
						thisSession.LastUpdated = policy.LastUpdated
					}
				}

				if policy.Partitions.Acl {
					// ACL
					log.Debug("Applying partition: ACL")
					thisSession.AccessRights = policy.AccessRights
					thisSession.HMACEnabled = policy.HMACEnabled
				}

			} else {
				// This is not a partitioned policy, apply everything
				log.Debug("Applying regular policy")
				// Quotas
				thisSession.QuotaMax = policy.QuotaMax
				thisSession.QuotaRenewalRate = policy.QuotaRenewalRate

				// Rate limting
				thisSession.Allowance = policy.Rate // This is a legacy thing, merely to make sure output is consistent. Needs to be purged
				thisSession.Rate = policy.Rate
				thisSession.Per = policy.Per
				if policy.LastUpdated != "" {
					thisSession.LastUpdated = policy.LastUpdated
				}

				// ACL
				thisSession.AccessRights = policy.AccessRights
				thisSession.HMACEnabled = policy.HMACEnabled
			}

			// Required for all
			thisSession.IsInactive = policy.IsInactive
			thisSession.Tags = policy.Tags

			log.Debug("Policy Applied, Access rights are: ", thisSession.AccessRights)
			log.Debug("Policy Applied, Access rights were: ", policy.AccessRights)

			// Update the session in the session manager in case it gets called again
			t.Spec.SessionManager.UpdateSession(key, *thisSession, t.Spec.APIDefinition.SessionLifetime)
			log.Debug("Policy applied to key")
		}
	}
}

// CheckSessionAndIdentityForValidKey will check first the Session store for a valid key, if not found, it will try
// the Auth Handler, if not found it will fail
func (t TykMiddleware) CheckSessionAndIdentityForValidKey(key string) (SessionState, bool) {
	// Try and get the session from the session store
	var thisSession SessionState
	var found bool

	// Check in-memory cache
	if !config.LocalSessionCache.DisableCacheSessionState {
		cachedVal, found := SessionCache.Get(key)
		if found {
			log.Debug("Key found in local cache")
			thisSession = cachedVal.(SessionState)
			t.ApplyPolicyIfExists(key, &thisSession)
			return thisSession, true
		}
	}

	// Check session store
	log.Debug("Querying keystore")
	thisSession, found = t.Spec.SessionManager.GetSessionDetail(key)
	if found {
		// If exists, assume it has been authorized and pass on
		// cache it
		go SessionCache.Set(key, thisSession, cache.DefaultExpiration)

		// Check for a policy, if there is a policy, pull it and overwrite the session values
		t.ApplyPolicyIfExists(key, &thisSession)
		log.Debug("Got key")
		return thisSession, true
	}

	// 2. If not there, get it from the AuthorizationHandler
	thisSession, found = t.Spec.AuthManager.IsKeyAuthorised(key)
	if found {
		// If not in Session, and got it from AuthHandler, create a session with a new TTL
		log.Info("Recreating session for key: ", key)

		// cache it
		go SessionCache.Set(key, thisSession, cache.DefaultExpiration)

		// Check for a policy, if there is a policy, pull it and overwrite the session values
		t.ApplyPolicyIfExists(key, &thisSession)
		t.Spec.SessionManager.UpdateSession(key, thisSession, t.Spec.APIDefinition.SessionLifetime)
	}

	return thisSession, found
}

// SuccessHandler represents the final ServeHTTP() request for a proxied API request
type SuccessHandler struct {
	*TykMiddleware
}

func (s SuccessHandler) RecordHit(w http.ResponseWriter, r *http.Request, timing int64, code int, requestCopy *http.Request, responseCopy *http.Response) {

	if s.Spec.DoNotTrack {
		return
	}

	if config.StoreAnalytics(r) {

		t := time.Now()

		// Track the key ID if it exists
		authHeaderValue := context.Get(r, AuthHeaderValue)
		keyName := ""
		if authHeaderValue != nil {
			keyName = authHeaderValue.(string)
		}

		// Track version data
		version := s.Spec.getVersionFromRequest(r)
		if version == "" {
			version = "Non Versioned"
		}

		// If OAuth, we need to grab it from the session, which may or may not exist
		OauthClientID := ""
		tags := make([]string, 0)
		var alias string
		thisSessionState := context.Get(r, SessionData)

		if thisSessionState != nil {
			OauthClientID = thisSessionState.(SessionState).OauthClientID
			tags = thisSessionState.(SessionState).Tags
			alias = thisSessionState.(SessionState).Alias
		}

		rawRequest := ""
		rawResponse := ""
		if RecordDetail(r) {
			if requestCopy != nil {
				// Get the wire format representation
				var wireFormatReq bytes.Buffer
				requestCopy.Write(&wireFormatReq)
				rawRequest = b64.StdEncoding.EncodeToString(wireFormatReq.Bytes())
			}
			if responseCopy != nil {
				// Get the wire format representation
				var wireFormatRes bytes.Buffer
				responseCopy.Write(&wireFormatRes)
				rawResponse = b64.StdEncoding.EncodeToString(wireFormatRes.Bytes())
			}
		}

		thisRecord := AnalyticsRecord{
			r.Method,
			r.URL.Path,
			r.URL.Path,
			r.ContentLength,
			r.Header.Get("User-Agent"),
			t.Day(),
			t.Month(),
			t.Year(),
			t.Hour(),
			code,
			keyName,
			t,
			version,
			s.Spec.APIDefinition.Name,
			s.Spec.APIDefinition.APIID,
			s.Spec.APIDefinition.OrgID,
			OauthClientID,
			timing,
			rawRequest,
			rawResponse,
			GetIPFromRequest(r),
			GeoData{},
			tags,
			alias,
			time.Now(),
		}

		thisRecord.GetGeo(GetIPFromRequest(r))

		expiresAfter := s.Spec.ExpireAnalyticsAfter
		if config.EnforceOrgDataAge {
			thisOrg := s.Spec.OrgID
			orgExpireDataTime := s.GetOrgSessionExpiry(thisOrg)

			if orgExpireDataTime > 0 {
				expiresAfter = orgExpireDataTime
			}
		}

		thisRecord.SetExpiry(expiresAfter)

		if config.AnalyticsConfig.NormaliseUrls.Enabled {
			thisRecord.NormalisePath()
		}

		go analytics.RecordHit(thisRecord)
	}

	// Report in health check
	ReportHealthCheckValue(s.Spec.Health, RequestLog, strconv.FormatInt(int64(timing), 10))

	if doMemoryProfile {
		pprof.WriteHeapProfile(profileFile)
	}

	context.Clear(r)
}

// ServeHTTP will store the request details in the analytics store if necessary and proxy the request to it's
// final destination, this is invoked by the ProxyHandler or right at the start of a request chain if the URL
// Spec states the path is Ignored
func (s SuccessHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) *http.Response {
	log.Debug("Started proxy")
	// Make sure we get the correct target URL
	if s.Spec.APIDefinition.Proxy.StripListenPath {
		log.Debug("Stripping: ", s.Spec.Proxy.ListenPath)
		r.URL.Path = strings.Replace(r.URL.Path, s.Spec.Proxy.ListenPath, "", 1)
		log.Debug("Upstream Path is: ", r.URL.Path)
	}

	var copiedRequest *http.Request
	if RecordDetail(r) {
		copiedRequest = CopyHttpRequest(r)
	}

	t1 := time.Now()
	resp := s.Proxy.ServeHTTP(w, r)
	t2 := time.Now()

	millisec := float64(t2.UnixNano()-t1.UnixNano()) * 0.000001
	log.Debug("Upstream request took (ms): ", millisec)

	if resp != nil {
		var copiedResponse *http.Response
		if RecordDetail(r) {
			copiedResponse = CopyHttpResponse(resp)
		}
		s.RecordHit(w, r, int64(millisec), resp.StatusCode, copiedRequest, copiedResponse)
	}
	log.Debug("Done proxy")
	return nil
}

// ServeHTTPWithCache will store the request details in the analytics store if necessary and proxy the request to it's
// final destination, this is invoked by the ProxyHandler or right at the start of a request chain if the URL
// Spec states the path is Ignored Itwill also return a response object for the cache
func (s SuccessHandler) ServeHTTPWithCache(w http.ResponseWriter, r *http.Request) *http.Response {
	// Make sure we get the correct target URL
	if s.Spec.APIDefinition.Proxy.StripListenPath {
		r.URL.Path = strings.Replace(r.URL.Path, s.Spec.Proxy.ListenPath, "", 1)
	}

	var copiedRequest *http.Request
	if RecordDetail(r) {
		copiedRequest = CopyHttpRequest(r)
	}

	t1 := time.Now()
	inRes := s.Proxy.ServeHTTPForCache(w, r)
	t2 := time.Now()

	var copiedResponse *http.Response
	if RecordDetail(r) {
		copiedResponse = CopyHttpResponse(inRes)
	}

	millisec := float64(t2.UnixNano()-t1.UnixNano()) * 0.000001
	log.Debug("Upstream request took (ms): ", millisec)

	if inRes != nil {
		s.RecordHit(w, r, int64(millisec), inRes.StatusCode, copiedRequest, copiedResponse)
	}

	return inRes
}
<commit_msg>Append a const for skipping CP auth.<commit_after>package main

import (
	"bytes"
	b64 "encoding/base64"
	"github.com/gorilla/context"
	"github.com/pmylund/go-cache"
	"io"
	"net/http"
	"runtime/pprof"
	"strconv"
	"strings"
	"time"
)

// ContextKey is a key type to avoid collisions
type ContextKey int

// Enums for keys to be stored in a session context - this is how gorilla expects
// these to be implemented and is lifted pretty much from docs
const (
	SessionData       = 0
	AuthHeaderValue   = 1
	VersionData       = 2
	VersionKeyContext = 3
	OrgSessionContext = 4
	ContextData       = 5
	RetainHost        = 6
	SkipCoProcessAuth	= 7
)

var SessionCache *cache.Cache = cache.New(10*time.Second, 5*time.Second)
var ExpiryCache *cache.Cache = cache.New(600*time.Second, 5*time.Second)

type ReturningHttpHandler interface {
	ServeHTTP(http.ResponseWriter, *http.Request) *http.Response
	ServeHTTPForCache(http.ResponseWriter, *http.Request) *http.Response
	CopyResponse(io.Writer, io.Reader)
	New(interface{}, *APISpec) (TykResponseHandler, error)
}

// TykMiddleware wraps up the ApiSpec and Proxy objects to be included in a
// middleware handler, this can probably be handled better.
type TykMiddleware struct {
	Spec  *APISpec
	Proxy ReturningHttpHandler
}

func SetUpSessionCache() *cache.Cache {
	sessionLength := 10
	evictionTime := 5
	if config.LocalSessionCache.CachedSessionTimeout > 0 {
		sessionLength = config.LocalSessionCache.CachedSessionTimeout
	}
	if config.LocalSessionCache.CacheSessionEviction > 0 {
		evictionTime = config.LocalSessionCache.CacheSessionEviction
	}

	return cache.New(time.Duration(sessionLength)*time.Second, time.Duration(evictionTime)*time.Second)
}

func (t TykMiddleware) GetOrgSession(key string) (SessionState, bool) {
	// Try and get the session from the session store
	var thisSession SessionState
	var found bool

	thisSession, found = t.Spec.OrgSessionManager.GetSessionDetail(key)
	if found {
		// If exists, assume it has been authorized and pass on
		if config.EnforceOrgDataAge {
			// We cache org expiry data
			log.Debug("Setting data expiry: ", thisSession.OrgID)
			go t.SetOrgExpiry(thisSession.OrgID, thisSession.DataExpires)
		}
		return thisSession, true
	}

	return thisSession, found
}

func (t TykMiddleware) SetOrgExpiry(orgid string, expiry int64) {
	ExpiryCache.Set(orgid, expiry, cache.DefaultExpiration)
}

func (t TykMiddleware) GetOrgSessionExpiry(orgid string) int64 {
	log.Debug("Checking: ", orgid)
	cachedVal, found := ExpiryCache.Get(orgid)
	if !found {
		log.Debug("no cached entry found, returning 7 days")
		return 604800
	}

	return cachedVal.(int64)
}

// ApplyPolicyIfExists will check if a policy is loaded, if it is, it will overwrite the session state to use the policy values
func (t TykMiddleware) ApplyPolicyIfExists(key string, thisSession *SessionState) {
	if thisSession.ApplyPolicyID != "" {
		log.Debug("Session has policy, checking")
		policy, ok := Policies[thisSession.ApplyPolicyID]
		if ok {
			// Check ownership, policy org owner must be the same as API,
			// otherwise youcould overwrite a session key with a policy from a different org!
			if policy.OrgID != t.Spec.APIDefinition.OrgID {
				log.Error("Attempting to apply policy from different organisation to key, skipping")
				return
			}

			log.Debug("Found policy, applying")

			if policy.Partitions.Quota || policy.Partitions.RateLimit || policy.Partitions.Acl {
				// This is a partitioned policy, only apply what is active
				log.Debug("Applying partitioned policy")

				if policy.Partitions.Quota {
					// Quotas
					log.Debug("Applying partition: Quota")
					thisSession.QuotaMax = policy.QuotaMax
					thisSession.QuotaRenewalRate = policy.QuotaRenewalRate
				}

				if policy.Partitions.RateLimit {
					// Rate limting
					log.Debug("Applying partition: Rate Limit")
					thisSession.Allowance = policy.Rate // This is a legacy thing, merely to make sure output is consistent. Needs to be purged
					thisSession.Rate = policy.Rate
					thisSession.Per = policy.Per
					if policy.LastUpdated != "" {
						thisSession.LastUpdated = policy.LastUpdated
					}
				}

				if policy.Partitions.Acl {
					// ACL
					log.Debug("Applying partition: ACL")
					thisSession.AccessRights = policy.AccessRights
					thisSession.HMACEnabled = policy.HMACEnabled
				}

			} else {
				// This is not a partitioned policy, apply everything
				log.Debug("Applying regular policy")
				// Quotas
				thisSession.QuotaMax = policy.QuotaMax
				thisSession.QuotaRenewalRate = policy.QuotaRenewalRate

				// Rate limting
				thisSession.Allowance = policy.Rate // This is a legacy thing, merely to make sure output is consistent. Needs to be purged
				thisSession.Rate = policy.Rate
				thisSession.Per = policy.Per
				if policy.LastUpdated != "" {
					thisSession.LastUpdated = policy.LastUpdated
				}

				// ACL
				thisSession.AccessRights = policy.AccessRights
				thisSession.HMACEnabled = policy.HMACEnabled
			}

			// Required for all
			thisSession.IsInactive = policy.IsInactive
			thisSession.Tags = policy.Tags

			log.Debug("Policy Applied, Access rights are: ", thisSession.AccessRights)
			log.Debug("Policy Applied, Access rights were: ", policy.AccessRights)

			// Update the session in the session manager in case it gets called again
			t.Spec.SessionManager.UpdateSession(key, *thisSession, t.Spec.APIDefinition.SessionLifetime)
			log.Debug("Policy applied to key")
		}
	}
}

// CheckSessionAndIdentityForValidKey will check first the Session store for a valid key, if not found, it will try
// the Auth Handler, if not found it will fail
func (t TykMiddleware) CheckSessionAndIdentityForValidKey(key string) (SessionState, bool) {
	// Try and get the session from the session store
	var thisSession SessionState
	var found bool

	// Check in-memory cache
	if !config.LocalSessionCache.DisableCacheSessionState {
		cachedVal, found := SessionCache.Get(key)
		if found {
			log.Debug("Key found in local cache")
			thisSession = cachedVal.(SessionState)
			t.ApplyPolicyIfExists(key, &thisSession)
			return thisSession, true
		}
	}

	// Check session store
	log.Debug("Querying keystore")
	thisSession, found = t.Spec.SessionManager.GetSessionDetail(key)
	if found {
		// If exists, assume it has been authorized and pass on
		// cache it
		go SessionCache.Set(key, thisSession, cache.DefaultExpiration)

		// Check for a policy, if there is a policy, pull it and overwrite the session values
		t.ApplyPolicyIfExists(key, &thisSession)
		log.Debug("Got key")
		return thisSession, true
	}

	// 2. If not there, get it from the AuthorizationHandler
	thisSession, found = t.Spec.AuthManager.IsKeyAuthorised(key)
	if found {
		// If not in Session, and got it from AuthHandler, create a session with a new TTL
		log.Info("Recreating session for key: ", key)

		// cache it
		go SessionCache.Set(key, thisSession, cache.DefaultExpiration)

		// Check for a policy, if there is a policy, pull it and overwrite the session values
		t.ApplyPolicyIfExists(key, &thisSession)
		t.Spec.SessionManager.UpdateSession(key, thisSession, t.Spec.APIDefinition.SessionLifetime)
	}

	return thisSession, found
}

// SuccessHandler represents the final ServeHTTP() request for a proxied API request
type SuccessHandler struct {
	*TykMiddleware
}

func (s SuccessHandler) RecordHit(w http.ResponseWriter, r *http.Request, timing int64, code int, requestCopy *http.Request, responseCopy *http.Response) {

	if s.Spec.DoNotTrack {
		return
	}

	if config.StoreAnalytics(r) {

		t := time.Now()

		// Track the key ID if it exists
		authHeaderValue := context.Get(r, AuthHeaderValue)
		keyName := ""
		if authHeaderValue != nil {
			keyName = authHeaderValue.(string)
		}

		// Track version data
		version := s.Spec.getVersionFromRequest(r)
		if version == "" {
			version = "Non Versioned"
		}

		// If OAuth, we need to grab it from the session, which may or may not exist
		OauthClientID := ""
		tags := make([]string, 0)
		var alias string
		thisSessionState := context.Get(r, SessionData)

		if thisSessionState != nil {
			OauthClientID = thisSessionState.(SessionState).OauthClientID
			tags = thisSessionState.(SessionState).Tags
			alias = thisSessionState.(SessionState).Alias
		}

		rawRequest := ""
		rawResponse := ""
		if RecordDetail(r) {
			if requestCopy != nil {
				// Get the wire format representation
				var wireFormatReq bytes.Buffer
				requestCopy.Write(&wireFormatReq)
				rawRequest = b64.StdEncoding.EncodeToString(wireFormatReq.Bytes())
			}
			if responseCopy != nil {
				// Get the wire format representation
				var wireFormatRes bytes.Buffer
				responseCopy.Write(&wireFormatRes)
				rawResponse = b64.StdEncoding.EncodeToString(wireFormatRes.Bytes())
			}
		}

		thisRecord := AnalyticsRecord{
			r.Method,
			r.URL.Path,
			r.URL.Path,
			r.ContentLength,
			r.Header.Get("User-Agent"),
			t.Day(),
			t.Month(),
			t.Year(),
			t.Hour(),
			code,
			keyName,
			t,
			version,
			s.Spec.APIDefinition.Name,
			s.Spec.APIDefinition.APIID,
			s.Spec.APIDefinition.OrgID,
			OauthClientID,
			timing,
			rawRequest,
			rawResponse,
			GetIPFromRequest(r),
			GeoData{},
			tags,
			alias,
			time.Now(),
		}

		thisRecord.GetGeo(GetIPFromRequest(r))

		expiresAfter := s.Spec.ExpireAnalyticsAfter
		if config.EnforceOrgDataAge {
			thisOrg := s.Spec.OrgID
			orgExpireDataTime := s.GetOrgSessionExpiry(thisOrg)

			if orgExpireDataTime > 0 {
				expiresAfter = orgExpireDataTime
			}
		}

		thisRecord.SetExpiry(expiresAfter)

		if config.AnalyticsConfig.NormaliseUrls.Enabled {
			thisRecord.NormalisePath()
		}

		go analytics.RecordHit(thisRecord)
	}

	// Report in health check
	ReportHealthCheckValue(s.Spec.Health, RequestLog, strconv.FormatInt(int64(timing), 10))

	if doMemoryProfile {
		pprof.WriteHeapProfile(profileFile)
	}

	context.Clear(r)
}

// ServeHTTP will store the request details in the analytics store if necessary and proxy the request to it's
// final destination, this is invoked by the ProxyHandler or right at the start of a request chain if the URL
// Spec states the path is Ignored
func (s SuccessHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) *http.Response {
	log.Debug("Started proxy")
	// Make sure we get the correct target URL
	if s.Spec.APIDefinition.Proxy.StripListenPath {
		log.Debug("Stripping: ", s.Spec.Proxy.ListenPath)
		r.URL.Path = strings.Replace(r.URL.Path, s.Spec.Proxy.ListenPath, "", 1)
		log.Debug("Upstream Path is: ", r.URL.Path)
	}

	var copiedRequest *http.Request
	if RecordDetail(r) {
		copiedRequest = CopyHttpRequest(r)
	}

	t1 := time.Now()
	resp := s.Proxy.ServeHTTP(w, r)
	t2 := time.Now()

	millisec := float64(t2.UnixNano()-t1.UnixNano()) * 0.000001
	log.Debug("Upstream request took (ms): ", millisec)

	if resp != nil {
		var copiedResponse *http.Response
		if RecordDetail(r) {
			copiedResponse = CopyHttpResponse(resp)
		}
		s.RecordHit(w, r, int64(millisec), resp.StatusCode, copiedRequest, copiedResponse)
	}
	log.Debug("Done proxy")
	return nil
}

// ServeHTTPWithCache will store the request details in the analytics store if necessary and proxy the request to it's
// final destination, this is invoked by the ProxyHandler or right at the start of a request chain if the URL
// Spec states the path is Ignored Itwill also return a response object for the cache
func (s SuccessHandler) ServeHTTPWithCache(w http.ResponseWriter, r *http.Request) *http.Response {
	// Make sure we get the correct target URL
	if s.Spec.APIDefinition.Proxy.StripListenPath {
		r.URL.Path = strings.Replace(r.URL.Path, s.Spec.Proxy.ListenPath, "", 1)
	}

	var copiedRequest *http.Request
	if RecordDetail(r) {
		copiedRequest = CopyHttpRequest(r)
	}

	t1 := time.Now()
	inRes := s.Proxy.ServeHTTPForCache(w, r)
	t2 := time.Now()

	var copiedResponse *http.Response
	if RecordDetail(r) {
		copiedResponse = CopyHttpResponse(inRes)
	}

	millisec := float64(t2.UnixNano()-t1.UnixNano()) * 0.000001
	log.Debug("Upstream request took (ms): ", millisec)

	if inRes != nil {
		s.RecordHit(w, r, int64(millisec), inRes.StatusCode, copiedRequest, copiedResponse)
	}

	return inRes
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"fmt"
	"os/exec"
	"syscall"
	"path/filepath"
	"log"
	"flag"
	"bytes"
	"code.google.com/p/go.crypto/ssh/terminal"
)

func main() {
	var parallel = flag.Bool("p", false, "Run hooks in parallel")
	var trace = flag.Bool("x", false, "Trace mode")
	flag.Parse()

	if len(os.Getenv("PLUGINHOOK_TRACE")) > 0 {
		*trace = true
	}

	pluginPath := os.Getenv("PLUGIN_PATH")
	if pluginPath == "" {
		log.Fatal("[ERROR] Unable to locate plugins: set $PLUGIN_PATH\n")
		os.Exit(1)
	}
	if flag.NArg() < 1 {
		log.Fatal("[ERROR] Hook name argument is required\n")
		os.Exit(1)
	}
	cmds := make([]exec.Cmd, 0)
	var matches, _ = filepath.Glob(fmt.Sprintf("%s/*/%s", pluginPath, flag.Arg(0)))
	for _, hook := range matches {
		cmd := exec.Command(hook, flag.Args()[1:]...)
		cmds = append(cmds, *cmd)
	}
	for i := len(cmds)-1; i >= 0; i-- {
		cmds[i].Stderr = os.Stderr

		if i == len(cmds)-1 {
			cmds[i].Stdout = os.Stdout
		}
		if i > 0 {
			if *parallel {
				stdout, err := cmds[i-1].StdoutPipe()
				if err != nil {
					log.Fatal(err)
				}
				cmds[i].Stdin = stdout
			} else {
				var buf bytes.Buffer
				cmds[i-1].Stdout = &buf
				cmds[i].Stdin = &buf
			}
		}
		if i == 0 && !terminal.IsTerminal(syscall.Stdin) {
			cmds[i].Stdin = os.Stdin
		}
	}

	if *parallel {
		done := make(chan bool, len(cmds))

		for i := 0; i < len(cmds); i++ {
			go func(cmd exec.Cmd) {
				if *trace {
					fmt.Fprintln(os.Stderr, "Executing : ", cmd.Args)
				}
				err := cmd.Run()
				if msg, ok := err.(*exec.ExitError); ok { // there is error code
					os.Exit(msg.Sys().(syscall.WaitStatus).ExitStatus())
				}
				done <- true
			}(cmds[i])
		}
		for i := 0; i < len(cmds); i++ {
			<-done
		}
	} else {
		for i := 0; i < len(cmds); i++ {
			if *trace {
				fmt.Fprintln(os.Stderr, "Executing : ", cmds[i].Args)
			}
			err := cmds[i].Run()
			if msg, ok := err.(*exec.ExitError); ok { // there is error code
				os.Exit(msg.Sys().(syscall.WaitStatus).ExitStatus())
			}
		}
	}
}
<commit_msg>Trace output more similar to bash's -x<commit_after>package main

import (
	"os"
	"fmt"
	"os/exec"
	"syscall"
	"path/filepath"
	"log"
	"flag"
	"bytes"
	"strings"
	"code.google.com/p/go.crypto/ssh/terminal"
)

func main() {
	var parallel = flag.Bool("p", false, "Run hooks in parallel")
	var trace = flag.Bool("x", false, "Trace mode")
	flag.Parse()

	if len(os.Getenv("PLUGINHOOK_TRACE")) > 0 {
		*trace = true
	}

	pluginPath := os.Getenv("PLUGIN_PATH")
	if pluginPath == "" {
		log.Fatal("[ERROR] Unable to locate plugins: set $PLUGIN_PATH\n")
		os.Exit(1)
	}
	if flag.NArg() < 1 {
		log.Fatal("[ERROR] Hook name argument is required\n")
		os.Exit(1)
	}
	cmds := make([]exec.Cmd, 0)
	var matches, _ = filepath.Glob(fmt.Sprintf("%s/*/%s", pluginPath, flag.Arg(0)))
	for _, hook := range matches {
		cmd := exec.Command(hook, flag.Args()[1:]...)
		cmds = append(cmds, *cmd)
	}
	for i := len(cmds)-1; i >= 0; i-- {
		cmds[i].Stderr = os.Stderr

		if i == len(cmds)-1 {
			cmds[i].Stdout = os.Stdout
		}
		if i > 0 {
			if *parallel {
				stdout, err := cmds[i-1].StdoutPipe()
				if err != nil {
					log.Fatal(err)
				}
				cmds[i].Stdin = stdout
			} else {
				var buf bytes.Buffer
				cmds[i-1].Stdout = &buf
				cmds[i].Stdin = &buf
			}
		}
		if i == 0 && !terminal.IsTerminal(syscall.Stdin) {
			cmds[i].Stdin = os.Stdin
		}
	}

	if *parallel {
		done := make(chan bool, len(cmds))

		for i := 0; i < len(cmds); i++ {
			go func(cmd exec.Cmd) {
				if *trace {
					fmt.Fprintln(os.Stderr, "+", strings.Join(cmds[i].Args, " "))
				}
				err := cmd.Run()
				if msg, ok := err.(*exec.ExitError); ok { // there is error code
					os.Exit(msg.Sys().(syscall.WaitStatus).ExitStatus())
				}
				done <- true
			}(cmds[i])
		}
		for i := 0; i < len(cmds); i++ {
			<-done
		}
	} else {
		for i := 0; i < len(cmds); i++ {
			if *trace {
				fmt.Fprintln(os.Stderr, "+", strings.Join(cmds[i].Args, " "))
			}
			err := cmds[i].Run()
			if msg, ok := err.(*exec.ExitError); ok { // there is error code
				os.Exit(msg.Sys().(syscall.WaitStatus).ExitStatus())
			}
		}
	}
}
<|endoftext|>


<commit_before>package rsa

import (
	"fmt"
	"godot/pkcs1"
	"godot/rand"
	"godot/rsa/pss"
	"godot/usage"
	"godot/util"
	"godot/x509"
	"math/big"
	"os"
)

func createRSA(l int) *pkcs1.RSAPrivateKey {
	var rsa = new(pkcs1.RSAPrivateKey)

	rsa.Version = big.NewInt(0)
	rsa.Prime1 = rand.Prime(l/2) // prime p
	rsa.Prime2 = rand.Prime(l/2) // prime q
	rsa.Modulus = new(big.Int).Mul(rsa.Prime1, rsa.Prime2)
	rsa.PublicExponent = big.NewInt(65537)

	pMinus := new(big.Int).Sub(rsa.Prime1, big.NewInt(1))
	qMinus := new(big.Int).Sub(rsa.Prime1, big.NewInt(1))
	phi := new(big.Int).Mul(pMinus, qMinus)

	rsa.PrivateExponent = new(big.Int).ModInverse(rsa.PublicExponent, phi)
	// CRT auxiliary parameters
	rsa.Exponent1 = new(big.Int).Mod(rsa.PrivateExponent, pMinus)
	rsa.Exponent2 = new(big.Int).Mod(rsa.PrivateExponent, qMinus)
	rsa.Coefficient = new(big.Int).ModInverse(rsa.Prime2, rsa.Prime1)

	return rsa
}

// getArg() retrieves a token from 'args' at index i + 1. The token must exist.
func getArg(args []string, i *int) string {
	if *i + 1 >= len(args) {
		usage.Print();
		os.Exit(1);
	}
	*i++;

	return args[*i]
}

func Verify(args []string) {
	var in, key *os.File

	for i := 0; i < len(args); i++ {
		switch args[i] {
		case "-i":
			fallthrough
		case "--in":
			util.OpenFile(&in, getArg(args, &i))
		case "-k":
			fallthrough
		case "--key":
			util.OpenFile(&key, getArg(args, &i))
		default:
			usage.Print();
			os.Exit(1);
		}
	}

	if key == nil {
		usage.Print();
		os.Exit(1);
	}
	if in == nil {
		in = os.Stdin
	}
}

func Sign(args []string) {
	var in, out, key *os.File

	for i := 0; i < len(args); i++ {
		switch args[i] {
		case "-i":
			fallthrough
		case "--in":
			util.OpenFile(&in, getArg(args, &i))
		case "-k":
			fallthrough
		case "--key":
			util.OpenKey(&key, getArg(args, &i))
		case "-o":
			fallthrough
		case "--out":
			util.CreateFile(&out, getArg(args, &i))
		default:
			usage.Print();
			os.Exit(1);
		}
	}

	if key == nil {
		usage.Print();
		os.Exit(1);
	}
	if in == nil {
		in = os.Stdin
	}
	if out == nil {
		out = os.Stdout
	}

	rsa := pkcs1.ReadRSA(key)
	if len(rsa.Modulus.Bytes()) != 512 ||
	   len(rsa.PrivateExponent.Bytes()) != 512 {
		fmt.Fprintf(os.Stderr, "%s: invalid key size\n", args[1])
		os.Exit(1)
	}

	m := pss.Encode(in, 4095)
	out.Write(new(big.Int).Exp(m, rsa.PrivateExponent, rsa.Modulus).Bytes())

	util.CloseFile(in)
	util.CloseFile(out)
}

func Pub(args []string) {
	var in, out *os.File

	for i := 0; i < len(args); i++ {
		switch args[i] {
		case "-i":
			fallthrough
		case "--in":
			util.OpenKey(&in, getArg(args, &i))
		case "-o":
			fallthrough
		case "--out":
			util.CreateFile(&out, getArg(args, &i))
		default:
			usage.Print();
			os.Exit(1);
		}
	}

	if in == nil {
		in = os.Stdin
	}
	if out == nil {
		out = os.Stdout
	}

	x509.WriteRSA(pkcs1.ReadRSA(in), out)
	util.CloseFile(in);
	util.CloseFile(out);
}

func New(args []string) {
	var out *os.File

	for i := 0; i < len(args); i++ {
		switch args[i] {
		case "-o":
			fallthrough
		case "--out":
			util.CreateFile(&out, getArg(args, &i))
		default:
			usage.Print();
			os.Exit(1);
		}
	}

	if out == nil {
		out = os.Stdout
	}

	pkcs1.WriteRSA(createRSA(4096), out)
	util.CloseFile(out)
}
<commit_msg>use GetArg() from util module<commit_after>package rsa

import (
	"fmt"
	"godot/pkcs1"
	"godot/rand"
	"godot/rsa/pss"
	"godot/usage"
	"godot/util"
	"godot/x509"
	"math/big"
	"os"
)

func createRSA(l int) *pkcs1.RSAPrivateKey {
	var rsa = new(pkcs1.RSAPrivateKey)

	rsa.Version = big.NewInt(0)
	rsa.Prime1 = rand.Prime(l/2) // prime p
	rsa.Prime2 = rand.Prime(l/2) // prime q
	rsa.Modulus = new(big.Int).Mul(rsa.Prime1, rsa.Prime2)
	rsa.PublicExponent = big.NewInt(65537)

	pMinus := new(big.Int).Sub(rsa.Prime1, big.NewInt(1))
	qMinus := new(big.Int).Sub(rsa.Prime1, big.NewInt(1))
	phi := new(big.Int).Mul(pMinus, qMinus)

	rsa.PrivateExponent = new(big.Int).ModInverse(rsa.PublicExponent, phi)
	// CRT auxiliary parameters
	rsa.Exponent1 = new(big.Int).Mod(rsa.PrivateExponent, pMinus)
	rsa.Exponent2 = new(big.Int).Mod(rsa.PrivateExponent, qMinus)
	rsa.Coefficient = new(big.Int).ModInverse(rsa.Prime2, rsa.Prime1)

	return rsa
}

func Verify(args []string) {
	var in, key *os.File

	for i := 0; i < len(args); i++ {
		switch args[i] {
		case "-i":
			fallthrough
		case "--in":
			util.OpenFile(&in, util.GetArg(args, &i))
		case "-k":
			fallthrough
		case "--key":
			util.OpenFile(&key, util.GetArg(args, &i))
		default:
			usage.Print();
			os.Exit(1);
		}
	}

	if key == nil {
		usage.Print();
		os.Exit(1);
	}
	if in == nil {
		in = os.Stdin
	}
}

func Sign(args []string) {
	var in, out, key *os.File

	for i := 0; i < len(args); i++ {
		switch args[i] {
		case "-i":
			fallthrough
		case "--in":
			util.OpenFile(&in, util.GetArg(args, &i))
		case "-k":
			fallthrough
		case "--key":
			util.OpenKey(&key, util.GetArg(args, &i))
		case "-o":
			fallthrough
		case "--out":
			util.CreateFile(&out, util.GetArg(args, &i))
		default:
			usage.Print();
			os.Exit(1);
		}
	}

	if key == nil {
		usage.Print();
		os.Exit(1);
	}
	if in == nil {
		in = os.Stdin
	}
	if out == nil {
		out = os.Stdout
	}

	rsa := pkcs1.ReadRSA(key)
	if len(rsa.Modulus.Bytes()) != 512 ||
	   len(rsa.PrivateExponent.Bytes()) != 512 {
		fmt.Fprintf(os.Stderr, "%s: invalid key size\n", args[1])
		os.Exit(1)
	}

	m := pss.Encode(in, 4095)
	out.Write(new(big.Int).Exp(m, rsa.PrivateExponent, rsa.Modulus).Bytes())

	util.CloseFile(in)
	util.CloseFile(out)
}

func Pub(args []string) {
	var in, out *os.File

	for i := 0; i < len(args); i++ {
		switch args[i] {
		case "-i":
			fallthrough
		case "--in":
			util.OpenKey(&in, util.GetArg(args, &i))
		case "-o":
			fallthrough
		case "--out":
			util.CreateFile(&out, util.GetArg(args, &i))
		default:
			usage.Print();
			os.Exit(1);
		}
	}

	if in == nil {
		in = os.Stdin
	}
	if out == nil {
		out = os.Stdout
	}

	x509.WriteRSA(pkcs1.ReadRSA(in), out)
	util.CloseFile(in);
	util.CloseFile(out);
}

func New(args []string) {
	var out *os.File

	for i := 0; i < len(args); i++ {
		switch args[i] {
		case "-o":
			fallthrough
		case "--out":
			util.CreateFile(&out, util.GetArg(args, &i))
		default:
			usage.Print();
			os.Exit(1);
		}
	}

	if out == nil {
		out = os.Stdout
	}

	pkcs1.WriteRSA(createRSA(4096), out)
	util.CloseFile(out)
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cachingfs_test

import (
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"
	"testing"
	"time"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/samples/cachingfs"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
)

func TestHelloFS(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type CachingFSTest struct {
	dir string
	mfs *fuse.MountedFileSystem
}

var _ TearDownInterface = &CachingFSTest{}

func (t *CachingFSTest) setUp(
	lookupEntryTimeout time.Duration,
	getattrTimeout time.Duration) {
	var err error

	// Set up a temporary directory for mounting.
	t.dir, err = ioutil.TempDir("", "caching_fs_test")
	AssertEq(nil, err)

	// Create a file system.
	fs, err := cachingfs.NewCachingFS(lookupEntryTimeout, getattrTimeout)
	AssertEq(nil, err)

	// Mount it.
	t.mfs, err = fuse.Mount(t.dir, fs)
	AssertEq(nil, err)

	err = t.mfs.WaitForReady(context.Background())
	AssertEq(nil, err)
}

func (t *CachingFSTest) TearDown() {
	// Was the file system mounted?
	if t.mfs == nil {
		return
	}

	// Unmount the file system. Try again on "resource busy" errors.
	delay := 10 * time.Millisecond
	for {
		err := t.mfs.Unmount()
		if err == nil {
			break
		}

		if strings.Contains(err.Error(), "resource busy") {
			log.Println("Resource busy error while unmounting; trying again")
			time.Sleep(delay)
			delay = time.Duration(1.3 * float64(delay))
			continue
		}

		panic("MountedFileSystem.Unmount: " + err.Error())
	}

	if err := t.mfs.Join(context.Background()); err != nil {
		panic("MountedFileSystem.Join: " + err.Error())
	}
}

////////////////////////////////////////////////////////////////////////
// Basics
////////////////////////////////////////////////////////////////////////

type BasicsTest struct {
	CachingFSTest
}

var _ SetUpInterface = &BasicsTest{}

func init() { RegisterTestSuite(&BasicsTest{}) }

func (t *BasicsTest) SetUp(ti *TestInfo) {
	const (
		lookupEntryTimeout = 0
		getattrTimeout     = 0
	)

	t.CachingFSTest.setUp(lookupEntryTimeout, getattrTimeout)
}

func (t *BasicsTest) StatNonexistent_Root() {
	names := []string{
		"blah",
		"bar", // Wrong directory
	}

	for _, n := range names {
		_, err := os.Stat(path.Join(t.dir, n))

		AssertNe(nil, err)
		ExpectTrue(os.IsNotExist(err), "n: %s, err: %v", n, err)
	}
}

func (t *BasicsTest) StatNonexistent_Dir() {
	AssertTrue(false, "TODO")
}

func (t *BasicsTest) StatFoo() {
	AssertTrue(false, "TODO")
}

func (t *BasicsTest) StatDir() {
	AssertTrue(false, "TODO")
}

func (t *BasicsTest) StatBar() {
	AssertTrue(false, "TODO")
}

////////////////////////////////////////////////////////////////////////
// No caching
////////////////////////////////////////////////////////////////////////

type NoCachingTest struct {
	CachingFSTest
}

var _ SetUpInterface = &NoCachingTest{}

func init() { RegisterTestSuite(&NoCachingTest{}) }

func (t *NoCachingTest) SetUp(ti *TestInfo) {
	const (
		lookupEntryTimeout = 0
		getattrTimeout     = 0
	)

	t.CachingFSTest.setUp(lookupEntryTimeout, getattrTimeout)
}

func (t *NoCachingTest) StatStat() {
	AssertTrue(false, "TODO")
}

func (t *NoCachingTest) StatRenumberStat() {
	AssertTrue(false, "TODO")
}

func (t *NoCachingTest) StatMtimeStat() {
	AssertTrue(false, "TODO")
}

func (t *NoCachingTest) StatRenumberMtimeStat() {
	AssertTrue(false, "TODO")
}
<commit_msg>BasicsTest.StatNonexistent<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cachingfs_test

import (
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"
	"testing"
	"time"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/samples/cachingfs"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
)

func TestHelloFS(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type CachingFSTest struct {
	dir string
	mfs *fuse.MountedFileSystem
}

var _ TearDownInterface = &CachingFSTest{}

func (t *CachingFSTest) setUp(
	lookupEntryTimeout time.Duration,
	getattrTimeout time.Duration) {
	var err error

	// Set up a temporary directory for mounting.
	t.dir, err = ioutil.TempDir("", "caching_fs_test")
	AssertEq(nil, err)

	// Create a file system.
	fs, err := cachingfs.NewCachingFS(lookupEntryTimeout, getattrTimeout)
	AssertEq(nil, err)

	// Mount it.
	t.mfs, err = fuse.Mount(t.dir, fs)
	AssertEq(nil, err)

	err = t.mfs.WaitForReady(context.Background())
	AssertEq(nil, err)
}

func (t *CachingFSTest) TearDown() {
	// Was the file system mounted?
	if t.mfs == nil {
		return
	}

	// Unmount the file system. Try again on "resource busy" errors.
	delay := 10 * time.Millisecond
	for {
		err := t.mfs.Unmount()
		if err == nil {
			break
		}

		if strings.Contains(err.Error(), "resource busy") {
			log.Println("Resource busy error while unmounting; trying again")
			time.Sleep(delay)
			delay = time.Duration(1.3 * float64(delay))
			continue
		}

		panic("MountedFileSystem.Unmount: " + err.Error())
	}

	if err := t.mfs.Join(context.Background()); err != nil {
		panic("MountedFileSystem.Join: " + err.Error())
	}
}

////////////////////////////////////////////////////////////////////////
// Basics
////////////////////////////////////////////////////////////////////////

type BasicsTest struct {
	CachingFSTest
}

var _ SetUpInterface = &BasicsTest{}

func init() { RegisterTestSuite(&BasicsTest{}) }

func (t *BasicsTest) SetUp(ti *TestInfo) {
	const (
		lookupEntryTimeout = 0
		getattrTimeout     = 0
	)

	t.CachingFSTest.setUp(lookupEntryTimeout, getattrTimeout)
}

func (t *BasicsTest) StatNonexistent() {
	names := []string{
		"blah",
		"bar",
		"dir/blah",
		"dir/dir",
		"dir/foo",
	}

	for _, n := range names {
		_, err := os.Stat(path.Join(t.dir, n))

		AssertNe(nil, err)
		ExpectTrue(os.IsNotExist(err), "n: %s, err: %v", n, err)
	}
}

func (t *BasicsTest) StatFoo() {
	AssertTrue(false, "TODO")
}

func (t *BasicsTest) StatDir() {
	AssertTrue(false, "TODO")
}

func (t *BasicsTest) StatBar() {
	AssertTrue(false, "TODO")
}

////////////////////////////////////////////////////////////////////////
// No caching
////////////////////////////////////////////////////////////////////////

type NoCachingTest struct {
	CachingFSTest
}

var _ SetUpInterface = &NoCachingTest{}

func init() { RegisterTestSuite(&NoCachingTest{}) }

func (t *NoCachingTest) SetUp(ti *TestInfo) {
	const (
		lookupEntryTimeout = 0
		getattrTimeout     = 0
	)

	t.CachingFSTest.setUp(lookupEntryTimeout, getattrTimeout)
}

func (t *NoCachingTest) StatStat() {
	AssertTrue(false, "TODO")
}

func (t *NoCachingTest) StatRenumberStat() {
	AssertTrue(false, "TODO")
}

func (t *NoCachingTest) StatMtimeStat() {
	AssertTrue(false, "TODO")
}

func (t *NoCachingTest) StatRenumberMtimeStat() {
	AssertTrue(false, "TODO")
}
<|endoftext|>


<commit_before>package hocon

import (
	"bytes"
	"fmt"
	"strings"
)

const (
	HoconNotInUnquotedKey  = "$\"{}[]:=,#`^?!@*&\\."
	HoconNotInUnquotedText = "$\"{}[]:=,#`^?!@*&\\"
)

type Tokenizer struct {
	text       string
	index      int
	indexStack *Stack
}

func NewTokenizer(text string) *Tokenizer {
	return &Tokenizer{
		indexStack: NewStack(),
		text:       text,
	}
}

func (p *Tokenizer) Push() {
	p.indexStack.Push(p.index)
}

func (p *Tokenizer) Pop() {
	index, err := p.indexStack.Pop()
	if err != nil {
		panic(err)
	}
	p.index = index
}

func (p *Tokenizer) EOF() bool {
	return p.index >= len(p.text)
}

func (p *Tokenizer) Matches(pattern string) bool {

	if len(pattern)+p.index > len(p.text) {
		return false
	}

	selected := string(p.text[p.index : p.index+len(pattern)])

	if selected == pattern {
		return true
	}

	return false
}

func (p *Tokenizer) MatchesMore(patterns []string) bool {
	for _, pattern := range patterns {
		if len(pattern)+p.index >= len(p.text) {
			continue
		}

		if string(p.text[p.index:p.index+len(pattern)]) == pattern {
			return true
		}
	}
	return false
}

func (p *Tokenizer) Take(length int) string {
	if p.index+length > len(p.text) {
		return ""
	}

	str := string(p.text[p.index : p.index+length])
	p.index += length
	return str
}

func (p *Tokenizer) Peek() byte {
	if p.EOF() {
		return 0
	}

	return p.text[p.index]
}

func (p *Tokenizer) TakeOne() byte {
	if p.EOF() {
		return 0
	}

	b := p.text[p.index]
	p.index += 1
	return b
}

func (p *Tokenizer) PullWhitespace() {
	for !p.EOF() && isWhitespace(p.Peek()) {
		p.TakeOne()
	}
}

type HoconTokenizer struct {
	*Tokenizer
}

func NewHoconTokenizer(text string) *HoconTokenizer {
	return &HoconTokenizer{NewTokenizer(text)}
}

func (p *HoconTokenizer) PullWhitespaceAndComments() {
	for {
		p.PullWhitespace()
		for p.IsStartOfComment() {
			p.PullComment()
		}

		if !p.IsWhitespace() {
			break
		}
	}
}

func (p *HoconTokenizer) PullRestOfLine() string {
	buf := bytes.NewBuffer(nil)

	for !p.EOF() {
		c := p.TakeOne()
		if c == '\n' {
			break
		}

		if c == '\r' {
			continue
		}
		if err := buf.WriteByte(c); err != nil {
			panic(err)
		}
	}

	return strings.TrimSpace(buf.String())
}

func (p *HoconTokenizer) PullNext() *Token {
	p.PullWhitespaceAndComments()

	if p.IsDot() {
		return p.PullDot()
	}

	if p.IsObjectStart() {
		return p.PullStartOfObject()
	}

	if p.IsEndOfObject() {
		return p.PullEndOfObject()
	}

	if p.IsAssignment() {
		return p.PullAssignment()
	}

	if p.IsInclude() {
		return p.PullInclude()
	}

	if p.isStartOfQuotedKey() {
		return p.PullQuotedKey()
	}

	if p.IsUnquotedKeyStart() {
		return p.PullUnquotedKey()
	}

	if p.IsArrayStart() {
		return p.PullArrayStart()
	}

	if p.IsArrayEnd() {
		return p.PullArrayEnd()
	}

	if p.EOF() {
		return NewToken(TokenTypeEoF)
	}

	panic("unknown token")
}

func (p *HoconTokenizer) isStartOfQuotedKey() bool {
	return p.Matches("\"")
}

func (p *HoconTokenizer) PullArrayEnd() *Token {
	p.TakeOne()
	return NewToken(TokenTypeArrayEnd)
}

func (p *HoconTokenizer) IsArrayEnd() bool {
	return p.Matches("]")
}

func (p *HoconTokenizer) IsArrayStart() bool {
	return p.Matches("[")
}

func (p *HoconTokenizer) PullArrayStart() *Token {
	p.TakeOne()
	return NewToken(TokenTypeArrayStart)
}

func (p *HoconTokenizer) PullDot() *Token {
	p.TakeOne()
	return NewToken(TokenTypeDot)
}

func (p *HoconTokenizer) PullComma() *Token {
	p.TakeOne()
	return NewToken(TokenTypeComma)
}

func (p *HoconTokenizer) PullStartOfObject() *Token {
	p.TakeOne()
	return NewToken(TokenTypeObjectStart)
}

func (p *HoconTokenizer) PullEndOfObject() *Token {
	p.TakeOne()
	return NewToken(TokenTypeObjectEnd)
}

func (p *HoconTokenizer) PullAssignment() *Token {
	p.TakeOne()
	return NewToken(TokenTypeAssign)
}

func (p *HoconTokenizer) IsComma() bool {
	return p.Matches(",")
}

func (p *HoconTokenizer) IsDot() bool {
	return p.Matches(".")
}

func (p *HoconTokenizer) IsObjectStart() bool {
	return p.Matches("{")
}

func (p *HoconTokenizer) IsEndOfObject() bool {
	return p.Matches("}")
}

func (p *HoconTokenizer) IsAssignment() bool {
	return p.MatchesMore([]string{"=", ":"})
}

func (p *HoconTokenizer) IsStartOfQuotedText() bool {
	return p.Matches("\"")
}

func (p *HoconTokenizer) IsStartOfTripleQuotedText() bool {
	return p.Matches("\"\"\"")
}

func (p *HoconTokenizer) PullComment() *Token {
	p.PullRestOfLine()
	return NewToken(TokenTypeComment)
}

func (p *HoconTokenizer) PullUnquotedKey() *Token {
	buf := bytes.NewBuffer(nil)
	for !p.EOF() && p.IsUnquotedKey() {
		if err := buf.WriteByte(p.TakeOne()); err != nil {
			panic(err)
		}
	}
	return DefaultToken.Key(strings.TrimSpace(buf.String()))
}

func (p *HoconTokenizer) IsUnquotedKey() bool {
	return !p.EOF() && !p.IsStartOfComment() && (strings.IndexByte(HoconNotInUnquotedKey, p.Peek()) == -1)
}

func (p *HoconTokenizer) IsUnquotedKeyStart() bool {
	return !p.EOF() && !p.IsWhitespace() && !p.IsStartOfComment() && (strings.IndexByte(HoconNotInUnquotedKey, p.Peek()) == -1)
}

func (p *HoconTokenizer) IsWhitespace() bool {
	return isWhitespace(p.Peek())
}

func (p *HoconTokenizer) IsWhitespaceOrComment() bool {
	return p.IsWhitespace() || p.IsStartOfComment()
}

func (p *HoconTokenizer) PullTripleQuotedText() *Token {
	buf := bytes.NewBuffer(nil)
	p.Take(3)
	for !p.EOF() && !p.Matches("\"\"\"") {
		if err := buf.WriteByte(p.Peek()); err != nil {
			panic(err)
		}
		p.TakeOne()
	}
	p.Take(3)
	return DefaultToken.LiteralValue(buf.String())
}

func (p *HoconTokenizer) PullQuotedText() *Token {
	buf := bytes.NewBuffer(nil)
	p.TakeOne()
	for !p.EOF() && !p.Matches("\"") {
		if p.Matches("\\") {
			if _, err := buf.WriteString(p.pullEscapeSequence()); err != nil {
				panic(err)
			}
		} else {
			if err := buf.WriteByte(p.Peek()); err != nil {
				panic(err)
			}
			p.TakeOne()
		}
	}
	p.TakeOne()
	return DefaultToken.LiteralValue(buf.String())
}

func (p *HoconTokenizer) PullQuotedKey() *Token {
	buf := bytes.NewBuffer(nil)
	p.TakeOne()
	for !p.EOF() && !p.Matches("\"") {
		if p.Matches("\\") {
			if _, err := buf.WriteString(p.pullEscapeSequence()); err != nil {
				panic(err)
			}
		} else {
			if err := buf.WriteByte(p.Peek()); err != nil {
				panic(err)
			}
			p.TakeOne()
		}
	}
	p.TakeOne()
	return DefaultToken.Key(buf.String())
}

func (p *HoconTokenizer) PullInclude() *Token {
	p.Take(len("include"))
	p.PullWhitespaceAndComments()
	rest := p.PullQuotedText()
	unQuote := rest.value
	return DefaultToken.Include(unQuote)
}

func (p *HoconTokenizer) pullEscapeSequence() string {
	p.TakeOne()
	escaped := p.TakeOne()
	switch escaped {
	case '"':
		return ("\"")
	case '\\':
		return ("\\")
	case '/':
		return ("/")
	case 'b':
		return ("\b")
	case 'f':
		return ("\f")
	case 'n':
		return ("\n")
	case 'r':
		return ("\r")
	case 't':
		return ("\t")
	case 'u':
		utf8Code := "\\u" + strings.ToLower(p.Take(4))
		utf8Str := ""
		if _, err := fmt.Sscanf(utf8Code, "%s", &utf8Str); err != nil {
			panic(err)
		}
		return utf8Str
	default:
		panic(fmt.Errorf("Unknown escape code: %v", escaped))
	}

	return ""
}

func (p *HoconTokenizer) IsStartOfComment() bool {
	return p.MatchesMore([]string{"#", "//"})
}

func (p *HoconTokenizer) PullValue() *Token {
	if p.IsObjectStart() {
		return p.PullStartOfObject()
	}

	if p.IsStartOfTripleQuotedText() {
		return p.PullTripleQuotedText()
	}

	if p.IsStartOfQuotedText() {
		return p.PullQuotedText()
	}

	if p.isUnquotedText() {
		return p.pullUnquotedText()
	}

	if p.IsArrayStart() {
		return p.PullArrayStart()
	}

	if p.IsArrayEnd() {
		return p.PullArrayEnd()
	}

	if p.IsSubstitutionStart() {
		return p.pullSubstitution()
	}

	panic(fmt.Errorf("Expected value: Null literal, Array, Quoted Text, Unquoted Text, Triple quoted Text, Object or End of array"))
}

func (p *HoconTokenizer) IsSubstitutionStart() bool {
	return p.MatchesMore([]string{"${", "${?"})
}

func (p *HoconTokenizer) IsInclude() bool {
	p.Push()
	defer func() {
		recover()
		p.Pop()
	}()
	if p.Matches("include") {
		p.Take(len("include"))
		if p.IsWhitespaceOrComment() {
			p.PullWhitespaceAndComments()
			if p.IsStartOfQuotedText() {
				p.PullQuotedText()
				return true
			}
		}
	}

	return false
}

func (p *HoconTokenizer) pullSubstitution() *Token {
	buf := bytes.NewBuffer(nil)
	p.Take(2)
	if p.Peek() == '?' {
		p.TakeOne()
	}

	for !p.EOF() && p.isUnquotedText() {
		if err := buf.WriteByte(p.TakeOne()); err != nil {
			panic(err)
		}
	}
	p.TakeOne()
	return DefaultToken.Substitution(buf.String())
}

func (p *HoconTokenizer) IsSpaceOrTab() bool {
	return p.MatchesMore([]string{" ", "\t"})
}

func (p *HoconTokenizer) IsStartSimpleValue() bool {
	if p.IsSpaceOrTab() {
		return true
	}

	if p.isUnquotedText() {
		return true
	}

	return false
}

func (p *HoconTokenizer) PullSpaceOrTab() *Token {
	buf := bytes.NewBuffer(nil)
	for p.IsSpaceOrTab() {
		if err := buf.WriteByte(p.TakeOne()); err != nil {
			panic(err)
		}
	}
	return DefaultToken.LiteralValue(buf.String())
}

func (p *HoconTokenizer) pullUnquotedText() *Token {
	buf := bytes.NewBuffer(nil)
	for !p.EOF() && p.isUnquotedText() {
		if err := buf.WriteByte(p.TakeOne()); err != nil {
			panic(err)
		}
	}
	return DefaultToken.LiteralValue(buf.String())
}

func (p *HoconTokenizer) isUnquotedText() bool {
	return !p.EOF() && !p.IsWhitespace() && !p.IsStartOfComment() && strings.IndexByte(HoconNotInUnquotedText, p.Peek()) == -1
}

func (p *HoconTokenizer) PullSimpleValue() *Token {
	if p.IsSpaceOrTab() {
		return p.PullSpaceOrTab()
	}

	if p.isUnquotedText() {
		return p.pullUnquotedText()
	}
	panic("No simple value found")
}

func (p *HoconTokenizer) isValue() bool {

	if p.IsArrayStart() ||
		p.IsObjectStart() ||
		p.IsStartOfTripleQuotedText() ||
		p.IsSubstitutionStart() ||
		p.IsStartOfQuotedText() ||
		p.isUnquotedText() {
		return true
	}
	return false
}

func isWhitespace(c byte) bool {
	if c == ' ' || c == '\r' || c == '\n' || c == '\t' {
		return true
	}
	return false
}
<commit_msg>improve isWhitespace<commit_after>package hocon

import (
	"bytes"
	"fmt"
	"strings"
)

const (
	HoconNotInUnquotedKey  = "$\"{}[]:=,#`^?!@*&\\."
	HoconNotInUnquotedText = "$\"{}[]:=,#`^?!@*&\\"
)

type Tokenizer struct {
	text       string
	index      int
	indexStack *Stack
}

func NewTokenizer(text string) *Tokenizer {
	return &Tokenizer{
		indexStack: NewStack(),
		text:       text,
	}
}

func (p *Tokenizer) Push() {
	p.indexStack.Push(p.index)
}

func (p *Tokenizer) Pop() {
	index, err := p.indexStack.Pop()
	if err != nil {
		panic(err)
	}
	p.index = index
}

func (p *Tokenizer) EOF() bool {
	return p.index >= len(p.text)
}

func (p *Tokenizer) Matches(pattern string) bool {

	if len(pattern)+p.index > len(p.text) {
		return false
	}

	selected := string(p.text[p.index : p.index+len(pattern)])

	if selected == pattern {
		return true
	}

	return false
}

func (p *Tokenizer) MatchesMore(patterns []string) bool {
	for _, pattern := range patterns {
		if len(pattern)+p.index >= len(p.text) {
			continue
		}

		if string(p.text[p.index:p.index+len(pattern)]) == pattern {
			return true
		}
	}
	return false
}

func (p *Tokenizer) Take(length int) string {
	if p.index+length > len(p.text) {
		return ""
	}

	str := string(p.text[p.index : p.index+length])
	p.index += length
	return str
}

func (p *Tokenizer) Peek() byte {
	if p.EOF() {
		return 0
	}

	return p.text[p.index]
}

func (p *Tokenizer) TakeOne() byte {
	if p.EOF() {
		return 0
	}

	b := p.text[p.index]
	p.index += 1
	return b
}

func (p *Tokenizer) PullWhitespace() {
	for !p.EOF() && isWhitespace(p.Peek()) {
		p.TakeOne()
	}
}

type HoconTokenizer struct {
	*Tokenizer
}

func NewHoconTokenizer(text string) *HoconTokenizer {
	return &HoconTokenizer{NewTokenizer(text)}
}

func (p *HoconTokenizer) PullWhitespaceAndComments() {
	for {
		p.PullWhitespace()
		for p.IsStartOfComment() {
			p.PullComment()
		}

		if !p.IsWhitespace() {
			break
		}
	}
}

func (p *HoconTokenizer) PullRestOfLine() string {
	buf := bytes.NewBuffer(nil)

	for !p.EOF() {
		c := p.TakeOne()
		if c == '\n' {
			break
		}

		if c == '\r' {
			continue
		}
		if err := buf.WriteByte(c); err != nil {
			panic(err)
		}
	}

	return strings.TrimSpace(buf.String())
}

func (p *HoconTokenizer) PullNext() *Token {
	p.PullWhitespaceAndComments()

	if p.IsDot() {
		return p.PullDot()
	}

	if p.IsObjectStart() {
		return p.PullStartOfObject()
	}

	if p.IsEndOfObject() {
		return p.PullEndOfObject()
	}

	if p.IsAssignment() {
		return p.PullAssignment()
	}

	if p.IsInclude() {
		return p.PullInclude()
	}

	if p.isStartOfQuotedKey() {
		return p.PullQuotedKey()
	}

	if p.IsUnquotedKeyStart() {
		return p.PullUnquotedKey()
	}

	if p.IsArrayStart() {
		return p.PullArrayStart()
	}

	if p.IsArrayEnd() {
		return p.PullArrayEnd()
	}

	if p.EOF() {
		return NewToken(TokenTypeEoF)
	}

	panic("unknown token")
}

func (p *HoconTokenizer) isStartOfQuotedKey() bool {
	return p.Matches("\"")
}

func (p *HoconTokenizer) PullArrayEnd() *Token {
	p.TakeOne()
	return NewToken(TokenTypeArrayEnd)
}

func (p *HoconTokenizer) IsArrayEnd() bool {
	return p.Matches("]")
}

func (p *HoconTokenizer) IsArrayStart() bool {
	return p.Matches("[")
}

func (p *HoconTokenizer) PullArrayStart() *Token {
	p.TakeOne()
	return NewToken(TokenTypeArrayStart)
}

func (p *HoconTokenizer) PullDot() *Token {
	p.TakeOne()
	return NewToken(TokenTypeDot)
}

func (p *HoconTokenizer) PullComma() *Token {
	p.TakeOne()
	return NewToken(TokenTypeComma)
}

func (p *HoconTokenizer) PullStartOfObject() *Token {
	p.TakeOne()
	return NewToken(TokenTypeObjectStart)
}

func (p *HoconTokenizer) PullEndOfObject() *Token {
	p.TakeOne()
	return NewToken(TokenTypeObjectEnd)
}

func (p *HoconTokenizer) PullAssignment() *Token {
	p.TakeOne()
	return NewToken(TokenTypeAssign)
}

func (p *HoconTokenizer) IsComma() bool {
	return p.Matches(",")
}

func (p *HoconTokenizer) IsDot() bool {
	return p.Matches(".")
}

func (p *HoconTokenizer) IsObjectStart() bool {
	return p.Matches("{")
}

func (p *HoconTokenizer) IsEndOfObject() bool {
	return p.Matches("}")
}

func (p *HoconTokenizer) IsAssignment() bool {
	return p.MatchesMore([]string{"=", ":"})
}

func (p *HoconTokenizer) IsStartOfQuotedText() bool {
	return p.Matches("\"")
}

func (p *HoconTokenizer) IsStartOfTripleQuotedText() bool {
	return p.Matches("\"\"\"")
}

func (p *HoconTokenizer) PullComment() *Token {
	p.PullRestOfLine()
	return NewToken(TokenTypeComment)
}

func (p *HoconTokenizer) PullUnquotedKey() *Token {
	buf := bytes.NewBuffer(nil)
	for !p.EOF() && p.IsUnquotedKey() {
		if err := buf.WriteByte(p.TakeOne()); err != nil {
			panic(err)
		}
	}
	return DefaultToken.Key(strings.TrimSpace(buf.String()))
}

func (p *HoconTokenizer) IsUnquotedKey() bool {
	return !p.EOF() && !p.IsStartOfComment() && (strings.IndexByte(HoconNotInUnquotedKey, p.Peek()) == -1)
}

func (p *HoconTokenizer) IsUnquotedKeyStart() bool {
	return !p.EOF() && !p.IsWhitespace() && !p.IsStartOfComment() && (strings.IndexByte(HoconNotInUnquotedKey, p.Peek()) == -1)
}

func (p *HoconTokenizer) IsWhitespace() bool {
	return isWhitespace(p.Peek())
}

func (p *HoconTokenizer) IsWhitespaceOrComment() bool {
	return p.IsWhitespace() || p.IsStartOfComment()
}

func (p *HoconTokenizer) PullTripleQuotedText() *Token {
	buf := bytes.NewBuffer(nil)
	p.Take(3)
	for !p.EOF() && !p.Matches("\"\"\"") {
		if err := buf.WriteByte(p.Peek()); err != nil {
			panic(err)
		}
		p.TakeOne()
	}
	p.Take(3)
	return DefaultToken.LiteralValue(buf.String())
}

func (p *HoconTokenizer) PullQuotedText() *Token {
	buf := bytes.NewBuffer(nil)
	p.TakeOne()
	for !p.EOF() && !p.Matches("\"") {
		if p.Matches("\\") {
			if _, err := buf.WriteString(p.pullEscapeSequence()); err != nil {
				panic(err)
			}
		} else {
			if err := buf.WriteByte(p.Peek()); err != nil {
				panic(err)
			}
			p.TakeOne()
		}
	}
	p.TakeOne()
	return DefaultToken.LiteralValue(buf.String())
}

func (p *HoconTokenizer) PullQuotedKey() *Token {
	buf := bytes.NewBuffer(nil)
	p.TakeOne()
	for !p.EOF() && !p.Matches("\"") {
		if p.Matches("\\") {
			if _, err := buf.WriteString(p.pullEscapeSequence()); err != nil {
				panic(err)
			}
		} else {
			if err := buf.WriteByte(p.Peek()); err != nil {
				panic(err)
			}
			p.TakeOne()
		}
	}
	p.TakeOne()
	return DefaultToken.Key(buf.String())
}

func (p *HoconTokenizer) PullInclude() *Token {
	p.Take(len("include"))
	p.PullWhitespaceAndComments()
	rest := p.PullQuotedText()
	unQuote := rest.value
	return DefaultToken.Include(unQuote)
}

func (p *HoconTokenizer) pullEscapeSequence() string {
	p.TakeOne()
	escaped := p.TakeOne()
	switch escaped {
	case '"':
		return ("\"")
	case '\\':
		return ("\\")
	case '/':
		return ("/")
	case 'b':
		return ("\b")
	case 'f':
		return ("\f")
	case 'n':
		return ("\n")
	case 'r':
		return ("\r")
	case 't':
		return ("\t")
	case 'u':
		utf8Code := "\\u" + strings.ToLower(p.Take(4))
		utf8Str := ""
		if _, err := fmt.Sscanf(utf8Code, "%s", &utf8Str); err != nil {
			panic(err)
		}
		return utf8Str
	default:
		panic(fmt.Errorf("Unknown escape code: %v", escaped))
	}

	return ""
}

func (p *HoconTokenizer) IsStartOfComment() bool {
	return p.MatchesMore([]string{"#", "//"})
}

func (p *HoconTokenizer) PullValue() *Token {
	if p.IsObjectStart() {
		return p.PullStartOfObject()
	}

	if p.IsStartOfTripleQuotedText() {
		return p.PullTripleQuotedText()
	}

	if p.IsStartOfQuotedText() {
		return p.PullQuotedText()
	}

	if p.isUnquotedText() {
		return p.pullUnquotedText()
	}

	if p.IsArrayStart() {
		return p.PullArrayStart()
	}

	if p.IsArrayEnd() {
		return p.PullArrayEnd()
	}

	if p.IsSubstitutionStart() {
		return p.pullSubstitution()
	}

	panic(fmt.Errorf("Expected value: Null literal, Array, Quoted Text, Unquoted Text, Triple quoted Text, Object or End of array"))
}

func (p *HoconTokenizer) IsSubstitutionStart() bool {
	return p.MatchesMore([]string{"${", "${?"})
}

func (p *HoconTokenizer) IsInclude() bool {
	p.Push()
	defer func() {
		recover()
		p.Pop()
	}()
	if p.Matches("include") {
		p.Take(len("include"))
		if p.IsWhitespaceOrComment() {
			p.PullWhitespaceAndComments()
			if p.IsStartOfQuotedText() {
				p.PullQuotedText()
				return true
			}
		}
	}

	return false
}

func (p *HoconTokenizer) pullSubstitution() *Token {
	buf := bytes.NewBuffer(nil)
	p.Take(2)
	if p.Peek() == '?' {
		p.TakeOne()
	}

	for !p.EOF() && p.isUnquotedText() {
		if err := buf.WriteByte(p.TakeOne()); err != nil {
			panic(err)
		}
	}
	p.TakeOne()
	return DefaultToken.Substitution(buf.String())
}

func (p *HoconTokenizer) IsSpaceOrTab() bool {
	return p.MatchesMore([]string{" ", "\t"})
}

func (p *HoconTokenizer) IsStartSimpleValue() bool {
	if p.IsSpaceOrTab() {
		return true
	}

	if p.isUnquotedText() {
		return true
	}

	return false
}

func (p *HoconTokenizer) PullSpaceOrTab() *Token {
	buf := bytes.NewBuffer(nil)
	for p.IsSpaceOrTab() {
		if err := buf.WriteByte(p.TakeOne()); err != nil {
			panic(err)
		}
	}
	return DefaultToken.LiteralValue(buf.String())
}

func (p *HoconTokenizer) pullUnquotedText() *Token {
	buf := bytes.NewBuffer(nil)
	for !p.EOF() && p.isUnquotedText() {
		if err := buf.WriteByte(p.TakeOne()); err != nil {
			panic(err)
		}
	}
	return DefaultToken.LiteralValue(buf.String())
}

func (p *HoconTokenizer) isUnquotedText() bool {
	return !p.EOF() && !p.IsWhitespace() && !p.IsStartOfComment() && strings.IndexByte(HoconNotInUnquotedText, p.Peek()) == -1
}

func (p *HoconTokenizer) PullSimpleValue() *Token {
	if p.IsSpaceOrTab() {
		return p.PullSpaceOrTab()
	}

	if p.isUnquotedText() {
		return p.pullUnquotedText()
	}
	panic("No simple value found")
}

func (p *HoconTokenizer) isValue() bool {

	if p.IsArrayStart() ||
		p.IsObjectStart() ||
		p.IsStartOfTripleQuotedText() ||
		p.IsSubstitutionStart() ||
		p.IsStartOfQuotedText() ||
		p.isUnquotedText() {
		return true
	}
	return false
}

func isWhitespace(c byte) bool {
	str := string(c)
	if str == " " || str == "\r" || str == "\n" || str == "\t" || str == "\u00A0" || str == "\u2007" || str == "\u202F" || str == "\uFEFF" {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2013 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package commands

import (
	. "github.com/limetext/lime/backend"
	. "github.com/limetext/text"
	"reflect"
	"testing"
)

type findTest struct {
	text string
	in   []Region
	exp  []Region
	fw   bool
}

func runFindTest(tests []findTest, t *testing.T, commands ...string) {
	ed := GetEditor()
	w := ed.NewWindow()
	defer w.Close()

	v := w.NewFile()
	defer func() {
		v.SetScratch(true)
		v.Close()
	}()

	for i, test := range tests {
		e := v.BeginEdit()
		v.Insert(e, 0, test.text)
		v.EndEdit(e)
		v.Sel().Clear()
		for _, r := range test.in {
			v.Sel().Add(r)
		}

		v.Settings().Set("find_wrap", tests[i].fw)

		for _, command := range commands {
			ed.CommandHandler().RunTextCommand(v, command, nil)
		}
		if sr := v.Sel().Regions(); !reflect.DeepEqual(sr, test.exp) {
			t.Errorf("Test %d: Expected %s, but got %s", i, test.exp, sr)
		}
		e = v.BeginEdit()
		v.Erase(e, Region{0, v.Buffer().Size()})
		v.EndEdit(e)
	}
}

func TestFindUnderExpand(t *testing.T) {
	tests := []findTest{
		{
			"Hello World!\nTest123123\nAbrakadabra\n",
			[]Region{{0, 0}},
			[]Region{{0, 5}},
			true,
		},
		{
			"Hello World!\nTest123123\nAbrakadabra\n",
			[]Region{{19, 20}},
			[]Region{{19, 20}, {22, 23}},
			true,
		},
	}

	runFindTest(tests, t, "find_under_expand")
}

func TestFindNext(t *testing.T) {
	tests := []findTest{
		{
			"Hello World!\nTest123123\nAbrakadabra\n",
			[]Region{{17, 20}},
			[]Region{{17, 20}},
			true,
		},
		// test find_wrap setting true
		{
			"Hello World!\nTest123123\nAbrakadabra\n",
			[]Region{{21, 23}},
			[]Region{{18, 20}},
			true,
		},
		// test find_wrap setting false
		{
			"Hello World!\nTest123123\nAbrakadabra\n",
			[]Region{{21, 23}},
			[]Region{{21, 23}},
			false,
		},
	}

	runFindTest(tests, t, "find_under_expand", "find_next")
}

type replaceTest struct {
	cursors []Region
	in      string
	exp     string
	fw      bool
}

func runReplaceTest(tests []replaceTest, t *testing.T, commands ...string) {
	ed := GetEditor()
	w := ed.NewWindow()
	defer w.Close()

	v := w.NewFile()
	defer func() {
		v.SetScratch(true)
		v.Close()
	}()

	for i, test := range tests {
		e := v.BeginEdit()
		v.Insert(e, 0, test.in)
		v.EndEdit(e)
		v.Sel().Clear()

		for _, r := range test.cursors {
			v.Sel().Add(r)
		}

		v.Settings().Set("find_wrap", tests[i].fw)

		replaceText = "f"
		for _, command := range commands {
			ed.CommandHandler().RunTextCommand(v, command, nil)
		}
		if out := v.Buffer().Substr(Region{0, v.Buffer().Size()}); out != test.exp {
			t.Errorf("Test %d failed: %s, %+v", i, out, test)
		}
		e = v.BeginEdit()
		v.Erase(e, Region{0, v.Buffer().Size()})
		v.EndEdit(e)
	}
}

func TestReplaceNext(t *testing.T) {
	tests := []replaceTest{
		{
			[]Region{{1, 1}, {2, 2}, {3, 3}},
			"abc abc bac abc abc",
			"abc f bac abc abc",
			true,
		},
		{
			[]Region{{0, 0}, {4, 4}, {8, 8}, {12, 13}},
			"abc abc bac abc abc",
			"abc abc bac abc f",
			true,
		},
		{
			[]Region{{12, 13}, {8, 8}, {4, 4}, {1, 0}},
			"abc abc bac abc abc",
			"abc abc bac abc f",
			true,
		},
		{
			[]Region{{15, 15}},
			"abc abc bac abc abc",
			"abc abc bac abc f",
			true,
		},
		{
			[]Region{{0, 0}},
			"abc abc bac abc abc",
			"abc f bac abc abc",
			true,
		},
		// test find_wrap setting true
		{
			[]Region{{16, 19}},
			"abc abc bac abc abc",
			"f abc bac abc abc",
			true,
		},
		// test find_wrap setting false
		{
			[]Region{{16, 19}},
			"abc abc bac abc abc",
			"abc abc bac abc abc",
			false,
		},
	}

	runReplaceTest(tests, t, "find_under_expand", "replace_next")
}
<commit_msg>using the value by range instead of the index<commit_after>// Copyright 2013 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package commands

import (
	. "github.com/limetext/lime/backend"
	. "github.com/limetext/text"
	"reflect"
	"testing"
)

type findTest struct {
	text string
	in   []Region
	exp  []Region
	fw   bool
}

func runFindTest(tests []findTest, t *testing.T, commands ...string) {
	ed := GetEditor()
	w := ed.NewWindow()
	defer w.Close()

	v := w.NewFile()
	defer func() {
		v.SetScratch(true)
		v.Close()
	}()

	for i, test := range tests {
		e := v.BeginEdit()
		v.Insert(e, 0, test.text)
		v.EndEdit(e)
		v.Sel().Clear()
		for _, r := range test.in {
			v.Sel().Add(r)
		}

		v.Settings().Set("find_wrap", test.fw)

		for _, command := range commands {
			ed.CommandHandler().RunTextCommand(v, command, nil)
		}
		if sr := v.Sel().Regions(); !reflect.DeepEqual(sr, test.exp) {
			t.Errorf("Test %d: Expected %s, but got %s", i, test.exp, sr)
		}
		e = v.BeginEdit()
		v.Erase(e, Region{0, v.Buffer().Size()})
		v.EndEdit(e)
	}
}

func TestFindUnderExpand(t *testing.T) {
	tests := []findTest{
		{
			"Hello World!\nTest123123\nAbrakadabra\n",
			[]Region{{0, 0}},
			[]Region{{0, 5}},
			true,
		},
		{
			"Hello World!\nTest123123\nAbrakadabra\n",
			[]Region{{19, 20}},
			[]Region{{19, 20}, {22, 23}},
			true,
		},
	}

	runFindTest(tests, t, "find_under_expand")
}

func TestFindNext(t *testing.T) {
	tests := []findTest{
		{
			"Hello World!\nTest123123\nAbrakadabra\n",
			[]Region{{17, 20}},
			[]Region{{17, 20}},
			true,
		},
		// test find_wrap setting true
		{
			"Hello World!\nTest123123\nAbrakadabra\n",
			[]Region{{21, 23}},
			[]Region{{18, 20}},
			true,
		},
		// test find_wrap setting false
		{
			"Hello World!\nTest123123\nAbrakadabra\n",
			[]Region{{21, 23}},
			[]Region{{21, 23}},
			false,
		},
	}

	runFindTest(tests, t, "find_under_expand", "find_next")
}

type replaceTest struct {
	cursors []Region
	in      string
	exp     string
	fw      bool
}

func runReplaceTest(tests []replaceTest, t *testing.T, commands ...string) {
	ed := GetEditor()
	w := ed.NewWindow()
	defer w.Close()

	v := w.NewFile()
	defer func() {
		v.SetScratch(true)
		v.Close()
	}()

	for i, test := range tests {
		e := v.BeginEdit()
		v.Insert(e, 0, test.in)
		v.EndEdit(e)
		v.Sel().Clear()

		for _, r := range test.cursors {
			v.Sel().Add(r)
		}

		v.Settings().Set("find_wrap", test.fw)

		replaceText = "f"
		for _, command := range commands {
			ed.CommandHandler().RunTextCommand(v, command, nil)
		}
		if out := v.Buffer().Substr(Region{0, v.Buffer().Size()}); out != test.exp {
			t.Errorf("Test %d failed: %s, %+v", i, out, test)
		}
		e = v.BeginEdit()
		v.Erase(e, Region{0, v.Buffer().Size()})
		v.EndEdit(e)
	}
}

func TestReplaceNext(t *testing.T) {
	tests := []replaceTest{
		{
			[]Region{{1, 1}, {2, 2}, {3, 3}},
			"abc abc bac abc abc",
			"abc f bac abc abc",
			true,
		},
		{
			[]Region{{0, 0}, {4, 4}, {8, 8}, {12, 13}},
			"abc abc bac abc abc",
			"abc abc bac abc f",
			true,
		},
		{
			[]Region{{12, 13}, {8, 8}, {4, 4}, {1, 0}},
			"abc abc bac abc abc",
			"abc abc bac abc f",
			true,
		},
		{
			[]Region{{15, 15}},
			"abc abc bac abc abc",
			"abc abc bac abc f",
			true,
		},
		{
			[]Region{{0, 0}},
			"abc abc bac abc abc",
			"abc f bac abc abc",
			true,
		},
		// test find_wrap setting true
		{
			[]Region{{16, 19}},
			"abc abc bac abc abc",
			"f abc bac abc abc",
			true,
		},
		// test find_wrap setting false
		{
			[]Region{{16, 19}},
			"abc abc bac abc abc",
			"abc abc bac abc abc",
			false,
		},
	}

	runReplaceTest(tests, t, "find_under_expand", "replace_next")
}
<|endoftext|>


<commit_before>package httpclient

import (
	"bytes"
	"crypto/rand"
	"crypto/tls"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"strings"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/net/context/ctxhttp"

	"github.com/rightscale/rsc/log"
	"github.com/rightscale/rsc/recording"
)

const (
	// NoDump is the default value for DumpFormat.
	NoDump Format = 1 << iota
	// Debug formats the dumps in human readable format, the use of this flag is exclusive with
	// JSON.
	Debug
	// JSON formats the dumps in JSON, the use of this flag is exclusive with Debug.
	JSON
	// Verbose enables the dumps for all requests and auth headers.
	Verbose
	// Record causes the dumps to be written to the recorder file descriptor (used by tests).
	Record
)

const (
	noRedirectError = "no redirect"
	requestIdHeader = "X-Request-Id"
)

var (
	// DumpFormat dictates how HTTP requests and responses are logged: NoDump prevents logging
	// altogether, Debug generates logs in human readable format and JSON in JSON format.
	// Verbose causes all headers to be logged - including sensitive ones.
	DumpFormat Format

	// Insecure dictates whether HTTP (true) or HTTPS (false) should be used to connect to the
	// API endpoints.
	Insecure bool

	// NoCertCheck dictates whether the SSL handshakes should bypass X509 certificate
	// validation (true) or not (false).
	NoCertCheck bool

	// ResponseHeaderTimeout if non-zero, specifies the amount of
	// time to wait in seconds for a server's response headers after fully
	// writing the request (including its body, if any). This
	// time does not include the time to read the response body.
	ResponseHeaderTimeout = 300 * time.Second

	// HiddenHeaders lists headers that should not be logged unless DumpFormat is Verbose.
	HiddenHeaders = map[string]bool{"Authorization": true, "Cookie": true}
)

// For tests
var (
	OsStderr io.Writer = os.Stderr
)

type (
	// HTTPClient makes it easier to stub HTTP clients for testing.
	HTTPClient interface {
		// Do makes a regular http request and returns the response/error.
		Do(req *http.Request) (*http.Response, error)

		// DoWithContext performs a request and is context-aware.
		DoWithContext(ctx context.Context, req *http.Request) (*http.Response, error)

		// DoHidden prevents logging, useful for requests made during authorization.
		DoHidden(req *http.Request) (*http.Response, error)

		// DoHiddenWithContext prevents logging and performs a context-aware request.
		DoHiddenWithContext(ctx context.Context, req *http.Request) (*http.Response, error)
	}

	// Format is the request/response dump format.
	Format int

	// HTTP client that optionally dumps requests and responses.
	// This client also disables the default http client redirect handling.
	dumpClient struct {
		Client *http.Client
		Format Format
	}
)

// Default DumpFormat to NoDump
func init() {
	DumpFormat = NoDump
}

// New returns an HTTP client using the settings specified by this package variables.
func New() HTTPClient {
	return &dumpClient{Client: newRawClient(false)}
}

// NewNoRedirect returns an HTTP client that does not follow redirects.
func NewNoRedirect() HTTPClient {
	return &dumpClient{Client: newRawClient(true)}
}

// ShortToken creates a 6 bytes unique string.
// Not meant to be cryptographically unique but good enough for logs.
func ShortToken() string {
	b := make([]byte, 6)
	io.ReadFull(rand.Reader, b)
	return base64.StdEncoding.EncodeToString(b)
}

// newRawClient creates an http package Client taking into account both the parameters and package
// variables.
func newRawClient(noredirect bool) *http.Client {
	tr := http.Transport{ResponseHeaderTimeout: ResponseHeaderTimeout, Proxy: http.ProxyFromEnvironment}
	tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: NoCertCheck}
	c := http.Client{Transport: &tr}
	if noredirect {
		c.CheckRedirect = func(*http.Request, []*http.Request) error {
			return fmt.Errorf(noRedirectError)
		}
	}
	return &c
}

// IsDebug is a convenience wrapper that returns true if the Debug bit is set on the flag.
func (f Format) IsDebug() bool {
	return f&Debug != 0
}

// IsJSON is a convenience wrapper that returns true if the JSON bit is set on the flag.
func (f Format) IsJSON() bool {
	return f&JSON != 0
}

// IsVerbose is a convenience wrapper that returns true if the Verbose bit is set on the flag.
func (f Format) IsVerbose() bool {
	return f&Verbose != 0
}

// IsRecord is a convenience wrapper that returns true if the Record bit is set on the flag.
func (f Format) IsRecord() bool {
	return f&Record != 0
}

// DoHidden is equivalent to Do with the exception that nothing gets logged unless DumpFormat is
// set to Verbose.
func (d *dumpClient) DoHidden(req *http.Request) (*http.Response, error) {
	return d.doImp(req, true, nil)
}

// Do dumps the request, makes the request and dumps the response as specified by DumpFormat.
func (d *dumpClient) Do(req *http.Request) (*http.Response, error) {
	return d.doImp(req, false, nil)
}

func (d *dumpClient) DoWithContext(ctx context.Context, req *http.Request) (*http.Response, error) {
	return d.doImp(req, true, ctx)
}

func (d *dumpClient) DoHiddenWithContext(ctx context.Context, req *http.Request) (*http.Response, error) {
	return d.doImp(req, false, ctx)
}

// doImp actually performs the HTTP request logging according to the various settings.
func (d *dumpClient) doImp(req *http.Request, hidden bool, ctx context.Context) (*http.Response, error) {
	if req.URL.Scheme == "" {
		if Insecure {
			req.URL.Scheme = "http"
		} else {
			req.URL.Scheme = "https"
		}
	}
	req.Header.Set("User-Agent", UA)

	var reqBody []byte
	startedAt := time.Now()

	// prefer the X-Request-Id header as request token for logging, if present.
	id := req.Header.Get(requestIdHeader)
	if id == "" {
		id = ShortToken()
	}
	log.Info("started", "id", id, req.Method, req.URL.String())
	hide := (DumpFormat == NoDump) || (hidden && !DumpFormat.IsVerbose())
	if !hide {
		startedAt = time.Now()
		reqBody = dumpRequest(req)
	}
	var resp *http.Response
	var err error
	if ctx == nil {
		resp, err = d.Client.Do(req)
	} else {
		resp, err = ctxhttp.Do(ctx, d.getClientWithoutTimeout(), req)
	}
	if urlError, ok := err.(*url.Error); ok {
		if urlError.Err.Error() == noRedirectError {
			err = nil
		}
	}
	if err != nil {
		return nil, err
	}
	if !hide {
		dumpResponse(resp, req, reqBody)
	}
	log.Info("completed", "id", id, "status", resp.Status, "time", time.Since(startedAt).String())

	return resp, nil
}

// getClientWithoutTimeout returns a modified client that doesn't have the ResponseHeaderTimeout field set
// in its Transport.
func (d *dumpClient) getClientWithoutTimeout() *http.Client {
	// Get a copy of the client and modify as multiple concurrent go routines can be using this client.
	client := *d.Client
	tr := &http.Transport{Proxy: http.ProxyFromEnvironment}
	tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: NoCertCheck}
	client.Transport = tr
	return &client
}

// Dump request if needed.
// Return request serialized as JSON if DumpFormat is JSON, nil otherwise.
func dumpRequest(req *http.Request) []byte {
	if DumpFormat == NoDump {
		return nil
	}
	reqBody, err := dumpReqBody(req)
	if err != nil {
		log.Error("Failed to load request body for dump", "error", err.Error())
	}
	if DumpFormat.IsDebug() {
		var buffer bytes.Buffer
		buffer.WriteString(req.Method + " " + req.URL.String() + "\n")
		writeHeaders(&buffer, req.Header)
		if reqBody != nil {
			buffer.WriteString("\n")
			buffer.Write(reqBody)
			buffer.WriteString("\n")
		}
		fmt.Fprint(OsStderr, buffer.String())
	} else if DumpFormat.IsJSON() {
		return reqBody
	}
	return nil
}

// dumpResponse dumps the response and optionally the request (in case of JSON format) according to
// DumpFormat.
// It also checks whether the special recorder pipe is opened and if so writes the dump to it.
func dumpResponse(resp *http.Response, req *http.Request, reqBody []byte) {
	if DumpFormat == NoDump {
		return
	}
	respBody, _ := dumpRespBody(resp)
	if DumpFormat.IsDebug() {
		var buffer bytes.Buffer
		buffer.WriteString("==> " + resp.Proto + " " + resp.Status + "\n")
		writeHeaders(&buffer, resp.Header)
		if respBody != nil {
			buffer.WriteString("\n")
			buffer.Write(respBody)
			buffer.WriteString("\n")
		}
		fmt.Fprint(OsStderr, buffer.String())
	} else if DumpFormat.IsJSON() {
		reqHeaders := make(http.Header)
		filterHeaders(req.Header, func(name string, value []string) {
			reqHeaders[name] = value
		})
		respHeaders := make(http.Header)
		filterHeaders(resp.Header, func(name string, value []string) {
			respHeaders[name] = value
		})
		dumped := recording.RequestResponse{
			Verb:       req.Method,
			URI:        req.URL.String(),
			ReqHeader:  reqHeaders,
			ReqBody:    string(reqBody),
			Status:     resp.StatusCode,
			RespHeader: respHeaders,
			RespBody:   string(respBody),
		}
		b, err := json.MarshalIndent(dumped, "", "    ")
		if err != nil {
			log.Error("Failed to dump request content", "error", err.Error())
			return
		}
		if DumpFormat.IsRecord() {
			f := os.NewFile(10, "fd10")
			_, err = f.Stat()
			if err == nil {
				// fd 10 is open, dump to it (used by recorder)
				fmt.Fprintf(f, "%s\n", string(b))
			}
		}
		fmt.Fprint(OsStderr, string(b))
	}
}

// writeHeaders is a helper function that writes the given HTTP headers to the given buffer as
// human readable strings. If DumpFormat is not Verbose then writeHeaders filters out headers whose
// names are keys of HiddenHeaders.
func writeHeaders(buffer *bytes.Buffer, headers http.Header) {
	filterHeaders(headers, func(name string, value []string) {
		buffer.WriteString(name)
		buffer.WriteString(": ")
		buffer.WriteString(strings.Join(value, ", "))
		buffer.WriteString("\n")
	})
}

// Dump request body, strongly inspired from httputil.DumpRequest
func dumpReqBody(req *http.Request) ([]byte, error) {
	if req.Body == nil {
		return nil, nil
	}
	var save io.ReadCloser
	var err error
	save, req.Body, err = drainBody(req.Body)
	if err != nil {
		return nil, err
	}
	var b bytes.Buffer
	var dest io.Writer = &b
	chunked := len(req.TransferEncoding) > 0 && req.TransferEncoding[0] == "chunked"
	if chunked {
		dest = httputil.NewChunkedWriter(dest)
	}
	_, err = io.Copy(dest, req.Body)
	if chunked {
		dest.(io.Closer).Close()
		io.WriteString(&b, "\r\n")
	}
	req.Body = save
	return b.Bytes(), err
}

// Dump response body, strongly inspired from httputil.DumpResponse
func dumpRespBody(resp *http.Response) ([]byte, error) {
	if resp.Body == nil {
		return nil, nil
	}
	var b bytes.Buffer
	savecl := resp.ContentLength
	var save io.ReadCloser
	var err error
	save, resp.Body, err = drainBody(resp.Body)
	if err != nil {
		return nil, err
	}
	_, err = io.Copy(&b, resp.Body)
	if err != nil {
		return nil, err
	}
	resp.Body = save
	resp.ContentLength = savecl
	if err != nil {
		return nil, err
	}
	return b.Bytes(), nil
}

// One of the copies, say from b to r2, could be avoided by using a more
// elaborate trick where the other copy is made during Request/Response.Write.
// This would complicate things too much, given that these functions are for
// debugging only.
func drainBody(b io.ReadCloser) (r1, r2 io.ReadCloser, err error) {
	var buf bytes.Buffer
	if _, err = buf.ReadFrom(b); err != nil {
		return nil, nil, err
	}
	if err = b.Close(); err != nil {
		return nil, nil, err
	}
	return ioutil.NopCloser(&buf), ioutil.NopCloser(bytes.NewReader(buf.Bytes())), nil
}

// headerIterator is a HTTP header iterator.
type headerIterator func(name string, value []string)

// filterHeaders iterates through the headers skipping hidden headers unless DumpFormat is Verbose.
// It calls the given iterator for each header name/value pair. The values are serialized as
// strings.
func filterHeaders(headers http.Header, iterator headerIterator) {
	for k, v := range headers {
		if !DumpFormat.IsVerbose() {
			if _, ok := HiddenHeaders[k]; ok {
				continue
			}
		}
		iterator(k, v)
	}
}
<commit_msg>SS-4415 fix for loss of configured InsecureSkipVerify flag when request executes in httpclient<commit_after>package httpclient

import (
	"bytes"
	"crypto/rand"
	"crypto/tls"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"strings"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/net/context/ctxhttp"

	"github.com/rightscale/rsc/log"
	"github.com/rightscale/rsc/recording"
)

const (
	// NoDump is the default value for DumpFormat.
	NoDump Format = 1 << iota
	// Debug formats the dumps in human readable format, the use of this flag is exclusive with
	// JSON.
	Debug
	// JSON formats the dumps in JSON, the use of this flag is exclusive with Debug.
	JSON
	// Verbose enables the dumps for all requests and auth headers.
	Verbose
	// Record causes the dumps to be written to the recorder file descriptor (used by tests).
	Record
)

const (
	noRedirectError = "no redirect"
	requestIdHeader = "X-Request-Id"
)

var (
	// DumpFormat dictates how HTTP requests and responses are logged: NoDump prevents logging
	// altogether, Debug generates logs in human readable format and JSON in JSON format.
	// Verbose causes all headers to be logged - including sensitive ones.
	DumpFormat Format

	// Insecure dictates whether HTTP (true) or HTTPS (false) should be used to connect to the
	// API endpoints.
	Insecure bool

	// NoCertCheck dictates whether the SSL handshakes should bypass X509 certificate
	// validation (true) or not (false).
	NoCertCheck bool

	// ResponseHeaderTimeout if non-zero, specifies the amount of
	// time to wait in seconds for a server's response headers after fully
	// writing the request (including its body, if any). This
	// time does not include the time to read the response body.
	ResponseHeaderTimeout = 300 * time.Second

	// HiddenHeaders lists headers that should not be logged unless DumpFormat is Verbose.
	HiddenHeaders = map[string]bool{"Authorization": true, "Cookie": true}
)

// For tests
var (
	OsStderr io.Writer = os.Stderr
)

type (
	// HTTPClient makes it easier to stub HTTP clients for testing.
	HTTPClient interface {
		// Do makes a regular http request and returns the response/error.
		Do(req *http.Request) (*http.Response, error)

		// DoWithContext performs a request and is context-aware.
		DoWithContext(ctx context.Context, req *http.Request) (*http.Response, error)

		// DoHidden prevents logging, useful for requests made during authorization.
		DoHidden(req *http.Request) (*http.Response, error)

		// DoHiddenWithContext prevents logging and performs a context-aware request.
		DoHiddenWithContext(ctx context.Context, req *http.Request) (*http.Response, error)
	}

	// Format is the request/response dump format.
	Format int

	// HTTP client that optionally dumps requests and responses.
	// This client also disables the default http client redirect handling.
	dumpClient struct {
		Client *http.Client
		Format Format
	}
)

// Default DumpFormat to NoDump
func init() {
	DumpFormat = NoDump
}

// New returns an HTTP client using the settings specified by this package variables.
func New() HTTPClient {
	return &dumpClient{Client: newRawClient(false)}
}

// NewNoRedirect returns an HTTP client that does not follow redirects.
func NewNoRedirect() HTTPClient {
	return &dumpClient{Client: newRawClient(true)}
}

// ShortToken creates a 6 bytes unique string.
// Not meant to be cryptographically unique but good enough for logs.
func ShortToken() string {
	b := make([]byte, 6)
	io.ReadFull(rand.Reader, b)
	return base64.StdEncoding.EncodeToString(b)
}

// newRawClient creates an http package Client taking into account both the parameters and package
// variables.
func newRawClient(noredirect bool) *http.Client {
	tr := http.Transport{ResponseHeaderTimeout: ResponseHeaderTimeout, Proxy: http.ProxyFromEnvironment}
	tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: NoCertCheck}
	c := http.Client{Transport: &tr}
	if noredirect {
		c.CheckRedirect = func(*http.Request, []*http.Request) error {
			return fmt.Errorf(noRedirectError)
		}
	}
	return &c
}

// IsDebug is a convenience wrapper that returns true if the Debug bit is set on the flag.
func (f Format) IsDebug() bool {
	return f&Debug != 0
}

// IsJSON is a convenience wrapper that returns true if the JSON bit is set on the flag.
func (f Format) IsJSON() bool {
	return f&JSON != 0
}

// IsVerbose is a convenience wrapper that returns true if the Verbose bit is set on the flag.
func (f Format) IsVerbose() bool {
	return f&Verbose != 0
}

// IsRecord is a convenience wrapper that returns true if the Record bit is set on the flag.
func (f Format) IsRecord() bool {
	return f&Record != 0
}

// DoHidden is equivalent to Do with the exception that nothing gets logged unless DumpFormat is
// set to Verbose.
func (d *dumpClient) DoHidden(req *http.Request) (*http.Response, error) {
	return d.doImp(req, true, nil)
}

// Do dumps the request, makes the request and dumps the response as specified by DumpFormat.
func (d *dumpClient) Do(req *http.Request) (*http.Response, error) {
	return d.doImp(req, false, nil)
}

func (d *dumpClient) DoWithContext(ctx context.Context, req *http.Request) (*http.Response, error) {
	return d.doImp(req, true, ctx)
}

func (d *dumpClient) DoHiddenWithContext(ctx context.Context, req *http.Request) (*http.Response, error) {
	return d.doImp(req, false, ctx)
}

// doImp actually performs the HTTP request logging according to the various settings.
func (d *dumpClient) doImp(req *http.Request, hidden bool, ctx context.Context) (*http.Response, error) {
	if req.URL.Scheme == "" {
		if Insecure {
			req.URL.Scheme = "http"
		} else {
			req.URL.Scheme = "https"
		}
	}
	req.Header.Set("User-Agent", UA)

	var reqBody []byte
	startedAt := time.Now()

	// prefer the X-Request-Id header as request token for logging, if present.
	id := req.Header.Get(requestIdHeader)
	if id == "" {
		id = ShortToken()
	}
	log.Info("started", "id", id, req.Method, req.URL.String())
	hide := (DumpFormat == NoDump) || (hidden && !DumpFormat.IsVerbose())
	if !hide {
		startedAt = time.Now()
		reqBody = dumpRequest(req)
	}
	var resp *http.Response
	var err error
	if ctx == nil {
		resp, err = d.Client.Do(req)
	} else {
		resp, err = ctxhttp.Do(ctx, d.getClientWithoutTimeout(), req)
	}
	if urlError, ok := err.(*url.Error); ok {
		if urlError.Err.Error() == noRedirectError {
			err = nil
		}
	}
	if err != nil {
		return nil, err
	}
	if !hide {
		dumpResponse(resp, req, reqBody)
	}
	log.Info("completed", "id", id, "status", resp.Status, "time", time.Since(startedAt).String())

	return resp, nil
}

// getClientWithoutTimeout returns a modified client that doesn't have the ResponseHeaderTimeout field set
// in its Transport.
func (d *dumpClient) getClientWithoutTimeout() *http.Client {
	// Get a copy of the client and modify as multiple concurrent go routines can be using this client.
	client := *d.Client
	tr, ok := client.Transport.(*http.Transport)
	if ok {
		trCopy := *tr
		trCopy.ResponseHeaderTimeout = 0
		tr = &trCopy
	} else {
		// note that the following code has a known issue in that it depends on the
		// current value of the NoCertCheck global. if that global changes after
		// creation of this client then the behavior is undefined.
		tr = &http.Transport{Proxy: http.ProxyFromEnvironment}
		tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: NoCertCheck}
	}
	client.Transport = tr
	return &client
}

// Dump request if needed.
// Return request serialized as JSON if DumpFormat is JSON, nil otherwise.
func dumpRequest(req *http.Request) []byte {
	if DumpFormat == NoDump {
		return nil
	}
	reqBody, err := dumpReqBody(req)
	if err != nil {
		log.Error("Failed to load request body for dump", "error", err.Error())
	}
	if DumpFormat.IsDebug() {
		var buffer bytes.Buffer
		buffer.WriteString(req.Method + " " + req.URL.String() + "\n")
		writeHeaders(&buffer, req.Header)
		if reqBody != nil {
			buffer.WriteString("\n")
			buffer.Write(reqBody)
			buffer.WriteString("\n")
		}
		fmt.Fprint(OsStderr, buffer.String())
	} else if DumpFormat.IsJSON() {
		return reqBody
	}
	return nil
}

// dumpResponse dumps the response and optionally the request (in case of JSON format) according to
// DumpFormat.
// It also checks whether the special recorder pipe is opened and if so writes the dump to it.
func dumpResponse(resp *http.Response, req *http.Request, reqBody []byte) {
	if DumpFormat == NoDump {
		return
	}
	respBody, _ := dumpRespBody(resp)
	if DumpFormat.IsDebug() {
		var buffer bytes.Buffer
		buffer.WriteString("==> " + resp.Proto + " " + resp.Status + "\n")
		writeHeaders(&buffer, resp.Header)
		if respBody != nil {
			buffer.WriteString("\n")
			buffer.Write(respBody)
			buffer.WriteString("\n")
		}
		fmt.Fprint(OsStderr, buffer.String())
	} else if DumpFormat.IsJSON() {
		reqHeaders := make(http.Header)
		filterHeaders(req.Header, func(name string, value []string) {
			reqHeaders[name] = value
		})
		respHeaders := make(http.Header)
		filterHeaders(resp.Header, func(name string, value []string) {
			respHeaders[name] = value
		})
		dumped := recording.RequestResponse{
			Verb:       req.Method,
			URI:        req.URL.String(),
			ReqHeader:  reqHeaders,
			ReqBody:    string(reqBody),
			Status:     resp.StatusCode,
			RespHeader: respHeaders,
			RespBody:   string(respBody),
		}
		b, err := json.MarshalIndent(dumped, "", "    ")
		if err != nil {
			log.Error("Failed to dump request content", "error", err.Error())
			return
		}
		if DumpFormat.IsRecord() {
			f := os.NewFile(10, "fd10")
			_, err = f.Stat()
			if err == nil {
				// fd 10 is open, dump to it (used by recorder)
				fmt.Fprintf(f, "%s\n", string(b))
			}
		}
		fmt.Fprint(OsStderr, string(b))
	}
}

// writeHeaders is a helper function that writes the given HTTP headers to the given buffer as
// human readable strings. If DumpFormat is not Verbose then writeHeaders filters out headers whose
// names are keys of HiddenHeaders.
func writeHeaders(buffer *bytes.Buffer, headers http.Header) {
	filterHeaders(headers, func(name string, value []string) {
		buffer.WriteString(name)
		buffer.WriteString(": ")
		buffer.WriteString(strings.Join(value, ", "))
		buffer.WriteString("\n")
	})
}

// Dump request body, strongly inspired from httputil.DumpRequest
func dumpReqBody(req *http.Request) ([]byte, error) {
	if req.Body == nil {
		return nil, nil
	}
	var save io.ReadCloser
	var err error
	save, req.Body, err = drainBody(req.Body)
	if err != nil {
		return nil, err
	}
	var b bytes.Buffer
	var dest io.Writer = &b
	chunked := len(req.TransferEncoding) > 0 && req.TransferEncoding[0] == "chunked"
	if chunked {
		dest = httputil.NewChunkedWriter(dest)
	}
	_, err = io.Copy(dest, req.Body)
	if chunked {
		dest.(io.Closer).Close()
		io.WriteString(&b, "\r\n")
	}
	req.Body = save
	return b.Bytes(), err
}

// Dump response body, strongly inspired from httputil.DumpResponse
func dumpRespBody(resp *http.Response) ([]byte, error) {
	if resp.Body == nil {
		return nil, nil
	}
	var b bytes.Buffer
	savecl := resp.ContentLength
	var save io.ReadCloser
	var err error
	save, resp.Body, err = drainBody(resp.Body)
	if err != nil {
		return nil, err
	}
	_, err = io.Copy(&b, resp.Body)
	if err != nil {
		return nil, err
	}
	resp.Body = save
	resp.ContentLength = savecl
	if err != nil {
		return nil, err
	}
	return b.Bytes(), nil
}

// One of the copies, say from b to r2, could be avoided by using a more
// elaborate trick where the other copy is made during Request/Response.Write.
// This would complicate things too much, given that these functions are for
// debugging only.
func drainBody(b io.ReadCloser) (r1, r2 io.ReadCloser, err error) {
	var buf bytes.Buffer
	if _, err = buf.ReadFrom(b); err != nil {
		return nil, nil, err
	}
	if err = b.Close(); err != nil {
		return nil, nil, err
	}
	return ioutil.NopCloser(&buf), ioutil.NopCloser(bytes.NewReader(buf.Bytes())), nil
}

// headerIterator is a HTTP header iterator.
type headerIterator func(name string, value []string)

// filterHeaders iterates through the headers skipping hidden headers unless DumpFormat is Verbose.
// It calls the given iterator for each header name/value pair. The values are serialized as
// strings.
func filterHeaders(headers http.Header, iterator headerIterator) {
	for k, v := range headers {
		if !DumpFormat.IsVerbose() {
			if _, ok := HiddenHeaders[k]; ok {
				continue
			}
		}
		iterator(k, v)
	}
}
<|endoftext|>


<commit_before>package main
func main() {
    return 1 || 1
}
<commit_msg>GoVetBearTest: Make vet_bad.go testfile worse<commit_after>package main
func_misspelt main() {
    return 1 || 1
}
<|endoftext|>