<commit_before>package main

import (
	"bytes"
	"crypto/md5"
	"encoding/gob"
	"fmt"
	"io"
	"net/http"

	log "github.com/Sirupsen/logrus"
	"github.com/elazarl/goproxy"
	"io/ioutil"
)

// DBClient provides access to cache, http client and configuration
type DBClient struct {
	cache Cache
	http  *http.Client
	cfg   *Configuration
}

// request holds structure for request
type request struct {
	details requestDetails
}

var emptyResp = &http.Response{}

// requestDetails stores information about request, it's used for creating unique hash and also as a payload structure
type requestDetails struct {
	Path        string              `json:"path"`
	Method      string              `json:"method"`
	Destination string              `json:"destination"`
	Scheme      string              `json:"scheme"`
	Query       string              `json:"query"`
	Body        string              `json:"body"`
	RemoteAddr  string              `json:"remoteAddr"`
	Headers     map[string][]string `json:"headers"`
}

func (r *request) concatenate() string {
	var buffer bytes.Buffer

	buffer.WriteString(r.details.Destination)
	buffer.WriteString(r.details.Path)
	buffer.WriteString(r.details.Method)
	buffer.WriteString(r.details.Query)
	buffer.WriteString(r.details.Body)

	return buffer.String()
}

// hash returns unique hash key for request
func (r *request) hash() string {
	h := md5.New()
	io.WriteString(h, r.concatenate())
	return fmt.Sprintf("%x", h.Sum(nil))
}

// res structure hold response body from external service, body is not decoded and is supposed
// to be bytes, however headers should provide all required information for later decoding
// by the client.
type response struct {
	Status  int                 `json:"status"`
	Body    string              `json:"body"`
	Headers map[string][]string `json:"headers"`
}

// Payload structure holds request and response structure
type Payload struct {
	Response response       `json:"response"`
	Request  requestDetails `json:"request"`
	ID       string         `json:"id"`
}

// encode method encodes all exported Payload fields to bytes
func (p *Payload) encode() ([]byte, error) {
	buf := new(bytes.Buffer)
	enc := gob.NewEncoder(buf)
	err := enc.Encode(p)
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// decodePayload decodes supplied bytes into Payload structure
func decodePayload(data []byte) (*Payload, error) {
	var p *Payload
	buf := bytes.NewBuffer(data)
	dec := gob.NewDecoder(buf)
	err := dec.Decode(&p)
	if err != nil {
		return nil, err
	}
	return p, nil
}

// captureRequest saves request for later playback
func (d *DBClient) captureRequest(req *http.Request) (*http.Response, error) {

	// this is mainly for testing, since when you create
	if req.Body == nil {
		req.Body = ioutil.NopCloser(bytes.NewBuffer([]byte("")))
	}

	reqBody, err := ioutil.ReadAll(req.Body)

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
			"mode":  "capture",
		}).Error("Got error when reading request body")
	}

	// outputting request body if verbose logging is set
	log.WithFields(log.Fields{
		"body": string(reqBody),
		"mode": "capture",
	}).Debug("got request body")

	req.Body = ioutil.NopCloser(bytes.NewBuffer(reqBody))

	// forwarding request
	resp, err := d.doRequest(req)

	if err == nil {
		respBody, err := extractBody(resp)

		if err != nil {

			log.WithFields(log.Fields{
				"error": err.Error(),
				"mode":  "capture",
			}).Error("Failed to copy response body.")

			return resp, err
		}

		// saving response body with request/response meta to cache
		d.save(req, reqBody, resp, respBody)
	}

	// return new response or error here
	return resp, err
}

func copyBody(body io.ReadCloser) (resp1, resp2 io.ReadCloser, err error) {
	var buf bytes.Buffer
	if _, err = buf.ReadFrom(body); err != nil {
		return nil, nil, err
	}
	if err = body.Close(); err != nil {
		return nil, nil, err
	}
	return ioutil.NopCloser(&buf), ioutil.NopCloser(bytes.NewReader(buf.Bytes())), nil
}

func extractBody(resp *http.Response) (extract []byte, err error) {
	save := resp.Body
	savecl := resp.ContentLength

	save, resp.Body, err = copyBody(resp.Body)

	if err != nil {
		return
	}
	defer resp.Body.Close()
	extract, err = ioutil.ReadAll(resp.Body)

	resp.Body = save
	resp.ContentLength = savecl
	if err != nil {
		return nil, err
	}
	return extract, nil
}

// doRequest performs original request and returns response that should be returned to client and error (if there is one)
func (d *DBClient) doRequest(request *http.Request) (*http.Response, error) {

	// We can't have this set. And it only contains "/pkg/net/http/" anyway
	request.RequestURI = ""

	if d.cfg.middleware != "" {
		var payload Payload

		c := NewConstructor(request, payload)
		err := c.ApplyMiddleware(d.cfg.middleware)

		if err != nil {
			log.WithFields(log.Fields{
				"mode":   d.cfg.mode,
				"error":  err.Error(),
				"host":   request.Host,
				"method": request.Method,
				"path":   request.URL.Path,
			}).Error("could not forward request, middleware failed to modify request.")
			return nil, err
		}

		request = c.reconstructRequest()
	}

	resp, err := d.http.Do(request)

	if err != nil {
		log.WithFields(log.Fields{
			"mode":   d.cfg.mode,
			"error":  err.Error(),
			"host":   request.Host,
			"method": request.Method,
			"path":   request.URL.Path,
		}).Error("could not forward request, failed to do an HTTP request.")
		return nil, err
	}

	log.WithFields(log.Fields{
		"mode":   d.cfg.mode,
		"host":   request.Host,
		"method": request.Method,
		"path":   request.URL.Path,
	}).Debug("response from external service got successfuly!")

	resp.Header.Set("hoverfly", "Was-Here")
	return resp, nil

}

// save gets request fingerprint, extracts request body, status code and headers, then saves it to cache
func (d *DBClient) save(req *http.Request, reqBody []byte, resp *http.Response, respBody []byte) {
	// record request here
	key := getRequestFingerprint(req, reqBody)

	if resp == nil {
		resp = emptyResp
	} else {
		responseObj := response{
			Status:  resp.StatusCode,
			Body:    string(respBody),
			Headers: resp.Header,
		}

		log.WithFields(log.Fields{
			"path":          req.URL.Path,
			"rawQuery":      req.URL.RawQuery,
			"requestMethod": req.Method,
			"bodyLen":       len(reqBody),
			"destination":   req.Host,
			"hashKey":       key,
		}).Debug("Capturing")

		requestObj := requestDetails{
			Path:        req.URL.Path,
			Method:      req.Method,
			Destination: req.Host,
			Scheme:      req.URL.Scheme,
			Query:       req.URL.RawQuery,
			Body:        string(reqBody),
			RemoteAddr:  req.RemoteAddr,
			Headers:     req.Header,
		}

		payload := Payload{
			Response: responseObj,
			Request:  requestObj,
			ID:       key,
		}

		bts, err := payload.encode()
		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Error("Failed to serialize payload")
		} else {
			d.cache.Set([]byte(key), bts)
		}
	}
}

// getRequestFingerprint returns request hash
func getRequestFingerprint(req *http.Request, requestBody []byte) string {
	details := requestDetails{
		Path:        req.URL.Path,
		Method:      req.Method,
		Destination: req.Host,
		Query:       req.URL.RawQuery,
		Body:        string(requestBody),
	}

	r := request{details: details}
	return r.hash()
}

// getResponse returns stored response from cache
func (d *DBClient) getResponse(req *http.Request) *http.Response {

	reqBody, err := ioutil.ReadAll(req.Body)

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Got error when reading request body")
	}

	key := getRequestFingerprint(req, reqBody)

	payloadBts, err := d.cache.Get([]byte(key))

	if err == nil {
		// getting cache response
		payload, err := decodePayload(payloadBts)
		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Error("Failed to decode payload")
		}

		c := NewConstructor(req, *payload)

		if d.cfg.middleware != "" {
			_ = c.ApplyMiddleware(d.cfg.middleware)
		}

		response := c.reconstructResponse()

		log.WithFields(log.Fields{
			"key":        key,
			"status":     payload.Response.Status,
			"bodyLength": response.ContentLength,
		}).Info("Response found, returning")

		return response

	}

	log.WithFields(log.Fields{
		"error":       err.Error(),
		"query":       req.URL.RawQuery,
		"path":        req.URL.RawPath,
		"destination": req.Host,
		"method":      req.Method,
	}).Warn("Failed to retrieve response from cache")
	// return error? if we return nil - proxy forwards request to original destination
	return goproxy.NewResponse(req,
		goproxy.ContentTypeText, http.StatusPreconditionFailed,
		"Coudldn't find recorded request, please record it first!")

}

// modifyRequestResponse modifies outgoing request and then modifies incoming response, neither request nor response
// is saved to cache.
func (d *DBClient) modifyRequestResponse(req *http.Request, middleware string) (*http.Response, error) {

	// modifying request
	resp, err := d.doRequest(req)

	if err != nil {
		return nil, err
	}

	// preparing payload
	bodyBytes, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		log.WithFields(log.Fields{
			"error":      err.Error(),
			"middleware": middleware,
		}).Error("Failed to read response body after sending modified request")
		return nil, err
	}

	r := response{
		Status:  resp.StatusCode,
		Body:    string(bodyBytes),
		Headers: resp.Header,
	}
	payload := Payload{Response: r}

	c := NewConstructor(req, payload)
	// applying middleware to modify response
	err = c.ApplyMiddleware(middleware)

	if err != nil {
		return nil, err
	}

	newResponse := c.reconstructResponse()

	log.WithFields(log.Fields{
		"status":     newResponse.StatusCode,
		"middleware": middleware,
	}).Info("Response modified, returning")

	return newResponse, nil

}
<commit_msg>a bit more information during virtualize mode<commit_after>package main

import (
	"bytes"
	"crypto/md5"
	"encoding/gob"
	"fmt"
	"io"
	"net/http"

	log "github.com/Sirupsen/logrus"
	"github.com/elazarl/goproxy"
	"io/ioutil"
)

// DBClient provides access to cache, http client and configuration
type DBClient struct {
	cache Cache
	http  *http.Client
	cfg   *Configuration
}

// request holds structure for request
type request struct {
	details requestDetails
}

var emptyResp = &http.Response{}

// requestDetails stores information about request, it's used for creating unique hash and also as a payload structure
type requestDetails struct {
	Path        string              `json:"path"`
	Method      string              `json:"method"`
	Destination string              `json:"destination"`
	Scheme      string              `json:"scheme"`
	Query       string              `json:"query"`
	Body        string              `json:"body"`
	RemoteAddr  string              `json:"remoteAddr"`
	Headers     map[string][]string `json:"headers"`
}

func (r *request) concatenate() string {
	var buffer bytes.Buffer

	buffer.WriteString(r.details.Destination)
	buffer.WriteString(r.details.Path)
	buffer.WriteString(r.details.Method)
	buffer.WriteString(r.details.Query)
	buffer.WriteString(r.details.Body)

	return buffer.String()
}

// hash returns unique hash key for request
func (r *request) hash() string {
	h := md5.New()
	io.WriteString(h, r.concatenate())
	return fmt.Sprintf("%x", h.Sum(nil))
}

// res structure hold response body from external service, body is not decoded and is supposed
// to be bytes, however headers should provide all required information for later decoding
// by the client.
type response struct {
	Status  int                 `json:"status"`
	Body    string              `json:"body"`
	Headers map[string][]string `json:"headers"`
}

// Payload structure holds request and response structure
type Payload struct {
	Response response       `json:"response"`
	Request  requestDetails `json:"request"`
	ID       string         `json:"id"`
}

// encode method encodes all exported Payload fields to bytes
func (p *Payload) encode() ([]byte, error) {
	buf := new(bytes.Buffer)
	enc := gob.NewEncoder(buf)
	err := enc.Encode(p)
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// decodePayload decodes supplied bytes into Payload structure
func decodePayload(data []byte) (*Payload, error) {
	var p *Payload
	buf := bytes.NewBuffer(data)
	dec := gob.NewDecoder(buf)
	err := dec.Decode(&p)
	if err != nil {
		return nil, err
	}
	return p, nil
}

// captureRequest saves request for later playback
func (d *DBClient) captureRequest(req *http.Request) (*http.Response, error) {

	// this is mainly for testing, since when you create
	if req.Body == nil {
		req.Body = ioutil.NopCloser(bytes.NewBuffer([]byte("")))
	}

	reqBody, err := ioutil.ReadAll(req.Body)

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
			"mode":  "capture",
		}).Error("Got error when reading request body")
	}

	// outputting request body if verbose logging is set
	log.WithFields(log.Fields{
		"body": string(reqBody),
		"mode": "capture",
	}).Debug("got request body")

	req.Body = ioutil.NopCloser(bytes.NewBuffer(reqBody))

	// forwarding request
	resp, err := d.doRequest(req)

	if err == nil {
		respBody, err := extractBody(resp)

		if err != nil {

			log.WithFields(log.Fields{
				"error": err.Error(),
				"mode":  "capture",
			}).Error("Failed to copy response body.")

			return resp, err
		}

		// saving response body with request/response meta to cache
		d.save(req, reqBody, resp, respBody)
	}

	// return new response or error here
	return resp, err
}

func copyBody(body io.ReadCloser) (resp1, resp2 io.ReadCloser, err error) {
	var buf bytes.Buffer
	if _, err = buf.ReadFrom(body); err != nil {
		return nil, nil, err
	}
	if err = body.Close(); err != nil {
		return nil, nil, err
	}
	return ioutil.NopCloser(&buf), ioutil.NopCloser(bytes.NewReader(buf.Bytes())), nil
}

func extractBody(resp *http.Response) (extract []byte, err error) {
	save := resp.Body
	savecl := resp.ContentLength

	save, resp.Body, err = copyBody(resp.Body)

	if err != nil {
		return
	}
	defer resp.Body.Close()
	extract, err = ioutil.ReadAll(resp.Body)

	resp.Body = save
	resp.ContentLength = savecl
	if err != nil {
		return nil, err
	}
	return extract, nil
}

// doRequest performs original request and returns response that should be returned to client and error (if there is one)
func (d *DBClient) doRequest(request *http.Request) (*http.Response, error) {

	// We can't have this set. And it only contains "/pkg/net/http/" anyway
	request.RequestURI = ""

	if d.cfg.middleware != "" {
		var payload Payload

		c := NewConstructor(request, payload)
		err := c.ApplyMiddleware(d.cfg.middleware)

		if err != nil {
			log.WithFields(log.Fields{
				"mode":   d.cfg.mode,
				"error":  err.Error(),
				"host":   request.Host,
				"method": request.Method,
				"path":   request.URL.Path,
			}).Error("could not forward request, middleware failed to modify request.")
			return nil, err
		}

		request = c.reconstructRequest()
	}

	resp, err := d.http.Do(request)

	if err != nil {
		log.WithFields(log.Fields{
			"mode":   d.cfg.mode,
			"error":  err.Error(),
			"host":   request.Host,
			"method": request.Method,
			"path":   request.URL.Path,
		}).Error("could not forward request, failed to do an HTTP request.")
		return nil, err
	}

	log.WithFields(log.Fields{
		"mode":   d.cfg.mode,
		"host":   request.Host,
		"method": request.Method,
		"path":   request.URL.Path,
	}).Debug("response from external service got successfuly!")

	resp.Header.Set("hoverfly", "Was-Here")
	return resp, nil

}

// save gets request fingerprint, extracts request body, status code and headers, then saves it to cache
func (d *DBClient) save(req *http.Request, reqBody []byte, resp *http.Response, respBody []byte) {
	// record request here
	key := getRequestFingerprint(req, reqBody)

	if resp == nil {
		resp = emptyResp
	} else {
		responseObj := response{
			Status:  resp.StatusCode,
			Body:    string(respBody),
			Headers: resp.Header,
		}

		log.WithFields(log.Fields{
			"path":          req.URL.Path,
			"rawQuery":      req.URL.RawQuery,
			"requestMethod": req.Method,
			"bodyLen":       len(reqBody),
			"destination":   req.Host,
			"hashKey":       key,
		}).Debug("Capturing")

		requestObj := requestDetails{
			Path:        req.URL.Path,
			Method:      req.Method,
			Destination: req.Host,
			Scheme:      req.URL.Scheme,
			Query:       req.URL.RawQuery,
			Body:        string(reqBody),
			RemoteAddr:  req.RemoteAddr,
			Headers:     req.Header,
		}

		payload := Payload{
			Response: responseObj,
			Request:  requestObj,
			ID:       key,
		}

		bts, err := payload.encode()
		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Error("Failed to serialize payload")
		} else {
			d.cache.Set([]byte(key), bts)
		}
	}
}

// getRequestFingerprint returns request hash
func getRequestFingerprint(req *http.Request, requestBody []byte) string {
	details := requestDetails{
		Path:        req.URL.Path,
		Method:      req.Method,
		Destination: req.Host,
		Query:       req.URL.RawQuery,
		Body:        string(requestBody),
	}

	r := request{details: details}
	return r.hash()
}

// getResponse returns stored response from cache
func (d *DBClient) getResponse(req *http.Request) *http.Response {

	reqBody, err := ioutil.ReadAll(req.Body)

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Got error when reading request body")
	}

	key := getRequestFingerprint(req, reqBody)

	payloadBts, err := d.cache.Get([]byte(key))

	if err == nil {
		// getting cache response
		payload, err := decodePayload(payloadBts)
		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
				"value": string(payloadBts),
				"key":   key,
			}).Error("Failed to decode payload")
			return goproxy.NewResponse(req,
				goproxy.ContentTypeText, http.StatusInternalServerError,
				fmt.Sprintf("Failed to virtualize, got error: %s \n", err.Error()))
		}

		c := NewConstructor(req, *payload)

		if d.cfg.middleware != "" {
			_ = c.ApplyMiddleware(d.cfg.middleware)
		}

		response := c.reconstructResponse()

		log.WithFields(log.Fields{
			"key":         key,
			"mode":        "virtualize",
			"middleware":  d.cfg.middleware,
			"path":        req.URL.Path,
			"rawQuery":    req.URL.RawQuery,
			"method":      req.Method,
			"destination": req.Host,
			"status":      payload.Response.Status,
			"bodyLength":  response.ContentLength,
		}).Info("Response found, returning")

		return response

	}

	log.WithFields(log.Fields{
		"key":         key,
		"error":       err.Error(),
		"query":       req.URL.RawQuery,
		"path":        req.URL.RawPath,
		"destination": req.Host,
		"method":      req.Method,
	}).Warn("Failed to retrieve response from cache")
	// return error? if we return nil - proxy forwards request to original destination
	return goproxy.NewResponse(req,
		goproxy.ContentTypeText, http.StatusPreconditionFailed,
		"Could not find recorded request, please record it first!\n")

}

// modifyRequestResponse modifies outgoing request and then modifies incoming response, neither request nor response
// is saved to cache.
func (d *DBClient) modifyRequestResponse(req *http.Request, middleware string) (*http.Response, error) {

	// modifying request
	resp, err := d.doRequest(req)

	if err != nil {
		return nil, err
	}

	// preparing payload
	bodyBytes, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		log.WithFields(log.Fields{
			"error":      err.Error(),
			"middleware": middleware,
		}).Error("Failed to read response body after sending modified request")
		return nil, err
	}

	r := response{
		Status:  resp.StatusCode,
		Body:    string(bodyBytes),
		Headers: resp.Header,
	}
	payload := Payload{Response: r}

	c := NewConstructor(req, payload)
	// applying middleware to modify response
	err = c.ApplyMiddleware(middleware)

	if err != nil {
		return nil, err
	}

	newResponse := c.reconstructResponse()

	log.WithFields(log.Fields{
		"status":     newResponse.StatusCode,
		"middleware": middleware,
	}).Info("Response modified, returning")

	return newResponse, nil

}
<|endoftext|>


<commit_before>// Copyright 2011 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package twittergo

import (
	"compress/gzip"
	"fmt"
	"github.com/kurrik/json"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

const (
	H_LIMIT              = "X-Rate-Limit-Limit"
	H_LIMIT_REMAIN       = "X-Rate-Limit-Remaining"
	H_LIMIT_RESET        = "X-Rate-Limit-Reset"
	H_MEDIA_LIMIT        = "X-MediaRateLimit-Limit"
	H_MEDIA_LIMIT_REMAIN = "X-MediaRateLimit-Remaining"
	H_MEDIA_LIMIT_RESET  = "X-MediaRateLimit-Reset"
)

const (
	STATUS_OK           = 200
	STATUS_CREATED      = 201
	STATUS_ACCEPTED     = 202
	STATUS_NO_CONTENT   = 204
	STATUS_INVALID      = 400
	STATUS_UNAUTHORIZED = 401
	STATUS_FORBIDDEN    = 403
	STATUS_NOTFOUND     = 404
	STATUS_LIMIT        = 429
	STATUS_GATEWAY      = 502
)

// Error returned if there was an issue parsing the response body.
type ResponseError struct {
	Body string
	Code int
}

func NewResponseError(code int, body string) ResponseError {
	return ResponseError{Code: code, Body: body}
}

func (e ResponseError) Error() string {
	return fmt.Sprintf(
		"Unable to handle response (status code %d): `%v`",
		e.Code,
		e.Body)
}

type Error map[string]interface{}

func (e Error) Code() int64 {
	return e["code"].(int64)
}

func (e Error) Message() string {
	return e["message"].(string)
}

func (e Error) Error() string {
	msg := "Error %v: %v"
	return fmt.Sprintf(msg, e.Code(), e.Message())
}

type Errors map[string]interface{}

func (e Errors) Error() string {
	var (
		msg string = ""
		err Error
		ok  bool
	)
	for _, val := range e["errors"].([]interface{}) {
		if err, ok = val.(map[string]interface{}); ok {
			msg += err.Error() + ". "
		}
	}
	return msg
}

func (e Errors) String() string {
	return e.Error()
}

func (e Errors) Errors() []Error {
	var errs = e["errors"].([]interface{})
	var out = make([]Error, len(errs))
	for i, val := range errs {
		out[i] = Error(val.(map[string]interface{}))
	}
	return out
}

type RateLimitError struct {
	Limit     uint32
	Remaining uint32
	Reset     time.Time
}

func (e RateLimitError) Error() string {
	msg := "Rate limit: %v, Remaining: %v, Reset: %v"
	return fmt.Sprintf(msg, e.Limit, e.Remaining, e.Reset)
}

type APIResponse http.Response

func (r APIResponse) HasRateLimit() bool {
	return r.Header.Get(H_LIMIT) != ""
}

func (r APIResponse) RateLimit() uint32 {
	h := r.Header.Get(H_LIMIT)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) RateLimitRemaining() uint32 {
	h := r.Header.Get(H_LIMIT_REMAIN)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) RateLimitReset() time.Time {
	h := r.Header.Get(H_LIMIT_RESET)
	i, _ := strconv.ParseUint(h, 10, 32)
	t := time.Unix(int64(i), 0)
	return t
}

func (r APIResponse) HasMediaRateLimit() bool {
	return r.Header.Get(H_MEDIA_LIMIT) != ""
}

func (r APIResponse) MediaRateLimit() uint32 {
	h := r.Header.Get(H_MEDIA_LIMIT)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) MediaRateLimitRemaining() uint32 {
	h := r.Header.Get(H_MEDIA_LIMIT_REMAIN)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) MediaRateLimitReset() time.Time {
	h := r.Header.Get(H_MEDIA_LIMIT_RESET)
	i, _ := strconv.ParseUint(h, 10, 32)
	t := time.Unix(int64(i), 0)
	return t
}

func (r APIResponse) readBody() (b []byte, err error) {
	var (
		header string
		reader io.Reader
	)
	defer r.Body.Close()
	header = strings.ToLower(r.Header.Get("Content-Encoding"))
	if header == "" || strings.Index(header, "gzip") == -1 {
		reader = r.Body
	} else {
		if reader, err = gzip.NewReader(r.Body); err != nil {
			return
		}
	}
	b, err = ioutil.ReadAll(reader)
	return
}

func (r APIResponse) ReadBody() string {
	var (
		b   []byte
		err error
	)
	if b, err = r.readBody(); err != nil {
		return ""
	}
	return string(b)
}

// Parses a JSON encoded HTTP response into the supplied interface.
func (r APIResponse) Parse(out interface{}) (err error) {
	var b []byte
	switch r.StatusCode {
	case STATUS_UNAUTHORIZED:
		fallthrough
	case STATUS_NOTFOUND:
		fallthrough
	case STATUS_GATEWAY:
		fallthrough
	case STATUS_FORBIDDEN:
		fallthrough
	case STATUS_INVALID:
		e := &Errors{}
		if b, err = r.readBody(); err != nil {
			return
		}
		if err = json.Unmarshal(b, e); err != nil {
			err = NewResponseError(r.StatusCode, string(b))
		} else {
			err = *e
		}
		return
	case STATUS_LIMIT:
		err = RateLimitError{
			Limit:     r.RateLimit(),
			Remaining: r.RateLimitRemaining(),
			Reset:     r.RateLimitReset(),
		}
		// consume the request body even if we don't need it
		r.readBody()
		return
	case STATUS_NO_CONTENT:
		return
	case STATUS_CREATED:
		fallthrough
	case STATUS_ACCEPTED:
		fallthrough
	case STATUS_OK:
		if b, err = r.readBody(); err != nil {
			return
		}
		err = json.Unmarshal(b, out)
		if err == io.EOF {
			err = nil
		}
	default:
		if b, err = r.readBody(); err != nil {
			return
		}
		err = NewResponseError(r.StatusCode, string(b))
	}
	return
}

// It's a user!
type User map[string]interface{}

func (u User) Id() uint64 {
	id, _ := strconv.ParseUint(u["id_str"].(string), 10, 64)
	return id
}

func (u User) IdStr() string {
	return u["id_str"].(string)
}

func (u User) Name() string {
	return u["name"].(string)
}

func (u User) ScreenName() string {
	return u["screen_name"].(string)
}

// It's a Tweet! (Adorably referred to by the API as a "status").
type Tweet map[string]interface{}

func (t Tweet) Id() (id uint64) {
	var (
		err error
		src = t["id_str"].(string)
	)
	if id, err = strconv.ParseUint(src, 10, 64); err != nil {
		panic(fmt.Sprintf("Could not parse ID: %v", err))
	}
	return
}

func (t Tweet) IdStr() string {
	return t["id_str"].(string)
}

func (t Tweet) Text() string {
	return t["text"].(string)
}

func (t Tweet) User() User {
	return User(t["user"].(map[string]interface{}))
}

func (t Tweet) Language() string {
	return t["lang"].(string)
}

func (t Tweet) CreatedAt() (out time.Time) {
	var (
		err error
		src = t["created_at"].(string)
	)
	if out, err = time.Parse(time.RubyDate, src); err != nil {
		panic(fmt.Sprintf("Could not parse time: %v", err))
	}
	return
}

// It's a less structured list of Tweets!
type Timeline []Tweet

// It's a structured list of Tweets!
type SearchResults map[string]interface{}

func (sr SearchResults) Statuses() []Tweet {
	var a []interface{} = sr["statuses"].([]interface{})
	b := make([]Tweet, len(a))
	for i, v := range a {
		b[i] = v.(map[string]interface{})
	}
	return b
}

func (sr SearchResults) SearchMetadata() map[string]interface{} {
	a := sr["search_metadata"].(map[string]interface{})
	return a
}

func (sr SearchResults) NextQuery() (val url.Values, err error) {
	var (
		sm   map[string]interface{}
		n    interface{}
		next string
		ok   bool
	)
	sm = sr.SearchMetadata()
	if n, ok = sm["next_results"]; !ok {
		err = fmt.Errorf("Could not get next_results from search")
		return
	}
	if next, ok = n.(string); !ok {
		err = fmt.Errorf("Could not convert next_results to str: %v", n)
		return
	}
	if next[0] == '?' {
		next = next[1:]
	}
	val, err = url.ParseQuery(next)
	return
}

// A List!
type List map[string]interface{}

func (l List) User() User {
	return User(l["user"].(map[string]interface{}))
}

func (l List) Id() (id uint64) {
	var (
		err error
		src = l["id_str"].(string)
	)
	if id, err = strconv.ParseUint(src, 10, 64); err != nil {
		panic(fmt.Sprintf("Could not parse ID: %v", err))
	}
	return
}

func (l List) IdStr() string {
	return l["id_str"].(string)
}

func (l List) Mode() string {
	return l["mode"].(string)
}

func (l List) Name() string {
	return l["name"].(string)
}

func (l List) Slug() string {
	return l["slug"].(string)
}

func (l List) SubscriberCount() int64 {
	return l["subscriber_count"].(int64)
}

func (l List) MemberCount() int64 {
	return l["member_count"].(int64)
}

// It's a less structured list of Lists!
type Lists []List

// It's a cursored list of Lists!

type CursoredLists map[string]interface{}

func (cl CursoredLists) NextCursorStr() string {
	return cl["next_cursor_str"].(string)
}

func (cl CursoredLists) PreviousCursorStr() string {
	return cl["previous_cursor_str"].(string)
}

func (cl CursoredLists) Lists() Lists {
	var a []interface{} = cl["lists"].([]interface{})
	b := make([]List, len(a))
	for i, v := range a {
		b[i] = v.(map[string]interface{})
	}
	return b
}

// Nested response structure for video uploads.
type VideoUpload map[string]interface{}

func (v VideoUpload) Type() string {
	return v["video_type"].(string)
}

// Response for media upload requests.
type MediaResponse map[string]interface{}

func (r MediaResponse) MediaId() int64 {
	return r["media_id"].(int64)
}

func (r MediaResponse) Size() int64 {
	return r["size"].(int64)
}

func (r MediaResponse) ExpiresAfterSecs() int32 {
	return r["expires_after_secs"].(int32)
}

func (r MediaResponse) Video() VideoUpload {
	return VideoUpload(r["video"].(map[string]interface{}))
}
<commit_msg>use encoding/json library instead of kurrik/json<commit_after>// Copyright 2011 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package twittergo

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

const (
	H_LIMIT              = "X-Rate-Limit-Limit"
	H_LIMIT_REMAIN       = "X-Rate-Limit-Remaining"
	H_LIMIT_RESET        = "X-Rate-Limit-Reset"
	H_MEDIA_LIMIT        = "X-MediaRateLimit-Limit"
	H_MEDIA_LIMIT_REMAIN = "X-MediaRateLimit-Remaining"
	H_MEDIA_LIMIT_RESET  = "X-MediaRateLimit-Reset"
)

const (
	STATUS_OK           = 200
	STATUS_CREATED      = 201
	STATUS_ACCEPTED     = 202
	STATUS_NO_CONTENT   = 204
	STATUS_INVALID      = 400
	STATUS_UNAUTHORIZED = 401
	STATUS_FORBIDDEN    = 403
	STATUS_NOTFOUND     = 404
	STATUS_LIMIT        = 429
	STATUS_GATEWAY      = 502
)

// Error returned if there was an issue parsing the response body.
type ResponseError struct {
	Body string
	Code int
}

func NewResponseError(code int, body string) ResponseError {
	return ResponseError{Code: code, Body: body}
}

func (e ResponseError) Error() string {
	return fmt.Sprintf(
		"Unable to handle response (status code %d): `%v`",
		e.Code,
		e.Body)
}

type Error map[string]interface{}

func (e Error) Code() int64 {
	return e["code"].(int64)
}

func (e Error) Message() string {
	return e["message"].(string)
}

func (e Error) Error() string {
	msg := "Error %v: %v"
	return fmt.Sprintf(msg, e.Code(), e.Message())
}

type Errors map[string]interface{}

func (e Errors) Error() string {
	var (
		msg string = ""
		err Error
		ok  bool
	)
	for _, val := range e["errors"].([]interface{}) {
		if err, ok = val.(map[string]interface{}); ok {
			msg += err.Error() + ". "
		}
	}
	return msg
}

func (e Errors) String() string {
	return e.Error()
}

func (e Errors) Errors() []Error {
	var errs = e["errors"].([]interface{})
	var out = make([]Error, len(errs))
	for i, val := range errs {
		out[i] = Error(val.(map[string]interface{}))
	}
	return out
}

type RateLimitError struct {
	Limit     uint32
	Remaining uint32
	Reset     time.Time
}

func (e RateLimitError) Error() string {
	msg := "Rate limit: %v, Remaining: %v, Reset: %v"
	return fmt.Sprintf(msg, e.Limit, e.Remaining, e.Reset)
}

type APIResponse http.Response

func (r APIResponse) HasRateLimit() bool {
	return r.Header.Get(H_LIMIT) != ""
}

func (r APIResponse) RateLimit() uint32 {
	h := r.Header.Get(H_LIMIT)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) RateLimitRemaining() uint32 {
	h := r.Header.Get(H_LIMIT_REMAIN)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) RateLimitReset() time.Time {
	h := r.Header.Get(H_LIMIT_RESET)
	i, _ := strconv.ParseUint(h, 10, 32)
	t := time.Unix(int64(i), 0)
	return t
}

func (r APIResponse) HasMediaRateLimit() bool {
	return r.Header.Get(H_MEDIA_LIMIT) != ""
}

func (r APIResponse) MediaRateLimit() uint32 {
	h := r.Header.Get(H_MEDIA_LIMIT)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) MediaRateLimitRemaining() uint32 {
	h := r.Header.Get(H_MEDIA_LIMIT_REMAIN)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) MediaRateLimitReset() time.Time {
	h := r.Header.Get(H_MEDIA_LIMIT_RESET)
	i, _ := strconv.ParseUint(h, 10, 32)
	t := time.Unix(int64(i), 0)
	return t
}

func (r APIResponse) readBody() (b []byte, err error) {
	var (
		header string
		reader io.Reader
	)
	defer r.Body.Close()
	header = strings.ToLower(r.Header.Get("Content-Encoding"))
	if header == "" || strings.Index(header, "gzip") == -1 {
		reader = r.Body
	} else {
		if reader, err = gzip.NewReader(r.Body); err != nil {
			return
		}
	}
	b, err = ioutil.ReadAll(reader)
	return
}

func (r APIResponse) ReadBody() string {
	var (
		b   []byte
		err error
	)
	if b, err = r.readBody(); err != nil {
		return ""
	}
	return string(b)
}

// Parses a JSON encoded HTTP response into the supplied interface.
func (r APIResponse) Parse(out interface{}) (err error) {
	var b []byte
	switch r.StatusCode {
	case STATUS_UNAUTHORIZED:
		fallthrough
	case STATUS_NOTFOUND:
		fallthrough
	case STATUS_GATEWAY:
		fallthrough
	case STATUS_FORBIDDEN:
		fallthrough
	case STATUS_INVALID:
		e := &Errors{}
		if b, err = r.readBody(); err != nil {
			return
		}
		if err = json.Unmarshal(b, e); err != nil {
			err = NewResponseError(r.StatusCode, string(b))
		} else {
			err = *e
		}
		return
	case STATUS_LIMIT:
		err = RateLimitError{
			Limit:     r.RateLimit(),
			Remaining: r.RateLimitRemaining(),
			Reset:     r.RateLimitReset(),
		}
		// consume the request body even if we don't need it
		r.readBody()
		return
	case STATUS_NO_CONTENT:
		return
	case STATUS_CREATED:
		fallthrough
	case STATUS_ACCEPTED:
		fallthrough
	case STATUS_OK:
		if b, err = r.readBody(); err != nil {
			return
		}
		err = json.Unmarshal(b, out)
		if err == io.EOF {
			err = nil
		}
	default:
		if b, err = r.readBody(); err != nil {
			return
		}
		err = NewResponseError(r.StatusCode, string(b))
	}
	return
}

// It's a user!
type User map[string]interface{}

func (u User) Id() uint64 {
	id, _ := strconv.ParseUint(u["id_str"].(string), 10, 64)
	return id
}

func (u User) IdStr() string {
	return u["id_str"].(string)
}

func (u User) Name() string {
	return u["name"].(string)
}

func (u User) ScreenName() string {
	return u["screen_name"].(string)
}

// It's a Tweet! (Adorably referred to by the API as a "status").
type Tweet map[string]interface{}

func (t Tweet) Id() (id uint64) {
	var (
		err error
		src = t["id_str"].(string)
	)
	if id, err = strconv.ParseUint(src, 10, 64); err != nil {
		panic(fmt.Sprintf("Could not parse ID: %v", err))
	}
	return
}

func (t Tweet) IdStr() string {
	return t["id_str"].(string)
}

func (t Tweet) Text() string {
	return t["text"].(string)
}

func (t Tweet) User() User {
	return User(t["user"].(map[string]interface{}))
}

func (t Tweet) Language() string {
	return t["lang"].(string)
}

func (t Tweet) CreatedAt() (out time.Time) {
	var (
		err error
		src = t["created_at"].(string)
	)
	if out, err = time.Parse(time.RubyDate, src); err != nil {
		panic(fmt.Sprintf("Could not parse time: %v", err))
	}
	return
}

// It's a less structured list of Tweets!
type Timeline []Tweet

// It's a structured list of Tweets!
type SearchResults map[string]interface{}

func (sr SearchResults) Statuses() []Tweet {
	var a []interface{} = sr["statuses"].([]interface{})
	b := make([]Tweet, len(a))
	for i, v := range a {
		b[i] = v.(map[string]interface{})
	}
	return b
}

func (sr SearchResults) SearchMetadata() map[string]interface{} {
	a := sr["search_metadata"].(map[string]interface{})
	return a
}

func (sr SearchResults) NextQuery() (val url.Values, err error) {
	var (
		sm   map[string]interface{}
		n    interface{}
		next string
		ok   bool
	)
	sm = sr.SearchMetadata()
	if n, ok = sm["next_results"]; !ok {
		err = fmt.Errorf("Could not get next_results from search")
		return
	}
	if next, ok = n.(string); !ok {
		err = fmt.Errorf("Could not convert next_results to str: %v", n)
		return
	}
	if next[0] == '?' {
		next = next[1:]
	}
	val, err = url.ParseQuery(next)
	return
}

// A List!
type List map[string]interface{}

func (l List) User() User {
	return User(l["user"].(map[string]interface{}))
}

func (l List) Id() (id uint64) {
	var (
		err error
		src = l["id_str"].(string)
	)
	if id, err = strconv.ParseUint(src, 10, 64); err != nil {
		panic(fmt.Sprintf("Could not parse ID: %v", err))
	}
	return
}

func (l List) IdStr() string {
	return l["id_str"].(string)
}

func (l List) Mode() string {
	return l["mode"].(string)
}

func (l List) Name() string {
	return l["name"].(string)
}

func (l List) Slug() string {
	return l["slug"].(string)
}

func (l List) SubscriberCount() int64 {
	return l["subscriber_count"].(int64)
}

func (l List) MemberCount() int64 {
	return l["member_count"].(int64)
}

// It's a less structured list of Lists!
type Lists []List

// It's a cursored list of Lists!

type CursoredLists map[string]interface{}

func (cl CursoredLists) NextCursorStr() string {
	return cl["next_cursor_str"].(string)
}

func (cl CursoredLists) PreviousCursorStr() string {
	return cl["previous_cursor_str"].(string)
}

func (cl CursoredLists) Lists() Lists {
	var a []interface{} = cl["lists"].([]interface{})
	b := make([]List, len(a))
	for i, v := range a {
		b[i] = v.(map[string]interface{})
	}
	return b
}

// Nested response structure for video uploads.
type VideoUpload map[string]interface{}

func (v VideoUpload) Type() string {
	return v["video_type"].(string)
}

// Response for media upload requests.
type MediaResponse map[string]interface{}

func (r MediaResponse) MediaId() int64 {
	return r["media_id"].(int64)
}

func (r MediaResponse) Size() int64 {
	return r["size"].(int64)
}

func (r MediaResponse) ExpiresAfterSecs() int32 {
	return r["expires_after_secs"].(int32)
}

func (r MediaResponse) Video() VideoUpload {
	return VideoUpload(r["video"].(map[string]interface{}))
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"

	log "github.com/Sirupsen/logrus"
	"github.com/elazarl/goproxy"
	"github.com/garyburd/redigo/redis"
)

type DBClient struct {
	cache Cache
	http  *http.Client
}

// request holds structure for request
type request struct {
	details requestDetails
}

var emptyResp = &http.Response{}

// requestDetails stores information about request, it's used for creating unique hash and also as a payload structure
type requestDetails struct {
	Path        string              `json:"path"`
	Method      string              `json:"method"`
	Destination string              `json:"destination"`
	Query       string              `json:"query"`
	Body        string              `json:"body"`
	RemoteAddr  string              `json:"remoteAddr"`
	Headers     map[string][]string `json:"headers"`
}

// hash returns unique hash key for request
func (r *request) hash() string {
	h := md5.New()
	io.WriteString(h, fmt.Sprintf("%s%s%s%s", r.details.Destination, r.details.Path, r.details.Method, r.details.Query))
	return fmt.Sprintf("%x", h.Sum(nil))
}

// res structure hold response body from external service, body is not decoded and is supposed
// to be bytes, however headers should provide all required information for later decoding
// by the client.
type response struct {
	Status  int                 `json:"status"`
	Body    string              `json:"body"`
	Headers map[string][]string `json:"headers"`
}

// Payload structure holds request and response structure
type Payload struct {
	Response response       `json:"response"`
	Request  requestDetails `json:"request"`
	ID       string         `json:"id"`
}

// recordRequest saves request for later playback
func (d *DBClient) captureRequest(req *http.Request) (*http.Response, error) {

	// forwarding request
	resp, err := d.doRequest(req)

	if err == nil {

		// getting response body
		body, err := httputil.DumpResponse(resp, true)
		if err != nil {
			// copying the response body did not work
			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
				}).Error("Failed to copy response body.")
			}
		}

		// saving response body with request/response meta to cache
		go d.save(req, resp, body)
	}

	// return new response or error here
	return resp, err
}

// doRequest performs original request and returns response that should be returned to client and error (if there is one)
func (d *DBClient) doRequest(request *http.Request) (*http.Response, error) {
	// We can't have this set. And it only contains "/pkg/net/http/" anyway
	request.RequestURI = ""

	resp, err := d.http.Do(request)

	if err != nil {
		log.WithFields(log.Fields{
			"error":  err.Error(),
			"host":   request.Host,
			"method": request.Method,
			"path":   request.URL.Path,
		}).Error("Could not forward request.")
		return nil, err
	}

	log.WithFields(log.Fields{
		"host":   request.Host,
		"method": request.Method,
		"path":   request.URL.Path,
	}).Info("Response got successfuly!")

	resp.Header.Set("Gen-proxy", "Was-Here")
	return resp, nil

}

// save gets request fingerprint, extracts request body, status code and headers, then saves it to cache
func (d *DBClient) save(req *http.Request, resp *http.Response, respBody []byte) {
	// record request here
	key := getRequestFingerprint(req)

	if resp == nil {
		resp = emptyResp
	} else {
		responseObj := response{
			Status:  resp.StatusCode,
			Body:    string(respBody),
			Headers: resp.Header,
		}

		log.WithFields(log.Fields{
			"path":          req.URL.Path,
			"rawQuery":      req.URL.RawQuery,
			"requestMethod": req.Method,
			"destination":   req.Host,
			"hashKey":       key,
		}).Info("Recording")

		requestObj := requestDetails{
			Path:        req.URL.Path,
			Method:      req.Method,
			Destination: req.Host,
			Query:       req.URL.RawQuery,
		}

		payload := Payload{
			Response: responseObj,
			Request:  requestObj,
			ID:       key,
		}
		// converting it to json bytes
		bts, err := json.Marshal(payload)

		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Error("Failed to marshal json")
		} else {
			d.cache.set(key, bts)
		}
	}

}

// getAllRecordsRaw returns raw (json string) for all records
func (d *DBClient) getAllRecordsRaw() ([]string, error) {
	keys, err := d.cache.getAllKeys()

	if err == nil {

		// checking if there are any records
		if len(keys) == 0 {
			return nil, nil
		}

		jsonStrs, err := d.cache.getAllValues(keys)

		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Error("Failed to get all values (raw)")
			return nil, err
		} else {
			return jsonStrs, nil
		}

	} else {
		return nil, err
	}
}

// getAllRecords returns all stored
func (d *DBClient) getAllRecords() ([]Payload, error) {
	var payloads []Payload

	jsonStrs, err := d.getAllRecordsRaw()

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Failed to get all values")
	} else {

		if jsonStrs != nil {
			for _, v := range jsonStrs {
				var pl Payload
				err = json.Unmarshal([]byte(v), &pl)
				if err != nil {
					log.WithFields(log.Fields{
						"error": err.Error(),
						"json":  v,
					}).Warning("Failed to deserialize json")
				} else {
					payloads = append(payloads, pl)
				}
			}
		}
	}

	return payloads, err

}

// deleteAllRecords deletes all recorded requests
func (d *DBClient) deleteAllRecords() error {
	keys, err := d.cache.getAllKeys()

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Warning("Failed to get keys, cannot delete all records")
		return err
	} else {
		for _, v := range keys {
			d.cache.delete(v)
		}
		return nil
	}
}

// getRequestFingerprint returns request hash
func getRequestFingerprint(req *http.Request) string {
	details := requestDetails{Path: req.URL.Path, Method: req.Method, Destination: req.Host, Query: req.URL.RawQuery}
	r := request{details: details}
	return r.hash()
}

// getResponse returns stored response from cache
func (d *DBClient) getResponse(req *http.Request) *http.Response {
	log.Info("Returning response")

	key := getRequestFingerprint(req)
	var payload Payload

	payloadBts, err := redis.Bytes(d.cache.get(key))

	if err == nil {
		log.Info("Decoding bytes")
		// getting cache response
		err = json.Unmarshal(payloadBts, &payload)
		if err != nil {
			log.Error(err)
			// what now?
		}

		c := NewConstructor(req, payload)

		if AppConfig.middleware != "" {
			_ = c.ApplyMiddleware(AppConfig.middleware)
		}

		response := c.reconstructResponse()

		log.WithFields(log.Fields{
			"key":        key,
			"status":     payload.Response.Status,
			"bodyLength": response.ContentLength,
		}).Info("Response found, returning")

		return response

	} else {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Failed to retrieve response from cache")
		// return error? if we return nil - proxy forwards request to original destination
		return goproxy.NewResponse(req,
			goproxy.ContentTypeText, http.StatusPreconditionFailed,
			"Coudldn't find recorded request, please record it first!")
	}

}
<commit_msg>cleanup<commit_after>package main

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/http/httputil"

	log "github.com/Sirupsen/logrus"
	"github.com/elazarl/goproxy"
	"github.com/garyburd/redigo/redis"
)

type DBClient struct {
	cache Cache
	http  *http.Client
}

// request holds structure for request
type request struct {
	details requestDetails
}

var emptyResp = &http.Response{}

// requestDetails stores information about request, it's used for creating unique hash and also as a payload structure
type requestDetails struct {
	Path        string              `json:"path"`
	Method      string              `json:"method"`
	Destination string              `json:"destination"`
	Query       string              `json:"query"`
	Body        string              `json:"body"`
	RemoteAddr  string              `json:"remoteAddr"`
	Headers     map[string][]string `json:"headers"`
}

// hash returns unique hash key for request
func (r *request) hash() string {
	h := md5.New()
	io.WriteString(h, fmt.Sprintf("%s%s%s%s", r.details.Destination, r.details.Path, r.details.Method, r.details.Query))
	return fmt.Sprintf("%x", h.Sum(nil))
}

// res structure hold response body from external service, body is not decoded and is supposed
// to be bytes, however headers should provide all required information for later decoding
// by the client.
type response struct {
	Status  int                 `json:"status"`
	Body    string              `json:"body"`
	Headers map[string][]string `json:"headers"`
}

// Payload structure holds request and response structure
type Payload struct {
	Response response       `json:"response"`
	Request  requestDetails `json:"request"`
	ID       string         `json:"id"`
}

// recordRequest saves request for later playback
func (d *DBClient) captureRequest(req *http.Request) (*http.Response, error) {

	// forwarding request
	resp, err := d.doRequest(req)

	if err == nil {

		// getting response body
		body, err := httputil.DumpResponse(resp, true)
		if err != nil {
			// copying the response body did not work
			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
				}).Error("Failed to copy response body.")
			}
		}

		// saving response body with request/response meta to cache
		go d.save(req, resp, body)
	}

	// return new response or error here
	return resp, err
}

// doRequest performs original request and returns response that should be returned to client and error (if there is one)
func (d *DBClient) doRequest(request *http.Request) (*http.Response, error) {
	// We can't have this set. And it only contains "/pkg/net/http/" anyway
	request.RequestURI = ""

	resp, err := d.http.Do(request)

	if err != nil {
		log.WithFields(log.Fields{
			"error":  err.Error(),
			"host":   request.Host,
			"method": request.Method,
			"path":   request.URL.Path,
		}).Error("Could not forward request.")
		return nil, err
	}

	log.WithFields(log.Fields{
		"host":   request.Host,
		"method": request.Method,
		"path":   request.URL.Path,
	}).Info("Response got successfuly!")

	resp.Header.Set("Gen-proxy", "Was-Here")
	return resp, nil

}

// save gets request fingerprint, extracts request body, status code and headers, then saves it to cache
func (d *DBClient) save(req *http.Request, resp *http.Response, respBody []byte) {
	// record request here
	key := getRequestFingerprint(req)

	if resp == nil {
		resp = emptyResp
	} else {
		responseObj := response{
			Status:  resp.StatusCode,
			Body:    string(respBody),
			Headers: resp.Header,
		}

		log.WithFields(log.Fields{
			"path":          req.URL.Path,
			"rawQuery":      req.URL.RawQuery,
			"requestMethod": req.Method,
			"destination":   req.Host,
			"hashKey":       key,
		}).Info("Recording")

		requestObj := requestDetails{
			Path:        req.URL.Path,
			Method:      req.Method,
			Destination: req.Host,
			Query:       req.URL.RawQuery,
		}

		payload := Payload{
			Response: responseObj,
			Request:  requestObj,
			ID:       key,
		}
		// converting it to json bytes
		bts, err := json.Marshal(payload)

		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Error("Failed to marshal json")
		} else {
			d.cache.set(key, bts)
		}
	}

}

// getAllRecordsRaw returns raw (json string) for all records
func (d *DBClient) getAllRecordsRaw() ([]string, error) {
	keys, err := d.cache.getAllKeys()

	if err == nil {

		// checking if there are any records
		if len(keys) == 0 {
			return nil, nil
		}

		jsonStrs, err := d.cache.getAllValues(keys)

		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Error("Failed to get all values (raw)")
			return nil, err
		} else {
			return jsonStrs, nil
		}

	} else {
		return nil, err
	}
}

// getAllRecords returns all stored
func (d *DBClient) getAllRecords() ([]Payload, error) {
	var payloads []Payload

	jsonStrs, err := d.getAllRecordsRaw()

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Failed to get all values")
	} else {

		if jsonStrs != nil {
			for _, v := range jsonStrs {
				var pl Payload
				err = json.Unmarshal([]byte(v), &pl)
				if err != nil {
					log.WithFields(log.Fields{
						"error": err.Error(),
						"json":  v,
					}).Warning("Failed to deserialize json")
				} else {
					payloads = append(payloads, pl)
				}
			}
		}
	}

	return payloads, err

}

// deleteAllRecords deletes all recorded requests
func (d *DBClient) deleteAllRecords() error {
	keys, err := d.cache.getAllKeys()

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Warning("Failed to get keys, cannot delete all records")
		return err
	} else {
		for _, v := range keys {
			d.cache.delete(v)
		}
		return nil
	}
}

// getRequestFingerprint returns request hash
func getRequestFingerprint(req *http.Request) string {
	details := requestDetails{Path: req.URL.Path, Method: req.Method, Destination: req.Host, Query: req.URL.RawQuery}
	r := request{details: details}
	return r.hash()
}

// getResponse returns stored response from cache
func (d *DBClient) getResponse(req *http.Request) *http.Response {
	log.Info("Returning response")

	key := getRequestFingerprint(req)
	var payload Payload

	payloadBts, err := redis.Bytes(d.cache.get(key))

	if err == nil {
		log.Info("Decoding bytes")
		// getting cache response
		err = json.Unmarshal(payloadBts, &payload)
		if err != nil {
			log.Error(err)
			// what now?
		}

		c := NewConstructor(req, payload)

		if AppConfig.middleware != "" {
			_ = c.ApplyMiddleware(AppConfig.middleware)
		}

		response := c.reconstructResponse()

		log.WithFields(log.Fields{
			"key":        key,
			"status":     payload.Response.Status,
			"bodyLength": response.ContentLength,
		}).Info("Response found, returning")

		return response

	} else {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Failed to retrieve response from cache")
		// return error? if we return nil - proxy forwards request to original destination
		return goproxy.NewResponse(req,
			goproxy.ContentTypeText, http.StatusPreconditionFailed,
			"Coudldn't find recorded request, please record it first!")
	}

}
<|endoftext|>


<commit_before>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"strings"
	"time"

	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
)

var conn dbox.IConnection
var count int
var ratioTableName string

var (
	ratiototrf      = float64(0.392)
	sourcetablename = "salespls-summary"
	calctablename   = "salespls-summary"
	desttablename   = "salespls-summary"
	t0              time.Time
	masters         = toolkit.M{}
	sgaalloc        = map[string]float64{
		"EXP": 0.08,
		"I4":  0.08,
		"I6":  0.105,
	}
)

type plalloc struct {
	ID                     string `bson:"_id" json:"_id"`
	Key                    string
	Key1, Key2, Key3       string
	Ref1                   float64
	Current                float64
	Expect                 float64
	Absorbed               float64
	Ratio1, Ratio2, Ratio3 float64
}

type allocmap map[string]*plalloc

var (
	rdfiscals    = allocmap{}
	branchtotals = allocmap{}
	trxsrc       = "nakulrd"
)

func main() {
	setinitialconnection()
	conn.NewQuery().From(desttablename).Where(dbox.Eq("key.trxsrc", trxsrc)).Delete().Exec(nil)

	prepmastercalc()
	//buildratio()
	processTable()
}

func buildratio() {
	connratio, _ := modules.GetDboxIConnection("db_godrej")
	defer connratio.Close()

	ctrx, _ := connratio.NewQuery().From(calctablename).
		Where(dbox.Eq("key.customer_channelid", "I1")).
		Select().Cursor(nil)
	i := 0
	count := ctrx.Count()
	t0 := time.Now()
	mstone := 0
	for {
		mr := toolkit.M{}
		if ef := ctrx.Fetch(&mr, 1, false); ef != nil {
			break
		}
		i++
		makeProgressLog("Update ratio sales", i, count, 5, &mstone, t0)

		key := mr.Get("key", toolkit.M{}).(toolkit.M)
		fiscal := key.GetString("date_fiscal")
		channelid := key.GetString("customer_channelid")
		//kaid := key.GetString("customer_customergroup")
		//brand := key.GetString("product_brand")
		branchid := key.GetString("customer_branchid")
		subchannel := key.GetString("customer_reportsubchannel")

		sales := mr.GetFloat64("PL8A")
		disc := mr.GetFloat64("PL7A")
		spg := mr.GetFloat64("PL31")
		promo := mr.GetFloat64("PL29A")

		//-- non rd and has data pls make trfrd
		if channelid != "I1" && (disc != 0 || spg != 0 || promo != 0) {

		} else
		//-- RD
		if channelid == "I1" {
			adjustAllocs(&branchtotals, fiscal+"_"+branchid+"_"+subchannel,
				0, 0, 0, sales)
			adjustAllocs(&rdfiscals, fiscal, 0, 0, 0, sales)
		}
	}

	for k, v := range branchtotals {
		fiscal := strings.Split(k, "_")[0]
		rdfiscal := rdfiscals[fiscal]
		v.Ratio1 = toolkit.Div(v.Ref1, rdfiscal.Ref1)
	}
}

func processTable() {
	connsave, _ := modules.GetDboxIConnection("db_godrej")
	defer connsave.Close()
	qsave := connsave.NewQuery().SetConfig("multiexec", true).From(desttablename).Save()

	connselect, _ := modules.GetDboxIConnection("db_godrej")
	defer connselect.Close()

	cursor, _ := connselect.NewQuery().
		From(calctablename).
		Where(dbox.Ne("key.customer_channelid", "I1"), dbox.Ne("key.trxsrc", trxsrc)).
		Select().Cursor(nil)
	defer cursor.Close()

	i := 0
	count := cursor.Count()
	mstone := 0
	t0 = time.Now()
	for {
		mr := toolkit.M{}
		e := cursor.Fetch(&mr, 1, false)
		if e != nil || i >= count {
			break
		}
		i++
		makeProgressLog("Processing", i, count, 5, &mstone, t0)

		mrid := mr.GetString("_id")
		key := mr.Get("key", toolkit.M{}).(toolkit.M)
		branchid := key.GetString("customer_branchid")
		branchname := key.GetString("customer_branchname")

		discount := mr.GetFloat64("PL7A") * ratiototrf
		spg := mr.GetFloat64("PL31") * ratiototrf
		promo := mr.GetFloat64("Pl29A") * ratiototrf

		if discount != 0 || spg != 0 || promo != 0 {
			if branchid != "CD02" || branchid != "CD04" || branchid != "CD11" ||
				branchid != "CD09" {
				branchid = "CD02"
				branchname = "Jakarta"
			}

			mreverse := toolkit.M{}
			mreverse.Set("_id", mrid+"_rd_reverse")
			mreversekey := toolkit.M{}
			for k, v := range key {
				mreversekey.Set(k, v)
			}
			mreversekey.Set("customer_customergroup", "")
			mreversekey.Set("customer_customergroupname", "")
			mreversekey.Set("trxsrc", trxsrc)
			mreverse.Set("key", mreversekey)
			mreverse.Set("PL7A", -discount)
			mreverse.Set("PL29A32", -spg)
			mreverse.Set("PL31C", -promo)
			gdrj.CalcSum(mreverse, masters)
			esave := qsave.Exec(toolkit.M{}.Set("data", mreverse))
			if esave != nil {
				toolkit.Printfn("Error: " + esave.Error())
				os.Exit(100)
			}

			mrd := toolkit.M{}
			mrd.Set("_id", mrid+"_rd")
			mrdkey := toolkit.M{}
			for k, v := range key {
				mrdkey.Set(k, v)
			}
			mrdkey.Set("customer_channelid", "I1")
			mrdkey.Set("customer_reportchannel", "RD")
			mrdkey.Set("customer_customertype", "RD")
			mrdkey.Set("customer_channelname", "RD")
			mrdkey.Set("customer_branchid", branchid)
			mrdkey.Set("customer_branchname", branchname)
			mrdkey.Set("customer_branchgroup", branchname)
			mrdkey.Set("customer_reportsubchannel", "SPG & Promo Allocation")
			mrdkey.Set("customer_customergroup", "")
			mrdkey.Set("customer_customergroupname", "")
			mrdkey.Set("trxsrc", trxsrc)

			mrd.Set("key", mrdkey)
			mrd.Set("PL7A", discount)
			mrd.Set("PL29A32", spg)
			mrd.Set("PL31C", promo)
			gdrj.CalcSum(mrd, masters)
			esave = qsave.Exec(toolkit.M{}.Set("data", mrd))
			if esave != nil {
				toolkit.Printfn("Error: " + esave.Error())
				os.Exit(100)
			}
		}
	}
}

func adjustAllocs(allocsmap *allocmap, key string, current, expect, absorbed, ref1 float64) {
	allocs := *allocsmap
	alloc := allocs[key]
	if alloc == nil {
		alloc = new(plalloc)
		alloc.Key = key
		alloc.ID = key
	}
	alloc.Current += current
	alloc.Expect += expect
	alloc.Ref1 += ref1
	alloc.Absorbed += absorbed
	allocs[key] = alloc
	*allocsmap = allocs
}

func makeProgressLog(reference string, i, count, step int, current *int, tstart time.Time) int {
	perstep := count * step / 100
	icurrent := *current
	if icurrent == 0 {
		icurrent = perstep
	}
	pct := i * 100 / count
	if i >= icurrent {
		toolkit.Printfn("%s, %d of %d [%d pct] in %s",
			reference, i, count, pct, time.Since(tstart).String())
		icurrent += perstep
	}
	*current = icurrent
	return icurrent
}

func buildmap(holder interface{},
	fnModel func() orm.IModel,
	filter *dbox.Filter,
	fnIter func(holder interface{}, obj interface{})) interface{} {
	crx, ecrx := gdrj.Find(fnModel(), filter, nil)
	if ecrx != nil {
		toolkit.Printfn("Cursor Error: %s", ecrx.Error())
		os.Exit(100)
	}
	defer crx.Close()
	for {
		s := fnModel()
		e := crx.Fetch(s, 1, false)
		if e != nil {
			break
		}
		fnIter(holder, s)
	}
	return holder
}

func prepmastercalc() {
	toolkit.Println("--> PL MODEL")
	masters.Set("plmodel", buildmap(map[string]*gdrj.PLModel{},
		func() orm.IModel {
			return new(gdrj.PLModel)
		},
		nil,
		func(holder, obj interface{}) {
			h := holder.(map[string]*gdrj.PLModel)
			o := obj.(*gdrj.PLModel)
			h[o.ID] = o
		}).(map[string]*gdrj.PLModel))
}

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}
<commit_msg>test ae<commit_after>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"strings"
	"time"

	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
)

var conn dbox.IConnection
var count int
var ratioTableName string

var (
	ratiototrf      = float64(0.392)
	sourcetablename = "salespls-summary"
	calctablename   = "salespls-summary"
	desttablename   = "salespls-summary"
	t0              time.Time
	masters         = toolkit.M{}
	sgaalloc        = map[string]float64{
		"EXP": 0.08,
		"I4":  0.08,
		"I6":  0.105,
	}
)

type plalloc struct {
	ID                     string `bson:"_id" json:"_id"`
	Key                    string
	Key1, Key2, Key3       string
	Ref1                   float64
	Current                float64
	Expect                 float64
	Absorbed               float64
	Ratio1, Ratio2, Ratio3 float64
}

type allocmap map[string]*plalloc

var (
	rdfiscals    = allocmap{}
	branchtotals = allocmap{}
	trxsrc       = "nakulrd"
)

func main() {
	setinitialconnection()
	conn.NewQuery().From(desttablename).Where(dbox.Eq("key.trxsrc", trxsrc)).Delete().Exec(nil)

	prepmastercalc()
	//buildratio()
	//processTable()
}

func buildratio() {
	connratio, _ := modules.GetDboxIConnection("db_godrej")
	defer connratio.Close()

	ctrx, _ := connratio.NewQuery().From(calctablename).
		Where(dbox.Eq("key.customer_channelid", "I1")).
		Select().Cursor(nil)
	i := 0
	count := ctrx.Count()
	t0 := time.Now()
	mstone := 0
	for {
		mr := toolkit.M{}
		if ef := ctrx.Fetch(&mr, 1, false); ef != nil {
			break
		}
		i++
		makeProgressLog("Update ratio sales", i, count, 5, &mstone, t0)

		key := mr.Get("key", toolkit.M{}).(toolkit.M)
		fiscal := key.GetString("date_fiscal")
		channelid := key.GetString("customer_channelid")
		//kaid := key.GetString("customer_customergroup")
		//brand := key.GetString("product_brand")
		branchid := key.GetString("customer_branchid")
		subchannel := key.GetString("customer_reportsubchannel")

		sales := mr.GetFloat64("PL8A")
		disc := mr.GetFloat64("PL7A")
		spg := mr.GetFloat64("PL31")
		promo := mr.GetFloat64("PL29A")

		//-- non rd and has data pls make trfrd
		if channelid != "I1" && (disc != 0 || spg != 0 || promo != 0) {

		} else
		//-- RD
		if channelid == "I1" {
			adjustAllocs(&branchtotals, fiscal+"_"+branchid+"_"+subchannel,
				0, 0, 0, sales)
			adjustAllocs(&rdfiscals, fiscal, 0, 0, 0, sales)
		}
	}

	for k, v := range branchtotals {
		fiscal := strings.Split(k, "_")[0]
		rdfiscal := rdfiscals[fiscal]
		v.Ratio1 = toolkit.Div(v.Ref1, rdfiscal.Ref1)
	}
}

func processTable() {
	connsave, _ := modules.GetDboxIConnection("db_godrej")
	defer connsave.Close()
	qsave := connsave.NewQuery().SetConfig("multiexec", true).From(desttablename).Save()

	connselect, _ := modules.GetDboxIConnection("db_godrej")
	defer connselect.Close()

	cursor, _ := connselect.NewQuery().
		From(calctablename).
		Where(dbox.Ne("key.customer_channelid", "I1"), dbox.Ne("key.trxsrc", trxsrc)).
		Select().Cursor(nil)
	defer cursor.Close()

	i := 0
	count := cursor.Count()
	mstone := 0
	t0 = time.Now()
	for {
		mr := toolkit.M{}
		e := cursor.Fetch(&mr, 1, false)
		if e != nil || i >= count {
			break
		}
		i++
		makeProgressLog("Processing", i, count, 5, &mstone, t0)

		mrid := mr.GetString("_id")
		key := mr.Get("key", toolkit.M{}).(toolkit.M)
		branchid := key.GetString("customer_branchid")
		branchname := key.GetString("customer_branchname")

		discount := mr.GetFloat64("PL7A") * ratiototrf
		spg := mr.GetFloat64("PL31") * ratiototrf
		promo := mr.GetFloat64("Pl29A") * ratiototrf

		if discount != 0 || spg != 0 || promo != 0 {
			if branchid != "CD02" || branchid != "CD04" || branchid != "CD11" ||
				branchid != "CD09" {
				branchid = "CD02"
				branchname = "Jakarta"
			}

			mreverse := toolkit.M{}
			mreverse.Set("_id", mrid+"_rd_reverse")
			mreversekey := toolkit.M{}
			for k, v := range key {
				mreversekey.Set(k, v)
			}
			mreversekey.Set("customer_customergroup", "")
			mreversekey.Set("customer_customergroupname", "")
			mreversekey.Set("trxsrc", trxsrc)
			mreverse.Set("key", mreversekey)
			mreverse.Set("PL7A", -discount)
			mreverse.Set("PL29A32", -spg)
			mreverse.Set("PL31C", -promo)
			gdrj.CalcSum(mreverse, masters)
			esave := qsave.Exec(toolkit.M{}.Set("data", mreverse))
			if esave != nil {
				toolkit.Printfn("Error: " + esave.Error())
				os.Exit(100)
			}

			mrd := toolkit.M{}
			mrd.Set("_id", mrid+"_rd")
			mrdkey := toolkit.M{}
			for k, v := range key {
				mrdkey.Set(k, v)
			}
			mrdkey.Set("customer_channelid", "I1")
			mrdkey.Set("customer_reportchannel", "RD")
			mrdkey.Set("customer_customertype", "RD")
			mrdkey.Set("customer_channelname", "RD")
			mrdkey.Set("customer_branchid", branchid)
			mrdkey.Set("customer_branchname", branchname)
			mrdkey.Set("customer_branchgroup", branchname)
			mrdkey.Set("customer_reportsubchannel", "SPG & Promo Allocation")
			mrdkey.Set("customer_customergroup", "")
			mrdkey.Set("customer_customergroupname", "")
			mrdkey.Set("trxsrc", trxsrc)

			mrd.Set("key", mrdkey)
			mrd.Set("PL7A", discount)
			mrd.Set("PL29A32", spg)
			mrd.Set("PL31C", promo)
			gdrj.CalcSum(mrd, masters)
			esave = qsave.Exec(toolkit.M{}.Set("data", mrd))
			if esave != nil {
				toolkit.Printfn("Error: " + esave.Error())
				os.Exit(100)
			}
		}
	}
}

func adjustAllocs(allocsmap *allocmap, key string, current, expect, absorbed, ref1 float64) {
	allocs := *allocsmap
	alloc := allocs[key]
	if alloc == nil {
		alloc = new(plalloc)
		alloc.Key = key
		alloc.ID = key
	}
	alloc.Current += current
	alloc.Expect += expect
	alloc.Ref1 += ref1
	alloc.Absorbed += absorbed
	allocs[key] = alloc
	*allocsmap = allocs
}

func makeProgressLog(reference string, i, count, step int, current *int, tstart time.Time) int {
	perstep := count * step / 100
	icurrent := *current
	if icurrent == 0 {
		icurrent = perstep
	}
	pct := i * 100 / count
	if i >= icurrent {
		toolkit.Printfn("%s, %d of %d [%d pct] in %s",
			reference, i, count, pct, time.Since(tstart).String())
		icurrent += perstep
	}
	*current = icurrent
	return icurrent
}

func buildmap(holder interface{},
	fnModel func() orm.IModel,
	filter *dbox.Filter,
	fnIter func(holder interface{}, obj interface{})) interface{} {
	crx, ecrx := gdrj.Find(fnModel(), filter, nil)
	if ecrx != nil {
		toolkit.Printfn("Cursor Error: %s", ecrx.Error())
		os.Exit(100)
	}
	defer crx.Close()
	for {
		s := fnModel()
		e := crx.Fetch(s, 1, false)
		if e != nil {
			break
		}
		fnIter(holder, s)
	}
	return holder
}

func prepmastercalc() {
	toolkit.Println("--> PL MODEL")
	masters.Set("plmodel", buildmap(map[string]*gdrj.PLModel{},
		func() orm.IModel {
			return new(gdrj.PLModel)
		},
		nil,
		func(holder, obj interface{}) {
			h := holder.(map[string]*gdrj.PLModel)
			o := obj.(*gdrj.PLModel)
			h[o.ID] = o
		}).(map[string]*gdrj.PLModel))
}

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>//
// pipeline.go
//
// Created by Frederic DELBOS - fred@hyperboloide.com on Nov  8 2014.
//

package rw

import (
	"errors"
	"fmt"
	"io"
)

type Pipeline struct {
	lastReader io.Reader
}

type EncodingPipeline struct {
	Encoders []Encoder
	Output   Outputer
}

type DecodingPipeline struct {
	Decoders []Decoder
	Input    Inputer
}

func (p *Pipeline) Exec(w io.Writer) error {
	done := make(chan int)

	go func() {
		_, err := io.Copy(w, p.lastReader)

		if err != nil {
			fmt.Println(err)
		}
		done <- 1
	}()
	<-done

	return nil
}

func NewEncoding(encoders []Encoder, r io.Reader, d *Data) (*Pipeline, error) {
	if len(encoders) == 0 {
		return nil, errors.New("pipeline: A pipeline should have at lease 1 operation")
	}
	p := &Pipeline{
		lastReader: r,
	}
	for _, e := range encoders {
		nextReader, writer := io.Pipe()

		go func(e Encoder, r io.Reader, w *io.PipeWriter, d *Data) {
			e.Encode(r, w, d)
			w.Close()
		}(e, p.lastReader, writer, d)

		p.lastReader = nextReader
	}
	return p, nil
}

func NewDecoding(decoders []Decoder, r io.Reader, d *Data) (*Pipeline, error) {
	if len(decoders) == 0 {
		return nil, errors.New("pipeline: A pipeline should have at lease 1 operation")
	}
	p := &Pipeline{
		lastReader: r,
	}
	for _, e := range decoders {
		nextReader, writer := io.Pipe()

		go func(e Decoder, r io.Reader, w *io.PipeWriter, d *Data) {
			e.Decode(r, w, d)
			w.Close()
		}(e, p.lastReader, writer, d)

		p.lastReader = nextReader
	}
	return p, nil
}
<commit_msg>reformat<commit_after>//
// pipeline.go
//
// Created by Frederic DELBOS - fred@hyperboloide.com on Nov  8 2014.
//

package rw

import (
	"errors"
	"fmt"
	"io"
)

type Pipeline struct {
	lastReader io.Reader
}

type EncodingPipeline struct {
	Encoders []Encoder
	Output   Outputer
}

type DecodingPipeline struct {
	Decoders []Decoder
	Input    Inputer
}

func (p *Pipeline) Exec(w io.Writer) error {
	done := make(chan int)

	go func() {
		_, err := io.Copy(w, p.lastReader)

		if err != nil {
			fmt.Println(err)
		}
		done <- 1
	}()
	<-done

	return nil
}

func NewEncoding(encoders []Encoder, r io.Reader, d *Data) (*Pipeline, error) {
	if len(encoders) == 0 {
		return nil, errors.New("pipeline: A pipeline should have at lease 1 operation")
	}
	p := &Pipeline{
		lastReader: r,
	}
	for _, e := range encoders {
		nextReader, writer := io.Pipe()

		go func(e Encoder, r io.Reader, w *io.PipeWriter, d *Data) {
			e.Encode(r, w, d)
			w.Close()
		}(e, p.lastReader, writer, d)

		p.lastReader = nextReader
	}
	return p, nil
}

func NewDecoding(decoders []Decoder, r io.Reader, d *Data) (*Pipeline, error) {
	if len(decoders) == 0 {
		return nil, errors.New("pipeline: A pipeline should have at lease 1 operation")
	}
	p := &Pipeline{
		lastReader: r,
	}
	for _, e := range decoders {
		nextReader, writer := io.Pipe()

		go func(e Decoder, r io.Reader, w *io.PipeWriter, d *Data) {
			e.Decode(r, w, d)
			w.Close()
		}(e, p.lastReader, writer, d)

		p.lastReader = nextReader
	}
	return p, nil
}
<|endoftext|>


<commit_before>package libkbfs

import (
	"errors"
	"testing"

	"code.google.com/p/gomock/gomock"
	libkb "github.com/keybase/client/go/libkb"
)

type TestBlock struct {
	A int
}

var NullCtxt BlockContext = &BlockPointer{QuotaSize: 4}

func blockOpsInit(t *testing.T) (mockCtrl *gomock.Controller,
	config *ConfigMock) {
	mockCtrl = gomock.NewController(t)
	config = NewConfigMock(mockCtrl)
	bops := &BlockOpsStandard{config}
	config.SetBlockOps(bops)
	return
}

func expectBlockDecode(config *ConfigMock, packedData []byte,
	gotData *TestBlock, block TestBlock, err error) {
	config.mockCodec.EXPECT().Decode(packedData, gotData).
		Do(func(buf []byte, obj interface{}) {
		v := obj.(*TestBlock)
		*v = block
	}).Return(err)
}

func TestBlockOpsGetSuccess(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// expect one call to fetch a block, and one to decrypt it
	id := BlockId{1}
	encData := []byte{1, 2, 3, 4}
	config.mockBserv.EXPECT().Get(id, NullCtxt).Return(encData, nil)
	decData := TestBlock{42}
	packedData := []byte{4, 3, 2, 1}
	key := NullKey
	config.mockCrypto.EXPECT().Decrypt(encData, key).Return(packedData, nil)
	var gotData TestBlock
	expectBlockDecode(config, packedData, &gotData, decData, nil)

	if err := config.BlockOps().Get(
		id, NullCtxt, NullKey, &gotData); err != nil {
		t.Errorf("Got error on get: %v", err)
	} else if gotData != decData {
		t.Errorf("Got back wrong data on get: %v", gotData)
	}
}

func TestBlockOpsGetFailGet(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// fail the fetch call
	id := BlockId{1}
	err := errors.New("Fake fail")
	config.mockBserv.EXPECT().Get(id, NullCtxt).Return(nil, err)

	var gotData TestBlock
	if err2 := config.BlockOps().Get(
		id, NullCtxt, NullKey, &gotData); err2 != err {
		t.Errorf("Got bad error: %v", err2)
	}
}

func TestBlockOpsGetFailDecrypt(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// expect one call to fetch a block, then fail to decrypt i
	id := BlockId{1}
	encData := []byte{1, 2, 3, 4}
	config.mockBserv.EXPECT().Get(id, NullCtxt).Return(encData, nil)
	err := errors.New("Fake fail")
	key := NullKey
	config.mockCrypto.EXPECT().Decrypt(encData, key).Return(nil, err)

	var gotData TestBlock
	if err2 := config.BlockOps().Get(
		id, NullCtxt, NullKey, &gotData); err2 != err {
		t.Errorf("Got bad error: %v", err2)
	}
}

func TestBlockOpsGetFailDecode(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// expect one call to fetch a block, and one to decrypt it
	id := BlockId{1}
	encData := []byte{1, 2, 3, 4}
	config.mockBserv.EXPECT().Get(id, NullCtxt).Return(encData, nil)
	decData := TestBlock{42}
	packedData := []byte{4, 3, 2, 1}
	key := NullKey
	config.mockCrypto.EXPECT().Decrypt(encData, key).Return(packedData, nil)
	var gotData TestBlock
	err := errors.New("Fake fail")
	expectBlockDecode(config, packedData, &gotData, decData, err)

	if err2 := config.BlockOps().Get(
		id, NullCtxt, NullKey, &gotData); err2 != err {
		t.Errorf("Got unexpected error on get: %v", err2)
	}
}

func TestBlockOpsReadySuccess(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// expect one call to encrypt a block, one to hash it
	decData := TestBlock{42}
	packedData := []byte{4, 3, 2, 1}
	encData := []byte{1, 2, 3, 4}
	key := NullKey
	id := BlockId{1}

	config.mockCodec.EXPECT().Encode(decData).Return(packedData, nil)
	config.mockCrypto.EXPECT().Encrypt(packedData, key).Return(encData, nil)
	config.mockCrypto.EXPECT().Hash(encData).Return(
		libkb.NodeHashShort(id), nil)

	//var data []byte
	if id2, _, err := config.BlockOps().Ready(decData, key); err != nil {
		t.Errorf("Got error on ready: %v", err)
	} else if id2 != id {
		t.Errorf("Got back wrong id on ready: %v", id)
		//	} else if data != encData {
		//		t.Errorf("Got back wrong data on get: %v", data)
	}
}

func TestBlockOpsReadyFailEncode(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// expect one call to encrypt a block, one to hash it
	decData := TestBlock{42}
	err := errors.New("Fake fail")
	key := NullKey

	config.mockCodec.EXPECT().Encode(decData).Return(nil, err)

	if _, _, err2 := config.BlockOps().Ready(decData, key); err2 != err {
		t.Errorf("Got bad error on ready: %v", err2)
	}
}

func TestBlockOpsReadyFailEncrypt(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// expect one call to encrypt a block, one to hash it
	decData := TestBlock{42}
	packedData := []byte{4, 3, 2, 1}
	err := errors.New("Fake fail")
	key := NullKey

	config.mockCodec.EXPECT().Encode(decData).Return(packedData, nil)
	config.mockCrypto.EXPECT().Encrypt(packedData, key).Return(nil, err)

	if _, _, err2 := config.BlockOps().Ready(decData, key); err2 != err {
		t.Errorf("Got bad error on ready: %v", err2)
	}
}

func TestBlockOpsReadyFailHash(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// expect one call to encrypt a block, one to hash it
	decData := TestBlock{42}
	packedData := []byte{4, 3, 2, 1}
	encData := []byte{1, 2, 3, 4}
	key := NullKey
	err := errors.New("Fake fail")

	config.mockCodec.EXPECT().Encode(decData).Return(packedData, nil)
	config.mockCrypto.EXPECT().Encrypt(packedData, key).Return(encData, nil)
	config.mockCrypto.EXPECT().Hash(encData).Return(nil, err)

	if _, _, err2 := config.BlockOps().Ready(decData, key); err2 != err {
		t.Errorf("Got bad error on ready: %v", err2)
	}
}

func TestBlockOpsReadyFailCast(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// expect one call to encrypt a block, one to hash it
	decData := TestBlock{42}
	packedData := []byte{4, 3, 2, 1}
	encData := []byte{1, 2, 3, 4}
	key := NullKey
	badId := libkb.NodeHashLong{0}

	config.mockCodec.EXPECT().Encode(decData).Return(packedData, nil)
	config.mockCrypto.EXPECT().Encrypt(packedData, key).Return(encData, nil)
	config.mockCrypto.EXPECT().Hash(encData).Return(badId, nil)

	err := &BadCryptoError{BlockId{0}}
	if _, _, err2 :=
		config.BlockOps().Ready(decData, key); err2.Error() != err.Error() {
		t.Errorf("Got bad error on ready: %v (expected %v)", err2, err)
	}
}

func TestBlockOpsPutSuccess(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// expect one call to put a block
	id := BlockId{1}
	encData := []byte{1, 2, 3, 4}
	config.mockBserv.EXPECT().Put(id, NullCtxt, encData).Return(nil)

	if err := config.BlockOps().Put(id, NullCtxt, encData); err != nil {
		t.Errorf("Got error on put: %v", err)
	}
}

func TestBlockOpsPutFail(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// fail the put call
	id := BlockId{1}
	encData := []byte{1, 2, 3, 4}
	err := errors.New("Fake fail")
	config.mockBserv.EXPECT().Put(id, NullCtxt, encData).Return(err)

	if err2 := config.BlockOps().Put(id, NullCtxt, encData); err2 != err {
		t.Errorf("Got bad error on put: %v", err2)
	}
}

func TestBlockOpsDeleteSuccess(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// expect one call to delete a block
	id := BlockId{1}
	config.mockBserv.EXPECT().Delete(id, NullCtxt).Return(nil)

	if err := config.BlockOps().Delete(id, NullCtxt); err != nil {
		t.Errorf("Got error on put: %v", err)
	}
}

func TestBlockOpsDeleteFail(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// fail the delete call
	id := BlockId{1}
	err := errors.New("Fake fail")
	config.mockBserv.EXPECT().Delete(id, NullCtxt).Return(err)

	if err2 := config.BlockOps().Delete(id, NullCtxt); err2 != err {
		t.Errorf("Got bad error on put: %v", err2)
	}
}
<commit_msg>Use better contexts in block_ops_test.go<commit_after>package libkbfs

import (
	"errors"
	"testing"

	"code.google.com/p/gomock/gomock"
	libkb "github.com/keybase/client/go/libkb"
)

type TestBlock struct {
	A int
}

func blockOpsInit(t *testing.T) (mockCtrl *gomock.Controller,
	config *ConfigMock) {
	mockCtrl = gomock.NewController(t)
	config = NewConfigMock(mockCtrl)
	bops := &BlockOpsStandard{config}
	config.SetBlockOps(bops)
	return
}

func expectBlockDecode(config *ConfigMock, packedData []byte,
	gotData *TestBlock, block TestBlock, err error) {
	config.mockCodec.EXPECT().Decode(packedData, gotData).
		Do(func(buf []byte, obj interface{}) {
		v := obj.(*TestBlock)
		*v = block
	}).Return(err)
}

func makeContext(encData []byte) BlockContext {
	return &BlockPointer{QuotaSize: uint32(len(encData))}
}

func TestBlockOpsGetSuccess(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// expect one call to fetch a block, and one to decrypt it
	id := BlockId{1}
	encData := []byte{1, 2, 3, 4}
	ctxt := makeContext(encData)
	config.mockBserv.EXPECT().Get(id, ctxt).Return(encData, nil)
	decData := TestBlock{42}
	packedData := []byte{4, 3, 2, 1}
	key := NullKey
	config.mockCrypto.EXPECT().Decrypt(encData, key).Return(packedData, nil)
	var gotData TestBlock
	expectBlockDecode(config, packedData, &gotData, decData, nil)

	if err := config.BlockOps().Get(
		id, ctxt, NullKey, &gotData); err != nil {
		t.Errorf("Got error on get: %v", err)
	} else if gotData != decData {
		t.Errorf("Got back wrong data on get: %v", gotData)
	}
}

func TestBlockOpsGetFailGet(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// fail the fetch call
	id := BlockId{1}
	err := errors.New("Fake fail")
	ctxt := makeContext(nil)
	config.mockBserv.EXPECT().Get(id, ctxt).Return(nil, err)

	var gotData TestBlock
	if err2 := config.BlockOps().Get(
		id, ctxt, NullKey, &gotData); err2 != err {
		t.Errorf("Got bad error: %v", err2)
	}
}

func TestBlockOpsGetFailDecrypt(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// expect one call to fetch a block, then fail to decrypt i
	id := BlockId{1}
	encData := []byte{1, 2, 3, 4}
	ctxt := makeContext(encData)
	config.mockBserv.EXPECT().Get(id, ctxt).Return(encData, nil)
	err := errors.New("Fake fail")
	key := NullKey
	config.mockCrypto.EXPECT().Decrypt(encData, key).Return(nil, err)

	var gotData TestBlock
	if err2 := config.BlockOps().Get(
		id, ctxt, NullKey, &gotData); err2 != err {
		t.Errorf("Got bad error: %v", err2)
	}
}

func TestBlockOpsGetFailDecode(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// expect one call to fetch a block, and one to decrypt it
	id := BlockId{1}
	encData := []byte{1, 2, 3, 4}
	ctxt := makeContext(encData)
	config.mockBserv.EXPECT().Get(id, ctxt).Return(encData, nil)
	decData := TestBlock{42}
	packedData := []byte{4, 3, 2, 1}
	key := NullKey
	config.mockCrypto.EXPECT().Decrypt(encData, key).Return(packedData, nil)
	var gotData TestBlock
	err := errors.New("Fake fail")
	expectBlockDecode(config, packedData, &gotData, decData, err)

	if err2 := config.BlockOps().Get(
		id, ctxt, NullKey, &gotData); err2 != err {
		t.Errorf("Got unexpected error on get: %v", err2)
	}
}

func TestBlockOpsReadySuccess(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// expect one call to encrypt a block, one to hash it
	decData := TestBlock{42}
	packedData := []byte{4, 3, 2, 1}
	encData := []byte{1, 2, 3, 4}
	key := NullKey
	id := BlockId{1}

	config.mockCodec.EXPECT().Encode(decData).Return(packedData, nil)
	config.mockCrypto.EXPECT().Encrypt(packedData, key).Return(encData, nil)
	config.mockCrypto.EXPECT().Hash(encData).Return(
		libkb.NodeHashShort(id), nil)

	//var data []byte
	if id2, _, err := config.BlockOps().Ready(decData, key); err != nil {
		t.Errorf("Got error on ready: %v", err)
	} else if id2 != id {
		t.Errorf("Got back wrong id on ready: %v", id)
		//	} else if data != encData {
		//		t.Errorf("Got back wrong data on get: %v", data)
	}
}

func TestBlockOpsReadyFailEncode(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// expect one call to encrypt a block, one to hash it
	decData := TestBlock{42}
	err := errors.New("Fake fail")
	key := NullKey

	config.mockCodec.EXPECT().Encode(decData).Return(nil, err)

	if _, _, err2 := config.BlockOps().Ready(decData, key); err2 != err {
		t.Errorf("Got bad error on ready: %v", err2)
	}
}

func TestBlockOpsReadyFailEncrypt(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// expect one call to encrypt a block, one to hash it
	decData := TestBlock{42}
	packedData := []byte{4, 3, 2, 1}
	err := errors.New("Fake fail")
	key := NullKey

	config.mockCodec.EXPECT().Encode(decData).Return(packedData, nil)
	config.mockCrypto.EXPECT().Encrypt(packedData, key).Return(nil, err)

	if _, _, err2 := config.BlockOps().Ready(decData, key); err2 != err {
		t.Errorf("Got bad error on ready: %v", err2)
	}
}

func TestBlockOpsReadyFailHash(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// expect one call to encrypt a block, one to hash it
	decData := TestBlock{42}
	packedData := []byte{4, 3, 2, 1}
	encData := []byte{1, 2, 3, 4}
	key := NullKey
	err := errors.New("Fake fail")

	config.mockCodec.EXPECT().Encode(decData).Return(packedData, nil)
	config.mockCrypto.EXPECT().Encrypt(packedData, key).Return(encData, nil)
	config.mockCrypto.EXPECT().Hash(encData).Return(nil, err)

	if _, _, err2 := config.BlockOps().Ready(decData, key); err2 != err {
		t.Errorf("Got bad error on ready: %v", err2)
	}
}

func TestBlockOpsReadyFailCast(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// expect one call to encrypt a block, one to hash it
	decData := TestBlock{42}
	packedData := []byte{4, 3, 2, 1}
	encData := []byte{1, 2, 3, 4}
	key := NullKey
	badId := libkb.NodeHashLong{0}

	config.mockCodec.EXPECT().Encode(decData).Return(packedData, nil)
	config.mockCrypto.EXPECT().Encrypt(packedData, key).Return(encData, nil)
	config.mockCrypto.EXPECT().Hash(encData).Return(badId, nil)

	err := &BadCryptoError{BlockId{0}}
	if _, _, err2 :=
		config.BlockOps().Ready(decData, key); err2.Error() != err.Error() {
		t.Errorf("Got bad error on ready: %v (expected %v)", err2, err)
	}
}

func TestBlockOpsPutSuccess(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// expect one call to put a block
	id := BlockId{1}
	encData := []byte{1, 2, 3, 4}
	ctxt := makeContext(encData)
	config.mockBserv.EXPECT().Put(id, ctxt, encData).Return(nil)

	if err := config.BlockOps().Put(id, ctxt, encData); err != nil {
		t.Errorf("Got error on put: %v", err)
	}
}

func TestBlockOpsPutFail(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// fail the put call
	id := BlockId{1}
	encData := []byte{1, 2, 3, 4}
	ctxt := makeContext(encData)
	err := errors.New("Fake fail")
	config.mockBserv.EXPECT().Put(id, ctxt, encData).Return(err)

	if err2 := config.BlockOps().Put(id, ctxt, encData); err2 != err {
		t.Errorf("Got bad error on put: %v", err2)
	}
}

func TestBlockOpsDeleteSuccess(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// expect one call to delete a block
	id := BlockId{1}
	ctxt := makeContext(nil)
	config.mockBserv.EXPECT().Delete(id, ctxt).Return(nil)

	if err := config.BlockOps().Delete(id, ctxt); err != nil {
		t.Errorf("Got error on put: %v", err)
	}
}

func TestBlockOpsDeleteFail(t *testing.T) {
	mockCtrl, config := blockOpsInit(t)
	defer mockCtrl.Finish()

	// fail the delete call
	id := BlockId{1}
	err := errors.New("Fake fail")
	ctxt := makeContext(nil)
	config.mockBserv.EXPECT().Delete(id, ctxt).Return(err)

	if err2 := config.BlockOps().Delete(id, ctxt); err2 != err {
		t.Errorf("Got bad error on put: %v", err2)
	}
}
<|endoftext|>


<commit_before>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package cmd

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/ikravets/errs"
	"github.com/jessevdk/go-flags"

	"my/ev/efh"
)

type cmdEfhSuite struct {
	Exchange    string   `long:"exch"  short:"e" value-name:"EXCHANGE" default:"nasdaq" description:"nasdaq, bats, etc."`
	Suites      []string `long:"suite" short:"s" value-name:"SUITE"`
	Tests       []string `long:"test"  short:"t" value-name:"TEST"`
	Speed       int      `long:"speed" value-name:"NUM" default:"50000"`
	Limit       int      `long:"limit" value-name:"NUM"`
	Local       bool     `long:"local"`
	EfhLoglevel int      `long:"efh-loglevel" default:"6"`
	EfhProf     bool     `long:"efh-prof"`

	shouldExecute bool
	topOutDirName string
	testRunsTotal int
	testRunsOk    int
}

func (c *cmdEfhSuite) Execute(args []string) error {
	c.shouldExecute = true
	return nil
}

func (c *cmdEfhSuite) ConfigParser(parser *flags.Parser) {
	parser.AddCommand("efh_suite", "run test_efh test suite", "", c)
}

func (c *cmdEfhSuite) ParsingFinished() (err error) {
	defer errs.Catch(func(ce errs.CheckerError) {
		log.Printf("caught %s\n", ce)
	})
	if !c.shouldExecute {
		return
	}
	c.topOutDirName = time.Now().Format("efh_regression.2006-01-02-15:04:05")
	suitesDirName := fmt.Sprintf("/local/dumps/%s/regression", c.Exchange)

	if len(c.Suites) == 0 || c.Suites[0] == "?" {
		suites, err2 := listDirs(suitesDirName)
		errs.CheckE(err2)
		if len(c.Suites) > 0 {
			fmt.Printf("suites: %s\n", suites)
			return
		}
		if len(suites) == 0 {
			log.Println("Warning: no suites found")
			return
		}
		c.Suites = suites
	}

	for _, suite := range c.Suites {
		var suiteDirName string
		if strings.HasPrefix(suite, "/") {
			suiteDirName = suite
		} else if strings.HasPrefix(suite, "./") {
			cwd, err := os.Getwd()
			errs.CheckE(err)
			suiteDirName = filepath.Join(cwd, suite)
		} else {
			suiteDirName = filepath.Join(suitesDirName, suite)
		}
		var tests []string
		if len(c.Tests) == 0 || c.Tests[0] == "?" {
			tests, err = listDirs(suiteDirName)
			errs.CheckE(err)
			if len(c.Tests) > 0 {
				fmt.Printf("suite %s tests: %v\n", suiteDirName, tests)
				continue
			}
		} else {
			tests = c.Tests
		}
		if len(tests) == 0 {
			log.Printf("Warning: no tests found in suite %s\n", suite)
			continue
		}

		for _, testName := range tests {
			testDirName := filepath.Join(suiteDirName, testName)
			subscriptionFileNames, err := filepath.Glob(filepath.Join(testDirName, "subscription*"))
			errs.CheckE(err)
			if len(subscriptionFileNames) == 0 {
				c.RunTest(testDirName, nil)
			} else {
				for _, sfn := range subscriptionFileNames {
					a := strings.SplitAfter(sfn, "/subscription")
					suf := ""
					if len(a) > 0 {
						suf = a[len(a)-1]
					}
					c.RunTest(testDirName, &suf)
				}
			}
		}
	}
	log.Printf("Tests OK/Total: %d/%d\n", c.testRunsOk, c.testRunsTotal)
	fmt.Printf("Tests OK/Total: %d/%d\n", c.testRunsOk, c.testRunsTotal)
	return
}
func (c *cmdEfhSuite) RunTest(testDirName string, suffix *string) (err error) {
	defer errs.Catch(func(ce errs.CheckerError) {
		log.Printf("caught %s\n", ce)
		err = ce
	})
	c.testRunsTotal++
	testRunName := time.Now().Format("2006-01-02-15:04:05.")
	tdnDir, tdnFile := filepath.Split(testDirName)
	tdnDir = filepath.Base(tdnDir)
	testRunName += tdnDir + "-" + tdnFile
	if suffix != nil {
		testRunName += *suffix
	}
	fmt.Printf("run %s\n", testRunName)
	outDirName := filepath.Join(c.topOutDirName, testRunName)
	errs.CheckE(os.MkdirAll(outDirName, 0777))
	errs.CheckE(ioutil.WriteFile(filepath.Join(outDirName, "fail"), nil, 0666))
	expoutName := filepath.Join(testDirName, "expout-efh-orders")
	if suffix != nil {
		expoutName += *suffix
	}
	inputPcapName := filepath.Join(testDirName, "dump.pcap")
	_, err = os.Stat(expoutName)
	errs.CheckE(err)
	_, err = os.Stat(inputPcapName)
	errs.CheckE(err)
	efhDumpName := filepath.Join(outDirName, "expout_orders")
	errs.CheckE(os.Symlink(expoutName, efhDumpName))
	errs.CheckE(os.Symlink(testDirName, filepath.Join(outDirName, "dump_dir")))
	errs.CheckE(os.Symlink(inputPcapName, filepath.Join(outDirName, "dump.pcap")))
	conf := efh.ReplayConfig{
		InputFileName:   inputPcapName,
		OutputInterface: "eth1",
		Pps:             c.Speed,
		Limit:           c.Limit,
		Loop:            1,
		EfhLoglevel:     c.EfhLoglevel,
		EfhIgnoreGap:    true,
		EfhDump:         "expout_orders",
		EfhChannel:      c.genEfhChannels(testDirName),
		EfhProf:         c.EfhProf,
		TestEfh:         "/usr/libexec/test_efh",
		Local:           c.Local,
	}
	if suffix != nil {
		subscr := "subscription" + *suffix
		errs.CheckE(os.Symlink(filepath.Join(testDirName, subscr), filepath.Join(outDirName, subscr)))
		conf.EfhSubscribe = []string{subscr}
	}
	origWd, err := os.Getwd()
	errs.CheckE(err)
	errs.CheckE(os.Chdir(outDirName))
	er := efh.NewEfhReplay(conf)
	efhReplayErr := er.Run()
	errs.CheckE(os.Chdir(origWd))
	errs.CheckE(efhReplayErr)
	errs.CheckE(ioutil.WriteFile(filepath.Join(outDirName, "ok"), nil, 0666))
	errs.CheckE(os.Remove(filepath.Join(outDirName, "fail")))
	c.testRunsOk++
	return
}
func (c *cmdEfhSuite) genEfhChannels(testDirName string) (channels []string) {
	// TODO support testDirName/channels
	switch c.Exchange {
	case "nasdaq":
		for i := 0; i < 4; i++ {
			channels = append(channels, fmt.Sprintf("233.54.12.%d:%d", 1+i, 18001+i))
		}
	case "bats":
		for i := 0; i < 32; i++ {
			channels = append(channels, fmt.Sprintf("224.0.131.%d:%d", i/4, 30101+i))
		}
	default:
		errs.Check(false)
	}
	return
}

func listDirs(parent string) (children []string, err error) {
	defer errs.PassE(&err)
	matches, err := filepath.Glob(parent + "/*")
	errs.CheckE(err)
	for _, f := range matches {
		fi, err := os.Stat(f)
		errs.CheckE(err)
		if fi.IsDir() {
			children = append(children, fi.Name())
		}
	}
	return
}

func init() {
	var c cmdEfhSuite
	Registry.Register(&c)
}
<commit_msg>cmd:efh_suite: add --test-efh<commit_after>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package cmd

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/ikravets/errs"
	"github.com/jessevdk/go-flags"

	"my/ev/efh"
)

type cmdEfhSuite struct {
	Exchange    string   `long:"exch"  short:"e" value-name:"EXCHANGE" default:"nasdaq" description:"nasdaq, bats, etc."`
	Suites      []string `long:"suite" short:"s" value-name:"SUITE"`
	Tests       []string `long:"test"  short:"t" value-name:"TEST"`
	Speed       int      `long:"speed" value-name:"NUM" default:"50000"`
	Limit       int      `long:"limit" value-name:"NUM"`
	TestEfh     string   `long:"test-efh (default: system-installed version)"`
	Local       bool     `long:"local"`
	EfhLoglevel int      `long:"efh-loglevel" default:"6"`
	EfhProf     bool     `long:"efh-prof"`

	shouldExecute bool
	topOutDirName string
	testRunsTotal int
	testRunsOk    int
}

func (c *cmdEfhSuite) Execute(args []string) error {
	c.shouldExecute = true
	return nil
}

func (c *cmdEfhSuite) ConfigParser(parser *flags.Parser) {
	parser.AddCommand("efh_suite", "run test_efh test suite", "", c)
}

func (c *cmdEfhSuite) ParsingFinished() (err error) {
	defer errs.Catch(func(ce errs.CheckerError) {
		log.Printf("caught %s\n", ce)
	})
	if !c.shouldExecute {
		return
	}
	c.topOutDirName = time.Now().Format("efh_regression.2006-01-02-15:04:05")
	suitesDirName := fmt.Sprintf("/local/dumps/%s/regression", c.Exchange)

	if len(c.Suites) == 0 || c.Suites[0] == "?" {
		suites, err2 := listDirs(suitesDirName)
		errs.CheckE(err2)
		if len(c.Suites) > 0 {
			fmt.Printf("suites: %s\n", suites)
			return
		}
		if len(suites) == 0 {
			log.Println("Warning: no suites found")
			return
		}
		c.Suites = suites
	}

	for _, suite := range c.Suites {
		var suiteDirName string
		if strings.HasPrefix(suite, "/") {
			suiteDirName = suite
		} else if strings.HasPrefix(suite, "./") {
			cwd, err := os.Getwd()
			errs.CheckE(err)
			suiteDirName = filepath.Join(cwd, suite)
		} else {
			suiteDirName = filepath.Join(suitesDirName, suite)
		}
		var tests []string
		if len(c.Tests) == 0 || c.Tests[0] == "?" {
			tests, err = listDirs(suiteDirName)
			errs.CheckE(err)
			if len(c.Tests) > 0 {
				fmt.Printf("suite %s tests: %v\n", suiteDirName, tests)
				continue
			}
		} else {
			tests = c.Tests
		}
		if len(tests) == 0 {
			log.Printf("Warning: no tests found in suite %s\n", suite)
			continue
		}

		for _, testName := range tests {
			testDirName := filepath.Join(suiteDirName, testName)
			subscriptionFileNames, err := filepath.Glob(filepath.Join(testDirName, "subscription*"))
			errs.CheckE(err)
			if len(subscriptionFileNames) == 0 {
				c.RunTest(testDirName, nil)
			} else {
				for _, sfn := range subscriptionFileNames {
					a := strings.SplitAfter(sfn, "/subscription")
					suf := ""
					if len(a) > 0 {
						suf = a[len(a)-1]
					}
					c.RunTest(testDirName, &suf)
				}
			}
		}
	}
	log.Printf("Tests OK/Total: %d/%d\n", c.testRunsOk, c.testRunsTotal)
	fmt.Printf("Tests OK/Total: %d/%d\n", c.testRunsOk, c.testRunsTotal)
	return
}
func (c *cmdEfhSuite) RunTest(testDirName string, suffix *string) (err error) {
	defer errs.Catch(func(ce errs.CheckerError) {
		log.Printf("caught %s\n", ce)
		err = ce
	})
	c.testRunsTotal++
	testRunName := time.Now().Format("2006-01-02-15:04:05.")
	tdnDir, tdnFile := filepath.Split(testDirName)
	tdnDir = filepath.Base(tdnDir)
	testRunName += tdnDir + "-" + tdnFile
	if suffix != nil {
		testRunName += *suffix
	}
	fmt.Printf("run %s\n", testRunName)
	outDirName := filepath.Join(c.topOutDirName, testRunName)
	errs.CheckE(os.MkdirAll(outDirName, 0777))
	errs.CheckE(ioutil.WriteFile(filepath.Join(outDirName, "fail"), nil, 0666))
	expoutName := filepath.Join(testDirName, "expout-efh-orders")
	if suffix != nil {
		expoutName += *suffix
	}
	inputPcapName := filepath.Join(testDirName, "dump.pcap")
	_, err = os.Stat(expoutName)
	errs.CheckE(err)
	_, err = os.Stat(inputPcapName)
	errs.CheckE(err)
	efhDumpName := filepath.Join(outDirName, "expout_orders")
	errs.CheckE(os.Symlink(expoutName, efhDumpName))
	errs.CheckE(os.Symlink(testDirName, filepath.Join(outDirName, "dump_dir")))
	errs.CheckE(os.Symlink(inputPcapName, filepath.Join(outDirName, "dump.pcap")))
	conf := efh.ReplayConfig{
		InputFileName:   inputPcapName,
		OutputInterface: "eth1",
		Pps:             c.Speed,
		Limit:           c.Limit,
		Loop:            1,
		EfhLoglevel:     c.EfhLoglevel,
		EfhIgnoreGap:    true,
		EfhDump:         "expout_orders",
		EfhChannel:      c.genEfhChannels(testDirName),
		EfhProf:         c.EfhProf,
		TestEfh:         c.TestEfh,
		Local:           c.Local,
	}
	if suffix != nil {
		subscr := "subscription" + *suffix
		errs.CheckE(os.Symlink(filepath.Join(testDirName, subscr), filepath.Join(outDirName, subscr)))
		conf.EfhSubscribe = []string{subscr}
	}
	origWd, err := os.Getwd()
	errs.CheckE(err)
	errs.CheckE(os.Chdir(outDirName))
	er := efh.NewEfhReplay(conf)
	efhReplayErr := er.Run()
	errs.CheckE(os.Chdir(origWd))
	errs.CheckE(efhReplayErr)
	errs.CheckE(ioutil.WriteFile(filepath.Join(outDirName, "ok"), nil, 0666))
	errs.CheckE(os.Remove(filepath.Join(outDirName, "fail")))
	c.testRunsOk++
	return
}
func (c *cmdEfhSuite) genEfhChannels(testDirName string) (channels []string) {
	// TODO support testDirName/channels
	switch c.Exchange {
	case "nasdaq":
		for i := 0; i < 4; i++ {
			channels = append(channels, fmt.Sprintf("233.54.12.%d:%d", 1+i, 18001+i))
		}
	case "bats":
		for i := 0; i < 32; i++ {
			channels = append(channels, fmt.Sprintf("224.0.131.%d:%d", i/4, 30101+i))
		}
	default:
		errs.Check(false)
	}
	return
}

func listDirs(parent string) (children []string, err error) {
	defer errs.PassE(&err)
	matches, err := filepath.Glob(parent + "/*")
	errs.CheckE(err)
	for _, f := range matches {
		fi, err := os.Stat(f)
		errs.CheckE(err)
		if fi.IsDir() {
			children = append(children, fi.Name())
		}
	}
	return
}

func init() {
	var c cmdEfhSuite
	Registry.Register(&c)
}
<|endoftext|>