<commit_before>package main

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"net/http"

	log "github.com/Sirupsen/logrus"
	"github.com/elazarl/goproxy"
	"github.com/garyburd/redigo/redis"
	"io/ioutil"
)

type DBClient struct {
	cache Cache
	http  *http.Client
}

// request holds structure for request
type request struct {
	details requestDetails
}

var emptyResp = &http.Response{}

// requestDetails stores information about request, it's used for creating unique hash and also as a payload structure
type requestDetails struct {
	Path        string              `json:"path"`
	Method      string              `json:"method"`
	Destination string              `json:"destination"`
	Query       string              `json:"query"`
	Body        string              `json:"body"`
	RemoteAddr  string              `json:"remoteAddr"`
	Headers     map[string][]string `json:"headers"`
}

// hash returns unique hash key for request
func (r *request) hash() string {
	h := md5.New()
	io.WriteString(h, fmt.Sprintf("%s%s%s%s", r.details.Destination, r.details.Path, r.details.Method, r.details.Query))
	return fmt.Sprintf("%x", h.Sum(nil))
}

// res structure hold response body from external service, body is not decoded and is supposed
// to be bytes, however headers should provide all required information for later decoding
// by the client.
type response struct {
	Status  int                 `json:"status"`
	Body    string              `json:"body"`
	Headers map[string][]string `json:"headers"`
}

// Payload structure holds request and response structure
type Payload struct {
	Response response       `json:"response"`
	Request  requestDetails `json:"request"`
	ID       string         `json:"id"`
}

// recordRequest saves request for later playback
func (d *DBClient) captureRequest(req *http.Request) (*http.Response, error) {

	// this is mainly for testing, since when you create
	if req.Body == nil {
		req.Body = ioutil.NopCloser(bytes.NewBuffer([]byte("")))
	}

	reqBody, err := ioutil.ReadAll(req.Body)

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
			"mode":  AppConfig.mode,
		}).Error("Got error when reading request body")
	}
	log.WithFields(log.Fields{
		"body": string(reqBody),
	}).Info("got request body")
	req.Body = ioutil.NopCloser(bytes.NewBuffer(reqBody))

	// forwarding request
	resp, err := d.doRequest(req)

	if err == nil {
		respBody, err := extractBody(resp)
		if err != nil {
			// copying the response body did not work
			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
				}).Error("Failed to copy response body.")
			}
		}

		// saving response body with request/response meta to cache
		go d.save(req, resp, respBody, reqBody)
	}

	// return new response or error here
	return resp, err
}

func copyBody(body io.ReadCloser) (resp1, resp2 io.ReadCloser, err error) {
	var buf bytes.Buffer
	if _, err = buf.ReadFrom(body); err != nil {
		return nil, nil, err
	}
	if err = body.Close(); err != nil {
		return nil, nil, err
	}
	return ioutil.NopCloser(&buf), ioutil.NopCloser(bytes.NewReader(buf.Bytes())), nil
}

func extractBody(resp *http.Response) (extract []byte, err error) {
	save := resp.Body
	savecl := resp.ContentLength

	save, resp.Body, err = copyBody(resp.Body)

	if err != nil {
		return
	}
	defer resp.Body.Close()
	extract, err = ioutil.ReadAll(resp.Body)

	resp.Body = save
	resp.ContentLength = savecl
	if err != nil {
		return nil, err
	}
	return extract, nil
}

// doRequest performs original request and returns response that should be returned to client and error (if there is one)
func (d *DBClient) doRequest(request *http.Request) (*http.Response, error) {
	// We can't have this set. And it only contains "/pkg/net/http/" anyway
	request.RequestURI = ""

	if AppConfig.middleware != "" {
		var payload Payload

		c := NewConstructor(request, payload)
		c.ApplyMiddleware(AppConfig.middleware)

		request = c.reconstructRequest()

	}

	resp, err := d.http.Do(request)

	if err != nil {
		log.WithFields(log.Fields{
			"error":  err.Error(),
			"host":   request.Host,
			"method": request.Method,
			"path":   request.URL.Path,
		}).Error("Could not forward request.")
		return nil, err
	}

	log.WithFields(log.Fields{
		"host":   request.Host,
		"method": request.Method,
		"path":   request.URL.Path,
	}).Info("Response got successfuly!")

	resp.Header.Set("hoverfly", "Was-Here")
	return resp, nil

}

// save gets request fingerprint, extracts request body, status code and headers, then saves it to cache
func (d *DBClient) save(req *http.Request, resp *http.Response, respBody []byte, reqBody []byte) {
	// record request here
	key := getRequestFingerprint(req)

	if resp == nil {
		resp = emptyResp
	} else {
		responseObj := response{
			Status:  resp.StatusCode,
			Body:    string(respBody),
			Headers: resp.Header,
		}

		log.WithFields(log.Fields{
			"path":          req.URL.Path,
			"rawQuery":      req.URL.RawQuery,
			"requestMethod": req.Method,
			"bodyLen":       len(reqBody),
			"destination":   req.Host,
			"hashKey":       key,
		}).Info("Capturing")

		requestObj := requestDetails{
			Path:        req.URL.Path,
			Method:      req.Method,
			Destination: req.Host,
			Query:       req.URL.RawQuery,
			Body:        string(reqBody),
			RemoteAddr:  req.RemoteAddr,
			Headers:     req.Header,
		}

		payload := Payload{
			Response: responseObj,
			Request:  requestObj,
			ID:       key,
		}
		// converting it to json bytes
		bts, err := json.Marshal(payload)

		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Error("Failed to marshal json")
		} else {
			d.cache.set(key, bts)
		}
	}

}

// getAllRecordsRaw returns raw (json string) for all records
func (d *DBClient) getAllRecordsRaw() ([]string, error) {
	keys, err := d.cache.getAllKeys()

	if err == nil {

		// checking if there are any records
		if len(keys) == 0 {
			return nil, nil
		}

		jsonStrs, err := d.cache.getAllValues(keys)

		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Error("Failed to get all values (raw)")
			return nil, err
		} else {
			return jsonStrs, nil
		}

	} else {
		return nil, err
	}
}

// getAllRecords returns all stored
func (d *DBClient) getAllRecords() ([]Payload, error) {
	var payloads []Payload

	jsonStrs, err := d.getAllRecordsRaw()

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Failed to get all values")
	} else {

		if jsonStrs != nil {
			for _, v := range jsonStrs {
				var pl Payload
				err = json.Unmarshal([]byte(v), &pl)
				if err != nil {
					log.WithFields(log.Fields{
						"error": err.Error(),
						"json":  v,
					}).Warning("Failed to deserialize json")
				} else {
					payloads = append(payloads, pl)
				}
			}
		}
	}

	return payloads, err

}

// deleteAllRecords deletes all recorded requests
func (d *DBClient) deleteAllRecords() error {
	keys, err := d.cache.getAllKeys()

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Warning("Failed to get keys, cannot delete all records")
		return err
	} else {
		for _, v := range keys {
			err := d.cache.delete(v)
			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
					"key":   v,
				}).Warning("Failed to delete key...")
			}
		}
		return nil
	}
}

// getRequestFingerprint returns request hash
func getRequestFingerprint(req *http.Request) string {
	details := requestDetails{Path: req.URL.Path, Method: req.Method, Destination: req.Host, Query: req.URL.RawQuery}
	r := request{details: details}
	return r.hash()
}

// getResponse returns stored response from cache
func (d *DBClient) getResponse(req *http.Request) *http.Response {

	key := getRequestFingerprint(req)
	var payload Payload

	payloadBts, err := redis.Bytes(d.cache.get(key))

	if err == nil {
		// getting cache response
		err = json.Unmarshal(payloadBts, &payload)
		if err != nil {
			log.Error(err)
			// what now?
		}

		c := NewConstructor(req, payload)

		if AppConfig.middleware != "" {
			_ = c.ApplyMiddleware(AppConfig.middleware)
		}

		response := c.reconstructResponse()

		log.WithFields(log.Fields{
			"key":        key,
			"status":     payload.Response.Status,
			"bodyLength": response.ContentLength,
			"mode":       AppConfig.mode,
		}).Info("Response found, returning")

		return response

	} else {
		log.WithFields(log.Fields{
			"error": err.Error(),
			"mode":  AppConfig.mode,
		}).Error("Failed to retrieve response from cache")
		// return error? if we return nil - proxy forwards request to original destination
		return goproxy.NewResponse(req,
			goproxy.ContentTypeText, http.StatusPreconditionFailed,
			"Coudldn't find recorded request, please record it first!")
	}

}

// modifyRequestResponse modifies outgoing request and then modifies incoming response, neither request nor response
// is saved to cache.
func (d *DBClient) modifyRequestResponse(req *http.Request, middleware string) (*http.Response, error) {

	// modifying request
	resp, err := d.doRequest(req)

	if err != nil {
		return nil, err
	}

	// preparing payload
	bodyBytes, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		log.WithFields(log.Fields{
			"error":      err.Error(),
			"middleware": middleware,
		}).Error("Failed to read response body after sending modified request")
		return nil, err
	}

	r := response{
		Status:  resp.StatusCode,
		Body:    string(bodyBytes),
		Headers: resp.Header,
	}
	payload := Payload{Response: r}

	c := NewConstructor(req, payload)
	// applying middleware to modify response
	err = c.ApplyMiddleware(middleware)

	if err != nil {
		return nil, err
	}

	newResponse := c.reconstructResponse()

	log.WithFields(log.Fields{
		"status":     newResponse.StatusCode,
		"middleware": middleware,
		"mode":       AppConfig.mode,
	}).Info("Response modified, returning")

	return newResponse, nil

}
<commit_msg>a bit more information when hoverfly couldn't find match for request<commit_after>package main

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"net/http"

	log "github.com/Sirupsen/logrus"
	"github.com/elazarl/goproxy"
	"github.com/garyburd/redigo/redis"
	"io/ioutil"
)

type DBClient struct {
	cache Cache
	http  *http.Client
}

// request holds structure for request
type request struct {
	details requestDetails
}

var emptyResp = &http.Response{}

// requestDetails stores information about request, it's used for creating unique hash and also as a payload structure
type requestDetails struct {
	Path        string              `json:"path"`
	Method      string              `json:"method"`
	Destination string              `json:"destination"`
	Query       string              `json:"query"`
	Body        string              `json:"body"`
	RemoteAddr  string              `json:"remoteAddr"`
	Headers     map[string][]string `json:"headers"`
}

// hash returns unique hash key for request
func (r *request) hash() string {
	h := md5.New()
	io.WriteString(h, fmt.Sprintf("%s%s%s%s", r.details.Destination, r.details.Path, r.details.Method, r.details.Query))
	return fmt.Sprintf("%x", h.Sum(nil))
}

// res structure hold response body from external service, body is not decoded and is supposed
// to be bytes, however headers should provide all required information for later decoding
// by the client.
type response struct {
	Status  int                 `json:"status"`
	Body    string              `json:"body"`
	Headers map[string][]string `json:"headers"`
}

// Payload structure holds request and response structure
type Payload struct {
	Response response       `json:"response"`
	Request  requestDetails `json:"request"`
	ID       string         `json:"id"`
}

// recordRequest saves request for later playback
func (d *DBClient) captureRequest(req *http.Request) (*http.Response, error) {

	// this is mainly for testing, since when you create
	if req.Body == nil {
		req.Body = ioutil.NopCloser(bytes.NewBuffer([]byte("")))
	}

	reqBody, err := ioutil.ReadAll(req.Body)

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
			"mode":  AppConfig.mode,
		}).Error("Got error when reading request body")
	}
	log.WithFields(log.Fields{
		"body": string(reqBody),
	}).Info("got request body")
	req.Body = ioutil.NopCloser(bytes.NewBuffer(reqBody))

	// forwarding request
	resp, err := d.doRequest(req)

	if err == nil {
		respBody, err := extractBody(resp)
		if err != nil {
			// copying the response body did not work
			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
				}).Error("Failed to copy response body.")
			}
		}

		// saving response body with request/response meta to cache
		go d.save(req, resp, respBody, reqBody)
	}

	// return new response or error here
	return resp, err
}

func copyBody(body io.ReadCloser) (resp1, resp2 io.ReadCloser, err error) {
	var buf bytes.Buffer
	if _, err = buf.ReadFrom(body); err != nil {
		return nil, nil, err
	}
	if err = body.Close(); err != nil {
		return nil, nil, err
	}
	return ioutil.NopCloser(&buf), ioutil.NopCloser(bytes.NewReader(buf.Bytes())), nil
}

func extractBody(resp *http.Response) (extract []byte, err error) {
	save := resp.Body
	savecl := resp.ContentLength

	save, resp.Body, err = copyBody(resp.Body)

	if err != nil {
		return
	}
	defer resp.Body.Close()
	extract, err = ioutil.ReadAll(resp.Body)

	resp.Body = save
	resp.ContentLength = savecl
	if err != nil {
		return nil, err
	}
	return extract, nil
}

// doRequest performs original request and returns response that should be returned to client and error (if there is one)
func (d *DBClient) doRequest(request *http.Request) (*http.Response, error) {
	// We can't have this set. And it only contains "/pkg/net/http/" anyway
	request.RequestURI = ""

	if AppConfig.middleware != "" {
		var payload Payload

		c := NewConstructor(request, payload)
		c.ApplyMiddleware(AppConfig.middleware)

		request = c.reconstructRequest()

	}

	resp, err := d.http.Do(request)

	if err != nil {
		log.WithFields(log.Fields{
			"error":  err.Error(),
			"host":   request.Host,
			"method": request.Method,
			"path":   request.URL.Path,
		}).Error("Could not forward request.")
		return nil, err
	}

	log.WithFields(log.Fields{
		"host":   request.Host,
		"method": request.Method,
		"path":   request.URL.Path,
	}).Info("Response got successfuly!")

	resp.Header.Set("hoverfly", "Was-Here")
	return resp, nil

}

// save gets request fingerprint, extracts request body, status code and headers, then saves it to cache
func (d *DBClient) save(req *http.Request, resp *http.Response, respBody []byte, reqBody []byte) {
	// record request here
	key := getRequestFingerprint(req)

	if resp == nil {
		resp = emptyResp
	} else {
		responseObj := response{
			Status:  resp.StatusCode,
			Body:    string(respBody),
			Headers: resp.Header,
		}

		log.WithFields(log.Fields{
			"path":          req.URL.Path,
			"rawQuery":      req.URL.RawQuery,
			"requestMethod": req.Method,
			"bodyLen":       len(reqBody),
			"destination":   req.Host,
			"hashKey":       key,
		}).Info("Capturing")

		requestObj := requestDetails{
			Path:        req.URL.Path,
			Method:      req.Method,
			Destination: req.Host,
			Query:       req.URL.RawQuery,
			Body:        string(reqBody),
			RemoteAddr:  req.RemoteAddr,
			Headers:     req.Header,
		}

		payload := Payload{
			Response: responseObj,
			Request:  requestObj,
			ID:       key,
		}
		// converting it to json bytes
		bts, err := json.Marshal(payload)

		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Error("Failed to marshal json")
		} else {
			d.cache.set(key, bts)
		}
	}

}

// getAllRecordsRaw returns raw (json string) for all records
func (d *DBClient) getAllRecordsRaw() ([]string, error) {
	keys, err := d.cache.getAllKeys()

	if err == nil {

		// checking if there are any records
		if len(keys) == 0 {
			return nil, nil
		}

		jsonStrs, err := d.cache.getAllValues(keys)

		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Error("Failed to get all values (raw)")
			return nil, err
		} else {
			return jsonStrs, nil
		}

	} else {
		return nil, err
	}
}

// getAllRecords returns all stored
func (d *DBClient) getAllRecords() ([]Payload, error) {
	var payloads []Payload

	jsonStrs, err := d.getAllRecordsRaw()

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Failed to get all values")
	} else {

		if jsonStrs != nil {
			for _, v := range jsonStrs {
				var pl Payload
				err = json.Unmarshal([]byte(v), &pl)
				if err != nil {
					log.WithFields(log.Fields{
						"error": err.Error(),
						"json":  v,
					}).Warning("Failed to deserialize json")
				} else {
					payloads = append(payloads, pl)
				}
			}
		}
	}

	return payloads, err

}

// deleteAllRecords deletes all recorded requests
func (d *DBClient) deleteAllRecords() error {
	keys, err := d.cache.getAllKeys()

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Warning("Failed to get keys, cannot delete all records")
		return err
	} else {
		for _, v := range keys {
			err := d.cache.delete(v)
			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
					"key":   v,
				}).Warning("Failed to delete key...")
			}
		}
		return nil
	}
}

// getRequestFingerprint returns request hash
func getRequestFingerprint(req *http.Request) string {
	details := requestDetails{Path: req.URL.Path, Method: req.Method, Destination: req.Host, Query: req.URL.RawQuery}
	r := request{details: details}
	return r.hash()
}

// getResponse returns stored response from cache
func (d *DBClient) getResponse(req *http.Request) *http.Response {

	key := getRequestFingerprint(req)
	var payload Payload

	payloadBts, err := redis.Bytes(d.cache.get(key))

	if err == nil {
		// getting cache response
		err = json.Unmarshal(payloadBts, &payload)
		if err != nil {
			log.Error(err)
			// what now?
		}

		c := NewConstructor(req, payload)

		if AppConfig.middleware != "" {
			_ = c.ApplyMiddleware(AppConfig.middleware)
		}

		response := c.reconstructResponse()

		log.WithFields(log.Fields{
			"key":        key,
			"status":     payload.Response.Status,
			"bodyLength": response.ContentLength,
			"mode":       AppConfig.mode,
		}).Info("Response found, returning")

		return response

	} else {
		log.WithFields(log.Fields{
			"error":       err.Error(),
			"mode":        AppConfig.mode,
			"query":       req.URL.RawQuery,
			"path":        req.URL.RawPath,
			"destination": req.Host,
			"method":      req.Method,
		}).Warn("Failed to retrieve response from cache")
		// return error? if we return nil - proxy forwards request to original destination
		return goproxy.NewResponse(req,
			goproxy.ContentTypeText, http.StatusPreconditionFailed,
			"Coudldn't find recorded request, please record it first!")
	}

}

// modifyRequestResponse modifies outgoing request and then modifies incoming response, neither request nor response
// is saved to cache.
func (d *DBClient) modifyRequestResponse(req *http.Request, middleware string) (*http.Response, error) {

	// modifying request
	resp, err := d.doRequest(req)

	if err != nil {
		return nil, err
	}

	// preparing payload
	bodyBytes, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		log.WithFields(log.Fields{
			"error":      err.Error(),
			"middleware": middleware,
		}).Error("Failed to read response body after sending modified request")
		return nil, err
	}

	r := response{
		Status:  resp.StatusCode,
		Body:    string(bodyBytes),
		Headers: resp.Header,
	}
	payload := Payload{Response: r}

	c := NewConstructor(req, payload)
	// applying middleware to modify response
	err = c.ApplyMiddleware(middleware)

	if err != nil {
		return nil, err
	}

	newResponse := c.reconstructResponse()

	log.WithFields(log.Fields{
		"status":     newResponse.StatusCode,
		"middleware": middleware,
		"mode":       AppConfig.mode,
	}).Info("Response modified, returning")

	return newResponse, nil

}
<|endoftext|>


<commit_before>package main

import (
	"time"

	"gopkg.in/mgo.v2/bson"
)

type Repository struct {
	ID         int    `json:"id"`
	Owner      string `json:"owner"`
	OwnerIsOrg bool   `json:"owner_is_org"`
	Name       string `json:"name"`
	FullName   string `json:"full_name"`
	URL        string `json:"url"`
}

type Docs struct {
	Repository    Repository `json:"repository"`
	ReadmeExists  bool       `json:"readme_exists"`
	ReadmeURL     string     `json:"readme_url"`
	LicenseExists bool       `json:"license_exists"`
	LicenseURL    string     `json:"license_url"`
	LicenseName   string     `json:"license_name"`
}

type ResponseTimes struct {
	Repository           Repository            `json:"repository"`
	AverageResponseTimes []AverageResponseTime `json:"average_response_times"`
}

type AverageResponseTime struct {
	Contributor           string    `json:"contributor"`
	AverageResponseTime   float32   `json:"average_response_time"`
	FirstContributionWeek time.Time `json:"first_contribution_week"`
}

type Indicator struct {
	ID   bson.ObjectId `bson:"_id" json:"id"`
	Name string        `bson:"name" json:"name"`
	Key  string        `bson:"key" json:"key"`
}

type PullRequests struct {
	Repository         Repository `json:"repository"`
	TotalPullRequests  string     `json:"total_pull_requests"`
	MergedPullRequests string     `json:"merged_pull_requests"`
	SentVsMerged       struct {
		PerWeek  string `json:"per_week"`
		PerMonth string `json:"per_month"`
	} `json:"sent_vs_merged"`
}

type Issues struct {
	Repository         Repository `json:"repository"`
	TotalPullRequests  string     `json:"total_pull_requests"`
	MergedPullRequests string     `json:"merged_pull_requests"`
	SentVsMerged       struct {
		PerWeek  string `json:"per_week"`
		PerMonth string `json:"per_month"`
	} `json:"sent_vs_merged"`
}

type Commits struct {
	Repository         Repository `json:"repository"`
	TotalPullRequests  string     `json:"total_pull_requests"`
	MergedPullRequests string     `json:"merged_pull_requests"`
	SentVsMerged       struct {
		PerWeek  string `json:"per_week"`
		PerMonth string `json:"per_month"`
	} `json:"sent_vs_merged"`
}
<commit_msg>Add granularity to models<commit_after>package main

import (
	"time"

	"gopkg.in/mgo.v2/bson"
)

type Indicator struct {
	ID   bson.ObjectId `bson:"_id" json:"id"`
	Name string        `bson:"name" json:"name"`
	Key  string        `bson:"key" json:"key"`
}

type Repository struct {
	ID         int    `json:"id"`
	Owner      string `json:"owner"`
	OwnerIsOrg bool   `json:"owner_is_org"`
	Name       string `json:"name"`
	FullName   string `json:"full_name"`
	URL        string `json:"url"`
}

type Docs struct {
	Repository    Repository `json:"repository"`
	ReadmeExists  bool       `json:"readme_exists"`
	ReadmeURL     string     `json:"readme_url"`
	LicenseExists bool       `json:"license_exists"`
	LicenseURL    string     `json:"license_url"`
	LicenseName   string     `json:"license_name"`
}

type ResponseTimes struct {
	Repository           Repository            `json:"repository"`
	AverageResponseTimes []AverageResponseTime `json:"average_response_times"`
}

type AverageResponseTime struct {
	Contributor           string    `json:"contributor"`
	AverageResponseTime   float32   `json:"average_response_time"`
	FirstContributionWeek time.Time `json:"first_contribution_week"`
}

type PullRequests struct {
	Repository         Repository `json:"repository"`
	TotalPullRequests  string     `json:"total_pull_requests"`
	MergedPullRequests string     `json:"merged_pull_requests"`
	SentVsMerged       Frequency  `json:"sent_vs_merged"`
}

type Issues struct {
	Repository         Repository `json:"repository"`
	TotalIssues        string     `json:"total_issues"`
	AverageTimeToClose string     `json:"average_time_to_close"`
	OpenVsClosed       Frequency  `json:"open_vs_closed"`
}

type Commits struct {
	Repository    Repository `json:"repository"`
	TotalCommits  int        `json:"total_commits"`
	FirstCommitAt time.Time  `json:"first_commit_at"`
	LastCommitAt  time.Time  `json:"last_commit_at"`
	CodeFrequency Frequency  `json:"code_frequency"`
}

type Frequency struct {
	PerWeek  string `json:"per_week"`
	PerMonth string `json:"per_month"`
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package maas

import (
	"fmt"
	"net/url"
	"strings"

	"github.com/juju/errors"
	"github.com/juju/gomaasapi"
	"github.com/juju/utils/set"

	"github.com/juju/juju/constraints"
	"github.com/juju/juju/network"
)

var unsupportedConstraints = []string{
	constraints.CpuPower,
	constraints.InstanceType,
	constraints.VirtType,
}

// ConstraintsValidator is defined on the Environs interface.
func (environ *maasEnviron) ConstraintsValidator() (constraints.Validator, error) {
	validator := constraints.NewValidator()
	validator.RegisterUnsupported(unsupportedConstraints)
	supportedArches, err := environ.SupportedArchitectures()
	if err != nil {
		return nil, err
	}
	validator.RegisterVocabulary(constraints.Arch, supportedArches)
	return validator, nil
}

// convertConstraints converts the given constraints into an url.Values object
// suitable to pass to MAAS when acquiring a node. CpuPower is ignored because
// it cannot be translated into something meaningful for MAAS right now.
func convertConstraints(cons constraints.Value) url.Values {
	params := url.Values{}
	if cons.Arch != nil {
		// Note: Juju and MAAS use the same architecture names.
		// MAAS also accepts a subarchitecture (e.g. "highbank"
		// for ARM), which defaults to "generic" if unspecified.
		params.Add("arch", *cons.Arch)
	}
	if cons.CpuCores != nil {
		params.Add("cpu_count", fmt.Sprintf("%d", *cons.CpuCores))
	}
	if cons.Mem != nil {
		params.Add("mem", fmt.Sprintf("%d", *cons.Mem))
	}
	convertTagsToParams(params, cons.Tags)
	if cons.CpuPower != nil {
		logger.Warningf("ignoring unsupported constraint 'cpu-power'")
	}
	return params
}

// convertConstraints2 converts the given constraints into a
// gomaasapi.AllocateMachineArgs for paasing to MAAS 2.
func convertConstraints2(cons constraints.Value) gomaasapi.AllocateMachineArgs {
	params := gomaasapi.AllocateMachineArgs{}
	if cons.Arch != nil {
		params.Architecture = *cons.Arch
	}
	if cons.CpuCores != nil {
		params.MinCPUCount = int(*cons.CpuCores)
	}
	if cons.Mem != nil {
		params.MinMemory = int(*cons.Mem)
	}
	if cons.Tags != nil {
		positives, negatives := parseDelimitedValues(*cons.Tags)
		if len(positives) > 0 {
			params.Tags = positives
		}
		if len(negatives) > 0 {
			params.NotTags = negatives
		}
	}
	if cons.CpuPower != nil {
		logger.Warningf("ignoring unsupported constraint 'cpu-power'")
	}
	return params
}

// convertTagsToParams converts a list of positive/negative tags from
// constraints into two comma-delimited lists of values, which can then be
// passed to MAAS using the "tags" and "not_tags" arguments to acquire. If
// either list of tags is empty, the respective argument is not added to params.
func convertTagsToParams(params url.Values, tags *[]string) {
	if tags == nil || len(*tags) == 0 {
		return
	}
	positives, negatives := parseDelimitedValues(*tags)
	if len(positives) > 0 {
		params.Add("tags", strings.Join(positives, ","))
	}
	if len(negatives) > 0 {
		params.Add("not_tags", strings.Join(negatives, ","))
	}
}

// convertSpacesFromConstraints extracts spaces from constraints and converts
// them to two lists of positive and negative spaces.
func convertSpacesFromConstraints(spaces *[]string) ([]string, []string) {
	if spaces == nil || len(*spaces) == 0 {
		return nil, nil
	}
	return parseDelimitedValues(*spaces)
}

// parseDelimitedValues parses a slice of raw values coming from constraints
// (Tags or Spaces). The result is split into two slices - positives and
// negatives (prefixed with "^"). Empty values are ignored.
func parseDelimitedValues(rawValues []string) (positives, negatives []string) {
	for _, value := range rawValues {
		if value == "" || value == "^" {
			// Neither of these cases should happen in practise, as constraints
			// are validated before setting them and empty names for spaces or
			// tags are not allowed.
			continue
		}
		if strings.HasPrefix(value, "^") {
			negatives = append(negatives, strings.TrimPrefix(value, "^"))
		} else {
			positives = append(positives, value)
		}
	}
	return positives, negatives
}

// interfaceBinding defines a requirement that a node interface must satisfy in
// order for that node to get selected and started, based on deploy-time
// bindings of a service.
//
// TODO(dimitern): Once the services have bindings defined in state, a version
// of this should go to the network package (needs to be non-MAAS-specifc
// first). Also, we need to transform Juju space names from constraints into
// MAAS space provider IDs.
type interfaceBinding struct {
	Name            string
	SpaceProviderId string

	// add more as needed.
}

// numericLabelLimit is a sentinel value used in addInterfaces to limit the
// number of disabmiguation inner loop iterations in case named labels clash
// with numeric labels for spaces coming from constraints. It's defined here to
// facilitate testing this behavior.
var numericLabelLimit uint = 0xffff

// addInterfaces converts a slice of interface bindings, postiveSpaces and
// negativeSpaces coming from constraints to the format MAAS expects for the
// "interfaces" and "not_networks" arguments to acquire node. Returns an error
// satisfying errors.IsNotValid() if the bindings contains duplicates, empty
// Name/SpaceProviderId, or if negative spaces clash with specified bindings.
// Duplicates between specified bindings and positiveSpaces are silently
// skipped.
func addInterfaces(
	params url.Values,
	bindings []interfaceBinding,
	positiveSpaces, negativeSpaces []network.SpaceInfo,
) error {
	combinedBindings, negatives, err := getBindings(bindings, positiveSpaces, negativeSpaces)
	if err != nil {
		return errors.Trace(err)
	}
	if len(combinedBindings) > 0 {
		combinedBindingsString := make([]string, len(combinedBindings))
		for i, binding := range combinedBindings {
			combinedBindingsString[i] = fmt.Sprintf("%s:space=%s", binding.Name, binding.SpaceProviderId)
		}
		params.Add("interfaces", strings.Join(combinedBindingsString, ";"))
	}
	if len(negatives) > 0 {
		negativesString := make([]string, len(negatives))
		for i, binding := range negatives {
			negativesString[i] = fmt.Sprintf("space:%s", binding.SpaceProviderId)
		}
		params.Add("not_networks", strings.Join(negativesString, ","))
	}
	return nil
}

func getBindings(
	bindings []interfaceBinding,
	positiveSpaces, negativeSpaces []network.SpaceInfo,
) ([]interfaceBinding, []interfaceBinding, error) {
	var (
		index            uint
		combinedBindings []interfaceBinding
	)
	namesSet := set.NewStrings()
	spacesSet := set.NewStrings()
	for _, binding := range bindings {
		switch {
		case binding.Name == "":
			return nil, nil, errors.NewNotValid(nil, "interface bindings cannot have empty names")
		case binding.SpaceProviderId == "":
			return nil, nil, errors.NewNotValid(nil, fmt.Sprintf(
				"invalid interface binding %q: space provider ID is required",
				binding.Name,
			))
		case namesSet.Contains(binding.Name):
			return nil, nil, errors.NewNotValid(nil, fmt.Sprintf(
				"duplicated interface binding %q",
				binding.Name,
			))
		}
		namesSet.Add(binding.Name)
		spacesSet.Add(binding.SpaceProviderId)

		combinedBindings = append(combinedBindings, binding)
	}

	for _, space := range positiveSpaces {
		if spacesSet.Contains(string(space.ProviderId)) {
			// Skip duplicates in positiveSpaces.
			continue
		}
		spacesSet.Add(string(space.ProviderId))
		// Make sure we pick a label that doesn't clash with possible bindings.
		var label string
		for {
			label = fmt.Sprintf("%v", index)
			if !namesSet.Contains(label) {
				break
			}
			if index > numericLabelLimit { // ...just to make sure we won't loop forever.
				return nil, nil, errors.Errorf("too many conflicting numeric labels, giving up.")
			}
			index++
		}
		namesSet.Add(label)
		combinedBindings = append(combinedBindings, interfaceBinding{label, string(space.ProviderId)})
		index++
	}

	var negatives []interfaceBinding
	for _, space := range negativeSpaces {
		if spacesSet.Contains(string(space.ProviderId)) {
			return nil, nil, errors.NewNotValid(nil, fmt.Sprintf(
				"negative space %q from constraints clashes with interface bindings",
				space.Name,
			))
		}
		var label string
		for {
			label = fmt.Sprintf("%v", index)
			if !namesSet.Contains(label) {
				break
			}
			if index > numericLabelLimit { // ...just to make sure we won't loop forever.
				return nil, nil, errors.Errorf("too many conflicting numeric labels, giving up.")
			}
			index++
		}
		namesSet.Add(label)
		negatives = append(negatives, interfaceBinding{label, string(space.ProviderId)})
		index++
	}
	return combinedBindings, negatives, nil
}

func addInterfaces2(
	params *gomaasapi.AllocateMachineArgs,
	bindings []interfaceBinding,
	positiveSpaces, negativeSpaces []network.SpaceInfo,
) error {
	combinedBindings, negatives, err := getBindings(bindings, positiveSpaces, negativeSpaces)
	if err != nil {
		return errors.Trace(err)
	}

	if len(combinedBindings) > 0 {
		interfaceSpecs := make([]gomaasapi.InterfaceSpec, len(combinedBindings))
		for i, space := range combinedBindings {
			interfaceSpecs[i] = gomaasapi.InterfaceSpec{space.Name, space.SpaceProviderId}
		}
		params.Interfaces = interfaceSpecs
	}
	if len(negatives) > 0 {
		negativeStrings := make([]string, len(negatives))
		for i, space := range negatives {
			negativeStrings[i] = space.SpaceProviderId
		}
		params.NotSpace = negativeStrings
	}
	return nil
}

// addStorage converts volume information into url.Values object suitable to
// pass to MAAS when acquiring a node.
func addStorage(params url.Values, volumes []volumeInfo) {
	if len(volumes) == 0 {
		return
	}
	// Requests for specific values are passed to the acquire URL
	// as a storage URL parameter of the form:
	// [volume-name:]sizeinGB[tag,...]
	// See http://maas.ubuntu.com/docs/api.html#nodes

	// eg storage=root:0(ssd),data:20(magnetic,5400rpm),45
	makeVolumeParams := func(v volumeInfo) string {
		var params string
		if v.name != "" {
			params = v.name + ":"
		}
		params += fmt.Sprintf("%d", v.sizeInGB)
		if len(v.tags) > 0 {
			params += fmt.Sprintf("(%s)", strings.Join(v.tags, ","))
		}
		return params
	}
	var volParms []string
	for _, v := range volumes {
		params := makeVolumeParams(v)
		volParms = append(volParms, params)
	}
	params.Add("storage", strings.Join(volParms, ","))
}

// addStorage2 adds volume information onto a gomaasapi.AllocateMachineArgs
// object suitable to pass to MAAS 2 when acquiring a node.
func addStorage(params *gomaasapi.AllocateMachineArgs, volumes []volumeInfo) {
	if len(volumes) == 0 {
		return
	}
	// Requests for specific values are passed to the acquire URL
	// as a storage URL parameter of the form:
	// [volume-name:]sizeinGB[tag,...]
	// See http://maas.ubuntu.com/docs/api.html#nodes

	// eg storage=root:0(ssd),data:20(magnetic,5400rpm),45
	makeVolumeParams := func(v volumeInfo) string {
		var params string
		if v.name != "" {
			params = v.name + ":"
		}
		params += fmt.Sprintf("%d", v.sizeInGB)
		if len(v.tags) > 0 {
			params += fmt.Sprintf("(%s)", strings.Join(v.tags, ","))
		}
		return params
	}
	var volParms []string
	for _, v := range volumes {
		params := makeVolumeParams(v)
		volParms = append(volParms, params)
	}
	params.Add("storage", strings.Join(volParms, ","))
}
<commit_msg>addStorage2<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package maas

import (
	"fmt"
	"net/url"
	"strings"

	"github.com/juju/errors"
	"github.com/juju/gomaasapi"
	"github.com/juju/utils/set"

	"github.com/juju/juju/constraints"
	"github.com/juju/juju/network"
)

var unsupportedConstraints = []string{
	constraints.CpuPower,
	constraints.InstanceType,
	constraints.VirtType,
}

// ConstraintsValidator is defined on the Environs interface.
func (environ *maasEnviron) ConstraintsValidator() (constraints.Validator, error) {
	validator := constraints.NewValidator()
	validator.RegisterUnsupported(unsupportedConstraints)
	supportedArches, err := environ.SupportedArchitectures()
	if err != nil {
		return nil, err
	}
	validator.RegisterVocabulary(constraints.Arch, supportedArches)
	return validator, nil
}

// convertConstraints converts the given constraints into an url.Values object
// suitable to pass to MAAS when acquiring a node. CpuPower is ignored because
// it cannot be translated into something meaningful for MAAS right now.
func convertConstraints(cons constraints.Value) url.Values {
	params := url.Values{}
	if cons.Arch != nil {
		// Note: Juju and MAAS use the same architecture names.
		// MAAS also accepts a subarchitecture (e.g. "highbank"
		// for ARM), which defaults to "generic" if unspecified.
		params.Add("arch", *cons.Arch)
	}
	if cons.CpuCores != nil {
		params.Add("cpu_count", fmt.Sprintf("%d", *cons.CpuCores))
	}
	if cons.Mem != nil {
		params.Add("mem", fmt.Sprintf("%d", *cons.Mem))
	}
	convertTagsToParams(params, cons.Tags)
	if cons.CpuPower != nil {
		logger.Warningf("ignoring unsupported constraint 'cpu-power'")
	}
	return params
}

// convertConstraints2 converts the given constraints into a
// gomaasapi.AllocateMachineArgs for paasing to MAAS 2.
func convertConstraints2(cons constraints.Value) gomaasapi.AllocateMachineArgs {
	params := gomaasapi.AllocateMachineArgs{}
	if cons.Arch != nil {
		params.Architecture = *cons.Arch
	}
	if cons.CpuCores != nil {
		params.MinCPUCount = int(*cons.CpuCores)
	}
	if cons.Mem != nil {
		params.MinMemory = int(*cons.Mem)
	}
	if cons.Tags != nil {
		positives, negatives := parseDelimitedValues(*cons.Tags)
		if len(positives) > 0 {
			params.Tags = positives
		}
		if len(negatives) > 0 {
			params.NotTags = negatives
		}
	}
	if cons.CpuPower != nil {
		logger.Warningf("ignoring unsupported constraint 'cpu-power'")
	}
	return params
}

// convertTagsToParams converts a list of positive/negative tags from
// constraints into two comma-delimited lists of values, which can then be
// passed to MAAS using the "tags" and "not_tags" arguments to acquire. If
// either list of tags is empty, the respective argument is not added to params.
func convertTagsToParams(params url.Values, tags *[]string) {
	if tags == nil || len(*tags) == 0 {
		return
	}
	positives, negatives := parseDelimitedValues(*tags)
	if len(positives) > 0 {
		params.Add("tags", strings.Join(positives, ","))
	}
	if len(negatives) > 0 {
		params.Add("not_tags", strings.Join(negatives, ","))
	}
}

// convertSpacesFromConstraints extracts spaces from constraints and converts
// them to two lists of positive and negative spaces.
func convertSpacesFromConstraints(spaces *[]string) ([]string, []string) {
	if spaces == nil || len(*spaces) == 0 {
		return nil, nil
	}
	return parseDelimitedValues(*spaces)
}

// parseDelimitedValues parses a slice of raw values coming from constraints
// (Tags or Spaces). The result is split into two slices - positives and
// negatives (prefixed with "^"). Empty values are ignored.
func parseDelimitedValues(rawValues []string) (positives, negatives []string) {
	for _, value := range rawValues {
		if value == "" || value == "^" {
			// Neither of these cases should happen in practise, as constraints
			// are validated before setting them and empty names for spaces or
			// tags are not allowed.
			continue
		}
		if strings.HasPrefix(value, "^") {
			negatives = append(negatives, strings.TrimPrefix(value, "^"))
		} else {
			positives = append(positives, value)
		}
	}
	return positives, negatives
}

// interfaceBinding defines a requirement that a node interface must satisfy in
// order for that node to get selected and started, based on deploy-time
// bindings of a service.
//
// TODO(dimitern): Once the services have bindings defined in state, a version
// of this should go to the network package (needs to be non-MAAS-specifc
// first). Also, we need to transform Juju space names from constraints into
// MAAS space provider IDs.
type interfaceBinding struct {
	Name            string
	SpaceProviderId string

	// add more as needed.
}

// numericLabelLimit is a sentinel value used in addInterfaces to limit the
// number of disabmiguation inner loop iterations in case named labels clash
// with numeric labels for spaces coming from constraints. It's defined here to
// facilitate testing this behavior.
var numericLabelLimit uint = 0xffff

// addInterfaces converts a slice of interface bindings, postiveSpaces and
// negativeSpaces coming from constraints to the format MAAS expects for the
// "interfaces" and "not_networks" arguments to acquire node. Returns an error
// satisfying errors.IsNotValid() if the bindings contains duplicates, empty
// Name/SpaceProviderId, or if negative spaces clash with specified bindings.
// Duplicates between specified bindings and positiveSpaces are silently
// skipped.
func addInterfaces(
	params url.Values,
	bindings []interfaceBinding,
	positiveSpaces, negativeSpaces []network.SpaceInfo,
) error {
	combinedBindings, negatives, err := getBindings(bindings, positiveSpaces, negativeSpaces)
	if err != nil {
		return errors.Trace(err)
	}
	if len(combinedBindings) > 0 {
		combinedBindingsString := make([]string, len(combinedBindings))
		for i, binding := range combinedBindings {
			combinedBindingsString[i] = fmt.Sprintf("%s:space=%s", binding.Name, binding.SpaceProviderId)
		}
		params.Add("interfaces", strings.Join(combinedBindingsString, ";"))
	}
	if len(negatives) > 0 {
		negativesString := make([]string, len(negatives))
		for i, binding := range negatives {
			negativesString[i] = fmt.Sprintf("space:%s", binding.SpaceProviderId)
		}
		params.Add("not_networks", strings.Join(negativesString, ","))
	}
	return nil
}

func getBindings(
	bindings []interfaceBinding,
	positiveSpaces, negativeSpaces []network.SpaceInfo,
) ([]interfaceBinding, []interfaceBinding, error) {
	var (
		index            uint
		combinedBindings []interfaceBinding
	)
	namesSet := set.NewStrings()
	spacesSet := set.NewStrings()
	for _, binding := range bindings {
		switch {
		case binding.Name == "":
			return nil, nil, errors.NewNotValid(nil, "interface bindings cannot have empty names")
		case binding.SpaceProviderId == "":
			return nil, nil, errors.NewNotValid(nil, fmt.Sprintf(
				"invalid interface binding %q: space provider ID is required",
				binding.Name,
			))
		case namesSet.Contains(binding.Name):
			return nil, nil, errors.NewNotValid(nil, fmt.Sprintf(
				"duplicated interface binding %q",
				binding.Name,
			))
		}
		namesSet.Add(binding.Name)
		spacesSet.Add(binding.SpaceProviderId)

		combinedBindings = append(combinedBindings, binding)
	}

	for _, space := range positiveSpaces {
		if spacesSet.Contains(string(space.ProviderId)) {
			// Skip duplicates in positiveSpaces.
			continue
		}
		spacesSet.Add(string(space.ProviderId))
		// Make sure we pick a label that doesn't clash with possible bindings.
		var label string
		for {
			label = fmt.Sprintf("%v", index)
			if !namesSet.Contains(label) {
				break
			}
			if index > numericLabelLimit { // ...just to make sure we won't loop forever.
				return nil, nil, errors.Errorf("too many conflicting numeric labels, giving up.")
			}
			index++
		}
		namesSet.Add(label)
		combinedBindings = append(combinedBindings, interfaceBinding{label, string(space.ProviderId)})
		index++
	}

	var negatives []interfaceBinding
	for _, space := range negativeSpaces {
		if spacesSet.Contains(string(space.ProviderId)) {
			return nil, nil, errors.NewNotValid(nil, fmt.Sprintf(
				"negative space %q from constraints clashes with interface bindings",
				space.Name,
			))
		}
		var label string
		for {
			label = fmt.Sprintf("%v", index)
			if !namesSet.Contains(label) {
				break
			}
			if index > numericLabelLimit { // ...just to make sure we won't loop forever.
				return nil, nil, errors.Errorf("too many conflicting numeric labels, giving up.")
			}
			index++
		}
		namesSet.Add(label)
		negatives = append(negatives, interfaceBinding{label, string(space.ProviderId)})
		index++
	}
	return combinedBindings, negatives, nil
}

func addInterfaces2(
	params *gomaasapi.AllocateMachineArgs,
	bindings []interfaceBinding,
	positiveSpaces, negativeSpaces []network.SpaceInfo,
) error {
	combinedBindings, negatives, err := getBindings(bindings, positiveSpaces, negativeSpaces)
	if err != nil {
		return errors.Trace(err)
	}

	if len(combinedBindings) > 0 {
		interfaceSpecs := make([]gomaasapi.InterfaceSpec, len(combinedBindings))
		for i, space := range combinedBindings {
			interfaceSpecs[i] = gomaasapi.InterfaceSpec{space.Name, space.SpaceProviderId}
		}
		params.Interfaces = interfaceSpecs
	}
	if len(negatives) > 0 {
		negativeStrings := make([]string, len(negatives))
		for i, space := range negatives {
			negativeStrings[i] = space.SpaceProviderId
		}
		params.NotSpace = negativeStrings
	}
	return nil
}

// addStorage converts volume information into url.Values object suitable to
// pass to MAAS when acquiring a node.
func addStorage(params url.Values, volumes []volumeInfo) {
	if len(volumes) == 0 {
		return
	}
	// Requests for specific values are passed to the acquire URL
	// as a storage URL parameter of the form:
	// [volume-name:]sizeinGB[tag,...]
	// See http://maas.ubuntu.com/docs/api.html#nodes

	// eg storage=root:0(ssd),data:20(magnetic,5400rpm),45
	makeVolumeParams := func(v volumeInfo) string {
		var params string
		if v.name != "" {
			params = v.name + ":"
		}
		params += fmt.Sprintf("%d", v.sizeInGB)
		if len(v.tags) > 0 {
			params += fmt.Sprintf("(%s)", strings.Join(v.tags, ","))
		}
		return params
	}
	var volParms []string
	for _, v := range volumes {
		params := makeVolumeParams(v)
		volParms = append(volParms, params)
	}
	params.Add("storage", strings.Join(volParms, ","))
}

// addStorage2 adds volume information onto a gomaasapi.AllocateMachineArgs
// object suitable to pass to MAAS 2 when acquiring a node.
func addStorage2(params *gomaasapi.AllocateMachineArgs, volumes []volumeInfo) {
	if len(volumes) == 0 {
		return
	}
	// Requests for specific values are passed to the acquire URL
	// as a storage URL parameter of the form:
	// [volume-name:]sizeinGB[tag,...]
	// See http://maas.ubuntu.com/docs/api.html#nodes

	// eg storage=root:0(ssd),data:20(magnetic,5400rpm),45
	makeVolumeParams := func(v volumeInfo) (params gomaasapi.StorageSpec) {
		params.Label = v.name
		params.Size = int(v.sizeInGB)
		params.Tags = v.tags
		return params
	}
	var volParams []gomaasapi.StorageSpec
	for _, v := range volumes {
		params := makeVolumeParams(v)
		volParams = append(volParams, params)
	}
	params.Storage = volParams
}
<|endoftext|>


<commit_before>package aws

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/docker/libmachete/provisioners/api"
	"github.com/docker/libmachete/provisioners/aws/mock"
	"github.com/golang/mock/gomock"
	"github.com/stretchr/testify/require"
	"gopkg.in/yaml.v2"
	"reflect"
	"testing"
	"time"
)

//go:generate mockgen -package mock -destination mock/mock_ec2iface.go github.com/aws/aws-sdk-go/service/ec2/ec2iface EC2API

func noSleep(time.Duration) {
	// no-op - don't sleep in tests
}

func TestCreateInstanceSync(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	clientMock := mock.NewMockEC2API(ctrl)

	request := new(CreateInstanceRequest)
	require.Nil(t, json.Unmarshal([]byte(testCreateSync[0]), request))

	reservation := ec2.Reservation{
		Instances: []*ec2.Instance{{InstanceId: aws.String("test-id")}}}
	// Validates command against a known-good value.
	matcher := func(input *ec2.RunInstancesInput) {
		expectedInput := new(ec2.RunInstancesInput)
		require.Nil(t, json.Unmarshal([]byte(testCreateSync[1]), expectedInput))
		if !reflect.DeepEqual(expectedInput, input) {
			t.Error("Expected and actual did not match.", expectedInput, input)
		}
	}
	clientMock.EXPECT().RunInstances(gomock.Any()).Do(matcher).Return(&reservation, nil)

	fmt.Println(">>>>>>", *request)

	instance, err := createInstanceSync(clientMock, *request)

	require.Nil(t, err)
	require.NotNil(t, instance)
}

type WrongRequestType struct {
}

func (w WrongRequestType) Name() string {
	return "nope"
}

func (w WrongRequestType) ProvisionerName() string {
	return "nope"
}

func (w WrongRequestType) Version() string {
	return "nope"
}

func (w WrongRequestType) ProvisionWorkflow() []api.TaskName {
	return []api.TaskName{}
}

func TestCreateIncompatibleType(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	clientMock := mock.NewMockEC2API(ctrl)

	p := &provisioner{client: clientMock, sleepFunction: noSleep, config: defaultConfig()}
	_, err := p.CreateInstance(&WrongRequestType{})
	require.NotNil(t, err)
}

// TODO(wfarner): Inline this function.
func makeDescribeOutput(instanceState string) *ec2.DescribeInstancesOutput {
	return &ec2.DescribeInstancesOutput{
		Reservations: []*ec2.Reservation{
			{Instances: []*ec2.Instance{
				{State: &ec2.InstanceState{Name: &instanceState}}},
			},
		},
	}
}

func collectCreateInstanceEvents(
	eventChan <-chan api.CreateInstanceEvent) []api.CreateInstanceEvent {

	var events []api.CreateInstanceEvent
	for event := range eventChan {
		events = append(events, event)
	}
	return events
}

func expectDescribeCall(
	clientMock *mock.MockEC2API,
	instanceID string,
	returnedState string) {

	clientMock.EXPECT().
		DescribeInstances(&ec2.DescribeInstancesInput{InstanceIds: []*string{&instanceID}}).
		Return(makeDescribeOutput(returnedState), nil)
}

func TestCreateInstanceSuccess(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	clientMock := mock.NewMockEC2API(ctrl)

	instanceID := "test-id"
	reservation := ec2.Reservation{Instances: []*ec2.Instance{{InstanceId: &instanceID}}}
	clientMock.EXPECT().RunInstances(gomock.Any()).Return(&reservation, nil)

	// Simulate the instance not being available yet, the first describe returns nothing.
	expectDescribeCall(clientMock, instanceID, ec2.InstanceStateNamePending)

	// The instance is now running.
	expectDescribeCall(clientMock, instanceID, ec2.InstanceStateNameRunning)

	// The instance is now running.  Second call to get the ip address
	expectDescribeCall(clientMock, instanceID, ec2.InstanceStateNameRunning)

	tagRequest := ec2.CreateTagsInput{
		Resources: []*string{&instanceID},
		Tags: []*ec2.Tag{
			{Key: aws.String("test"), Value: aws.String("test2")},
			{Key: aws.String("Name"), Value: aws.String("test-instance")},
		},
	}
	clientMock.EXPECT().CreateTags(&tagRequest).Return(&ec2.CreateTagsOutput{}, nil)

	provisioner := provisioner{client: clientMock, sleepFunction: noSleep, config: defaultConfig()}
	request := &CreateInstanceRequest{
		BaseMachineRequest: api.BaseMachineRequest{MachineName: "test-instance"},
		Tags:               map[string]string{"test": "test2"},
	}
	eventChan, err := provisioner.CreateInstance(request)

	require.Nil(t, err)
	expectedEvents := []api.CreateInstanceEvent{
		{Type: api.CreateInstanceStarted},
		{Type: api.CreateInstanceCompleted, InstanceID: instanceID, Machine: request}}
	require.Equal(t, expectedEvents, collectCreateInstanceEvents(eventChan))
}

func TestCreateInstanceError(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	clientMock := mock.NewMockEC2API(ctrl)

	runError := errors.New("request failed")
	clientMock.EXPECT().RunInstances(gomock.Any()).Return(&ec2.Reservation{}, runError)

	provisioner := provisioner{client: clientMock, sleepFunction: noSleep, config: defaultConfig()}
	eventChan, err := provisioner.CreateInstance(&CreateInstanceRequest{})

	require.Nil(t, err)
	expectedEvents := []api.CreateInstanceEvent{
		{Type: api.CreateInstanceStarted},
		{Type: api.CreateInstanceError, Error: runError}}
	require.Equal(t, expectedEvents, collectCreateInstanceEvents(eventChan))
}

func collectDestroyInstanceEvents(
	eventChan <-chan api.DestroyInstanceEvent) []api.DestroyInstanceEvent {

	var events []api.DestroyInstanceEvent
	for event := range eventChan {
		events = append(events, event)
	}
	return events
}

func TestDestroyInstanceSuccess(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	clientMock := mock.NewMockEC2API(ctrl)

	instanceID := "test-id"

	clientMock.EXPECT().
		TerminateInstances(
			&ec2.TerminateInstancesInput{InstanceIds: []*string{&instanceID}}).
		Return(&ec2.TerminateInstancesOutput{
			TerminatingInstances: []*ec2.InstanceStateChange{{
				InstanceId: &instanceID,
			}}}, nil)

	// Instance is in terminating state, not yet terminated.
	expectDescribeCall(clientMock, instanceID, ec2.InstanceStateNameStopping)

	expectDescribeCall(clientMock, instanceID, ec2.InstanceStateNameTerminated)

	provisioner := provisioner{client: clientMock, sleepFunction: noSleep, config: defaultConfig()}
	eventChan, err := provisioner.DestroyInstance(instanceID)

	require.Nil(t, err)
	expectedEvents := []api.DestroyInstanceEvent{
		{Type: api.DestroyInstanceStarted},
		{Type: api.DestroyInstanceCompleted}}
	require.Equal(t, expectedEvents, collectDestroyInstanceEvents(eventChan))
}

func TestDestroyInstanceError(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	clientMock := mock.NewMockEC2API(ctrl)

	runError := errors.New("request failed")
	clientMock.EXPECT().TerminateInstances(gomock.Any()).
		Return(&ec2.TerminateInstancesOutput{}, runError)

	provisioner := provisioner{client: clientMock, sleepFunction: noSleep, config: defaultConfig()}
	eventChan, err := provisioner.DestroyInstance("test-id")

	require.Nil(t, err)
	expectedEvents := []api.DestroyInstanceEvent{
		{Type: api.DestroyInstanceStarted},
		{Type: api.DestroyInstanceError, Error: runError}}
	require.Equal(t, expectedEvents, collectDestroyInstanceEvents(eventChan))
}

const yamlDoc = `name: database
availability_zone: us-west-2a
image_id: ami-5
block_device_name: /dev/sdb
root_size: 64
volume_type: gp2
delete_on_termination: true
security_group_ids: [sg-1, sg-2]
subnet_id: my-subnet-id
instance_type: t2.micro
private_ip_address: 127.0.0.1
associate_public_ip_address: true
private_ip_only: true
ebs_optimized: true
iam_instance_profile: my-iam-profile
tags:
  Name: unit-test-create
  test: aws-create-test
key_name: dev
vpc_id: my-vpc-id
monitoring: true`

func TestYamlSpec(t *testing.T) {
	expected := CreateInstanceRequest{
		BaseMachineRequest:       api.BaseMachineRequest{MachineName: "database"},
		AvailabilityZone:         "us-west-2a",
		ImageID:                  "ami-5",
		BlockDeviceName:          "/dev/sdb",
		RootSize:                 64,
		VolumeType:               "gp2",
		DeleteOnTermination:      true,
		SecurityGroupIds:         []string{"sg-1", "sg-2"},
		SubnetID:                 "my-subnet-id",
		InstanceType:             "t2.micro",
		PrivateIPAddress:         "127.0.0.1",
		AssociatePublicIPAddress: true,
		PrivateIPOnly:            true,
		EbsOptimized:             true,
		IamInstanceProfile:       "my-iam-profile",
		Tags: map[string]string{
			"Name": "unit-test-create",
			"test": "aws-create-test"},
		KeyName:    "dev",
		VpcID:      "my-vpc-id",
		Monitoring: true,
	}
	actual := CreateInstanceRequest{}
	err := yaml.Unmarshal([]byte(yamlDoc), &actual)
	require.Nil(t, err)
	require.Equal(t, expected, actual)
}
<commit_msg>Re-enable test cases in machine_test. (#49)<commit_after>package aws

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/docker/libmachete/provisioners/api"
	"github.com/docker/libmachete/provisioners/aws/mock"
	"github.com/golang/mock/gomock"
	"github.com/stretchr/testify/require"
	"gopkg.in/yaml.v2"
	"reflect"
	"testing"
	"time"
)

//go:generate mockgen -package mock -destination mock/mock_ec2iface.go github.com/aws/aws-sdk-go/service/ec2/ec2iface EC2API

func noSleep(time.Duration) {
	// no-op - don't sleep in tests
}

func TestCreateInstanceSync(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	clientMock := mock.NewMockEC2API(ctrl)

	request := new(CreateInstanceRequest)
	require.Nil(t, json.Unmarshal([]byte(testCreateSync[0]), request))

	reservation := ec2.Reservation{
		Instances: []*ec2.Instance{{InstanceId: aws.String("test-id")}}}
	// Validates command against a known-good value.
	matcher := func(input *ec2.RunInstancesInput) {
		expectedInput := new(ec2.RunInstancesInput)
		require.Nil(t, json.Unmarshal([]byte(testCreateSync[1]), expectedInput))
		if !reflect.DeepEqual(expectedInput, input) {
			t.Error("Expected and actual did not match.", expectedInput, input)
		}
	}
	clientMock.EXPECT().RunInstances(gomock.Any()).Do(matcher).Return(&reservation, nil)

	fmt.Println(">>>>>>", *request)

	instance, err := createInstanceSync(clientMock, *request)

	require.Nil(t, err)
	require.NotNil(t, instance)
}

type WrongRequestType struct {
	api.BaseMachineRequest
}

func (w WrongRequestType) ProvisionWorkflow() []api.TaskName {
	return []api.TaskName{}
}

func TestCreateIncompatibleType(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	clientMock := mock.NewMockEC2API(ctrl)

	p := &provisioner{client: clientMock, sleepFunction: noSleep, config: defaultConfig()}
	_, err := p.CreateInstance(&WrongRequestType{})
	require.NotNil(t, err)
}

// TODO(wfarner): Inline this function.
func makeDescribeOutput(instanceState string) *ec2.DescribeInstancesOutput {
	return &ec2.DescribeInstancesOutput{
		Reservations: []*ec2.Reservation{
			{Instances: []*ec2.Instance{
				{State: &ec2.InstanceState{Name: &instanceState}}},
			},
		},
	}
}

func collectCreateInstanceEvents(
	eventChan <-chan api.CreateInstanceEvent) []api.CreateInstanceEvent {

	var events []api.CreateInstanceEvent
	for event := range eventChan {
		events = append(events, event)
	}
	return events
}

func expectDescribeCall(
	clientMock *mock.MockEC2API,
	instanceID string,
	returnedState string) {

	clientMock.EXPECT().
		DescribeInstances(&ec2.DescribeInstancesInput{InstanceIds: []*string{&instanceID}}).
		Return(makeDescribeOutput(returnedState), nil)
}

func TestCreateInstanceSuccess(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	clientMock := mock.NewMockEC2API(ctrl)

	instanceID := "test-id"
	reservation := ec2.Reservation{Instances: []*ec2.Instance{{InstanceId: &instanceID}}}
	clientMock.EXPECT().RunInstances(gomock.Any()).Return(&reservation, nil)

	// Simulate the instance not being available yet, the first describe returns nothing.
	expectDescribeCall(clientMock, instanceID, ec2.InstanceStateNamePending)

	// The instance is now running.
	expectDescribeCall(clientMock, instanceID, ec2.InstanceStateNameRunning)

	// The instance is now running.  Second call to get the ip address
	expectDescribeCall(clientMock, instanceID, ec2.InstanceStateNameRunning)

	tagRequest := ec2.CreateTagsInput{
		Resources: []*string{&instanceID},
		Tags: []*ec2.Tag{
			{Key: aws.String("test"), Value: aws.String("test2")},
			{Key: aws.String("Name"), Value: aws.String("test-instance")},
		},
	}
	clientMock.EXPECT().CreateTags(&tagRequest).Return(&ec2.CreateTagsOutput{}, nil)

	provisioner := provisioner{client: clientMock, sleepFunction: noSleep, config: defaultConfig()}
	request := &CreateInstanceRequest{
		BaseMachineRequest: api.BaseMachineRequest{MachineName: "test-instance"},
		Tags:               map[string]string{"test": "test2"},
	}
	eventChan, err := provisioner.CreateInstance(request)

	require.Nil(t, err)
	expectedEvents := []api.CreateInstanceEvent{
		{Type: api.CreateInstanceStarted},
		{Type: api.CreateInstanceCompleted, InstanceID: instanceID, Machine: request}}
	require.Equal(t, expectedEvents, collectCreateInstanceEvents(eventChan))
}

func TestCreateInstanceError(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	clientMock := mock.NewMockEC2API(ctrl)

	runError := errors.New("request failed")
	clientMock.EXPECT().RunInstances(gomock.Any()).Return(&ec2.Reservation{}, runError)

	provisioner := provisioner{client: clientMock, sleepFunction: noSleep, config: defaultConfig()}
	eventChan, err := provisioner.CreateInstance(&CreateInstanceRequest{})

	require.Nil(t, err)
	expectedEvents := []api.CreateInstanceEvent{
		{Type: api.CreateInstanceStarted},
		{Type: api.CreateInstanceError, Error: runError}}
	require.Equal(t, expectedEvents, collectCreateInstanceEvents(eventChan))
}

func collectDestroyInstanceEvents(
	eventChan <-chan api.DestroyInstanceEvent) []api.DestroyInstanceEvent {

	var events []api.DestroyInstanceEvent
	for event := range eventChan {
		events = append(events, event)
	}
	return events
}

func TestDestroyInstanceSuccess(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	clientMock := mock.NewMockEC2API(ctrl)

	instanceID := "test-id"

	clientMock.EXPECT().
		TerminateInstances(
			&ec2.TerminateInstancesInput{InstanceIds: []*string{&instanceID}}).
		Return(&ec2.TerminateInstancesOutput{
			TerminatingInstances: []*ec2.InstanceStateChange{{
				InstanceId: &instanceID,
			}}}, nil)

	// Instance is in terminating state, not yet terminated.
	expectDescribeCall(clientMock, instanceID, ec2.InstanceStateNameStopping)

	expectDescribeCall(clientMock, instanceID, ec2.InstanceStateNameTerminated)

	provisioner := provisioner{client: clientMock, sleepFunction: noSleep, config: defaultConfig()}
	eventChan, err := provisioner.DestroyInstance(instanceID)

	require.Nil(t, err)
	expectedEvents := []api.DestroyInstanceEvent{
		{Type: api.DestroyInstanceStarted},
		{Type: api.DestroyInstanceCompleted}}
	require.Equal(t, expectedEvents, collectDestroyInstanceEvents(eventChan))
}

func TestDestroyInstanceError(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	clientMock := mock.NewMockEC2API(ctrl)

	runError := errors.New("request failed")
	clientMock.EXPECT().TerminateInstances(gomock.Any()).
		Return(&ec2.TerminateInstancesOutput{}, runError)

	provisioner := provisioner{client: clientMock, sleepFunction: noSleep, config: defaultConfig()}
	eventChan, err := provisioner.DestroyInstance("test-id")

	require.Nil(t, err)
	expectedEvents := []api.DestroyInstanceEvent{
		{Type: api.DestroyInstanceStarted},
		{Type: api.DestroyInstanceError, Error: runError}}
	require.Equal(t, expectedEvents, collectDestroyInstanceEvents(eventChan))
}

const yamlDoc = `name: database
availability_zone: us-west-2a
image_id: ami-5
block_device_name: /dev/sdb
root_size: 64
volume_type: gp2
delete_on_termination: true
security_group_ids: [sg-1, sg-2]
subnet_id: my-subnet-id
instance_type: t2.micro
private_ip_address: 127.0.0.1
associate_public_ip_address: true
private_ip_only: true
ebs_optimized: true
iam_instance_profile: my-iam-profile
tags:
  Name: unit-test-create
  test: aws-create-test
key_name: dev
vpc_id: my-vpc-id
monitoring: true`

func TestYamlSpec(t *testing.T) {
	expected := CreateInstanceRequest{
		BaseMachineRequest:       api.BaseMachineRequest{MachineName: "database"},
		AvailabilityZone:         "us-west-2a",
		ImageID:                  "ami-5",
		BlockDeviceName:          "/dev/sdb",
		RootSize:                 64,
		VolumeType:               "gp2",
		DeleteOnTermination:      true,
		SecurityGroupIds:         []string{"sg-1", "sg-2"},
		SubnetID:                 "my-subnet-id",
		InstanceType:             "t2.micro",
		PrivateIPAddress:         "127.0.0.1",
		AssociatePublicIPAddress: true,
		PrivateIPOnly:            true,
		EbsOptimized:             true,
		IamInstanceProfile:       "my-iam-profile",
		Tags: map[string]string{
			"Name": "unit-test-create",
			"test": "aws-create-test"},
		KeyName:    "dev",
		VpcID:      "my-vpc-id",
		Monitoring: true,
	}
	actual := CreateInstanceRequest{}
	err := yaml.Unmarshal([]byte(yamlDoc), &actual)
	require.Nil(t, err)
	require.Equal(t, expected, actual)
}
<|endoftext|>


<commit_before>package monico

type Moniter struct {
	path string
}
<commit_msg>Define NewMoniter<commit_after>package monico

type Moniter struct {
	path string
}

func NewMoniter(path string) (*Moniter, error) {
	return &Moniter{}, nil
}
<|endoftext|>


<commit_before>package composition

import (
	"github.com/tarent/lib-compose/logging"
	"io"
	"net/http"
	"strconv"
	"strings"
)

// A ContentFetcherFactory returns a configured fetch job for a request
// which can return the fetch results.
type ContentFetcherFactory func(r *http.Request) FetchResultSupplier

type CompositionHandler struct {
	contentFetcherFactory ContentFetcherFactory
	contentMergerFactory  func(metaJSON map[string]interface{}) ContentMerger
	cache                 Cache
}

// NewCompositionHandler creates a new Handler with the supplied defaultData,
// which is used for each request.
func NewCompositionHandler(contentFetcherFactory ContentFetcherFactory) *CompositionHandler {
	return &CompositionHandler{
		contentFetcherFactory: contentFetcherFactory,
		contentMergerFactory: func(metaJSON map[string]interface{}) ContentMerger {
			return NewContentMerge(metaJSON)
		},
		cache: nil,
	}
}

func NewCompositionHandlerWithCache(contentFetcherFactory ContentFetcherFactory, cache Cache) *CompositionHandler {
	return &CompositionHandler{
		contentFetcherFactory: contentFetcherFactory,
		contentMergerFactory: func(metaJSON map[string]interface{}) ContentMerger {
			return NewContentMerge(metaJSON)
		},
		cache: cache,
	}
}

func (agg *CompositionHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	fetcher := agg.contentFetcherFactory(r)

	if agg.handleEmptyFetcher(fetcher, w, r) {
		return
	}

	// fetch all contents
	results := fetcher.WaitForResults()

	// Allow HEAD requests and disable composition of body fragments
	if agg.handleHeadRequests(results, w, r) {
		return
	}

	mergeContext := agg.contentMergerFactory(fetcher.MetaJSON())

	for _, res := range results {
		if res.Err == nil && res.Content != nil {

			if agg.handleForwardingRequests(res, w, r) {
				// Return if it's a forwarded request
				return
			}

			if agg.handleRequestsWithBody(res, w, r) {
				// Return if it's a request body
				return
			}

			mergeContext.AddContent(res)

		} else if res.Def.Required {
			LogFetchResultLoadingError(res, w, r)
			return
		} else {
			logging.Application(r.Header).WithField("fetchResult", res).Warnf("optional content not loaded: %v", res.Def.URL)
		}
	}

	status := agg.extractStatusCode(results, w, r)

	// Overwrite Content-Type to ensure, that the encoding is correct
	w.Header().Set("Content-Type", "text/html; charset=utf-8")

	html, err := agg.processHtml(mergeContext, w, r)
	// Return if an error occured within the html aggregation
	if err {
		return
	}

	w.Header().Set("Content-Length", strconv.Itoa(len(html)))
	w.WriteHeader(status)
	w.Write(html)
}

func (agg *CompositionHandler) extractStatusCode(results []*FetchResult, w http.ResponseWriter, r *http.Request) (statusCode int) {
	// Take status code and headers from first fetch definition
	if len(results) > 0 {
		copyHeaders(results[0].Content.HttpHeader(), w.Header(), ForwardResponseHeaders)
		if results[0].Content.HttpStatusCode() != 0 {
			return results[0].Content.HttpStatusCode()
		}
	}
	return 200
}

func (agg *CompositionHandler) processHtml(mergeContext ContentMerger, w http.ResponseWriter, r *http.Request) ([]byte, bool) {
	html, err := mergeContext.GetHtml()
	if err != nil {
		if agg.cache != nil {
			agg.cache.PurgeEntries(mergeContext.GetHashes())
		}
		logging.Application(r.Header).Error(err.Error())
		http.Error(w, "Internal Server Error: "+err.Error(), 500)
		return nil, true
	}
	return html, false
}

func (agg *CompositionHandler) handleHeadRequests(results []*FetchResult, w http.ResponseWriter, r *http.Request) bool {
	if r.Method == "HEAD" && len(results) > 0 {
		copyHeaders(results[0].Content.HttpHeader(), w.Header(), ForwardResponseHeaders)
		w.WriteHeader(results[0].Content.HttpStatusCode())
		return true
	}
	return false
}

func (agg *CompositionHandler) handleEmptyFetcher(fetcher FetchResultSupplier, w http.ResponseWriter, r *http.Request) bool {
	if fetcher.Empty() {
		w.WriteHeader(500)
		w.Write([]byte("Internal server error"))
		logging.Application(r.Header).Error("No fetchers available for composition, throwing error 500")
		return true
	}
	return false
}

func (agg *CompositionHandler) handleForwardingRequests(result *FetchResult, w http.ResponseWriter, r *http.Request) bool {
	if result.Content.HttpStatusCode() >= 300 && result.Content.HttpStatusCode() <= 308 {
		copyHeaders(result.Content.HttpHeader(), w.Header(), ForwardResponseHeaders)
		w.WriteHeader(result.Content.HttpStatusCode())
		return true
	}
	return false
}

func (agg *CompositionHandler) handleRequestsWithBody(result *FetchResult, w http.ResponseWriter, r *http.Request) bool {
	if result.Content.Reader() != nil {
		copyHeaders(result.Content.HttpHeader(), w.Header(), ForwardResponseHeaders)
		w.WriteHeader(result.Content.HttpStatusCode())
		io.Copy(w, result.Content.Reader())
		result.Content.Reader().Close()
		return true
	}
	return false
}

func LogFetchResultLoadingError(res *FetchResult, w http.ResponseWriter, r *http.Request) {
	// 404 and 502 Error already become logged in logger.go
	if res.Content.HttpStatusCode() != 404 && res.Content.HttpStatusCode() != 502 {
		logging.Application(r.Header).WithField("fetchResult", res).Errorf("error loading content from: %v", res.Def.URL)
	}
	res.Def.ErrHandler.Handle(res.Err, res.Content.HttpStatusCode(), w, r)
}

func MetadataForRequest(r *http.Request) map[string]interface{} {
	return map[string]interface{}{
		"host":     getHostFromRequest(r),
		"base_url": getBaseUrlFromRequest(r),
		"params":   r.URL.Query(),
	}
}

func getBaseUrlFromRequest(r *http.Request) string {
	proto := "http"
	if r.TLS != nil {
		proto = "https"
	}
	if xfph := r.Header.Get("X-Forwarded-Proto"); xfph != "" {
		protoParts := strings.SplitN(xfph, ",", 2)
		proto = protoParts[0]
	}

	return proto + "://" + getHostFromRequest(r)
}

func getHostFromRequest(r *http.Request) string {
	host := r.Host
	if xffh := r.Header.Get("X-Forwarded-For"); xffh != "" {
		hostParts := strings.SplitN(xffh, ",", 2)
		host = hostParts[0]
	}
	return host
}

func hasPrioritySetting(results []*FetchResult) bool {
	for _, res := range results {
		if res.Def.Priority > 0 {
			return true
		}
	}
	return false
}
<commit_msg>* Refactored methods * go format<commit_after>package composition

import (
	"github.com/tarent/lib-compose/logging"
	"io"
	"net/http"
	"strconv"
	"strings"
)

// A ContentFetcherFactory returns a configured fetch job for a request
// which can return the fetch results.
type ContentFetcherFactory func(r *http.Request) FetchResultSupplier

type CompositionHandler struct {
	contentFetcherFactory ContentFetcherFactory
	contentMergerFactory  func(metaJSON map[string]interface{}) ContentMerger
	cache                 Cache
}

// NewCompositionHandler creates a new Handler with the supplied defaultData,
// which is used for each request.
func NewCompositionHandler(contentFetcherFactory ContentFetcherFactory) *CompositionHandler {
	return &CompositionHandler{
		contentFetcherFactory: contentFetcherFactory,
		contentMergerFactory: func(metaJSON map[string]interface{}) ContentMerger {
			return NewContentMerge(metaJSON)
		},
		cache: nil,
	}
}

func NewCompositionHandlerWithCache(contentFetcherFactory ContentFetcherFactory, cache Cache) *CompositionHandler {
	return &CompositionHandler{
		contentFetcherFactory: contentFetcherFactory,
		contentMergerFactory: func(metaJSON map[string]interface{}) ContentMerger {
			return NewContentMerge(metaJSON)
		},
		cache: cache,
	}
}

func (agg *CompositionHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	fetcher := agg.contentFetcherFactory(r)

	if agg.handleEmptyFetcher(fetcher, w, r) {
		return
	}

	// fetch all contents
	results := fetcher.WaitForResults()

	// Allow HEAD requests and disable composition of body fragments
	if agg.handleHeadRequests(results, w, r) {
		return
	}

	mergeContext := agg.contentMergerFactory(fetcher.MetaJSON())

	for _, res := range results {
		if res.Err == nil && res.Content != nil {

			// Handle responses with 30x status code or with response bodies
			if agg.handleNonMergeableResponses(res, w, r) {
				return
			}

			mergeContext.AddContent(res)

		} else if res.Def.Required {
			LogFetchResultLoadingError(res, w, r)
			return
		} else {
			logging.Application(r.Header).WithField("fetchResult", res).Warnf("optional content not loaded: %v", res.Def.URL)
		}
	}

	status := agg.extractStatusCode(results, w, r)

	agg.copyHeadersIfNeeded(results, w, r)

	// Overwrite Content-Type to ensure, that the encoding is correct
	w.Header().Set("Content-Type", "text/html; charset=utf-8")

	html, err := agg.processHtml(mergeContext, w, r)
	// Return if an error occured within the html aggregation
	if err {
		return
	}

	w.Header().Set("Content-Length", strconv.Itoa(len(html)))
	w.WriteHeader(status)
	w.Write(html)
}

func (agg *CompositionHandler) handleNonMergeableResponses(result *FetchResult, w http.ResponseWriter, r *http.Request) bool {

	if agg.handle30xResponses(result, w, r) {
		// Return if it's a forwarded status code
		return true
	}

	if agg.handleStreamResponses(result, w, r) {
		// Return if it's a response with body
		return true
	}

	return false
}

func (agg *CompositionHandler) extractStatusCode(results []*FetchResult, w http.ResponseWriter, r *http.Request) (statusCode int) {
	if len(results) > 0 {
		if results[0].Content.HttpStatusCode() != 0 {
			return results[0].Content.HttpStatusCode()
		}
	}
	return 200
}

func (agg *CompositionHandler) copyHeadersIfNeeded(results []*FetchResult, w http.ResponseWriter, r *http.Request) {
	// Take status code and headers from first fetch definition
	if len(results) > 0 {
		copyHeaders(results[0].Content.HttpHeader(), w.Header(), ForwardResponseHeaders)
	}
}

func (agg *CompositionHandler) processHtml(mergeContext ContentMerger, w http.ResponseWriter, r *http.Request) ([]byte, bool) {
	html, err := mergeContext.GetHtml()
	if err != nil {
		if agg.cache != nil {
			agg.cache.PurgeEntries(mergeContext.GetHashes())
		}
		logging.Application(r.Header).Error(err.Error())
		http.Error(w, "Internal Server Error: "+err.Error(), 500)
		return nil, true
	}
	return html, false
}

func (agg *CompositionHandler) handleHeadRequests(results []*FetchResult, w http.ResponseWriter, r *http.Request) bool {
	if r.Method == "HEAD" && len(results) > 0 {
		copyHeaders(results[0].Content.HttpHeader(), w.Header(), ForwardResponseHeaders)
		w.WriteHeader(results[0].Content.HttpStatusCode())
		return true
	}
	return false
}

func (agg *CompositionHandler) handleEmptyFetcher(fetcher FetchResultSupplier, w http.ResponseWriter, r *http.Request) bool {
	if fetcher.Empty() {
		w.WriteHeader(500)
		w.Write([]byte("Internal server error"))
		logging.Application(r.Header).Error("No fetchers available for composition, throwing error 500")
		return true
	}
	return false
}

func (agg *CompositionHandler) handle30xResponses(result *FetchResult, w http.ResponseWriter, r *http.Request) bool {
	if result.Content.HttpStatusCode() >= 300 && result.Content.HttpStatusCode() <= 308 {
		copyHeaders(result.Content.HttpHeader(), w.Header(), ForwardResponseHeaders)
		w.WriteHeader(result.Content.HttpStatusCode())
		return true
	}
	return false
}

func (agg *CompositionHandler) handleStreamResponses(result *FetchResult, w http.ResponseWriter, r *http.Request) bool {
	if result.Content.Reader() != nil {
		copyHeaders(result.Content.HttpHeader(), w.Header(), ForwardResponseHeaders)
		w.WriteHeader(result.Content.HttpStatusCode())
		io.Copy(w, result.Content.Reader())
		result.Content.Reader().Close()
		return true
	}
	return false
}

func LogFetchResultLoadingError(res *FetchResult, w http.ResponseWriter, r *http.Request) {
	// 404 and 502 Error already become logged in logger.go
	if res.Content.HttpStatusCode() != 404 && res.Content.HttpStatusCode() != 502 {
		logging.Application(r.Header).WithField("fetchResult", res).Errorf("error loading content from: %v", res.Def.URL)
	}
	res.Def.ErrHandler.Handle(res.Err, res.Content.HttpStatusCode(), w, r)
}

func MetadataForRequest(r *http.Request) map[string]interface{} {
	return map[string]interface{}{
		"host":     getHostFromRequest(r),
		"base_url": getBaseUrlFromRequest(r),
		"params":   r.URL.Query(),
	}
}

func getBaseUrlFromRequest(r *http.Request) string {
	proto := "http"
	if r.TLS != nil {
		proto = "https"
	}
	if xfph := r.Header.Get("X-Forwarded-Proto"); xfph != "" {
		protoParts := strings.SplitN(xfph, ",", 2)
		proto = protoParts[0]
	}

	return proto + "://" + getHostFromRequest(r)
}

func getHostFromRequest(r *http.Request) string {
	host := r.Host
	if xffh := r.Header.Get("X-Forwarded-For"); xffh != "" {
		hostParts := strings.SplitN(xffh, ",", 2)
		host = hostParts[0]
	}
	return host
}

func hasPrioritySetting(results []*FetchResult) bool {
	for _, res := range results {
		if res.Def.Priority > 0 {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Package lfs brings together the core LFS functionality
// NOTE: Subject to change, do not rely on this package from outside git-lfs source
package lfs

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/github/git-lfs/config"
	"github.com/github/git-lfs/localstorage"
	"github.com/github/git-lfs/tools"
	"github.com/rubyist/tracerx"
)

const (
	Version = "1.2.1"
)

var (
	LargeSizeThreshold = 5 * 1024 * 1024
)

// LocalMediaDir returns the root of lfs objects
func LocalMediaDir() string {
	if localstorage.Objects() != nil {
		return localstorage.Objects().RootDir
	}
	return ""
}

func LocalObjectTempDir() string {
	if localstorage.Objects() != nil {
		return localstorage.Objects().TempDir
	}
	return ""
}

func TempDir() string {
	return localstorage.TempDir
}

func TempFile(prefix string) (*os.File, error) {
	return localstorage.TempFile(prefix)
}

func LocalMediaPath(oid string) (string, error) {
	return localstorage.Objects().BuildObjectPath(oid)
}

func LocalMediaPathReadOnly(oid string) string {
	return localstorage.Objects().ObjectPath(oid)
}

func LocalReferencePath(sha string) string {
	if config.LocalReferenceDir == "" {
		return ""
	}
	return filepath.Join(config.LocalReferenceDir, sha[0:2], sha[2:4], sha)
}

func ObjectExistsOfSize(oid string, size int64) bool {
	path := localstorage.Objects().ObjectPath(oid)
	return tools.FileExistsOfSize(path, size)
}

func Environ() []string {
	osEnviron := os.Environ()
	env := make([]string, 0, len(osEnviron)+7)
	env = append(env,
		fmt.Sprintf("LocalWorkingDir=%s", config.LocalWorkingDir),
		fmt.Sprintf("LocalGitDir=%s", config.LocalGitDir),
		fmt.Sprintf("LocalGitStorageDir=%s", config.LocalGitStorageDir),
		fmt.Sprintf("LocalMediaDir=%s", LocalMediaDir()),
		fmt.Sprintf("LocalReferenceDir=%s", config.LocalReferenceDir),
		fmt.Sprintf("TempDir=%s", TempDir()),
		fmt.Sprintf("ConcurrentTransfers=%d", config.Config.ConcurrentTransfers()),
		fmt.Sprintf("BasicTransfersOnly=%v", config.Config.BasicTransfersOnly()),
		fmt.Sprintf("BatchTransfer=%v", config.Config.BatchTransfer()),
		fmt.Sprintf("SkipDownloadErrors=%v", config.Config.SkipDownloadErrors()),
		fmt.Sprintf("FetchRecentAlways=%v", config.Config.FetchPruneConfig().FetchRecentAlways),
		fmt.Sprintf("FetchRecentRefsDays=%d", config.Config.FetchPruneConfig().FetchRecentRefsDays),
		fmt.Sprintf("FetchRecentCommitsDays=%d", config.Config.FetchPruneConfig().FetchRecentCommitsDays),
		fmt.Sprintf("FetchRecentRefsIncludeRemotes=%v", config.Config.FetchPruneConfig().FetchRecentRefsIncludeRemotes),
		fmt.Sprintf("PruneOffsetDays=%d", config.Config.FetchPruneConfig().PruneOffsetDays),
		fmt.Sprintf("PruneVerifyRemoteAlways=%v", config.Config.FetchPruneConfig().PruneVerifyRemoteAlways),
		fmt.Sprintf("PruneRemoteName=%s", config.Config.FetchPruneConfig().PruneRemoteName),
		fmt.Sprintf("AccessDownload=%s", config.Config.Access("download")),
		fmt.Sprintf("AccessUpload=%s", config.Config.Access("upload")),
	)
	if len(config.Config.FetchExcludePaths()) > 0 {
		env = append(env, fmt.Sprintf("FetchExclude=%s", strings.Join(config.Config.FetchExcludePaths(), ", ")))
	}
	if len(config.Config.FetchIncludePaths()) > 0 {
		env = append(env, fmt.Sprintf("FetchInclude=%s", strings.Join(config.Config.FetchIncludePaths(), ", ")))
	}
	for _, ext := range config.Config.Extensions() {
		env = append(env, fmt.Sprintf("Extension[%d]=%s", ext.Priority, ext.Name))
	}

	for _, e := range osEnviron {
		if !strings.Contains(e, "GIT_") {
			continue
		}
		env = append(env, e)
	}

	return env
}

func InRepo() bool {
	return config.LocalGitDir != ""
}

func ClearTempObjects() error {
	if localstorage.Objects() == nil {
		return nil
	}
	return localstorage.Objects().ClearTempObjects()
}

func ScanObjectsChan() <-chan localstorage.Object {
	return localstorage.Objects().ScanObjectsChan()
}

func init() {
	tracerx.DefaultKey = "GIT"
	tracerx.Prefix = "trace git-lfs: "

	localstorage.ResolveDirs()
}

const (
	gitExt       = ".git"
	gitPtrPrefix = "gitdir: "
)

// only used in tests
func AllObjects() []localstorage.Object {
	return localstorage.Objects().AllObjects()
}

func LinkOrCopyFromReference(oid string, size int64) error {
	if ObjectExistsOfSize(oid, size) {
		return nil
	}
	altMediafile := LocalReferencePath(oid)
	mediafile, err := LocalMediaPath(oid)
	if err != nil {
		return err
	}
	if altMediafile != "" && tools.FileExistsOfSize(altMediafile, size) {
		return LinkOrCopy(altMediafile, mediafile)
	}
	return nil
}
<commit_msg>spit out TusTransfers in 'git lfs env'<commit_after>// Package lfs brings together the core LFS functionality
// NOTE: Subject to change, do not rely on this package from outside git-lfs source
package lfs

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/github/git-lfs/config"
	"github.com/github/git-lfs/localstorage"
	"github.com/github/git-lfs/tools"
	"github.com/rubyist/tracerx"
)

const (
	Version = "1.2.1"
)

var (
	LargeSizeThreshold = 5 * 1024 * 1024
)

// LocalMediaDir returns the root of lfs objects
func LocalMediaDir() string {
	if localstorage.Objects() != nil {
		return localstorage.Objects().RootDir
	}
	return ""
}

func LocalObjectTempDir() string {
	if localstorage.Objects() != nil {
		return localstorage.Objects().TempDir
	}
	return ""
}

func TempDir() string {
	return localstorage.TempDir
}

func TempFile(prefix string) (*os.File, error) {
	return localstorage.TempFile(prefix)
}

func LocalMediaPath(oid string) (string, error) {
	return localstorage.Objects().BuildObjectPath(oid)
}

func LocalMediaPathReadOnly(oid string) string {
	return localstorage.Objects().ObjectPath(oid)
}

func LocalReferencePath(sha string) string {
	if config.LocalReferenceDir == "" {
		return ""
	}
	return filepath.Join(config.LocalReferenceDir, sha[0:2], sha[2:4], sha)
}

func ObjectExistsOfSize(oid string, size int64) bool {
	path := localstorage.Objects().ObjectPath(oid)
	return tools.FileExistsOfSize(path, size)
}

func Environ() []string {
	osEnviron := os.Environ()
	env := make([]string, 0, len(osEnviron)+7)
	env = append(env,
		fmt.Sprintf("LocalWorkingDir=%s", config.LocalWorkingDir),
		fmt.Sprintf("LocalGitDir=%s", config.LocalGitDir),
		fmt.Sprintf("LocalGitStorageDir=%s", config.LocalGitStorageDir),
		fmt.Sprintf("LocalMediaDir=%s", LocalMediaDir()),
		fmt.Sprintf("LocalReferenceDir=%s", config.LocalReferenceDir),
		fmt.Sprintf("TempDir=%s", TempDir()),
		fmt.Sprintf("ConcurrentTransfers=%d", config.Config.ConcurrentTransfers()),
		fmt.Sprintf("TusTransfers=%v", config.Config.TusTransfersAllowed()),
		fmt.Sprintf("BasicTransfersOnly=%v", config.Config.BasicTransfersOnly()),
		fmt.Sprintf("BatchTransfer=%v", config.Config.BatchTransfer()),
		fmt.Sprintf("SkipDownloadErrors=%v", config.Config.SkipDownloadErrors()),
		fmt.Sprintf("FetchRecentAlways=%v", config.Config.FetchPruneConfig().FetchRecentAlways),
		fmt.Sprintf("FetchRecentRefsDays=%d", config.Config.FetchPruneConfig().FetchRecentRefsDays),
		fmt.Sprintf("FetchRecentCommitsDays=%d", config.Config.FetchPruneConfig().FetchRecentCommitsDays),
		fmt.Sprintf("FetchRecentRefsIncludeRemotes=%v", config.Config.FetchPruneConfig().FetchRecentRefsIncludeRemotes),
		fmt.Sprintf("PruneOffsetDays=%d", config.Config.FetchPruneConfig().PruneOffsetDays),
		fmt.Sprintf("PruneVerifyRemoteAlways=%v", config.Config.FetchPruneConfig().PruneVerifyRemoteAlways),
		fmt.Sprintf("PruneRemoteName=%s", config.Config.FetchPruneConfig().PruneRemoteName),
		fmt.Sprintf("AccessDownload=%s", config.Config.Access("download")),
		fmt.Sprintf("AccessUpload=%s", config.Config.Access("upload")),
	)
	if len(config.Config.FetchExcludePaths()) > 0 {
		env = append(env, fmt.Sprintf("FetchExclude=%s", strings.Join(config.Config.FetchExcludePaths(), ", ")))
	}
	if len(config.Config.FetchIncludePaths()) > 0 {
		env = append(env, fmt.Sprintf("FetchInclude=%s", strings.Join(config.Config.FetchIncludePaths(), ", ")))
	}
	for _, ext := range config.Config.Extensions() {
		env = append(env, fmt.Sprintf("Extension[%d]=%s", ext.Priority, ext.Name))
	}

	for _, e := range osEnviron {
		if !strings.Contains(e, "GIT_") {
			continue
		}
		env = append(env, e)
	}

	return env
}

func InRepo() bool {
	return config.LocalGitDir != ""
}

func ClearTempObjects() error {
	if localstorage.Objects() == nil {
		return nil
	}
	return localstorage.Objects().ClearTempObjects()
}

func ScanObjectsChan() <-chan localstorage.Object {
	return localstorage.Objects().ScanObjectsChan()
}

func init() {
	tracerx.DefaultKey = "GIT"
	tracerx.Prefix = "trace git-lfs: "

	localstorage.ResolveDirs()
}

const (
	gitExt       = ".git"
	gitPtrPrefix = "gitdir: "
)

// only used in tests
func AllObjects() []localstorage.Object {
	return localstorage.Objects().AllObjects()
}

func LinkOrCopyFromReference(oid string, size int64) error {
	if ObjectExistsOfSize(oid, size) {
		return nil
	}
	altMediafile := LocalReferencePath(oid)
	mediafile, err := LocalMediaPath(oid)
	if err != nil {
		return err
	}
	if altMediafile != "" && tools.FileExistsOfSize(altMediafile, size) {
		return LinkOrCopy(altMediafile, mediafile)
	}
	return nil
}
<|endoftext|>