<commit_before>package sandbox

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"text/template"

	"github.com/freneticmonkey/migrate/go/config"
	"github.com/freneticmonkey/migrate/go/table"
	"github.com/freneticmonkey/migrate/go/util"
	"github.com/freneticmonkey/migrate/go/yaml"
	"github.com/spf13/afero"
)

func stringSlice(input string, from, to int) string {
	return input[from:to]
}

func removeHungarian(input string) string {
	return strings.TrimLeft(input, "abcdefghijklnmopqrstuvxyz")
}

func underscoreDelimit(input string) string {
	pattern, err := regexp.Compile("(.)([A-Z][a-z])")
	if err != nil {
		util.LogErrorf("Regex Compile Error: %v", err)
	}
	return pattern.ReplaceAllString(input, "${1}_$2")
}

func scanGeneratedFiles(path string) (files []string, err error) {

	visit := func(path string, f os.FileInfo, err error) error {
		files = append(files, path)
		return nil
	}

	err = filepath.Walk(path, visit)

	return files, err
}

// Generate Serialise the YAML Table(s) using a configuration defined template
func Generate(conf config.Config, tableName string) (err error) {

	err = yaml.ReadTables(conf)
	if util.ErrorCheck(err) {
		return fmt.Errorf("Generate failed. Unable to read YAML Tables")
	}

	// Filter by tableName in the YAML Schema
	if tableName != "*" {
		tgtTbl := []table.Table{}

		for _, tbl := range yaml.Schema {
			if tbl.Name == tableName {
				tgtTbl = append(tgtTbl, tbl)
				break
			}
		}
		// Reduce the YAML schema to the single target table
		yaml.Schema = tgtTbl
	}

	if len(yaml.Schema) == 0 {
		return fmt.Errorf("Generate: YAML Table not found: %s", tableName)
	}

	return writeTables(conf, yaml.Schema)
}

// GenerateTable Serialise a table using the configured template
func GenerateTable(conf config.Config, tbl table.Table) (err error) {

	return writeTables(conf, []table.Table{tbl})
}

func writeTables(conf config.Config, tables []table.Table) (err error) {
	var data []byte
	var tmpl *template.Template
	var exists bool
	var files []string

	util.SetVerbose(true)

	// Define the custom template functions
	funcMap := template.FuncMap{
		"content":           contentSlice,
		"removeHungarian":   removeHungarian,
		"toUpper":           strings.ToUpper,
		"trimSuffix":        strings.TrimSuffix,
		"replace":           strings.Replace,
		"title":             strings.Title,
		"contains":          strings.Contains,
		"slice":             stringSlice,
		"underscoreDelimit": underscoreDelimit,
	}

	for _, tmpOptions := range conf.Options.Generation.Templates {
		util.LogInfof("Generating Schema for Template: %s", tmpOptions.Name)

		if tmpOptions.Path == "" || tmpOptions.File == "" || tmpOptions.FileFormat == "" {
			return fmt.Errorf("Generate Table: Badly configured template options")
		}

		templateFile := util.WorkingSubDir(tmpOptions.File)

		// Generation path
		genPath := util.WorkingSubDir(tmpOptions.Path)

		// Ensure that the path exists
		exists, err = util.DirExists(genPath)
		if err != nil {
			return err
		}

		if !exists {
			err = util.Mkdir(genPath, 0755)
			if err != nil {
				return err
			}
		}

		data, err = util.ReadFile(templateFile)
		if err != nil {
			return err
		}
		tblTmpl := string(data)

		// Configure the table template
		tmpl, err = template.New(tmpOptions.Name).Funcs(funcMap).Parse(tblTmpl)
		if err != nil {
			return err
		}

		files, err = scanGeneratedFiles(genPath)

		// Generate each table
		for _, tbl := range tables {
			var f afero.File
			// Build a filename and path for the table file

			// Set the file format for the output file
			tbl.Namespace.SetTableFilename(tmpOptions.FileFormat)
			// Search for an existing file to adopt its naming case
			tbl.Namespace.SetExistingFilename(files)
			// Generate the final filename
			tblFilename := tbl.Namespace.GenerateFilename(tmpOptions.Ext)
			// Relative to the working directory
			tblFilename = filepath.Join(genPath, tblFilename)

			// Create Directory if not exists
			dir := filepath.Dir(tblFilename)
			exists, err = util.DirExists(dir)
			if err != nil {
				return err
			}
			if !exists {
				util.Mkdir(dir, 0755)
			}

			// If the file exists, read it and build any content sections
			exists, err = util.FileExists(tblFilename)
			if exists {
				data, err = util.ReadFile(tblFilename)
				if err != nil {
					return fmt.Errorf("Generate Table: Problem reading target file for replacement. Error: %v", err)
				}
				// Read the current file for content sections
				err = parseContent(string(data))
			}

			// Create a file
			f, err = util.Create(tblFilename)
			if err != nil {
				return err
			}

			// Generate the contents from the template
			err = tmpl.Execute(f, tbl)
			if err != nil {
				f.Close()
				return err
			}
			f.Close()
		}

	}

	return err

}
<commit_msg>Switched Generation Configuration to reside inside Project configuration<commit_after>package sandbox

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"text/template"

	"github.com/freneticmonkey/migrate/go/config"
	"github.com/freneticmonkey/migrate/go/table"
	"github.com/freneticmonkey/migrate/go/util"
	"github.com/freneticmonkey/migrate/go/yaml"
	"github.com/spf13/afero"
)

func stringSlice(input string, from, to int) string {
	return input[from:to]
}

func removeHungarian(input string) string {
	return strings.TrimLeft(input, "abcdefghijklnmopqrstuvxyz")
}

func underscoreDelimit(input string) string {
	pattern, err := regexp.Compile("(.)([A-Z][a-z])")
	if err != nil {
		util.LogErrorf("Regex Compile Error: %v", err)
	}
	return pattern.ReplaceAllString(input, "${1}_$2")
}

func scanGeneratedFiles(path string) (files []string, err error) {

	visit := func(path string, f os.FileInfo, err error) error {
		files = append(files, path)
		return nil
	}

	err = filepath.Walk(path, visit)

	return files, err
}

// Generate Serialise the YAML Table(s) using a configuration defined template
func Generate(conf config.Config, tableName string) (err error) {

	err = yaml.ReadTables(conf)
	if util.ErrorCheck(err) {
		return fmt.Errorf("Generate failed. Unable to read YAML Tables")
	}

	// Filter by tableName in the YAML Schema
	if tableName != "*" {
		tgtTbl := []table.Table{}

		for _, tbl := range yaml.Schema {
			if tbl.Name == tableName {
				tgtTbl = append(tgtTbl, tbl)
				break
			}
		}
		// Reduce the YAML schema to the single target table
		yaml.Schema = tgtTbl
	}

	if len(yaml.Schema) == 0 {
		return fmt.Errorf("Generate: YAML Table not found: %s", tableName)
	}

	return writeTables(conf, yaml.Schema)
}

// GenerateTable Serialise a table using the configured template
func GenerateTable(conf config.Config, tbl table.Table) (err error) {

	return writeTables(conf, []table.Table{tbl})
}

func writeTables(conf config.Config, tables []table.Table) (err error) {
	var data []byte
	var tmpl *template.Template
	var exists bool
	var files []string

	util.SetVerbose(true)

	// Define the custom template functions
	funcMap := template.FuncMap{
		"content":           contentSlice,
		"removeHungarian":   removeHungarian,
		"toUpper":           strings.ToUpper,
		"trimSuffix":        strings.TrimSuffix,
		"replace":           strings.Replace,
		"title":             strings.Title,
		"contains":          strings.Contains,
		"slice":             stringSlice,
		"underscoreDelimit": underscoreDelimit,
	}

	for _, tmpOptions := range conf.Project.Generation.Templates {
		util.LogInfof("Generating Schema for Template: %s", tmpOptions.Name)

		if tmpOptions.Path == "" || tmpOptions.File == "" || tmpOptions.FileFormat == "" {
			return fmt.Errorf("Generate Table: Badly configured template options")
		}

		templateFile := util.WorkingSubDir(tmpOptions.File)

		// Generation path
		genPath := util.WorkingSubDir(tmpOptions.Path)

		// Ensure that the path exists
		exists, err = util.DirExists(genPath)
		if err != nil {
			return err
		}

		if !exists {
			err = util.Mkdir(genPath, 0755)
			if err != nil {
				return err
			}
		}

		data, err = util.ReadFile(templateFile)
		if err != nil {
			return err
		}
		tblTmpl := string(data)

		// Configure the table template
		tmpl, err = template.New(tmpOptions.Name).Funcs(funcMap).Parse(tblTmpl)
		if err != nil {
			return err
		}

		files, err = scanGeneratedFiles(genPath)

		// Generate each table
		for _, tbl := range tables {
			var f afero.File
			// Build a filename and path for the table file

			// Set the file format for the output file
			tbl.Namespace.SetTableFilename(tmpOptions.FileFormat)
			// Search for an existing file to adopt its naming case
			tbl.Namespace.SetExistingFilename(files)
			// Generate the final filename
			tblFilename := tbl.Namespace.GenerateFilename(tmpOptions.Ext)
			// Relative to the working directory
			tblFilename = filepath.Join(genPath, tblFilename)

			// Create Directory if not exists
			dir := filepath.Dir(tblFilename)
			exists, err = util.DirExists(dir)
			if err != nil {
				return err
			}
			if !exists {
				util.Mkdir(dir, 0755)
			}

			// If the file exists, read it and build any content sections
			exists, err = util.FileExists(tblFilename)
			if exists {
				data, err = util.ReadFile(tblFilename)
				if err != nil {
					return fmt.Errorf("Generate Table: Problem reading target file for replacement. Error: %v", err)
				}
				// Read the current file for content sections
				err = parseContent(string(data))
			}

			// Create a file
			f, err = util.Create(tblFilename)
			if err != nil {
				return err
			}

			// Generate the contents from the template
			err = tmpl.Execute(f, tbl)
			if err != nil {
				f.Close()
				return err
			}
			f.Close()
		}

	}

	return err

}
<|endoftext|>


<commit_before>package gofetcher

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"reflect"
	"strconv"
	"time"

	"github.com/cocaine/cocaine-framework-go/cocaine"
	"github.com/ugorji/go/codec"
)

const (
	DefaultTimeout         = 5000
	DefaultFollowRedirects = true
	KeepAliveTimeout       = 30
)

// took from httputil/reverseproxy.go
// Hop-by-hop headers. These are removed when sent to the backend.
// http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html
var hopHeaders = []string{
	"Connection",
	"Keep-Alive",
	"Proxy-Authenticate",
	"Proxy-Authorization",
	"Te", // canonicalized version of "TE"
	"Trailers",
	"Transfer-Encoding",
	"Upgrade",
}

type WarnError struct {
	err error
}

func (s *WarnError) Error() string { return s.err.Error() }

func NewWarn(err error) *WarnError {
	return &WarnError{err: err}
}

type Gofetcher struct {
	Logger    *cocaine.Logger
	Transport http.RoundTripper

	UserAgent string
}

type Cookies map[string]string

type Request struct {
	Method          string
	URL             string
	Body            io.Reader
	Timeout         int64
	Cookies         Cookies
	Headers         http.Header
	FollowRedirects bool
}

type responseAndError struct {
	res *http.Response
	err error
}

type Response struct {
	httpResponse *http.Response
	body         []byte
	header       http.Header
	runtime      time.Duration
}

func NewGofetcher() *Gofetcher {
	logger, err := cocaine.NewLogger()
	if err != nil {
		fmt.Printf("Could not initialize logger due to error: %v", err)
		return nil
	}
	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: KeepAliveTimeout * time.Second,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
	}
	gofetcher := Gofetcher{logger, transport, ""}
	return &gofetcher
}

func (gofetcher *Gofetcher) SetUserAgent(userAgent string) {
	gofetcher.UserAgent = userAgent
}

func noRedirect(_ *http.Request, via []*http.Request) error {
	if len(via) > 0 {
		return errors.New("stopped after first redirect")
	}
	return nil
}

func (gofetcher *Gofetcher) PrepareRequest(request *Request) (*http.Request, *http.Client, error) {
	var (
		err            error
		httpRequest    *http.Request
		requestTimeout time.Duration = time.Duration(request.Timeout) * time.Millisecond
	)

	httpClient := &http.Client{
		Transport: gofetcher.Transport,
		Timeout:   requestTimeout,
	}
	if request.FollowRedirects == false {
		httpClient.CheckRedirect = noRedirect
	}

	httpRequest, err = http.NewRequest(request.Method, request.URL, request.Body)
	if err != nil {
		return nil, nil, err
	}
	for name, value := range request.Cookies {
		httpRequest.AddCookie(&http.Cookie{Name: name, Value: value})
	}
	httpRequest.Header = request.Headers

	// Remove hop-by-hop headers to the backend.  Especially
	// important is "Connection" because we want a persistent
	// connection, regardless of what the client sent to us.  This
	// is modifying the same underlying map from req (shallow
	// copied above) so we only copy it if necessary.
	for _, h := range hopHeaders {
		httpRequest.Header.Del(h)
	}
	httpRequest.Header.Add("Connection", "keep-alive")
	httpRequest.Header.Add("Keep-Alive", fmt.Sprintf("%d", KeepAliveTimeout))

	if gofetcher.UserAgent != "" && len(httpRequest.Header["User-Agent"]) == 0 {
		httpRequest.Header.Set("User-Agent", gofetcher.UserAgent)
	}

	return httpRequest, httpClient, nil
}

func (gofetcher *Gofetcher) ExecuteRequest(req *http.Request, client *http.Client, attempt int) (*http.Response, error) {
	var (
		httpResponse *http.Response
		err          error
	)

	gofetcher.Logger.Infof("Requested url: %s, Method: %s, timeout: %d, headers: %v, attempt: %d",
		req.URL.String(), req.Method, client.Timeout, req.Header, attempt)

	resultChan := make(chan responseAndError)
	started := time.Now()
	go func() {
		res, err := client.Do(req)
		resultChan <- responseAndError{res, err}
	}()
	// http connection stay active after timeout exceeded in go <1.3, cause we can't close it using current client api.
	// Read more about timeouts: https://code.google.com/p/go/issues/detail?id=3362
	//
	select {
	case result := <-resultChan:
		httpResponse, err = result.res, result.err
	case <-time.After(client.Timeout):
		err = errors.New(fmt.Sprintf("Request timeout[%s] exceeded", client.Timeout.String()))
		go func() {
			// close httpResponse when it ready
			result := <-resultChan
			if result.res != nil {
				result.res.Body.Close()
			}
		}()
	}
	if err != nil {
		// special case for redirect failure (returns both response and error)
		// read more https://code.google.com/p/go/issues/detail?id=3795
		if httpResponse == nil {
			if urlError, ok := err.(*url.Error); ok {
				// golang bug: golang.org/issue/3514
				if urlError.Err == io.EOF {
					gofetcher.Logger.Infof("Got EOF error while loading %s, attempt(%d)", req.URL.String(), attempt)
					if attempt == 1 {
						return gofetcher.ExecuteRequest(req, client, attempt+1)
					}
				}
			}
			return nil, NewWarn(err)
		}
	}

	for _, h := range hopHeaders {
		httpResponse.Header.Del(h)
	}

	runtime := time.Since(started)
	gofetcher.Logger.Info(fmt.Sprintf("Response code: %d, url: %s, runtime: %v",
		httpResponse.StatusCode, req.URL.String(), runtime))
	return httpResponse, nil

}

// Normal methods

func parseHeaders(rawHeaders map[string]interface{}) http.Header {
	headers := make(http.Header)
	for name, values := range rawHeaders {
		for _, value := range values.([]interface{}) {
			headers.Add(name, string(value.([]uint8))) // to transform in canonical form
		}
	}
	return headers
}

func parseCookies(rawCookie map[string]interface{}) Cookies {
	cookies := Cookies{}
	for key, value := range rawCookie {
		cookies[key] = string(value.([]uint8))
	}
	return cookies
}

func parseTimeout(rawTimeout interface{}) (timeout int64) {
	// is it possible to got timeout in int64 instead of uint64?
	switch rawTimeout.(type) {
	case uint64:
		timeout = int64(rawTimeout.(uint64))
	case int64:
		timeout = rawTimeout.(int64)
	}
	return timeout
}

func (gofetcher *Gofetcher) ParseRequest(method string, requestBody []byte) (request *Request) {
	var (
		mh              codec.MsgpackHandle
		h                     = &mh
		timeout         int64 = DefaultTimeout
		cookies         Cookies
		headers              = make(http.Header)
		followRedirects bool = DefaultFollowRedirects
		body            *bytes.Buffer
	)
	mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
	var res []interface{}
	codec.NewDecoderBytes(requestBody, h).Decode(&res)
	url := string(res[0].([]uint8))
	switch {
	case method == "GET" || method == "HEAD" || method == "DELETE":
		if len(res) > 1 {
			timeout = parseTimeout(res[1])
		}
		if len(res) > 2 {
			cookies = parseCookies(res[2].(map[string]interface{}))
		}
		if len(res) > 3 {
			headers = parseHeaders(res[3].(map[string]interface{}))
		}
		if len(res) > 4 {
			followRedirects = res[4].(bool)
		}
	case method == "POST" || method == "PUT" || method == "PATCH":
		if len(res) > 1 {
			body = bytes.NewBuffer(res[1].([]byte))
		}
		if len(res) > 2 {
			timeout = parseTimeout(res[2])
		}
		if len(res) > 3 {
			cookies = parseCookies(res[3].(map[string]interface{}))
		}
		if len(res) > 4 {
			headers = parseHeaders(res[4].(map[string]interface{}))
		}
		if len(res) > 5 {
			followRedirects = res[5].(bool)
		}
	}

	request = &Request{Method: method, URL: url, Timeout: timeout,
		FollowRedirects: followRedirects,
		Cookies:         cookies, Headers: headers}
	if body != nil {
		request.Body = body
	}
	return request
}

func (gofetcher *Gofetcher) WriteError(response *cocaine.Response, request *Request, err error) {
	if _, casted := err.(*WarnError); casted {
		gofetcher.Logger.Warnf("Error occured: %v, while downloading %s",
			err.Error(), request.URL)
	} else {
		gofetcher.Logger.Errf("Error occured: %v, while downloading %s",
			err.Error(), request.URL)
	}
	response.Write([]interface{}{false, err.Error(), 0, http.Header{}})
}

func (gofetcher *Gofetcher) WriteResponse(response *cocaine.Response, request *Request, resp *http.Response, body []byte) {
	response.Write([]interface{}{true, body, resp.StatusCode, resp.Header})
}

func (gofetcher *Gofetcher) handler(Method string, request *cocaine.Request, response *cocaine.Response) {
	defer response.Close()

	requestBody := <-request.Read()
	httpRequest := gofetcher.ParseRequest(Method, requestBody)

	req, client, err := gofetcher.PrepareRequest(httpRequest)
	if client != nil {
		gofetcher.WriteError(response, httpRequest, err)
		return
	}

	resp, err := gofetcher.ExecuteRequest(req, client, 1)
	if err != nil {
		gofetcher.WriteError(response, httpRequest, err)
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		gofetcher.WriteError(response, httpRequest, err)
		return
	}

	gofetcher.WriteResponse(response, httpRequest, resp, body)
}

func (gofetcher *Gofetcher) GetHandler(Method string) func(request *cocaine.Request, response *cocaine.Response) {
	return func(request *cocaine.Request, response *cocaine.Response) {
		gofetcher.handler(Method, request, response)
	}
}

// Http methods

func (gofetcher *Gofetcher) HttpProxy(res http.ResponseWriter, req *http.Request) {
	var (
		timeout int64 = DefaultTimeout
		resp    *http.Response
	)
	url := req.FormValue("url")
	timeoutArg := req.FormValue("timeout")
	if timeoutArg != "" {
		tout, _ := strconv.Atoi(timeoutArg)
		timeout = int64(tout)
	}
	httpRequest := &Request{Method: req.Method, URL: url, Timeout: timeout,
		FollowRedirects: DefaultFollowRedirects, Headers: req.Header, Body: req.Body}
	prepReq, prepClient, err := gofetcher.PrepareRequest(httpRequest)
	if err == nil {
		resp, err = gofetcher.ExecuteRequest(prepReq, prepClient, 1)
	}

	if err != nil {
		res.Header().Set("Content-Type", "text/html")
		res.WriteHeader(500)
		res.Write([]byte(err.Error()))
		if _, casted := err.(*WarnError); casted {
			gofetcher.Logger.Warnf("Gofetcher error: %v", err)
		} else {
			gofetcher.Logger.Errf("Gofetcher error: %v", err)
		}

	} else {
		for key, values := range resp.Header {
			for _, value := range values {
				res.Header().Add(key, value)
			}
		}
		res.WriteHeader(200)
		if _, err := io.Copy(res, resp.Body); err != nil {
			gofetcher.Logger.Errf("Error: %v", err)
		}
	}
}

func (gofetcher *Gofetcher) HttpEcho(res http.ResponseWriter, req *http.Request) {
	gofetcher.Logger.Info("Http echo handler requested")
	text := req.FormValue("text")
	res.Header().Set("Content-Type", "text/html")
	res.WriteHeader(200)
	res.Write([]byte(text))
}
<commit_msg>fixed capitalization of "method" argument<commit_after>package gofetcher

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"reflect"
	"strconv"
	"time"

	"github.com/cocaine/cocaine-framework-go/cocaine"
	"github.com/ugorji/go/codec"
)

const (
	DefaultTimeout         = 5000
	DefaultFollowRedirects = true
	KeepAliveTimeout       = 30
)

// took from httputil/reverseproxy.go
// Hop-by-hop headers. These are removed when sent to the backend.
// http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html
var hopHeaders = []string{
	"Connection",
	"Keep-Alive",
	"Proxy-Authenticate",
	"Proxy-Authorization",
	"Te", // canonicalized version of "TE"
	"Trailers",
	"Transfer-Encoding",
	"Upgrade",
}

type WarnError struct {
	err error
}

func (s *WarnError) Error() string { return s.err.Error() }

func NewWarn(err error) *WarnError {
	return &WarnError{err: err}
}

type Gofetcher struct {
	Logger    *cocaine.Logger
	Transport http.RoundTripper

	UserAgent string
}

type Cookies map[string]string

type Request struct {
	Method          string
	URL             string
	Body            io.Reader
	Timeout         int64
	Cookies         Cookies
	Headers         http.Header
	FollowRedirects bool
}

type responseAndError struct {
	res *http.Response
	err error
}

type Response struct {
	httpResponse *http.Response
	body         []byte
	header       http.Header
	runtime      time.Duration
}

func NewGofetcher() *Gofetcher {
	logger, err := cocaine.NewLogger()
	if err != nil {
		fmt.Printf("Could not initialize logger due to error: %v", err)
		return nil
	}
	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: KeepAliveTimeout * time.Second,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
	}
	gofetcher := Gofetcher{logger, transport, ""}
	return &gofetcher
}

func (gofetcher *Gofetcher) SetUserAgent(userAgent string) {
	gofetcher.UserAgent = userAgent
}

func noRedirect(_ *http.Request, via []*http.Request) error {
	if len(via) > 0 {
		return errors.New("stopped after first redirect")
	}
	return nil
}

func (gofetcher *Gofetcher) PrepareRequest(request *Request) (*http.Request, *http.Client, error) {
	var (
		err            error
		httpRequest    *http.Request
		requestTimeout time.Duration = time.Duration(request.Timeout) * time.Millisecond
	)

	httpClient := &http.Client{
		Transport: gofetcher.Transport,
		Timeout:   requestTimeout,
	}
	if request.FollowRedirects == false {
		httpClient.CheckRedirect = noRedirect
	}

	httpRequest, err = http.NewRequest(request.Method, request.URL, request.Body)
	if err != nil {
		return nil, nil, err
	}
	for name, value := range request.Cookies {
		httpRequest.AddCookie(&http.Cookie{Name: name, Value: value})
	}
	httpRequest.Header = request.Headers

	// Remove hop-by-hop headers to the backend.  Especially
	// important is "Connection" because we want a persistent
	// connection, regardless of what the client sent to us.  This
	// is modifying the same underlying map from req (shallow
	// copied above) so we only copy it if necessary.
	for _, h := range hopHeaders {
		httpRequest.Header.Del(h)
	}
	httpRequest.Header.Add("Connection", "keep-alive")
	httpRequest.Header.Add("Keep-Alive", fmt.Sprintf("%d", KeepAliveTimeout))

	if gofetcher.UserAgent != "" && len(httpRequest.Header["User-Agent"]) == 0 {
		httpRequest.Header.Set("User-Agent", gofetcher.UserAgent)
	}

	return httpRequest, httpClient, nil
}

func (gofetcher *Gofetcher) ExecuteRequest(req *http.Request, client *http.Client, attempt int) (*http.Response, error) {
	var (
		httpResponse *http.Response
		err          error
	)

	gofetcher.Logger.Infof("Requested url: %s, method: %s, timeout: %d, headers: %v, attempt: %d",
		req.URL.String(), req.Method, client.Timeout, req.Header, attempt)

	resultChan := make(chan responseAndError)
	started := time.Now()
	go func() {
		res, err := client.Do(req)
		resultChan <- responseAndError{res, err}
	}()
	// http connection stay active after timeout exceeded in go <1.3, cause we can't close it using current client api.
	// Read more about timeouts: https://code.google.com/p/go/issues/detail?id=3362
	//
	select {
	case result := <-resultChan:
		httpResponse, err = result.res, result.err
	case <-time.After(client.Timeout):
		err = errors.New(fmt.Sprintf("Request timeout[%s] exceeded", client.Timeout.String()))
		go func() {
			// close httpResponse when it ready
			result := <-resultChan
			if result.res != nil {
				result.res.Body.Close()
			}
		}()
	}
	if err != nil {
		// special case for redirect failure (returns both response and error)
		// read more https://code.google.com/p/go/issues/detail?id=3795
		if httpResponse == nil {
			if urlError, ok := err.(*url.Error); ok {
				// golang bug: golang.org/issue/3514
				if urlError.Err == io.EOF {
					gofetcher.Logger.Infof("Got EOF error while loading %s, attempt(%d)", req.URL.String(), attempt)
					if attempt == 1 {
						return gofetcher.ExecuteRequest(req, client, attempt+1)
					}
				}
			}
			return nil, NewWarn(err)
		}
	}

	for _, h := range hopHeaders {
		httpResponse.Header.Del(h)
	}

	runtime := time.Since(started)
	gofetcher.Logger.Info(fmt.Sprintf("Response code: %d, url: %s, runtime: %v",
		httpResponse.StatusCode, req.URL.String(), runtime))
	return httpResponse, nil

}

// Normal methods

func parseHeaders(rawHeaders map[string]interface{}) http.Header {
	headers := make(http.Header)
	for name, values := range rawHeaders {
		for _, value := range values.([]interface{}) {
			headers.Add(name, string(value.([]uint8))) // to transform in canonical form
		}
	}
	return headers
}

func parseCookies(rawCookie map[string]interface{}) Cookies {
	cookies := Cookies{}
	for key, value := range rawCookie {
		cookies[key] = string(value.([]uint8))
	}
	return cookies
}

func parseTimeout(rawTimeout interface{}) (timeout int64) {
	// is it possible to got timeout in int64 instead of uint64?
	switch rawTimeout.(type) {
	case uint64:
		timeout = int64(rawTimeout.(uint64))
	case int64:
		timeout = rawTimeout.(int64)
	}
	return timeout
}

func (gofetcher *Gofetcher) ParseRequest(method string, requestBody []byte) (request *Request) {
	var (
		mh              codec.MsgpackHandle
		h                     = &mh
		timeout         int64 = DefaultTimeout
		cookies         Cookies
		headers              = make(http.Header)
		followRedirects bool = DefaultFollowRedirects
		body            *bytes.Buffer
	)
	mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
	var res []interface{}
	codec.NewDecoderBytes(requestBody, h).Decode(&res)
	url := string(res[0].([]uint8))
	switch {
	case method == "GET" || method == "HEAD" || method == "DELETE":
		if len(res) > 1 {
			timeout = parseTimeout(res[1])
		}
		if len(res) > 2 {
			cookies = parseCookies(res[2].(map[string]interface{}))
		}
		if len(res) > 3 {
			headers = parseHeaders(res[3].(map[string]interface{}))
		}
		if len(res) > 4 {
			followRedirects = res[4].(bool)
		}
	case method == "POST" || method == "PUT" || method == "PATCH":
		if len(res) > 1 {
			body = bytes.NewBuffer(res[1].([]byte))
		}
		if len(res) > 2 {
			timeout = parseTimeout(res[2])
		}
		if len(res) > 3 {
			cookies = parseCookies(res[3].(map[string]interface{}))
		}
		if len(res) > 4 {
			headers = parseHeaders(res[4].(map[string]interface{}))
		}
		if len(res) > 5 {
			followRedirects = res[5].(bool)
		}
	}

	request = &Request{Method: method, URL: url, Timeout: timeout,
		FollowRedirects: followRedirects,
		Cookies:         cookies, Headers: headers}
	if body != nil {
		request.Body = body
	}
	return request
}

func (gofetcher *Gofetcher) WriteError(response *cocaine.Response, request *Request, err error) {
	if _, casted := err.(*WarnError); casted {
		gofetcher.Logger.Warnf("Error occured: %v, while downloading %s",
			err.Error(), request.URL)
	} else {
		gofetcher.Logger.Errf("Error occured: %v, while downloading %s",
			err.Error(), request.URL)
	}
	response.Write([]interface{}{false, err.Error(), 0, http.Header{}})
}

func (gofetcher *Gofetcher) WriteResponse(response *cocaine.Response, request *Request, resp *http.Response, body []byte) {
	response.Write([]interface{}{true, body, resp.StatusCode, resp.Header})
}

func (gofetcher *Gofetcher) handler(method string, request *cocaine.Request, response *cocaine.Response) {
	defer response.Close()

	requestBody := <-request.Read()
	httpRequest := gofetcher.ParseRequest(method, requestBody)

	req, client, err := gofetcher.PrepareRequest(httpRequest)
	if client != nil {
		gofetcher.WriteError(response, httpRequest, err)
		return
	}

	resp, err := gofetcher.ExecuteRequest(req, client, 1)
	if err != nil {
		gofetcher.WriteError(response, httpRequest, err)
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		gofetcher.WriteError(response, httpRequest, err)
		return
	}

	gofetcher.WriteResponse(response, httpRequest, resp, body)
}

func (gofetcher *Gofetcher) GetHandler(method string) func(request *cocaine.Request, response *cocaine.Response) {
	return func(request *cocaine.Request, response *cocaine.Response) {
		gofetcher.handler(method, request, response)
	}
}

// Http methods

func (gofetcher *Gofetcher) HttpProxy(res http.ResponseWriter, req *http.Request) {
	var (
		timeout int64 = DefaultTimeout
		resp    *http.Response
	)
	url := req.FormValue("url")
	timeoutArg := req.FormValue("timeout")
	if timeoutArg != "" {
		tout, _ := strconv.Atoi(timeoutArg)
		timeout = int64(tout)
	}
	httpRequest := &Request{Method: req.Method, URL: url, Timeout: timeout,
		FollowRedirects: DefaultFollowRedirects, Headers: req.Header, Body: req.Body}
	prepReq, prepClient, err := gofetcher.PrepareRequest(httpRequest)
	if err == nil {
		resp, err = gofetcher.ExecuteRequest(prepReq, prepClient, 1)
	}

	if err != nil {
		res.Header().Set("Content-Type", "text/html")
		res.WriteHeader(500)
		res.Write([]byte(err.Error()))
		if _, casted := err.(*WarnError); casted {
			gofetcher.Logger.Warnf("Gofetcher error: %v", err)
		} else {
			gofetcher.Logger.Errf("Gofetcher error: %v", err)
		}

	} else {
		for key, values := range resp.Header {
			for _, value := range values {
				res.Header().Add(key, value)
			}
		}
		res.WriteHeader(200)
		if _, err := io.Copy(res, resp.Body); err != nil {
			gofetcher.Logger.Errf("Error: %v", err)
		}
	}
}

func (gofetcher *Gofetcher) HttpEcho(res http.ResponseWriter, req *http.Request) {
	gofetcher.Logger.Info("Http echo handler requested")
	text := req.FormValue("text")
	res.Header().Set("Content-Type", "text/html")
	res.WriteHeader(200)
	res.Write([]byte(text))
}
<|endoftext|>


<commit_before>package stick

import (
	"fmt"
	"reflect"
	"sync"
)

// Field is dynamically accessible field.
type Field struct {
	Index int
	Type  reflect.Type
}

// Accessor provides dynamic access to a structs fields.
type Accessor struct {
	Name   string
	Fields map[string]*Field
}

// Accessible is a type that has dynamically accessible fields.
type Accessible interface {
	GetAccessor(interface{}) *Accessor
}

// GetAccessor is a short-hand to retrieve the accessor of an accessible.
func GetAccessor(acc Accessible) *Accessor {
	return acc.GetAccessor(acc)
}

var accessMutex sync.Mutex
var accessCache = map[reflect.Type]*Accessor{}

// BasicAccess may be embedded in a struct to provide basic accessibility.
type BasicAccess struct{}

// GetAccessor implements the Accessible interface.
func (a *BasicAccess) GetAccessor(v interface{}) *Accessor {
	// get type
	typ := reflect.TypeOf(v).Elem()

	// acquire mutex
	accessMutex.Lock()
	defer accessMutex.Unlock()

	// check if accessor has already been cached
	accessor, ok := accessCache[typ]
	if ok {
		return accessor
	}

	// build accessor
	accessor = BuildAccessor(v.(Accessible), "BasicAccess")

	// cache accessor
	accessCache[typ] = accessor

	return accessor
}

// BuildAccessor will build an accessor for the provided type.
func BuildAccessor(v Accessible, ignore ...string) *Accessor {
	// get type
	typ := reflect.TypeOf(v).Elem()

	// prepare accessor
	accessor := &Accessor{
		Name:   typ.String(),
		Fields: map[string]*Field{},
	}

	// iterate through all fields
	for i := 0; i < typ.NumField(); i++ {
		// get field
		field := typ.Field(i)

		// check field
		var skip bool
		for _, item := range ignore {
			if item == field.Name {
				skip = true
			}
		}
		if skip {
			continue
		}

		// add field
		accessor.Fields[field.Name] = &Field{
			Index: i,
			Type:  field.Type,
		}
	}

	return accessor
}

// Get will look up the specified field on the accessible and return its value
// and whether the field was found at all.
func Get(acc Accessible, name string) (interface{}, bool) {
	// find field
	field := GetAccessor(acc).Fields[name]
	if field == nil {
		return nil, false
	}

	// get value
	value := reflect.ValueOf(acc).Elem().Field(field.Index).Interface()

	return value, true
}

// GetRaw will look up the specified field on the accessible and return its raw
// value and whether the field was found at all.
func GetRaw(acc Accessible, name string) (reflect.Value, bool) {
	// find field
	field := GetAccessor(acc).Fields[name]
	if field == nil {
		return reflect.Value{}, false
	}

	// get value
	value := reflect.ValueOf(acc).Elem().Field(field.Index)

	return value, true
}

// Set will set the specified field on the accessible with the provided value
// and return whether the field has been found and the value has been set.
func Set(acc Accessible, name string, value interface{}) bool {
	// find field
	field := GetAccessor(acc).Fields[name]
	if field == nil {
		return false
	}

	// get value
	fieldValue := reflect.ValueOf(acc).Elem().Field(field.Index)

	// get value value
	valueValue := reflect.ValueOf(value)

	// correct untyped nil values
	if value == nil && field.Type.Kind() == reflect.Ptr {
		valueValue = reflect.Zero(field.Type)
	}

	// check type
	if fieldValue.Type() != valueValue.Type() {
		return false
	}

	// set value
	fieldValue.Set(valueValue)

	return true
}

// MustGet will call Get and panic if the operation failed.
func MustGet(acc Accessible, name string) interface{} {
	// get value
	value, ok := Get(acc, name)
	if !ok {
		panic(fmt.Sprintf(`stick: could not get field "%s" on "%s"`, name, GetAccessor(acc).Name))
	}

	return value
}

// MustGetRaw will call GetRaw and panic if the operation failed.
func MustGetRaw(acc Accessible, name string) reflect.Value {
	// get raw value
	value, ok := GetRaw(acc, name)
	if !ok {
		panic(fmt.Sprintf(`stick: could not get field "%s" on "%s"`, name, GetAccessor(acc).Name))
	}

	return value
}

// MustSet will call Set and panic if the operation failed.
func MustSet(acc Accessible, name string, value interface{}) {
	// get value
	ok := Set(acc, name, value)
	if !ok {
		panic(fmt.Sprintf(`stick: could not set "%s" on "%s"`, name, GetAccessor(acc).Name))
	}
}
<commit_msg>added helpers to find struct type and value<commit_after>package stick

import (
	"fmt"
	"reflect"
	"sync"
)

// Field is dynamically accessible field.
type Field struct {
	Index int
	Type  reflect.Type
}

// Accessor provides dynamic access to a structs fields.
type Accessor struct {
	Name   string
	Fields map[string]*Field
}

// Accessible is a type that has dynamically accessible fields.
type Accessible interface {
	GetAccessor(interface{}) *Accessor
}

// GetAccessor is a short-hand to retrieve the accessor of an accessible.
func GetAccessor(acc Accessible) *Accessor {
	return acc.GetAccessor(acc)
}

var accessMutex sync.Mutex
var accessCache = map[reflect.Type]*Accessor{}

// BasicAccess may be embedded in a struct to provide basic accessibility.
type BasicAccess struct{}

// GetAccessor implements the Accessible interface.
func (a *BasicAccess) GetAccessor(v interface{}) *Accessor {
	// get type
	typ := structType(v)

	// acquire mutex
	accessMutex.Lock()
	defer accessMutex.Unlock()

	// check if accessor has already been cached
	accessor, ok := accessCache[typ]
	if ok {
		return accessor
	}

	// build accessor
	accessor = BuildAccessor(v.(Accessible), "BasicAccess")

	// cache accessor
	accessCache[typ] = accessor

	return accessor
}

// BuildAccessor will build an accessor for the provided type.
func BuildAccessor(v Accessible, ignore ...string) *Accessor {
	// get type
	typ := structType(v)

	// prepare accessor
	accessor := &Accessor{
		Name:   typ.String(),
		Fields: map[string]*Field{},
	}

	// iterate through all fields
	for i := 0; i < typ.NumField(); i++ {
		// get field
		field := typ.Field(i)

		// check field
		var skip bool
		for _, item := range ignore {
			if item == field.Name {
				skip = true
			}
		}
		if skip {
			continue
		}

		// add field
		accessor.Fields[field.Name] = &Field{
			Index: i,
			Type:  field.Type,
		}
	}

	return accessor
}

// Get will look up the specified field on the accessible and return its value
// and whether the field was found at all.
func Get(acc Accessible, name string) (interface{}, bool) {
	// find field
	field := GetAccessor(acc).Fields[name]
	if field == nil {
		return nil, false
	}

	// get value
	value := structValue(acc).Field(field.Index).Interface()

	return value, true
}

// GetRaw will look up the specified field on the accessible and return its raw
// value and whether the field was found at all.
func GetRaw(acc Accessible, name string) (reflect.Value, bool) {
	// find field
	field := GetAccessor(acc).Fields[name]
	if field == nil {
		return reflect.Value{}, false
	}

	// get value
	value := structValue(acc).Field(field.Index)

	return value, true
}

// Set will set the specified field on the accessible with the provided value
// and return whether the field has been found and the value has been set.
func Set(acc Accessible, name string, value interface{}) bool {
	// find field
	field := GetAccessor(acc).Fields[name]
	if field == nil {
		return false
	}

	// get value
	fieldValue := structValue(acc).Field(field.Index)

	// get value value
	valueValue := reflect.ValueOf(value)

	// correct untyped nil values
	if value == nil && field.Type.Kind() == reflect.Ptr {
		valueValue = reflect.Zero(field.Type)
	}

	// check type
	if fieldValue.Type() != valueValue.Type() {
		return false
	}

	// set value
	fieldValue.Set(valueValue)

	return true
}

// MustGet will call Get and panic if the operation failed.
func MustGet(acc Accessible, name string) interface{} {
	// get value
	value, ok := Get(acc, name)
	if !ok {
		panic(fmt.Sprintf(`stick: could not get field "%s" on "%s"`, name, GetAccessor(acc).Name))
	}

	return value
}

// MustGetRaw will call GetRaw and panic if the operation failed.
func MustGetRaw(acc Accessible, name string) reflect.Value {
	// get raw value
	value, ok := GetRaw(acc, name)
	if !ok {
		panic(fmt.Sprintf(`stick: could not get field "%s" on "%s"`, name, GetAccessor(acc).Name))
	}

	return value
}

// MustSet will call Set and panic if the operation failed.
func MustSet(acc Accessible, name string, value interface{}) {
	// get value
	ok := Set(acc, name, value)
	if !ok {
		panic(fmt.Sprintf(`stick: could not set "%s" on "%s"`, name, GetAccessor(acc).Name))
	}
}

func structType(v interface{}) reflect.Type {
	typ := reflect.TypeOf(v)
	for typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
	}
	if typ.Kind() != reflect.Struct {
		panic("stick: expected struct")
	}
	return typ
}

func structValue(v interface{}) reflect.Value {
	val := reflect.ValueOf(v)
	for val.Type().Kind() == reflect.Ptr {
		val = val.Elem()
	}
	if val.Kind() != reflect.Struct {
		panic("stick: expected struct")
	}
	return val
}
<|endoftext|>


<commit_before>package storage

import (
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/anacrolix/missinggo"
	"github.com/edsrzf/mmap-go"

	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/mmap_span"
)

type mmapStorage struct {
	baseDir string
	pc      PieceCompletion
}

func NewMMap(baseDir string) ClientImpl {
	return NewMMapWithCompletion(baseDir, pieceCompletionForDir(baseDir))
}

func NewMMapWithCompletion(baseDir string, completion PieceCompletion) ClientImpl {
	return &mmapStorage{
		baseDir: baseDir,
		pc:      completion,
	}
}

func (s *mmapStorage) OpenTorrent(info *metainfo.Info, infoHash metainfo.Hash) (t TorrentImpl, err error) {
	span, err := mMapTorrent(info, s.baseDir)
	t = &mmapTorrentStorage{
		infoHash: infoHash,
		span:     span,
		pc:       s.pc,
	}
	return
}

func (s *mmapStorage) Close() error {
	return s.pc.Close()
}

type mmapTorrentStorage struct {
	infoHash metainfo.Hash
	span     mmap_span.MMapSpan
	pc       PieceCompletion
}

func (ts *mmapTorrentStorage) Piece(p metainfo.Piece) PieceImpl {
	return mmapStoragePiece{
		pc:       ts.pc,
		p:        p,
		ih:       ts.infoHash,
		ReaderAt: io.NewSectionReader(ts.span, p.Offset(), p.Length()),
		WriterAt: missinggo.NewSectionWriter(ts.span, p.Offset(), p.Length()),
	}
}

func (ts *mmapTorrentStorage) Close() error {
	ts.pc.Close()
	return ts.span.Close()
}

type mmapStoragePiece struct {
	pc PieceCompletion
	p  metainfo.Piece
	ih metainfo.Hash
	io.ReaderAt
	io.WriterAt
}

func (me mmapStoragePiece) pieceKey() metainfo.PieceKey {
	return metainfo.PieceKey{me.ih, me.p.Index()}
}

func (sp mmapStoragePiece) GetIsComplete() (ret bool) {
	ret, _ = sp.pc.Get(sp.pieceKey())
	return
}

func (sp mmapStoragePiece) MarkComplete() error {
	sp.pc.Set(sp.pieceKey(), true)
	return nil
}

func (sp mmapStoragePiece) MarkNotComplete() error {
	sp.pc.Set(sp.pieceKey(), false)
	return nil
}

func mMapTorrent(md *metainfo.Info, location string) (mms mmap_span.MMapSpan, err error) {
	defer func() {
		if err != nil {
			mms.Close()
		}
	}()
	for _, miFile := range md.UpvertedFiles() {
		fileName := filepath.Join(append([]string{location, md.Name}, miFile.Path...)...)
		err = os.MkdirAll(filepath.Dir(fileName), 0777)
		if err != nil {
			err = fmt.Errorf("error creating data directory %q: %s", filepath.Dir(fileName), err)
			return
		}
		var file *os.File
		file, err = os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, 0666)
		if err != nil {
			return
		}
		func() {
			defer file.Close()
			var fi os.FileInfo
			fi, err = file.Stat()
			if err != nil {
				return
			}
			if fi.Size() < miFile.Length {
				err = file.Truncate(miFile.Length)
				if err != nil {
					return
				}
			}
			if miFile.Length == 0 {
				// Can't mmap() regions with length 0.
				return
			}
			var mMap mmap.MMap
			mMap, err = mmap.MapRegion(file,
				int(miFile.Length), // Probably not great on <64 bit systems.
				mmap.RDWR, 0, 0)
			if err != nil {
				err = fmt.Errorf("error mapping file %q, length %d: %s", file.Name(), miFile.Length, err)
				return
			}
			if int64(len(mMap)) != miFile.Length {
				panic("mmap has wrong length")
			}
			mms.Append(mMap)
		}()
		if err != nil {
			return
		}
	}
	return
}
<commit_msg>Improve mmap storage error messages<commit_after>package storage

import (
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/anacrolix/missinggo"
	"github.com/edsrzf/mmap-go"

	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/mmap_span"
)

type mmapStorage struct {
	baseDir string
	pc      PieceCompletion
}

func NewMMap(baseDir string) ClientImpl {
	return NewMMapWithCompletion(baseDir, pieceCompletionForDir(baseDir))
}

func NewMMapWithCompletion(baseDir string, completion PieceCompletion) ClientImpl {
	return &mmapStorage{
		baseDir: baseDir,
		pc:      completion,
	}
}

func (s *mmapStorage) OpenTorrent(info *metainfo.Info, infoHash metainfo.Hash) (t TorrentImpl, err error) {
	span, err := mMapTorrent(info, s.baseDir)
	t = &mmapTorrentStorage{
		infoHash: infoHash,
		span:     span,
		pc:       s.pc,
	}
	return
}

func (s *mmapStorage) Close() error {
	return s.pc.Close()
}

type mmapTorrentStorage struct {
	infoHash metainfo.Hash
	span     mmap_span.MMapSpan
	pc       PieceCompletion
}

func (ts *mmapTorrentStorage) Piece(p metainfo.Piece) PieceImpl {
	return mmapStoragePiece{
		pc:       ts.pc,
		p:        p,
		ih:       ts.infoHash,
		ReaderAt: io.NewSectionReader(ts.span, p.Offset(), p.Length()),
		WriterAt: missinggo.NewSectionWriter(ts.span, p.Offset(), p.Length()),
	}
}

func (ts *mmapTorrentStorage) Close() error {
	ts.pc.Close()
	return ts.span.Close()
}

type mmapStoragePiece struct {
	pc PieceCompletion
	p  metainfo.Piece
	ih metainfo.Hash
	io.ReaderAt
	io.WriterAt
}

func (me mmapStoragePiece) pieceKey() metainfo.PieceKey {
	return metainfo.PieceKey{me.ih, me.p.Index()}
}

func (sp mmapStoragePiece) GetIsComplete() (ret bool) {
	ret, _ = sp.pc.Get(sp.pieceKey())
	return
}

func (sp mmapStoragePiece) MarkComplete() error {
	sp.pc.Set(sp.pieceKey(), true)
	return nil
}

func (sp mmapStoragePiece) MarkNotComplete() error {
	sp.pc.Set(sp.pieceKey(), false)
	return nil
}

func mMapTorrent(md *metainfo.Info, location string) (mms mmap_span.MMapSpan, err error) {
	defer func() {
		if err != nil {
			mms.Close()
		}
	}()
	for _, miFile := range md.UpvertedFiles() {
		fileName := filepath.Join(append([]string{location, md.Name}, miFile.Path...)...)
		var mm mmap.MMap
		mm, err = mmapFile(fileName, miFile.Length)
		if err != nil {
			err = fmt.Errorf("file %q: %s", miFile.DisplayPath(md), err)
			return
		}
		mms.Append(mm)
	}
	return
}

func mmapFile(name string, size int64) (ret mmap.MMap, err error) {
	dir := filepath.Dir(name)
	err = os.MkdirAll(dir, 0777)
	if err != nil {
		err = fmt.Errorf("making directory %q: %s", dir, err)
		return
	}
	var file *os.File
	file, err = os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		return
	}
	defer file.Close()
	var fi os.FileInfo
	fi, err = file.Stat()
	if err != nil {
		return
	}
	if fi.Size() < size {
		// I think this is necessary on HFS+. Maybe Linux will SIGBUS too if
		// you overmap a file but I'm not sure.
		err = file.Truncate(size)
		if err != nil {
			return
		}
	}
	if size == 0 {
		// Can't mmap() regions with length 0.
		return
	}
	ret, err = mmap.MapRegion(file,
		int(size), // Probably not great on <64 bit systems.
		mmap.RDWR, 0, 0)
	if err != nil {
		err = fmt.Errorf("mapping file %q, length %d: %s", file.Name(), size, err)
		return
	}
	if int64(len(ret)) != size {
		panic("mmap has wrong length")
	}
	return
}
<|endoftext|>


<commit_before>package storage

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/base64"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"
)

func (c Client) computeHmac256(message string) string {
	h := hmac.New(sha256.New, c.accountKey)
	h.Write([]byte(message))
	return base64.StdEncoding.EncodeToString(h.Sum(nil))
}

func currentTimeRfc1123Formatted() string {
	return timeRfc1123Formatted(time.Now().UTC())
}

func timeRfc1123Formatted(t time.Time) string {
	return t.Format(http.TimeFormat)
}

func mergeParams(v1, v2 url.Values) url.Values {
	out := url.Values{}
	for k, v := range v1 {
		out[k] = v
	}
	for k, v := range v2 {
		vals, ok := out[k]
		if ok {
			vals = append(vals, v...)
			out[k] = vals
		} else {
			out[k] = v
		}
	}
	return out
}

func prepareBlockListRequest(blocks []Block) string {
	s := `<?xml version="1.0" encoding="utf-8"?><BlockList>`
	for _, v := range blocks {
		s += fmt.Sprintf("<%s>%s</%s>", v.Status, v.ID, v.Status)
	}
	s += `</BlockList>`
	return s
}

func xmlUnmarshal(body io.ReadCloser, v interface{}) error {
	data, err := ioutil.ReadAll(body)
	if err != nil {
		return err
	}
	defer body.Close()
	return xml.Unmarshal(data, v)
}

func xmlMarshal(v interface{}) (io.Reader, int, error) {
	b, err := xml.Marshal(v)
	if err != nil {
		return nil, 0, err
	}
	return bytes.NewReader(b), len(b), nil
}
<commit_msg>[Util] Close body even if ReadAll fails<commit_after>package storage

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/base64"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"
)

func (c Client) computeHmac256(message string) string {
	h := hmac.New(sha256.New, c.accountKey)
	h.Write([]byte(message))
	return base64.StdEncoding.EncodeToString(h.Sum(nil))
}

func currentTimeRfc1123Formatted() string {
	return timeRfc1123Formatted(time.Now().UTC())
}

func timeRfc1123Formatted(t time.Time) string {
	return t.Format(http.TimeFormat)
}

func mergeParams(v1, v2 url.Values) url.Values {
	out := url.Values{}
	for k, v := range v1 {
		out[k] = v
	}
	for k, v := range v2 {
		vals, ok := out[k]
		if ok {
			vals = append(vals, v...)
			out[k] = vals
		} else {
			out[k] = v
		}
	}
	return out
}

func prepareBlockListRequest(blocks []Block) string {
	s := `<?xml version="1.0" encoding="utf-8"?><BlockList>`
	for _, v := range blocks {
		s += fmt.Sprintf("<%s>%s</%s>", v.Status, v.ID, v.Status)
	}
	s += `</BlockList>`
	return s
}

func xmlUnmarshal(body io.ReadCloser, v interface{}) error {
	defer body.Close()
	data, err := ioutil.ReadAll(body)
	if err != nil {
		return err
	}
	return xml.Unmarshal(data, v)
}

func xmlMarshal(v interface{}) (io.Reader, int, error) {
	b, err := xml.Marshal(v)
	if err != nil {
		return nil, 0, err
	}
	return bytes.NewReader(b), len(b), nil
}
<|endoftext|>


<commit_before>package nhc_test

import (
	"fmt"
	"net"
	"reflect"
	"testing"
	"time"

	"github.com/mch1307/gomotics/log"
	. "github.com/mch1307/gomotics/nhc"
	"github.com/mch1307/gomotics/testutil"
)

func stubNHCTCP() {
	// listen to incoming tcp connections
	l, err := net.Listen("tcp", "0.0.0.0:8000")
	if err != nil {
		fmt.Println(err)
	}
	defer l.Close()
	_, err = l.Accept()
	if err != nil {
		fmt.Println(err)
	}
}

func stubNHCUDP() {
	// listen to incoming udp packets
	fmt.Println("starting UDP stub")
	pc, err := net.ListenPacket("udp", "0.0.0.0:10000")
	if err != nil {
		log.Fatal(err)
	}
	defer pc.Close()

	//simple read
	buffer := make([]byte, 1024)
	var addr net.Addr
	_, addr, _ = pc.ReadFrom(buffer)

	//simple write
	pc.WriteTo([]byte("NHC Stub"), addr)
}

func getOutboundIP() net.IP {
	conn, err := net.Dial("udp", "8.8.8.8:80")
	if err != nil {
		log.Fatal(err)
	}
	defer conn.Close()

	localAddr := conn.LocalAddr().(*net.UDPAddr)

	return localAddr.IP
}

func TestDiscover(t *testing.T) {

	tests := []struct {
		name string
		want net.IP
	}{
		{"no nhc on LAN", nil},
		{"stub nhc", getOutboundIP()},
	}
	portCheckIteration := 0
	for _, tt := range tests {
		fmt.Println("starting test ", tt.name)
		if tt.want != nil {
			go stubNHCUDP()
			go stubNHCTCP()
		}
		t.Run(tt.name, func(t *testing.T) {
		GotoTestPort:
			if testutil.IsTCPPortAvailable(18043) {
				if got := Discover(); !reflect.DeepEqual(got, tt.want) {
					t.Errorf("Discover() = %v, want %v", got, tt.want)
				}
			} else {
				portCheckIteration++
				if portCheckIteration < 21 {
					fmt.Printf("UDP 18043 busy, %v retry", portCheckIteration)
					time.Sleep(time.Millisecond * 500)
					goto GotoTestPort
				} else {
					t.Error("Discover failed to get UDP port 18043, test failed")
				}

			}
		})
	}
}
<commit_msg>comment one of the 2 tests<commit_after>package nhc_test

import (
	"fmt"
	"net"
	"reflect"
	"testing"
	"time"

	"github.com/mch1307/gomotics/log"
	. "github.com/mch1307/gomotics/nhc"
	"github.com/mch1307/gomotics/testutil"
)

func stubNHCTCP() {
	// listen to incoming tcp connections
	l, err := net.Listen("tcp", "0.0.0.0:8000")
	if err != nil {
		fmt.Println(err)
	}
	defer l.Close()
	_, err = l.Accept()
	if err != nil {
		fmt.Println(err)
	}
}

func stubNHCUDP() {
	// listen to incoming udp packets
	fmt.Println("starting UDP stub")
	pc, err := net.ListenPacket("udp", "0.0.0.0:10000")
	if err != nil {
		log.Fatal(err)
	}
	defer pc.Close()

	//simple read
	buffer := make([]byte, 1024)
	var addr net.Addr
	_, addr, _ = pc.ReadFrom(buffer)

	//simple write
	pc.WriteTo([]byte("NHC Stub"), addr)
}

func getOutboundIP() net.IP {
	conn, err := net.Dial("udp", "8.8.8.8:80")
	if err != nil {
		log.Fatal(err)
	}
	defer conn.Close()

	localAddr := conn.LocalAddr().(*net.UDPAddr)

	return localAddr.IP
}

func TestDiscover(t *testing.T) {

	tests := []struct {
		name string
		want net.IP
	}{
		//{"no nhc on LAN", nil},
		{"stub nhc", getOutboundIP()},
	}
	portCheckIteration := 0
	for _, tt := range tests {
		fmt.Println("starting test ", tt.name)
		if tt.want != nil {
			go stubNHCUDP()
			go stubNHCTCP()
		}
		t.Run(tt.name, func(t *testing.T) {
		GotoTestPort:
			if testutil.IsTCPPortAvailable(18043) {
				if got := Discover(); !reflect.DeepEqual(got, tt.want) {
					t.Errorf("Discover() = %v, want %v", got, tt.want)
				}
			} else {
				portCheckIteration++
				if portCheckIteration < 21 {
					fmt.Printf("UDP 18043 busy, %v retry", portCheckIteration)
					time.Sleep(time.Millisecond * 500)
					goto GotoTestPort
				} else {
					t.Error("Discover failed to get UDP port 18043, test failed")
				}

			}
		})
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package label

import (
	"bytes"
	"fmt"
	"io"
	"sync"

	"github.com/golang/glog"
	"k8s.io/apiserver/pkg/admission"
	utilfeature "k8s.io/apiserver/pkg/util/feature"
	api "k8s.io/kubernetes/pkg/apis/core"
	"k8s.io/kubernetes/pkg/cloudprovider"
	"k8s.io/kubernetes/pkg/cloudprovider/providers/aws"
	"k8s.io/kubernetes/pkg/cloudprovider/providers/azure"
	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
	"k8s.io/kubernetes/pkg/features"
	kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
	vol "k8s.io/kubernetes/pkg/volume"
	volumeutil "k8s.io/kubernetes/pkg/volume/util"
)

const (
	// PluginName is the name of persistent volume label admission plugin
	PluginName = "PersistentVolumeLabel"
)

// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		persistentVolumeLabelAdmission := newPersistentVolumeLabel()
		return persistentVolumeLabelAdmission, nil
	})
}

var _ = admission.Interface(&persistentVolumeLabel{})

type persistentVolumeLabel struct {
	*admission.Handler

	mutex            sync.Mutex
	ebsVolumes       aws.Volumes
	cloudConfig      []byte
	gceCloudProvider *gce.GCECloud
	azureProvider    *azure.Cloud
}

var _ admission.MutationInterface = &persistentVolumeLabel{}
var _ kubeapiserveradmission.WantsCloudConfig = &persistentVolumeLabel{}

// newPersistentVolumeLabel returns an admission.Interface implementation which adds labels to PersistentVolume CREATE requests,
// based on the labels provided by the underlying cloud provider.
//
// As a side effect, the cloud provider may block invalid or non-existent volumes.
func newPersistentVolumeLabel() *persistentVolumeLabel {
	// DEPRECATED: cloud-controller-manager will now start NewPersistentVolumeLabelController
	// which does exactly what this admission controller used to do. So once GCE, AWS and AZURE can
	// run externally, we can remove this admission controller.
	glog.Warning("PersistentVolumeLabel admission controller is deprecated. " +
		"Please remove this controller from your configuration files and scripts.")
	return &persistentVolumeLabel{
		Handler: admission.NewHandler(admission.Create),
	}
}

func (l *persistentVolumeLabel) SetCloudConfig(cloudConfig []byte) {
	l.cloudConfig = cloudConfig
}

func nodeSelectorRequirementKeysExistInNodeSelectorTerms(reqs []api.NodeSelectorRequirement, terms []api.NodeSelectorTerm) bool {
	for _, req := range reqs {
		for _, term := range terms {
			for _, r := range term.MatchExpressions {
				if r.Key == req.Key {
					return true
				}
			}
		}
	}
	return false
}

func (l *persistentVolumeLabel) Admit(a admission.Attributes) (err error) {
	if a.GetResource().GroupResource() != api.Resource("persistentvolumes") {
		return nil
	}
	obj := a.GetObject()
	if obj == nil {
		return nil
	}
	volume, ok := obj.(*api.PersistentVolume)
	if !ok {
		return nil
	}

	var volumeLabels map[string]string
	if volume.Spec.AWSElasticBlockStore != nil {
		labels, err := l.findAWSEBSLabels(volume)
		if err != nil {
			return admission.NewForbidden(a, fmt.Errorf("error querying AWS EBS volume %s: %v", volume.Spec.AWSElasticBlockStore.VolumeID, err))
		}
		volumeLabels = labels
	}
	if volume.Spec.GCEPersistentDisk != nil {
		labels, err := l.findGCEPDLabels(volume)
		if err != nil {
			return admission.NewForbidden(a, fmt.Errorf("error querying GCE PD volume %s: %v", volume.Spec.GCEPersistentDisk.PDName, err))
		}
		volumeLabels = labels
	}
	if volume.Spec.AzureDisk != nil {
		labels, err := l.findAzureDiskLabels(volume)
		if err != nil {
			return admission.NewForbidden(a, fmt.Errorf("error querying AzureDisk volume %s: %v", volume.Spec.AzureDisk.DiskName, err))
		}
		volumeLabels = labels
	}

	requirements := make([]api.NodeSelectorRequirement, 0)
	if len(volumeLabels) != 0 {
		if volume.Labels == nil {
			volume.Labels = make(map[string]string)
		}
		for k, v := range volumeLabels {
			// We (silently) replace labels if they are provided.
			// This should be OK because they are in the kubernetes.io namespace
			// i.e. we own them
			volume.Labels[k] = v

			// Set NodeSelectorRequirements based on the labels
			var values []string
			if k == kubeletapis.LabelZoneFailureDomain {
				zones, err := volumeutil.LabelZonesToSet(v)
				if err != nil {
					return admission.NewForbidden(a, fmt.Errorf("failed to convert label string for Zone: %s to a Set", v))
				}
				values = zones.UnsortedList()
			} else {
				values = []string{v}
			}
			requirements = append(requirements, api.NodeSelectorRequirement{Key: k, Operator: api.NodeSelectorOpIn, Values: values})
		}

		if utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) {
			if volume.Spec.NodeAffinity == nil {
				volume.Spec.NodeAffinity = new(api.VolumeNodeAffinity)
			}
			if volume.Spec.NodeAffinity.Required == nil {
				volume.Spec.NodeAffinity.Required = new(api.NodeSelector)
			}
			if len(volume.Spec.NodeAffinity.Required.NodeSelectorTerms) == 0 {
				// Need atleast one term pre-allocated whose MatchExpressions can be appended to
				volume.Spec.NodeAffinity.Required.NodeSelectorTerms = make([]api.NodeSelectorTerm, 1)
			}
			if nodeSelectorRequirementKeysExistInNodeSelectorTerms(requirements, volume.Spec.NodeAffinity.Required.NodeSelectorTerms) {
				glog.V(4).Info("NodeSelectorRequirements for cloud labels %v conflict with existing NodeAffinity %v. Skipping addition of NodeSelectorRequirements for cloud labels.",
					requirements, volume.Spec.NodeAffinity)
			} else {
				for _, req := range requirements {
					for i := range volume.Spec.NodeAffinity.Required.NodeSelectorTerms {
						volume.Spec.NodeAffinity.Required.NodeSelectorTerms[i].MatchExpressions = append(volume.Spec.NodeAffinity.Required.NodeSelectorTerms[i].MatchExpressions, req)
					}
				}
			}
		}
	}

	return nil
}

func (l *persistentVolumeLabel) findAWSEBSLabels(volume *api.PersistentVolume) (map[string]string, error) {
	// Ignore any volumes that are being provisioned
	if volume.Spec.AWSElasticBlockStore.VolumeID == vol.ProvisionedVolumeName {
		return nil, nil
	}
	ebsVolumes, err := l.getEBSVolumes()
	if err != nil {
		return nil, err
	}
	if ebsVolumes == nil {
		return nil, fmt.Errorf("unable to build AWS cloud provider for EBS")
	}

	// TODO: GetVolumeLabels is actually a method on the Volumes interface
	// If that gets standardized we can refactor to reduce code duplication
	spec := aws.KubernetesVolumeID(volume.Spec.AWSElasticBlockStore.VolumeID)
	labels, err := ebsVolumes.GetVolumeLabels(spec)
	if err != nil {
		return nil, err
	}

	return labels, nil
}

// getEBSVolumes returns the AWS Volumes interface for ebs
func (l *persistentVolumeLabel) getEBSVolumes() (aws.Volumes, error) {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	if l.ebsVolumes == nil {
		var cloudConfigReader io.Reader
		if len(l.cloudConfig) > 0 {
			cloudConfigReader = bytes.NewReader(l.cloudConfig)
		}
		cloudProvider, err := cloudprovider.GetCloudProvider("aws", cloudConfigReader)
		if err != nil || cloudProvider == nil {
			return nil, err
		}
		awsCloudProvider, ok := cloudProvider.(*aws.Cloud)
		if !ok {
			// GetCloudProvider has gone very wrong
			return nil, fmt.Errorf("error retrieving AWS cloud provider")
		}
		l.ebsVolumes = awsCloudProvider
	}
	return l.ebsVolumes, nil
}

func (l *persistentVolumeLabel) findGCEPDLabels(volume *api.PersistentVolume) (map[string]string, error) {
	// Ignore any volumes that are being provisioned
	if volume.Spec.GCEPersistentDisk.PDName == vol.ProvisionedVolumeName {
		return nil, nil
	}

	provider, err := l.getGCECloudProvider()
	if err != nil {
		return nil, err
	}
	if provider == nil {
		return nil, fmt.Errorf("unable to build GCE cloud provider for PD")
	}

	// If the zone is already labeled, honor the hint
	zone := volume.Labels[kubeletapis.LabelZoneFailureDomain]

	labels, err := provider.GetAutoLabelsForPD(volume.Spec.GCEPersistentDisk.PDName, zone)
	if err != nil {
		return nil, err
	}

	return labels, nil
}

// getGCECloudProvider returns the GCE cloud provider, for use for querying volume labels
func (l *persistentVolumeLabel) getGCECloudProvider() (*gce.GCECloud, error) {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	if l.gceCloudProvider == nil {
		var cloudConfigReader io.Reader
		if len(l.cloudConfig) > 0 {
			cloudConfigReader = bytes.NewReader(l.cloudConfig)
		}
		cloudProvider, err := cloudprovider.GetCloudProvider("gce", cloudConfigReader)
		if err != nil || cloudProvider == nil {
			return nil, err
		}
		gceCloudProvider, ok := cloudProvider.(*gce.GCECloud)
		if !ok {
			// GetCloudProvider has gone very wrong
			return nil, fmt.Errorf("error retrieving GCE cloud provider")
		}
		l.gceCloudProvider = gceCloudProvider
	}
	return l.gceCloudProvider, nil
}

// getAzureCloudProvider returns the Azure cloud provider, for use for querying volume labels
func (l *persistentVolumeLabel) getAzureCloudProvider() (*azure.Cloud, error) {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	if l.azureProvider == nil {
		var cloudConfigReader io.Reader
		if len(l.cloudConfig) > 0 {
			cloudConfigReader = bytes.NewReader(l.cloudConfig)
		}
		cloudProvider, err := cloudprovider.GetCloudProvider("azure", cloudConfigReader)
		if err != nil || cloudProvider == nil {
			return nil, err
		}
		azureProvider, ok := cloudProvider.(*azure.Cloud)
		if !ok {
			// GetCloudProvider has gone very wrong
			return nil, fmt.Errorf("error retrieving Azure cloud provider")
		}
		l.azureProvider = azureProvider
	}

	return l.azureProvider, nil
}

func (l *persistentVolumeLabel) findAzureDiskLabels(volume *api.PersistentVolume) (map[string]string, error) {
	// Ignore any volumes that are being provisioned
	if volume.Spec.AzureDisk.DiskName == vol.ProvisionedVolumeName {
		return nil, nil
	}

	provider, err := l.getAzureCloudProvider()
	if err != nil {
		return nil, err
	}
	if provider == nil {
		return nil, fmt.Errorf("unable to build Azure cloud provider for AzureDisk")
	}

	return provider.GetAzureDiskLabels(volume.Spec.AzureDisk.DataDiskURI)
}
<commit_msg>fix vet error in plugin/pkg/admission/storage/persistentvolume/label/admission.go<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package label

import (
	"bytes"
	"fmt"
	"io"
	"sync"

	"github.com/golang/glog"
	"k8s.io/apiserver/pkg/admission"
	utilfeature "k8s.io/apiserver/pkg/util/feature"
	api "k8s.io/kubernetes/pkg/apis/core"
	"k8s.io/kubernetes/pkg/cloudprovider"
	"k8s.io/kubernetes/pkg/cloudprovider/providers/aws"
	"k8s.io/kubernetes/pkg/cloudprovider/providers/azure"
	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
	"k8s.io/kubernetes/pkg/features"
	kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
	vol "k8s.io/kubernetes/pkg/volume"
	volumeutil "k8s.io/kubernetes/pkg/volume/util"
)

const (
	// PluginName is the name of persistent volume label admission plugin
	PluginName = "PersistentVolumeLabel"
)

// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		persistentVolumeLabelAdmission := newPersistentVolumeLabel()
		return persistentVolumeLabelAdmission, nil
	})
}

var _ = admission.Interface(&persistentVolumeLabel{})

type persistentVolumeLabel struct {
	*admission.Handler

	mutex            sync.Mutex
	ebsVolumes       aws.Volumes
	cloudConfig      []byte
	gceCloudProvider *gce.GCECloud
	azureProvider    *azure.Cloud
}

var _ admission.MutationInterface = &persistentVolumeLabel{}
var _ kubeapiserveradmission.WantsCloudConfig = &persistentVolumeLabel{}

// newPersistentVolumeLabel returns an admission.Interface implementation which adds labels to PersistentVolume CREATE requests,
// based on the labels provided by the underlying cloud provider.
//
// As a side effect, the cloud provider may block invalid or non-existent volumes.
func newPersistentVolumeLabel() *persistentVolumeLabel {
	// DEPRECATED: cloud-controller-manager will now start NewPersistentVolumeLabelController
	// which does exactly what this admission controller used to do. So once GCE, AWS and AZURE can
	// run externally, we can remove this admission controller.
	glog.Warning("PersistentVolumeLabel admission controller is deprecated. " +
		"Please remove this controller from your configuration files and scripts.")
	return &persistentVolumeLabel{
		Handler: admission.NewHandler(admission.Create),
	}
}

func (l *persistentVolumeLabel) SetCloudConfig(cloudConfig []byte) {
	l.cloudConfig = cloudConfig
}

func nodeSelectorRequirementKeysExistInNodeSelectorTerms(reqs []api.NodeSelectorRequirement, terms []api.NodeSelectorTerm) bool {
	for _, req := range reqs {
		for _, term := range terms {
			for _, r := range term.MatchExpressions {
				if r.Key == req.Key {
					return true
				}
			}
		}
	}
	return false
}

func (l *persistentVolumeLabel) Admit(a admission.Attributes) (err error) {
	if a.GetResource().GroupResource() != api.Resource("persistentvolumes") {
		return nil
	}
	obj := a.GetObject()
	if obj == nil {
		return nil
	}
	volume, ok := obj.(*api.PersistentVolume)
	if !ok {
		return nil
	}

	var volumeLabels map[string]string
	if volume.Spec.AWSElasticBlockStore != nil {
		labels, err := l.findAWSEBSLabels(volume)
		if err != nil {
			return admission.NewForbidden(a, fmt.Errorf("error querying AWS EBS volume %s: %v", volume.Spec.AWSElasticBlockStore.VolumeID, err))
		}
		volumeLabels = labels
	}
	if volume.Spec.GCEPersistentDisk != nil {
		labels, err := l.findGCEPDLabels(volume)
		if err != nil {
			return admission.NewForbidden(a, fmt.Errorf("error querying GCE PD volume %s: %v", volume.Spec.GCEPersistentDisk.PDName, err))
		}
		volumeLabels = labels
	}
	if volume.Spec.AzureDisk != nil {
		labels, err := l.findAzureDiskLabels(volume)
		if err != nil {
			return admission.NewForbidden(a, fmt.Errorf("error querying AzureDisk volume %s: %v", volume.Spec.AzureDisk.DiskName, err))
		}
		volumeLabels = labels
	}

	requirements := make([]api.NodeSelectorRequirement, 0)
	if len(volumeLabels) != 0 {
		if volume.Labels == nil {
			volume.Labels = make(map[string]string)
		}
		for k, v := range volumeLabels {
			// We (silently) replace labels if they are provided.
			// This should be OK because they are in the kubernetes.io namespace
			// i.e. we own them
			volume.Labels[k] = v

			// Set NodeSelectorRequirements based on the labels
			var values []string
			if k == kubeletapis.LabelZoneFailureDomain {
				zones, err := volumeutil.LabelZonesToSet(v)
				if err != nil {
					return admission.NewForbidden(a, fmt.Errorf("failed to convert label string for Zone: %s to a Set", v))
				}
				values = zones.UnsortedList()
			} else {
				values = []string{v}
			}
			requirements = append(requirements, api.NodeSelectorRequirement{Key: k, Operator: api.NodeSelectorOpIn, Values: values})
		}

		if utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) {
			if volume.Spec.NodeAffinity == nil {
				volume.Spec.NodeAffinity = new(api.VolumeNodeAffinity)
			}
			if volume.Spec.NodeAffinity.Required == nil {
				volume.Spec.NodeAffinity.Required = new(api.NodeSelector)
			}
			if len(volume.Spec.NodeAffinity.Required.NodeSelectorTerms) == 0 {
				// Need atleast one term pre-allocated whose MatchExpressions can be appended to
				volume.Spec.NodeAffinity.Required.NodeSelectorTerms = make([]api.NodeSelectorTerm, 1)
			}
			if nodeSelectorRequirementKeysExistInNodeSelectorTerms(requirements, volume.Spec.NodeAffinity.Required.NodeSelectorTerms) {
				glog.V(4).Infof("NodeSelectorRequirements for cloud labels %v conflict with existing NodeAffinity %v. Skipping addition of NodeSelectorRequirements for cloud labels.",
					requirements, volume.Spec.NodeAffinity)
			} else {
				for _, req := range requirements {
					for i := range volume.Spec.NodeAffinity.Required.NodeSelectorTerms {
						volume.Spec.NodeAffinity.Required.NodeSelectorTerms[i].MatchExpressions = append(volume.Spec.NodeAffinity.Required.NodeSelectorTerms[i].MatchExpressions, req)
					}
				}
			}
		}
	}

	return nil
}

func (l *persistentVolumeLabel) findAWSEBSLabels(volume *api.PersistentVolume) (map[string]string, error) {
	// Ignore any volumes that are being provisioned
	if volume.Spec.AWSElasticBlockStore.VolumeID == vol.ProvisionedVolumeName {
		return nil, nil
	}
	ebsVolumes, err := l.getEBSVolumes()
	if err != nil {
		return nil, err
	}
	if ebsVolumes == nil {
		return nil, fmt.Errorf("unable to build AWS cloud provider for EBS")
	}

	// TODO: GetVolumeLabels is actually a method on the Volumes interface
	// If that gets standardized we can refactor to reduce code duplication
	spec := aws.KubernetesVolumeID(volume.Spec.AWSElasticBlockStore.VolumeID)
	labels, err := ebsVolumes.GetVolumeLabels(spec)
	if err != nil {
		return nil, err
	}

	return labels, nil
}

// getEBSVolumes returns the AWS Volumes interface for ebs
func (l *persistentVolumeLabel) getEBSVolumes() (aws.Volumes, error) {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	if l.ebsVolumes == nil {
		var cloudConfigReader io.Reader
		if len(l.cloudConfig) > 0 {
			cloudConfigReader = bytes.NewReader(l.cloudConfig)
		}
		cloudProvider, err := cloudprovider.GetCloudProvider("aws", cloudConfigReader)
		if err != nil || cloudProvider == nil {
			return nil, err
		}
		awsCloudProvider, ok := cloudProvider.(*aws.Cloud)
		if !ok {
			// GetCloudProvider has gone very wrong
			return nil, fmt.Errorf("error retrieving AWS cloud provider")
		}
		l.ebsVolumes = awsCloudProvider
	}
	return l.ebsVolumes, nil
}

func (l *persistentVolumeLabel) findGCEPDLabels(volume *api.PersistentVolume) (map[string]string, error) {
	// Ignore any volumes that are being provisioned
	if volume.Spec.GCEPersistentDisk.PDName == vol.ProvisionedVolumeName {
		return nil, nil
	}

	provider, err := l.getGCECloudProvider()
	if err != nil {
		return nil, err
	}
	if provider == nil {
		return nil, fmt.Errorf("unable to build GCE cloud provider for PD")
	}

	// If the zone is already labeled, honor the hint
	zone := volume.Labels[kubeletapis.LabelZoneFailureDomain]

	labels, err := provider.GetAutoLabelsForPD(volume.Spec.GCEPersistentDisk.PDName, zone)
	if err != nil {
		return nil, err
	}

	return labels, nil
}

// getGCECloudProvider returns the GCE cloud provider, for use for querying volume labels
func (l *persistentVolumeLabel) getGCECloudProvider() (*gce.GCECloud, error) {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	if l.gceCloudProvider == nil {
		var cloudConfigReader io.Reader
		if len(l.cloudConfig) > 0 {
			cloudConfigReader = bytes.NewReader(l.cloudConfig)
		}
		cloudProvider, err := cloudprovider.GetCloudProvider("gce", cloudConfigReader)
		if err != nil || cloudProvider == nil {
			return nil, err
		}
		gceCloudProvider, ok := cloudProvider.(*gce.GCECloud)
		if !ok {
			// GetCloudProvider has gone very wrong
			return nil, fmt.Errorf("error retrieving GCE cloud provider")
		}
		l.gceCloudProvider = gceCloudProvider
	}
	return l.gceCloudProvider, nil
}

// getAzureCloudProvider returns the Azure cloud provider, for use for querying volume labels
func (l *persistentVolumeLabel) getAzureCloudProvider() (*azure.Cloud, error) {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	if l.azureProvider == nil {
		var cloudConfigReader io.Reader
		if len(l.cloudConfig) > 0 {
			cloudConfigReader = bytes.NewReader(l.cloudConfig)
		}
		cloudProvider, err := cloudprovider.GetCloudProvider("azure", cloudConfigReader)
		if err != nil || cloudProvider == nil {
			return nil, err
		}
		azureProvider, ok := cloudProvider.(*azure.Cloud)
		if !ok {
			// GetCloudProvider has gone very wrong
			return nil, fmt.Errorf("error retrieving Azure cloud provider")
		}
		l.azureProvider = azureProvider
	}

	return l.azureProvider, nil
}

func (l *persistentVolumeLabel) findAzureDiskLabels(volume *api.PersistentVolume) (map[string]string, error) {
	// Ignore any volumes that are being provisioned
	if volume.Spec.AzureDisk.DiskName == vol.ProvisionedVolumeName {
		return nil, nil
	}

	provider, err := l.getAzureCloudProvider()
	if err != nil {
		return nil, err
	}
	if provider == nil {
		return nil, fmt.Errorf("unable to build Azure cloud provider for AzureDisk")
	}

	return provider.GetAzureDiskLabels(volume.Spec.AzureDisk.DataDiskURI)
}
<|endoftext|>


<commit_before>// Copyright 2014 Chadev. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/danryan/hal"
)

var eventHandler = hal.Hear(`.events`, func(res *hal.Response) error {
	events, err := getCalendarEvents()
	if err != nil {
		return res.Send(err.Error())
	}

	return res.Send(events)
})

var (
	baseURL     = "https://www.googleapis.com/calendar/v3/calendars"
	accessToken AccessToken
)

// AccessToken contains the current oauth2 token and its expiry time.
type AccessToken struct {
	Token string
	Valid time.Time
}

func (a *AccessToken) getExpireTime(o int) {
	n := time.Now()
	d := time.Duration(o) * time.Second

	a.Valid = n.Add(d)
}

func (a *AccessToken) expiredToken() bool {
	n := time.Now()
	if a.Valid.After(n) {
		return true
	}

	return false
}

// Event is the top-level of the JSON return from the calendar API.
type Event struct {
	// Kind is the type of collection ("calendar#events")
	Kind string `json:"kind"`
	// Etag is the etag of the collection
	Etag string `json:"etag"`
	// Summary is the name of the calendar
	Summary string `json:"summary"`
	// Description is the description of the calendar
	Description string `json:"descritpion"`
	// Updated is the last modified time for the calendar
	Updated string `json:"updated"`
	// TimeZone is the timezone for the calendar
	TimeZone string `json:"timeZone"`
	// AccessRole is the role of the current user.
	// possible values include: "none", "freeBuzyReader", "reader", "writer", "owner"
	AccessRole string `json:"accessRole"`
	// NextPageToken is used to access the next page of this result. Omitted if no
	// further results are available, in which case nextSyncToken is provided.
	NextPageToken string `json:"nextPageToken,omitempty"`
	// Items is the list of events on the calendar.
	Items []EventItem `json:"items"`
	// NextSyncToken is used at a later point in time to retrieve only the entries
	// that have changed since this result was returned. Omitted if further results
	// are available, in which case nextPageToken is provided.
	NextSyncToken string `json:"nextSyncToken"`
}

// EventItem contains items with-in the "items" list of the JSON.
type EventItem struct {
	// Kind is the type of collection ("calendar#event")
	Kind string `json:"kind"`
	// Etag string `json:"etag"`
	Etag string `json:"etag"`
	// ID is the ID of the event
	ID string `json:"id"`
	// Status is the status of the event
	Status string `json:"status"`
	// HTMLLink is the link to the event on the calendar
	HTMLLink string `json:"htmlLink"`
	// Created is the datetime stamp that the event was created
	Created string `json:"created"`
	// Updated is the datetame stamp that the event was last modified
	Updated string `json:"updated"`
	// Summary is the name of the event
	Summary string `json:"summary"`
	// Description is the body of the event
	Description string `json:"description"`
	// Location is the address for the event
	Location string `json:"location"`
	// Creator is the user that created the event
	Creator EventCreator `json:"creator"`
	// Organizer is the organizer information
	organizer EventOrganizer `json:"organizer"`
	// Start is the start date and time
	Start EventDateTime `json:"start"`
	// End is the end date and time
	End EventDateTime `json:"end"`
	// ICalUID is the iCal ID for the event
	ICalUID string `json:"uCalUID"`
	// Sequence denotes if this is a repeating event
	Sequence int `json:"sequence"`
	// Reminders are the event reminders
	Reminders EventReminder `json:"reminders"`
}

// EventCreator contains the fields for the creator object.
type EventCreator struct {
	// Email address for the event creator
	Email string `json:"email"`
	// DisplayName is the real name of the creator
	DisplayName string `json:"displayName"`
}

// EventOrganizer contains the fields for the organizer object.
type EventOrganizer struct {
	// Email address for the event organizer
	Email string `json:"email"`
	// DisplayName is the name of the event organizer
	DisplayName string `json:"displayName"`
	// Self denotes if the organizer is the current user
	Self bool `json:self"`
}

// EventDateTime contains the fields for the "start" and "end" objects.
type EventDateTime struct {
	// DateTime is the start/end datetime in  RFC 3339 format
	DateTime string `json:"dateTime"`
	// TimeZone is the Timezone for the event
	TimeZone string `json:"timeZone,omitempty"`
}

//EventReminder contains the fields for the reminders object.
type EventReminder struct {
	// UseDefault denotes if Google Calendar default reminders are used
	UseDefault bool `json:"useDefault"`
}

func getCalendarEvents() (string, error) {
	var err error

	if accessToken.Token == "" ||
		accessToken.expiredToken() {
		accessToken, err = getOauth2Token()
		if err != nil {
			return "", err
		}
	}

	URL := fmt.Sprintf("%s/4qc3thgj9ocunpfist563utr6g@group.calendar.google.com/events?access_token=%s&singleEvents=true&orderBy=startTime&timeMin=%s&maxResults=7",
		baseURL, url.QueryEscape(accessToken.Token),
		url.QueryEscape(time.Now().Format("2006-01-02T15:04:05Z")))
	resp, err := http.Get(URL)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	var Events Event

	err = json.Unmarshal(body, &Events)
	if err != nil {
		return "", err
	}

	eventList := getEventList(Events)

	return eventList, nil
}

func getOauth2Token() (AccessToken, error) {
	type Responce struct {
		Token   string `json:"access_token"`
		Expires int    `json:"expires_in"`
	}

	clientID := os.Getenv("CHADEV_ID")
	clientSecret := os.Getenv("CHADEV_SECRET")
	refreshToken := os.Getenv("CHADEV_TOKEN")

	var r Responce

	body := fmt.Sprintf("client_id=%s&client_secret=%s&refresh_token=%s&grant_type=refresh_token",
		clientID, clientSecret, refreshToken)
	b := strings.NewReader(body)

	resp, err := http.Post("https://accounts.google.com/o/oauth2/token", "application/x-www-form-urlencoded", b)
	if err != nil {
		return accessToken, err
	}
	defer resp.Body.Close()

	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return accessToken, err
	}

	err = json.Unmarshal(respBody, &r)
	if err != nil {
		return accessToken, err
	}

	accessToken.Token = r.Token
	accessToken.getExpireTime(r.Expires)

	return accessToken, nil
}

func getEventList(events Event) string {
	var output string

	output = "Next 7 events: "
	for key, event := range events.Items {
		if key == 0 {
			output += event.Summary
		} else {
			output += ", " + event.Summary
		}
		output += " (" + formatDate(event.Start.DateTime) + ")"
	}

	return output
}

func formatDate(s string) string {
	date, err := time.Parse(time.RFC3339, s)
	if err != nil {
		return ""
	}

	return date.Format("01/02")
}
<commit_msg>Upon an error happening when fetching events, log it<commit_after>// Copyright 2014 Chadev. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/danryan/hal"
)

var eventHandler = hal.Hear(`.events`, func(res *hal.Response) error {
	events, err := getCalendarEvents()
	if err != nil {
		log.Printf("failed to call Calendar API: %v\n", err)
		return res.Send("Could not fetch data from Google Calendar API, please try again later")
	}

	return res.Send(events)
})

var (
	baseURL     = "https://www.googleapis.com/calendar/v3/calendars"
	accessToken AccessToken
)

// AccessToken contains the current oauth2 token and its expiry time.
type AccessToken struct {
	Token string
	Valid time.Time
}

func (a *AccessToken) getExpireTime(o int) {
	n := time.Now()
	d := time.Duration(o) * time.Second

	a.Valid = n.Add(d)
}

func (a *AccessToken) expiredToken() bool {
	n := time.Now()
	if a.Valid.After(n) {
		return true
	}

	return false
}

// Event is the top-level of the JSON return from the calendar API.
type Event struct {
	// Kind is the type of collection ("calendar#events")
	Kind string `json:"kind"`
	// Etag is the etag of the collection
	Etag string `json:"etag"`
	// Summary is the name of the calendar
	Summary string `json:"summary"`
	// Description is the description of the calendar
	Description string `json:"descritpion"`
	// Updated is the last modified time for the calendar
	Updated string `json:"updated"`
	// TimeZone is the timezone for the calendar
	TimeZone string `json:"timeZone"`
	// AccessRole is the role of the current user.
	// possible values include: "none", "freeBuzyReader", "reader", "writer", "owner"
	AccessRole string `json:"accessRole"`
	// NextPageToken is used to access the next page of this result. Omitted if no
	// further results are available, in which case nextSyncToken is provided.
	NextPageToken string `json:"nextPageToken,omitempty"`
	// Items is the list of events on the calendar.
	Items []EventItem `json:"items"`
	// NextSyncToken is used at a later point in time to retrieve only the entries
	// that have changed since this result was returned. Omitted if further results
	// are available, in which case nextPageToken is provided.
	NextSyncToken string `json:"nextSyncToken"`
}

// EventItem contains items with-in the "items" list of the JSON.
type EventItem struct {
	// Kind is the type of collection ("calendar#event")
	Kind string `json:"kind"`
	// Etag string `json:"etag"`
	Etag string `json:"etag"`
	// ID is the ID of the event
	ID string `json:"id"`
	// Status is the status of the event
	Status string `json:"status"`
	// HTMLLink is the link to the event on the calendar
	HTMLLink string `json:"htmlLink"`
	// Created is the datetime stamp that the event was created
	Created string `json:"created"`
	// Updated is the datetame stamp that the event was last modified
	Updated string `json:"updated"`
	// Summary is the name of the event
	Summary string `json:"summary"`
	// Description is the body of the event
	Description string `json:"description"`
	// Location is the address for the event
	Location string `json:"location"`
	// Creator is the user that created the event
	Creator EventCreator `json:"creator"`
	// Organizer is the organizer information
	organizer EventOrganizer `json:"organizer"`
	// Start is the start date and time
	Start EventDateTime `json:"start"`
	// End is the end date and time
	End EventDateTime `json:"end"`
	// ICalUID is the iCal ID for the event
	ICalUID string `json:"uCalUID"`
	// Sequence denotes if this is a repeating event
	Sequence int `json:"sequence"`
	// Reminders are the event reminders
	Reminders EventReminder `json:"reminders"`
}

// EventCreator contains the fields for the creator object.
type EventCreator struct {
	// Email address for the event creator
	Email string `json:"email"`
	// DisplayName is the real name of the creator
	DisplayName string `json:"displayName"`
}

// EventOrganizer contains the fields for the organizer object.
type EventOrganizer struct {
	// Email address for the event organizer
	Email string `json:"email"`
	// DisplayName is the name of the event organizer
	DisplayName string `json:"displayName"`
	// Self denotes if the organizer is the current user
	Self bool `json:self"`
}

// EventDateTime contains the fields for the "start" and "end" objects.
type EventDateTime struct {
	// DateTime is the start/end datetime in  RFC 3339 format
	DateTime string `json:"dateTime"`
	// TimeZone is the Timezone for the event
	TimeZone string `json:"timeZone,omitempty"`
}

//EventReminder contains the fields for the reminders object.
type EventReminder struct {
	// UseDefault denotes if Google Calendar default reminders are used
	UseDefault bool `json:"useDefault"`
}

func getCalendarEvents() (string, error) {
	var err error

	if accessToken.Token == "" ||
		accessToken.expiredToken() {
		accessToken, err = getOauth2Token()
		if err != nil {
			return "", err
		}
	}

	URL := fmt.Sprintf("%s/4qc3thgj9ocunpfist563utr6g@group.calendar.google.com/events?access_token=%s&singleEvents=true&orderBy=startTime&timeMin=%s&maxResults=7",
		baseURL, url.QueryEscape(accessToken.Token),
		url.QueryEscape(time.Now().Format("2006-01-02T15:04:05Z")))
	resp, err := http.Get(URL)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	var Events Event

	err = json.Unmarshal(body, &Events)
	if err != nil {
		return "", err
	}

	eventList := getEventList(Events)

	return eventList, nil
}

func getOauth2Token() (AccessToken, error) {
	type Responce struct {
		Token   string `json:"access_token"`
		Expires int    `json:"expires_in"`
	}

	clientID := os.Getenv("CHADEV_ID")
	clientSecret := os.Getenv("CHADEV_SECRET")
	refreshToken := os.Getenv("CHADEV_TOKEN")

	var r Responce

	body := fmt.Sprintf("client_id=%s&client_secret=%s&refresh_token=%s&grant_type=refresh_token",
		clientID, clientSecret, refreshToken)
	b := strings.NewReader(body)

	resp, err := http.Post("https://accounts.google.com/o/oauth2/token", "application/x-www-form-urlencoded", b)
	if err != nil {
		return accessToken, err
	}
	defer resp.Body.Close()

	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return accessToken, err
	}

	err = json.Unmarshal(respBody, &r)
	if err != nil {
		return accessToken, err
	}

	accessToken.Token = r.Token
	accessToken.getExpireTime(r.Expires)

	return accessToken, nil
}

func getEventList(events Event) string {
	var output string

	output = "Next 7 events: "
	for key, event := range events.Items {
		if key == 0 {
			output += event.Summary
		} else {
			output += ", " + event.Summary
		}
		output += " (" + formatDate(event.Start.DateTime) + ")"
	}

	return output
}

func formatDate(s string) string {
	date, err := time.Parse(time.RFC3339, s)
	if err != nil {
		return ""
	}

	return date.Format("01/02")
}
<|endoftext|>