<commit_before>/*
   DNS-over-HTTPS
   Copyright (C) 2017-2018 Star Brilliant <m13253@hotmail.com>

   Permission is hereby granted, free of charge, to any person obtaining a
   copy of this software and associated documentation files (the "Software"),
   to deal in the Software without restriction, including without limitation
   the rights to use, copy, modify, merge, publish, distribute, sublicense,
   and/or sell copies of the Software, and to permit persons to whom the
   Software is furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
   FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
   DEALINGS IN THE SOFTWARE.
*/

package main

import (
	"fmt"
	"log"
	"math/rand"
	"net"
	"net/http"
	"os"
	"strings"
	"time"

	"../json-dns"
	"github.com/gorilla/handlers"
	"github.com/miekg/dns"
)

type Server struct {
	conf      *config
	udpClient *dns.Client
	tcpClient *dns.Client
	servemux  *http.ServeMux
}

type DNSRequest struct {
	request    *dns.Msg
	response   *dns.Msg
	isTailored bool
	errcode    int
	errtext    string
}

func NewServer(conf *config) (s *Server) {
	s = &Server{
		conf: conf,
		udpClient: &dns.Client{
			Net:     "udp",
			Timeout: time.Duration(conf.Timeout) * time.Second,
		},
		tcpClient: &dns.Client{
			Net:     "tcp",
			Timeout: time.Duration(conf.Timeout) * time.Second,
		},
		servemux: http.NewServeMux(),
	}
	s.servemux.HandleFunc(conf.Path, s.handlerFunc)
	return
}

func (s *Server) Start() error {
	servemux := http.Handler(s.servemux)
	if s.conf.Verbose {
		servemux = handlers.CombinedLoggingHandler(os.Stdout, servemux)
	}
	if s.conf.Cert != "" || s.conf.Key != "" {
		return http.ListenAndServeTLS(s.conf.Listen, s.conf.Cert, s.conf.Key, servemux)
	}
	return http.ListenAndServe(s.conf.Listen, servemux)
}

func (s *Server) handlerFunc(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Server", "DNS-over-HTTPS/1.1 (+https://github.com/m13253/dns-over-https)")
	w.Header().Set("X-Powered-By", "DNS-over-HTTPS/1.1 (+https://github.com/m13253/dns-over-https)")

	if r.Form == nil {
		const maxMemory = 32 << 20 // 32 MB
		r.ParseMultipartForm(maxMemory)
	}
	contentType := r.Header.Get("Content-Type")
	if ct := r.FormValue("ct"); ct != "" {
		contentType = ct
	}
	if contentType == "" {
		// Guess request Content-Type based on other parameters
		if r.FormValue("name") != "" {
			contentType = "application/x-www-form-urlencoded"
		} else if r.FormValue("dns") != "" {
			contentType = "application/dns-udpwireformat"
		}
	}
	var responseType string
	for _, responseCandidate := range strings.Split(r.Header.Get("Accept"), ",") {
		responseCandidate = strings.ToLower(strings.SplitN(responseCandidate, ";", 2)[0])
		if responseCandidate == "application/json" {
			responseType = "application/json"
			break
		} else if responseCandidate == "application/dns-udpwireformat" {
			responseType = "application/dns-udpwireformat"
			break
		}
	}
	if responseType == "" {
		// Guess response Content-Type based on request Content-Type
		if contentType == "application/x-www-form-urlencoded" {
			responseType = "application/json"
		} else if contentType == "application/dns-udpwireformat" {
			responseType = "application/dns-udpwireformat"
		}
	}

	var req *DNSRequest
	if contentType == "application/x-www-form-urlencoded" {
		req = s.parseRequestGoogle(w, r)
	} else if contentType == "application/dns-udpwireformat" {
		req = s.parseRequestIETF(w, r)
	} else {
		jsonDNS.FormatError(w, fmt.Sprintf("Invalid argument value: \"ct\" = %q", contentType), 400)
		return
	}
	if req.errcode != 0 {
		jsonDNS.FormatError(w, req.errtext, req.errcode)
		return
	}

	var err error
	req.response, err = s.doDNSQuery(req.request)
	if err != nil {
		jsonDNS.FormatError(w, fmt.Sprintf("DNS query failure (%s)", err.Error()), 503)
		return
	}

	if contentType == "application/x-www-form-urlencoded" {
		s.generateResponseGoogle(w, r, req)
	} else if contentType == "application/dns-udpwireformat" {
		s.generateResponseIETF(w, r, req)
	}
}

func (s *Server) findClientIP(r *http.Request) net.IP {
	XForwardedFor := r.Header.Get("X-Forwarded-For")
	if XForwardedFor != "" {
		for _, addr := range strings.Split(XForwardedFor, ",") {
			addr = strings.TrimSpace(addr)
			ip := net.ParseIP(addr)
			if jsonDNS.IsGlobalIP(ip) {
				return ip
			}
		}
	}
	XRealIP := r.Header.Get("X-Real-IP")
	if XRealIP != "" {
		addr := strings.TrimSpace(XRealIP)
		ip := net.ParseIP(addr)
		if jsonDNS.IsGlobalIP(ip) {
			return ip
		}
	}
	remoteAddr, err := net.ResolveTCPAddr("tcp", r.RemoteAddr)
	if err != nil {
		return nil
	}
	if ip := remoteAddr.IP; jsonDNS.IsGlobalIP(ip) {
		return ip
	}
	return nil
}

func (s *Server) doDNSQuery(msg *dns.Msg) (resp *dns.Msg, err error) {
	numServers := len(s.conf.Upstream)
	for i := uint(0); i < s.conf.Tries; i++ {
		server := s.conf.Upstream[rand.Intn(numServers)]
		if !s.conf.TCPOnly {
			resp, _, err = s.udpClient.Exchange(msg, server)
			if err == dns.ErrTruncated {
				log.Println(err)
				resp, _, err = s.tcpClient.Exchange(msg, server)
			}
		} else {
			resp, _, err = s.tcpClient.Exchange(msg, server)
		}
		if err == nil {
			return
		}
		log.Println(err)
	}
	return
}
<commit_msg>Fix Accept header detection<commit_after>/*
   DNS-over-HTTPS
   Copyright (C) 2017-2018 Star Brilliant <m13253@hotmail.com>

   Permission is hereby granted, free of charge, to any person obtaining a
   copy of this software and associated documentation files (the "Software"),
   to deal in the Software without restriction, including without limitation
   the rights to use, copy, modify, merge, publish, distribute, sublicense,
   and/or sell copies of the Software, and to permit persons to whom the
   Software is furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
   FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
   DEALINGS IN THE SOFTWARE.
*/

package main

import (
	"fmt"
	"log"
	"math/rand"
	"net"
	"net/http"
	"os"
	"strings"
	"time"

	"../json-dns"
	"github.com/gorilla/handlers"
	"github.com/miekg/dns"
)

type Server struct {
	conf      *config
	udpClient *dns.Client
	tcpClient *dns.Client
	servemux  *http.ServeMux
}

type DNSRequest struct {
	request    *dns.Msg
	response   *dns.Msg
	isTailored bool
	errcode    int
	errtext    string
}

func NewServer(conf *config) (s *Server) {
	s = &Server{
		conf: conf,
		udpClient: &dns.Client{
			Net:     "udp",
			Timeout: time.Duration(conf.Timeout) * time.Second,
		},
		tcpClient: &dns.Client{
			Net:     "tcp",
			Timeout: time.Duration(conf.Timeout) * time.Second,
		},
		servemux: http.NewServeMux(),
	}
	s.servemux.HandleFunc(conf.Path, s.handlerFunc)
	return
}

func (s *Server) Start() error {
	servemux := http.Handler(s.servemux)
	if s.conf.Verbose {
		servemux = handlers.CombinedLoggingHandler(os.Stdout, servemux)
	}
	if s.conf.Cert != "" || s.conf.Key != "" {
		return http.ListenAndServeTLS(s.conf.Listen, s.conf.Cert, s.conf.Key, servemux)
	}
	return http.ListenAndServe(s.conf.Listen, servemux)
}

func (s *Server) handlerFunc(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Server", "DNS-over-HTTPS/1.1 (+https://github.com/m13253/dns-over-https)")
	w.Header().Set("X-Powered-By", "DNS-over-HTTPS/1.1 (+https://github.com/m13253/dns-over-https)")

	if r.Form == nil {
		const maxMemory = 32 << 20 // 32 MB
		r.ParseMultipartForm(maxMemory)
	}
	contentType := r.Header.Get("Content-Type")
	if ct := r.FormValue("ct"); ct != "" {
		contentType = ct
	}
	if contentType == "" {
		// Guess request Content-Type based on other parameters
		if r.FormValue("name") != "" {
			contentType = "application/x-www-form-urlencoded"
		} else if r.FormValue("dns") != "" {
			contentType = "application/dns-udpwireformat"
		}
	}
	var responseType string
	for _, responseCandidate := range strings.Split(r.Header.Get("Accept"), ",") {
		responseCandidate = strings.ToLower(strings.SplitN(responseCandidate, ";", 2)[0])
		if responseCandidate == "application/json" {
			responseType = "application/json"
			break
		} else if responseCandidate == "application/dns-udpwireformat" {
			responseType = "application/dns-udpwireformat"
			break
		}
	}
	if responseType == "" {
		// Guess response Content-Type based on request Content-Type
		if contentType == "application/x-www-form-urlencoded" {
			responseType = "application/json"
		} else if contentType == "application/dns-udpwireformat" {
			responseType = "application/dns-udpwireformat"
		}
	}

	var req *DNSRequest
	if contentType == "application/x-www-form-urlencoded" {
		req = s.parseRequestGoogle(w, r)
	} else if contentType == "application/dns-udpwireformat" {
		req = s.parseRequestIETF(w, r)
	} else {
		jsonDNS.FormatError(w, fmt.Sprintf("Invalid argument value: \"ct\" = %q", contentType), 400)
		return
	}
	if req.errcode != 0 {
		jsonDNS.FormatError(w, req.errtext, req.errcode)
		return
	}

	var err error
	req.response, err = s.doDNSQuery(req.request)
	if err != nil {
		jsonDNS.FormatError(w, fmt.Sprintf("DNS query failure (%s)", err.Error()), 503)
		return
	}

	if responseType == "application/json" {
		s.generateResponseGoogle(w, r, req)
	} else if contentType == "application/dns-udpwireformat" {
		s.generateResponseIETF(w, r, req)
	}
}

func (s *Server) findClientIP(r *http.Request) net.IP {
	XForwardedFor := r.Header.Get("X-Forwarded-For")
	if XForwardedFor != "" {
		for _, addr := range strings.Split(XForwardedFor, ",") {
			addr = strings.TrimSpace(addr)
			ip := net.ParseIP(addr)
			if jsonDNS.IsGlobalIP(ip) {
				return ip
			}
		}
	}
	XRealIP := r.Header.Get("X-Real-IP")
	if XRealIP != "" {
		addr := strings.TrimSpace(XRealIP)
		ip := net.ParseIP(addr)
		if jsonDNS.IsGlobalIP(ip) {
			return ip
		}
	}
	remoteAddr, err := net.ResolveTCPAddr("tcp", r.RemoteAddr)
	if err != nil {
		return nil
	}
	if ip := remoteAddr.IP; jsonDNS.IsGlobalIP(ip) {
		return ip
	}
	return nil
}

func (s *Server) doDNSQuery(msg *dns.Msg) (resp *dns.Msg, err error) {
	numServers := len(s.conf.Upstream)
	for i := uint(0); i < s.conf.Tries; i++ {
		server := s.conf.Upstream[rand.Intn(numServers)]
		if !s.conf.TCPOnly {
			resp, _, err = s.udpClient.Exchange(msg, server)
			if err == dns.ErrTruncated {
				log.Println(err)
				resp, _, err = s.tcpClient.Exchange(msg, server)
			}
		} else {
			resp, _, err = s.tcpClient.Exchange(msg, server)
		}
		if err == nil {
			return
		}
		log.Println(err)
	}
	return
}
<|endoftext|>


<commit_before>package download

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"strings"

	"github.com/subutai-io/base/agent/log"
	"github.com/subutai-io/gorjun/db"
)

var (
	path = "/tmp/"
)

type AptItem struct {
	Architecture string `json:"architecture,omitempty"`
	Description  string `json:"description,omitempty"`
	Filename     string `json:"filename,omitempty"`
	Md5Sum       string `json:"md5Sum,omitempty"`
	Name         string `json:"name,omitempty"`
	Package      string `json:"package,omitempty"`
	Version      string `json:"version,omitempty"`
	Size         int64  `json:"size"`
}

type RawItem struct {
	Md5Sum  string `json:"md5Sum,omitempty"`
	Name    string `json:"name,omitempty"`
	Package string `json:"package,omitempty"`
	Version string `json:"version,omitempty"`
	Size    int64  `json:"size"`
	ID      string `json:"id"`
}

type ListItem struct {
	Architecture   string `json:"architecture"`
	ConfigContents string `json:"configContents"`
	Extra          struct {
		Lxc_idMap           string `json:"lxc.id_map"`
		Lxc_include         string `json:"lxc.include"`
		Lxc_mount           string `json:"lxc.mount"`
		Lxc_mount_entry     string `json:"lxc.mount.entry"`
		Lxc_network_flags   string `json:"lxc.network.flags"`
		Lxc_network_hwaddr  string `json:"lxc.network.hwaddr"`
		Lxc_network_link    string `json:"lxc.network.link"`
		Lxc_network_type    string `json:"lxc.network.type"`
		Lxc_rootfs          string `json:"lxc.rootfs"`
		Subutai_config_path string `json:"subutai.config.path"`
		Subutai_git_branch  string `json:"subutai.git.branch"`
	} `json:"extra"`
	ID               string `json:"id"`
	Md5Sum           string `json:"md5Sum"`
	Name             string `json:"name"`
	OwnerFprint      string `json:"ownerFprint"`
	Package          string `json:"package"`
	PackagesContents string `json:"packagesContents"`
	Parent           string `json:"parent"`
	Size             int64  `json:"size"`
	Version          string `json:"version"`
}

func Handler(w http.ResponseWriter, r *http.Request) {
	hash := r.URL.Query().Get("hash")
	name := r.URL.Query().Get("name")
	id := r.URL.Query().Get("id")
	if len(id) > 0 {
		hash = id
		tmp := strings.Split(id, ".")
		if len(tmp) == 2 {
			hash = tmp[1]
		}
	}
	if len(hash) == 0 && len(name) == 0 {
		w.Write([]byte("Please specify hash or name"))
		return
	} else if len(name) != 0 {
		hash = db.LastHash(name)
	}
	w.Header().Set("Content-Disposition", "attachment; filename="+db.Read(hash))
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	f, err := os.Open(path + hash)
	log.Check(log.WarnLevel, "Opening file "+path+hash, err)
	fi, _ := f.Stat()
	w.Header().Set("Content-Length", fmt.Sprint(fi.Size()))
	defer f.Close()
	io.Copy(w, f)
}

func List(repo string, w http.ResponseWriter, r *http.Request) {
	w.Write([]byte(""))
	for k, v := range db.List() {
		if db.Info(k)["type"] == repo {
			w.Write([]byte("
" + v + "
"))
		}
	}
	w.Write([]byte(""))
}

func Search(repo string, w http.ResponseWriter, r *http.Request) {
	query := r.URL.Query().Get("q")
	w.Write([]byte(""))
	for k, v := range db.Search(query) {
		w.Write([]byte("
" + v + "
"))
	}
	w.Write([]byte(""))
}

func Info(repo string, r *http.Request) []byte {
	var item, js []byte
	name := r.URL.Query().Get("name")
	version := r.URL.Query().Get("version")
	rtype := r.URL.Query().Get("type")
	id := r.URL.Query().Get("id")

	ids := strings.Split(id, ".")
	if len(ids) > 1 {
		name = db.Read(ids[1])
	}

	counter := 0
	for k, _ := range db.Search(name) {
		info := db.Info(k)
		if info["type"] == repo {
			counter++
			if rtype == "text" && repo == "template" {
				if strings.HasPrefix(info["name"], name) && (len(version) == 0 || info["version"] == version) {
					return ([]byte("public." + k))
				}
				return []byte("")
			}
			f, err := os.Open(path + k)
			defer f.Close()
			log.Check(log.WarnLevel, "Opening file "+path+k, err)
			fi, _ := f.Stat()
			switch repo {
			case "template":
				item, _ = json.Marshal(ListItem{
					Name:         strings.Split(info["name"], "-")[0],
					ID:           info["owner"] + "." + k,
					OwnerFprint:  info["owner"],
					Parent:       info["parent"],
					Version:      info["version"],
					Architecture: strings.ToUpper(info["arch"]),
					Size:         fi.Size(),
					Md5Sum:       k,
				})
			case "apt":
				item, _ = json.Marshal(AptItem{
					Name:         info["name"],
					Md5Sum:       info["Md5Sum"],
					Description:  info["Description"],
					Architecture: info["Architecture"],
					Package:      info["Package"],
					Version:      info["Version"],
					Size:         fi.Size(),
				})
			case "raw":
				item, _ = json.Marshal(RawItem{
					Name:    info["name"],
					ID:      k,
					Md5Sum:  k,
					Package: info["package"],
					Version: info["version"],
					Size:    fi.Size(),
				})
			}
			if counter > 1 {
				js = append(js, []byte(",")[0])
			}
			js = append(js, item...)
			if name == strings.Split(info["name"], "-subutai-template")[0] {
				return item
			}
		}
	}
	if counter > 1 {
		js = append([]byte("["), js...)
		js = append(js, []byte("]")[0])
	}
	return js
}
<commit_msg>Fixed multiple output for raw info<commit_after>package download

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"strings"

	"github.com/subutai-io/base/agent/log"
	"github.com/subutai-io/gorjun/db"
)

var (
	path = "/tmp/"
)

type AptItem struct {
	Architecture string `json:"architecture,omitempty"`
	Description  string `json:"description,omitempty"`
	Filename     string `json:"filename,omitempty"`
	Md5Sum       string `json:"md5Sum,omitempty"`
	Name         string `json:"name,omitempty"`
	Package      string `json:"package,omitempty"`
	Version      string `json:"version,omitempty"`
	Size         int64  `json:"size"`
}

type RawItem struct {
	Md5Sum  string `json:"md5Sum,omitempty"`
	Name    string `json:"name,omitempty"`
	Package string `json:"package,omitempty"`
	Version string `json:"version,omitempty"`
	Size    int64  `json:"size"`
	ID      string `json:"id"`
}

type ListItem struct {
	Architecture   string `json:"architecture"`
	ConfigContents string `json:"configContents"`
	Extra          struct {
		Lxc_idMap           string `json:"lxc.id_map"`
		Lxc_include         string `json:"lxc.include"`
		Lxc_mount           string `json:"lxc.mount"`
		Lxc_mount_entry     string `json:"lxc.mount.entry"`
		Lxc_network_flags   string `json:"lxc.network.flags"`
		Lxc_network_hwaddr  string `json:"lxc.network.hwaddr"`
		Lxc_network_link    string `json:"lxc.network.link"`
		Lxc_network_type    string `json:"lxc.network.type"`
		Lxc_rootfs          string `json:"lxc.rootfs"`
		Subutai_config_path string `json:"subutai.config.path"`
		Subutai_git_branch  string `json:"subutai.git.branch"`
	} `json:"extra"`
	ID               string `json:"id"`
	Md5Sum           string `json:"md5Sum"`
	Name             string `json:"name"`
	OwnerFprint      string `json:"ownerFprint"`
	Package          string `json:"package"`
	PackagesContents string `json:"packagesContents"`
	Parent           string `json:"parent"`
	Size             int64  `json:"size"`
	Version          string `json:"version"`
}

func Handler(w http.ResponseWriter, r *http.Request) {
	hash := r.URL.Query().Get("hash")
	name := r.URL.Query().Get("name")
	id := r.URL.Query().Get("id")
	if len(id) > 0 {
		hash = id
		tmp := strings.Split(id, ".")
		if len(tmp) == 2 {
			hash = tmp[1]
		}
	}
	if len(hash) == 0 && len(name) == 0 {
		w.Write([]byte("Please specify hash or name"))
		return
	} else if len(name) != 0 {
		hash = db.LastHash(name)
	}
	w.Header().Set("Content-Disposition", "attachment; filename="+db.Read(hash))
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	f, err := os.Open(path + hash)
	log.Check(log.WarnLevel, "Opening file "+path+hash, err)
	fi, _ := f.Stat()
	w.Header().Set("Content-Length", fmt.Sprint(fi.Size()))
	defer f.Close()
	io.Copy(w, f)
}

func List(repo string, w http.ResponseWriter, r *http.Request) {
	w.Write([]byte(""))
	for k, v := range db.List() {
		if db.Info(k)["type"] == repo {
			w.Write([]byte("
" + v + "
"))
		}
	}
	w.Write([]byte(""))
}

func Search(repo string, w http.ResponseWriter, r *http.Request) {
	query := r.URL.Query().Get("q")
	w.Write([]byte(""))
	for k, v := range db.Search(query) {
		w.Write([]byte("
" + v + "
"))
	}
	w.Write([]byte(""))
}

func Info(repo string, r *http.Request) []byte {
	var item, js []byte
	name := r.URL.Query().Get("name")
	version := r.URL.Query().Get("version")
	rtype := r.URL.Query().Get("type")
	id := r.URL.Query().Get("id")

	ids := strings.Split(id, ".")
	if len(ids) > 1 {
		name = db.Read(ids[1])
	}

	counter := 0
	for k, _ := range db.Search(name) {
		info := db.Info(k)
		if info["type"] == repo {
			counter++
			if rtype == "text" && repo == "template" {
				if strings.HasPrefix(info["name"], name) && (len(version) == 0 || info["version"] == version) {
					return ([]byte("public." + k))
				}
				return []byte("")
			}
			f, err := os.Open(path + k)
			defer f.Close()
			log.Check(log.WarnLevel, "Opening file "+path+k, err)
			fi, _ := f.Stat()
			switch repo {
			case "template":
				item, _ = json.Marshal(ListItem{
					Name:         strings.Split(info["name"], "-")[0],
					ID:           info["owner"] + "." + k,
					OwnerFprint:  info["owner"],
					Parent:       info["parent"],
					Version:      info["version"],
					Architecture: strings.ToUpper(info["arch"]),
					Size:         fi.Size(),
					Md5Sum:       k,
				})
			case "apt":
				item, _ = json.Marshal(AptItem{
					Name:         info["name"],
					Md5Sum:       info["Md5Sum"],
					Description:  info["Description"],
					Architecture: info["Architecture"],
					Package:      info["Package"],
					Version:      info["Version"],
					Size:         fi.Size(),
				})
			case "raw":
				item, _ = json.Marshal(RawItem{
					Name:    info["name"],
					ID:      k,
					Md5Sum:  k,
					Package: info["package"],
					Version: info["version"],
					Size:    fi.Size(),
				})
			}
			if counter > 1 {
				js = append(js, []byte(",")[0])
			}
			js = append(js, item...)
			if name == strings.Split(info["name"], "-subutai-template")[0] || name == info["name"] {
				return item
			}
		}
	}
	if counter > 1 {
		js = append([]byte("["), js...)
		js = append(js, []byte("]")[0])
	}
	return js
}
<|endoftext|>


<commit_before>package global

import (
	"code.cloudfoundry.org/cli/integration/helpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("rename buildpack command", func() {
	BeforeEach(func() {
		Skip("until #158613755 is complete")
	})

	Describe("help", func() {
		Context("when --help flag is set", func() {
			It("Displays command usage to output", func() {
				session := helpers.CF("rename-buildpack", "--help")
				Eventually(session).Should(Say("NAME:"))
				Eventually(session).Should(Say("rename-buildpack - Rename a buildpack"))
				Eventually(session).Should(Say("USAGE:"))
				Eventually(session).Should(Say("cf rename-buildpack BUILDPACK_NAME NEW_BUILDPACK_NAME"))
				Eventually(session).Should(Say("SEE ALSO:"))
				Eventually(session).Should(Say("update-buildpack"))
				Eventually(session).Should(Exit(0))
			})
		})
	})

	Context("when the environment is not setup correctly", func() {
		It("fails with the appropriate errors", func() {
			helpers.CheckEnvironmentTargetedCorrectly(false, false, ReadOnlyOrg, "rename-buildpack", "fake-buildpack", "some-name")
		})
	})

	Context("when the user is logged in", func() {
		var (
			oldBuildpackName      string
			newBuildpackName      string
			existingBuildpackName string
		)

		BeforeEach(func() {
			helpers.LoginCF()
		})

		Context("when the buildpack exists", func() {
			BeforeEach(func() {
				oldBuildpackName = helpers.NewBuildpack()
				newBuildpackName = helpers.NewBuildpack()

				helpers.BuildpackWithStack(func(buildpackPath string) {
					session := helpers.CF("create-buildpack", oldBuildpackName, buildpackPath, "1")
					Eventually(session).Should(Exit(0))
				}, "")
			})

			Context("when renaming to a unique name", func() {
				It("successfully renames buildpack", func() {
					session := helpers.CF("rename-buildpack", oldBuildpackName, newBuildpackName)
					Eventually(session).Should(Say("Renaming buildpack %s to %s", oldBuildpackName, newBuildpackName))
					Eventually(session).Should(Say("OK"))
					Eventually(session).Should(Exit(0))
				})
			})

			Context("when renaming to the same name", func() {
				It("successfully renames buildpack", func() {
					session := helpers.CF("rename-buildpack", oldBuildpackName, oldBuildpackName)
					Eventually(session).Should(Say("Renaming buildpack %s to %s", oldBuildpackName, oldBuildpackName))
					Eventually(session).Should(Say("OK"))
					Eventually(session).Should(Exit(0))
				})
			})

			Context("when renaming to the same name as another buildpack", func() {
				BeforeEach(func() {
					existingBuildpackName = helpers.NewBuildpack()

					helpers.BuildpackWithStack(func(buildpackPath string) {
						session := helpers.CF("create-buildpack", existingBuildpackName, buildpackPath, "1")
						Eventually(session).Should(Exit(0))
					}, "")
				})

				It("returns the appropriate error", func() {
					session := helpers.CF("rename-buildpack", oldBuildpackName, existingBuildpackName)
					Eventually(session).Should(Say("Renaming buildpack %s to %s", oldBuildpackName, existingBuildpackName))
					Eventually(session).Should(Say("FAILED"))
					Eventually(session.Err).Should(Say("Buildpack %s already exists without a stack", existingBuildpackName))
					Eventually(session).Should(Exit(1))
				})
			})
		})

		Context("when there are multiple ambiguous buildpacks", func() {
			BeforeEach(func() {
				helpers.SkipIfOneStack()

				stacks := helpers.FetchStacks()

				oldBuildpackName = helpers.NewBuildpack()
				newBuildpackName = helpers.NewBuildpack()

				helpers.BuildpackWithStack(func(buildpackPath string) {
					session := helpers.CF("create-buildpack", oldBuildpackName, buildpackPath, "1")
					Eventually(session).Should(Exit(0))
				}, stacks[0])
				helpers.BuildpackWithStack(func(buildpackPath string) {
					session := helpers.CF("create-buildpack", oldBuildpackName, buildpackPath, "1")
					Eventually(session).Should(Exit(0))
				}, stacks[1])
			})

			It("returns multiple buildpacks found error", func() {
				session := helpers.CF("rename-buildpack", oldBuildpackName, newBuildpackName)
				Eventually(session).Should(Say("Renaming buildpack %s to %s", oldBuildpackName, newBuildpackName))
				Eventually(session).Should(Say("FAILED"))
				Eventually(session.Err).Should(Say("Multiple buildpacks named %s found.", oldBuildpackName))
				Eventually(session).Should(Exit(1))
			})
		})
	})
})
<commit_msg>I (Nick) acted too fast on previous commit.  The test failure that happened on concourse is not reproducible locally, and this test does not need to be skipped.<commit_after>package global

import (
	"code.cloudfoundry.org/cli/integration/helpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("rename buildpack command", func() {
	Describe("help", func() {
		Context("when --help flag is set", func() {
			It("Displays command usage to output", func() {
				session := helpers.CF("rename-buildpack", "--help")
				Eventually(session).Should(Say("NAME:"))
				Eventually(session).Should(Say("rename-buildpack - Rename a buildpack"))
				Eventually(session).Should(Say("USAGE:"))
				Eventually(session).Should(Say("cf rename-buildpack BUILDPACK_NAME NEW_BUILDPACK_NAME"))
				Eventually(session).Should(Say("SEE ALSO:"))
				Eventually(session).Should(Say("update-buildpack"))
				Eventually(session).Should(Exit(0))
			})
		})
	})

	Context("when the environment is not setup correctly", func() {
		It("fails with the appropriate errors", func() {
			helpers.CheckEnvironmentTargetedCorrectly(false, false, ReadOnlyOrg, "rename-buildpack", "fake-buildpack", "some-name")
		})
	})

	Context("when the user is logged in", func() {
		var (
			oldBuildpackName      string
			newBuildpackName      string
			existingBuildpackName string
		)

		BeforeEach(func() {
			helpers.LoginCF()
		})

		Context("when the buildpack exists", func() {
			BeforeEach(func() {
				oldBuildpackName = helpers.NewBuildpack()
				newBuildpackName = helpers.NewBuildpack()

				helpers.BuildpackWithStack(func(buildpackPath string) {
					session := helpers.CF("create-buildpack", oldBuildpackName, buildpackPath, "1")
					Eventually(session).Should(Exit(0))
				}, "")
			})

			Context("when renaming to a unique name", func() {
				It("successfully renames buildpack", func() {
					session := helpers.CF("rename-buildpack", oldBuildpackName, newBuildpackName)
					Eventually(session).Should(Say("Renaming buildpack %s to %s", oldBuildpackName, newBuildpackName))
					Eventually(session).Should(Say("OK"))
					Eventually(session).Should(Exit(0))
				})
			})

			Context("when renaming to the same name", func() {
				It("successfully renames buildpack", func() {
					session := helpers.CF("rename-buildpack", oldBuildpackName, oldBuildpackName)
					Eventually(session).Should(Say("Renaming buildpack %s to %s", oldBuildpackName, oldBuildpackName))
					Eventually(session).Should(Say("OK"))
					Eventually(session).Should(Exit(0))
				})
			})

			Context("when renaming to the same name as another buildpack", func() {
				BeforeEach(func() {
					existingBuildpackName = helpers.NewBuildpack()

					helpers.BuildpackWithStack(func(buildpackPath string) {
						session := helpers.CF("create-buildpack", existingBuildpackName, buildpackPath, "1")
						Eventually(session).Should(Exit(0))
					}, "")
				})

				It("returns the appropriate error", func() {
					session := helpers.CF("rename-buildpack", oldBuildpackName, existingBuildpackName)
					Eventually(session).Should(Say("Renaming buildpack %s to %s", oldBuildpackName, existingBuildpackName))
					Eventually(session).Should(Say("FAILED"))
					Eventually(session.Err).Should(Say("Buildpack %s already exists without a stack", existingBuildpackName))
					Eventually(session).Should(Exit(1))
				})
			})
		})

		Context("when there are multiple ambiguous buildpacks", func() {
			BeforeEach(func() {
				helpers.SkipIfOneStack()

				stacks := helpers.FetchStacks()

				oldBuildpackName = helpers.NewBuildpack()
				newBuildpackName = helpers.NewBuildpack()

				helpers.BuildpackWithStack(func(buildpackPath string) {
					session := helpers.CF("create-buildpack", oldBuildpackName, buildpackPath, "1")
					Eventually(session).Should(Exit(0))
				}, stacks[0])
				helpers.BuildpackWithStack(func(buildpackPath string) {
					session := helpers.CF("create-buildpack", oldBuildpackName, buildpackPath, "1")
					Eventually(session).Should(Exit(0))
				}, stacks[1])
			})

			It("returns multiple buildpacks found error", func() {
				session := helpers.CF("rename-buildpack", oldBuildpackName, newBuildpackName)
				Eventually(session).Should(Say("Renaming buildpack %s to %s", oldBuildpackName, newBuildpackName))
				Eventually(session).Should(Say("FAILED"))
				Eventually(session.Err).Should(Say("Multiple buildpacks named %s found.", oldBuildpackName))
				Eventually(session).Should(Exit(1))
			})
		})
	})
})
<|endoftext|>


<commit_before>package cache

import (
	"log"
	"sync"
	"time"

	"github.com/micro/go-micro/registry"
	"github.com/micro/go-micro/selector"
)

/*
	Cache selector is a selector which uses the registry.Watcher to Cache service entries.
	It defaults to a TTL for 1 minute and causes a cache miss on the next request.
*/

type cacheSelector struct {
	so  selector.Options
	ttl time.Duration

	// registry cache
	sync.Mutex
	cache map[string][]*registry.Service
	ttls  map[string]time.Time

	// used to close or reload watcher
	reload chan bool
	exit   chan bool
}

var (
	DefaultTTL = time.Minute
)

func (c *cacheSelector) quit() bool {
	select {
	case <-c.exit:
		return true
	default:
		return false
	}
}

// cp copies a service. Because we're caching handing back pointers would
// create a race condition, so we do this instead
// its fast enough
func (c *cacheSelector) cp(current []*registry.Service) []*registry.Service {
	var services []*registry.Service

	for _, service := range current {
		// copy service
		s := new(registry.Service)
		*s = *service

		// copy nodes
		var nodes []*registry.Node
		for _, node := range service.Nodes {
			n := new(registry.Node)
			*n = *node
			nodes = append(nodes, n)
		}
		s.Nodes = nodes

		// copy endpoints
		var eps []*registry.Endpoint
		for _, ep := range service.Endpoints {
			e := new(registry.Endpoint)
			*e = *ep
			eps = append(eps, e)
		}
		s.Endpoints = eps

		// append service
		services = append(services, s)
	}

	return services
}

func (c *cacheSelector) del(service string) {
	delete(c.cache, service)
	delete(c.ttls, service)
}

func (c *cacheSelector) get(service string) ([]*registry.Service, error) {
	c.Lock()
	defer c.Unlock()

	// check the cache first
	services, ok := c.cache[service]
	ttl, kk := c.ttls[service]

	// got results, copy and return
	if ok && len(services) > 0 {
		// only return if its less than the ttl
		if kk && time.Since(ttl) < c.ttl {
			return c.cp(services), nil
		}
	}

	// cache miss or ttl expired

	// now ask the registry
	services, err := c.so.Registry.GetService(service)
	if err != nil {
		return nil, err
	}

	// we didn't have any results so cache
	c.cache[service] = c.cp(services)
	c.ttls[service] = time.Now().Add(c.ttl)
	return services, nil
}

func (c *cacheSelector) set(service string, services []*registry.Service) {
	c.cache[service] = services
	c.ttls[service] = time.Now().Add(c.ttl)
}

func (c *cacheSelector) update(res *registry.Result) {
	if res == nil || res.Service == nil {
		return
	}

	c.Lock()
	defer c.Unlock()

	services, ok := c.cache[res.Service.Name]
	if !ok {
		// we're not going to cache anything
		// unless there was already a lookup
		return
	}

	if len(res.Service.Nodes) == 0 {
		switch res.Action {
		case "delete":
			c.del(res.Service.Name)
		}
		return
	}

	// existing service found
	var service *registry.Service
	var index int
	for i, s := range services {
		if s.Version == res.Service.Version {
			service = s
			index = i
		}
	}

	switch res.Action {
	case "create", "update":
		if service == nil {
			c.set(res.Service.Name, append(services, res.Service))
			return
		}

		// append old nodes to new service
		for _, cur := range service.Nodes {
			var seen bool
			for _, node := range res.Service.Nodes {
				if cur.Id == node.Id {
					seen = true
					break
				}
			}
			if !seen {
				res.Service.Nodes = append(res.Service.Nodes, cur)
			}
		}

		services[index] = res.Service
		c.set(res.Service.Name, services)
	case "delete":
		if service == nil {
			return
		}

		var nodes []*registry.Node

		// filter cur nodes to remove the dead one
		for _, cur := range service.Nodes {
			var seen bool
			for _, del := range res.Service.Nodes {
				if del.Id == cur.Id {
					seen = true
					break
				}
			}
			if !seen {
				nodes = append(nodes, cur)
			}
		}

		// still got nodes, save and return
		if len(nodes) > 0 {
			service.Nodes = nodes
			services[index] = service
			c.set(service.Name, services)
			return
		}

		// zero nodes left

		// only have one thing to delete
		// nuke the thing
		if len(services) == 1 {
			c.del(service.Name)
			return
		}

		// still have more than 1 service
		// check the version and keep what we know
		var srvs []*registry.Service
		for _, s := range services {
			if s.Version != service.Version {
				srvs = append(srvs, s)
			}
		}

		// save
		c.set(service.Name, srvs)
	}
}

// run starts the cache watcher loop
// it creates a new watcher if there's a problem
// reloads the watcher if Init is called
// and returns when Close is called
func (c *cacheSelector) run() {
	go c.tick()

	for {
		// exit early if already dead
		if c.quit() {
			return
		}

		// create new watcher
		w, err := c.so.Registry.Watch()
		if err != nil {
			log.Println(err)
			time.Sleep(time.Second)
			continue
		}

		// watch for events
		if err := c.watch(w); err != nil {
			log.Println(err)
			continue
		}
	}
}

// check cache and expire on each tick
func (c *cacheSelector) tick() {
	t := time.NewTicker(time.Minute)

	for {
		select {
		case <-t.C:
			c.Lock()
			for service, expiry := range c.ttls {
				if d := time.Since(expiry); d > c.ttl {
					// TODO: maybe refresh the cache rather than blowing it away
					c.del(service)
				}
			}
			c.Unlock()
		case <-c.exit:
			return
		}
	}
}

// watch loops the next event and calls update
// it returns if there's an error
func (c *cacheSelector) watch(w registry.Watcher) error {
	defer w.Stop()

	// manage this loop
	go func() {
		// wait for exit or reload signal
		select {
		case <-c.exit:
		case <-c.reload:
		}

		// stop the watcher
		w.Stop()
	}()

	for {
		res, err := w.Next()
		if err != nil {
			return err
		}
		c.update(res)
	}
}

func (c *cacheSelector) Init(opts ...selector.Option) error {
	for _, o := range opts {
		o(&c.so)
	}

	// reload the watcher
	go func() {
		select {
		case <-c.exit:
			return
		default:
			c.reload <- true
		}
	}()

	return nil
}

func (c *cacheSelector) Options() selector.Options {
	return c.so
}

func (c *cacheSelector) Select(service string, opts ...selector.SelectOption) (selector.Next, error) {
	sopts := selector.SelectOptions{
		Strategy: c.so.Strategy,
	}

	for _, opt := range opts {
		opt(&sopts)
	}

	// get the service
	// try the cache first
	// if that fails go directly to the registry
	services, err := c.get(service)
	if err != nil {
		return nil, err
	}

	// apply the filters
	for _, filter := range sopts.Filters {
		services = filter(services)
	}

	// if there's nothing left, return
	if len(services) == 0 {
		return nil, selector.ErrNotFound
	}

	return sopts.Strategy(services), nil
}

func (c *cacheSelector) Mark(service string, node *registry.Node, err error) {
	return
}

func (c *cacheSelector) Reset(service string) {
	c.Lock()
	c.del(service)
	c.Unlock()
}

// Close stops the watcher and destroys the cache
func (c *cacheSelector) Close() error {
	c.Lock()
	c.cache = make(map[string][]*registry.Service)
	c.Unlock()

	select {
	case <-c.exit:
		return nil
	default:
		close(c.exit)
	}
	return nil
}

func (c *cacheSelector) String() string {
	return "cache"
}

func NewSelector(opts ...selector.Option) selector.Selector {
	sopts := selector.Options{
		Strategy: selector.Random,
	}

	for _, opt := range opts {
		opt(&sopts)
	}

	if sopts.Registry == nil {
		sopts.Registry = registry.DefaultRegistry
	}

	ttl := DefaultTTL

	if sopts.Context != nil {
		if t, ok := sopts.Context.Value(ttlKey{}).(time.Duration); ok {
			ttl = t
		}
	}

	c := &cacheSelector{
		so:     sopts,
		ttl:    ttl,
		cache:  make(map[string][]*registry.Service),
		ttls:   make(map[string]time.Time),
		reload: make(chan bool, 1),
		exit:   make(chan bool),
	}

	go c.run()
	return c
}
<commit_msg>Add blacklist to cache<commit_after>package cache

import (
	"log"
	"sync"
	"time"

	"github.com/micro/go-micro/registry"
	"github.com/micro/go-micro/selector"
	"github.com/micro/go-micro/selector/internal/blacklist"
)

/*
	Cache selector is a selector which uses the registry.Watcher to Cache service entries.
	It defaults to a TTL for 1 minute and causes a cache miss on the next request.
*/

type cacheSelector struct {
	so  selector.Options
	ttl time.Duration

	// registry cache
	sync.Mutex
	cache map[string][]*registry.Service
	ttls  map[string]time.Time

	// used to close or reload watcher
	reload chan bool
	exit   chan bool

	// blacklist
	bl *blacklist.BlackList
}

var (
	DefaultTTL = time.Minute
)

func (c *cacheSelector) quit() bool {
	select {
	case <-c.exit:
		return true
	default:
		return false
	}
}

// cp copies a service. Because we're caching handing back pointers would
// create a race condition, so we do this instead
// its fast enough
func (c *cacheSelector) cp(current []*registry.Service) []*registry.Service {
	var services []*registry.Service

	for _, service := range current {
		// copy service
		s := new(registry.Service)
		*s = *service

		// copy nodes
		var nodes []*registry.Node
		for _, node := range service.Nodes {
			n := new(registry.Node)
			*n = *node
			nodes = append(nodes, n)
		}
		s.Nodes = nodes

		// copy endpoints
		var eps []*registry.Endpoint
		for _, ep := range service.Endpoints {
			e := new(registry.Endpoint)
			*e = *ep
			eps = append(eps, e)
		}
		s.Endpoints = eps

		// append service
		services = append(services, s)
	}

	return services
}

func (c *cacheSelector) del(service string) {
	delete(c.cache, service)
	delete(c.ttls, service)
}

func (c *cacheSelector) get(service string) ([]*registry.Service, error) {
	c.Lock()
	defer c.Unlock()

	// check the cache first
	services, ok := c.cache[service]
	ttl, kk := c.ttls[service]

	// got results, copy and return
	if ok && len(services) > 0 {
		// only return if its less than the ttl
		if kk && time.Since(ttl) < c.ttl {
			return c.cp(services), nil
		}
	}

	// cache miss or ttl expired

	// now ask the registry
	services, err := c.so.Registry.GetService(service)
	if err != nil {
		return nil, err
	}

	// we didn't have any results so cache
	c.cache[service] = c.cp(services)
	c.ttls[service] = time.Now().Add(c.ttl)
	return services, nil
}

func (c *cacheSelector) set(service string, services []*registry.Service) {
	c.cache[service] = services
	c.ttls[service] = time.Now().Add(c.ttl)
}

func (c *cacheSelector) update(res *registry.Result) {
	if res == nil || res.Service == nil {
		return
	}

	c.Lock()
	defer c.Unlock()

	services, ok := c.cache[res.Service.Name]
	if !ok {
		// we're not going to cache anything
		// unless there was already a lookup
		return
	}

	if len(res.Service.Nodes) == 0 {
		switch res.Action {
		case "delete":
			c.del(res.Service.Name)
		}
		return
	}

	// existing service found
	var service *registry.Service
	var index int
	for i, s := range services {
		if s.Version == res.Service.Version {
			service = s
			index = i
		}
	}

	switch res.Action {
	case "create", "update":
		if service == nil {
			c.set(res.Service.Name, append(services, res.Service))
			return
		}

		// append old nodes to new service
		for _, cur := range service.Nodes {
			var seen bool
			for _, node := range res.Service.Nodes {
				if cur.Id == node.Id {
					seen = true
					break
				}
			}
			if !seen {
				res.Service.Nodes = append(res.Service.Nodes, cur)
			}
		}

		services[index] = res.Service
		c.set(res.Service.Name, services)
	case "delete":
		if service == nil {
			return
		}

		var nodes []*registry.Node

		// filter cur nodes to remove the dead one
		for _, cur := range service.Nodes {
			var seen bool
			for _, del := range res.Service.Nodes {
				if del.Id == cur.Id {
					seen = true
					break
				}
			}
			if !seen {
				nodes = append(nodes, cur)
			}
		}

		// still got nodes, save and return
		if len(nodes) > 0 {
			service.Nodes = nodes
			services[index] = service
			c.set(service.Name, services)
			return
		}

		// zero nodes left

		// only have one thing to delete
		// nuke the thing
		if len(services) == 1 {
			c.del(service.Name)
			return
		}

		// still have more than 1 service
		// check the version and keep what we know
		var srvs []*registry.Service
		for _, s := range services {
			if s.Version != service.Version {
				srvs = append(srvs, s)
			}
		}

		// save
		c.set(service.Name, srvs)
	}
}

// run starts the cache watcher loop
// it creates a new watcher if there's a problem
// reloads the watcher if Init is called
// and returns when Close is called
func (c *cacheSelector) run() {
	go c.tick()

	for {
		// exit early if already dead
		if c.quit() {
			return
		}

		// create new watcher
		w, err := c.so.Registry.Watch()
		if err != nil {
			log.Println(err)
			time.Sleep(time.Second)
			continue
		}

		// watch for events
		if err := c.watch(w); err != nil {
			log.Println(err)
			continue
		}
	}
}

// check cache and expire on each tick
func (c *cacheSelector) tick() {
	t := time.NewTicker(time.Minute)

	for {
		select {
		case <-t.C:
			c.Lock()
			for service, expiry := range c.ttls {
				if d := time.Since(expiry); d > c.ttl {
					// TODO: maybe refresh the cache rather than blowing it away
					c.del(service)
				}
			}
			c.Unlock()
		case <-c.exit:
			return
		}
	}
}

// watch loops the next event and calls update
// it returns if there's an error
func (c *cacheSelector) watch(w registry.Watcher) error {
	defer w.Stop()

	// manage this loop
	go func() {
		// wait for exit or reload signal
		select {
		case <-c.exit:
		case <-c.reload:
		}

		// stop the watcher
		w.Stop()
	}()

	for {
		res, err := w.Next()
		if err != nil {
			return err
		}
		c.update(res)
	}
}

func (c *cacheSelector) Init(opts ...selector.Option) error {
	for _, o := range opts {
		o(&c.so)
	}

	// reload the watcher
	go func() {
		select {
		case <-c.exit:
			return
		default:
			c.reload <- true
		}
	}()

	return nil
}

func (c *cacheSelector) Options() selector.Options {
	return c.so
}

func (c *cacheSelector) Select(service string, opts ...selector.SelectOption) (selector.Next, error) {
	sopts := selector.SelectOptions{
		Strategy: c.so.Strategy,
	}

	for _, opt := range opts {
		opt(&sopts)
	}

	// get the service
	// try the cache first
	// if that fails go directly to the registry
	services, err := c.get(service)
	if err != nil {
		return nil, err
	}

	// apply the filters
	for _, filter := range sopts.Filters {
		services = filter(services)
	}

	services, err = c.bl.Filter(services)
	if err != nil {
		return nil, err
	}

	// if there's nothing left, return
	if len(services) == 0 {
		return nil, selector.ErrNotFound
	}

	return sopts.Strategy(services), nil
}

func (c *cacheSelector) Mark(service string, node *registry.Node, err error) {
	c.bl.Mark(service, node, err)
}

func (c *cacheSelector) Reset(service string) {
	c.Lock()
	c.del(service)
	c.Unlock()
	c.bl.Reset(service)
}

// Close stops the watcher and destroys the cache
func (c *cacheSelector) Close() error {
	c.Lock()
	c.cache = make(map[string][]*registry.Service)
	c.Unlock()

	select {
	case <-c.exit:
		return nil
	default:
		close(c.exit)
		c.bl.Close()
	}
	return nil
}

func (c *cacheSelector) String() string {
	return "cache"
}

func NewSelector(opts ...selector.Option) selector.Selector {
	sopts := selector.Options{
		Strategy: selector.Random,
	}

	for _, opt := range opts {
		opt(&sopts)
	}

	if sopts.Registry == nil {
		sopts.Registry = registry.DefaultRegistry
	}

	ttl := DefaultTTL

	if sopts.Context != nil {
		if t, ok := sopts.Context.Value(ttlKey{}).(time.Duration); ok {
			ttl = t
		}
	}

	c := &cacheSelector{
		so:     sopts,
		ttl:    ttl,
		cache:  make(map[string][]*registry.Service),
		ttls:   make(map[string]time.Time),
		reload: make(chan bool, 1),
		exit:   make(chan bool),
		bl:     blacklist.New(),
	}

	go c.run()
	return c
}
<|endoftext|>


<commit_before>package config

type Repo struct {
	Path   string   `json:"path"`
	Name   string   `json:"name"`
	Refs   []string `json:"refs"`
	Github string   `json:"github"`
}

type Backend struct {
	Id        string `json:"id"`
	Name      string `json:"name"`
	Addr      string `json:"addr"`
	IndexPath string `json:"index"`
	Repos     []Repo `json:"repos"`
}

type Config struct {
	// Location of the directory containing templates and static
	// assets. This should point at the "web" directory of the
	// repository.
	DocRoot string `json:"docroot"`

	Feedback struct {
		// The mailto address for the "feedback" url.
		Mailto string `json:"mailto"`
	} `json:"feedback"`

	GoogleAnalyticsId string `json:"google_analytics_id"`
	// Should we respect X-Real-Ip, X-Real-Proto, and X-Forwarded-Host?
	ReverseProxy bool `json:"reverse_proxy"`

	// List of backends to connect to. Each backend must minimally
	// include the "id" and "addr" fields; All other fields are
	// optional and will be replaced by values reported by the
	// backend server once we successfully connect.
	Backends []Backend `json:"backends"`
}
<commit_msg>MailTo, not Mailto.<commit_after>package config

type Repo struct {
	Path   string   `json:"path"`
	Name   string   `json:"name"`
	Refs   []string `json:"refs"`
	Github string   `json:"github"`
}

type Backend struct {
	Id        string `json:"id"`
	Name      string `json:"name"`
	Addr      string `json:"addr"`
	IndexPath string `json:"index"`
	Repos     []Repo `json:"repos"`
}

type Config struct {
	// Location of the directory containing templates and static
	// assets. This should point at the "web" directory of the
	// repository.
	DocRoot string `json:"docroot"`

	Feedback struct {
		// The mailto address for the "feedback" url.
		MailTo string `json:"mailto"`
	} `json:"feedback"`

	GoogleAnalyticsId string `json:"google_analytics_id"`
	// Should we respect X-Real-Ip, X-Real-Proto, and X-Forwarded-Host?
	ReverseProxy bool `json:"reverse_proxy"`

	// List of backends to connect to. Each backend must minimally
	// include the "id" and "addr" fields; All other fields are
	// optional and will be replaced by values reported by the
	// backend server once we successfully connect.
	Backends []Backend `json:"backends"`
}
<|endoftext|>


<commit_before>package endpoint

import (
	"errors"

	"github.com/golang/protobuf/proto"
	"github.com/vinceprignano/bunny/transport"
)

type JsonEndpoint struct {
	EndpointName string
	Transport    transport.Transport
	Handler      func(delivery *transport.Request) (proto.Message, error)
}

func (j *JsonEndpoint) Name() string {
	return j.EndpointName
}

func (j *JsonEndpoint) HandleRequest(req *transport.Request) ([]byte, error) {
	res, err := j.Handler(req)
	if err != nil {
		return nil, errors.New("Failed")
	}
	return proto.Marshal(res)
}
<commit_msg>Fix json return<commit_after>package endpoint

import (
	"errors"

	"github.com/vinceprignano/bunny/transport"
)

type JsonEndpoint struct {
	EndpointName string
	Transport    transport.Transport
	Handler      func(delivery *transport.Request) ([]byte, error)
}

func (j *JsonEndpoint) Name() string {
	return j.EndpointName
}

func (j *JsonEndpoint) HandleRequest(req *transport.Request) ([]byte, error) {
	res, err := j.Handler(req)
	if err != nil {
		return nil, errors.New("Failed")
	}
	return res
}
<|endoftext|>


<commit_before>// Copyright 2016,2017,2018 Yaacov Zamir <kobi.zamir@gmail.com>
// and other contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package apiErrors API errors
package apiErrors

import (
	"fmt"
	"net/http"
)

// ErrBadMetricID a new error with bad metrics id message
var ErrBadMetricID = fmt.Errorf("Bad metrics ID")

// Error represent an error that occurred and its status code.
type Error struct {
	code    int
	message string
}

func (e Error) Error() string {
	return e.message
}

// JSON writes Error as JSON to http.ResponseWriter
func (e Error) JSON(w http.ResponseWriter) {
	w.WriteHeader(e.code)
	w.Write([]byte(fmt.Sprintf(`{"code":"%d","message":"%s"}`, e.code, e.message)))
}

// BadRequest creates new Error with status code 400 from error.
func BadRequest(err error) Error {
	return Error{code: http.StatusBadRequest, message: err.Error()}
}

// InternalError creates new Error with status code 500 from error.
func InternalError(err error) Error {
	return Error{code: http.StatusInternalServerError, message: err.Error()}
}
<commit_msg>use errors explicitly<commit_after>// Copyright 2016,2017,2018 Yaacov Zamir <kobi.zamir@gmail.com>
// and other contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package apiErrors API errors
package apiErrors

import (
	"errors"
	"fmt"
	"net/http"
)

// ErrBadMetricID a new error with bad metrics id message
var ErrBadMetricID = errors.New("Bad metrics ID")

// Error represent an error that occurred and its status code.
type Error struct {
	code    int
	message string
}

func (e Error) Error() string {
	return e.message
}

// JSON writes Error as JSON to http.ResponseWriter
func (e Error) JSON(w http.ResponseWriter) {
	w.WriteHeader(e.code)
	w.Write([]byte(fmt.Sprintf(`{"code":"%d","message":"%s"}`, e.code, e.message)))
}

// BadRequest creates new Error with status code 400 from error.
func BadRequest(err error) Error {
	return Error{code: http.StatusBadRequest, message: err.Error()}
}

// InternalError creates new Error with status code 500 from error.
func InternalError(err error) Error {
	return Error{code: http.StatusInternalServerError, message: err.Error()}
}
<|endoftext|>


<commit_before>package router

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	"golang.org/x/net/context"

	"github.com/AlexanderChen1989/xrest"
	"github.com/AlexanderChen1989/xrest/plugs/hello"
)

func TestRouter(t *testing.T) {

	router := New()

	sr := router.SubRouter("/api")

	auth := sr.SubRouter("/auth")
	noauth := sr.SubRouter("/noauth")

	auth.Plug(hello.NewHelloHandler("Hello1"))
	auth.Plug(hello.NewHelloHandler("Hello2"))
	noauth.Plug(hello.NewHelloHandler("World1"))
	noauth.Plug(hello.NewHelloHandler("World2"))

	auth.Get("/files", xrest.HandleFunc(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
		fmt.Println("Auth files Done!")
	}))
	noauth.Post("/login", xrest.HandleFunc(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
		fmt.Println("Notauth login Done!")
	}))

	authReq, _ := http.NewRequest("GET", "/api/auth/files", nil)
	noauthReq, _ := http.NewRequest("POST", "/api/noauth/login", nil)

	pipe := xrest.NewPipeline().Plug(router)

	w := httptest.NewRecorder()

	pipe.HTTPHandler().ServeHTTP(w, authReq)
	pipe.HTTPHandler().ServeHTTP(w, noauthReq)
}
<commit_msg>fix router test<commit_after>package router

import (
	"net/http"
	"testing"

	"golang.org/x/net/context"

	"github.com/AlexanderChen1989/xrest"
	"github.com/stretchr/testify/assert"
)

type testPlug struct {
	name string
	next xrest.Handler
}

func newTestPlug(name string) *testPlug {
	return &testPlug{name: name}
}

func (plug *testPlug) Plug(h xrest.Handler) xrest.Handler {
	plug.next = h
	return plug
}

var ctxTestPlug uint8

func (plug *testPlug) ServeHTTP(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	vals, ok := ctx.Value(&ctxTestPlug).([]string)
	if !ok {
		vals = []string{}
	}
	vals = append(vals, plug.name)
	ctx = context.WithValue(ctx, &ctxTestPlug, vals)
	plug.next.ServeHTTP(ctx, w, r)
}

func TestRouter(t *testing.T) {

	router := New()

	sr := router.SubRouter("/api")

	auth := sr.SubRouter("/auth")
	noauth := sr.SubRouter("/noauth")

	authVals := []string{"Hello1", "Hello2", "Hello3"}
	for _, val := range authVals {
		auth.Plug(newTestPlug(val))
	}

	noauthVals := []string{"World1", "World2", "World3"}
	for _, val := range noauthVals {
		noauth.Plug(newTestPlug(val))
	}

	auth.Get("/files", xrest.HandleFunc(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
		assert.EqualValues(t, authVals, ctx.Value(&ctxTestPlug))
	}))
	noauth.Post("/login", xrest.HandleFunc(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
		assert.EqualValues(t, noauthVals, ctx.Value(&ctxTestPlug))
	}))

	authReq, _ := http.NewRequest("GET", "/api/auth/files", nil)
	noauthReq, _ := http.NewRequest("POST", "/api/noauth/login", nil)

	pipe := xrest.NewPipeline().Plug(router)

	pipe.HTTPHandler().ServeHTTP(nil, authReq)
	pipe.HTTPHandler().ServeHTTP(nil, noauthReq)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/mrsaints/go-nx/nx"
	"log"
)

func main() {
	nxf, err := nx.NewFile("../../data/Base.nx", true)
	failOnError(err)

	nxh, err := nxf.Header()
	failOnError(err)
	log.Printf("File header: %+v", nxh)

	root, err := nxf.Root()
	failOnError(err)
	log.Printf("Root: %+v", root)
	c, err := root.Children()
	failOnError(err)
	log.Printf("Root's children: %+v", c)

	//nd, err := nx.NewNode(nxf, 0)
	//failOnError(err)
	//err = nd.Parse()
	//failOnError(err)
	//log.Printf("Node: %+v", nd)
	//c, err := nd.Children()
	//failOnError(err)
	//log.Printf("Node's children: %+v", c)

	// Blocking operation
	printChildren(c)

	log.Printf("Total nodes: %+v", nxh.NodeCount)
}

func printChildren(c *nx.Children) {
	for i, nd := range c.Nodes {
		log.Printf("%+v: %+v | %+v | %+v", i, nd.Id, nd.Name, nd.Type)

		if nd.Count > 0 {
			c2, err := nd.Children()
			failOnError(err)
			printChildren(c2)
		}
	}
}

func failOnError(err error) {
	if err != nil {
		log.Fatalln(err)
	}
}
<commit_msg>Fixed example<commit_after>package main

import (
	"github.com/mrsaints/go-nx/nx"
	"log"
)

func main() {
	nxf, err := nx.NewFile("../../data/Base.nx", true)
	failOnError(err)

	nxh, err := nxf.Header()
	failOnError(err)
	log.Printf("File header: %+v", nxh)

	root, err := nxf.Root()
	failOnError(err)
	log.Printf("Root: %+v", root)
	c, err := root.Children()
	failOnError(err)
	log.Printf("Root's children: %+v", c)

	//nd, err := nx.NewNode(nxf, 0)
	//failOnError(err)
	//err = nd.Parse()
	//failOnError(err)
	//log.Printf("Node: %+v", nd)
	//c, err := nd.Children()
	//failOnError(err)
	//log.Printf("Node's children: %+v", c)

	// Blocking operation
	printChildren(c)

	log.Printf("Total nodes: %+v", nxh.NodeCount)
}

func printChildren(c *nx.Children) {
	for i, nd := range c.Nodes {
		log.Printf("%+v: %+v | %+v | %+v", i, nd.ID, nd.Name, nd.Type)

		if nd.Count > 0 {
			c2, err := nd.Children()
			failOnError(err)
			printChildren(c2)
		}
	}
}

func failOnError(err error) {
	if err != nil {
		log.Fatalln(err)
	}
}
<|endoftext|>


<commit_before>//go:generate bitfanDoc
// Example
// ```
// input{
//   webhook{
//         uri => "toto/titi"
//         pipeline=> "test.conf"
//         codec => plain{
//             role => "decoder"
//         }
//         codec => plain{
//             role => "encoder"
//             format=> "<h1>Hello {{.request.querystring.name}}</h1>"
//         }
//         headers => {
//             "Content-Type" => "text/html"
//         }
//     }
// }
// ```
package webfan

import (
	"io"
	"net/http"
	"net/http/httputil"
	"sync"

	"github.com/vjeantet/bitfan/codecs"
	"github.com/vjeantet/bitfan/core"
	"github.com/vjeantet/bitfan/entrypoint"
	"github.com/vjeantet/bitfan/processors"
)

func New() processors.Processor {
	return &processor{opt: &options{}}
}

type options struct {
	processors.CommonOptions `mapstructure:",squash"`

	// The codec used for posted data. Input codecs are a convenient method for decoding
	// your data before it enters the pipeline, without needing a separate filter in your bitfan pipeline
	//
	// Default decode http request as plain text, response is json encoded.
	// Set multiple codec with role to customize
	// @ExampleLS codec => plain { role=>"encoder"} codec => json { role=>"decoder"}
	// @Type codec
	Codec codecs.CodecCollection

	// URI path /_/path
	Uri string `mapstructure:"uri" validate:"required"`

	// Path to pipeline's configuration to execute on request
	// This configuration should contains only a filter section an a output like ```output{pass{}}```
	Pipeline string `mapstructure:"pipeline" validate:"required"`

	// Headers to send back into outgoing response
	// @ExampleLS {"X-Processor" => "bitfan"}
	Headers map[string]string `mapstructure:"headers"`
}

// Reads events from standard input
type processor struct {
	processors.Base

	opt *options
	wg  *sync.WaitGroup
	ep  *entrypoint.Entrypoint
}

func (p *processor) Configure(ctx processors.ProcessorContext, conf map[string]interface{}) error {
	defaults := options{}
	p.opt = &defaults
	err := p.ConfigureAndValidate(ctx, conf, p.opt)
	if err != nil {
		return err
	}

	if p.opt.Codec.Enc == nil {
		p.opt.Codec.Enc = codecs.New("json", nil, ctx.Log(), ctx.ConfigWorkingLocation())
	}

	if p.opt.Codec.Dec == nil && p.opt.Codec.Default == nil {
		p.opt.Codec.Dec = codecs.New("plain", nil, ctx.Log(), ctx.ConfigWorkingLocation())
	}

	return err
}
func (p *processor) Start(e processors.IPacket) error {
	p.wg = &sync.WaitGroup{}
	p.WebHook.AddShort(p.opt.Uri, p.HttpHandler)

	var err error
	p.ep, err = entrypoint.New(p.opt.Pipeline, p.ConfigWorkingLocation, entrypoint.CONTENT_REF)
	if err != nil {
		p.Logger.Errorf("Error with entrypoint %s", p.opt.Pipeline)
	}

	return err
}

// Handle Request received by bitfan for this agent (url hook should be registered during p.Start)
func (p *processor) HttpHandler(w http.ResponseWriter, r *http.Request) {
	p.wg.Add(1)
	defer p.wg.Done()
	p.Logger.Debug("reading request")

	// Build Pipeline
	ppl, err := p.ep.Pipeline()
	if err != nil {
		p.Logger.Errorf("%s", err)
		w.WriteHeader(http.StatusInternalServerError)
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.Write([]byte(err.Error()))
		return
	}

	// pp.Println("ppl-->", ppl)

	orderedAgentConfList := core.Sort(ppl.Agents(), core.SortInputsFirst)

	// Find Last Agent
	firstAgent := orderedAgentConfList[0]
	lastAgent := orderedAgentConfList[len(orderedAgentConfList)-1]

	// When no pass output set, add one
	if lastAgent.Label != "pass" {
		lastEp, _ := entrypoint.New("output{pass{}}", firstAgent.Wd, entrypoint.CONTENT_INLINE)
		lastPpl, _ := lastEp.Pipeline()
		for _, a := range lastPpl.Agents() {
			a.AgentSources = core.PortList{core.Port{AgentID: lastAgent.ID, PortNumber: 0}}
			ppl.AddAgent(*a)
			lastAgent = a
			break
		}
	}

	back := make(chan processors.IPacket)
	lastAgent.Options["chan"] = back

	// Create a reader
	var dec codecs.Decoder
	if dec, err = p.opt.Codec.NewDecoder(r.Body); err != nil {
		p.Logger.Errorln("decoder error : ", err.Error())
		close(back)
		return
	}
	// Create a writer
	var enc codecs.Encoder
	enc, err = p.opt.Codec.NewEncoder(w)
	if err != nil {
		p.Logger.Errorln("codec error : ", err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.Write([]byte(err.Error()))
		return
	}

	done := make(chan bool)
	go func(back chan processors.IPacket) {
		defer close(done)
		firstPass := true
		for e := range back {
			if firstPass {
				w.Header().Set("Content-Type", "text/plain; charset=utf-8")
				for hn, hv := range p.opt.Headers {
					w.Header().Set(hn, hv)
					p.Logger.Debugf("added header : %s -> %s", hn, hv)
				}
				w.WriteHeader(http.StatusAccepted)
				firstPass = false
			}
			enc.Encode(e.Fields().Old())
			w.(http.Flusher).Flush()
		}
	}(back)

	_, err = ppl.Start()
	if err != nil {
		p.Logger.Errorf("Can not start webfan request pipeline : %s", err)
		w.WriteHeader(http.StatusInternalServerError)
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.Write([]byte(err.Error()))
		close(back) // will close done chan
		return
	}

	headersBytes, _ := httputil.DumpRequest(r, false)
	headers := string(headersBytes)
	req := map[string]interface{}{
		"remoteAddr":  r.RemoteAddr,
		"rawHeaders":  headers,
		"method":      r.Method,
		"requestURI":  r.RequestURI,
		"proto":       r.Proto,
		"host":        r.Host,
		"requestPath": r.URL.Path,
	}

	req["querystring"] = map[string]interface{}{}
	for i, v := range r.URL.Query() {
		if len(v) == 1 {
			req["querystring"].(map[string]interface{})[i] = v[0]
		} else {
			req["querystring"].(map[string]interface{})[i] = v
		}
	}

	req["headers"] = map[string]interface{}{}
	for i, v := range r.Header {
		if len(v) == 1 {
			req["headers"].(map[string]interface{})[i] = v[0]
		} else {
			req["headers"].(map[string]interface{})[i] = v
		}
	}

	if r.Method == "POST" {
		r.ParseForm()
		req["formvalues"] = map[string]interface{}{}
		for i, v := range r.PostForm {
			if len(v) == 1 {
				req["formvalues"].(map[string]interface{})[i] = v[0]
			} else {
				req["formvalues"].(map[string]interface{})[i] = v
			}
		}
	}

	p.Logger.Debug("request = ", req)
	p.Logger.Debug("start reading body content")

	for dec.More() {
		var record interface{}

		if err = dec.Decode(&record); err != nil {
			if err == io.EOF {
				p.Logger.Debugln("error while http read docoding : ", err)
			} else {
				p.Logger.Errorln("error while http read docoding : ", err)
				w.WriteHeader(http.StatusInternalServerError)
				w.Header().Set("Content-Type", "text/plain; charset=utf-8")
				w.Write([]byte(err.Error()))
				close(back)
				break
			}
		}

		var e processors.IPacket
		switch v := record.(type) {
		case nil:
			e = p.NewPacket(map[string]interface{}{
				"request": req,
			})
		case string:
			e = p.NewPacket(map[string]interface{}{
				"message": v,
				"request": req,
			})
		case map[string]interface{}:
			e = p.NewPacket(v)
			e.Fields().SetValueForPath(req, "request")
		case []interface{}:
			e = p.NewPacket(map[string]interface{}{
				"request": req,
				"data":    v,
			})
		default:
			p.Logger.Errorf("Unknow structure %#v", v)
		}

		p.opt.ProcessCommonOptions(e.Fields())

		firstAgent.Processor().Receive(e)
	}

	ppl.Stop()
	<-done
}

func (p *processor) Stop(e processors.IPacket) error {
	p.wg.Wait()
	return nil
}
<commit_msg>processors: webfan - pass received event to next processor in main pipeline<commit_after>//go:generate bitfanDoc
// Example
// ```
// input{
//   webhook{
//         uri => "toto/titi"
//         pipeline=> "test.conf"
//         codec => plain{
//             role => "decoder"
//         }
//         codec => plain{
//             role => "encoder"
//             format=> "<h1>Hello {{.request.querystring.name}}</h1>"
//         }
//         headers => {
//             "Content-Type" => "text/html"
//         }
//     }
// }
// ```
package webfan

import (
	"io"
	"net/http"
	"net/http/httputil"
	"sync"

	"github.com/vjeantet/bitfan/codecs"
	"github.com/vjeantet/bitfan/core"
	"github.com/vjeantet/bitfan/entrypoint"
	"github.com/vjeantet/bitfan/processors"
)

func New() processors.Processor {
	return &processor{opt: &options{}}
}

type options struct {
	processors.CommonOptions `mapstructure:",squash"`

	// The codec used for posted data. Input codecs are a convenient method for decoding
	// your data before it enters the pipeline, without needing a separate filter in your bitfan pipeline
	//
	// Default decode http request as plain text, response is json encoded.
	// Set multiple codec with role to customize
	// @ExampleLS codec => plain { role=>"encoder"} codec => json { role=>"decoder"}
	// @Type codec
	Codec codecs.CodecCollection

	// URI path /_/path
	Uri string `mapstructure:"uri" validate:"required"`

	// Path to pipeline's configuration to execute on request
	// This configuration should contains only a filter section an a output like ```output{pass{}}```
	Pipeline string `mapstructure:"pipeline" validate:"required"`

	// Headers to send back into outgoing response
	// @ExampleLS {"X-Processor" => "bitfan"}
	Headers map[string]string `mapstructure:"headers"`
}

// Reads events from standard input
type processor struct {
	processors.Base

	opt *options
	wg  *sync.WaitGroup
	ep  *entrypoint.Entrypoint
}

func (p *processor) Configure(ctx processors.ProcessorContext, conf map[string]interface{}) error {
	defaults := options{}
	p.opt = &defaults
	err := p.ConfigureAndValidate(ctx, conf, p.opt)
	if err != nil {
		return err
	}

	if p.opt.Codec.Enc == nil {
		p.opt.Codec.Enc = codecs.New("json", nil, ctx.Log(), ctx.ConfigWorkingLocation())
	}

	if p.opt.Codec.Dec == nil && p.opt.Codec.Default == nil {
		p.opt.Codec.Dec = codecs.New("plain", nil, ctx.Log(), ctx.ConfigWorkingLocation())
	}

	return err
}
func (p *processor) Start(e processors.IPacket) error {
	p.wg = &sync.WaitGroup{}
	p.WebHook.AddShort(p.opt.Uri, p.HttpHandler)

	var err error
	p.ep, err = entrypoint.New(p.opt.Pipeline, p.ConfigWorkingLocation, entrypoint.CONTENT_REF)
	if err != nil {
		p.Logger.Errorf("Error with entrypoint %s", p.opt.Pipeline)
	}

	return err
}

// Handle Request received by bitfan for this agent (url hook should be registered during p.Start)
func (p *processor) HttpHandler(w http.ResponseWriter, r *http.Request) {
	p.wg.Add(1)
	defer p.wg.Done()
	p.Logger.Debug("reading request")

	// Build Pipeline
	ppl, err := p.ep.Pipeline()
	if err != nil {
		p.Logger.Errorf("%s", err)
		w.WriteHeader(http.StatusInternalServerError)
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.Write([]byte(err.Error()))
		return
	}

	// pp.Println("ppl-->", ppl)

	orderedAgentConfList := core.Sort(ppl.Agents(), core.SortInputsFirst)

	// Find Last Agent
	firstAgent := orderedAgentConfList[0]
	lastAgent := orderedAgentConfList[len(orderedAgentConfList)-1]

	// When no pass output set, add one
	if lastAgent.Label != "pass" {
		lastEp, _ := entrypoint.New("output{pass{}}", firstAgent.Wd, entrypoint.CONTENT_INLINE)
		lastPpl, _ := lastEp.Pipeline()
		for _, a := range lastPpl.Agents() {
			a.AgentSources = core.PortList{core.Port{AgentID: lastAgent.ID, PortNumber: 0}}
			ppl.AddAgent(*a)
			lastAgent = a
			break
		}
	}

	back := make(chan processors.IPacket)
	lastAgent.Options["chan"] = back

	// Create a reader
	var dec codecs.Decoder
	if dec, err = p.opt.Codec.NewDecoder(r.Body); err != nil {
		p.Logger.Errorln("decoder error : ", err.Error())
		close(back)
		return
	}
	// Create a writer
	var enc codecs.Encoder
	enc, err = p.opt.Codec.NewEncoder(w)
	if err != nil {
		p.Logger.Errorln("codec error : ", err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.Write([]byte(err.Error()))
		return
	}

	done := make(chan bool)
	go func(back chan processors.IPacket) {
		defer close(done)
		firstPass := true
		for e := range back {
			if firstPass {
				w.Header().Set("Content-Type", "text/plain; charset=utf-8")
				for hn, hv := range p.opt.Headers {
					w.Header().Set(hn, hv)
					p.Logger.Debugf("added header : %s -> %s", hn, hv)
				}
				w.WriteHeader(http.StatusAccepted)
				firstPass = false
			}
			enc.Encode(e.Fields().Old())
			w.(http.Flusher).Flush()
		}
	}(back)

	_, err = ppl.Start()
	if err != nil {
		p.Logger.Errorf("Can not start webfan request pipeline : %s", err)
		w.WriteHeader(http.StatusInternalServerError)
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.Write([]byte(err.Error()))
		close(back) // will close done chan
		return
	}

	headersBytes, _ := httputil.DumpRequest(r, false)
	headers := string(headersBytes)
	req := map[string]interface{}{
		"remoteAddr":  r.RemoteAddr,
		"rawHeaders":  headers,
		"method":      r.Method,
		"requestURI":  r.RequestURI,
		"proto":       r.Proto,
		"host":        r.Host,
		"requestPath": r.URL.Path,
	}

	req["querystring"] = map[string]interface{}{}
	for i, v := range r.URL.Query() {
		if len(v) == 1 {
			req["querystring"].(map[string]interface{})[i] = v[0]
		} else {
			req["querystring"].(map[string]interface{})[i] = v
		}
	}

	req["headers"] = map[string]interface{}{}
	for i, v := range r.Header {
		if len(v) == 1 {
			req["headers"].(map[string]interface{})[i] = v[0]
		} else {
			req["headers"].(map[string]interface{})[i] = v
		}
	}

	if r.Method == "POST" {
		r.ParseForm()
		req["formvalues"] = map[string]interface{}{}
		for i, v := range r.PostForm {
			if len(v) == 1 {
				req["formvalues"].(map[string]interface{})[i] = v[0]
			} else {
				req["formvalues"].(map[string]interface{})[i] = v
			}
		}
	}

	p.Logger.Debug("request = ", req)
	p.Logger.Debug("start reading body content")

	for dec.More() {
		var record interface{}

		if err = dec.Decode(&record); err != nil {
			if err == io.EOF {
				p.Logger.Debugln("error while http read docoding : ", err)
			} else {
				p.Logger.Errorln("error while http read docoding : ", err)
				w.WriteHeader(http.StatusInternalServerError)
				w.Header().Set("Content-Type", "text/plain; charset=utf-8")
				w.Write([]byte(err.Error()))
				close(back)
				break
			}
		}

		var e processors.IPacket
		switch v := record.(type) {
		case nil:
			e = p.NewPacket(map[string]interface{}{
				"request": req,
			})
		case string:
			e = p.NewPacket(map[string]interface{}{
				"message": v,
				"request": req,
			})
		case map[string]interface{}:
			e = p.NewPacket(v)
			e.Fields().SetValueForPath(req, "request")
		case []interface{}:
			e = p.NewPacket(map[string]interface{}{
				"request": req,
				"data":    v,
			})
		default:
			p.Logger.Errorf("Unknow structure %#v", v)
		}

		p.opt.ProcessCommonOptions(e.Fields())

		firstAgent.Processor().Receive(e)
		p.Send(e)
	}

	ppl.Stop()
	<-done
}

func (p *processor) Stop(e processors.IPacket) error {
	p.wg.Wait()
	return nil
}
<|endoftext|>


<commit_before>package diagnostic

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"sync"
	"time"

	"crypto/rand"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"

	net "github.com/jbenet/go-ipfs/net"
	msg "github.com/jbenet/go-ipfs/net/message"
	peer "github.com/jbenet/go-ipfs/peer"
	util "github.com/jbenet/go-ipfs/util"
)

var log = util.Logger("diagnostics")

const ResponseTimeout = time.Second * 10

type Diagnostics struct {
	network net.Network
	sender  net.Sender
	self    *peer.Peer

	diagLock sync.Mutex
	diagMap  map[string]time.Time
	birth    time.Time
}

func NewDiagnostics(self *peer.Peer, inet net.Network, sender net.Sender) *Diagnostics {
	return &Diagnostics{
		network: inet,
		sender:  sender,
		self:    self,
		diagMap: make(map[string]time.Time),
		birth:   time.Now(),
	}
}

type connDiagInfo struct {
	Latency time.Duration
	ID      string
}

type diagInfo struct {
	ID          string
	Connections []connDiagInfo
	Keys        []string
	LifeSpan    time.Duration
	CodeVersion string
}

func (di *diagInfo) Marshal() []byte {
	b, err := json.Marshal(di)
	if err != nil {
		panic(err)
	}
	//TODO: also consider compressing this. There will be a lot of these
	return b
}

func (d *Diagnostics) getPeers() []*peer.Peer {
	return d.network.GetPeerList()
}

func (d *Diagnostics) getDiagInfo() *diagInfo {
	di := new(diagInfo)
	di.CodeVersion = "github.com/jbenet/go-ipfs"
	di.ID = d.self.ID.Pretty()
	di.LifeSpan = time.Since(d.birth)
	di.Keys = nil // Currently no way to query datastore

	for _, p := range d.getPeers() {
		di.Connections = append(di.Connections, connDiagInfo{p.GetLatency(), p.ID.Pretty()})
	}
	return di
}

func newID() string {
	id := make([]byte, 4)
	rand.Read(id)
	return string(id)
}

func (d *Diagnostics) GetDiagnostic(timeout time.Duration) ([]*diagInfo, error) {
	log.Debug("Getting diagnostic.")
	ctx, _ := context.WithTimeout(context.TODO(), timeout)

	diagID := newID()
	d.diagLock.Lock()
	d.diagMap[diagID] = time.Now()
	d.diagLock.Unlock()

	log.Debug("Begin Diagnostic")

	peers := d.getPeers()
	log.Debug("Sending diagnostic request to %d peers.", len(peers))

	var out []*diagInfo
	di := d.getDiagInfo()
	out = append(out, di)

	pmes := newMessage(diagID)

	respdata := make(chan []byte)
	sends := 0
	for _, p := range peers {
		log.Debug("Sending getDiagnostic to: %s", p)
		sends++
		go func(p *peer.Peer) {
			data, err := d.getDiagnosticFromPeer(ctx, p, pmes)
			if err != nil {
				log.Error("GetDiagnostic error: %v", err)
				respdata <- nil
				return
			}
			respdata <- data
		}(p)
	}

	for i := 0; i < sends; i++ {
		data := <-respdata
		if data == nil {
			continue
		}
		out = AppendDiagnostics(data, out)
	}
	return out, nil
}

func AppendDiagnostics(data []byte, cur []*diagInfo) []*diagInfo {
	buf := bytes.NewBuffer(data)
	dec := json.NewDecoder(buf)
	for {
		di := new(diagInfo)
		err := dec.Decode(di)
		if err != nil {
			if err != io.EOF {
				log.Error("error decoding diagInfo: %v", err)
			}
			break
		}
		cur = append(cur, di)
	}
	return cur
}

// TODO: this method no longer needed.
func (d *Diagnostics) getDiagnosticFromPeer(ctx context.Context, p *peer.Peer, mes *Message) ([]byte, error) {
	rpmes, err := d.sendRequest(ctx, p, mes)
	if err != nil {
		return nil, err
	}
	return rpmes.GetData(), nil
}

func newMessage(diagID string) *Message {
	pmes := new(Message)
	pmes.DiagID = proto.String(diagID)
	return pmes
}

func (d *Diagnostics) sendRequest(ctx context.Context, p *peer.Peer, pmes *Message) (*Message, error) {

	mes, err := msg.FromObject(p, pmes)
	if err != nil {
		return nil, err
	}

	start := time.Now()

	rmes, err := d.sender.SendRequest(ctx, mes)
	if err != nil {
		return nil, err
	}
	if rmes == nil {
		return nil, errors.New("no response to request")
	}

	rtt := time.Since(start)
	log.Info("diagnostic request took: %s", rtt.String())

	rpmes := new(Message)
	if err := proto.Unmarshal(rmes.Data(), rpmes); err != nil {
		return nil, err
	}

	return rpmes, nil
}

func (d *Diagnostics) handleDiagnostic(p *peer.Peer, pmes *Message) (*Message, error) {
	resp := newMessage(pmes.GetDiagID())
	d.diagLock.Lock()
	_, found := d.diagMap[pmes.GetDiagID()]
	if found {
		d.diagLock.Unlock()
		return resp, nil
	}
	d.diagMap[pmes.GetDiagID()] = time.Now()
	d.diagLock.Unlock()

	buf := new(bytes.Buffer)
	di := d.getDiagInfo()
	buf.Write(di.Marshal())

	ctx, _ := context.WithTimeout(context.TODO(), ResponseTimeout)

	respdata := make(chan []byte)
	sendcount := 0
	for _, p := range d.getPeers() {
		log.Debug("Sending diagnostic request to peer: %s", p)
		go func(p *peer.Peer) {
			out, err := d.getDiagnosticFromPeer(ctx, p, pmes)
			if err != nil {
				log.Error("getDiagnostic error: %v", err)
				respdata <- nil
				return
			}
			respdata <- out
		}(p)
	}

	for i := 0; i < sendcount; i++ {
		out := <-respdata
		_, err := buf.Write(out)
		if err != nil {
			log.Error("getDiagnostic write output error: %v", err)
			continue
		}
	}

	resp.Data = buf.Bytes()
	return resp, nil
}

func (d *Diagnostics) HandleMessage(ctx context.Context, mes msg.NetMessage) msg.NetMessage {
	mData := mes.Data()
	if mData == nil {
		log.Error("message did not include Data")
		return nil
	}

	mPeer := mes.Peer()
	if mPeer == nil {
		log.Error("message did not include a Peer")
		return nil
	}

	// deserialize msg
	pmes := new(Message)
	err := proto.Unmarshal(mData, pmes)
	if err != nil {
		log.Error("Failed to decode protobuf message: %v", err)
		return nil
	}

	// Print out diagnostic
	log.Info("[peer: %s] Got message from [%s]\n",
		d.self.ID.Pretty(), mPeer.ID.Pretty())

	// dispatch handler.
	rpmes, err := d.handleDiagnostic(mPeer, pmes)
	if err != nil {
		log.Error("handleDiagnostic error: %s", err)
		return nil
	}

	// if nil response, return it before serializing
	if rpmes == nil {
		return nil
	}

	// serialize response msg
	rmes, err := msg.FromObject(mPeer, rpmes)
	if err != nil {
		log.Error("Failed to encode protobuf message: %v", err)
		return nil
	}

	return rmes
}
<commit_msg>add diagnostic logging<commit_after>package diagnostic

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"sync"
	"time"

	"crypto/rand"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"

	net "github.com/jbenet/go-ipfs/net"
	msg "github.com/jbenet/go-ipfs/net/message"
	peer "github.com/jbenet/go-ipfs/peer"
	util "github.com/jbenet/go-ipfs/util"
)

var log = util.Logger("diagnostics")

const ResponseTimeout = time.Second * 10

type Diagnostics struct {
	network net.Network
	sender  net.Sender
	self    *peer.Peer

	diagLock sync.Mutex
	diagMap  map[string]time.Time
	birth    time.Time
}

func NewDiagnostics(self *peer.Peer, inet net.Network, sender net.Sender) *Diagnostics {
	return &Diagnostics{
		network: inet,
		sender:  sender,
		self:    self,
		diagMap: make(map[string]time.Time),
		birth:   time.Now(),
	}
}

type connDiagInfo struct {
	Latency time.Duration
	ID      string
}

type diagInfo struct {
	ID          string
	Connections []connDiagInfo
	Keys        []string
	LifeSpan    time.Duration
	CodeVersion string
}

func (di *diagInfo) Marshal() []byte {
	b, err := json.Marshal(di)
	if err != nil {
		panic(err)
	}
	//TODO: also consider compressing this. There will be a lot of these
	return b
}

func (d *Diagnostics) getPeers() []*peer.Peer {
	return d.network.GetPeerList()
}

func (d *Diagnostics) getDiagInfo() *diagInfo {
	di := new(diagInfo)
	di.CodeVersion = "github.com/jbenet/go-ipfs"
	di.ID = d.self.ID.Pretty()
	di.LifeSpan = time.Since(d.birth)
	di.Keys = nil // Currently no way to query datastore

	for _, p := range d.getPeers() {
		di.Connections = append(di.Connections, connDiagInfo{p.GetLatency(), p.ID.Pretty()})
	}
	return di
}

func newID() string {
	id := make([]byte, 4)
	rand.Read(id)
	return string(id)
}

func (d *Diagnostics) GetDiagnostic(timeout time.Duration) ([]*diagInfo, error) {
	log.Debug("Getting diagnostic.")
	ctx, _ := context.WithTimeout(context.TODO(), timeout)

	diagID := newID()
	d.diagLock.Lock()
	d.diagMap[diagID] = time.Now()
	d.diagLock.Unlock()

	log.Debug("Begin Diagnostic")

	peers := d.getPeers()
	log.Debug("Sending diagnostic request to %d peers.", len(peers))

	var out []*diagInfo
	di := d.getDiagInfo()
	out = append(out, di)

	pmes := newMessage(diagID)

	respdata := make(chan []byte)
	sends := 0
	for _, p := range peers {
		log.Debug("Sending getDiagnostic to: %s", p)
		sends++
		go func(p *peer.Peer) {
			data, err := d.getDiagnosticFromPeer(ctx, p, pmes)
			if err != nil {
				log.Error("GetDiagnostic error: %v", err)
				respdata <- nil
				return
			}
			respdata <- data
		}(p)
	}

	for i := 0; i < sends; i++ {
		data := <-respdata
		if data == nil {
			continue
		}
		out = AppendDiagnostics(data, out)
	}
	return out, nil
}

func AppendDiagnostics(data []byte, cur []*diagInfo) []*diagInfo {
	buf := bytes.NewBuffer(data)
	dec := json.NewDecoder(buf)
	for {
		di := new(diagInfo)
		err := dec.Decode(di)
		if err != nil {
			if err != io.EOF {
				log.Error("error decoding diagInfo: %v", err)
			}
			break
		}
		cur = append(cur, di)
	}
	return cur
}

// TODO: this method no longer needed.
func (d *Diagnostics) getDiagnosticFromPeer(ctx context.Context, p *peer.Peer, mes *Message) ([]byte, error) {
	rpmes, err := d.sendRequest(ctx, p, mes)
	if err != nil {
		return nil, err
	}
	return rpmes.GetData(), nil
}

func newMessage(diagID string) *Message {
	pmes := new(Message)
	pmes.DiagID = proto.String(diagID)
	return pmes
}

func (d *Diagnostics) sendRequest(ctx context.Context, p *peer.Peer, pmes *Message) (*Message, error) {

	mes, err := msg.FromObject(p, pmes)
	if err != nil {
		return nil, err
	}

	start := time.Now()

	rmes, err := d.sender.SendRequest(ctx, mes)
	if err != nil {
		return nil, err
	}
	if rmes == nil {
		return nil, errors.New("no response to request")
	}

	rtt := time.Since(start)
	log.Info("diagnostic request took: %s", rtt.String())

	rpmes := new(Message)
	if err := proto.Unmarshal(rmes.Data(), rpmes); err != nil {
		return nil, err
	}

	return rpmes, nil
}

func (d *Diagnostics) handleDiagnostic(p *peer.Peer, pmes *Message) (*Message, error) {
	log.Debug("HandleDiagnostic from %s for id = %s", p, pmes.GetDiagID())
	resp := newMessage(pmes.GetDiagID())
	d.diagLock.Lock()
	_, found := d.diagMap[pmes.GetDiagID()]
	if found {
		d.diagLock.Unlock()
		return resp, nil
	}
	d.diagMap[pmes.GetDiagID()] = time.Now()
	d.diagLock.Unlock()

	buf := new(bytes.Buffer)
	di := d.getDiagInfo()
	buf.Write(di.Marshal())

	ctx, _ := context.WithTimeout(context.TODO(), ResponseTimeout)

	respdata := make(chan []byte)
	sendcount := 0
	for _, p := range d.getPeers() {
		log.Debug("Sending diagnostic request to peer: %s", p)
		go func(p *peer.Peer) {
			out, err := d.getDiagnosticFromPeer(ctx, p, pmes)
			if err != nil {
				log.Error("getDiagnostic error: %v", err)
				respdata <- nil
				return
			}
			respdata <- out
		}(p)
	}

	for i := 0; i < sendcount; i++ {
		out := <-respdata
		_, err := buf.Write(out)
		if err != nil {
			log.Error("getDiagnostic write output error: %v", err)
			continue
		}
	}

	resp.Data = buf.Bytes()
	return resp, nil
}

func (d *Diagnostics) HandleMessage(ctx context.Context, mes msg.NetMessage) msg.NetMessage {
	mData := mes.Data()
	if mData == nil {
		log.Error("message did not include Data")
		return nil
	}

	mPeer := mes.Peer()
	if mPeer == nil {
		log.Error("message did not include a Peer")
		return nil
	}

	// deserialize msg
	pmes := new(Message)
	err := proto.Unmarshal(mData, pmes)
	if err != nil {
		log.Error("Failed to decode protobuf message: %v", err)
		return nil
	}

	// Print out diagnostic
	log.Info("[peer: %s] Got message from [%s]\n",
		d.self.ID.Pretty(), mPeer.ID.Pretty())

	// dispatch handler.
	rpmes, err := d.handleDiagnostic(mPeer, pmes)
	if err != nil {
		log.Error("handleDiagnostic error: %s", err)
		return nil
	}

	// if nil response, return it before serializing
	if rpmes == nil {
		return nil
	}

	// serialize response msg
	rmes, err := msg.FromObject(mPeer, rpmes)
	if err != nil {
		log.Error("Failed to encode protobuf message: %v", err)
		return nil
	}

	return rmes
}
<|endoftext|>