<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package ssh

import (
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/Unknwon/com"
	"golang.org/x/crypto/ssh"
	log "gopkg.in/clog.v1"

	"github.com/gogits/gogs/models"
	"github.com/gogits/gogs/pkg/setting"
)

func cleanCommand(cmd string) string {
	i := strings.Index(cmd, "git")
	if i == -1 {
		return cmd
	}
	return cmd[i:]
}

func handleServerConn(keyID string, chans <-chan ssh.NewChannel) {
	for newChan := range chans {
		if newChan.ChannelType() != "session" {
			newChan.Reject(ssh.UnknownChannelType, "unknown channel type")
			continue
		}

		ch, reqs, err := newChan.Accept()
		if err != nil {
			log.Error(3, "Error accepting channel: %v", err)
			continue
		}

		go func(in <-chan *ssh.Request) {
			defer ch.Close()
			for req := range in {
				payload := cleanCommand(string(req.Payload))
				switch req.Type {
				case "env":
					args := strings.Split(strings.Replace(payload, "\x00", "", -1), "\v")
					if len(args) != 2 {
						log.Warn("SSH: Invalid env arguments: '%#v'", args)
						continue
					}
					args[0] = strings.TrimLeft(args[0], "\x04")
					_, _, err := com.ExecCmdBytes("env", args[0]+"="+args[1])
					if err != nil {
						log.Error(3, "env: %v", err)
						return
					}
				case "exec":
					cmdName := strings.TrimLeft(payload, "'()")
					log.Trace("SSH: Payload: %v", cmdName)

					args := []string{"serv", "key-" + keyID, "--config=" + setting.CustomConf}
					log.Trace("SSH: Arguments: %v", args)
					cmd := exec.Command(setting.AppPath, args...)
					cmd.Env = append(os.Environ(), "SSH_ORIGINAL_COMMAND="+cmdName)

					stdout, err := cmd.StdoutPipe()
					if err != nil {
						log.Error(3, "SSH: StdoutPipe: %v", err)
						return
					}
					stderr, err := cmd.StderrPipe()
					if err != nil {
						log.Error(3, "SSH: StderrPipe: %v", err)
						return
					}
					input, err := cmd.StdinPipe()
					if err != nil {
						log.Error(3, "SSH: StdinPipe: %v", err)
						return
					}

					// FIXME: check timeout
					if err = cmd.Start(); err != nil {
						log.Error(3, "SSH: Start: %v", err)
						return
					}

					req.Reply(true, nil)
					go io.Copy(input, ch)
					io.Copy(ch, stdout)
					io.Copy(ch.Stderr(), stderr)

					if err = cmd.Wait(); err != nil {
						log.Error(3, "SSH: Wait: %v", err)
						return
					}

					ch.SendRequest("exit-status", false, []byte{0, 0, 0, 0})
					return
				default:
				}
			}
		}(reqs)
	}
}

func listen(config *ssh.ServerConfig, host string, port int) {
	listener, err := net.Listen("tcp", host+":"+com.ToStr(port))
	if err != nil {
		log.Fatal(4, "Fail to start SSH server: %v", err)
	}
	for {
		// Once a ServerConfig has been configured, connections can be accepted.
		conn, err := listener.Accept()
		if err != nil {
			log.Error(3, "SSH: Error accepting incoming connection: %v", err)
			continue
		}

		// Before use, a handshake must be performed on the incoming net.Conn.
		// It must be handled in a separate goroutine,
		// otherwise one user could easily block entire loop.
		// For example, user could be asked to trust server key fingerprint and hangs.
		go func() {
			log.Trace("SSH: Handshaking for %s", conn.RemoteAddr())
			sConn, chans, reqs, err := ssh.NewServerConn(conn, config)
			if err != nil {
				if err == io.EOF {
					log.Warn("SSH: Handshaking was terminated: %v", err)
				} else {
					log.Error(3, "SSH: Error on handshaking: %v", err)
				}
				return
			}

			log.Trace("SSH: Connection from %s (%s)", sConn.RemoteAddr(), sConn.ClientVersion())
			// The incoming Request channel must be serviced.
			go ssh.DiscardRequests(reqs)
			go handleServerConn(sConn.Permissions.Extensions["key-id"], chans)
		}()
	}
}

// Listen starts a SSH server listens on given port.
func Listen(host string, port int, ciphers []string) {
	config := &ssh.ServerConfig{
		Config: ssh.Config{
			Ciphers: ciphers,
		},
		PublicKeyCallback: func(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
			pkey, err := models.SearchPublicKeyByContent(strings.TrimSpace(string(ssh.MarshalAuthorizedKey(key))))
			if err != nil {
				log.Error(3, "SearchPublicKeyByContent: %v", err)
				return nil, err
			}
			return &ssh.Permissions{Extensions: map[string]string{"key-id": com.ToStr(pkey.ID)}}, nil
		},
	}

	keyPath := filepath.Join(setting.AppDataPath, "ssh/gogs.rsa")
	if !com.IsExist(keyPath) {
		os.MkdirAll(filepath.Dir(keyPath), os.ModePerm)
		_, stderr, err := com.ExecCmd(setting.SSH.KeygenPath, "-f", keyPath, "-t", "rsa", "-N", "")
		if err != nil {
			panic(fmt.Sprintf("Fail to generate private key: %v - %s", err, stderr))
		}
		log.Trace("SSH: New private key is generateed: %s", keyPath)
	}

	privateBytes, err := ioutil.ReadFile(keyPath)
	if err != nil {
		panic("SSH: Fail to load private key")
	}
	private, err := ssh.ParsePrivateKey(privateBytes)
	if err != nil {
		panic("SSH: Fail to parse private key")
	}
	config.AddHostKey(private)

	go listen(config, host, port)
}
<commit_msg>[annex] return exit value with go ssh server<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package ssh

import (
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/Unknwon/com"
	"golang.org/x/crypto/ssh"
	log "gopkg.in/clog.v1"

	"github.com/gogits/gogs/models"
	"github.com/gogits/gogs/pkg/setting"
	"syscall"
)

func cleanCommand(cmd string) string {
	i := strings.Index(cmd, "git")
	if i == -1 {
		return cmd
	}
	return cmd[i:]
}

func handleServerConn(keyID string, chans <-chan ssh.NewChannel) {
	for newChan := range chans {
		if newChan.ChannelType() != "session" {
			newChan.Reject(ssh.UnknownChannelType, "unknown channel type")
			continue
		}

		ch, reqs, err := newChan.Accept()
		if err != nil {
			log.Error(3, "Error accepting channel: %v", err)
			continue
		}

		go func(in <-chan *ssh.Request) {
			defer ch.Close()
			for req := range in {
				payload := cleanCommand(string(req.Payload))
				switch req.Type {
				case "env":
					args := strings.Split(strings.Replace(payload, "\x00", "", -1), "\v")
					if len(args) != 2 {
						log.Warn("SSH: Invalid env arguments: '%#v'", args)
						continue
					}
					args[0] = strings.TrimLeft(args[0], "\x04")
					_, _, err := com.ExecCmdBytes("env", args[0]+"="+args[1])
					if err != nil {
						log.Error(3, "env: %v", err)
						return
					}
				case "exec":
					cmdName := strings.TrimLeft(payload, "'()")
					log.Trace("SSH: Payload: %v", cmdName)

					args := []string{"serv", "key-" + keyID, "--config=" + setting.CustomConf}
					log.Trace("SSH: Arguments: %v", args)
					cmd := exec.Command(setting.AppPath, args...)
					cmd.Env = append(os.Environ(), "SSH_ORIGINAL_COMMAND="+cmdName)

					stdout, err := cmd.StdoutPipe()
					if err != nil {
						log.Error(3, "SSH: StdoutPipe: %v", err)
						return
					}
					stderr, err := cmd.StderrPipe()
					if err != nil {
						log.Error(3, "SSH: StderrPipe: %v", err)
						return
					}
					input, err := cmd.StdinPipe()
					if err != nil {
						log.Error(3, "SSH: StdinPipe: %v", err)
						return
					}

					// FIXME: check timeout
					if err = cmd.Start(); err != nil {
						log.Error(3, "SSH: Start: %v", err)
						return
					}

					req.Reply(true, nil)
					go io.Copy(input, ch)
					io.Copy(ch, stdout)
					io.Copy(ch.Stderr(), stderr)

					if err = cmd.Wait(); err != nil {
						log.Error(3, "SSH: Wait: %v", err)
						// Fix 255 default return value error
						if t, ok := err.(*exec.ExitError); ok {
							log.Info("t:%s", t)

							es := t.Sys().(syscall.WaitStatus).ExitStatus()
							if es == 1 {
								ch.SendRequest("exit-status", false, []byte{0, 0, 0, 1})
							}
						}
						return
					}

					ch.SendRequest("exit-status", false, []byte{0, 0, 0, 0})
					return
				default:
				}
			}
		}(reqs)
	}
}

func listen(config *ssh.ServerConfig, host string, port int) {
	listener, err := net.Listen("tcp", host+":"+com.ToStr(port))
	if err != nil {
		log.Fatal(4, "Fail to start SSH server: %v", err)
	}
	for {
		// Once a ServerConfig has been configured, connections can be accepted.
		conn, err := listener.Accept()
		if err != nil {
			log.Error(3, "SSH: Error accepting incoming connection: %v", err)
			continue
		}

		// Before use, a handshake must be performed on the incoming net.Conn.
		// It must be handled in a separate goroutine,
		// otherwise one user could easily block entire loop.
		// For example, user could be asked to trust server key fingerprint and hangs.
		go func() {
			log.Trace("SSH: Handshaking for %s", conn.RemoteAddr())
			sConn, chans, reqs, err := ssh.NewServerConn(conn, config)
			if err != nil {
				if err == io.EOF {
					log.Warn("SSH: Handshaking was terminated: %v", err)
				} else {
					log.Error(3, "SSH: Error on handshaking: %v", err)
				}
				return
			}

			log.Trace("SSH: Connection from %s (%s)", sConn.RemoteAddr(), sConn.ClientVersion())
			// The incoming Request channel must be serviced.
			go ssh.DiscardRequests(reqs)
			go handleServerConn(sConn.Permissions.Extensions["key-id"], chans)
		}()
	}
}

// Listen starts a SSH server listens on given port.
func Listen(host string, port int, ciphers []string) {
	config := &ssh.ServerConfig{
		Config: ssh.Config{
			Ciphers: ciphers,
		},
		PublicKeyCallback: func(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
			pkey, err := models.SearchPublicKeyByContent(strings.TrimSpace(string(ssh.MarshalAuthorizedKey(key))))
			if err != nil {
				log.Error(3, "SearchPublicKeyByContent: %v", err)
				return nil, err
			}
			return &ssh.Permissions{Extensions: map[string]string{"key-id": com.ToStr(pkey.ID)}}, nil
		},
	}

	keyPath := filepath.Join(setting.AppDataPath, "ssh/gogs.rsa")
	if !com.IsExist(keyPath) {
		os.MkdirAll(filepath.Dir(keyPath), os.ModePerm)
		_, stderr, err := com.ExecCmd(setting.SSH.KeygenPath, "-f", keyPath, "-t", "rsa", "-N", "")
		if err != nil {
			panic(fmt.Sprintf("Fail to generate private key: %v - %s", err, stderr))
		}
		log.Trace("SSH: New private key is generateed: %s", keyPath)
	}

	privateBytes, err := ioutil.ReadFile(keyPath)
	if err != nil {
		panic("SSH: Fail to load private key")
	}
	private, err := ssh.ParsePrivateKey(privateBytes)
	if err != nil {
		panic("SSH: Fail to parse private key")
	}
	config.AddHostKey(private)

	go listen(config, host, port)
}
<|endoftext|>


<commit_before>package alertsAPI

// Author: Joseph Herlant
//
// This File contains all the struct definitions needed for the xml parser for
// catchpoint Alert Push API according to the xsd they provide as of 2015-07-06

import (
	"encoding/xml"
)

type NodeThreshold struct {
	TypeId     uint8
	NodeCount  int64
	NodePct    float32
	OrOperator string
}

type InsightDataSource struct {
	id           int `xml:"id,attr"`
	Name         string
	DataSourceId uint8
}

type TriggerInsight struct {
	Indicator  InsightDataSource
	Tracepoint InsightDataSource
}

type TriggerZone struct {
	id   int `xml:"id,attr"`
	Name string
}

type Trigger struct {
	TypeId             uint8
	HistoricalInterval uint16
	OperatorId         uint8
	Warning            float32
	Critical           float32
	RegexMatch         string
	Insight            TriggerInsight
	Zone               TriggerZone
	CompareId          uint8
	CompareModifier    float32
}

type Setting struct {
	AlertGroupId                  int
	AlertGroupItemId              int
	AlertTypeId                   uint8
	AlertSubTypeId                uint8
	AlertGroupItemFilterTypeId    uint8
	AlertGroupItemFilterTypeValue string
	NodeThreshold                 NodeThreshold
	TimeThreshold                 uint16 // Unit = minutes
	Trigger                       Trigger
}

type TimestampData struct {
	Local, Utc string
}

type AlertTimestamp struct {
	Create, ReportInterval TimestampData
	ProcessingUtc          string
}

type AlertTestDetail struct {
	Name          string
	MonitorTypeId uint8
	TypeId        uint8
	Path          string
	Url           string
	ClientName    string
	DivisionName  string
	ProductName   string
}

type AlertNodeTriggered struct {
	Counter, Max, Min      int
	Mean, Median, Trailing float32
}

type AlertNodePageFailure struct {
	ErrorCode, HttpStatusCode int
}

type AlertNodeSuspect struct {
	Url                          string
	ObjectResponse, PageResponse int // Unit = milliseconds
}

type AlertNodeHostFailure struct {
	HostsFailed                 uint32
	WorstHost, WorstHostDetails string
}

type AlertNode struct {
	XmlName                                      xml.Name `xml:"Node"`
	Id                                           int      `xml:"id,attr"`
	Name, IpAddress, RemoteIpAddress, IsCritical string
	TransactionStepIndex                         uint8
	ProbableCauseId                              int8
	Counter                                      int
	Mean                                         float32
	Triggered                                    AlertNodeTriggered   // Applies for 'Response', 'ByteLength' and 'Insight (Indicator)' alerts
	PageFailure                                  AlertNodePageFailure // Applies only for 'PageFaliure' alert
	Suspect                                      AlertNodeSuspect     // Applies only for 'ResponseTimeTotalPageLoadWithSuspect alert sub-type
	HostFailure                                  AlertNodeHostFailure // Applies only for 'HostFailure' alert
}

type ConditionRuns struct {
	Detected, Expected int
}

type AlertCondition struct {
	NodeCount                  int64
	NodePct                    float32
	AverageAcrossNodes         float64
	AverageAcrossNodesTrailing float64
	TransactionStepIndex       uint8
	Runs                       ConditionRuns
	Nodes                      []AlertNode `xml:"Nodes>Node"`
}

type PingGroupPacketSent struct {
	Total, Failed  int8
	RoundTripTimes []int `xml:"RoundTripTimes>RoundTripTime"` // Unit = milliseconds (version since Aug-2011)
}

type PingGroup struct {
	V                         uint `xml:"v,attr"`
	Address, Host, Asn        string
	BufferSize, FailureStatus int16
	FromDebugPrimaryHost      string
	Duration                  int // Unit = milliseconds
	PacketsSent               PingGroupPacketSent
}

type TraceRouteGroup struct {
	V                        uint `xml:"v,attr"`
	Address, Host, Timestamp string
	ErrorCode                int8
	FromDebugPrimaryHost     string
	Duration                 int         // Unit = milliseconds
	Hops                     []PingGroup `xml:"Hops>Hop"`
}

type DnsServer struct {
	Address, HostName, Name string
	Port                    uint8
}

type DnsQuery struct {
	XmlName      xml.Name `xml:"Query"`
	V            uint     `xml:"v,attr"`
	Server       DnsServer
	ReturnCode   uint
	ResponseTime int
	ErrorMessage string
	Ping         PingGroup
	TraceRoute   []TraceRouteGroup `xml:"TraceRoute>TraceRouteGroup"`
}

type DnsResponse struct {
	XmlName                xml.Name `xml:"Response"`
	V                      uint     `xml:"v,attr"`
	Name, Info, Address    string
	Ttl                    uint
	Class                  uint8
	Type, InnerResolveTime uint16
	InnerResolveErrorCode  int8
}
type DnsGroup struct {
	XmlName   xml.Name      `xml:"Group"`
	Queries   []DnsQuery    `xml:"Queries>Query"`
	Responses []DnsResponse `xml:"Responses>Response"`
}

type DiagnosticDnsTraversalLastLevel struct {
	// For 'QueryType' enumerations see:  http://www.iana.org/assignments/dns-parameters
	QueryType uint16
	V         uint `xml:"v,attr"`
	level     int8
	Groups    []DnsGroup `xml:"Groups>Group"`
}

type AlertDiagnostic struct {
	Ping                  PingGroup
	TraceRoute            TraceRouteGroup
	DnsTraversalLastLevel DiagnosticDnsTraversalLastLevel
}

type Alert struct {
	XmlName             xml.Name `xml:"Alert"`
	Version             uint     `xml:"version,attr"`
	V                   uint     `xml:"v,attr"`
	TestId              uint64   `xml:"testId,attr"`
	NotificationLevelId uint8    `xml:"notificationLevelId,attr"`
	DivisionId          int      `xml:"divisionId,attr"`
	ProductId           int      `xml:"productId,attr"`
	Setting             Setting
	Timestamp           AlertTimestamp
	TestDetail          AlertTestDetail
	Condition           AlertCondition
	Diagnostic          AlertDiagnostic
}
<commit_msg>Adding some code documentation<commit_after>package alertsAPI

// This File contains all the struct definitions needed for the xml parser for
// catchpoint Alert Push API according to the xsd they provide as of 2015-07-06

import (
	"encoding/xml"
)

type NodeThreshold struct {
	TypeId     uint8
	NodeCount  int64
	NodePct    float32
	OrOperator string
}

type InsightDataSource struct {
	id           int `xml:"id,attr"`
	Name         string
	DataSourceId uint8
}

type TriggerInsight struct {
	Indicator  InsightDataSource
	Tracepoint InsightDataSource
}

type TriggerZone struct {
	id   int `xml:"id,attr"`
	Name string
}

type Trigger struct {
	TypeId             uint8
	HistoricalInterval uint16
	OperatorId         uint8
	Warning            float32
	Critical           float32
	RegexMatch         string
	Insight            TriggerInsight
	Zone               TriggerZone
	CompareId          uint8
	CompareModifier    float32
}

type Setting struct {
	AlertGroupId                  int
	AlertGroupItemId              int
	AlertTypeId                   uint8
	AlertSubTypeId                uint8
	AlertGroupItemFilterTypeId    uint8
	AlertGroupItemFilterTypeValue string
	NodeThreshold                 NodeThreshold
	TimeThreshold                 uint16 // Unit = minutes
	Trigger                       Trigger
}

type TimestampData struct {
	Local, Utc string
}

type AlertTimestamp struct {
	Create, ReportInterval TimestampData
	ProcessingUtc          string
}

type AlertTestDetail struct {
	Name          string
	MonitorTypeId uint8
	TypeId        uint8
	Path          string
	Url           string
	ClientName    string
	DivisionName  string
	ProductName   string
}

type AlertNodeTriggered struct {
	Counter, Max, Min      int
	Mean, Median, Trailing float32
}

type AlertNodePageFailure struct {
	ErrorCode, HttpStatusCode int
}

type AlertNodeSuspect struct {
	Url                          string
	ObjectResponse, PageResponse int // Unit = milliseconds
}

type AlertNodeHostFailure struct {
	HostsFailed                 uint32
	WorstHost, WorstHostDetails string
}

type AlertNode struct {
	XmlName                                      xml.Name `xml:"Node"`
	Id                                           int      `xml:"id,attr"`
	Name, IpAddress, RemoteIpAddress, IsCritical string
	TransactionStepIndex                         uint8
	ProbableCauseId                              int8
	Counter                                      int
	Mean                                         float32
	Triggered                                    AlertNodeTriggered   // Applies for 'Response', 'ByteLength' and 'Insight (Indicator)' alerts
	PageFailure                                  AlertNodePageFailure // Applies only for 'PageFaliure' alert
	Suspect                                      AlertNodeSuspect     // Applies only for 'ResponseTimeTotalPageLoadWithSuspect alert sub-type
	HostFailure                                  AlertNodeHostFailure // Applies only for 'HostFailure' alert
}

type ConditionRuns struct {
	Detected, Expected int
}

type AlertCondition struct {
	NodeCount                  int64
	NodePct                    float32
	AverageAcrossNodes         float64
	AverageAcrossNodesTrailing float64
	TransactionStepIndex       uint8
	Runs                       ConditionRuns
	Nodes                      []AlertNode `xml:"Nodes>Node"`
}

type PingGroupPacketSent struct {
	Total, Failed  int8
	RoundTripTimes []int `xml:"RoundTripTimes>RoundTripTime"` // Unit = milliseconds (version since Aug-2011)
}

type PingGroup struct {
	V                         uint `xml:"v,attr"`
	Address, Host, Asn        string
	BufferSize, FailureStatus int16
	FromDebugPrimaryHost      string
	Duration                  int // Unit = milliseconds
	PacketsSent               PingGroupPacketSent
}

type TraceRouteGroup struct {
	V                        uint `xml:"v,attr"`
	Address, Host, Timestamp string
	ErrorCode                int8
	FromDebugPrimaryHost     string
	Duration                 int         // Unit = milliseconds
	Hops                     []PingGroup `xml:"Hops>Hop"`
}

type DnsServer struct {
	Address, HostName, Name string
	Port                    uint8
}

type DnsQuery struct {
	XmlName      xml.Name `xml:"Query"`
	V            uint     `xml:"v,attr"`
	Server       DnsServer
	ReturnCode   uint
	ResponseTime int
	ErrorMessage string
	Ping         PingGroup
	TraceRoute   []TraceRouteGroup `xml:"TraceRoute>TraceRouteGroup"`
}

type DnsResponse struct {
	XmlName                xml.Name `xml:"Response"`
	V                      uint     `xml:"v,attr"`
	Name, Info, Address    string
	Ttl                    uint
	Class                  uint8
	Type, InnerResolveTime uint16
	InnerResolveErrorCode  int8
}
type DnsGroup struct {
	XmlName   xml.Name      `xml:"Group"`
	Queries   []DnsQuery    `xml:"Queries>Query"`
	Responses []DnsResponse `xml:"Responses>Response"`
}

type DiagnosticDnsTraversalLastLevel struct {
	// For 'QueryType' enumerations see:  http://www.iana.org/assignments/dns-parameters
	QueryType uint16
	V         uint `xml:"v,attr"`
	level     int8
	Groups    []DnsGroup `xml:"Groups>Group"`
}

type AlertDiagnostic struct {
	Ping                  PingGroup
	TraceRoute            TraceRouteGroup
	DnsTraversalLastLevel DiagnosticDnsTraversalLastLevel
}

type Alert struct {
	XmlName             xml.Name `xml:"Alert"`
	Version             uint     `xml:"version,attr"`
	V                   uint     `xml:"v,attr"`
	TestId              uint64   `xml:"testId,attr"`
	NotificationLevelId uint8    `xml:"notificationLevelId,attr"`
	DivisionId          int      `xml:"divisionId,attr"`
	ProductId           int      `xml:"productId,attr"`
	Setting             Setting
	Timestamp           AlertTimestamp
	TestDetail          AlertTestDetail
	Condition           AlertCondition
	Diagnostic          AlertDiagnostic
}
<|endoftext|>


<commit_before>package httpmux

import (
	"fmt"
	"net/http"
	"path"
	"regexp"
	"strings"

	"golang.org/x/net/context"
)

var pathParamContextKey = new(struct{})

type Mux struct {
	handlers []handler
}

func New() *Mux {
	return new(Mux)
}

type handler struct {
	path        *regexp.Regexp
	userHandler http.Handler
}

func (me *Mux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	matches := me.matchingHandlers(r)
	if len(matches) == 0 {
		http.NotFound(w, r)
		return
	}
	m := matches[0]
	r = r.WithContext(context.WithValue(r.Context(), pathParamContextKey, &PathParams{m}))
	m.handler.userHandler.ServeHTTP(w, r)
}

type match struct {
	handler    handler
	submatches []string
}

func (me *Mux) matchingHandlers(r *http.Request) (ret []match) {
	for _, h := range me.handlers {
		subs := h.path.FindStringSubmatch(r.URL.Path)
		if subs == nil {
			continue
		}
		ret = append(ret, match{h, subs})
	}
	return
}

func (me *Mux) distinctHandlerRegexp(r *regexp.Regexp) bool {
	for _, h := range me.handlers {
		if h.path.String() == r.String() {
			return false
		}
	}
	return true
}

func (me *Mux) Handle(path string, h http.Handler) {
	expr := "^" + path
	if !strings.HasSuffix(expr, "$") {
		expr += "$"
	}
	re, err := regexp.Compile(expr)
	if err != nil {
		panic(err)
	}
	if !me.distinctHandlerRegexp(re) {
		panic(fmt.Sprintf("path %q is not distinct", path))
	}
	me.handlers = append(me.handlers, handler{re, h})
}

func (me *Mux) HandleFunc(path string, hf func(http.ResponseWriter, *http.Request)) {
	me.Handle(path, http.HandlerFunc(hf))
}

func Path(parts ...string) string {
	return path.Join(parts...)
}

type PathParams struct {
	match match
}

func (me *PathParams) ByName(name string) string {
	for i, sn := range me.match.handler.path.SubexpNames()[1:] {
		if sn == name {
			return me.match.submatches[i+1]
		}
	}
	return ""
}

func RequestPathParams(r *http.Request) *PathParams {
	ctx := r.Context()
	return ctx.Value(pathParamContextKey).(*PathParams)
}

func PathRegexpParam(name string, re string) string {
	return fmt.Sprintf("(?P<%s>%s)", name, re)
}

func Param(name string) string {
	return fmt.Sprintf("(?P<%s>[^/]+)", name)
}

func RestParam(name string) string {
	return fmt.Sprintf("(?P<%s>.*)$", name)
}

func NonEmptyRestParam(name string) string {
	return fmt.Sprintf("(?P<%s>.+)$", name)
}
<commit_msg>httpmux: Log pattern when handler panics<commit_after>package httpmux

import (
	"fmt"
	"log"
	"net/http"
	"path"
	"regexp"
	"strings"

	"golang.org/x/net/context"
)

var pathParamContextKey = new(struct{})

type Mux struct {
	handlers []handler
}

func New() *Mux {
	return new(Mux)
}

type handler struct {
	path        *regexp.Regexp
	userHandler http.Handler
}

func (me *Mux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	matches := me.matchingHandlers(r)
	if len(matches) == 0 {
		http.NotFound(w, r)
		return
	}
	m := matches[0]
	r = r.WithContext(context.WithValue(r.Context(), pathParamContextKey, &PathParams{m}))
	defer func() {
		r := recover()
		if r == nil {
			return
		}
		log.Printf("panic while handling %v", m)
		panic(r)
	}()
	m.handler.userHandler.ServeHTTP(w, r)
}

type match struct {
	handler    handler
	submatches []string
}

func (me *Mux) matchingHandlers(r *http.Request) (ret []match) {
	for _, h := range me.handlers {
		subs := h.path.FindStringSubmatch(r.URL.Path)
		if subs == nil {
			continue
		}
		ret = append(ret, match{h, subs})
	}
	return
}

func (me *Mux) distinctHandlerRegexp(r *regexp.Regexp) bool {
	for _, h := range me.handlers {
		if h.path.String() == r.String() {
			return false
		}
	}
	return true
}

func (me *Mux) Handle(path string, h http.Handler) {
	expr := "^" + path
	if !strings.HasSuffix(expr, "$") {
		expr += "$"
	}
	re, err := regexp.Compile(expr)
	if err != nil {
		panic(err)
	}
	if !me.distinctHandlerRegexp(re) {
		panic(fmt.Sprintf("path %q is not distinct", path))
	}
	me.handlers = append(me.handlers, handler{re, h})
}

func (me *Mux) HandleFunc(path string, hf func(http.ResponseWriter, *http.Request)) {
	me.Handle(path, http.HandlerFunc(hf))
}

func Path(parts ...string) string {
	return path.Join(parts...)
}

type PathParams struct {
	match match
}

func (me *PathParams) ByName(name string) string {
	for i, sn := range me.match.handler.path.SubexpNames()[1:] {
		if sn == name {
			return me.match.submatches[i+1]
		}
	}
	return ""
}

func RequestPathParams(r *http.Request) *PathParams {
	ctx := r.Context()
	return ctx.Value(pathParamContextKey).(*PathParams)
}

func PathRegexpParam(name string, re string) string {
	return fmt.Sprintf("(?P<%s>%s)", name, re)
}

func Param(name string) string {
	return fmt.Sprintf("(?P<%s>[^/]+)", name)
}

func RestParam(name string) string {
	return fmt.Sprintf("(?P<%s>.*)$", name)
}

func NonEmptyRestParam(name string) string {
	return fmt.Sprintf("(?P<%s>.+)$", name)
}
<|endoftext|>


<commit_before>/* Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"archive/zip"
	"bytes"
	"encoding/xml"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"sort"
	"strings"
)

// Repo is an F-Droid repository holding apps and apks
type Repo struct {
	Apps []App `xml:"application"`
}

type CommaList []string

func (cl *CommaList) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	var content string
	if err := d.DecodeElement(&content, &start); err != nil {
		return err
	}
	*cl = strings.Split(content, ",")
	return nil
}

// App is an Android application
type App struct {
	ID       string    `xml:"id"`
	Name     string    `xml:"name"`
	Summary  string    `xml:"summary"`
	Desc     string    `xml:"desc"`
	License  string    `xml:"license"`
	Categs   CommaList `xml:"categories"`
	Website  string    `xml:"web"`
	Source   string    `xml:"source"`
	Tracker  string    `xml:"tracker"`
	Donate   string    `xml:"donate"`
	Bitcoin  string    `xml:"bitcoin"`
	Litecoin string    `xml:"litecoin"`
	Dogecoin string    `xml:"dogecoin"`
	FlattrID string    `xml:"flattr"`
	Apks     []Apk     `xml:"package"`
	CVName   string    `xml:"marketversion"`
	CVCode   uint      `xml:"marketvercode"`
	CurApk   *Apk
}

// Apk is an Android package
type Apk struct {
	VName  string    `xml:"version"`
	VCode  uint      `xml:"versioncode"`
	Size   int       `xml:"size"`
	MinSdk int       `xml:"sdkver"`
	MaxSdk int       `xml:"maxsdkver"`
	ABIs   CommaList `xml:"nativecode"`
}

func (app *App) calcCurApk() {
	for _, apk := range app.Apks {
		app.CurApk = &apk
		if app.CVCode >= apk.VCode {
			break
		}
	}
}

func (app *App) writeTextDesc(w io.Writer) {
	reader := strings.NewReader(app.Desc)
	decoder := xml.NewDecoder(reader)
	firstParagraph := true
	linePrefix := ""
	colsUsed := 0
	for {
		token, err := decoder.Token()
		if err == io.EOF || token == nil {
			break
		}
		switch t := token.(type) {
		case xml.StartElement:
			switch t.Name.Local {
			case "p":
				if firstParagraph {
					firstParagraph = false
				} else {
					fmt.Fprintln(w)
				}
				linePrefix = ""
				colsUsed = 0
			case "li":
				fmt.Fprint(w, "\n *")
				linePrefix = "   "
				colsUsed = 0
			}
		case xml.EndElement:
			switch t.Name.Local {
			case "p":
				fmt.Fprintln(w)
			case "ul":
				fmt.Fprintln(w)
			case "ol":
				fmt.Fprintln(w)
			}
		case xml.CharData:
			left := string(t)
			limit := 80 - len(linePrefix) - colsUsed
			firstLine := true
			for len(left) > limit {
				last := 0
				for i, c := range left {
					if i >= limit {
						break
					}
					if c == ' ' {
						last = i
					}
				}
				if firstLine {
					firstLine = false
					limit += colsUsed
				} else {
					fmt.Fprint(w, linePrefix)
				}
				fmt.Fprintln(w, left[:last])
				left = left[last+1:]
				colsUsed = 0
			}
			if firstLine {
				firstLine = false
			} else {
				fmt.Fprint(w, linePrefix)
			}
			fmt.Fprint(w, left)
			colsUsed += len(left)
		}
	}
}

func (app *App) prepareData() {
	app.calcCurApk()
}

func (app *App) writeShort(w io.Writer) {
	fmt.Fprintf(w, "%s | %s %s\n", app.ID, app.Name, app.CurApk.VName)
	fmt.Fprintf(w, "    %s\n", app.Summary)
}

func (app *App) writeDetailed(w io.Writer) {
	p := func(title string, format string, args ...interface{}) {
		if format == "" {
			fmt.Fprintln(w, title)
		} else {
			fmt.Fprintf(w, "%s %s\n", title, fmt.Sprintf(format, args...))
		}
	}
	p("Name             :", "%s", app.Name)
	p("Summary          :", "%s", app.Summary)
	p("Current Version  :", "%s (%d)", app.CurApk.VName, app.CurApk.VCode)
	p("Upstream Version :", "%s (%d)", app.CVName, app.CVCode)
	p("License          :", "%s", app.License)
	if app.Categs != nil {
		p("Categories       :", "%s", strings.Join(app.Categs, ", "))
	}
	if app.Website != "" {
		p("Website          :", "%s", app.Website)
	}
	if app.Source != "" {
		p("Source           :", "%s", app.Source)
	}
	if app.Tracker != "" {
		p("Tracker          :", "%s", app.Tracker)
	}
	if app.Donate != "" {
		p("Donate           :", "%s", app.Donate)
	}
	if app.Bitcoin != "" {
		p("Bitcoin          :", "bitcoin:%s", app.Bitcoin)
	}
	if app.Litecoin != "" {
		p("Litecoin         :", "litecoin:%s", app.Litecoin)
	}
	if app.Dogecoin != "" {
		p("Dogecoin         :", "dogecoin:%s", app.Dogecoin)
	}
	if app.FlattrID != "" {
		p("Flattr           :", "https://flattr.com/thing/%s", app.FlattrID)
	}
	fmt.Println()
	p("Description :", "")
	fmt.Println()
	app.writeTextDesc(w)
	fmt.Println()
	p("Available Versions :", "")
	for _, apk := range app.Apks {
		fmt.Println()
		p("    Name   :", "%s (%d)", apk.VName, apk.VCode)
		p("    Size   :", "%d", apk.Size)
		p("    MinSdk :", "%d", apk.MinSdk)
		if apk.MaxSdk > 0 {
			p("    MaxSdk :", "%d", apk.MaxSdk)
		}
		if apk.ABIs != nil {
			p("    ABIs   :", "%s", strings.Join(apk.ABIs, ", "))
		}
	}
}

var ErrNotModified = errors.New("etag matches, file was not modified")

func downloadEtag(url, path string) error {
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)

	etagPath := path + "-etag"
	if _, err := os.Stat(path); err == nil {
		etag, _ := ioutil.ReadFile(etagPath)
		req.Header.Add("If-None-Match", string(etag))
	}

	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusNotModified {
		return ErrNotModified
	}
	jar, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(path, jar, 0644)
	err2 := ioutil.WriteFile(etagPath, []byte(resp.Header["Etag"][0]), 0644)
	if err != nil {
		return err
	}
	if err2 != nil {
		return err2
	}
	return nil
}

const indexName = "index.jar"

func updateIndex() {
	url := fmt.Sprintf("%s/%s", *repoURL, indexName)
	log.Printf("Downloading %s", url)
	err := downloadEtag(url, indexName)
	if err == ErrNotModified {
		log.Printf("Index is already up to date")
	} else if err != nil {
		log.Fatalf("Could not update index: %s", err)
	}
}

func loadApps() map[string]App {
	r, err := zip.OpenReader(indexName)
	if err != nil {
		log.Fatal(err)
	}
	defer r.Close()
	buf := new(bytes.Buffer)

	for _, f := range r.File {
		if f.Name != "index.xml" {
			continue
		}
		rc, err := f.Open()
		if err != nil {
			log.Fatal(err)
		}
		if _, err = io.Copy(buf, rc); err != nil {
			log.Fatal(err)
		}
		rc.Close()
		break
	}

	var repo Repo
	if err := xml.Unmarshal(buf.Bytes(), &repo); err != nil {
		log.Fatalf("Could not read xml: %s", err)
	}
	apps := make(map[string]App)

	for i := range repo.Apps {
		app := repo.Apps[i]
		app.prepareData()
		apps[app.ID] = app
	}
	return apps
}

func appMatches(fields []string, terms []string) bool {
	for _, field := range fields {
		for _, term := range terms {
			if !strings.Contains(field, term) {
				goto next
			}
		}
		return true
	next:
	}
	return false
}

func filterAppsSearch(apps *map[string]App, terms []string) {
	for _, term := range terms {
		term = strings.ToLower(term)
	}
	for appID, app := range *apps {
		fields := []string{
			strings.ToLower(app.ID),
			strings.ToLower(app.Name),
			strings.ToLower(app.Summary),
			strings.ToLower(app.Desc),
		}
		if !appMatches(fields, terms) {
			delete(*apps, appID)
		}
	}
}

type appList []App

func (al appList) Len() int           { return len(al) }
func (al appList) Swap(i, j int)      { al[i], al[j] = al[j], al[i] }
func (al appList) Less(i, j int) bool { return al[i].ID < al[j].ID }

func sortedApps(apps map[string]App) []App {
	list := make(appList, 0, len(apps))
	for appID := range apps {
		list = append(list, apps[appID])
	}
	sort.Sort(list)
	return list
}

var repoURL = flag.String("r", "https://f-droid.org/repo", "repository address")

func init() {
	flag.Usage = func() {
		p := func(args ...interface{}) {
			fmt.Fprintln(os.Stderr, args...)
		}
		p("Usage: fdroidcl [-h] [-r <repo address>] <command> [<args>]")
		p()
		p("Available commands:")
		p("   update           Update the index")
		p("   list             List all available apps")
		p("   search <term...> Search available apps")
		p("   show <appid...>   Show detailed info of an app")
	}
}

func main() {
	flag.Parse()
	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(2)
	}

	cmd := flag.Args()[0]
	args := flag.Args()[1:]

	switch cmd {
	case "update":
		updateIndex()
	case "list":
		apps := loadApps()
		for _, app := range sortedApps(apps) {
			app.writeShort(os.Stdout)
		}
	case "search":
		apps := loadApps()
		filterAppsSearch(&apps, args)
		for _, app := range sortedApps(apps) {
			app.writeShort(os.Stdout)
		}
	case "show":
		apps := loadApps()
		for _, appID := range args {
			app, e := apps[appID]
			if !e {
				log.Fatalf("Could not find app with ID '%s'", appID)
			}
			app.writeDetailed(os.Stdout)
		}
	default:
		fmt.Fprintf(os.Stderr, "Unrecognised command '%s'\n\n", cmd)
		flag.Usage()
		os.Exit(2)
	}
}
<commit_msg>Add link support to description<commit_after>/* Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"archive/zip"
	"bytes"
	"encoding/xml"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"sort"
	"strings"
)

// Repo is an F-Droid repository holding apps and apks
type Repo struct {
	Apps []App `xml:"application"`
}

type CommaList []string

func (cl *CommaList) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	var content string
	if err := d.DecodeElement(&content, &start); err != nil {
		return err
	}
	*cl = strings.Split(content, ",")
	return nil
}

// App is an Android application
type App struct {
	ID       string    `xml:"id"`
	Name     string    `xml:"name"`
	Summary  string    `xml:"summary"`
	Desc     string    `xml:"desc"`
	License  string    `xml:"license"`
	Categs   CommaList `xml:"categories"`
	Website  string    `xml:"web"`
	Source   string    `xml:"source"`
	Tracker  string    `xml:"tracker"`
	Donate   string    `xml:"donate"`
	Bitcoin  string    `xml:"bitcoin"`
	Litecoin string    `xml:"litecoin"`
	Dogecoin string    `xml:"dogecoin"`
	FlattrID string    `xml:"flattr"`
	Apks     []Apk     `xml:"package"`
	CVName   string    `xml:"marketversion"`
	CVCode   uint      `xml:"marketvercode"`
	CurApk   *Apk
}

// Apk is an Android package
type Apk struct {
	VName  string    `xml:"version"`
	VCode  uint      `xml:"versioncode"`
	Size   int       `xml:"size"`
	MinSdk int       `xml:"sdkver"`
	MaxSdk int       `xml:"maxsdkver"`
	ABIs   CommaList `xml:"nativecode"`
}

func (app *App) calcCurApk() {
	for _, apk := range app.Apks {
		app.CurApk = &apk
		if app.CVCode >= apk.VCode {
			break
		}
	}
}

func (app *App) writeTextDesc(w io.Writer) {
	reader := strings.NewReader(app.Desc)
	decoder := xml.NewDecoder(reader)
	firstParagraph := true
	linePrefix := ""
	colsUsed := 0
	var links []string
	linked := false
	for {
		token, err := decoder.Token()
		if err == io.EOF || token == nil {
			break
		}
		switch t := token.(type) {
		case xml.StartElement:
			switch t.Name.Local {
			case "p":
				if firstParagraph {
					firstParagraph = false
				} else {
					fmt.Fprintln(w)
				}
				linePrefix = ""
				colsUsed = 0
			case "li":
				fmt.Fprint(w, "\n *")
				linePrefix = "   "
				colsUsed = 0
			case "a":
				for _, attr := range t.Attr {
					if attr.Name.Local == "href" {
						links = append(links, attr.Value)
						linked = true
						break
					}
				}
			}
		case xml.EndElement:
			switch t.Name.Local {
			case "p":
				fmt.Fprintln(w)
			case "ul":
				fmt.Fprintln(w)
			case "ol":
				fmt.Fprintln(w)
			}
		case xml.CharData:
			left := string(t)
			if linked {
				left += fmt.Sprintf("[%d]", len(links)-1)
				linked = false
			}
			limit := 80 - len(linePrefix) - colsUsed
			firstLine := true
			for len(left) > limit {
				last := 0
				for i, c := range left {
					if i >= limit {
						break
					}
					if c == ' ' {
						last = i
					}
				}
				if firstLine {
					firstLine = false
					limit += colsUsed
				} else {
					fmt.Fprint(w, linePrefix)
				}
				fmt.Fprintln(w, left[:last])
				left = left[last+1:]
				colsUsed = 0
			}
			if firstLine {
				firstLine = false
			} else {
				fmt.Fprint(w, linePrefix)
			}
			fmt.Fprint(w, left)
			colsUsed += len(left)
		}
	}
	if len(links) > 0 {
		fmt.Fprintln(w)
		for i, link := range links {
			fmt.Fprintf(w, "[%d] %s\n", i, link)
		}
	}
}

func (app *App) prepareData() {
	app.calcCurApk()
}

func (app *App) writeShort(w io.Writer) {
	fmt.Fprintf(w, "%s | %s %s\n", app.ID, app.Name, app.CurApk.VName)
	fmt.Fprintf(w, "    %s\n", app.Summary)
}

func (app *App) writeDetailed(w io.Writer) {
	p := func(title string, format string, args ...interface{}) {
		if format == "" {
			fmt.Fprintln(w, title)
		} else {
			fmt.Fprintf(w, "%s %s\n", title, fmt.Sprintf(format, args...))
		}
	}
	p("Name             :", "%s", app.Name)
	p("Summary          :", "%s", app.Summary)
	p("Current Version  :", "%s (%d)", app.CurApk.VName, app.CurApk.VCode)
	p("Upstream Version :", "%s (%d)", app.CVName, app.CVCode)
	p("License          :", "%s", app.License)
	if app.Categs != nil {
		p("Categories       :", "%s", strings.Join(app.Categs, ", "))
	}
	if app.Website != "" {
		p("Website          :", "%s", app.Website)
	}
	if app.Source != "" {
		p("Source           :", "%s", app.Source)
	}
	if app.Tracker != "" {
		p("Tracker          :", "%s", app.Tracker)
	}
	if app.Donate != "" {
		p("Donate           :", "%s", app.Donate)
	}
	if app.Bitcoin != "" {
		p("Bitcoin          :", "bitcoin:%s", app.Bitcoin)
	}
	if app.Litecoin != "" {
		p("Litecoin         :", "litecoin:%s", app.Litecoin)
	}
	if app.Dogecoin != "" {
		p("Dogecoin         :", "dogecoin:%s", app.Dogecoin)
	}
	if app.FlattrID != "" {
		p("Flattr           :", "https://flattr.com/thing/%s", app.FlattrID)
	}
	fmt.Println()
	p("Description :", "")
	fmt.Println()
	app.writeTextDesc(w)
	fmt.Println()
	p("Available Versions :", "")
	for _, apk := range app.Apks {
		fmt.Println()
		p("    Name   :", "%s (%d)", apk.VName, apk.VCode)
		p("    Size   :", "%d", apk.Size)
		p("    MinSdk :", "%d", apk.MinSdk)
		if apk.MaxSdk > 0 {
			p("    MaxSdk :", "%d", apk.MaxSdk)
		}
		if apk.ABIs != nil {
			p("    ABIs   :", "%s", strings.Join(apk.ABIs, ", "))
		}
	}
}

var ErrNotModified = errors.New("etag matches, file was not modified")

func downloadEtag(url, path string) error {
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)

	etagPath := path + "-etag"
	if _, err := os.Stat(path); err == nil {
		etag, _ := ioutil.ReadFile(etagPath)
		req.Header.Add("If-None-Match", string(etag))
	}

	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusNotModified {
		return ErrNotModified
	}
	jar, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(path, jar, 0644)
	err2 := ioutil.WriteFile(etagPath, []byte(resp.Header["Etag"][0]), 0644)
	if err != nil {
		return err
	}
	if err2 != nil {
		return err2
	}
	return nil
}

const indexName = "index.jar"

func updateIndex() {
	url := fmt.Sprintf("%s/%s", *repoURL, indexName)
	log.Printf("Downloading %s", url)
	err := downloadEtag(url, indexName)
	if err == ErrNotModified {
		log.Printf("Index is already up to date")
	} else if err != nil {
		log.Fatalf("Could not update index: %s", err)
	}
}

func loadApps() map[string]App {
	r, err := zip.OpenReader(indexName)
	if err != nil {
		log.Fatal(err)
	}
	defer r.Close()
	buf := new(bytes.Buffer)

	for _, f := range r.File {
		if f.Name != "index.xml" {
			continue
		}
		rc, err := f.Open()
		if err != nil {
			log.Fatal(err)
		}
		if _, err = io.Copy(buf, rc); err != nil {
			log.Fatal(err)
		}
		rc.Close()
		break
	}

	var repo Repo
	if err := xml.Unmarshal(buf.Bytes(), &repo); err != nil {
		log.Fatalf("Could not read xml: %s", err)
	}
	apps := make(map[string]App)

	for i := range repo.Apps {
		app := repo.Apps[i]
		app.prepareData()
		apps[app.ID] = app
	}
	return apps
}

func appMatches(fields []string, terms []string) bool {
	for _, field := range fields {
		for _, term := range terms {
			if !strings.Contains(field, term) {
				goto next
			}
		}
		return true
	next:
	}
	return false
}

func filterAppsSearch(apps *map[string]App, terms []string) {
	for _, term := range terms {
		term = strings.ToLower(term)
	}
	for appID, app := range *apps {
		fields := []string{
			strings.ToLower(app.ID),
			strings.ToLower(app.Name),
			strings.ToLower(app.Summary),
			strings.ToLower(app.Desc),
		}
		if !appMatches(fields, terms) {
			delete(*apps, appID)
		}
	}
}

type appList []App

func (al appList) Len() int           { return len(al) }
func (al appList) Swap(i, j int)      { al[i], al[j] = al[j], al[i] }
func (al appList) Less(i, j int) bool { return al[i].ID < al[j].ID }

func sortedApps(apps map[string]App) []App {
	list := make(appList, 0, len(apps))
	for appID := range apps {
		list = append(list, apps[appID])
	}
	sort.Sort(list)
	return list
}

var repoURL = flag.String("r", "https://f-droid.org/repo", "repository address")

func init() {
	flag.Usage = func() {
		p := func(args ...interface{}) {
			fmt.Fprintln(os.Stderr, args...)
		}
		p("Usage: fdroidcl [-h] [-r <repo address>] <command> [<args>]")
		p()
		p("Available commands:")
		p("   update           Update the index")
		p("   list             List all available apps")
		p("   search <term...> Search available apps")
		p("   show <appid...>   Show detailed info of an app")
	}
}

func main() {
	flag.Parse()
	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(2)
	}

	cmd := flag.Args()[0]
	args := flag.Args()[1:]

	switch cmd {
	case "update":
		updateIndex()
	case "list":
		apps := loadApps()
		for _, app := range sortedApps(apps) {
			app.writeShort(os.Stdout)
		}
	case "search":
		apps := loadApps()
		filterAppsSearch(&apps, args)
		for _, app := range sortedApps(apps) {
			app.writeShort(os.Stdout)
		}
	case "show":
		apps := loadApps()
		for _, appID := range args {
			app, e := apps[appID]
			if !e {
				log.Fatalf("Could not find app with ID '%s'", appID)
			}
			app.writeDetailed(os.Stdout)
		}
	default:
		fmt.Fprintf(os.Stderr, "Unrecognised command '%s'\n\n", cmd)
		flag.Usage()
		os.Exit(2)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The LUCI Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package tumble

import (
	"fmt"
	"math"
	"time"

	ds "github.com/luci/gae/service/datastore"
	"github.com/luci/gae/service/info"
	tq "github.com/luci/gae/service/taskqueue"
	"github.com/luci/luci-go/common/clock"
	"github.com/luci/luci-go/common/errors"
	"github.com/luci/luci-go/common/logging"

	"golang.org/x/net/context"
)

type timestamp int64

const minTS timestamp = math.MinInt64

// TaskNamespace is the namespace used to store and dispatch Tumble task queue
// tasks.
const TaskNamespace = "__tumble"

func (t timestamp) Unix() time.Time {
	return time.Unix((int64)(t), 0).UTC()
}

func mkTimestamp(cfg *Config, t time.Time) timestamp {
	trf := time.Duration(cfg.TemporalRoundFactor)
	eta := t.UTC().Add(time.Duration(cfg.TemporalMinDelay) + trf).Round(trf)
	return timestamp(eta.Unix())
}

type taskShard struct {
	shard uint64
	time  timestamp
}

func fireTasks(c context.Context, cfg *Config, shards map[taskShard]struct{}) bool {
	if len(shards) == 0 {
		return true
	}

	// If namespacing is enabled, Tumble will fire tasks into the Tumble task
	// namespace.
	if cfg.Namespaced {
		c = info.MustNamespace(c, TaskNamespace)
	}

	nextSlot := mkTimestamp(cfg, clock.Now(c).UTC())
	logging.Fields{
		"slot": nextSlot,
	}.Debugf(c, "got next slot")

	tasks := make([]*tq.Task, 0, len(shards))

	for shard := range shards {
		eta := nextSlot
		if cfg.DelayedMutations && shard.time > eta {
			eta = shard.time
		}
		tsk := &tq.Task{
			Name: fmt.Sprintf("%d_%d", eta, shard.shard),

			Path: processURL(eta, shard.shard),

			ETA: eta.Unix(),

			// TODO(riannucci): Tune RetryOptions?
		}
		tasks = append(tasks, tsk)
		logging.Infof(c, "added task %q %s %s", tsk.Name, tsk.Path, tsk.ETA)
	}

	if err := errors.Filter(tq.Add(ds.WithoutTransaction(c), baseName, tasks...), tq.ErrTaskAlreadyAdded); err != nil {
		logging.Warningf(c, "attempted to fire tasks %v, but failed: %s", shards, err)
		return false
	}
	return true
}
<commit_msg>Use batch add for Tumble task queue tasks.<commit_after>// Copyright 2015 The LUCI Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package tumble

import (
	"fmt"
	"math"
	"time"

	ds "github.com/luci/gae/service/datastore"
	"github.com/luci/gae/service/info"
	tq "github.com/luci/gae/service/taskqueue"
	"github.com/luci/luci-go/common/clock"
	"github.com/luci/luci-go/common/errors"
	"github.com/luci/luci-go/common/logging"

	"golang.org/x/net/context"
)

type timestamp int64

const minTS timestamp = math.MinInt64

// TaskNamespace is the namespace used to store and dispatch Tumble task queue
// tasks.
const TaskNamespace = "__tumble"

func (t timestamp) Unix() time.Time {
	return time.Unix((int64)(t), 0).UTC()
}

func mkTimestamp(cfg *Config, t time.Time) timestamp {
	trf := time.Duration(cfg.TemporalRoundFactor)
	eta := t.UTC().Add(time.Duration(cfg.TemporalMinDelay) + trf).Round(trf)
	return timestamp(eta.Unix())
}

type taskShard struct {
	shard uint64
	time  timestamp
}

func fireTasks(c context.Context, cfg *Config, shards map[taskShard]struct{}) bool {
	if len(shards) == 0 {
		return true
	}

	// If namespacing is enabled, Tumble will fire tasks into the Tumble task
	// namespace.
	if cfg.Namespaced {
		c = info.MustNamespace(c, TaskNamespace)
	}

	nextSlot := mkTimestamp(cfg, clock.Now(c).UTC())
	logging.Fields{
		"slot": nextSlot,
	}.Debugf(c, "got next slot")

	tasks := make([]*tq.Task, 0, len(shards))

	for shard := range shards {
		eta := nextSlot
		if cfg.DelayedMutations && shard.time > eta {
			eta = shard.time
		}
		tsk := &tq.Task{
			Name: fmt.Sprintf("%d_%d", eta, shard.shard),

			Path: processURL(eta, shard.shard),

			ETA: eta.Unix(),

			// TODO(riannucci): Tune RetryOptions?
		}
		tasks = append(tasks, tsk)
		logging.Infof(c, "added task %q %s %s", tsk.Name, tsk.Path, tsk.ETA)
	}

	b := tq.Batcher{}
	if err := errors.Filter(b.Add(ds.WithoutTransaction(c), baseName, tasks...), tq.ErrTaskAlreadyAdded); err != nil {
		logging.Warningf(c, "attempted to fire tasks %v, but failed: %s", shards, err)
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package mch

import (
	"time"
	"strconv"
	"fmt"
)

type QueryOrderRequest struct {
	TransactionID string // 微信的订单号，优先使用
	OutTradeNo    string // 商户系统内部的订单号，当没提供transaction_id时需要传这个
}

type OrderInfo struct {
	DeviceInfo     string // 终端设备号
	OpenID         string // 用户在商户appid下的唯一标识
	IsSubscribe    bool   // 用户是否关注公众账号，仅在公众账号类型支付有效
	SubOpenID      string // 用户在子商户appid下的唯一标识
	SubIsSubscribe bool   // 用户是否关注子公众账号，仅在公众账号类型支付有效

	TradeType string
	BankType  string // 银行类型，采用字符串类型的银行标识

	TotalFee           int    // 订单总金额，单位为分
	FeeType            string // 货币类型，符合ISO 4217标准的三位字母代码，默认人民币：CNY
	CashFee            int    // 订单现金支付金额，单位为分
	CashFeeType        string // 货币类型，符合ISO 4217标准的三位字母代码，默认人民币：CNY
	SettlementTotalFee int    // 应结订单金额=订单金额-非充值代金券金额，应结订单金额<=订单金额
	CouponFee          int    // 代金券或立减优惠金额<=订单总金额，订单总金额-代金券或立减优惠金额=现金支付金额
	CouponCount        int    // 代金券或立减优惠使用数量
	// TODO: coupon list

	TransactionID string    // 微信支付订单号
	OutTradeNo    string    // 商户系统内部订单号，要求32个字符内，只能是数字、大小写字母_-|*@ ，且在同一个商户号下唯一
	Attach        string    // 附加数据，在查询API和支付通知中原样返回，该字段主要用于商户携带订单的自定义数据
	TimeEnd       time.Time // 支付完成时间，格式为yyyyMMddHHmmss，如2009年12月25日9点10分10秒表示为20091225091010
}

func getOrderInfo(req map[string]string) (*OrderInfo, error) {
	totalFee, err := strconv.Atoi(req["total_fee"])
	if err != nil {
		return nil, err
	}
	cashFee, err := strconv.Atoi(req["cash_fee"])
	if err != nil {
		return nil, err
	}
	settlementTotalFee, err := strconv.Atoi(req["settlement_total_fee"])
	if err != nil {
		return nil, err
	}
	couponFee, err := strconv.Atoi(req["coupon_fee"])
	if err != nil {
		return nil, err
	}
	couponCount, err := strconv.Atoi(req["coupon_count"])
	if err != nil {
		return nil, err
	}
	timeEnd, err := ParseTime(req["time_end"])
	if err != nil {
		return nil, err
	}

	return &OrderInfo{
		DeviceInfo:         req["device_info"],
		OpenID:             req["openid"],
		IsSubscribe:        req["is_subscribe"] == "Y",
		SubOpenID:          req["sub_openid"],
		SubIsSubscribe:     req["sub_is_subscribe"] == "Y",
		TradeType:          req["trade_type"],
		BankType:           req["bank_type"],
		TotalFee:           totalFee,
		FeeType:            req["fee_type"],
		CashFee:            cashFee,
		CashFeeType:        req["cash_fee_type"],
		SettlementTotalFee: settlementTotalFee,
		CouponFee:          couponFee,
		CouponCount:        couponCount,
		TransactionID:      req["transaction_id"],
		OutTradeNo:         req["out_trade_no"],
		Attach:             req["attach"],
		TimeEnd:            timeEnd,
	}, nil
}

type QueryOrderResponse struct {
	TradeState            // 交易状态
	TradeStateDesc string // 对当前查询订单状态的描述和下一步操作的指引

	OrderInfo

	Detail string // TODO: 商品详细列表
}

func (client *Client) QueryOrder(req *QueryOrderRequest) (rep *QueryOrderResponse, err error) {
	reqMap := make(map[string]string)
	if req.TransactionID != "" {
		reqMap["transaction_id"] = req.TransactionID
	}
	if req.OutTradeNo != "" {
		reqMap["out_trade_no"] = req.OutTradeNo
	}

	repMap, err := client.PostXML("/pay/orderquery", reqMap)

	tradeState := TradeState(repMap["trade_state"])
	if tradeState != TradeStateSUCCESS {
		rep = &QueryOrderResponse{
			TradeState:     tradeState,
			TradeStateDesc: repMap["trade_state_desc"],
			OrderInfo: OrderInfo{
				OutTradeNo: repMap["out_trade_no"],
				Attach:     repMap["attach"],
			},
		}
		return rep, nil
	}

	orderInfo, err := getOrderInfo(repMap)
	if err != nil {
		return nil, err
	}

	rep = &QueryOrderResponse{
		TradeState:     tradeState,
		TradeStateDesc: repMap["trade_state_desc"],
		OrderInfo:      *orderInfo,
		Detail:         repMap["detail"],
	}

	if req.TransactionID != "" && rep.TransactionID != "" && req.TransactionID != rep.TransactionID {
		err = fmt.Errorf("transaction_id mismatch, have: %s, want: %s", rep.TransactionID, req.TransactionID)
		return nil, err
	}
	if req.OutTradeNo != "" && rep.OutTradeNo != "" && req.OutTradeNo != rep.OutTradeNo {
		err = fmt.Errorf("out_trade_no mismatch, have: %s, want: %s", rep.OutTradeNo, req.OutTradeNo)
		return nil, err
	}

	return rep, nil
}
<commit_msg>Update source files<commit_after>package mch

import (
	"time"
	"strconv"
	"fmt"
)

type QueryOrderRequest struct {
	TransactionID string // 微信的订单号，优先使用
	OutTradeNo    string // 商户系统内部的订单号，当没提供transaction_id时需要传这个
}

type OrderInfo struct {
	DeviceInfo     string // 终端设备号
	OpenID         string // 用户在商户appid下的唯一标识
	IsSubscribe    bool   // 用户是否关注公众账号，仅在公众账号类型支付有效
	SubOpenID      string // 用户在子商户appid下的唯一标识
	SubIsSubscribe bool   // 用户是否关注子公众账号，仅在公众账号类型支付有效

	TradeType string
	BankType  string // 银行类型，采用字符串类型的银行标识

	TotalFee           int    // 订单总金额，单位为分
	FeeType            string // 货币类型，符合ISO 4217标准的三位字母代码，默认人民币：CNY
	CashFee            int    // 订单现金支付金额，单位为分
	CashFeeType        string // 货币类型，符合ISO 4217标准的三位字母代码，默认人民币：CNY
	SettlementTotalFee int    // 应结订单金额=订单金额-非充值代金券金额，应结订单金额<=订单金额
	CouponFee          int    // 代金券或立减优惠金额<=订单总金额，订单总金额-代金券或立减优惠金额=现金支付金额
	CouponCount        int    // 代金券或立减优惠使用数量
	// TODO: coupon list

	TransactionID string    // 微信支付订单号
	OutTradeNo    string    // 商户系统内部订单号，要求32个字符内，只能是数字、大小写字母_-|*@ ，且在同一个商户号下唯一
	Attach        string    // 附加数据，在查询API和支付通知中原样返回，该字段主要用于商户携带订单的自定义数据
	TimeEnd       time.Time // 支付完成时间，格式为yyyyMMddHHmmss，如2009年12月25日9点10分10秒表示为20091225091010
}

func getOrderInfo(req map[string]string) (*OrderInfo, error) {
	totalFee, err := strconv.Atoi(req["total_fee"])
	if err != nil {
		return nil, err
	}
	cashFee, err := strconv.Atoi(req["cash_fee"])
	if err != nil {
		return nil, err
	}
	settlementTotalFee, err := strconv.Atoi(req["settlement_total_fee"])
	if err != nil {
		return nil, err
	}
	couponFee, err := strconv.Atoi(req["coupon_fee"])
	if err != nil {
		return nil, err
	}
	couponCount, err := strconv.Atoi(req["coupon_count"])
	if err != nil {
		return nil, err
	}
	timeEnd, err := ParseTime(req["time_end"])
	if err != nil {
		return nil, err
	}

	return &OrderInfo{
		DeviceInfo:         req["device_info"],
		OpenID:             req["openid"],
		IsSubscribe:        req["is_subscribe"] == "Y",
		SubOpenID:          req["sub_openid"],
		SubIsSubscribe:     req["sub_is_subscribe"] == "Y",
		TradeType:          req["trade_type"],
		BankType:           req["bank_type"],
		TotalFee:           totalFee,
		FeeType:            req["fee_type"],
		CashFee:            cashFee,
		CashFeeType:        req["cash_fee_type"],
		SettlementTotalFee: settlementTotalFee,
		CouponFee:          couponFee,
		CouponCount:        couponCount,
		TransactionID:      req["transaction_id"],
		OutTradeNo:         req["out_trade_no"],
		Attach:             req["attach"],
		TimeEnd:            timeEnd,
	}, nil
}

type QueryOrderResponse struct {
	TradeState            // 交易状态
	TradeStateDesc string // 对当前查询订单状态的描述和下一步操作的指引

	OrderInfo

	Detail string // TODO: 商品详细列表
}

func (client *Client) QueryOrder(req *QueryOrderRequest) (rep *QueryOrderResponse, err error) {
	reqMap := make(map[string]string)
	if req.TransactionID != "" {
		reqMap["transaction_id"] = req.TransactionID
	}
	if req.OutTradeNo != "" {
		reqMap["out_trade_no"] = req.OutTradeNo
	}

	repMap, err := client.PostXML("/pay/orderquery", reqMap)
	fmt.Printf("orderquery: %s", repMap)

	tradeState := TradeState(repMap["trade_state"])
	if tradeState != TradeStateSUCCESS {
		rep = &QueryOrderResponse{
			TradeState:     tradeState,
			TradeStateDesc: repMap["trade_state_desc"],
			OrderInfo: OrderInfo{
				OutTradeNo: repMap["out_trade_no"],
				Attach:     repMap["attach"],
			},
		}
		return rep, nil
	}

	orderInfo, err := getOrderInfo(repMap)
	if err != nil {
		return nil, err
	}

	rep = &QueryOrderResponse{
		TradeState:     tradeState,
		TradeStateDesc: repMap["trade_state_desc"],
		OrderInfo:      *orderInfo,
		Detail:         repMap["detail"],
	}

	if req.TransactionID != "" && rep.TransactionID != "" && req.TransactionID != rep.TransactionID {
		err = fmt.Errorf("transaction_id mismatch, have: %s, want: %s", rep.TransactionID, req.TransactionID)
		return nil, err
	}
	if req.OutTradeNo != "" && rep.OutTradeNo != "" && req.OutTradeNo != rep.OutTradeNo {
		err = fmt.Errorf("out_trade_no mismatch, have: %s, want: %s", rep.OutTradeNo, req.OutTradeNo)
		return nil, err
	}

	return rep, nil
}
<|endoftext|>


<commit_before>package crawler

import (
	"bytes"
	"net/url"
	"testing"

	"golang.org/x/net/html"

	. "github.com/smartystreets/goconvey/convey"
)

func helperRunDataExtrator(htmlStr string) *HTMLMetadata {
	baseURL, _ := url.Parse("http://testhost1/test/")
	node, err := html.Parse(bytes.NewReader([]byte(htmlStr)))
	So(err, ShouldBeNil)

	meta, err := RunDataExtrator(node, baseURL)
	So(err, ShouldBeNil)

	return meta
}

func TestURLs(t *testing.T) {
	Convey("TestURLs", t, func() {
		meta := helperRunDataExtrator(`
<link rel="next" href="/link1">
<link rel="prev" href="link2">
<link rel="previous" href="link3/link3">

text

text

text
                        text
                        text

<noindex>
text
</noindex>
text

`)

		expectedURLs := make(map[string]string)
		expectedURLs["http://testhost1/link1"] = "testhost1"
		expectedURLs["http://testhost1/test/link2"] = "testhost1"
		expectedURLs["http://testhost1/test/link3/link3"] = "testhost1"
		expectedURLs["http://testhost1/link4"] = "testhost1"
		expectedURLs["http://testhost1/test/link5"] = "testhost1"
		expectedURLs["http://testhost2/link6"] = "testhost2"
		expectedURLs["http://testhost1/test/link7"] = "testhost1"
		So(meta.URLs, ShouldResemble, expectedURLs)

		expectedWrongURLs := make(map[string]string)
		expectedWrongURLs["/wrong%9"] = `parse /wrong%9: invalid URL escape "%9"`
		So(meta.WrongURLs, ShouldResemble, expectedWrongURLs)
	})

	Convey("TestFrameURLs", t, func() {
		meta := helperRunDataExtrator(`
  <frameset><frame src="link1">
    <frameset>
      <frame src="/link2">
    </frameset>
  </frameset>
`)

		expectedURLs := make(map[string]string)
		expectedURLs["http://testhost1/test/link1"] = "testhost1"
		expectedURLs["http://testhost1/link2"] = "testhost1"
		So(meta.URLs, ShouldResemble, expectedURLs)

		So(len(meta.WrongURLs), ShouldEqual, 0)
	})

	Convey("TestIFrameURLs", t, func() {
		meta := helperRunDataExtrator(`

`)

		expectedURLs := make(map[string]string)
		expectedURLs["http://testhost1/test/link1"] = "testhost1"
		So(meta.URLs, ShouldResemble, expectedURLs)

		So(len(meta.WrongURLs), ShouldEqual, 0)
	})
}

func TestTitle(t *testing.T) {
	Convey("Title as url", t, func() {
		meta := helperRunDataExtrator(`<html><head></head><body><div></div></body></html>`)
		So(meta.Title, ShouldEqual, "http://testhost1/test/")
	})

	Convey("Title from tag title", t, func() {
		meta := helperRunDataExtrator(`<html><head><title>Title text</title></head><body><div></div></body></html>`)
		So(meta.Title, ShouldEqual, "Title text")
	})

	Convey("Title from tag meta", t, func() {
		meta := helperRunDataExtrator(`
<meta name="title" content="Title text">

`)
		So(meta.Title, ShouldEqual, "Title text")
	})

	Convey("Title from tags meta and title", t, func() {
		meta := helperRunDataExtrator(`
<meta name="title" content="Title text 1">
<title>Title text 2</title>

`)
		So(meta.Title, ShouldEqual, "Title text 2")
	})

	Convey("Long title", t, func() {
		meta := helperRunDataExtrator(`
<title>!0000000000111111111122222222223333333333444444444455555555556666666666777777777788888888889999999999</title>

`)
		So(meta.Title, ShouldEqual, "!000000000011111111112222222222333333333344444444445555555555666666666677777777778888888888999999...")
	})
}

func TestMeta(t *testing.T) {
	Convey("NoIndex robots", t, func() {
		meta := helperRunDataExtrator(`
<meta name="robots" content="noIndex">

`)
		So(meta.MetaTagIndex, ShouldEqual, false)
	})

	Convey("NoIndex googlebot", t, func() {
		meta := helperRunDataExtrator(`
<meta name="googlebot" content="noindex">

`)
		So(meta.MetaTagIndex, ShouldEqual, false)
	})

	Convey("NoIndex yandex", t, func() {
		meta := helperRunDataExtrator(`
<meta name="yandex" content="Noindex">

`)
		So(meta.MetaTagIndex, ShouldEqual, true)
	})

	Convey("Index robots", t, func() {
		meta := helperRunDataExtrator(`
<meta name="robots" content="index">

`)
		So(meta.MetaTagIndex, ShouldEqual, true)
	})

	Convey("Nofollow robots", t, func() {
		meta := helperRunDataExtrator(`
<meta name="robots" content="Nofollow">

`)
		So(meta.MetaTagIndex, ShouldEqual, true)
		So(len(meta.URLs), ShouldEqual, 0)
	})

	Convey("Index and Nofollow robots", t, func() {
		meta := helperRunDataExtrator(`
<meta name="robots" content="index, Nofollow">

`)
		So(meta.MetaTagIndex, ShouldEqual, true)
		So(len(meta.URLs), ShouldEqual, 0)
	})

	Convey("None robots", t, func() {
		meta := helperRunDataExtrator(`
<meta name="robots" content="none">

`)
		So(meta.MetaTagIndex, ShouldEqual, false)
		So(len(meta.URLs), ShouldEqual, 0)
	})
}

func TestErrorDataExtrator(t *testing.T) {
	Convey("Test error node type", t, func() {
		baseURL, err := url.Parse("http://testhost1/test/")
		So(err, ShouldBeNil)

		node, err := html.Parse(bytes.NewReader([]byte(`<html><head></head><body></body></html>`)))
		So(err, ShouldBeNil)

		node.FirstChild.Type = html.ErrorNode
		_, err = RunDataExtrator(node, baseURL)
		So(err, ShouldNotBeNil)
		So(err.Error(), ShouldEqual, ErrDataExtractorUnexpectedNodeType.Error())
	})
}
<commit_msg>использовал встроенные функции проверки словаря на пустоту в тестах<commit_after>package crawler

import (
	"bytes"
	"net/url"
	"testing"

	"golang.org/x/net/html"

	. "github.com/smartystreets/goconvey/convey"
)

func helperRunDataExtrator(htmlStr string) *HTMLMetadata {
	baseURL, _ := url.Parse("http://testhost1/test/")
	node, err := html.Parse(bytes.NewReader([]byte(htmlStr)))
	So(err, ShouldBeNil)

	meta, err := RunDataExtrator(node, baseURL)
	So(err, ShouldBeNil)

	return meta
}

func TestURLs(t *testing.T) {
	Convey("TestURLs", t, func() {
		meta := helperRunDataExtrator(`
<link rel="next" href="/link1">
<link rel="prev" href="link2">
<link rel="previous" href="link3/link3">

text

text

text
                        text
                        text

<noindex>
text
</noindex>
text

`)

		expectedURLs := make(map[string]string)
		expectedURLs["http://testhost1/link1"] = "testhost1"
		expectedURLs["http://testhost1/test/link2"] = "testhost1"
		expectedURLs["http://testhost1/test/link3/link3"] = "testhost1"
		expectedURLs["http://testhost1/link4"] = "testhost1"
		expectedURLs["http://testhost1/test/link5"] = "testhost1"
		expectedURLs["http://testhost2/link6"] = "testhost2"
		expectedURLs["http://testhost1/test/link7"] = "testhost1"
		So(meta.URLs, ShouldResemble, expectedURLs)

		expectedWrongURLs := make(map[string]string)
		expectedWrongURLs["/wrong%9"] = `parse /wrong%9: invalid URL escape "%9"`
		So(meta.WrongURLs, ShouldResemble, expectedWrongURLs)
	})

	Convey("TestFrameURLs", t, func() {
		meta := helperRunDataExtrator(`
  <frameset><frame src="link1">
    <frameset>
      <frame src="/link2">
    </frameset>
  </frameset>
`)

		expectedURLs := make(map[string]string)
		expectedURLs["http://testhost1/test/link1"] = "testhost1"
		expectedURLs["http://testhost1/link2"] = "testhost1"
		So(meta.URLs, ShouldResemble, expectedURLs)

		So(meta.WrongURLs, ShouldBeEmpty)
	})

	Convey("TestIFrameURLs", t, func() {
		meta := helperRunDataExtrator(`

`)

		expectedURLs := make(map[string]string)
		expectedURLs["http://testhost1/test/link1"] = "testhost1"
		So(meta.URLs, ShouldResemble, expectedURLs)

		So(meta.WrongURLs, ShouldBeEmpty)
	})
}

func TestTitle(t *testing.T) {
	Convey("Title as url", t, func() {
		meta := helperRunDataExtrator(`<html><head></head><body><div></div></body></html>`)
		So(meta.Title, ShouldEqual, "http://testhost1/test/")
	})

	Convey("Title from tag title", t, func() {
		meta := helperRunDataExtrator(`<html><head><title>Title text</title></head><body><div></div></body></html>`)
		So(meta.Title, ShouldEqual, "Title text")
	})

	Convey("Title from tag meta", t, func() {
		meta := helperRunDataExtrator(`
<meta name="title" content="Title text">

`)
		So(meta.Title, ShouldEqual, "Title text")
	})

	Convey("Title from tags meta and title", t, func() {
		meta := helperRunDataExtrator(`
<meta name="title" content="Title text 1">
<title>Title text 2</title>

`)
		So(meta.Title, ShouldEqual, "Title text 2")
	})

	Convey("Long title", t, func() {
		meta := helperRunDataExtrator(`
<title>!0000000000111111111122222222223333333333444444444455555555556666666666777777777788888888889999999999</title>

`)
		So(meta.Title, ShouldEqual, "!000000000011111111112222222222333333333344444444445555555555666666666677777777778888888888999999...")
	})
}

func TestMeta(t *testing.T) {
	Convey("NoIndex robots", t, func() {
		meta := helperRunDataExtrator(`
<meta name="robots" content="noIndex">

`)
		So(meta.MetaTagIndex, ShouldEqual, false)
	})

	Convey("NoIndex googlebot", t, func() {
		meta := helperRunDataExtrator(`
<meta name="googlebot" content="noindex">

`)
		So(meta.MetaTagIndex, ShouldEqual, false)
	})

	Convey("NoIndex yandex", t, func() {
		meta := helperRunDataExtrator(`
<meta name="yandex" content="Noindex">

`)
		So(meta.MetaTagIndex, ShouldEqual, true)
	})

	Convey("Index robots", t, func() {
		meta := helperRunDataExtrator(`
<meta name="robots" content="index">

`)
		So(meta.MetaTagIndex, ShouldEqual, true)
	})

	Convey("Nofollow robots", t, func() {
		meta := helperRunDataExtrator(`
<meta name="robots" content="Nofollow">

`)
		So(meta.MetaTagIndex, ShouldEqual, true)
		So(meta.URLs, ShouldBeEmpty)
	})

	Convey("Index and Nofollow robots", t, func() {
		meta := helperRunDataExtrator(`
<meta name="robots" content="index, Nofollow">

`)
		So(meta.MetaTagIndex, ShouldEqual, true)
		So(meta.URLs, ShouldBeEmpty)
	})

	Convey("None robots", t, func() {
		meta := helperRunDataExtrator(`
<meta name="robots" content="none">

`)
		So(meta.MetaTagIndex, ShouldEqual, false)
		So(meta.URLs, ShouldBeEmpty)
	})
}

func TestErrorDataExtrator(t *testing.T) {
	Convey("Test error node type", t, func() {
		baseURL, err := url.Parse("http://testhost1/test/")
		So(err, ShouldBeNil)

		node, err := html.Parse(bytes.NewReader([]byte(`<html><head></head><body></body></html>`)))
		So(err, ShouldBeNil)

		node.FirstChild.Type = html.ErrorNode
		_, err = RunDataExtrator(node, baseURL)
		So(err, ShouldNotBeNil)
		So(err.Error(), ShouldEqual, ErrDataExtractorUnexpectedNodeType.Error())
	})
}
<|endoftext|>


<commit_before>package subnets

import (
	"strconv"

	"github.com/racker/perigee"
	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/openstack/utils"
	"github.com/rackspace/gophercloud/pagination"
)

// ListOpts allows the filtering and sorting of paginated collections through
// the API. Filtering is achieved by passing in struct field values that map to
// the subnet attributes you want to see returned. SortKey allows you to sort
// by a particular subnet attribute. SortDir sets the direction, and is either
// `asc' or `desc'. Marker and Limit are used for pagination.
type ListOpts struct {
	Name       string
	EnableDHCP *bool
	NetworkID  string
	TenantID   string
	IPVersion  int
	GatewayIP  string
	CIDR       string
	ID         string
	Limit      int
	Marker     string
	SortKey    string
	SortDir    string
}

// List returns a Pager which allows you to iterate over a collection of
// subnets. It accepts a ListOpts struct, which allows you to filter and sort
// the returned collection for greater efficiency.
//
// Default policy settings return only those subnets that are owned by the tenant
// who submits the request, unless the request is submitted by an user with
// administrative rights.
func List(c *gophercloud.ServiceClient, opts ListOpts) pagination.Pager {
	// Build query parameters
	q := make(map[string]string)
	if opts.Name != "" {
		q["name"] = opts.Name
	}
	if opts.EnableDHCP != nil {
		q["enable_dhcp"] = strconv.FormatBool(*opts.EnableDHCP)
	}
	if opts.NetworkID != "" {
		q["network_id"] = opts.NetworkID
	}
	if opts.TenantID != "" {
		q["tenant_id"] = opts.TenantID
	}
	if opts.IPVersion != 0 {
		q["ip_version"] = strconv.Itoa(opts.IPVersion)
	}
	if opts.GatewayIP != "" {
		q["gateway_ip"] = opts.GatewayIP
	}
	if opts.CIDR != "" {
		q["cidr"] = opts.CIDR
	}
	if opts.ID != "" {
		q["id"] = opts.ID
	}
	if opts.Limit != 0 {
		q["limit"] = strconv.Itoa(opts.Limit)
	}
	if opts.Marker != "" {
		q["marker"] = opts.Marker
	}
	if opts.SortKey != "" {
		q["sort_key"] = opts.SortKey
	}
	if opts.SortDir != "" {
		q["sort_dir"] = opts.SortDir
	}

	u := listURL(c) + utils.BuildQuery(q)
	return pagination.NewPager(c, u, func(r pagination.LastHTTPResponse) pagination.Page {
		return SubnetPage{pagination.LinkedPageBase{LastHTTPResponse: r}}
	})
}

// Get retrieves a specific subnet based on its unique ID.
func Get(c *gophercloud.ServiceClient, id string) GetResult {
	var res GetResult
	_, res.Err = perigee.Request("GET", getURL(c, id), perigee.Options{
		MoreHeaders: c.Provider.AuthenticatedHeaders(),
		Results:     &res.Resp,
		OkCodes:     []int{200},
	})
	return res
}

// Valid IP types
const (
	IPv4 = 4
	IPv6 = 6
)

// CreateOpts represents the attributes used when creating a new subnet.
type CreateOpts struct {
	// Required
	NetworkID string
	CIDR      string
	// Optional
	Name            string
	TenantID        string
	AllocationPools []AllocationPool
	GatewayIP       string
	IPVersion       int
	EnableDHCP      *bool
	DNSNameservers  []string
	HostRoutes      []interface{}
}

// Create accepts a CreateOpts struct and creates a new subnet using the values
// provided. You must remember to provide a valid NetworkID, CIDR and IP version.
func Create(c *gophercloud.ServiceClient, opts CreateOpts) CreateResult {
	var res CreateResult

	// Validate required options
	if opts.NetworkID == "" {
		res.Err = errNetworkIDRequired
		return res
	}
	if opts.CIDR == "" {
		res.Err = errCIDRRequired
		return res
	}
	if opts.IPVersion != 0 && opts.IPVersion != IPv4 && opts.IPVersion != IPv6 {
		res.Err = errInvalidIPType
		return res
	}

	type subnet struct {
		NetworkID       string           `json:"network_id"`
		CIDR            string           `json:"cidr"`
		Name            *string          `json:"name,omitempty"`
		TenantID        *string          `json:"tenant_id,omitempty"`
		AllocationPools []AllocationPool `json:"allocation_pools,omitempty"`
		GatewayIP       *string          `json:"gateway_ip,omitempty"`
		IPVersion       int              `json:"ip_version,omitempty"`
		EnableDHCP      *bool            `json:"enable_dhcp,omitempty"`
		DNSNameservers  []string         `json:"dns_nameservers,omitempty"`
		HostRoutes      []interface{}    `json:"host_routes,omitempty"`
	}
	type request struct {
		Subnet subnet `json:"subnet"`
	}

	reqBody := request{Subnet: subnet{
		NetworkID:  opts.NetworkID,
		CIDR:       opts.CIDR,
		Name:       gophercloud.MaybeString(opts.Name),
		TenantID:   gophercloud.MaybeString(opts.TenantID),
		GatewayIP:  gophercloud.MaybeString(opts.GatewayIP),
		EnableDHCP: opts.EnableDHCP,
	}}

	if opts.IPVersion != 0 {
		reqBody.Subnet.IPVersion = opts.IPVersion
	}
	if len(opts.AllocationPools) != 0 {
		reqBody.Subnet.AllocationPools = opts.AllocationPools
	}
	if len(opts.DNSNameservers) != 0 {
		reqBody.Subnet.DNSNameservers = opts.DNSNameservers
	}
	if len(opts.HostRoutes) != 0 {
		reqBody.Subnet.HostRoutes = opts.HostRoutes
	}

	_, res.Err = perigee.Request("POST", createURL(c), perigee.Options{
		MoreHeaders: c.Provider.AuthenticatedHeaders(),
		ReqBody:     &reqBody,
		Results:     &res.Resp,
		OkCodes:     []int{201},
	})

	return res
}

// UpdateOpts represents the attributes used when updating an existing subnet.
type UpdateOpts struct {
	Name           string
	GatewayIP      string
	DNSNameservers []string
	HostRoutes     []interface{}
	EnableDHCP     *bool
}

// Update accepts a UpdateOpts struct and updates an existing subnet using the
// values provided.
func Update(c *gophercloud.ServiceClient, id string, opts UpdateOpts) UpdateResult {
	type subnet struct {
		Name           *string       `json:"name,omitempty"`
		GatewayIP      *string       `json:"gateway_ip,omitempty"`
		DNSNameservers []string      `json:"dns_nameservers,omitempty"`
		HostRoutes     []interface{} `json:"host_routes,omitempty"`
		EnableDHCP     *bool         `json:"enable_dhcp,omitempty"`
	}
	type request struct {
		Subnet subnet `json:"subnet"`
	}

	reqBody := request{Subnet: subnet{
		Name:       gophercloud.MaybeString(opts.Name),
		GatewayIP:  gophercloud.MaybeString(opts.GatewayIP),
		EnableDHCP: opts.EnableDHCP,
	}}

	if len(opts.DNSNameservers) != 0 {
		reqBody.Subnet.DNSNameservers = opts.DNSNameservers
	}

	if len(opts.HostRoutes) != 0 {
		reqBody.Subnet.HostRoutes = opts.HostRoutes
	}

	var res UpdateResult
	_, res.Err = perigee.Request("PUT", updateURL(c, id), perigee.Options{
		MoreHeaders: c.Provider.AuthenticatedHeaders(),
		ReqBody:     &reqBody,
		Results:     &res.Resp,
		OkCodes:     []int{200, 201},
	})

	return res
}

// Delete accepts a unique ID and deletes the subnet associated with it.
func Delete(c *gophercloud.ServiceClient, id string) DeleteResult {
	var res DeleteResult
	_, res.Err = perigee.Request("DELETE", deleteURL(c, id), perigee.Options{
		MoreHeaders: c.Provider.AuthenticatedHeaders(),
		OkCodes:     []int{204},
	})
	return res
}
<commit_msg>Beginning work on query cleanup<commit_after>package subnets

import (
	"github.com/racker/perigee"
	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/openstack/utils"
	"github.com/rackspace/gophercloud/pagination"
)

// ListOpts allows the filtering and sorting of paginated collections through
// the API. Filtering is achieved by passing in struct field values that map to
// the subnet attributes you want to see returned. SortKey allows you to sort
// by a particular subnet attribute. SortDir sets the direction, and is either
// `asc' or `desc'. Marker and Limit are used for pagination.
type ListOpts struct {
	Name       string `q:"name"`
	EnableDHCP *bool  `q:"enable_dhcp"`
	NetworkID  string `q:"network_id"`
	TenantID   string `q:"tenant_id"`
	IPVersion  int    `q:"ip_version"`
	GatewayIP  string `q:"gateway_ip"`
	CIDR       string `q:"cidr"`
	ID         string `q:"id"`
	Limit      int    `q:"limit"`
	Marker     string `q:"marker"`
	SortKey    string `q:"sort_key"`
	SortDir    string `q:"sort_dir"`
}

// List returns a Pager which allows you to iterate over a collection of
// subnets. It accepts a ListOpts struct, which allows you to filter and sort
// the returned collection for greater efficiency.
//
// Default policy settings return only those subnets that are owned by the tenant
// who submits the request, unless the request is submitted by an user with
// administrative rights.
func List(c *gophercloud.ServiceClient, opts ListOpts) pagination.Pager {
	// Build query parameters

	u := listURL(c) + utils.BuildQuery(q)
	return pagination.NewPager(c, u, func(r pagination.LastHTTPResponse) pagination.Page {
		return SubnetPage{pagination.LinkedPageBase{LastHTTPResponse: r}}
	})
}

// Get retrieves a specific subnet based on its unique ID.
func Get(c *gophercloud.ServiceClient, id string) GetResult {
	var res GetResult
	_, res.Err = perigee.Request("GET", getURL(c, id), perigee.Options{
		MoreHeaders: c.Provider.AuthenticatedHeaders(),
		Results:     &res.Resp,
		OkCodes:     []int{200},
	})
	return res
}

// Valid IP types
const (
	IPv4 = 4
	IPv6 = 6
)

// CreateOpts represents the attributes used when creating a new subnet.
type CreateOpts struct {
	// Required
	NetworkID string
	CIDR      string
	// Optional
	Name            string
	TenantID        string
	AllocationPools []AllocationPool
	GatewayIP       string
	IPVersion       int
	EnableDHCP      *bool
	DNSNameservers  []string
	HostRoutes      []interface{}
}

// Create accepts a CreateOpts struct and creates a new subnet using the values
// provided. You must remember to provide a valid NetworkID, CIDR and IP version.
func Create(c *gophercloud.ServiceClient, opts CreateOpts) CreateResult {
	var res CreateResult

	// Validate required options
	if opts.NetworkID == "" {
		res.Err = errNetworkIDRequired
		return res
	}
	if opts.CIDR == "" {
		res.Err = errCIDRRequired
		return res
	}
	if opts.IPVersion != 0 && opts.IPVersion != IPv4 && opts.IPVersion != IPv6 {
		res.Err = errInvalidIPType
		return res
	}

	type subnet struct {
		NetworkID       string           `json:"network_id"`
		CIDR            string           `json:"cidr"`
		Name            *string          `json:"name,omitempty"`
		TenantID        *string          `json:"tenant_id,omitempty"`
		AllocationPools []AllocationPool `json:"allocation_pools,omitempty"`
		GatewayIP       *string          `json:"gateway_ip,omitempty"`
		IPVersion       int              `json:"ip_version,omitempty"`
		EnableDHCP      *bool            `json:"enable_dhcp,omitempty"`
		DNSNameservers  []string         `json:"dns_nameservers,omitempty"`
		HostRoutes      []interface{}    `json:"host_routes,omitempty"`
	}
	type request struct {
		Subnet subnet `json:"subnet"`
	}

	reqBody := request{Subnet: subnet{
		NetworkID:  opts.NetworkID,
		CIDR:       opts.CIDR,
		Name:       gophercloud.MaybeString(opts.Name),
		TenantID:   gophercloud.MaybeString(opts.TenantID),
		GatewayIP:  gophercloud.MaybeString(opts.GatewayIP),
		EnableDHCP: opts.EnableDHCP,
	}}

	if opts.IPVersion != 0 {
		reqBody.Subnet.IPVersion = opts.IPVersion
	}
	if len(opts.AllocationPools) != 0 {
		reqBody.Subnet.AllocationPools = opts.AllocationPools
	}
	if len(opts.DNSNameservers) != 0 {
		reqBody.Subnet.DNSNameservers = opts.DNSNameservers
	}
	if len(opts.HostRoutes) != 0 {
		reqBody.Subnet.HostRoutes = opts.HostRoutes
	}

	_, res.Err = perigee.Request("POST", createURL(c), perigee.Options{
		MoreHeaders: c.Provider.AuthenticatedHeaders(),
		ReqBody:     &reqBody,
		Results:     &res.Resp,
		OkCodes:     []int{201},
	})

	return res
}

// UpdateOpts represents the attributes used when updating an existing subnet.
type UpdateOpts struct {
	Name           string
	GatewayIP      string
	DNSNameservers []string
	HostRoutes     []interface{}
	EnableDHCP     *bool
}

// Update accepts a UpdateOpts struct and updates an existing subnet using the
// values provided.
func Update(c *gophercloud.ServiceClient, id string, opts UpdateOpts) UpdateResult {
	type subnet struct {
		Name           *string       `json:"name,omitempty"`
		GatewayIP      *string       `json:"gateway_ip,omitempty"`
		DNSNameservers []string      `json:"dns_nameservers,omitempty"`
		HostRoutes     []interface{} `json:"host_routes,omitempty"`
		EnableDHCP     *bool         `json:"enable_dhcp,omitempty"`
	}
	type request struct {
		Subnet subnet `json:"subnet"`
	}

	reqBody := request{Subnet: subnet{
		Name:       gophercloud.MaybeString(opts.Name),
		GatewayIP:  gophercloud.MaybeString(opts.GatewayIP),
		EnableDHCP: opts.EnableDHCP,
	}}

	if len(opts.DNSNameservers) != 0 {
		reqBody.Subnet.DNSNameservers = opts.DNSNameservers
	}

	if len(opts.HostRoutes) != 0 {
		reqBody.Subnet.HostRoutes = opts.HostRoutes
	}

	var res UpdateResult
	_, res.Err = perigee.Request("PUT", updateURL(c, id), perigee.Options{
		MoreHeaders: c.Provider.AuthenticatedHeaders(),
		ReqBody:     &reqBody,
		Results:     &res.Resp,
		OkCodes:     []int{200, 201},
	})

	return res
}

// Delete accepts a unique ID and deletes the subnet associated with it.
func Delete(c *gophercloud.ServiceClient, id string) DeleteResult {
	var res DeleteResult
	_, res.Err = perigee.Request("DELETE", deleteURL(c, id), perigee.Options{
		MoreHeaders: c.Provider.AuthenticatedHeaders(),
		OkCodes:     []int{204},
	})
	return res
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/gorilla/mux"
	"io"
	"koding/db/models"
	"koding/db/mongodb"
	"koding/kontrol/kontroldaemon/workerconfig"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"log"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"time"
)

type ApiWorker struct {
	Name               string    `json:"name"`
	ServiceGenericName string    `json:"serviceGenericName"`
	ServiceUniqueName  string    `json:"serviceUniqueName"`
	Uuid               string    `json:"uuid"`
	Hostname           string    `json:"hostname"`
	Version            int       `json:"version"`
	Timestamp          time.Time `json:"timestamp"`
	Pid                int       `json:"pid"`
	State              string    `json:"state"`
	Uptime             int       `json:"uptime"`
	Port               int       `json:"port"`
}

type Workers []ApiWorker

var StatusCode = map[models.WorkerStatus]string{
	models.Started: "started",
	models.Waiting: "waiting",
	models.Killed:  "dead",
	models.Dead:    "dead",
}

func GetWorkers(writer http.ResponseWriter, req *http.Request) {
	queries, _ := url.ParseQuery(req.URL.RawQuery)

	var latestVersion bool
	var sortFields []string // not initialized means do not sort
	query := bson.M{}
	for key, value := range queries {
		switch key {
		case "version", "pid":
			if value[0] == "latest" {
				latestVersion = true
				break
			}
			v, _ := strconv.Atoi(value[0])
			query[key] = v
		case "state":
			for status, state := range StatusCode {
				if value[0] == state {
					query["status"] = status
				}
			}
		case "sort":
			sortFields = []string{value[0]}
			// override "state" with status, they are not the same in db
			if value[0] == "state" {
				sortFields = []string{"status"}
			}
		default:
			if key == "name" {
				name := value[0]
				if counts := strings.Count(value[0], "-"); counts > 0 {
					s := strings.Split(value[0], "-")
					name = s[0]
				}
				// if searched for social-1, social-2, then return all workers
				// that begins with social
				query[key] = bson.RegEx{Pattern: "^" + name, Options: "i"}
			} else {
				query[key] = value[0]
			}
		}
	}

	matchedWorkers := queryResult(query, latestVersion, sortFields)
	data, err := json.MarshalIndent(matchedWorkers, "", "  ")
	if err != nil {
		io.WriteString(writer, fmt.Sprintf("{\"err\":\"%s\"}\n", err))
		return
	}
	writer.Write(data)

}

func GetWorker(writer http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	uuid := vars["uuid"]

	query := bson.M{"uuid": uuid}
	matchedWorkers := queryResult(query, false, nil)
	data, err := json.MarshalIndent(matchedWorkers, "", "  ")
	if err != nil {
		io.WriteString(writer, fmt.Sprintf("{\"err\":\"%s\"}\n", err))
		return
	}
	writer.Write(data)
}

func UpdateWorker(writer http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	uuid, action := vars["uuid"], vars["action"]

	buildSendCmd(action, uuid)
	resp := fmt.Sprintf("worker: '%s' is updated in db", uuid)
	io.WriteString(writer, resp)
}

func DeleteWorker(writer http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	uuid := vars["uuid"]

	buildSendCmd("delete", uuid)
	resp := fmt.Sprintf("worker: '%s' is deleted from db", uuid)
	io.WriteString(writer, resp)
}

func queryResult(query bson.M, latestVersion bool, sortFields []string) Workers {
	workers := make(Workers, 0)
	worker := models.Worker{}

	queryFunc := func(c *mgo.Collection) error {
		// sorting is no-op when sortFields is empty
		iter := c.Find(query).Sort(sortFields...).Iter()
		for iter.Next(&worker) {
			apiWorker := &ApiWorker{
				worker.Name,
				worker.ServiceGenericName,
				worker.ServiceUniqueName,
				worker.Uuid,
				worker.Hostname,
				worker.Version,
				worker.Timestamp,
				worker.Pid,
				StatusCode[worker.Status],
				worker.Monitor.Uptime,
				worker.Port,
			}

			workers = append(workers, *apiWorker)
		}
		return nil
	}

	mongodb.Run("jKontrolWorkers", queryFunc)

	// finding the largest number of a field in mongo is kinda problematic.
	// therefore we are doing it on our side
	if latestVersion {
		versions := make([]int, len(workers))

		if len(workers) == 0 {
			return workers
		}

		for i, val := range workers {
			versions[i] = val.Version
		}

		sort.Ints(versions)
		maxVersion := versions[len(versions)-1] // get largest version number

		filteredWorkers := make(Workers, 0)
		for _, val := range workers {
			if maxVersion == val.Version {
				filteredWorkers = append(filteredWorkers, val)
			}
		}

		return filteredWorkers
	}

	return workers
}

func buildSendCmd(action, uuid string) {
	cmd := workerconfig.ApiRequest{Uuid: uuid, Command: action}
	data, err := json.Marshal(cmd)
	if err != nil {
		log.Println("Json marshall error", data)
	}

	log.Println("Sending cmd to kontrold:", cmd)
	amqpWrapper.Publish(data)
}
<commit_msg>kontrolapi: fix kontrol db<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"koding/db/models"
	"koding/db/mongodb"
	"koding/kontrol/kontroldaemon/workerconfig"
	"koding/tools/config"
	"log"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/gorilla/mux"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

type ApiWorker struct {
	Name               string    `json:"name"`
	ServiceGenericName string    `json:"serviceGenericName"`
	ServiceUniqueName  string    `json:"serviceUniqueName"`
	Uuid               string    `json:"uuid"`
	Hostname           string    `json:"hostname"`
	Version            int       `json:"version"`
	Timestamp          time.Time `json:"timestamp"`
	Pid                int       `json:"pid"`
	State              string    `json:"state"`
	Uptime             int       `json:"uptime"`
	Port               int       `json:"port"`
}

type Workers []ApiWorker

var StatusCode = map[models.WorkerStatus]string{
	models.Started: "started",
	models.Waiting: "waiting",
	models.Killed:  "dead",
	models.Dead:    "dead",
}

const (
	WorkersCollection = "jKontrolWorkers"
	WorkersDB         = "kontrol"
)

var kontrolDB = mongodb.NewMongoDB(config.Current.MongoKontrol)

func GetWorkers(writer http.ResponseWriter, req *http.Request) {
	queries, _ := url.ParseQuery(req.URL.RawQuery)

	var latestVersion bool
	var sortFields []string // not initialized means do not sort
	query := bson.M{}
	for key, value := range queries {
		switch key {
		case "version", "pid":
			if value[0] == "latest" {
				latestVersion = true
				break
			}
			v, _ := strconv.Atoi(value[0])
			query[key] = v
		case "state":
			for status, state := range StatusCode {
				if value[0] == state {
					query["status"] = status
				}
			}
		case "sort":
			sortFields = []string{value[0]}
			// override "state" with status, they are not the same in db
			if value[0] == "state" {
				sortFields = []string{"status"}
			}
		default:
			if key == "name" {
				name := value[0]
				if counts := strings.Count(value[0], "-"); counts > 0 {
					s := strings.Split(value[0], "-")
					name = s[0]
				}
				// if searched for social-1, social-2, then return all workers
				// that begins with social
				query[key] = bson.RegEx{Pattern: "^" + name, Options: "i"}
			} else {
				query[key] = value[0]
			}
		}
	}

	matchedWorkers := queryResult(query, latestVersion, sortFields)
	data, err := json.MarshalIndent(matchedWorkers, "", "  ")
	if err != nil {
		io.WriteString(writer, fmt.Sprintf("{\"err\":\"%s\"}\n", err))
		return
	}
	writer.Write(data)

}

func GetWorker(writer http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	uuid := vars["uuid"]

	query := bson.M{"uuid": uuid}
	matchedWorkers := queryResult(query, false, nil)
	data, err := json.MarshalIndent(matchedWorkers, "", "  ")
	if err != nil {
		io.WriteString(writer, fmt.Sprintf("{\"err\":\"%s\"}\n", err))
		return
	}
	writer.Write(data)
}

func UpdateWorker(writer http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	uuid, action := vars["uuid"], vars["action"]

	buildSendCmd(action, uuid)
	resp := fmt.Sprintf("worker: '%s' is updated in db", uuid)
	io.WriteString(writer, resp)
}

func DeleteWorker(writer http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	uuid := vars["uuid"]

	buildSendCmd("delete", uuid)
	resp := fmt.Sprintf("worker: '%s' is deleted from db", uuid)
	io.WriteString(writer, resp)
}

func queryResult(query bson.M, latestVersion bool, sortFields []string) Workers {
	workers := make(Workers, 0)
	worker := models.Worker{}

	queryFunc := func(c *mgo.Collection) error {
		// sorting is no-op when sortFields is empty
		iter := c.Find(query).Sort(sortFields...).Iter()
		for iter.Next(&worker) {
			apiWorker := &ApiWorker{
				worker.Name,
				worker.ServiceGenericName,
				worker.ServiceUniqueName,
				worker.Uuid,
				worker.Hostname,
				worker.Version,
				worker.Timestamp,
				worker.Pid,
				StatusCode[worker.Status],
				worker.Monitor.Uptime,
				worker.Port,
			}

			workers = append(workers, *apiWorker)
		}
		return nil
	}

	kontrolDB.RunOnDatabase(WorkersDB, WorkersCollection, queryFunc)

	// finding the largest number of a field in mongo is kinda problematic.
	// therefore we are doing it on our side
	if latestVersion {
		versions := make([]int, len(workers))

		if len(workers) == 0 {
			return workers
		}

		for i, val := range workers {
			versions[i] = val.Version
		}

		sort.Ints(versions)
		maxVersion := versions[len(versions)-1] // get largest version number

		filteredWorkers := make(Workers, 0)
		for _, val := range workers {
			if maxVersion == val.Version {
				filteredWorkers = append(filteredWorkers, val)
			}
		}

		return filteredWorkers
	}

	return workers
}

func buildSendCmd(action, uuid string) {
	cmd := workerconfig.ApiRequest{Uuid: uuid, Command: action}
	data, err := json.Marshal(cmd)
	if err != nil {
		log.Println("Json marshall error", data)
	}

	log.Println("Sending cmd to kontrold:", cmd)
	amqpWrapper.Publish(data)
}
<|endoftext|>