<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// cl-fetch fetches and tags CLs from Gerrit.
package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"regexp"
	"strings"

	"golang.org/x/build/gerrit"
)

var (
	flagOutgoing = flag.Bool("outgoing", false, "fetch outgoing CLs")
	flagIncoming = flag.Bool("incoming", false, "fetch incoming CLs")
	flagQuery    = flag.String("q", "", "fetch CLs matching `query`")
	flagVerbose  = flag.Bool("v", false, "verbose output")
	flagDry      = flag.Bool("dry-run", false, "print but do not execute commands")
)

var clRe = regexp.MustCompile("[0-9]+|I[0-9a-f]{40}")

type Tag struct {
	tag    string
	commit *gerrit.CommitInfo
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [flags] [CLs...]\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()

	queryParts := []string{}
	if *flagOutgoing {
		queryParts = append(queryParts, "is:open owner:self")
	}
	if *flagIncoming {
		queryParts = append(queryParts, "is:open reviewer:self -owner:self")
	}
	if *flagQuery != "" {
		queryParts = append(queryParts, *flagQuery)
	}
	for _, arg := range flag.Args() {
		if !clRe.MatchString(arg) {
			fmt.Fprintf(os.Stderr, "CL must be a CL number or Change-Id")
			os.Exit(2)
		}
		queryParts = append(queryParts, "change:"+arg)
	}
	if len(queryParts) == 0 {
		fmt.Fprintf(os.Stderr, "must specify something to fetch\n")
		os.Exit(2)
	}
	query := "(" + strings.Join(queryParts, ") OR (") + ")"

	if *flagVerbose {
		log.Printf("query: %s", query)
	}

	// Get the origin so we don't pull CLs for other repositories
	// in to this one.
	origin := gitOutput("config", "remote.origin.url")

	// Get the existing CL tags.
	haveTags := map[string]bool{}
	for _, tag := range strings.Split(gitOutput("tag"), "\n") {
		haveTags[tag] = true
	}

	c := gerrit.NewClient("https://go-review.googlesource.com", gerrit.GitCookiesAuth())

	cls, err := c.QueryChanges(context.Background(), query, gerrit.QueryChangesOpt{
		Fields: []string{"CURRENT_REVISION", "CURRENT_COMMIT"},
	})
	if err != nil {
		log.Fatal(err)
	}

	if *flagVerbose {
		v, _ := json.MarshalIndent(cls, "", "  ")
		log.Printf("Query response:\n%s\n", v)
	}

	// Collect git fetch and tag commands.
	fetchCmd := []string{"fetch", "--", origin}
	tags := make(map[string]*Tag)
	hashOrder := []string{}
	for _, cl := range cls {
		for commitID, rev := range cl.Revisions {
			tag := fmt.Sprintf("cl/%d/%d", cl.ChangeNumber, rev.PatchSetNumber)
			if !haveTags[tag] {
				any := false
				for _, fetch := range rev.Fetch {
					if fetch.URL == origin {
						fetchCmd = append(fetchCmd, fetch.Ref)
						any = true
						break
					}
				}
				if !any {
					continue
				}
			}

			tags[commitID] = &Tag{
				tag:    tag,
				commit: rev.Commit,
			}

			hashOrder = append(hashOrder, commitID)
		}
	}

	// Execute git fetch and tag commands.
	if len(fetchCmd) != 3 {
		git(fetchCmd...)
		fmt.Println()
	}
	for commitID, tag := range tags {
		if !haveTags[tag.tag] {
			git("tag", tag.tag, commitID)
		}
	}
	if *flagDry {
		// Separate command from printed tags.
		fmt.Println()
	}

	// Print tags.
	leafs := make(map[string]bool)
	for commitID, _ := range tags {
		leafs[commitID] = true
	}
	for _, tag := range tags {
		for _, parent := range tag.commit.Parents {
			leafs[parent.CommitID] = false
		}
	}

	printed := make(map[string]bool)
	needBlank := false
	for i := range hashOrder {
		commitID := hashOrder[len(hashOrder)-i-1]
		if !leafs[commitID] {
			continue
		}
		if needBlank {
			fmt.Println()
		}
		needBlank = printChain(tags, commitID, printed)
	}
}

func git(args ...string) {
	if *flagDry {
		fmt.Printf("git %s\n", strings.Join(args, " "))
		return
	}

	cmd := exec.Command("git", args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		log.Fatalf("git %s failed: %s", strings.Join(args, " "), err)
	}
}

func gitOutput(args ...string) string {
	if *flagDry {
		fmt.Printf("git %s\n", strings.Join(args, " "))
	}

	cmd := exec.Command("git", args...)
	cmd.Stderr = os.Stderr
	out, err := cmd.Output()
	if err != nil {
		log.Fatalf("git %s failed: %s", strings.Join(args, " "), err)
	}
	return strings.TrimRight(string(out), "\n")
}

func printChain(tags map[string]*Tag, commitID string, printed map[string]bool) bool {
	if printed[commitID] {
		return false
	}
	printed[commitID] = true

	tag := tags[commitID]
	for _, parent := range tag.commit.Parents {
		if tags[parent.CommitID] != nil {
			printChain(tags, parent.CommitID, printed)
		}
	}
	fmt.Printf("%s %s\n", tag.tag, tag.commit.Subject)
	return true
}
<commit_msg>cl-fetch: make argument regexp strict<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// cl-fetch fetches and tags CLs from Gerrit.
package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"regexp"
	"strings"

	"golang.org/x/build/gerrit"
)

var (
	flagOutgoing = flag.Bool("outgoing", false, "fetch outgoing CLs")
	flagIncoming = flag.Bool("incoming", false, "fetch incoming CLs")
	flagQuery    = flag.String("q", "", "fetch CLs matching `query`")
	flagVerbose  = flag.Bool("v", false, "verbose output")
	flagDry      = flag.Bool("dry-run", false, "print but do not execute commands")
)

var clRe = regexp.MustCompile("^[0-9]+$|^I[0-9a-f]{40}$")

type Tag struct {
	tag    string
	commit *gerrit.CommitInfo
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [flags] [CLs...]\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()

	queryParts := []string{}
	if *flagOutgoing {
		queryParts = append(queryParts, "is:open owner:self")
	}
	if *flagIncoming {
		queryParts = append(queryParts, "is:open reviewer:self -owner:self")
	}
	if *flagQuery != "" {
		queryParts = append(queryParts, *flagQuery)
	}
	for _, arg := range flag.Args() {
		if !clRe.MatchString(arg) {
			fmt.Fprintf(os.Stderr, "CL must be a CL number or Change-Id\n")
			os.Exit(2)
		}
		queryParts = append(queryParts, "change:"+arg)
	}
	if len(queryParts) == 0 {
		fmt.Fprintf(os.Stderr, "must specify something to fetch\n")
		os.Exit(2)
	}
	query := "(" + strings.Join(queryParts, ") OR (") + ")"

	if *flagVerbose {
		log.Printf("query: %s", query)
	}

	// Get the origin so we don't pull CLs for other repositories
	// in to this one.
	origin := gitOutput("config", "remote.origin.url")

	// Get the existing CL tags.
	haveTags := map[string]bool{}
	for _, tag := range strings.Split(gitOutput("tag"), "\n") {
		haveTags[tag] = true
	}

	c := gerrit.NewClient("https://go-review.googlesource.com", gerrit.GitCookiesAuth())

	cls, err := c.QueryChanges(context.Background(), query, gerrit.QueryChangesOpt{
		Fields: []string{"CURRENT_REVISION", "CURRENT_COMMIT"},
	})
	if err != nil {
		log.Fatal(err)
	}

	if *flagVerbose {
		v, _ := json.MarshalIndent(cls, "", "  ")
		log.Printf("Query response:\n%s\n", v)
	}

	// Collect git fetch and tag commands.
	fetchCmd := []string{"fetch", "--", origin}
	tags := make(map[string]*Tag)
	hashOrder := []string{}
	for _, cl := range cls {
		for commitID, rev := range cl.Revisions {
			tag := fmt.Sprintf("cl/%d/%d", cl.ChangeNumber, rev.PatchSetNumber)
			if !haveTags[tag] {
				any := false
				for _, fetch := range rev.Fetch {
					if fetch.URL == origin {
						fetchCmd = append(fetchCmd, fetch.Ref)
						any = true
						break
					}
				}
				if !any {
					continue
				}
			}

			tags[commitID] = &Tag{
				tag:    tag,
				commit: rev.Commit,
			}

			hashOrder = append(hashOrder, commitID)
		}
	}

	// Execute git fetch and tag commands.
	if len(fetchCmd) != 3 {
		git(fetchCmd...)
		fmt.Println()
	}
	for commitID, tag := range tags {
		if !haveTags[tag.tag] {
			git("tag", tag.tag, commitID)
		}
	}
	if *flagDry {
		// Separate command from printed tags.
		fmt.Println()
	}

	// Print tags.
	leafs := make(map[string]bool)
	for commitID, _ := range tags {
		leafs[commitID] = true
	}
	for _, tag := range tags {
		for _, parent := range tag.commit.Parents {
			leafs[parent.CommitID] = false
		}
	}

	printed := make(map[string]bool)
	needBlank := false
	for i := range hashOrder {
		commitID := hashOrder[len(hashOrder)-i-1]
		if !leafs[commitID] {
			continue
		}
		if needBlank {
			fmt.Println()
		}
		needBlank = printChain(tags, commitID, printed)
	}
}

func git(args ...string) {
	if *flagDry {
		fmt.Printf("git %s\n", strings.Join(args, " "))
		return
	}

	cmd := exec.Command("git", args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		log.Fatalf("git %s failed: %s", strings.Join(args, " "), err)
	}
}

func gitOutput(args ...string) string {
	if *flagDry {
		fmt.Printf("git %s\n", strings.Join(args, " "))
	}

	cmd := exec.Command("git", args...)
	cmd.Stderr = os.Stderr
	out, err := cmd.Output()
	if err != nil {
		log.Fatalf("git %s failed: %s", strings.Join(args, " "), err)
	}
	return strings.TrimRight(string(out), "\n")
}

func printChain(tags map[string]*Tag, commitID string, printed map[string]bool) bool {
	if printed[commitID] {
		return false
	}
	printed[commitID] = true

	tag := tags[commitID]
	for _, parent := range tag.commit.Parents {
		if tags[parent.CommitID] != nil {
			printChain(tags, parent.CommitID, printed)
		}
	}
	fmt.Printf("%s %s\n", tag.tag, tag.commit.Subject)
	return true
}
<|endoftext|>


<commit_before>// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.
// HTTP客户端请求.

package ghttp

import (
    "time"
    "bytes"
    "strings"
    "net/http"
    "mime/multipart"
    "os"
    "io"
)

// http客户端
type Client struct {
    http.Client              // 底层http client对象
    header map[string]string // header
}

// http客户端对象指针
func NewClient() (*Client) {
    return &Client{
        http.Client {
            Transport: &http.Transport {
                DisableKeepAlives: true,
            },
        },
        make(map[string]string),
    }
}

// 设置HTTP Headerss
func (c *Client) SetHeader(key, value string) {
    c.header[key] = value
}

// 设置请求过期时间
func (c *Client) SetTimeOut(t time.Duration)  {
    c.Timeout = t
}

// GET请求
func (c *Client) Get(url string) (*ClientResponse, error) {
    return c.DoRequest("GET", url, []byte(""))
}

// PUT请求
func (c *Client) Put(url, data string) (*ClientResponse, error) {
    return c.DoRequest("PUT", url, []byte(data))
}

// POST请求提交数据
// 支持文件上传，需要字段格式为：FieldName=@file:
func (c *Client) Post(url, data string) (*ClientResponse, error) {
    var req *http.Request
    if strings.Contains(data, "@file:") {
        buffer := new(bytes.Buffer)
        writer := multipart.NewWriter(buffer)
        for _, item := range strings.Split(data, "&") {
            array := strings.Split(item, "=")
            if len(array[1]) > 6 && strings.Compare(array[1][0:6], "@file:") == 0 {
                if file, err := writer.CreateFormFile(array[0], array[1][6:]); err == nil {
                    if f, err := os.Open(array[1][6:]); err == nil {
                        defer f.Close()
                        if _, err = io.Copy(file, f); err != nil {
                            return nil, err
                        }
                    } else {
                        return nil, err
                    }
                } else {
                    return nil, err
                }
            } else {
                writer.WriteField(array[0], array[1])
            }
        }
        writer.Close()
        if r, err := http.NewRequest("POST", url, buffer); err != nil {
            return nil, err
        } else {
            req = r
            req.Header.Set("Content-Type", writer.FormDataContentType())
        }
    } else {
        if r, err := http.NewRequest("POST", url, bytes.NewReader([]byte(data))); err != nil {
            return nil, err
        } else {
            req = r
            req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
        }
    }
    // 自定义header
    if len(c.header) > 0 {
        for k, v := range c.header {
            req.Header.Set(k, v)
        }
    }
    // 执行请求
    resp, err := c.Do(req)
    if err != nil {
        return nil, err
    }
    r := &ClientResponse{}
    r.Response = *resp
    return r, nil
}

// DELETE请求
func (c *Client) Delete(url, data string) (*ClientResponse, error) {
    return c.DoRequest("DELETE", url, []byte(data))
}

func (c *Client) Head(url, data string) (*ClientResponse, error) {
    return c.DoRequest("HEAD", url, []byte(data))
}

func (c *Client) Patch(url, data string) (*ClientResponse, error) {
    return c.DoRequest("PATCH", url, []byte(data))
}

func (c *Client) Connect(url, data string) (*ClientResponse, error) {
    return c.DoRequest("CONNECT", url, []byte(data))
}

func (c *Client) Options(url, data string) (*ClientResponse, error) {
    return c.DoRequest("OPTIONS", url, []byte(data))
}

func (c *Client) Trace(url, data string) (*ClientResponse, error) {
    return c.DoRequest("TRACE", url, []byte(data))
}

// 请求并返回response对象，该方法支持二进制提交数据
func (c *Client) DoRequest(method, url string, data []byte) (*ClientResponse, error) {
    if strings.Compare("POST", strings.ToUpper(method)) == 0 {
        return c.Post(url, string(data))
    }
    req, err := http.NewRequest(strings.ToUpper(method), url, bytes.NewReader(data))
    if err != nil {
        return nil, err
    }
    // 自定义header
    if len(c.header) > 0 {
        for k, v := range c.header {
            req.Header.Set(k, v)
        }
    }
    // 执行请求
    resp, err := c.Do(req)
    if err != nil {
        return nil, err
    }
    r := &ClientResponse{}
    r.Response = *resp
    return r, nil
}

func Get(url string) (*ClientResponse, error) {
    return DoRequest("GET", url, []byte(""))
}

func Put(url, data string) (*ClientResponse, error) {
    return DoRequest("PUT", url, []byte(data))
}

func Post(url, data string) (*ClientResponse, error) {
    return DoRequest("POST", url, []byte(data))
}

func Delete(url, data string) (*ClientResponse, error) {
    return DoRequest("DELETE", url, []byte(data))
}

func Head(url, data string) (*ClientResponse, error) {
    return DoRequest("HEAD", url, []byte(data))
}

func Patch(url, data string) (*ClientResponse, error) {
    return DoRequest("PATCH", url, []byte(data))
}

func Connect(url, data string) (*ClientResponse, error) {
    return DoRequest("CONNECT", url, []byte(data))
}

func Options(url, data string) (*ClientResponse, error) {
    return DoRequest("OPTIONS", url, []byte(data))
}

func Trace(url, data string) (*ClientResponse, error) {
    return DoRequest("TRACE", url, []byte(data))
}

// 该方法支持二进制提交数据
func DoRequest(method, url string, data []byte) (*ClientResponse, error) {
    return NewClient().DoRequest(method, url, data)
}
<commit_msg>ghttp.Client增加HTTP账号密码验证功能<commit_after>// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.
// HTTP客户端请求.

package ghttp

import (
    "time"
    "bytes"
    "strings"
    "net/http"
    "mime/multipart"
    "os"
    "io"
)

// http客户端
type Client struct {
    http.Client                // 底层http client对象
    header   map[string]string // header
    authUser string            // HTTP基本权限设置：名称
    authPass string            // HTTP基本权限设置：密码
}

// http客户端对象指针
func NewClient() (*Client) {
    return &Client{
        Client : http.Client {
            Transport: &http.Transport {
                DisableKeepAlives: true,
            },
        },
        header : make(map[string]string),
    }
}

// 设置HTTP Headerss
func (c *Client) SetHeader(key, value string) {
    c.header[key] = value
}

// 设置请求过期时间
func (c *Client) SetTimeOut(t time.Duration)  {
    c.Timeout = t
}

// 设置HTTP访问账号密码
func (c *Client) SetBasicAuth(user, pass string) {
    c.authUser = user
    c.authPass = pass
}

// GET请求
func (c *Client) Get(url string) (*ClientResponse, error) {
    return c.DoRequest("GET", url, []byte(""))
}

// PUT请求
func (c *Client) Put(url, data string) (*ClientResponse, error) {
    return c.DoRequest("PUT", url, []byte(data))
}

// POST请求提交数据
// 支持文件上传，需要字段格式为：FieldName=@file:
func (c *Client) Post(url, data string) (*ClientResponse, error) {
    var req *http.Request
    if strings.Contains(data, "@file:") {
        buffer := new(bytes.Buffer)
        writer := multipart.NewWriter(buffer)
        for _, item := range strings.Split(data, "&") {
            array := strings.Split(item, "=")
            if len(array[1]) > 6 && strings.Compare(array[1][0:6], "@file:") == 0 {
                if file, err := writer.CreateFormFile(array[0], array[1][6:]); err == nil {
                    if f, err := os.Open(array[1][6:]); err == nil {
                        defer f.Close()
                        if _, err = io.Copy(file, f); err != nil {
                            return nil, err
                        }
                    } else {
                        return nil, err
                    }
                } else {
                    return nil, err
                }
            } else {
                writer.WriteField(array[0], array[1])
            }
        }
        writer.Close()
        if r, err := http.NewRequest("POST", url, buffer); err != nil {
            return nil, err
        } else {
            req = r
            req.Header.Set("Content-Type", writer.FormDataContentType())
        }
    } else {
        if r, err := http.NewRequest("POST", url, bytes.NewReader([]byte(data))); err != nil {
            return nil, err
        } else {
            req = r
            req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
        }
    }
    // 自定义header
    if len(c.header) > 0 {
        for k, v := range c.header {
            req.Header.Set(k, v)
        }
    }
    // HTTP账号密码
    if len(c.authUser) > 0 {
        req.SetBasicAuth(c.authUser, c.authPass)
    }
    // 执行请求
    resp, err := c.Do(req)
    if err != nil {
        return nil, err
    }
    r := &ClientResponse{}
    r.Response = *resp
    return r, nil
}

// DELETE请求
func (c *Client) Delete(url, data string) (*ClientResponse, error) {
    return c.DoRequest("DELETE", url, []byte(data))
}

func (c *Client) Head(url, data string) (*ClientResponse, error) {
    return c.DoRequest("HEAD", url, []byte(data))
}

func (c *Client) Patch(url, data string) (*ClientResponse, error) {
    return c.DoRequest("PATCH", url, []byte(data))
}

func (c *Client) Connect(url, data string) (*ClientResponse, error) {
    return c.DoRequest("CONNECT", url, []byte(data))
}

func (c *Client) Options(url, data string) (*ClientResponse, error) {
    return c.DoRequest("OPTIONS", url, []byte(data))
}

func (c *Client) Trace(url, data string) (*ClientResponse, error) {
    return c.DoRequest("TRACE", url, []byte(data))
}

// 请求并返回response对象，该方法支持二进制提交数据
func (c *Client) DoRequest(method, url string, data []byte) (*ClientResponse, error) {
    if strings.Compare("POST", strings.ToUpper(method)) == 0 {
        return c.Post(url, string(data))
    }
    req, err := http.NewRequest(strings.ToUpper(method), url, bytes.NewReader(data))
    if err != nil {
        return nil, err
    }
    // 自定义header
    if len(c.header) > 0 {
        for k, v := range c.header {
            req.Header.Set(k, v)
        }
    }
    // 执行请求
    resp, err := c.Do(req)
    if err != nil {
        return nil, err
    }
    r := &ClientResponse{}
    r.Response = *resp
    return r, nil
}

func Get(url string) (*ClientResponse, error) {
    return DoRequest("GET", url, []byte(""))
}

func Put(url, data string) (*ClientResponse, error) {
    return DoRequest("PUT", url, []byte(data))
}

func Post(url, data string) (*ClientResponse, error) {
    return DoRequest("POST", url, []byte(data))
}

func Delete(url, data string) (*ClientResponse, error) {
    return DoRequest("DELETE", url, []byte(data))
}

func Head(url, data string) (*ClientResponse, error) {
    return DoRequest("HEAD", url, []byte(data))
}

func Patch(url, data string) (*ClientResponse, error) {
    return DoRequest("PATCH", url, []byte(data))
}

func Connect(url, data string) (*ClientResponse, error) {
    return DoRequest("CONNECT", url, []byte(data))
}

func Options(url, data string) (*ClientResponse, error) {
    return DoRequest("OPTIONS", url, []byte(data))
}

func Trace(url, data string) (*ClientResponse, error) {
    return DoRequest("TRACE", url, []byte(data))
}

// 该方法支持二进制提交数据
func DoRequest(method, url string, data []byte) (*ClientResponse, error) {
    return NewClient().DoRequest(method, url, data)
}
<|endoftext|>


<commit_before>package colonycore

import (
	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
)

type WebGrabber struct {
	orm.ModelBase
	ID                string `json:"_id",bson:"_id"`
	CallType          string
	SourceType        string
	IntervalType      string
	GrabInterval      int32
	TimeoutInterval   int32
	URL               string
	LogConfiguration  *LogConfiguration
	DataSetting       *DataSetting
	GrabConfiguration toolkit.M
	Parameter         []*Parameter
}

func (ds *WebGrabber) TableName() string {
	return "webgrabbers"
}

func (ds *WebGrabber) RecordID() interface{} {
	return ds.ID
}

type LogConfiguration struct {
	FileName    string
	FilePattern string
	LogPath     string
}

type ConnectionInfo struct {
	dbox.ConnectionInfo
	Collection string
}

type DataSetting struct {
	RowSelector     string
	FilterCondition toolkit.M
	ColumnSettings  []*ColumnSetting

	RowDeleteCondition  toolkit.M
	RowIncludeCondition toolkit.M

	ConnectionInfo  *ConnectionInfo
	DestinationType string
	Name            string
}

type ColumnSetting struct {
	Alias     string
	Index     int
	Selector  string
	ValueType string
}

type Parameter struct {
	Format  string
	Key     string
	Pattern string
	Value   interface{}
}
<commit_msg>no message<commit_after>package colonycore

import (
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
)

type WebGrabber struct {
	orm.ModelBase
	ID                string            `json:"nameid",bson:"nameid"`
	CallType          string            `json:"calltype",bson:"calltype"`
	SourceType        string            `json:"sourcetype",bson:"sourcetype"`
	IntervalType      string            `json:"intervaltype",bson:"intervaltype"`
	GrabInterval      int32             `json:"grabinterval",bson:"grabinterval"`
	TimeoutInterval   int32             `json:"timeoutinterval",bson:"timeoutinterval"`
	URL               string            `json:"url",bson:"url"`
	LogConfiguration  *LogConfiguration `json:"logconf",bson:"logconf"`
	DataSettings      []*DataSetting    `json:"datasettings",bson:"datasettings"`
	GrabConfiguration toolkit.M         `json:"grabconf",bson:"grabconf"`
}

func (ds *WebGrabber) TableName() string {
	return "webgrabbers"
}

func (ds *WebGrabber) RecordID() interface{} {
	return ds.ID
}

type LogConfiguration struct {
	FileName    string `json:"filename",bson:"filename"`
	FilePattern string `json:"filepattern",bson:"filepattern"`
	LogPath     string `json:"logpath",bson:"logpath"`
}

type ConnectionInfo struct {
	Host       string    `json:"host",bson:"host"`
	Database   string    `json:"database",bson:"database"`
	UserName   string    `json:"username",bson:"username"`
	Password   string    `json:"password",bson:"password"`
	Settings   toolkit.M `json:"settings",bson:"settings"`
	Collection string    `json:"collection",bson:"collection"`
}

type DataSetting struct {
	RowSelector         string           `json:"rowselector",bson:"rowselector"`
	FilterCondition     toolkit.M        `json:"rowselector",bson:"rowselector"`
	ColumnSettings      []*ColumnSetting `json:"columnsettings",bson:"columnsettings"`
	RowDeleteCondition  toolkit.M        `json:"rowdeletecond",bson:"rowdeletecond"`
	RowIncludeCondition toolkit.M        `json:"rowincludecond",bson:"rowincludecond"`
	ConnectionInfo      *ConnectionInfo  `json:"connectioninfo",bson:"connectioninfo"`
	DestinationType     string           `json:"desttype",bson:"desttype"`
	Name                string           `json:"name",bson:"name"`
}

type ColumnSetting struct {
	Alias     string `json:"alias",bson:"alias"`
	Index     int    `json:"index",bson:"index"`
	Selector  string `json:"selector",bson:"selector"`
	ValueType string `json:"valuetype",bson:"valuetype"`
}
<|endoftext|>


<commit_before>package distVoteGen

import (
	"fmt"

	"github.com/marcuswestin/go-tp/gen/distVoteGen"
)

func Example() {
	err := run()
	if err != nil {
		panic(err)
	}
}

func run() (err error) {
	model := distVoteGen.NewModel(10)
	model.ProcessDocument("the cat sat on the mat")
	model.ProcessDocument("the cat hopes to see the hat")
	// model.PrintDistFreqs()
	model.Freeze()
	printSuggestions(model, "the cat")
	return
}

func printSuggestions(model *distVoteGen.Model, query string) {
	fmt.Println("Query:", query)
	suggestions := model.SuggestWord(query, 3)
	for _, suggestion := range suggestions {
		fmt.Println("\t", query, suggestion.String)
	}
}
<commit_msg>fix distVoteGen_test<commit_after>package distVoteGen

import (
	"fmt"
	"testing"

	"github.com/marcuswestin/go-tp/gen/distVoteGen"
)

func TestExample(t *testing.T) {
	model := distVoteGen.NewModel(10)
	model.ProcessDocument("the cat sat on the mat")
	model.ProcessDocument("the cat hopes to see the hat")
	// model.PrintDistFreqs()
	model.Freeze()
	printSuggestions(model, "the cat")
}

func printSuggestions(model *distVoteGen.Model, query string) {
	fmt.Println("Query:", query)
	suggestions := model.SuggestWord(query, 3)
	for _, suggestion := range suggestions {
		fmt.Println("\t", query, suggestion.String)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package doc

import (
	"bytes"
	"errors"
	"fmt"
	"net/http"
	"path"
	"regexp"
	"strings"
	"time"

	"github.com/Unknwon/com"

	"github.com/Unknwon/gowalker/models"
	"github.com/Unknwon/gowalker/modules/base"
	"github.com/Unknwon/gowalker/modules/setting"
)

var (
	githubRawHeader       = http.Header{"Accept": {"application/vnd.github-blob.raw"}}
	githubRevisionPattern = regexp.MustCompile(`data-clipboard-text="[a-z0-9A-Z]+`)
	githubPattern         = regexp.MustCompile(`^github\.com/(?P<owner>[a-z0-9A-Z_.\-]+)/(?P<repo>[a-z0-9A-Z_.\-]+)(?P<dir>/[a-z0-9A-Z_.\-/]*)?$`)
)

func getGithubRevision(importPath, tag string) (string, error) {
	data, err := com.HttpGetBytes(Client, fmt.Sprintf("https://%s/commits/"+tag, importPath), nil)
	if err != nil {
		return "", fmt.Errorf("fetch revision page: %v", err)
	}

	i := bytes.Index(data, []byte(`btn-outline`))
	if i == -1 {
		return "", errors.New("find revision locater: not found")
	}
	data = data[i+1:]
	m := githubRevisionPattern.FindSubmatch(data)
	if m == nil {
		return "", fmt.Errorf("find revision: not found")
	}
	return strings.TrimPrefix(string(m[0]), `data-clipboard-text="`), nil
}

type RepoInfo struct {
	DefaultBranch string `json:"default_branch"`
	Fork          bool   `json:"fork"`
	Parent        struct {
		FullName string `json:"full_name"`
	} `json:"parent"`
}

type RepoCommit struct {
	Commit struct {
		Committer struct {
			Date time.Time `json:"date"`
		} `json:"committer"`
	} `json:"commit"`
}

func getGithubDoc(match map[string]string, etag string) (_ *Package, err error) {
	match["cred"] = setting.GitHubCredentials

	repoInfo := new(RepoInfo)
	if err := com.HttpGetJSON(Client, com.Expand("https://api.github.com/repos/{owner}/{repo}?{cred}", match), repoInfo); err != nil {
		return nil, fmt.Errorf("get repo default branch: %v", err)
	}

	// Set default branch if not presented.
	if len(match["tag"]) == 0 {
		match["tag"] = repoInfo.DefaultBranch
	}

	// Check if last commit time is behind upstream for fork repository.
	if repoInfo.Fork {
		url := com.Expand("https://api.github.com/repos/{owner}/{repo}/commits?per_page=1&{cred}", match)
		forkCommits := make([]*RepoCommit, 0, 1)
		if err := com.HttpGetJSON(Client, url, &forkCommits); err != nil {
			return nil, fmt.Errorf("get fork repository commits: %v", err)
		}
		if len(forkCommits) == 0 {
			return nil, fmt.Errorf("unexpected zero number of fork repository commits: %s", url)
		}

		match["parent"] = repoInfo.Parent.FullName
		url = com.Expand("https://api.github.com/repos/{parent}/commits?per_page=1&{cred}", match)
		parentCommits := make([]*RepoCommit, 0, 1)
		if err := com.HttpGetJSON(Client, url, &parentCommits); err != nil {
			return nil, fmt.Errorf("get parent repository commits: %v", err)
		}
		if len(parentCommits) == 0 {
			return nil, fmt.Errorf("unexpected zero number of parent repository commits: %s", url)
		}

		if parentCommits[0].Commit.Committer.Date.After(forkCommits[0].Commit.Committer.Date) {
			return nil, fmt.Errorf("commits of this fork repository are behind or equal to its parent: %s", repoInfo.Parent.FullName)
		}
	}

	// Check revision.
	var commit string
	if strings.HasPrefix(match["importPath"], "gopkg.in") {
		// FIXME: get commit ID of gopkg.in indepdently.
		var obj struct {
			Sha string `json:"sha"`
		}

		if err := com.HttpGetJSON(Client,
			com.Expand("https://gopm.io/api/v1/revision?pkgname={importPath}", match), &obj); err != nil {
			return nil, fmt.Errorf("get gopkg.in revision: %v", err)
		}

		commit = obj.Sha
		match["tag"] = commit
		fmt.Println(commit)
	} else {
		commit, err = getGithubRevision(com.Expand("github.com/{owner}/{repo}", match), match["tag"])
		if err != nil {
			return nil, fmt.Errorf("get revision: %v", err)
		}
		if commit == etag {
			return nil, ErrPackageNotModified
		}
	}

	// Get files.
	var tree struct {
		Tree []struct {
			Url  string
			Path string
			Type string
		}
		Url string
	}

	if err := com.HttpGetJSON(Client,
		com.Expand("https://api.github.com/repos/{owner}/{repo}/git/trees/{tag}?recursive=1&{cred}", match), &tree); err != nil {
		return nil, fmt.Errorf("get tree: %v", err)
	}

	// Because Github API URLs are case-insensitive, we need to check that the
	// userRepo returned from Github matches the one that we are requesting.
	if !strings.HasPrefix(tree.Url, com.Expand("https://api.github.com/repos/{owner}/{repo}/", match)) {
		return nil, errors.New("GitHub import path has incorrect case")
	}

	// Get source file data and subdirectories.
	dirPrefix := match["dir"]
	if dirPrefix != "" {
		dirPrefix = dirPrefix[1:] + "/"
	}
	dirLevel := len(strings.Split(dirPrefix, "/"))
	dirLength := len(dirPrefix)
	dirMap := make(map[string]bool)
	files := make([]com.RawFile, 0, 10)

	for _, node := range tree.Tree {
		// Skip directories and files in wrong directories, get them later.
		if node.Type != "blob" || !strings.HasPrefix(node.Path, dirPrefix) {
			continue
		}

		// Get files and check if directories have acceptable files.
		if d, f := path.Split(node.Path); base.IsDocFile(f) {
			// Check if file is in the directory that is corresponding to import path.
			if d == dirPrefix {
				files = append(files, &Source{
					SrcName:   f,
					BrowseUrl: com.Expand("github.com/{owner}/{repo}/blob/{tag}/{0}", match, node.Path),
					RawSrcUrl: com.Expand("https://raw.github.com/{owner}/{repo}/{tag}/{0}?{1}", match, node.Path, setting.GitHubCredentials),
				})
				continue
			}

			// Otherwise, check if it's a direct sub-directory of import path.
			if len(strings.Split(d, "/"))-dirLevel == 1 {
				dirMap[d[dirLength:len(d)-1]] = true
				continue
			}
		}
	}

	dirs := base.MapToSortedStrings(dirMap)

	if len(files) == 0 && len(dirs) == 0 {
		return nil, ErrPackageNoGoFile
	} else if err := com.FetchFiles(Client, files, githubRawHeader); err != nil {
		return nil, fmt.Errorf("fetch files: %v", err)
	}

	// Start generating data.
	// IsGoSubrepo check has been placed to crawl.getDynamic.
	w := &Walker{
		LineFmt: "#L%d",
		Pdoc: &Package{
			PkgInfo: &models.PkgInfo{
				ImportPath:  match["importPath"],
				ProjectPath: com.Expand("github.com/{owner}/{repo}", match),
				ViewDirPath: com.Expand("github.com/{owner}/{repo}/tree/{tag}/{importPath}", match),
				Etag:        commit,
				Subdirs:     strings.Join(dirs, "|"),
			},
		},
	}

	srcs := make([]*Source, 0, len(files))
	srcMap := make(map[string]*Source)
	for _, f := range files {
		s, _ := f.(*Source)
		srcs = append(srcs, s)

		if !strings.HasSuffix(f.Name(), "_test.go") {
			srcMap[f.Name()] = s
		}
	}

	pdoc, err := w.Build(&WalkRes{
		WalkDepth: WD_All,
		WalkType:  WT_Memory,
		WalkMode:  WM_All,
		Srcs:      srcs,
	})
	if err != nil {
		return nil, fmt.Errorf("error walking package: %v", err)
	}

	// Get stars.
	var repoTree struct {
		Stars int64 `json:"watchers"`
	}
	if err := com.HttpGetJSON(Client,
		com.Expand("https://api.github.com/repos/{owner}/{repo}?{cred}", match), &repoTree); err != nil {
		return nil, fmt.Errorf("get repoTree: %v", err)
	}
	pdoc.Stars = repoTree.Stars

	return pdoc, nil
}
<commit_msg>fix opposite logic<commit_after>// Copyright 2015 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package doc

import (
	"bytes"
	"errors"
	"fmt"
	"net/http"
	"path"
	"regexp"
	"strings"
	"time"

	"github.com/Unknwon/com"

	"github.com/Unknwon/gowalker/models"
	"github.com/Unknwon/gowalker/modules/base"
	"github.com/Unknwon/gowalker/modules/setting"
)

var (
	githubRawHeader       = http.Header{"Accept": {"application/vnd.github-blob.raw"}}
	githubRevisionPattern = regexp.MustCompile(`data-clipboard-text="[a-z0-9A-Z]+`)
	githubPattern         = regexp.MustCompile(`^github\.com/(?P<owner>[a-z0-9A-Z_.\-]+)/(?P<repo>[a-z0-9A-Z_.\-]+)(?P<dir>/[a-z0-9A-Z_.\-/]*)?$`)
)

func getGithubRevision(importPath, tag string) (string, error) {
	data, err := com.HttpGetBytes(Client, fmt.Sprintf("https://%s/commits/"+tag, importPath), nil)
	if err != nil {
		return "", fmt.Errorf("fetch revision page: %v", err)
	}

	i := bytes.Index(data, []byte(`btn-outline`))
	if i == -1 {
		return "", errors.New("find revision locater: not found")
	}
	data = data[i+1:]
	m := githubRevisionPattern.FindSubmatch(data)
	if m == nil {
		return "", fmt.Errorf("find revision: not found")
	}
	return strings.TrimPrefix(string(m[0]), `data-clipboard-text="`), nil
}

type RepoInfo struct {
	DefaultBranch string `json:"default_branch"`
	Fork          bool   `json:"fork"`
	Parent        struct {
		FullName string `json:"full_name"`
	} `json:"parent"`
}

type RepoCommit struct {
	Commit struct {
		Committer struct {
			Date time.Time `json:"date"`
		} `json:"committer"`
	} `json:"commit"`
}

func getGithubDoc(match map[string]string, etag string) (_ *Package, err error) {
	match["cred"] = setting.GitHubCredentials

	repoInfo := new(RepoInfo)
	if err := com.HttpGetJSON(Client, com.Expand("https://api.github.com/repos/{owner}/{repo}?{cred}", match), repoInfo); err != nil {
		return nil, fmt.Errorf("get repo default branch: %v", err)
	}

	// Set default branch if not presented.
	if len(match["tag"]) == 0 {
		match["tag"] = repoInfo.DefaultBranch
	}

	// Check if last commit time is behind upstream for fork repository.
	if repoInfo.Fork {
		url := com.Expand("https://api.github.com/repos/{owner}/{repo}/commits?per_page=1&{cred}", match)
		forkCommits := make([]*RepoCommit, 0, 1)
		if err := com.HttpGetJSON(Client, url, &forkCommits); err != nil {
			return nil, fmt.Errorf("get fork repository commits: %v", err)
		}
		if len(forkCommits) == 0 {
			return nil, fmt.Errorf("unexpected zero number of fork repository commits: %s", url)
		}

		match["parent"] = repoInfo.Parent.FullName
		url = com.Expand("https://api.github.com/repos/{parent}/commits?per_page=1&{cred}", match)
		parentCommits := make([]*RepoCommit, 0, 1)
		if err := com.HttpGetJSON(Client, url, &parentCommits); err != nil {
			return nil, fmt.Errorf("get parent repository commits: %v", err)
		}
		if len(parentCommits) == 0 {
			return nil, fmt.Errorf("unexpected zero number of parent repository commits: %s", url)
		}

		if !parentCommits[0].Commit.Committer.Date.After(forkCommits[0].Commit.Committer.Date) {
			return nil, fmt.Errorf("commits of this fork repository are behind or equal to its parent: %s", repoInfo.Parent.FullName)
		}
	}

	// Check revision.
	var commit string
	if strings.HasPrefix(match["importPath"], "gopkg.in") {
		// FIXME: get commit ID of gopkg.in indepdently.
		var obj struct {
			Sha string `json:"sha"`
		}

		if err := com.HttpGetJSON(Client,
			com.Expand("https://gopm.io/api/v1/revision?pkgname={importPath}", match), &obj); err != nil {
			return nil, fmt.Errorf("get gopkg.in revision: %v", err)
		}

		commit = obj.Sha
		match["tag"] = commit
		fmt.Println(commit)
	} else {
		commit, err = getGithubRevision(com.Expand("github.com/{owner}/{repo}", match), match["tag"])
		if err != nil {
			return nil, fmt.Errorf("get revision: %v", err)
		}
		if commit == etag {
			return nil, ErrPackageNotModified
		}
	}

	// Get files.
	var tree struct {
		Tree []struct {
			Url  string
			Path string
			Type string
		}
		Url string
	}

	if err := com.HttpGetJSON(Client,
		com.Expand("https://api.github.com/repos/{owner}/{repo}/git/trees/{tag}?recursive=1&{cred}", match), &tree); err != nil {
		return nil, fmt.Errorf("get tree: %v", err)
	}

	// Because Github API URLs are case-insensitive, we need to check that the
	// userRepo returned from Github matches the one that we are requesting.
	if !strings.HasPrefix(tree.Url, com.Expand("https://api.github.com/repos/{owner}/{repo}/", match)) {
		return nil, errors.New("GitHub import path has incorrect case")
	}

	// Get source file data and subdirectories.
	dirPrefix := match["dir"]
	if dirPrefix != "" {
		dirPrefix = dirPrefix[1:] + "/"
	}
	dirLevel := len(strings.Split(dirPrefix, "/"))
	dirLength := len(dirPrefix)
	dirMap := make(map[string]bool)
	files := make([]com.RawFile, 0, 10)

	for _, node := range tree.Tree {
		// Skip directories and files in wrong directories, get them later.
		if node.Type != "blob" || !strings.HasPrefix(node.Path, dirPrefix) {
			continue
		}

		// Get files and check if directories have acceptable files.
		if d, f := path.Split(node.Path); base.IsDocFile(f) {
			// Check if file is in the directory that is corresponding to import path.
			if d == dirPrefix {
				files = append(files, &Source{
					SrcName:   f,
					BrowseUrl: com.Expand("github.com/{owner}/{repo}/blob/{tag}/{0}", match, node.Path),
					RawSrcUrl: com.Expand("https://raw.github.com/{owner}/{repo}/{tag}/{0}?{1}", match, node.Path, setting.GitHubCredentials),
				})
				continue
			}

			// Otherwise, check if it's a direct sub-directory of import path.
			if len(strings.Split(d, "/"))-dirLevel == 1 {
				dirMap[d[dirLength:len(d)-1]] = true
				continue
			}
		}
	}

	dirs := base.MapToSortedStrings(dirMap)

	if len(files) == 0 && len(dirs) == 0 {
		return nil, ErrPackageNoGoFile
	} else if err := com.FetchFiles(Client, files, githubRawHeader); err != nil {
		return nil, fmt.Errorf("fetch files: %v", err)
	}

	// Start generating data.
	// IsGoSubrepo check has been placed to crawl.getDynamic.
	w := &Walker{
		LineFmt: "#L%d",
		Pdoc: &Package{
			PkgInfo: &models.PkgInfo{
				ImportPath:  match["importPath"],
				ProjectPath: com.Expand("github.com/{owner}/{repo}", match),
				ViewDirPath: com.Expand("github.com/{owner}/{repo}/tree/{tag}/{importPath}", match),
				Etag:        commit,
				Subdirs:     strings.Join(dirs, "|"),
			},
		},
	}

	srcs := make([]*Source, 0, len(files))
	srcMap := make(map[string]*Source)
	for _, f := range files {
		s, _ := f.(*Source)
		srcs = append(srcs, s)

		if !strings.HasSuffix(f.Name(), "_test.go") {
			srcMap[f.Name()] = s
		}
	}

	pdoc, err := w.Build(&WalkRes{
		WalkDepth: WD_All,
		WalkType:  WT_Memory,
		WalkMode:  WM_All,
		Srcs:      srcs,
	})
	if err != nil {
		return nil, fmt.Errorf("error walking package: %v", err)
	}

	// Get stars.
	var repoTree struct {
		Stars int64 `json:"watchers"`
	}
	if err := com.HttpGetJSON(Client,
		com.Expand("https://api.github.com/repos/{owner}/{repo}?{cred}", match), &repoTree); err != nil {
		return nil, fmt.Errorf("get repoTree: %v", err)
	}
	pdoc.Stars = repoTree.Stars

	return pdoc, nil
}
<|endoftext|>


<commit_before>// This file is part of Monsti, a web content management system.
// Copyright 2012-2013 Christian Neumann
//
// Monsti is free software: you can redistribute it and/or modify it under the
// terms of the GNU Affero General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option) any
// later version.
//
// Monsti is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
// A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
// details.
//
// You should have received a copy of the GNU Affero General Public License
// along with Monsti.  If not, see <http://www.gnu.org/licenses/>.

/*
 Monsti is a simple and resource efficient CMS.

 This package implements the contactform node type.
*/
package main

import (
	"flag"
	"fmt"
	"github.com/chrneumann/mimemail"
	"pkg.monsti.org/form"
	"pkg.monsti.org/service"
	"pkg.monsti.org/util"
	"pkg.monsti.org/util/l10n"
	"pkg.monsti.org/util/template"
	htmlT "html/template"
	"log"
	"os"
)

var settings struct {
	Monsti util.MonstiSettings
}

var logger *log.Logger
var renderer template.Renderer

type contactFormData struct {
	Name, Email, Subject, Message string
}

func view(req service.Request, res *service.Response, s *service.Session) {
	G := l10n.UseCatalog(req.Session.Locale)
	data := contactFormData{}
	form := form.NewForm(&data, form.Fields{
		"Name":    form.Field{G("Name"), "", form.Required(G("Required.")), nil},
		"Email":   form.Field{G("Email"), "", form.Required(G("Required.")), nil},
		"Subject": form.Field{G("Subject"), "", form.Required(G("Required.")), nil},
		"Message": form.Field{G("Message"), "", form.Required(G("Required.")),
			new(form.TextArea)}})
	context := template.Context{}
	switch req.Method {
	case "GET":
		if _, submitted := req.Query["submitted"]; submitted {
			context["Submitted"] = 1
		}
	case "POST":
		if form.Fill(req.FormData) {
			mail := mimemail.Mail{
				From:    mimemail.Address{data.Name, data.Email},
				Subject: data.Subject,
				Body:    []byte(data.Message)}
			site := settings.Monsti.Sites[req.Site]
			owner := mimemail.Address{site.Owner.Name, site.Owner.Email}
			mail.To = []mimemail.Address{owner}
			mail = mimemail.Mail{}
			err := s.Mail().SendMail(&mail)
			if err != nil {
				panic("Could not send mail: " + err.Error())
			}
			res.Redirect = req.Node.Path + "/?submitted"
			return
		}
	default:
		panic("Request method not supported: " + req.Method)
	}
	res.Node = &req.Node
	body, err := s.Data().GetNodeData(req.Site, req.Node.Path, "body.html")
	if err != nil {
		panic("Could not get node data: " + err.Error())
	}
	context["Body"] = htmlT.HTML(string(body))
	context["Form"] = form.RenderData()
	fmt.Fprint(res, renderer.Render("contactform/view", context,
		req.Session.Locale, ""))
}

type editFormData struct {
	Title, Body string
}

func edit(req service.Request, res *service.Response, s *service.Session) {
	G := l10n.UseCatalog(req.Session.Locale)
	data := editFormData{}
	form := form.NewForm(&data, form.Fields{
		"Title": form.Field{G("Title"), "", form.Required(G("Required.")), nil},
		"Body": form.Field{G("Body"), "", form.Required(G("Required.")),
			new(form.AlohaEditor)}})
	dataCli := s.Data()
	switch req.Method {
	case "GET":
		data.Title = req.Node.Title
		nodeData, err := dataCli.GetNodeData(req.Site, req.Node.Path, "body.html")
		if err != nil {
			panic("Could not get node data: " + err.Error())
		}
		data.Body = string(nodeData)
	case "POST":
		if form.Fill(req.FormData) {
			node := req.Node
			node.Title = data.Title
			if err := dataCli.UpdateNode(req.Site, node); err != nil {
				panic("Could not update node: " + err.Error())
			}
			if err := dataCli.WriteNodeData(req.Site, req.Node.Path, "body.html",
				data.Body); err != nil {
				panic("Could not update node data: " + err.Error())
			}
			res.Redirect = req.Node.Path
			return
		}
	default:
		panic("Request method not supported: " + req.Method)
	}
	fmt.Fprint(res, renderer.Render("contactform/edit",
		template.Context{"Form": form.RenderData()},
		req.Session.Locale, ""))
}

func main() {
	logger = log.New(os.Stderr, "contactform ", log.LstdFlags)
	// Load configuration
	flag.Parse()
	if flag.NArg() != 1 {
		logger.Fatal("Expecting configuration path.")
	}
	cfgPath := util.GetConfigPath(flag.Arg(0))
	if err := util.LoadModuleSettings("contactform", cfgPath, &settings); err != nil {
		logger.Fatal("Could not load settings: ", err)
	}
	if err := settings.Monsti.LoadSiteSettings(); err != nil {
		logger.Fatal("Could not load site settings: ", err)
	}

	infoPath := settings.Monsti.GetServicePath(service.Info.String())

	l10n.Setup("monsti-contactform", settings.Monsti.GetLocalePath())
	renderer.Root = settings.Monsti.GetTemplatesPath()

	provider := service.NewNodeProvider(logger, infoPath)
	contactform := service.NodeTypeHandler{
		Name:       "ContactForm",
		ViewAction: view,
		EditAction: edit,
	}
	provider.AddNodeType(&contactform)
	if err := provider.Serve(settings.Monsti.GetServicePath(
		service.Node.String() + "_contactform")); err != nil {
		panic("Could not setup node provider: " + err.Error())
	}
}
<commit_msg>Use new API.<commit_after>// This file is part of Monsti, a web content management system.
// Copyright 2012-2013 Christian Neumann
//
// Monsti is free software: you can redistribute it and/or modify it under the
// terms of the GNU Affero General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option) any
// later version.
//
// Monsti is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
// A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
// details.
//
// You should have received a copy of the GNU Affero General Public License
// along with Monsti.  If not, see <http://www.gnu.org/licenses/>.

/*
 Monsti is a simple and resource efficient CMS.

 This package implements the contactform node type.
*/
package main

import (
	"flag"
	"fmt"
	htmlT "html/template"
	"log"
	"os"

	"github.com/chrneumann/mimemail"
	"pkg.monsti.org/form"
	"pkg.monsti.org/service"
	"pkg.monsti.org/util"
	"pkg.monsti.org/util/l10n"
	"pkg.monsti.org/util/template"
)

var settings struct {
	Monsti util.MonstiSettings
}

var logger *log.Logger
var renderer template.Renderer

type contactFormData struct {
	Name, Email, Subject, Message string
}

func view(req service.Request, res *service.Response, s *service.Session) error {
	G := l10n.UseCatalog(req.Session.Locale)
	data := contactFormData{}
	form := form.NewForm(&data, form.Fields{
		"Name":    form.Field{G("Name"), "", form.Required(G("Required.")), nil},
		"Email":   form.Field{G("Email"), "", form.Required(G("Required.")), nil},
		"Subject": form.Field{G("Subject"), "", form.Required(G("Required.")), nil},
		"Message": form.Field{G("Message"), "", form.Required(G("Required.")),
			new(form.TextArea)}})
	context := template.Context{}
	switch req.Method {
	case service.GetRequest:
		if _, submitted := req.Query["submitted"]; submitted {
			context["Submitted"] = 1
		}
	case service.PostRequest:
		if form.Fill(req.FormData) {
			mail := mimemail.Mail{
				From:    mimemail.Address{data.Name, data.Email},
				Subject: data.Subject,
				Body:    []byte(data.Message)}
			site := settings.Monsti.Sites[req.Site]
			owner := mimemail.Address{site.Owner.Name, site.Owner.Email}
			mail.To = []mimemail.Address{owner}
			err := s.Mail().SendMail(&mail)
			if err != nil {
				return fmt.Errorf("Could not send mail: %v", err)
			}
			res.Redirect = req.Node.Path + "/?submitted"
			return nil
		}
	default:
		return fmt.Errorf("Request method not supported: %v", req.Method)
	}
	res.Node = &req.Node
	body, err := s.Data().GetNodeData(req.Site, req.Node.Path, "body.html")
	if err != nil {
		return fmt.Errorf("Could not get node data: %v", err)
	}
	context["Body"] = htmlT.HTML(string(body))
	context["Form"] = form.RenderData()
	rendered, err := renderer.Render("contactform/view", context,
		req.Session.Locale, "")
	if err != nil {
		return fmt.Errorf("Could not render template: %v", err)
	}
	fmt.Fprint(res, rendered)
	return nil
}

type editFormData struct {
	Title, Body string
}

func edit(req service.Request, res *service.Response, s *service.Session) error {
	G := l10n.UseCatalog(req.Session.Locale)
	data := editFormData{}
	form := form.NewForm(&data, form.Fields{
		"Title": form.Field{G("Title"), "", form.Required(G("Required.")), nil},
		"Body": form.Field{G("Body"), "", form.Required(G("Required.")),
			new(form.AlohaEditor)}})
	dataCli := s.Data()
	switch req.Method {
	case service.GetRequest:
		data.Title = req.Node.Title
		nodeData, err := dataCli.GetNodeData(req.Site, req.Node.Path, "body.html")
		if err != nil {
			return fmt.Errorf("Could not get node data: %v", err)
		}
		data.Body = string(nodeData)
	case service.PostRequest:
		if form.Fill(req.FormData) {
			node := req.Node
			node.Title = data.Title
			if err := dataCli.UpdateNode(req.Site, node); err != nil {
				return fmt.Errorf("Could not update node: %v", err)
			}
			if err := dataCli.WriteNodeData(req.Site, req.Node.Path, "body.html",
				data.Body); err != nil {
				return fmt.Errorf("Could not update node data: %v", err)
			}
			res.Redirect = req.Node.Path
			return nil
		}
	default:
		return fmt.Errorf("Request method not supported: %v", req.Method)
	}
	rendered, err := renderer.Render("contactform/edit",
		template.Context{"Form": form.RenderData()},
		req.Session.Locale, "")
	if err != nil {
		return fmt.Errorf("Could not render template: %v", err)
	}
	fmt.Fprint(res, rendered)
	return nil
}

func main() {
	logger = log.New(os.Stderr, "contactform ", log.LstdFlags)
	// Load configuration
	flag.Parse()
	if flag.NArg() != 1 {
		logger.Fatal("Expecting configuration path.")
	}
	cfgPath := util.GetConfigPath(flag.Arg(0))
	if err := util.LoadModuleSettings("contactform", cfgPath, &settings); err != nil {
		logger.Fatal("Could not load settings: ", err)
	}
	if err := settings.Monsti.LoadSiteSettings(); err != nil {
		logger.Fatal("Could not load site settings: ", err)
	}

	infoPath := settings.Monsti.GetServicePath(service.Info.String())

	l10n.Setup("monsti-contactform", settings.Monsti.GetLocalePath())
	renderer.Root = settings.Monsti.GetTemplatesPath()

	provider := service.NewNodeProvider(logger, infoPath)
	contactform := service.NodeTypeHandler{
		Name:       "ContactForm",
		ViewAction: view,
		EditAction: edit,
	}
	provider.AddNodeType(&contactform)
	if err := provider.Serve(settings.Monsti.GetServicePath(
		service.Node.String() + "_contactform")); err != nil {
		panic("Could not setup node provider: " + err.Error())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net"
	"net/http"
	"net/rpc"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"regexp"
	"runtime"
	"syscall"
)

func createPidFile(filename string) (*os.File, error) {

	file, err := os.OpenFile(filename,
		os.O_WRONLY|os.O_CREATE|os.O_EXCL|os.O_SYNC,
		syscall.S_IRUSR|syscall.S_IWUSR)

	if err != nil {
		return nil, err
	}

	err = syscall.Flock(int(file.Fd()), syscall.F_WRLCK)
	if err != nil {
		file.Close()
		return nil, err
	}

	_, err = file.WriteString(fmt.Sprintf("%d", os.Getpid()))
	if err != nil {
		file.Close()
		return nil, err
	}

	return file, nil
}

func removePidFile(file *os.File) {

	err := syscall.Flock(int(file.Fd()), syscall.F_UNLCK)
	if err != nil {
		return
	}

	file.Close()
	os.Remove(file.Name())
}

type TranscodeQueue struct {
	Requests chan string
}

func NewTranscodeQueue() *TranscodeQueue {
	return &TranscodeQueue{
		Requests: make(chan string, 100),
	}
}

func (this *TranscodeQueue) DoTranscode(file string, reply *bool) error {
	log.Println("Asked to transcode:", file)
	this.Requests <- file
	*reply = true
	return nil
}

func performTranscode(infile, outfile string, isDone chan<- error) {

	transcodeCommand := exec.Command(
		"/usr/local/bin/HandBrakeCLI",
		"-O", "-I",
		"-f", "mp4",
		"--encoder", "x264",
		"--x264-preset", "faster",
		"--x264-tune", "film",
		"--h264-profile", "auto",
		"--h264-level", "auto",
		"--quality", "20",
		"--large-file",
		"--aencoder", "ca_aac,copy:ac3",
		"-B", "160",
		"--mixdown", "dpl2",
		"--aname", "English",
		"--loose-anamorphic",
		"--decomb",
		"--modulus", "2",
		"-i", infile,
		"-o", outfile)

	transferCommand := exec.Command("/usr/bin/scp", "-B", "-C", "-q", outfile,
		"guy@mediaserver.local:/srv/Media/Movies/0\\ -\\ Inbox/TV")

	log.Println("Performing transcode of", infile)
	log.Println("Writing file", outfile)

	err := transcodeCommand.Run()
	if err != nil {
		log.Println(err.Error())
		isDone <- err
		return
	}

	log.Println("Done transcoding file")
	log.Println("Transfering file to server")

	err = transferCommand.Run()
	if err == nil {
		os.Remove(infile)
		//os.Remove(outfile) // leave commented out until im sure i understand the scp problem
	} else {
		log.Println(err.Error())
		isDone <- err
		return
	}

	log.Println("Done transfering file to server")
	isDone <- nil
}

func requestTranscode(infile, unixSocket string) (reply bool, err error) {

	client, err := rpc.DialHTTPPath("unix", unixSocket, "/transcode")
	if err != nil {
		return
	}
	err = client.Call("TranscodeQueue.DoTranscode", infile, &reply)
	return
}

func createTranscodeServer(pidfile *os.File, unixSocket string) error {

	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)
	defer signal.Stop(sigc)

	q := NewTranscodeQueue()
	server := rpc.NewServer()
	server.Register(q)
	server.HandleHTTP("/transcode", "/transcode-debug")

	l, err := net.Listen("unix", unixSocket)
	if err != nil {
		return err
	}

	defer l.Close()

	go http.Serve(l, nil)

	isDone := make(chan error)
	rxp, err := regexp.Compile(`(.*)(\.mpg)$`)

	if err != nil {
		return err
	}

	q.Requests <- os.Args[1]

	for {
		select {
		case <-sigc:
			return nil
		case req := <-q.Requests:

			outfile := rxp.ReplaceAllString(req, "$1.m4v")
			go performTranscode(req, outfile, isDone)

			select {
			case <-sigc:
				return nil
			case e := <-isDone:
				if e != nil {
					return e
				}
			}
		default:
			return nil
		}
	}
}

func main() {

	if len(os.Args) != 2 {
		return
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	log.Println("Starting...")

	toTranscode := os.Args[1]
	pidFileName := path.Join(os.TempDir(), "transcodequeue.pid")
	unixSocket := path.Join(os.TempDir(), "transcodequeue.sock")

	pidFile, err := createPidFile(pidFileName)

	if err != nil {
		log.Println("Sending server request to transcode", toTranscode)

		reply, err := requestTranscode(toTranscode, unixSocket)
		if err != nil {
			log.Panicln(err.Error())
		}

		if reply {
			log.Println("Successfully requested transcode")
		}
	} else {
		log.Println("Creating Server")

		defer removePidFile(pidFile)
		err = createTranscodeServer(pidFile, unixSocket)
		if err != nil {
			log.Panicln(err.Error())
		}
	}

	log.Println("Finished")
}
<commit_msg>Cleaned up main.go a bit<commit_after>package main

import (
	"fmt"
	"log"
	"net"
	"net/http"
	"net/rpc"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"regexp"
	"runtime"
	"syscall"
)

func createPidFile(filename string) (*os.File, error) {

	file, err := os.OpenFile(filename,
		os.O_WRONLY|os.O_CREATE|os.O_EXCL|os.O_SYNC,
		syscall.S_IRUSR|syscall.S_IWUSR)

	if err != nil {
		return nil, err
	}

	err = syscall.Flock(int(file.Fd()), syscall.F_WRLCK)
	if err != nil {
		file.Close()
		return nil, err
	}

	_, err = file.WriteString(fmt.Sprintf("%d", os.Getpid()))
	if err != nil {
		file.Close()
		return nil, err
	}

	return file, nil
}

func removePidFile(file *os.File) {

	err := syscall.Flock(int(file.Fd()), syscall.F_UNLCK)
	if err != nil {
		return
	}

	file.Close()
	os.Remove(file.Name())
}

type TranscodeQueue struct {
	Requests chan string
}

func NewTranscodeQueue() *TranscodeQueue {
	return &TranscodeQueue{
		Requests: make(chan string, 100),
	}
}

func (this *TranscodeQueue) DoTranscode(file string, reply *bool) error {
	log.Println("Asked to transcode:", file)
	this.Requests <- file
	*reply = true
	return nil
}

func performTranscode(infile, outfile string, isDone chan<- error) {

	transcodeCommand := exec.Command(
		"/usr/local/bin/HandBrakeCLI",
		"-O", "-I",
		"-f", "mp4",
		"--encoder", "x264",
		"--x264-preset", "faster",
		"--x264-tune", "film",
		"--h264-profile", "auto",
		"--h264-level", "auto",
		"--quality", "20",
		"--large-file",
		"--aencoder", "ca_aac,copy:ac3",
		"-B", "160",
		"--mixdown", "dpl2",
		"--aname", "English",
		"--loose-anamorphic",
		"--decomb",
		"--modulus", "2",
		"-i", infile,
		"-o", outfile)

	transferCommand := exec.Command("/usr/bin/scp", "-B", "-C", "-q", outfile,
		"guy@mediaserver.local:/srv/Media/Movies/0\\ -\\ Inbox/TV")

	log.Println("Performing transcode of", infile)
	log.Println("Writing file", outfile)

	err := transcodeCommand.Run()
	if err != nil {
		log.Println(err.Error())
		isDone <- err
		return
	}

	log.Println("Done transcoding file")
	log.Println("Transfering file to server")

	err = transferCommand.Run()
	if err == nil {
		os.Remove(infile)
		//os.Remove(outfile) // leave commented out until im sure i understand the scp problem
	} else {
		log.Println(err.Error())
		isDone <- err
		return
	}

	log.Println("Done transfering file to server")
	isDone <- nil
}

func requestTranscode(infile, unixSocket string) (reply bool, err error) {

	client, err := rpc.DialHTTPPath("unix", unixSocket, "/transcode")
	if err != nil {
		return
	}
	err = client.Call("TranscodeQueue.DoTranscode", infile, &reply)
	return
}

func createTranscodeServer(pidfile *os.File, unixSocket string) error {

	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)
	defer signal.Stop(sigc)

	q := NewTranscodeQueue()
	server := rpc.NewServer()
	server.Register(q)
	server.HandleHTTP("/transcode", "/transcode-debug")

	l, err := net.Listen("unix", unixSocket)
	if err != nil {
		return err
	}

	defer l.Close()

	go http.Serve(l, nil)

	isDone := make(chan error)
	rxp, err := regexp.Compile(`(.*)(\.mpg)$`)

	if err != nil {
		return err
	}

	q.Requests <- os.Args[1]

	for {
		select {
		case <-sigc:
			return nil
		case req := <-q.Requests:

			outfile := rxp.ReplaceAllString(req, "$1.m4v")
			go performTranscode(req, outfile, isDone)

			select {
			case <-sigc:
				return nil
			case e := <-isDone:
				if e != nil {
					return e
				}
			}
		default:
			return nil
		}
	}
}

func main() {

	if len(os.Args) != 2 {
		return
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	log.Println("Starting...")

	toTranscode := os.Args[1]
	pidFileName := path.Join(os.TempDir(), "transcodequeue.pid")
	unixSocket := path.Join(os.TempDir(), "transcodequeue.sock")

	pidFile, err := createPidFile(pidFileName)

	if err != nil {
		log.Println("Sending server request to transcode", toTranscode)

		reply, err := requestTranscode(toTranscode, unixSocket)
		if err != nil {
			log.Panicln(err.Error())
		}

		if reply {
			log.Println("Successfully requested transcode")
		}
	} else {
		log.Println("Creating Server")

		defer removePidFile(pidFile)

		err = createTranscodeServer(pidFile, unixSocket)
		if err != nil {
			log.Panicln(err.Error())
		}
	}

	log.Println("Finished")
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package heap provides heap operations for any type that implements
// heap.Interface. A heap is a tree with the property that each node is the
// minimum-valued node in its subtree.
//
// The minimum element in the tree is the root, at index 0.
//
// A heap is a common way to implement a priority queue. To build a priority
// queue, implement the Heap interface with the (negative) priority as the
// ordering for the Less method, so Push adds items while Pop removes the
// highest-priority item from the queue. The Examples include such an
// implementation; the file example_pq_test.go has the complete source.
//
package heap

import "sort"

// Any type that implements heap.Interface may be used as a
// min-heap with the following invariants (established after
// Init has been called or if the data is empty or sorted):
//
//	!h.Less(j, i) for 0 <= i < h.Len() and 2*i+1 <= j <= 2*i+2 and j < h.Len()
//
// Note that Push and Pop in this interface are for package heap's
// implementation to call. To add and remove things from the heap,
// use heap.Push and heap.Pop.
type Interface interface {
	sort.Interface
	Push(x interface{}) // add x as element Len()
	Pop() interface{}   // remove and return element Len() - 1.
}

// A heap must be initialized before any of the heap operations
// can be used. Init is idempotent with respect to the heap invariants
// and may be called whenever the heap invariants may have been invalidated.
// Its complexity is O(n) where n = h.Len().
//
func Init(h Interface) {
	// heapify
	n := h.Len()
	for i := n/2 - 1; i >= 0; i-- {
		down(h, i, n)
	}
}

// Push pushes the element x onto the heap. The complexity is
// O(log(n)) where n = h.Len().
//
func Push(h Interface, x interface{}) {
	h.Push(x)
	up(h, h.Len()-1)
}

// Pop removes the minimum element (according to Less) from the heap
// and returns it. The complexity is O(log(n)) where n = h.Len().
// It is equivalent to Remove(h, 0).
//
func Pop(h Interface) interface{} {
	n := h.Len() - 1
	h.Swap(0, n)
	down(h, 0, n)
	return h.Pop()
}

// Remove removes the element at index i from the heap.
// The complexity is O(log(n)) where n = h.Len().
//
func Remove(h Interface, i int) interface{} {
	n := h.Len() - 1
	if n != i {
		h.Swap(i, n)
		if !down(h, i, n) {
			up(h, i)
		}
	}
	return h.Pop()
}

// Fix re-establishes the heap ordering after the element at index i has changed its value.
// Changing the value of the element at index i and then calling Fix is equivalent to,
// but less expensive than, calling Remove(h, i) followed by a Push of the new value.
// The complexity is O(log(n)) where n = h.Len().
func Fix(h Interface, i int) {
	if !down(h, i, h.Len()) {
		up(h, i)
	}
}

func up(h Interface, j int) {
	for {
		i := (j - 1) / 2 // parent
		if i == j || !h.Less(j, i) {
			break
		}
		h.Swap(i, j)
		j = i
	}
}

func down(h Interface, i0, n int) bool {
	i := i0
	for {
		j1 := 2*i + 1
		if j1 >= n || j1 < 0 { // j1 < 0 after int overflow
			break
		}
		j := j1 // left child
		if j2 := j1 + 1; j2 < n && h.Less(j2, j1) {
			j = j2 // = 2*i + 2  // right child
		}
		if !h.Less(j, i) {
			break
		}
		h.Swap(i, j)
		i = j
	}
	return i > i0
}
<commit_msg>container/heap: fix comments style<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package heap provides heap operations for any type that implements
// heap.Interface. A heap is a tree with the property that each node is the
// minimum-valued node in its subtree.
//
// The minimum element in the tree is the root, at index 0.
//
// A heap is a common way to implement a priority queue. To build a priority
// queue, implement the Heap interface with the (negative) priority as the
// ordering for the Less method, so Push adds items while Pop removes the
// highest-priority item from the queue. The Examples include such an
// implementation; the file example_pq_test.go has the complete source.
//
package heap

import "sort"

// The Interface type describes the requirements
// for a type using the routines in this package.
// Any type that implements it may be used as a
// min-heap with the following invariants (established after
// Init has been called or if the data is empty or sorted):
//
//	!h.Less(j, i) for 0 <= i < h.Len() and 2*i+1 <= j <= 2*i+2 and j < h.Len()
//
// Note that Push and Pop in this interface are for package heap's
// implementation to call. To add and remove things from the heap,
// use heap.Push and heap.Pop.
type Interface interface {
	sort.Interface
	Push(x interface{}) // add x as element Len()
	Pop() interface{}   // remove and return element Len() - 1.
}

// Init establishes the heap invariants required by the other routines in this package.
// Init is idempotent with respect to the heap invariants
// and may be called whenever the heap invariants may have been invalidated.
// Its complexity is O(n) where n = h.Len().
func Init(h Interface) {
	// heapify
	n := h.Len()
	for i := n/2 - 1; i >= 0; i-- {
		down(h, i, n)
	}
}

// Push pushes the element x onto the heap. The complexity is
// O(log(n)) where n = h.Len().
//
func Push(h Interface, x interface{}) {
	h.Push(x)
	up(h, h.Len()-1)
}

// Pop removes the minimum element (according to Less) from the heap
// and returns it. The complexity is O(log(n)) where n = h.Len().
// It is equivalent to Remove(h, 0).
//
func Pop(h Interface) interface{} {
	n := h.Len() - 1
	h.Swap(0, n)
	down(h, 0, n)
	return h.Pop()
}

// Remove removes the element at index i from the heap.
// The complexity is O(log(n)) where n = h.Len().
//
func Remove(h Interface, i int) interface{} {
	n := h.Len() - 1
	if n != i {
		h.Swap(i, n)
		if !down(h, i, n) {
			up(h, i)
		}
	}
	return h.Pop()
}

// Fix re-establishes the heap ordering after the element at index i has changed its value.
// Changing the value of the element at index i and then calling Fix is equivalent to,
// but less expensive than, calling Remove(h, i) followed by a Push of the new value.
// The complexity is O(log(n)) where n = h.Len().
func Fix(h Interface, i int) {
	if !down(h, i, h.Len()) {
		up(h, i)
	}
}

func up(h Interface, j int) {
	for {
		i := (j - 1) / 2 // parent
		if i == j || !h.Less(j, i) {
			break
		}
		h.Swap(i, j)
		j = i
	}
}

func down(h Interface, i0, n int) bool {
	i := i0
	for {
		j1 := 2*i + 1
		if j1 >= n || j1 < 0 { // j1 < 0 after int overflow
			break
		}
		j := j1 // left child
		if j2 := j1 + 1; j2 < n && h.Less(j2, j1) {
			j = j2 // = 2*i + 2  // right child
		}
		if !h.Less(j, i) {
			break
		}
		h.Swap(i, j)
		i = j
	}
	return i > i0
}
<|endoftext|>


<commit_before>package vars

import (
	"fmt"
	"regexp"
	"sort"
	"strings"

	"github.com/hashicorp/go-multierror"
	"github.com/pkg/errors"
	"gopkg.in/yaml.v2"
)

type Template struct {
	bytes []byte
}

type EvaluateOpts struct {
	ExpectAllKeys     bool
	ExpectAllVarsUsed bool
}

func NewTemplate(bytes []byte) Template {
	return Template{bytes: bytes}
}

func (t Template) ExtraVarNames() []string {
	return interpolator{}.extractVarNames(string(t.bytes))
}

func (t Template) Evaluate(vars Variables, opts EvaluateOpts) ([]byte, error) {
	var obj interface{}

	err := yaml.Unmarshal(t.bytes, &obj)
	if err != nil {
		return []byte{}, err
	}

	obj, err = t.interpolateRoot(obj, newVarsTracker(vars, opts.ExpectAllKeys, opts.ExpectAllVarsUsed))
	if err != nil {
		return []byte{}, err
	}

	bytes, err := yaml.Marshal(obj)
	if err != nil {
		return []byte{}, err
	}

	return bytes, nil
}

func (t Template) interpolateRoot(obj interface{}, tracker varsTracker) (interface{}, error) {
	var err error
	obj, err = interpolator{}.Interpolate(obj, varsLookup{tracker})
	if err != nil {
		return nil, err
	}

	return obj, tracker.Error()
}

type interpolator struct{}

var (
	interpolationRegex         = regexp.MustCompile(`\(\((!?([-/\.\w\pL]+\:)?[-/\.\w\pL]+)\)\)`)
	interpolationAnchoredRegex = regexp.MustCompile("\\A" + interpolationRegex.String() + "\\z")
)

func (i interpolator) Interpolate(node interface{}, varsLookup varsLookup) (interface{}, error) {
	switch typedNode := node.(type) {
	case map[interface{}]interface{}:
		for k, v := range typedNode {
			evaluatedValue, err := i.Interpolate(v, varsLookup)
			if err != nil {
				return nil, err
			}

			evaluatedKey, err := i.Interpolate(k, varsLookup)
			if err != nil {
				return nil, err
			}

			delete(typedNode, k) // delete in case key has changed
			typedNode[evaluatedKey] = evaluatedValue
		}

	case []interface{}:
		for idx, x := range typedNode {
			var err error
			typedNode[idx], err = i.Interpolate(x, varsLookup)
			if err != nil {
				return nil, err
			}
		}

	case string:
		for _, name := range i.extractVarNames(typedNode) {
			foundVal, found, err := varsLookup.Get(name)
			if err != nil {
				return nil, errors.WithMessagef(err, "Finding variable '%s'", name)
			}

			if found {
				// ensure that value type is preserved when replacing the entire field
				if interpolationAnchoredRegex.MatchString(typedNode) {
					return foundVal, nil
				}

				switch foundVal.(type) {
				case string, int, int16, int32, int64, uint, uint16, uint32, uint64:
					foundValStr := fmt.Sprintf("%v", foundVal)
					typedNode = strings.Replace(typedNode, fmt.Sprintf("((%s))", name), foundValStr, -1)
					typedNode = strings.Replace(typedNode, fmt.Sprintf("((!%s))", name), foundValStr, -1)
				default:
					return nil, InvalidInterpolationError{
						Path:  name,
						Value: foundVal,
					}
				}
			}
		}

		return typedNode, nil
	}

	return node, nil
}

func (i interpolator) extractVarNames(value string) []string {
	var names []string

	for _, match := range interpolationRegex.FindAllSubmatch([]byte(value), -1) {
		names = append(names, strings.TrimPrefix(string(match[1]), "!"))
	}

	return names
}

type varsLookup struct {
	varsTracker
}

var ErrEmptyVar = errors.New("empty var")

// Get value of a var. Name can be the following formats: 1) 'foo', where foo
// is var name; 2) 'foo:bar', where foo is var source name, and bar is var name;
// 3) '.:foo', where . means a local var, foo is var name.
func (l varsLookup) Get(name string) (interface{}, bool, error) {
	var splitName []string
	if strings.Index(name, ":") > 0 {
		parts := strings.Split(name, ":")
		splitName = strings.Split(parts[1], ".")
		splitName[0] = fmt.Sprintf("%s:%s", parts[0], splitName[0])
	} else {
		splitName = strings.Split(name, ".")
	}

	// this should be impossible since interpolationRegex only matches non-empty
	// vars, but better to error than to panic
	if len(splitName) == 0 {
		return nil, false, ErrEmptyVar
	}

	val, found, err := l.varsTracker.Get(splitName[0])
	if !found || err != nil {
		return val, found, err
	}

	for _, seg := range splitName[1:] {
		switch v := val.(type) {
		case map[interface{}]interface{}:
			var found bool
			val, found = v[seg]
			if !found {
				return nil, false, MissingFieldError{
					Path:  name,
					Field: seg,
				}
			}
		case map[string]interface{}:
			var found bool
			val, found = v[seg]
			if !found {
				return nil, false, MissingFieldError{
					Path:  name,
					Field: seg,
				}
			}
		default:
			return nil, false, InvalidFieldError{
				Path:  name,
				Field: seg,
				Value: val,
			}
		}
	}

	return val, true, err
}

type varsTracker struct {
	vars Variables

	expectAllFound bool
	expectAllUsed  bool

	missing    map[string]struct{} // track missing var names
	visited    map[string]struct{}
	visitedAll map[string]struct{} // track all var names that were accessed
}

func newVarsTracker(vars Variables, expectAllFound, expectAllUsed bool) varsTracker {
	return varsTracker{
		vars:           vars,
		expectAllFound: expectAllFound,
		expectAllUsed:  expectAllUsed,
		missing:        map[string]struct{}{},
		visited:        map[string]struct{}{},
		visitedAll:     map[string]struct{}{},
	}
}

func (t varsTracker) Get(name string) (interface{}, bool, error) {
	t.visitedAll[name] = struct{}{}

	val, found, err := t.vars.Get(VariableDefinition{Name: name})
	if !found {
		t.missing[name] = struct{}{}
	}

	return val, found, err
}

func (t varsTracker) Error() error {
	missingErr := t.MissingError()
	extraErr := t.ExtraError()
	if missingErr != nil && extraErr != nil {
		return multierror.Append(missingErr, extraErr)
	} else if missingErr != nil {
		return missingErr
	} else if extraErr != nil {
		return extraErr
	}

	return nil
}

func (t varsTracker) MissingError() error {
	if !t.expectAllFound || len(t.missing) == 0 {
		return nil
	}

	return UndefinedVarsError{Vars: names(t.missing)}
}

func (t varsTracker) ExtraError() error {
	if !t.expectAllUsed {
		return nil
	}

	allDefs, err := t.vars.List()
	if err != nil {
		return err
	}

	unusedNames := map[string]struct{}{}

	for _, def := range allDefs {
		if _, found := t.visitedAll[def.Name]; !found {
			unusedNames[def.Name] = struct{}{}
		}
	}

	if len(unusedNames) == 0 {
		return nil
	}

	return UnusedVarsError{Vars: names(unusedNames)}
}

func names(mapWithNames map[string]struct{}) []string {
	var names []string
	for name, _ := range mapWithNames {
		names = append(names, name)
	}

	sort.Strings(names)

	return names
}
<commit_msg>vars: use go std err wrapping<commit_after>package vars

import (
	"errors"
	"fmt"
	"regexp"
	"sort"
	"strings"

	"github.com/hashicorp/go-multierror"
	"gopkg.in/yaml.v2"
)

type Template struct {
	bytes []byte
}

type EvaluateOpts struct {
	ExpectAllKeys     bool
	ExpectAllVarsUsed bool
}

func NewTemplate(bytes []byte) Template {
	return Template{bytes: bytes}
}

func (t Template) ExtraVarNames() []string {
	return interpolator{}.extractVarNames(string(t.bytes))
}

func (t Template) Evaluate(vars Variables, opts EvaluateOpts) ([]byte, error) {
	var obj interface{}

	err := yaml.Unmarshal(t.bytes, &obj)
	if err != nil {
		return []byte{}, err
	}

	obj, err = t.interpolateRoot(obj, newVarsTracker(vars, opts.ExpectAllKeys, opts.ExpectAllVarsUsed))
	if err != nil {
		return []byte{}, err
	}

	bytes, err := yaml.Marshal(obj)
	if err != nil {
		return []byte{}, err
	}

	return bytes, nil
}

func (t Template) interpolateRoot(obj interface{}, tracker varsTracker) (interface{}, error) {
	var err error
	obj, err = interpolator{}.Interpolate(obj, varsLookup{tracker})
	if err != nil {
		return nil, err
	}

	return obj, tracker.Error()
}

type interpolator struct{}

var (
	interpolationRegex         = regexp.MustCompile(`\(\((!?([-/\.\w\pL]+\:)?[-/\.\w\pL]+)\)\)`)
	interpolationAnchoredRegex = regexp.MustCompile("\\A" + interpolationRegex.String() + "\\z")
)

func (i interpolator) Interpolate(node interface{}, varsLookup varsLookup) (interface{}, error) {
	switch typedNode := node.(type) {
	case map[interface{}]interface{}:
		for k, v := range typedNode {
			evaluatedValue, err := i.Interpolate(v, varsLookup)
			if err != nil {
				return nil, err
			}

			evaluatedKey, err := i.Interpolate(k, varsLookup)
			if err != nil {
				return nil, err
			}

			delete(typedNode, k) // delete in case key has changed
			typedNode[evaluatedKey] = evaluatedValue
		}

	case []interface{}:
		for idx, x := range typedNode {
			var err error
			typedNode[idx], err = i.Interpolate(x, varsLookup)
			if err != nil {
				return nil, err
			}
		}

	case string:
		for _, name := range i.extractVarNames(typedNode) {
			foundVal, found, err := varsLookup.Get(name)
			if err != nil {
				return nil, fmt.Errorf("var lookup '%s': %w", name, err)
			}

			if found {
				// ensure that value type is preserved when replacing the entire field
				if interpolationAnchoredRegex.MatchString(typedNode) {
					return foundVal, nil
				}

				switch foundVal.(type) {
				case string, int, int16, int32, int64, uint, uint16, uint32, uint64:
					foundValStr := fmt.Sprintf("%v", foundVal)
					typedNode = strings.Replace(typedNode, fmt.Sprintf("((%s))", name), foundValStr, -1)
					typedNode = strings.Replace(typedNode, fmt.Sprintf("((!%s))", name), foundValStr, -1)
				default:
					return nil, InvalidInterpolationError{
						Path:  name,
						Value: foundVal,
					}
				}
			}
		}

		return typedNode, nil
	}

	return node, nil
}

func (i interpolator) extractVarNames(value string) []string {
	var names []string

	for _, match := range interpolationRegex.FindAllSubmatch([]byte(value), -1) {
		names = append(names, strings.TrimPrefix(string(match[1]), "!"))
	}

	return names
}

type varsLookup struct {
	varsTracker
}

var ErrEmptyVar = errors.New("empty var")

// Get value of a var. Name can be the following formats: 1) 'foo', where foo
// is var name; 2) 'foo:bar', where foo is var source name, and bar is var name;
// 3) '.:foo', where . means a local var, foo is var name.
func (l varsLookup) Get(name string) (interface{}, bool, error) {
	var splitName []string
	if strings.Index(name, ":") > 0 {
		parts := strings.Split(name, ":")
		splitName = strings.Split(parts[1], ".")
		splitName[0] = fmt.Sprintf("%s:%s", parts[0], splitName[0])
	} else {
		splitName = strings.Split(name, ".")
	}

	// this should be impossible since interpolationRegex only matches non-empty
	// vars, but better to error than to panic
	if len(splitName) == 0 {
		return nil, false, ErrEmptyVar
	}

	val, found, err := l.varsTracker.Get(splitName[0])
	if !found || err != nil {
		return val, found, err
	}

	for _, seg := range splitName[1:] {
		switch v := val.(type) {
		case map[interface{}]interface{}:
			var found bool
			val, found = v[seg]
			if !found {
				return nil, false, MissingFieldError{
					Path:  name,
					Field: seg,
				}
			}
		case map[string]interface{}:
			var found bool
			val, found = v[seg]
			if !found {
				return nil, false, MissingFieldError{
					Path:  name,
					Field: seg,
				}
			}
		default:
			return nil, false, InvalidFieldError{
				Path:  name,
				Field: seg,
				Value: val,
			}
		}
	}

	return val, true, err
}

type varsTracker struct {
	vars Variables

	expectAllFound bool
	expectAllUsed  bool

	missing    map[string]struct{} // track missing var names
	visited    map[string]struct{}
	visitedAll map[string]struct{} // track all var names that were accessed
}

func newVarsTracker(vars Variables, expectAllFound, expectAllUsed bool) varsTracker {
	return varsTracker{
		vars:           vars,
		expectAllFound: expectAllFound,
		expectAllUsed:  expectAllUsed,
		missing:        map[string]struct{}{},
		visited:        map[string]struct{}{},
		visitedAll:     map[string]struct{}{},
	}
}

func (t varsTracker) Get(name string) (interface{}, bool, error) {
	t.visitedAll[name] = struct{}{}

	val, found, err := t.vars.Get(VariableDefinition{Name: name})
	if !found {
		t.missing[name] = struct{}{}
	}

	return val, found, err
}

func (t varsTracker) Error() error {
	missingErr := t.MissingError()
	extraErr := t.ExtraError()
	if missingErr != nil && extraErr != nil {
		return multierror.Append(missingErr, extraErr)
	} else if missingErr != nil {
		return missingErr
	} else if extraErr != nil {
		return extraErr
	}

	return nil
}

func (t varsTracker) MissingError() error {
	if !t.expectAllFound || len(t.missing) == 0 {
		return nil
	}

	return UndefinedVarsError{Vars: names(t.missing)}
}

func (t varsTracker) ExtraError() error {
	if !t.expectAllUsed {
		return nil
	}

	allDefs, err := t.vars.List()
	if err != nil {
		return err
	}

	unusedNames := map[string]struct{}{}

	for _, def := range allDefs {
		if _, found := t.visitedAll[def.Name]; !found {
			unusedNames[def.Name] = struct{}{}
		}
	}

	if len(unusedNames) == 0 {
		return nil
	}

	return UnusedVarsError{Vars: names(unusedNames)}
}

func names(mapWithNames map[string]struct{}) []string {
	var names []string
	for name, _ := range mapWithNames {
		names = append(names, name)
	}

	sort.Strings(names)

	return names
}
<|endoftext|>


<commit_before>package redis

import (
	"bytes"
	"errors"
	"fmt"
	"net"
	"reflect"
	"time"
)

const (
	bufSize int = 4096
)

// ErrPipelineEmpty is returned from PipeResp() to indicate that all commands
// which were put into the pipeline have had their responses read
var ErrPipelineEmpty = errors.New("pipeline queue empty")

// Client describes a Redis client.
type Client struct {
	conn         net.Conn
	respReader   *RespReader
	timeout      time.Duration
	pending      []request
	writeScratch []byte
	writeBuf     *bytes.Buffer

	completed, completedHead []*Resp

	// The network/address of the redis instance this client is connected to.
	// These will be whatever strings were passed into the Dial function when
	// creating this connection
	Network, Addr string

	// The most recent critical network error which occurred when either reading
	// or writing. A critical network error is one in which the connection was
	// found to be no longer usable; in essence, any error except a timeout.
	// Close is automatically called on the client when it encounters a critical
	// network error
	LastCritical error
}

// request describes a client's request to the redis server
type request struct {
	cmd  string
	args []interface{}
}

// DialTimeout connects to the given Redis server with the given timeout, which
// will be used as the read/write timeout when communicating with redis
func DialTimeout(network, addr string, timeout time.Duration) (*Client, error) {
	// establish a connection
	conn, err := net.DialTimeout(network, addr, timeout)
	if err != nil {
		return nil, err
	}

	completed := make([]*Resp, 0, 10)
	return &Client{
		conn:          conn,
		respReader:    NewRespReader(conn),
		timeout:       timeout,
		writeScratch:  make([]byte, 0, 128),
		writeBuf:      bytes.NewBuffer(make([]byte, 0, 128)),
		completed:     completed,
		completedHead: completed,
		Network:       network,
		Addr:          addr,
	}, nil
}

// Dial connects to the given Redis server.
func Dial(network, addr string) (*Client, error) {
	return DialTimeout(network, addr, time.Duration(0))
}

// Close closes the connection.
func (c *Client) Close() error {
	return c.conn.Close()
}

// Cmd calls the given Redis command.
func (c *Client) Cmd(cmd string, args ...interface{}) *Resp {
	err := c.writeRequest(request{cmd, args})
	if err != nil {
		return newRespIOErr(err)
	}
	return c.ReadResp()
}

// PipeAppend adds the given call to the pipeline queue.
// Use PipeResp() to read the response.
func (c *Client) PipeAppend(cmd string, args ...interface{}) {
	c.pending = append(c.pending, request{cmd, args})
}

// PipeResp returns the reply for the next request in the pipeline queue. Err
// with ErrPipelineEmpty is returned if the pipeline queue is empty.
func (c *Client) PipeResp() *Resp {
	if len(c.completed) > 0 {
		r := c.completed[0]
		c.completed = c.completed[1:]
		return r
	}

	if len(c.pending) == 0 {
		return NewResp(ErrPipelineEmpty)
	}

	nreqs := len(c.pending)
	err := c.writeRequest(c.pending...)
	c.pending = nil
	if err != nil {
		return newRespIOErr(err)
	}
	c.completed = c.completedHead
	for i := 0; i < nreqs; i++ {
		r := c.ReadResp()
		c.completed = append(c.completed, r)
	}

	// At this point c.completed should have something in it
	return c.PipeResp()
}

// PipeClear clears the contents of the current pipeline queue, both commands
// queued by PipeAppend which have yet to be sent and responses which have yet
// to be retrieved through PipeResp. The first returned int will be the number
// of pending commands dropped, the second will be the number of pending
// responses dropped
func (c *Client) PipeClear() (int, int) {
	callCount, replyCount := len(c.pending), len(c.completed)
	if callCount > 0 {
		c.pending = nil
	}
	if replyCount > 0 {
		c.completed = nil
	}
	return callCount, replyCount
}

// ReadResp will read a Resp off of the connection without sending anything
// first (useful after you've sent a SUSBSCRIBE command). This will block until
// a reply is received or the timeout is reached (returning the IOErr). You can
// use IsTimeout to check if the Resp is due to a Timeout
//
// Note: this is a more low-level function, you really shouldn't have to
// actually use it unless you're writing your own pub/sub code
func (c *Client) ReadResp() *Resp {
	if c.timeout != 0 {
		c.conn.SetReadDeadline(time.Now().Add(c.timeout))
	}
	r := c.respReader.Read()
	if r.IsType(IOErr) && !IsTimeout(r) {
		c.LastCritical = r.Err
		c.Close()
	}
	return r
}

func (c *Client) writeRequest(requests ...request) error {
	if c.timeout != 0 {
		c.conn.SetWriteDeadline(time.Now().Add(c.timeout))
	}
	var err error
outer:
	for i := range requests {
		c.writeBuf.Reset()
		elems := flattenedLength(requests[i].args...) + 1
		_, err = writeArrayHeader(c.writeBuf, c.writeScratch, int64(elems))
		if err != nil {
			break
		}

		_, err = writeTo(c.writeBuf, c.writeScratch, requests[i].cmd, true, true)
		if err != nil {
			break
		}

		for _, arg := range requests[i].args {
			_, err = writeTo(c.writeBuf, c.writeScratch, arg, true, true)
			if err != nil {
				break outer
			}
		}

		if _, err = c.writeBuf.WriteTo(c.conn); err != nil {
			break
		}
	}
	if err != nil {
		c.LastCritical = err
		c.Close()
		return err
	}
	return nil
}

var errBadCmdNoKey = errors.New("bad command, no key")

// KeyFromArgs is a helper function which other library packages which wrap this
// one might find useful. It takes in a set of arguments which might be passed
// into Cmd and returns the first key for the command. Since radix supports
// complicated arguments (like slices, slices of slices, maps, etc...) this is
// not always as straightforward as it might seem, so this helper function is
// provided.
//
// An error is returned if no key can be determined
func KeyFromArgs(args ...interface{}) (string, error) {
	if len(args) == 0 {
		return "", errBadCmdNoKey
	}
	arg := args[0]
	switch argv := arg.(type) {
	case string:
		return argv, nil
	case []byte:
		return string(argv), nil
	default:
		switch reflect.TypeOf(arg).Kind() {
		case reflect.Slice:
			argVal := reflect.ValueOf(arg)
			if argVal.Len() < 1 {
				return "", errBadCmdNoKey
			}
			first := argVal.Index(0).Interface()
			return KeyFromArgs(first)
		case reflect.Map:
			// Maps have no order, we can't possibly choose a key out of one
			return "", errBadCmdNoKey
		default:
			return fmt.Sprint(arg), nil
		}
	}
}
<commit_msg>make network timeouts count as critical network errors<commit_after>package redis

import (
	"bytes"
	"errors"
	"fmt"
	"net"
	"reflect"
	"time"
)

// ErrPipelineEmpty is returned from PipeResp() to indicate that all commands
// which were put into the pipeline have had their responses read
var ErrPipelineEmpty = errors.New("pipeline queue empty")

// Client describes a Redis client.
type Client struct {
	conn         net.Conn
	respReader   *RespReader
	timeout      time.Duration
	pending      []request
	writeScratch []byte
	writeBuf     *bytes.Buffer

	completed, completedHead []*Resp

	// The network/address of the redis instance this client is connected to.
	// These will be whatever strings were passed into the Dial function when
	// creating this connection
	Network, Addr string

	// The most recent network error which occurred when either reading
	// or writing. A critical network error is basically any non-application
	// level error, e.g. a timeout, disconnect, etc... Close is automatically
	// called on the client when it encounters a network error
	LastCritical error
}

// request describes a client's request to the redis server
type request struct {
	cmd  string
	args []interface{}
}

// DialTimeout connects to the given Redis server with the given timeout, which
// will be used as the read/write timeout when communicating with redis
func DialTimeout(network, addr string, timeout time.Duration) (*Client, error) {
	// establish a connection
	conn, err := net.DialTimeout(network, addr, timeout)
	if err != nil {
		return nil, err
	}

	completed := make([]*Resp, 0, 10)
	return &Client{
		conn:          conn,
		respReader:    NewRespReader(conn),
		timeout:       timeout,
		writeScratch:  make([]byte, 0, 128),
		writeBuf:      bytes.NewBuffer(make([]byte, 0, 128)),
		completed:     completed,
		completedHead: completed,
		Network:       network,
		Addr:          addr,
	}, nil
}

// Dial connects to the given Redis server.
func Dial(network, addr string) (*Client, error) {
	return DialTimeout(network, addr, time.Duration(0))
}

// Close closes the connection.
func (c *Client) Close() error {
	return c.conn.Close()
}

// Cmd calls the given Redis command.
func (c *Client) Cmd(cmd string, args ...interface{}) *Resp {
	err := c.writeRequest(request{cmd, args})
	if err != nil {
		return newRespIOErr(err)
	}
	return c.ReadResp()
}

// PipeAppend adds the given call to the pipeline queue.
// Use PipeResp() to read the response.
func (c *Client) PipeAppend(cmd string, args ...interface{}) {
	c.pending = append(c.pending, request{cmd, args})
}

// PipeResp returns the reply for the next request in the pipeline queue. Err
// with ErrPipelineEmpty is returned if the pipeline queue is empty.
func (c *Client) PipeResp() *Resp {
	if len(c.completed) > 0 {
		r := c.completed[0]
		c.completed = c.completed[1:]
		return r
	}

	if len(c.pending) == 0 {
		return NewResp(ErrPipelineEmpty)
	}

	nreqs := len(c.pending)
	err := c.writeRequest(c.pending...)
	c.pending = nil
	if err != nil {
		return newRespIOErr(err)
	}
	c.completed = c.completedHead
	for i := 0; i < nreqs; i++ {
		r := c.ReadResp()
		c.completed = append(c.completed, r)
	}

	// At this point c.completed should have something in it
	return c.PipeResp()
}

// PipeClear clears the contents of the current pipeline queue, both commands
// queued by PipeAppend which have yet to be sent and responses which have yet
// to be retrieved through PipeResp. The first returned int will be the number
// of pending commands dropped, the second will be the number of pending
// responses dropped
func (c *Client) PipeClear() (int, int) {
	callCount, replyCount := len(c.pending), len(c.completed)
	if callCount > 0 {
		c.pending = nil
	}
	if replyCount > 0 {
		c.completed = nil
	}
	return callCount, replyCount
}

// ReadResp will read a Resp off of the connection without sending anything
// first (useful after you've sent a SUSBSCRIBE command). This will block until
// a reply is received or the timeout is reached (returning the IOErr). You can
// use IsTimeout to check if the Resp is due to a Timeout
//
// Note: this is a more low-level function, you really shouldn't have to
// actually use it unless you're writing your own pub/sub code
func (c *Client) ReadResp() *Resp {
	if c.timeout != 0 {
		c.conn.SetReadDeadline(time.Now().Add(c.timeout))
	}
	r := c.respReader.Read()
	if r.IsType(IOErr) {
		c.LastCritical = r.Err
		c.Close()
	}
	return r
}

func (c *Client) writeRequest(requests ...request) error {
	if c.timeout != 0 {
		c.conn.SetWriteDeadline(time.Now().Add(c.timeout))
	}
	var err error
outer:
	for i := range requests {
		c.writeBuf.Reset()
		elems := flattenedLength(requests[i].args...) + 1
		_, err = writeArrayHeader(c.writeBuf, c.writeScratch, int64(elems))
		if err != nil {
			break
		}

		_, err = writeTo(c.writeBuf, c.writeScratch, requests[i].cmd, true, true)
		if err != nil {
			break
		}

		for _, arg := range requests[i].args {
			_, err = writeTo(c.writeBuf, c.writeScratch, arg, true, true)
			if err != nil {
				break outer
			}
		}

		if _, err = c.writeBuf.WriteTo(c.conn); err != nil {
			break
		}
	}
	if err != nil {
		c.LastCritical = err
		c.Close()
		return err
	}
	return nil
}

var errBadCmdNoKey = errors.New("bad command, no key")

// KeyFromArgs is a helper function which other library packages which wrap this
// one might find useful. It takes in a set of arguments which might be passed
// into Cmd and returns the first key for the command. Since radix supports
// complicated arguments (like slices, slices of slices, maps, etc...) this is
// not always as straightforward as it might seem, so this helper function is
// provided.
//
// An error is returned if no key can be determined
func KeyFromArgs(args ...interface{}) (string, error) {
	if len(args) == 0 {
		return "", errBadCmdNoKey
	}
	arg := args[0]
	switch argv := arg.(type) {
	case string:
		return argv, nil
	case []byte:
		return string(argv), nil
	default:
		switch reflect.TypeOf(arg).Kind() {
		case reflect.Slice:
			argVal := reflect.ValueOf(arg)
			if argVal.Len() < 1 {
				return "", errBadCmdNoKey
			}
			first := argVal.Index(0).Interface()
			return KeyFromArgs(first)
		case reflect.Map:
			// Maps have no order, we can't possibly choose a key out of one
			return "", errBadCmdNoKey
		default:
			return fmt.Sprint(arg), nil
		}
	}
}
<|endoftext|>


<commit_before>package metrics

import (
	"fmt"
	"github.com/DataDog/datadog-go/statsd"
	"github.com/pkg/errors"
	"github.com/stitchfix/flotilla-os/config"
)

type DatadogStatsdMetricsClient struct {
	client *statsd.Client
}

func (dd *DatadogStatsdMetricsClient) Init(conf config.Config) error {
	if !conf.IsSet("metrics.dogstatsd.address") {
		return errors.Errorf("Unable to initialize DatadogMetricsClient: metrics.dogstatsd.address must be set in the config.")
	}

	addr := conf.GetString("metrics.dogstatsd.address")
	client, err := statsd.New(addr)
	if err != nil {
		return err
	}

	// Set global namespace if set in config.
	if conf.IsSet("metrics.dogstatsd.namespace") {
		client.Namespace = conf.GetString("metrics.dogstatsd.namespace")
	}

	// Set global tags if set in config.
	if conf.IsSet("metrics.dogstatsd.tags") {
		tags := conf.GetStringMapString("metrics.dogstatsd.tags")
		for k, v := range tags {
			client.Tags = append(client.Tags, fmt.Sprintf("%s=%s", k, v))
		}
	}

	dd.client = client

	return nil
}

func (dd *DatadogStatsdMetricsClient) Decrement(name Metric, tags []string, rate float64) error {
	return dd.client.Decr(string(name), tags, rate)
}

func (dd *DatadogStatsdMetricsClient) Increment(name Metric, tags []string, rate float64) error {
	return dd.client.Incr(string(name), tags, rate)
}

func (dd *DatadogStatsdMetricsClient) Histogram(name Metric, value float64, tags []string, rate float64) error {
	return dd.client.Histogram(string(name), value, tags, rate)
}

func (dd *DatadogStatsdMetricsClient) Distribution(name Metric, value float64, tags []string, rate float64) error {
	return dd.client.Distribution(string(name), value, tags, rate)
}

func (dd *DatadogStatsdMetricsClient) Set(name Metric, value string, tags []string, rate float64) error {
	return dd.client.Set(string(name), value, tags, rate)
}

func (dd *DatadogStatsdMetricsClient) Event(e event) error {
	se := statsd.NewEvent(e.Title, e.Text)
	se.Tags = e.Tags
	return dd.client.Event(se)
}
<commit_msg>fix tags<commit_after>package metrics

import (
	"fmt"
	"github.com/DataDog/datadog-go/statsd"
	"github.com/pkg/errors"
	"github.com/stitchfix/flotilla-os/config"
)

type DatadogStatsdMetricsClient struct {
	client *statsd.Client
}

func (dd *DatadogStatsdMetricsClient) Init(conf config.Config) error {
	if !conf.IsSet("metrics.dogstatsd.address") {
		return errors.Errorf("Unable to initialize DatadogMetricsClient: metrics.dogstatsd.address must be set in the config.")
	}

	addr := conf.GetString("metrics.dogstatsd.address")
	client, err := statsd.New(addr)
	if err != nil {
		return err
	}

	// Set global namespace if set in config.
	if conf.IsSet("metrics.dogstatsd.namespace") {
		client.Namespace = conf.GetString("metrics.dogstatsd.namespace")
	}

	// Set global tags if set in config.
	if conf.IsSet("metrics.dogstatsd.tags") {
		tags := conf.GetStringMapString("metrics.dogstatsd.tags")
		for k, v := range tags {
			client.Tags = append(client.Tags, fmt.Sprintf("%s:%s", k, v))
		}
	}

	dd.client = client

	return nil
}

func (dd *DatadogStatsdMetricsClient) Decrement(name Metric, tags []string, rate float64) error {
	return dd.client.Decr(string(name), tags, rate)
}

func (dd *DatadogStatsdMetricsClient) Increment(name Metric, tags []string, rate float64) error {
	return dd.client.Incr(string(name), tags, rate)
}

func (dd *DatadogStatsdMetricsClient) Histogram(name Metric, value float64, tags []string, rate float64) error {
	return dd.client.Histogram(string(name), value, tags, rate)
}

func (dd *DatadogStatsdMetricsClient) Distribution(name Metric, value float64, tags []string, rate float64) error {
	return dd.client.Distribution(string(name), value, tags, rate)
}

func (dd *DatadogStatsdMetricsClient) Set(name Metric, value string, tags []string, rate float64) error {
	return dd.client.Set(string(name), value, tags, rate)
}

func (dd *DatadogStatsdMetricsClient) Event(e event) error {
	se := statsd.NewEvent(e.Title, e.Text)
	se.Tags = e.Tags
	return dd.client.Event(se)
}
<|endoftext|>