<commit_before>package output

import (
	"strings"

	//	"fmt"

	"github.com/gohugoio/hugo/docshelper"
)

// This is is just some helpers used to create some JSON used in the Hugo docs.
func init() {
	docsProvider := func() map[string]interface{} {
		docs := make(map[string]interface{})

		docs["formats"] = DefaultFormats
		docs["layouts"] = createLayoutExamples()
		return docs
	}

	docshelper.AddDocProvider("output", docsProvider)
}

func createLayoutExamples() interface{} {

	type Example struct {
		Example      string
		Kind         string
		OutputFormat string
		Suffix       string
		Layouts      []string `json:"Template Lookup Order"`
	}

	var (
		basicExamples []Example
		demoLayout    = "demolayout"
		demoType      = "demotype"
	)

	for _, example := range []struct {
		name     string
		d        LayoutDescriptor
		hasTheme bool
		f        Format
	}{
		// Taxonomy output.LayoutDescriptor={categories category taxonomy en  false Type Section
		{"Single page in \"posts\" section", LayoutDescriptor{Kind: "page", Type: "posts"}, false, HTMLFormat},
		{"Single page in \"posts\" section with layout set", LayoutDescriptor{Kind: "page", Type: "posts", Layout: demoLayout}, false, HTMLFormat},
		{"Single page in \"posts\" section with theme", LayoutDescriptor{Kind: "page", Type: "posts"}, true, HTMLFormat},
		{"AMP single page", LayoutDescriptor{Kind: "page", Type: "posts"}, false, AMPFormat},
		{"AMP single page, French language", LayoutDescriptor{Kind: "page", Type: "posts", Lang: "fr"}, false, AMPFormat},
		// All section or typeless pages gets "page" as type
		{"Home page", LayoutDescriptor{Kind: "home", Type: "page"}, false, HTMLFormat},
		{"Home page with type set", LayoutDescriptor{Kind: "home", Type: demoType}, false, HTMLFormat},
		{"Home page with layout set", LayoutDescriptor{Kind: "home", Type: "page", Layout: demoLayout}, false, HTMLFormat},
		{`Home page with theme`, LayoutDescriptor{Kind: "home", Type: "page"}, true, HTMLFormat},
		{`AMP home, French language"`, LayoutDescriptor{Kind: "home", Type: "page", Lang: "fr"}, false, AMPFormat},
		{"JSON home", LayoutDescriptor{Kind: "home", Type: "page"}, false, JSONFormat},
		{"RSS home", LayoutDescriptor{Kind: "home", Type: "page"}, false, RSSFormat},

		{"Section list for \"posts\" section", LayoutDescriptor{Kind: "section", Type: "posts", Section: "posts"}, false, HTMLFormat},
		{"Section list for \"posts\" section with type set to \"blog\"", LayoutDescriptor{Kind: "section", Type: "blog", Section: "posts"}, false, HTMLFormat},
		{"Section list for \"posts\" section with layout set to \"demoLayout\"", LayoutDescriptor{Kind: "section", Layout: demoLayout, Section: "posts"}, false, HTMLFormat},

		{"Taxonomy list in categories", LayoutDescriptor{Kind: "taxonomy", Type: "categories", Section: "category"}, false, HTMLFormat},
		{"Taxonomy term in categories", LayoutDescriptor{Kind: "taxonomyTerm", Type: "categories", Section: "category"}, false, HTMLFormat},
	} {

		l := NewLayoutHandler(example.hasTheme)
		layouts, _ := l.For(example.d, example.f)

		basicExamples = append(basicExamples, Example{
			Example:      example.name,
			Kind:         example.d.Kind,
			OutputFormat: example.f.Name,
			Suffix:       example.f.MediaType.Suffix,
			Layouts:      makeLayoutsPresentable(layouts)})
	}

	return basicExamples

}

func makeLayoutsPresentable(l []string) []string {
	var filtered []string
	for _, ll := range l {
		ll = strings.TrimPrefix(ll, "_text/")
		if strings.Contains(ll, "theme/") {
			ll = strings.Replace(ll, "theme/", "demoTheme/layouts/", -1)
		} else {
			ll = "layouts/" + ll
		}
		if !strings.Contains(ll, "indexes") {
			filtered = append(filtered, ll)
		}
	}

	return filtered
}
<commit_msg>outputs: Adjust the layout examples<commit_after>package output

import (
	"fmt"
	"strings"

	//	"fmt"

	"github.com/gohugoio/hugo/docshelper"
)

// This is is just some helpers used to create some JSON used in the Hugo docs.
func init() {
	docsProvider := func() map[string]interface{} {
		docs := make(map[string]interface{})

		docs["formats"] = DefaultFormats
		docs["layouts"] = createLayoutExamples()
		return docs
	}

	docshelper.AddDocProvider("output", docsProvider)
}

func createLayoutExamples() interface{} {

	type Example struct {
		Example      string
		Kind         string
		OutputFormat string
		Suffix       string
		Layouts      []string `json:"Template Lookup Order"`
	}

	var (
		basicExamples []Example
		demoLayout    = "demolayout"
		demoType      = "demotype"
	)

	for _, example := range []struct {
		name     string
		d        LayoutDescriptor
		hasTheme bool
		f        Format
	}{
		// Taxonomy output.LayoutDescriptor={categories category taxonomy en  false Type Section
		{"Single page in \"posts\" section", LayoutDescriptor{Kind: "page", Type: "posts"}, false, HTMLFormat},
		{"Single page in \"posts\" section with layout set", LayoutDescriptor{Kind: "page", Type: "posts", Layout: demoLayout}, false, HTMLFormat},
		{"Single page in \"posts\" section with theme", LayoutDescriptor{Kind: "page", Type: "posts"}, true, HTMLFormat},
		{"AMP single page", LayoutDescriptor{Kind: "page", Type: "posts"}, false, AMPFormat},
		{"AMP single page, French language", LayoutDescriptor{Kind: "page", Type: "posts", Lang: "fr"}, false, AMPFormat},
		// All section or typeless pages gets "page" as type
		{"Home page", LayoutDescriptor{Kind: "home", Type: "page"}, false, HTMLFormat},
		{"Home page with type set", LayoutDescriptor{Kind: "home", Type: demoType}, false, HTMLFormat},
		{"Home page with layout set", LayoutDescriptor{Kind: "home", Type: "page", Layout: demoLayout}, false, HTMLFormat},
		{`Home page with theme`, LayoutDescriptor{Kind: "home", Type: "page"}, true, HTMLFormat},
		{`AMP home, French language"`, LayoutDescriptor{Kind: "home", Type: "page", Lang: "fr"}, false, AMPFormat},
		{"JSON home", LayoutDescriptor{Kind: "home", Type: "page"}, false, JSONFormat},
		{"RSS home with theme", LayoutDescriptor{Kind: "home", Type: "page"}, true, RSSFormat},
		{"RSS section posts", LayoutDescriptor{Kind: "section", Type: "posts"}, false, RSSFormat},
		{"Taxonomy list in categories", LayoutDescriptor{Kind: "taxonomy", Type: "categories", Section: "category"}, false, RSSFormat},
		{"Taxonomy terms in categories", LayoutDescriptor{Kind: "taxonomyTerm", Type: "categories", Section: "category"}, false, RSSFormat},
		{"Section list for \"posts\" section", LayoutDescriptor{Kind: "section", Type: "posts", Section: "posts"}, false, HTMLFormat},
		{"Section list for \"posts\" section with type set to \"blog\"", LayoutDescriptor{Kind: "section", Type: "blog", Section: "posts"}, false, HTMLFormat},
		{"Section list for \"posts\" section with layout set to \"demoLayout\"", LayoutDescriptor{Kind: "section", Layout: demoLayout, Section: "posts"}, false, HTMLFormat},

		{"Taxonomy list in categories", LayoutDescriptor{Kind: "taxonomy", Type: "categories", Section: "category"}, false, HTMLFormat},
		{"Taxonomy term in categories", LayoutDescriptor{Kind: "taxonomyTerm", Type: "categories", Section: "category"}, false, HTMLFormat},
	} {

		l := NewLayoutHandler(example.hasTheme)
		layouts, _ := l.For(example.d, example.f)

		basicExamples = append(basicExamples, Example{
			Example:      example.name,
			Kind:         example.d.Kind,
			OutputFormat: example.f.Name,
			Suffix:       example.f.MediaType.Suffix,
			Layouts:      makeLayoutsPresentable(layouts)})
	}

	return basicExamples

}

func makeLayoutsPresentable(l []string) []string {
	var filtered []string
	for _, ll := range l {
		if strings.Contains(ll, "page/") {
			// This is a valid lookup, but it's more confusing than useful.
			continue
		}
		ll = strings.TrimPrefix(ll, "_text/")
		if strings.Contains(ll, "theme/") {
			ll = strings.Replace(ll, "theme/", "demoTheme/layouts/", -1)
		} else {
			ll = "layouts/" + ll
		}
		if !strings.Contains(ll, "indexes") {
			filtered = append(filtered, ll)
		}
	}

	return filtered
}
<|endoftext|>


<commit_before>package paths

import (
	"bytes"
	"fmt"
	"io"
	"testing"

	. "github.com/VonC/godbg"
	"github.com/VonC/senvgo/prgs"
	. "github.com/smartystreets/goconvey/convey"
)

func TestPath(t *testing.T) {

	Convey("Tests for Path", t, func() {
		Convey("An empty path remains empty", func() {
			SetBuffers(nil)
			p := NewPath("")
			So(p.path, ShouldEqual, "")
		})

		Convey("An http path remains unchanged", func() {
			SetBuffers(nil)
			p := NewPath(`http://a\b/../c`)
			So(p.path, ShouldEqual, `http://a\b/../c`)
		})
		Convey("A path without trailing / must have one if it is an existing folder", func() {
			SetBuffers(nil)
			p := NewPath(`../paths`)
			So(p.path, ShouldEqual, `..\paths\`)
		})
		Convey("A path without trailing / must keep it even if it is not an existing folder", func() {
			SetBuffers(nil)
			p := NewPath(`xxx\`)
			p = NewPath(`xxx/`)
			So(p.path, ShouldEqual, `xxx\`)
		})
	})
}

type testPrg struct{ name string }

func (tp *testPrg) Name() string { return tp.name }

type testPathWriter struct{ b *bytes.Buffer }

type testWriter struct{ w io.Writer }

func (tw *testWriter) Write(p []byte) (n int, err error) {
	s := string(p)
	if s == "prg2" {
		return 0, fmt.Errorf("Error writing '%s'", s)
	}
	return tw.w.Write(p)
}

func (tpw *testPathWriter) WritePath(prgs []prgs.Prg, w io.Writer) error {
	if err := pw.WritePath(prgs, w); err != nil {
		return err
	}
	return nil
}

func TestMain(t *testing.T) {
	tpw := &testPathWriter{b: bytes.NewBuffer(nil)}
	prgs := []prgs.Prg{&testPrg{name: "prg1"}, &testPrg{name: "prg2"}}
	Convey("Tests for Path Writer", t, func() {

		Convey("A Path writer writes any empty path if no prgs", func() {
			SetBuffers(nil)
			err := tpw.WritePath(prgs, tpw.b)
			So(err, ShouldBeNil)
			So(tpw.b.String(), ShouldEqual, "prg1prg2")
		})

		Convey("A Path writer can report error during writing", func() {
			SetBuffers(nil)
			tpw.b = bytes.NewBuffer(nil)
			tw := &testWriter{w: tpw.b}
			err := tpw.WritePath(prgs, tw)
			So(err.Error(), ShouldEqual, "Error writing 'prg2'")
			So(tpw.b.String(), ShouldEqual, "prg1")
		})
	})
}
<commit_msg>Focus on Path.IsDir() test<commit_after>package paths

import (
	"bytes"
	"fmt"
	"io"
	"testing"

	. "github.com/VonC/godbg"
	"github.com/VonC/senvgo/prgs"
	. "github.com/smartystreets/goconvey/convey"
)

func TestPath(t *testing.T) {

	FocusConvey("Tests for Path", t, func() {
		Convey("An empty path remains empty", func() {
			SetBuffers(nil)
			p := NewPath("")
			So(p.path, ShouldEqual, "")
		})

		Convey("An http path remains unchanged", func() {
			SetBuffers(nil)
			p := NewPath(`http://a\b/../c`)
			So(p.path, ShouldEqual, `http://a\b/../c`)
		})
		Convey("A path without trailing / must have one if it is an existing folder", func() {
			SetBuffers(nil)
			p := NewPath(`../paths`)
			So(p.path, ShouldEqual, `..\paths\`)
		})
		Convey("A path without trailing / must keep it even if it is not an existing folder", func() {
			SetBuffers(nil)
			p := NewPath(`xxx\`)
			p = NewPath(`xxx/`)
			So(p.path, ShouldEqual, `xxx\`)
		})

		FocusConvey("A Path can test if it is a Dir", func() {
			SetBuffers(nil)
			p := NewPath("")
			So(p.IsDir(), ShouldBeFalse)
		})
	})
}

type testPrg struct{ name string }

func (tp *testPrg) Name() string { return tp.name }

type testPathWriter struct{ b *bytes.Buffer }

type testWriter struct{ w io.Writer }

func (tw *testWriter) Write(p []byte) (n int, err error) {
	s := string(p)
	if s == "prg2" {
		return 0, fmt.Errorf("Error writing '%s'", s)
	}
	return tw.w.Write(p)
}

func (tpw *testPathWriter) WritePath(prgs []prgs.Prg, w io.Writer) error {
	if err := pw.WritePath(prgs, w); err != nil {
		return err
	}
	return nil
}

func TestMain(t *testing.T) {
	tpw := &testPathWriter{b: bytes.NewBuffer(nil)}
	prgs := []prgs.Prg{&testPrg{name: "prg1"}, &testPrg{name: "prg2"}}
	Convey("Tests for Path Writer", t, func() {

		Convey("A Path writer writes any empty path if no prgs", func() {
			SetBuffers(nil)
			err := tpw.WritePath(prgs, tpw.b)
			So(err, ShouldBeNil)
			So(tpw.b.String(), ShouldEqual, "prg1prg2")
		})

		Convey("A Path writer can report error during writing", func() {
			SetBuffers(nil)
			tpw.b = bytes.NewBuffer(nil)
			tw := &testWriter{w: tpw.b}
			err := tpw.WritePath(prgs, tw)
			So(err.Error(), ShouldEqual, "Error writing 'prg2'")
			So(tpw.b.String(), ShouldEqual, "prg1")
		})
	})
}
<|endoftext|>


<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package optimizer

import (
	"github.com/pingcap/tidb/ast"
	"github.com/pingcap/tidb/parser/opcode"
	"github.com/pingcap/tidb/terror"
)

// Optimizer error codes.
const (
	CodeOneColumn terror.ErrCode = iota + 1
	CodeRowColumns
	CodeSameColumns
	CodeMultiWildCard
)

// Optimizer base errors.
var (
	ErrOneColumn     = terror.ClassOptimizer.New(CodeOneColumn, "Operand should contain 1 column(s)")
	ErrRowColumns    = terror.ClassOptimizer.New(CodeRowColumns, "Operand should contain >= 2 columns for Row")
	ErrSameColumns   = terror.ClassOptimizer.New(CodeRowColumns, "Operands should contain same columns")
	ErrMultiWildCard = terror.ClassOptimizer.New(CodeMultiWildCard, "wildcard field exist more than once")
)

func validate(node ast.Node) error {
	var v validator
	node.Accept(&v)
	return v.err
}

// validator is an ast.Visitor that validates
// ast Nodes parsed from parser.
type validator struct {
	err           error
	wildCardCount int
}

func (v *validator) Enter(in ast.Node) (out ast.Node, skipChildren bool) {
	return in, false
}

func (v *validator) Leave(in ast.Node) (out ast.Node, ok bool) {
	switch x := in.(type) {
	case *ast.IsNullExpr:
		v.checkAllOneColumn(x.Expr)
	case *ast.IsTruthExpr:
		v.checkAllOneColumn(x.Expr)
	case *ast.BetweenExpr:
		v.checkAllOneColumn(x.Expr, x.Left, x.Right)
	case *ast.RowExpr:
		if len(x.Values) < 2 {
			v.err = ErrRowColumns
		}
	case *ast.BinaryOperationExpr:
		v.checkBinaryOperation(x)
	case *ast.PatternInExpr:
		v.checkSameColumns(append(x.List, x.Expr)...)
	case *ast.FieldList:
		v.checkFieldList(x)
	case *ast.ByItem:
		v.checkAllOneColumn(x.Expr)
	}
	return in, v.err == nil
}

func (v *validator) checkAllOneColumn(exprs ...ast.ExprNode) {
	for _, expr := range exprs {
		switch x := expr.(type) {
		case *ast.RowExpr:
			v.err = ErrOneColumn
		case *ast.SubqueryExpr:
			if len(x.Query.GetResultFields()) != 1 {
				v.err = ErrOneColumn
			}
		}
	}
	return
}

func (v *validator) checkBinaryOperation(x *ast.BinaryOperationExpr) {
	// row constructor only supports comparison operation.
	switch x.Op {
	case opcode.LT, opcode.LE, opcode.GE, opcode.GT, opcode.EQ, opcode.NE, opcode.NullEQ:
		v.checkSameColumns(x.L, x.R)
	default:
		v.checkAllOneColumn(x.L, x.R)
	}
}

func columnCount(ex ast.ExprNode) int {
	switch x := ex.(type) {
	case *ast.RowExpr:
		return len(x.Values)
	case *ast.SubqueryExpr:
		return len(x.Query.GetResultFields())
	default:
		return 1
	}
}

func (v *validator) checkSameColumns(exprs ...ast.ExprNode) {
	if len(exprs) == 0 {
		return
	}
	count := columnCount(exprs[0])
	for i := 1; i < len(exprs); i++ {
		if columnCount(exprs[i]) != count {
			v.err = ErrSameColumns
			return
		}
	}
}

func (v *validator) checkFieldList(x *ast.FieldList) {
	var hasWildCard bool
	for _, val := range x.Fields {
		if val.WildCard != nil && val.WildCard.Table.L == "" {
			if hasWildCard {
				v.err = ErrMultiWildCard
				return
			}
			hasWildCard = true
		}
		v.checkAllOneColumn(val.Expr)
		if v.err != nil {
			return
		}
	}
}
<commit_msg>optimizer: add more comments for validator.<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package optimizer

import (
	"github.com/pingcap/tidb/ast"
	"github.com/pingcap/tidb/parser/opcode"
	"github.com/pingcap/tidb/terror"
)

// Optimizer error codes.
const (
	CodeOneColumn terror.ErrCode = iota + 1
	CodeRowColumns
	CodeSameColumns
	CodeMultiWildCard
)

// Optimizer base errors.
var (
	ErrOneColumn     = terror.ClassOptimizer.New(CodeOneColumn, "Operand should contain 1 column(s)")
	ErrRowColumns    = terror.ClassOptimizer.New(CodeRowColumns, "Operand should contain >= 2 columns for Row")
	ErrSameColumns   = terror.ClassOptimizer.New(CodeRowColumns, "Operands should contain same columns")
	ErrMultiWildCard = terror.ClassOptimizer.New(CodeMultiWildCard, "wildcard field exist more than once")
)

func validate(node ast.Node) error {
	var v validator
	node.Accept(&v)
	return v.err
}

// validator is an ast.Visitor that validates
// ast Nodes parsed from parser.
type validator struct {
	err           error
	wildCardCount int
}

func (v *validator) Enter(in ast.Node) (out ast.Node, skipChildren bool) {
	return in, false
}

func (v *validator) Leave(in ast.Node) (out ast.Node, ok bool) {
	switch x := in.(type) {
	case *ast.IsNullExpr:
		v.checkAllOneColumn(x.Expr)
	case *ast.IsTruthExpr:
		v.checkAllOneColumn(x.Expr)
	case *ast.BetweenExpr:
		v.checkAllOneColumn(x.Expr, x.Left, x.Right)
	case *ast.RowExpr:
		if len(x.Values) < 2 {
			v.err = ErrRowColumns
		}
	case *ast.BinaryOperationExpr:
		v.checkBinaryOperation(x)
	case *ast.PatternInExpr:
		v.checkSameColumns(append(x.List, x.Expr)...)
	case *ast.FieldList:
		v.checkFieldList(x)
	case *ast.ByItem:
		v.checkAllOneColumn(x.Expr)
	}
	return in, v.err == nil
}

// checkAllOneColumn checks all expression has one column,
// expression may has more than one column when it is rowExpr or
// Subquery with more than one result fields.
func (v *validator) checkAllOneColumn(exprs ...ast.ExprNode) {
	for _, expr := range exprs {
		switch x := expr.(type) {
		case *ast.RowExpr:
			v.err = ErrOneColumn
		case *ast.SubqueryExpr:
			if len(x.Query.GetResultFields()) != 1 {
				v.err = ErrOneColumn
			}
		}
	}
	return
}

func (v *validator) checkBinaryOperation(x *ast.BinaryOperationExpr) {
	// row constructor only supports comparison operation.
	switch x.Op {
	case opcode.LT, opcode.LE, opcode.GE, opcode.GT, opcode.EQ, opcode.NE, opcode.NullEQ:
		v.checkSameColumns(x.L, x.R)
	default:
		v.checkAllOneColumn(x.L, x.R)
	}
}

func columnCount(ex ast.ExprNode) int {
	switch x := ex.(type) {
	case *ast.RowExpr:
		return len(x.Values)
	case *ast.SubqueryExpr:
		return len(x.Query.GetResultFields())
	default:
		return 1
	}
}

func (v *validator) checkSameColumns(exprs ...ast.ExprNode) {
	if len(exprs) == 0 {
		return
	}
	count := columnCount(exprs[0])
	for i := 1; i < len(exprs); i++ {
		if columnCount(exprs[i]) != count {
			v.err = ErrSameColumns
			return
		}
	}
}

// checkFieldList checks there is only one '*" and each field has only one column,
func (v *validator) checkFieldList(x *ast.FieldList) {
	var hasWildCard bool
	for _, val := range x.Fields {
		if val.WildCard != nil && val.WildCard.Table.L == "" {
			if hasWildCard {
				v.err = ErrMultiWildCard
				return
			}
			hasWildCard = true
		}
		v.checkAllOneColumn(val.Expr)
		if v.err != nil {
			return
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"strings"
)

func GetUserByNick(nick string) *User {
	nick = strings.ToLower(nick)
	if nick == strings.ToLower(SystemUser.nick) {
		return SystemUser
	}
	for _, k := range userlist {
		if strings.ToLower(k.nick) == nick {
			return k
		}
	}
	return nil
}

func SetUserIPInfo(user *User) {
	ip := user.connection.RemoteAddr().String()
	if !strings.HasPrefix(ip, "[") {
		//ipv4
		user.ConnType = "IP4"
		user.realip = strings.Split(ip, ":")[0]
	} else {
		user.ConnType = "IP6"
		ip = strings.Split(ip, "]")[0]
		ip = strings.TrimPrefix(ip, "[")
		user.realip = ip
	}
}

func GetChannelByName(name string) *Channel {
	return chanlist[strings.ToLower(name)]
}

func SendToMany(msg string, list []*User) {
	users := make(map[*User]int)
	for _, j := range list {
		users[j] = 0
	}
	for j, _ := range users {
		j.SendLine(msg)
	}
}

func ValidChanName(name string) bool {
	if ChanHasBadChars(name) {
		return false
	}
	for _, k := range valid_chan_prefix {
		if strings.HasPrefix(name, k) {
			return true
		}
	}
	return false
}

//IMPORTANT: args must ABSOLUTELY be a valid privmsg command, or this will not work
//validity does not depend on leading ":", I don't care that much
func FormatMessageArgs(args []string) string {
	msg := strings.Join(args[2:], " ")
	msg = strings.TrimPrefix(msg, ":")
	return msg
}

func NickHasBadChars(nick string) bool {
	for _, k := range global_bad_chars {
		if strings.Contains(nick, k) {
			return true
		}
	}
	for _, k := range valid_chan_prefix {
		if strings.Contains(nick, k) {
			return true
		}
	}
	return false
}

func ChanHasBadChars(nick string) bool {
	for _, k := range global_bad_chars {
		if strings.Contains(nick, k) {
			return true
		}
	}
	return false
}

func ChanUserNone(name string) int {
	if GetChannelByName(name) != nil {
		return 1
	} else if GetUserByNick(name) != nil {
		return 2
	} else {
		return 0
	}
}

func WildcardMatch(text string, pattern string) bool {
	cards := strings.Split(pattern, "*")
	for _, card := range cards {
		index := strings.Index(text, card)
		if index == -1 {
			return false
		}
		text = text[index+len(card):]
	}
	return true
}

func SetupSytemUser() {
	for _, k := range config.LogChannels {
		SystemUser.JoinHandler([]string{"JOIN", k})
		SystemUser.ModeHandler([]string{"MODE", k, "+A"})
	}
}
<commit_msg>refactor SendToMany<commit_after>package main

import (
	"strings"
)

func GetUserByNick(nick string) *User {
	nick = strings.ToLower(nick)
	if nick == strings.ToLower(SystemUser.nick) {
		return SystemUser
	}
	for _, k := range userlist {
		if strings.ToLower(k.nick) == nick {
			return k
		}
	}
	return nil
}

func SetUserIPInfo(user *User) {
	ip := user.connection.RemoteAddr().String()
	if !strings.HasPrefix(ip, "[") {
		//ipv4
		user.ConnType = "IP4"
		user.realip = strings.Split(ip, ":")[0]
	} else {
		user.ConnType = "IP6"
		ip = strings.Split(ip, "]")[0]
		ip = strings.TrimPrefix(ip, "[")
		user.realip = ip
	}
}

func GetChannelByName(name string) *Channel {
	return chanlist[strings.ToLower(name)]
}

func SendToMany(msg string, list []*User) {
	users := make(map[*User]int)
	for _, j := range list {
		if users[j] != 1 {
			j.SendLine(msg)
		} else {
			users[j] = 1
		}
	}
}

func ValidChanName(name string) bool {
	if ChanHasBadChars(name) {
		return false
	}
	for _, k := range valid_chan_prefix {
		if strings.HasPrefix(name, k) {
			return true
		}
	}
	return false
}

//IMPORTANT: args must ABSOLUTELY be a valid privmsg command, or this will not work
//validity does not depend on leading ":", I don't care that much
func FormatMessageArgs(args []string) string {
	msg := strings.Join(args[2:], " ")
	msg = strings.TrimPrefix(msg, ":")
	return msg
}

func NickHasBadChars(nick string) bool {
	for _, k := range global_bad_chars {
		if strings.Contains(nick, k) {
			return true
		}
	}
	for _, k := range valid_chan_prefix {
		if strings.Contains(nick, k) {
			return true
		}
	}
	return false
}

func ChanHasBadChars(nick string) bool {
	for _, k := range global_bad_chars {
		if strings.Contains(nick, k) {
			return true
		}
	}
	return false
}

func ChanUserNone(name string) int {
	if GetChannelByName(name) != nil {
		return 1
	} else if GetUserByNick(name) != nil {
		return 2
	} else {
		return 0
	}
}

func WildcardMatch(text string, pattern string) bool {
	cards := strings.Split(pattern, "*")
	for _, card := range cards {
		index := strings.Index(text, card)
		if index == -1 {
			return false
		}
		text = text[index+len(card):]
	}
	return true
}

func SetupSytemUser() {
	for _, k := range config.LogChannels {
		SystemUser.JoinHandler([]string{"JOIN", k})
		SystemUser.ModeHandler([]string{"MODE", k, "+A"})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
)

type Person struct {
	name string
	age  int32
}

func main() {
	p := Person{"Viktor", 23}
	p2 := Person{age: 21, name: "Zsu"}
	fmt.Println("Person:", p)
	fmt.Println("P2:", p2)

	// create with default values
	p3 := new(Person)
	fmt.Println("P3:", p3)
}
<commit_msg>ref by name: file comment and utility function Printfln<commit_after>/*
Reference by name

A simple struct test, creating objects from structs with either positional parameters or through named parameters.
*/

package main

import (
	"fmt"
)

type Person struct {
	name string
	age  int32
}

//
// Base Println doesn't support the formatting Printf does
//  but adding \n to every Printf call is not DRY (and not fun at all)
// So this function does exactly this: you can use
//  exactly the same way as you can Printf but it will
//  automatically add a newline to the end of the line
//
func Printfln(format string, a ...interface{}) (n int, err error) {
	format = format + "\n"
	return fmt.Printf(format, a...)
}

func main() {
	p := Person{"Viktor", 23}
	p2 := Person{age: 21, name: "Zsu"}
	Printfln("Person: %#v", p)
	Printfln("P2: %#v", p2)

	// create with default values
	p3 := new(Person)
	Printfln("P3: %v", p3)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/karrick/godirwalk"
	"github.com/pkg/errors"
)

func main() {
	if len(os.Args) < 2 {
		fmt.Fprintf(os.Stderr, "usage: %s dir1 [dir2 [dir3...]]\n", filepath.Base(os.Args[0]))
		os.Exit(2)
	}

	scratchBuffer := make([]byte, 32768) // allocate once and re-use
	var count, total int
	var err error

	for _, arg := range os.Args[1:] {
		count, err = pruneEmptyDirectories(arg, scratchBuffer)
		total += count
		if err != nil {
			break
		}
	}

	fmt.Fprintf(os.Stderr, "Removed %d empty directories\n", total)
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: %s\n", err)
		os.Exit(1)
	}
}

func pruneEmptyDirectories(osDirname string, scratchBuffer []byte) (int, error) {
	var count int

	err := godirwalk.Walk(osDirname, &godirwalk.Options{
		Unsorted:      true,
		ScratchBuffer: scratchBuffer,
		Callback: func(_ string, _ *godirwalk.Dirent) error {
			// no-op while diving in; all the fun happens in PostChildrenCallback
			return nil
		},
		PostChildrenCallback: func(osPathname string, _ *godirwalk.Dirent) error {
			deChildren, err := godirwalk.ReadDirents(osPathname, scratchBuffer)
			if err != nil {
				return errors.Wrap(err, "cannot ReadDirents")
			}
			// NOTE: ReadDirents skips "." and ".."
			if len(deChildren) > 0 {
				return nil // this directory has children; no additional work here
			}
			if osPathname == osDirname {
				return nil // do not remove provided root directory
			}
			err = os.Remove(osPathname)
			if err == nil {
				count++
			}
			return err
		},
	})

	return count, err
}
<commit_msg>use larger scratch buffer<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/karrick/godirwalk"
	"github.com/pkg/errors"
)

func main() {
	if len(os.Args) < 2 {
		fmt.Fprintf(os.Stderr, "usage: %s dir1 [dir2 [dir3...]]\n", filepath.Base(os.Args[0]))
		os.Exit(2)
	}

	scratchBuffer := make([]byte, 64*1024) // allocate once and re-use each time
	var count, total int
	var err error

	for _, arg := range os.Args[1:] {
		count, err = pruneEmptyDirectories(arg, scratchBuffer)
		total += count
		if err != nil {
			break
		}
	}

	fmt.Fprintf(os.Stderr, "Removed %d empty directories\n", total)
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: %s\n", err)
		os.Exit(1)
	}
}

func pruneEmptyDirectories(osDirname string, scratchBuffer []byte) (int, error) {
	var count int

	err := godirwalk.Walk(osDirname, &godirwalk.Options{
		Unsorted:      true,
		ScratchBuffer: scratchBuffer,
		Callback: func(_ string, _ *godirwalk.Dirent) error {
			// no-op while diving in; all the fun happens in PostChildrenCallback
			return nil
		},
		PostChildrenCallback: func(osPathname string, _ *godirwalk.Dirent) error {
			deChildren, err := godirwalk.ReadDirents(osPathname, scratchBuffer)
			if err != nil {
				return errors.Wrap(err, "cannot ReadDirents")
			}
			// NOTE: ReadDirents skips "." and ".."
			if len(deChildren) > 0 {
				return nil // this directory has children; no additional work here
			}
			if osPathname == osDirname {
				return nil // do not remove provided root directory
			}
			err = os.Remove(osPathname)
			if err == nil {
				count++
			}
			return err
		},
	})

	return count, err
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"io"
	"path/filepath"

	"os"

	"github.com/apprenda/kismatic/pkg/install"
	"github.com/apprenda/kismatic/pkg/util"
	"github.com/spf13/cobra"
)

type validateOpts struct {
	generatedAssetsDir string
	planFile           string
	verbose            bool
	outputFormat       string
	skipPreFlight      bool
}

// NewCmdValidate creates a new install validate command
func NewCmdValidate(out io.Writer, installOpts *installOpts) *cobra.Command {
	opts := &validateOpts{}
	cmd := &cobra.Command{
		Use:   "validate",
		Short: "validate your plan file",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) != 0 {
				return fmt.Errorf("Unexpected args: %v", args)
			}
			planner := &install.FilePlanner{File: installOpts.planFilename}
			opts.planFile = installOpts.planFilename
			return doValidate(out, planner, opts)
		},
	}
	cmd.Flags().StringVar(&opts.generatedAssetsDir, "generated-assets-dir", "generated", "path to the directory where assets generated during the installation process will be stored")
	cmd.Flags().BoolVar(&opts.verbose, "verbose", false, "enable verbose logging from the installation")
	cmd.Flags().StringVarP(&opts.outputFormat, "output", "o", "simple", "installation output format (options simple|raw)")
	cmd.Flags().BoolVar(&opts.skipPreFlight, "skip-preflight", false, "skip pre-flight checks")
	return cmd
}

func doValidate(out io.Writer, planner install.Planner, opts *validateOpts) error {
	util.PrintHeader(out, "Validating", '=')
	// Check if plan file exists
	if !planner.PlanExists() {
		util.PrettyPrintErr(out, "Reading installation plan file [ERROR]")
		fmt.Fprintln(out, "Run \"kismatic install plan\" to generate it")
		return fmt.Errorf("plan does not exist")
	}
	plan, err := planner.Read()
	if err != nil {
		util.PrettyPrintErr(out, "Reading installation plan file %q", opts.planFile)
		return fmt.Errorf("error reading plan file: %v", err)
	}
	util.PrettyPrintOk(out, "Reading installation plan file %q", opts.planFile)

	// Validate plan file
	ok, errs := install.ValidatePlan(plan)
	if !ok {
		util.PrettyPrintErr(out, "Validating installation plan file")
		util.PrintValidationErrors(out, errs)
		return fmt.Errorf("Plan file validation error prevents installation from proceeding")
	}
	util.PrettyPrintOk(out, "Validating installation plan file")

	// Validate SSH connections
	ok, errs = install.ValidatePlanSSHConnections(plan)
	if !ok {
		util.PrettyPrintErr(out, "Validating SSH connections to nodes")
		util.PrintValidationErrors(out, errs)
		return fmt.Errorf("SSH connectivity validation error prevents installation from proceeding")
	}

	// get a new pki
	pki, err := newPKI(out, opts)
	if err != nil {
		return err
	}
	// Validate Certificates
	ok, errs = install.ValidateCertificates(plan, pki)
	if !ok {
		util.PrettyPrintErr(out, "Validating cluster certificates")
		util.PrintValidationErrors(out, errs)
		return fmt.Errorf("Cluster certificates validation error prevents installation from proceeding")
	}

	if opts.skipPreFlight {
		return nil
	}
	// Run pre-flight
	options := install.ExecutorOptions{
		OutputFormat: opts.outputFormat,
		Verbose:      opts.verbose,
	}
	e, err := install.NewPreFlightExecutor(out, os.Stderr, options)
	if err != nil {
		return err
	}
	if err = e.RunPreFlightCheck(plan); err != nil {
		return err
	}
	return nil
}

// TODO this should really not be here
func newPKI(stdout io.Writer, options *validateOpts) (*install.LocalPKI, error) {
	ansibleDir := "ansible"
	if options.generatedAssetsDir == "" {
		return nil, fmt.Errorf("GeneratedAssetsDirectory option cannot be empty")
	}
	certsDir := filepath.Join(options.generatedAssetsDir, "keys")
	pki := &install.LocalPKI{
		CACsr:                   filepath.Join(ansibleDir, "playbooks", "tls", "ca-csr.json"),
		CAConfigFile:            filepath.Join(ansibleDir, "playbooks", "tls", "ca-config.json"),
		CASigningProfile:        "kubernetes",
		GeneratedCertsDirectory: certsDir,
		Log: stdout,
	}

	return pki, nil
}
<commit_msg>Inform the user that we are validating SSH connectivity<commit_after>package cli

import (
	"fmt"
	"io"
	"path/filepath"

	"os"

	"github.com/apprenda/kismatic/pkg/install"
	"github.com/apprenda/kismatic/pkg/util"
	"github.com/spf13/cobra"
)

type validateOpts struct {
	generatedAssetsDir string
	planFile           string
	verbose            bool
	outputFormat       string
	skipPreFlight      bool
}

// NewCmdValidate creates a new install validate command
func NewCmdValidate(out io.Writer, installOpts *installOpts) *cobra.Command {
	opts := &validateOpts{}
	cmd := &cobra.Command{
		Use:   "validate",
		Short: "validate your plan file",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) != 0 {
				return fmt.Errorf("Unexpected args: %v", args)
			}
			planner := &install.FilePlanner{File: installOpts.planFilename}
			opts.planFile = installOpts.planFilename
			return doValidate(out, planner, opts)
		},
	}
	cmd.Flags().StringVar(&opts.generatedAssetsDir, "generated-assets-dir", "generated", "path to the directory where assets generated during the installation process will be stored")
	cmd.Flags().BoolVar(&opts.verbose, "verbose", false, "enable verbose logging from the installation")
	cmd.Flags().StringVarP(&opts.outputFormat, "output", "o", "simple", "installation output format (options simple|raw)")
	cmd.Flags().BoolVar(&opts.skipPreFlight, "skip-preflight", false, "skip pre-flight checks")
	return cmd
}

func doValidate(out io.Writer, planner install.Planner, opts *validateOpts) error {
	util.PrintHeader(out, "Validating", '=')
	// Check if plan file exists
	if !planner.PlanExists() {
		util.PrettyPrintErr(out, "Reading installation plan file [ERROR]")
		fmt.Fprintln(out, "Run \"kismatic install plan\" to generate it")
		return fmt.Errorf("plan does not exist")
	}
	plan, err := planner.Read()
	if err != nil {
		util.PrettyPrintErr(out, "Reading installation plan file %q", opts.planFile)
		return fmt.Errorf("error reading plan file: %v", err)
	}
	util.PrettyPrintOk(out, "Reading installation plan file %q", opts.planFile)

	// Validate plan file
	ok, errs := install.ValidatePlan(plan)
	if !ok {
		util.PrettyPrintErr(out, "Validating installation plan file")
		util.PrintValidationErrors(out, errs)
		return fmt.Errorf("Plan file validation error prevents installation from proceeding")
	}
	util.PrettyPrintOk(out, "Validating installation plan file")

	// Validate SSH connections
	ok, errs = install.ValidatePlanSSHConnections(plan)
	if !ok {
		util.PrettyPrintErr(out, "Validating SSH connectivity to nodes")
		util.PrintValidationErrors(out, errs)
		return fmt.Errorf("SSH connectivity validation error prevents installation from proceeding")
	}
	util.PrettyPrintOk(out, "Validating SSH connectivity to nodes")

	// get a new pki
	pki, err := newPKI(out, opts)
	if err != nil {
		return err
	}
	// Validate Certificates
	ok, errs = install.ValidateCertificates(plan, pki)
	if !ok {
		util.PrettyPrintErr(out, "Validating cluster certificates")
		util.PrintValidationErrors(out, errs)
		return fmt.Errorf("Cluster certificates validation error prevents installation from proceeding")
	}

	if opts.skipPreFlight {
		return nil
	}
	// Run pre-flight
	options := install.ExecutorOptions{
		OutputFormat: opts.outputFormat,
		Verbose:      opts.verbose,
	}
	e, err := install.NewPreFlightExecutor(out, os.Stderr, options)
	if err != nil {
		return err
	}
	if err = e.RunPreFlightCheck(plan); err != nil {
		return err
	}
	return nil
}

// TODO this should really not be here
func newPKI(stdout io.Writer, options *validateOpts) (*install.LocalPKI, error) {
	ansibleDir := "ansible"
	if options.generatedAssetsDir == "" {
		return nil, fmt.Errorf("GeneratedAssetsDirectory option cannot be empty")
	}
	certsDir := filepath.Join(options.generatedAssetsDir, "keys")
	pki := &install.LocalPKI{
		CACsr:                   filepath.Join(ansibleDir, "playbooks", "tls", "ca-csr.json"),
		CAConfigFile:            filepath.Join(ansibleDir, "playbooks", "tls", "ca-config.json"),
		CASigningProfile:        "kubernetes",
		GeneratedCertsDirectory: certsDir,
		Log: stdout,
	}

	return pki, nil
}
<|endoftext|>


<commit_before>/*
 * Minio Client (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package s3

import (
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/minio/mc/pkg/client"
	"github.com/minio/minio-go"
	"github.com/minio/minio/pkg/iodine"
)

// Config - see http://docs.amazonwebservices.com/AmazonS3/latest/dev/index.html?RESTAuthentication.html
type Config struct {
	AccessKeyID     string
	SecretAccessKey string
	HostURL         string
	AppName         string
	AppVersion      string
	AppComments     []string
	Debug           bool

	// Used for SSL transport layer
	CertPEM string
	KeyPEM  string
}

// TLSConfig - TLS cert and key configuration
type TLSConfig struct {
	CertPEMBlock []byte
	KeyPEMBlock  []byte
}

type s3Client struct {
	api     minio.API
	hostURL *client.URL
}

// New returns an initialized s3Client structure. if debug use a internal trace transport
func New(config *Config) (client.Client, error) {
	u, err := client.Parse(config.HostURL)
	if err != nil {
		return nil, iodine.New(err, nil)
	}
	// convert any virtual host styled requests
	match, _ := filepath.Match("*.s3*.amazonaws.com", u.Host)
	if match {
		hostSplits := strings.SplitN(u.Host, ".", 2)
		u.Path = hostSplits[0]
		u.Host = hostSplits[1]
	}
	var transport http.RoundTripper
	switch {
	case config.Debug == true:
		transport = GetNewTraceTransport(NewTrace(), http.DefaultTransport)
	default:
		transport = http.DefaultTransport
	}
	s3Conf := minio.Config{
		AccessKeyID:     config.AccessKeyID,
		SecretAccessKey: config.SecretAccessKey,
		Transport:       transport,
		Endpoint:        u.Scheme + "://" + u.Host,
	}
	s3Conf.AccessKeyID = config.AccessKeyID
	s3Conf.SecretAccessKey = config.SecretAccessKey
	s3Conf.Transport = transport
	s3Conf.SetUserAgent(config.AppName, config.AppVersion, config.AppComments...)
	s3Conf.Endpoint = u.Scheme + "://" + u.Host
	api, err := minio.New(s3Conf)
	if err != nil {
		return nil, err
	}
	return &s3Client{api: api, hostURL: u}, nil
}

// URL get url
func (c *s3Client) URL() *client.URL {
	return c.hostURL
}

// GetObject - get object
func (c *s3Client) GetObject(offset, length int64) (io.ReadCloser, int64, error) {
	bucket, object := c.url2BucketAndObject()
	reader, metadata, err := c.api.GetPartialObject(bucket, object, offset, length)
	if err != nil {
		return nil, length, iodine.New(err, nil)
	}
	return reader, metadata.Size, nil
}

// ObjectAlreadyExists - typed return for MethodNotAllowed
type ObjectAlreadyExists struct {
	Object string
}

func (e ObjectAlreadyExists) Error() string {
	return "Object #" + e.Object + " already exists."
}

// PutObject - put object
func (c *s3Client) PutObject(size int64, data io.Reader) error {
	// md5 is purposefully ignored since AmazonS3 does not return proper md5sum
	// for a multipart upload and there is no need to cross verify,
	// invidual parts are properly verified
	bucket, object := c.url2BucketAndObject()
	err := c.api.PutObject(bucket, object, "application/octet-stream", size, data)
	if err != nil {
		if minio.ToErrorResponse(err).Code == "MethodNotAllowed" {
			return iodine.New(ObjectAlreadyExists{Object: object}, nil)
		}
		return iodine.New(err, nil)
	}
	return nil
}

// MakeBucket - make a new bucket
func (c *s3Client) MakeBucket() error {
	bucket, object := c.url2BucketAndObject()
	if object != "" {
		return iodine.New(client.InvalidQueryURL{URL: c.hostURL.String()}, nil)
	}
	// location string is intentionally left out
	err := c.api.MakeBucket(bucket, minio.BucketACL("private"))
	return iodine.New(err, nil)
}

// SetBucketACL add canned acl's on a bucket
func (c *s3Client) SetBucketACL(acl string) error {
	bucket, object := c.url2BucketAndObject()
	if object != "" {
		return iodine.New(client.InvalidQueryURL{URL: c.hostURL.String()}, nil)
	}
	err := c.api.SetBucketACL(bucket, minio.BucketACL(acl))
	return iodine.New(err, nil)
}

// Stat - send a 'HEAD' on a bucket or object to get its metadata
func (c *s3Client) Stat() (*client.Content, error) {
	objectMetadata := new(client.Content)
	bucket, object := c.url2BucketAndObject()
	switch {
	// valid case for s3:...
	case bucket == "" && object == "":
		for bucket := range c.api.ListBuckets() {
			if bucket.Err != nil {
				return nil, iodine.New(bucket.Err, nil)
			}
			return &client.Content{Type: os.ModeDir}, nil
		}
	}
	if object != "" {
		metadata, err := c.api.StatObject(bucket, object)
		if err != nil {
			errResponse := minio.ToErrorResponse(err)
			if errResponse != nil {
				if errResponse.Code == "NoSuchKey" {
					for content := range c.List(false) {
						if content.Err != nil {
							return nil, iodine.New(err, nil)
						}
						content.Content.Type = os.ModeDir
						content.Content.Name = object
						content.Content.Size = 0
						return content.Content, nil
					}
				}
			}
			return nil, iodine.New(err, nil)
		}
		objectMetadata.Name = metadata.Key
		objectMetadata.Time = metadata.LastModified
		objectMetadata.Size = metadata.Size
		objectMetadata.Type = os.FileMode(0664)
		return objectMetadata, nil
	}
	err := c.api.BucketExists(bucket)
	if err != nil {
		return nil, iodine.New(err, nil)
	}
	bucketMetadata := new(client.Content)
	bucketMetadata.Name = bucket
	bucketMetadata.Type = os.ModeDir
	return bucketMetadata, nil
}

// url2BucketAndObject gives bucketName and objectName from URL path
func (c *s3Client) url2BucketAndObject() (bucketName, objectName string) {
	splits := strings.SplitN(c.hostURL.Path, string(c.hostURL.Separator), 3)
	switch len(splits) {
	case 0, 1:
		bucketName = ""
		objectName = ""
	case 2:
		bucketName = splits[1]
		objectName = ""
	case 3:
		bucketName = splits[1]
		objectName = splits[2]
	}
	return bucketName, objectName
}

/// Bucket API operations

// List - list at delimited path, if not recursive
func (c *s3Client) List(recursive bool) <-chan client.ContentOnChannel {
	contentCh := make(chan client.ContentOnChannel)
	switch recursive {
	case true:
		go c.listRecursiveInRoutine(contentCh)
	default:
		go c.listInRoutine(contentCh)
	}
	return contentCh
}

func (c *s3Client) listInRoutine(contentCh chan client.ContentOnChannel) {
	defer close(contentCh)
	b, o := c.url2BucketAndObject()
	switch {
	case b == "" && o == "":
		for bucket := range c.api.ListBuckets() {
			if bucket.Err != nil {
				contentCh <- client.ContentOnChannel{
					Content: nil,
					Err:     bucket.Err,
				}
				return
			}
			content := new(client.Content)
			content.Name = bucket.Stat.Name
			content.Size = 0
			content.Time = bucket.Stat.CreationDate
			content.Type = os.ModeDir
			contentCh <- client.ContentOnChannel{
				Content: content,
				Err:     nil,
			}
		}
	default:
		metadata, err := c.api.StatObject(b, o)
		switch err.(type) {
		case nil:
			content := new(client.Content)
			content.Name = metadata.Key
			content.Time = metadata.LastModified
			content.Size = metadata.Size
			content.Type = os.FileMode(0664)
			contentCh <- client.ContentOnChannel{
				Content: content,
				Err:     nil,
			}
		default:
			for object := range c.api.ListObjects(b, o, false) {
				if object.Err != nil {
					contentCh <- client.ContentOnChannel{
						Content: nil,
						Err:     object.Err,
					}
					return
				}
				content := new(client.Content)
				normalizedPrefix := strings.TrimSuffix(o, string(c.hostURL.Separator)) + string(c.hostURL.Separator)
				normalizedKey := object.Stat.Key
				if normalizedPrefix != object.Stat.Key && strings.HasPrefix(object.Stat.Key, normalizedPrefix) {
					normalizedKey = strings.TrimPrefix(object.Stat.Key, normalizedPrefix)
				}
				content.Name = normalizedKey
				switch {
				case strings.HasSuffix(object.Stat.Key, string(c.hostURL.Separator)):
					content.Time = time.Now()
					content.Type = os.ModeDir
				default:
					content.Size = object.Stat.Size
					content.Time = object.Stat.LastModified
					content.Type = os.FileMode(0664)
				}
				contentCh <- client.ContentOnChannel{
					Content: content,
					Err:     nil,
				}
			}
		}
	}
}

func (c *s3Client) listRecursiveInRoutine(contentCh chan client.ContentOnChannel) {
	defer close(contentCh)
	b, o := c.url2BucketAndObject()
	switch {
	case b == "" && o == "":
		for bucket := range c.api.ListBuckets() {
			if bucket.Err != nil {
				contentCh <- client.ContentOnChannel{
					Content: nil,
					Err:     bucket.Err,
				}
				return
			}
			for object := range c.api.ListObjects(bucket.Stat.Name, o, true) {
				if object.Err != nil {
					contentCh <- client.ContentOnChannel{
						Content: nil,
						Err:     object.Err,
					}
					return
				}
				content := new(client.Content)
				content.Name = filepath.Join(bucket.Stat.Name, object.Stat.Key)
				content.Size = object.Stat.Size
				content.Time = object.Stat.LastModified
				content.Type = os.FileMode(0664)
				contentCh <- client.ContentOnChannel{
					Content: content,
					Err:     nil,
				}
			}
		}
	default:
		for object := range c.api.ListObjects(b, o, true) {
			if object.Err != nil {
				contentCh <- client.ContentOnChannel{
					Content: nil,
					Err:     object.Err,
				}
				return
			}
			content := new(client.Content)
			normalizedKey := object.Stat.Key
			switch {
			case o == "":
				// if no prefix provided and also URL is not delimited then we add bucket back into object name
				if strings.LastIndex(c.hostURL.Path, string(c.hostURL.Separator)) == 0 {
					if c.hostURL.String()[:strings.LastIndex(c.hostURL.String(), string(c.hostURL.Separator))+1] != b {
						normalizedKey = filepath.Join(b, object.Stat.Key)
					}
				}
			default:
				if strings.HasSuffix(o, string(c.hostURL.Separator)) {
					normalizedKey = strings.TrimPrefix(object.Stat.Key, o)
				}
			}
			content.Name = normalizedKey
			content.Size = object.Stat.Size
			content.Time = object.Stat.LastModified
			content.Type = os.FileMode(0664)
			contentCh <- client.ContentOnChannel{
				Content: content,
				Err:     nil,
			}
		}
	}
}
<commit_msg>Fix path when bucket is specified as part of domain name<commit_after>/*
 * Minio Client (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package s3

import (
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/minio/mc/pkg/client"
	"github.com/minio/minio-go"
	"github.com/minio/minio/pkg/iodine"
)

// Config - see http://docs.amazonwebservices.com/AmazonS3/latest/dev/index.html?RESTAuthentication.html
type Config struct {
	AccessKeyID     string
	SecretAccessKey string
	HostURL         string
	AppName         string
	AppVersion      string
	AppComments     []string
	Debug           bool

	// Used for SSL transport layer
	CertPEM string
	KeyPEM  string
}

// TLSConfig - TLS cert and key configuration
type TLSConfig struct {
	CertPEMBlock []byte
	KeyPEMBlock  []byte
}

type s3Client struct {
	api     minio.API
	hostURL *client.URL
}

// New returns an initialized s3Client structure. if debug use a internal trace transport
func New(config *Config) (client.Client, error) {
	u, err := client.Parse(config.HostURL)
	if err != nil {
		return nil, iodine.New(err, nil)
	}
	// convert any virtual host styled requests
	match, _ := filepath.Match("*.s3*.amazonaws.com", u.Host)
	if match {
		hostSplits := strings.SplitN(u.Host, ".", 2)
		u.Path = "/" + hostSplits[0] + u.Path
		u.Host = hostSplits[1]
	}
	var transport http.RoundTripper
	switch {
	case config.Debug == true:
		transport = GetNewTraceTransport(NewTrace(), http.DefaultTransport)
	default:
		transport = http.DefaultTransport
	}
	s3Conf := minio.Config{
		AccessKeyID:     config.AccessKeyID,
		SecretAccessKey: config.SecretAccessKey,
		Transport:       transport,
		Endpoint:        u.Scheme + "://" + u.Host,
	}
	s3Conf.AccessKeyID = config.AccessKeyID
	s3Conf.SecretAccessKey = config.SecretAccessKey
	s3Conf.Transport = transport
	s3Conf.SetUserAgent(config.AppName, config.AppVersion, config.AppComments...)
	s3Conf.Endpoint = u.Scheme + "://" + u.Host
	api, err := minio.New(s3Conf)
	if err != nil {
		return nil, err
	}
	return &s3Client{api: api, hostURL: u}, nil
}

// URL get url
func (c *s3Client) URL() *client.URL {
	return c.hostURL
}

// GetObject - get object
func (c *s3Client) GetObject(offset, length int64) (io.ReadCloser, int64, error) {
	bucket, object := c.url2BucketAndObject()
	reader, metadata, err := c.api.GetPartialObject(bucket, object, offset, length)
	if err != nil {
		return nil, length, iodine.New(err, nil)
	}
	return reader, metadata.Size, nil
}

// ObjectAlreadyExists - typed return for MethodNotAllowed
type ObjectAlreadyExists struct {
	Object string
}

func (e ObjectAlreadyExists) Error() string {
	return "Object #" + e.Object + " already exists."
}

// PutObject - put object
func (c *s3Client) PutObject(size int64, data io.Reader) error {
	// md5 is purposefully ignored since AmazonS3 does not return proper md5sum
	// for a multipart upload and there is no need to cross verify,
	// invidual parts are properly verified
	bucket, object := c.url2BucketAndObject()
	err := c.api.PutObject(bucket, object, "application/octet-stream", size, data)
	if err != nil {
		if minio.ToErrorResponse(err).Code == "MethodNotAllowed" {
			return iodine.New(ObjectAlreadyExists{Object: object}, nil)
		}
		return iodine.New(err, nil)
	}
	return nil
}

// MakeBucket - make a new bucket
func (c *s3Client) MakeBucket() error {
	bucket, object := c.url2BucketAndObject()
	if object != "" {
		return iodine.New(client.InvalidQueryURL{URL: c.hostURL.String()}, nil)
	}
	// location string is intentionally left out
	err := c.api.MakeBucket(bucket, minio.BucketACL("private"))
	return iodine.New(err, nil)
}

// SetBucketACL add canned acl's on a bucket
func (c *s3Client) SetBucketACL(acl string) error {
	bucket, object := c.url2BucketAndObject()
	if object != "" {
		return iodine.New(client.InvalidQueryURL{URL: c.hostURL.String()}, nil)
	}
	err := c.api.SetBucketACL(bucket, minio.BucketACL(acl))
	return iodine.New(err, nil)
}

// Stat - send a 'HEAD' on a bucket or object to get its metadata
func (c *s3Client) Stat() (*client.Content, error) {
	objectMetadata := new(client.Content)
	bucket, object := c.url2BucketAndObject()
	switch {
	// valid case for s3:...
	case bucket == "" && object == "":
		for bucket := range c.api.ListBuckets() {
			if bucket.Err != nil {
				return nil, iodine.New(bucket.Err, nil)
			}
			return &client.Content{Type: os.ModeDir}, nil
		}
	}
	if object != "" {
		metadata, err := c.api.StatObject(bucket, object)
		if err != nil {
			errResponse := minio.ToErrorResponse(err)
			if errResponse != nil {
				if errResponse.Code == "NoSuchKey" {
					for content := range c.List(false) {
						if content.Err != nil {
							return nil, iodine.New(err, nil)
						}
						content.Content.Type = os.ModeDir
						content.Content.Name = object
						content.Content.Size = 0
						return content.Content, nil
					}
				}
			}
			return nil, iodine.New(err, nil)
		}
		objectMetadata.Name = metadata.Key
		objectMetadata.Time = metadata.LastModified
		objectMetadata.Size = metadata.Size
		objectMetadata.Type = os.FileMode(0664)
		return objectMetadata, nil
	}
	err := c.api.BucketExists(bucket)
	if err != nil {
		return nil, iodine.New(err, nil)
	}
	bucketMetadata := new(client.Content)
	bucketMetadata.Name = bucket
	bucketMetadata.Type = os.ModeDir
	return bucketMetadata, nil
}

// url2BucketAndObject gives bucketName and objectName from URL path
func (c *s3Client) url2BucketAndObject() (bucketName, objectName string) {
	splits := strings.SplitN(c.hostURL.Path, string(c.hostURL.Separator), 3)
	switch len(splits) {
	case 0, 1:
		bucketName = ""
		objectName = ""
	case 2:
		bucketName = splits[1]
		objectName = ""
	case 3:
		bucketName = splits[1]
		objectName = splits[2]
	}
	return bucketName, objectName
}

/// Bucket API operations

// List - list at delimited path, if not recursive
func (c *s3Client) List(recursive bool) <-chan client.ContentOnChannel {
	contentCh := make(chan client.ContentOnChannel)
	switch recursive {
	case true:
		go c.listRecursiveInRoutine(contentCh)
	default:
		go c.listInRoutine(contentCh)
	}
	return contentCh
}

func (c *s3Client) listInRoutine(contentCh chan client.ContentOnChannel) {
	defer close(contentCh)
	b, o := c.url2BucketAndObject()
	switch {
	case b == "" && o == "":
		for bucket := range c.api.ListBuckets() {
			if bucket.Err != nil {
				contentCh <- client.ContentOnChannel{
					Content: nil,
					Err:     bucket.Err,
				}
				return
			}
			content := new(client.Content)
			content.Name = bucket.Stat.Name
			content.Size = 0
			content.Time = bucket.Stat.CreationDate
			content.Type = os.ModeDir
			contentCh <- client.ContentOnChannel{
				Content: content,
				Err:     nil,
			}
		}
	default:
		metadata, err := c.api.StatObject(b, o)
		switch err.(type) {
		case nil:
			content := new(client.Content)
			content.Name = metadata.Key
			content.Time = metadata.LastModified
			content.Size = metadata.Size
			content.Type = os.FileMode(0664)
			contentCh <- client.ContentOnChannel{
				Content: content,
				Err:     nil,
			}
		default:
			for object := range c.api.ListObjects(b, o, false) {
				if object.Err != nil {
					contentCh <- client.ContentOnChannel{
						Content: nil,
						Err:     object.Err,
					}
					return
				}
				content := new(client.Content)
				normalizedPrefix := strings.TrimSuffix(o, string(c.hostURL.Separator)) + string(c.hostURL.Separator)
				normalizedKey := object.Stat.Key
				if normalizedPrefix != object.Stat.Key && strings.HasPrefix(object.Stat.Key, normalizedPrefix) {
					normalizedKey = strings.TrimPrefix(object.Stat.Key, normalizedPrefix)
				}
				content.Name = normalizedKey
				switch {
				case strings.HasSuffix(object.Stat.Key, string(c.hostURL.Separator)):
					content.Time = time.Now()
					content.Type = os.ModeDir
				default:
					content.Size = object.Stat.Size
					content.Time = object.Stat.LastModified
					content.Type = os.FileMode(0664)
				}
				contentCh <- client.ContentOnChannel{
					Content: content,
					Err:     nil,
				}
			}
		}
	}
}

func (c *s3Client) listRecursiveInRoutine(contentCh chan client.ContentOnChannel) {
	defer close(contentCh)
	b, o := c.url2BucketAndObject()
	switch {
	case b == "" && o == "":
		for bucket := range c.api.ListBuckets() {
			if bucket.Err != nil {
				contentCh <- client.ContentOnChannel{
					Content: nil,
					Err:     bucket.Err,
				}
				return
			}
			for object := range c.api.ListObjects(bucket.Stat.Name, o, true) {
				if object.Err != nil {
					contentCh <- client.ContentOnChannel{
						Content: nil,
						Err:     object.Err,
					}
					return
				}
				content := new(client.Content)
				content.Name = filepath.Join(bucket.Stat.Name, object.Stat.Key)
				content.Size = object.Stat.Size
				content.Time = object.Stat.LastModified
				content.Type = os.FileMode(0664)
				contentCh <- client.ContentOnChannel{
					Content: content,
					Err:     nil,
				}
			}
		}
	default:
		for object := range c.api.ListObjects(b, o, true) {
			if object.Err != nil {
				contentCh <- client.ContentOnChannel{
					Content: nil,
					Err:     object.Err,
				}
				return
			}
			content := new(client.Content)
			normalizedKey := object.Stat.Key
			switch {
			case o == "":
				// if no prefix provided and also URL is not delimited then we add bucket back into object name
				if strings.LastIndex(c.hostURL.Path, string(c.hostURL.Separator)) == 0 {
					if c.hostURL.String()[:strings.LastIndex(c.hostURL.String(), string(c.hostURL.Separator))+1] != b {
						normalizedKey = filepath.Join(b, object.Stat.Key)
					}
				}
			default:
				if strings.HasSuffix(o, string(c.hostURL.Separator)) {
					normalizedKey = strings.TrimPrefix(object.Stat.Key, o)
				}
			}
			content.Name = normalizedKey
			content.Size = object.Stat.Size
			content.Time = object.Stat.LastModified
			content.Type = os.FileMode(0664)
			contentCh <- client.ContentOnChannel{
				Content: content,
				Err:     nil,
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"strconv"
	"strings"
	"text/template"
)

func rootpage(w http.ResponseWriter, req *http.Request) {
  // The "/" pattern matches everything, so we need to check
  // that we're at the root here.
  if req.URL.Path != "/" {
	  http.NotFound(w, req)
	  return
  }

	// get rid of :port
	host := req.Host
	if i := strings.Index(host, ":"); i != -1 {
		host = host[:i]
	}

	templ.Execute(w, struct {
		KodiURL string
		KodiRPC string
	}{
		KodiURL: "http://" + host + ":" + strconv.Itoa(*kodiHttpPort),
		KodiRPC: host + ":" + strconv.Itoa(*kodiRpcPort),
	})
}

var templ = template.Must(template.New("").Parse(content))

// %%%%content=rootpage.htm%%%%
// START OF AUTOGENERATED CONTENT - DO NOT CHANGE ANYTHING BELOW
// GENERATED: 2018-03-20 18:28:50.3567943 +0100 CET m=+0.016000201
var content = `<!doctype html><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1"><title>Kodi</title>
<form>
<input name=url id=link class=InputAddOn-field placeholder=http://... autocomplete=off type=url size=50 autofocus required>
<input class=InputAddOn-item id=send-link type=submit onclick=sendUrl()>

<input type=range min=1 max=100 class=slider id=progress onchange=seek(this.value)>

Kodi Web Interface
</form>
`<commit_msg>go fmt<commit_after>package main

import (
	"net/http"
	"strconv"
	"strings"
	"text/template"
)

func rootpage(w http.ResponseWriter, req *http.Request) {
	// The "/" pattern matches everything, so we need to check
	// that we're at the root here.
	if req.URL.Path != "/" {
		http.NotFound(w, req)
		return
	}

	// get rid of :port
	host := req.Host
	if i := strings.Index(host, ":"); i != -1 {
		host = host[:i]
	}

	templ.Execute(w, struct {
		KodiURL string
		KodiRPC string
	}{
		KodiURL: "http://" + host + ":" + strconv.Itoa(*kodiHttpPort),
		KodiRPC: host + ":" + strconv.Itoa(*kodiRpcPort),
	})
}

var templ = template.Must(template.New("").Parse(content))

// %%%%content=rootpage.htm%%%%
// START OF AUTOGENERATED CONTENT - DO NOT CHANGE ANYTHING BELOW
// GENERATED: 2018-03-20 18:28:50.3567943 +0100 CET m=+0.016000201
var content = `<!doctype html><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1"><title>Kodi</title>
<form>
<input name=url id=link class=InputAddOn-field placeholder=http://... autocomplete=off type=url size=50 autofocus required>
<input class=InputAddOn-item id=send-link type=submit onclick=sendUrl()>

<input type=range min=1 max=100 class=slider id=progress onchange=seek(this.value)>

Kodi Web Interface
</form>
`<|endoftext|>


<commit_before>// Application that does metrics analysis as described in the design doc:
// go/ct_metrics_analysis
//
// Can be tested locally with:
// $ go run go/worker_scripts/metrics_analysis/main.go --start_range=1 --num=3 --run_id=rmistry-test1 --benchmark_extra_args="--output-format=csv" --metric_name="loadingMetric" --logtostderr=true --local --chromium_hash=cab48032c45695c73b202af7d472a52808d551ca
//
package main

import (
	"bytes"
	"context"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"go.skia.org/infra/go/sklog"

	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/ct/go/worker_scripts/worker_common"
	"go.skia.org/infra/go/common"
	skutil "go.skia.org/infra/go/util"
)

const (
	// The number of goroutines that will run in parallel to download traces and
	// run metrics analysis.
	WORKER_POOL_SIZE = 10

	METRICS_BENCHMARK_TIMEOUT_SECS = 300

	TRACE_OUTPUT_BUCKET = "chrome-telemetry-output"
)

var (
	startRange         = flag.Int("start_range", 1, "The number this worker will run metrics analysis from.")
	num                = flag.Int("num", 100, "The total number of traces to run metrics analysis from the start_range.")
	runID              = flag.String("run_id", "", "The unique run id (typically requester + timestamp).")
	benchmarkExtraArgs = flag.String("benchmark_extra_args", "", "The extra arguments that are passed to the specified benchmark.")
	metricName         = flag.String("metric_name", "", "The metric to parse the traces with. Eg: loadingMetric")
	chromiumHash       = flag.String("chromium_hash", "", "The chromium commit hash to sync the local checkout to. Done so that all slaves operate on the same revision.")
)

func metricsAnalysis() error {
	defer common.LogPanic()
	worker_common.Init()
	if !*worker_common.Local {
		defer util.CleanTmpDir()
	}
	defer util.TimeTrack(time.Now(), "Metrics Analysis")
	defer sklog.Flush()

	// Validate required arguments.
	if *runID == "" {
		return errors.New("Must specify --run_id")
	}
	if *metricName == "" {
		return errors.New("Must specify --metric_name")
	}
	if *chromiumHash == "" {
		return errors.New("Must specify --chromium_hash")
	}

	ctx := context.Background()

	// Reset the local chromium checkout.
	if err := util.ResetChromiumCheckout(ctx, util.ChromiumSrcDir); err != nil {
		return fmt.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err)
	}
	// Sync the local chromium checkout.
	if err := util.SyncDir(ctx, util.ChromiumSrcDir, map[string]string{"src": *chromiumHash}, []string{}); err != nil {
		return fmt.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
	}

	// Instantiate GcsUtil object.
	gs, err := util.NewGcsUtil(nil)
	if err != nil {
		return fmt.Errorf("GcsUtil instantiation failed: %s", err)
	}

	// Download the trace URLs for this run from Google storage.
	tracesFilename := *runID + ".traces.csv"
	tmpDir, err := ioutil.TempDir(util.PagesetsDir, "traces")
	if err != nil {
		return fmt.Errorf("Could not create tmpdir: %s", err)
	}
	defer skutil.RemoveAll(tmpDir)
	remotePatchesDir := filepath.Join(util.BenchmarkRunsDir, *runID)

	// Download the catapult patch for this run from Google storage.
	catapultPatchName := *runID + ".catapult.patch"
	if err := util.DownloadAndApplyPatch(ctx, catapultPatchName, tmpDir, remotePatchesDir, util.CatapultSrcDir, gs); err != nil {
		return fmt.Errorf("Could not apply %s: %s", catapultPatchName, err)
	}

	// Download the chromium patch for this run from Google storage.
	chromiumPatchName := *runID + ".chromium.patch"
	if err := util.DownloadAndApplyPatch(ctx, chromiumPatchName, tmpDir, remotePatchesDir, util.ChromiumSrcDir, gs); err != nil {
		return fmt.Errorf("Could not apply %s: %s", chromiumPatchName, err)
	}

	// Download traces.
	if _, err := util.DownloadPatch(filepath.Join(tmpDir, tracesFilename), filepath.Join(remotePatchesDir, tracesFilename), gs); err != nil {
		return fmt.Errorf("Could not download %s: %s", tracesFilename, err)
	}
	traces, err := util.GetCustomPages(filepath.Join(tmpDir, tracesFilename))
	if err != nil {
		return fmt.Errorf("Could not read custom traces file %s: %s", tracesFilename, err)
	}
	if len(traces) == 0 {
		return errors.New(fmt.Sprintf("No traces found in %s", tracesFilename))
	}
	traces = util.GetCustomPagesWithinRange(*startRange, *num, traces)
	sklog.Infof("Using %d traces", len(traces))

	// Establish output paths for trace downloads and metrics.
	traceDownloadDir := filepath.Join(util.StorageDir, util.TraceDownloadsDir, *runID)
	skutil.RemoveAll(traceDownloadDir)
	skutil.MkdirAll(traceDownloadDir, 0700)
	defer skutil.RemoveAll(traceDownloadDir)

	localOutputDir := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runID)
	skutil.RemoveAll(localOutputDir)
	skutil.MkdirAll(localOutputDir, 0700)
	defer skutil.RemoveAll(localOutputDir)
	remoteDir := filepath.Join(util.BenchmarkRunsDir, *runID)

	sklog.Infof("===== Going to run the task with %d parallel goroutines =====", WORKER_POOL_SIZE)
	// Create channel that contains all trace ULs. This channel will
	// be consumed by the worker pool.
	traceRequests := getClosedChannelOfTraces(traces)

	var wg sync.WaitGroup

	// If not a single benchmark run succeeds then throw at error at the end.
	atleastOneBenchmarkSucceeded := false
	// Gather traceURLs that could not be downloaded.
	erroredTraces := []string{}
	// Mutex to control access to the above slice.
	var erroredTracesMutex sync.Mutex

	// Loop through workers in the worker pool.
	for i := 0; i < WORKER_POOL_SIZE; i++ {
		// Increment the WaitGroup counter.
		wg.Add(1)

		// Create and run a goroutine closure that runs the analysis benchmark.
		go func() {
			// Decrement the WaitGroup counter when the goroutine completes.
			defer wg.Done()

			for t := range traceRequests {
				sklog.Infof("========== Downloading trace %s ==========", t)
				downloadedTrace, err := downloadTrace(t, traceDownloadDir, gs)
				if err != nil {
					sklog.Errorf("Could not download %s: %s", t, err)
					erroredTracesMutex.Lock()
					erroredTraces = append(erroredTraces, t)
					erroredTracesMutex.Unlock()
					continue
				}

				sklog.Infof("========== Processing %s ==========", t)
				if err := runMetricsAnalysisBenchmark(ctx, localOutputDir, downloadedTrace, t); err != nil {
					sklog.Errorf("Error during run_benchmark: %s", err)
				} else {
					atleastOneBenchmarkSucceeded = true
				}
			}
		}()
	}

	// Wait for all spawned goroutines to complete.
	wg.Wait()

	// Summarize errors.
	if len(erroredTraces) > 0 {
		sklog.Error("The following traces could not be downloaded:")
		for _, erroredTrace := range erroredTraces {
			sklog.Errorf("\t%s", erroredTrace)
		}
	}
	if !atleastOneBenchmarkSucceeded {
		return errors.New("Not a single benchmark run was successful. Something is wrong.")
	}

	// If "--output-format=csv" was specified then merge all CSV files and upload.
	if strings.Contains(*benchmarkExtraArgs, "--output-format=csv") {
		// Construct path to CT's python scripts.
		pathToPyFiles := util.GetPathToPyFiles(!*worker_common.Local)
		if err := util.MergeUploadCSVFilesOnWorkers(ctx, localOutputDir, pathToPyFiles, *runID, remoteDir, gs, *startRange, true /* handleStrings */, false /* addRanks */, map[string]map[string]string{} /* pageRankToAdditionalFields */); err != nil {
			return fmt.Errorf("Error while processing withpatch CSV files: %s", err)
		}
	}

	return nil
}

// runMetricsAnalysisBenchmark runs the analysis_metrics_ct benchmark on the provided trace.
func runMetricsAnalysisBenchmark(ctx context.Context, outputPath, downloadedTrace, cloudTraceLink string) error {
	args := []string{
		filepath.Join(util.TelemetryBinariesDir, util.BINARY_RUN_BENCHMARK),
		util.BENCHMARK_METRICS_ANALYSIS,
		"--local-trace-path", fmt.Sprintf("file://%s", downloadedTrace),
		"--cloud-trace-link", cloudTraceLink,
		"--metric-name", *metricName,
		"--browser", "system", // No browser is brought up but unfortunately this is needed by the framework.
		"--output-dir", filepath.Join(outputPath, getTraceName(downloadedTrace)),
	}
	// Split benchmark args if not empty and append to args.
	if *benchmarkExtraArgs != "" {
		args = append(args, strings.Fields(*benchmarkExtraArgs)...)
	}
	// Set the PYTHONPATH to the telemetry dirs.
	env := []string{
		fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHONPATH", util.TelemetryBinariesDir, util.TelemetrySrcDir, util.CatapultSrcDir),
		"DISPLAY=:0",
	}
	// Append the original environment as well.
	for _, e := range os.Environ() {
		env = append(env, e)
	}

	// Create buffer for capturing the stdout and stderr of the benchmark run.
	var b bytes.Buffer
	if _, err := b.WriteString(fmt.Sprintf("========== Stdout and stderr for %s ==========\n", downloadedTrace)); err != nil {
		return fmt.Errorf("Error writing to output buffer: %s", err)
	}
	if err := util.ExecuteCmdWithConfigurableLogging(ctx, "python", args, env, METRICS_BENCHMARK_TIMEOUT_SECS*time.Second, &b, &b, false, false); err != nil {
		output, getErr := util.GetRunBenchmarkOutput(b)
		skutil.LogErr(getErr)
		fmt.Println(output)
		return fmt.Errorf("Run benchmark command failed with: %s", err)
	}
	output, err := util.GetRunBenchmarkOutput(b)
	if err != nil {
		return fmt.Errorf("Could not get run benchmark output: %s", err)
	}
	// Print the output and return.
	fmt.Println(output)
	return nil
}

// downloadTrace downloads the traceURL from google storage into the specified
// destDir.
func downloadTrace(traceURL, destDir string, gs *util.GcsUtil) (string, error) {
	traceName := getTraceName(traceURL)
	traceDest := filepath.Join(destDir, traceName)
	if err := gs.DownloadRemoteFileFromBucket(TRACE_OUTPUT_BUCKET, traceName, traceDest); err != nil {
		return "", fmt.Errorf("Error downloading %s from %s to %s: %s", traceName, TRACE_OUTPUT_BUCKET, traceDest, err)
	}
	return traceDest, nil
}

// getTraceName parses the provided traceURI and returns the name of the trace.
// traceURI could be a file path or a URL.
func getTraceName(traceURI string) string {
	traceTokens := strings.Split(traceURI, "/")
	return traceTokens[len(traceTokens)-1]
}

// getClosedChannelOfTraces returns a channel that contains all trace URLs.
func getClosedChannelOfTraces(traces []string) chan string {
	tracesChannel := make(chan string, len(traces))
	for _, t := range traces {
		tracesChannel <- t
	}
	close(tracesChannel)
	return tracesChannel
}

func main() {
	retCode := 0
	if err := metricsAnalysis(); err != nil {
		sklog.Errorf("Error while running metrics analysis: %s", err)
		retCode = 255
	}
	os.Exit(retCode)
}
<commit_msg>[CT] Half the number of parallel goroutines to try to prevent OOMs<commit_after>// Application that does metrics analysis as described in the design doc:
// go/ct_metrics_analysis
//
// Can be tested locally with:
// $ go run go/worker_scripts/metrics_analysis/main.go --start_range=1 --num=3 --run_id=rmistry-test1 --benchmark_extra_args="--output-format=csv" --metric_name="loadingMetric" --logtostderr=true --local --chromium_hash=cab48032c45695c73b202af7d472a52808d551ca
//
package main

import (
	"bytes"
	"context"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"go.skia.org/infra/go/sklog"

	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/ct/go/worker_scripts/worker_common"
	"go.skia.org/infra/go/common"
	skutil "go.skia.org/infra/go/util"
)

const (
	// The number of goroutines that will run in parallel to download traces and
	// run metrics analysis.
	WORKER_POOL_SIZE = 5

	METRICS_BENCHMARK_TIMEOUT_SECS = 300

	TRACE_OUTPUT_BUCKET = "chrome-telemetry-output"
)

var (
	startRange         = flag.Int("start_range", 1, "The number this worker will run metrics analysis from.")
	num                = flag.Int("num", 100, "The total number of traces to run metrics analysis from the start_range.")
	runID              = flag.String("run_id", "", "The unique run id (typically requester + timestamp).")
	benchmarkExtraArgs = flag.String("benchmark_extra_args", "", "The extra arguments that are passed to the specified benchmark.")
	metricName         = flag.String("metric_name", "", "The metric to parse the traces with. Eg: loadingMetric")
	chromiumHash       = flag.String("chromium_hash", "", "The chromium commit hash to sync the local checkout to. Done so that all slaves operate on the same revision.")
)

func metricsAnalysis() error {
	defer common.LogPanic()
	worker_common.Init()
	if !*worker_common.Local {
		defer util.CleanTmpDir()
	}
	defer util.TimeTrack(time.Now(), "Metrics Analysis")
	defer sklog.Flush()

	// Validate required arguments.
	if *runID == "" {
		return errors.New("Must specify --run_id")
	}
	if *metricName == "" {
		return errors.New("Must specify --metric_name")
	}
	if *chromiumHash == "" {
		return errors.New("Must specify --chromium_hash")
	}

	ctx := context.Background()

	// Reset the local chromium checkout.
	if err := util.ResetChromiumCheckout(ctx, util.ChromiumSrcDir); err != nil {
		return fmt.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err)
	}
	// Sync the local chromium checkout.
	if err := util.SyncDir(ctx, util.ChromiumSrcDir, map[string]string{"src": *chromiumHash}, []string{}); err != nil {
		return fmt.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
	}

	// Instantiate GcsUtil object.
	gs, err := util.NewGcsUtil(nil)
	if err != nil {
		return fmt.Errorf("GcsUtil instantiation failed: %s", err)
	}

	// Download the trace URLs for this run from Google storage.
	tracesFilename := *runID + ".traces.csv"
	tmpDir, err := ioutil.TempDir(util.PagesetsDir, "traces")
	if err != nil {
		return fmt.Errorf("Could not create tmpdir: %s", err)
	}
	defer skutil.RemoveAll(tmpDir)
	remotePatchesDir := filepath.Join(util.BenchmarkRunsDir, *runID)

	// Download the catapult patch for this run from Google storage.
	catapultPatchName := *runID + ".catapult.patch"
	if err := util.DownloadAndApplyPatch(ctx, catapultPatchName, tmpDir, remotePatchesDir, util.CatapultSrcDir, gs); err != nil {
		return fmt.Errorf("Could not apply %s: %s", catapultPatchName, err)
	}

	// Download the chromium patch for this run from Google storage.
	chromiumPatchName := *runID + ".chromium.patch"
	if err := util.DownloadAndApplyPatch(ctx, chromiumPatchName, tmpDir, remotePatchesDir, util.ChromiumSrcDir, gs); err != nil {
		return fmt.Errorf("Could not apply %s: %s", chromiumPatchName, err)
	}

	// Download traces.
	if _, err := util.DownloadPatch(filepath.Join(tmpDir, tracesFilename), filepath.Join(remotePatchesDir, tracesFilename), gs); err != nil {
		return fmt.Errorf("Could not download %s: %s", tracesFilename, err)
	}
	traces, err := util.GetCustomPages(filepath.Join(tmpDir, tracesFilename))
	if err != nil {
		return fmt.Errorf("Could not read custom traces file %s: %s", tracesFilename, err)
	}
	if len(traces) == 0 {
		return errors.New(fmt.Sprintf("No traces found in %s", tracesFilename))
	}
	traces = util.GetCustomPagesWithinRange(*startRange, *num, traces)
	sklog.Infof("Using %d traces", len(traces))

	// Establish output paths for trace downloads and metrics.
	traceDownloadDir := filepath.Join(util.StorageDir, util.TraceDownloadsDir, *runID)
	skutil.RemoveAll(traceDownloadDir)
	skutil.MkdirAll(traceDownloadDir, 0700)
	defer skutil.RemoveAll(traceDownloadDir)

	localOutputDir := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runID)
	skutil.RemoveAll(localOutputDir)
	skutil.MkdirAll(localOutputDir, 0700)
	defer skutil.RemoveAll(localOutputDir)
	remoteDir := filepath.Join(util.BenchmarkRunsDir, *runID)

	sklog.Infof("===== Going to run the task with %d parallel goroutines =====", WORKER_POOL_SIZE)
	// Create channel that contains all trace ULs. This channel will
	// be consumed by the worker pool.
	traceRequests := getClosedChannelOfTraces(traces)

	var wg sync.WaitGroup

	// If not a single benchmark run succeeds then throw at error at the end.
	atleastOneBenchmarkSucceeded := false
	// Gather traceURLs that could not be downloaded.
	erroredTraces := []string{}
	// Mutex to control access to the above slice.
	var erroredTracesMutex sync.Mutex

	// Loop through workers in the worker pool.
	for i := 0; i < WORKER_POOL_SIZE; i++ {
		// Increment the WaitGroup counter.
		wg.Add(1)

		// Create and run a goroutine closure that runs the analysis benchmark.
		go func() {
			// Decrement the WaitGroup counter when the goroutine completes.
			defer wg.Done()

			for t := range traceRequests {
				sklog.Infof("========== Downloading trace %s ==========", t)
				downloadedTrace, err := downloadTrace(t, traceDownloadDir, gs)
				if err != nil {
					sklog.Errorf("Could not download %s: %s", t, err)
					erroredTracesMutex.Lock()
					erroredTraces = append(erroredTraces, t)
					erroredTracesMutex.Unlock()
					continue
				}

				sklog.Infof("========== Processing %s ==========", t)
				if err := runMetricsAnalysisBenchmark(ctx, localOutputDir, downloadedTrace, t); err != nil {
					sklog.Errorf("Error during run_benchmark: %s", err)
				} else {
					atleastOneBenchmarkSucceeded = true
				}
			}
		}()
	}

	// Wait for all spawned goroutines to complete.
	wg.Wait()

	// Summarize errors.
	if len(erroredTraces) > 0 {
		sklog.Error("The following traces could not be downloaded:")
		for _, erroredTrace := range erroredTraces {
			sklog.Errorf("\t%s", erroredTrace)
		}
	}
	if !atleastOneBenchmarkSucceeded {
		return errors.New("Not a single benchmark run was successful. Something is wrong.")
	}

	// If "--output-format=csv" was specified then merge all CSV files and upload.
	if strings.Contains(*benchmarkExtraArgs, "--output-format=csv") {
		// Construct path to CT's python scripts.
		pathToPyFiles := util.GetPathToPyFiles(!*worker_common.Local)
		if err := util.MergeUploadCSVFilesOnWorkers(ctx, localOutputDir, pathToPyFiles, *runID, remoteDir, gs, *startRange, true /* handleStrings */, false /* addRanks */, map[string]map[string]string{} /* pageRankToAdditionalFields */); err != nil {
			return fmt.Errorf("Error while processing withpatch CSV files: %s", err)
		}
	}

	return nil
}

// runMetricsAnalysisBenchmark runs the analysis_metrics_ct benchmark on the provided trace.
func runMetricsAnalysisBenchmark(ctx context.Context, outputPath, downloadedTrace, cloudTraceLink string) error {
	args := []string{
		filepath.Join(util.TelemetryBinariesDir, util.BINARY_RUN_BENCHMARK),
		util.BENCHMARK_METRICS_ANALYSIS,
		"--local-trace-path", fmt.Sprintf("file://%s", downloadedTrace),
		"--cloud-trace-link", cloudTraceLink,
		"--metric-name", *metricName,
		"--browser", "system", // No browser is brought up but unfortunately this is needed by the framework.
		"--output-dir", filepath.Join(outputPath, getTraceName(downloadedTrace)),
	}
	// Split benchmark args if not empty and append to args.
	if *benchmarkExtraArgs != "" {
		args = append(args, strings.Fields(*benchmarkExtraArgs)...)
	}
	// Set the PYTHONPATH to the telemetry dirs.
	env := []string{
		fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHONPATH", util.TelemetryBinariesDir, util.TelemetrySrcDir, util.CatapultSrcDir),
		"DISPLAY=:0",
	}
	// Append the original environment as well.
	for _, e := range os.Environ() {
		env = append(env, e)
	}

	// Create buffer for capturing the stdout and stderr of the benchmark run.
	var b bytes.Buffer
	if _, err := b.WriteString(fmt.Sprintf("========== Stdout and stderr for %s ==========\n", downloadedTrace)); err != nil {
		return fmt.Errorf("Error writing to output buffer: %s", err)
	}
	if err := util.ExecuteCmdWithConfigurableLogging(ctx, "python", args, env, METRICS_BENCHMARK_TIMEOUT_SECS*time.Second, &b, &b, false, false); err != nil {
		output, getErr := util.GetRunBenchmarkOutput(b)
		skutil.LogErr(getErr)
		fmt.Println(output)
		return fmt.Errorf("Run benchmark command failed with: %s", err)
	}
	output, err := util.GetRunBenchmarkOutput(b)
	if err != nil {
		return fmt.Errorf("Could not get run benchmark output: %s", err)
	}
	// Print the output and return.
	fmt.Println(output)
	return nil
}

// downloadTrace downloads the traceURL from google storage into the specified
// destDir.
func downloadTrace(traceURL, destDir string, gs *util.GcsUtil) (string, error) {
	traceName := getTraceName(traceURL)
	traceDest := filepath.Join(destDir, traceName)
	if err := gs.DownloadRemoteFileFromBucket(TRACE_OUTPUT_BUCKET, traceName, traceDest); err != nil {
		return "", fmt.Errorf("Error downloading %s from %s to %s: %s", traceName, TRACE_OUTPUT_BUCKET, traceDest, err)
	}
	return traceDest, nil
}

// getTraceName parses the provided traceURI and returns the name of the trace.
// traceURI could be a file path or a URL.
func getTraceName(traceURI string) string {
	traceTokens := strings.Split(traceURI, "/")
	return traceTokens[len(traceTokens)-1]
}

// getClosedChannelOfTraces returns a channel that contains all trace URLs.
func getClosedChannelOfTraces(traces []string) chan string {
	tracesChannel := make(chan string, len(traces))
	for _, t := range traces {
		tracesChannel <- t
	}
	close(tracesChannel)
	return tracesChannel
}

func main() {
	retCode := 0
	if err := metricsAnalysis(); err != nil {
		sklog.Errorf("Error while running metrics analysis: %s", err)
		retCode = 255
	}
	os.Exit(retCode)
}
<|endoftext|>