<commit_before>package images

import (
	"fmt"
	"time"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	exutil "github.com/openshift/origin/test/extended/util"
)

type SampleRepoConfig struct {
	repoName               string
	templateURL            string
	buildConfigName        string
	serviceName            string
	deploymentConfigName   string
	expectedString         string
	appPath                string
	dbDeploymentConfigName string
}

// NewSampleRepoTest creates a function for a new ginkgo test case that will instantiate a template
// from a url, kick off the buildconfig defined in that template, wait for the build/deploy,
// and then confirm the application is serving an expected string value.
func NewSampleRepoTest(c SampleRepoConfig) func() {
	return func() {
		defer g.GinkgoRecover()
		var oc = exutil.NewCLI(c.repoName+"-repo-test", exutil.KubeConfigPath())

		g.JustBeforeEach(func() {
			g.By("Waiting for builder service account")
			err := exutil.WaitForBuilderAccount(oc.KubeREST().ServiceAccounts(oc.Namespace()))
			o.Expect(err).NotTo(o.HaveOccurred())
		})

		g.Describe("Building "+c.repoName+" app from new-app", func() {
			g.It(fmt.Sprintf("should build a "+c.repoName+" image and run it in a pod"), func() {
				oc.SetOutputDir(exutil.TestContext.OutputDir)

				exutil.CheckOpenShiftNamespaceImageStreams(oc)
				g.By(fmt.Sprintf("calling oc new-app with the " + c.repoName + " example template"))
				err := oc.Run("new-app").Args("-f", c.templateURL).Execute()
				o.Expect(err).NotTo(o.HaveOccurred())

				// all the templates automatically start a build.
				buildName := c.buildConfigName + "-1"

				g.By("expecting the build is in the Complete phase")
				err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), buildName, exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
				if err != nil {
					exutil.DumpBuildLogs(c.buildConfigName, oc)
				}
				o.Expect(err).NotTo(o.HaveOccurred())

				g.By("expecting the app deployment to be complete")
				err = exutil.WaitForADeploymentToComplete(oc.KubeREST().ReplicationControllers(oc.Namespace()), c.deploymentConfigName, oc)
				o.Expect(err).NotTo(o.HaveOccurred())

				if len(c.dbDeploymentConfigName) > 0 {
					g.By("expecting the db deployment to be complete")
					err = exutil.WaitForADeploymentToComplete(oc.KubeREST().ReplicationControllers(oc.Namespace()), c.dbDeploymentConfigName, oc)
					o.Expect(err).NotTo(o.HaveOccurred())
				}

				g.By("expecting the service is available")
				serviceIP, err := oc.Run("get").Args("service", c.serviceName).Template("{{ .spec.clusterIP }}").Output()
				o.Expect(err).NotTo(o.HaveOccurred())
				o.Expect(serviceIP).ShouldNot(o.Equal(""))

				g.By("expecting an endpoint is available")
				err = oc.KubeFramework().WaitForAnEndpoint(c.serviceName)
				o.Expect(err).NotTo(o.HaveOccurred())

				g.By("verifying string from app request")
				response, err := exutil.FetchURL("http://"+serviceIP+":8080"+c.appPath, time.Duration(30*time.Second))
				o.Expect(err).NotTo(o.HaveOccurred())
				o.Expect(response).Should(o.ContainSubstring(c.expectedString))
			})
		})
	}
}

var _ = g.Describe("[images][Slow] openshift sample application repositories", func() {

	g.Describe("[images][ruby] test ruby images with rails-ex db repo", NewSampleRepoTest(
		SampleRepoConfig{
			"rails-postgresql",
			"https://raw.githubusercontent.com/openshift/rails-ex/master/openshift/templates/rails-postgresql.json",
			"rails-postgresql-example",
			"rails-postgresql-example",
			"rails-postgresql-example",
			"Listing articles",
			"/articles",
			"postgresql",
		},
	))

	g.Describe("[images][python] test python images with django-ex db repo", NewSampleRepoTest(
		SampleRepoConfig{
			"django-psql",
			"https://raw.githubusercontent.com/openshift/django-ex/master/openshift/templates/django-postgresql.json",
			"django-psql-example",
			"django-psql-example",
			"django-psql-example",
			"Page views: 1",
			"",
			"postgresql",
		},
	))

	g.Describe("[images][nodejs] test nodejs images with nodejs-ex db repo", NewSampleRepoTest(
		SampleRepoConfig{
			"nodejs-mongodb",
			"https://raw.githubusercontent.com/openshift/nodejs-ex/master/openshift/templates/nodejs-mongodb.json",
			"nodejs-mongodb-example",
			"nodejs-mongodb-example",
			"nodejs-mongodb-example",
			"1",
			"",
			"mongodb",
		},
	))

	var _ = g.Describe("[images][php] test php images with cakephp-ex db repo", NewSampleRepoTest(
		SampleRepoConfig{
			"cakephp-mysql",
			"https://raw.githubusercontent.com/openshift/cakephp-ex/master/openshift/templates/cakephp-mysql.json",
			"cakephp-mysql-example",
			"cakephp-mysql-example",
			"cakephp-mysql-example",
			"1",
			"",
			"mysql",
		},
	))

	var _ = g.Describe("[images][perl] test perl images with dancer-ex db repo", NewSampleRepoTest(
		SampleRepoConfig{
			"dancer-mysql",
			"https://raw.githubusercontent.com/openshift/dancer-ex/master/openshift/templates/dancer-mysql.json",
			"dancer-mysql-example",
			"dancer-mysql-example",
			"dancer-mysql-example",
			"1",
			"",
			"database",
		},
	))

	// test the no-db templates too
	g.Describe("[images][python] test python images with django-ex repo", NewSampleRepoTest(
		SampleRepoConfig{
			"django",
			"https://raw.githubusercontent.com/openshift/django-ex/master/openshift/templates/django.json",
			"django-example",
			"django-example",
			"django-example",
			"Welcome",
			"",
			"",
		},
	))

	g.Describe("[images][nodejs] images with nodejs-ex repo", NewSampleRepoTest(
		SampleRepoConfig{
			"nodejs",
			"https://raw.githubusercontent.com/openshift/nodejs-ex/master/openshift/templates/nodejs.json",
			"nodejs-example",
			"nodejs-example",
			"nodejs-example",
			"Welcome",
			"",
			"",
		},
	))

	var _ = g.Describe("[images][php] test php images with cakephp-ex repo", NewSampleRepoTest(
		SampleRepoConfig{
			"cakephp",
			"https://raw.githubusercontent.com/openshift/cakephp-ex/master/openshift/templates/cakephp.json",
			"cakephp-example",
			"cakephp-example",
			"cakephp-example",
			"Welcome",
			"",
			"",
		},
	))

	var _ = g.Describe("[images][perl] test perl images with dancer-ex repo", NewSampleRepoTest(
		SampleRepoConfig{
			"dancer",
			"https://raw.githubusercontent.com/openshift/dancer-ex/master/openshift/templates/dancer.json",
			"dancer-example",
			"dancer-example",
			"dancer-example",
			"Welcome",
			"",
			"",
		},
	))

})
<commit_msg>make sure the db has an endpoint before testing the app<commit_after>package images

import (
	"fmt"
	"time"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	exutil "github.com/openshift/origin/test/extended/util"
)

type SampleRepoConfig struct {
	repoName               string
	templateURL            string
	buildConfigName        string
	serviceName            string
	deploymentConfigName   string
	expectedString         string
	appPath                string
	dbDeploymentConfigName string
	dbServiceName          string
}

// NewSampleRepoTest creates a function for a new ginkgo test case that will instantiate a template
// from a url, kick off the buildconfig defined in that template, wait for the build/deploy,
// and then confirm the application is serving an expected string value.
func NewSampleRepoTest(c SampleRepoConfig) func() {
	return func() {
		defer g.GinkgoRecover()
		var oc = exutil.NewCLI(c.repoName+"-repo-test", exutil.KubeConfigPath())

		g.JustBeforeEach(func() {
			g.By("Waiting for builder service account")
			err := exutil.WaitForBuilderAccount(oc.KubeREST().ServiceAccounts(oc.Namespace()))
			o.Expect(err).NotTo(o.HaveOccurred())
		})

		g.Describe("Building "+c.repoName+" app from new-app", func() {
			g.It(fmt.Sprintf("should build a "+c.repoName+" image and run it in a pod"), func() {
				oc.SetOutputDir(exutil.TestContext.OutputDir)

				exutil.CheckOpenShiftNamespaceImageStreams(oc)
				g.By(fmt.Sprintf("calling oc new-app with the " + c.repoName + " example template"))
				err := oc.Run("new-app").Args("-f", c.templateURL).Execute()
				o.Expect(err).NotTo(o.HaveOccurred())

				// all the templates automatically start a build.
				buildName := c.buildConfigName + "-1"

				g.By("expecting the build is in the Complete phase")
				err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), buildName, exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
				if err != nil {
					exutil.DumpBuildLogs(c.buildConfigName, oc)
				}
				o.Expect(err).NotTo(o.HaveOccurred())

				g.By("expecting the app deployment to be complete")
				err = exutil.WaitForADeploymentToComplete(oc.KubeREST().ReplicationControllers(oc.Namespace()), c.deploymentConfigName, oc)
				o.Expect(err).NotTo(o.HaveOccurred())

				if len(c.dbDeploymentConfigName) > 0 {
					g.By("expecting the db deployment to be complete")
					err = exutil.WaitForADeploymentToComplete(oc.KubeREST().ReplicationControllers(oc.Namespace()), c.dbDeploymentConfigName, oc)
					o.Expect(err).NotTo(o.HaveOccurred())

					g.By("expecting the db service is available")
					serviceIP, err := oc.Run("get").Args("service", c.dbServiceName).Template("{{ .spec.clusterIP }}").Output()
					o.Expect(err).NotTo(o.HaveOccurred())
					o.Expect(serviceIP).ShouldNot(o.Equal(""))

					g.By("expecting a db endpoint is available")
					err = oc.KubeFramework().WaitForAnEndpoint(c.dbServiceName)
					o.Expect(err).NotTo(o.HaveOccurred())
				}

				g.By("expecting the app service is available")
				serviceIP, err := oc.Run("get").Args("service", c.serviceName).Template("{{ .spec.clusterIP }}").Output()
				o.Expect(err).NotTo(o.HaveOccurred())
				o.Expect(serviceIP).ShouldNot(o.Equal(""))

				g.By("expecting an app endpoint is available")
				err = oc.KubeFramework().WaitForAnEndpoint(c.serviceName)
				o.Expect(err).NotTo(o.HaveOccurred())

				g.By("verifying string from app request")
				response, err := exutil.FetchURL("http://"+serviceIP+":8080"+c.appPath, time.Duration(30*time.Second))
				o.Expect(err).NotTo(o.HaveOccurred())
				o.Expect(response).Should(o.ContainSubstring(c.expectedString))
			})
		})
	}
}

var _ = g.Describe("[images][Slow] openshift sample application repositories", func() {

	g.Describe("[images][ruby] test ruby images with rails-ex db repo", NewSampleRepoTest(
		SampleRepoConfig{
			"rails-postgresql",
			"https://raw.githubusercontent.com/openshift/rails-ex/master/openshift/templates/rails-postgresql.json",
			"rails-postgresql-example",
			"rails-postgresql-example",
			"rails-postgresql-example",
			"Listing articles",
			"/articles",
			"postgresql",
			"postgresql",
		},
	))

	g.Describe("[images][python] test python images with django-ex db repo", NewSampleRepoTest(
		SampleRepoConfig{
			"django-psql",
			"https://raw.githubusercontent.com/openshift/django-ex/master/openshift/templates/django-postgresql.json",
			"django-psql-example",
			"django-psql-example",
			"django-psql-example",
			"Page views: 1",
			"",
			"postgresql",
			"postgresql",
		},
	))

	g.Describe("[images][nodejs] test nodejs images with nodejs-ex db repo", NewSampleRepoTest(
		SampleRepoConfig{
			"nodejs-mongodb",
			"https://raw.githubusercontent.com/openshift/nodejs-ex/master/openshift/templates/nodejs-mongodb.json",
			"nodejs-mongodb-example",
			"nodejs-mongodb-example",
			"nodejs-mongodb-example",
			"1",
			"",
			"mongodb",
			"mongodb",
		},
	))

	var _ = g.Describe("[images][php] test php images with cakephp-ex db repo", NewSampleRepoTest(
		SampleRepoConfig{
			"cakephp-mysql",
			"https://raw.githubusercontent.com/openshift/cakephp-ex/master/openshift/templates/cakephp-mysql.json",
			"cakephp-mysql-example",
			"cakephp-mysql-example",
			"cakephp-mysql-example",
			"1",
			"",
			"mysql",
			"mysql",
		},
	))

	var _ = g.Describe("[images][perl] test perl images with dancer-ex db repo", NewSampleRepoTest(
		SampleRepoConfig{
			"dancer-mysql",
			"https://raw.githubusercontent.com/openshift/dancer-ex/master/openshift/templates/dancer-mysql.json",
			"dancer-mysql-example",
			"dancer-mysql-example",
			"dancer-mysql-example",
			"1",
			"",
			"database",
			"database",
		},
	))

	// test the no-db templates too
	g.Describe("[images][python] test python images with django-ex repo", NewSampleRepoTest(
		SampleRepoConfig{
			"django",
			"https://raw.githubusercontent.com/openshift/django-ex/master/openshift/templates/django.json",
			"django-example",
			"django-example",
			"django-example",
			"Welcome",
			"",
			"",
			"",
		},
	))

	g.Describe("[images][nodejs] images with nodejs-ex repo", NewSampleRepoTest(
		SampleRepoConfig{
			"nodejs",
			"https://raw.githubusercontent.com/openshift/nodejs-ex/master/openshift/templates/nodejs.json",
			"nodejs-example",
			"nodejs-example",
			"nodejs-example",
			"Welcome",
			"",
			"",
			"",
		},
	))

	var _ = g.Describe("[images][php] test php images with cakephp-ex repo", NewSampleRepoTest(
		SampleRepoConfig{
			"cakephp",
			"https://raw.githubusercontent.com/openshift/cakephp-ex/master/openshift/templates/cakephp.json",
			"cakephp-example",
			"cakephp-example",
			"cakephp-example",
			"Welcome",
			"",
			"",
			"",
		},
	))

	var _ = g.Describe("[images][perl] test perl images with dancer-ex repo", NewSampleRepoTest(
		SampleRepoConfig{
			"dancer",
			"https://raw.githubusercontent.com/openshift/dancer-ex/master/openshift/templates/dancer.json",
			"dancer-example",
			"dancer-example",
			"dancer-example",
			"Welcome",
			"",
			"",
			"",
		},
	))

})
<|endoftext|>


<commit_before>// Package colorine provides a simple feature to print messages to console.
//
// A log line consists of a prefix and a message. The prefix is colored by its
// content (for example below, "create" is green, "exist" is blue, and "error"
// is red, and so on.) and the message is not colored.
//	$ your_program
//	    create path/to/file
//	     exist path/to/another/file
//	     error something went wrong!
//
// By default, no prefix color is defined. You must specify prefix-to-color
// mapping when you create a Logger. See the Logger's example.
package colorine

import (
	"fmt"
	"io"
	"os"
	"sync"

	"github.com/daviddengcn/go-colortext"
)

// A Logger handles the printing stuff. You can create one with NewLogger().
type Logger struct {
	// Prefix string to TextStyle mapping.
	Prefixes Prefixes
	output   io.Writer
	mu       sync.RWMutex
}

func (logger *Logger) getOutput() io.Writer {
	logger.mu.RLock()
	defer logger.mu.RUnlock()
	if logger.output == nil {
		return os.Stdout
	}
	return logger.output
}

// SetOutput sets the output destination for the logger.
func (logger *Logger) SetOutput(w io.Writer) {
	logger.mu.Lock()
	defer logger.mu.Unlock()
	logger.output = w
}

// A TextStyle represents the style that strings are displayed with. Consists
// of a foreground and a background, each of which is one of the text color constants.
type TextStyle struct {
	Foreground textColor
	Background textColor
}

type textColor struct {
	Color  ct.Color
	Bright bool
}

// A Prefixes is a map of prefix string to TextStyle. Prefix strings of log lines are colored according to this mapping.
// The TextStyle for an empty prefix ("") is used for default TextStyle.
type Prefixes map[string]TextStyle

// Text color constants. Use these to build a TextStyle.
var (
	None = textColor{ct.None, false}

	Black   = textColor{ct.Black, false}
	Red     = textColor{ct.Red, false}
	Green   = textColor{ct.Green, false}
	Yellow  = textColor{ct.Yellow, false}
	Blue    = textColor{ct.Blue, false}
	Magenta = textColor{ct.Magenta, false}
	Cyan    = textColor{ct.Cyan, false}
	White   = textColor{ct.White, false}

	BrightBlack   = textColor{ct.Black, true}
	BrightRed     = textColor{ct.Red, true}
	BrightGreen   = textColor{ct.Green, true}
	BrightYellow  = textColor{ct.Yellow, true}
	BrightBlue    = textColor{ct.Blue, true}
	BrightMagenta = textColor{ct.Magenta, true}
	BrightCyan    = textColor{ct.Cyan, true}
	BrightWhite   = textColor{ct.White, true}
)

// Predefined TextStyles. Of course you can ignore these.
var (
	Verbose = TextStyle{White, None}
	Info    = TextStyle{Green, None}
	Notice  = TextStyle{Blue, None}
	Warn    = TextStyle{Yellow, None}
	Error   = TextStyle{Red, None}
	Default = TextStyle{Green, None}
)

// NewLogger creates a new Logger. prefixes is a prefix-string-to-text-style
// mapping. The log prefix is colored using this table. If no entry is found in
// prefixes, defaultStyle is used.
//
// By default, no prefix is registered to any text style.
func NewLogger(prefixes Prefixes, defaultStyle TextStyle) *Logger {
	prefixes[""] = defaultStyle
	return &Logger{Prefixes: prefixes, output: os.Stdout}
}

// Log prints messages to console. prefix is colored using the logger.prefixes
// table.
func (logger *Logger) Log(prefix, message string) {
	textStyle, ok := logger.Prefixes[prefix]
	if !ok {
		textStyle, ok = logger.Prefixes[""]
		if !ok {
			textStyle = Default
		}
	}

	ct.ChangeColor(
		textStyle.Foreground.Color,
		textStyle.Foreground.Bright,
		textStyle.Background.Color,
		textStyle.Background.Bright,
	)

	out := logger.getOutput()

	logger.mu.Lock()
	defer logger.mu.Unlock()
	fmt.Fprintf(out, "%10s", prefix)

	ct.ResetColor()

	fmt.Fprintf(out, " %s\n", message)
}
<commit_msg>consider ct.Writer<commit_after>// Package colorine provides a simple feature to print messages to console.
//
// A log line consists of a prefix and a message. The prefix is colored by its
// content (for example below, "create" is green, "exist" is blue, and "error"
// is red, and so on.) and the message is not colored.
//	$ your_program
//	    create path/to/file
//	     exist path/to/another/file
//	     error something went wrong!
//
// By default, no prefix color is defined. You must specify prefix-to-color
// mapping when you create a Logger. See the Logger's example.
package colorine

import (
	"fmt"
	"io"
	"os"
	"sync"

	"github.com/daviddengcn/go-colortext"
)

// A Logger handles the printing stuff. You can create one with NewLogger().
type Logger struct {
	// Prefix string to TextStyle mapping.
	Prefixes Prefixes
	output   io.Writer
	mu       sync.RWMutex
}

func (logger *Logger) getOutput() io.Writer {
	logger.mu.RLock()
	defer logger.mu.RUnlock()
	if logger.output == nil {
		return os.Stdout
	}
	return logger.output
}

// SetOutput sets the output destination for the logger.
func (logger *Logger) SetOutput(w io.Writer) {
	logger.mu.Lock()
	defer logger.mu.Unlock()
	ct.Writer = w
	logger.output = w
}

// A TextStyle represents the style that strings are displayed with. Consists
// of a foreground and a background, each of which is one of the text color constants.
type TextStyle struct {
	Foreground textColor
	Background textColor
}

type textColor struct {
	Color  ct.Color
	Bright bool
}

// A Prefixes is a map of prefix string to TextStyle. Prefix strings of log lines are colored according to this mapping.
// The TextStyle for an empty prefix ("") is used for default TextStyle.
type Prefixes map[string]TextStyle

// Text color constants. Use these to build a TextStyle.
var (
	None = textColor{ct.None, false}

	Black   = textColor{ct.Black, false}
	Red     = textColor{ct.Red, false}
	Green   = textColor{ct.Green, false}
	Yellow  = textColor{ct.Yellow, false}
	Blue    = textColor{ct.Blue, false}
	Magenta = textColor{ct.Magenta, false}
	Cyan    = textColor{ct.Cyan, false}
	White   = textColor{ct.White, false}

	BrightBlack   = textColor{ct.Black, true}
	BrightRed     = textColor{ct.Red, true}
	BrightGreen   = textColor{ct.Green, true}
	BrightYellow  = textColor{ct.Yellow, true}
	BrightBlue    = textColor{ct.Blue, true}
	BrightMagenta = textColor{ct.Magenta, true}
	BrightCyan    = textColor{ct.Cyan, true}
	BrightWhite   = textColor{ct.White, true}
)

// Predefined TextStyles. Of course you can ignore these.
var (
	Verbose = TextStyle{White, None}
	Info    = TextStyle{Green, None}
	Notice  = TextStyle{Blue, None}
	Warn    = TextStyle{Yellow, None}
	Error   = TextStyle{Red, None}
	Default = TextStyle{Green, None}
)

// NewLogger creates a new Logger. prefixes is a prefix-string-to-text-style
// mapping. The log prefix is colored using this table. If no entry is found in
// prefixes, defaultStyle is used.
//
// By default, no prefix is registered to any text style.
func NewLogger(prefixes Prefixes, defaultStyle TextStyle) *Logger {
	prefixes[""] = defaultStyle
	return &Logger{Prefixes: prefixes, output: os.Stdout}
}

// Log prints messages to console. prefix is colored using the logger.prefixes
// table.
func (logger *Logger) Log(prefix, message string) {
	textStyle, ok := logger.Prefixes[prefix]
	if !ok {
		textStyle, ok = logger.Prefixes[""]
		if !ok {
			textStyle = Default
		}
	}

	out := logger.getOutput()

	logger.mu.Lock()
	defer logger.mu.Unlock()

	ct.ChangeColor(
		textStyle.Foreground.Color,
		textStyle.Foreground.Bright,
		textStyle.Background.Color,
		textStyle.Background.Bright,
	)

	fmt.Fprintf(out, "%10s", prefix)

	ct.ResetColor()

	fmt.Fprintf(out, " %s\n", message)
}
<|endoftext|>


<commit_before>// Copyright 2012-2015 Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.

package elastic

// FunctionScoreQuery allows you to modify the score of documents that
// are retrieved by a query. This can be useful if, for example,
// a score function is computationally expensive and it is sufficient
// to compute the score on a filtered set of documents.
//
// For more details, see
// https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-function-score-query.html
type FunctionScoreQuery struct {
	query      Query
	filter     Query
	boost      *float64
	maxBoost   *float64
	scoreMode  string
	boostMode  string
	filters    []Query
	scoreFuncs []ScoreFunction
	minScore   *float64
	weight     *float64
}

// NewFunctionScoreQuery creates and initializes a new function score query.
func NewFunctionScoreQuery() *FunctionScoreQuery {
	return &FunctionScoreQuery{
		filters:    make([]Query, 0),
		scoreFuncs: make([]ScoreFunction, 0),
	}
}

// Query sets the query for the function score query.
func (q *FunctionScoreQuery) Query(query Query) *FunctionScoreQuery {
	q.query = query
	q.filter = nil
	return q
}

// Filter sets the filter for the function score query.
func (q *FunctionScoreQuery) Filter(filter Query) *FunctionScoreQuery {
	q.query = nil
	q.filter = filter
	return q
}

// Add adds a score function that will execute on all the documents
// matching the filter.
func (q *FunctionScoreQuery) Add(filter Query, scoreFunc ScoreFunction) *FunctionScoreQuery {
	q.filters = append(q.filters, filter)
	q.scoreFuncs = append(q.scoreFuncs, scoreFunc)
	return q
}

// AddScoreFunc adds a score function that will execute the function on all documents.
func (q *FunctionScoreQuery) AddScoreFunc(scoreFunc ScoreFunction) *FunctionScoreQuery {
	q.filters = append(q.filters, nil)
	q.scoreFuncs = append(q.scoreFuncs, scoreFunc)
	return q
}

// ScoreMode defines how results of individual score functions will be aggregated.
// Can be first, avg, max, sum, min, or multiply.
func (q *FunctionScoreQuery) ScoreMode(scoreMode string) *FunctionScoreQuery {
	q.scoreMode = scoreMode
	return q
}

// BoostMode defines how the combined result of score functions will
// influence the final score together with the sub query score.
func (q *FunctionScoreQuery) BoostMode(boostMode string) *FunctionScoreQuery {
	q.boostMode = boostMode
	return q
}

// MaxBoost is the maximum boost that will be applied by function score.
func (q *FunctionScoreQuery) MaxBoost(maxBoost float64) *FunctionScoreQuery {
	q.maxBoost = &maxBoost
	return q
}

// Boost sets the boost for this query. Documents matching this query will
// (in addition to the normal weightings) have their score multiplied by the
// boost provided.
func (q *FunctionScoreQuery) Boost(boost float64) *FunctionScoreQuery {
	q.boost = &boost
	return q
}

// MinScore sets the minimum score.
func (q *FunctionScoreQuery) MinScore(minScore float64) *FunctionScoreQuery {
	q.minScore = &minScore
	return q
}

// Source returns JSON for the function score query.
func (q *FunctionScoreQuery) Source() (interface{}, error) {
	source := make(map[string]interface{})
	query := make(map[string]interface{})
	source["function_score"] = query

	if q.query != nil {
		src, err := q.query.Source()
		if err != nil {
			return nil, err
		}
		query["query"] = src
	} else if q.filter != nil {
		src, err := q.filter.Source()
		if err != nil {
			return nil, err
		}
		query["filter"] = src
	}

	if len(q.filters) == 1 && q.filters[0] == nil {
		// Weight needs to be serialized on this level.
		if weight := q.scoreFuncs[0].GetWeight(); weight != nil {
			query["weight"] = weight
		}
		// Serialize the score function
		src, err := q.scoreFuncs[0].Source()
		if err != nil {
			return nil, err
		}
		query[q.scoreFuncs[0].Name()] = src
	} else {
		funcs := make([]interface{}, len(q.filters))
		for i, filter := range q.filters {
			hsh := make(map[string]interface{})
			if filter != nil {
				src, err := filter.Source()
				if err != nil {
					return nil, err
				}
				hsh["filter"] = src
			}
			// Weight needs to be serialized on this level.
			if weight := q.scoreFuncs[i].GetWeight(); weight != nil {
				hsh["weight"] = weight
			}
			// Serialize the score function
			src, err := q.scoreFuncs[i].Source()
			if err != nil {
				return nil, err
			}
			hsh[q.scoreFuncs[i].Name()] = src
			funcs[i] = hsh
		}
		query["functions"] = funcs
	}

	if q.scoreMode != "" {
		query["score_mode"] = q.scoreMode
	}
	if q.boostMode != "" {
		query["boost_mode"] = q.boostMode
	}
	if q.maxBoost != nil {
		query["max_boost"] = *q.maxBoost
	}
	if q.boost != nil {
		query["boost"] = *q.boost
	}
	if q.minScore != nil {
		query["min_score"] = *q.minScore
	}

	return source, nil
}
<commit_msg>Remove `Filter` from `FunctionScoreQuery`<commit_after>// Copyright 2012-2015 Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.

package elastic

// FunctionScoreQuery allows you to modify the score of documents that
// are retrieved by a query. This can be useful if, for example,
// a score function is computationally expensive and it is sufficient
// to compute the score on a filtered set of documents.
//
// For more details, see
// https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-function-score-query.html
type FunctionScoreQuery struct {
	query      Query
	boost      *float64
	maxBoost   *float64
	scoreMode  string
	boostMode  string
	filters    []Query
	scoreFuncs []ScoreFunction
	minScore   *float64
	weight     *float64
}

// NewFunctionScoreQuery creates and initializes a new function score query.
func NewFunctionScoreQuery() *FunctionScoreQuery {
	return &FunctionScoreQuery{
		filters:    make([]Query, 0),
		scoreFuncs: make([]ScoreFunction, 0),
	}
}

// Query sets the query for the function score query.
func (q *FunctionScoreQuery) Query(query Query) *FunctionScoreQuery {
	q.query = query
	return q
}

// Add adds a score function that will execute on all the documents
// matching the filter.
func (q *FunctionScoreQuery) Add(filter Query, scoreFunc ScoreFunction) *FunctionScoreQuery {
	q.filters = append(q.filters, filter)
	q.scoreFuncs = append(q.scoreFuncs, scoreFunc)
	return q
}

// AddScoreFunc adds a score function that will execute the function on all documents.
func (q *FunctionScoreQuery) AddScoreFunc(scoreFunc ScoreFunction) *FunctionScoreQuery {
	q.filters = append(q.filters, nil)
	q.scoreFuncs = append(q.scoreFuncs, scoreFunc)
	return q
}

// ScoreMode defines how results of individual score functions will be aggregated.
// Can be first, avg, max, sum, min, or multiply.
func (q *FunctionScoreQuery) ScoreMode(scoreMode string) *FunctionScoreQuery {
	q.scoreMode = scoreMode
	return q
}

// BoostMode defines how the combined result of score functions will
// influence the final score together with the sub query score.
func (q *FunctionScoreQuery) BoostMode(boostMode string) *FunctionScoreQuery {
	q.boostMode = boostMode
	return q
}

// MaxBoost is the maximum boost that will be applied by function score.
func (q *FunctionScoreQuery) MaxBoost(maxBoost float64) *FunctionScoreQuery {
	q.maxBoost = &maxBoost
	return q
}

// Boost sets the boost for this query. Documents matching this query will
// (in addition to the normal weightings) have their score multiplied by the
// boost provided.
func (q *FunctionScoreQuery) Boost(boost float64) *FunctionScoreQuery {
	q.boost = &boost
	return q
}

// MinScore sets the minimum score.
func (q *FunctionScoreQuery) MinScore(minScore float64) *FunctionScoreQuery {
	q.minScore = &minScore
	return q
}

// Source returns JSON for the function score query.
func (q *FunctionScoreQuery) Source() (interface{}, error) {
	source := make(map[string]interface{})
	query := make(map[string]interface{})
	source["function_score"] = query

	if q.query != nil {
		src, err := q.query.Source()
		if err != nil {
			return nil, err
		}
		query["query"] = src
	}

	if len(q.filters) == 1 && q.filters[0] == nil {
		// Weight needs to be serialized on this level.
		if weight := q.scoreFuncs[0].GetWeight(); weight != nil {
			query["weight"] = weight
		}
		// Serialize the score function
		src, err := q.scoreFuncs[0].Source()
		if err != nil {
			return nil, err
		}
		query[q.scoreFuncs[0].Name()] = src
	} else {
		funcs := make([]interface{}, len(q.filters))
		for i, filter := range q.filters {
			hsh := make(map[string]interface{})
			if filter != nil {
				src, err := filter.Source()
				if err != nil {
					return nil, err
				}
				hsh["filter"] = src
			}
			// Weight needs to be serialized on this level.
			if weight := q.scoreFuncs[i].GetWeight(); weight != nil {
				hsh["weight"] = weight
			}
			// Serialize the score function
			src, err := q.scoreFuncs[i].Source()
			if err != nil {
				return nil, err
			}
			hsh[q.scoreFuncs[i].Name()] = src
			funcs[i] = hsh
		}
		query["functions"] = funcs
	}

	if q.scoreMode != "" {
		query["score_mode"] = q.scoreMode
	}
	if q.boostMode != "" {
		query["boost_mode"] = q.boostMode
	}
	if q.maxBoost != nil {
		query["max_boost"] = *q.maxBoost
	}
	if q.boost != nil {
		query["boost"] = *q.boost
	}
	if q.minScore != nil {
		query["min_score"] = *q.minScore
	}

	return source, nil
}
<|endoftext|>


<commit_before>package main

import (
  "log"
  "fmt"
  "os"
  "time"
  "github.com/codegangsta/cli"
  "github.com/awslabs/aws-sdk-go/aws"
  "github.com/awslabs/aws-sdk-go/aws/awsutil"
  "github.com/awslabs/aws-sdk-go/service/ec2"
  "github.com/awslabs/aws-sdk-go/service/iam"
)

var Commands = []cli.Command{
  commandList_ipaddress,
  commandEnter_instance,
  commandList_users,
}

var commandList_ipaddress = cli.Command{
  Name:  "list-ipaddress",
  Aliases:  []string{"li"},
  Usage: "get ipaddress list by NameTag",
  Description: `
`,
  Flags: []cli.Flag {
    cli.StringFlag{
      Name: "nametag, n",
      Usage: "NameTag",
    },
  },
  Action: doList_ipaddress,
}

var commandEnter_instance = cli.Command{
  Name:  "enter-instance",
  Aliases:  []string{"ei"},
  Usage: "",
  Description: `
  aaaa hogehoge mogemoge
`,
  Action: doEnter_instance,
}

var commandList_users = cli.Command{
  Name:  "list-users",
  Aliases:  []string{"lu"},
  Usage: "",
  Description: `
  List IAM Users
`,
  Action: doList_users,
}

func debug(v ...interface{}) {
  if os.Getenv("DEBUG") != "" {
    log.Println(v...)
  }
}

func assert(err error) {
  if err != nil {
    log.Fatal(err)
  }
}

func doList_ipaddress(c *cli.Context) {
  profile := c.GlobalString("profile")
  if profile == "" {
    fmt.Println("'--profile' is required")
    os.Exit(1)
  }
  name := c.String("nametag")
  if name == "" {
    fmt.Println("'--nametag' is required")
    os.Exit(1)
  }
  prov, _ := aws.ProfileCreds("", profile, 5 * time.Minute)
  svc := ec2.New(&aws.Config{Credentials: prov, Region: "ap-northeast-1"})
  params := ec2.DescribeInstancesInput{
    Filters: []ec2.Filter{
      Name: "tag:Name",
      Values: "*" + name + "*",
    },
  }
  res, err := svc.DescribeInstances(params)

  if awserr := aws.Error(err); awserr != nil {
    // A service error occurred.
    fmt.Println("Error:", awserr.Code, awserr.Message)
  } else if err != nil {
    // A non-service error occurred.
    panic(err)
  }

  for _, r := range res.Reservations {
    for _, i := range r.Instances {
      var nt string
      for _, t := range i.Tags {
        if *t.Key == "Name" {
          nt = *t.Value
          break
        }
      }
      fmt.Println(nt, *i.PrivateIPAddress)
    }
  }
  //fmt.Println(awsutil.StringValue(res))
}

func doList_users(c *cli.Context) {
  profile := c.GlobalString("profile")
  if profile == "" {
    fmt.Println("'--profile' is required")
    os.Exit(1)
  }
  prov, _ := aws.ProfileCreds("", profile, 5 * time.Minute)
  svc := iam.New(&aws.Config{Credentials: prov})
  params := &iam.ListUsersInput{
    //Marker:     aws.String("markerType"),
    //MaxItems:   aws.Long(1),
    //PathPrefix: aws.String("/"),
  }
  resp, err := svc.ListUsers(params)
  if awserr := aws.Error(err); awserr != nil {
    // A service error occurred.
    fmt.Println("Error:", awserr.Code, awserr.Message)
  } else if err != nil {
    // A non-service error occurred.
    panic(err)
  }
  fmt.Println(awsutil.StringValue(resp))
}

func doEnter_instance(c *cli.Context) {
}

<commit_msg>ver 0.0.1beta list-ipaddress command<commit_after>package main

import (
  "log"
  "fmt"
  "os"
  "time"
  "github.com/codegangsta/cli"
  "github.com/awslabs/aws-sdk-go/aws"
  "github.com/awslabs/aws-sdk-go/aws/awsutil"
  "github.com/awslabs/aws-sdk-go/service/ec2"
  "github.com/awslabs/aws-sdk-go/service/iam"
)

var Commands = []cli.Command{
  commandList_ipaddress,
  commandEnter_instance,
  commandList_users,
}

var commandList_ipaddress = cli.Command{
  Name:  "list-ipaddress",
  Aliases:  []string{"li"},
  Usage: "get ipaddress list by NameTag",
  Description: `
`,
  Flags: []cli.Flag {
    &cli.StringFlag{ Name: "nametag, n", Usage: "NameTag", },
    &cli.BoolFlag{ Name: "all, a", Usage: "Get all status instances", },
  },
  Action: doList_ipaddress,
}

var commandEnter_instance = cli.Command{
  Name:  "enter-instance",
  Aliases:  []string{"ei"},
  Usage: "",
  Description: `
  aaaa hogehoge mogemoge
`,
  Action: doEnter_instance,
}

var commandList_users = cli.Command{
  Name:  "list-users",
  Aliases:  []string{"lu"},
  Usage: "",
  Description: `
  List IAM Users
`,
  Action: doList_users,
}

func debug(v ...interface{}) {
  if os.Getenv("DEBUG") != "" {
    log.Println(v...)
  }
}

func assert(err error) {
  if err != nil {
    log.Fatal(err)
  }
}

func doList_ipaddress(c *cli.Context) {
  profile := c.GlobalString("profile")
  if profile == "" {
    fmt.Println("'--profile' is required")
    os.Exit(1)
  }
  name := c.String("nametag")
  if name == "" {
    fmt.Println("'--nametag' is required")
    os.Exit(1)
  }
  prov, _ := aws.ProfileCreds("", profile, 5 * time.Minute)
  svc := ec2.New(&aws.Config{Credentials: prov, Region: "ap-northeast-1"})
  params := &ec2.DescribeInstancesInput{
    Filters: []*ec2.Filter{
      &ec2.Filter{
        Name: aws.String("tag:Name"),
        Values: []*string{
          aws.String(name),
        },
      },
    },
  }
  if c.Bool("all") == false {
    sf := &ec2.Filter{
      Name: aws.String("instance-state-name"),
      Values: []*string{
        aws.String("running"),
      },
    }
    params.Filters = append(params.Filters, sf)
  }
  res, err := svc.DescribeInstances(params)

  if awserr := aws.Error(err); awserr != nil {
    // A service error occurred.
    fmt.Println("Error:", awserr.Code, awserr.Message)
  } else if err != nil {
    // A non-service error occurred.
    panic(err)
  }

  for _, r := range res.Reservations {
    for _, i := range r.Instances {
      var nt string
      for _, t := range i.Tags {
        if *t.Key == "Name" {
          nt = *t.Value
          break
        }
      }
      fmt.Println(nt, *i.PrivateIPAddress, *i.State.Name)
    }
  }
  //fmt.Println(awsutil.StringValue(res))
}

func doList_users(c *cli.Context) {
  profile := c.GlobalString("profile")
  if profile == "" {
    fmt.Println("'--profile' is required")
    os.Exit(1)
  }
  prov, _ := aws.ProfileCreds("", profile, 5 * time.Minute)
  svc := iam.New(&aws.Config{Credentials: prov})
  params := &iam.ListUsersInput{
    //Marker:     aws.String("markerType"),
    //MaxItems:   aws.Long(1),
    //PathPrefix: aws.String("/"),
  }
  resp, err := svc.ListUsers(params)
  if awserr := aws.Error(err); awserr != nil {
    // A service error occurred.
    fmt.Println("Error:", awserr.Code, awserr.Message)
  } else if err != nil {
    // A non-service error occurred.
    panic(err)
  }
  fmt.Println(awsutil.StringValue(resp))
}

func doEnter_instance(c *cli.Context) {
}

<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/url"
	"os"
	"strings"
	"syscall"

	"github.com/codegangsta/cli"
	"github.com/google/go-github/github"
	"github.com/motemen/ghq/pocket"
	"github.com/motemen/ghq/utils"
)

var Commands = []cli.Command{
	commandGet,
	commandList,
	commandLook,
	commandImport,
}

var commandGet = cli.Command{
	Name:  "get",
	Usage: "Clone/sync with a remote repository",
	Description: `
    Clone a GitHub repository under ghq root direcotry. If the repository is
    already cloned to local, nothing will happen unless '-u' ('--update')
    flag is supplied, in which case 'git remote update' is executed.
`,
	Action: doGet,
	Flags: []cli.Flag{
		cli.BoolFlag{"update, u", "Update local repository if cloned already"},
	},
}

var commandList = cli.Command{
	Name:  "list",
	Usage: "List local repositories",
	Description: `
    List locally cloned repositories. If a query argument is given, only
    repositories whose names contain that query text are listed. '-e'
    ('--exact') forces the match to be an exact one (i.e. the query equals to
    _project_ or _user_/_project_) If '-p' ('--full-path') is given, the full paths
    to the repository root are printed instead of relative ones.
`,
	Action: doList,
	Flags: []cli.Flag{
		cli.BoolFlag{"exact, e", "Perform an exact match"},
		cli.BoolFlag{"full-path, p", "Print full paths"},
		cli.BoolFlag{"unique", "Print unique subpaths"},
	},
}

var commandLook = cli.Command{
	Name:  "look",
	Usage: "Look into a local repository",
	Description: `
    Look into a locally cloned repository with the shell.
`,
	Action: doLook,
}

var commandImport = cli.Command{
	Name:  "import",
	Usage: "Import repositories from other web services",
	Subcommands: []cli.Command{
		commandImportStarred,
		commandImportPocket,
	},
}

var commandImportStarred = cli.Command{
	Name:  "starred",
	Usage: "Get all starred GitHub repositories",
	Description: `
    Retrieves GitHub repositories that are starred by the user specified and
    performs 'get' for each of them.
`,
	Action: doImportStarred,
	Flags: []cli.Flag{
		cli.BoolFlag{"update, u", "Update local repository if cloned already"},
	},
}

var commandImportPocket = cli.Command{
	Name:  "pocket",
	Usage: "Get all github.com entries in Pocket",
	Description: `
    Retrieves Pocket <http://getpocket.com/> entries of github.com and
    performs 'get' for each of them.
`,
	Action: doImportPocket,
	Flags: []cli.Flag{
		cli.BoolFlag{"update, u", "Update local repository if cloned already"},
	},
}

type commandDoc struct {
	Parent    string
	Arguments string
}

var commandDocs = map[string]commandDoc{
	"get":     {"", "[-u] <repository URL> | <user>/<project>"},
	"list":    {"", "[-p] [-e] [<query>]"},
	"look":    {"", "<project> | <user>/<project> | <host>/<user>/<project>"},
	"import":  {"", "[-u] starred <user> | pocket"},
	"starred": {"import", "[-u] <user>"},
	"pocket":  {"import", "[-u]"},
}

// Makes template conditionals to generate per-command documents.
func mkCommandsTemplate(genTemplate func(commandDoc) string) string {
	template := "{{if false}}"
	for _, command := range append(Commands, commandImportStarred, commandImportPocket) {
		template = template + fmt.Sprintf("{{else if (eq .Name %q)}}%s", command.Name, genTemplate(commandDocs[command.Name]))
	}
	return template + "{{end}}"
}

func init() {
	argsTemplate := mkCommandsTemplate(func(doc commandDoc) string { return doc.Arguments })
	parentTemplate := mkCommandsTemplate(func(doc commandDoc) string { return string(strings.TrimLeft(doc.Parent+" ", " ")) })

	cli.CommandHelpTemplate = `NAME:
    {{.Name}} - {{.Usage}}

USAGE:
    ghq ` + parentTemplate + `{{.Name}} ` + argsTemplate + `
{{if (len .Description)}}
DESCRIPTION: {{.Description}}
{{end}}{{if (len .Flags)}}
OPTIONS:
    {{range .Flags}}{{.}}
    {{end}}
{{end}}`
}

func doGet(c *cli.Context) {
	argURL := c.Args().Get(0)
	doUpdate := c.Bool("update")

	if argURL == "" {
		cli.ShowCommandHelp(c, "get")
		os.Exit(1)
	}

	url, err := url.Parse(argURL)
	utils.DieIf(err)

	if !url.IsAbs() {
		url.Scheme = "https"
		url.Host = "github.com"
		if url.Path[0] != '/' {
			url.Path = "/" + url.Path
		}
	}

	remote, err := NewRemoteRepository(url)
	utils.DieIf(err)

	if remote.IsValid() == false {
		utils.Log("error", fmt.Sprintf("Not a valid repository: %s", url))
		os.Exit(1)
	}

	getRemoteRepository(remote, doUpdate)
}

func getRemoteRepository(remote RemoteRepository, doUpdate bool) {
	remoteURL := remote.URL()
	local := LocalRepositoryFromURL(remoteURL)

	path := local.FullPath
	newPath := false

	_, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			newPath = true
			err = nil
		}
		utils.PanicIf(err)
	}

	if newPath {
		utils.Log("clone", fmt.Sprintf("%s -> %s", remoteURL, path))
		remote.VCS().Clone(remoteURL, path)
	} else {
		if doUpdate {
			utils.Log("update", path)
			local.VCS().Update(path)
		} else {
			utils.Log("exists", path)
		}
	}
}

func doList(c *cli.Context) {
	query := c.Args().First()
	exact := c.Bool("exact")
	printFullPaths := c.Bool("full-path")
	printUniquePaths := c.Bool("unique")

	var filterFn func(*LocalRepository) bool
	if query == "" {
		filterFn = func(_ *LocalRepository) bool {
			return true
		}
	} else if exact {
		filterFn = func(repo *LocalRepository) bool {
			return repo.Matches(query)
		}
	} else {
		filterFn = func(repo *LocalRepository) bool {
			return strings.Contains(repo.NonHostPath(), query)
		}
	}

	repos := []*LocalRepository{}

	walkLocalRepositories(func(repo *LocalRepository) {
		if filterFn(repo) == false {
			return
		}

		repos = append(repos, repo)
	})

	if printUniquePaths {
		subpathCount := map[string]int{} // Count duplicated subpaths (ex. foo/dotfiles and bar/dotfiles)
		reposCount := map[string]int{}   // Check duplicated repositories among roots

		// Primary first
		for _, repo := range repos {
			if reposCount[repo.RelPath] == 0 {
				for _, p := range repo.Subpaths() {
					subpathCount[p] = subpathCount[p] + 1
				}
			}

			reposCount[repo.RelPath] = reposCount[repo.RelPath] + 1
		}

		for _, repo := range repos {
			if reposCount[repo.RelPath] > 1 && repo.IsUnderPrimaryRoot() == false {
				continue
			}

			for _, p := range repo.Subpaths() {
				if subpathCount[p] == 1 {
					fmt.Println(p)
					break
				}
			}
		}
	} else {
		for _, repo := range repos {
			if printFullPaths {
				fmt.Println(repo.FullPath)
			} else {
				fmt.Println(repo.RelPath)
			}
		}
	}
}

func doLook(c *cli.Context) {
	name := c.Args().First()

	if name == "" {
		cli.ShowCommandHelp(c, "look")
		os.Exit(1)
	}

	reposFound := []*LocalRepository{}
	walkLocalRepositories(func(repo *LocalRepository) {
		if repo.Matches(name) {
			reposFound = append(reposFound, repo)
		}
	})

	switch len(reposFound) {
	case 0:
		utils.Log("error", "No repository found")

	case 1:
		shell := os.Getenv("SHELL")
		if shell == "" {
			shell = "/bin/sh"
		}

		utils.Log("cd", reposFound[0].FullPath)
		err := os.Chdir(reposFound[0].FullPath)
		utils.PanicIf(err)

		syscall.Exec(shell, []string{shell}, syscall.Environ())

	default:
		utils.Log("error", "More than one repositories are found; Try more precise name")
		for _, repo := range reposFound {
			utils.Log("error", "- "+strings.Join(repo.PathParts, "/"))
		}
	}
}

func doImportStarred(c *cli.Context) {
	user := c.Args().First()

	if user == "" {
		cli.ShowCommandHelp(c, "starred")
		os.Exit(1)
	}

	client := github.NewClient(nil)
	options := &github.ActivityListStarredOptions{Sort: "created"}

	for page := 1; ; page++ {
		options.Page = page

		repositories, res, err := client.Activity.ListStarred(user, options)
		utils.DieIf(err)

		utils.Log("page", fmt.Sprintf("%d/%d", page, res.LastPage))
		for _, repo := range repositories {
			url, err := url.Parse(*repo.HTMLURL)
			if err != nil {
				utils.Log("error", fmt.Sprintf("Could not parse URL <%s>: %s", repo.HTMLURL, err))
				continue
			}

			remote, err := NewRemoteRepository(url)
			if utils.ErrorIf(err) {
				continue
			}

			if remote.IsValid() == false {
				utils.Log("skip", fmt.Sprintf("Not a valid repository: %s", url))
				continue
			}

			getRemoteRepository(remote, c.Bool("update"))
		}

		if page >= res.LastPage {
			break
		}
	}
}

func doImportPocket(c *cli.Context) {
	if pocket.ConsumerKey == "" {
		utils.Log("error", "Built without consumer key set")
		return
	}

	accessToken, err := GitConfig("ghq.pocket.token")
	utils.PanicIf(err)

	if accessToken == "" {
		receiverURL, ch, err := pocket.StartAccessTokenReceiver()
		utils.PanicIf(err)

		utils.Log("pocket", "Waiting for Pocket authentication callback at "+receiverURL)

		utils.Log("pocket", "Obtaining request token")
		authRequest, err := pocket.ObtainRequestToken(receiverURL)
		utils.DieIf(err)

		url := pocket.GenerateAuthorizationURL(authRequest.Code, receiverURL)
		utils.Log("open", url)

		<-ch

		utils.Log("pocket", "Obtaining access token")
		authorized, err := pocket.ObtainAccessToken(authRequest.Code)
		utils.DieIf(err)

		utils.Log("authorized", authorized.Username)

		accessToken = authorized.AccessToken
		utils.Run("git", "config", "ghq.pocket.token", authorized.AccessToken)
	}

	utils.Log("pocket", "Retrieving github.com entries")
	res, err := pocket.RetrieveGitHubEntries(accessToken)
	utils.DieIf(err)

	for _, item := range res.List {
		url, err := url.Parse(item.ResolvedURL)
		if err != nil {
			utils.Log("error", fmt.Sprintf("Could not parse URL <%s>: %s", item.ResolvedURL, err))
			continue
		}

		remote, err := NewRemoteRepository(url)
		if utils.ErrorIf(err) {
			continue
		}

		if remote.IsValid() == false {
			utils.Log("skip", fmt.Sprintf("Not a valid repository: %s", url))
			continue
		}

		getRemoteRepository(remote, c.Bool("update"))
	}
}
<commit_msg>Use `NewURL()` instead of raw `url.Parse()`.<commit_after>package main

import (
	"fmt"
	"net/url"
	"os"
	"strings"
	"syscall"

	"github.com/codegangsta/cli"
	"github.com/google/go-github/github"
	"github.com/motemen/ghq/pocket"
	"github.com/motemen/ghq/utils"
)

var Commands = []cli.Command{
	commandGet,
	commandList,
	commandLook,
	commandImport,
}

var commandGet = cli.Command{
	Name:  "get",
	Usage: "Clone/sync with a remote repository",
	Description: `
    Clone a GitHub repository under ghq root direcotry. If the repository is
    already cloned to local, nothing will happen unless '-u' ('--update')
    flag is supplied, in which case 'git remote update' is executed.
`,
	Action: doGet,
	Flags: []cli.Flag{
		cli.BoolFlag{"update, u", "Update local repository if cloned already"},
	},
}

var commandList = cli.Command{
	Name:  "list",
	Usage: "List local repositories",
	Description: `
    List locally cloned repositories. If a query argument is given, only
    repositories whose names contain that query text are listed. '-e'
    ('--exact') forces the match to be an exact one (i.e. the query equals to
    _project_ or _user_/_project_) If '-p' ('--full-path') is given, the full paths
    to the repository root are printed instead of relative ones.
`,
	Action: doList,
	Flags: []cli.Flag{
		cli.BoolFlag{"exact, e", "Perform an exact match"},
		cli.BoolFlag{"full-path, p", "Print full paths"},
		cli.BoolFlag{"unique", "Print unique subpaths"},
	},
}

var commandLook = cli.Command{
	Name:  "look",
	Usage: "Look into a local repository",
	Description: `
    Look into a locally cloned repository with the shell.
`,
	Action: doLook,
}

var commandImport = cli.Command{
	Name:  "import",
	Usage: "Import repositories from other web services",
	Subcommands: []cli.Command{
		commandImportStarred,
		commandImportPocket,
	},
}

var commandImportStarred = cli.Command{
	Name:  "starred",
	Usage: "Get all starred GitHub repositories",
	Description: `
    Retrieves GitHub repositories that are starred by the user specified and
    performs 'get' for each of them.
`,
	Action: doImportStarred,
	Flags: []cli.Flag{
		cli.BoolFlag{"update, u", "Update local repository if cloned already"},
	},
}

var commandImportPocket = cli.Command{
	Name:  "pocket",
	Usage: "Get all github.com entries in Pocket",
	Description: `
    Retrieves Pocket <http://getpocket.com/> entries of github.com and
    performs 'get' for each of them.
`,
	Action: doImportPocket,
	Flags: []cli.Flag{
		cli.BoolFlag{"update, u", "Update local repository if cloned already"},
	},
}

type commandDoc struct {
	Parent    string
	Arguments string
}

var commandDocs = map[string]commandDoc{
	"get":     {"", "[-u] <repository URL> | <user>/<project>"},
	"list":    {"", "[-p] [-e] [<query>]"},
	"look":    {"", "<project> | <user>/<project> | <host>/<user>/<project>"},
	"import":  {"", "[-u] starred <user> | pocket"},
	"starred": {"import", "[-u] <user>"},
	"pocket":  {"import", "[-u]"},
}

// Makes template conditionals to generate per-command documents.
func mkCommandsTemplate(genTemplate func(commandDoc) string) string {
	template := "{{if false}}"
	for _, command := range append(Commands, commandImportStarred, commandImportPocket) {
		template = template + fmt.Sprintf("{{else if (eq .Name %q)}}%s", command.Name, genTemplate(commandDocs[command.Name]))
	}
	return template + "{{end}}"
}

func init() {
	argsTemplate := mkCommandsTemplate(func(doc commandDoc) string { return doc.Arguments })
	parentTemplate := mkCommandsTemplate(func(doc commandDoc) string { return string(strings.TrimLeft(doc.Parent+" ", " ")) })

	cli.CommandHelpTemplate = `NAME:
    {{.Name}} - {{.Usage}}

USAGE:
    ghq ` + parentTemplate + `{{.Name}} ` + argsTemplate + `
{{if (len .Description)}}
DESCRIPTION: {{.Description}}
{{end}}{{if (len .Flags)}}
OPTIONS:
    {{range .Flags}}{{.}}
    {{end}}
{{end}}`
}

func doGet(c *cli.Context) {
	argURL := c.Args().Get(0)
	doUpdate := c.Bool("update")

	if argURL == "" {
		cli.ShowCommandHelp(c, "get")
		os.Exit(1)
	}

	url, err := NewURL(argURL)
	utils.DieIf(err)

	if !url.IsAbs() {
		url.Scheme = "https"
		url.Host = "github.com"
		if url.Path[0] != '/' {
			url.Path = "/" + url.Path
		}
	}

	remote, err := NewRemoteRepository(url)
	utils.DieIf(err)

	if remote.IsValid() == false {
		utils.Log("error", fmt.Sprintf("Not a valid repository: %s", url))
		os.Exit(1)
	}

	getRemoteRepository(remote, doUpdate)
}

func getRemoteRepository(remote RemoteRepository, doUpdate bool) {
	remoteURL := remote.URL()
	local := LocalRepositoryFromURL(remoteURL)

	path := local.FullPath
	newPath := false

	_, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			newPath = true
			err = nil
		}
		utils.PanicIf(err)
	}

	if newPath {
		utils.Log("clone", fmt.Sprintf("%s -> %s", remoteURL, path))
		remote.VCS().Clone(remoteURL, path)
	} else {
		if doUpdate {
			utils.Log("update", path)
			local.VCS().Update(path)
		} else {
			utils.Log("exists", path)
		}
	}
}

func doList(c *cli.Context) {
	query := c.Args().First()
	exact := c.Bool("exact")
	printFullPaths := c.Bool("full-path")
	printUniquePaths := c.Bool("unique")

	var filterFn func(*LocalRepository) bool
	if query == "" {
		filterFn = func(_ *LocalRepository) bool {
			return true
		}
	} else if exact {
		filterFn = func(repo *LocalRepository) bool {
			return repo.Matches(query)
		}
	} else {
		filterFn = func(repo *LocalRepository) bool {
			return strings.Contains(repo.NonHostPath(), query)
		}
	}

	repos := []*LocalRepository{}

	walkLocalRepositories(func(repo *LocalRepository) {
		if filterFn(repo) == false {
			return
		}

		repos = append(repos, repo)
	})

	if printUniquePaths {
		subpathCount := map[string]int{} // Count duplicated subpaths (ex. foo/dotfiles and bar/dotfiles)
		reposCount := map[string]int{}   // Check duplicated repositories among roots

		// Primary first
		for _, repo := range repos {
			if reposCount[repo.RelPath] == 0 {
				for _, p := range repo.Subpaths() {
					subpathCount[p] = subpathCount[p] + 1
				}
			}

			reposCount[repo.RelPath] = reposCount[repo.RelPath] + 1
		}

		for _, repo := range repos {
			if reposCount[repo.RelPath] > 1 && repo.IsUnderPrimaryRoot() == false {
				continue
			}

			for _, p := range repo.Subpaths() {
				if subpathCount[p] == 1 {
					fmt.Println(p)
					break
				}
			}
		}
	} else {
		for _, repo := range repos {
			if printFullPaths {
				fmt.Println(repo.FullPath)
			} else {
				fmt.Println(repo.RelPath)
			}
		}
	}
}

func doLook(c *cli.Context) {
	name := c.Args().First()

	if name == "" {
		cli.ShowCommandHelp(c, "look")
		os.Exit(1)
	}

	reposFound := []*LocalRepository{}
	walkLocalRepositories(func(repo *LocalRepository) {
		if repo.Matches(name) {
			reposFound = append(reposFound, repo)
		}
	})

	switch len(reposFound) {
	case 0:
		utils.Log("error", "No repository found")

	case 1:
		shell := os.Getenv("SHELL")
		if shell == "" {
			shell = "/bin/sh"
		}

		utils.Log("cd", reposFound[0].FullPath)
		err := os.Chdir(reposFound[0].FullPath)
		utils.PanicIf(err)

		syscall.Exec(shell, []string{shell}, syscall.Environ())

	default:
		utils.Log("error", "More than one repositories are found; Try more precise name")
		for _, repo := range reposFound {
			utils.Log("error", "- "+strings.Join(repo.PathParts, "/"))
		}
	}
}

func doImportStarred(c *cli.Context) {
	user := c.Args().First()

	if user == "" {
		cli.ShowCommandHelp(c, "starred")
		os.Exit(1)
	}

	client := github.NewClient(nil)
	options := &github.ActivityListStarredOptions{Sort: "created"}

	for page := 1; ; page++ {
		options.Page = page

		repositories, res, err := client.Activity.ListStarred(user, options)
		utils.DieIf(err)

		utils.Log("page", fmt.Sprintf("%d/%d", page, res.LastPage))
		for _, repo := range repositories {
			url, err := url.Parse(*repo.HTMLURL)
			if err != nil {
				utils.Log("error", fmt.Sprintf("Could not parse URL <%s>: %s", repo.HTMLURL, err))
				continue
			}

			remote, err := NewRemoteRepository(url)
			if utils.ErrorIf(err) {
				continue
			}

			if remote.IsValid() == false {
				utils.Log("skip", fmt.Sprintf("Not a valid repository: %s", url))
				continue
			}

			getRemoteRepository(remote, c.Bool("update"))
		}

		if page >= res.LastPage {
			break
		}
	}
}

func doImportPocket(c *cli.Context) {
	if pocket.ConsumerKey == "" {
		utils.Log("error", "Built without consumer key set")
		return
	}

	accessToken, err := GitConfig("ghq.pocket.token")
	utils.PanicIf(err)

	if accessToken == "" {
		receiverURL, ch, err := pocket.StartAccessTokenReceiver()
		utils.PanicIf(err)

		utils.Log("pocket", "Waiting for Pocket authentication callback at "+receiverURL)

		utils.Log("pocket", "Obtaining request token")
		authRequest, err := pocket.ObtainRequestToken(receiverURL)
		utils.DieIf(err)

		url := pocket.GenerateAuthorizationURL(authRequest.Code, receiverURL)
		utils.Log("open", url)

		<-ch

		utils.Log("pocket", "Obtaining access token")
		authorized, err := pocket.ObtainAccessToken(authRequest.Code)
		utils.DieIf(err)

		utils.Log("authorized", authorized.Username)

		accessToken = authorized.AccessToken
		utils.Run("git", "config", "ghq.pocket.token", authorized.AccessToken)
	}

	utils.Log("pocket", "Retrieving github.com entries")
	res, err := pocket.RetrieveGitHubEntries(accessToken)
	utils.DieIf(err)

	for _, item := range res.List {
		url, err := url.Parse(item.ResolvedURL)
		if err != nil {
			utils.Log("error", fmt.Sprintf("Could not parse URL <%s>: %s", item.ResolvedURL, err))
			continue
		}

		remote, err := NewRemoteRepository(url)
		if utils.ErrorIf(err) {
			continue
		}

		if remote.IsValid() == false {
			utils.Log("skip", fmt.Sprintf("Not a valid repository: %s", url))
			continue
		}

		getRemoteRepository(remote, c.Bool("update"))
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go-Commander Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// Based on the original work by The Go Authors:
// Copyright 2011 The Go Authors.  All rights reserved.

// commander helps creating command line programs whose arguments are flags,
// commands and subcommands.
package commander

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"sort"
	"strings"
	"text/template"

	"github.com/gonuts/flag"
)

// A Commander holds the configuration for the command line tool.
type Commander struct {
	// Name is the command name, usually the executable's name.
	Name string
	// Short is a short description of the commander
	Short string
	// Commands is the list of commands supported by this commander program.
	Commands []*Command
	// Flag is a set of flags for the whole commander. It should not be
	// changed after Run() is called.
	Flag *flag.FlagSet
	// Parent is the parent commander of this commander
	Parent *Commander
	// Commanders is the list of sub-commanders supported by this commander program.
	Commanders []*Commander
}

// Type to allow us to use sort.Sort on a slice of Commanders
type CommanderSlice []*Commander

func (c CommanderSlice) Len() int {
	return len(c)
}

func (c CommanderSlice) Less(i, j int) bool {
	return c[i].Name < c[j].Name
}

func (c CommanderSlice) Swap(i, j int) {
	c[i], c[j] = c[j], c[i]
}

type CommandSlice []*Command

func (c CommandSlice) Len() int {
	return len(c)
}

func (c CommandSlice) Less(i, j int) bool {
	return c[i].Name() < c[j].Name()
}

func (c CommandSlice) Swap(i, j int) {
	c[i], c[j] = c[j], c[i]
}

// Sort the subcommanders.
func (c *Commander) SortCommanders() {
	sort.Sort(CommanderSlice(c.Commanders))
}

// Sort the commanders
func (c *Commander) SortCommands() {
	sort.Sort(CommandSlice(c.Commands))
}

// Run executes the commander using the provided arguments. The command
// matching the first argument is executed and it receives the remaining
// arguments.
func (c *Commander) Run(args []string) error {
	if c == nil {
		return fmt.Errorf("Called Run() on a nil Commander")
	}
	// setup hierarchy...
	for _, cmd := range c.Commanders {
		cmd.Parent = c
	}

	if c.Flag == nil {
		c.Flag = flag.NewFlagSet(c.Name, flag.ExitOnError)
	}
	if c.Flag.Usage == nil {
		c.Flag.Usage = func() {
			if err := c.usage(); err != nil {
				fmt.Fprintln(os.Stderr, err)
			}
		}
	}
	if !c.Flag.Parsed() {
		if err := c.Flag.Parse(args); err != nil {
			return fmt.Errorf("Commander.Main flag parsing failure: %v", err)
		}
	}
	if len(args) < 1 {
		if err := c.usage(); err != nil {
			return err
		}
		return fmt.Errorf("Not enough arguments provided")
	}

	if args[0] == "help" {
		return c.help(args[1:])
	}

	// first, try a sub-commander
	for _, cmd := range c.Commanders {
		n := cmd.Name
		if n == args[0] {
			return cmd.Run(args[1:])
		}
	}

	// then, try an internal command
	for _, cmd := range c.Commands {
		if cmd.Name() == args[0] && cmd.Runnable() {
			cmd.Flag.Usage = func() { cmd.Usage() }
			if cmd.CustomFlags {
				args = args[1:]
			} else {
				cmd.Flag.Parse(args[1:])
				args = cmd.Flag.Args()
			}
			cmd.Run(cmd, args)
			return nil
		}
	}

	// then try out an external one
	bin, err := exec.LookPath(c.FullName() + "-" + args[0])
	if err == nil {
		cmd := exec.Command(bin, args[1:]...)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		return cmd.Run()
	}

	// TODO: try an alias
	//...

	return fmt.Errorf("unknown subcommand %q\nRun 'help' for usage.\n", args[0])
}

func (c *Commander) usage() error {
	c.SortCommanders()
	c.SortCommands()
	err := tmpl(os.Stderr, usageTemplate, c)
	if err != nil {
		fmt.Println(err)
	}
	return err
}

// help implements the 'help' command.
func (c *Commander) help(args []string) error {
	if len(args) == 0 {
		return c.usage()
	}
	if len(args) != 1 {
		return fmt.Errorf("usage: %v help command\n\nToo many arguments given.\n", c.Name)
	}

	arg := args[0]

	for _, cmd := range c.Commanders {
		n := cmd.Name
		if strings.HasPrefix(n, c.Name+"-") {
			n = n[len(c.Name+"-"):]
		}
		if n == arg {
			return cmd.help(args[1:])
		}
	}

	for _, cmd := range c.Commands {
		if cmd.Name() == arg {
			c := struct {
				*Command
				ProgramName string
			}{cmd, c.FullSpacedName()}
			return tmpl(os.Stdout, helpTemplate, c)
		}
	}

	return fmt.Errorf("Unknown help topic %#q.  Run '%v help'.\n", arg, c.Name)
}

// FlagOptions returns the flag's options as a string
func (c *Commander) FlagOptions() string {
	var buf bytes.Buffer
	c.Flag.SetOutput(&buf)
	fmt.Fprintf(&buf, "\noptions:\n")
    c.Flag.PrintDefaults()
	return string(buf.Bytes())
}

func (c *Commander) MaxLen() (res int) {
	res = 0
	for _, cmd := range c.Commands {
		i := len(cmd.Name())
		if i > res {
			res = i
		}
	}
	for _, cmd := range c.Commanders {
		i := len(cmd.Name)
		if i > res {
			res = i
		}
	}
	return
}

// ColFormat returns the column header size format for printing in the template
func (c *Commander) ColFormat() string {
	sz := c.MaxLen()
	if sz < 11 {
		sz = 11
	}
	return fmt.Sprintf("%%-%ds", sz)
}

// FullName returns the full name of the commander, prefixed with its parent commanders, if any.
func (c *Commander) FullName() string {
	n := c.Name
	if c.Parent != nil {
		n = c.Parent.FullName() + "-" + n
	}
	return n
}

// FullSpacedName returns the full name of the commander, with subcommand names seperated by spaces.
func (c *Commander) FullSpacedName() string {
	n := c.Name
	if c.Parent != nil {
		n = c.Parent.FullSpacedName() + " " + n
	}
	return n
}

// A Command is an implementation of a subcommand.
type Command struct {
	// Run runs the command.
	// The args are the arguments after the command name.
	Run func(cmd *Command, args []string)

	// UsageLine is the one-line usage message.
	// The first word in the line is taken to be the command name.
	UsageLine string

	// Short is the short description shown in the 'help' output.
	Short string

	// Long is the long message shown in the 'help <this-command>' output.
	Long string

	// Flag is a set of flags specific to this command.
	Flag flag.FlagSet

	// CustomFlags indicates that the command will do its own
	// flag parsing.
	CustomFlags bool
}

// Name returns the command's name: the first word in the usage line.
func (c *Command) Name() string {
	name := c.UsageLine
	i := strings.Index(name, " ")
	if i >= 0 {
		name = name[:i]
	}
	return name
}

// Usage prints the usage details to the standard error output.
func (c *Command) Usage() {
	fmt.Fprintf(os.Stderr, "usage: %s\n\n", c.UsageLine)
	fmt.Fprintf(os.Stderr, "%s\n", strings.TrimSpace(c.Long))
}

// FlagOptions returns the flag's options as a string
func (c *Command) FlagOptions() string {
	var buf bytes.Buffer
	c.Flag.SetOutput(&buf)
	fmt.Fprintf(&buf, "\noptions:\n")
	if c.Flag.Usage != nil {
		c.Flag.Usage()
	} else {
		c.Flag.PrintDefaults()
	}
	return string(buf.Bytes())
}

// Runnable reports whether the command can be run; otherwise
// it is a documentation pseudo-command such as importpath.
func (c *Command) Runnable() bool {
	return c.Run != nil
}

var usageTemplate = `Usage: {{.FullSpacedName}} command [arguments]

Commands:
{{range .Commands}}{{if .Runnable}}    {{.Name | printf (colfmt)}} {{.Short}}{{end}}
{{end}}

Subcommands:
{{range .Commanders}}    {{.Name | printf (colfmt)}} {{.Short}}
{{end}}
Use "{{.Name}} help [command]" for more information about a command.
{{.FlagOptions}}

Additional help topics:
{{range .Commands}}{{if not .Runnable}}
    {{.Name | printf (colfmt)}} {{.Short}}{{end}}{{end}}
Use "{{.Name}} help [topic]" for more information about that topic.
`

var helpTemplate = `{{if .Runnable}}Usage: {{.ProgramName}} {{.UsageLine}}

{{end}}{{.Long | trim}}
{{.FlagOptions}}
`

// tmpl executes the given template text on data, writing the result to w.
func tmpl(w io.Writer, text string, data interface{}) error {
	t := template.New("top")
	t.Funcs(template.FuncMap{
		"trim":   strings.TrimSpace,
		"colfmt": func() string { return data.(*Commander).ColFormat() },
	})
	template.Must(t.Parse(text))
	return t.Execute(w, data)
}
<commit_msg>Meld the Commander and Command abstractions.<commit_after>// Copyright 2012 The Go-Commander Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// Based on the original work by The Go Authors:
// Copyright 2011 The Go Authors.  All rights reserved.

// commander helps creating command line programs whose arguments are flags,
// commands and subcommands.
package commander

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"sort"
	"strings"
	"text/template"

	"github.com/gonuts/flag"
)

// A Command is an implementation of a subcommand.
type Command struct {

	// UsageLine is the short usage message.
	// The first word in the line is taken to be the command name.
	UsageLine string

	// Short is the short description line shown in command lists.
	Short string

	// Long is the long description shown in the 'help <this-command>' output.
	Long string

	// Unlested reports whether this command should not be listed in Usage.
	Unlisted bool

	// Run runs the command.
	// The args are the arguments after the command name.
	Run func(cmd *Command, args []string) error

	// Flag is a set of flags specific to this command.
	Flag flag.FlagSet

	// CustomFlags indicates that the command will do its own
	// flag parsing.
	CustomFlags bool

	// Subcommands are dispatched from this command
	Subcommands []*Command

	// Parent command, nil for root.
	Parent *Command

	// UsageTemplate formats the usage (short) information displayed to the user
	// (leave empty for default)
	UsageTemplate string

	// HelpTemplate formats the help (long) information displayed to the user
	// (leave empty for default)
	HelpTemplate string
}

// Name returns the command's name: the first word in the usage line.
func (c *Command) Name() string {
	name := c.UsageLine
	i := strings.Index(name, " ")
	if i >= 0 {
		name = name[:i]
	}
	return name
}

// Usage prints the usage details to the standard error output.
func (c *Command) Usage() {
	c.usage()
}

// FlagOptions returns the flag's options as a string
func (c *Command) FlagOptions() string {
	var buf bytes.Buffer
	c.Flag.SetOutput(&buf)
	c.Flag.PrintDefaults()

	str := string(buf.Bytes())
	if len(str) > 0 {
		return fmt.Sprintf("\nOptions:\n%s", str)
	}
	return ""
}

// Runnable reports whether the command can be run; otherwise
// it is a documentation pseudo-command such as importpath.
func (c *Command) Runnable() bool {
	return c.Run != nil
}

// Type to allow us to use sort.Sort on a slice of Commands
type CommandSlice []*Command

func (c CommandSlice) Len() int {
	return len(c)
}

func (c CommandSlice) Less(i, j int) bool {
	return c[i].Name() < c[j].Name()
}

func (c CommandSlice) Swap(i, j int) {
	c[i], c[j] = c[j], c[i]
}

// Sort the commands
func (c *Command) SortCommands() {
	sort.Sort(CommandSlice(c.Subcommands))
}

// Init the command
func (c *Command) init() {
	if c.Parent != nil {
		return // already initialized.
	}

	// setup strings
	if len(c.UsageLine) < 1 {
		c.UsageLine = Defaults.UsageLine
	}
	if len(c.UsageTemplate) < 1 {
		c.UsageTemplate = Defaults.UsageTemplate
	}
	if len(c.HelpTemplate) < 1 {
		c.HelpTemplate = Defaults.HelpTemplate
	}

	// init subcommands
	for _, cmd := range c.Subcommands {
		cmd.init()
	}

	// init hierarchy...
	for _, cmd := range c.Subcommands {
		cmd.Parent = c
	}
}

// Dispatch executes the command using the provided arguments.
// If a subcommand exists matching the first argument, it is dispatched.
// Otherwise, the command's Run function is called.
func (c *Command) Dispatch(args []string) error {
	if c == nil {
		return fmt.Errorf("Called Run() on a nil Command")
	}

	// Ensure command is initialized.
	c.init()

	// First, try a sub-command
	if len(args) > 0 {
		for _, cmd := range c.Subcommands {
			n := cmd.Name()
			if n == args[0] {
				return cmd.Dispatch(args[1:])
			}
		}

		// help is builtin (but after, to allow overriding)
		if args[0] == "help" {
			return c.help(args[1:])
		}

		// then, try out an external binary (git-style)
		bin, err := exec.LookPath(c.FullName() + "-" + args[0])
		if err == nil {
			cmd := exec.Command(bin, args[1:]...)
			cmd.Stdin = os.Stdin
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			return cmd.Run()
		}
	}

	// then, try running this command
	if c.Runnable() {
		if !c.CustomFlags {
			var err = error(nil)
			c.Flag.Usage = func() {
				c.Usage()
				err = fmt.Errorf("Failed to parse flags.")
			}
			c.Flag.Parse(args)
			if err != nil {
				return err
			}
			args = c.Flag.Args()
		}
		return c.Run(c, args)
	}

	// TODO: try an alias
	//...

	// Last, print usage
	if err := c.usage(); err != nil {
		return err
	}
	return nil
}

func (c *Command) usage() error {
	c.SortCommands()
	err := tmpl(os.Stderr, c.UsageTemplate, c)
	if err != nil {
		fmt.Println(err)
	}
	return err
}

// help implements the 'help' command.
func (c *Command) help(args []string) error {

	// help exactly for this command?
	if len(args) == 0 {
		if len(c.Long) > 0 {
			return tmpl(os.Stdout, c.HelpTemplate, c)
		} else {
			return c.usage()
		}
	}

	arg := args[0]

	// is this help for a subcommand?
	for _, cmd := range c.Subcommands {
		n := cmd.Name()
		// strip out "<parent>-"" name
		if strings.HasPrefix(n, c.Name()+"-") {
			n = n[len(c.Name()+"-"):]
		}
		if n == arg {
			return cmd.help(args[1:])
		}
	}

	return fmt.Errorf("Unknown help topic %#q.  Run '%v help'.\n", arg, c.Name())
}

func (c *Command) MaxLen() (res int) {
	res = 0
	for _, cmd := range c.Subcommands {
		i := len(cmd.Name())
		if i > res {
			res = i
		}
	}
	return
}

// ColFormat returns the column header size format for printing in the template
func (c *Command) ColFormat() string {
	sz := c.MaxLen()
	if sz < 11 {
		sz = 11
	}
	return fmt.Sprintf("%%-%ds", sz)
}

// FullName returns the full name of the command, prefixed with parent commands
func (c *Command) FullName() string {
	n := c.Name()
	if c.Parent != nil {
		n = c.Parent.FullName() + "-" + n
	}
	return n
}

// FullSpacedName returns the full name of the command, with ' ' instead of '-'
func (c *Command) FullSpacedName() string {
	n := c.Name()
	if c.Parent != nil {
		n = c.Parent.FullSpacedName() + " " + n
	}
	return n
}

func (c *Command) HasHelpTopics() bool {
	for _, cmd := range c.Subcommands {
		if cmd.Unlisted {
			return true
		}
	}
	return false
}

var Defaults = Command{
	UsageTemplate: `{{if .Runnable}}Usage: {{.UsageLine}}

{{end}}{{.FullSpacedName}} - {{.Short}}

{{if .Subcommands}}Commands:
{{range .Subcommands}}{{if not .Unlisted}}    {{.Name | printf (colfmt)}} {{.Short}}{{end}}
{{end}}
Use "{{.Name}} help <command>" for more information about a command.

{{end}}{{.FlagOptions}}{{if .HasHelpTopics}}
Additional help topics:
{{range .Subcommands}}{{if .Unlisted}}
    {{.Name | printf (colfmt)}} {{.Short}}{{end}}{{end}}
Use "{{.Name}} help <topic>" for more information about that topic.

{{end}}`,

	HelpTemplate: `{{if .Runnable}}Usage: {{.UsageLine}}

{{end}}{{.Long | trim}}
{{.FlagOptions}}
`,
}

// tmpl executes the given template text on data, writing the result to w.
func tmpl(w io.Writer, text string, data interface{}) error {
	t := template.New("top")
	t.Funcs(template.FuncMap{
		"trim":   strings.TrimSpace,
		"colfmt": func() string { return data.(*Command).ColFormat() },
	})
	template.Must(t.Parse(text))
	return t.Execute(w, data)
}
<|endoftext|>


<commit_before>/*
 * ZLint Copyright 2020 Regents of the University of Michigan
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy
 * of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

package apple

import (
	"time"

	"github.com/zmap/zcrypto/x509"
	"github.com/zmap/zlint/v2/lint"
	"github.com/zmap/zlint/v2/util"
)

type serverCertValidityTooLong struct{}

func (l *serverCertValidityTooLong) Initialize() error {
	return nil
}

func (l *serverCertValidityTooLong) CheckApplies(c *x509.Certificate) bool {
	return util.IsServerAuthCert(c)
}

func (l *serverCertValidityTooLong) Execute(c *x509.Certificate) *lint.LintResult {
	// "398 days is measured with a day being equal to 86,400 seconds. Any time
	// greater than this indicates an additional day of validity."
	dayLength := 86400 * time.Second
	// "TLS server certificates issued on or after September 1, 2020 00:00 GMT/UTC
	// must not have a validity period greater than 398 days."
	maxValidity := 398 * dayLength
	// "We recommend that certificates be issued with a maximum validity of 397 days."
	warnValidity := 397 * dayLength

	// Calculate the date that the certificate NotAfter must be less than
	errEndDate := c.NotBefore.Add(maxValidity)
	warnEndDate := c.NotBefore.Add(warnValidity)

	if c.NotAfter.After(errEndDate) {
		return &lint.LintResult{Status: lint.Error}
	} else if c.NotAfter.After(warnEndDate) {
		return &lint.LintResult{
			// RFC 2119 has SHOULD and RECOMMENDED as equal. Since Apple recommends
			// 397 days we treat this as a lint.Warn result as a violation of
			// a SHOULD.
			Status: lint.Warn,
			Details: "Apple recommends that certificates be issued with a maximum " +
				"validity of 397 days.",
		}
	}

	return &lint.LintResult{Status: lint.Pass}
}

func init() {
	lint.RegisterLint(&lint.Lint{
		Name: "e_tls_server_cert_valid_time_longer_than_398_days",
		Description: "TLS server certificates issued on or after September 1, 2020 " +
			"00:00 GMT/UTC must not have a validity period greater than 398 days",
		Citation: "https://support.apple.com/en-us/HT211025",
		// TODO(@cpu): The Source should be `lint.ApplePolicy` or something similar.
		// The "CT" bit is too specific. Unfortunately since the constant is
		// exported by the `util` package we can't change it without bumping the
		// major version. See https://github.com/zmap/zlint/issues/418
		Source:        lint.AppleCTPolicy,
		EffectiveDate: util.AppleReducedLifetimeDate,
		Lint:          &serverCertValidityTooLong{},
	})
}
<commit_msg>this lint shouldn't apply to CA certs (#456)<commit_after>/*
 * ZLint Copyright 2020 Regents of the University of Michigan
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy
 * of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

package apple

import (
	"time"

	"github.com/zmap/zcrypto/x509"
	"github.com/zmap/zlint/v2/lint"
	"github.com/zmap/zlint/v2/util"
)

type serverCertValidityTooLong struct{}

func (l *serverCertValidityTooLong) Initialize() error {
	return nil
}

func (l *serverCertValidityTooLong) CheckApplies(c *x509.Certificate) bool {
	return util.IsServerAuthCert(c) && !c.IsCA
}

func (l *serverCertValidityTooLong) Execute(c *x509.Certificate) *lint.LintResult {
	// "398 days is measured with a day being equal to 86,400 seconds. Any time
	// greater than this indicates an additional day of validity."
	dayLength := 86400 * time.Second
	// "TLS server certificates issued on or after September 1, 2020 00:00 GMT/UTC
	// must not have a validity period greater than 398 days."
	maxValidity := 398 * dayLength
	// "We recommend that certificates be issued with a maximum validity of 397 days."
	warnValidity := 397 * dayLength

	// Calculate the date that the certificate NotAfter must be less than
	errEndDate := c.NotBefore.Add(maxValidity)
	warnEndDate := c.NotBefore.Add(warnValidity)

	if c.NotAfter.After(errEndDate) {
		return &lint.LintResult{Status: lint.Error}
	} else if c.NotAfter.After(warnEndDate) {
		return &lint.LintResult{
			// RFC 2119 has SHOULD and RECOMMENDED as equal. Since Apple recommends
			// 397 days we treat this as a lint.Warn result as a violation of
			// a SHOULD.
			Status: lint.Warn,
			Details: "Apple recommends that certificates be issued with a maximum " +
				"validity of 397 days.",
		}
	}

	return &lint.LintResult{Status: lint.Pass}
}

func init() {
	lint.RegisterLint(&lint.Lint{
		Name: "e_tls_server_cert_valid_time_longer_than_398_days",
		Description: "TLS server certificates issued on or after September 1, 2020 " +
			"00:00 GMT/UTC must not have a validity period greater than 398 days",
		Citation: "https://support.apple.com/en-us/HT211025",
		// TODO(@cpu): The Source should be `lint.ApplePolicy` or something similar.
		// The "CT" bit is too specific. Unfortunately since the constant is
		// exported by the `util` package we can't change it without bumping the
		// major version. See https://github.com/zmap/zlint/issues/418
		Source:        lint.AppleCTPolicy,
		EffectiveDate: util.AppleReducedLifetimeDate,
		Lint:          &serverCertValidityTooLong{},
	})
}
<|endoftext|>


<commit_before>package app

import (
	gcconfig "k8s.io/kubernetes/pkg/controller/garbagecollector/config"

	"k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
)

func applyOpenShiftGCConfig(controllerManager *config.Config) error {
	// TODO make this configurable or discoverable.  This is going to prevent us from running the stock GC controller
	// IF YOU ADD ANYTHING TO THIS LIST, MAKE SURE THAT YOU UPDATE THEIR STRATEGIES TO PREVENT GC FINALIZERS
	controllerManager.ComponentConfig.GarbageCollectorController.GCIgnoredResources = append(controllerManager.ComponentConfig.GarbageCollectorController.GCIgnoredResources,
		// explicitly disabled from GC for now - not enough value to track them
		gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "rolebindingrestrictions"},
		gcconfig.GroupResource{Group: "network.openshift.io", Resource: "clusternetworks"},
		gcconfig.GroupResource{Group: "network.openshift.io", Resource: "egressnetworkpolicies"},
		gcconfig.GroupResource{Group: "network.openshift.io", Resource: "hostsubnets"},
		gcconfig.GroupResource{Group: "network.openshift.io", Resource: "netnamespaces"},
		gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthclientauthorizations"},
		gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthclients"},
		gcconfig.GroupResource{Group: "quota.openshift.io", Resource: "clusterresourcequotas"},
		gcconfig.GroupResource{Group: "user.openshift.io", Resource: "groups"},
		gcconfig.GroupResource{Group: "user.openshift.io", Resource: "identities"},
		gcconfig.GroupResource{Group: "user.openshift.io", Resource: "users"},
		gcconfig.GroupResource{Group: "image.openshift.io", Resource: "images"},

		// virtual resource
		gcconfig.GroupResource{Group: "project.openshift.io", Resource: "projects"},
		// virtual and unwatchable resource, surfaced via rbac.authorization.k8s.io objects
		gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "clusterroles"},
		gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "clusterrolebindings"},
		gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "roles"},
		gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "rolebindings"},
		// these resources contain security information in their names, and we don't need to track them
		gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthaccesstokens"},
		gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthauthorizetokens"},
		// exposed as autoscaling v1
		gcconfig.GroupResource{Group: "extensions", Resource: "horizontalpodautoscalers"},
		// exposed as security.openshift.io v1
		gcconfig.GroupResource{Group: "", Resource: "securitycontextconstraints"},
	)

	return nil
}
<commit_msg>UPSTREAM: <carry>: GC doesn't ignore hpa and securitycontextconstraints<commit_after>package app

import (
	gcconfig "k8s.io/kubernetes/pkg/controller/garbagecollector/config"

	"k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
)

func applyOpenShiftGCConfig(controllerManager *config.Config) error {
	// TODO make this configurable or discoverable.  This is going to prevent us from running the stock GC controller
	// IF YOU ADD ANYTHING TO THIS LIST, MAKE SURE THAT YOU UPDATE THEIR STRATEGIES TO PREVENT GC FINALIZERS
	controllerManager.ComponentConfig.GarbageCollectorController.GCIgnoredResources = append(controllerManager.ComponentConfig.GarbageCollectorController.GCIgnoredResources,
		// explicitly disabled from GC for now - not enough value to track them
		gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "rolebindingrestrictions"},
		gcconfig.GroupResource{Group: "network.openshift.io", Resource: "clusternetworks"},
		gcconfig.GroupResource{Group: "network.openshift.io", Resource: "egressnetworkpolicies"},
		gcconfig.GroupResource{Group: "network.openshift.io", Resource: "hostsubnets"},
		gcconfig.GroupResource{Group: "network.openshift.io", Resource: "netnamespaces"},
		gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthclientauthorizations"},
		gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthclients"},
		gcconfig.GroupResource{Group: "quota.openshift.io", Resource: "clusterresourcequotas"},
		gcconfig.GroupResource{Group: "user.openshift.io", Resource: "groups"},
		gcconfig.GroupResource{Group: "user.openshift.io", Resource: "identities"},
		gcconfig.GroupResource{Group: "user.openshift.io", Resource: "users"},
		gcconfig.GroupResource{Group: "image.openshift.io", Resource: "images"},

		// virtual resource
		gcconfig.GroupResource{Group: "project.openshift.io", Resource: "projects"},
		// virtual and unwatchable resource, surfaced via rbac.authorization.k8s.io objects
		gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "clusterroles"},
		gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "clusterrolebindings"},
		gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "roles"},
		gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "rolebindings"},
		// these resources contain security information in their names, and we don't need to track them
		gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthaccesstokens"},
		gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthauthorizetokens"},
	)

	return nil
}
<|endoftext|>


<commit_before>package couchdb2_goclient

import (
	"fmt"
	"net/http"
	"time"
)

type Documents interface {
	Headers(db string, id string) (size *int, rev string, err error)
	Document(db string, id string) ([]byte, error)
	UpdateDocument(db string, id string, req map[string]interface{}) (*OkKoResponse, error)
	DeleteDocument(db string, id string) (*OkKoResponse, error)
}

type DocumentsClient struct {
	*CouchDb2ConnDetails
}

func (d *DocumentsClient) Headers(db string, id string) (size *int, rev string, err error) {
	panic("not implemented")
}

func (d *DocumentsClient) Document(db string, id string) ([]byte, error) {
	return d.bytesRequester(http.MethodGet, fmt.Sprintf("/%s/%s", db, id), nil)
}

func (d *DocumentsClient) UpdateDocument(db string, id string, req map[string]interface{}) (*OkKoResponse, error) {
	panic("not implemented")
}

func (d *DocumentsClient) DeleteDocument(db string, id string) (*OkKoResponse, error) {
	panic("not implemented")
}

func NewDocumentsWithConnection(conn *CouchDb2ConnDetails) (doc Documents) {
	doc = &DocumentsClient{conn}

	return
}

func NewDocuments(timeout time.Duration, addr string, user, pass string, secured bool) (doc Documents) {
	doc = &DocumentsClient{
		CouchDb2ConnDetails: NewConnection(timeout, addr, user, pass, secured),
	}

	return
}<commit_msg>Added "DocumentWithType" query method<commit_after>package couchdb2_goclient

import (
	"encoding/json"
	"fmt"
	"net/http"
	"time"
)

type Documents interface {
	Headers(db string, id string) (size *int, rev string, err error)
	Document(db string, id string) ([]byte, error)
	DocumentWithType(db string, id string, t interface{}) error
	UpdateDocument(db string, id string, req map[string]interface{}) (*OkKoResponse, error)
	DeleteDocument(db string, id string) (*OkKoResponse, error)
}

type DocumentsClient struct {
	*CouchDb2ConnDetails
}

func (d *DocumentsClient) Headers(db string, id string) (size *int, rev string, err error) {
	panic("not implemented")
}

func (d *DocumentsClient) DocumentWithType(db string, id string, t interface{}) (err error) {
	var byt []byte
	if byt, err = d.bytesRequester(http.MethodGet, fmt.Sprintf("/%s/%s", db, id), nil); err != nil {
		return
	}

	var dbresult DbResult
	if err = json.Unmarshal(byt, &dbresult); err != nil {
		return
	}

	if dbresult.ErrorResponse != nil {
		err = dbresult.ErrorResponse
		return
	}

	err = json.Unmarshal(byt, &t)

	return
}

func (d *DocumentsClient) Document(db string, id string) ([]byte, error) {
	return d.bytesRequester(http.MethodGet, fmt.Sprintf("/%s/%s", db, id), nil)
}

func (d *DocumentsClient) UpdateDocument(db string, id string, req map[string]interface{}) (*OkKoResponse, error) {
	panic("not implemented")
}

func (d *DocumentsClient) DeleteDocument(db string, id string) (*OkKoResponse, error) {
	panic("not implemented")
}

func NewDocumentsWithConnection(conn *CouchDb2ConnDetails) (doc Documents) {
	doc = &DocumentsClient{conn}

	return
}

func NewDocuments(timeout time.Duration, addr string, user, pass string, secured bool) (doc Documents) {
	doc = &DocumentsClient{
		CouchDb2ConnDetails: NewConnection(timeout, addr, user, pass, secured),
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2015 Robert S. Gerus. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package bot

import (
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/transform"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"regexp"
	"strings"
	"unicode/utf8"

	"github.com/arachnist/gorepost/irc"
)

var trimTitle = regexp.MustCompile("[\\s]+")
var trimLink = regexp.MustCompile("^.*?http")
var enc = charmap.ISO8859_2

func youtube(vid string) string {
	var dat map[string]interface{}
	link := fmt.Sprintf("https://www.youtube.com/oembed?format=json&url=http://www.youtube.com/watch?v=%+v", vid)
	data, err := httpGet(link)
	if err != nil {
		return "error getting data from youtube"
	}

	err = json.Unmarshal(data, &dat)
	if err != nil {
		return "error decoding data from youtube"
	}
	return dat["title"].(string)
}

func youtubeLong(l string) string {
	pattern := regexp.MustCompile(`/watch[?]v[=](?P<vid>[a-zA-Z0-9-_]+)`)
	res := []byte{}
	for _, s := range pattern.FindAllSubmatchIndex([]byte(l), -1) {
		res = pattern.ExpandString(res, "$vid", l, s)
	}
	return youtube(string(res))
}

func youtubeShort(l string) string {
	pattern := regexp.MustCompile(`youtu.be/(?P<vid>[a-zA-Z0-9-_]+)`)
	res := []byte{}
	for _, s := range pattern.FindAllSubmatchIndex([]byte(l), -1) {
		res = pattern.ExpandString(res, "$vid", l, s)
	}
	return youtube(string(res))
}

func fourchanscrape(l string) string {
	h := sha1.New()
	t, e := ioutil.TempFile("", "4scrape_")
	ext := path.Ext(l)
	if e != nil {
		return "error creating temp file"
	}
	multiwriter := io.MultiWriter(h, t)

	response, err := http.Get(l)
	if err != nil {
		return "error while downloading url"
	}
	defer response.Body.Close()

	if response.StatusCode != 200 {
		return "no title"
	}

	_, err = io.Copy(multiwriter, response.Body)
	if err != nil {
		return "error while reading response"
	}

	old := t.Name()
	t.Close()

	filename := fmt.Sprintf("%x%s", h.Sum(nil), ext)
	dest := path.Join(cfg.LookupString(nil, "FourChanDir"), filename)

	err = os.Rename(old, dest)
	if err != nil {
		return "error while renaming tempfile"
	}

	return path.Join(cfg.LookupString(nil, "FourChanLinkBase"), filename)
}

func genericURLTitle(l string) string {
	title, err := httpGetXpath(l, "//head/title")
	if err == errElementNotFound {
		return "no title"
	} else if err != nil {
		return fmt.Sprint("error:", err)
	}

	title = string(trimTitle.ReplaceAll([]byte(title), []byte{' '})[:])
	if !utf8.ValidString(title) {
		title, _, err = transform.String(enc.NewDecoder(), title)
		if err != nil {
			return fmt.Sprint("error:", err)
		}
	}

	return title
}

var customDataFetchers = []struct {
	re      *regexp.Regexp
	fetcher func(l string) string
}{
	{
		re:      regexp.MustCompile("//(www.)?youtube.com/watch"),
		fetcher: youtubeLong,
	},
	{
		re:      regexp.MustCompile("//youtu.be/"),
		fetcher: youtubeShort,
	},
	{
		re:      regexp.MustCompile("//i[.]4cdn[.]org/"),
		fetcher: fourchanscrape,
	},
	{
		re:      regexp.MustCompile(".*"),
		fetcher: genericURLTitle,
	},
}

func linktitle(output func(irc.Message), msg irc.Message) {
	var r []string

	for _, s := range strings.Split(strings.Trim(msg.Trailing, "\001"), " ") {
		if s == "notitle" {
			return
		}

		b, _ := regexp.Match("https?://", []byte(s))

		s = string(trimLink.ReplaceAll([]byte(s), []byte("http"))[:])

		if b {
		FetchersLoop:
			for _, d := range customDataFetchers {
				if d.re.MatchString(s) {
					t := d.fetcher(s)
					if t != "no title" {
						r = append(r, t)
					}
					break FetchersLoop
				}
			}
		}
	}

	if len(r) > 0 {
		t := cfg.LookupString(msg.Context, "LinkTitlePrefix") + strings.Join(r, cfg.LookupString(msg.Context, "LinkTitleDelimiter"))

		output(reply(msg, t))
	}
}

func init() {
	addCallback("PRIVMSG", "LINKTITLE", linktitle)
}
<commit_msg>Fix permisions on downloaded files.<commit_after>// Copyright 2015 Robert S. Gerus. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package bot

import (
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/transform"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"regexp"
	"strings"
	"unicode/utf8"

	"github.com/arachnist/gorepost/irc"
)

var trimTitle = regexp.MustCompile("[\\s]+")
var trimLink = regexp.MustCompile("^.*?http")
var enc = charmap.ISO8859_2

func youtube(vid string) string {
	var dat map[string]interface{}
	link := fmt.Sprintf("https://www.youtube.com/oembed?format=json&url=http://www.youtube.com/watch?v=%+v", vid)
	data, err := httpGet(link)
	if err != nil {
		return "error getting data from youtube"
	}

	err = json.Unmarshal(data, &dat)
	if err != nil {
		return "error decoding data from youtube"
	}
	return dat["title"].(string)
}

func youtubeLong(l string) string {
	pattern := regexp.MustCompile(`/watch[?]v[=](?P<vid>[a-zA-Z0-9-_]+)`)
	res := []byte{}
	for _, s := range pattern.FindAllSubmatchIndex([]byte(l), -1) {
		res = pattern.ExpandString(res, "$vid", l, s)
	}
	return youtube(string(res))
}

func youtubeShort(l string) string {
	pattern := regexp.MustCompile(`youtu.be/(?P<vid>[a-zA-Z0-9-_]+)`)
	res := []byte{}
	for _, s := range pattern.FindAllSubmatchIndex([]byte(l), -1) {
		res = pattern.ExpandString(res, "$vid", l, s)
	}
	return youtube(string(res))
}

func fourchanscrape(l string) string {
	h := sha1.New()
	t, e := ioutil.TempFile("", "4scrape_")
	ext := path.Ext(l)
	if e != nil {
		return "error creating temp file"
	}
	multiwriter := io.MultiWriter(h, t)

	response, err := http.Get(l)
	if err != nil {
		return "error while downloading url"
	}
	defer response.Body.Close()

	if response.StatusCode != 200 {
		return "no title"
	}

	_, err = io.Copy(multiwriter, response.Body)
	if err != nil {
		return "error while reading response"
	}

	old := t.Name()
	t.Close()

	filename := fmt.Sprintf("%x%s", h.Sum(nil), ext)
	dest := path.Join(cfg.LookupString(nil, "FourChanDir"), filename)

	err = os.Rename(old, dest)
	if err != nil {
		return "error while renaming tempfile"
	}

	err = os.Chmod(dest, 0644)
	if err != nil {
		return "error while correcting permisions"
	}

	return path.Join(cfg.LookupString(nil, "FourChanLinkBase"), filename)
}

func genericURLTitle(l string) string {
	title, err := httpGetXpath(l, "//head/title")
	if err == errElementNotFound {
		return "no title"
	} else if err != nil {
		return fmt.Sprint("error:", err)
	}

	title = string(trimTitle.ReplaceAll([]byte(title), []byte{' '})[:])
	if !utf8.ValidString(title) {
		title, _, err = transform.String(enc.NewDecoder(), title)
		if err != nil {
			return fmt.Sprint("error:", err)
		}
	}

	return title
}

var customDataFetchers = []struct {
	re      *regexp.Regexp
	fetcher func(l string) string
}{
	{
		re:      regexp.MustCompile("//(www.)?youtube.com/watch"),
		fetcher: youtubeLong,
	},
	{
		re:      regexp.MustCompile("//youtu.be/"),
		fetcher: youtubeShort,
	},
	{
		re:      regexp.MustCompile("//i[.]4cdn[.]org/"),
		fetcher: fourchanscrape,
	},
	{
		re:      regexp.MustCompile(".*"),
		fetcher: genericURLTitle,
	},
}

func linktitle(output func(irc.Message), msg irc.Message) {
	var r []string

	for _, s := range strings.Split(strings.Trim(msg.Trailing, "\001"), " ") {
		if s == "notitle" {
			return
		}

		b, _ := regexp.Match("https?://", []byte(s))

		s = string(trimLink.ReplaceAll([]byte(s), []byte("http"))[:])

		if b {
		FetchersLoop:
			for _, d := range customDataFetchers {
				if d.re.MatchString(s) {
					t := d.fetcher(s)
					if t != "no title" {
						r = append(r, t)
					}
					break FetchersLoop
				}
			}
		}
	}

	if len(r) > 0 {
		t := cfg.LookupString(msg.Context, "LinkTitlePrefix") + strings.Join(r, cfg.LookupString(msg.Context, "LinkTitleDelimiter"))

		output(reply(msg, t))
	}
}

func init() {
	addCallback("PRIVMSG", "LINKTITLE", linktitle)
}
<|endoftext|>