<commit_before>package main

import (
	"errors"
	"fmt"
	"strings"

	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

type tokenSource struct {
	token *oauth2.Token
}

func (t *tokenSource) Token() (*oauth2.Token, error) {
	return t.token, nil
}

// Github holds github token and client
type Github struct {
	token  string
	client *github.Client
}

func newGithub(token string) Github {
	// TODO: https://github.com/sourcegraph/apiproxy
	client := github.NewClient(oauth2.NewClient(oauth2.NoContext, &tokenSource{
		&oauth2.Token{
			AccessToken: token,
			TokenType:   "token",
		},
	}))

	//* DEBUG: print rate-limit.
	fmt.Println("<DEBUG>")
	rate, _, err := client.RateLimit()
	if err != nil {
		fmt.Printf("Error fetching GitHub API Rate Limit: %#v\n", err)
	} else {
		fmt.Printf("GitHub API Rate Limit: %#v\n", rate)
	}
	fmt.Println("</DEBUG>")
	// END DEBUG. */

	return Github{client: client, token: token}
}

func (g *Github) getTeamID(orgName string) (int, error) {
	orgNameArray := strings.Split(orgName, "/")
	org := orgNameArray[0]
	name := orgNameArray[1]
	teams, _, err := g.client.Organizations.ListTeams(org, nil)
	if err != nil {
		return 0, err
	}
	for _, team := range teams {
		if strings.EqualFold(*team.Name, name) {
			return *team.ID, nil
		}
	}
	return 0, errors.New("Team ID not found.")
}

var githubTeamMembers = map[string][]string{}

func (g *Github) getMembersOfTeam(orgName string) ([]string, error) {
	members, ok := githubTeamMembers[orgName]
	if ok {
		return members, nil
	}
	teamID, err := g.getTeamID(orgName)
	if err != nil {
		return members, err
	}
	githubUsers, _, err := g.client.Organizations.ListTeamMembers(teamID, nil)
	if err != nil {
		return members, err
	}
	for _, githubUser := range githubUsers {
		members = append(members, *githubUser.Login)
	}
	githubTeamMembers[orgName] = members
	return members, nil
}

var githubUserKeys = map[string][]string{}

func (g *Github) getKeysOfUser(user string) ([]string, error) {
	keys, ok := githubUserKeys[user]
	if ok {
		return keys, nil
	}
	githubKeys, _, err := g.client.Users.ListKeys(user, nil)
	if err != nil {
		return keys, err
	}
	for _, githubKey := range githubKeys {
		keys = append(keys, *githubKey.Key)
	}
	githubUserKeys[user] = keys
	return keys, nil
}

func appendUserIfMissing(users []string, newUser string) []string {
	for _, user := range users {
		if user == newUser {
			return users
		}
	}
	return append(users, newUser)
}

func (g *Github) getKeys(users, teams []string) []string {
	var keys []string
	// add members of teams to array of users
	teamMembersChan := make(chan []string)
	for _, orgName := range teams {
		go func(orgName string) {
			members, _ := g.getMembersOfTeam(orgName)
			// TODO: handle error
			teamMembersChan <- members
		}(orgName)
	}
	for i := 0; i < len(teams); i++ {
		members := <-teamMembersChan
		for _, member := range members {
			users = appendUserIfMissing(users, member)
		}
	}
	// get keys of each user
	userKeysChan := make(chan []string)
	for _, user := range users {
		go func(user string) {
			userKeys, _ := g.getKeysOfUser(user)
			// TODO: handle error
			userKeysChan <- userKeys
		}(user)
	}
	for i := 0; i < len(users); i++ {
		userKeys := <-userKeysChan
		keys = append(keys, userKeys...)
	}
	return keys
}
<commit_msg>Comment out rate limit printing<commit_after>package main

import (
	"errors"
	"strings"

	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

type tokenSource struct {
	token *oauth2.Token
}

func (t *tokenSource) Token() (*oauth2.Token, error) {
	return t.token, nil
}

// Github holds github token and client
type Github struct {
	token  string
	client *github.Client
}

func newGithub(token string) Github {
	// TODO: https://github.com/sourcegraph/apiproxy
	client := github.NewClient(oauth2.NewClient(oauth2.NoContext, &tokenSource{
		&oauth2.Token{
			AccessToken: token,
			TokenType:   "token",
		},
	}))

	/* DEBUG: print rate-limit.
	fmt.Println("<DEBUG>")
	rate, _, err := client.RateLimit()
	if err != nil {
		fmt.Printf("Error fetching GitHub API Rate Limit: %#v\n", err)
	} else {
		fmt.Printf("GitHub API Rate Limit: %#v\n", rate)
	}
	fmt.Println("</DEBUG>")
	// END DEBUG. */

	return Github{client: client, token: token}
}

func (g *Github) getTeamID(orgName string) (int, error) {
	orgNameArray := strings.Split(orgName, "/")
	org := orgNameArray[0]
	name := orgNameArray[1]
	teams, _, err := g.client.Organizations.ListTeams(org, nil)
	if err != nil {
		return 0, err
	}
	for _, team := range teams {
		if strings.EqualFold(*team.Name, name) {
			return *team.ID, nil
		}
	}
	return 0, errors.New("Team ID not found.")
}

var githubTeamMembers = map[string][]string{}

func (g *Github) getMembersOfTeam(orgName string) ([]string, error) {
	members, ok := githubTeamMembers[orgName]
	if ok {
		return members, nil
	}
	teamID, err := g.getTeamID(orgName)
	if err != nil {
		return members, err
	}
	githubUsers, _, err := g.client.Organizations.ListTeamMembers(teamID, nil)
	if err != nil {
		return members, err
	}
	for _, githubUser := range githubUsers {
		members = append(members, *githubUser.Login)
	}
	githubTeamMembers[orgName] = members
	return members, nil
}

var githubUserKeys = map[string][]string{}

func (g *Github) getKeysOfUser(user string) ([]string, error) {
	keys, ok := githubUserKeys[user]
	if ok {
		return keys, nil
	}
	githubKeys, _, err := g.client.Users.ListKeys(user, nil)
	if err != nil {
		return keys, err
	}
	for _, githubKey := range githubKeys {
		keys = append(keys, *githubKey.Key)
	}
	githubUserKeys[user] = keys
	return keys, nil
}

func appendUserIfMissing(users []string, newUser string) []string {
	for _, user := range users {
		if user == newUser {
			return users
		}
	}
	return append(users, newUser)
}

func (g *Github) getKeys(users, teams []string) []string {
	var keys []string
	// add members of teams to array of users
	teamMembersChan := make(chan []string)
	for _, orgName := range teams {
		go func(orgName string) {
			members, _ := g.getMembersOfTeam(orgName)
			// TODO: handle error
			teamMembersChan <- members
		}(orgName)
	}
	for i := 0; i < len(teams); i++ {
		members := <-teamMembersChan
		for _, member := range members {
			users = appendUserIfMissing(users, member)
		}
	}
	// get keys of each user
	userKeysChan := make(chan []string)
	for _, user := range users {
		go func(user string) {
			userKeys, _ := g.getKeysOfUser(user)
			// TODO: handle error
			userKeysChan <- userKeys
		}(user)
	}
	for i := 0; i < len(users); i++ {
		userKeys := <-userKeysChan
		keys = append(keys, userKeys...)
	}
	return keys
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package generators

import (
	"fmt"
	"path"
	"path/filepath"
	"strings"

	"k8s.io/gengo/args"
	"k8s.io/gengo/generator"
	"k8s.io/gengo/namer"
	"k8s.io/gengo/types"
	"k8s.io/klog"

	"k8s.io/code-generator/cmd/client-gen/generators/util"
	clientgentypes "k8s.io/code-generator/cmd/client-gen/types"
	informergenargs "k8s.io/code-generator/cmd/informer-gen/args"
)

// NameSystems returns the name system used by the generators in this package.
func NameSystems() namer.NameSystems {
	pluralExceptions := map[string]string{
		"Endpoints": "Endpoints",
	}
	return namer.NameSystems{
		"public":             namer.NewPublicNamer(0),
		"private":            namer.NewPrivateNamer(0),
		"raw":                namer.NewRawNamer("", nil),
		"publicPlural":       namer.NewPublicPluralNamer(pluralExceptions),
		"allLowercasePlural": namer.NewAllLowercasePluralNamer(pluralExceptions),
		"lowercaseSingular":  &lowercaseSingularNamer{},
	}
}

// lowercaseSingularNamer implements Namer
type lowercaseSingularNamer struct{}

// Name returns t's name in all lowercase.
func (n *lowercaseSingularNamer) Name(t *types.Type) string {
	return strings.ToLower(t.Name.Name)
}

// DefaultNameSystem returns the default name system for ordering the types to be
// processed by the generators in this package.
func DefaultNameSystem() string {
	return "public"
}

// objectMetaForPackage returns the type of ObjectMeta used by package p.
func objectMetaForPackage(p *types.Package) (*types.Type, bool, error) {
	generatingForPackage := false
	for _, t := range p.Types {
		if !util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...)).GenerateClient {
			continue
		}
		generatingForPackage = true
		for _, member := range t.Members {
			if member.Name == "ObjectMeta" {
				return member.Type, isInternal(member), nil
			}
		}
	}
	if generatingForPackage {
		return nil, false, fmt.Errorf("unable to find ObjectMeta for any types in package %s", p.Path)
	}
	return nil, false, nil
}

// isInternal returns true if the tags for a member do not contain a json tag
func isInternal(m types.Member) bool {
	return !strings.Contains(m.Tags, "json")
}

func packageForInternalInterfaces(base string) string {
	return filepath.Join(base, "internalinterfaces")
}

func vendorless(p string) string {
	if pos := strings.LastIndex(p, "/vendor/"); pos != -1 {
		return p[pos+len("/vendor/"):]
	}
	return p
}

// Packages makes the client package definition.
func Packages(context *generator.Context, arguments *args.GeneratorArgs) generator.Packages {
	boilerplate, err := arguments.LoadGoBoilerplate()
	if err != nil {
		klog.Fatalf("Failed loading boilerplate: %v", err)
	}

	customArgs, ok := arguments.CustomArgs.(*informergenargs.CustomArgs)
	if !ok {
		klog.Fatalf("Wrong CustomArgs type: %T", arguments.CustomArgs)
	}

	internalVersionPackagePath := filepath.Join(arguments.OutputPackagePath)
	externalVersionPackagePath := filepath.Join(arguments.OutputPackagePath)
	if !customArgs.SingleDirectory {
		internalVersionPackagePath = filepath.Join(arguments.OutputPackagePath, "internalversion")
		externalVersionPackagePath = filepath.Join(arguments.OutputPackagePath, "externalversions")
	}

	var packageList generator.Packages
	typesForGroupVersion := make(map[clientgentypes.GroupVersion][]*types.Type)

	externalGroupVersions := make(map[string]clientgentypes.GroupVersions)
	internalGroupVersions := make(map[string]clientgentypes.GroupVersions)
	groupGoNames := make(map[string]string)
	for _, inputDir := range arguments.InputDirs {
		p := context.Universe.Package(vendorless(inputDir))

		objectMeta, internal, err := objectMetaForPackage(p)
		if err != nil {
			klog.Fatal(err)
		}
		if objectMeta == nil {
			// no types in this package had genclient
			continue
		}

		var gv clientgentypes.GroupVersion
		var targetGroupVersions map[string]clientgentypes.GroupVersions

		if internal {
			lastSlash := strings.LastIndex(p.Path, "/")
			if lastSlash == -1 {
				klog.Fatalf("error constructing internal group version for package %q", p.Path)
			}
			gv.Group = clientgentypes.Group(p.Path[lastSlash+1:])
			targetGroupVersions = internalGroupVersions
		} else {
			parts := strings.Split(p.Path, "/")
			gv.Group = clientgentypes.Group(parts[len(parts)-2])
			gv.Version = clientgentypes.Version(parts[len(parts)-1])
			targetGroupVersions = externalGroupVersions
		}
		groupPackageName := gv.Group.NonEmpty()
		gvPackage := path.Clean(p.Path)

		// If there's a comment of the form "// +groupName=somegroup" or
		// "// +groupName=somegroup.foo.bar.io", use the first field (somegroup) as the name of the
		// group when generating.
		if override := types.ExtractCommentTags("+", p.Comments)["groupName"]; override != nil {
			gv.Group = clientgentypes.Group(override[0])
		}

		// If there's a comment of the form "// +groupGoName=SomeUniqueShortName", use that as
		// the Go group identifier in CamelCase. It defaults
		groupGoNames[groupPackageName] = namer.IC(strings.Split(gv.Group.NonEmpty(), ".")[0])
		if override := types.ExtractCommentTags("+", p.Comments)["groupGoName"]; override != nil {
			groupGoNames[groupPackageName] = namer.IC(override[0])
		}

		var typesToGenerate []*types.Type
		for _, t := range p.Types {
			tags := util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...))
			if !tags.GenerateClient || tags.NoVerbs || !tags.HasVerb("list") || !tags.HasVerb("watch") {
				continue
			}

			typesToGenerate = append(typesToGenerate, t)

			if _, ok := typesForGroupVersion[gv]; !ok {
				typesForGroupVersion[gv] = []*types.Type{}
			}
			typesForGroupVersion[gv] = append(typesForGroupVersion[gv], t)
		}
		if len(typesToGenerate) == 0 {
			continue
		}

		groupVersionsEntry, ok := targetGroupVersions[groupPackageName]
		if !ok {
			groupVersionsEntry = clientgentypes.GroupVersions{
				PackageName: groupPackageName,
				Group:       gv.Group,
			}
		}
		groupVersionsEntry.Versions = append(groupVersionsEntry.Versions, clientgentypes.PackageVersion{Version: gv.Version, Package: gvPackage})
		targetGroupVersions[groupPackageName] = groupVersionsEntry

		orderer := namer.Orderer{Namer: namer.NewPrivateNamer(0)}
		typesToGenerate = orderer.OrderTypes(typesToGenerate)

		if internal {
			packageList = append(packageList, versionPackage(internalVersionPackagePath, groupPackageName, gv, groupGoNames[groupPackageName], boilerplate, typesToGenerate, customArgs.InternalClientSetPackage, customArgs.ListersPackage))
		} else {
			packageList = append(packageList, versionPackage(externalVersionPackagePath, groupPackageName, gv, groupGoNames[groupPackageName], boilerplate, typesToGenerate, customArgs.VersionedClientSetPackage, customArgs.ListersPackage))
		}
	}

	if len(externalGroupVersions) != 0 {
		packageList = append(packageList, factoryInterfacePackage(externalVersionPackagePath, boilerplate, customArgs.VersionedClientSetPackage))
		packageList = append(packageList, factoryPackage(externalVersionPackagePath, boilerplate, groupGoNames, externalGroupVersions, customArgs.VersionedClientSetPackage, typesForGroupVersion))
		for _, gvs := range externalGroupVersions {
			packageList = append(packageList, groupPackage(externalVersionPackagePath, gvs, boilerplate))
		}
	}

	if len(internalGroupVersions) != 0 {
		packageList = append(packageList, factoryInterfacePackage(internalVersionPackagePath, boilerplate, customArgs.InternalClientSetPackage))
		packageList = append(packageList, factoryPackage(internalVersionPackagePath, boilerplate, groupGoNames, internalGroupVersions, customArgs.InternalClientSetPackage, typesForGroupVersion))
		for _, gvs := range internalGroupVersions {
			packageList = append(packageList, groupPackage(internalVersionPackagePath, gvs, boilerplate))
		}
	}

	return packageList
}

func factoryPackage(basePackage string, boilerplate []byte, groupGoNames map[string]string, groupVersions map[string]clientgentypes.GroupVersions, clientSetPackage string, typesForGroupVersion map[clientgentypes.GroupVersion][]*types.Type) generator.Package {
	return &generator.DefaultPackage{
		PackageName: filepath.Base(basePackage),
		PackagePath: basePackage,
		HeaderText:  boilerplate,
		GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) {
			generators = append(generators, &factoryGenerator{
				DefaultGen: generator.DefaultGen{
					OptionalName: "factory",
				},
				outputPackage:             basePackage,
				imports:                   generator.NewImportTracker(),
				groupVersions:             groupVersions,
				clientSetPackage:          clientSetPackage,
				internalInterfacesPackage: packageForInternalInterfaces(basePackage),
				gvGoNames:                 groupGoNames,
			})

			generators = append(generators, &genericGenerator{
				DefaultGen: generator.DefaultGen{
					OptionalName: "generic",
				},
				outputPackage:        basePackage,
				imports:              generator.NewImportTracker(),
				groupVersions:        groupVersions,
				typesForGroupVersion: typesForGroupVersion,
				groupGoNames:         groupGoNames,
			})

			return generators
		},
	}
}

func factoryInterfacePackage(basePackage string, boilerplate []byte, clientSetPackage string) generator.Package {
	packagePath := packageForInternalInterfaces(basePackage)

	return &generator.DefaultPackage{
		PackageName: filepath.Base(packagePath),
		PackagePath: packagePath,
		HeaderText:  boilerplate,
		GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) {
			generators = append(generators, &factoryInterfaceGenerator{
				DefaultGen: generator.DefaultGen{
					OptionalName: "factory_interfaces",
				},
				outputPackage:    packagePath,
				imports:          generator.NewImportTracker(),
				clientSetPackage: clientSetPackage,
			})

			return generators
		},
	}
}

func groupPackage(basePackage string, groupVersions clientgentypes.GroupVersions, boilerplate []byte) generator.Package {
	packagePath := filepath.Join(basePackage, groupVersions.PackageName)

	return &generator.DefaultPackage{
		PackageName: groupVersions.PackageName,
		PackagePath: packagePath,
		HeaderText:  boilerplate,
		GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) {
			generators = append(generators, &groupInterfaceGenerator{
				DefaultGen: generator.DefaultGen{
					OptionalName: "interface",
				},
				outputPackage:             packagePath,
				groupVersions:             groupVersions,
				imports:                   generator.NewImportTracker(),
				internalInterfacesPackage: packageForInternalInterfaces(basePackage),
			})
			return generators
		},
		FilterFunc: func(c *generator.Context, t *types.Type) bool {
			tags := util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...))
			return tags.GenerateClient && tags.HasVerb("list") && tags.HasVerb("watch")
		},
	}
}

func versionPackage(basePackage string, groupPkgName string, gv clientgentypes.GroupVersion, groupGoName string, boilerplate []byte, typesToGenerate []*types.Type, clientSetPackage, listersPackage string) generator.Package {
	packagePath := filepath.Join(basePackage, groupPkgName, strings.ToLower(gv.Version.NonEmpty()))

	return &generator.DefaultPackage{
		PackageName: strings.ToLower(gv.Version.NonEmpty()),
		PackagePath: packagePath,
		HeaderText:  boilerplate,
		GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) {
			generators = append(generators, &versionInterfaceGenerator{
				DefaultGen: generator.DefaultGen{
					OptionalName: "interface",
				},
				outputPackage:             packagePath,
				imports:                   generator.NewImportTracker(),
				types:                     typesToGenerate,
				internalInterfacesPackage: packageForInternalInterfaces(basePackage),
			})

			for _, t := range typesToGenerate {
				generators = append(generators, &informerGenerator{
					DefaultGen: generator.DefaultGen{
						OptionalName: strings.ToLower(t.Name.Name),
					},
					outputPackage:             packagePath,
					groupPkgName:              groupPkgName,
					groupVersion:              gv,
					groupGoName:               groupGoName,
					typeToGenerate:            t,
					imports:                   generator.NewImportTracker(),
					clientSetPackage:          clientSetPackage,
					listersPackage:            listersPackage,
					internalInterfacesPackage: packageForInternalInterfaces(basePackage),
				})
			}
			return generators
		},
		FilterFunc: func(c *generator.Context, t *types.Type) bool {
			tags := util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...))
			return tags.GenerateClient && tags.HasVerb("list") && tags.HasVerb("watch")
		},
	}
}
<commit_msg>informer-gen: allow package names containing dots<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package generators

import (
	"fmt"
	"path"
	"path/filepath"
	"strings"

	"k8s.io/gengo/args"
	"k8s.io/gengo/generator"
	"k8s.io/gengo/namer"
	"k8s.io/gengo/types"
	"k8s.io/klog"

	"k8s.io/code-generator/cmd/client-gen/generators/util"
	clientgentypes "k8s.io/code-generator/cmd/client-gen/types"
	informergenargs "k8s.io/code-generator/cmd/informer-gen/args"
)

// NameSystems returns the name system used by the generators in this package.
func NameSystems() namer.NameSystems {
	pluralExceptions := map[string]string{
		"Endpoints": "Endpoints",
	}
	return namer.NameSystems{
		"public":             namer.NewPublicNamer(0),
		"private":            namer.NewPrivateNamer(0),
		"raw":                namer.NewRawNamer("", nil),
		"publicPlural":       namer.NewPublicPluralNamer(pluralExceptions),
		"allLowercasePlural": namer.NewAllLowercasePluralNamer(pluralExceptions),
		"lowercaseSingular":  &lowercaseSingularNamer{},
	}
}

// lowercaseSingularNamer implements Namer
type lowercaseSingularNamer struct{}

// Name returns t's name in all lowercase.
func (n *lowercaseSingularNamer) Name(t *types.Type) string {
	return strings.ToLower(t.Name.Name)
}

// DefaultNameSystem returns the default name system for ordering the types to be
// processed by the generators in this package.
func DefaultNameSystem() string {
	return "public"
}

// objectMetaForPackage returns the type of ObjectMeta used by package p.
func objectMetaForPackage(p *types.Package) (*types.Type, bool, error) {
	generatingForPackage := false
	for _, t := range p.Types {
		if !util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...)).GenerateClient {
			continue
		}
		generatingForPackage = true
		for _, member := range t.Members {
			if member.Name == "ObjectMeta" {
				return member.Type, isInternal(member), nil
			}
		}
	}
	if generatingForPackage {
		return nil, false, fmt.Errorf("unable to find ObjectMeta for any types in package %s", p.Path)
	}
	return nil, false, nil
}

// isInternal returns true if the tags for a member do not contain a json tag
func isInternal(m types.Member) bool {
	return !strings.Contains(m.Tags, "json")
}

func packageForInternalInterfaces(base string) string {
	return filepath.Join(base, "internalinterfaces")
}

func vendorless(p string) string {
	if pos := strings.LastIndex(p, "/vendor/"); pos != -1 {
		return p[pos+len("/vendor/"):]
	}
	return p
}

// Packages makes the client package definition.
func Packages(context *generator.Context, arguments *args.GeneratorArgs) generator.Packages {
	boilerplate, err := arguments.LoadGoBoilerplate()
	if err != nil {
		klog.Fatalf("Failed loading boilerplate: %v", err)
	}

	customArgs, ok := arguments.CustomArgs.(*informergenargs.CustomArgs)
	if !ok {
		klog.Fatalf("Wrong CustomArgs type: %T", arguments.CustomArgs)
	}

	internalVersionPackagePath := filepath.Join(arguments.OutputPackagePath)
	externalVersionPackagePath := filepath.Join(arguments.OutputPackagePath)
	if !customArgs.SingleDirectory {
		internalVersionPackagePath = filepath.Join(arguments.OutputPackagePath, "internalversion")
		externalVersionPackagePath = filepath.Join(arguments.OutputPackagePath, "externalversions")
	}

	var packageList generator.Packages
	typesForGroupVersion := make(map[clientgentypes.GroupVersion][]*types.Type)

	externalGroupVersions := make(map[string]clientgentypes.GroupVersions)
	internalGroupVersions := make(map[string]clientgentypes.GroupVersions)
	groupGoNames := make(map[string]string)
	for _, inputDir := range arguments.InputDirs {
		p := context.Universe.Package(vendorless(inputDir))

		objectMeta, internal, err := objectMetaForPackage(p)
		if err != nil {
			klog.Fatal(err)
		}
		if objectMeta == nil {
			// no types in this package had genclient
			continue
		}

		var gv clientgentypes.GroupVersion
		var targetGroupVersions map[string]clientgentypes.GroupVersions

		if internal {
			lastSlash := strings.LastIndex(p.Path, "/")
			if lastSlash == -1 {
				klog.Fatalf("error constructing internal group version for package %q", p.Path)
			}
			gv.Group = clientgentypes.Group(p.Path[lastSlash+1:])
			targetGroupVersions = internalGroupVersions
		} else {
			parts := strings.Split(p.Path, "/")
			gv.Group = clientgentypes.Group(parts[len(parts)-2])
			gv.Version = clientgentypes.Version(parts[len(parts)-1])
			targetGroupVersions = externalGroupVersions
		}
		groupPackageName := gv.Group.NonEmpty()
		gvPackage := path.Clean(p.Path)

		// If there's a comment of the form "// +groupName=somegroup" or
		// "// +groupName=somegroup.foo.bar.io", use the first field (somegroup) as the name of the
		// group when generating.
		if override := types.ExtractCommentTags("+", p.Comments)["groupName"]; override != nil {
			gv.Group = clientgentypes.Group(override[0])
		}

		// If there's a comment of the form "// +groupGoName=SomeUniqueShortName", use that as
		// the Go group identifier in CamelCase. It defaults
		groupGoNames[groupPackageName] = namer.IC(strings.Split(gv.Group.NonEmpty(), ".")[0])
		if override := types.ExtractCommentTags("+", p.Comments)["groupGoName"]; override != nil {
			groupGoNames[groupPackageName] = namer.IC(override[0])
		}

		var typesToGenerate []*types.Type
		for _, t := range p.Types {
			tags := util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...))
			if !tags.GenerateClient || tags.NoVerbs || !tags.HasVerb("list") || !tags.HasVerb("watch") {
				continue
			}

			typesToGenerate = append(typesToGenerate, t)

			if _, ok := typesForGroupVersion[gv]; !ok {
				typesForGroupVersion[gv] = []*types.Type{}
			}
			typesForGroupVersion[gv] = append(typesForGroupVersion[gv], t)
		}
		if len(typesToGenerate) == 0 {
			continue
		}

		groupVersionsEntry, ok := targetGroupVersions[groupPackageName]
		if !ok {
			groupVersionsEntry = clientgentypes.GroupVersions{
				PackageName: groupPackageName,
				Group:       gv.Group,
			}
		}
		groupVersionsEntry.Versions = append(groupVersionsEntry.Versions, clientgentypes.PackageVersion{Version: gv.Version, Package: gvPackage})
		targetGroupVersions[groupPackageName] = groupVersionsEntry

		orderer := namer.Orderer{Namer: namer.NewPrivateNamer(0)}
		typesToGenerate = orderer.OrderTypes(typesToGenerate)

		if internal {
			packageList = append(packageList, versionPackage(internalVersionPackagePath, groupPackageName, gv, groupGoNames[groupPackageName], boilerplate, typesToGenerate, customArgs.InternalClientSetPackage, customArgs.ListersPackage))
		} else {
			packageList = append(packageList, versionPackage(externalVersionPackagePath, groupPackageName, gv, groupGoNames[groupPackageName], boilerplate, typesToGenerate, customArgs.VersionedClientSetPackage, customArgs.ListersPackage))
		}
	}

	if len(externalGroupVersions) != 0 {
		packageList = append(packageList, factoryInterfacePackage(externalVersionPackagePath, boilerplate, customArgs.VersionedClientSetPackage))
		packageList = append(packageList, factoryPackage(externalVersionPackagePath, boilerplate, groupGoNames, externalGroupVersions, customArgs.VersionedClientSetPackage, typesForGroupVersion))
		for _, gvs := range externalGroupVersions {
			packageList = append(packageList, groupPackage(externalVersionPackagePath, gvs, boilerplate))
		}
	}

	if len(internalGroupVersions) != 0 {
		packageList = append(packageList, factoryInterfacePackage(internalVersionPackagePath, boilerplate, customArgs.InternalClientSetPackage))
		packageList = append(packageList, factoryPackage(internalVersionPackagePath, boilerplate, groupGoNames, internalGroupVersions, customArgs.InternalClientSetPackage, typesForGroupVersion))
		for _, gvs := range internalGroupVersions {
			packageList = append(packageList, groupPackage(internalVersionPackagePath, gvs, boilerplate))
		}
	}

	return packageList
}

func factoryPackage(basePackage string, boilerplate []byte, groupGoNames map[string]string, groupVersions map[string]clientgentypes.GroupVersions, clientSetPackage string, typesForGroupVersion map[clientgentypes.GroupVersion][]*types.Type) generator.Package {
	return &generator.DefaultPackage{
		PackageName: filepath.Base(basePackage),
		PackagePath: basePackage,
		HeaderText:  boilerplate,
		GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) {
			generators = append(generators, &factoryGenerator{
				DefaultGen: generator.DefaultGen{
					OptionalName: "factory",
				},
				outputPackage:             basePackage,
				imports:                   generator.NewImportTracker(),
				groupVersions:             groupVersions,
				clientSetPackage:          clientSetPackage,
				internalInterfacesPackage: packageForInternalInterfaces(basePackage),
				gvGoNames:                 groupGoNames,
			})

			generators = append(generators, &genericGenerator{
				DefaultGen: generator.DefaultGen{
					OptionalName: "generic",
				},
				outputPackage:        basePackage,
				imports:              generator.NewImportTracker(),
				groupVersions:        groupVersions,
				typesForGroupVersion: typesForGroupVersion,
				groupGoNames:         groupGoNames,
			})

			return generators
		},
	}
}

func factoryInterfacePackage(basePackage string, boilerplate []byte, clientSetPackage string) generator.Package {
	packagePath := packageForInternalInterfaces(basePackage)

	return &generator.DefaultPackage{
		PackageName: filepath.Base(packagePath),
		PackagePath: packagePath,
		HeaderText:  boilerplate,
		GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) {
			generators = append(generators, &factoryInterfaceGenerator{
				DefaultGen: generator.DefaultGen{
					OptionalName: "factory_interfaces",
				},
				outputPackage:    packagePath,
				imports:          generator.NewImportTracker(),
				clientSetPackage: clientSetPackage,
			})

			return generators
		},
	}
}

func groupPackage(basePackage string, groupVersions clientgentypes.GroupVersions, boilerplate []byte) generator.Package {
	packagePath := filepath.Join(basePackage, groupVersions.PackageName)
	groupPkgName := strings.Split(string(groupVersions.PackageName), ".")[0]

	return &generator.DefaultPackage{
		PackageName: groupPkgName,
		PackagePath: packagePath,
		HeaderText:  boilerplate,
		GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) {
			generators = append(generators, &groupInterfaceGenerator{
				DefaultGen: generator.DefaultGen{
					OptionalName: "interface",
				},
				outputPackage:             packagePath,
				groupVersions:             groupVersions,
				imports:                   generator.NewImportTracker(),
				internalInterfacesPackage: packageForInternalInterfaces(basePackage),
			})
			return generators
		},
		FilterFunc: func(c *generator.Context, t *types.Type) bool {
			tags := util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...))
			return tags.GenerateClient && tags.HasVerb("list") && tags.HasVerb("watch")
		},
	}
}

func versionPackage(basePackage string, groupPkgName string, gv clientgentypes.GroupVersion, groupGoName string, boilerplate []byte, typesToGenerate []*types.Type, clientSetPackage, listersPackage string) generator.Package {
	packagePath := filepath.Join(basePackage, groupPkgName, strings.ToLower(gv.Version.NonEmpty()))

	return &generator.DefaultPackage{
		PackageName: strings.ToLower(gv.Version.NonEmpty()),
		PackagePath: packagePath,
		HeaderText:  boilerplate,
		GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) {
			generators = append(generators, &versionInterfaceGenerator{
				DefaultGen: generator.DefaultGen{
					OptionalName: "interface",
				},
				outputPackage:             packagePath,
				imports:                   generator.NewImportTracker(),
				types:                     typesToGenerate,
				internalInterfacesPackage: packageForInternalInterfaces(basePackage),
			})

			for _, t := range typesToGenerate {
				generators = append(generators, &informerGenerator{
					DefaultGen: generator.DefaultGen{
						OptionalName: strings.ToLower(t.Name.Name),
					},
					outputPackage:             packagePath,
					groupPkgName:              groupPkgName,
					groupVersion:              gv,
					groupGoName:               groupGoName,
					typeToGenerate:            t,
					imports:                   generator.NewImportTracker(),
					clientSetPackage:          clientSetPackage,
					listersPackage:            listersPackage,
					internalInterfacesPackage: packageForInternalInterfaces(basePackage),
				})
			}
			return generators
		},
		FilterFunc: func(c *generator.Context, t *types.Type) bool {
			tags := util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...))
			return tags.GenerateClient && tags.HasVerb("list") && tags.HasVerb("watch")
		},
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"encoding/json"
	"fmt"
	"os"
	"strconv"
	"strings"

	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/driver"
	"k8s.io/minikube/pkg/minikube/exit"
	"k8s.io/minikube/pkg/minikube/machine"
	"k8s.io/minikube/pkg/minikube/out"
	"k8s.io/minikube/pkg/minikube/reason"
	"k8s.io/minikube/pkg/minikube/style"

	"github.com/docker/machine/libmachine"
	"github.com/docker/machine/libmachine/state"
	"github.com/olekukonko/tablewriter"
	"github.com/spf13/cobra"

	"k8s.io/klog/v2"
)

var output string
var isLight bool

var profileListCmd = &cobra.Command{
	Use:   "list",
	Short: "Lists all minikube profiles.",
	Long:  "Lists all valid minikube profiles and detects all possible invalid profiles.",
	Run: func(cmd *cobra.Command, args []string) {
		switch strings.ToLower(output) {
		case "json":
			printProfilesJSON()
		case "table":
			printProfilesTable()
		default:
			exit.Message(reason.Usage, fmt.Sprintf("invalid output format: %s. Valid values: 'table', 'json'", output))
		}
	},
}

func listProfiles() (validProfiles, invalidProfiles []*config.Profile, err error) {
	if isLight {
		validProfiles, err = config.ListValidProfiles()
	} else {
		validProfiles, invalidProfiles, err = config.ListProfiles()
	}

	return validProfiles, invalidProfiles, err
}

func printProfilesTable() {
	validProfiles, invalidProfiles, err := listProfiles()

	if err != nil {
		klog.Warningf("error loading profiles: %v", err)
	}

	if len(validProfiles) == 0 {
		exit.Message(reason.UsageNoProfileRunning, "No minikube profile was found. ")
	}

	updateProfilesStatus(validProfiles)
	renderProfilesTable(profilesToTableData(validProfiles))
	warnInvalidProfiles(invalidProfiles)
}

func updateProfilesStatus(profiles []*config.Profile) {
	if isLight {
		for _, p := range profiles {
			p.Status = "Skipped"
		}
		return
	}

	api, err := machine.NewAPIClient()
	if err != nil {
		klog.Errorf("failed to get machine api client %v", err)
	}
	defer api.Close()

	for _, p := range profiles {
		p.Status = profileStatus(p, api)
	}
}

func profileStatus(p *config.Profile, api libmachine.API) string {
	cp, err := config.PrimaryControlPlane(p.Config)
	if err != nil {
		exit.Error(reason.GuestCpConfig, "error getting primary control plane", err)
	}

	host, err := machine.LoadHost(api, config.MachineName(*p.Config, cp))
	if err != nil {
		klog.Warningf("error loading profiles: %v", err)
		return "Unknown"
	}

	// The machine isn't running, no need to check inside
	s, err := host.Driver.GetState()
	if err != nil {
		klog.Warningf("error getting host state: %v", err)
		return "Unknown"
	}
	if s != state.Running {
		return s.String()
	}

	cr, err := machine.CommandRunner(host)
	if err != nil {
		klog.Warningf("error loading profiles: %v", err)
		return "Unknown"
	}

	hostname, _, port, err := driver.ControlPlaneEndpoint(p.Config, &cp, host.DriverName)
	if err != nil {
		klog.Warningf("error loading profiles: %v", err)
		return "Unknown"
	}

	status, err := kverify.APIServerStatus(cr, hostname, port)
	if err != nil {
		klog.Warningf("error getting apiserver status for %s: %v", p.Name, err)
		return "Unknown"
	}
	return status.String()
}

func renderProfilesTable(ps [][]string) {
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Profile", "Driver", "Runtime", "IP", "Port", "Version", "Status", "Nodes"})
	table.SetAutoFormatHeaders(false)
	table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
	table.SetCenterSeparator("|")
	table.AppendBulk(ps)
	table.Render()
}

func profilesToTableData(profiles []*config.Profile) [][]string {
	var data [][]string
	for _, p := range profiles {
		cp, err := config.PrimaryControlPlane(p.Config)
		if err != nil {
			exit.Error(reason.GuestCpConfig, "error getting primary control plane", err)
		}

		data = append(data, []string{p.Name, p.Config.Driver, p.Config.KubernetesConfig.ContainerRuntime, cp.IP, strconv.Itoa(cp.Port), p.Config.KubernetesConfig.KubernetesVersion, p.Status, strconv.Itoa(len(p.Config.Nodes))})
	}
	return data
}

func warnInvalidProfiles(invalidProfiles []*config.Profile) {
	if invalidProfiles == nil {
		return
	}

	out.WarningT("Found {{.number}} invalid profile(s) ! ", out.V{"number": len(invalidProfiles)})
	for _, p := range invalidProfiles {
		out.ErrT(style.Empty, "\t "+p.Name)
	}

	out.ErrT(style.Tip, "You can delete them using the following command(s): ")
	for _, p := range invalidProfiles {
		out.Err(fmt.Sprintf("\t $ minikube delete -p %s \n", p.Name))
	}
}

func printProfilesJSON() {
	validProfiles, invalidProfiles, err := listProfiles()

	updateProfilesStatus(validProfiles)

	var body = map[string]interface{}{}
	if err == nil || config.IsNotExist(err) {
		body["valid"] = profilesOrDefault(validProfiles)
		body["invalid"] = profilesOrDefault(invalidProfiles)
		jsonString, _ := json.Marshal(body)
		out.String(string(jsonString))
	} else {
		body["error"] = err
		jsonString, _ := json.Marshal(body)
		out.String(string(jsonString))
		os.Exit(reason.ExGuestError)
	}
}

func profilesOrDefault(profiles []*config.Profile) []*config.Profile {
	if profiles != nil {
		return profiles
	}
	return []*config.Profile{}
}

func init() {
	profileListCmd.Flags().StringVarP(&output, "output", "o", "table", "The output format. One of 'json', 'table'")
	profileListCmd.Flags().BoolVarP(&isLight, "light", "l", false, "If true, returns list of profiles faster by skipping validating the status of the cluster.")
	ProfileCmd.AddCommand(profileListCmd)
}
<commit_msg>revert unrelated change<commit_after>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"encoding/json"
	"fmt"
	"os"
	"strconv"
	"strings"

	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/driver"
	"k8s.io/minikube/pkg/minikube/exit"
	"k8s.io/minikube/pkg/minikube/machine"
	"k8s.io/minikube/pkg/minikube/out"
	"k8s.io/minikube/pkg/minikube/reason"
	"k8s.io/minikube/pkg/minikube/style"

	"github.com/docker/machine/libmachine"
	"github.com/docker/machine/libmachine/state"
	"github.com/olekukonko/tablewriter"
	"github.com/spf13/cobra"

	"k8s.io/klog/v2"
)

var output string
var isLight bool

var profileListCmd = &cobra.Command{
	Use:   "list",
	Short: "Lists all minikube profiles.",
	Long:  "Lists all valid minikube profiles and detects all possible invalid profiles.",
	Run: func(cmd *cobra.Command, args []string) {
		switch strings.ToLower(output) {
		case "json":
			printProfilesJSON()
		case "table":
			printProfilesTable()
		default:
			exit.Message(reason.Usage, fmt.Sprintf("invalid output format: %s. Valid values: 'table', 'json'", output))
		}
	},
}

func listProfiles() (validProfiles, invalidProfiles []*config.Profile, err error) {
	if isLight {
		validProfiles, err = config.ListValidProfiles()
	} else {
		validProfiles, invalidProfiles, err = config.ListProfiles()
	}

	return validProfiles, invalidProfiles, err
}

func printProfilesTable() {
	validProfiles, invalidProfiles, err := listProfiles()

	if err != nil {
		klog.Warningf("error loading profiles: %v", err)
	}

	if len(validProfiles) == 0 {
		exit.Message(reason.UsageNoProfileRunning, "No minikube profile was found. ")
	}

	updateProfilesStatus(validProfiles)
	renderProfilesTable(profilesToTableData(validProfiles))
	warnInvalidProfiles(invalidProfiles)
}

func updateProfilesStatus(profiles []*config.Profile) {
	if isLight {
		for _, p := range profiles {
			p.Status = "Skipped"
		}
		return
	}

	api, err := machine.NewAPIClient()
	if err != nil {
		klog.Errorf("failed to get machine api client %v", err)
	}
	defer api.Close()

	for _, p := range profiles {
		p.Status = profileStatus(p, api)
	}
}

func profileStatus(p *config.Profile, api libmachine.API) string {
	cp, err := config.PrimaryControlPlane(p.Config)
	if err != nil {
		exit.Error(reason.GuestCpConfig, "error getting primary control plane", err)
	}

	host, err := machine.LoadHost(api, config.MachineName(*p.Config, cp))
	if err != nil {
		klog.Warningf("error loading profiles: %v", err)
		return "Unknown"
	}

	// The machine isn't running, no need to check inside
	s, err := host.Driver.GetState()
	if err != nil {
		klog.Warningf("error getting host state: %v", err)
		return "Unknown"
	}
	if s != state.Running {
		return s.String()
	}

	cr, err := machine.CommandRunner(host)
	if err != nil {
		klog.Warningf("error loading profiles: %v", err)
		return "Unknown"
	}

	hostname, _, port, err := driver.ControlPlaneEndpoint(p.Config, &cp, host.DriverName)
	if err != nil {
		klog.Warningf("error loading profiles: %v", err)
		return "Unknown"
	}

	status, err := kverify.APIServerStatus(cr, hostname, port)
	if err != nil {
		klog.Warningf("error getting apiserver status for %s: %v", p.Name, err)
		return "Unknown"
	}
	return status.String()
}

func renderProfilesTable(ps [][]string) {
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Profile", "VM Driver", "Runtime", "IP", "Port", "Version", "Status", "Nodes"})
	table.SetAutoFormatHeaders(false)
	table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
	table.SetCenterSeparator("|")
	table.AppendBulk(ps)
	table.Render()
}

func profilesToTableData(profiles []*config.Profile) [][]string {
	var data [][]string
	for _, p := range profiles {
		cp, err := config.PrimaryControlPlane(p.Config)
		if err != nil {
			exit.Error(reason.GuestCpConfig, "error getting primary control plane", err)
		}

		data = append(data, []string{p.Name, p.Config.Driver, p.Config.KubernetesConfig.ContainerRuntime, cp.IP, strconv.Itoa(cp.Port), p.Config.KubernetesConfig.KubernetesVersion, p.Status, strconv.Itoa(len(p.Config.Nodes))})
	}
	return data
}

func warnInvalidProfiles(invalidProfiles []*config.Profile) {
	if invalidProfiles == nil {
		return
	}

	out.WarningT("Found {{.number}} invalid profile(s) ! ", out.V{"number": len(invalidProfiles)})
	for _, p := range invalidProfiles {
		out.ErrT(style.Empty, "\t "+p.Name)
	}

	out.ErrT(style.Tip, "You can delete them using the following command(s): ")
	for _, p := range invalidProfiles {
		out.Err(fmt.Sprintf("\t $ minikube delete -p %s \n", p.Name))
	}
}

func printProfilesJSON() {
	validProfiles, invalidProfiles, err := listProfiles()

	updateProfilesStatus(validProfiles)

	var body = map[string]interface{}{}
	if err == nil || config.IsNotExist(err) {
		body["valid"] = profilesOrDefault(validProfiles)
		body["invalid"] = profilesOrDefault(invalidProfiles)
		jsonString, _ := json.Marshal(body)
		out.String(string(jsonString))
	} else {
		body["error"] = err
		jsonString, _ := json.Marshal(body)
		out.String(string(jsonString))
		os.Exit(reason.ExGuestError)
	}
}

func profilesOrDefault(profiles []*config.Profile) []*config.Profile {
	if profiles != nil {
		return profiles
	}
	return []*config.Profile{}
}

func init() {
	profileListCmd.Flags().StringVarP(&output, "output", "o", "table", "The output format. One of 'json', 'table'")
	profileListCmd.Flags().BoolVarP(&isLight, "light", "l", false, "If true, returns list of profiles faster by skipping validating the status of the cluster.")
	ProfileCmd.AddCommand(profileListCmd)
}
<|endoftext|>


<commit_before>package worker

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net/http"
	"sync"
	"time"

	gocontext "context"

	"github.com/Sirupsen/logrus"
	"github.com/jtacoma/uritemplates"
	"github.com/pkg/errors"
	"github.com/travis-ci/worker/context"
)

type httpLogPart struct {
	JobID   uint64 `json:"id"`
	Content string `json:"log"`
	Number  int    `json:"number"`
	UUID    string `json:"uuid"`
	Final   bool   `json:"final"`
}

type httpLogWriter struct {
	ctx   gocontext.Context
	jobID uint64

	closeChan chan struct{}

	bufferMutex   sync.Mutex
	buffer        *bytes.Buffer
	logPartNumber int

	bytesWritten int
	maxLength    int

	httpClient *http.Client
	baseURL    string
	authToken  string

	timer   *time.Timer
	timeout time.Duration
}

func newHTTPLogWriter(ctx gocontext.Context, url string, authToken string, jobID uint64, timeout time.Duration) (*httpLogWriter, error) {
	writer := &httpLogWriter{
		ctx:        context.FromComponent(ctx, "log_writer"),
		jobID:      jobID,
		closeChan:  make(chan struct{}),
		buffer:     new(bytes.Buffer),
		timer:      time.NewTimer(time.Hour),
		timeout:    timeout,
		httpClient: &http.Client{},
		baseURL:    url,
		authToken:  authToken,
	}

	go writer.flushRegularly(ctx)

	return writer, nil
}

func (w *httpLogWriter) Write(p []byte) (int, error) {
	if w.closed() {
		return 0, fmt.Errorf("attempted write to closed log")
	}

	logger := context.LoggerFromContext(w.ctx).WithField("self", "http_log_writer")

	logger.WithFields(logrus.Fields{
		"length": len(p),
		"bytes":  string(p),
	}).Debug("writing bytes")

	w.timer.Reset(w.timeout)

	w.bytesWritten += len(p)
	if w.bytesWritten > w.maxLength {
		_, err := w.WriteAndClose([]byte(fmt.Sprintf("\n\nThe log length has exceeded the limit of %d MB (this usually means that the test suite is raising the same exception over and over).\n\nThe job has been terminated\n", w.maxLength/1000/1000)))
		if err != nil {
			logger.WithField("err", err).Error("couldn't write 'log length exceeded' error message to log")
		}
		return 0, ErrWrotePastMaxLogLength
	}

	w.bufferMutex.Lock()
	defer w.bufferMutex.Unlock()
	return w.buffer.Write(p)
}

func (w *httpLogWriter) Close() error {
	if w.closed() {
		return nil
	}

	w.timer.Stop()

	close(w.closeChan)
	w.flush()

	part := httpLogPart{
		JobID:  w.jobID,
		Number: w.logPartNumber,
		Final:  true,
	}
	w.logPartNumber++
	return w.publishLogPart(part)
}

func (w *httpLogWriter) Timeout() <-chan time.Time {
	return w.timer.C
}

func (w *httpLogWriter) SetMaxLogLength(bytes int) {
	w.maxLength = bytes
}

func (w *httpLogWriter) WriteAndClose(p []byte) (int, error) {
	if w.closed() {
		return 0, fmt.Errorf("log already closed")
	}

	w.timer.Stop()

	close(w.closeChan)

	w.bufferMutex.Lock()
	n, err := w.buffer.Write(p)
	w.bufferMutex.Unlock()
	if err != nil {
		return n, err
	}

	w.flush()

	part := httpLogPart{
		JobID:  w.jobID,
		Number: w.logPartNumber,
		Final:  true,
	}
	w.logPartNumber++

	err = w.publishLogPart(part)
	return n, err
}

func (w *httpLogWriter) closed() bool {
	select {
	case <-w.closeChan:
		return true
	default:
		return false
	}
}

func (w *httpLogWriter) flushRegularly(ctx gocontext.Context) {
	ticker := time.NewTicker(LogWriterTick)
	defer ticker.Stop()
	for {
		select {
		case <-w.closeChan:
			return
		case <-ticker.C:
			w.flush()
		case <-ctx.Done():
			return
		}
	}
}

func (w *httpLogWriter) flush() {
	if w.buffer.Len() <= 0 {
		return
	}

	buf := make([]byte, LogChunkSize)

	for w.buffer.Len() > 0 {
		w.bufferMutex.Lock()
		n, err := w.buffer.Read(buf)
		w.bufferMutex.Unlock()
		if err != nil {
			// According to documentation, err should only be non-nil if
			// there's no data in the buffer. We've checked for this, so
			// this means that err should never be nil. Something is very
			// wrong if this happens, so let's abort!
			panic("non-empty buffer shouldn't return an error on Read")
		}

		part := httpLogPart{
			JobID:   w.jobID,
			Content: string(buf[0:n]),
			Number:  w.logPartNumber,
		}
		w.logPartNumber++

		err = w.publishLogPart(part)
		if err != nil {
			context.LoggerFromContext(w.ctx).WithFields(logrus.Fields{
				"err":  err,
				"self": "http_log_writer",
			}).Error("couldn't publish log part")
		}
	}
}

func (w *httpLogWriter) publishLogPart(part httpLogPart) error {
	part.UUID, _ = context.UUIDFromContext(w.ctx)

	type logPartPayload struct {
		Type     string `json:"@type"`
		Final    bool   `json:"final"`
		Content  string `json:"content"`
		Encoding string `json:"encoding"`
	}

	payload := logPartPayload{
		Type:     "log_part",
		Final:    part.Final,
		Content:  base64.StdEncoding.EncodeToString([]byte(part.Content)),
		Encoding: "base64",
	}

	template, err := uritemplates.Parse(w.baseURL)
	if err != nil {
		return errors.Wrap(err, "couldn't parse base URL template")
	}

	partURL, err := template.Expand(map[string]interface{}{
		"job_id":      w.jobID,
		"log_part_id": part.Number,
	})
	if err != nil {
		return errors.Wrap(err, "couldn't expand base URL template")
	}

	partBody, err := json.Marshal(payload)
	if err != nil {
		return errors.Wrap(err, "couldn't marshal JSON")
	}

	req, err := http.NewRequest("PUT", partURL, bytes.NewReader(partBody))
	if err != nil {
		return errors.Wrap(err, "couldn't create request")
	}

	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", w.authToken))

	resp, err := w.httpClient.Do(req)
	if err != nil {
		return errors.Wrap(err, "error making request")
	}

	if resp.StatusCode != http.StatusNoContent {
		return errors.Errorf("expected %d but got %d", http.StatusNoContent, resp.StatusCode)
	}

	return nil
}
<commit_msg>Batch together HTTP log parts writes<commit_after>package worker

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net/http"
	"sync"
	"time"

	gocontext "context"

	"github.com/Sirupsen/logrus"
	"github.com/jtacoma/uritemplates"
	"github.com/pkg/errors"
	"github.com/travis-ci/worker/context"
)

type httpLogPart struct {
	JobID   uint64 `json:"id"`
	Content string `json:"log"`
	Number  int    `json:"number"`
	Final   bool   `json:"final"`
}

type httpLogWriter struct {
	ctx   gocontext.Context
	jobID uint64

	closeChan chan struct{}

	bufferMutex   sync.Mutex
	buffer        *bytes.Buffer
	logPartNumber int

	bytesWritten int
	maxLength    int
	maxBatchSize int

	httpClient *http.Client
	baseURL    string
	authToken  string

	timer   *time.Timer
	timeout time.Duration
}

type httpLogPartPayload struct {
	Type     string `json:"@type"`
	Final    bool   `json:"final"`
	Content  string `json:"content"`
	Encoding string `json:"encoding"`
}

func newHTTPLogWriter(ctx gocontext.Context, url string, authToken string, jobID uint64, timeout time.Duration) (*httpLogWriter, error) {
	writer := &httpLogWriter{
		ctx:          context.FromComponent(ctx, "log_writer"),
		jobID:        jobID,
		closeChan:    make(chan struct{}),
		buffer:       new(bytes.Buffer),
		timer:        time.NewTimer(time.Hour),
		timeout:      timeout,
		maxBatchSize: 16,
		httpClient:   &http.Client{},
		baseURL:      url,
		authToken:    authToken,
	}

	go writer.flushRegularly(ctx)

	return writer, nil
}

func (w *httpLogWriter) Write(p []byte) (int, error) {
	if w.closed() {
		return 0, fmt.Errorf("attempted write to closed log")
	}

	logger := context.LoggerFromContext(w.ctx).WithField("self", "http_log_writer")

	logger.WithFields(logrus.Fields{
		"length": len(p),
		"bytes":  string(p),
	}).Debug("writing bytes")

	w.timer.Reset(w.timeout)

	w.bytesWritten += len(p)
	if w.bytesWritten > w.maxLength {
		_, err := w.WriteAndClose([]byte(fmt.Sprintf("\n\nThe log length has exceeded the limit of %d MB (this usually means that the test suite is raising the same exception over and over).\n\nThe job has been terminated\n", w.maxLength/1000/1000)))
		if err != nil {
			logger.WithField("err", err).Error("couldn't write 'log length exceeded' error message to log")
		}
		return 0, ErrWrotePastMaxLogLength
	}

	w.bufferMutex.Lock()
	defer w.bufferMutex.Unlock()
	return w.buffer.Write(p)
}

func (w *httpLogWriter) Close() error {
	if w.closed() {
		return nil
	}

	w.timer.Stop()

	close(w.closeChan)
	w.flush()

	part := &httpLogPart{
		JobID:  w.jobID,
		Number: w.logPartNumber,
		Final:  true,
	}
	w.logPartNumber++
	return w.publishLogPart(part)
}

func (w *httpLogWriter) Timeout() <-chan time.Time {
	return w.timer.C
}

func (w *httpLogWriter) SetMaxLogLength(bytes int) {
	w.maxLength = bytes
}

func (w *httpLogWriter) WriteAndClose(p []byte) (int, error) {
	if w.closed() {
		return 0, fmt.Errorf("log already closed")
	}

	w.timer.Stop()

	close(w.closeChan)

	w.bufferMutex.Lock()
	n, err := w.buffer.Write(p)
	w.bufferMutex.Unlock()
	if err != nil {
		return n, err
	}

	w.flush()

	part := &httpLogPart{
		JobID:  w.jobID,
		Number: w.logPartNumber,
		Final:  true,
	}
	w.logPartNumber++

	err = w.publishLogPart(part)
	return n, err
}

func (w *httpLogWriter) closed() bool {
	select {
	case <-w.closeChan:
		return true
	default:
		return false
	}
}

func (w *httpLogWriter) flushRegularly(ctx gocontext.Context) {
	ticker := time.NewTicker(LogWriterTick)
	defer ticker.Stop()
	for {
		select {
		case <-w.closeChan:
			return
		case <-ticker.C:
			w.flush()
		case <-ctx.Done():
			return
		}
	}
}

func (w *httpLogWriter) flush() {
	if w.buffer.Len() <= 0 {
		return
	}

	logPartsPayload := []*httpLogPart{}
	buf := make([]byte, LogChunkSize)

	for w.buffer.Len() > 0 {
		w.bufferMutex.Lock()
		n, err := w.buffer.Read(buf)
		w.bufferMutex.Unlock()
		if err != nil {
			// According to documentation, err should only be non-nil if
			// there's no data in the buffer. We've checked for this, so
			// this means that err should never be nil. Something is very
			// wrong if this happens, so let's abort!
			panic("non-empty buffer shouldn't return an error on Read")
		}

		logPartsPayload = append(logPartsPayload, &httpLogPart{
			JobID:   w.jobID,
			Content: string(buf[0:n]),
			Number:  w.logPartNumber,
		})
		w.logPartNumber++

		if len(logPartsPayload) >= w.maxBatchSize {
			w.logFlushError(w.publishLogParts(logPartsPayload))
			logPartsPayload = []*httpLogPart{}
		}
	}

	w.logFlushError(w.publishLogParts(logPartsPayload))
}

func (w *httpLogWriter) logFlushError(err error) {
	if err == nil {
		return
	}

	context.LoggerFromContext(w.ctx).WithFields(logrus.Fields{
		"err":  err,
		"self": "http_log_writer",
	}).Error("couldn't publish log parts")
}

func (w *httpLogWriter) publishLogPart(part *httpLogPart) error {
	payload := &httpLogPartPayload{
		Type:     "log_part",
		Final:    part.Final,
		Content:  base64.StdEncoding.EncodeToString([]byte(part.Content)),
		Encoding: "base64",
	}

	return w.publishLogPartOrParts(payload, "PUT", fmt.Sprintf("%d", part.Number))
}

func (w *httpLogWriter) publishLogParts(parts []*httpLogPart) error {
	payload := []*httpLogPartPayload{}

	for _, part := range parts {
		payload = append(payload, &httpLogPartPayload{
			Type:     "log_part",
			Final:    part.Final,
			Content:  base64.StdEncoding.EncodeToString([]byte(part.Content)),
			Encoding: "base64",
		})
	}

	return w.publishLogPartOrParts(payload, "POST", "multi")
}

func (w *httpLogWriter) publishLogPartOrParts(payload interface{}, publishMethod, payloadID string) error {
	template, err := uritemplates.Parse(w.baseURL)
	if err != nil {
		return errors.Wrap(err, "couldn't parse base URL template")
	}

	partURL, err := template.Expand(map[string]interface{}{
		"job_id":      w.jobID,
		"log_part_id": payloadID,
	})
	if err != nil {
		return errors.Wrap(err, "couldn't expand base URL template")
	}

	partBody, err := json.Marshal(payload)
	if err != nil {
		return errors.Wrap(err, "couldn't marshal JSON")
	}

	req, err := http.NewRequest(publishMethod, partURL, bytes.NewReader(partBody))
	if err != nil {
		return errors.Wrap(err, "couldn't create request")
	}

	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", w.authToken))

	resp, err := w.httpClient.Do(req)
	if err != nil {
		return errors.Wrap(err, "error making request")
	}

	if resp.StatusCode != http.StatusNoContent {
		return errors.Errorf("expected %d but got %d", http.StatusNoContent, resp.StatusCode)
	}

	return nil
}
<|endoftext|>


<commit_before>package httpapi

import (
	"encoding/json"
	"github.com/boivie/lovebeat-go/service"
	"github.com/gorilla/mux"
	"github.com/op/go-logging"
	"io"
	"net/http"
	"strconv"
	"time"
)

var (
	svcs   *service.Services
	client service.ServiceIf
)

func now() int64 { return time.Now().Unix() }

var log = logging.MustGetLogger("lovebeat")

func ServiceHandler(c http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	name := params["name"]

	var err = r.ParseForm()
	if err != nil {
		log.Error("error parsing form ", err)
		return
	}

	var errtmo, warntmo = r.FormValue("err-tmo"), r.FormValue("warn-tmo")

	client.Beat(name)

	if val, err := strconv.Atoi(errtmo); err == nil {
		client.SetErrorTimeout(name, val)
	}

	if val, err := strconv.Atoi(warntmo); err == nil {
		client.SetWarningTimeout(name, val)
	}

	c.Header().Add("Content-Type", "application/json")
	c.Header().Add("Content-Length", "3")
	io.WriteString(c, "{}\n")
}

func DeleteServiceHandler(c http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	name := params["name"]

	client.DeleteService(name)

	c.Header().Add("Content-Type", "application/json")
	c.Header().Add("Content-Length", "3")
	io.WriteString(c, "{}\n")
}

func CreateViewHandler(c http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	view_name := params["name"]
	var expr = r.FormValue("regexp")
	if expr == "" {
		log.Error("No regexp provided")
		return
	}

	client.CreateOrUpdateView(view_name, expr)

	c.Header().Add("Content-Type", "application/json")
	c.Header().Add("Content-Length", "3")
	io.WriteString(c, "{}\n")
}

func DeleteViewHandler(c http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	name := params["name"]

	client.DeleteView(name)

	c.Header().Add("Content-Type", "application/json")
	c.Header().Add("Content-Length", "3")
	io.WriteString(c, "{}\n")
}

type JsonView struct {
	Name  string `json:"name"`
	State string `json:"state"`
}

func GetViewsHandler(c http.ResponseWriter, r *http.Request) {
	var ret = make([]JsonView, 0)
	for _, v := range client.GetViews() {
		js := JsonView{
			Name:  v.Name,
			State: v.State,
		}
		ret = append(ret, js)
	}
	var encoded, _ = json.MarshalIndent(ret, "", "  ")

	c.Header().Add("Content-Type", "application/json")
	c.Header().Add("Content-Length", strconv.Itoa(len(encoded)+1))
	c.Write(encoded)
	io.WriteString(c, "\n")
}

type JsonService struct {
	Name           string `json:"name"`
	LastBeat       int64  `json:"last_beat"`
	LastBeatDelta  int64  `json:"last_beat_delta"`
	WarningTimeout int64  `json:"warning_timeout"`
	ErrorTimeout   int64  `json:"error_timeout"`
	State          string `json:"state"`
}

func GetServicesHandler(c http.ResponseWriter, r *http.Request) {
	viewName := "all"

	if val, ok := r.URL.Query()["view"]; ok {
		viewName = val[0]
	}
	var now = now()
	var ret = make([]JsonService, 0)
	for _, s := range client.GetServices(viewName) {
		js := JsonService{
			Name:           s.Name,
			LastBeat:       s.LastBeat,
			LastBeatDelta:  now - s.LastBeat,
			WarningTimeout: s.WarningTimeout,
			ErrorTimeout:   s.ErrorTimeout,
			State:          s.State,
		}
		ret = append(ret, js)
	}
	var encoded, _ = json.MarshalIndent(ret, "", "  ")

	c.Header().Add("Content-Type", "application/json")
	c.Header().Add("Content-Length", strconv.Itoa(len(encoded)+1))
	c.Write(encoded)
	io.WriteString(c, "\n")
}

func GetServiceHandler(c http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	name := params["name"]

	var now = now()
	var s = client.GetService(name)
	js := JsonService{
		Name:           s.Name,
		LastBeat:       s.LastBeat,
		LastBeatDelta:  now - s.LastBeat,
		WarningTimeout: s.WarningTimeout,
		ErrorTimeout:   s.ErrorTimeout,
		State:          s.State,
	}
	var encoded, _ = json.MarshalIndent(js, "", "  ")

	c.Header().Add("Content-Type", "application/json")
	c.Header().Add("Content-Length", strconv.Itoa(len(encoded)+1))
	c.Write(encoded)
	io.WriteString(c, "\n")
}

func Register(rtr *mux.Router, client_ service.ServiceIf) {
	client = client_
	rtr.HandleFunc("/api/services/",
		GetServicesHandler).Methods("GET")
	rtr.HandleFunc("/api/services/{name:[a-z0-9.-]+}",
		ServiceHandler).Methods("POST")
	rtr.HandleFunc("/api/services/{name:[a-z0-9.-]+}",
		GetServiceHandler).Methods("GET")
	rtr.HandleFunc("/api/services/{name:[a-z0-9.-]+}", DeleteServiceHandler).Methods("DELETE")
	rtr.HandleFunc("/api/views/", GetViewsHandler).Methods("GET")
	rtr.HandleFunc("/api/views/{name:[a-z0-9.-]+}", CreateViewHandler).Methods("POST")
	rtr.HandleFunc("/api/views/{name:[a-z0-9.-]+}", DeleteViewHandler).Methods("DELETE")
}
<commit_msg>httpapi: "Get Service" with details returns matching views<commit_after>package httpapi

import (
	"encoding/json"
	"github.com/boivie/lovebeat-go/service"
	"github.com/gorilla/mux"
	"github.com/op/go-logging"
	"io"
	"net/http"
	"regexp"
	"strconv"
	"time"
)

var (
	svcs   *service.Services
	client service.ServiceIf
)

func now() int64 { return time.Now().Unix() }

var log = logging.MustGetLogger("lovebeat")

func ServiceHandler(c http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	name := params["name"]

	var err = r.ParseForm()
	if err != nil {
		log.Error("error parsing form ", err)
		return
	}

	var errtmo, warntmo = r.FormValue("err-tmo"), r.FormValue("warn-tmo")

	client.Beat(name)

	if val, err := strconv.Atoi(errtmo); err == nil {
		client.SetErrorTimeout(name, val)
	}

	if val, err := strconv.Atoi(warntmo); err == nil {
		client.SetWarningTimeout(name, val)
	}

	c.Header().Add("Content-Type", "application/json")
	c.Header().Add("Content-Length", "3")
	io.WriteString(c, "{}\n")
}

func DeleteServiceHandler(c http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	name := params["name"]

	client.DeleteService(name)

	c.Header().Add("Content-Type", "application/json")
	c.Header().Add("Content-Length", "3")
	io.WriteString(c, "{}\n")
}

func CreateViewHandler(c http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	view_name := params["name"]
	var expr = r.FormValue("regexp")
	if expr == "" {
		log.Error("No regexp provided")
		return
	}

	client.CreateOrUpdateView(view_name, expr)

	c.Header().Add("Content-Type", "application/json")
	c.Header().Add("Content-Length", "3")
	io.WriteString(c, "{}\n")
}

func DeleteViewHandler(c http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	name := params["name"]

	client.DeleteView(name)

	c.Header().Add("Content-Type", "application/json")
	c.Header().Add("Content-Length", "3")
	io.WriteString(c, "{}\n")
}

type JsonView struct {
	Name  string `json:"name"`
	State string `json:"state"`
}

func GetViewsHandler(c http.ResponseWriter, r *http.Request) {
	var ret = make([]JsonView, 0)
	for _, v := range client.GetViews() {
		js := JsonView{
			Name:  v.Name,
			State: v.State,
		}
		ret = append(ret, js)
	}
	var encoded, _ = json.MarshalIndent(ret, "", "  ")

	c.Header().Add("Content-Type", "application/json")
	c.Header().Add("Content-Length", strconv.Itoa(len(encoded)+1))
	c.Write(encoded)
	io.WriteString(c, "\n")
}

type JsonViewRef struct {
	Name string `json:"name"`
}

type JsonService struct {
	Name           string        `json:"name"`
	LastBeat       int64         `json:"last_beat"`
	LastBeatDelta  int64         `json:"last_beat_delta"`
	WarningTimeout int64         `json:"warning_timeout"`
	ErrorTimeout   int64         `json:"error_timeout"`
	State          string        `json:"state"`
	Views          []JsonViewRef `json:"views,omitempty"`
}

func GetServicesHandler(c http.ResponseWriter, r *http.Request) {
	viewName := "all"

	if val, ok := r.URL.Query()["view"]; ok {
		viewName = val[0]
	}
	var now = now()
	var ret = make([]JsonService, 0)
	for _, s := range client.GetServices(viewName) {
		js := JsonService{
			Name:           s.Name,
			LastBeat:       s.LastBeat,
			LastBeatDelta:  now - s.LastBeat,
			WarningTimeout: s.WarningTimeout,
			ErrorTimeout:   s.ErrorTimeout,
			State:          s.State,
		}
		ret = append(ret, js)
	}
	var encoded, _ = json.MarshalIndent(ret, "", "  ")

	c.Header().Add("Content-Type", "application/json")
	c.Header().Add("Content-Length", strconv.Itoa(len(encoded)+1))
	c.Write(encoded)
	io.WriteString(c, "\n")
}

func GetServiceHandler(c http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	name := params["name"]

	var now = now()
	var s = client.GetService(name)
	js := JsonService{
		Name:           s.Name,
		LastBeat:       s.LastBeat,
		LastBeatDelta:  now - s.LastBeat,
		WarningTimeout: s.WarningTimeout,
		ErrorTimeout:   s.ErrorTimeout,
		State:          s.State,
	}

	if _, ok := r.URL.Query()["details"]; ok {
		js.Views = make([]JsonViewRef, 0)
		for _, view := range client.GetViews() {
			matched, err := regexp.MatchString(view.Regexp, s.Name)
			if err == nil && matched {
				js.Views = append(js.Views, JsonViewRef{Name: view.Name})
			}
		}
	}
	var encoded, _ = json.MarshalIndent(js, "", "  ")

	c.Header().Add("Content-Type", "application/json")
	c.Header().Add("Content-Length", strconv.Itoa(len(encoded)+1))
	c.Write(encoded)
	io.WriteString(c, "\n")
}

func Register(rtr *mux.Router, client_ service.ServiceIf) {
	client = client_
	rtr.HandleFunc("/api/services/",
		GetServicesHandler).Methods("GET")
	rtr.HandleFunc("/api/services/{name:[a-z0-9.-]+}",
		ServiceHandler).Methods("POST")
	rtr.HandleFunc("/api/services/{name:[a-z0-9.-]+}",
		GetServiceHandler).Methods("GET")
	rtr.HandleFunc("/api/services/{name:[a-z0-9.-]+}", DeleteServiceHandler).Methods("DELETE")
	rtr.HandleFunc("/api/views/", GetViewsHandler).Methods("GET")
	rtr.HandleFunc("/api/views/{name:[a-z0-9.-]+}", CreateViewHandler).Methods("POST")
	rtr.HandleFunc("/api/views/{name:[a-z0-9.-]+}", DeleteViewHandler).Methods("DELETE")
}
<|endoftext|>


<commit_before>package reviewdog

import (
	"context"
	"fmt"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"

	"github.com/xanzy/go-gitlab"
	"golang.org/x/sync/errgroup"
)

var _ CommentService = &GitLabMergeRequest{}
var _ DiffService = &GitLabMergeRequest{}

// GitLabMergeRequest is a comment and diff service for GitLab MergeRequest.
//
// API:
//  https://docs.gitlab.com/ce/api/commits.html#post-comment-to-commit
//  POST /projects/:id/repository/commits/:sha/comments
type GitLabMergeRequest struct {
	cli      *gitlab.Client
	owner    string
	repo     string
	pr       int
	sha      string
	projects string

	muComments   sync.Mutex
	postComments []*Comment

	postedcs postedcomments

	// wd is working directory relative to root of repository.
	wd string
}

// NewGitLabMergeReqest returns a new GitLabMergeRequest service.
// GitLabMergeRequest service needs git command in $PATH.
func NewGitLabMergeReqest(cli *gitlab.Client, owner, repo string, pr int, sha string) (*GitLabMergeRequest, error) {
	workDir, err := gitRelWorkdir()
	if err != nil {
		return nil, fmt.Errorf("GitLabMergeRequest needs 'git' command: %v", err)
	}
	return &GitLabMergeRequest{
		cli:      cli,
		owner:    owner,
		repo:     repo,
		pr:       pr,
		sha:      sha,
		projects: owner + "/" + repo,
		wd:       workDir,
	}, nil
}

// Post accepts a comment and holds it. Flush method actually posts comments to
// GitLab in parallel.
func (g *GitLabMergeRequest) Post(_ context.Context, c *Comment) error {
	c.Path = filepath.Join(g.wd, c.Path)
	g.muComments.Lock()
	defer g.muComments.Unlock()
	g.postComments = append(g.postComments, c)
	return nil
}

// Flush posts comments which has not been posted yet.
func (g *GitLabMergeRequest) Flush(ctx context.Context) error {
	defer g.muComments.Unlock()
	g.muComments.Lock()

	if err := g.setPostedComment(ctx); err != nil {
		return err
	}

	return g.postCommentsForEach(ctx)
}

func (g *GitLabMergeRequest) postCommentsForEach(ctx context.Context) error {
	var eg errgroup.Group
	for _, c := range g.postComments {
		comment := c
		if g.postedcs.IsPosted(comment, comment.Lnum) {
			continue
		}
		eg.Go(func() error {
			commitID, err := g.getLastCommitsID(comment.Path, comment.Lnum)
			if err != nil {
				commitID = g.sha
			}
			body := commentBody(comment)
			ltype := "new"
			prcomment := &gitlab.PostCommitCommentOptions{
				Note:     &body,
				Path:     &comment.Path,
				Line:     &comment.Lnum,
				LineType: &ltype,
			}
			_, _, err = g.cli.Commits.PostCommitComment(g.projects, commitID, prcomment, nil)
			return err
		})
	}
	return eg.Wait()
}

func (g *GitLabMergeRequest) getLastCommitsID(path string, line int) (string, error) {
	lineFormat := fmt.Sprintf("%d,%d", line, line)
	s, err := exec.Command("git", "blame", "-l", "-L", lineFormat, path).Output()
	if err != nil {
		return "", fmt.Errorf("failed to get commitID: %v", err)
	}
	commitID := strings.Split(string(s), " ")[0]
	return commitID, nil
}

func (g *GitLabMergeRequest) setPostedComment(ctx context.Context) error {
	g.postedcs = make(postedcomments)
	cs, err := g.comment(ctx)
	if err != nil {
		return err
	}
	for _, c := range cs {
		if c.Line == 0 || c.Path == "" || c.Note == "" {
			// skip resolved comments. Or comments which do not have "path" nor
			// "body".
			continue
		}
		path := c.Path
		pos := c.Line
		body := c.Note
		if _, ok := g.postedcs[path]; !ok {
			g.postedcs[path] = make(map[int][]string)
		}
		if _, ok := g.postedcs[path][pos]; !ok {
			g.postedcs[path][pos] = make([]string, 0)
		}
		g.postedcs[path][pos] = append(g.postedcs[path][pos], body)
	}
	return nil
}

// Diff returns a diff of MergeRequest. It runs `git diff` locally instead of
// diff_url of GitLab Pull Request because diff of diff_url is not suited for
// comment API in a sense that diff of diff_url is equivalent to
// `git diff --no-renames`, we want diff which is equivalent to
// `git diff --find-renames`.
func (g *GitLabMergeRequest) Diff(ctx context.Context) ([]byte, error) {
	mr, _, err := g.cli.MergeRequests.GetMergeRequest(g.projects, g.pr, nil)
	if err != nil {
		return nil, err
	}
	targetBranch, _, err := g.cli.Branches.GetBranch(mr.TargetProjectID, mr.TargetBranch, nil)
	if err != nil {
		return nil, err
	}
	return g.gitDiff(ctx, g.sha, targetBranch.Commit.ID)
}

func (g *GitLabMergeRequest) gitDiff(ctx context.Context, baseSha string, targetSha string) ([]byte, error) {
	b, err := exec.Command("git", "merge-base", targetSha, baseSha).Output()
	if err != nil {
		return nil, fmt.Errorf("failed to get merge-base commit: %v", err)
	}
	mergeBase := strings.Trim(string(b), "\n")
	relArg := fmt.Sprintf("--relative=%s", g.wd)
	bytes, err := exec.Command("git", "diff", relArg, "--find-renames", mergeBase, baseSha).Output()
	if err != nil {
		return nil, fmt.Errorf("failed to run git diff: %v", err)
	}
	return bytes, nil
}

// Strip returns 1 as a strip of git diff.
func (g *GitLabMergeRequest) Strip() int {
	return 1
}

func (g *GitLabMergeRequest) comment(ctx context.Context) ([]*gitlab.CommitComment, error) {
	commits, _, err := g.cli.MergeRequests.GetMergeRequestCommits(g.projects, g.pr, nil)
	if err != nil {
		return nil, err
	}
	comments := make([]*gitlab.CommitComment, 0)
	for _, c := range commits {
		tmpComments, _, err := g.cli.Commits.GetCommitComments(g.projects, c.ID, nil)
		if err != nil {
			continue
		}
		comments = append(comments, tmpComments...)
	}
	return comments, nil
}
<commit_msg>reword: s/GitLab Pull Request/GitLab Merge Request/<commit_after>package reviewdog

import (
	"context"
	"fmt"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"

	"github.com/xanzy/go-gitlab"
	"golang.org/x/sync/errgroup"
)

var _ CommentService = &GitLabMergeRequest{}
var _ DiffService = &GitLabMergeRequest{}

// GitLabMergeRequest is a comment and diff service for GitLab MergeRequest.
//
// API:
//  https://docs.gitlab.com/ce/api/commits.html#post-comment-to-commit
//  POST /projects/:id/repository/commits/:sha/comments
type GitLabMergeRequest struct {
	cli      *gitlab.Client
	owner    string
	repo     string
	pr       int
	sha      string
	projects string

	muComments   sync.Mutex
	postComments []*Comment

	postedcs postedcomments

	// wd is working directory relative to root of repository.
	wd string
}

// NewGitLabMergeReqest returns a new GitLabMergeRequest service.
// GitLabMergeRequest service needs git command in $PATH.
func NewGitLabMergeReqest(cli *gitlab.Client, owner, repo string, pr int, sha string) (*GitLabMergeRequest, error) {
	workDir, err := gitRelWorkdir()
	if err != nil {
		return nil, fmt.Errorf("GitLabMergeRequest needs 'git' command: %v", err)
	}
	return &GitLabMergeRequest{
		cli:      cli,
		owner:    owner,
		repo:     repo,
		pr:       pr,
		sha:      sha,
		projects: owner + "/" + repo,
		wd:       workDir,
	}, nil
}

// Post accepts a comment and holds it. Flush method actually posts comments to
// GitLab in parallel.
func (g *GitLabMergeRequest) Post(_ context.Context, c *Comment) error {
	c.Path = filepath.Join(g.wd, c.Path)
	g.muComments.Lock()
	defer g.muComments.Unlock()
	g.postComments = append(g.postComments, c)
	return nil
}

// Flush posts comments which has not been posted yet.
func (g *GitLabMergeRequest) Flush(ctx context.Context) error {
	defer g.muComments.Unlock()
	g.muComments.Lock()

	if err := g.setPostedComment(ctx); err != nil {
		return err
	}

	return g.postCommentsForEach(ctx)
}

func (g *GitLabMergeRequest) postCommentsForEach(ctx context.Context) error {
	var eg errgroup.Group
	for _, c := range g.postComments {
		comment := c
		if g.postedcs.IsPosted(comment, comment.Lnum) {
			continue
		}
		eg.Go(func() error {
			commitID, err := g.getLastCommitsID(comment.Path, comment.Lnum)
			if err != nil {
				commitID = g.sha
			}
			body := commentBody(comment)
			ltype := "new"
			prcomment := &gitlab.PostCommitCommentOptions{
				Note:     &body,
				Path:     &comment.Path,
				Line:     &comment.Lnum,
				LineType: &ltype,
			}
			_, _, err = g.cli.Commits.PostCommitComment(g.projects, commitID, prcomment, nil)
			return err
		})
	}
	return eg.Wait()
}

func (g *GitLabMergeRequest) getLastCommitsID(path string, line int) (string, error) {
	lineFormat := fmt.Sprintf("%d,%d", line, line)
	s, err := exec.Command("git", "blame", "-l", "-L", lineFormat, path).Output()
	if err != nil {
		return "", fmt.Errorf("failed to get commitID: %v", err)
	}
	commitID := strings.Split(string(s), " ")[0]
	return commitID, nil
}

func (g *GitLabMergeRequest) setPostedComment(ctx context.Context) error {
	g.postedcs = make(postedcomments)
	cs, err := g.comment(ctx)
	if err != nil {
		return err
	}
	for _, c := range cs {
		if c.Line == 0 || c.Path == "" || c.Note == "" {
			// skip resolved comments. Or comments which do not have "path" nor
			// "body".
			continue
		}
		path := c.Path
		pos := c.Line
		body := c.Note
		if _, ok := g.postedcs[path]; !ok {
			g.postedcs[path] = make(map[int][]string)
		}
		if _, ok := g.postedcs[path][pos]; !ok {
			g.postedcs[path][pos] = make([]string, 0)
		}
		g.postedcs[path][pos] = append(g.postedcs[path][pos], body)
	}
	return nil
}

// Diff returns a diff of MergeRequest. It runs `git diff` locally instead of
// diff_url of GitLab Merge Request because diff of diff_url is not suited for
// comment API in a sense that diff of diff_url is equivalent to
// `git diff --no-renames`, we want diff which is equivalent to
// `git diff --find-renames`.
func (g *GitLabMergeRequest) Diff(ctx context.Context) ([]byte, error) {
	mr, _, err := g.cli.MergeRequests.GetMergeRequest(g.projects, g.pr, nil)
	if err != nil {
		return nil, err
	}
	targetBranch, _, err := g.cli.Branches.GetBranch(mr.TargetProjectID, mr.TargetBranch, nil)
	if err != nil {
		return nil, err
	}
	return g.gitDiff(ctx, g.sha, targetBranch.Commit.ID)
}

func (g *GitLabMergeRequest) gitDiff(ctx context.Context, baseSha string, targetSha string) ([]byte, error) {
	b, err := exec.Command("git", "merge-base", targetSha, baseSha).Output()
	if err != nil {
		return nil, fmt.Errorf("failed to get merge-base commit: %v", err)
	}
	mergeBase := strings.Trim(string(b), "\n")
	relArg := fmt.Sprintf("--relative=%s", g.wd)
	bytes, err := exec.Command("git", "diff", relArg, "--find-renames", mergeBase, baseSha).Output()
	if err != nil {
		return nil, fmt.Errorf("failed to run git diff: %v", err)
	}
	return bytes, nil
}

// Strip returns 1 as a strip of git diff.
func (g *GitLabMergeRequest) Strip() int {
	return 1
}

func (g *GitLabMergeRequest) comment(ctx context.Context) ([]*gitlab.CommitComment, error) {
	commits, _, err := g.cli.MergeRequests.GetMergeRequestCommits(g.projects, g.pr, nil)
	if err != nil {
		return nil, err
	}
	comments := make([]*gitlab.CommitComment, 0)
	for _, c := range commits {
		tmpComments, _, err := g.cli.Commits.GetCommitComments(g.projects, c.ID, nil)
		if err != nil {
			continue
		}
		comments = append(comments, tmpComments...)
	}
	return comments, nil
}
<|endoftext|>


<commit_before>package hwaflib

func (ctx *Context) Version() string {
	version := "20130819"
	return version
}

func (ctx *Context) Revision() string {
	revision := "c980dad"
	return revision
}

// EOF

<commit_msg>version: 20130819<commit_after>package hwaflib

func (ctx *Context) Version() string {
	version := "20130819"
	return version
}

func (ctx *Context) Revision() string {
	revision := "ad0d582"
	return revision
}

// EOF

<|endoftext|>


<commit_before>package godong

import (
	"fmt"
	"net/http"
	"reflect"
	"regexp"
	"strings"
)

const (
	slash              string = "/"
	UrlModeDashed      int    = 0
	UrlModeCapitalized int    = 1
)

var (
	Debug         bool   = false
	Prefix        string = "Action"
	Separator     string = "_"
	DefaultAction string = ""
	UrlMode       int    = UrlModeDashed
)

func Route(o interface{}) {
	var reflectType = reflect.TypeOf(o)
	var reflectValue = reflect.ValueOf(o)

	var controllerName = reflect.Indirect(reflectValue).Type().Name()

	for i := 0; i < reflectType.NumMethod(); i++ {
		method := reflectType.Method(i)
		methodName := getMethodName(method)
		methodBody := reflectValue.MethodByName(method.Name)
		routePath := getRoutePath(slash, controllerName, methodName)

		if i == 0 && DefaultAction == strings.Replace(methodName, slash, "", -1) {
			handleRoute(slash, methodBody)
		}

		handleRoute(routePath, methodBody)
	}
}

func getMethodName(method reflect.Method) string {
	methodName := method.Name

	methodName = strings.Replace(methodName, Prefix, "", -1)
	methodName = strings.Replace(methodName, Separator, slash, -1)

	return methodName
}

func getRoutePath(slash string, controllerName string, methodName string) string {
	routePath := slash + controllerName + methodName

	if UrlMode == UrlModeDashed {
		reg, err := regexp.Compile("([a-z])([A-Z])")
		if err != nil {
			return routePath
		}

		routePath = strings.ToLower(reg.ReplaceAllString(routePath, "$1-$2"))
	}

	return routePath
}

func handleRoute(routePath string, methodBody reflect.Value) {
	http.HandleFunc(routePath, func(w http.ResponseWriter, r *http.Request) {
		methodHandler := methodBody.Interface().(func(w http.ResponseWriter, r *http.Request))
		methodHandler(w, r)
	})

	if Debug {
		fmt.Println("route", routePath)
	}
}
<commit_msg>no message<commit_after>package godong

import (
	"fmt"
	"net/http"
	"reflect"
	"regexp"
	"strings"
)

const (
	slash              string = "/"
	UrlModeDashed      int    = 0
	UrlModeCapitalized int    = 1
)

var (
	Debug         bool   = false
	Prefix        string = "Action"
	Separator     string = "_"
	DefaultAction string = ""
	UrlMode       int    = UrlModeDashed
)

func Route(o interface{}) {
	var reflectType = reflect.TypeOf(o)
	var reflectValue = reflect.ValueOf(o)

	var controllerName = reflect.Indirect(reflectValue).Type().Name()

	for i := 0; i < reflectType.NumMethod(); i++ {
		method := reflectType.Method(i)
		methodName := getMethodName(method)
		methodBody := reflectValue.MethodByName(method.Name)
		routePath := getRoutePath(slash, controllerName, methodName)
		actionMap := strings.Join([]string{controllerName, method.Name}, ".")

		if actionMap == DefaultAction {
			handleRoute(slash, methodBody, actionMap)
		}

		handleRoute(routePath, methodBody, actionMap)
	}
}

func getMethodName(method reflect.Method) string {
	methodName := method.Name

	methodName = strings.Replace(methodName, Prefix, "", -1)
	methodName = strings.Replace(methodName, Separator, slash, -1)

	return methodName
}

func getRoutePath(slash string, controllerName string, methodName string) string {
	routePath := slash + controllerName + methodName

	if UrlMode == UrlModeDashed {
		reg, err := regexp.Compile("([a-z])([A-Z])")
		if err != nil {
			return routePath
		}

		routePath = strings.ToLower(reg.ReplaceAllString(routePath, "$1-$2"))
	}

	return routePath
}

func handleRoute(routePath string, methodBody reflect.Value, actionMap string) {
	methodHandler := methodBody.Interface().(func(w http.ResponseWriter, r *http.Request))
	http.HandleFunc(routePath, methodHandler)

	if Debug {
		fmt.Println("route", routePath)
		fmt.Println("   ->", actionMap)
	}
}
<|endoftext|>


<commit_before>/*
gogrep implements a simple parallel grep like application.

Inspired by ack and the silver searcher.
*/
package main

import (
	"bufio"
	"container/list"
	"fmt"
	"github.com/docopt/docopt.go"
	"github.com/mgutz/ansi"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"regexp"
	"runtime/pprof"
	"strconv"
	"sync"
	"time"
)

type HitPrinter func(int, int, string)

type Hit struct {
	Line, Col int
	Text      string
}

var query regexp.Regexp
var printLock sync.Mutex

var ignore_types = map[string]bool{
	"application/octet-stream": true,
}

var ignore_files = map[string]bool{
	".svn":              true,
	".agignore":         true,
	".gitignore":        true,
	".git":              true,
	".git/info/exclude": true,
	".hgignore":         true,
}

func noColorPrinter(line int, col int, text string) {
	fmt.Printf("%d: %s\n", line, text)
}

var redColor = ansi.ColorCode("red+b")
var yellowColor = ansi.ColorCode("yellow")
var resetColor = ansi.ColorCode("reset")

func colorPrinter(line int, col int, text string) {
	msg := redColor + strconv.Itoa(line) + ": " + resetColor + yellowColor + text + resetColor
	fmt.Println(msg)
}

// searchInFile does the actual search for the regex inside the file
func searchInFile(filename string, printer HitPrinter) {
	file, err := os.Open(filename)
	if err != nil {
		log.Fatal(os.Stderr, "Failed opening file:", err)
		return
	}

	file_header := make([]byte, 512)
	file.Read(file_header)

	if ignore_types[http.DetectContentType(file_header)] {
		return
	}
	hits := list.New()
	scanner := bufio.NewScanner(file)
	for i := 1; scanner.Scan(); i++ {
		if query.Match(scanner.Bytes()) {
			hits.PushBack(Hit{i, 0, scanner.Text()})
		}
	}
	if hits.Len() > 0 {
		// We serielize the printing of the results for grouping per file
		//
		printLock.Lock()
		fmt.Println(filename)
		for e := hits.Front(); e != nil; e = e.Next() {
			printer(e.Value.(Hit).Line, 0, e.Value.(Hit).Text)
		}
		fmt.Println("")
		printLock.Unlock()
	}
}

// searchInDir walks a directory and generated files to search. For every
// discovered subdirectory, it launches a new go routine (recursively).
func searchInDir(dir string, files chan string) {
	p, _ := ioutil.ReadDir(dir)
	for i := 0; i < len(p); i++ {
		if ignore_files[p[i].Name()] {
			continue
		}

		if p[i].IsDir() {
			searchInDir(path.Join(dir, p[i].Name()), files)
		} else {
			files <- path.Join(dir, p[i].Name())
		}
	}
}

func initializeIgnoreList() {
}

func searchFiles(files chan string) {
	for i := range files {
		go searchInFile(i, colorPrinter)
	}

}

func searchPaths(paths []string, files chan string) {
	defer close(files)
	if len(paths) == 0 {
		pwd, _ := os.Getwd()
		searchInDir(pwd, files)
	} else {
		for _, path := range paths {
			fi, _ := os.Stat(path)
			if fi.IsDir() {
				searchInDir(path, files)
			} else {
				files <- path
			}
		}
	}
}

func main() {
	start := time.Now()
	usage := `Recursively search for PATTERN in PATH

    usage: gogrep [options] PATTERN [PATH...]

	Arguments:
	  PATTERN  go regular expression

	Options:
	  -h --help            show this help message and exit
	  --version            show version and exit
	  -i --ignore-case     Match case insensitively
	  --ignore=PATTERNS    exclude files or directories which match these comma
	                       separated patterns [default: .svn,CVS,.bzr,.hg,.git]
	  --profile			   Run the go profiler on this run`

	arguments, _ := docopt.Parse(usage, nil, true, "1.0.0rc2", false)
	if arguments["--profile"].(bool) {
		f, err := os.Create("gogrep.prof")
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	query = *regexp.MustCompile(arguments["PATTERN"].(string))

	initializeIgnoreList()

	files := make(chan string)
	go searchPaths(arguments["PATH"].([]string), files)
	searchFiles(files)
	fmt.Println("Search finished in: ", time.Since(start))
}
<commit_msg>Fixed concurrency issues using a pool<commit_after>/*
gogrep implements a simple parallel grep like application.

Inspired by ack and the silver searcher.
*/
package main

import (
	"bufio"
	"container/list"
	"fmt"
	"github.com/docopt/docopt.go"
	"github.com/mgutz/ansi"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"regexp"
	"runtime"
	"runtime/pprof"
	"strconv"
	"sync"
	"time"
)

type HitPrinter func(int, int, string)

type Hit struct {
	Line, Col int
	Text      string
}

var query regexp.Regexp
var printLock sync.Mutex

var ignore_types = map[string]bool{
	"application/octet-stream": true,
}

var ignore_files = map[string]bool{
	".svn":              true,
	".agignore":         true,
	".gitignore":        true,
	".git":              true,
	".git/info/exclude": true,
	".hgignore":         true,
}

func noColorPrinter(line int, col int, text string) {
	fmt.Printf("%d: %s\n", line, text)
}

var redColor = ansi.ColorCode("red+b")
var yellowColor = ansi.ColorCode("yellow")
var resetColor = ansi.ColorCode("reset")

func colorPrinter(line int, col int, text string) {
	msg := redColor + strconv.Itoa(line) + ": " + resetColor + yellowColor + text + resetColor
	fmt.Println(msg)
}

// searchInFile does the actual search for the regex inside the file
func searchInFile(filename string, printer HitPrinter) {
	file, err := os.Open(filename)
	defer file.Close()
	if err != nil {
		log.Fatal(os.Stderr, "Failed opening file:", err)
		return
	}

	file_header := make([]byte, 512)
	file.Read(file_header)

	if ignore_types[http.DetectContentType(file_header)] {
		return
	}
	hits := list.New()
	scanner := bufio.NewScanner(file)
	for i := 1; scanner.Scan(); i++ {
		if query.Match(scanner.Bytes()) {
			hits.PushBack(Hit{i, 0, scanner.Text()})
		}
	}
	if hits.Len() > 0 {
		// We serielize the printing of the results for grouping per file
		//
		printLock.Lock()
		fmt.Println(filename)
		for e := hits.Front(); e != nil; e = e.Next() {
			printer(e.Value.(Hit).Line, 0, e.Value.(Hit).Text)
		}
		fmt.Println("")
		printLock.Unlock()
	}
}

// searchInDir walks a directory and generated files to search.
//
func searchInDir(dir string, files chan string) {
	p, err := ioutil.ReadDir(dir)
	if err != nil {
		log.Fatal(os.Stderr, "Failed opening file:", err)
		return
	}
	for i := 0; i < len(p); i++ {
		if ignore_files[p[i].Name()] {
			continue
		}

		if p[i].IsDir() {
			searchInDir(path.Join(dir, p[i].Name()), files)
		} else {
			files <- path.Join(dir, p[i].Name())
		}
	}
}

func initializeIgnoreList() {
}

func searchFiles(files chan string) {
	// Use a pool of goroutines to search the files, otherwise we might reach
	// the max open files permitted
	//
	var wg sync.WaitGroup
	const OPEN_FILES = 64
	for i := 0; i < OPEN_FILES; i++ {
		wg.Add(1)
		go func() {
			for i := range files {
				searchInFile(i, colorPrinter)
			}
			wg.Done()
		}()
	}
	// wait for the workers to finish
	wg.Wait()
}

func searchPaths(paths []string, files chan string) {
	// Close the files channel
	defer close(files)
	if len(paths) == 0 {
		pwd, _ := os.Getwd()
		searchInDir(pwd, files)
	} else {
		for _, path := range paths {
			fi, _ := os.Stat(path)
			if fi.IsDir() {
				searchInDir(path, files)
			} else {
				files <- path
			}
		}
	}
}

func main() {
	start := time.Now()
	usage := `Recursively search for PATTERN in PATH

    usage: gogrep [options] PATTERN [PATH...]

	Arguments:
	  PATTERN  go regular expression

	Options:
	  -h --help            show this help message and exit
	  --version            show version and exit
	  -i --ignore-case     Match case insensitively
	  --ignore=PATTERNS    exclude files or directories which match these comma
	                       separated patterns [default: .svn,CVS,.bzr,.hg,.git]
	  --profile			   Run the go profiler on this run`

	arguments, _ := docopt.Parse(usage, nil, true, "0.1", false)
	if arguments["--profile"].(bool) {
		f, err := os.Create("gogrep.prof")
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	// Since we are mostly IO bound, use also the hyper threads
	// Should benchmark this though...
	//
	runtime.GOMAXPROCS(runtime.NumCPU() * 2)

	query = *regexp.MustCompile(arguments["PATTERN"].(string))

	initializeIgnoreList()

	files := make(chan string)
	go searchPaths(arguments["PATH"].([]string), files)
	searchFiles(files)

	fmt.Println("Search finished in: ", time.Since(start))
}
<|endoftext|>