<commit_before>package awsutil

import (
	"sort"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/arn"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sts"
)

type sessionResult struct {
	accountName string
	accountId   string
	awsSession  *session.Session
	regions     []string
	err         error
}

func tryLoadCredentials() (*CredentialsStore, map[string]error, error) {
	accountNames, err := listAccountNames()
	if err != nil {
		return nil, nil, err
	}
	cs, unloadableAccounts := createCredentials(accountNames)
	return cs, unloadableAccounts, nil
}

func loadCredentials() (*CredentialsStore, error) {
	accountNames, err := listAccountNames()
	if err != nil {
		return nil, err
	}
	cs, unloadableAccounts := createCredentials(accountNames)
	for _, err := range unloadableAccounts {
		return cs, err
	}
	return cs, nil
}

func createCredentials(accountNames []string) (
	*CredentialsStore, map[string]error) {
	sort.Strings(accountNames)
	cs := &CredentialsStore{
		accountNames:    accountNames,
		sessionMap:      make(map[string]*session.Session),
		accountIdToName: make(map[string]string),
		accountNameToId: make(map[string]string),
		accountRegions:  make(map[string][]string),
	}
	resultsChannel := make(chan sessionResult, len(accountNames))
	for _, accountName := range accountNames {
		go func(accountName string) {
			resultsChannel <- createSession(accountName)
		}(accountName)
	}
	unloadableAccounts := make(map[string]error)
	for range accountNames {
		result := <-resultsChannel
		if result.err != nil {
			unloadableAccounts[result.accountName] = result.err
		} else {
			cs.sessionMap[result.accountName] = result.awsSession
			cs.accountIdToName[result.accountId] = result.accountName
			cs.accountNameToId[result.accountName] = result.accountId
			cs.accountRegions[result.accountName] = result.regions
		}
	}
	close(resultsChannel)
	return cs, unloadableAccounts
}

func createSession(accountName string) sessionResult {
	awsSession, err := CreateSession(accountName)
	if err != nil {
		return sessionResult{err: err, accountName: accountName}
	}
	stsService := sts.New(awsSession)
	inp := &sts.GetCallerIdentityInput{}
	var accountId string
	if out, err := stsService.GetCallerIdentity(inp); err != nil {
		return sessionResult{err: err, accountName: accountName}
	} else {
		if arnV, err := arn.Parse(aws.StringValue(out.Arn)); err != nil {
			return sessionResult{err: err, accountName: accountName}
		} else {
			accountId = arnV.AccountID
		}
	}
	regions, err := listRegions(CreateService(awsSession, "us-east-1"))
	if err != nil {
		return sessionResult{err: err, accountName: accountName}
	}
	return sessionResult{
		accountName: accountName,
		accountId:   accountId,
		awsSession:  awsSession,
		regions:     regions,
	}
}
<commit_msg>Richards suggestions.<commit_after>package awsutil

import (
	"sort"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/arn"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sts"
)

type sessionResult struct {
	accountName string
	accountId   string
	awsSession  *session.Session
	regions     []string
	err         error
}

func tryLoadCredentials() (*CredentialsStore, map[string]error, error) {
	accountNames, err := listAccountNames()
	if err != nil {
		return nil, nil, err
	}
	cs, unloadableAccounts := createCredentials(accountNames)
	return cs, unloadableAccounts, nil
}

func loadCredentials() (*CredentialsStore, error) {
	accountNames, err := listAccountNames()
	if err != nil {
		return nil, err
	}
	cs, unloadableAccounts := createCredentials(accountNames)
	for _, err := range unloadableAccounts {
		return nil, err
	}
	return cs, nil
}

func createCredentials(accountNames []string) (
	*CredentialsStore, map[string]error) {
	sort.Strings(accountNames)
	cs := &CredentialsStore{
		accountNames:    accountNames,
		sessionMap:      make(map[string]*session.Session),
		accountIdToName: make(map[string]string),
		accountNameToId: make(map[string]string),
		accountRegions:  make(map[string][]string),
	}
	resultsChannel := make(chan sessionResult, len(accountNames))
	for _, accountName := range accountNames {
		go func(accountName string) {
			resultsChannel <- createSession(accountName)
		}(accountName)
	}
	unloadableAccounts := make(map[string]error)
	for range accountNames {
		result := <-resultsChannel
		if result.err != nil {
			unloadableAccounts[result.accountName] = result.err
		} else {
			cs.sessionMap[result.accountName] = result.awsSession
			cs.accountIdToName[result.accountId] = result.accountName
			cs.accountNameToId[result.accountName] = result.accountId
			cs.accountRegions[result.accountName] = result.regions
		}
	}
	close(resultsChannel)
	return cs, unloadableAccounts
}

func createSession(accountName string) sessionResult {
	awsSession, err := CreateSession(accountName)
	if err != nil {
		return sessionResult{err: err, accountName: accountName}
	}
	stsService := sts.New(awsSession)
	inp := &sts.GetCallerIdentityInput{}
	var accountId string
	if out, err := stsService.GetCallerIdentity(inp); err != nil {
		return sessionResult{err: err, accountName: accountName}
	} else {
		if arnV, err := arn.Parse(aws.StringValue(out.Arn)); err != nil {
			return sessionResult{err: err, accountName: accountName}
		} else {
			accountId = arnV.AccountID
		}
	}
	regions, err := listRegions(CreateService(awsSession, "us-east-1"))
	if err != nil {
		return sessionResult{err: err, accountName: accountName}
	}
	return sessionResult{
		accountName: accountName,
		accountId:   accountId,
		awsSession:  awsSession,
		regions:     regions,
	}
}
<|endoftext|>


<commit_before>/*
                       WARNING WARNING WARNING

                Attention all potential contributors

   This testfile is not in the best state. We've been slowly transitioning
   from the built in "testing" package to using Ginkgo. As you can see, we've
   changed the format, but a lot of the setup, test body, descriptions, etc
   are either hardcoded, completely lacking, or misleading.

   For example:

   Describe("Testing with ginkgo"...)      // This is not a great description
   It("TestDoesSoemthing"...)              // This is a horrible description

   Describe("create-user command"...       // Describe the actual object under test
   It("creates a user when provided ..."   // this is more descriptive

   For good examples of writing Ginkgo tests for the cli, refer to

   src/github.com/cloudfoundry/cli/cf/commands/application/delete_app_test.go
   src/github.com/cloudfoundry/cli/cf/terminal/ui_test.go
   src/github.com/cloudfoundry/loggregator_consumer/consumer_test.go
*/

package user_test

import (
	. "github.com/cloudfoundry/cli/cf/commands/user"
	"github.com/cloudfoundry/cli/cf/models"
	testapi "github.com/cloudfoundry/cli/testhelpers/api"
	testcmd "github.com/cloudfoundry/cli/testhelpers/commands"
	testconfig "github.com/cloudfoundry/cli/testhelpers/configuration"
	testreq "github.com/cloudfoundry/cli/testhelpers/requirements"
	testterm "github.com/cloudfoundry/cli/testhelpers/terminal"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	. "github.com/cloudfoundry/cli/testhelpers/matchers"
)

func callSpaceUsers(args []string, requirementsFactory *testreq.FakeReqFactory, spaceRepo *testapi.FakeSpaceRepository, userRepo *testapi.FakeUserRepository) (ui *testterm.FakeUI) {
	ui = new(testterm.FakeUI)
	config := testconfig.NewRepositoryWithDefaults()
	cmd := NewSpaceUsers(ui, config, spaceRepo, userRepo)
	testcmd.RunCommand(cmd, args, requirementsFactory)
	return
}

var _ = Describe("Testing with ginkgo", func() {
	It("TestSpaceUsersFailsWithUsage", func() {
		requirementsFactory := &testreq.FakeReqFactory{}
		spaceRepo := &testapi.FakeSpaceRepository{}
		userRepo := &testapi.FakeUserRepository{}

		ui := callSpaceUsers([]string{}, requirementsFactory, spaceRepo, userRepo)
		Expect(ui.FailedWithUsage).To(BeTrue())

		ui = callSpaceUsers([]string{"my-org"}, requirementsFactory, spaceRepo, userRepo)
		Expect(ui.FailedWithUsage).To(BeTrue())

		ui = callSpaceUsers([]string{"my-org", "my-space"}, requirementsFactory, spaceRepo, userRepo)
		Expect(ui.FailedWithUsage).To(BeFalse())
	})
	It("TestSpaceUsersRequirements", func() {

		requirementsFactory := &testreq.FakeReqFactory{}
		spaceRepo := &testapi.FakeSpaceRepository{}
		userRepo := &testapi.FakeUserRepository{}
		args := []string{"my-org", "my-space"}

		requirementsFactory.LoginSuccess = false
		callSpaceUsers(args, requirementsFactory, spaceRepo, userRepo)
		Expect(testcmd.CommandDidPassRequirements).To(BeFalse())

		requirementsFactory.LoginSuccess = true
		callSpaceUsers(args, requirementsFactory, spaceRepo, userRepo)
		Expect(testcmd.CommandDidPassRequirements).To(BeTrue())

		Expect("my-org").To(Equal(requirementsFactory.OrganizationName))
	})
	It("TestSpaceUsers", func() {

		org := models.Organization{}
		org.Name = "Org1"
		org.Guid = "org1-guid"
		space := models.Space{}
		space.Name = "Space1"
		space.Guid = "space1-guid"

		requirementsFactory := &testreq.FakeReqFactory{LoginSuccess: true, Organization: org}
		spaceRepo := &testapi.FakeSpaceRepository{FindByNameInOrgSpace: space}
		userRepo := &testapi.FakeUserRepository{}

		user := models.UserFields{}
		user.Username = "user1"
		user2 := models.UserFields{}
		user2.Username = "user2"
		user3 := models.UserFields{}
		user3.Username = "user3"
		user4 := models.UserFields{}
		user4.Username = "user4"
		userRepo.ListUsersByRole = map[string][]models.UserFields{
			models.SPACE_MANAGER:   []models.UserFields{user, user2},
			models.SPACE_DEVELOPER: []models.UserFields{user4},
			models.SPACE_AUDITOR:   []models.UserFields{user3},
		}

		ui := callSpaceUsers([]string{"my-org", "my-space"}, requirementsFactory, spaceRepo, userRepo)

		Expect(spaceRepo.FindByNameInOrgName).To(Equal("my-space"))
		Expect(spaceRepo.FindByNameInOrgOrgGuid).To(Equal("org1-guid"))
		Expect(userRepo.ListUsersSpaceGuid).To(Equal("space1-guid"))

		Expect(ui.Outputs).To(ContainSubstrings(
			[]string{"Getting users in org", "Org1", "Space1", "my-user"},
			[]string{"SPACE MANAGER"},
			[]string{"user1"},
			[]string{"user2"},
			[]string{"SPACE DEVELOPER"},
			[]string{"user4"},
			[]string{"SPACE AUDITOR"},
			[]string{"user3"},
		))
	})
})
<commit_msg>Cleanup space-users test<commit_after>package user_test

import (
	. "github.com/cloudfoundry/cli/cf/commands/user"
	"github.com/cloudfoundry/cli/cf/configuration"
	"github.com/cloudfoundry/cli/cf/models"
	testapi "github.com/cloudfoundry/cli/testhelpers/api"
	testcmd "github.com/cloudfoundry/cli/testhelpers/commands"
	testconfig "github.com/cloudfoundry/cli/testhelpers/configuration"
	testreq "github.com/cloudfoundry/cli/testhelpers/requirements"
	testterm "github.com/cloudfoundry/cli/testhelpers/terminal"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	. "github.com/cloudfoundry/cli/testhelpers/matchers"
)

var _ = Describe("space-users command", func() {
	var (
		ui                  *testterm.FakeUI
		requirementsFactory *testreq.FakeReqFactory
		spaceRepo           *testapi.FakeSpaceRepository
		userRepo            *testapi.FakeUserRepository
		config              configuration.ReadWriter
	)

	BeforeEach(func() {
		config = testconfig.NewRepositoryWithDefaults()
		ui = &testterm.FakeUI{}
		requirementsFactory = &testreq.FakeReqFactory{}
		spaceRepo = &testapi.FakeSpaceRepository{}
		userRepo = &testapi.FakeUserRepository{}
	})

	runCommand := func(args ...string) {
		testcmd.RunCommand(NewSpaceUsers(ui, config, spaceRepo, userRepo), args, requirementsFactory)
	}

	Describe("requirements", func() {
		It("fails when not logged in", func() {
			runCommand("my-org", "my-space")

			Expect(testcmd.CommandDidPassRequirements).To(BeFalse())
		})

		It("succeeds when logged in", func() {
			requirementsFactory.LoginSuccess = true
			runCommand("some-org", "whatever-space")

			Expect(testcmd.CommandDidPassRequirements).To(BeTrue())
			Expect("some-org").To(Equal(requirementsFactory.OrganizationName))
		})
	})

	It("fails with usage when not invoked with exactly two args", func() {
		runCommand("my-org")
		Expect(ui.FailedWithUsage).To(BeTrue())
	})

	Context("when logged in and given some users in the org and space", func() {
		BeforeEach(func() {
			requirementsFactory.LoginSuccess = true

			org := models.Organization{}
			org.Name = "Org1"
			org.Guid = "org1-guid"
			space := models.Space{}
			space.Name = "Space1"
			space.Guid = "space1-guid"

			requirementsFactory.Organization = org
			spaceRepo.FindByNameInOrgSpace = space

			user := models.UserFields{}
			user.Username = "user1"
			user2 := models.UserFields{}
			user2.Username = "user2"
			user3 := models.UserFields{}
			user3.Username = "user3"
			user4 := models.UserFields{}
			user4.Username = "user4"
			userRepo.ListUsersByRole = map[string][]models.UserFields{
				models.SPACE_MANAGER:   []models.UserFields{user, user2},
				models.SPACE_DEVELOPER: []models.UserFields{user4},
				models.SPACE_AUDITOR:   []models.UserFields{user3},
			}
		})

		It("tells you about the space users in the given space", func() {
			runCommand("my-org", "my-space")

			Expect(spaceRepo.FindByNameInOrgName).To(Equal("my-space"))
			Expect(spaceRepo.FindByNameInOrgOrgGuid).To(Equal("org1-guid"))
			Expect(userRepo.ListUsersSpaceGuid).To(Equal("space1-guid"))

			Expect(ui.Outputs).To(ContainSubstrings(
				[]string{"Getting users in org", "Org1", "Space1", "my-user"},
				[]string{"SPACE MANAGER"},
				[]string{"user1"},
				[]string{"user2"},
				[]string{"SPACE DEVELOPER"},
				[]string{"user4"},
				[]string{"SPACE AUDITOR"},
				[]string{"user3"},
			))
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2017 Bret Jordan, All rights reserved.
//
// Use of this source code is governed by an Apache 2.0 license
// that can be found in the LICENSE file in the root of the source
// tree.

package config

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/freetaxii/libstix2/resources"
	"github.com/gologme/log"
	"github.com/gorilla/mux"
)

/*
ServerConfigType - This type defines the configuration for the entire server.
*/
type ServerConfigType struct {
	Router *mux.Router
	Global struct {
		Prefix             string
		Listen             string
		Protocol           string
		TLSDir             string
		TLSKey             string
		TLSCrt             string
		DbConfig           bool
		DbType             string
		DbFile             string
		MaxNumberOfObjects int
	}
	HTML struct {
		HTMLConfigType
	}
	Logging struct {
		Enabled bool
		LogFile string
	}
	DiscoveryServer struct {
		Enabled  bool
		Services []DiscoveryServiceType
	}
	APIRootServer struct {
		Enabled  bool
		Services []APIRootServiceType
	}
	DiscoveryResources  map[string]resources.DiscoveryType  // The key in the map is the ResourceID
	APIRootResources    map[string]resources.APIRootType    // The key in the map is the ResourceID
	CollectionResources map[string]resources.CollectionType // The key in the map is the ResourceID
}

/*
BaseServiceType - This struct represents the common properties between the
Discovery and API-Root services.

Name          - A name for this service, this is used as part of the URL Path
Enabled       - Is this service enabled
ResourceID    - A unique ID for the resource that this service is using
ResourcePath  - The actual full URL path for the resource
HTML          - The configuration for generating HTML output
*/
type BaseServiceType struct {
	Name         string         // User defined in configuration file
	Enabled      bool           // User defined in configuration file
	ResourceID   string         // User defined in configuration file
	ResourcePath string         // Set in verifyDiscoveryConfig() or verifyAPIRootConfig()
	HTML         HTMLConfigType // User defined in configuration file or set in the verify scripts.
}

/*
DiscoveryServiceType - This struct represents an instance of a Discovery server.
If someone tries to set the 'resourcepath' directive in the configuration file it
will get overwritten in code.
*/
type DiscoveryServiceType struct {
	BaseServiceType
}

/*
APIRootServiceType - This struct represents an instance of an API Root server.
If someone tries to set the 'path' directive in the configuration file it
will just get overwritten in code.
*/
type APIRootServiceType struct {
	BaseServiceType
	Collections struct {
		ResourceIDs  []string // User defined in configuration file. A list of collections that are members of this API Root
		Enabled      bool     // User defined in configuration file
		ResourcePath string   // Set in verifyAPIRootConfig()
	}
}

/*
HTMLConfigType - This struct holds the configuration elements for generating HTML
output. This is used at the top level of the configuration file as well as in
each individual service. This means individual services can have a different
HTML configuration.

Enabled       - Is HTML enabled for this service
TemplateDir   - The location of the template files relative to the base of the application (prefix)
TemplatePath  - The full path of the template directory (prefix + TemplateDir)
TemplateFiles - The HTML template filenames in the template directory for the following services
*/
type HTMLConfigType struct {
	Enabled       bool   // User defined in configuration file or set in verifyHTMLConfig()
	TemplateDir   string // User defined in configuration file or set in verifyHTMLConfig()
	TemplatePath  string // Set in verifyHTMLConfig()
	TemplateFiles struct {
		Discovery   string // User defined in configuration file or set in verifyHTMLConfig()
		APIRoot     string
		Collections string
		Collection  string
		Objects     string
		Manifest    string
	}
}

// ----------------------------------------------------------------------
//
// Public Create Functions
//
// ----------------------------------------------------------------------

/*
New - This function will return a ServerConfigType, load the current configuration
from a file, and verify that the configuration is correct.
*/
func New(filename string) (ServerConfigType, error) {
	var c ServerConfigType
	err1 := c.loadServerConfig(filename)
	if err1 != nil {
		return c, err1
	}

	// In addition to checking the configuration for completeness the verify
	// process will also populate some of the helper values.
	err2 := c.verifyServerConfig()
	if err2 != nil {
		return c, err2
	}
	return c, nil
}

// --------------------------------------------------
//
// Load Configuration File, Parse JSON, and Verify
//
// --------------------------------------------------

/*
loadServerConfig - This methods takes in a string value representing a
filename of the configuration file and loads the configuration into memory.
*/
func (c *ServerConfigType) loadServerConfig(filename string) error {
	// TODO - Need to make make a validation check for the configuration file

	// Open and read configuration file
	sysConfigFileData, err1 := os.Open(filename)
	defer sysConfigFileData.Close()
	if err1 != nil {
		return fmt.Errorf("error opening configuration file: %v", err1)
	}

	// --------------------------------------------------
	// Decode JSON configuration file
	// --------------------------------------------------
	// Use decoder instead of unmarshal so we can handle stream data
	decoder := json.NewDecoder(sysConfigFileData)
	err2 := decoder.Decode(c)

	if err2 != nil {
		return fmt.Errorf("error parsing the configuration file: %v", err2)
	}

	log.Debugln("DEBUG LoadServerConfig(): System Configuration Dump")
	log.Debugf("%+v\n", c)
	return nil
}

/*
verifyServerConfig - This method will verify that the configuration file has
what it needs.
TODO finish fleshing this out
*/
func (c *ServerConfigType) verifyServerConfig() error {
	var err error

	// --------------------------------------------------
	//
	// Global Configuration
	//
	// --------------------------------------------------

	err = c.verifyGlobalConfig()
	if err != nil {
		return err
	}

	err = c.verifyHTMLConfig()
	if err != nil {
		return err
	}

	// --------------------------------------------------
	// Discovery Server
	// --------------------------------------------------

	// Only verify the Discovery server configuration if it is enabled.
	if c.DiscoveryServer.Enabled == true {
		err = c.verifyDiscoveryConfig()
	} else {
		log.Infoln("CONFIG: The Discovery Server is not enabled in the configuration file")
	}

	if err != nil {
		return err
	}

	// --------------------------------------------------
	// API Root Server
	// --------------------------------------------------

	// Only verify the API Root server configuration if it is enabled.
	if c.APIRootServer.Enabled == true {
		err = c.verifyAPIRootConfig()
	} else {
		log.Infoln("CONFIG: The API Root Server is not enabled in the configuration file")
	}

	if err != nil {
		return err
	}
	return nil
}

func (config *ServerConfigType) exists(name string) bool {
	if _, err := os.Stat(name); err != nil {
		if os.IsNotExist(err) {
			return false
		}
		return false
	}
	return true
}
<commit_msg>json custom types<commit_after>// Copyright 2017 Bret Jordan, All rights reserved.
//
// Use of this source code is governed by an Apache 2.0 license
// that can be found in the LICENSE file in the root of the source
// tree.

package config

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/freetaxii/libstix2/resources"
	"github.com/gologme/log"
	"github.com/gorilla/mux"
)

/*
JSONbool - A boolean type for use with JSON that will track if a value is
undefined or set to null.
*/
type JSONbool struct {
	Value bool
	Valid bool
	Set   bool
}

/*
JSONstring - A string type for use with JSON that will track if a value is
undefined or set to null.
*/
type JSONstring struct {
	Value string
	Valid bool
	Set   bool
}

/*
UnmarshalJSON - This method will handle the unmarshalling of content for the
JSONbool type
*/
func (b *JSONbool) UnmarshalJSON(data []byte) error {
	// If this method was called, the value was set.
	b.Set = true

	if string(data) == "null" {
		// The key was set to null
		b.Valid = false
		return nil
	}

	// The key isn't set to null
	var temp bool
	if err := json.Unmarshal(data, &temp); err != nil {
		return err
	}
	b.Value = temp
	b.Valid = true
	return nil
}

/*
UnmarshalJSON - This method will handle the unmarshalling of content for the
JSONbool type
*/
func (s *JSONstring) UnmarshalJSON(data []byte) error {
	// If this method was called, the value was set.
	s.Set = true

	if string(data) == "null" {
		// The key was set to null
		s.Valid = false
		return nil
	}

	// The key isn't set to null
	var temp bool
	if err := json.Unmarshal(data, &temp); err != nil {
		return err
	}
	s.Value = temp
	s.Valid = true
	return nil
}

/*
ServerConfigType - This type defines the configuration for the entire server.
*/
type ServerConfigType struct {
	Router *mux.Router
	Global struct {
		Prefix             string
		Listen             string
		Protocol           string
		TLSDir             string
		TLSKey             string
		TLSCrt             string
		DbConfig           bool
		DbType             string
		DbFile             string
		MaxNumberOfObjects int
	}
	HTML struct {
		HTMLConfigType
	}
	Logging struct {
		Enabled bool
		LogFile string
	}
	DiscoveryServer struct {
		Enabled  bool
		Services []DiscoveryServiceType
	}
	APIRootServer struct {
		Enabled  bool
		Services []APIRootServiceType
	}
	DiscoveryResources  map[string]resources.DiscoveryType  // The key in the map is the ResourceID
	APIRootResources    map[string]resources.APIRootType    // The key in the map is the ResourceID
	CollectionResources map[string]resources.CollectionType // The key in the map is the ResourceID
}

/*
BaseServiceType - This struct represents the common properties between the
Discovery and API-Root services.

Name          - A name for this service, this is used as part of the URL Path
Enabled       - Is this service enabled
ResourceID    - A unique ID for the resource that this service is using
ResourcePath  - The actual full URL path for the resource
HTML          - The configuration for generating HTML output
*/
type BaseServiceType struct {
	Name         string         // User defined in configuration file
	Enabled      bool           // User defined in configuration file
	ResourceID   string         // User defined in configuration file
	ResourcePath string         // Set in verifyDiscoveryConfig() or verifyAPIRootConfig()
	HTML         HTMLConfigType // User defined in configuration file or set in the verify scripts.
}

/*
DiscoveryServiceType - This struct represents an instance of a Discovery server.
If someone tries to set the 'resourcepath' directive in the configuration file it
will get overwritten in code.
*/
type DiscoveryServiceType struct {
	BaseServiceType
}

/*
APIRootServiceType - This struct represents an instance of an API Root server.
If someone tries to set the 'path' directive in the configuration file it
will just get overwritten in code.
*/
type APIRootServiceType struct {
	BaseServiceType
	Collections struct {
		ResourceIDs  []string // User defined in configuration file. A list of collections that are members of this API Root
		Enabled      bool     // User defined in configuration file
		ResourcePath string   // Set in verifyAPIRootConfig()
	}
}

/*
HTMLConfigType - This struct holds the configuration elements for generating HTML
output. This is used at the top level of the configuration file as well as in
each individual service. This means individual services can have a different
HTML configuration.

Enabled       - Is HTML enabled for this service
TemplateDir   - The location of the template files relative to the base of the application (prefix)
TemplatePath  - The full path of the template directory (prefix + TemplateDir)
TemplateFiles - The HTML template filenames in the template directory for the following services
*/
type HTMLConfigType struct {
	Enabled       JSONbool   // User defined in configuration file or set in verifyHTMLConfig()
	TemplateDir   JSONstring // User defined in configuration file or set in verifyHTMLConfig()
	TemplatePath  JSONstring // Set in verifyHTMLConfig()
	TemplateFiles struct {
		Discovery   JSONstring // User defined in configuration file or set in verifyHTMLConfig()
		APIRoot     JSONstring
		Collections JSONstring
		Collection  JSONstring
		Objects     JSONstring
		Manifest    JSONstring
	}
}

// ----------------------------------------------------------------------
//
// Public Create Functions
//
// ----------------------------------------------------------------------

/*
New - This function will return a ServerConfigType, load the current configuration
from a file, and verify that the configuration is correct.
*/
func New(filename string) (ServerConfigType, error) {
	var c ServerConfigType
	err1 := c.loadServerConfig(filename)
	if err1 != nil {
		return c, err1
	}

	// In addition to checking the configuration for completeness the verify
	// process will also populate some of the helper values.
	err2 := c.verifyServerConfig()
	if err2 != nil {
		return c, err2
	}
	return c, nil
}

// --------------------------------------------------
//
// Load Configuration File, Parse JSON, and Verify
//
// --------------------------------------------------

/*
loadServerConfig - This methods takes in a string value representing a
filename of the configuration file and loads the configuration into memory.
*/
func (c *ServerConfigType) loadServerConfig(filename string) error {
	// TODO - Need to make make a validation check for the configuration file

	// Open and read configuration file
	sysConfigFileData, err1 := os.Open(filename)
	defer sysConfigFileData.Close()
	if err1 != nil {
		return fmt.Errorf("error opening configuration file: %v", err1)
	}

	// --------------------------------------------------
	// Decode JSON configuration file
	// --------------------------------------------------
	// Use decoder instead of unmarshal so we can handle stream data
	decoder := json.NewDecoder(sysConfigFileData)
	err2 := decoder.Decode(c)

	if err2 != nil {
		return fmt.Errorf("error parsing the configuration file: %v", err2)
	}

	log.Debugln("DEBUG LoadServerConfig(): System Configuration Dump")
	log.Debugf("%+v\n", c)
	return nil
}

/*
verifyServerConfig - This method will verify that the configuration file has
what it needs.
TODO finish fleshing this out
*/
func (c *ServerConfigType) verifyServerConfig() error {
	var err error

	// --------------------------------------------------
	//
	// Global Configuration
	//
	// --------------------------------------------------

	err = c.verifyGlobalConfig()
	if err != nil {
		return err
	}

	err = c.verifyHTMLConfig()
	if err != nil {
		return err
	}

	// --------------------------------------------------
	// Discovery Server
	// --------------------------------------------------

	// Only verify the Discovery server configuration if it is enabled.
	if c.DiscoveryServer.Enabled == true {
		err = c.verifyDiscoveryConfig()
	} else {
		log.Infoln("CONFIG: The Discovery Server is not enabled in the configuration file")
	}

	if err != nil {
		return err
	}

	// --------------------------------------------------
	// API Root Server
	// --------------------------------------------------

	// Only verify the API Root server configuration if it is enabled.
	if c.APIRootServer.Enabled == true {
		err = c.verifyAPIRootConfig()
	} else {
		log.Infoln("CONFIG: The API Root Server is not enabled in the configuration file")
	}

	if err != nil {
		return err
	}
	return nil
}

func (config *ServerConfigType) exists(name string) bool {
	if _, err := os.Stat(name); err != nil {
		if os.IsNotExist(err) {
			return false
		}
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package bubbles

import (
	"io/ioutil"
	"log"
	"testing"
	"time"
)

func init() {
	log.SetOutput(ioutil.Discard)
}

func TestIndex(t *testing.T) {
	es := newMockES(t, func() string {
		return `{"took":7,"items":[{"create":{"_index":"test","_type":"type1","_id":"1","_version":1}}]}`
	})
	defer es.Stop()

	b := New([]string{es.Addr()}, OptConnCount(2), OptFlush(10*time.Millisecond))

	ins := Action{
		Type: Index,
		MetaData: MetaData{
			Index: "test",
			Type:  "type1",
			ID:    "1",
		},
		Document: `{"field1": "value1"}`,
	}

	b.Enqueue() <- ins
	time.Sleep(15 * time.Millisecond)
	pending := b.Stop()
	if have, want := len(pending), 0; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
}

func TestIndexNoES(t *testing.T) {
	// Index without an ES
	b := New([]string{"localhost:4321"}, OptConnCount(2), OptFlush(10*time.Millisecond))

	ins := Action{
		Type: Index,
		MetaData: MetaData{
			Index: "test",
			Type:  "type1",
			ID:    "1",
		},
		Document: `{"field1": "value1"}`,
	}

	b.Enqueue() <- ins
	time.Sleep(20 * time.Millisecond)
	pending := b.Stop()
	if have, want := len(pending), 1; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
	if pending[0] != ins {
		t.Errorf("Wrong pending object returned")
	}
}

func TestIndexErr(t *testing.T) {
	es := newMockES(
		t,
		func() string {
			return `{"took":8,"errors":true,"items":[{"index":{"_index":"index","_type":"type1","_id":"1","_version":5,"status":200}},{"index":{"_index":"index","_type":"type1","_id":"2","status":400,"error":"MapperParsingException[failed to parse]; nested: JsonParseException[Unexpected end-of-input within/between OBJECT entries\n at [Source: [B@5f72a900; line: 1, column: 160]]; "}}]}`
		},
	)
	defer es.Stop()

	errs := make(chan ActionError)
	b := New([]string{es.Addr()},
		OptConnCount(2),
		OptFlush(10*time.Millisecond),
		OptError(func(e ActionError) { errs <- e }),
	)

	ins1 := Action{
		Type: Index,
		MetaData: MetaData{
			Index: "test",
			Type:  "type1",
			ID:    "1",
		},
		Document: `{"field1": "value1"}`,
	}
	ins2 := Action{
		Type: Index,
		MetaData: MetaData{
			Index: "test",
			Type:  "type1",
			ID:    "2",
		},
		Document: `{"field1": `, // fake an error
	}

	b.Enqueue() <- ins1
	b.Enqueue() <- ins2
	var aerr ActionError
	select {
	case <-time.After(1 * time.Second):
		t.Fatalf("timeout")
	case aerr = <-errs:
	}
	if have, want := aerr.Action, ins2; have != want {
		t.Fatalf("wrong err. have %v, want %v", have, want)
	}
	pending := b.Stop()
	if have, want := len(pending), 0; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
}

func TestShutdownTimeout(t *testing.T) {
	es := newMockES(t, func() string {
		time.Sleep(10 * time.Second)
		return "{}"
	})
	defer es.Stop()

	b := New([]string{es.Addr()},
		OptConnCount(1),
		OptFlush(10*time.Millisecond),
		OptServerTimeout(100*time.Millisecond),
	)

	ins := Action{
		Type: Index,
		MetaData: MetaData{
			Index: "test",
			Type:  "type1",
			ID:    "1",
		},
		Document: `{"field1": "value1"}`,
	}
	b.Enqueue() <- ins

	time.Sleep(20 * time.Millisecond)
	now := time.Now()
	pending := b.Stop()
	if time.Since(now) > 1*time.Second {
		t.Fatalf("Stop() took too long")
	}
	if have, want := len(pending), 1; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
	if pending[0] != ins {
		t.Errorf("Wrong pending object returned")
	}
}
<commit_msg>More documents for shutdown test, triggering bug.<commit_after>package bubbles

import (
	"io/ioutil"
	"log"
	"testing"
	"time"
)

func init() {
	log.SetOutput(ioutil.Discard)
}

func TestIndex(t *testing.T) {
	es := newMockES(t, func() string {
		return `{"took":7,"items":[{"create":{"_index":"test","_type":"type1","_id":"1","_version":1}}]}`
	})
	defer es.Stop()

	b := New([]string{es.Addr()}, OptConnCount(2), OptFlush(10*time.Millisecond))

	ins := Action{
		Type: Index,
		MetaData: MetaData{
			Index: "test",
			Type:  "type1",
			ID:    "1",
		},
		Document: `{"field1": "value1"}`,
	}

	b.Enqueue() <- ins
	time.Sleep(15 * time.Millisecond)
	pending := b.Stop()
	if have, want := len(pending), 0; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
}

func TestIndexNoES(t *testing.T) {
	// Index without an ES
	b := New([]string{"localhost:4321"}, OptConnCount(2), OptFlush(10*time.Millisecond))

	ins := Action{
		Type: Index,
		MetaData: MetaData{
			Index: "test",
			Type:  "type1",
			ID:    "1",
		},
		Document: `{"field1": "value1"}`,
	}

	b.Enqueue() <- ins
	time.Sleep(20 * time.Millisecond)
	pending := b.Stop()
	if have, want := len(pending), 1; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
	if pending[0] != ins {
		t.Errorf("Wrong pending object returned")
	}
}

func TestIndexErr(t *testing.T) {
	es := newMockES(
		t,
		func() string {
			return `{"took":8,"errors":true,"items":[{"index":{"_index":"index","_type":"type1","_id":"1","_version":5,"status":200}},{"index":{"_index":"index","_type":"type1","_id":"2","status":400,"error":"MapperParsingException[failed to parse]; nested: JsonParseException[Unexpected end-of-input within/between OBJECT entries\n at [Source: [B@5f72a900; line: 1, column: 160]]; "}}]}`
		},
	)
	defer es.Stop()

	errs := make(chan ActionError)
	b := New([]string{es.Addr()},
		OptConnCount(2),
		OptFlush(10*time.Millisecond),
		OptError(func(e ActionError) { errs <- e }),
	)

	ins1 := Action{
		Type: Index,
		MetaData: MetaData{
			Index: "test",
			Type:  "type1",
			ID:    "1",
		},
		Document: `{"field1": "value1"}`,
	}
	ins2 := Action{
		Type: Index,
		MetaData: MetaData{
			Index: "test",
			Type:  "type1",
			ID:    "2",
		},
		Document: `{"field1": `, // fake an error
	}

	b.Enqueue() <- ins1
	b.Enqueue() <- ins2
	var aerr ActionError
	select {
	case <-time.After(1 * time.Second):
		t.Fatalf("timeout")
	case aerr = <-errs:
	}
	if have, want := aerr.Action, ins2; have != want {
		t.Fatalf("wrong err. have %v, want %v", have, want)
	}
	pending := b.Stop()
	if have, want := len(pending), 0; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
}

func TestShutdownTimeout(t *testing.T) {
	es := newMockES(t, func() string {
		time.Sleep(10 * time.Second)
		return "{}"
	})
	defer es.Stop()

	maxDocs := 5
	b := New([]string{es.Addr()},
		OptConnCount(1),
		OptFlush(10*time.Millisecond),
		OptServerTimeout(100*time.Millisecond),
		OptMaxDocs(5),
	)

	ins := Action{
		Type: Index,
		MetaData: MetaData{
			Index: "test",
			Type:  "type1",
			ID:    "1",
		},
		Document: `{"field1": "value1"}`,
	}
	docs := maxDocs
	for i := 0; i < docs; i++ {
		b.Enqueue() <- ins
	}

	time.Sleep(20 * time.Millisecond)
	p := make(chan []Action)
	go func() {
		p <- b.Stop()
	}()
	var pending []Action
	select {
	case pending = <-p:
	case <-time.After(1 * time.Second):
		t.Fatalf("Stop() took too long")
	}
	if have, want := len(pending), docs; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
	if pending[0] != ins {
		t.Errorf("Wrong pending object returned")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"runtime"
)

const (
	queueLength       = 14000
	workerConcurrency = 7
	divider           = "--------------------------------------------------------"
)

var (
	logF *os.File
)

func init() {
	logF, err := os.OpenFile(
		getEnv("GR_BATCH_LOG", "batch.log"),
		os.O_RDWR|os.O_CREATE|os.O_APPEND,
		0644,
	)
	if err != nil {
		panic(err)
	}

	log.SetOutput(logF)
}

func main() {
	defer logF.Close()
	defer db.Close()

	runtime.GOMAXPROCS(workerConcurrency + 1)
	idQueue := make(chan int, queueLength)

	log.Println(divider)
	for i := 0; i < workerConcurrency; i++ {
		go workerLoop(i, idQueue)
	}
	schedulerLoop(idQueue)
}
<commit_msg>Apply required concurrency<commit_after>package main

import (
	"log"
	"os"
	"runtime"
)

const (
	queueLength       = 14000
	workerConcurrency = 45
	divider           = "--------------------------------------------------------"
)

var (
	logF *os.File
)

func init() {
	logF, err := os.OpenFile(
		getEnv("GR_BATCH_LOG", "batch.log"),
		os.O_RDWR|os.O_CREATE|os.O_APPEND,
		0644,
	)
	if err != nil {
		panic(err)
	}

	log.SetOutput(logF)
}

func main() {
	defer logF.Close()
	defer db.Close()

	runtime.GOMAXPROCS(workerConcurrency + 1)
	idQueue := make(chan int, queueLength)

	log.Println(divider)
	for i := 0; i < workerConcurrency; i++ {
		go workerLoop(i, idQueue)
	}
	schedulerLoop(idQueue)
}
<|endoftext|>


<commit_before>package gobatch

import (
	"errors"
	"testing"
)

func TestMust(t *testing.T) {
	batch := &MockBatch{}
	if Must(batch, nil) != batch {
		t.Error("Must(batch, nil) != batch")
	}

	var panics bool
	func() {
		defer func() {
			if p := recover(); p != nil {
				panics = true
			}
		}()
		_ = Must(&MockBatch{}, errors.New("error"))
	}()

	if !panics {
		t.Error("Must(batch, err) doesn't panic")
	}
}
<commit_msg>Fix imports in test<commit_after>package gobatch

import (
	"errors"
	"testing"

	"github.com/MasterOfBinary/gobatch/mocks"
)

func TestMust(t *testing.T) {
	batch := &mocks.Batch{}
	if Must(batch, nil) != batch {
		t.Error("Must(batch, nil) != batch")
	}

	var panics bool
	func() {
		defer func() {
			if p := recover(); p != nil {
				panics = true
			}
		}()
		_ = Must(&mocks.Batch{}, errors.New("error"))
	}()

	if !panics {
		t.Error("Must(batch, err) doesn't panic")
	}
}
<|endoftext|>


<commit_before>package fire

import (
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"github.com/256dpi/serve"

	"github.com/256dpi/fire/coal"
)

var textBody = strings.Repeat("X", 100)

func BenchmarkList(b *testing.B) {
	store := mongoStore

	b.Run("00X", func(b *testing.B) {
		listBenchmark(b, store, 20, 0)
	})

	b.Run("01X", func(b *testing.B) {
		listBenchmark(b, store, 20, 1)
	})

	b.Run("10X", func(b *testing.B) {
		listBenchmark(b, store, 20, 10)
	})

	b.Run("50X", func(b *testing.B) {
		listBenchmark(b, store, 20, 50)
	})
}

func BenchmarkFind(b *testing.B) {
	store := mongoStore

	b.Run("00X", func(b *testing.B) {
		findBenchmark(b, store, 0)
	})

	b.Run("01X", func(b *testing.B) {
		findBenchmark(b, store, 1)
	})

	b.Run("10X", func(b *testing.B) {
		findBenchmark(b, store, 10)
	})

	b.Run("50X", func(b *testing.B) {
		findBenchmark(b, store, 50)
	})
}

func BenchmarkCreate(b *testing.B) {
	store := mongoStore

	b.Run("00X", func(b *testing.B) {
		createBenchmark(b, store, 0)
	})

	b.Run("01X", func(b *testing.B) {
		createBenchmark(b, store, 1)
	})

	b.Run("10X", func(b *testing.B) {
		createBenchmark(b, store, 10)
	})

	b.Run("50X", func(b *testing.B) {
		createBenchmark(b, store, 50)
	})
}

func listBenchmark(b *testing.B, store *coal.Store, items, parallelism int) {
	tester := NewTester(store, modelList...)
	tester.Clean()

	tester.Assign("", &Controller{
		Model: &postModel{},
		Store: tester.Store,
	}, &Controller{
		Model: &commentModel{},
		Store: tester.Store,
	}, &Controller{
		Model: &selectionModel{},
		Store: tester.Store,
	}, &Controller{
		Model: &noteModel{},
		Store: tester.Store,
	})

	tester.Handler = serve.Compose(
		serve.Throttle(100),
		tester.Handler,
	)

	for i := 0; i < items; i++ {
		tester.Insert(&postModel{
			Title:    "Hello World!",
			TextBody: textBody,
		})
	}

	parallelBenchmark(b, parallelism, func() {
		tester.Request("GET", "posts", "", func(r *httptest.ResponseRecorder, rq *http.Request) {
			if r.Code != http.StatusOK {
				panic("not ok")
			}
		})
	})
}

func findBenchmark(b *testing.B, store *coal.Store, parallelism int) {
	tester := NewTester(store, modelList...)
	tester.Clean()

	tester.Assign("", &Controller{
		Model: &postModel{},
		Store: tester.Store,
	}, &Controller{
		Model: &commentModel{},
		Store: tester.Store,
	}, &Controller{
		Model: &selectionModel{},
		Store: tester.Store,
	}, &Controller{
		Model: &noteModel{},
		Store: tester.Store,
	})

	tester.Handler = serve.Compose(
		serve.Throttle(100),
		tester.Handler,
	)

	id := tester.Insert(&postModel{
		Title:    "Hello World!",
		TextBody: textBody,
	}).ID()

	parallelBenchmark(b, parallelism, func() {
		tester.Request("GET", "posts/"+id.Hex(), "", func(r *httptest.ResponseRecorder, rq *http.Request) {
			if r.Code != http.StatusOK {
				panic("not ok")
			}
		})
	})
}

func createBenchmark(b *testing.B, store *coal.Store, parallelism int) {
	tester := NewTester(store, modelList...)
	tester.Clean()

	tester.Assign("", &Controller{
		Model: &postModel{},
		Store: tester.Store,
	}, &Controller{
		Model: &commentModel{},
		Store: tester.Store,
	}, &Controller{
		Model: &selectionModel{},
		Store: tester.Store,
	}, &Controller{
		Model: &noteModel{},
		Store: tester.Store,
	})

	tester.Handler = serve.Compose(
		serve.Throttle(100),
		tester.Handler,
	)

	parallelBenchmark(b, parallelism, func() {
		tester.Request("POST", "posts", `{
			"data": {
				"type": "posts",
				"attributes": {
					"title": "Post 1"
				}
			}
		}`, func(r *httptest.ResponseRecorder, rq *http.Request) {
			if r.Code != http.StatusCreated {
				panic("not ok")
			}
		})
	})
}

func parallelBenchmark(b *testing.B, parallelism int, fn func()) {
	if parallelism != 0 {
		b.SetParallelism(parallelism)
	}

	b.ReportAllocs()
	b.ResetTimer()

	now := time.Now()

	if parallelism != 0 {
		b.RunParallel(func(pb *testing.PB) {
			for pb.Next() {
				fn()
			}
		})
	} else {
		for i := 0; i < b.N; i++ {
			fn()
		}
	}

	b.StopTimer()

	nsPerOp := float64(time.Since(now)) / float64(b.N)
	opsPerS := float64(time.Second) / nsPerOp
	b.ReportMetric(opsPerS, "ops/s")
}
<commit_msg>added 100X<commit_after>package fire

import (
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"github.com/256dpi/serve"

	"github.com/256dpi/fire/coal"
)

var textBody = strings.Repeat("X", 100)

func BenchmarkList(b *testing.B) {
	store := mongoStore

	b.Run("00X", func(b *testing.B) {
		listBenchmark(b, store, 20, 0)
	})

	b.Run("01X", func(b *testing.B) {
		listBenchmark(b, store, 20, 1)
	})

	b.Run("10X", func(b *testing.B) {
		listBenchmark(b, store, 20, 10)
	})

	b.Run("50X", func(b *testing.B) {
		listBenchmark(b, store, 20, 50)
	})

	b.Run("100X", func(b *testing.B) {
		listBenchmark(b, store, 20, 100)
	})
}

func BenchmarkFind(b *testing.B) {
	store := mongoStore

	b.Run("00X", func(b *testing.B) {
		findBenchmark(b, store, 0)
	})

	b.Run("01X", func(b *testing.B) {
		findBenchmark(b, store, 1)
	})

	b.Run("10X", func(b *testing.B) {
		findBenchmark(b, store, 10)
	})

	b.Run("50X", func(b *testing.B) {
		findBenchmark(b, store, 50)
	})

	b.Run("100X", func(b *testing.B) {
		findBenchmark(b, store, 100)
	})
}

func BenchmarkCreate(b *testing.B) {
	store := mongoStore

	b.Run("00X", func(b *testing.B) {
		createBenchmark(b, store, 0)
	})

	b.Run("01X", func(b *testing.B) {
		createBenchmark(b, store, 1)
	})

	b.Run("10X", func(b *testing.B) {
		createBenchmark(b, store, 10)
	})

	b.Run("50X", func(b *testing.B) {
		createBenchmark(b, store, 50)
	})

	b.Run("100X", func(b *testing.B) {
		createBenchmark(b, store, 100)
	})
}

func listBenchmark(b *testing.B, store *coal.Store, items, parallelism int) {
	tester := NewTester(store, modelList...)
	tester.Clean()

	tester.Assign("", &Controller{
		Model: &postModel{},
		Store: tester.Store,
	}, &Controller{
		Model: &commentModel{},
		Store: tester.Store,
	}, &Controller{
		Model: &selectionModel{},
		Store: tester.Store,
	}, &Controller{
		Model: &noteModel{},
		Store: tester.Store,
	})

	tester.Handler = serve.Compose(
		serve.Throttle(100),
		tester.Handler,
	)

	for i := 0; i < items; i++ {
		tester.Insert(&postModel{
			Title:    "Hello World!",
			TextBody: textBody,
		})
	}

	parallelBenchmark(b, parallelism, func() {
		tester.Request("GET", "posts", "", func(r *httptest.ResponseRecorder, rq *http.Request) {
			if r.Code != http.StatusOK {
				panic("not ok")
			}
		})
	})
}

func findBenchmark(b *testing.B, store *coal.Store, parallelism int) {
	tester := NewTester(store, modelList...)
	tester.Clean()

	tester.Assign("", &Controller{
		Model: &postModel{},
		Store: tester.Store,
	}, &Controller{
		Model: &commentModel{},
		Store: tester.Store,
	}, &Controller{
		Model: &selectionModel{},
		Store: tester.Store,
	}, &Controller{
		Model: &noteModel{},
		Store: tester.Store,
	})

	tester.Handler = serve.Compose(
		serve.Throttle(100),
		tester.Handler,
	)

	id := tester.Insert(&postModel{
		Title:    "Hello World!",
		TextBody: textBody,
	}).ID()

	parallelBenchmark(b, parallelism, func() {
		tester.Request("GET", "posts/"+id.Hex(), "", func(r *httptest.ResponseRecorder, rq *http.Request) {
			if r.Code != http.StatusOK {
				panic("not ok")
			}
		})
	})
}

func createBenchmark(b *testing.B, store *coal.Store, parallelism int) {
	tester := NewTester(store, modelList...)
	tester.Clean()

	tester.Assign("", &Controller{
		Model: &postModel{},
		Store: tester.Store,
	}, &Controller{
		Model: &commentModel{},
		Store: tester.Store,
	}, &Controller{
		Model: &selectionModel{},
		Store: tester.Store,
	}, &Controller{
		Model: &noteModel{},
		Store: tester.Store,
	})

	tester.Handler = serve.Compose(
		serve.Throttle(100),
		tester.Handler,
	)

	parallelBenchmark(b, parallelism, func() {
		tester.Request("POST", "posts", `{
			"data": {
				"type": "posts",
				"attributes": {
					"title": "Post 1"
				}
			}
		}`, func(r *httptest.ResponseRecorder, rq *http.Request) {
			if r.Code != http.StatusCreated {
				panic("not ok")
			}
		})
	})
}

func parallelBenchmark(b *testing.B, parallelism int, fn func()) {
	if parallelism != 0 {
		b.SetParallelism(parallelism)
	}

	b.ReportAllocs()
	b.ResetTimer()

	now := time.Now()

	if parallelism != 0 {
		b.RunParallel(func(pb *testing.PB) {
			for pb.Next() {
				fn()
			}
		})
	} else {
		for i := 0; i < b.N; i++ {
			fn()
		}
	}

	b.StopTimer()

	nsPerOp := float64(time.Since(now)) / float64(b.N)
	opsPerS := float64(time.Second) / nsPerOp
	b.ReportMetric(opsPerS, "ops/s")
}
<|endoftext|>


<commit_before>package lz4_test

import (
	"bytes"
	"io"
	"io/ioutil"
	"testing"

	"github.com/pierrec/lz4/v4"
	"github.com/pierrec/lz4/v4/internal/lz4block"
)

func BenchmarkCompress(b *testing.B) {
	buf := make([]byte, len(pg1661))

	b.ReportAllocs()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		_, _ = lz4block.CompressBlock(pg1661, buf, nil)
	}
}

func BenchmarkCompressRandom(b *testing.B) {
	buf := make([]byte, len(randomLZ4))

	b.ReportAllocs()
	b.SetBytes(int64(len(random)))
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		_, _ = lz4block.CompressBlock(random, buf, nil)
	}
}

func BenchmarkCompressHC(b *testing.B) {
	buf := make([]byte, len(pg1661))

	b.ReportAllocs()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		_, _ = lz4block.CompressBlockHC(pg1661, buf, 16, nil, nil)
	}
}

func BenchmarkUncompress(b *testing.B) {
	buf := make([]byte, len(pg1661))

	b.ReportAllocs()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		_, _ = lz4block.UncompressBlock(pg1661LZ4, buf)
	}
}

func mustLoadFile(f string) []byte {
	b, err := ioutil.ReadFile(f)
	if err != nil {
		panic(err)
	}
	return b
}

var (
	pg1661    = mustLoadFile("testdata/pg1661.txt")
	digits    = mustLoadFile("testdata/e.txt")
	twain     = mustLoadFile("testdata/Mark.Twain-Tom.Sawyer.txt")
	random    = mustLoadFile("testdata/random.data")
	pg1661LZ4 = mustLoadFile("testdata/pg1661.txt.lz4")
	digitsLZ4 = mustLoadFile("testdata/e.txt.lz4")
	twainLZ4  = mustLoadFile("testdata/Mark.Twain-Tom.Sawyer.txt.lz4")
	randomLZ4 = mustLoadFile("testdata/random.data.lz4")
)

func benchmarkUncompress(b *testing.B, compressed []byte) {
	r := bytes.NewReader(compressed)
	zr := lz4.NewReader(r)

	// Determine the uncompressed size of testfile.
	uncompressedSize, err := io.Copy(ioutil.Discard, zr)
	if err != nil {
		b.Fatal(err)
	}

	b.SetBytes(uncompressedSize)
	b.ReportAllocs()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		r.Reset(compressed)
		zr.Reset(r)
		_, _ = io.Copy(ioutil.Discard, zr)
	}
}

func BenchmarkUncompressPg1661(b *testing.B) { benchmarkUncompress(b, pg1661LZ4) }
func BenchmarkUncompressDigits(b *testing.B) { benchmarkUncompress(b, digitsLZ4) }
func BenchmarkUncompressTwain(b *testing.B)  { benchmarkUncompress(b, twainLZ4) }
func BenchmarkUncompressRand(b *testing.B)   { benchmarkUncompress(b, randomLZ4) }

func benchmarkCompress(b *testing.B, uncompressed []byte) {
	w := bytes.NewBuffer(nil)
	zw := lz4.NewWriter(w)
	r := bytes.NewReader(uncompressed)

	// Determine the compressed size of testfile.
	compressedSize, err := io.Copy(zw, r)
	if err != nil {
		b.Fatal(err)
	}
	if err := zw.Close(); err != nil {
		b.Fatal(err)
	}

	b.SetBytes(compressedSize)
	b.ReportAllocs()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		r.Reset(uncompressed)
		zw.Reset(w)
		_, _ = io.Copy(zw, r)
	}
}

func BenchmarkCompressPg1661(b *testing.B) { benchmarkCompress(b, pg1661) }
func BenchmarkCompressDigits(b *testing.B) { benchmarkCompress(b, digits) }
func BenchmarkCompressTwain(b *testing.B)  { benchmarkCompress(b, twain) }
func BenchmarkCompressRand(b *testing.B)   { benchmarkCompress(b, random) }

// Benchmark to check reallocations upon Reset().
// See issue https://github.com/pierrec/lz4/issues/52.
func BenchmarkWriterReset(b *testing.B) {
	b.ReportAllocs()

	zw := lz4.NewWriter(nil)
	src := mustLoadFile("testdata/gettysburg.txt")
	var buf bytes.Buffer

	for n := 0; n < b.N; n++ {
		buf.Reset()
		zw.Reset(&buf)

		_, _ = zw.Write(src)
		_ = zw.Close()
	}
}
<commit_msg>Report compressed sizes from compression benchmarks<commit_after>package lz4_test

import (
	"bytes"
	"io"
	"io/ioutil"
	"testing"

	"github.com/pierrec/lz4/v4"
	"github.com/pierrec/lz4/v4/internal/lz4block"
)

func BenchmarkCompress(b *testing.B) {
	buf := make([]byte, len(pg1661))

	n, _ := lz4block.CompressBlock(pg1661, buf, nil)

	b.ReportAllocs()
	b.ResetTimer()
	b.ReportMetric(float64(n), "outbytes")

	for i := 0; i < b.N; i++ {
		_, _ = lz4block.CompressBlock(pg1661, buf, nil)
	}
}

func BenchmarkCompressRandom(b *testing.B) {
	buf := make([]byte, len(randomLZ4))

	n, _ := lz4block.CompressBlock(pg1661, buf, nil)

	b.ReportAllocs()
	b.SetBytes(int64(len(random)))
	b.ResetTimer()
	b.ReportMetric(float64(n), "outbytes")

	for i := 0; i < b.N; i++ {
		_, _ = lz4block.CompressBlock(random, buf, nil)
	}
}

func BenchmarkCompressHC(b *testing.B) {
	buf := make([]byte, len(pg1661))

	n, _ := lz4block.CompressBlockHC(pg1661, buf, 16, nil, nil)

	b.ReportAllocs()
	b.ResetTimer()
	b.ReportMetric(float64(n), "outbytes")

	for i := 0; i < b.N; i++ {
		_, _ = lz4block.CompressBlockHC(pg1661, buf, 16, nil, nil)
	}
}

func BenchmarkUncompress(b *testing.B) {
	buf := make([]byte, len(pg1661))

	b.ReportAllocs()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		_, _ = lz4block.UncompressBlock(pg1661LZ4, buf)
	}
}

func mustLoadFile(f string) []byte {
	b, err := ioutil.ReadFile(f)
	if err != nil {
		panic(err)
	}
	return b
}

var (
	pg1661    = mustLoadFile("testdata/pg1661.txt")
	digits    = mustLoadFile("testdata/e.txt")
	twain     = mustLoadFile("testdata/Mark.Twain-Tom.Sawyer.txt")
	random    = mustLoadFile("testdata/random.data")
	pg1661LZ4 = mustLoadFile("testdata/pg1661.txt.lz4")
	digitsLZ4 = mustLoadFile("testdata/e.txt.lz4")
	twainLZ4  = mustLoadFile("testdata/Mark.Twain-Tom.Sawyer.txt.lz4")
	randomLZ4 = mustLoadFile("testdata/random.data.lz4")
)

func benchmarkUncompress(b *testing.B, compressed []byte) {
	r := bytes.NewReader(compressed)
	zr := lz4.NewReader(r)

	// Decompress once to determine the uncompressed size of testfile.
	_, err := io.Copy(ioutil.Discard, zr)
	if err != nil {
		b.Fatal(err)
	}

	b.SetBytes(int64(len(compressed)))
	b.ReportAllocs()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		r.Reset(compressed)
		zr.Reset(r)
		_, _ = io.Copy(ioutil.Discard, zr)
	}
}

func BenchmarkUncompressPg1661(b *testing.B) { benchmarkUncompress(b, pg1661LZ4) }
func BenchmarkUncompressDigits(b *testing.B) { benchmarkUncompress(b, digitsLZ4) }
func BenchmarkUncompressTwain(b *testing.B)  { benchmarkUncompress(b, twainLZ4) }
func BenchmarkUncompressRand(b *testing.B)   { benchmarkUncompress(b, randomLZ4) }

func benchmarkCompress(b *testing.B, uncompressed []byte) {
	w := bytes.NewBuffer(nil)
	zw := lz4.NewWriter(w)
	r := bytes.NewReader(uncompressed)

	// Compress once to determine the compressed size of testfile.
	_, err := io.Copy(zw, r)
	if err != nil {
		b.Fatal(err)
	}
	if err := zw.Close(); err != nil {
		b.Fatal(err)
	}

	b.SetBytes(int64(len(uncompressed)))
	b.ReportAllocs()
	b.ResetTimer()
	b.ReportMetric(float64(w.Len()), "outbytes")

	for i := 0; i < b.N; i++ {
		r.Reset(uncompressed)
		zw.Reset(w)
		_, _ = io.Copy(zw, r)
	}
}

func BenchmarkCompressPg1661(b *testing.B) { benchmarkCompress(b, pg1661) }
func BenchmarkCompressDigits(b *testing.B) { benchmarkCompress(b, digits) }
func BenchmarkCompressTwain(b *testing.B)  { benchmarkCompress(b, twain) }
func BenchmarkCompressRand(b *testing.B)   { benchmarkCompress(b, random) }

// Benchmark to check reallocations upon Reset().
// See issue https://github.com/pierrec/lz4/issues/52.
func BenchmarkWriterReset(b *testing.B) {
	b.ReportAllocs()

	zw := lz4.NewWriter(nil)
	src := mustLoadFile("testdata/gettysburg.txt")
	var buf bytes.Buffer

	for n := 0; n < b.N; n++ {
		buf.Reset()
		zw.Reset(&buf)

		_, _ = zw.Write(src)
		_ = zw.Close()
	}
}
<|endoftext|>


<commit_before>package polygen

import (
	"image/color"
	"math/rand"
	"log"
	"image"
	"github.com/llgcode/draw2d/draw2dimg"
	"fmt"
)

const (
	MutationColor = iota
	MutationPoint = iota
	MutationAddPoint = iota
	MutationDeletePoint = iota
)

const (
	MutationChance = 0.05
	PopulationCount = 10
	PolygonsPerIndividual = 100
	MaxPolygonPoints = 6
	MinPolygonPoints = 3
)

var (
	Mutations = []int{MutationColor, MutationPoint, MutationAddPoint, MutationDeletePoint}
)

type Candidate struct {
	w, h     int
	Polygons []*Polygon
	Fitness  int64
	img      *image.RGBA
}

type Point struct {
	X, Y int
}

type Polygon struct {
	Points []*Point
	color.Color
}

func RandomCandidate(w, h int) *Candidate {
	result := &Candidate{w: w, h: h, Polygons: make([]*Polygon, PolygonsPerIndividual)}
	for j := 0; j < len(result.Polygons); j++ {
		result.Polygons[j] = RandomPolygon(w, h)
	}

	result.RenderImage()

	return result
}

func RandomPolygon(maxW, maxH int) *Polygon {
	result := &Polygon{}
	result.Color = color.RGBA{uint8(rand.Intn(0xff)), uint8(rand.Intn(0xff)), uint8(rand.Intn(0xff)), uint8(rand.Intn(0xff))}

	numPoints := RandomInt(MinPolygonPoints, MaxPolygonPoints)

	for i := 0; i < numPoints; i++ {
		result.AddPoint(&Point{rand.Intn(maxW), rand.Intn(maxH)})
	}

	return result
}

func (m1 *Candidate) Mate(m2 *Candidate) *Candidate {
	w, h := m1.w, m1.h
	crossover := rand.Intn(len(m1.Polygons))
	polygons := make([]*Polygon, len(m1.Polygons))

	for i := 0; i < len(polygons); i++ {
		var p Polygon

		if i <= crossover {
			p = *m1.Polygons[i]  // NB copy the polygon, not the pointer
		} else {
			p = *m2.Polygons[i]
		}

		if shouldMutate() {
			p.Mutate(w, h)
		}

		polygons[i] = &p
	}

	result := &Candidate{w: m1.w, h: m1.h, Polygons: polygons}
	result.RenderImage()
	return result
}

func (p *Polygon) AddPoint(point *Point) {
	p.Points = append(p.Points, point)
}

func (p *Polygon) Mutate(maxW, maxH int) {
	switch randomMutation() {
	case MutationColor:
		orig := p.Color
		p.Color = MutateColor(p.Color)
		log.Printf("MutationColor: %v -> %v", orig, p.Color)

	case MutationPoint:
		i := rand.Intn(len(p.Points))
		orig := *p.Points[i]
		mutated := MutatePoint(orig, maxW, maxH)
		p.Points[i] = &mutated
		log.Printf("MutationPoint: %v -> %v", orig, mutated)

	case MutationAddPoint:
		log.Printf("MutationAddPoint")

	case MutationDeletePoint:
		log.Printf("MutationDeletePoint")
	}

}

// NB: operates on copy of p
func MutatePoint(p Point, maxW, maxH int) Point {
	result := p

	i := rand.Intn(2)
	switch i {
	case 0:
		result.X = rand.Intn(maxW)
	case 1:
		result.Y = rand.Intn(maxH)
	}

	return result
}

func MutateColor(c color.Color) color.Color {
	// get the non-premultiplied rgba values
	nrgba := color.NRGBAModel.Convert(c).(color.NRGBA)

	// randomly select one of the r/g/b/a values to mutate
	i := rand.Intn(4)
	val := uint8(rand.Intn(255))

	switch i {
	case 0:
		nrgba.R = val
	case 1:
		nrgba.G = val
	case 2:
		nrgba.B = val
	case 3:
		nrgba.A = val
	}

	return color.RGBAModel.Convert(nrgba)
}

func randomMutation() int {
	return Mutations[rand.Int() % len(Mutations)]
}

func (cd *Candidate) RenderImage() {
	cd.img = image.NewRGBA(image.Rect(0, 0, cd.w, cd.h))
	gc := draw2dimg.NewGraphicContext(cd.img)

	gc.SetLineWidth(1)

	for _, polygon := range cd.Polygons {
		gc.SetStrokeColor(polygon.Color)
		gc.SetFillColor(polygon.Color)

		firstPoint := polygon.Points[0]
		gc.MoveTo(float64(firstPoint.X), float64(firstPoint.Y))

		for _, point := range polygon.Points[1:] {
			gc.LineTo(float64(point.X), float64(point.Y))
		}

		gc.Close()
		gc.FillStroke()
	}
}

func (cd *Candidate) DrawAndSave(destFile string) {
	cd.RenderImage()
	draw2dimg.SaveToPngFile(destFile, cd.img)
}

func (cd *Candidate) String() string {
	return fmt.Sprintf("fitness: %d", cd.Fitness)
}

func shouldMutate() bool {
	return rand.Float32() < MutationChance
}

type ByFitness []*Candidate
func (cds ByFitness) Len() int           { return len(cds) }
func (cds ByFitness) Swap(i, j int)      { cds[i], cds[j] = cds[j], cds[i] }
func (cds ByFitness) Less(i, j int) bool { return cds[i].Fitness < cds[j].Fitness }
<commit_msg>fold MutationAddPoint / MutationDeletePoint into a single operation<commit_after>package polygen

import (
	"image/color"
	"math/rand"
	"log"
	"image"
	"github.com/llgcode/draw2d/draw2dimg"
	"fmt"
)

const (
	MutationColor = iota
	MutationPoint = iota
	MutationAddOrDeletePoint = iota
)

const (
	MutationChance = 0.05
	PopulationCount = 10
	PolygonsPerIndividual = 100
	MaxPolygonPoints = 6
	MinPolygonPoints = 3
)

var (
	Mutations = []int{MutationColor, MutationPoint, MutationAddOrDeletePoint}
)

type Candidate struct {
	w, h     int
	Polygons []*Polygon
	Fitness  int64
	img      *image.RGBA
}

type Point struct {
	X, Y int
}

type Polygon struct {
	Points []*Point
	color.Color
}

func RandomCandidate(w, h int) *Candidate {
	result := &Candidate{w: w, h: h, Polygons: make([]*Polygon, PolygonsPerIndividual)}
	for j := 0; j < len(result.Polygons); j++ {
		result.Polygons[j] = RandomPolygon(w, h)
	}

	result.RenderImage()

	return result
}

func RandomPolygon(maxW, maxH int) *Polygon {
	result := &Polygon{}
	result.Color = color.RGBA{uint8(rand.Intn(0xff)), uint8(rand.Intn(0xff)), uint8(rand.Intn(0xff)), uint8(rand.Intn(0xff))}

	numPoints := RandomInt(MinPolygonPoints, MaxPolygonPoints)

	for i := 0; i < numPoints; i++ {
		result.AddPoint(RandomPoint(maxW, maxH))
	}

	return result
}

func RandomPoint(maxW, maxH int) *Point {
	return &Point{rand.Intn(maxW), rand.Intn(maxH)}
}

func (m1 *Candidate) Mate(m2 *Candidate) *Candidate {
	w, h := m1.w, m1.h
	crossover := rand.Intn(len(m1.Polygons))
	polygons := make([]*Polygon, len(m1.Polygons))

	for i := 0; i < len(polygons); i++ {
		var p Polygon

		if i <= crossover {
			p = *m1.Polygons[i]  // NB copy the polygon, not the pointer
		} else {
			p = *m2.Polygons[i]
		}

		if shouldMutate() {
			p.Mutate(w, h)
		}

		polygons[i] = &p
	}

	result := &Candidate{w: m1.w, h: m1.h, Polygons: polygons}
	result.RenderImage()
	return result
}

func (p *Polygon) AddPoint(point *Point) {
	p.Points = append(p.Points, point)
}

func (p *Polygon) Mutate(maxW, maxH int) {
	switch randomMutation() {
	case MutationColor:
		orig := p.Color
		p.Color = MutateColor(p.Color)
		log.Printf("MutationColor: %v -> %v", orig, p.Color)

	case MutationPoint:
		i := rand.Intn(len(p.Points))
		orig := *p.Points[i]
		mutated := MutatePoint(orig, maxW, maxH)
		p.Points[i] = &mutated
		log.Printf("MutationPoint: %v -> %v", orig, mutated)

	case MutationAddOrDeletePoint:
		origPointCount := len(p.Points)

		if len(p.Points) == MinPolygonPoints {
			// can't delete
			p.AddPoint(RandomPoint(maxW, maxH))
		} else if len(p.Points) == MaxPolygonPoints {
			// can't add
			p.DeleteRandomPoint()
		} else {
			// we can do either add or delete
			switch rand.Intn(2) {
			case 0:
				p.AddPoint(RandomPoint(maxW, maxH))
			case 1:
				p.DeleteRandomPoint()
			}
		}

		newPointCount := len(p.Points)

		log.Printf("MutationAddOrDeletePoint: %d -> %d points", origPointCount, newPointCount)
	}
}

func (p *Polygon) DeleteRandomPoint() {
	i := rand.Intn(len(p.Points))
	p.Points = append(p.Points[:i], p.Points[i+1:]...)
}

// NB: operates on copy of p
func MutatePoint(p Point, maxW, maxH int) Point {
	result := p

	i := rand.Intn(2)
	switch i {
	case 0:
		result.X = rand.Intn(maxW)
	case 1:
		result.Y = rand.Intn(maxH)
	}

	return result
}

func MutateColor(c color.Color) color.Color {
	// get the non-premultiplied rgba values
	nrgba := color.NRGBAModel.Convert(c).(color.NRGBA)

	// randomly select one of the r/g/b/a values to mutate
	i := rand.Intn(4)
	val := uint8(rand.Intn(255))

	switch i {
	case 0:
		nrgba.R = val
	case 1:
		nrgba.G = val
	case 2:
		nrgba.B = val
	case 3:
		nrgba.A = val
	}

	return color.RGBAModel.Convert(nrgba)
}

func randomMutation() int {
	return Mutations[rand.Int() % len(Mutations)]
}

func (cd *Candidate) RenderImage() {
	cd.img = image.NewRGBA(image.Rect(0, 0, cd.w, cd.h))
	gc := draw2dimg.NewGraphicContext(cd.img)

	gc.SetLineWidth(1)

	for _, polygon := range cd.Polygons {
		gc.SetStrokeColor(polygon.Color)
		gc.SetFillColor(polygon.Color)

		firstPoint := polygon.Points[0]
		gc.MoveTo(float64(firstPoint.X), float64(firstPoint.Y))

		for _, point := range polygon.Points[1:] {
			gc.LineTo(float64(point.X), float64(point.Y))
		}

		gc.Close()
		gc.FillStroke()
	}
}

func (cd *Candidate) DrawAndSave(destFile string) {
	cd.RenderImage()
	draw2dimg.SaveToPngFile(destFile, cd.img)
}

func (cd *Candidate) String() string {
	return fmt.Sprintf("fitness: %d", cd.Fitness)
}

func shouldMutate() bool {
	return rand.Float32() < MutationChance
}

type ByFitness []*Candidate
func (cds ByFitness) Len() int           { return len(cds) }
func (cds ByFitness) Swap(i, j int)      { cds[i], cds[j] = cds[j], cds[i] }
func (cds ByFitness) Less(i, j int) bool { return cds[i].Fitness < cds[j].Fitness }
<|endoftext|>


<commit_before>package gin_test

import (
	"github.com/codegangsta/gin/lib"
	"os"
	"path/filepath"
	"runtime"
	"testing"
)

func Test_Builder_Build_Success(t *testing.T) {
	wd := filepath.Join("test_fixtures", "build_success")
	bin := "build_success"
	if runtime.GOOS == "windows" {
		bin += ".exe"
	}

	builder := gin.NewBuilder(wd, bin, false)
	err := builder.Build()
	expect(t, err, nil)

	file, err := os.Open(filepath.Join(wd, bin))
	if err != nil {
		t.Fatalf("File has not been written: %v", err)
	}

	refute(t, file, nil)
}
<commit_msg>fix test<commit_after>package gin_test

import (
	"github.com/codegangsta/gin/lib"
	"os"
	"path/filepath"
	"runtime"
	"testing"
)

func Test_Builder_Build_Success(t *testing.T) {
	wd := filepath.Join("test_fixtures", "build_success")
	bin := "build_success"
	if runtime.GOOS == "windows" {
		bin += ".exe"
	}

	builder := gin.NewBuilder(wd, wd, bin, false)
	err := builder.Build()
	expect(t, err, nil)

	file, err := os.Open(filepath.Join(wd, bin))
	if err != nil {
		t.Fatalf("File has not been written: %v", err)
	}

	refute(t, file, nil)
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/gdamore/tcell/v2"
)

type styleMap map[string]tcell.Style

func parseStyles() styleMap {
	if env := os.Getenv("LS_COLORS"); env != "" {
		return parseStylesGNU(env)
	}

	if env := os.Getenv("LSCOLORS"); env != "" {
		return parseStylesBSD(env)
	}

	// default values from dircolors with removed background colors
	defaultColors := []string{
		// "rs=0",
		"di=01;34",
		"ln=01;36",
		// "mh=00",
		"pi=33", // "pi=40;33",
		"so=01;35",
		"do=01;35",
		"bd=33;01", // "bd=40;33;01",
		"cd=33;01", // "cd=40;33;01",
		"or=31;01", // "or=40;31;01",
		// "mi=00",
		"su=01;32", // "su=37;41",
		"sg=01;32", // "sg=30;43",
		// "ca=30;41",
		"tw=01;34", // "tw=30;42",
		"ow=01;34", // "ow=34;42",
		"st=01;34", // "st=37;44",
		"ex=01;32",
	}

	return parseStylesGNU(strings.Join(defaultColors, ":"))
}

func applyAnsiCodes(s string, st tcell.Style) tcell.Style {
	toks := strings.Split(s, ";")

	var nums []int
	for _, tok := range toks {
		if tok == "" {
			nums = append(nums, 0)
			continue
		}
		n, err := strconv.Atoi(tok)
		if err != nil {
			log.Printf("converting escape code: %s", err)
			continue
		}
		nums = append(nums, n)
	}

	// ECMA-48 details the standard
	// TODO: should we support turning off attributes?
	//    Probably because this is used for previewers too
	for i := 0; i < len(nums); i++ {
		n := nums[i]
		switch {
		case n == 0:
			st = tcell.StyleDefault
		case n == 1:
			st = st.Bold(true)
		case n == 2:
			st = st.Dim(true)
		case n == 4:
			st = st.Underline(true)
		case n == 5 || n == 6:
			st = st.Blink(true)
		case n == 7:
			st = st.Reverse(true)
		case n == 8:
			// TODO: tcell PR for proper conceal
			_, bg, _ := st.Decompose()
			st = st.Foreground(bg)
		case n == 9:
			st = st.StrikeThrough(true)
		case n >= 30 && n <= 37:
			st = st.Foreground(tcell.PaletteColor(n - 30))
		case n == 38:
			if i+3 <= len(nums) && nums[i+1] == 5 {
				st = st.Foreground(tcell.PaletteColor(nums[i+2]))
				i += 2
			} else if i+5 <= len(nums) && nums[i+1] == 2 {
				st = st.Foreground(tcell.NewRGBColor(
					int32(nums[i+2]),
					int32(nums[i+3]),
					int32(nums[i+4])))
				i += 4
			} else {
				log.Printf("unknown ansi code or incorrect form: %d", n)
			}
		case n >= 40 && n <= 47:
			st = st.Background(tcell.PaletteColor(n - 40))
		case n == 48:
			if i+3 <= len(nums) && nums[i+1] == 5 {
				st = st.Background(tcell.PaletteColor(nums[i+2]))
				i += 2
			} else if i+5 <= len(nums) && nums[i+1] == 2 {
				st = st.Background(tcell.NewRGBColor(
					int32(nums[i+2]),
					int32(nums[i+3]),
					int32(nums[i+4])))
				i += 4
			} else {
				log.Printf("unknown ansi code or incorrect form: %d", n)
			}
		default:
			log.Printf("unknown ansi code: %d", n)
		}
	}

	return st
}

// This function parses $LS_COLORS environment variable.
func parseStylesGNU(env string) styleMap {
	styles := make(styleMap)

	entries := strings.Split(env, ":")
	for _, entry := range entries {
		if entry == "" {
			continue
		}
		pair := strings.Split(entry, "=")
		if len(pair) != 2 {
			log.Printf("invalid $LS_COLORS entry: %s", entry)
			return styles
		}
		key, val := pair[0], pair[1]
		styles[key] = applyAnsiCodes(val, tcell.StyleDefault)
	}

	return styles
}

// This function parses $LSCOLORS environment variable.
func parseStylesBSD(env string) styleMap {
	styles := make(styleMap)

	if len(env) != 22 {
		log.Printf("invalid $LSCOLORS variable: %s", env)
		return styles
	}

	colorNames := []string{"di", "ln", "so", "pi", "ex", "bd", "cd", "su", "sg", "tw", "ow"}

	getStyle := func(r1, r2 byte) tcell.Style {
		st := tcell.StyleDefault

		switch {
		case r1 == 'x':
			st = st.Foreground(tcell.ColorDefault)
		case 'A' <= r1 && r1 <= 'H':
			st = st.Foreground(tcell.Color(r1 - 'A')).Bold(true)
		case 'a' <= r1 && r1 <= 'h':
			st = st.Foreground(tcell.Color(r1 - 'a'))
		default:
			log.Printf("invalid $LSCOLORS entry: %c", r1)
			return tcell.StyleDefault
		}

		switch {
		case r2 == 'x':
			st = st.Background(tcell.ColorDefault)
		case 'a' <= r2 && r2 <= 'h':
			st = st.Background(tcell.Color(r2 - 'a'))
		default:
			log.Printf("invalid $LSCOLORS entry: %c", r2)
			return tcell.StyleDefault
		}

		return st
	}

	for i, key := range colorNames {
		styles[key] = getStyle(env[i*2], env[i*2+1])
	}

	return styles
}

func (cm styleMap) get(f *file) tcell.Style {
	var key string

	switch {
	case f.linkState == working:
		key = "ln"
	case f.linkState == broken:
		key = "or"
	case f.IsDir() && f.Mode()&os.ModeSticky != 0 && f.Mode()&0002 != 0:
		key = "tw"
	case f.IsDir() && f.Mode()&os.ModeSticky != 0:
		key = "st"
	case f.IsDir() && f.Mode()&0002 != 0:
		key = "ow"
	case f.IsDir():
		key = "di"
	case f.Mode()&os.ModeNamedPipe != 0:
		key = "pi"
	case f.Mode()&os.ModeSocket != 0:
		key = "so"
	case f.Mode()&os.ModeCharDevice != 0:
		key = "cd"
	case f.Mode()&os.ModeDevice != 0:
		key = "bd"
	case f.Mode()&os.ModeSetuid != 0:
		key = "su"
	case f.Mode()&os.ModeSetgid != 0:
		key = "sg"
	case f.Mode().IsRegular() && f.Mode()&0111 != 0:
		key = "ex"
	default:
		key = "*" + filepath.Ext(f.Name())
	}

	if val, ok := cm[key]; ok {
		return val
	}

	if val, ok := cm["fi"]; ok {
		return val
	}

	return tcell.StyleDefault
}
<commit_msg>Fix colors for BSD style LSCOLORS for tcell/v2<commit_after>package main

import (
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/gdamore/tcell/v2"
)

type styleMap map[string]tcell.Style

func parseStyles() styleMap {
	if env := os.Getenv("LS_COLORS"); env != "" {
		return parseStylesGNU(env)
	}

	if env := os.Getenv("LSCOLORS"); env != "" {
		return parseStylesBSD(env)
	}

	// default values from dircolors with removed background colors
	defaultColors := []string{
		// "rs=0",
		"di=01;34",
		"ln=01;36",
		// "mh=00",
		"pi=33", // "pi=40;33",
		"so=01;35",
		"do=01;35",
		"bd=33;01", // "bd=40;33;01",
		"cd=33;01", // "cd=40;33;01",
		"or=31;01", // "or=40;31;01",
		// "mi=00",
		"su=01;32", // "su=37;41",
		"sg=01;32", // "sg=30;43",
		// "ca=30;41",
		"tw=01;34", // "tw=30;42",
		"ow=01;34", // "ow=34;42",
		"st=01;34", // "st=37;44",
		"ex=01;32",
	}

	return parseStylesGNU(strings.Join(defaultColors, ":"))
}

func applyAnsiCodes(s string, st tcell.Style) tcell.Style {
	toks := strings.Split(s, ";")

	var nums []int
	for _, tok := range toks {
		if tok == "" {
			nums = append(nums, 0)
			continue
		}
		n, err := strconv.Atoi(tok)
		if err != nil {
			log.Printf("converting escape code: %s", err)
			continue
		}
		nums = append(nums, n)
	}

	// ECMA-48 details the standard
	// TODO: should we support turning off attributes?
	//    Probably because this is used for previewers too
	for i := 0; i < len(nums); i++ {
		n := nums[i]
		switch {
		case n == 0:
			st = tcell.StyleDefault
		case n == 1:
			st = st.Bold(true)
		case n == 2:
			st = st.Dim(true)
		case n == 4:
			st = st.Underline(true)
		case n == 5 || n == 6:
			st = st.Blink(true)
		case n == 7:
			st = st.Reverse(true)
		case n == 8:
			// TODO: tcell PR for proper conceal
			_, bg, _ := st.Decompose()
			st = st.Foreground(bg)
		case n == 9:
			st = st.StrikeThrough(true)
		case n >= 30 && n <= 37:
			st = st.Foreground(tcell.PaletteColor(n - 30))
		case n == 38:
			if i+3 <= len(nums) && nums[i+1] == 5 {
				st = st.Foreground(tcell.PaletteColor(nums[i+2]))
				i += 2
			} else if i+5 <= len(nums) && nums[i+1] == 2 {
				st = st.Foreground(tcell.NewRGBColor(
					int32(nums[i+2]),
					int32(nums[i+3]),
					int32(nums[i+4])))
				i += 4
			} else {
				log.Printf("unknown ansi code or incorrect form: %d", n)
			}
		case n >= 40 && n <= 47:
			st = st.Background(tcell.PaletteColor(n - 40))
		case n == 48:
			if i+3 <= len(nums) && nums[i+1] == 5 {
				st = st.Background(tcell.PaletteColor(nums[i+2]))
				i += 2
			} else if i+5 <= len(nums) && nums[i+1] == 2 {
				st = st.Background(tcell.NewRGBColor(
					int32(nums[i+2]),
					int32(nums[i+3]),
					int32(nums[i+4])))
				i += 4
			} else {
				log.Printf("unknown ansi code or incorrect form: %d", n)
			}
		default:
			log.Printf("unknown ansi code: %d", n)
		}
	}

	return st
}

// This function parses $LS_COLORS environment variable.
func parseStylesGNU(env string) styleMap {
	styles := make(styleMap)

	entries := strings.Split(env, ":")
	for _, entry := range entries {
		if entry == "" {
			continue
		}
		pair := strings.Split(entry, "=")
		if len(pair) != 2 {
			log.Printf("invalid $LS_COLORS entry: %s", entry)
			return styles
		}
		key, val := pair[0], pair[1]
		styles[key] = applyAnsiCodes(val, tcell.StyleDefault)
	}

	return styles
}

// This function parses $LSCOLORS environment variable.
func parseStylesBSD(env string) styleMap {
	styles := make(styleMap)

	if len(env) != 22 {
		log.Printf("invalid $LSCOLORS variable: %s", env)
		return styles
	}

	colorNames := []string{"di", "ln", "so", "pi", "ex", "bd", "cd", "su", "sg", "tw", "ow"}

	getStyle := func(r1, r2 byte) tcell.Style {
		st := tcell.StyleDefault

		switch {
		case r1 == 'x':
			st = st.Foreground(tcell.ColorDefault)
		case 'A' <= r1 && r1 <= 'H':
			st = st.Foreground(tcell.PaletteColor(int(r1 - 'A'))).Bold(true)
		case 'a' <= r1 && r1 <= 'h':
			st = st.Foreground(tcell.PaletteColor(int(r1 - 'a')))
		default:
			log.Printf("invalid $LSCOLORS entry: %c", r1)
			return tcell.StyleDefault
		}

		switch {
		case r2 == 'x':
			st = st.Background(tcell.ColorDefault)
		case 'a' <= r2 && r2 <= 'h':
			st = st.Background(tcell.PaletteColor(int(r2 - 'a')))
		default:
			log.Printf("invalid $LSCOLORS entry: %c", r2)
			return tcell.StyleDefault
		}

		return st
	}

	for i, key := range colorNames {
		styles[key] = getStyle(env[i*2], env[i*2+1])
	}

	return styles
}

func (cm styleMap) get(f *file) tcell.Style {
	var key string

	switch {
	case f.linkState == working:
		key = "ln"
	case f.linkState == broken:
		key = "or"
	case f.IsDir() && f.Mode()&os.ModeSticky != 0 && f.Mode()&0002 != 0:
		key = "tw"
	case f.IsDir() && f.Mode()&os.ModeSticky != 0:
		key = "st"
	case f.IsDir() && f.Mode()&0002 != 0:
		key = "ow"
	case f.IsDir():
		key = "di"
	case f.Mode()&os.ModeNamedPipe != 0:
		key = "pi"
	case f.Mode()&os.ModeSocket != 0:
		key = "so"
	case f.Mode()&os.ModeCharDevice != 0:
		key = "cd"
	case f.Mode()&os.ModeDevice != 0:
		key = "bd"
	case f.Mode()&os.ModeSetuid != 0:
		key = "su"
	case f.Mode()&os.ModeSetgid != 0:
		key = "sg"
	case f.Mode().IsRegular() && f.Mode()&0111 != 0:
		key = "ex"
	default:
		key = "*" + filepath.Ext(f.Name())
	}

	if val, ok := cm[key]; ok {
		return val
	}

	if val, ok := cm["fi"]; ok {
		return val
	}

	return tcell.StyleDefault
}
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/minio/cli"
	"github.com/minio/mc/pkg/console"
)

var serverCmd = cli.Command{
	Name:  "server",
	Usage: "Start object storage server.",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "address",
			Value: ":9000",
		},
	},
	Action: serverMain,
	CustomHelpTemplate: `NAME:
  minio {{.Name}} - {{.Usage}}

USAGE:
  minio {{.Name}} [OPTIONS] PATH [PATH...]

OPTIONS:
  {{range .Flags}}{{.}}
  {{end}}
ENVIRONMENT VARIABLES:
  MINIO_ACCESS_KEY: Access key string of 5 to 20 characters in length.
  MINIO_SECRET_KEY: Secret key string of 8 to 40 characters in length.

EXAMPLES:
  1. Start minio server.
      $ minio {{.Name}} /home/shared

  2. Start minio server bound to a specific IP:PORT, when you have multiple network interfaces.
      $ minio {{.Name}} --address 192.168.1.101:9000 /home/shared

  3. Start minio server on Windows.
      $ minio {{.Name}} C:\MyShare

  4. Start minio server 12 disks to enable erasure coded layer with 6 data and 6 parity.
      $ minio {{.Name}} /mnt/export1/backend /mnt/export2/backend /mnt/export3/backend /mnt/export4/backend \
          /mnt/export5/backend /mnt/export6/backend /mnt/export7/backend /mnt/export8/backend /mnt/export9/backend \
          /mnt/export10/backend /mnt/export11/backend /mnt/export12/backend
`,
}

type serverCmdConfig struct {
	serverAddr  string
	exportPaths []string
}

// configureServer configure a new server instance
func configureServer(srvCmdConfig serverCmdConfig) *http.Server {
	// Minio server config
	apiServer := &http.Server{
		Addr: srvCmdConfig.serverAddr,
		// Adding timeout of 10 minutes for unresponsive client connections.
		ReadTimeout:    10 * time.Minute,
		WriteTimeout:   10 * time.Minute,
		Handler:        configureServerHandler(srvCmdConfig),
		MaxHeaderBytes: 1 << 20,
	}

	// Returns configured HTTP server.
	return apiServer
}

// getListenIPs - gets all the ips to listen on.
func getListenIPs(httpServerConf *http.Server) (hosts []string, port string) {
	host, port, err := net.SplitHostPort(httpServerConf.Addr)
	fatalIf(err, "Unable to parse host port.")

	switch {
	case host != "":
		hosts = append(hosts, host)
	default:
		addrs, err := net.InterfaceAddrs()
		fatalIf(err, "Unable to determine network interface address.")
		for _, addr := range addrs {
			if addr.Network() == "ip+net" {
				host := strings.Split(addr.String(), "/")[0]
				if ip := net.ParseIP(host); ip.To4() != nil {
					hosts = append(hosts, host)
				}
			}
		}
	}
	return hosts, port
}

// Print listen ips.
func printListenIPs(tls bool, hosts []string, port string) {
	for _, host := range hosts {
		if tls {
			console.Printf("    https://%s:%s\n", host, port)
		} else {
			console.Printf("    http://%s:%s\n", host, port)
		}
	}
}

// initServerConfig initialize server config.
func initServerConfig(c *cli.Context) {
	// Save new config.
	err := serverConfig.Save()
	fatalIf(err, "Unable to save config.")

	// Fetch max conn limit from environment variable.
	if maxConnStr := os.Getenv("MINIO_MAXCONN"); maxConnStr != "" {
		// We need to parse to its integer value.
		var err error
		globalMaxConn, err = strconv.Atoi(maxConnStr)
		fatalIf(err, "Unable to convert MINIO_MAXCONN=%s environment variable into its integer value.", maxConnStr)
	}

	// Fetch access keys from environment variables if any and update the config.
	accessKey := os.Getenv("MINIO_ACCESS_KEY")
	secretKey := os.Getenv("MINIO_SECRET_KEY")

	// Validate if both keys are specified and they are valid save them.
	if accessKey != "" && secretKey != "" {
		if !isValidAccessKey.MatchString(accessKey) {
			fatalIf(errInvalidArgument, "Invalid access key.")
		}
		if !isValidSecretKey.MatchString(secretKey) {
			fatalIf(errInvalidArgument, "Invalid secret key.")
		}
		serverConfig.SetCredential(credential{
			AccessKeyID:     accessKey,
			SecretAccessKey: secretKey,
		})
	}

	// Set maxOpenFiles, This is necessary since default operating
	// system limits of 1024, 2048 are not enough for Minio server.
	setMaxOpenFiles()
	// Do not fail if this is not allowed, lower limits are fine as well.
}

// Check server arguments.
func checkServerSyntax(c *cli.Context) {
	if !c.Args().Present() || c.Args().First() == "help" {
		cli.ShowCommandHelpAndExit(c, "server", 1)
	}
}

// Extract port number from address address should be of the form host:port.
func getPort(address string) int {
	_, portStr, err := net.SplitHostPort(address)
	fatalIf(err, "Unable to parse host port.")
	portInt, err := strconv.Atoi(portStr)
	fatalIf(err, "Invalid port number.")
	return portInt
}

// Make sure that none of the other processes are listening on the
// specified port on any of the interfaces.
//
// On linux if a process is listening on 127.0.0.1:9000 then Listen()
// on ":9000" fails with the error "port already in use".
// However on Mac OSX Listen() on ":9000" falls back to the IPv6 address.
// This causes confusion on Mac OSX that minio server is not reachable
// on 127.0.0.1 even though minio server is running. So before we start
// the minio server we make sure that the port is free on all the IPs.
func checkPortAvailability(port int) {
	isAddrInUse := func(err error) bool {
		// Check if the syscall error is EADDRINUSE.
		// EADDRINUSE is the system call error if another process is
		// already listening at the specified port.
		neterr, ok := err.(*net.OpError)
		if !ok {
			return false
		}
		osErr, ok := neterr.Err.(*os.SyscallError)
		if !ok {
			return false
		}
		sysErr, ok := osErr.Err.(syscall.Errno)
		if !ok {
			return false
		}
		if sysErr != syscall.EADDRINUSE {
			return false
		}
		return true
	}
	ifcs, err := net.Interfaces()
	if err != nil {
		fatalIf(err, "Unable to list interfaces.")
	}
	for _, ifc := range ifcs {
		addrs, err := ifc.Addrs()
		if err != nil {
			fatalIf(err, "Unable to list addresses on interface %s.", ifc.Name)
		}
		for _, addr := range addrs {
			ipnet, ok := addr.(*net.IPNet)
			if !ok {
				errorIf(errors.New(""), "Failed to assert type on (*net.IPNet) interface.")
				continue
			}
			ip := ipnet.IP
			network := "tcp4"
			if ip.To4() == nil {
				network = "tcp6"
			}
			tcpAddr := net.TCPAddr{IP: ip, Port: port, Zone: ifc.Name}
			l, err := net.ListenTCP(network, &tcpAddr)
			if err != nil {
				if isAddrInUse(err) {
					// Fail if port is already in use.
					fatalIf(err, "Unable to listen on %s:%.d.", tcpAddr.IP, tcpAddr.Port)
				} else {
					// Ignore other errors.
					continue
				}
			}
			if err = l.Close(); err != nil {
				fatalIf(err, "Unable to close listener on %s:%.d.", tcpAddr.IP, tcpAddr.Port)
			}
		}
	}
}

func serverMain(c *cli.Context) {
	// check 'server' cli arguments.
	checkServerSyntax(c)

	// Initialize server config.
	initServerConfig(c)

	// Server address.
	serverAddress := c.String("address")

	host, port, _ := net.SplitHostPort(serverAddress)
	// If port empty, default to port '80'
	if port == "" {
		port = "80"
		// if SSL is enabled, choose port as "443" instead.
		if isSSL() {
			port = "443"
		}
	}

	// Check if requested port is available.
	checkPortAvailability(getPort(net.JoinHostPort(host, port)))

	// Save all command line args as export paths.
	exportPaths := c.Args()

	// Configure server.
	apiServer := configureServer(serverCmdConfig{
		serverAddr:  serverAddress,
		exportPaths: exportPaths,
	})

	// Credential.
	cred := serverConfig.GetCredential()

	// Region.
	region := serverConfig.GetRegion()

	// Print credentials and region.
	console.Println("\n" + cred.String() + "  " + colorMagenta("Region: ") + colorWhite(region))

	hosts, port := getListenIPs(apiServer) // get listen ips and port.
	tls := apiServer.TLSConfig != nil      // 'true' if TLS is enabled.

	console.Println("\nMinio Object Storage:")
	// Print api listen ips.
	printListenIPs(tls, hosts, port)

	console.Println("\nMinio Browser:")
	// Print browser listen ips.
	printListenIPs(tls, hosts, port)

	console.Println("\nTo configure Minio Client:")

	// Figure out right endpoint for 'mc'.
	endpoint := fmt.Sprintf("http://%s:%s", hosts[0], port)
	if tls {
		endpoint = fmt.Sprintf("https://%s:%s", hosts[0], port)
	}

	// Download 'mc' info.
	if runtime.GOOS == "windows" {
		console.Printf("    Download 'mc' from https://dl.minio.io/client/mc/release/%s-%s/mc.exe\n", runtime.GOOS, runtime.GOARCH)
		console.Printf("    $ mc.exe config host add myminio %s %s %s\n", endpoint, cred.AccessKeyID, cred.SecretAccessKey)
	} else {
		console.Printf("    $ wget https://dl.minio.io/client/mc/release/%s-%s/mc\n", runtime.GOOS, runtime.GOARCH)
		console.Printf("    $ chmod 755 mc\n")
		console.Printf("    $ ./mc config host add myminio %s %s %s\n", endpoint, cred.AccessKeyID, cred.SecretAccessKey)
	}

	// Start server.
	var err error
	// Configure TLS if certs are available.
	if isSSL() {
		err = apiServer.ListenAndServeTLS(mustGetCertFile(), mustGetKeyFile())
	} else {
		// Fallback to http.
		err = apiServer.ListenAndServe()
	}
	fatalIf(err, "Failed to start minio server.")
}
<commit_msg>Show "https" in the "minio server export/" output if certificates are available. (#2065)<commit_after>/*
 * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/minio/cli"
	"github.com/minio/mc/pkg/console"
)

var serverCmd = cli.Command{
	Name:  "server",
	Usage: "Start object storage server.",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "address",
			Value: ":9000",
		},
	},
	Action: serverMain,
	CustomHelpTemplate: `NAME:
  minio {{.Name}} - {{.Usage}}

USAGE:
  minio {{.Name}} [OPTIONS] PATH [PATH...]

OPTIONS:
  {{range .Flags}}{{.}}
  {{end}}
ENVIRONMENT VARIABLES:
  MINIO_ACCESS_KEY: Access key string of 5 to 20 characters in length.
  MINIO_SECRET_KEY: Secret key string of 8 to 40 characters in length.

EXAMPLES:
  1. Start minio server.
      $ minio {{.Name}} /home/shared

  2. Start minio server bound to a specific IP:PORT, when you have multiple network interfaces.
      $ minio {{.Name}} --address 192.168.1.101:9000 /home/shared

  3. Start minio server on Windows.
      $ minio {{.Name}} C:\MyShare

  4. Start minio server 12 disks to enable erasure coded layer with 6 data and 6 parity.
      $ minio {{.Name}} /mnt/export1/backend /mnt/export2/backend /mnt/export3/backend /mnt/export4/backend \
          /mnt/export5/backend /mnt/export6/backend /mnt/export7/backend /mnt/export8/backend /mnt/export9/backend \
          /mnt/export10/backend /mnt/export11/backend /mnt/export12/backend
`,
}

type serverCmdConfig struct {
	serverAddr  string
	exportPaths []string
}

// configureServer configure a new server instance
func configureServer(srvCmdConfig serverCmdConfig) *http.Server {
	// Minio server config
	apiServer := &http.Server{
		Addr: srvCmdConfig.serverAddr,
		// Adding timeout of 10 minutes for unresponsive client connections.
		ReadTimeout:    10 * time.Minute,
		WriteTimeout:   10 * time.Minute,
		Handler:        configureServerHandler(srvCmdConfig),
		MaxHeaderBytes: 1 << 20,
	}

	// Returns configured HTTP server.
	return apiServer
}

// getListenIPs - gets all the ips to listen on.
func getListenIPs(httpServerConf *http.Server) (hosts []string, port string) {
	host, port, err := net.SplitHostPort(httpServerConf.Addr)
	fatalIf(err, "Unable to parse host port.")

	switch {
	case host != "":
		hosts = append(hosts, host)
	default:
		addrs, err := net.InterfaceAddrs()
		fatalIf(err, "Unable to determine network interface address.")
		for _, addr := range addrs {
			if addr.Network() == "ip+net" {
				host := strings.Split(addr.String(), "/")[0]
				if ip := net.ParseIP(host); ip.To4() != nil {
					hosts = append(hosts, host)
				}
			}
		}
	}
	return hosts, port
}

// Print listen ips.
func printListenIPs(tls bool, hosts []string, port string) {
	for _, host := range hosts {
		if tls {
			console.Printf("    https://%s:%s\n", host, port)
		} else {
			console.Printf("    http://%s:%s\n", host, port)
		}
	}
}

// initServerConfig initialize server config.
func initServerConfig(c *cli.Context) {
	// Save new config.
	err := serverConfig.Save()
	fatalIf(err, "Unable to save config.")

	// Fetch max conn limit from environment variable.
	if maxConnStr := os.Getenv("MINIO_MAXCONN"); maxConnStr != "" {
		// We need to parse to its integer value.
		var err error
		globalMaxConn, err = strconv.Atoi(maxConnStr)
		fatalIf(err, "Unable to convert MINIO_MAXCONN=%s environment variable into its integer value.", maxConnStr)
	}

	// Fetch access keys from environment variables if any and update the config.
	accessKey := os.Getenv("MINIO_ACCESS_KEY")
	secretKey := os.Getenv("MINIO_SECRET_KEY")

	// Validate if both keys are specified and they are valid save them.
	if accessKey != "" && secretKey != "" {
		if !isValidAccessKey.MatchString(accessKey) {
			fatalIf(errInvalidArgument, "Invalid access key.")
		}
		if !isValidSecretKey.MatchString(secretKey) {
			fatalIf(errInvalidArgument, "Invalid secret key.")
		}
		serverConfig.SetCredential(credential{
			AccessKeyID:     accessKey,
			SecretAccessKey: secretKey,
		})
	}

	// Set maxOpenFiles, This is necessary since default operating
	// system limits of 1024, 2048 are not enough for Minio server.
	setMaxOpenFiles()
	// Do not fail if this is not allowed, lower limits are fine as well.
}

// Check server arguments.
func checkServerSyntax(c *cli.Context) {
	if !c.Args().Present() || c.Args().First() == "help" {
		cli.ShowCommandHelpAndExit(c, "server", 1)
	}
}

// Extract port number from address address should be of the form host:port.
func getPort(address string) int {
	_, portStr, err := net.SplitHostPort(address)
	fatalIf(err, "Unable to parse host port.")
	portInt, err := strconv.Atoi(portStr)
	fatalIf(err, "Invalid port number.")
	return portInt
}

// Make sure that none of the other processes are listening on the
// specified port on any of the interfaces.
//
// On linux if a process is listening on 127.0.0.1:9000 then Listen()
// on ":9000" fails with the error "port already in use".
// However on Mac OSX Listen() on ":9000" falls back to the IPv6 address.
// This causes confusion on Mac OSX that minio server is not reachable
// on 127.0.0.1 even though minio server is running. So before we start
// the minio server we make sure that the port is free on all the IPs.
func checkPortAvailability(port int) {
	isAddrInUse := func(err error) bool {
		// Check if the syscall error is EADDRINUSE.
		// EADDRINUSE is the system call error if another process is
		// already listening at the specified port.
		neterr, ok := err.(*net.OpError)
		if !ok {
			return false
		}
		osErr, ok := neterr.Err.(*os.SyscallError)
		if !ok {
			return false
		}
		sysErr, ok := osErr.Err.(syscall.Errno)
		if !ok {
			return false
		}
		if sysErr != syscall.EADDRINUSE {
			return false
		}
		return true
	}
	ifcs, err := net.Interfaces()
	if err != nil {
		fatalIf(err, "Unable to list interfaces.")
	}
	for _, ifc := range ifcs {
		addrs, err := ifc.Addrs()
		if err != nil {
			fatalIf(err, "Unable to list addresses on interface %s.", ifc.Name)
		}
		for _, addr := range addrs {
			ipnet, ok := addr.(*net.IPNet)
			if !ok {
				errorIf(errors.New(""), "Failed to assert type on (*net.IPNet) interface.")
				continue
			}
			ip := ipnet.IP
			network := "tcp4"
			if ip.To4() == nil {
				network = "tcp6"
			}
			tcpAddr := net.TCPAddr{IP: ip, Port: port, Zone: ifc.Name}
			l, err := net.ListenTCP(network, &tcpAddr)
			if err != nil {
				if isAddrInUse(err) {
					// Fail if port is already in use.
					fatalIf(err, "Unable to listen on %s:%.d.", tcpAddr.IP, tcpAddr.Port)
				} else {
					// Ignore other errors.
					continue
				}
			}
			if err = l.Close(); err != nil {
				fatalIf(err, "Unable to close listener on %s:%.d.", tcpAddr.IP, tcpAddr.Port)
			}
		}
	}
}

func serverMain(c *cli.Context) {
	// check 'server' cli arguments.
	checkServerSyntax(c)

	// Initialize server config.
	initServerConfig(c)

	// If https.
	tls := isSSL()

	// Server address.
	serverAddress := c.String("address")

	host, port, _ := net.SplitHostPort(serverAddress)
	// If port empty, default to port '80'
	if port == "" {
		port = "80"
		// if SSL is enabled, choose port as "443" instead.
		if tls {
			port = "443"
		}
	}

	// Check if requested port is available.
	checkPortAvailability(getPort(net.JoinHostPort(host, port)))

	// Save all command line args as export paths.
	exportPaths := c.Args()

	// Configure server.
	apiServer := configureServer(serverCmdConfig{
		serverAddr:  serverAddress,
		exportPaths: exportPaths,
	})

	// Credential.
	cred := serverConfig.GetCredential()

	// Region.
	region := serverConfig.GetRegion()

	// Print credentials and region.
	console.Println("\n" + cred.String() + "  " + colorMagenta("Region: ") + colorWhite(region))

	hosts, port := getListenIPs(apiServer) // get listen ips and port.

	console.Println("\nMinio Object Storage:")
	// Print api listen ips.
	printListenIPs(tls, hosts, port)

	console.Println("\nMinio Browser:")
	// Print browser listen ips.
	printListenIPs(tls, hosts, port)

	console.Println("\nTo configure Minio Client:")

	// Figure out right endpoint for 'mc'.
	endpoint := fmt.Sprintf("http://%s:%s", hosts[0], port)
	if tls {
		endpoint = fmt.Sprintf("https://%s:%s", hosts[0], port)
	}

	// Download 'mc' info.
	if runtime.GOOS == "windows" {
		console.Printf("    Download 'mc' from https://dl.minio.io/client/mc/release/%s-%s/mc.exe\n", runtime.GOOS, runtime.GOARCH)
		console.Printf("    $ mc.exe config host add myminio %s %s %s\n", endpoint, cred.AccessKeyID, cred.SecretAccessKey)
	} else {
		console.Printf("    $ wget https://dl.minio.io/client/mc/release/%s-%s/mc\n", runtime.GOOS, runtime.GOARCH)
		console.Printf("    $ chmod 755 mc\n")
		console.Printf("    $ ./mc config host add myminio %s %s %s\n", endpoint, cred.AccessKeyID, cred.SecretAccessKey)
	}

	// Start server.
	var err error
	// Configure TLS if certs are available.
	if isSSL() {
		err = apiServer.ListenAndServeTLS(mustGetCertFile(), mustGetKeyFile())
	} else {
		// Fallback to http.
		err = apiServer.ListenAndServe()
	}
	fatalIf(err, "Failed to start minio server.")
}
<|endoftext|>


<commit_before>package evaluation

import (
	"strings"
)

func MakeCollection(t Type, elems []Object, ctx *Context) Object {
	switch t {
	case ARRAY:
		return &Array{Value: elems}
	case TUPLE:
		return &Tuple{Value: elems}
	case STRING:
		var strs []string

		for _, elem := range elems {
			strs = append(strs, elem.String())
		}

		return &String{Value: strings.Join(strs, "")}
	default:
		return Err(ctx, "could not form a collection of type %s", "TypeError", t)
	}
}
<commit_msg>Add comment to MakeCollection()<commit_after>package evaluation

import (
	"strings"
)

// MakeCollection creates a collection of type t with the elements provided
func MakeCollection(t Type, elems []Object, ctx *Context) Object {
	switch t {
	case ARRAY:
		return &Array{Value: elems}
	case TUPLE:
		return &Tuple{Value: elems}
	case STRING:
		var strs []string

		for _, elem := range elems {
			strs = append(strs, elem.String())
		}

		return &String{Value: strings.Join(strs, "")}
	default:
		return Err(ctx, "could not form a collection of type %s", "TypeError", t)
	}
}
<|endoftext|>