<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"net/url"
	"os"
	"time"

	uaaclient "github.com/cloudfoundry-incubator/uaa-go-client"
	uaaconfig "github.com/cloudfoundry-incubator/uaa-go-client/config"

	"github.com/cloudfoundry-incubator/cf-lager"
	"github.com/cloudfoundry-incubator/routing-api"
	"github.com/cloudfoundry-incubator/routing-api-cli/commands"
	"github.com/cloudfoundry-incubator/routing-api/db"
	trace "github.com/cloudfoundry-incubator/trace-logger"
	"github.com/codegangsta/cli"
	"github.com/pivotal-golang/clock"
)

const (
	RTR_TRACE                      = "RTR_TRACE"
	DefaultTokenFetchRetryInterval = 5 * time.Second
	DefaultTokenFetchNumRetries    = uint32(1)
	DefaultExpirationBufferTime    = int64(30)
)

var flags = []cli.Flag{
	cli.StringFlag{
		Name:  "api",
		Usage: "Endpoint for the routing-api. (required)",
	},
	cli.StringFlag{
		Name:  "client-id",
		Usage: "Id of the OAuth client. (required)",
	},
	cli.StringFlag{
		Name:  "client-secret",
		Usage: "Secret for OAuth client. (required)",
	},
	cli.StringFlag{
		Name:  "oauth-url",
		Usage: "URL for OAuth client. (required)",
	},
	cli.BoolFlag{
		Name:  "skip-oauth-tls-verification",
		Usage: "Skip OAuth TLS Verification (optional)",
	},
}

var eventsFlags = []cli.Flag{
	cli.BoolFlag{
		Name:  "http",
		Usage: "Stream HTTP events",
	},
	cli.BoolFlag{
		Name:  "tcp",
		Usage: "Stream TCP events",
	},
}

var cliCommands = []cli.Command{
	{
		Name:  "register",
		Usage: "Registers routes with the routing-api",
		Description: `Routes must be specified in JSON format, like so:
'[{"route":"foo.com", "port":12345, "ip":"1.2.3.4", "ttl":5, "log_guid":"log-guid"}]'`,
		Action: registerRoutes,
		Flags:  flags,
	},
	{
		Name:  "unregister",
		Usage: "Unregisters routes with the routing-api",
		Description: `Routes must be specified in JSON format, like so:
'[{"route":"foo.com", "port":12345, "ip":"1.2.3.4"]'`,
		Action: unregisterRoutes,
		Flags:  flags,
	},
	{
		Name:   "list",
		Usage:  "Lists the currently registered routes",
		Action: listRoutes,
		Flags:  flags,
	},
	{
		Name:   "events",
		Usage:  "Stream events from the Routing API",
		Action: streamEvents,
		Flags:  append(flags, eventsFlags...),
	},
}

var environmentVariableHelp = `ENVIRONMENT VARIABLES:
   RTR_TRACE=true	Print API request diagnostics to stdout`

func main() {
	cf_lager.AddFlags(flag.CommandLine)
	fmt.Println()
	app := cli.NewApp()
	app.Name = "rtr"
	app.Usage = "A CLI for the Router API server."
	authors := []cli.Author{cli.Author{Name: "Cloud Foundry Routing Team", Email: "cf-dev@lists.cloudfoundry.org"}}
	app.Authors = authors
	app.Commands = cliCommands
	app.CommandNotFound = commandNotFound
	app.Version = "2.3.0"

	cli.AppHelpTemplate = cli.AppHelpTemplate + environmentVariableHelp + "\n"

	trace.NewLogger(os.Getenv(RTR_TRACE))

	app.Run(os.Args)
	os.Exit(0)
}

func registerRoutes(c *cli.Context) {
	issues := checkFlags(c)
	errorMessage := "route registration failed:"
	issues = append(issues, checkArguments(c, "register")...)

	if len(issues) > 0 {
		printHelpForCommand(c, issues, "register")
	}

	desiredRoutes := c.Args().First()
	var routes []db.Route

	err := json.Unmarshal([]byte(desiredRoutes), &routes)
	checkError(errorMessage, err)

	client, err := newRoutingApiClient(c)
	checkError(errorMessage, err)

	err = commands.Register(client, routes)
	checkError(errorMessage, err)

	fmt.Printf("Successfully registered routes: %s\n", desiredRoutes)
}

func unregisterRoutes(c *cli.Context) {
	issues := checkFlags(c)
	errorMessage := "route unregistration failed:"
	issues = append(issues, checkArguments(c, "unregister")...)

	if len(issues) > 0 {
		printHelpForCommand(c, issues, "unregister")
	}

	desiredRoutes := c.Args().First()
	var routes []db.Route
	err := json.Unmarshal([]byte(desiredRoutes), &routes)
	checkError(errorMessage, err)

	client, err := newRoutingApiClient(c)
	checkError(errorMessage, err)

	err = commands.UnRegister(client, routes)
	checkError(errorMessage, err)

	fmt.Printf("Successfully unregistered routes: %s\n", desiredRoutes)
}

func listRoutes(c *cli.Context) {
	errorMessage := "listing routes failed:"
	issues := checkFlags(c)
	issues = append(issues, checkArguments(c, "list")...)

	if len(issues) > 0 {
		printHelpForCommand(c, issues, "list")
	}

	client, err := newRoutingApiClient(c)
	checkError(errorMessage, err)

	routes, err := commands.List(client)
	if err != nil {
		fmt.Println("listing routes failed:", err)
		os.Exit(3)
	}

	prettyRoutes, _ := json.Marshal(routes)

	fmt.Printf("%v\n", string(prettyRoutes))
}

func streamEvents(c *cli.Context) {
	issues := checkFlags(c)
	issues = append(issues, checkArguments(c, "events")...)

	if len(issues) > 0 {
		printHelpForCommand(c, issues, "events")
	}

	streamHttp := c.Bool("http")
	streamTcp := c.Bool("tcp")

	if !streamHttp && !streamTcp {
		streamHttp = true
		streamTcp = true
	}

	client, err := newRoutingApiClient(c)
	checkError("streaming events failed:", err)
	errorChan := make(chan error)
	eventChan := make(chan string)

	numOfSubscriptions := 0

	if streamHttp {
		numOfSubscriptions++
		go streamHttpEvents(client, eventChan, errorChan)
	}

	if streamTcp {
		numOfSubscriptions++
		go streamTcpEvents(client, eventChan, errorChan)
	}

	errorCount := 0

loop:
	for {
		select {
		case eventMessage := <-eventChan:
			fmt.Println(eventMessage)
		case err := <-errorChan:
			errorCount++
			fmt.Printf("Connection closed: %s", err.Error())
			if errorCount >= numOfSubscriptions {
				break loop
			}
		}
	}
}

func streamHttpEvents(client routing_api.Client, eventChan chan string, errorChan chan error) {
	eventSource, err := client.SubscribeToEvents()
	if err != nil {
		fmt.Println("streaming events failed:", err)
		return
	}
	for {
		e, err := eventSource.Next()
		if err != nil {
			errorChan <- err
			break
		}

		event, _ := json.Marshal(e)
		eventChan <- fmt.Sprintf("%v\n", string(event))
	}
}

func streamTcpEvents(client routing_api.Client, eventChan chan string, errorChan chan error) {
	eventSource, err := client.SubscribeToTcpEvents()
	if err != nil {
		fmt.Println("streaming events failed:", err)
		return
	}
	for {
		e, err := eventSource.Next()
		if err != nil {
			errorChan <- err
			break
		}

		event, _ := json.Marshal(e)
		eventChan <- fmt.Sprintf("%v\n", string(event))
	}
}

func buildOauthConfig(c *cli.Context) *uaaconfig.Config {

	return &uaaconfig.Config{
		UaaEndpoint:           c.String("oauth-url"),
		SkipVerification:      c.Bool("skip-oauth-tls-verification"),
		ClientName:            c.String("client-id"),
		ClientSecret:          c.String("client-secret"),
		MaxNumberOfRetries:    3,
		RetryInterval:         500 * time.Millisecond,
		ExpirationBufferInSec: 30,
	}

}

func checkFlags(c *cli.Context) []string {
	var issues []string

	if c.String("api") == "" {
		issues = append(issues, "Must provide an API endpoint for the routing-api component.")
	}

	if c.String("client-id") == "" {
		issues = append(issues, "Must provide the id of an OAuth client.")
	}

	if c.String("client-secret") == "" {
		issues = append(issues, "Must provide an OAuth secret.")
	}

	if c.String("oauth-url") == "" {
		issues = append(issues, "Must provide an URL to the OAuth client.")
	}

	_, err := url.Parse(c.String("oauth-url"))
	if err != nil {
		issues = append(issues, "Invalid OAuth client URL")
	}

	return issues
}

func checkArguments(c *cli.Context, cmd string) []string {
	var issues []string

	switch cmd {
	case "register", "unregister":
		if len(c.Args()) > 1 {
			issues = append(issues, "Unexpected arguments.")
		} else if len(c.Args()) < 1 {
			issues = append(issues, "Must provide routes JSON.")
		}
	case "list", "events":
		if len(c.Args()) > 0 {
			issues = append(issues, "Unexpected arguments.")
		}
	}

	return issues
}

func printHelpForCommand(c *cli.Context, issues []string, cmd string) {
	for _, issue := range issues {
		fmt.Println(issue)
	}
	fmt.Println()
	cli.ShowCommandHelp(c, cmd)
	os.Exit(1)
}

func commandNotFound(c *cli.Context, cmd string) {
	fmt.Println("Not a valid command:", cmd)
	os.Exit(1)
}

func newRoutingApiClient(c *cli.Context) (routing_api.Client, error) {

	uaaClient, err := newUaaClient(c)
	if err != nil {
		return nil, err
	}

	token, err := uaaClient.FetchToken(true)
	if err != nil {
		return nil, err
	}

	routingApiClient := routing_api.NewClient(c.String("api"))

	routingApiClient.SetToken(token.AccessToken)

	return routingApiClient, nil

}

func checkError(message string, err error) {
	if err != nil {
		fmt.Println(message, err.Error())
		os.Exit(3)
	}
}

func newUaaClient(c *cli.Context) (uaaclient.Client, error) {

	logger, _ := cf_lager.New("rtr")
	cfg := buildOauthConfig(c)
	klok := clock.NewClock()

	uaaClient, err := uaaclient.NewClient(logger, cfg, klok)
	if err != nil {
		return nil, err
	}
	_, err = uaaClient.FetchToken(true)

	return uaaClient, nil
}
<commit_msg>Increment api version<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"net/url"
	"os"
	"time"

	uaaclient "github.com/cloudfoundry-incubator/uaa-go-client"
	uaaconfig "github.com/cloudfoundry-incubator/uaa-go-client/config"

	"github.com/cloudfoundry-incubator/cf-lager"
	"github.com/cloudfoundry-incubator/routing-api"
	"github.com/cloudfoundry-incubator/routing-api-cli/commands"
	"github.com/cloudfoundry-incubator/routing-api/db"
	trace "github.com/cloudfoundry-incubator/trace-logger"
	"github.com/codegangsta/cli"
	"github.com/pivotal-golang/clock"
)

const (
	RTR_TRACE                      = "RTR_TRACE"
	DefaultTokenFetchRetryInterval = 5 * time.Second
	DefaultTokenFetchNumRetries    = uint32(1)
	DefaultExpirationBufferTime    = int64(30)
)

var flags = []cli.Flag{
	cli.StringFlag{
		Name:  "api",
		Usage: "Endpoint for the routing-api. (required)",
	},
	cli.StringFlag{
		Name:  "client-id",
		Usage: "Id of the OAuth client. (required)",
	},
	cli.StringFlag{
		Name:  "client-secret",
		Usage: "Secret for OAuth client. (required)",
	},
	cli.StringFlag{
		Name:  "oauth-url",
		Usage: "URL for OAuth client. (required)",
	},
	cli.BoolFlag{
		Name:  "skip-oauth-tls-verification",
		Usage: "Skip OAuth TLS Verification (optional)",
	},
}

var eventsFlags = []cli.Flag{
	cli.BoolFlag{
		Name:  "http",
		Usage: "Stream HTTP events",
	},
	cli.BoolFlag{
		Name:  "tcp",
		Usage: "Stream TCP events",
	},
}

var cliCommands = []cli.Command{
	{
		Name:  "register",
		Usage: "Registers routes with the routing-api",
		Description: `Routes must be specified in JSON format, like so:
'[{"route":"foo.com", "port":12345, "ip":"1.2.3.4", "ttl":5, "log_guid":"log-guid"}]'`,
		Action: registerRoutes,
		Flags:  flags,
	},
	{
		Name:  "unregister",
		Usage: "Unregisters routes with the routing-api",
		Description: `Routes must be specified in JSON format, like so:
'[{"route":"foo.com", "port":12345, "ip":"1.2.3.4"]'`,
		Action: unregisterRoutes,
		Flags:  flags,
	},
	{
		Name:   "list",
		Usage:  "Lists the currently registered routes",
		Action: listRoutes,
		Flags:  flags,
	},
	{
		Name:   "events",
		Usage:  "Stream events from the Routing API",
		Action: streamEvents,
		Flags:  append(flags, eventsFlags...),
	},
}

var environmentVariableHelp = `ENVIRONMENT VARIABLES:
   RTR_TRACE=true	Print API request diagnostics to stdout`

func main() {
	cf_lager.AddFlags(flag.CommandLine)
	fmt.Println()
	app := cli.NewApp()
	app.Name = "rtr"
	app.Usage = "A CLI for the Router API server."
	authors := []cli.Author{cli.Author{Name: "Cloud Foundry Routing Team", Email: "cf-dev@lists.cloudfoundry.org"}}
	app.Authors = authors
	app.Commands = cliCommands
	app.CommandNotFound = commandNotFound
	app.Version = "2.4.0"

	cli.AppHelpTemplate = cli.AppHelpTemplate + environmentVariableHelp + "\n"

	trace.NewLogger(os.Getenv(RTR_TRACE))

	app.Run(os.Args)
	os.Exit(0)
}

func registerRoutes(c *cli.Context) {
	issues := checkFlags(c)
	errorMessage := "route registration failed:"
	issues = append(issues, checkArguments(c, "register")...)

	if len(issues) > 0 {
		printHelpForCommand(c, issues, "register")
	}

	desiredRoutes := c.Args().First()
	var routes []db.Route

	err := json.Unmarshal([]byte(desiredRoutes), &routes)
	checkError(errorMessage, err)

	client, err := newRoutingApiClient(c)
	checkError(errorMessage, err)

	err = commands.Register(client, routes)
	checkError(errorMessage, err)

	fmt.Printf("Successfully registered routes: %s\n", desiredRoutes)
}

func unregisterRoutes(c *cli.Context) {
	issues := checkFlags(c)
	errorMessage := "route unregistration failed:"
	issues = append(issues, checkArguments(c, "unregister")...)

	if len(issues) > 0 {
		printHelpForCommand(c, issues, "unregister")
	}

	desiredRoutes := c.Args().First()
	var routes []db.Route
	err := json.Unmarshal([]byte(desiredRoutes), &routes)
	checkError(errorMessage, err)

	client, err := newRoutingApiClient(c)
	checkError(errorMessage, err)

	err = commands.UnRegister(client, routes)
	checkError(errorMessage, err)

	fmt.Printf("Successfully unregistered routes: %s\n", desiredRoutes)
}

func listRoutes(c *cli.Context) {
	errorMessage := "listing routes failed:"
	issues := checkFlags(c)
	issues = append(issues, checkArguments(c, "list")...)

	if len(issues) > 0 {
		printHelpForCommand(c, issues, "list")
	}

	client, err := newRoutingApiClient(c)
	checkError(errorMessage, err)

	routes, err := commands.List(client)
	if err != nil {
		fmt.Println("listing routes failed:", err)
		os.Exit(3)
	}

	prettyRoutes, _ := json.Marshal(routes)

	fmt.Printf("%v\n", string(prettyRoutes))
}

func streamEvents(c *cli.Context) {
	issues := checkFlags(c)
	issues = append(issues, checkArguments(c, "events")...)

	if len(issues) > 0 {
		printHelpForCommand(c, issues, "events")
	}

	streamHttp := c.Bool("http")
	streamTcp := c.Bool("tcp")

	if !streamHttp && !streamTcp {
		streamHttp = true
		streamTcp = true
	}

	client, err := newRoutingApiClient(c)
	checkError("streaming events failed:", err)
	errorChan := make(chan error)
	eventChan := make(chan string)

	numOfSubscriptions := 0

	if streamHttp {
		numOfSubscriptions++
		go streamHttpEvents(client, eventChan, errorChan)
	}

	if streamTcp {
		numOfSubscriptions++
		go streamTcpEvents(client, eventChan, errorChan)
	}

	errorCount := 0

loop:
	for {
		select {
		case eventMessage := <-eventChan:
			fmt.Println(eventMessage)
		case err := <-errorChan:
			errorCount++
			fmt.Printf("Connection closed: %s", err.Error())
			if errorCount >= numOfSubscriptions {
				break loop
			}
		}
	}
}

func streamHttpEvents(client routing_api.Client, eventChan chan string, errorChan chan error) {
	eventSource, err := client.SubscribeToEvents()
	if err != nil {
		fmt.Println("streaming events failed:", err)
		return
	}
	for {
		e, err := eventSource.Next()
		if err != nil {
			errorChan <- err
			break
		}

		event, _ := json.Marshal(e)
		eventChan <- fmt.Sprintf("%v\n", string(event))
	}
}

func streamTcpEvents(client routing_api.Client, eventChan chan string, errorChan chan error) {
	eventSource, err := client.SubscribeToTcpEvents()
	if err != nil {
		fmt.Println("streaming events failed:", err)
		return
	}
	for {
		e, err := eventSource.Next()
		if err != nil {
			errorChan <- err
			break
		}

		event, _ := json.Marshal(e)
		eventChan <- fmt.Sprintf("%v\n", string(event))
	}
}

func buildOauthConfig(c *cli.Context) *uaaconfig.Config {

	return &uaaconfig.Config{
		UaaEndpoint:           c.String("oauth-url"),
		SkipVerification:      c.Bool("skip-oauth-tls-verification"),
		ClientName:            c.String("client-id"),
		ClientSecret:          c.String("client-secret"),
		MaxNumberOfRetries:    3,
		RetryInterval:         500 * time.Millisecond,
		ExpirationBufferInSec: 30,
	}

}

func checkFlags(c *cli.Context) []string {
	var issues []string

	if c.String("api") == "" {
		issues = append(issues, "Must provide an API endpoint for the routing-api component.")
	}

	if c.String("client-id") == "" {
		issues = append(issues, "Must provide the id of an OAuth client.")
	}

	if c.String("client-secret") == "" {
		issues = append(issues, "Must provide an OAuth secret.")
	}

	if c.String("oauth-url") == "" {
		issues = append(issues, "Must provide an URL to the OAuth client.")
	}

	_, err := url.Parse(c.String("oauth-url"))
	if err != nil {
		issues = append(issues, "Invalid OAuth client URL")
	}

	return issues
}

func checkArguments(c *cli.Context, cmd string) []string {
	var issues []string

	switch cmd {
	case "register", "unregister":
		if len(c.Args()) > 1 {
			issues = append(issues, "Unexpected arguments.")
		} else if len(c.Args()) < 1 {
			issues = append(issues, "Must provide routes JSON.")
		}
	case "list", "events":
		if len(c.Args()) > 0 {
			issues = append(issues, "Unexpected arguments.")
		}
	}

	return issues
}

func printHelpForCommand(c *cli.Context, issues []string, cmd string) {
	for _, issue := range issues {
		fmt.Println(issue)
	}
	fmt.Println()
	cli.ShowCommandHelp(c, cmd)
	os.Exit(1)
}

func commandNotFound(c *cli.Context, cmd string) {
	fmt.Println("Not a valid command:", cmd)
	os.Exit(1)
}

func newRoutingApiClient(c *cli.Context) (routing_api.Client, error) {

	uaaClient, err := newUaaClient(c)
	if err != nil {
		return nil, err
	}

	token, err := uaaClient.FetchToken(true)
	if err != nil {
		return nil, err
	}

	routingApiClient := routing_api.NewClient(c.String("api"))

	routingApiClient.SetToken(token.AccessToken)

	return routingApiClient, nil

}

func checkError(message string, err error) {
	if err != nil {
		fmt.Println(message, err.Error())
		os.Exit(3)
	}
}

func newUaaClient(c *cli.Context) (uaaclient.Client, error) {

	logger, _ := cf_lager.New("rtr")
	cfg := buildOauthConfig(c)
	klok := clock.NewClock()

	uaaClient, err := uaaclient.NewClient(logger, cfg, klok)
	if err != nil {
		return nil, err
	}
	_, err = uaaClient.FetchToken(true)

	return uaaClient, nil
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"runtime"

	"github.com/Solher/zest/infrastructure"
	"github.com/Solher/zest/interfaces"
	"github.com/Solher/zest/middlewares"
	"github.com/Solher/zest/ressources"
	"github.com/Solher/zest/usecases"
	"github.com/codegangsta/negroni"
	"github.com/dimfeld/httptreemux"

	_ "github.com/clipperhouse/typewriter" // Forced to allow vendoring.
)

var Environment, Port, DatabaseURL string

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	env := os.Getenv("GOENV")
	if env == "development" || env == "production" {
		Environment = env
	} else {
		Environment = "development"
	}

	port := os.Getenv("PORT")
	if port != "" {
		Port = ":" + os.Getenv("PORT")
	} else {
		Port = ":3000"
	}

	DatabaseURL = os.Getenv("DATABASE_URL")
}

func main() {
	mustExit := handleOsArgs()
	if mustExit {
		return
	}

	app := negroni.New()
	router := httptreemux.New()
	router.RedirectBehavior = httptreemux.UseHandler
	render := infrastructure.NewRender()
	store := infrastructure.NewGormStore()
	sessionCache := infrastructure.NewLRUCacheStore(1024)
	roleCache := infrastructure.NewCacheStore()
	aclCache := infrastructure.NewCacheStore()

	initApp(app, router, render, store, sessionCache, roleCache, aclCache)
	defer closeApp(store)

	app.Run(Port)
}

func handleOsArgs() bool {
	if len(os.Args) > 1 {
		switch os.Args[1] {
		case "resetDB":
			resetDatabase()
			return true
		case "migrateDB":
			migrateDatabase()
			return true
		}
	}

	return false
}

func initApp(app *negroni.Negroni, router *httptreemux.TreeMux, render *infrastructure.Render,
	store *infrastructure.GormStore, lruCacheStore *infrastructure.LRUCacheStore, roleCacheStore, aclCacheStore *infrastructure.CacheStore) {

	err := connectDB(store)
	if err != nil {
		panic("Could not connect to database.")
	}

	userRepository := ressources.NewUserRepo(store)
	sessionRepository := ressources.NewSessionRepo(store)
	accountRepository := ressources.NewAccountRepo(store)
	roleMappingRepository := ressources.NewRoleMappingRepo(store)
	roleRepository := ressources.NewRoleRepo(store)
	aclMappingRepository := ressources.NewAclMappingRepo(store)
	aclRepository := ressources.NewAclRepo(store)

	sessionCacheInter := usecases.NewSessionCacheInter(sessionRepository, lruCacheStore)
	permissionCacheInter := usecases.NewPermissionCacheInter(accountRepository, aclRepository, roleCacheStore, aclCacheStore)
	sessionCacheInter.Refresh()
	permissionCacheInter.Refresh()

	userInteractor := ressources.NewUserInter(userRepository)
	sessionInteractor := ressources.NewSessionInter(sessionRepository)
	accountInteractor := ressources.NewAccountInter(accountRepository, userInteractor, sessionInteractor, sessionCacheInter, permissionCacheInter)
	roleMappingInteractor := ressources.NewRoleMappingInter(roleMappingRepository)
	roleInteractor := ressources.NewRoleInter(roleRepository)
	aclMappingInteractor := ressources.NewAclMappingInter(aclMappingRepository)
	aclInteractor := ressources.NewAclInter(aclRepository)

	routes := interfaces.NewRouteDirectory(accountInteractor, render)

	ressources.NewUserCtrl(userInteractor, render, routes)
	ressources.NewSessionCtrl(sessionInteractor, render, routes)
	ressources.NewAccountCtrl(accountInteractor, render, routes)
	ressources.NewRoleMappingCtrl(roleMappingInteractor, render, routes)
	ressources.NewRoleCtrl(roleInteractor, render, routes)
	ressources.NewAclMappingCtrl(aclMappingInteractor, render, routes)
	ressources.NewAclCtrl(aclInteractor, render, routes)

	routes.Register(router)

	app.Use(negroni.NewLogger())
	app.Use(negroni.NewRecovery())
	app.Use(middlewares.NewSessions(accountInteractor))

	app.UseHandler(router)
}

func closeApp(store *infrastructure.GormStore) {
	store.Close()
}

func connectDB(store *infrastructure.GormStore) error {
	var err error

	if DatabaseURL != "" {
		err = store.Connect("postgres", DatabaseURL)
	} else {
		err = store.Connect("sqlite3", "database.db")
	}

	return err
}
<commit_msg>CORS middleware added.<commit_after>package main

import (
	"os"
	"runtime"

	"github.com/Solher/zest/infrastructure"
	"github.com/Solher/zest/interfaces"
	"github.com/Solher/zest/middlewares"
	"github.com/Solher/zest/ressources"
	"github.com/Solher/zest/usecases"
	"github.com/codegangsta/negroni"
	"github.com/dimfeld/httptreemux"
	"github.com/rs/cors"

	_ "github.com/clipperhouse/typewriter" // Forced to allow vendoring.
)

var Environment, Port, DatabaseURL string

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	env := os.Getenv("GOENV")
	if env == "development" || env == "production" {
		Environment = env
	} else {
		Environment = "development"
	}

	port := os.Getenv("PORT")
	if port != "" {
		Port = ":" + os.Getenv("PORT")
	} else {
		Port = ":3000"
	}

	DatabaseURL = os.Getenv("DATABASE_URL")
}

func main() {
	mustExit := handleOsArgs()
	if mustExit {
		return
	}

	app := negroni.New()
	router := httptreemux.New()
	router.RedirectBehavior = httptreemux.UseHandler
	render := infrastructure.NewRender()
	store := infrastructure.NewGormStore()
	sessionCache := infrastructure.NewLRUCacheStore(1024)
	roleCache := infrastructure.NewCacheStore()
	aclCache := infrastructure.NewCacheStore()

	initApp(app, router, render, store, sessionCache, roleCache, aclCache)
	defer closeApp(store)

	app.Run(Port)
}

func handleOsArgs() bool {
	if len(os.Args) > 1 {
		switch os.Args[1] {
		case "resetDB":
			resetDatabase()
			return true
		case "migrateDB":
			migrateDatabase()
			return true
		}
	}

	return false
}

func initApp(app *negroni.Negroni, router *httptreemux.TreeMux, render *infrastructure.Render,
	store *infrastructure.GormStore, lruCacheStore *infrastructure.LRUCacheStore, roleCacheStore, aclCacheStore *infrastructure.CacheStore) {

	err := connectDB(store)
	if err != nil {
		panic("Could not connect to database.")
	}

	userRepository := ressources.NewUserRepo(store)
	sessionRepository := ressources.NewSessionRepo(store)
	accountRepository := ressources.NewAccountRepo(store)
	roleMappingRepository := ressources.NewRoleMappingRepo(store)
	roleRepository := ressources.NewRoleRepo(store)
	aclMappingRepository := ressources.NewAclMappingRepo(store)
	aclRepository := ressources.NewAclRepo(store)

	sessionCacheInter := usecases.NewSessionCacheInter(sessionRepository, lruCacheStore)
	permissionCacheInter := usecases.NewPermissionCacheInter(accountRepository, aclRepository, roleCacheStore, aclCacheStore)
	sessionCacheInter.Refresh()
	permissionCacheInter.Refresh()

	userInteractor := ressources.NewUserInter(userRepository)
	sessionInteractor := ressources.NewSessionInter(sessionRepository)
	accountInteractor := ressources.NewAccountInter(accountRepository, userInteractor, sessionInteractor, sessionCacheInter, permissionCacheInter)
	roleMappingInteractor := ressources.NewRoleMappingInter(roleMappingRepository)
	roleInteractor := ressources.NewRoleInter(roleRepository)
	aclMappingInteractor := ressources.NewAclMappingInter(aclMappingRepository)
	aclInteractor := ressources.NewAclInter(aclRepository)

	routes := interfaces.NewRouteDirectory(accountInteractor, render)

	ressources.NewUserCtrl(userInteractor, render, routes)
	ressources.NewSessionCtrl(sessionInteractor, render, routes)
	ressources.NewAccountCtrl(accountInteractor, render, routes)
	ressources.NewRoleMappingCtrl(roleMappingInteractor, render, routes)
	ressources.NewRoleCtrl(roleInteractor, render, routes)
	ressources.NewAclMappingCtrl(aclMappingInteractor, render, routes)
	ressources.NewAclCtrl(aclInteractor, render, routes)

	routes.Register(router)

	app.Use(negroni.NewLogger())
	app.Use(negroni.NewRecovery())
	app.Use(cors.Default())
	app.Use(middlewares.NewSessions(accountInteractor))

	app.UseHandler(router)
}

func closeApp(store *infrastructure.GormStore) {
	store.Close()
}

func connectDB(store *infrastructure.GormStore) error {
	var err error

	if DatabaseURL != "" {
		err = store.Connect("postgres", DatabaseURL)
	} else {
		err = store.Connect("sqlite3", "database.db")
	}

	return err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
	"sort"
	"time"
)

func main() {
	start := time.Now()
	// 処理
	var i float64
	i = 0
	for {
		i++
		end := time.Now()
		diff := float64(end.Sub(start) / 1000 / 1000 / 1000)
		m := int(i / diff)
		out := 0
		if m < 0 {
		} else {
			out = m
		}
		fmt.Printf("\r")
		fmt.Printf("\r%v回試行  %v秒経過 %v回/秒", i, diff, out)
		if submain() {
			break
		}
	}
	fmt.Printf("\n")
}

func submain() bool {
	/*
		fmt.Print("started.")
		// チャネル
		sleep1_finished := make(chan bool)

		go func() {
			// 0.2秒かかるコマンド
			fmt.Print("sleep1 started.")
			time.Sleep(200 * 1000 * 1000 * time.Nanosecond)
			fmt.Print("sleep1 finished.")
			sleep1_finished <- true
		}()

		// 終わるまで待つ
		<- sleep1_finished
	*/

	// 136枚の中から14枚返したい
	list := shuffled_cards()

	// 出力
	result := solve(list)
	string_output(list)
	return result
}

// http://d.hatena.ne.jp/hake/20150930/p1
func shuffle(list []int) {
	for i := len(list); i > 1; i-- {
		j := rand.Intn(i) // 0　.. i-1 の乱数発生
		list[i-1], list[j] = list[j], list[i-1]
	}
}

func shuffled_cards() []int {
	rand.Seed(time.Now().UnixNano())

	// データ要素数指定、および初期データ作成
	size := 136
	list := make([]int, size, size)
	for i := 0; i < size; i++ {
		list[i] = i
	}

	// シャッフル
	shuffle(list)

	return list[:14]
}

// 牌の出力
func string_output(list []int) {
	// http://qiita.com/ruiu/items/2bb83b29baeae2433a79
	// サイズ0、内部バッファの長さ69の[]byteの値を割り当てる
	b := make([]byte, 0, 70)

	// bに文字列を追加
	for j := 0; j < 14; j++ {
		// 126976 is 'ton'
		// https://codepoints.net/U+1F000
		b = append(b, string(list[j]/4+126976)...) // ...が必要
		b = append(b, string(32)...)               // ...が必要
	}
	fmt.Printf("%v", string(b))
}

// solve
// 字マンソーピンのリストをつくる
func solve(list []int) bool {
	if is_chitoitsu(list) {
		return true
	}
	matrix := [][]int{{}, {}, {}, {}}
	for _, value := range list {
		group(matrix, value)
	}
	//fmt.Printf("%v", matrix)

	return group_scan(matrix)
}

// スート分類してくれる
func group(m [][]int, j int) {
	// 4で割ると本来のインデックスに
	i := j / 4
	switch {
	case i < 7:
		m[0] = append(m[0], i)
	case i < 7+(9*1):
		m[1] = append(m[1], i-7)
	case i < 7+(9*2):
		m[2] = append(m[2], i-7-(9*1))
	case i < 7+(9*3):
		m[3] = append(m[3], i-7-(9*2))
	}
}

func group_scan(m [][]int) bool {
	if !valid_mod3(m) {
		return false
	}
	if !valid_33332(m) {
		return false
	}
	return true
}

func valid_mod3(m [][]int) bool {
	//スートのサイズを3で割った時
	//あまりが2であるすーとグループが1つであること
	c := 0
	for _, a := range m {
		switch len(a) % 3 {
		case 0:
			// noop
		case 1:
			return false
		case 2:
			c++
		}
	}
	return c == 1
}

func valid_33332(m [][]int) bool {
	for i := 0; i < 4; i++ {
		if !valid_suit_group(m[i], i) {
			return false
		}
	}
	return true
}

func valid_suit_group(a []int, i int) bool {
	// 第二引数は字牌のとき0
	for _, v := range a {
		// 4で割ると本来のインデックスに
		v = v / 4
	}

	//ソート
	sort.Ints(a)
	if len(a)%3 == 2 {
		//ペアを探す
		pair_numbers := pairable_numbers(a)
		//ペア候補がなかったらぬける
		if len(pair_numbers) == 0 {
			return false
		}
		//ペア候補毎に繰り返し処理
		for _, v := range pair_numbers {
			//ペアとなる２枚を除去
			rest := []int{}
			c := 2
			for _, w := range a {
				// ペア候補以外は新スライスに入れる
				// ペア候補は３枚目以降は新スライスに入れる
				if w != v || c <= 0 {
					rest = append(rest, w)
				}
				if w == v {
					c--
				}
			}
			if valid_3cards(rest, i) {
				return true
			} else {
				continue
			}
		}
		return false
	} else if len(a)%3 == 0 {
		return valid_3cards(a, i)
	}
	// 来ないはず
	return false
}

func valid_3cards(a []int, i int) bool {
	// 刻子や順子のみで構成されている場合true
	// a is sorted
	// a.size % 3 is0
	// 第二引数は字牌のとき0
	ok := false
	for {
		a, ok = remove_kotsu(a)
		if ok {
			continue
		}
		if i > 0 {
			a, ok = remove_shuntsu(a)
			if ok {
				continue
			}
		}
		return len(a) == 0
	}
}

func remove_kotsu(a []int) ([]int, bool) {
	// 刻子を除去できればtrue
	// a is sorted
	retval := a
	if len(a) < 3 {
		return retval, false
	}
	if a[0] == a[1] && a[0] == a[2] {
		retval = a[3:]
		return retval, true
	}
	return retval, false
}

func remove_shuntsu(a []int) ([]int, bool) {
	// 順子を除去できればtrue
	// a is sorted
	rest := []int{}
	first := -1
	second := -1
	found := false
	for _, v := range a {
		if found {
			rest = append(rest, v)
			continue
		}
		if first == -1 {
			first = v
		} else if second == -1 && first+1 == v {
			second = v
		} else if second != -1 && first+2 == v {
			//flush
			first = -1
			second = -1
			found = true
		} else {
			rest = append(rest, v)
		}
	}
	return rest, found
}

func pairable_numbers(sorted []int) []int {
	//カウンタ
	retval := []int{}
	a := 999 // 2つ前
	b := 999 // 1つ前
	for _, v := range sorted {
		if b == v && a != v {
			retval = append(retval, v)
		} else {
			b = v
		}
	}
	return retval
}

func is_chitoitsu(list []int) bool {
	//カウンタ
	c := map[int]int{}

	//コピー
	l := list

	for _, v := range l {
		// 4で割ると本来のインデックスに
		v = v / 4
		count, ok := c[v]
		if ok {
			if count == 1 {
				c[v] = 2
			} else {
				// c[v] == 2
				return false
			}
		} else {
			c[v] = 1
		}
		//8個チェック
		if len(c) >= 8 {
			return false
		}
	}
	return true
}
<commit_msg>Refactor 冗長な割り算<commit_after>package main

import (
	"fmt"
	"math/rand"
	"sort"
	"time"
)

func main() {
	start := time.Now()
	// 処理
	var i float64
	i = 0
	for {
		i++
		end := time.Now()
		diff := float64(end.Sub(start) / 1000 / 1000 / 1000)
		m := int(i / diff)
		out := 0
		if m < 0 {
		} else {
			out = m
		}
		fmt.Printf("\r")
		fmt.Printf("\r%v回試行  %v秒経過 %v回/秒", i, diff, out)
		if submain() {
			break
		}
	}
	fmt.Printf("\n")
}

func submain() bool {
	/*
		fmt.Print("started.")
		// チャネル
		sleep1_finished := make(chan bool)

		go func() {
			// 0.2秒かかるコマンド
			fmt.Print("sleep1 started.")
			time.Sleep(200 * 1000 * 1000 * time.Nanosecond)
			fmt.Print("sleep1 finished.")
			sleep1_finished <- true
		}()

		// 終わるまで待つ
		<- sleep1_finished
	*/

	// 136枚の中から14枚返したい
	list := shuffled_cards()

	// 出力
	result := solve(list)
	string_output(list)
	return result
}

// http://d.hatena.ne.jp/hake/20150930/p1
func shuffle(list []int) {
	for i := len(list); i > 1; i-- {
		j := rand.Intn(i) // 0　.. i-1 の乱数発生
		list[i-1], list[j] = list[j], list[i-1]
	}
}

func shuffled_cards() []int {
	rand.Seed(time.Now().UnixNano())

	// データ要素数指定、および初期データ作成
	size := 136
	list := make([]int, size, size)
	for i := 0; i < size; i++ {
		list[i] = i / 4
	}

	// シャッフル
	shuffle(list)

	return list[:14]
}

// 牌の出力
func string_output(list []int) {
	// http://qiita.com/ruiu/items/2bb83b29baeae2433a79
	// サイズ0、内部バッファの長さ69の[]byteの値を割り当てる
	b := make([]byte, 0, 70)

	// bに文字列を追加
	for j := 0; j < 14; j++ {
		// 126976 is 'ton'
		// https://codepoints.net/U+1F000
		b = append(b, string(list[j]+126976)...) // ...が必要
		b = append(b, string(32)...)             // ...が必要
	}
	fmt.Printf("%v", string(b))
}

// solve
// 字マンソーピンのリストをつくる
func solve(list []int) bool {
	if is_chitoitsu(list) {
		return true
	}
	matrix := [][]int{{}, {}, {}, {}}
	for _, value := range list {
		group(matrix, value)
	}
	//fmt.Printf("%v", matrix)

	return group_scan(matrix)
}

// スート分類してくれる
func group(m [][]int, i int) {
	switch {
	case i < 7:
		m[0] = append(m[0], i)
	case i < 7+(9*1):
		m[1] = append(m[1], i-7)
	case i < 7+(9*2):
		m[2] = append(m[2], i-7-(9*1))
	case i < 7+(9*3):
		m[3] = append(m[3], i-7-(9*2))
	}
}

func group_scan(m [][]int) bool {
	if !valid_mod3(m) {
		return false
	}
	if !valid_33332(m) {
		return false
	}
	return true
}

func valid_mod3(m [][]int) bool {
	//スートのサイズを3で割った時
	//あまりが2であるすーとグループが1つであること
	c := 0
	for _, a := range m {
		switch len(a) % 3 {
		case 0:
			// noop
		case 1:
			return false
		case 2:
			c++
		}
	}
	return c == 1
}

func valid_33332(m [][]int) bool {
	for i := 0; i < 4; i++ {
		if !valid_suit_group(m[i], i) {
			return false
		}
	}
	return true
}

func valid_suit_group(a []int, i int) bool {
	// 第二引数は字牌のとき0

	//ソート
	sort.Ints(a)
	if len(a)%3 == 2 {
		//ペアを探す
		pair_numbers := pairable_numbers(a)
		//ペア候補がなかったらぬける
		if len(pair_numbers) == 0 {
			return false
		}
		//ペア候補毎に繰り返し処理
		for _, v := range pair_numbers {
			//ペアとなる２枚を除去
			rest := []int{}
			c := 2
			for _, w := range a {
				// ペア候補以外は新スライスに入れる
				// ペア候補は３枚目以降は新スライスに入れる
				if w != v || c <= 0 {
					rest = append(rest, w)
				}
				if w == v {
					c--
				}
			}
			if valid_3cards(rest, i) {
				return true
			} else {
				continue
			}
		}
		return false
	} else if len(a)%3 == 0 {
		return valid_3cards(a, i)
	}
	// 来ないはず
	return false
}

func valid_3cards(a []int, i int) bool {
	// 刻子や順子のみで構成されている場合true
	// a is sorted
	// a.size % 3 is0
	// 第二引数は字牌のとき0
	ok := false
	for {
		a, ok = remove_kotsu(a)
		if ok {
			continue
		}
		if i > 0 {
			a, ok = remove_shuntsu(a)
			if ok {
				continue
			}
		}
		return len(a) == 0
	}
}

func remove_kotsu(a []int) ([]int, bool) {
	// 刻子を除去できればtrue
	// a is sorted
	retval := a
	if len(a) < 3 {
		return retval, false
	}
	if a[0] == a[1] && a[0] == a[2] {
		retval = a[3:]
		return retval, true
	}
	return retval, false
}

func remove_shuntsu(a []int) ([]int, bool) {
	// 順子を除去できればtrue
	// a is sorted
	rest := []int{}
	first := -1
	second := -1
	found := false
	for _, v := range a {
		if found {
			rest = append(rest, v)
			continue
		}
		if first == -1 {
			first = v
		} else if second == -1 && first+1 == v {
			second = v
		} else if second != -1 && first+2 == v {
			//flush
			first = -1
			second = -1
			found = true
		} else {
			rest = append(rest, v)
		}
	}
	return rest, found
}

func pairable_numbers(sorted []int) []int {
	//カウンタ
	retval := []int{}
	a := 999 // 2つ前
	b := 999 // 1つ前
	for _, v := range sorted {
		if b == v && a != v {
			retval = append(retval, v)
		} else {
			b = v
		}
	}
	return retval
}

func is_chitoitsu(list []int) bool {
	//カウンタ
	c := map[int]int{}

	//コピー
	l := list

	for _, v := range l {
		count, ok := c[v]
		if ok {
			if count == 1 {
				c[v] = 2
			} else {
				// c[v] == 2
				return false
			}
		} else {
			c[v] = 1
		}
		//8個チェック
		if len(c) >= 8 {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/evolsnow/robot/conn"
	"github.com/gorilla/websocket"
	"log"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"
)

var upgrader = websocket.Upgrader{CheckOrigin: func(r *http.Request) bool {
	return true
}} // use default options for webSocket
var visitor = 0

func main() {
	var configFile string
	var debug bool

	flag.StringVar(&configFile, "c", "config.json", "specify config file")
	flag.BoolVar(&debug, "d", false, "debug mode")
	flag.Parse()
	config, err := ParseConfig(configFile)
	if err != nil {
		log.Fatal("a vailid json config file must exist")
	}

	//connect to redis
	redisPort := strconv.Itoa(config.RedisPort)
	redisServer := net.JoinHostPort(config.RedisAddress, redisPort)
	if !conn.Ping(redisServer, config.RedisPassword) {
		log.Fatal("connect to redis server failed")
	}
	conn.Pool = conn.NewPool(redisServer, config.RedisPassword, config.RedisDB)

	//create robot and run
	robot := newRobot(config.RobotToken, config.RobotName, config.WebHookUrl)
	robot.bot.Debug = debug
	go robot.run()

	//run server and web samaritan
	srvPort := strconv.Itoa(config.Port)
	http.HandleFunc("/ajax", ajax)
	http.HandleFunc("/websocket", socketHandler)
	http.HandleFunc("/groupTalk", groupTalk)
	log.Fatal(http.ListenAndServeTLS(net.JoinHostPort(config.Server, srvPort), config.Cert, config.CertKey, nil))

}

func groupTalk(w http.ResponseWriter, r *http.Request) {
	c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Print("upgrade:", err)
		return
	}
	defer c.Close()
	visitor++
	tlChan := make(chan string, 5)
	qinChan := make(chan string, 5)
	iceChan := make(chan string, 5)
	initSentence := "你好"
	tlChan <- qinAI(initSentence)
	go func() {
		for {
			if visitor > 0 {
				msgToTl := <-tlChan
				replyFromTl := tlAI(msgToTl)
				qinChan <- replyFromTl
				if cap(iceChan) > 0 {
					iceChan <- replyFromTl
				}
				c.WriteMessage(websocket.TextMessage, []byte("samaritan: "+replyFromTl))

			}
		}
	}()

	go func() {
		for {
			if visitor > 0 {
				msgToQin := <-qinChan
				replyFromQin := qinAI(msgToQin)
				tlChan <- replyFromQin
				if cap(iceChan) > 0 {
					iceChan <- replyFromQin
				}
				c.WriteMessage(websocket.TextMessage, []byte("菲菲: "+replyFromQin))

			}
		}
	}()

	go func() {
		for {
			msgToIce := <-iceChan
			replyFromIce := iceAI(msgToIce)
			tlChan <- replyFromIce
			qinChan <- replyFromIce
			c.WriteMessage(websocket.TextMessage, []byte("小冰: "+replyFromIce))

		}
	}()
	for {

		_, _, err := c.ReadMessage()
		if err != nil {
			log.Println("read:", err)
			visitor--
			break
		}

	}
}

//used for web samaritan robot
func socketHandler(w http.ResponseWriter, r *http.Request) {
	c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Print("upgrade:", err)
		return
	}
	defer c.Close()
	for {
		var in []byte
		var ret []string
		mt, in, err := c.ReadMessage()
		if err != nil {
			log.Println("read:", err)
			break
		}
		ret = receive(string(in))
		for i := range ret {
			c.WriteMessage(mt, []byte(ret[i]))
			time.Sleep(time.Second)
		}
		c.WriteMessage(mt, []byte(""))
	}
}

//when webSocket unavailable, fallback to ajax long polling
func ajax(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Access-Control-Allow-Origin", "*")
	var messages = make(chan string)
	if r.Method == "GET" {
		w.Write([]byte(<-messages))
	} else {
		body := r.FormValue("text")
		if body != "" {
			go func(string) {
				ret := receive(body)
				for i := range ret {
					messages <- ret[i]
					time.Sleep(time.Second)
				}
				messages <- ""
			}(body)
		}
	}
}

//receive from client
func receive(in string) (ret []string) {
	defer func() {
		if p := recover(); p != nil {
			err := fmt.Errorf("client closed error: %v", p)
			log.Println(err)
		}
	}()
	fmt.Printf("Received: %s\n", in)
	var response string
	var answer = make(chan string)
	sf := func(c rune) bool {
		return c == ',' || c == '，' || c == ';' || c == '。' || c == '.' || c == '？' || c == '?'
	}
	if chinese(in) {
		go func() {
			answer <- iceAI(in)
		}()
		go func() {
			answer <- tlAI(in)
		}()
		go func() {
			ret := qinAI(in)
			if ret != "" {
				answer <- strings.Replace(ret, "Jarvis", "samaritan", -1)
			}
		}()
		response = <-answer
		// Separate into fields with func.
		ret = strings.FieldsFunc(response, sf)

	} else {
		response = mitAI(in)
		ret = strings.FieldsFunc(response, sf)
	}
	return
}
<commit_msg>len<cap<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/evolsnow/robot/conn"
	"github.com/gorilla/websocket"
	"log"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"
)

var upgrader = websocket.Upgrader{CheckOrigin: func(r *http.Request) bool {
	return true
}} // use default options for webSocket
var visitor = 0

func main() {
	var configFile string
	var debug bool

	flag.StringVar(&configFile, "c", "config.json", "specify config file")
	flag.BoolVar(&debug, "d", false, "debug mode")
	flag.Parse()
	config, err := ParseConfig(configFile)
	if err != nil {
		log.Fatal("a vailid json config file must exist")
	}

	//connect to redis
	redisPort := strconv.Itoa(config.RedisPort)
	redisServer := net.JoinHostPort(config.RedisAddress, redisPort)
	if !conn.Ping(redisServer, config.RedisPassword) {
		log.Fatal("connect to redis server failed")
	}
	conn.Pool = conn.NewPool(redisServer, config.RedisPassword, config.RedisDB)

	//create robot and run
	robot := newRobot(config.RobotToken, config.RobotName, config.WebHookUrl)
	robot.bot.Debug = debug
	go robot.run()

	//run server and web samaritan
	srvPort := strconv.Itoa(config.Port)
	http.HandleFunc("/ajax", ajax)
	http.HandleFunc("/websocket", socketHandler)
	http.HandleFunc("/groupTalk", groupTalk)
	log.Fatal(http.ListenAndServeTLS(net.JoinHostPort(config.Server, srvPort), config.Cert, config.CertKey, nil))

}

func groupTalk(w http.ResponseWriter, r *http.Request) {
	c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Print("upgrade:", err)
		return
	}
	defer c.Close()
	visitor++
	tlChan := make(chan string, 5)
	qinChan := make(chan string, 5)
	iceChan := make(chan string, 5)
	initSentence := "你好"
	tlChan <- qinAI(initSentence)
	go func() {
		for {
			if visitor > 0 {
				msgToTl := <-tlChan
				replyFromTl := tlAI(msgToTl)
				qinChan <- replyFromTl
				if len(iceChan) < cap(iceChan) {
					iceChan <- replyFromTl
				}
				c.WriteMessage(websocket.TextMessage, []byte("samaritan: "+replyFromTl))

			}
		}
	}()

	go func() {
		for {
			if visitor > 0 {
				msgToQin := <-qinChan
				replyFromQin := qinAI(msgToQin)
				tlChan <- replyFromQin
				if len(iceChan) < cap(iceChan) {
					iceChan <- replyFromQin
				}
				c.WriteMessage(websocket.TextMessage, []byte("菲菲: "+replyFromQin))

			}
		}
	}()

	go func() {
		for {
			if visitor > 0 {
				msgToIce := <-iceChan
				replyFromIce := iceAI(msgToIce)
				tlChan <- replyFromIce
				qinChan <- replyFromIce
				c.WriteMessage(websocket.TextMessage, []byte("小冰: "+replyFromIce))

			}
		}
	}()
	for {

		_, _, err := c.ReadMessage()
		if err != nil {
			log.Println("read:", err)
			visitor--
			break
		}

	}
}

//used for web samaritan robot
func socketHandler(w http.ResponseWriter, r *http.Request) {
	c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Print("upgrade:", err)
		return
	}
	defer c.Close()
	for {
		var in []byte
		var ret []string
		mt, in, err := c.ReadMessage()
		if err != nil {
			log.Println("read:", err)
			break
		}
		ret = receive(string(in))
		for i := range ret {
			c.WriteMessage(mt, []byte(ret[i]))
			time.Sleep(time.Second)
		}
		c.WriteMessage(mt, []byte(""))
	}
}

//when webSocket unavailable, fallback to ajax long polling
func ajax(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Access-Control-Allow-Origin", "*")
	var messages = make(chan string)
	if r.Method == "GET" {
		w.Write([]byte(<-messages))
	} else {
		body := r.FormValue("text")
		if body != "" {
			go func(string) {
				ret := receive(body)
				for i := range ret {
					messages <- ret[i]
					time.Sleep(time.Second)
				}
				messages <- ""
			}(body)
		}
	}
}

//receive from client
func receive(in string) (ret []string) {
	defer func() {
		if p := recover(); p != nil {
			err := fmt.Errorf("client closed error: %v", p)
			log.Println(err)
		}
	}()
	fmt.Printf("Received: %s\n", in)
	var response string
	var answer = make(chan string)
	sf := func(c rune) bool {
		return c == ',' || c == '，' || c == ';' || c == '。' || c == '.' || c == '？' || c == '?'
	}
	if chinese(in) {
		go func() {
			answer <- iceAI(in)
		}()
		go func() {
			answer <- tlAI(in)
		}()
		go func() {
			ret := qinAI(in)
			if ret != "" {
				answer <- strings.Replace(ret, "Jarvis", "samaritan", -1)
			}
		}()
		response = <-answer
		// Separate into fields with func.
		ret = strings.FieldsFunc(response, sf)

	} else {
		response = mitAI(in)
		ret = strings.FieldsFunc(response, sf)
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2014 The gofsh Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"log"
	"net/http"
)

var (
	addr = flag.String("addr", ":6060", "listening address")
	root = flag.String("root", ".", "root folder")
)

func main() {
	flag.Parse()
	log.Printf("Listening on %s (root=%s)\n", *addr, *root)
	if err := http.ListenAndServe(*addr, http.HandlerFunc(logHandler)); err != nil {
		log.Fatalln("Error:", err)
	}
}

func logHandler(w http.ResponseWriter, r *http.Request) {
	log.Printf("%s - %s\n", r.RemoteAddr, r.RequestURI)
	fileHandler := http.FileServer(http.Dir(*root))
	fileHandler.ServeHTTP(w, r)
}
<commit_msg>Refactoring<commit_after>// Copyright 2014 The gofsh Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"log"
	"net/http"
)

var (
	addr = flag.String("addr", ":6060", "listening address")
	root = flag.String("root", ".", "root folder")
)

func main() {
	flag.Parse()
	log.Printf("Listening on %s (root=%s)\n", *addr, *root)
	fileHandler := http.FileServer(http.Dir(*root))
	if err := http.ListenAndServe(*addr, handlerLog(fileHandler)); err != nil {
		log.Fatalln("Error:", err)
	}
}

func handlerLog(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Printf("%s - %s %s\n", r.RemoteAddr, r.Method, r.RequestURI)
		handler.ServeHTTP(w, r)
	})
}
<|endoftext|>


<commit_before>package main

import (
	"html/template"
	"io/ioutil"
	"regexp"
	"log"
	"sync"
	"encoding/json"

	"net/http"

	"flag"
	//"github.com/FogCreek/mini"

	"github.com/JackKnifed/blackfriday"
	//"github.com/russross/blackfriday"
)

type WikiPage struct {
	Title string
	//Body  stringo
	Body template.HTML
}

var templates = template.Must(template.ParseFiles("wiki.html"))

var validWiki = regexp.MustCompile("^/([a-zA-Z0-9_ ]+)$")

var (
	configFile = flag.String("config", "", "specify a configuration file")
	config = map[]interface{}
)

//var validFiles = regexp.MustCompile("^/raw/([a-zA-Z0-9]+)\\.(jpg|gif|jpeg|md|png)$")

//var validSupport = regexp.MustCompile("^/global/([a-zA-Z0-9]+)\\.(css|js)$")

func markdownHandler(responsePipe http.ResponseWriter, request *http.Request) {
	validRequest := validWiki.FindStringSubmatch(request.URL.Path)
	config := stormcall.GetConfig()
	if validRequest == nil {
		http.Error(responsePipe, "Request not allowed", 403)
	}
	contents, err := loadFile(config.StringFromSection("global", "wiki", "./") + validRequest[1] + ".md")
	if err != nil {
		http.Error(responsePipe, err.Error(), 500)
	}
	// parse any markdown in the input
	body := template.HTML(blackfriday.MarkdownCommon(contents))

	response := WikiPage{Title: validRequest[1], Body: body}
	err = templates.ExecuteTemplate(responsePipe, "wiki.html", response)
	if err != nil {
		http.Error(responsePipe, err.Error(), http.StatusInternalServerError)
	}
}

func GetConfig() *map[]interface{} {
	configLock.RLock()
	defer configLock.RUnlock()
	return config
}

func LoadConfig(configFile string) bool {

	if configFile == "" {
		log.Println("no configuration file specified, using ./config.json")
		// return an empty config file
		configFile = "config.json"
	}
	//temp, err := mini.LoadConfiguration(configFile)

	// have to read in the line into a byte[] array
	if configContents, err := io.ReadFile(configFile); err != nil {
		log.Println("Problem loading config file: ", err)
	}

	// UnMarshal the config file that was read in
	temp := new(interface{})
	err := json.Unmarshal(configContents, temp)

	//Make sure you were able to read it in
	if err != nil {
		log.Println("parse config error: ", err)
		return false
	}

	configLock.Lock()
	config = temp
	configLock.Unlock()

	return true
}

/*
func contentHandler(responsePipe http.ResponseWriter, request *http.Request) {
	validRequest := validFiles.FindStringSubmatch(request.URL.Path)
	if validRequest == nil {
		http.Error(responsePipe, "Request not allowed", 403)
	}
	strippedRequest := http.StripPrefix("/raw/", validRequest[0])
	http.ServeFile(responsePipe, request, strippedRequest)
	http.FileServer()
	contents, err := loadFile(validRequest)
	if err != nil {
		http.Error(responsePipe, err, 500)
	}
	response := contents
	err := templates.ExecuteTemplate(responsePipe, "markdown.html", p)
	if err != nil {
		http.Error(responsePipe, err.Error(), http.StatusInternalServerError)
	}

}
*/

func loadFile(filename string) ([]byte, error) {
	contents, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	return contents, nil
}

func main() {
	flag.Parse()

	stormcall.LoadConfig(*configFile)

	config := stormcall.GetConfig()

	rawFiles := http.FileServer(http.Dir(config.StringFromSection("global", "raw", "./")))
	siteFiles := http.FileServer(http.Dir(config.StringFromSection("global", "site", "./")))

	http.Handle("/raw/", http.StripPrefix("/raw/", rawFiles))
	http.Handle("/site/", http.StripPrefix("/site/", siteFiles))
	http.HandleFunc("/", markdownHandler)

	log.Println(http.ListenAndServe(":"+config.StringFromSection("global", "port", "80"), nil))
}
<commit_msg>finished getting the initial workings of the config.json stuff going<commit_after>package main

import (
	"html/template"
	"io/ioutil"
	"regexp"
	"log"
	"sync"
	"encoding/json"

	"net/http"

	"flag"
	//"github.com/FogCreek/mini"

	"github.com/JackKnifed/blackfriday"
	//"github.com/russross/blackfriday"
)

type WikiPage struct {
	Title string
	//Body  stringo
	Body template.HTML
}

var templates = template.Must(template.ParseFiles("wiki.html"))

var validWiki = regexp.MustCompile("^/([a-zA-Z0-9_ ]+)$")

var	configFile = flag.String("config", "", "specify a configuration file")

var staticConfig *configStruct
var configLock = new(sync.RWMutex)

//var validFiles = regexp.MustCompile("^/raw/([a-zA-Z0-9]+)\\.(jpg|gif|jpeg|md|png)$")

//var validSupport = regexp.MustCompile("^/global/([a-zA-Z0-9]+)\\.(css|js)$")
type globalConfigStruct struct{
	port string
	hostname string
}

type serverConfigStruct struct {
	path string
	prefix string
	defaultPage string
	serverType string
	restricted []string
}

type configStruct struct {
	global globalConfigStruct
	mainserver serverConfigStruct
	server []serverConfigStruct
}

func GetConfig() *configStruct {
	configLock.RLock()
	defer configLock.RUnlock()
	return staticConfig
}

func LoadConfig(configFile string) bool {

	if configFile == "" {
		log.Println("no configuration file specified, using ./config.json")
		// return an empty config file
		configFile = "config.json"
	}
	//temp, err := mini.LoadConfiguration(configFile)

	//var fileContents []byte
	//var err error
	// have to read in the line into a byte[] array
	fileContents, err := ioutil.ReadFile(configFile); if err != nil {
		log.Println("Problem loading config file: ", err)
	}

	// UnMarshal the config file that was read in
	temp := new(configStruct)
	err = json.Unmarshal(fileContents, temp)

	//Make sure you were able to read it in
	if err != nil {
		log.Println("parse config error: ", err)
		return false
	}

	configLock.Lock()
	staticConfig = temp
	configLock.Unlock()

	return true
}

func markdownHandler(responsePipe http.ResponseWriter, request *http.Request) {
	validRequest := validWiki.FindStringSubmatch(request.URL.Path)
	config := GetConfig()
	if validRequest == nil {
		http.Error(responsePipe, "Request not allowed", 403)
	}
	contents, err := loadFile(config.mainserver.prefix + validRequest[1] + ".md")
	if err != nil {
		http.Error(responsePipe, err.Error(), 500)
	}
	// parse any markdown in the input
	body := template.HTML(blackfriday.MarkdownCommon(contents))

	response := WikiPage{Title: validRequest[1], Body: body}
	err = templates.ExecuteTemplate(responsePipe, "wiki.html", response)
	if err != nil {
		http.Error(responsePipe, err.Error(), http.StatusInternalServerError)
	}
}

/*
func contentHandler(responsePipe http.ResponseWriter, request *http.Request) {
	validRequest := validFiles.FindStringSubmatch(request.URL.Path)
	if validRequest == nil {
		http.Error(responsePipe, "Request not allowed", 403)
	}
	strippedRequest := http.StripPrefix("/raw/", validRequest[0])
	http.ServeFile(responsePipe, request, strippedRequest)
	http.FileServer()
	contents, err := loadFile(validRequest)
	if err != nil {
		http.Error(responsePipe, err, 500)
	}
	response := contents
	err := templates.ExecuteTemplate(responsePipe, "markdown.html", p)
	if err != nil {
		http.Error(responsePipe, err.Error(), http.StatusInternalServerError)
	}

}
*/

func loadFile(filename string) ([]byte, error) {
	contents, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	return contents, nil
}

func main() {
	flag.Parse()

	LoadConfig(*configFile)

	config := GetConfig()

	rawFiles := http.FileServer(http.Dir(config.mainserver.path))
	siteFiles := http.FileServer(http.Dir(config.mainserver.path))

	http.Handle("/raw/", http.StripPrefix("/raw/", rawFiles))
	http.Handle("/site/", http.StripPrefix("/site/", siteFiles))
	http.HandleFunc("/", markdownHandler)

	log.Println(http.ListenAndServe(":"+config.global.port, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	"golang.org/x/net/context"
)

// We assume the zip file contains entries for directories too.

var progName = filepath.Base(os.Args[0])

func usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", progName)
	fmt.Fprintf(os.Stderr, "  %s ZIP MOUNTPOINT\n", progName)
	flag.PrintDefaults()
}

func main() {
	log.SetFlags(0)
	log.SetPrefix(progName + ": ")

	flag.Usage = usage
	flag.Parse()

	if flag.NArg() != 2 {
		usage()
		os.Exit(2)
	}
	path := flag.Arg(0)
	mountpoint := flag.Arg(1)
	if err := mount(path, mountpoint); err != nil {
		log.Fatal(err)
	}
}

func mount(path, mountpoint string) error {
	archive, err := zip.OpenReader(path)
	if err != nil {
		return err
	}
	defer archive.Close()

	c, err := fuse.Mount(mountpoint)
	if err != nil {
		return err
	}
	defer c.Close()

	filesys := &FS{
		archive: &archive.Reader,
	}
	if err := fs.Serve(c, filesys); err != nil {
		return err
	}

	// check if the mount process has an error to report
	<-c.Ready
	if err := c.MountError; err != nil {
		return err
	}

	return nil
}

type FS struct {
	archive *zip.Reader
}

var _ fs.FS = (*FS)(nil)

func (f *FS) Root() (fs.Node, error) {
	n := &Dir{
		archive: f.archive,
	}
	return n, nil
}

type Dir struct {
	archive *zip.Reader
	// nil for the root directory, which has no entry in the zip
	file *zip.File
}

var _ fs.Node = (*Dir)(nil)

func zipAttr(f *zip.File, a *fuse.Attr) {
	a.Size = f.UncompressedSize64
	a.Mode = f.Mode()
	a.Mtime = f.ModTime()
	a.Ctime = f.ModTime()
	a.Crtime = f.ModTime()
}

func (d *Dir) Attr(ctx context.Context, a *fuse.Attr) error {
	if d.file == nil {
		// root directory
		a.Mode = os.ModeDir | 0755
		return nil
	}
	zipAttr(d.file, a)
	return nil
}

var _ = fs.NodeRequestLookuper(&Dir{})

func (d *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fs.Node, error) {
	path := req.Name
	if d.file != nil {
		path = d.file.Name + path
	}
	for _, f := range d.archive.File {
		switch {
		case f.Name == path:
			child := &File{
				file: f,
			}
			return child, nil
		case f.Name[:len(f.Name)-1] == path && f.Name[len(f.Name)-1] == '/':
			child := &Dir{
				archive: d.archive,
				file:    f,
			}
			return child, nil
		}
	}
	return nil, fuse.ENOENT
}

var _ = fs.HandleReadDirAller(&Dir{})

func (d *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
	prefix := ""
	if d.file != nil {
		prefix = d.file.Name
	}

	var res []fuse.Dirent
	for _, f := range d.archive.File {
		if !strings.HasPrefix(f.Name, prefix) {
			continue
		}
		name := f.Name[len(prefix):]
		if name == "" {
			// the dir itself, not a child
			continue
		}
		if strings.ContainsRune(name[:len(name)-1], '/') {
			// contains slash in the middle -> is in a deeper subdir
			continue
		}
		var de fuse.Dirent
		if name[len(name)-1] == '/' {
			// directory
			name = name[:len(name)-1]
			de.Type = fuse.DT_Dir
		}
		de.Name = name
		res = append(res, de)
	}
	return res, nil
}

type File struct {
	file *zip.File
}

var _ fs.Node = (*File)(nil)

func (f *File) Attr(ctx context.Context, a *fuse.Attr) error {
	zipAttr(f.file, a)
	return nil
}

var _ = fs.NodeOpener(&File{})

func (f *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {
	r, err := f.file.Open()
	if err != nil {
		return nil, err
	}
	// individual entries inside a zip file are not seekable
	resp.Flags |= fuse.OpenNonSeekable
	return &FileHandle{r: r}, nil
}

type FileHandle struct {
	r io.ReadCloser
}

var _ fs.Handle = (*FileHandle)(nil)

var _ fs.HandleReleaser = (*FileHandle)(nil)

func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
	return fh.r.Close()
}

var _ = fs.HandleReader(&FileHandle{})

func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
	// We don't actually enforce Offset to match where previous read
	// ended. Maybe we should, but that would mean'd we need to track
	// it. The kernel *should* do it for us, based on the
	// fuse.OpenNonSeekable flag.
	buf := make([]byte, req.Size)
	n, err := fh.r.Read(buf)
	resp.Data = buf[:n]
	return err
}
<commit_msg>Avoid partial reads, as page cache will re-read the whole page<commit_after>package main

import (
	"archive/zip"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	"golang.org/x/net/context"
)

// We assume the zip file contains entries for directories too.

var progName = filepath.Base(os.Args[0])

func usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", progName)
	fmt.Fprintf(os.Stderr, "  %s ZIP MOUNTPOINT\n", progName)
	flag.PrintDefaults()
}

func main() {
	log.SetFlags(0)
	log.SetPrefix(progName + ": ")

	flag.Usage = usage
	flag.Parse()

	if flag.NArg() != 2 {
		usage()
		os.Exit(2)
	}
	path := flag.Arg(0)
	mountpoint := flag.Arg(1)
	if err := mount(path, mountpoint); err != nil {
		log.Fatal(err)
	}
}

func mount(path, mountpoint string) error {
	archive, err := zip.OpenReader(path)
	if err != nil {
		return err
	}
	defer archive.Close()

	c, err := fuse.Mount(mountpoint)
	if err != nil {
		return err
	}
	defer c.Close()

	filesys := &FS{
		archive: &archive.Reader,
	}
	if err := fs.Serve(c, filesys); err != nil {
		return err
	}

	// check if the mount process has an error to report
	<-c.Ready
	if err := c.MountError; err != nil {
		return err
	}

	return nil
}

type FS struct {
	archive *zip.Reader
}

var _ fs.FS = (*FS)(nil)

func (f *FS) Root() (fs.Node, error) {
	n := &Dir{
		archive: f.archive,
	}
	return n, nil
}

type Dir struct {
	archive *zip.Reader
	// nil for the root directory, which has no entry in the zip
	file *zip.File
}

var _ fs.Node = (*Dir)(nil)

func zipAttr(f *zip.File, a *fuse.Attr) {
	a.Size = f.UncompressedSize64
	a.Mode = f.Mode()
	a.Mtime = f.ModTime()
	a.Ctime = f.ModTime()
	a.Crtime = f.ModTime()
}

func (d *Dir) Attr(ctx context.Context, a *fuse.Attr) error {
	if d.file == nil {
		// root directory
		a.Mode = os.ModeDir | 0755
		return nil
	}
	zipAttr(d.file, a)
	return nil
}

var _ = fs.NodeRequestLookuper(&Dir{})

func (d *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fs.Node, error) {
	path := req.Name
	if d.file != nil {
		path = d.file.Name + path
	}
	for _, f := range d.archive.File {
		switch {
		case f.Name == path:
			child := &File{
				file: f,
			}
			return child, nil
		case f.Name[:len(f.Name)-1] == path && f.Name[len(f.Name)-1] == '/':
			child := &Dir{
				archive: d.archive,
				file:    f,
			}
			return child, nil
		}
	}
	return nil, fuse.ENOENT
}

var _ = fs.HandleReadDirAller(&Dir{})

func (d *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
	prefix := ""
	if d.file != nil {
		prefix = d.file.Name
	}

	var res []fuse.Dirent
	for _, f := range d.archive.File {
		if !strings.HasPrefix(f.Name, prefix) {
			continue
		}
		name := f.Name[len(prefix):]
		if name == "" {
			// the dir itself, not a child
			continue
		}
		if strings.ContainsRune(name[:len(name)-1], '/') {
			// contains slash in the middle -> is in a deeper subdir
			continue
		}
		var de fuse.Dirent
		if name[len(name)-1] == '/' {
			// directory
			name = name[:len(name)-1]
			de.Type = fuse.DT_Dir
		}
		de.Name = name
		res = append(res, de)
	}
	return res, nil
}

type File struct {
	file *zip.File
}

var _ fs.Node = (*File)(nil)

func (f *File) Attr(ctx context.Context, a *fuse.Attr) error {
	zipAttr(f.file, a)
	return nil
}

var _ = fs.NodeOpener(&File{})

func (f *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {
	r, err := f.file.Open()
	if err != nil {
		return nil, err
	}
	// individual entries inside a zip file are not seekable
	resp.Flags |= fuse.OpenNonSeekable
	return &FileHandle{r: r}, nil
}

type FileHandle struct {
	r io.ReadCloser
}

var _ fs.Handle = (*FileHandle)(nil)

var _ fs.HandleReleaser = (*FileHandle)(nil)

func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
	return fh.r.Close()
}

var _ = fs.HandleReader(&FileHandle{})

func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
	// We don't actually enforce Offset to match where previous read
	// ended. Maybe we should, but that would mean'd we need to track
	// it. The kernel *should* do it for us, based on the
	// fuse.OpenNonSeekable flag.
	//
	// One exception to the abve is if we fail to fully populate a
	// page cache page; a read into page cache is always page aligned.
	// Make sure we never serve a partial read, to avoid that.
	buf := make([]byte, req.Size)
	n, err := io.ReadFull(fh.r, buf)
	if err == io.ErrUnexpectedEOF || err == io.EOF {
		err = nil
	}
	resp.Data = buf[:n]
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"path/filepath"
)

var (
	// directory where Jekyll will look to transform files
	source = flag.String("source", "", "")

	// directory where Jekyll will write files to
	destination = flag.String("destination", "_site", "")

	// fires up a server that will host your _site directory if True
	server = flag.Bool("server", false, "")

	// the port that the Jekyll server will run on
	port = flag.String("server_port", ":4000", "")

	// deploys the website to S3
	deploy = flag.Bool("s3", false, "")

	// runs Jekyll with verbose output if True
	verbose = flag.Bool("verbose", false, "")

	// displays the help / usage if True
	help = flag.Bool("help", false, "")
)

func main() {

	// Parse the input parameters
	flag.BoolVar(help, "h", false, "")
	flag.BoolVar(verbose, "v", false, "")
	flag.Usage = usage
	flag.Parse()

	if *help {
		flag.Usage()
		os.Exit(0)
	}

	// User may specify the source as a non-flag variable
	if flag.NArg() > 0 {
		source = &flag.Args()[0]
	}

	// Convert the directory to an absolute path
	src, _ := filepath.Abs(*source)
	dest, _ := filepath.Abs(*destination)

	// Change the working directory to the website's source directory
	os.Chdir(src)

	// Initialize the Jekyll website
	site, err := NewSite(src, dest)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// Generate the static website
	if err := site.Generate(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// Deploys the static website to S3
	if *deploy {
		// Read the S3 configuration details
		path := filepath.Join(site.Src, "_jekyll_s3.yml")
		conf, err := ParseDeployConfig(path)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}

		if err := site.Deploy(conf.Key, conf.Secret, conf.Bucket); err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	}

	// If the server option is enabled, launch a webserver
	if *server {

		// Change the working directory to the _site directory
		os.Chdir(dest)

		// Create the handler to serve from the filesystem
		http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			path := filepath.Clean(r.URL.Path)
			path = filepath.Join(dest, path)
			http.ServeFile(w, r, path)
		})

		// Serve the website from the _site directory
		fmt.Printf("Starting server on port %s\n", *port)
		if err := http.ListenAndServe(*port, nil); err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	}

	os.Exit(0)
}

func logf(msg string, args ...interface{}) {
	if *verbose {
		println(fmt.Sprintf(msg, args...))
	}
}

var usage = func() {
	fmt.Println(`Usage: jkl [OPTION]... [SOURCE]

      --source         changes the dir where Jekyll will look to transform files
      --destination    changes the dir where Jekyll will write files to
      --server         starts a server that will host your _site directory
      --server-port    changes the port that the Jekyll server will run on
      --s3             copies the _site directory to s3
  -v, --verbose        runs Jekyll with verbose output
  -h, --help           display this help and exit

Examples:
  jkl                 generates site from current working dir
  jkl --server        generates site and serves at localhost:4000
  jkl /path/to/site   generates site from source dir /path/to/site

Report bugs to <https://github.com/bradrydzewski/jkl/issues>
Jekyll home page: <https://github.com/bradrydzewski/jkl>`)
}
<commit_msg>added command line args for s3 key, secret and bucket<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"path/filepath"
)

var (
	// directory where Jekyll will look to transform files
	source = flag.String("source", "", "")

	// directory where Jekyll will write files to
	destination = flag.String("destination", "_site", "")

	// fires up a server that will host your _site directory if True
	server = flag.Bool("server", false, "")

	// the port that the Jekyll server will run on
	port = flag.String("server_port", ":4000", "")

	// deploys the website to S3
	deploy = flag.Bool("s3", false, "")

	// s3 access key
	s3key = flag.String("s3_key", "", "")

	// s3 secret key
	s3secret = flag.String("s3_secret", "", "")

	// s3 bucket name
	s3bucket = flag.String("s3_bucket", "", "")

	// runs Jekyll with verbose output if True
	verbose = flag.Bool("verbose", false, "")

	// displays the help / usage if True
	help = flag.Bool("help", false, "")
)

func main() {

	// Parse the input parameters
	flag.BoolVar(help, "h", false, "")
	flag.BoolVar(verbose, "v", false, "")
	flag.Usage = usage
	flag.Parse()

	if *help {
		flag.Usage()
		os.Exit(0)
	}

	// User may specify the source as a non-flag variable
	if flag.NArg() > 0 {
		source = &flag.Args()[0]
	}

	// Convert the directory to an absolute path
	src, _ := filepath.Abs(*source)
	dest, _ := filepath.Abs(*destination)

	// Change the working directory to the website's source directory
	os.Chdir(src)

	// Initialize the Jekyll website
	site, err := NewSite(src, dest)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// Generate the static website
	if err := site.Generate(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// Deploys the static website to S3
	if *deploy {

		var conf *DeployConfig
		// Read the S3 configuration details if not provided as
		// command line
		if *s3key == "" {
			path := filepath.Join(site.Src, "_jekyll_s3.yml")
			conf, err = ParseDeployConfig(path)
			if err != nil {
				fmt.Println(err)
				os.Exit(1)
			}
		} else {
			// else use the command line args
			conf = &DeployConfig{ *s3key, *s3secret, *s3bucket }
		}

		if err := site.Deploy(conf.Key, conf.Secret, conf.Bucket); err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	}

	// If the server option is enabled, launch a webserver
	if *server {

		// Change the working directory to the _site directory
		os.Chdir(dest)

		// Create the handler to serve from the filesystem
		http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			path := filepath.Clean(r.URL.Path)
			path = filepath.Join(dest, path)
			http.ServeFile(w, r, path)
		})

		// Serve the website from the _site directory
		fmt.Printf("Starting server on port %s\n", *port)
		if err := http.ListenAndServe(*port, nil); err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	}

	os.Exit(0)
}

func logf(msg string, args ...interface{}) {
	if *verbose {
		println(fmt.Sprintf(msg, args...))
	}
}

var usage = func() {
	fmt.Println(`Usage: jkl [OPTION]... [SOURCE]

      --source         changes the dir where Jekyll will look to transform files
      --destination    changes the dir where Jekyll will write files to
      --server         starts a server that will host your _site directory
      --server-port    changes the port that the Jekyll server will run on
      --s3             copies the _site directory to s3
      --s3_key         aws access key use for s3 authentication
      --s3_secret      aws secret key use for s3 authentication
      --s3_bucket      name of the s3 bucket
  -v, --verbose        runs Jekyll with verbose output
  -h, --help           display this help and exit

Examples:
  jkl                 generates site from current working directory
  jkl --server        generates site and serves at localhost:4000
  jkl /path/to/site   generates site from source dir /path/to/site

Report bugs to <https://github.com/bradrydzewski/jkl/issues>
Jekyll home page: <https://github.com/bradrydzewski/jkl>`)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/apex/log"
	"github.com/apex/log/handlers/cli"
	"github.com/apex/log/handlers/multi"
	"github.com/segmentio/ecs-logs-go"
	"github.com/segmentio/ecs-logs/lib"

	_ "github.com/segmentio/ecs-logs/lib/cloudwatchlogs"
	_ "github.com/segmentio/ecs-logs/lib/datadog"
	_ "github.com/segmentio/ecs-logs/lib/loggly"
	_ "github.com/segmentio/ecs-logs/lib/statsd"
	_ "github.com/segmentio/ecs-logs/lib/syslog"
)

type source struct {
	lib.Source
	name string
}

type destination struct {
	lib.Destination
	name string
}

type reader struct {
	lib.Reader
	name string
}

func main() {
	var err error
	var src string
	var dst string
	var hostname string
	var level = lib.LogLevel(log.InfoLevel)
	var maxBytes int
	var maxCount int
	var flushTimeout time.Duration
	var cacheTimeout time.Duration

	hostname, _ = os.Hostname()
	log.SetLevel(log.DebugLevel)
	log.SetHandler(cli.New(os.Stdout))

	flag.StringVar(&src, "src", "stdin", "A comma separated list of log sources from which messages will be read ["+strings.Join(lib.SourcesAvailable(), ", ")+"]")
	flag.StringVar(&dst, "dst", "stdout", "A comma separated list of log destinations to which messages will be written ["+strings.Join(lib.DestinationsAvailable(), ", ")+"]")
	flag.StringVar(&hostname, "hostname", hostname, "The hostname advertised by ecs-logs")
	flag.Var(&level, "log-level", "The minimum level of log messages shown by ecs-logs")
	flag.IntVar(&maxBytes, "max-batch-bytes", 1000000, "The maximum size in bytes of a message batch")
	flag.IntVar(&maxCount, "max-batch-size", 10000, "The maximum number of messages in a batch")
	flag.DurationVar(&flushTimeout, "flush-timeout", 5*time.Second, "How often messages will be flushed")
	flag.DurationVar(&cacheTimeout, "cache-timeout", 5*time.Minute, "How to wait before clearing unused internal cache")
	flag.Parse()

	var store = lib.NewStore()
	var sources []source
	var readers []reader
	var dests []destination

	if sources = getSources(strings.Split(src, ",")); len(sources) == 0 {
		log.Fatal("no or invalid log sources")
	}

	if dests = getDestinations(strings.Split(dst, ",")); len(dests) == 0 {
		log.Fatal("no or invalid log destinations")
	}

	if readers, err = openSources(sources); err != nil {
		log.WithError(err).Fatal("failed to open log sources readers")
	}

	join := &sync.WaitGroup{}

	limits := lib.StreamLimits{
		MaxCount: maxCount,
		MaxBytes: maxBytes,
		MaxTime:  flushTimeout,
	}

	logger := &lib.LogHandler{
		Group:    "ecs-logs",
		Stream:   hostname,
		Hostname: hostname,
		Queue:    lib.NewMessageQueue(),
	}
	log.SetLevel(log.Level(level))
	log.SetHandler(multi.New(cli.New(os.Stdout), logger))

	expchan := time.Tick(flushTimeout / 2)
	msgchan := make(chan lib.Message, len(readers))
	sigchan := make(chan os.Signal, 1)
	counter := int32(len(readers))
	startReaders(readers, msgchan, &counter, hostname)

	for {
		select {
		case msg, ok := <-msgchan:
			now := time.Now()

			if !ok {
				log.Info("waiting for all write operations to complete")
				limits.Force = true
				flushAll(dests, store, limits, now, join)
				flushQueue(dests, store, logger.Queue, limits, now, join)
				join.Wait()
				return
			}

			_, stream := store.Add(msg, now)
			flush(dests, stream, limits, now, join)

		case <-logger.Queue.C:
			now := time.Now()
			flushQueue(dests, store, logger.Queue, limits, now, join)

		case <-expchan:
			now := time.Now()
			flushAll(dests, store, limits, now, join)
			removeExpired(dests, store, cacheTimeout, now)

		case sig := <-sigchan:
			log.WithFields(log.Fields{"signal": sig.String()}).Info("closing message readers")
			stopReaders(readers)
		}
	}
}

func getSources(names []string) (sources []source) {
	for i, src := range lib.GetSources(names...) {
		sources = append(sources, source{
			Source: src,
			name:   names[i],
		})
	}
	return
}

func getDestinations(names []string) (destinations []destination) {
	for i, dst := range lib.GetDestinations(names...) {
		destinations = append(destinations, destination{
			Destination: dst,
			name:        names[i],
		})
	}
	return
}

func openSources(sources []source) (readers []reader, err error) {
	readers = make([]reader, 0, len(sources))

	for _, source := range sources {
		if r, e := source.Open(); e != nil {
			log.WithFields(log.Fields{
				"source": source.name,
				"error":  e,
			}).Error("failed to open log source")
		} else {
			readers = append(readers, reader{
				Reader: r,
				name:   source.name,
			})
		}
	}

	if len(readers) == 0 {
		err = fmt.Errorf("no sources to read from")
	}

	return
}

func startReaders(readers []reader, msgchan chan<- lib.Message, counter *int32, hostname string) {
	for _, reader := range readers {
		go read(reader, msgchan, counter, hostname)
	}
}

func stopReaders(readers []reader) {
	for _, reader := range readers {
		reader.Close()
	}
}

func term(c chan<- lib.Message, counter *int32) {
	if atomic.AddInt32(counter, -1) == 0 {
		close(c)
	}
}

func read(r reader, c chan<- lib.Message, counter *int32, hostname string) {
	defer term(c, counter)
	for {
		var msg lib.Message
		var err error

		if msg, err = r.ReadMessage(); err != nil {
			if err == io.EOF {
				break
			}
			log.WithFields(log.Fields{
				"reader": r.name,
				"error":  err,
			}).Error("the message reader failed")
			continue
		}

		if len(msg.Group) == 0 {
			log.WithFields(log.Fields{
				"reader":  r.name,
				"missing": "group",
			}).Warn("dropping message because the a required field wasn't set")
			continue
		}

		if len(msg.Stream) == 0 {
			log.WithFields(log.Fields{
				"reader":  r.name,
				"missing": "stream",
			}).Warn("dropping message because the a required field wasn't set")
			continue
		}

		if len(msg.Event.Info.Host) == 0 {
			msg.Event.Info.Host = hostname
		}

		if msg.Event.Time == (time.Time{}) {
			msg.Event.Time = time.Now()
		}

		if msg.Event.Data == nil {
			msg.Event.Data = ecslogs.EventData{}
		}

		c <- msg
	}
}

func write(dest destination, group string, stream string, batch lib.MessageBatch, join *sync.WaitGroup) {
	defer join.Done()

	var writer lib.Writer
	var err error

	if writer, err = dest.Open(group, stream); err != nil {
		logDropBatch(dest.name, group, stream, err, batch)
		return
	}
	defer writer.Close()

	if err = writer.WriteMessageBatch(batch); err != nil {
		logDropBatch(dest.name, group, stream, err, batch)
		return
	}
}

func flush(dests []destination, stream *lib.Stream, limits lib.StreamLimits, now time.Time, join *sync.WaitGroup) {
	for {
		batch, reason := stream.Flush(limits, now)

		if len(batch) == 0 {
			break
		}

		// Ensure all messages in the batch are sorted. Checking if the batch is
		// sorted is an optimization since in most cases the batch will be sorted
		// because we're reading events that are generated live (checking for a
		// sorted list is O(N) vs O(N*log(N)) for sorting it).
		// There are cases where some log entries do appear unordered and this is
		// causing issues with CloudWatchLogs.
		if !sort.IsSorted(batch) {
			sort.Stable(batch)
		}

		log.WithFields(log.Fields{
			"group":  stream.Group(),
			"stream": stream.Name(),
			"count":  len(batch),
			"reason": reason,
		}).Info("flushing message batch")

		for _, dest := range dests {
			join.Add(1)
			go write(dest, stream.Group(), stream.Name(), batch, join)
		}
	}
}

func flushAll(dests []destination, store *lib.Store, limits lib.StreamLimits, now time.Time, join *sync.WaitGroup) {
	store.ForEach(func(group *lib.Group) {
		group.ForEach(func(stream *lib.Stream) {
			flush(dests, stream, limits, now, join)
		})
	})
}

func flushQueue(dests []destination, store *lib.Store, queue *lib.MessageQueue, limits lib.StreamLimits, now time.Time, join *sync.WaitGroup) {
	streams := make(map[string]*lib.Stream)

	for _, msg := range queue.Flush() {
		_, stream := store.Add(msg, now)
		key := stream.Group() + ":" + stream.Name()

		if streams[key] == nil {
			streams[key] = stream
		}
	}

	for _, stream := range streams {
		flush(dests, stream, limits, now, join)
	}
}

func removeExpired(dests []destination, store *lib.Store, cacheTimeout time.Duration, now time.Time) {
	for _, stream := range store.RemoveExpired(cacheTimeout, now) {
		for _, dest := range dests {
			dest.Close(stream.Group(), stream.Name())
		}
		log.WithFields(log.Fields{
			"group":  stream.Group(),
			"stream": stream.Name(),
		}).Info("removed expired stream")
	}
}

func logDropBatch(dest string, group string, stream string, err error, batch lib.MessageBatch) {
	log.WithFields(log.Fields{
		"group":       group,
		"stream":      stream,
		"destination": dest,
		"error":       err,
		"count":       len(batch),
	}).Error("dropping message batch")

	for _, msg := range batch {
		log.WithFields(log.Fields{
			"group":  msg.Group,
			"stream": msg.Stream,
			"event":  msg.Event,
		}).Debug("dropped")
	}
}
<commit_msg>check for the hostname<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/apex/log"
	"github.com/apex/log/handlers/cli"
	"github.com/apex/log/handlers/multi"
	"github.com/segmentio/ecs-logs-go"
	"github.com/segmentio/ecs-logs/lib"

	_ "github.com/segmentio/ecs-logs/lib/cloudwatchlogs"
	_ "github.com/segmentio/ecs-logs/lib/datadog"
	_ "github.com/segmentio/ecs-logs/lib/loggly"
	_ "github.com/segmentio/ecs-logs/lib/statsd"
	_ "github.com/segmentio/ecs-logs/lib/syslog"
)

type source struct {
	lib.Source
	name string
}

type destination struct {
	lib.Destination
	name string
}

type reader struct {
	lib.Reader
	name string
}

func main() {
	var err error
	var src string
	var dst string
	var hostname string
	var level = lib.LogLevel(log.InfoLevel)
	var maxBytes int
	var maxCount int
	var flushTimeout time.Duration
	var cacheTimeout time.Duration

	hostname, _ = os.Hostname()
	log.SetLevel(log.DebugLevel)
	log.SetHandler(cli.New(os.Stdout))

	flag.StringVar(&src, "src", "stdin", "A comma separated list of log sources from which messages will be read ["+strings.Join(lib.SourcesAvailable(), ", ")+"]")
	flag.StringVar(&dst, "dst", "stdout", "A comma separated list of log destinations to which messages will be written ["+strings.Join(lib.DestinationsAvailable(), ", ")+"]")
	flag.StringVar(&hostname, "hostname", hostname, "The hostname advertised by ecs-logs")
	flag.Var(&level, "log-level", "The minimum level of log messages shown by ecs-logs")
	flag.IntVar(&maxBytes, "max-batch-bytes", 1000000, "The maximum size in bytes of a message batch")
	flag.IntVar(&maxCount, "max-batch-size", 10000, "The maximum number of messages in a batch")
	flag.DurationVar(&flushTimeout, "flush-timeout", 5*time.Second, "How often messages will be flushed")
	flag.DurationVar(&cacheTimeout, "cache-timeout", 5*time.Minute, "How to wait before clearing unused internal cache")
	flag.Parse()

	var store = lib.NewStore()
	var sources []source
	var readers []reader
	var dests []destination

	if len(hostname) == 0 {
		log.Fatal("no hostname configured")
	}

	if sources = getSources(strings.Split(src, ",")); len(sources) == 0 {
		log.Fatal("no or invalid log sources")
	}

	if dests = getDestinations(strings.Split(dst, ",")); len(dests) == 0 {
		log.Fatal("no or invalid log destinations")
	}

	if readers, err = openSources(sources); err != nil {
		log.WithError(err).Fatal("failed to open log sources readers")
	}

	join := &sync.WaitGroup{}

	limits := lib.StreamLimits{
		MaxCount: maxCount,
		MaxBytes: maxBytes,
		MaxTime:  flushTimeout,
	}

	logger := &lib.LogHandler{
		Group:    "ecs-logs",
		Stream:   hostname,
		Hostname: hostname,
		Queue:    lib.NewMessageQueue(),
	}
	log.SetLevel(log.Level(level))
	log.SetHandler(multi.New(cli.New(os.Stdout), logger))

	expchan := time.Tick(flushTimeout / 2)
	msgchan := make(chan lib.Message, len(readers))
	sigchan := make(chan os.Signal, 1)
	counter := int32(len(readers))
	startReaders(readers, msgchan, &counter, hostname)

	for {
		select {
		case msg, ok := <-msgchan:
			now := time.Now()

			if !ok {
				log.Info("waiting for all write operations to complete")
				limits.Force = true
				flushAll(dests, store, limits, now, join)
				flushQueue(dests, store, logger.Queue, limits, now, join)
				join.Wait()
				return
			}

			_, stream := store.Add(msg, now)
			flush(dests, stream, limits, now, join)

		case <-logger.Queue.C:
			now := time.Now()
			flushQueue(dests, store, logger.Queue, limits, now, join)

		case <-expchan:
			now := time.Now()
			flushAll(dests, store, limits, now, join)
			removeExpired(dests, store, cacheTimeout, now)

		case sig := <-sigchan:
			log.WithFields(log.Fields{"signal": sig.String()}).Info("closing message readers")
			stopReaders(readers)
		}
	}
}

func getSources(names []string) (sources []source) {
	for i, src := range lib.GetSources(names...) {
		sources = append(sources, source{
			Source: src,
			name:   names[i],
		})
	}
	return
}

func getDestinations(names []string) (destinations []destination) {
	for i, dst := range lib.GetDestinations(names...) {
		destinations = append(destinations, destination{
			Destination: dst,
			name:        names[i],
		})
	}
	return
}

func openSources(sources []source) (readers []reader, err error) {
	readers = make([]reader, 0, len(sources))

	for _, source := range sources {
		if r, e := source.Open(); e != nil {
			log.WithFields(log.Fields{
				"source": source.name,
				"error":  e,
			}).Error("failed to open log source")
		} else {
			readers = append(readers, reader{
				Reader: r,
				name:   source.name,
			})
		}
	}

	if len(readers) == 0 {
		err = fmt.Errorf("no sources to read from")
	}

	return
}

func startReaders(readers []reader, msgchan chan<- lib.Message, counter *int32, hostname string) {
	for _, reader := range readers {
		go read(reader, msgchan, counter, hostname)
	}
}

func stopReaders(readers []reader) {
	for _, reader := range readers {
		reader.Close()
	}
}

func term(c chan<- lib.Message, counter *int32) {
	if atomic.AddInt32(counter, -1) == 0 {
		close(c)
	}
}

func read(r reader, c chan<- lib.Message, counter *int32, hostname string) {
	defer term(c, counter)
	for {
		var msg lib.Message
		var err error

		if msg, err = r.ReadMessage(); err != nil {
			if err == io.EOF {
				break
			}
			log.WithFields(log.Fields{
				"reader": r.name,
				"error":  err,
			}).Error("the message reader failed")
			continue
		}

		if len(msg.Group) == 0 {
			log.WithFields(log.Fields{
				"reader":  r.name,
				"missing": "group",
			}).Warn("dropping message because the a required field wasn't set")
			continue
		}

		if len(msg.Stream) == 0 {
			log.WithFields(log.Fields{
				"reader":  r.name,
				"missing": "stream",
			}).Warn("dropping message because the a required field wasn't set")
			continue
		}

		if len(msg.Event.Info.Host) == 0 {
			msg.Event.Info.Host = hostname
		}

		if msg.Event.Time == (time.Time{}) {
			msg.Event.Time = time.Now()
		}

		if msg.Event.Data == nil {
			msg.Event.Data = ecslogs.EventData{}
		}

		c <- msg
	}
}

func write(dest destination, group string, stream string, batch lib.MessageBatch, join *sync.WaitGroup) {
	defer join.Done()

	var writer lib.Writer
	var err error

	if writer, err = dest.Open(group, stream); err != nil {
		logDropBatch(dest.name, group, stream, err, batch)
		return
	}
	defer writer.Close()

	if err = writer.WriteMessageBatch(batch); err != nil {
		logDropBatch(dest.name, group, stream, err, batch)
		return
	}
}

func flush(dests []destination, stream *lib.Stream, limits lib.StreamLimits, now time.Time, join *sync.WaitGroup) {
	for {
		batch, reason := stream.Flush(limits, now)

		if len(batch) == 0 {
			break
		}

		// Ensure all messages in the batch are sorted. Checking if the batch is
		// sorted is an optimization since in most cases the batch will be sorted
		// because we're reading events that are generated live (checking for a
		// sorted list is O(N) vs O(N*log(N)) for sorting it).
		// There are cases where some log entries do appear unordered and this is
		// causing issues with CloudWatchLogs.
		if !sort.IsSorted(batch) {
			sort.Stable(batch)
		}

		log.WithFields(log.Fields{
			"group":  stream.Group(),
			"stream": stream.Name(),
			"count":  len(batch),
			"reason": reason,
		}).Info("flushing message batch")

		for _, dest := range dests {
			join.Add(1)
			go write(dest, stream.Group(), stream.Name(), batch, join)
		}
	}
}

func flushAll(dests []destination, store *lib.Store, limits lib.StreamLimits, now time.Time, join *sync.WaitGroup) {
	store.ForEach(func(group *lib.Group) {
		group.ForEach(func(stream *lib.Stream) {
			flush(dests, stream, limits, now, join)
		})
	})
}

func flushQueue(dests []destination, store *lib.Store, queue *lib.MessageQueue, limits lib.StreamLimits, now time.Time, join *sync.WaitGroup) {
	streams := make(map[string]*lib.Stream)

	for _, msg := range queue.Flush() {
		_, stream := store.Add(msg, now)
		key := stream.Group() + ":" + stream.Name()

		if streams[key] == nil {
			streams[key] = stream
		}
	}

	for _, stream := range streams {
		flush(dests, stream, limits, now, join)
	}
}

func removeExpired(dests []destination, store *lib.Store, cacheTimeout time.Duration, now time.Time) {
	for _, stream := range store.RemoveExpired(cacheTimeout, now) {
		for _, dest := range dests {
			dest.Close(stream.Group(), stream.Name())
		}
		log.WithFields(log.Fields{
			"group":  stream.Group(),
			"stream": stream.Name(),
		}).Info("removed expired stream")
	}
}

func logDropBatch(dest string, group string, stream string, err error, batch lib.MessageBatch) {
	log.WithFields(log.Fields{
		"group":       group,
		"stream":      stream,
		"destination": dest,
		"error":       err,
		"count":       len(batch),
	}).Error("dropping message batch")

	for _, msg := range batch {
		log.WithFields(log.Fields{
			"group":  msg.Group,
			"stream": msg.Stream,
			"event":  msg.Event,
		}).Debug("dropped")
	}
}
<|endoftext|>


<commit_before>// Welcome! Let's start main package so we'll get executable binary.
package main

// This is how we import code from other packages.
import (
	"errors"
	"fmt"
	"log"
	"math"
	"math/rand"
	"runtime"
	"time"
)

// init is run when the file loads
func init() {
	// do nothing
}

// main is function runs when executable starts
func main() {
	fmt.Println("Here is how we print something to standard output.")
	log.Println("We can also print to standard error.")
}

// whatIsCheatsheetFor is private to package because it starts with lower case.
func whatIsCheatsheetFor() {
	fmt.Println("I need to learn go.")
}

// WhatIsGoFor is public; it can be imported by other packages.
func WhatIsGoFor() {
	fmt.Println("Go is great for server-side programming.")
}

// PI is a untyped numeric constant.
const Pi = 3.14159

// AUTHOR is a untyped string constant.
const Author = "Rafal Radulski"

// letsDefineVariables defines some variables.
func letsDefineVariables() {
	// int
	var a = 1
	var b int = 42
	var c, d int = 9, 27
	e := 3

	// char
	letter := 'l'

	// string
	abc := "text"

	// float
	x := 1.7

	// array of length 3
	var arr [3]int
	arr[0] = 91

	// slide of length 3, capacity 10
	s := make([]int, 3, 10)
	s = append(s, 20)

	// map of string -> int
	var m map[string]int
	m = make(map[string]int)
	m["zero"] = 0

	n := map[string]int{"zero": 0, "one": 1}
	delete(n, "one")

	// Local variables must be used or Go won't compile.
	fmt.Println(a, b, c, d, e, letter, abc, x)
}

// letsDoTypeConversion converts between char, string and int
func letsDoTypeConversion() {
	c := 'R'
	s := string(c)
	i := int(c)
	fmt.Println(c, s, i)
}

// letsDoConditional uses if statement. If can have two parts, init and condition
func letsDoConditional() {
	rand.Seed(time.Now().UTC().UnixNano())

	if r := rand.Intn(100); r > 9 {
		fmt.Println("We generated number greater than 9.")
	} else {
		fmt.Println("We generated a small number.")
	}
}

// letsDoLoops runs "for" statement a couple of ways
func letsDoLoops(start, stop int) int {
	sum := 0

	// we start with traditional for loop
	for i := start; i < stop; i++ {
		sum += i
	}

	// next is "while" loop
	for sum < 20 {
		sum += 1
	}

	// next is infinit loop
	for true {
		// but we are not going to wait forever
		break
	}

	// let's try range, which returns index and value
	list := [6]int{1, 2, 3, 5, 7, 11}
	for _, v := range list {
		sum += v
	}

	// or just index
	for i := range list {
		sum += list[i]
	}
	return sum
}

// letsDoSwitch uses switch statement
func letsDoSwitch() {
	// traditional way
	name := runtime.GOOS
	switch name {
	case "darwin":
		fmt.Println("We are on OS X.")
	default:
		fmt.Printf("We are on %s.", name)
	}

	// Go way
	switch os := runtime.GOOS; os {
	case "darwin":
		fmt.Println("We are on OS X.")
	default:
		fmt.Printf("We are on %s.", os)
	}
}

// letsUseDefer runs defered code at the end of a function
func letsUseDefer() {
	defer fmt.Print("world.\n")
	fmt.Print("Hello ")
}

// HowDoYouReturnErrors indicates error with return value
func HowDoYouReturnErrors(wantError bool) (string, err) {
	if wantError {
		return "", errors.New("if you must... here is an error")
	} else {
		return "No error this time!", nil
	}
}

// Vertex is a simple struct
type Vertex struct {
	X int // X cordinate (uppercase == public field)
	Y int // Y cordinate
}

// String is a method which implements Stringer interface and returns string representation of an object
func (v Vertex) String() string {
	return fmt.Sprintf("(%d, %d)", v.X, v.Y)
}

// abs is a method private to package (it's lower case)
func (v Vertex) abs() float64 {
	return math.Sqrt(float64(v.X)*float64(v.X) + float64(v.Y)*float64(v.Y))
}

// Currency is a custom type.
type Currency float64

// currency is another type, but it's private to pckage.
type currency float64

// GetNamer is interface which requires a single method GetName, which must take no parameters and return string
type GetNamer interface {
	GetName() string
}

// Animal is a new type, a struct
type Animal struct {
	name string // animal's name (lowercase == private field)
}

// GetName is a method on Animal
func (a Animal) GetName() string {
	return a.name
}

// Cat is a struct with anonymouse field. Fields and methods are inherited from Animal.
type Cat struct {
	Animal
	size int
}

// Mouse is another type. It has the same fields as Animal, but it has no methods.
type Mouse Animal

// TryToGetName accepts object of any type, and tries to get a name from object.
func TryToGetName(obj interface{}) string {
	switch o := obj.(type) {
	default:
		return "unnamed"
	case GetNamer:
		return o.GetName()
	case fmt.Stringer:
		return o.String()
	}
}
<commit_msg>Fix package comment<commit_after>/*
Welcome! Let's start main package so we'll get executable binary.

By the way, this is where we write package comment.
*/
package main

// This is how we import code from other packages.
import (
	"errors"
	"fmt"
	"log"
	"math"
	"math/rand"
	"runtime"
	"time"
)

// init is run when the file loads
func init() {
	// do nothing
}

// main is function runs when executable starts
func main() {
	fmt.Println("Here is how we print something to standard output.")
	log.Println("We can also print to standard error.")
}

// whatIsCheatsheetFor is private to package because it starts with lower case.
func whatIsCheatsheetFor() {
	fmt.Println("I need to learn go.")
}

// WhatIsGoFor is public; it can be imported by other packages.
func WhatIsGoFor() {
	fmt.Println("Go is great for server-side programming.")
}

// PI is a untyped numeric constant.
const Pi = 3.14159

// AUTHOR is a untyped string constant.
const Author = "Rafal Radulski"

// letsDefineVariables defines some variables.
func letsDefineVariables() {
	// int
	var a = 1
	var b int = 42
	var c, d int = 9, 27
	e := 3

	// char
	letter := 'l'

	// string
	abc := "text"

	// float
	x := 1.7

	// array of length 3
	var arr [3]int
	arr[0] = 91

	// slide of length 3, capacity 10
	s := make([]int, 3, 10)
	s = append(s, 20)

	// map of string -> int
	var m map[string]int
	m = make(map[string]int)
	m["zero"] = 0

	n := map[string]int{"zero": 0, "one": 1}
	delete(n, "one")

	// Local variables must be used or Go won't compile.
	fmt.Println(a, b, c, d, e, letter, abc, x)
}

// letsDoTypeConversion converts between char, string and int
func letsDoTypeConversion() {
	c := 'R'
	s := string(c)
	i := int(c)
	fmt.Println(c, s, i)
}

// letsDoConditional uses if statement. If can have two parts, init and condition
func letsDoConditional() {
	rand.Seed(time.Now().UTC().UnixNano())

	if r := rand.Intn(100); r > 9 {
		fmt.Println("We generated number greater than 9.")
	} else {
		fmt.Println("We generated a small number.")
	}
}

// letsDoLoops runs "for" statement a couple of ways
func letsDoLoops(start, stop int) int {
	sum := 0

	// we start with traditional for loop
	for i := start; i < stop; i++ {
		sum += i
	}

	// next is "while" loop
	for sum < 20 {
		sum += 1
	}

	// next is infinit loop
	for true {
		// but we are not going to wait forever
		break
	}

	// let's try range, which returns index and value
	list := [6]int{1, 2, 3, 5, 7, 11}
	for _, v := range list {
		sum += v
	}

	// or just index
	for i := range list {
		sum += list[i]
	}
	return sum
}

// letsDoSwitch uses switch statement
func letsDoSwitch() {
	// traditional way
	name := runtime.GOOS
	switch name {
	case "darwin":
		fmt.Println("We are on OS X.")
	default:
		fmt.Printf("We are on %s.", name)
	}

	// Go way
	switch os := runtime.GOOS; os {
	case "darwin":
		fmt.Println("We are on OS X.")
	default:
		fmt.Printf("We are on %s.", os)
	}
}

// letsUseDefer runs defered code at the end of a function
func letsUseDefer() {
	defer fmt.Print("world.\n")
	fmt.Print("Hello ")
}

// HowDoYouReturnErrors indicates error with return value
func HowDoYouReturnErrors(wantError bool) (string, err) {
	if wantError {
		return "", errors.New("if you must... here is an error")
	} else {
		return "No error this time!", nil
	}
}

// Vertex is a simple struct
type Vertex struct {
	X int // X cordinate (uppercase == public field)
	Y int // Y cordinate
}

// String is a method which implements Stringer interface and returns string representation of an object
func (v Vertex) String() string {
	return fmt.Sprintf("(%d, %d)", v.X, v.Y)
}

// abs is a method private to package (it's lower case)
func (v Vertex) abs() float64 {
	return math.Sqrt(float64(v.X)*float64(v.X) + float64(v.Y)*float64(v.Y))
}

// Currency is a custom type.
type Currency float64

// currency is another type, but it's private to pckage.
type currency float64

// GetNamer is interface which requires a single method GetName, which must take no parameters and return string
type GetNamer interface {
	GetName() string
}

// Animal is a new type, a struct
type Animal struct {
	name string // animal's name (lowercase == private field)
}

// GetName is a method on Animal
func (a Animal) GetName() string {
	return a.name
}

// Cat is a struct with anonymouse field. Fields and methods are inherited from Animal.
type Cat struct {
	Animal
	size int
}

// Mouse is another type. It has the same fields as Animal, but it has no methods.
type Mouse Animal

// TryToGetName accepts object of any type, and tries to get a name from object.
func TryToGetName(obj interface{}) string {
	switch o := obj.(type) {
	default:
		return "unnamed"
	case GetNamer:
		return o.GetName()
	case fmt.Stringer:
		return o.String()
	}
}
<|endoftext|>