<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"

	moocfetcher "github.com/moocfetcher/moocfetcher-appliance/backend/lib"
	"github.com/urfave/cli"

	"github.com/moocfetcher/moocfetcher-appliance/backend/lib/server"
)

func addCorsHeaders(h http.Handler) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if origin := r.Header.Get("Origin"); origin != "" {
			w.Header().Set("Access-Control-Allow-Origin", origin)
		}
		w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
		w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token")
		w.Header().Set("Access-Control-Allow-Credentials", "true")
		if r.Method == "OPTIONS" {
			return
		}
		h.ServeHTTP(w, r)
	}
}

func main() {
	app := cli.NewApp()
	app.Name = "moocfetcher-server"
	app.Usage = "MOOCFetcher Appliance Server"

	app.Flags = []cli.Flag{
		cli.IntFlag{
			Name:  "port, p",
			Value: 8080,
			Usage: "Run server on `PORT`",
		},
		cli.StringFlag{
			Name:  "course-metadata, m",
			Usage: "Load course metadata in JSON format from `FILE`",
		},
		cli.StringFlag{
			Name:  "courses-dir, d",
			Usage: "Location of courses on filesystem. Load courses from `DIRECTORY`.",
		},
		cli.StringFlag{
			Name:  "static-files-dir, s",
			Usage: "Load static files to be served from `DIRECTORY`",
		},
	}

	app.Action = func(c *cli.Context) error {
		courseMetadataFile := c.String("course-metadata")
		coursesDir := c.String("courses-dir")
		staticFilesDir := c.String("static-files-dir")
		port := c.Int("port")

		if courseMetadataFile == "" {
			return errors.New("course-metadata is required")
		}

		if coursesDir == "" {
			return errors.New("courses-directory is required")
		}

		if staticFilesDir == "" {
			return errors.New("static-files-dir is required")
		}

		// Parse Course Metadata
		cm, err := ioutil.ReadFile(courseMetadataFile)

		if err != nil {
			return errors.New(fmt.Sprintf("Error reading course metadata: %s", err))
		}

		var courseMetadata moocfetcher.CourseData
		err = json.Unmarshal(cm, &courseMetadata)
		if err != nil {
			return errors.New(fmt.Sprintf("Error parsing course metadata: %s", err))
		}

		s := server.NewServer(coursesDir, courseMetadata)

		// Add handler for static content
		s.Handle("/", http.FileServer(http.Dir(staticFilesDir)))

		http.ListenAndServe(fmt.Sprintf(":%d", port), addCorsHeaders(s))
		return nil
	}

	err := app.Run(os.Args)

	if err != nil {
		fmt.Println(err)
	}
}
<commit_msg>Adding chained handlers using Alice<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"

	"github.com/NYTimes/gziphandler"
	"github.com/justinas/alice"
	"github.com/moocfetcher/moocfetcher-appliance/backend/lib/server"
	"github.com/urfave/cli"

	moocfetcher "github.com/moocfetcher/moocfetcher-appliance/backend/lib"
)

func addCorsHeaders(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if origin := r.Header.Get("Origin"); origin != "" {
			w.Header().Set("Access-Control-Allow-Origin", origin)
		}
		w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
		w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token")
		w.Header().Set("Access-Control-Allow-Credentials", "true")
		if r.Method == "OPTIONS" {
			return
		}
		h.ServeHTTP(w, r)
	})
}

func main() {
	app := cli.NewApp()
	app.Name = "moocfetcher-server"
	app.Usage = "MOOCFetcher Appliance Server"

	app.Flags = []cli.Flag{
		cli.IntFlag{
			Name:  "port, p",
			Value: 8080,
			Usage: "Run server on `PORT`",
		},
		cli.StringFlag{
			Name:  "course-metadata, m",
			Usage: "Load course metadata in JSON format from `FILE`",
		},
		cli.StringFlag{
			Name:  "courses-dir, d",
			Usage: "Location of courses on filesystem. Load courses from `DIRECTORY`.",
		},
		cli.StringFlag{
			Name:  "static-files-dir, s",
			Usage: "Load static files to be served from `DIRECTORY`",
		},
	}

	app.Action = func(c *cli.Context) error {
		courseMetadataFile := c.String("course-metadata")
		coursesDir := c.String("courses-dir")
		staticFilesDir := c.String("static-files-dir")
		port := c.Int("port")

		if courseMetadataFile == "" {
			return errors.New("course-metadata is required")
		}

		if coursesDir == "" {
			return errors.New("courses-directory is required")
		}

		if staticFilesDir == "" {
			return errors.New("static-files-dir is required")
		}

		// Parse Course Metadata
		cm, err := ioutil.ReadFile(courseMetadataFile)

		if err != nil {
			return errors.New(fmt.Sprintf("Error reading course metadata: %s", err))
		}

		var courseMetadata moocfetcher.CourseData
		err = json.Unmarshal(cm, &courseMetadata)
		if err != nil {
			return errors.New(fmt.Sprintf("Error parsing course metadata: %s", err))
		}

		s := server.NewServer(coursesDir, courseMetadata)

		// Add handler for static content
		s.Handle("/", http.FileServer(http.Dir(staticFilesDir)))

		http.ListenAndServe(fmt.Sprintf(":%d", port), alice.New(gziphandler.GzipHandler, addCorsHeaders).Then(s))
		return nil
	}

	err := app.Run(os.Args)

	if err != nil {
		fmt.Println(err)
	}
}
<|endoftext|>


<commit_before>package architect

import (
	"context"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/asdine/storm"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
	"github.com/velocity-ci/velocity/backend/pkg/architect/rest"
	"github.com/velocity-ci/velocity/backend/pkg/domain"
	"github.com/velocity-ci/velocity/backend/pkg/domain/build"
	"github.com/velocity-ci/velocity/backend/pkg/domain/builder"
	"github.com/velocity-ci/velocity/backend/pkg/domain/githistory"
	"github.com/velocity-ci/velocity/backend/pkg/domain/knownhost"
	"github.com/velocity-ci/velocity/backend/pkg/domain/project"
	"github.com/velocity-ci/velocity/backend/pkg/domain/task"
	"github.com/velocity-ci/velocity/backend/pkg/domain/user"
	"github.com/velocity-ci/velocity/backend/pkg/velocity"
)

type Architect struct {
	Server   *echo.Echo
	workerWg sync.WaitGroup
	Workers  []domain.Worker
	DB       *storm.DB
	LogsPath string
}

func (a *Architect) Start() {
	a.Init()
	a.Server.Use(middleware.Logger())
	a.Server.Use(middleware.Recover())
	for _, w := range a.Workers {
		go w.StartWorker()
	}

	a.Server.Start(fmt.Sprintf(":%s", os.Getenv("PORT")))
}

func (a *Architect) Stop() error {
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	for _, w := range a.Workers {
		w.StopWorker()
	}

	return a.Server.Shutdown(ctx)
}

type App interface {
	Start()
	Stop() error
}

func New() *Architect {
	velocity.SetLogLevel()
	a := &Architect{
		Server:   echo.New(),
		LogsPath: "/var/velocityci/logs",
	}

	return a
}

func (a *Architect) Init() {
	if a.DB == nil {
		a.DB = domain.NewStormDB("/var/velocityci/architect.db")
	}
	validator, trans := domain.NewValidator()
	userManager := user.NewManager(a.DB, validator, trans)
	userManager.EnsureAdminUser()
	knownHostManager := knownhost.NewManager(a.DB, validator, trans, "")
	projectManager := project.NewManager(a.DB, validator, trans, velocity.GitClone)
	commitManager := githistory.NewCommitManager(a.DB)
	branchManager := githistory.NewBranchManager(a.DB)
	taskManager := task.NewManager(a.DB, projectManager, branchManager, commitManager)
	buildStepManager := build.NewStepManager(a.DB)
	buildStreamFileManager := build.NewStreamFileManager(&a.workerWg, a.LogsPath)
	buildStreamManager := build.NewStreamManager(a.DB, buildStreamFileManager)
	buildManager := build.NewBuildManager(a.DB, buildStepManager, buildStreamManager)
	builderManager := builder.NewManager(buildManager, knownHostManager, buildStepManager, buildStreamManager)

	a.Server.Use(middleware.CORS())
	rest.AddRoutes(
		a.Server,
		userManager,
		knownHostManager,
		projectManager,
		commitManager,
		branchManager,
		taskManager,
		buildStepManager,
		buildStreamManager,
		buildManager,
		builderManager,
	)

	a.Workers = []domain.Worker{
		builder.NewScheduler(builderManager, buildManager, &a.workerWg),
		buildStreamFileManager,
	}
}
<commit_msg>[backend] added CORS allowing all<commit_after>package architect

import (
	"context"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/asdine/storm"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
	"github.com/velocity-ci/velocity/backend/pkg/architect/rest"
	"github.com/velocity-ci/velocity/backend/pkg/domain"
	"github.com/velocity-ci/velocity/backend/pkg/domain/build"
	"github.com/velocity-ci/velocity/backend/pkg/domain/builder"
	"github.com/velocity-ci/velocity/backend/pkg/domain/githistory"
	"github.com/velocity-ci/velocity/backend/pkg/domain/knownhost"
	"github.com/velocity-ci/velocity/backend/pkg/domain/project"
	"github.com/velocity-ci/velocity/backend/pkg/domain/task"
	"github.com/velocity-ci/velocity/backend/pkg/domain/user"
	"github.com/velocity-ci/velocity/backend/pkg/velocity"
)

type Architect struct {
	Server   *echo.Echo
	workerWg sync.WaitGroup
	Workers  []domain.Worker
	DB       *storm.DB
	LogsPath string
}

func (a *Architect) Start() {
	a.Init()
	a.Server.Use(middleware.Logger())
	a.Server.Use(middleware.Recover())
	a.Server.Use(middleware.CORSWithConfig(middleware.DefaultCORSConfig))
	for _, w := range a.Workers {
		go w.StartWorker()
	}

	a.Server.Start(fmt.Sprintf(":%s", os.Getenv("PORT")))
}

func (a *Architect) Stop() error {
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	for _, w := range a.Workers {
		w.StopWorker()
	}

	return a.Server.Shutdown(ctx)
}

type App interface {
	Start()
	Stop() error
}

func New() *Architect {
	velocity.SetLogLevel()
	a := &Architect{
		Server:   echo.New(),
		LogsPath: "/var/velocityci/logs",
	}

	return a
}

func (a *Architect) Init() {
	if a.DB == nil {
		a.DB = domain.NewStormDB("/var/velocityci/architect.db")
	}
	validator, trans := domain.NewValidator()
	userManager := user.NewManager(a.DB, validator, trans)
	userManager.EnsureAdminUser()
	knownHostManager := knownhost.NewManager(a.DB, validator, trans, "")
	projectManager := project.NewManager(a.DB, validator, trans, velocity.GitClone)
	commitManager := githistory.NewCommitManager(a.DB)
	branchManager := githistory.NewBranchManager(a.DB)
	taskManager := task.NewManager(a.DB, projectManager, branchManager, commitManager)
	buildStepManager := build.NewStepManager(a.DB)
	buildStreamFileManager := build.NewStreamFileManager(&a.workerWg, a.LogsPath)
	buildStreamManager := build.NewStreamManager(a.DB, buildStreamFileManager)
	buildManager := build.NewBuildManager(a.DB, buildStepManager, buildStreamManager)
	builderManager := builder.NewManager(buildManager, knownHostManager, buildStepManager, buildStreamManager)

	a.Server.Use(middleware.CORS())
	rest.AddRoutes(
		a.Server,
		userManager,
		knownHostManager,
		projectManager,
		commitManager,
		branchManager,
		taskManager,
		buildStepManager,
		buildStreamManager,
		buildManager,
		builderManager,
	)

	a.Workers = []domain.Worker{
		builder.NewScheduler(builderManager, buildManager, &a.workerWg),
		buildStreamFileManager,
	}
}
<|endoftext|>


<commit_before>package contour

// Register contains all of contour's Register functions.Calling Register
// adds, or registers, the Settings information to the AppConfig variable.
// The setting value, if there is one, is not saved to its ironment
// variable at this point.
//
// This allows for
//
// These should be called at app startup to register all configuration
// Settings that the application uses.

import (
	"fmt"
)

// Config methods
// RegisterConfigFilename set's the configuration file's name. The name is
// parsed for a valid extension--one that is a supported format--and saves
// that value too. If it cannot be determined, the extension info is not set.
// These are considered core values and cannot be changed from command-line
// and configuration files. (IsCore == true).
func (c *Cfg) RegisterConfigFilename(k, v string) error {
	if v == "" {
		return fmt.Errorf("A config filename was expected, none received")
	}

	if k == "" {
		return fmt.Errorf("A key for the config filename setting was expected, none received")
	}

	c.RegisterStringCore(k, v)

	// Register it first. If a valid config format isn't found, an error
	// will be returned, so registering it afterwords would mean the
	// setting would not exist.
	c.RegisterString(CfgFormat, "")
	format, err := cfgFormat(v)
	if err != nil {
		return err
	}

	// Now we can update the format, since it wasn't set before, it can be
	// set now before it becomes read only.
	c.UpdateString(CfgFormat, format.String())
	fmt.Printf("FORMAT %s\n", format.String())
	return nil
}

// RegisterSetting checks to see if the entry already exists and adds the
// new setting if it does not.
func (c *Cfg) RegisterSetting(typ, name, short string, value, dflt interface{}, usage string, IsCore, IsCfg, IsFlag bool) {
	c.lock.RLock()
	_, ok := appCfg.settings[name]
	if ok {
		// Settings can't be re-registered.
		c.lock.RUnlock()
		return
	}

	c.lock.RUnlock()
	c.lock.Lock()
	defer c.lock.Unlock()

	// Add the setting
	c.settings[name] = &setting{
		Type:    typ,
		Name:    name,
		Short:   short,
		Value:   value,
		Default: dflt,
		Usage:   usage,
		IsCore:  IsCore,
		IsCfg:   IsCfg,
		IsFlag:  IsFlag,
	}

	// Keep track of whether or not a config is being used. If a setting is
	// registered as a config setting, it is assumed a configuration source
	// is being used.
	if IsCfg {
		c.useCfg = true
	}

	// Keep track of whether or not flags are being used. If a setting is
	// registered as a flag setting, it is assumed that flags are being
	// used.
	if IsFlag {
		c.useFlags = true
	}
}

// RegisterBoolCore adds the information to the AppsConfig struct, but does not
// save it to its ironment variable
func (c *Cfg) RegisterBoolCore(k string, v bool) {
	c.RegisterSetting("bool", k, "", v, v, "", true, false, false)
	return
}

// RegisterIntCore adds the information to the AppsConfig struct, but does not
// save it to its ironment variable
func (c *Cfg) RegisterIntCore(k string, v int) {
	c.RegisterSetting("int", k, "", v, v, "", true, false, false)
	return
}

// RegisterStringCore adds the information to the AppsConfig struct, but does not
// save it to its ironment variable
func (c *Cfg) RegisterStringCore(k, v string) {
	c.RegisterSetting("string", k, "", v, v, "", true, false, false)
	return
}

// RegisterBoolConf adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func (c *Cfg) RegisterBoolConf(k string, v bool) {
	c.RegisterSetting("bool", k, "", v, v, "", false, true, false)
	return
}

// RegisterIntConf adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func (c *Cfg) RegisterIntConf(k string, v bool) {
	c.RegisterSetting("int", k, "", v, v, "", false, true, false)
	return
}

// RegisterStringConf adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func (c *Cfg) RegisterStringConf(k string, v bool) {
	c.RegisterSetting("string", k, "", v, v, "", false, true, false)
	return
}

// RegisterBoolFlag adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func (c *Cfg) RegisterBoolFlag(k, s string, v, dflt bool, usage string) {
	c.RegisterSetting("bool", k, s, v, dflt, usage, false, true, true)
	return
}

// RegisterIntFlag adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func (c *Cfg) RegisterIntFlag(k, s string, v, dflt int, usage string) {
	c.RegisterSetting("int", k, s, v, dflt, usage, false, true, true)
	return
}

// RegisterStringFlag adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func (c *Cfg) RegisterStringFlag(k, s, v, dflt, usage string) {
	c.RegisterSetting("string", k, s, v, dflt, usage, false, true, true)
	return
}

// RegisterBool adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func (c *Cfg) RegisterBool(k string, v bool) {
	c.RegisterSetting("bool", k, "", v, v, "", false, false, false)
	return
}

// RegisterInt adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func (c *Cfg) RegisterInt(k string, v int) {
	c.RegisterSetting("int", k, "", v, v, "", false, false, false)
	return
}

// RegisterString adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func (c *Cfg) RegisterString(k, v string) {
	c.RegisterSetting("string", "", k, v, v, "", false, false, false)
	return
}

// Convenience functions for interacting with the configs[app] configuration.

// RegisterConfigFilename set's the configuration file's name. The name is
// parsed for a valid extension--one that is a supported format--and saves
// that value too. If it cannot be determined, the extension info is not set.
// These are considered core values and cannot be changed from command-line
// and configuration files. (IsCore == true).
func RegisterConfigFilename(k, v string) error {
	if v == "" {
		return fmt.Errorf("A config filename was expected, none received")
	}

	if k == "" {
		return fmt.Errorf("A key for the config filename setting was expected, none received")
	}

	appCfg.RegisterStringCore(k, v)

	// TODO redo this given new paradigm
	// Register it first. If a valid config format isn't found, an error
	// will be returned, so registering it afterwords would mean the
	// setting would not exist.
	appCfg.RegisterString(CfgFormat, "")
	format, err := cfgFormat(v)
	if err != nil {
		return err
	}

	appCfg.RegisterString(CfgFormat, format.String())

	return nil
}

// RegisterSetting checks to see if the entry already exists and adds the
// new setting if it does not.
func RegisterSetting(typ, name, short string, value, dflt interface{}, usage string, IsCore, IsCfg, IsFlag bool) {
	appCfg.RegisterSetting(typ, name, short, value, dflt, usage, IsCore, IsCfg, IsFlag)
}

// RegisterBoolCore adds the information to the AppsConfig struct, but does not
// save it to its ironment variable
func RegisterBoolCore(k string, v bool) {
	appCfg.RegisterBoolCore(k, v)
}

// RegisterIntCore adds the information to the AppsConfig struct, but does not
// save it to its ironment variable
func RegisterIntCore(k string, v int) {
	appCfg.RegisterIntCore(k, v)
}

// RegisterStringCore adds the information to the AppsConfig struct, but does not
// save it to its ironment variable
func RegisterStringCore(k, v string) {
	appCfg.RegisterStringCore(k, v)
}

// RegisterConfBool adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func RegisteeBoolCore(k string, v bool) {
	appCfg.RegisterBoolCore(k, v)
}

// RegisterIntConf adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func RegisterIntConf(k string, v bool) {
	appCfg.RegisterIntConf(k, v)
}

// RegisterStringConf adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func RegisterStringConf(k string, v bool) {
	appCfg.RegisterStringConf(k, v)
}

// RegisterBoolFlag adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func RegisterBoolFlag(k, s string, v, dflt bool, u string) {
	appCfg.RegisterBoolFlag(k, s, v, dflt, u)
}

// RegisterIntFlag adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func RegisterIntFlag(k, s string, v, dflt int, u string) {
	appCfg.RegisterIntFlag(k, s, v, dflt, u)
}

// RegisterStringFlag adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func RegisterStringFlag(k, s, v, dflt, u string) {
	appCfg.RegisterStringFlag(k, s, v, dflt, u)
}

// RegisterBool adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func RegisterBool(k string, v bool) {
	appCfg.RegisterBool(k, v)
}

// RegisterInt adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func RegisterInt(k string, v int) {
	appCfg.RegisterInt(k, v)
}

// RegisterString adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func RegisterString(k, v string) {
	appCfg.RegisterString(k, v)
}
<commit_msg>add RegisterCfgFilename func for appCfg<commit_after>package contour

// Register contains all of contour's Register functions.Calling Register
// adds, or registers, the Settings information to the AppConfig variable.
// The setting value, if there is one, is not saved to its ironment
// variable at this point.
//
// This allows for
//
// These should be called at app startup to register all configuration
// Settings that the application uses.

import (
	"fmt"
)

// Config methods
// RegisterConfigFilename set's the configuration file's name. The name is
// parsed for a valid extension--one that is a supported format--and saves
// that value too. If it cannot be determined, the extension info is not set.
// These are considered core values and cannot be changed from command-line
// and configuration files. (IsCore == true).
func (c *Cfg) RegisterCfgFilename(k, v string) error {
	if v == "" {
		return fmt.Errorf("A config filename was expected, none received")
	}

	if k == "" {
		return fmt.Errorf("A key for the config filename setting was expected, none received")
	}

	c.RegisterStringCore(k, v)

	// Register it first. If a valid config format isn't found, an error
	// will be returned, so registering it afterwords would mean the
	// setting would not exist.
	c.RegisterString(CfgFormat, "")
	format, err := cfgFormat(v)
	if err != nil {
		return err
	}

	// Now we can update the format, since it wasn't set before, it can be
	// set now before it becomes read only.
	c.UpdateString(CfgFormat, format.String())
	fmt.Printf("FORMAT %s\n", format.String())
	return nil
}

func RegisterCfgFilename(k, v string) error {
	return appCfg.RegisterCfgFilename(k, v)
}

// RegisterSetting checks to see if the entry already exists and adds the
// new setting if it does not.
func (c *Cfg) RegisterSetting(typ, name, short string, value, dflt interface{}, usage string, IsCore, IsCfg, IsFlag bool) {
	c.lock.RLock()
	_, ok := appCfg.settings[name]
	if ok {
		// Settings can't be re-registered.
		c.lock.RUnlock()
		return
	}

	c.lock.RUnlock()
	c.lock.Lock()
	defer c.lock.Unlock()

	// Add the setting
	c.settings[name] = &setting{
		Type:    typ,
		Name:    name,
		Short:   short,
		Value:   value,
		Default: dflt,
		Usage:   usage,
		IsCore:  IsCore,
		IsCfg:   IsCfg,
		IsFlag:  IsFlag,
	}

	// Keep track of whether or not a config is being used. If a setting is
	// registered as a config setting, it is assumed a configuration source
	// is being used.
	if IsCfg {
		c.useCfg = true
	}

	// Keep track of whether or not flags are being used. If a setting is
	// registered as a flag setting, it is assumed that flags are being
	// used.
	if IsFlag {
		c.useFlags = true
	}
}

// RegisterBoolCore adds the information to the AppsConfig struct, but does not
// save it to its ironment variable
func (c *Cfg) RegisterBoolCore(k string, v bool) {
	c.RegisterSetting("bool", k, "", v, v, "", true, false, false)
	return
}

// RegisterIntCore adds the information to the AppsConfig struct, but does not
// save it to its ironment variable
func (c *Cfg) RegisterIntCore(k string, v int) {
	c.RegisterSetting("int", k, "", v, v, "", true, false, false)
	return
}

// RegisterStringCore adds the information to the AppsConfig struct, but does not
// save it to its ironment variable
func (c *Cfg) RegisterStringCore(k, v string) {
	c.RegisterSetting("string", k, "", v, v, "", true, false, false)
	return
}

// RegisterBoolConf adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func (c *Cfg) RegisterBoolConf(k string, v bool) {
	c.RegisterSetting("bool", k, "", v, v, "", false, true, false)
	return
}

// RegisterIntConf adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func (c *Cfg) RegisterIntConf(k string, v bool) {
	c.RegisterSetting("int", k, "", v, v, "", false, true, false)
	return
}

// RegisterStringConf adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func (c *Cfg) RegisterStringConf(k string, v bool) {
	c.RegisterSetting("string", k, "", v, v, "", false, true, false)
	return
}

// RegisterBoolFlag adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func (c *Cfg) RegisterBoolFlag(k, s string, v, dflt bool, usage string) {
	c.RegisterSetting("bool", k, s, v, dflt, usage, false, true, true)
	return
}

// RegisterIntFlag adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func (c *Cfg) RegisterIntFlag(k, s string, v, dflt int, usage string) {
	c.RegisterSetting("int", k, s, v, dflt, usage, false, true, true)
	return
}

// RegisterStringFlag adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func (c *Cfg) RegisterStringFlag(k, s, v, dflt, usage string) {
	c.RegisterSetting("string", k, s, v, dflt, usage, false, true, true)
	return
}

// RegisterBool adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func (c *Cfg) RegisterBool(k string, v bool) {
	c.RegisterSetting("bool", k, "", v, v, "", false, false, false)
	return
}

// RegisterInt adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func (c *Cfg) RegisterInt(k string, v int) {
	c.RegisterSetting("int", k, "", v, v, "", false, false, false)
	return
}

// RegisterString adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func (c *Cfg) RegisterString(k, v string) {
	c.RegisterSetting("string", "", k, v, v, "", false, false, false)
	return
}

// Convenience functions for interacting with the configs[app] configuration.

// RegisterConfigFilename set's the configuration file's name. The name is
// parsed for a valid extension--one that is a supported format--and saves
// that value too. If it cannot be determined, the extension info is not set.
// These are considered core values and cannot be changed from command-line
// and configuration files. (IsCore == true).
func RegisterConfigFilename(k, v string) error {
	if v == "" {
		return fmt.Errorf("A config filename was expected, none received")
	}

	if k == "" {
		return fmt.Errorf("A key for the config filename setting was expected, none received")
	}

	appCfg.RegisterStringCore(k, v)

	// TODO redo this given new paradigm
	// Register it first. If a valid config format isn't found, an error
	// will be returned, so registering it afterwords would mean the
	// setting would not exist.
	appCfg.RegisterString(CfgFormat, "")
	format, err := cfgFormat(v)
	if err != nil {
		return err
	}

	appCfg.RegisterString(CfgFormat, format.String())

	return nil
}

// RegisterSetting checks to see if the entry already exists and adds the
// new setting if it does not.
func RegisterSetting(typ, name, short string, value, dflt interface{}, usage string, IsCore, IsCfg, IsFlag bool) {
	appCfg.RegisterSetting(typ, name, short, value, dflt, usage, IsCore, IsCfg, IsFlag)
}

// RegisterBoolCore adds the information to the AppsConfig struct, but does not
// save it to its ironment variable
func RegisterBoolCore(k string, v bool) {
	appCfg.RegisterBoolCore(k, v)
}

// RegisterIntCore adds the information to the AppsConfig struct, but does not
// save it to its ironment variable
func RegisterIntCore(k string, v int) {
	appCfg.RegisterIntCore(k, v)
}

// RegisterStringCore adds the information to the AppsConfig struct, but does not
// save it to its ironment variable
func RegisterStringCore(k, v string) {
	appCfg.RegisterStringCore(k, v)
}

// RegisterConfBool adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func RegisteeBoolCore(k string, v bool) {
	appCfg.RegisterBoolCore(k, v)
}

// RegisterIntConf adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func RegisterIntConf(k string, v bool) {
	appCfg.RegisterIntConf(k, v)
}

// RegisterStringConf adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func RegisterStringConf(k string, v bool) {
	appCfg.RegisterStringConf(k, v)
}

// RegisterBoolFlag adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func RegisterBoolFlag(k, s string, v, dflt bool, u string) {
	appCfg.RegisterBoolFlag(k, s, v, dflt, u)
}

// RegisterIntFlag adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func RegisterIntFlag(k, s string, v, dflt int, u string) {
	appCfg.RegisterIntFlag(k, s, v, dflt, u)
}

// RegisterStringFlag adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func RegisterStringFlag(k, s, v, dflt, u string) {
	appCfg.RegisterStringFlag(k, s, v, dflt, u)
}

// RegisterBool adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func RegisterBool(k string, v bool) {
	appCfg.RegisterBool(k, v)
}

// RegisterInt adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func RegisterInt(k string, v int) {
	appCfg.RegisterInt(k, v)
}

// RegisterString adds the information to the AppsConfig struct, but does not
// save it to its ironment variable.
func RegisterString(k, v string) {
	appCfg.RegisterString(k, v)
}
<|endoftext|>


<commit_before>package wikidump

import (
	"bytes"
	"errors"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"testing"
)

type mockTransport struct{}

func (t mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	path := "/scowiki/latest/scowiki-latest-pages-articles.xml.bz2"

	var msg string
	switch {
	case req.Method != "GET":
		msg = "not a GET request"
	case req.URL.Host != "dumps.wikimedia.org":
		msg = "wrong host"
	case req.URL.Path != path:
		msg = "wrong path"
	case req.Body != nil:
		msg = "non-nil Body"
	}
	if msg != "" {
		return nil, errors.New(msg)
	}

	content := []byte("all went well")
	resp := http.Response{
		Status: "200 OK",
		StatusCode: 200,
		Body: ioutil.NopCloser(bytes.NewBuffer(content)),
		ContentLength:  int64(len(content)),
		Request: req,
	}
	return &resp, nil
}

var (
	mockClient = http.Client{Transport: mockTransport{}}
)

func TestDownload(t *testing.T) {
	d, err := ioutil.TempDir("", "dumpparser-test")
	if err != nil {
		panic(err)
	}

	path, err := download("scowiki", d, false, &mockClient)
	if err != nil {
		t.Error(err)
	} else {
		base := filepath.Base(path)
		if base != "scowiki-latest-pages-articles.xml.bz2" {
			t.Errorf("unexpected filename: %s", base)
		}
	}

	err = os.Remove(path)
	if err != nil {
		panic(err)
	}
	err = os.Remove(d)
	if err != nil {
		panic(err)
	}
}
<commit_msg>go fmt<commit_after>package wikidump

import (
	"bytes"
	"errors"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"testing"
)

type mockTransport struct{}

func (t mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	path := "/scowiki/latest/scowiki-latest-pages-articles.xml.bz2"

	var msg string
	switch {
	case req.Method != "GET":
		msg = "not a GET request"
	case req.URL.Host != "dumps.wikimedia.org":
		msg = "wrong host"
	case req.URL.Path != path:
		msg = "wrong path"
	case req.Body != nil:
		msg = "non-nil Body"
	}
	if msg != "" {
		return nil, errors.New(msg)
	}

	content := []byte("all went well")
	resp := http.Response{
		Status:        "200 OK",
		StatusCode:    200,
		Body:          ioutil.NopCloser(bytes.NewBuffer(content)),
		ContentLength: int64(len(content)),
		Request:       req,
	}
	return &resp, nil
}

var (
	mockClient = http.Client{Transport: mockTransport{}}
)

func TestDownload(t *testing.T) {
	d, err := ioutil.TempDir("", "dumpparser-test")
	if err != nil {
		panic(err)
	}

	path, err := download("scowiki", d, false, &mockClient)
	if err != nil {
		t.Error(err)
	} else {
		base := filepath.Base(path)
		if base != "scowiki-latest-pages-articles.xml.bz2" {
			t.Errorf("unexpected filename: %s", base)
		}
	}

	err = os.Remove(path)
	if err != nil {
		panic(err)
	}
	err = os.Remove(d)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package application

import (
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	gocache "github.com/patrickmn/go-cache"

	"github.com/resourced/resourced-master/config"
	"github.com/resourced/resourced-master/messagebus"
	resourced_wire "github.com/resourced/resourced-wire"
)

// NewMessageBus creates a new MessageBus instance.
func (app *Application) NewMessageBus(generalConfig config.GeneralConfig) (*messagebus.MessageBus, error) {
	bus, err := messagebus.New(generalConfig.MessageBus.URL)
	if err != nil {
		return nil, err
	}

	err = bus.DialOthers(generalConfig.MessageBus.Peers)
	if err != nil {
		return nil, err
	}

	return bus, nil
}

func (app *Application) MessageBusHandlers() map[string]func(msg string) {
	peersHeartbeat := func(msg string) {
		fullAddr := resourced_wire.ParseSingle(msg).PlainContent()
		if strings.Contains(fullAddr, "Error") {
			logrus.WithFields(logrus.Fields{
				"Method": "app.MessageBusHandlers",
				"Error":  fullAddr,
			}).Error("Error when parsing content from peers-heartbeat topic")
		}

		if fullAddr != "" {
			peersLengthBeforeSet := len(app.Peers.Items())

			app.Peers.Set(fullAddr, true, gocache.DefaultExpiration)

			if len(app.Peers.Items()) != peersLengthBeforeSet {
				app.RefetchChecksChan <- true
			}
		}
	}

	checksRefetch := func(msg string) {
		content := resourced_wire.ParseSingle(msg).PlainContent()
		if strings.Contains(content, "Error") {
			logrus.WithFields(logrus.Fields{
				"Method": "app.MessageBusHandlers",
				"Error":  content,
			}).Error("Error when parsing content from checks-refetch topic")
		}

		app.RefetchChecksChan <- true
	}

	metricStream := func(msg string) {
		content := resourced_wire.ParseSingle(msg).JSONStringContent()
		if strings.Contains(content, "Error") {
			logrus.WithFields(logrus.Fields{
				"Method": "app.MessageBusHandlers",
				"Error":  content,
			}).Error("Error when parsing content from checks-refetch topic")
		}

		for clientChan, _ := range app.MessageBus.Clients {
			clientChan <- content
		}
	}

	return map[string]func(msg string){
		"peers-heartbeat": peersHeartbeat,
		"checks-refetch":  checksRefetch,
		"metric-":         metricStream,
	}
}

// sendHeartbeatOnce payload using the messagebus mechanism
func (app *Application) sendHeartbeatOnce() error {
	return app.MessageBus.Publish("peers-heartbeat", app.FullAddr())
}

// SendHeartbeat every 30 seconds over message bus.
func (app *Application) SendHeartbeat() {
	for range time.Tick(30 * time.Second) {
		err := app.sendHeartbeatOnce()
		if err != nil {
			logrus.WithFields(logrus.Fields{
				"Method": "app.SendHeartbeat",
				"Error":  err,
			}).Error("Error when sending heartbeat every 30 seconds")
		}
	}
}
<commit_msg>Always refetch checks when message bus says so.<commit_after>package application

import (
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	gocache "github.com/patrickmn/go-cache"

	"github.com/resourced/resourced-master/config"
	"github.com/resourced/resourced-master/messagebus"
	resourced_wire "github.com/resourced/resourced-wire"
)

// NewMessageBus creates a new MessageBus instance.
func (app *Application) NewMessageBus(generalConfig config.GeneralConfig) (*messagebus.MessageBus, error) {
	bus, err := messagebus.New(generalConfig.MessageBus.URL)
	if err != nil {
		return nil, err
	}

	err = bus.DialOthers(generalConfig.MessageBus.Peers)
	if err != nil {
		return nil, err
	}

	return bus, nil
}

func (app *Application) MessageBusHandlers() map[string]func(msg string) {
	peersHeartbeat := func(msg string) {
		fullAddr := resourced_wire.ParseSingle(msg).PlainContent()
		if strings.Contains(fullAddr, "Error") {
			logrus.WithFields(logrus.Fields{
				"Method": "app.MessageBusHandlers",
				"Error":  fullAddr,
			}).Error("Error when parsing content from peers-heartbeat topic")
		}

		if fullAddr != "" {
			app.Peers.Set(fullAddr, true, gocache.DefaultExpiration)
			app.RefetchChecksChan <- true
		}
	}

	checksRefetch := func(msg string) {
		content := resourced_wire.ParseSingle(msg).PlainContent()
		if strings.Contains(content, "Error") {
			logrus.WithFields(logrus.Fields{
				"Method": "app.MessageBusHandlers",
				"Error":  content,
			}).Error("Error when parsing content from checks-refetch topic")
		}

		app.RefetchChecksChan <- true
	}

	metricStream := func(msg string) {
		content := resourced_wire.ParseSingle(msg).JSONStringContent()
		if strings.Contains(content, "Error") {
			logrus.WithFields(logrus.Fields{
				"Method": "app.MessageBusHandlers",
				"Error":  content,
			}).Error("Error when parsing content from checks-refetch topic")
		}

		for clientChan, _ := range app.MessageBus.Clients {
			clientChan <- content
		}
	}

	return map[string]func(msg string){
		"peers-heartbeat": peersHeartbeat,
		"checks-refetch":  checksRefetch,
		"metric-":         metricStream,
	}
}

// sendHeartbeatOnce payload using the messagebus mechanism
func (app *Application) sendHeartbeatOnce() error {
	return app.MessageBus.Publish("peers-heartbeat", app.FullAddr())
}

// SendHeartbeat every 30 seconds over message bus.
func (app *Application) SendHeartbeat() {
	for range time.Tick(30 * time.Second) {
		err := app.sendHeartbeatOnce()
		if err != nil {
			logrus.WithFields(logrus.Fields{
				"Method": "app.SendHeartbeat",
				"Error":  err,
			}).Error("Error when sending heartbeat every 30 seconds")
		}
	}
}
<|endoftext|>


<commit_before>package backends

import (
	"bytes"
	_ "crypto/sha512"
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/schachmat/wego/iface"
)

type wwoCond struct {
	TmpCor        *int                     `json:"chanceofrain,string"`
	TmpCode       int                      `json:"weatherCode,string"`
	TmpDesc       []struct{ Value string } `json:"weatherDesc"`
	FeelsLikeC    *float32                 `json:",string"`
	PrecipMM      *float32                 `json:"precipMM,string"`
	TmpTempC      *float32                 `json:"tempC,string"`
	TmpTempC2     *float32                 `json:"temp_C,string"`
	TmpTime       *int                     `json:"time,string"`
	VisibleDistKM *float32                 `json:"visibility,string"`
	WindGustKmph  *float32                 `json:",string"`
	WinddirDegree *int                     `json:"winddirDegree,string"`
	WindspeedKmph *float32                 `json:"windspeedKmph,string"`
}

type wwoDay struct {
	Astronomy []struct {
		Moonrise string
		Moonset  string
		Sunrise  string
		Sunset   string
	}
	Date     string
	Hourly   []wwoCond
	MaxtempC *float32 `json:"maxtempC,string"`
	MintempC *float32 `json:"mintempC,string"`
}

type wwoResponse struct {
	Data struct {
		CurCond []wwoCond              `json:"current_condition"`
		Err     []struct{ Msg string } `json:"error"`
		Req     []struct {
			Query string `json:"query"`
			Type  string `json:"type"`
		} `json:"request"`
		Days []wwoDay `json:"weather"`
	} `json:"data"`
}

type wwoCoordinateResp struct {
	Search struct {
		Result []struct {
			Longitude *float32 `json:"longitude,string"`
			Latitude  *float32 `json:"latitude,string"`
		} `json:"result"`
	} `json:"search_api"`
}

type wwoConfig struct {
	apiKey   string
	language string
}

const (
	wwoSuri = "https://api.worldweatheronline.com/free/v2/search.ashx?"
	wwoWuri = "https://api.worldweatheronline.com/free/v2/weather.ashx?"
)

func wwoParseCond(cond wwoCond, date time.Time) (ret iface.Cond) {
	ret.ChanceOfRainPercent = cond.TmpCor

	codemap := map[int]iface.WeatherCode{
		113: iface.CodeSunny,
		116: iface.CodePartlyCloudy,
		119: iface.CodeCloudy,
		122: iface.CodeVeryCloudy,
		143: iface.CodeFog,
		176: iface.CodeLightShowers,
		179: iface.CodeLightSleetShowers,
		182: iface.CodeLightSleet,
		185: iface.CodeLightSleet,
		200: iface.CodeThunderyShowers,
		227: iface.CodeLightSnow,
		230: iface.CodeHeavySnow,
		248: iface.CodeFog,
		260: iface.CodeFog,
		263: iface.CodeLightShowers,
		266: iface.CodeLightRain,
		281: iface.CodeLightSleet,
		284: iface.CodeLightSleet,
		293: iface.CodeLightRain,
		296: iface.CodeLightRain,
		299: iface.CodeHeavyShowers,
		302: iface.CodeHeavyRain,
		305: iface.CodeHeavyShowers,
		308: iface.CodeHeavyRain,
		311: iface.CodeLightSleet,
		314: iface.CodeLightSleet,
		317: iface.CodeLightSleet,
		320: iface.CodeLightSnow,
		323: iface.CodeLightSnowShowers,
		326: iface.CodeLightSnowShowers,
		329: iface.CodeHeavySnow,
		332: iface.CodeHeavySnow,
		335: iface.CodeHeavySnowShowers,
		338: iface.CodeHeavySnow,
		350: iface.CodeLightSleet,
		353: iface.CodeLightShowers,
		356: iface.CodeHeavyShowers,
		359: iface.CodeHeavyRain,
		362: iface.CodeLightSleetShowers,
		365: iface.CodeLightSleetShowers,
		368: iface.CodeLightSnowShowers,
		371: iface.CodeHeavySnowShowers,
		374: iface.CodeLightSleetShowers,
		377: iface.CodeLightSleet,
		386: iface.CodeThunderyShowers,
		389: iface.CodeThunderyHeavyRain,
		392: iface.CodeThunderySnowShowers,
		395: iface.CodeHeavySnowShowers,
	}
	ret.Code = iface.CodeUnknown
	if val, ok := codemap[cond.TmpCode]; ok {
		ret.Code = val
	}

	if cond.TmpDesc != nil && len(cond.TmpDesc) > 0 {
		ret.Desc = cond.TmpDesc[0].Value
	}

	ret.TempC = cond.TmpTempC2
	if cond.TmpTempC != nil {
		ret.TempC = cond.TmpTempC
	}
	ret.FeelsLikeC = cond.FeelsLikeC

	if cond.PrecipMM != nil {
		ret.PrecipM = new(float32)
		*ret.PrecipM = *cond.PrecipMM / 1000
	}

	ret.Time = date
	if cond.TmpTime != nil {
		year, month, day := date.Date()
		hour, min := *cond.TmpTime/100, *cond.TmpTime%100
		ret.Time = time.Date(year, month, day, hour, min, 0, 0, time.UTC)
	}

	if cond.VisibleDistKM != nil {
		ret.VisibleDistM = new(float32)
		*ret.VisibleDistM = *cond.VisibleDistKM * 1000
	}

	if cond.WinddirDegree != nil && *cond.WinddirDegree >= 0 {
		ret.WinddirDegree = new(int)
		*ret.WinddirDegree = *cond.WinddirDegree % 360
	}

	ret.WindspeedKmph = cond.WindspeedKmph
	ret.WindGustKmph = cond.WindGustKmph

	return
}

func wwoParseDay(day wwoDay, index int) (ret iface.Day) {
	//TODO: Astronomy

	ret.Date = time.Now().Add(time.Hour * 24 * time.Duration(index))
	date, err := time.Parse("2006-01-02", day.Date)
	if err == nil {
		ret.Date = date
	}

	ret.MaxtempC = day.MaxtempC
	ret.MintempC = day.MintempC

	if day.Hourly != nil && len(day.Hourly) > 0 {
		for _, slot := range day.Hourly {
			ret.Slots = append(ret.Slots, wwoParseCond(slot, date))
		}
	}

	return
}

func wwoUnmarshalLang(body []byte, r *wwoResponse, lang string) error {
	var rv map[string]interface{}
	if err := json.Unmarshal(body, &rv); err != nil {
		return err
	}
	if data, ok := rv["data"].(map[string]interface{}); ok {
		if ccs, ok := data["current_condition"].([]interface{}); ok {
			for _, cci := range ccs {
				cc, ok := cci.(map[string]interface{})
				if !ok {
					continue
				}
				langs, ok := cc["lang_"+lang].([]interface{})
				if !ok || len(langs) == 0 {
					continue
				}
				weatherDesc, ok := cc["weatherDesc"].([]interface{})
				if !ok || len(weatherDesc) == 0 {
					continue
				}
				weatherDesc[0] = langs[0]
			}
		}
		if ws, ok := data["weather"].([]interface{}); ok {
			for _, wi := range ws {
				w, ok := wi.(map[string]interface{})
				if !ok {
					continue
				}
				if hs, ok := w["hourly"].([]interface{}); ok {
					for _, hi := range hs {
						h, ok := hi.(map[string]interface{})
						if !ok {
							continue
						}
						langs, ok := h["lang_"+lang].([]interface{})
						if !ok || len(langs) == 0 {
							continue
						}
						weatherDesc, ok := h["weatherDesc"].([]interface{})
						if !ok || len(weatherDesc) == 0 {
							continue
						}
						weatherDesc[0] = langs[0]
					}
				}
			}
		}
	}
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(rv); err != nil {
		return err
	}
	return json.NewDecoder(&buf).Decode(r)
}

func (c *wwoConfig) Setup() {
	flag.StringVar(&c.apiKey, "wwo-api-key", "", "wwo backend: the api `KEY` to use")
	flag.StringVar(&c.language, "wwo-lang", "en", "wwo backend: the `LANGUAGE` to request from wwo")
}

func getCoordinatesFromAPI(queryParams []string, c chan *iface.LatLon) {
	var coordResp wwoCoordinateResp
	res, err := http.Get(wwoSuri + strings.Join(queryParams, "&"))
	if err != nil {
		log.Println("Unable to fetch geo location:", err)
		c <- nil
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Println("Unable to read geo location data:", err)
		c <- nil
	}

	if err = json.Unmarshal(body, &coordResp); err != nil {
		log.Println("Unable to unmarshal geo location data:", err)
		c <- nil
	}

	r := coordResp.Search.Result
	if len(r) < 1 || r[0].Latitude == nil || r[0].Longitude == nil {
		log.Println("Malformed geo location response")
		c <- nil
	}

	c <- &iface.LatLon{Latitude: *r[0].Latitude, Longitude: *r[0].Longitude}
}

func (c *wwoConfig) Fetch(loc string, numdays int) iface.Data {
	var params []string
	var resp wwoResponse
	var ret iface.Data
	coordChan := make(chan *iface.LatLon)

	if len(c.apiKey) == 0 {
		log.Fatal("No API key specified. Setup instructions are in the README.")
	}
	params = append(params, "key="+c.apiKey)

	if len(loc) > 0 {
		params = append(params, "q="+url.QueryEscape(loc))
	}
	params = append(params, "format=json")
	params = append(params, "num_of_days="+strconv.Itoa(numdays))
	params = append(params, "tp=3")

	go getCoordinatesFromAPI(params, coordChan)

	if c.language != "" {
		params = append(params, "lang="+c.language)
	}

	res, err := http.Get(wwoWuri + strings.Join(params, "&"))
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}

	if c.language == "" {
		if err = json.Unmarshal(body, &resp); err != nil {
			log.Println(err)
		}
	} else {
		if err = wwoUnmarshalLang(body, &resp, c.language); err != nil {
			log.Println(err)
		}
	}

	if resp.Data.Req == nil || len(resp.Data.Req) < 1 {
		if resp.Data.Err != nil && len(resp.Data.Err) >= 1 {
			log.Fatal(resp.Data.Err[0].Msg)
		}
		log.Fatal("Malformed response.")
	}

	ret.Location = resp.Data.Req[0].Type + ": " + resp.Data.Req[0].Query
	ret.GeoLoc = <-coordChan

	if resp.Data.CurCond != nil && len(resp.Data.CurCond) > 0 {
		ret.Current = wwoParseCond(resp.Data.CurCond[0], time.Now())
	}

	if resp.Data.Days != nil && numdays > 0 {
		for i, day := range resp.Data.Days {
			ret.Forecast = append(ret.Forecast, wwoParseDay(day, i))
		}
	}

	return ret
}

func init() {
	iface.AllBackends["worldweatheronline.com"] = &wwoConfig{}
}
<commit_msg>wwo-backend: add debug flag to print raw response<commit_after>package backends

import (
	"bytes"
	_ "crypto/sha512"
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/schachmat/wego/iface"
)

type wwoCond struct {
	TmpCor        *int                     `json:"chanceofrain,string"`
	TmpCode       int                      `json:"weatherCode,string"`
	TmpDesc       []struct{ Value string } `json:"weatherDesc"`
	FeelsLikeC    *float32                 `json:",string"`
	PrecipMM      *float32                 `json:"precipMM,string"`
	TmpTempC      *float32                 `json:"tempC,string"`
	TmpTempC2     *float32                 `json:"temp_C,string"`
	TmpTime       *int                     `json:"time,string"`
	VisibleDistKM *float32                 `json:"visibility,string"`
	WindGustKmph  *float32                 `json:",string"`
	WinddirDegree *int                     `json:"winddirDegree,string"`
	WindspeedKmph *float32                 `json:"windspeedKmph,string"`
}

type wwoDay struct {
	Astronomy []struct {
		Moonrise string
		Moonset  string
		Sunrise  string
		Sunset   string
	}
	Date     string
	Hourly   []wwoCond
	MaxtempC *float32 `json:"maxtempC,string"`
	MintempC *float32 `json:"mintempC,string"`
}

type wwoResponse struct {
	Data struct {
		CurCond []wwoCond              `json:"current_condition"`
		Err     []struct{ Msg string } `json:"error"`
		Req     []struct {
			Query string `json:"query"`
			Type  string `json:"type"`
		} `json:"request"`
		Days []wwoDay `json:"weather"`
	} `json:"data"`
}

type wwoCoordinateResp struct {
	Search struct {
		Result []struct {
			Longitude *float32 `json:"longitude,string"`
			Latitude  *float32 `json:"latitude,string"`
		} `json:"result"`
	} `json:"search_api"`
}

type wwoConfig struct {
	apiKey   string
	language string
	debug    bool
}

const (
	wwoSuri = "https://api.worldweatheronline.com/free/v2/search.ashx?"
	wwoWuri = "https://api.worldweatheronline.com/free/v2/weather.ashx?"
)

func wwoParseCond(cond wwoCond, date time.Time) (ret iface.Cond) {
	ret.ChanceOfRainPercent = cond.TmpCor

	codemap := map[int]iface.WeatherCode{
		113: iface.CodeSunny,
		116: iface.CodePartlyCloudy,
		119: iface.CodeCloudy,
		122: iface.CodeVeryCloudy,
		143: iface.CodeFog,
		176: iface.CodeLightShowers,
		179: iface.CodeLightSleetShowers,
		182: iface.CodeLightSleet,
		185: iface.CodeLightSleet,
		200: iface.CodeThunderyShowers,
		227: iface.CodeLightSnow,
		230: iface.CodeHeavySnow,
		248: iface.CodeFog,
		260: iface.CodeFog,
		263: iface.CodeLightShowers,
		266: iface.CodeLightRain,
		281: iface.CodeLightSleet,
		284: iface.CodeLightSleet,
		293: iface.CodeLightRain,
		296: iface.CodeLightRain,
		299: iface.CodeHeavyShowers,
		302: iface.CodeHeavyRain,
		305: iface.CodeHeavyShowers,
		308: iface.CodeHeavyRain,
		311: iface.CodeLightSleet,
		314: iface.CodeLightSleet,
		317: iface.CodeLightSleet,
		320: iface.CodeLightSnow,
		323: iface.CodeLightSnowShowers,
		326: iface.CodeLightSnowShowers,
		329: iface.CodeHeavySnow,
		332: iface.CodeHeavySnow,
		335: iface.CodeHeavySnowShowers,
		338: iface.CodeHeavySnow,
		350: iface.CodeLightSleet,
		353: iface.CodeLightShowers,
		356: iface.CodeHeavyShowers,
		359: iface.CodeHeavyRain,
		362: iface.CodeLightSleetShowers,
		365: iface.CodeLightSleetShowers,
		368: iface.CodeLightSnowShowers,
		371: iface.CodeHeavySnowShowers,
		374: iface.CodeLightSleetShowers,
		377: iface.CodeLightSleet,
		386: iface.CodeThunderyShowers,
		389: iface.CodeThunderyHeavyRain,
		392: iface.CodeThunderySnowShowers,
		395: iface.CodeHeavySnowShowers,
	}
	ret.Code = iface.CodeUnknown
	if val, ok := codemap[cond.TmpCode]; ok {
		ret.Code = val
	}

	if cond.TmpDesc != nil && len(cond.TmpDesc) > 0 {
		ret.Desc = cond.TmpDesc[0].Value
	}

	ret.TempC = cond.TmpTempC2
	if cond.TmpTempC != nil {
		ret.TempC = cond.TmpTempC
	}
	ret.FeelsLikeC = cond.FeelsLikeC

	if cond.PrecipMM != nil {
		ret.PrecipM = new(float32)
		*ret.PrecipM = *cond.PrecipMM / 1000
	}

	ret.Time = date
	if cond.TmpTime != nil {
		year, month, day := date.Date()
		hour, min := *cond.TmpTime/100, *cond.TmpTime%100
		ret.Time = time.Date(year, month, day, hour, min, 0, 0, time.UTC)
	}

	if cond.VisibleDistKM != nil {
		ret.VisibleDistM = new(float32)
		*ret.VisibleDistM = *cond.VisibleDistKM * 1000
	}

	if cond.WinddirDegree != nil && *cond.WinddirDegree >= 0 {
		ret.WinddirDegree = new(int)
		*ret.WinddirDegree = *cond.WinddirDegree % 360
	}

	ret.WindspeedKmph = cond.WindspeedKmph
	ret.WindGustKmph = cond.WindGustKmph

	return
}

func wwoParseDay(day wwoDay, index int) (ret iface.Day) {
	//TODO: Astronomy

	ret.Date = time.Now().Add(time.Hour * 24 * time.Duration(index))
	date, err := time.Parse("2006-01-02", day.Date)
	if err == nil {
		ret.Date = date
	}

	ret.MaxtempC = day.MaxtempC
	ret.MintempC = day.MintempC

	if day.Hourly != nil && len(day.Hourly) > 0 {
		for _, slot := range day.Hourly {
			ret.Slots = append(ret.Slots, wwoParseCond(slot, date))
		}
	}

	return
}

func wwoUnmarshalLang(body []byte, r *wwoResponse, lang string) error {
	var rv map[string]interface{}
	if err := json.Unmarshal(body, &rv); err != nil {
		return err
	}
	if data, ok := rv["data"].(map[string]interface{}); ok {
		if ccs, ok := data["current_condition"].([]interface{}); ok {
			for _, cci := range ccs {
				cc, ok := cci.(map[string]interface{})
				if !ok {
					continue
				}
				langs, ok := cc["lang_"+lang].([]interface{})
				if !ok || len(langs) == 0 {
					continue
				}
				weatherDesc, ok := cc["weatherDesc"].([]interface{})
				if !ok || len(weatherDesc) == 0 {
					continue
				}
				weatherDesc[0] = langs[0]
			}
		}
		if ws, ok := data["weather"].([]interface{}); ok {
			for _, wi := range ws {
				w, ok := wi.(map[string]interface{})
				if !ok {
					continue
				}
				if hs, ok := w["hourly"].([]interface{}); ok {
					for _, hi := range hs {
						h, ok := hi.(map[string]interface{})
						if !ok {
							continue
						}
						langs, ok := h["lang_"+lang].([]interface{})
						if !ok || len(langs) == 0 {
							continue
						}
						weatherDesc, ok := h["weatherDesc"].([]interface{})
						if !ok || len(weatherDesc) == 0 {
							continue
						}
						weatherDesc[0] = langs[0]
					}
				}
			}
		}
	}
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(rv); err != nil {
		return err
	}
	return json.NewDecoder(&buf).Decode(r)
}

func (c *wwoConfig) Setup() {
	flag.StringVar(&c.apiKey, "wwo-api-key", "", "wwo backend: the api `KEY` to use")
	flag.StringVar(&c.language, "wwo-lang", "en", "wwo backend: the `LANGUAGE` to request from wwo")
	flag.BoolVar(&c.debug, "wwo-debug", false, "wwo backend: print raw requests and responses")
}

func (c *wwoConfig) getCoordinatesFromAPI(queryParams []string, res chan *iface.LatLon) {
	var coordResp wwoCoordinateResp
	requri := wwoSuri + strings.Join(queryParams, "&")
	hres, err := http.Get(requri)
	if err != nil {
		log.Println("Unable to fetch geo location:", err)
		res <- nil
	} else if hres.StatusCode != 200 {
		log.Println("Unable to fetch geo location: http status", hres.StatusCode)
		res <- nil
	}
	defer hres.Body.Close()

	body, err := ioutil.ReadAll(hres.Body)
	if err != nil {
		log.Println("Unable to read geo location data:", err)
		res <- nil
	}

	if c.debug {
		log.Println("Geo location request:", requri)
		log.Printf("Geo location response: %s\n", body)
	}

	if err = json.Unmarshal(body, &coordResp); err != nil {
		log.Println("Unable to unmarshal geo location data:", err)
		res <- nil
	}

	r := coordResp.Search.Result
	if len(r) < 1 || r[0].Latitude == nil || r[0].Longitude == nil {
		log.Println("Malformed geo location response")
		res <- nil
	}

	res <- &iface.LatLon{Latitude: *r[0].Latitude, Longitude: *r[0].Longitude}
}

func (c *wwoConfig) Fetch(loc string, numdays int) iface.Data {
	var params []string
	var resp wwoResponse
	var ret iface.Data
	coordChan := make(chan *iface.LatLon)

	if len(c.apiKey) == 0 {
		log.Fatal("No API key specified. Setup instructions are in the README.")
	}
	params = append(params, "key="+c.apiKey)

	if len(loc) > 0 {
		params = append(params, "q="+url.QueryEscape(loc))
	}
	params = append(params, "format=json")
	params = append(params, "num_of_days="+strconv.Itoa(numdays))
	params = append(params, "tp=3")

	go c.getCoordinatesFromAPI(params, coordChan)

	if c.language != "" {
		params = append(params, "lang="+c.language)
	}
	requri := wwoWuri + strings.Join(params, "&")

	res, err := http.Get(requri)
	if err != nil {
		log.Fatal("Unable to get weather data: ", err)
	} else if res.StatusCode != 200 {
		log.Fatal("Unable to get weather data: http status ", res.StatusCode)
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}

	if c.debug {
		log.Println("Weather request:", requri)
		log.Printf("Weather response: %s\n", body)
	}

	if c.language == "" {
		if err = json.Unmarshal(body, &resp); err != nil {
			log.Println(err)
		}
	} else {
		if err = wwoUnmarshalLang(body, &resp, c.language); err != nil {
			log.Println(err)
		}
	}

	if resp.Data.Req == nil || len(resp.Data.Req) < 1 {
		if resp.Data.Err != nil && len(resp.Data.Err) >= 1 {
			log.Fatal(resp.Data.Err[0].Msg)
		}
		log.Fatal("Malformed response.")
	}

	ret.Location = resp.Data.Req[0].Type + ": " + resp.Data.Req[0].Query
	ret.GeoLoc = <-coordChan

	if resp.Data.CurCond != nil && len(resp.Data.CurCond) > 0 {
		ret.Current = wwoParseCond(resp.Data.CurCond[0], time.Now())
	}

	if resp.Data.Days != nil && numdays > 0 {
		for i, day := range resp.Data.Days {
			ret.Forecast = append(ret.Forecast, wwoParseDay(day, i))
		}
	}

	return ret
}

func init() {
	iface.AllBackends["worldweatheronline.com"] = &wwoConfig{}
}
<|endoftext|>


<commit_before>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/couchbaselabs/indexing/api"
	"log"
	"net"
	"net/rpc"
	"net/rpc/jsonrpc"
	"sync"
)

type MutationManager struct {
	enginemap   map[string]api.Finder
	sequencemap api.IndexSequenceMap
	chmutation  chan *api.Mutation                       //buffered channel to store incoming mutations
	chworkers   [MAX_MUTATION_WORKERS]chan *api.Mutation //buffered channel for each worker
	chseq       chan seqNotification                     //buffered channel to store sequence notifications from workers
	chddl       chan ddlNotification                     //channel for incoming ddl notifications
}

type ddlNotification struct {
	indexinfo api.IndexInfo
	engine    api.Finder
	ddltype   api.RequestType
}

type seqNotification struct {
	engine  api.Finder
	indexid string
	seqno   uint64
	vbucket uint16
}

//error state flag
var indexerErrorState bool
var indexerErrorString string
var wg sync.WaitGroup
var chdrain chan bool

const MAX_INCOMING_QUEUE = 50000
const MAX_MUTATION_WORKERS = 8
const MAX_WORKER_QUEUE = 1000
const MAX_SEQUENCE_QUEUE = 50000
const META_DOC_ID = "."
const SEQ_MAP_PERSIST_INTERVAL = 10 //number of mutations after which sequence map is persisted

var mutationMgr MutationManager

//perf data
var mutationCount int64

//---Exported RPC methods which are available to remote clients

//This function returns a map of <Index, SequenceVector> based on the IndexList received in request
func (m *MutationManager) GetSequenceVectors(indexList api.IndexList, reply *api.IndexSequenceMap) error {

	// if indexer is in error state, let the error handing routines finish
	if indexerErrorState == true {
		wg.Wait()
		//reset the error state at each handshake
		indexerErrorState = false
		indexerErrorString = ""
	}

	//if indexList is nil, return the complete map
	if len(indexList) == 0 {
		*reply = m.sequencemap
		if options.debugLog {
			log.Printf("Mutation Manager returning complete SequenceMap %v", m.sequencemap)
		}
		return nil
	}

	//loop through the list of requested indexes and return the sequenceVector for those indexes
	var replyMap = make(api.IndexSequenceMap)
	for _, idx := range indexList {
		//if the requested index is not found, return an error
		v, ok := m.sequencemap[idx]
		if !ok {
			return errors.New("Requested Index Not Found")
		}

		//add to the reply map
		if options.debugLog {
			log.Printf("Mutation Manager returning sequence vector for index %v %v", idx, v)
		}
		replyMap[idx] = v
	}
	*reply = replyMap
	return nil

}

//This method takes as input an api.Mutation and copies into mutation queue for processing
func (m *MutationManager) ProcessSingleMutation(mutation *api.Mutation, reply *bool) error {
	if options.debugLog {
		log.Printf("Received Mutation Type %s Indexid %v, Docid %v, Vbucket %v, Seqno %v", mutation.Type, mutation.Indexid, mutation.Docid, mutation.Vbucket, mutation.Seqno)
	}

	//if there is any pending error, reply with that. This will force a handshake again.
	if indexerErrorState == true {
		*reply = false
	}

	//copy the mutation data and return
	m.chmutation <- mutation
	*reply = true
	return nil

}

//---End of exported RPC methods

//read incoming mutation and distribute it on worker queues based on vbucketid
func (m *MutationManager) manageMutationQueue() {

	for {
		select {
		case mut, ok := <-m.chmutation:
			if ok {
				m.chworkers[mut.Vbucket%MAX_MUTATION_WORKERS] <- mut
			}
		case <-chdrain:
			wg.Add(1)
			m.drainMutationChannel(m.chmutation)
		}
	}

}

//start a mutation worker which handles mutation on the specified workerId channel
func (m *MutationManager) startMutationWorker(workerId int) {

	for {
		select {
		case mutation := <-m.chworkers[workerId]:
			m.handleMutation(mutation)
		case <-chdrain:
			wg.Add(1)
			m.drainMutationChannel(m.chworkers[workerId])
		}
	}
}

func (m *MutationManager) handleMutation(mutation *api.Mutation) {

	if mutation.Type == api.INSERT {

		var key api.Key
		var value api.Value
		var err error

		if key, err = api.NewKey(mutation.SecondaryKey, mutation.Docid); err != nil {
			log.Printf("Error Generating Key From Mutation %v. Skipped.", err)
			return
		}

		if value, err = api.NewValue(mutation.SecondaryKey, mutation.Docid, mutation.Vbucket, mutation.Seqno); err != nil {
			log.Printf("Error Generating Value From Mutation %v. Skipped.", err)
			return
		}

		if engine, ok := m.enginemap[mutation.Indexid]; ok {
			if err := engine.InsertMutation(key, value); err != nil {
				log.Printf("Error from Engine during InsertMutation. Key %v. Index %v. Error %v", key, mutation.Docid, err)
			}
			//send notification for this seqno to be recorded in SeqVector
			seqnotify := seqNotification{engine: engine,
				indexid: mutation.Indexid,
				seqno:   mutation.Seqno,
				vbucket: mutation.Vbucket,
			}
			m.chseq <- seqnotify
		} else {
			err := fmt.Sprintf("Unknown Index %v or Engine not found", mutation.Indexid)
			m.initErrorState(err)
		}

	} else if mutation.Type == api.DELETE {

		if engine, ok := m.enginemap[mutation.Indexid]; ok {
			if err := engine.DeleteMutation(mutation.Docid); err != nil {
				log.Printf("Error from Engine during Delete Mutation. Key %v. Error %v", mutation.Docid, err)
				return
			}
			//send notification for this seqno to be recorded in SeqVector
			seqnotify := seqNotification{engine: engine,
				indexid: mutation.Indexid,
				seqno:   mutation.Seqno,
				vbucket: mutation.Vbucket,
			}
			m.chseq <- seqnotify
		} else {
			err := fmt.Sprintf("Unknown Index %v or Engine not found", mutation.Indexid)
			m.initErrorState(err)
		}
	}
}

func StartMutationManager(engineMap map[string]api.Finder) (chan ddlNotification, error) {

	var err error

	//init the mutation manager maps
	mutationMgr.sequencemap = make(api.IndexSequenceMap)
	//copy the inital map from the indexer
	mutationMgr.enginemap = engineMap
	mutationMgr.initSequenceMapFromPersistence()

	//create channel to receive notification for new sequence numbers
	//and start a goroutine to manage it
	mutationMgr.chseq = make(chan seqNotification, MAX_SEQUENCE_QUEUE)
	go mutationMgr.manageSeqNotification()

	//create a channel to receive notification from indexer
	//and start a goroutine to listen to it
	mutationMgr.chddl = make(chan ddlNotification)
	go mutationMgr.manageIndexerNotification()

	//init the channel for incoming mutations
	mutationMgr.chmutation = make(chan *api.Mutation, MAX_INCOMING_QUEUE)
	go mutationMgr.manageMutationQueue()

	//init the workers for processing mutations
	for w := 0; w < MAX_MUTATION_WORKERS; w++ {
		mutationMgr.chworkers[w] = make(chan *api.Mutation, MAX_WORKER_QUEUE)
		go mutationMgr.startMutationWorker(w)
	}

	//init error state
	indexerErrorState = false
	indexerErrorString = ""
	chdrain = make(chan bool)

	//start the rpc server
	if err = startRPCServer(); err != nil {
		return nil, err
	}

	return mutationMgr.chddl, nil
}

func startRPCServer() error {

	log.Println("Starting Mutation Manager")
	server := rpc.NewServer()
	server.Register(&mutationMgr)

	server.HandleHTTP(rpc.DefaultRPCPath, rpc.DefaultDebugPath)

	l, err := net.Listen("tcp", ":8096")
	if err != nil {
		return err
	}

	go func() {
		for {
			conn, err := l.Accept()
			if err != nil {
				log.Printf("Error in Accept %v. Shutting down")
				//FIXME Add a cleanup function
				return
			}
			go server.ServeCodec(jsonrpc.NewServerCodec(conn))
		}
	}()
	return nil

}

func (m *MutationManager) manageIndexerNotification() {

	ok := true
	var ddl ddlNotification
	for ok {
		ddl, ok = <-m.chddl
		if ok {
			switch ddl.ddltype {
			case api.CREATE:
				m.enginemap[ddl.indexinfo.Uuid] = ddl.engine
				//init sequence map of new index
				seqVec := make(api.SequenceVector, api.MAX_VBUCKETS)
				m.sequencemap[ddl.indexinfo.Uuid] = seqVec
			case api.DROP:
				delete(m.enginemap, ddl.indexinfo.Uuid)
				//FIXME : Delete index entry from sequence map
			default:
				log.Printf("Mutation Manager Received Unsupported Notification %v", ddl.ddltype)
			}
		}
	}
}

func (m *MutationManager) manageSeqNotification() {

	var seq seqNotification
	openSeqCount := 0
	ok := true
	var perfWriteCount int64

	for ok {
		select {
		case seq, ok = <-m.chseq:
			if ok {
				seqVector, exists := m.sequencemap[seq.indexid]
				if !exists {
					log.Printf("IndexId %v not found in Sequence Vector. INCONSISTENT INDEXER STATE!!!", seq.indexid)
					break
				}
				seqVector[seq.vbucket] = seq.seqno
				m.sequencemap[seq.indexid] = seqVector
				openSeqCount += 1
				perfWriteCount += 1
				//persist only after SEQ_MAP_PERSIST_INTERVAL
				if openSeqCount == SEQ_MAP_PERSIST_INTERVAL {
					m.persistSequenceMap()
					openSeqCount = 0
				}
				if perfWriteCount%10000 == 0 {
					log.Printf("Processed Mutation %v", perfWriteCount)
				}
			}
		case <-chdrain:
			wg.Add(1)
			m.drainSeqChannel(m.chseq)
		}
	}
}

func (m *MutationManager) initSequenceMapFromPersistence() {

	sequenceVector := make(api.SequenceVector, api.MAX_VBUCKETS)
	for idx, engine := range m.enginemap {
		metaval, err := engine.GetMeta(META_DOC_ID)
		if err != nil {
			log.Printf("Error retreiving Meta from Engine %v", err)
		}
		err = json.Unmarshal([]byte(metaval), &sequenceVector)
		if err != nil {
			log.Printf("Error unmarshalling SequenceVector %v", err)
		}
		m.sequencemap[idx] = sequenceVector
	}
}

func (m *MutationManager) persistSequenceMap() {

	for idx, seqm := range m.sequencemap {
		jsonval, err := json.Marshal(seqm)
		if err != nil {
			log.Printf("Error Marshalling SequenceMap %v", err)
		} else {
			//FIXME - Handle Error here
			m.enginemap[idx].InsertMeta(META_DOC_ID, string(jsonval))
		}
	}
}

func (m *MutationManager) initErrorState(err string) {

	indexerErrorState = true
	indexerErrorString = err

	//send drain signal to mutation queue
	chdrain <- true

	//send drain signal to sequence queue
	chdrain <- true

	//send drain signal to worker queues
	for w := 0; w < MAX_MUTATION_WORKERS; w++ {
		chdrain <- true
	}

}

func (m *MutationManager) drainMutationChannel(ch chan *api.Mutation) {

loop:
	for {
		select {
		case <-ch:
			continue
		default:
			break loop
		}
	}
	wg.Done()

}

func (m *MutationManager) drainSeqChannel(ch chan seqNotification) {

loop:
	for {
		select {
		case <-ch:
			continue
		default:
			break loop
		}
	}
	wg.Done()

}
<commit_msg>Persist sequencemap for every mutation<commit_after>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/couchbaselabs/indexing/api"
	"log"
	"net"
	"net/rpc"
	"net/rpc/jsonrpc"
	"sync"
)

type MutationManager struct {
	enginemap   map[string]api.Finder
	sequencemap api.IndexSequenceMap
	chmutation  chan *api.Mutation                       //buffered channel to store incoming mutations
	chworkers   [MAX_MUTATION_WORKERS]chan *api.Mutation //buffered channel for each worker
	chseq       chan seqNotification                     //buffered channel to store sequence notifications from workers
	chddl       chan ddlNotification                     //channel for incoming ddl notifications
}

type ddlNotification struct {
	indexinfo api.IndexInfo
	engine    api.Finder
	ddltype   api.RequestType
}

type seqNotification struct {
	engine  api.Finder
	indexid string
	seqno   uint64
	vbucket uint16
}

//error state flag
var indexerErrorState bool
var indexerErrorString string
var wg sync.WaitGroup
var chdrain chan bool

const MAX_INCOMING_QUEUE = 50000
const MAX_MUTATION_WORKERS = 8
const MAX_WORKER_QUEUE = 1000
const MAX_SEQUENCE_QUEUE = 50000
const META_DOC_ID = "."
const SEQ_MAP_PERSIST_INTERVAL = 1 //number of mutations after which sequence map is persisted

var mutationMgr MutationManager

//perf data
var mutationCount int64

//---Exported RPC methods which are available to remote clients

//This function returns a map of <Index, SequenceVector> based on the IndexList received in request
func (m *MutationManager) GetSequenceVectors(indexList api.IndexList, reply *api.IndexSequenceMap) error {

	// if indexer is in error state, let the error handing routines finish
	if indexerErrorState == true {
		wg.Wait()
		//reset the error state at each handshake
		indexerErrorState = false
		indexerErrorString = ""
	}

	//if indexList is nil, return the complete map
	if len(indexList) == 0 {
		*reply = m.sequencemap
		if options.debugLog {
			log.Printf("Mutation Manager returning complete SequenceMap %v", m.sequencemap)
		}
		return nil
	}

	//loop through the list of requested indexes and return the sequenceVector for those indexes
	var replyMap = make(api.IndexSequenceMap)
	for _, idx := range indexList {
		//if the requested index is not found, return an error
		v, ok := m.sequencemap[idx]
		if !ok {
			return errors.New("Requested Index Not Found")
		}

		//add to the reply map
		if options.debugLog {
			log.Printf("Mutation Manager returning sequence vector for index %v %v", idx, v)
		}
		replyMap[idx] = v
	}
	*reply = replyMap
	return nil

}

//This method takes as input an api.Mutation and copies into mutation queue for processing
func (m *MutationManager) ProcessSingleMutation(mutation *api.Mutation, reply *bool) error {
	if options.debugLog {
		log.Printf("Received Mutation Type %s Indexid %v, Docid %v, Vbucket %v, Seqno %v", mutation.Type, mutation.Indexid, mutation.Docid, mutation.Vbucket, mutation.Seqno)
	}

	//if there is any pending error, reply with that. This will force a handshake again.
	if indexerErrorState == true {
		*reply = false
	}

	//copy the mutation data and return
	m.chmutation <- mutation
	*reply = true
	return nil

}

//---End of exported RPC methods

//read incoming mutation and distribute it on worker queues based on vbucketid
func (m *MutationManager) manageMutationQueue() {

	for {
		select {
		case mut, ok := <-m.chmutation:
			if ok {
				m.chworkers[mut.Vbucket%MAX_MUTATION_WORKERS] <- mut
			}
		case <-chdrain:
			wg.Add(1)
			m.drainMutationChannel(m.chmutation)
		}
	}

}

//start a mutation worker which handles mutation on the specified workerId channel
func (m *MutationManager) startMutationWorker(workerId int) {

	for {
		select {
		case mutation := <-m.chworkers[workerId]:
			m.handleMutation(mutation)
		case <-chdrain:
			wg.Add(1)
			m.drainMutationChannel(m.chworkers[workerId])
		}
	}
}

func (m *MutationManager) handleMutation(mutation *api.Mutation) {

	if mutation.Type == api.INSERT {

		var key api.Key
		var value api.Value
		var err error

		if key, err = api.NewKey(mutation.SecondaryKey, mutation.Docid); err != nil {
			log.Printf("Error Generating Key From Mutation %v. Skipped.", err)
			return
		}

		if value, err = api.NewValue(mutation.SecondaryKey, mutation.Docid, mutation.Vbucket, mutation.Seqno); err != nil {
			log.Printf("Error Generating Value From Mutation %v. Skipped.", err)
			return
		}

		if engine, ok := m.enginemap[mutation.Indexid]; ok {
			if err := engine.InsertMutation(key, value); err != nil {
				log.Printf("Error from Engine during InsertMutation. Key %v. Index %v. Error %v", key, mutation.Docid, err)
			}
			//send notification for this seqno to be recorded in SeqVector
			seqnotify := seqNotification{engine: engine,
				indexid: mutation.Indexid,
				seqno:   mutation.Seqno,
				vbucket: mutation.Vbucket,
			}
			m.chseq <- seqnotify
		} else {
			err := fmt.Sprintf("Unknown Index %v or Engine not found", mutation.Indexid)
			m.initErrorState(err)
		}

	} else if mutation.Type == api.DELETE {

		if engine, ok := m.enginemap[mutation.Indexid]; ok {
			if err := engine.DeleteMutation(mutation.Docid); err != nil {
				log.Printf("Error from Engine during Delete Mutation. Key %v. Error %v", mutation.Docid, err)
				return
			}
			//send notification for this seqno to be recorded in SeqVector
			seqnotify := seqNotification{engine: engine,
				indexid: mutation.Indexid,
				seqno:   mutation.Seqno,
				vbucket: mutation.Vbucket,
			}
			m.chseq <- seqnotify
		} else {
			err := fmt.Sprintf("Unknown Index %v or Engine not found", mutation.Indexid)
			m.initErrorState(err)
		}
	}
}

func StartMutationManager(engineMap map[string]api.Finder) (chan ddlNotification, error) {

	var err error

	//init the mutation manager maps
	mutationMgr.sequencemap = make(api.IndexSequenceMap)
	//copy the inital map from the indexer
	mutationMgr.enginemap = engineMap
	mutationMgr.initSequenceMapFromPersistence()

	//create channel to receive notification for new sequence numbers
	//and start a goroutine to manage it
	mutationMgr.chseq = make(chan seqNotification, MAX_SEQUENCE_QUEUE)
	go mutationMgr.manageSeqNotification()

	//create a channel to receive notification from indexer
	//and start a goroutine to listen to it
	mutationMgr.chddl = make(chan ddlNotification)
	go mutationMgr.manageIndexerNotification()

	//init the channel for incoming mutations
	mutationMgr.chmutation = make(chan *api.Mutation, MAX_INCOMING_QUEUE)
	go mutationMgr.manageMutationQueue()

	//init the workers for processing mutations
	for w := 0; w < MAX_MUTATION_WORKERS; w++ {
		mutationMgr.chworkers[w] = make(chan *api.Mutation, MAX_WORKER_QUEUE)
		go mutationMgr.startMutationWorker(w)
	}

	//init error state
	indexerErrorState = false
	indexerErrorString = ""
	chdrain = make(chan bool)

	//start the rpc server
	if err = startRPCServer(); err != nil {
		return nil, err
	}

	return mutationMgr.chddl, nil
}

func startRPCServer() error {

	log.Println("Starting Mutation Manager")
	server := rpc.NewServer()
	server.Register(&mutationMgr)

	server.HandleHTTP(rpc.DefaultRPCPath, rpc.DefaultDebugPath)

	l, err := net.Listen("tcp", ":8096")
	if err != nil {
		return err
	}

	go func() {
		for {
			conn, err := l.Accept()
			if err != nil {
				log.Printf("Error in Accept %v. Shutting down")
				//FIXME Add a cleanup function
				return
			}
			go server.ServeCodec(jsonrpc.NewServerCodec(conn))
		}
	}()
	return nil

}

func (m *MutationManager) manageIndexerNotification() {

	ok := true
	var ddl ddlNotification
	for ok {
		ddl, ok = <-m.chddl
		if ok {
			switch ddl.ddltype {
			case api.CREATE:
				m.enginemap[ddl.indexinfo.Uuid] = ddl.engine
				//init sequence map of new index
				seqVec := make(api.SequenceVector, api.MAX_VBUCKETS)
				m.sequencemap[ddl.indexinfo.Uuid] = seqVec
			case api.DROP:
				delete(m.enginemap, ddl.indexinfo.Uuid)
				//FIXME : Delete index entry from sequence map
			default:
				log.Printf("Mutation Manager Received Unsupported Notification %v", ddl.ddltype)
			}
		}
	}
}

func (m *MutationManager) manageSeqNotification() {

	var seq seqNotification
	openSeqCount := 0
	ok := true
	var perfWriteCount int64

	for ok {
		select {
		case seq, ok = <-m.chseq:
			if ok {
				seqVector, exists := m.sequencemap[seq.indexid]
				if !exists {
					log.Printf("IndexId %v not found in Sequence Vector. INCONSISTENT INDEXER STATE!!!", seq.indexid)
					break
				}
				seqVector[seq.vbucket] = seq.seqno
				m.sequencemap[seq.indexid] = seqVector
				openSeqCount += 1
				perfWriteCount += 1
				//persist only after SEQ_MAP_PERSIST_INTERVAL
				if openSeqCount == SEQ_MAP_PERSIST_INTERVAL {
					m.persistSequenceMap()
					openSeqCount = 0
				}
				if perfWriteCount%10000 == 0 {
					log.Printf("Processed Mutation %v", perfWriteCount)
				}
			}
		case <-chdrain:
			wg.Add(1)
			m.drainSeqChannel(m.chseq)
		}
	}
}

func (m *MutationManager) initSequenceMapFromPersistence() {

	sequenceVector := make(api.SequenceVector, api.MAX_VBUCKETS)
	for idx, engine := range m.enginemap {
		metaval, err := engine.GetMeta(META_DOC_ID)
		if err != nil {
			log.Printf("Error retreiving Meta from Engine %v", err)
		}
		err = json.Unmarshal([]byte(metaval), &sequenceVector)
		if err != nil {
			log.Printf("Error unmarshalling SequenceVector %v", err)
		}
		m.sequencemap[idx] = sequenceVector
	}
}

func (m *MutationManager) persistSequenceMap() {

	for idx, seqm := range m.sequencemap {
		jsonval, err := json.Marshal(seqm)
		if err != nil {
			log.Printf("Error Marshalling SequenceMap %v", err)
		} else {
			//FIXME - Handle Error here
			m.enginemap[idx].InsertMeta(META_DOC_ID, string(jsonval))
		}
	}
}

func (m *MutationManager) initErrorState(err string) {

	indexerErrorState = true
	indexerErrorString = err

	//send drain signal to mutation queue
	chdrain <- true

	//send drain signal to sequence queue
	chdrain <- true

	//send drain signal to worker queues
	for w := 0; w < MAX_MUTATION_WORKERS; w++ {
		chdrain <- true
	}

}

func (m *MutationManager) drainMutationChannel(ch chan *api.Mutation) {

loop:
	for {
		select {
		case <-ch:
			continue
		default:
			break loop
		}
	}
	wg.Done()

}

func (m *MutationManager) drainSeqChannel(ch chan seqNotification) {

loop:
	for {
		select {
		case <-ch:
			continue
		default:
			break loop
		}
	}
	wg.Done()

}
<|endoftext|>


<commit_before>package kafkamdm

import (
	"time"

	"github.com/Shopify/sarama"
	"github.com/raintank/met"
	"github.com/raintank/raintank-metric/fake_metrics/out"
	"github.com/raintank/raintank-metric/schema"
)

type KafkaMdm struct {
	out.OutStats
	topic   string
	brokers []string
	config  *sarama.Config
	client  sarama.SyncProducer
}

func New(topic string, brokers []string, stats met.Backend) (*KafkaMdm, error) {
	// We are looking for strong consistency semantics.
	// Because we don't change the flush settings, sarama will try to produce messages
	// as fast as possible to keep latency low.
	config := sarama.NewConfig()
	config.Producer.RequiredAcks = sarama.WaitForAll // Wait for all in-sync replicas to ack the message
	config.Producer.Retry.Max = 10                   // Retry up to 10 times to produce the message
	err := config.Validate()
	if err != nil {
		return nil, err
	}

	client, err := sarama.NewSyncProducer(brokers, config)
	if err != nil {
		return nil, err
	}

	return &KafkaMdm{
		OutStats: out.NewStats(stats, "kafka-mdm"),
		topic:    topic,
		brokers:  brokers,
		config:   config,
		client:   client,
	}, nil
}

func (k *KafkaMdm) Close() error {
	return k.client.Close()
}

func (k *KafkaMdm) Flush(metrics []*schema.MetricData) error {
	preFlush := time.Now()
	if len(metrics) == 0 {
		k.FlushDuration.Value(time.Since(preFlush))
		return nil
	}

	k.MessageMetrics.Value(1)
	var data []byte

	for _, metric := range metrics {
		data, err := metric.MarshalMsg(data[:])
		if err != nil {
			return err
		}

		k.MessageBytes.Value(int64(len(data)))

		prePub := time.Now()

		// We are not setting a message key, which means that all messages will
		// be distributed randomly over the different partitions.
		_, _, err = k.client.SendMessage(&sarama.ProducerMessage{
			Topic: k.topic,
			Value: sarama.ByteEncoder(data),
		})
		if err != nil {
			k.PublishErrors.Inc(1)
			return err
		}

		k.PublishedMessages.Inc(1)
		k.PublishDuration.Value(time.Since(prePub))
	}
	k.PublishedMetrics.Inc(int64(len(metrics)))
	k.FlushDuration.Value(time.Since(preFlush))
	return nil
}
<commit_msg>use sendMessages to send metrics in batches to kafka.<commit_after>package kafkamdm

import (
	"time"

	"github.com/Shopify/sarama"
	"github.com/raintank/met"
	"github.com/raintank/raintank-metric/fake_metrics/out"
	"github.com/raintank/raintank-metric/schema"
)

type KafkaMdm struct {
	out.OutStats
	topic   string
	brokers []string
	config  *sarama.Config
	client  sarama.SyncProducer
}

func New(topic string, brokers []string, stats met.Backend) (*KafkaMdm, error) {
	// We are looking for strong consistency semantics.
	// Because we don't change the flush settings, sarama will try to produce messages
	// as fast as possible to keep latency low.
	config := sarama.NewConfig()
	config.Producer.RequiredAcks = sarama.WaitForAll // Wait for all in-sync replicas to ack the message
	config.Producer.Retry.Max = 10                   // Retry up to 10 times to produce the message
	err := config.Validate()
	if err != nil {
		return nil, err
	}

	client, err := sarama.NewSyncProducer(brokers, config)
	if err != nil {
		return nil, err
	}

	return &KafkaMdm{
		OutStats: out.NewStats(stats, "kafka-mdm"),
		topic:    topic,
		brokers:  brokers,
		config:   config,
		client:   client,
	}, nil
}

func (k *KafkaMdm) Close() error {
	return k.client.Close()
}

func (k *KafkaMdm) Flush(metrics []*schema.MetricData) error {
	preFlush := time.Now()
	if len(metrics) == 0 {
		k.FlushDuration.Value(time.Since(preFlush))
		return nil
	}

	k.MessageMetrics.Value(1)
	var data []byte

	payload := make([]*sarama.ProducerMessage, len(metrics))

	for i, metric := range metrics {
		data, err := metric.MarshalMsg(data[:])
		if err != nil {
			return err
		}

		k.MessageBytes.Value(int64(len(data)))
		// We are not setting a message key, which means that all messages will
		// be distributed randomly over the different partitions.
		payload[i] = &sarama.ProducerMessage{
			Topic: k.topic,
			Value: sarama.ByteEncoder(data),
		}

	}
	prePub := time.Now()
	err := k.client.SendMessages(payload)
	if err != nil {
		k.PublishErrors.Inc(1)
		return err
	}

	k.PublishedMessages.Inc(int64(len(metrics)))
	k.PublishDuration.Value(time.Since(prePub))
	k.PublishedMetrics.Inc(int64(len(metrics)))
	k.FlushDuration.Value(time.Since(preFlush))
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2020 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Concurrently read objects on GCS provided by stdin. The user must ensure
//    (1) all the objects come from the same bucket, and
//    (2) the script is authorized to read from the bucket.
// The stdin should contain N lines of object name, in the form of
// "gs://bucket-name/object-name".
//
// This benchmark only tests the internal reader implementation, which
// doesn't have FUSE involved.
//
// Usage Example:
// 	 gsutil ls 'gs://bucket/prefix*' | go run \
//      --conns_per_host=10 --reader=vendor ./benchmark/concurrent_read
//

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"strings"
	"time"
)

var fIterations = flag.Int(
	"iterations",
	1,
	"Number of iterations to read the files.",
)
var fHTTP = flag.String(
	"http",
	"1.1",
	"HTTP protocol version, 1.1 or 2.",
)
var fConnsPerHost = flag.Int(
	"conns_per_host",
	10,
	"Max number of TCP connections per host.",
)
var fReader = flag.String(
	"reader",
	"vendor",
	"Reader type: vendor, official.",
)

const (
	KB = 1024
	MB = 1024 * KB
)

func testReader(rf readerFactory, objectNames []string) (stats testStats) {
	reportDuration := 10 * time.Second
	ticker := time.NewTicker(reportDuration)
	defer ticker.Stop()

	doneBytes := make(chan int64)
	doneFiles := make(chan int)
	start := time.Now()

	// run readers concurrently
	for _, objectName := range objectNames {
		name := objectName
		go func() {
			reader := rf.NewReader(name)
			defer reader.Close()
			p := make([]byte, 128*1024)
			for {
				n, err := reader.Read(p)
				doneBytes <- int64(n)
				if err == io.EOF {
					break
				} else if err != nil {
					panic(fmt.Errorf("read %v fails: %v", name, err))
				}
			}
			doneFiles <- 1
			return
		}()
	}

	// collect test stats
	var lastTotalBytes int64
	for stats.totalFiles < len(objectNames) {
		select {
		case b := <-doneBytes:
			stats.totalBytes += b
		case f := <-doneFiles:
			stats.totalFiles += f
		case <-ticker.C:
			readBytes := stats.totalBytes - lastTotalBytes
			lastTotalBytes = stats.totalBytes
			mbps := float32(readBytes/MB) / float32(reportDuration/time.Second)
			stats.mbps = append(stats.mbps, mbps)
		}
	}
	stats.duration = time.Since(start)
	return
}

func run(bucketName string, objectNames []string) {
	protocols := map[string]string{
		"1.1": http1,
		"2":   http2,
	}
	httpVersion := protocols[*fHTTP]
	transport := getTransport(httpVersion, *fConnsPerHost)
	defer transport.CloseIdleConnections()

	readers := map[string]string{
		"vendor":   vendorClientReader,
		"official": officialClientReader,
	}
	readerVersion := readers[*fReader]
	rf := newReaderFactory(transport, readerVersion, bucketName)

	for i := 0; i < *fIterations; i++ {
		stats := testReader(rf, objectNames)
		stats.report(httpVersion, readerVersion)
	}
}

type testStats struct {
	totalBytes int64
	totalFiles int
	mbps       []float32
	duration   time.Duration
}

func (s testStats) throughput() float32 {
	mbs := float32(s.totalBytes) / float32(MB)
	seconds := float32(s.duration) / float32(time.Second)
	return mbs / seconds
}

func (s testStats) report(
	httpVersion string,
	readerVersion string,
) {
	fmt.Printf(
		"# TEST READER %s\n"+
			"Protocol: %s\n"+
			"Total bytes: %d\n"+
			"Total files: %d\n"+
			"Avg Throughput: %.1f MB/s\n\n",
		readerVersion,
		httpVersion,
		s.totalBytes,
		s.totalFiles,
		s.throughput(),
	)
}

func getLinesFromStdin() (lines []string) {
	reader := bufio.NewReader(os.Stdin)
	for {
		line, err := reader.ReadString('\n')
		if err != nil {
			if err == io.EOF {
				err = nil
				break
			}
			panic(fmt.Errorf("Stdin error: %v", err))
		}
		lines = append(lines, line)
	}
	return
}

func getObjectNames() (bucketName string, objectNames []string) {
	uris := getLinesFromStdin()
	for _, uri := range uris {
		path := strings.TrimLeft(uri, "gs://")
		path = strings.TrimRight(path, "\n")
		segs := strings.Split(path, "/")
		if len(segs) <= 1 {
			panic(fmt.Errorf("Not a file name: %v", uri))
		}

		if bucketName == "" {
			bucketName = segs[0]
		} else if bucketName != segs[0] {
			panic(fmt.Errorf("Multiple buckets: %v, %v", bucketName, segs[0]))
		}

		objectName := strings.Join(segs[1:], "/")
		objectNames = append(objectNames, objectName)
	}
	return
}

func main() {
	flag.Parse()
	bucketName, objectNames := getObjectNames()
	run(bucketName, objectNames)
	return
}
<commit_msg>Add connections per host to the printed log<commit_after>// Copyright 2020 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Concurrently read objects on GCS provided by stdin. The user must ensure
//    (1) all the objects come from the same bucket, and
//    (2) the script is authorized to read from the bucket.
// The stdin should contain N lines of object name, in the form of
// "gs://bucket-name/object-name".
//
// This benchmark only tests the internal reader implementation, which
// doesn't have FUSE involved.
//
// Usage Example:
// 	 gsutil ls 'gs://bucket/prefix*' | go run \
//      --conns_per_host=10 --reader=vendor ./benchmark/concurrent_read
//

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"strings"
	"time"
)

var fIterations = flag.Int(
	"iterations",
	1,
	"Number of iterations to read the files.",
)
var fHTTP = flag.String(
	"http",
	"1.1",
	"HTTP protocol version, 1.1 or 2.",
)
var fConnsPerHost = flag.Int(
	"conns_per_host",
	10,
	"Max number of TCP connections per host.",
)
var fReader = flag.String(
	"reader",
	"vendor",
	"Reader type: vendor, official.",
)

const (
	KB = 1024
	MB = 1024 * KB
)

func testReader(rf readerFactory, objectNames []string) (stats testStats) {
	reportDuration := 10 * time.Second
	ticker := time.NewTicker(reportDuration)
	defer ticker.Stop()

	doneBytes := make(chan int64)
	doneFiles := make(chan int)
	start := time.Now()

	// run readers concurrently
	for _, objectName := range objectNames {
		name := objectName
		go func() {
			reader := rf.NewReader(name)
			defer reader.Close()
			p := make([]byte, 128*1024)
			for {
				n, err := reader.Read(p)
				doneBytes <- int64(n)
				if err == io.EOF {
					break
				} else if err != nil {
					panic(fmt.Errorf("read %v fails: %v", name, err))
				}
			}
			doneFiles <- 1
			return
		}()
	}

	// collect test stats
	var lastTotalBytes int64
	for stats.totalFiles < len(objectNames) {
		select {
		case b := <-doneBytes:
			stats.totalBytes += b
		case f := <-doneFiles:
			stats.totalFiles += f
		case <-ticker.C:
			readBytes := stats.totalBytes - lastTotalBytes
			lastTotalBytes = stats.totalBytes
			mbps := float32(readBytes/MB) / float32(reportDuration/time.Second)
			stats.mbps = append(stats.mbps, mbps)
		}
	}
	stats.duration = time.Since(start)
	return
}

func run(bucketName string, objectNames []string) {
	protocols := map[string]string{
		"1.1": http1,
		"2":   http2,
	}
	httpVersion := protocols[*fHTTP]
	transport := getTransport(httpVersion, *fConnsPerHost)
	defer transport.CloseIdleConnections()

	readers := map[string]string{
		"vendor":   vendorClientReader,
		"official": officialClientReader,
	}
	readerVersion := readers[*fReader]
	rf := newReaderFactory(transport, readerVersion, bucketName)

	for i := 0; i < *fIterations; i++ {
		stats := testReader(rf, objectNames)
		stats.report(httpVersion, *fConnsPerHost, readerVersion)
	}
}

type testStats struct {
	totalBytes int64
	totalFiles int
	mbps       []float32
	duration   time.Duration
}

func (s testStats) throughput() float32 {
	mbs := float32(s.totalBytes) / float32(MB)
	seconds := float32(s.duration) / float32(time.Second)
	return mbs / seconds
}

func (s testStats) report(
	httpVersion string,
	maxConnsPerHost int,
	readerVersion string,
) {
	fmt.Printf(
		"# TEST READER %s\n"+
			"Protocol: %s (%v connections per host)\n"+
			"Total bytes: %d\n"+
			"Total files: %d\n"+
			"Avg Throughput: %.1f MB/s\n\n",
		readerVersion,
		httpVersion,
		maxConnsPerHost,
		s.totalBytes,
		s.totalFiles,
		s.throughput(),
	)
}

func getLinesFromStdin() (lines []string) {
	reader := bufio.NewReader(os.Stdin)
	for {
		line, err := reader.ReadString('\n')
		if err != nil {
			if err == io.EOF {
				err = nil
				break
			}
			panic(fmt.Errorf("Stdin error: %v", err))
		}
		lines = append(lines, line)
	}
	return
}

func getObjectNames() (bucketName string, objectNames []string) {
	uris := getLinesFromStdin()
	for _, uri := range uris {
		path := strings.TrimLeft(uri, "gs://")
		path = strings.TrimRight(path, "\n")
		segs := strings.Split(path, "/")
		if len(segs) <= 1 {
			panic(fmt.Errorf("Not a file name: %v", uri))
		}

		if bucketName == "" {
			bucketName = segs[0]
		} else if bucketName != segs[0] {
			panic(fmt.Errorf("Multiple buckets: %v, %v", bucketName, segs[0]))
		}

		objectName := strings.Join(segs[1:], "/")
		objectNames = append(objectNames, objectName)
	}
	return
}

func main() {
	flag.Parse()
	bucketName, objectNames := getObjectNames()
	run(bucketName, objectNames)
	return
}
<|endoftext|>


<commit_before>/* {{{ Copyright (c) Paul R. Tagliamonte <paultag@debian.org>, 2015
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE. }}} */

package reprepro

import (
	"os/exec"
)

type Repo struct {
	Basedir string
}

func (repo *Repo) Command(args ...string) *exec.Cmd {
	return exec.Command("reprepro", append([]string{
		"--basedir", repo.Basedir,
	}, args...)...)
}

func (repo *Repo) ProcessIncoming(rule string) error {
	cmd := repo.Command("processincoming", rule)
	return cmd.Run()
}

func (repo *Repo) Check() error {
	cmd := repo.Command("check")
	return cmd.Run()
}

func (repo *Repo) CheckPool() error {
	cmd := repo.Command("checkpool")
	return cmd.Run()
}

func (repo *Repo) Include(suite string, changes string) error {
	cmd := repo.Command("include", suite, changes)
	return cmd.Run()
}

// Create a new reprepro.Repo object given a filesystem path to the Repo.
func NewRepo(path string) *Repo {
	return &Repo{Basedir: path}
}

// vim: foldmethod=marker
<commit_msg>Cleanup errors, add args<commit_after>/* {{{ Copyright (c) Paul R. Tagliamonte <paultag@debian.org>, 2015
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE. }}} */

package reprepro

import (
	"bytes"
	"fmt"
	"os/exec"
	"strings"
)

type Repo struct {
	Basedir   string
	Arguments []string
}

func (repo *Repo) Command(args ...string) *exec.Cmd {
	return exec.Command("reprepro", append(repo.Arguments, append([]string{
		"--basedir", repo.Basedir,
	}, args...)...)...)
}

func proxyRun(cmd *exec.Cmd) error {
	buf := bytes.Buffer{}
	cmd.Stderr = &buf
	err := cmd.Run()
	out := strings.SplitN(buf.String(), "\n", 2)
	if err != nil {
		if len(out) == 0 {
			return err
		}
		return fmt.Errorf("%s (underlying error: %s)", out[0], err)
	}
	return nil
}

func (repo *Repo) ProcessIncoming(rule string) error {
	cmd := repo.Command("processincoming", rule)
	return proxyRun(cmd)
}

func (repo *Repo) Check() error {
	cmd := repo.Command("check")
	return proxyRun(cmd)
}

func (repo *Repo) Export() error {
	cmd := repo.Command("export")
	return proxyRun(cmd)
}

func (repo *Repo) CheckPool() error {
	cmd := repo.Command("checkpool")
	return proxyRun(cmd)
}

func (repo *Repo) Include(suite string, changes string) error {
	cmd := repo.Command("include", suite, changes)
	return proxyRun(cmd)
}

// Create a new reprepro.Repo object given a filesystem path to the Repo.
func NewRepo(path string, args ...string) *Repo {
	return &Repo{
		Basedir:   path,
		Arguments: args,
	}
}

// vim: foldmethod=marker
<|endoftext|>