<commit_before>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package container

import (
    "bytes"
	"fmt"
	"github.com/control-center/serviced/domain/service"
	"github.com/control-center/serviced/domain/servicedefinition"
	"github.com/zenoss/glog"
)

const (
	logstashContainerConfig = "/etc/filebeat.conf"
)

//createFields makes the map of tags for the logstash config including the type
func createFields(hostID string, service *service.Service, instanceID string, logConfig *servicedefinition.LogConfig) map[string]string {
	fields := make(map[string]string)
	fields["type"] = logConfig.Type
	fields["service"] = service.ID
	fields["instance"] = instanceID

	// CC-2234: Note that logstash is hardcoded to inject a field named 'host' into to every message, but when run from within
	// a docker container, the value is actually the container id, not the name of the docker host. So this tag is
	// named a little differently to distinguish it from the tag named 'host'
	fields["ccWorkerID"] = hostID

	for _, tag := range logConfig.LogTags {
		fields[tag.Name] = tag.Value
	}
	return fields
}

//formatTagsForConfFile takes the set of tags for a LogConfig and return json representing the tags
func formatTagsForConfFile(tags map[string]string) string {
	if len(tags) == 0 {
		return ""
	}
    var buffer bytes.Buffer
    buffer.WriteString("{")
    for k, v := range tags {
        buffer.WriteString(k + ": " + v + ", ")
    }
    buffer.WriteString("}")
	return buffer.String()
}

// writeLogstashAgentConfig creates the logstash forwarder config file
func writeLogstashAgentConfig(confPath string, hostID string, service *service.Service, instanceID string, resourcePath string) error {
	glog.Infof("Using logstash resourcePath: %s", resourcePath)

	// generate the json config.
	// TODO: Grab the structs from logstash-forwarder and marshal this instead of generating it
	logstashForwarderLogConf :=
`    -
      paths:
        - %s
      fields: %s`

	logstashForwarderLogConf = fmt.Sprintf(logstashForwarderLogConf, service.LogConfigs[0].Path, formatTagsForConfFile(createFields(hostID, service, instanceID, &service.LogConfigs[0])))
	for _, logConfig := range service.LogConfigs[1:] {
		logstashForwarderLogConf = logstashForwarderLogConf + `
    -
      paths:
        - %s
      fields: %s`

		logstashForwarderLogConf = fmt.Sprintf(logstashForwarderLogConf, logConfig.Path, formatTagsForConfFile(createFields(hostID, service, instanceID, &logConfig)))
	}

	logstashForwarderShipperConf :=
`filebeat:
  idle_timeout: 5s
  prospectors:
%s
output:
  logstash:
    enabled: true
    hosts:
      - %s
    tls:
      insecure: true
      certificate: %s
      certificate_key: %s
      certificate_authorities:
        - %s
      timeout: 15
logging:
  to_syslog: false`

	logstashForwarderShipperConf = fmt.Sprintf(logstashForwarderShipperConf,
        logstashForwarderLogConf,
//		"172.17.42.1:5043",
		"127.0.0.1:5043",
		resourcePath+"/filebeat.crt",
		resourcePath+"/filebeat.key",
		resourcePath+"/filebeat.crt",
		)

	config := servicedefinition.ConfigFile{
		Filename: confPath,
		Content:  logstashForwarderShipperConf,
	}
	err := writeConfFile(config)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>Fix unsupported time unit in filebeat config<commit_after>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package container

import (
    "bytes"
	"fmt"
	"github.com/control-center/serviced/domain/service"
	"github.com/control-center/serviced/domain/servicedefinition"
	"github.com/zenoss/glog"
)

const (
	logstashContainerConfig = "/etc/filebeat.conf"
)

//createFields makes the map of tags for the logstash config including the type
func createFields(hostID string, service *service.Service, instanceID string, logConfig *servicedefinition.LogConfig) map[string]string {
	fields := make(map[string]string)
	fields["type"] = logConfig.Type
	fields["service"] = service.ID
	fields["instance"] = instanceID

	// CC-2234: Note that logstash is hardcoded to inject a field named 'host' into to every message, but when run from within
	// a docker container, the value is actually the container id, not the name of the docker host. So this tag is
	// named a little differently to distinguish it from the tag named 'host'
	fields["ccWorkerID"] = hostID

	for _, tag := range logConfig.LogTags {
		fields[tag.Name] = tag.Value
	}
	return fields
}

//formatTagsForConfFile takes the set of tags for a LogConfig and return json representing the tags
func formatTagsForConfFile(tags map[string]string) string {
	if len(tags) == 0 {
		return ""
	}
    var buffer bytes.Buffer
    buffer.WriteString("{")
    for k, v := range tags {
        buffer.WriteString(k + ": " + v + ", ")
    }
    buffer.WriteString("}")
	return buffer.String()
}

// writeLogstashAgentConfig creates the logstash forwarder config file
func writeLogstashAgentConfig(confPath string, hostID string, service *service.Service, instanceID string, resourcePath string) error {
	glog.Infof("Using logstash resourcePath: %s", resourcePath)

	// generate the json config.
	// TODO: Grab the structs from logstash-forwarder and marshal this instead of generating it
	logstashForwarderLogConf :=
`    -
      ignore_older: 26280h
      paths:
        - %s
      fields: %s`

	logstashForwarderLogConf = fmt.Sprintf(logstashForwarderLogConf, service.LogConfigs[0].Path, formatTagsForConfFile(createFields(hostID, service, instanceID, &service.LogConfigs[0])))
	for _, logConfig := range service.LogConfigs[1:] {
		logstashForwarderLogConf = logstashForwarderLogConf + `
    -
      ignore_older: 3y
      paths:
        - %s
      fields: %s`

		logstashForwarderLogConf = fmt.Sprintf(logstashForwarderLogConf, logConfig.Path, formatTagsForConfFile(createFields(hostID, service, instanceID, &logConfig)))
	}

	logstashForwarderShipperConf :=
`filebeat:
  idle_timeout: 5s
  prospectors:
%s
output:
  logstash:
    enabled: true
    hosts:
      - %s
    tls:
      insecure: true
      certificate: %s
      certificate_key: %s
      certificate_authorities:
        - %s
      timeout: 15
logging:
  to_syslog: false`

	logstashForwarderShipperConf = fmt.Sprintf(logstashForwarderShipperConf,
        logstashForwarderLogConf,
//		"172.17.42.1:5043",
		"127.0.0.1:5043",
		resourcePath+"/filebeat.crt",
		resourcePath+"/filebeat.key",
		resourcePath+"/filebeat.crt",
		)

	config := servicedefinition.ConfigFile{
		Filename: confPath,
		Content:  logstashForwarderShipperConf,
	}
	err := writeConfFile(config)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2018 Anapaya Systems
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"math/rand"
	"os"
	"path/filepath"
	"time"

	"github.com/BurntSushi/toml"

	"github.com/scionproto/scion/go/lib/addr"
	"github.com/scionproto/scion/go/lib/common"
	"github.com/scionproto/scion/go/lib/env"
	"github.com/scionproto/scion/go/lib/infra"
	"github.com/scionproto/scion/go/lib/infra/infraenv"
	"github.com/scionproto/scion/go/lib/infra/modules/cleaner"
	"github.com/scionproto/scion/go/lib/infra/modules/itopo"
	"github.com/scionproto/scion/go/lib/infra/modules/trust"
	"github.com/scionproto/scion/go/lib/infra/modules/trust/trustdb"
	"github.com/scionproto/scion/go/lib/log"
	"github.com/scionproto/scion/go/lib/pathstorage"
	"github.com/scionproto/scion/go/lib/periodic"
	"github.com/scionproto/scion/go/path_srv/internal/handlers"
	"github.com/scionproto/scion/go/path_srv/internal/psconfig"
	"github.com/scionproto/scion/go/path_srv/internal/segsyncer"
	"github.com/scionproto/scion/go/proto"
)

type Config struct {
	General env.General
	Logging env.Logging
	Metrics env.Metrics
	Trust   env.Trust
	Infra   env.Infra
	PS      psconfig.Config
}

var (
	config      Config
	environment *env.Env
)

func init() {
	flag.Usage = env.Usage
}

// main initializes the path server and starts the dispatcher.
func main() {
	os.Exit(realMain())
}

func realMain() int {
	env.AddFlags()
	flag.Parse()
	if v, ok := env.CheckFlags(psconfig.Sample); !ok {
		return v
	}
	if err := setupBasic(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return 1
	}
	defer env.CleanupLog()
	if err := setup(); err != nil {
		log.Crit("Setup failed", "err", err)
		return 1
	}
	pathDB, revCache, err := pathstorage.NewPathStorage(config.PS.PathDB, config.PS.RevCache)
	if err != nil {
		log.Crit("Unable to initialize path storage", "err", err)
		return 1
	}
	trustDB, err := trustdb.New(config.Trust.TrustDB)
	if err != nil {
		log.Crit("Unable to initialize trustDB", "err", err)
		return 1
	}
	topo := itopo.GetCurrentTopology()
	trustConf := &trust.Config{
		ServiceType: proto.ServiceType_ps,
	}
	trustStore, err := trust.NewStore(trustDB, topo.ISD_AS,
		rand.Uint64(), trustConf, log.Root())
	if err != nil {
		log.Crit("Unable to initialize trust store", "err", err)
		return 1
	}
	err = trustStore.LoadAuthoritativeTRC(filepath.Join(config.General.ConfigDir, "certs"))
	if err != nil {
		log.Crit("TRC error", "err", err)
		return 1
	}
	topoAddress := topo.PS.GetById(config.General.ID)
	if topoAddress == nil {
		log.Crit("Unable to find topo address")
		return 1
	}
	msger, err := infraenv.InitMessenger(
		topo.ISD_AS,
		env.GetPublicSnetAddress(topo.ISD_AS, topoAddress),
		env.GetBindSnetAddress(topo.ISD_AS, topoAddress),
		addr.SvcPS,
		config.General.ReconnectToDispatcher,
		trustStore,
	)
	if err != nil {
		log.Crit(infraenv.ErrAppUnableToInitMessenger, "err", err)
		return 1
	}
	msger.AddHandler(infra.ChainRequest, trustStore.NewChainReqHandler(false))
	// TOOD(lukedirtwalker): with the new CP-PKI design the PS should no longer need to handle TRC
	// and cert requests.
	msger.AddHandler(infra.TRCRequest, trustStore.NewTRCReqHandler(false))
	args := handlers.HandlerArgs{
		PathDB:     pathDB,
		RevCache:   revCache,
		TrustStore: trustStore,
		Config:     config.PS,
		IA:         topo.ISD_AS,
	}
	core := topo.Core
	var segReqHandler infra.Handler
	deduper := handlers.NewGetSegsDeduper(msger)
	if core {
		segReqHandler = handlers.NewSegReqCoreHandler(args, deduper)
	} else {
		segReqHandler = handlers.NewSegReqNonCoreHandler(args, deduper)
	}
	msger.AddHandler(infra.SegRequest, segReqHandler)
	msger.AddHandler(infra.SegReg, handlers.NewSegRegHandler(args))
	msger.AddHandler(infra.IfStateInfos, handlers.NewIfStatInfoHandler(args))
	if config.PS.SegSync && core {
		// Old down segment sync mechanism
		msger.AddHandler(infra.SegSync, handlers.NewSyncHandler(args))
		segSyncers, err := segsyncer.StartAll(args, msger)
		if err != nil {
			log.Crit("Unable to start seg syncer", "err", err)
			return 1
		}
		for _, segsync := range segSyncers {
			defer segsync.Stop()
		}
	}
	msger.AddHandler(infra.SegRev, handlers.NewRevocHandler(args))
	// Create a channel where prometheus can signal fatal errors
	fatalC := make(chan error, 1)
	config.Metrics.StartPrometheus(fatalC)
	// Start handling requests/messages
	go func() {
		defer log.LogPanicAndExit()
		msger.ListenAndServe()
	}()
	cleaner := periodic.StartPeriodicTask(cleaner.New(pathDB),
		time.NewTicker(300*time.Second), 295*time.Second)
	defer cleaner.Stop()
	select {
	case <-environment.AppShutdownSignal:
		// Whenever we receive a SIGINT or SIGTERM we exit without an error.
		return 0
	case err := <-fatalC:
		// Prometheus encountered a fatal error, thus we exit.
		log.Crit("Unable to listen and serve", "err", err)
		return 1
	}
}

func setupBasic() error {
	if _, err := toml.DecodeFile(env.ConfigFile(), &config); err != nil {
		return err
	}
	if err := env.InitLogging(&config.Logging); err != nil {
		return err
	}
	env.LogSvcStarted(common.PS, config.General.ID)
	return nil
}

func setup() error {
	if err := env.InitGeneral(&config.General); err != nil {
		return err
	}
	itopo.SetCurrentTopology(config.General.Topology)
	environment = infraenv.InitInfraEnvironment(config.General.TopologyPath)
	config.PS.InitDefaults()
	return nil
}
<commit_msg>PS: Import net/http/pprof (#2086)<commit_after>// Copyright 2018 Anapaya Systems
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"math/rand"
	_ "net/http/pprof"
	"os"
	"path/filepath"
	"time"

	"github.com/BurntSushi/toml"

	"github.com/scionproto/scion/go/lib/addr"
	"github.com/scionproto/scion/go/lib/common"
	"github.com/scionproto/scion/go/lib/env"
	"github.com/scionproto/scion/go/lib/infra"
	"github.com/scionproto/scion/go/lib/infra/infraenv"
	"github.com/scionproto/scion/go/lib/infra/modules/cleaner"
	"github.com/scionproto/scion/go/lib/infra/modules/itopo"
	"github.com/scionproto/scion/go/lib/infra/modules/trust"
	"github.com/scionproto/scion/go/lib/infra/modules/trust/trustdb"
	"github.com/scionproto/scion/go/lib/log"
	"github.com/scionproto/scion/go/lib/pathstorage"
	"github.com/scionproto/scion/go/lib/periodic"
	"github.com/scionproto/scion/go/path_srv/internal/handlers"
	"github.com/scionproto/scion/go/path_srv/internal/psconfig"
	"github.com/scionproto/scion/go/path_srv/internal/segsyncer"
	"github.com/scionproto/scion/go/proto"
)

type Config struct {
	General env.General
	Logging env.Logging
	Metrics env.Metrics
	Trust   env.Trust
	Infra   env.Infra
	PS      psconfig.Config
}

var (
	config      Config
	environment *env.Env
)

func init() {
	flag.Usage = env.Usage
}

// main initializes the path server and starts the dispatcher.
func main() {
	os.Exit(realMain())
}

func realMain() int {
	env.AddFlags()
	flag.Parse()
	if v, ok := env.CheckFlags(psconfig.Sample); !ok {
		return v
	}
	if err := setupBasic(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return 1
	}
	defer env.CleanupLog()
	if err := setup(); err != nil {
		log.Crit("Setup failed", "err", err)
		return 1
	}
	pathDB, revCache, err := pathstorage.NewPathStorage(config.PS.PathDB, config.PS.RevCache)
	if err != nil {
		log.Crit("Unable to initialize path storage", "err", err)
		return 1
	}
	trustDB, err := trustdb.New(config.Trust.TrustDB)
	if err != nil {
		log.Crit("Unable to initialize trustDB", "err", err)
		return 1
	}
	topo := itopo.GetCurrentTopology()
	trustConf := &trust.Config{
		ServiceType: proto.ServiceType_ps,
	}
	trustStore, err := trust.NewStore(trustDB, topo.ISD_AS,
		rand.Uint64(), trustConf, log.Root())
	if err != nil {
		log.Crit("Unable to initialize trust store", "err", err)
		return 1
	}
	err = trustStore.LoadAuthoritativeTRC(filepath.Join(config.General.ConfigDir, "certs"))
	if err != nil {
		log.Crit("TRC error", "err", err)
		return 1
	}
	topoAddress := topo.PS.GetById(config.General.ID)
	if topoAddress == nil {
		log.Crit("Unable to find topo address")
		return 1
	}
	msger, err := infraenv.InitMessenger(
		topo.ISD_AS,
		env.GetPublicSnetAddress(topo.ISD_AS, topoAddress),
		env.GetBindSnetAddress(topo.ISD_AS, topoAddress),
		addr.SvcPS,
		config.General.ReconnectToDispatcher,
		trustStore,
	)
	if err != nil {
		log.Crit(infraenv.ErrAppUnableToInitMessenger, "err", err)
		return 1
	}
	msger.AddHandler(infra.ChainRequest, trustStore.NewChainReqHandler(false))
	// TOOD(lukedirtwalker): with the new CP-PKI design the PS should no longer need to handle TRC
	// and cert requests.
	msger.AddHandler(infra.TRCRequest, trustStore.NewTRCReqHandler(false))
	args := handlers.HandlerArgs{
		PathDB:     pathDB,
		RevCache:   revCache,
		TrustStore: trustStore,
		Config:     config.PS,
		IA:         topo.ISD_AS,
	}
	core := topo.Core
	var segReqHandler infra.Handler
	deduper := handlers.NewGetSegsDeduper(msger)
	if core {
		segReqHandler = handlers.NewSegReqCoreHandler(args, deduper)
	} else {
		segReqHandler = handlers.NewSegReqNonCoreHandler(args, deduper)
	}
	msger.AddHandler(infra.SegRequest, segReqHandler)
	msger.AddHandler(infra.SegReg, handlers.NewSegRegHandler(args))
	msger.AddHandler(infra.IfStateInfos, handlers.NewIfStatInfoHandler(args))
	if config.PS.SegSync && core {
		// Old down segment sync mechanism
		msger.AddHandler(infra.SegSync, handlers.NewSyncHandler(args))
		segSyncers, err := segsyncer.StartAll(args, msger)
		if err != nil {
			log.Crit("Unable to start seg syncer", "err", err)
			return 1
		}
		for _, segsync := range segSyncers {
			defer segsync.Stop()
		}
	}
	msger.AddHandler(infra.SegRev, handlers.NewRevocHandler(args))
	// Create a channel where prometheus can signal fatal errors
	fatalC := make(chan error, 1)
	config.Metrics.StartPrometheus(fatalC)
	// Start handling requests/messages
	go func() {
		defer log.LogPanicAndExit()
		msger.ListenAndServe()
	}()
	cleaner := periodic.StartPeriodicTask(cleaner.New(pathDB),
		time.NewTicker(300*time.Second), 295*time.Second)
	defer cleaner.Stop()
	select {
	case <-environment.AppShutdownSignal:
		// Whenever we receive a SIGINT or SIGTERM we exit without an error.
		return 0
	case err := <-fatalC:
		// Prometheus encountered a fatal error, thus we exit.
		log.Crit("Unable to listen and serve", "err", err)
		return 1
	}
}

func setupBasic() error {
	if _, err := toml.DecodeFile(env.ConfigFile(), &config); err != nil {
		return err
	}
	if err := env.InitLogging(&config.Logging); err != nil {
		return err
	}
	env.LogSvcStarted(common.PS, config.General.ID)
	return nil
}

func setup() error {
	if err := env.InitGeneral(&config.General); err != nil {
		return err
	}
	itopo.SetCurrentTopology(config.General.Topology)
	environment = infraenv.InitInfraEnvironment(config.General.TopologyPath)
	config.PS.InitDefaults()
	return nil
}
<|endoftext|>


<commit_before>package controllers

import (
	"github.com/gin-gonic/gin"
	"net/http"

	"github.com/eirka/eirka-libs/config"
	e "github.com/eirka/eirka-libs/errors"
	"github.com/eirka/eirka-libs/redis"
	"github.com/eirka/eirka-libs/user"

	"github.com/eirka/eirka-post/models"
	u "github.com/eirka/eirka-post/utils"
)

// AvatarController handles updating a users avatar
func AvatarController(c *gin.Context) {
	var err error
	req := c.Request

	// get userdata from session middleware
	userdata := c.MustGet("userdata").(user.User)

	// check size of content
	if req.ContentLength > int64(config.Settings.Limits.ImageMaxSize) {
		c.JSON(http.StatusExpectationFailed, gin.H{"error_message": e.ErrImageSize.Error()})
		c.Error(e.ErrImageSize).SetMeta("AvatarController.ContentLength")
		return
	}

	// set max bytes reader
	req.Body = http.MaxBytesReader(c.Writer, req.Body, int64(config.Settings.Limits.ImageMaxSize))

	image := u.ImageType{}

	// Check if theres a file
	image.File, image.Header, err = req.FormFile("file")
	if err == http.ErrMissingFile {
		c.JSON(http.StatusBadRequest, gin.H{"error_message": e.ErrNoImage.Error()})
		c.Error(e.ErrNoImage).SetMeta("ThreadController.FormFile")
		return
	}

	// set the user id as the ib ;D
	image.Ib = userdata.Id

	// Save the image to a file
	err = image.SaveAvatar()
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
		c.Error(err).SetMeta("AvatarController.SaveAvatar")
		return
	}

	c.Redirect(303, req.Referer())

	return

}
<commit_msg>add avatar controller<commit_after>package controllers

import (
	"github.com/gin-gonic/gin"
	"net/http"

	"github.com/eirka/eirka-libs/config"
	e "github.com/eirka/eirka-libs/errors"
	"github.com/eirka/eirka-libs/user"

	u "github.com/eirka/eirka-post/utils"
)

// AvatarController handles updating a users avatar
func AvatarController(c *gin.Context) {
	var err error
	req := c.Request

	// get userdata from session middleware
	userdata := c.MustGet("userdata").(user.User)

	// check size of content
	if req.ContentLength > int64(config.Settings.Limits.ImageMaxSize) {
		c.JSON(http.StatusExpectationFailed, gin.H{"error_message": e.ErrImageSize.Error()})
		c.Error(e.ErrImageSize).SetMeta("AvatarController.ContentLength")
		return
	}

	// set max bytes reader
	req.Body = http.MaxBytesReader(c.Writer, req.Body, int64(config.Settings.Limits.ImageMaxSize))

	image := u.ImageType{}

	// Check if theres a file
	image.File, image.Header, err = req.FormFile("file")
	if err == http.ErrMissingFile {
		c.JSON(http.StatusBadRequest, gin.H{"error_message": e.ErrNoImage.Error()})
		c.Error(e.ErrNoImage).SetMeta("ThreadController.FormFile")
		return
	}

	// set the user id as the ib ;D
	image.Ib = userdata.Id

	// Save the image to a file
	err = image.SaveAvatar()
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
		c.Error(err).SetMeta("AvatarController.SaveAvatar")
		return
	}

	c.Redirect(303, req.Referer())

	return

}
<|endoftext|>


<commit_before>package controllers

import (
	"github.com/gorilla/mux"
	"github.com/prest/prest/config"
	"github.com/prest/prest/config/router"
	"github.com/prest/prest/middlewares"
	"github.com/urfave/negroni"
)

// MakeHandler reagister all routes
func Routes() *negroni.Negroni {
	n := middlewares.GetApp()
	r := router.Get()
	// if auth is enabled
	if config.PrestConf.AuthEnabled {
		r.HandleFunc("/auth", Auth).Methods("POST")
	}
	r.HandleFunc("/databases", GetDatabases).Methods("GET")
	r.HandleFunc("/schemas", GetSchemas).Methods("GET")
	r.HandleFunc("/tables", GetTables).Methods("GET")
	r.HandleFunc("/_QUERIES/{queriesLocation}/{script}", ExecuteFromScripts)
	r.HandleFunc("/{database}/{schema}", GetTablesByDatabaseAndSchema).Methods("GET")
	r.HandleFunc("/show/{database}/{schema}/{table}", ShowTable).Methods("GET")
	crudRoutes := mux.NewRouter().PathPrefix("/").Subrouter().StrictSlash(true)
	crudRoutes.HandleFunc("/{database}/{schema}/{table}", SelectFromTables).Methods("GET")
	crudRoutes.HandleFunc("/{database}/{schema}/{table}", InsertInTables).Methods("POST")
	crudRoutes.HandleFunc("/batch/{database}/{schema}/{table}", BatchInsertInTables).Methods("POST")
	crudRoutes.HandleFunc("/{database}/{schema}/{table}", DeleteFromTable).Methods("DELETE")
	crudRoutes.HandleFunc("/{database}/{schema}/{table}", UpdateTable).Methods("PUT", "PATCH")
	r.PathPrefix("/").Handler(negroni.New(
		middlewares.AccessControl(),
		middlewares.AuthMiddleware(),
		negroni.Wrap(crudRoutes),
	))
	n.UseHandler(r)
	return n
}
<commit_msg>fixed typo function description<commit_after>package controllers

import (
	"github.com/gorilla/mux"
	"github.com/prest/prest/config"
	"github.com/prest/prest/config/router"
	"github.com/prest/prest/middlewares"
	"github.com/urfave/negroni"
)

// Routes reagister all routes
func Routes() *negroni.Negroni {
	n := middlewares.GetApp()
	r := router.Get()
	// if auth is enabled
	if config.PrestConf.AuthEnabled {
		r.HandleFunc("/auth", Auth).Methods("POST")
	}
	r.HandleFunc("/databases", GetDatabases).Methods("GET")
	r.HandleFunc("/schemas", GetSchemas).Methods("GET")
	r.HandleFunc("/tables", GetTables).Methods("GET")
	r.HandleFunc("/_QUERIES/{queriesLocation}/{script}", ExecuteFromScripts)
	r.HandleFunc("/{database}/{schema}", GetTablesByDatabaseAndSchema).Methods("GET")
	r.HandleFunc("/show/{database}/{schema}/{table}", ShowTable).Methods("GET")
	crudRoutes := mux.NewRouter().PathPrefix("/").Subrouter().StrictSlash(true)
	crudRoutes.HandleFunc("/{database}/{schema}/{table}", SelectFromTables).Methods("GET")
	crudRoutes.HandleFunc("/{database}/{schema}/{table}", InsertInTables).Methods("POST")
	crudRoutes.HandleFunc("/batch/{database}/{schema}/{table}", BatchInsertInTables).Methods("POST")
	crudRoutes.HandleFunc("/{database}/{schema}/{table}", DeleteFromTable).Methods("DELETE")
	crudRoutes.HandleFunc("/{database}/{schema}/{table}", UpdateTable).Methods("PUT", "PATCH")
	r.PathPrefix("/").Handler(negroni.New(
		middlewares.AccessControl(),
		middlewares.AuthMiddleware(),
		negroni.Wrap(crudRoutes),
	))
	n.UseHandler(r)
	return n
}
<|endoftext|>


<commit_before>package controllers

import (
	"crypto/md5"
	"fmt"
	"html/template"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"

	"github.com/nicomo/abacaxi/logger"
	"github.com/nicomo/abacaxi/models"
	"github.com/nicomo/abacaxi/views"
)

// UploadHandler manages upload of source file, checks extension
// then passes the file on to the appropriate controller
func UploadHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" { // you just arrived here, I'll give you a token
		crutime := time.Now().Unix()
		h := md5.New()
		io.WriteString(h, strconv.FormatInt(crutime, 10))
		token := fmt.Sprintf("%x", h.Sum(nil))

		t, _ := template.ParseFiles("upload.gtpl")
		t.Execute(w, token)
	} else {

		// our messages (errors, confirmation, etc) to the user & the template will be store in this map
		//FIXME: either userMessages struct of d[string] interface{} as used in other funcs, but not both...
		userM := make(userMessages)

		// parsing multipart file
		r.ParseMultipartForm(32 << 20)

		// get the ebook package name
		tsname := r.PostFormValue("pack")
		userM["myPackage"] = tsname
		file, handler, err := r.FormFile("uploadfile")
		if err != nil {
			logger.Error.Println(err)
			return
		}
		defer file.Close()

		// create new file with same name
		// FIXME: returns an error if dir /data does not exist, should use os.MkdirAll
		// see models/download.go:16
		path := "./data/" + handler.Filename
		f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0666)
		if err != nil {
			logger.Error.Println(err)
			return
		}
		defer f.Close()

		// copy uploaded file into new file
		io.Copy(f, file)

		// if xml pass on to xmlio, if csv, pass on to csvio, if neither, abort
		ext := filepath.Ext(handler.Filename)
		fmt.Println(ext)
		if ext == ".csv" {
			// pass on the name of the package and the name of the file to csvio package

			csvRecords, myTS, userM, err := csvIO(path, tsname, userM)
			if err != nil {
				logger.Error.Println(err)
			}
			createdCounter, updatedCounter, createUpdateErr := models.EbooksCreateOrUpdate(csvRecords)
			if createUpdateErr != nil {
				logger.Error.Println("EbooksCreateOrUpdate error: ", createUpdateErr)
			}

			userM["createdCounter"] = strconv.Itoa(createdCounter)
			userM["updatedCounter"] = strconv.Itoa(updatedCounter)

			// update the target service with last update date
			tsUpdateErr := models.TSUpdate(myTS)
			if tsUpdateErr != nil {
				logger.Error.Printf("couldn't update Target Service %v. Error: %v", myTS, tsUpdateErr)
			}

			// list of TS appearing in menu
			TSListing, _ := models.GetTargetServicesListing()
			userM["TSListing"] = TSListing

			views.RenderTmpl(w, "upload", userM)

		} else if ext == ".xml" {

			xmlRecords, myTS, userM, err := xmlIO(path, tsname, userM)
			if err != nil {
				logger.Error.Println(err)
			}
			createdCounter, updatedCounter, createUpdateErr := models.EbooksCreateOrUpdate(xmlRecords)
			if createUpdateErr != nil {
				logger.Error.Println("EbooksCreateOrUpdate error: ", createUpdateErr)
			}
			userM["createdCounter"] = strconv.Itoa(createdCounter)
			userM["updatedCounter"] = strconv.Itoa(updatedCounter)

			tsUpdateErr := models.TSUpdate(myTS)
			if tsUpdateErr != nil {
				logger.Error.Printf("couldn't update Target Service %v. Error: %v", myTS, tsUpdateErr)
			}

			// list of TS appearing in menu
			TSListing, _ := models.GetTargetServicesListing()
			userM["TSListing"] = TSListing

			views.RenderTmpl(w, "upload", userM)

		} else {

			// TODO : transmit either error or success message to user
			// manage case wrong file extension : message to the user with redirect to home
			logger.Error.Println("wrong file extension")
			// redirect to home
			http.Redirect(w, r, "/", 303)
		}

	}
}
<commit_msg>bugfix : create dir /data/ if it doesn’t exist<commit_after>package controllers

import (
	"crypto/md5"
	"fmt"
	"html/template"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"

	"github.com/nicomo/abacaxi/logger"
	"github.com/nicomo/abacaxi/models"
	"github.com/nicomo/abacaxi/views"
)

// UploadHandler manages upload of source file, checks extension
// then passes the file on to the appropriate controller
func UploadHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" { // you just arrived here, I'll give you a token
		crutime := time.Now().Unix()
		h := md5.New()
		io.WriteString(h, strconv.FormatInt(crutime, 10))
		token := fmt.Sprintf("%x", h.Sum(nil))

		t, _ := template.ParseFiles("upload.gtpl")
		t.Execute(w, token)
	} else {

		// our messages (errors, confirmation, etc) to the user & the template will be store in this map
		//FIXME: either userMessages struct of d[string] interface{} as used in other funcs, but not both...
		userM := make(userMessages)

		// parsing multipart file
		r.ParseMultipartForm(32 << 20)

		// get the ebook package name
		tsname := r.PostFormValue("pack")
		userM["myPackage"] = tsname
		file, handler, err := r.FormFile("uploadfile")
		if err != nil {
			logger.Error.Println(err)
			return
		}
		defer file.Close()

		// create new file with same name
		// FIXME: returns an error if dir /data does not exist, should use os.MkdirAll
		// see models/download.go:16
		// create dir if it doesn't exist
		path := "data"
		pathErr := os.MkdirAll("data", os.ModePerm)
		if pathErr != nil {
			logger.Error.Println(pathErr)
		}

		fpath := path + handler.Filename
		f, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE, 0666)
		if err != nil {
			logger.Error.Println(err)
			return
		}
		defer f.Close()

		// copy uploaded file into new file
		io.Copy(f, file)

		// if xml pass on to xmlio, if csv, pass on to csvio, if neither, abort
		ext := filepath.Ext(handler.Filename)
		fmt.Println(ext)
		if ext == ".csv" {
			// pass on the name of the package and the name of the file to csvio package

			csvRecords, myTS, userM, err := csvIO(path, tsname, userM)
			if err != nil {
				logger.Error.Println(err)
			}
			createdCounter, updatedCounter, createUpdateErr := models.EbooksCreateOrUpdate(csvRecords)
			if createUpdateErr != nil {
				logger.Error.Println("EbooksCreateOrUpdate error: ", createUpdateErr)
			}

			userM["createdCounter"] = strconv.Itoa(createdCounter)
			userM["updatedCounter"] = strconv.Itoa(updatedCounter)

			// update the target service with last update date
			tsUpdateErr := models.TSUpdate(myTS)
			if tsUpdateErr != nil {
				logger.Error.Printf("couldn't update Target Service %v. Error: %v", myTS, tsUpdateErr)
			}

			// list of TS appearing in menu
			TSListing, _ := models.GetTargetServicesListing()
			userM["TSListing"] = TSListing

			views.RenderTmpl(w, "upload", userM)

		} else if ext == ".xml" {

			xmlRecords, myTS, userM, err := xmlIO(path, tsname, userM)
			if err != nil {
				logger.Error.Println(err)
			}
			createdCounter, updatedCounter, createUpdateErr := models.EbooksCreateOrUpdate(xmlRecords)
			if createUpdateErr != nil {
				logger.Error.Println("EbooksCreateOrUpdate error: ", createUpdateErr)
			}
			userM["createdCounter"] = strconv.Itoa(createdCounter)
			userM["updatedCounter"] = strconv.Itoa(updatedCounter)

			tsUpdateErr := models.TSUpdate(myTS)
			if tsUpdateErr != nil {
				logger.Error.Printf("couldn't update Target Service %v. Error: %v", myTS, tsUpdateErr)
			}

			// list of TS appearing in menu
			TSListing, _ := models.GetTargetServicesListing()
			userM["TSListing"] = TSListing

			views.RenderTmpl(w, "upload", userM)

		} else {

			// TODO : transmit either error or success message to user
			// manage case wrong file extension : message to the user with redirect to home
			logger.Error.Println("wrong file extension")
			// redirect to home
			http.Redirect(w, r, "/", 303)
		}

	}
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/subutai-io/base/agent/agent"
	"github.com/subutai-io/base/agent/cli"
	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/log"

	"github.com/codegangsta/cli"
)

var TIMESTAMP string = "-unknown"

func init() {
	if os.Getuid() != 0 {
		log.Error("Please run as root")
	}
	os.Setenv("PATH", "/apps/subutai/current/bin:"+os.Getenv("PATH"))
	if len(os.Args) > 1 {
		if os.Args[1] == "-d" {
			log.Level(log.DebugLevel)
		}
	}
}

func main() {
	app := cli.NewApp()
	app.Name = "Subutai"
	app.Version = "4.0.0-RC9-SNAPSHOT" + TIMESTAMP
	app.Usage = "daemon and command line interface binary"

	app.Flags = []cli.Flag{cli.BoolFlag{
		Name:  "d",
		Usage: "debug mode"}}

	app.Commands = []cli.Command{{
		Name: "attach", Usage: "attach to container",
		Flags: []cli.Flag{
			cli.BoolFlag{Name: "c", Usage: "clear environment"},
			cli.BoolFlag{Name: "x", Usage: "use x86 personality"},
			cli.BoolFlag{Name: "r", Usage: "connect as regular user"}},
		Action: func(c *cli.Context) {
			lib.LxcAttach(c.Args().Get(0), c.Bool("c"), c.Bool("x"), c.Bool("r"))
		}}, {

		Name: "backup", Usage: "backup Subutai container",
		Flags: []cli.Flag{
			cli.BoolFlag{Name: "full", Usage: "make full backup"},
			cli.BoolFlag{Name: "stop", Usage: "stop container at the time of backup"}},
		Action: func(c *cli.Context) {
			lib.BackupContainer(c.Args().Get(0), c.Bool("full"), c.Bool("stop"))
		}}, {

		Name: "batch", Usage: "batch commands execution",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "json", Usage: "JSON string with commands"}},
		Action: func(c *cli.Context) {
			lib.Batch(c.String("json"))
		}}, {

		Name: "clone", Usage: "clone Subutai container",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "e", Usage: "set environment id for container"},
			cli.StringFlag{Name: "i", Usage: "set container IP address and VLAN"},
			cli.StringFlag{Name: "t", Usage: "token to verify with MH"}},
		Action: func(c *cli.Context) {
			lib.LxcClone(c.Args().Get(0), c.Args().Get(1), c.String("e"), c.String("i"), c.String("t"))
		}}, {

		Name: "cleanup", Usage: "clean Subutai environment",
		Action: func(c *cli.Context) {
			lib.Cleanup(c.Args().Get(0))
		}}, {

		Name: "config", Usage: "containerName add/del key value",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "o", Usage: "add/del key value"},
			cli.StringFlag{Name: "k", Usage: "add/del key value"},
			cli.StringFlag{Name: "v", Usage: "add/del key value"},
		},
		Action: func(c *cli.Context) {
			lib.LxcConfig(c.Args().Get(0), c.String("o"), c.String("k"), c.String("v"))
		}}, {

		Name: "daemon", Usage: "start an agent",
		Action: func(c *cli.Context) {
			config.InitAgentDebug()
			agent.Start(c)
		}}, {

		Name: "demote", Usage: "demote Subutai container",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "i", Usage: "network value ie 192.168.1.1/24"},
			cli.StringFlag{Name: "v", Usage: "vlan id"},
		},
		Action: func(c *cli.Context) {
			lib.LxcDemote(c.Args().Get(0), c.String("i"), c.String("v"))
		}}, {

		Name: "destroy", Usage: "destroy Subutai container",
		Action: func(c *cli.Context) {
			lib.LxcDestroy(c.Args().Get(0))
		}}, {

		Name: "export", Usage: "export Subutai container",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "v", Usage: "template version"}},
		Action: func(c *cli.Context) {
			lib.LxcExport(c.Args().Get(0), c.String("v"))
		}}, {

		Name: "import", Usage: "import Subutai template",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "v", Usage: "template version"},
			cli.StringFlag{Name: "t", Usage: "token to access kurjun repo"}},
		Action: func(c *cli.Context) {
			lib.LxcImport(c.Args().Get(0), c.String("v"), c.String("t"))
		}}, {

		Name: "list", Usage: "list Subutai container",
		Flags: []cli.Flag{
			cli.BoolFlag{Name: "c", Usage: "containers only"},
			cli.BoolFlag{Name: "t", Usage: "templates only"},
			cli.BoolFlag{Name: "i", Usage: "detailed container info"},
			cli.BoolFlag{Name: "a", Usage: "with ancestors"},
			cli.BoolFlag{Name: "p", Usage: "with parent"}},
		Action: func(c *cli.Context) {
			lib.LxcList(c.Args().Get(0), c.Bool("c"), c.Bool("t"), c.Bool("i"), c.Bool("a"), c.Bool("p"))
		}}, {

		Name: "management_network", Usage: "configure management network",
		Flags: []cli.Flag{
			cli.BoolFlag{Name: "listtunnel, l", Usage: "-l"},
			cli.StringFlag{Name: "createtunnel, c", Usage: "-c TUNNELPORTNAME TUNNELIPADDRESS TUNNELTYPE"},

			cli.BoolFlag{Name: "listvnimap, v", Usage: "-v"},
			cli.StringFlag{Name: "createvnimap, m", Usage: "-m TUNNELPORTNAME VNI VLANID ENV_ID"},
		},

		Subcommands: []cli.Command{{
			Name:  "tunnel",
			Usage: "tunnels operation",
			Flags: []cli.Flag{
				cli.StringFlag{Name: "create", Usage: "create tunnel (tunnel -c)"},
				cli.StringFlag{Name: "delete", Usage: "delete tunnel (tunnel -d)"},
				cli.BoolFlag{Name: "list", Usage: "list of tunnels (tunnel -l)"},

				cli.StringFlag{Name: "remoteip", Usage: "remote ip"},
				cli.StringFlag{Name: "vlan", Usage: "tunnel vlan"},
				cli.StringFlag{Name: "vni", Usage: "vni"},
			},
			Action: func(c *cli.Context) {
				lib.VxlanTunnel(c.String("create"), c.String("delete"), c.String("remoteip"), c.String("vlan"), c.String("vni"), c.Bool("list"))
			}}, {

			Name:  "p2p",
			Usage: "p2p network operation",
			Flags: []cli.Flag{
				cli.BoolFlag{Name: "c", Usage: "create p2p instance (p2p -c interfaceName hash key ttl localPeepIPAddr)"},
				cli.BoolFlag{Name: "d", Usage: "delete p2p instance (p2p -d hash)"},
				cli.BoolFlag{Name: "u", Usage: "update p2p instance encryption key (p2p -u hash newkey ttl)"},
				cli.BoolFlag{Name: "l", Usage: "list of p2p instances (p2p -l)"},
				cli.BoolFlag{Name: "p", Usage: "list of p2p participants (p2p -p hash)"}},
			Action: func(c *cli.Context) {
				lib.P2P(c.Bool("c"), c.Bool("d"), c.Bool("u"), c.Bool("l"), c.Bool("p"), os.Args)
			}}},

		Action: func(c *cli.Context) {
			lib.LxcManagementNetwork(os.Args)
		}}, {

		Name: "metrics", Usage: "list Subutai container",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "s", Usage: "start time"},
			cli.StringFlag{Name: "e", Usage: "end time"}},
		Action: func(c *cli.Context) {
			lib.HostMetrics(c.Args().Get(0), c.String("s"), c.String("e"))
		}}, {

		Name: "network", Usage: "containerName set/remove/list network vlan id",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "set, s", Usage: "IPADDRESS/NETMASK"},
			cli.StringFlag{Name: "vlan, v", Usage: "vlanid"},
			cli.BoolFlag{Name: "remove, r", Usage: ""},
			cli.BoolFlag{Name: "list, l", Usage: ""},
		},
		Action: func(c *cli.Context) {
			lib.LxcNetwork(c.Args().Get(0), c.String("s"), c.String("vlan"), c.Bool("r"), c.Bool("l"))
		}}, {

		Name: "promote", Usage: "promote Subutai container",
		Action: func(c *cli.Context) {
			lib.LxcPromote(c.Args().Get(0))
		}}, {

		Name: "proxy", Usage: "Subutai reverse proxy",
		Subcommands: []cli.Command{
			{
				Name:     "add",
				Usage:    "add reverse proxy component",
				HideHelp: true,
				Flags: []cli.Flag{
					cli.StringFlag{Name: "domain,d", Usage: "add domain to vlan"},
					cli.StringFlag{Name: "host, h", Usage: "add host to domain on vlan"},
					cli.StringFlag{Name: "policy, p", Usage: "set load balance policy (rr|lb|hash)"},
					cli.StringFlag{Name: "file, f", Usage: "specify pem certificate file"}},
				Action: func(c *cli.Context) {
					lib.ProxyAdd(c.Args().Get(0), c.String("d"), c.String("h"), c.String("p"), c.String("f"))
				},
			},
			{
				Name:     "del",
				Usage:    "del reverse proxy component",
				HideHelp: true,
				Flags: []cli.Flag{
					cli.BoolFlag{Name: "domain, d", Usage: "delete domain from vlan"},
					cli.StringFlag{Name: "host, h", Usage: "delete host from domain on vlan"}},
				Action: func(c *cli.Context) {
					lib.ProxyDel(c.Args().Get(0), c.String("h"), c.Bool("d"))
				},
			},
			{
				Name:     "check",
				Usage:    "check existing domain or host",
				HideHelp: true,
				Flags: []cli.Flag{
					cli.BoolFlag{Name: "domain, d", Usage: "check domains on vlan"},
					cli.StringFlag{Name: "host, h", Usage: "check hosts on vlan"}},
				Action: func(c *cli.Context) {
					lib.ProxyCheck(c.Args().Get(0), c.String("h"), c.Bool("d"))
				},
			},
		}}, {

		Name: "quota", Usage: "set quotas for Subutai container",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "s", Usage: "set quota for the specified resource type"},
			cli.StringFlag{Name: "t", Usage: "set alert threshold"}},
		Action: func(c *cli.Context) {
			lib.LxcQuota(c.Args().Get(0), c.Args().Get(1), c.String("s"), c.String("t"))
		}}, {

		Name: "register", Usage: "register Subutai container",
		Action: func(c *cli.Context) {
			lib.LxcRegister(c.Args().Get(0))
		}}, {

		Name: "rename", Usage: "rename Subutai container",
		Action: func(c *cli.Context) {
			lib.LxcRename(c.Args().Get(0), c.Args().Get(1))
		}}, {

		Name: "restore", Usage: "restore Subutai container",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "d", Usage: "date of backup snapshot"},
			cli.StringFlag{Name: "c", Usage: "name of new container"}},
		Action: func(c *cli.Context) {
			lib.RestoreContainer(c.Args().Get(0), c.Args().Get(1), c.Args().Get(2))
		}}, {

		Name: "stats", Usage: "statistics from host",
		Action: func(c *cli.Context) {
			lib.Stats(c.Args().Get(0), c.Args().Get(1), c.Args().Get(2))
		}}, {

		Name: "start", Usage: "start Subutai container",
		Action: func(c *cli.Context) {
			lib.LxcStart(c.Args().Get(0))
		}}, {

		Name: "stop", Usage: "stop Subutai container",
		Action: func(c *cli.Context) {
			lib.LxcStop(c.Args().Get(0))
		}}, {

		Name: "tunnel", Usage: "create SSH tunnel",
		Flags: []cli.Flag{
			cli.BoolFlag{Name: "g", Usage: "global accessible tunnel"}},
		Action: func(c *cli.Context) {
			lib.Tunnel(c.Args().Get(0), c.Args().Get(1), c.Args().Get(2), c.Bool("g"))
		}}, {

		Name: "unregister", Usage: "unregister Subutai container",
		Action: func(c *cli.Context) {
			lib.LxcUnregister(c.Args().Get(0))
		}}, {

		Name: "update", Usage: "update Subutai management, container or Resource host",
		Flags: []cli.Flag{
			cli.BoolFlag{Name: "c", Usage: "check for updates without installation"}},
		Action: func(c *cli.Context) {
			lib.Update(c.Args().Get(0), c.Bool("c"))
		}},
	}

	app.Run(os.Args)
}
<commit_msg>Changed version to 4.0.0-RC9.<commit_after>package main

import (
	"os"

	"github.com/subutai-io/base/agent/agent"
	"github.com/subutai-io/base/agent/cli"
	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/log"

	"github.com/codegangsta/cli"
)

var TIMESTAMP string = "-unknown"

func init() {
	if os.Getuid() != 0 {
		log.Error("Please run as root")
	}
	os.Setenv("PATH", "/apps/subutai/current/bin:"+os.Getenv("PATH"))
	if len(os.Args) > 1 {
		if os.Args[1] == "-d" {
			log.Level(log.DebugLevel)
		}
	}
}

func main() {
	app := cli.NewApp()
	app.Name = "Subutai"
	app.Version = "4.0.0-RC9"
	app.Usage = "daemon and command line interface binary"

	app.Flags = []cli.Flag{cli.BoolFlag{
		Name:  "d",
		Usage: "debug mode"}}

	app.Commands = []cli.Command{{
		Name: "attach", Usage: "attach to container",
		Flags: []cli.Flag{
			cli.BoolFlag{Name: "c", Usage: "clear environment"},
			cli.BoolFlag{Name: "x", Usage: "use x86 personality"},
			cli.BoolFlag{Name: "r", Usage: "connect as regular user"}},
		Action: func(c *cli.Context) {
			lib.LxcAttach(c.Args().Get(0), c.Bool("c"), c.Bool("x"), c.Bool("r"))
		}}, {

		Name: "backup", Usage: "backup Subutai container",
		Flags: []cli.Flag{
			cli.BoolFlag{Name: "full", Usage: "make full backup"},
			cli.BoolFlag{Name: "stop", Usage: "stop container at the time of backup"}},
		Action: func(c *cli.Context) {
			lib.BackupContainer(c.Args().Get(0), c.Bool("full"), c.Bool("stop"))
		}}, {

		Name: "batch", Usage: "batch commands execution",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "json", Usage: "JSON string with commands"}},
		Action: func(c *cli.Context) {
			lib.Batch(c.String("json"))
		}}, {

		Name: "clone", Usage: "clone Subutai container",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "e", Usage: "set environment id for container"},
			cli.StringFlag{Name: "i", Usage: "set container IP address and VLAN"},
			cli.StringFlag{Name: "t", Usage: "token to verify with MH"}},
		Action: func(c *cli.Context) {
			lib.LxcClone(c.Args().Get(0), c.Args().Get(1), c.String("e"), c.String("i"), c.String("t"))
		}}, {

		Name: "cleanup", Usage: "clean Subutai environment",
		Action: func(c *cli.Context) {
			lib.Cleanup(c.Args().Get(0))
		}}, {

		Name: "config", Usage: "containerName add/del key value",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "o", Usage: "add/del key value"},
			cli.StringFlag{Name: "k", Usage: "add/del key value"},
			cli.StringFlag{Name: "v", Usage: "add/del key value"},
		},
		Action: func(c *cli.Context) {
			lib.LxcConfig(c.Args().Get(0), c.String("o"), c.String("k"), c.String("v"))
		}}, {

		Name: "daemon", Usage: "start an agent",
		Action: func(c *cli.Context) {
			config.InitAgentDebug()
			agent.Start(c)
		}}, {

		Name: "demote", Usage: "demote Subutai container",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "i", Usage: "network value ie 192.168.1.1/24"},
			cli.StringFlag{Name: "v", Usage: "vlan id"},
		},
		Action: func(c *cli.Context) {
			lib.LxcDemote(c.Args().Get(0), c.String("i"), c.String("v"))
		}}, {

		Name: "destroy", Usage: "destroy Subutai container",
		Action: func(c *cli.Context) {
			lib.LxcDestroy(c.Args().Get(0))
		}}, {

		Name: "export", Usage: "export Subutai container",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "v", Usage: "template version"}},
		Action: func(c *cli.Context) {
			lib.LxcExport(c.Args().Get(0), c.String("v"))
		}}, {

		Name: "import", Usage: "import Subutai template",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "v", Usage: "template version"},
			cli.StringFlag{Name: "t", Usage: "token to access kurjun repo"}},
		Action: func(c *cli.Context) {
			lib.LxcImport(c.Args().Get(0), c.String("v"), c.String("t"))
		}}, {

		Name: "list", Usage: "list Subutai container",
		Flags: []cli.Flag{
			cli.BoolFlag{Name: "c", Usage: "containers only"},
			cli.BoolFlag{Name: "t", Usage: "templates only"},
			cli.BoolFlag{Name: "i", Usage: "detailed container info"},
			cli.BoolFlag{Name: "a", Usage: "with ancestors"},
			cli.BoolFlag{Name: "p", Usage: "with parent"}},
		Action: func(c *cli.Context) {
			lib.LxcList(c.Args().Get(0), c.Bool("c"), c.Bool("t"), c.Bool("i"), c.Bool("a"), c.Bool("p"))
		}}, {

		Name: "management_network", Usage: "configure management network",
		Flags: []cli.Flag{
			cli.BoolFlag{Name: "listtunnel, l", Usage: "-l"},
			cli.StringFlag{Name: "createtunnel, c", Usage: "-c TUNNELPORTNAME TUNNELIPADDRESS TUNNELTYPE"},

			cli.BoolFlag{Name: "listvnimap, v", Usage: "-v"},
			cli.StringFlag{Name: "createvnimap, m", Usage: "-m TUNNELPORTNAME VNI VLANID ENV_ID"},
		},

		Subcommands: []cli.Command{{
			Name:  "tunnel",
			Usage: "tunnels operation",
			Flags: []cli.Flag{
				cli.StringFlag{Name: "create", Usage: "create tunnel (tunnel -c)"},
				cli.StringFlag{Name: "delete", Usage: "delete tunnel (tunnel -d)"},
				cli.BoolFlag{Name: "list", Usage: "list of tunnels (tunnel -l)"},

				cli.StringFlag{Name: "remoteip", Usage: "remote ip"},
				cli.StringFlag{Name: "vlan", Usage: "tunnel vlan"},
				cli.StringFlag{Name: "vni", Usage: "vni"},
			},
			Action: func(c *cli.Context) {
				lib.VxlanTunnel(c.String("create"), c.String("delete"), c.String("remoteip"), c.String("vlan"), c.String("vni"), c.Bool("list"))
			}}, {

			Name:  "p2p",
			Usage: "p2p network operation",
			Flags: []cli.Flag{
				cli.BoolFlag{Name: "c", Usage: "create p2p instance (p2p -c interfaceName hash key ttl localPeepIPAddr)"},
				cli.BoolFlag{Name: "d", Usage: "delete p2p instance (p2p -d hash)"},
				cli.BoolFlag{Name: "u", Usage: "update p2p instance encryption key (p2p -u hash newkey ttl)"},
				cli.BoolFlag{Name: "l", Usage: "list of p2p instances (p2p -l)"},
				cli.BoolFlag{Name: "p", Usage: "list of p2p participants (p2p -p hash)"}},
			Action: func(c *cli.Context) {
				lib.P2P(c.Bool("c"), c.Bool("d"), c.Bool("u"), c.Bool("l"), c.Bool("p"), os.Args)
			}}},

		Action: func(c *cli.Context) {
			lib.LxcManagementNetwork(os.Args)
		}}, {

		Name: "metrics", Usage: "list Subutai container",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "s", Usage: "start time"},
			cli.StringFlag{Name: "e", Usage: "end time"}},
		Action: func(c *cli.Context) {
			lib.HostMetrics(c.Args().Get(0), c.String("s"), c.String("e"))
		}}, {

		Name: "network", Usage: "containerName set/remove/list network vlan id",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "set, s", Usage: "IPADDRESS/NETMASK"},
			cli.StringFlag{Name: "vlan, v", Usage: "vlanid"},
			cli.BoolFlag{Name: "remove, r", Usage: ""},
			cli.BoolFlag{Name: "list, l", Usage: ""},
		},
		Action: func(c *cli.Context) {
			lib.LxcNetwork(c.Args().Get(0), c.String("s"), c.String("vlan"), c.Bool("r"), c.Bool("l"))
		}}, {

		Name: "promote", Usage: "promote Subutai container",
		Action: func(c *cli.Context) {
			lib.LxcPromote(c.Args().Get(0))
		}}, {

		Name: "proxy", Usage: "Subutai reverse proxy",
		Subcommands: []cli.Command{
			{
				Name:     "add",
				Usage:    "add reverse proxy component",
				HideHelp: true,
				Flags: []cli.Flag{
					cli.StringFlag{Name: "domain,d", Usage: "add domain to vlan"},
					cli.StringFlag{Name: "host, h", Usage: "add host to domain on vlan"},
					cli.StringFlag{Name: "policy, p", Usage: "set load balance policy (rr|lb|hash)"},
					cli.StringFlag{Name: "file, f", Usage: "specify pem certificate file"}},
				Action: func(c *cli.Context) {
					lib.ProxyAdd(c.Args().Get(0), c.String("d"), c.String("h"), c.String("p"), c.String("f"))
				},
			},
			{
				Name:     "del",
				Usage:    "del reverse proxy component",
				HideHelp: true,
				Flags: []cli.Flag{
					cli.BoolFlag{Name: "domain, d", Usage: "delete domain from vlan"},
					cli.StringFlag{Name: "host, h", Usage: "delete host from domain on vlan"}},
				Action: func(c *cli.Context) {
					lib.ProxyDel(c.Args().Get(0), c.String("h"), c.Bool("d"))
				},
			},
			{
				Name:     "check",
				Usage:    "check existing domain or host",
				HideHelp: true,
				Flags: []cli.Flag{
					cli.BoolFlag{Name: "domain, d", Usage: "check domains on vlan"},
					cli.StringFlag{Name: "host, h", Usage: "check hosts on vlan"}},
				Action: func(c *cli.Context) {
					lib.ProxyCheck(c.Args().Get(0), c.String("h"), c.Bool("d"))
				},
			},
		}}, {

		Name: "quota", Usage: "set quotas for Subutai container",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "s", Usage: "set quota for the specified resource type"},
			cli.StringFlag{Name: "t", Usage: "set alert threshold"}},
		Action: func(c *cli.Context) {
			lib.LxcQuota(c.Args().Get(0), c.Args().Get(1), c.String("s"), c.String("t"))
		}}, {

		Name: "register", Usage: "register Subutai container",
		Action: func(c *cli.Context) {
			lib.LxcRegister(c.Args().Get(0))
		}}, {

		Name: "rename", Usage: "rename Subutai container",
		Action: func(c *cli.Context) {
			lib.LxcRename(c.Args().Get(0), c.Args().Get(1))
		}}, {

		Name: "restore", Usage: "restore Subutai container",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "d", Usage: "date of backup snapshot"},
			cli.StringFlag{Name: "c", Usage: "name of new container"}},
		Action: func(c *cli.Context) {
			lib.RestoreContainer(c.Args().Get(0), c.Args().Get(1), c.Args().Get(2))
		}}, {

		Name: "stats", Usage: "statistics from host",
		Action: func(c *cli.Context) {
			lib.Stats(c.Args().Get(0), c.Args().Get(1), c.Args().Get(2))
		}}, {

		Name: "start", Usage: "start Subutai container",
		Action: func(c *cli.Context) {
			lib.LxcStart(c.Args().Get(0))
		}}, {

		Name: "stop", Usage: "stop Subutai container",
		Action: func(c *cli.Context) {
			lib.LxcStop(c.Args().Get(0))
		}}, {

		Name: "tunnel", Usage: "create SSH tunnel",
		Flags: []cli.Flag{
			cli.BoolFlag{Name: "g", Usage: "global accessible tunnel"}},
		Action: func(c *cli.Context) {
			lib.Tunnel(c.Args().Get(0), c.Args().Get(1), c.Args().Get(2), c.Bool("g"))
		}}, {

		Name: "unregister", Usage: "unregister Subutai container",
		Action: func(c *cli.Context) {
			lib.LxcUnregister(c.Args().Get(0))
		}}, {

		Name: "update", Usage: "update Subutai management, container or Resource host",
		Flags: []cli.Flag{
			cli.BoolFlag{Name: "c", Usage: "check for updates without installation"}},
		Action: func(c *cli.Context) {
			lib.Update(c.Args().Get(0), c.Bool("c"))
		}},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package ai

import (
	"bytes"
	"log"
	"math/rand"
	"time"

	"github.com/nelhage/taktician/bitboard"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/tak"
)

const (
	maxEval      int64 = 1 << 30
	minEval            = -maxEval
	WinThreshold       = 1 << 29

	tableSize uint64 = (1 << 20)

	maxStack = 10
)

type EvaluationFunc func(m *MinimaxAI, p *tak.Position) int64

type MinimaxAI struct {
	cfg  MinimaxConfig
	rand *rand.Rand

	st Stats
	c  bitboard.Constants

	heatMap []uint64

	evaluate EvaluationFunc

	table []tableEntry
	stack [maxStack]struct {
		p     *tak.Position
		moves [100]tak.Move
	}
}

type tableEntry struct {
	hash  uint64
	depth int
	value int64
	bound boundType
	m     tak.Move
	p     *tak.Position
}

type boundType byte

const (
	lowerBound = iota
	exactBound = iota
	upperBound = iota
)

type Stats struct {
	Depth     int
	Generated uint64
	Evaluated uint64
	Terminal  uint64
	Visited   uint64

	CutNodes  uint64
	Cut0      uint64
	Cut1      uint64
	CutSearch uint64

	AllNodes uint64

	TTHits uint64
}

type MinimaxConfig struct {
	Size  int
	Depth int
	Debug int
	Seed  int64

	NoSort  bool
	NoTable bool

	Evaluate EvaluationFunc
}

func NewMinimax(cfg MinimaxConfig) *MinimaxAI {
	m := &MinimaxAI{cfg: cfg}
	m.precompute()
	m.evaluate = cfg.Evaluate
	if m.evaluate == nil {
		m.evaluate = DefaultEvaluate
	}
	m.heatMap = make([]uint64, m.cfg.Size*m.cfg.Size)
	m.table = make([]tableEntry, tableSize)
	for i := range m.stack {
		m.stack[i].p = tak.Alloc(m.cfg.Size)
	}
	return m
}

func (m *MinimaxAI) ttGet(h uint64) *tableEntry {
	if m.cfg.NoTable {
		return nil
	}
	te := &m.table[h%tableSize]
	if te.hash != h {
		return nil
	}
	return te
}

func (m *MinimaxAI) ttPut(h uint64) *tableEntry {
	return &m.table[h%tableSize]
}

func (m *MinimaxAI) precompute() {
	s := uint(m.cfg.Size)
	m.c = bitboard.Precompute(s)
}

func formatpv(ms []tak.Move) string {
	var out bytes.Buffer
	out.WriteString("[")
	for i, m := range ms {
		if i != 0 {
			out.WriteString(" ")
		}
		out.WriteString(ptn.FormatMove(&m))
	}
	out.WriteString("]")
	return out.String()
}

func (m *MinimaxAI) GetMove(p *tak.Position, limit time.Duration) tak.Move {
	ms, _, _ := m.Analyze(p, limit)
	return ms[0]
}

func (m *MinimaxAI) Analyze(p *tak.Position, limit time.Duration) ([]tak.Move, int64, Stats) {
	if m.cfg.Size != p.Size() {
		panic("Analyze: wrong size")
	}
	for i, v := range m.heatMap {
		m.heatMap[i] = v / 2
	}

	var seed = m.cfg.Seed
	if seed == 0 {
		seed = time.Now().Unix()
	}
	m.rand = rand.New(rand.NewSource(seed))
	if m.cfg.Debug > 0 {
		log.Printf("seed=%d", seed)
	}

	var ms []tak.Move
	var v int64
	top := time.Now()
	var prevEval uint64
	var branchSum uint64
	base := 0
	te := m.ttGet(p.Hash())
	if te != nil && te.bound == exactBound {
		base = te.depth
		ms = []tak.Move{te.m}
	}

	for i := 1; i+base <= m.cfg.Depth; i++ {
		m.st = Stats{Depth: i + base}
		start := time.Now()
		ms, v = m.minimax(p, 0, i+base, ms, minEval-1, maxEval+1)
		timeUsed := time.Now().Sub(top)
		timeMove := time.Now().Sub(start)
		if m.cfg.Debug > 0 {
			log.Printf("[minimax] deepen: depth=%d val=%d pv=%s time=%s total=%s evaluated=%d tt=%d branch=%d",
				base+i, v, formatpv(ms),
				timeMove,
				timeUsed,
				m.st.Evaluated,
				m.st.TTHits,
				m.st.Evaluated/(prevEval+1),
			)
		}
		if m.cfg.Debug > 1 {
			log.Printf("[minimax]  stats: visited=%d evaluated=%d terminal=%d cut=%d cut0=%d(%2.2f) cut1=%d(%2.2f) m/cut=%2.2f m/ms=%f all=%d",
				m.st.Visited,
				m.st.Evaluated,
				m.st.Terminal,
				m.st.CutNodes,
				m.st.Cut0,
				float64(m.st.Cut0)/float64(m.st.CutNodes+1),
				m.st.Cut1,
				float64(m.st.Cut0+m.st.Cut1)/float64(m.st.CutNodes+1),
				float64(m.st.CutSearch)/float64(m.st.CutNodes-m.st.Cut0-m.st.Cut1+1),
				float64(m.st.Visited+m.st.Evaluated)/float64(timeMove.Seconds()*1000),
				m.st.AllNodes)
		}
		if i > 1 {
			branchSum += m.st.Evaluated / (prevEval + 1)
		}
		prevEval = m.st.Evaluated
		if v > WinThreshold || v < -WinThreshold {
			break
		}
		if i+base != m.cfg.Depth && limit != 0 {
			var branch uint64
			if i > 2 {
				branch = branchSum / uint64(i-1)
			} else {
				// conservative estimate if we haven't
				// run enough plies to have one
				// yet. This can matter if the table
				// returns a deep move
				branch = 20
			}
			estimate := timeUsed + time.Now().Sub(start)*time.Duration(branch)
			if estimate > limit {
				if m.cfg.Debug > 0 {
					log.Printf("[minimax] time cutoff: depth=%d used=%s estimate=%s",
						i, timeUsed, estimate)
				}
				break
			}
		}
	}
	return ms, v, m.st
}

const debugTable = false

func (ai *MinimaxAI) minimax(
	p *tak.Position,
	ply, depth int,
	pv []tak.Move,
	α, β int64) ([]tak.Move, int64) {
	over, _ := p.GameOver()
	if depth == 0 || over {
		ai.st.Evaluated++
		if over {
			ai.st.Terminal++
		}
		return nil, ai.evaluate(ai, p)
	}

	ai.st.Visited++

	te := ai.ttGet(p.Hash())
	if te != nil {
		teSuffices := false
		if te.depth >= depth {
			if te.bound == exactBound ||
				(te.value < α && te.bound == upperBound) ||
				(te.value > β && te.bound == lowerBound) {
				teSuffices = true
			}
		}

		if te.bound == exactBound &&
			(te.value > WinThreshold || te.value < -WinThreshold) {
			teSuffices = true
		}
		if teSuffices {
			_, e := p.Move(&te.m)
			if e == nil {
				ai.st.TTHits++
				return []tak.Move{te.m}, te.value
			}
			te = nil
		}
	}
	mg := moveGenerator{
		ai:    ai,
		ply:   ply,
		depth: depth,
		p:     p,
		te:    te,
		pv:    pv,
	}

	best := make([]tak.Move, 0, depth)
	best = append(best, pv...)
	improved := false
	var i int
	for m, child := mg.Next(); child != nil; m, child = mg.Next() {
		i++
		var ms []tak.Move
		var newpv []tak.Move
		var v int64
		if len(best) != 0 {
			newpv = best[1:]
		}
		if i > 1 {
			ms, v = ai.minimax(child, ply+1, depth-1, newpv, -α-1, -α)
			if -v > α && -v < β {
				ms, v = ai.minimax(child, ply+1, depth-1, newpv, -β, -α)
			}
		} else {
			ms, v = ai.minimax(child, ply+1, depth-1, newpv, -β, -α)
		}
		v = -v
		if ai.cfg.Debug > 2 && ply == 0 {
			log.Printf("[minimax] search: depth=%d ply=%d m=%s pv=%s window=(%d,%d) ms=%s v=%d evaluated=%d",
				depth, ply, ptn.FormatMove(&m), formatpv(newpv), α, β, formatpv(ms), v, ai.st.Evaluated)
		}

		if len(best) == 0 {
			best = append(best[:0], m)
			best = append(best, ms...)
		}
		if v > α {
			improved = true
			best = append(best[:0], m)
			best = append(best, ms...)
			α = v
			if α >= β {
				ai.st.CutNodes++
				switch i {
				case 1:
					ai.st.Cut0++
				case 2:
					ai.st.Cut1++
				default:
					ai.st.CutSearch += uint64(i + 1)
				}
				ai.heatMap[m.X+m.Y*ai.cfg.Size] += (1 << uint(depth))
				if ai.cfg.Debug > 3 && i > 20 && depth >= 3 {
					var tm tak.Move
					td := 0
					if te != nil {
						tm = te.m
						td = te.depth
					}
					log.Printf("[minimax] late cutoff depth=%d m=%d pv=%s te=%d:%s killer=%s pos=%q",
						depth, i, formatpv(pv), td, ptn.FormatMove(&tm), ptn.FormatMove(&m), ptn.FormatTPS(p),
					)
				}
				break
			}
		}
	}

	if debugTable && te != nil &&
		te.depth == depth &&
		te.bound == exactBound &&
		!best[0].Equal(&te.m) {
		log.Printf("? ply=%d depth=%d found=[%s, %v] t=[%s, %v]",
			ply, depth,
			ptn.FormatMove(&best[0]), α,
			ptn.FormatMove(&te.m), te.value,
		)
		log.Printf(" p> %#v", p)
		log.Printf("tp> %#v", te.p)
	}

	te = ai.ttPut(p.Hash())
	te.hash = p.Hash()
	te.depth = depth
	te.m = best[0]
	te.value = α
	if debugTable {
		te.p = p
	}
	if !improved {
		te.bound = upperBound
		ai.st.AllNodes++
	} else if α >= β {
		te.bound = lowerBound
	} else {
		te.bound = exactBound
	}

	return best, α
}
<commit_msg>remove debugTable<commit_after>package ai

import (
	"bytes"
	"log"
	"math/rand"
	"time"

	"github.com/nelhage/taktician/bitboard"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/tak"
)

const (
	maxEval      int64 = 1 << 30
	minEval            = -maxEval
	WinThreshold       = 1 << 29

	tableSize uint64 = (1 << 20)

	maxStack = 10
)

type EvaluationFunc func(m *MinimaxAI, p *tak.Position) int64

type MinimaxAI struct {
	cfg  MinimaxConfig
	rand *rand.Rand

	st Stats
	c  bitboard.Constants

	heatMap []uint64

	evaluate EvaluationFunc

	table []tableEntry
	stack [maxStack]struct {
		p     *tak.Position
		moves [100]tak.Move
	}
}

type tableEntry struct {
	hash  uint64
	depth int
	value int64
	bound boundType
	m     tak.Move
	p     *tak.Position
}

type boundType byte

const (
	lowerBound = iota
	exactBound = iota
	upperBound = iota
)

type Stats struct {
	Depth     int
	Generated uint64
	Evaluated uint64
	Terminal  uint64
	Visited   uint64

	CutNodes  uint64
	Cut0      uint64
	Cut1      uint64
	CutSearch uint64

	AllNodes uint64

	TTHits uint64
}

type MinimaxConfig struct {
	Size  int
	Depth int
	Debug int
	Seed  int64

	NoSort  bool
	NoTable bool

	Evaluate EvaluationFunc
}

func NewMinimax(cfg MinimaxConfig) *MinimaxAI {
	m := &MinimaxAI{cfg: cfg}
	m.precompute()
	m.evaluate = cfg.Evaluate
	if m.evaluate == nil {
		m.evaluate = DefaultEvaluate
	}
	m.heatMap = make([]uint64, m.cfg.Size*m.cfg.Size)
	m.table = make([]tableEntry, tableSize)
	for i := range m.stack {
		m.stack[i].p = tak.Alloc(m.cfg.Size)
	}
	return m
}

func (m *MinimaxAI) ttGet(h uint64) *tableEntry {
	if m.cfg.NoTable {
		return nil
	}
	te := &m.table[h%tableSize]
	if te.hash != h {
		return nil
	}
	return te
}

func (m *MinimaxAI) ttPut(h uint64) *tableEntry {
	return &m.table[h%tableSize]
}

func (m *MinimaxAI) precompute() {
	s := uint(m.cfg.Size)
	m.c = bitboard.Precompute(s)
}

func formatpv(ms []tak.Move) string {
	var out bytes.Buffer
	out.WriteString("[")
	for i, m := range ms {
		if i != 0 {
			out.WriteString(" ")
		}
		out.WriteString(ptn.FormatMove(&m))
	}
	out.WriteString("]")
	return out.String()
}

func (m *MinimaxAI) GetMove(p *tak.Position, limit time.Duration) tak.Move {
	ms, _, _ := m.Analyze(p, limit)
	return ms[0]
}

func (m *MinimaxAI) Analyze(p *tak.Position, limit time.Duration) ([]tak.Move, int64, Stats) {
	if m.cfg.Size != p.Size() {
		panic("Analyze: wrong size")
	}
	for i, v := range m.heatMap {
		m.heatMap[i] = v / 2
	}

	var seed = m.cfg.Seed
	if seed == 0 {
		seed = time.Now().Unix()
	}
	m.rand = rand.New(rand.NewSource(seed))
	if m.cfg.Debug > 0 {
		log.Printf("seed=%d", seed)
	}

	var ms []tak.Move
	var v int64
	top := time.Now()
	var prevEval uint64
	var branchSum uint64
	base := 0
	te := m.ttGet(p.Hash())
	if te != nil && te.bound == exactBound {
		base = te.depth
		ms = []tak.Move{te.m}
	}

	for i := 1; i+base <= m.cfg.Depth; i++ {
		m.st = Stats{Depth: i + base}
		start := time.Now()
		ms, v = m.minimax(p, 0, i+base, ms, minEval-1, maxEval+1)
		timeUsed := time.Now().Sub(top)
		timeMove := time.Now().Sub(start)
		if m.cfg.Debug > 0 {
			log.Printf("[minimax] deepen: depth=%d val=%d pv=%s time=%s total=%s evaluated=%d tt=%d branch=%d",
				base+i, v, formatpv(ms),
				timeMove,
				timeUsed,
				m.st.Evaluated,
				m.st.TTHits,
				m.st.Evaluated/(prevEval+1),
			)
		}
		if m.cfg.Debug > 1 {
			log.Printf("[minimax]  stats: visited=%d evaluated=%d terminal=%d cut=%d cut0=%d(%2.2f) cut1=%d(%2.2f) m/cut=%2.2f m/ms=%f all=%d",
				m.st.Visited,
				m.st.Evaluated,
				m.st.Terminal,
				m.st.CutNodes,
				m.st.Cut0,
				float64(m.st.Cut0)/float64(m.st.CutNodes+1),
				m.st.Cut1,
				float64(m.st.Cut0+m.st.Cut1)/float64(m.st.CutNodes+1),
				float64(m.st.CutSearch)/float64(m.st.CutNodes-m.st.Cut0-m.st.Cut1+1),
				float64(m.st.Visited+m.st.Evaluated)/float64(timeMove.Seconds()*1000),
				m.st.AllNodes)
		}
		if i > 1 {
			branchSum += m.st.Evaluated / (prevEval + 1)
		}
		prevEval = m.st.Evaluated
		if v > WinThreshold || v < -WinThreshold {
			break
		}
		if i+base != m.cfg.Depth && limit != 0 {
			var branch uint64
			if i > 2 {
				branch = branchSum / uint64(i-1)
			} else {
				// conservative estimate if we haven't
				// run enough plies to have one
				// yet. This can matter if the table
				// returns a deep move
				branch = 20
			}
			estimate := timeUsed + time.Now().Sub(start)*time.Duration(branch)
			if estimate > limit {
				if m.cfg.Debug > 0 {
					log.Printf("[minimax] time cutoff: depth=%d used=%s estimate=%s",
						i, timeUsed, estimate)
				}
				break
			}
		}
	}
	return ms, v, m.st
}

func (ai *MinimaxAI) minimax(
	p *tak.Position,
	ply, depth int,
	pv []tak.Move,
	α, β int64) ([]tak.Move, int64) {
	over, _ := p.GameOver()
	if depth == 0 || over {
		ai.st.Evaluated++
		if over {
			ai.st.Terminal++
		}
		return nil, ai.evaluate(ai, p)
	}

	ai.st.Visited++

	te := ai.ttGet(p.Hash())
	if te != nil {
		teSuffices := false
		if te.depth >= depth {
			if te.bound == exactBound ||
				(te.value < α && te.bound == upperBound) ||
				(te.value > β && te.bound == lowerBound) {
				teSuffices = true
			}
		}

		if te.bound == exactBound &&
			(te.value > WinThreshold || te.value < -WinThreshold) {
			teSuffices = true
		}
		if teSuffices {
			_, e := p.Move(&te.m)
			if e == nil {
				ai.st.TTHits++
				return []tak.Move{te.m}, te.value
			}
			te = nil
		}
	}
	mg := moveGenerator{
		ai:    ai,
		ply:   ply,
		depth: depth,
		p:     p,
		te:    te,
		pv:    pv,
	}

	best := make([]tak.Move, 0, depth)
	best = append(best, pv...)
	improved := false
	var i int
	for m, child := mg.Next(); child != nil; m, child = mg.Next() {
		i++
		var ms []tak.Move
		var newpv []tak.Move
		var v int64
		if len(best) != 0 {
			newpv = best[1:]
		}
		if i > 1 {
			ms, v = ai.minimax(child, ply+1, depth-1, newpv, -α-1, -α)
			if -v > α && -v < β {
				ms, v = ai.minimax(child, ply+1, depth-1, newpv, -β, -α)
			}
		} else {
			ms, v = ai.minimax(child, ply+1, depth-1, newpv, -β, -α)
		}
		v = -v
		if ai.cfg.Debug > 2 && ply == 0 {
			log.Printf("[minimax] search: depth=%d ply=%d m=%s pv=%s window=(%d,%d) ms=%s v=%d evaluated=%d",
				depth, ply, ptn.FormatMove(&m), formatpv(newpv), α, β, formatpv(ms), v, ai.st.Evaluated)
		}

		if len(best) == 0 {
			best = append(best[:0], m)
			best = append(best, ms...)
		}
		if v > α {
			improved = true
			best = append(best[:0], m)
			best = append(best, ms...)
			α = v
			if α >= β {
				ai.st.CutNodes++
				switch i {
				case 1:
					ai.st.Cut0++
				case 2:
					ai.st.Cut1++
				default:
					ai.st.CutSearch += uint64(i + 1)
				}
				ai.heatMap[m.X+m.Y*ai.cfg.Size] += (1 << uint(depth))
				if ai.cfg.Debug > 3 && i > 20 && depth >= 3 {
					var tm tak.Move
					td := 0
					if te != nil {
						tm = te.m
						td = te.depth
					}
					log.Printf("[minimax] late cutoff depth=%d m=%d pv=%s te=%d:%s killer=%s pos=%q",
						depth, i, formatpv(pv), td, ptn.FormatMove(&tm), ptn.FormatMove(&m), ptn.FormatTPS(p),
					)
				}
				break
			}
		}
	}

	te = ai.ttPut(p.Hash())
	te.hash = p.Hash()
	te.depth = depth
	te.m = best[0]
	te.value = α
	if !improved {
		te.bound = upperBound
		ai.st.AllNodes++
	} else if α >= β {
		te.bound = lowerBound
	} else {
		te.bound = exactBound
	}

	return best, α
}
<|endoftext|>


<commit_before>package anomalyzer

import (
	"fmt"
	"github.com/drewlanenga/govector"
	"math"
)

type Algorithm func(govector.Vector, AnomalyzerConf) float64

var (
	Algorithms = map[string]Algorithm{
		"magnitude": MagnitudeTest,
		"diff":      DiffTest,
		"highrank":  RankTest,
		"lowrank":   ReverseRankTest,
		"cdf":       CDFTest,
		"fence":     FenceTest,
		"ks":        BootstrapKsTest,
	}
)

// Identity function
func identity(anything interface{}) interface{} {
	return anything
}

// Returns a value within a given window (xmin and xmax).
func cap(x, min, max float64) float64 {
	return math.Max(math.Min(x, max), min)
}

// Returns a contant
func constant(x float64) float64 {
	return 0.2
}

// Return integer math comparisons
func max(x, y int) int {
	if x > y {
		return x
	}
	return y
}

func min(x, y int) int {
	if y < x {
		return y
	}
	return x
}

// Return a vector slice for the active window and reference window.
// Some tests require different minimum thresholds for sizes of reference windows.
// This can be specified in the minRefSize parameter. If size isn't important, use -1
func extractWindows(vector govector.Vector, refSize, activeSize, minRefSize int) (govector.Vector, govector.Vector, error) {
	n := len(vector)
	activeSize = min(activeSize, n)
	refSize = min(refSize, n-activeSize)

	// make sure the reference size is at least as big as the active size
	// note that this penalty might be overly severe for some tests
	if refSize < minRefSize {
		return nil, nil, fmt.Errorf("Reference size must be at least as big as active size")
	}

	// return reference and active windows
	return vector[n-activeSize-refSize : n-activeSize], vector[n-activeSize:], nil
}

// This function can be used to test whether or not data is getting close to a
// specified upper or lower bound.
func FenceTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	// we don't really care about a reference window for this one
	_, active, _ := extractWindows(vector, conf.referenceSize, conf.ActiveSize, -1)

	x := active.Mean()

	distance := 0.0
	if conf.LowerBound == NA {
		// we only care about distance from the upper bound
		distance = x / conf.UpperBound
	} else {
		// we care about both bounds, so measure distance
		// from midpoint

		bound := (conf.UpperBound - conf.LowerBound) / 2
		mid := conf.LowerBound + bound

		distance = (math.Abs(x - mid)) / bound
	}
	return weightExp(cap(distance, 0, 1), 10)
}

// This is a function will sharply scale values between 0 and 1 such that
// smaller values are weighted more towards 0. A larger base value means a
// more horshoe type function.
func weightExp(x, base float64) float64 {
	return (math.Pow(base, x) - 1) / (math.Pow(base, 1) - 1)
}

// Generates permutations of reference and active window values to determine
// whether or not data is anomalous. The number of permutations desired has
// been set to 500 but can be increased for more precision.
func DiffTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	// Find the differences between neighboring elements and rank those differences.
	ranks := vector.RelDiff().Apply(math.Abs).Rank()

	// The indexing runs to length-1 because after applying .Diff(), We have
	// decreased the length of out vector by 1.
	_, active, err := extractWindows(ranks, conf.referenceSize-1, conf.ActiveSize, conf.ActiveSize)
	if err != nil {
		return NA
	}

	// Consider the sum of the ranks across the active data. This is the sum that
	// we will compare our permutations to.
	activeSum := active.Sum()

	i := 0
	significant := 0

	// Permute the active and reference data and compute the sums across the tail
	// (from the length of the reference data to the full length).
	for i < conf.PermCount {
		permRanks := vector.Shuffle().RelDiff().Apply(math.Abs).Rank()
		_, permActive, _ := extractWindows(permRanks, conf.referenceSize-1, conf.ActiveSize, conf.ActiveSize)

		// If we find a sum that is less than the initial sum across the active data,
		// this implies our initial sum might be uncharacteristically high. We increment
		// our count.
		if permActive.Sum() < activeSum {
			significant++
		}
		i++
	}
	// We return the percentage of the number of iterations where we found our initial
	// sum to be high.
	return float64(significant) / float64(conf.PermCount)
}

func RankTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	return rankTest(vector, conf, lessThan)
}

func ReverseRankTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	return rankTest(vector, conf, greaterThan)
}

type compare func(x, y float64) bool

func greaterThan(x, y float64) bool {
	if x > y {
		return true
	}
	return false
}

func lessThan(x, y float64) bool {
	if x < y {
		return true
	}
	return false
}

// Generates permutations of reference and active window values to determine
// whether or not data is anomalous. The number of permutations desired defaults
// to 500 but can be increased for more precision. A comparison function above
// can be specified to create Rank and ReverseRank tests.
func rankTest(vector govector.Vector, conf AnomalyzerConf, comparison compare) float64 {
	// Rank the elements of a vector
	ranks := vector.Rank()

	_, active, err := extractWindows(ranks, conf.referenceSize, conf.ActiveSize, conf.ActiveSize)
	if err != nil {
		return NA
	}

	// Consider the sum of the ranks across the active data. This is the sum that
	// we will compare our permutations to.
	activeSum := active.Sum()

	i := 0
	significant := 0

	// Permute the active and reference data and compute the sums across the tail
	// (from the length of the reference data to the full length).
	for i < conf.PermCount {
		permRanks := vector.Shuffle().Rank()
		_, permActive, _ := extractWindows(permRanks, conf.referenceSize, conf.ActiveSize, conf.ActiveSize)

		// If we find a sum that is less than the initial sum across the active data,
		// this implies our initial sum might be uncharacteristically high. We increment
		// our count.

		permSum := permActive.Sum()
		if comparison(permSum, activeSum) {
			significant++
		}
		i++
	}
	// We return the percentage of the number of iterations where we found our initial
	// sum to be high.
	return float64(significant) / float64(conf.PermCount)
}

// Generates the cumulative distribution function using the difference in the means
// for the data.
func CDFTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	diffs := vector.Diff().Apply(math.Abs)
	reference, active, err := extractWindows(diffs, conf.referenceSize-1, conf.ActiveSize, conf.ActiveSize)
	if err != nil {
		return NA
	}

	// Find the empircal distribution function using the reference window.
	refEcdf := reference.Ecdf()

	// Difference between the active and reference means.
	activeDiff := active.Mean() - reference.Mean()

	// Apply the empirical distribution function to that difference.
	percentile := refEcdf(activeDiff)

	// Scale so max probability is in tails and prob at 0.5 is 0.
	return (2 * math.Abs(0.5-percentile))
}

// Generates the percent difference between the means of the reference and active
// data. Returns a value scaled such that it lies between 0 and 1.
func MagnitudeTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	reference, active, err := extractWindows(vector, conf.referenceSize, conf.ActiveSize, 1)
	if err != nil {
		return NA
	}

	activeMean := active.Mean()
	refMean := reference.Mean()

	// If the baseline is 0, then the magnitude should be Inf, but we'll
	// round to 1.
	if refMean == 0 {
		return 1
	}
	pdiff := math.Abs(activeMean-refMean) / refMean
	//return weightExp(pdiff, 10)
	return pdiff
}

// Calculate a Kolmogorov-Smirnov test statistic.
func KsStat(vector govector.Vector, conf AnomalyzerConf) float64 {
	reference, active, err := extractWindows(vector, conf.referenceSize, conf.ActiveSize, conf.ActiveSize)
	if err != nil {
		return NA
	}
	n1 := len(reference)
	n2 := len(active)
	if n1%n2 != 0 {
		return NA
	}

	// First sort the active data and generate a cummulative distribution function
	// using that data. Do the same for the reference data.
	activeEcdf := active.Ecdf()
	refEcdf := reference.Ecdf()

	// We want the reference and active vectors to have the same length n, so we
	// consider the min and max for each and interpolated the points between.
	min := math.Min(reference.Min(), active.Min())
	max := math.Max(reference.Max(), active.Max())

	interpolated := interpolate(min, max, n1+n2)

	// Then we apply the distribution function over the interpolated data.
	activeDist := interpolated.Apply(activeEcdf)
	refDist := interpolated.Apply(refEcdf)

	// Find the maximum displacement between both distributions.
	d := 0.0
	for i := 0; i < n1+n2; i++ {
		d = math.Max(d, math.Abs(activeDist[i]-refDist[i]))
	}
	return d
}

func BootstrapKsTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	dist := KsStat(vector, conf)
	if dist == NA {
		return NA
	}

	i := 0
	significant := 0

	for i < conf.PermCount {
		permVector := vector.Shuffle()
		permDist := KsStat(permVector, conf)

		if permDist < dist {
			significant++
		}
		i++
	}
	return float64(significant) / float64(conf.PermCount)
}

// A helper function for KS that rescales a vector to the desired length npoints.
func interpolate(min, max float64, npoints int) govector.Vector {
	interp := make(govector.Vector, npoints)

	step := (max - min) / (float64(npoints) - 1)
	interp[0] = min
	i := 1
	for i < npoints {
		interp[i] = interp[i-1] + step
		i++
	}
	return interp
}
<commit_msg>If the average of both active and ref windows are 0, return 0 for the magnitude test<commit_after>package anomalyzer

import (
	"fmt"
	"github.com/drewlanenga/govector"
	"math"
)

type Algorithm func(govector.Vector, AnomalyzerConf) float64

var (
	Algorithms = map[string]Algorithm{
		"magnitude": MagnitudeTest,
		"diff":      DiffTest,
		"highrank":  RankTest,
		"lowrank":   ReverseRankTest,
		"cdf":       CDFTest,
		"fence":     FenceTest,
		"ks":        BootstrapKsTest,
	}
)

// Identity function
func identity(anything interface{}) interface{} {
	return anything
}

// Returns a value within a given window (xmin and xmax).
func cap(x, min, max float64) float64 {
	return math.Max(math.Min(x, max), min)
}

// Returns a contant
func constant(x float64) float64 {
	return 0.2
}

// Return integer math comparisons
func max(x, y int) int {
	if x > y {
		return x
	}
	return y
}

func min(x, y int) int {
	if y < x {
		return y
	}
	return x
}

// Return a vector slice for the active window and reference window.
// Some tests require different minimum thresholds for sizes of reference windows.
// This can be specified in the minRefSize parameter. If size isn't important, use -1
func extractWindows(vector govector.Vector, refSize, activeSize, minRefSize int) (govector.Vector, govector.Vector, error) {
	n := len(vector)
	activeSize = min(activeSize, n)
	refSize = min(refSize, n-activeSize)

	// make sure the reference size is at least as big as the active size
	// note that this penalty might be overly severe for some tests
	if refSize < minRefSize {
		return nil, nil, fmt.Errorf("Reference size must be at least as big as active size")
	}

	// return reference and active windows
	return vector[n-activeSize-refSize : n-activeSize], vector[n-activeSize:], nil
}

// This function can be used to test whether or not data is getting close to a
// specified upper or lower bound.
func FenceTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	// we don't really care about a reference window for this one
	_, active, _ := extractWindows(vector, conf.referenceSize, conf.ActiveSize, -1)

	x := active.Mean()

	distance := 0.0
	if conf.LowerBound == NA {
		// we only care about distance from the upper bound
		distance = x / conf.UpperBound
	} else {
		// we care about both bounds, so measure distance
		// from midpoint

		bound := (conf.UpperBound - conf.LowerBound) / 2
		mid := conf.LowerBound + bound

		distance = (math.Abs(x - mid)) / bound
	}
	return weightExp(cap(distance, 0, 1), 10)
}

// This is a function will sharply scale values between 0 and 1 such that
// smaller values are weighted more towards 0. A larger base value means a
// more horshoe type function.
func weightExp(x, base float64) float64 {
	return (math.Pow(base, x) - 1) / (math.Pow(base, 1) - 1)
}

// Generates permutations of reference and active window values to determine
// whether or not data is anomalous. The number of permutations desired has
// been set to 500 but can be increased for more precision.
func DiffTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	// Find the differences between neighboring elements and rank those differences.
	ranks := vector.RelDiff().Apply(math.Abs).Rank()

	// The indexing runs to length-1 because after applying .Diff(), We have
	// decreased the length of out vector by 1.
	_, active, err := extractWindows(ranks, conf.referenceSize-1, conf.ActiveSize, conf.ActiveSize)
	if err != nil {
		return NA
	}

	// Consider the sum of the ranks across the active data. This is the sum that
	// we will compare our permutations to.
	activeSum := active.Sum()

	i := 0
	significant := 0

	// Permute the active and reference data and compute the sums across the tail
	// (from the length of the reference data to the full length).
	for i < conf.PermCount {
		permRanks := vector.Shuffle().RelDiff().Apply(math.Abs).Rank()
		_, permActive, _ := extractWindows(permRanks, conf.referenceSize-1, conf.ActiveSize, conf.ActiveSize)

		// If we find a sum that is less than the initial sum across the active data,
		// this implies our initial sum might be uncharacteristically high. We increment
		// our count.
		if permActive.Sum() < activeSum {
			significant++
		}
		i++
	}
	// We return the percentage of the number of iterations where we found our initial
	// sum to be high.
	return float64(significant) / float64(conf.PermCount)
}

func RankTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	return rankTest(vector, conf, lessThan)
}

func ReverseRankTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	return rankTest(vector, conf, greaterThan)
}

type compare func(x, y float64) bool

func greaterThan(x, y float64) bool {
	if x > y {
		return true
	}
	return false
}

func lessThan(x, y float64) bool {
	if x < y {
		return true
	}
	return false
}

// Generates permutations of reference and active window values to determine
// whether or not data is anomalous. The number of permutations desired defaults
// to 500 but can be increased for more precision. A comparison function above
// can be specified to create Rank and ReverseRank tests.
func rankTest(vector govector.Vector, conf AnomalyzerConf, comparison compare) float64 {
	// Rank the elements of a vector
	ranks := vector.Rank()

	_, active, err := extractWindows(ranks, conf.referenceSize, conf.ActiveSize, conf.ActiveSize)
	if err != nil {
		return NA
	}

	// Consider the sum of the ranks across the active data. This is the sum that
	// we will compare our permutations to.
	activeSum := active.Sum()

	i := 0
	significant := 0

	// Permute the active and reference data and compute the sums across the tail
	// (from the length of the reference data to the full length).
	for i < conf.PermCount {
		permRanks := vector.Shuffle().Rank()
		_, permActive, _ := extractWindows(permRanks, conf.referenceSize, conf.ActiveSize, conf.ActiveSize)

		// If we find a sum that is less than the initial sum across the active data,
		// this implies our initial sum might be uncharacteristically high. We increment
		// our count.

		permSum := permActive.Sum()
		if comparison(permSum, activeSum) {
			significant++
		}
		i++
	}
	// We return the percentage of the number of iterations where we found our initial
	// sum to be high.
	return float64(significant) / float64(conf.PermCount)
}

// Generates the cumulative distribution function using the difference in the means
// for the data.
func CDFTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	diffs := vector.Diff().Apply(math.Abs)
	reference, active, err := extractWindows(diffs, conf.referenceSize-1, conf.ActiveSize, conf.ActiveSize)
	if err != nil {
		return NA
	}

	// Find the empircal distribution function using the reference window.
	refEcdf := reference.Ecdf()

	// Difference between the active and reference means.
	activeDiff := active.Mean() - reference.Mean()

	// Apply the empirical distribution function to that difference.
	percentile := refEcdf(activeDiff)

	// Scale so max probability is in tails and prob at 0.5 is 0.
	return (2 * math.Abs(0.5-percentile))
}

// Generates the percent difference between the means of the reference and active
// data. Returns a value scaled such that it lies between 0 and 1.
func MagnitudeTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	reference, active, err := extractWindows(vector, conf.referenceSize, conf.ActiveSize, 1)
	if err != nil {
		return NA
	}

	activeMean := active.Mean()
	refMean := reference.Mean()

	// If the baseline is 0, then the magnitude should be Inf, but we'll
	// round to 1.
	if refMean == 0 {
		if activeMean == 0 {
			return 0
		} else {
			return 1
		}
	}
	pdiff := math.Abs(activeMean-refMean) / refMean
	//return weightExp(pdiff, 10)
	return pdiff
}

// Calculate a Kolmogorov-Smirnov test statistic.
func KsStat(vector govector.Vector, conf AnomalyzerConf) float64 {
	reference, active, err := extractWindows(vector, conf.referenceSize, conf.ActiveSize, conf.ActiveSize)
	if err != nil {
		return NA
	}
	n1 := len(reference)
	n2 := len(active)
	if n1%n2 != 0 {
		return NA
	}

	// First sort the active data and generate a cummulative distribution function
	// using that data. Do the same for the reference data.
	activeEcdf := active.Ecdf()
	refEcdf := reference.Ecdf()

	// We want the reference and active vectors to have the same length n, so we
	// consider the min and max for each and interpolated the points between.
	min := math.Min(reference.Min(), active.Min())
	max := math.Max(reference.Max(), active.Max())

	interpolated := interpolate(min, max, n1+n2)

	// Then we apply the distribution function over the interpolated data.
	activeDist := interpolated.Apply(activeEcdf)
	refDist := interpolated.Apply(refEcdf)

	// Find the maximum displacement between both distributions.
	d := 0.0
	for i := 0; i < n1+n2; i++ {
		d = math.Max(d, math.Abs(activeDist[i]-refDist[i]))
	}
	return d
}

func BootstrapKsTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	dist := KsStat(vector, conf)
	if dist == NA {
		return NA
	}

	i := 0
	significant := 0

	for i < conf.PermCount {
		permVector := vector.Shuffle()
		permDist := KsStat(permVector, conf)

		if permDist < dist {
			significant++
		}
		i++
	}
	return float64(significant) / float64(conf.PermCount)
}

// A helper function for KS that rescales a vector to the desired length npoints.
func interpolate(min, max float64, npoints int) govector.Vector {
	interp := make(govector.Vector, npoints)

	step := (max - min) / (float64(npoints) - 1)
	interp[0] = min
	i := 1
	for i < npoints {
		interp[i] = interp[i-1] + step
		i++
	}
	return interp
}
<|endoftext|>


<commit_before>// Author  Raido Pahtma
// License MIT

package main

import "fmt"
import "os"
import "os/signal"
import "log"
import "time"

import "github.com/jessevdk/go-flags"
import "github.com/proactivity-lab/go-sfconnection"

const ApplicationVersionMajor = 0
const ApplicationVersionMinor = 1
const ApplicationVersionPatch = 2

var ApplicationBuildDate string
var ApplicationBuildDistro string

func main() {

	var opts struct {
		Positional struct {
			ConnectionString string `description:"Connectionstring sf@HOST:PORT"`
		} `positional-args:"yes"`

		Reconnect uint `long:"reconnect" default:"30" description:"Reconnect period, seconds"`

		Debug       []bool `short:"D" long:"debug" description:"Debug mode, print raw packets"`
		ShowVersion func() `short:"V" long:"version" description:"Show application version"`
	}

	opts.ShowVersion = func() {
		if ApplicationBuildDate == "" {
			ApplicationBuildDate = "YYYY-mm-dd_HH:MM:SS"
		}
		if ApplicationBuildDistro == "" {
			ApplicationBuildDistro = "unknown"
		}
		fmt.Printf("amlistener %d.%d.%d (%s %s)\n", ApplicationVersionMajor, ApplicationVersionMinor, ApplicationVersionPatch, ApplicationBuildDate, ApplicationBuildDistro)
		os.Exit(0)
	}

	_, err := flags.Parse(&opts)
	if err != nil {
		fmt.Printf("Argument parser error: %s\n", err)
		os.Exit(1)
	}

	host, port, err := sfconnection.ParseSfConnectionString(opts.Positional.ConnectionString)
	if err != nil {
		fmt.Printf("ERROR: %s\n", err)
		os.Exit(1)
	}

	dsp := sfconnection.NewMessageDispatcher(sfconnection.NewMessage(0, 0))
	receive := make(chan *sfconnection.Message)
	dsp.RegisterMessageSnooper(receive)

	sfc := sfconnection.NewSfConnection()
	sfc.AddDispatcher(dsp)

	// Configure logging
	logformat := log.Ldate | log.Ltime | log.Lmicroseconds
	var logger *log.Logger
	if len(opts.Debug) > 0 {
		if len(opts.Debug) > 1 {
			logformat = logformat | log.Lshortfile
		}
		logger = log.New(os.Stdout, "INFO:  ", logformat)
		sfc.SetDebugLogger(log.New(os.Stdout, "DEBUG: ", logformat))
		sfc.SetInfoLogger(logger)
	} else {
		logger = log.New(os.Stdout, "", logformat)
	}
	sfc.SetWarningLogger(log.New(os.Stdout, "WARN:  ", logformat))
	sfc.SetErrorLogger(log.New(os.Stdout, "ERROR: ", logformat))

	// Connect to the host
	sfc.Autoconnect(host, port, time.Duration(opts.Reconnect)*time.Second)

	// Set up signals to close nicely on Control+C
	signals := make(chan os.Signal)
	signal.Notify(signals, os.Interrupt, os.Kill)

	for interrupted := false; interrupted == false; {
		select {
		case msg := <-receive:
			logger.Printf("%s\n", msg)
		case sig := <-signals:
			signal.Stop(signals)
			logger.Printf("signal %s\n", sig)
			sfc.Disconnect()
			interrupted = true
		}
	}
}
<commit_msg>Help printed only once if requested and exit code is 0.<commit_after>// Author  Raido Pahtma
// License MIT

package main

import "fmt"
import "os"
import "os/signal"
import "log"
import "time"

import "github.com/jessevdk/go-flags"
import "github.com/proactivity-lab/go-sfconnection"

const ApplicationVersionMajor = 0
const ApplicationVersionMinor = 1
const ApplicationVersionPatch = 2

var ApplicationBuildDate string
var ApplicationBuildDistro string

func main() {

	var opts struct {
		Positional struct {
			ConnectionString string `description:"Connectionstring sf@HOST:PORT"`
		} `positional-args:"yes"`

		Reconnect uint `long:"reconnect" default:"30" description:"Reconnect period, seconds"`

		Debug       []bool `short:"D" long:"debug" description:"Debug mode, print raw packets"`
		ShowVersion func() `short:"V" long:"version" description:"Show application version"`
	}

	opts.ShowVersion = func() {
		if ApplicationBuildDate == "" {
			ApplicationBuildDate = "YYYY-mm-dd_HH:MM:SS"
		}
		if ApplicationBuildDistro == "" {
			ApplicationBuildDistro = "unknown"
		}
		fmt.Printf("amlistener %d.%d.%d (%s %s)\n", ApplicationVersionMajor, ApplicationVersionMinor, ApplicationVersionPatch, ApplicationBuildDate, ApplicationBuildDistro)
		os.Exit(0)
	}

	_, err := flags.Parse(&opts)
	if err != nil {
		flagserr := err.(*flags.Error)
		if flagserr.Type != flags.ErrHelp {
			if len(opts.Debug) > 0 {
				fmt.Printf("Argument parser error: %s\n", err)
			}
			os.Exit(1)
		}
		os.Exit(0)
	}

	host, port, err := sfconnection.ParseSfConnectionString(opts.Positional.ConnectionString)
	if err != nil {
		fmt.Printf("ERROR: %s\n", err)
		os.Exit(1)
	}

	dsp := sfconnection.NewMessageDispatcher(sfconnection.NewMessage(0, 0))
	receive := make(chan *sfconnection.Message)
	dsp.RegisterMessageSnooper(receive)

	sfc := sfconnection.NewSfConnection()
	sfc.AddDispatcher(dsp)

	// Configure logging
	logformat := log.Ldate | log.Ltime | log.Lmicroseconds
	var logger *log.Logger
	if len(opts.Debug) > 0 {
		if len(opts.Debug) > 1 {
			logformat = logformat | log.Lshortfile
		}
		logger = log.New(os.Stdout, "INFO:  ", logformat)
		sfc.SetDebugLogger(log.New(os.Stdout, "DEBUG: ", logformat))
		sfc.SetInfoLogger(logger)
	} else {
		logger = log.New(os.Stdout, "", logformat)
	}
	sfc.SetWarningLogger(log.New(os.Stdout, "WARN:  ", logformat))
	sfc.SetErrorLogger(log.New(os.Stdout, "ERROR: ", logformat))

	// Connect to the host
	sfc.Autoconnect(host, port, time.Duration(opts.Reconnect)*time.Second)

	// Set up signals to close nicely on Control+C
	signals := make(chan os.Signal)
	signal.Notify(signals, os.Interrupt, os.Kill)

	for interrupted := false; interrupted == false; {
		select {
		case msg := <-receive:
			logger.Printf("%s\n", msg)
		case sig := <-signals:
			signal.Stop(signals)
			logger.Printf("signal %s\n", sig)
			sfc.Disconnect()
			interrupted = true
		}
	}
}
<|endoftext|>


<commit_before>package immortal

import (
	"os/exec"
	"testing"
)

func TestWatchPid0(t *testing.T) {
	D := &Daemon{}
	ch := make(chan error, 1)
	D.watchPid(0, ch)
	err := <-ch
	if err != nil {
		if err.Error() != "PID NOT FOUND" {
			t.Error(err)
		}
	}
}

func TestWatchPidGetpid(t *testing.T) {
	D := &Daemon{}
	ch := make(chan error)

	cmd := exec.Command("go", "version")
	cmd.Start()
	pid := cmd.Process.Pid
	go func() {
		D.watchPid(pid, ch)
		ch <- cmd.Wait()
	}()

	select {
	case err := <-ch:
		if err != nil {
			if err.Error() != "EXIT" {
				t.Error(err)
			}
		}
	}
}

func TestWatchPidGetpidKill(t *testing.T) {
	D := &Daemon{}
	ch := make(chan error)

	cmd := exec.Command("sleep", "100")
	cmd.Start()
	pid := cmd.Process.Pid
	go func() {
		D.watchPid(pid, ch)
		ch <- cmd.Wait()
	}()

	if err := cmd.Process.Kill(); err != nil {
		t.Errorf("failed to kill: %s", err)
	}
	select {
	case err := <-ch:
		if err != nil {
			if err.Error() != "EXIT" {
				t.Error(err)
			}
		}
	}
}
<commit_msg>	modified:   watchpid_test.go<commit_after>package immortal

import (
	"os/exec"
	"testing"
	"time"
)

func TestWatchPid0(t *testing.T) {
	D := &Daemon{}
	ch := make(chan error, 1)
	D.watchPid(0, ch)
	err := <-ch
	if err != nil {
		if err.Error() != "PID NOT FOUND" {
			t.Error(err)
		}
	}
}

func TestWatchPidGetpid(t *testing.T) {
	D := &Daemon{}
	ch := make(chan error)

	cmd := exec.Command("go", "version")
	cmd.Start()
	pid := cmd.Process.Pid
	go func() {
		D.watchPid(pid, ch)
		ch <- cmd.Wait()
	}()

	select {
	case err := <-ch:
		if err != nil {
			if err.Error() != "EXIT" {
				t.Error(err)
			}
		}
	}
}

func TestWatchPidGetpidKill(t *testing.T) {
	D := &Daemon{}
	ch := make(chan error)

	cmd := exec.Command("sleep", "100")
	cmd.Start()
	pid := cmd.Process.Pid
	go func() {
		D.watchPid(pid, ch)
		ch <- cmd.Wait()
	}()

	select {
	case err := <-ch:
		if err != nil {
			if err.Error() != "EXIT" {
				t.Error(err)
			}
		}
	case <-time.After(1 * time.Millisecond):
		if err := cmd.Process.Kill(); err != nil {
			t.Errorf("failed to kill: %s", err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/github/codeql-go/extractor/dbscheme"

	"github.com/github/codeql-go/extractor"
)

func usage() {
	fmt.Fprintf(os.Stderr, "%s is a program for building a snapshot of a Go code base.\n\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "Usage:\n\n  %s [<flag>...] [<buildflag>...] [--] <file>...\n\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "Flags:\n\n")
	fmt.Fprintf(os.Stderr, "--help                Print this help.\n")
	fmt.Fprintf(os.Stderr, "--dbscheme string     Write dbscheme to this file.\n")
}

func parseFlags(args []string) ([]string, []string, string) {
	i := 0
	var dumpDbscheme string
	buildFlags := []string{}
	for i < len(args) && strings.HasPrefix(args[i], "-") {
		if args[i] == "--" {
			i += 1
			break
		}

		if strings.HasPrefix(args[i], "--dbscheme=") {
			dumpDbscheme = strings.TrimPrefix(args[i], "--dbscheme=")
		} else if args[i] == "--dbscheme" {
			i += 1
			dumpDbscheme = args[i]
		} else if args[i] == "--help" {
			usage()
			os.Exit(0)
		} else {
			buildFlags = append(buildFlags, args[i])
		}

		i += 1
	}

	return buildFlags, args[i:], dumpDbscheme
}

func main() {
	buildFlags, patterns, dumpDbscheme := parseFlags(os.Args[1:])

	if dumpDbscheme != "" {
		f, err := os.Create(dumpDbscheme)
		if err != nil {
			log.Fatalf("Unable to open file %s for writing.", dumpDbscheme)
		}
		dbscheme.PrintDbScheme(f)
		f.Close()
		log.Printf("Dbscheme written to file %s.", dumpDbscheme)
	}

	if len(patterns) == 0 {
		log.Println("Nothing to extract.")
	} else {
		err := extractor.ExtractWithFlags(buildFlags, patterns)
		if err != nil {
			log.Fatal(err)
		}
	}
}
<commit_msg>Add environment variables for dumping CPU and memory profiles.<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"strings"

	"github.com/github/codeql-go/extractor/dbscheme"

	"github.com/github/codeql-go/extractor"
)

var cpuprofile, memprofile string

func usage() {
	fmt.Fprintf(os.Stderr, "%s is a program for building a snapshot of a Go code base.\n\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "Usage:\n\n  %s [<flag>...] [<buildflag>...] [--] <file>...\n\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "Flags:\n\n")
	fmt.Fprintf(os.Stderr, "--help                Print this help.\n")
	fmt.Fprintf(os.Stderr, "--dbscheme string     Write dbscheme to this file.\n")
}

func parseFlags(args []string) ([]string, []string, string) {
	i := 0
	var dumpDbscheme string
	buildFlags := []string{}
	for i < len(args) && strings.HasPrefix(args[i], "-") {
		if args[i] == "--" {
			i++
			break
		}

		if strings.HasPrefix(args[i], "--dbscheme=") {
			dumpDbscheme = strings.TrimPrefix(args[i], "--dbscheme=")
		} else if args[i] == "--dbscheme" {
			i++
			dumpDbscheme = args[i]
		} else if args[i] == "--help" {
			usage()
			os.Exit(0)
		} else {
			buildFlags = append(buildFlags, args[i])
		}

		i++
	}

	cpuprofile = os.Getenv("CODEQL_EXTRACTOR_GO_CPU_PROFILE")
	memprofile = os.Getenv("CODEQL_EXTRACTOR_GO_MEM_PROFILE")

	return buildFlags, args[i:], dumpDbscheme
}

func main() {
	buildFlags, patterns, dumpDbscheme := parseFlags(os.Args[1:])

	if dumpDbscheme != "" {
		f, err := os.Create(dumpDbscheme)
		if err != nil {
			log.Fatalf("Unable to open file %s for writing.", dumpDbscheme)
		}
		dbscheme.PrintDbScheme(f)
		f.Close()
		log.Printf("Dbscheme written to file %s.", dumpDbscheme)
	}

	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatalf("Unable to create CPU profile: %v.", err)
		}
		defer f.Close()
		if err := pprof.StartCPUProfile(f); err != nil {
			log.Fatalf("Unable to start CPU profile: %v.", err)
		}
		defer pprof.StopCPUProfile()
	}

	if len(patterns) == 0 {
		log.Println("Nothing to extract.")
	} else {
		err := extractor.ExtractWithFlags(buildFlags, patterns)
		if err != nil {
			log.Fatal(err)
		}
	}

	if memprofile != "" {
		f, err := os.Create(memprofile)
		if err != nil {
			log.Fatalf("Unable to create memory profile: %v", err)
		}
		defer f.Close()
		runtime.GC() // get up-to-date statistics
		if err := pprof.WriteHeapProfile(f); err != nil {
			log.Fatal("Unable to write memory profile: ", err)
		}
	}
}
<|endoftext|>