<commit_before>package setup

import (
	"crypto/md5"
	"encoding/hex"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Setup database user and groups", func() {

	var (
		dbConnectionInfo = ConnectionDetails{
			Username: "sebastian",
			Database: "sebastian"}
		sampleConnString = "user=sebastian dbname=sebastian"

		testUserName  = "dbview"
		wrongUserName = "missing_user_for_this_database"

		createTempDBName = func() string {
			hasher := md5.New()
			hasher.Write([]byte(time.Now().Local().Format(time.UnixDate)))
			return "temp_" + hex.EncodeToString(hasher.Sum(nil))
		}
	)

	Context("When I connect to the database", func() {

		It("Should convert a the connection info to the lib/pq connection", func() {
			Expect(dbConnectionInfo.toString()).To(Equal(sampleConnString))
		})

		It("Should not set a value in the connection string when it is empty", func() {
			Expect(ConnectionDetails{Username: "sebastian", Host: ""}.toString()).To(Equal("user=sebastian"))
		})

		It("Should connect to a database", func() {
			dbConnectionInfo.SslMode = "disable"
			_, err := connect(dbConnectionInfo)

			Expect(err).To(BeNil())
		})

	})

	Context("When I Create a user at the database", func() {

		It("It check if user exists before try to create a new one", func() {
			_, err := checkIfUserExists(dbConnectionInfo, testUserName)
			Expect(err).To(BeNil())
		})

		It("Should create a user in the database", func() {
			err := CreateUser(dbConnectionInfo, testUserName, []string{"PASSWORD 'super_senha'", "SUPERUSER"})
			Expect(err).To(BeNil())
		})

		It("Should grant some roles to a new user", func() {
			err := GrantRolesToUser(dbConnectionInfo, "sebastian", []string{"dbview"})
			Expect(err).To(BeNil())
		})

		It("Should check if user exists before grant a role", func() {
			err := GrantRolesToUser(dbConnectionInfo, wrongUserName, []string{"dbview"})
			Expect(err).To(BeNil())
		})

		It("Should be possible to update the 'search_path' configuration for the user", func() {
			err := SetSearchPathForUser(dbConnectionInfo, "sebastian", []string{"dbview", "public"})
			Expect(err).To(BeNil())
		})

		It("Should check if user exists before update the 'search_path'", func() {
			err := SetSearchPathForUser(dbConnectionInfo, wrongUserName, []string{"dbview", "public"})
			Expect(err).To(BeNil())
		})

		It("Should drop a user if it exists", func() {

			err := DropUser(dbConnectionInfo, wrongUserName)
			Expect(err).To(BeNil())
		})

	})

	Context("When I create a database", func() {
		It("Should create a new database", func() {
			err := CreateNewDatabase(dbConnectionInfo, "dbview", nil)
			Expect(err).To(BeNil())
		})

		It("Should drop a database", func() {
			err := DropDatabase(dbConnectionInfo, "dbview")
			Expect(err).To(BeNil())
		})

		It("Should check if the database exists before create a new one", func() {
			_, err := checkIfDatabaseExists(dbConnectionInfo, "template1")
			Expect(err).To(BeNil())
		})

		It("Should create a some extensions", func() {
			err := CreateExtensionsInDatabase(dbConnectionInfo, []string{"plpgsql"})
			Expect(err).To(BeNil())
		})

		It("Should check if a schema exists", func() {
			_, err := CheckIfSchemaExists(dbConnectionInfo, "public")
			Expect(err).To(BeNil())
		})

		It("Should remove a schema and all of this contents", func() {
			err := RemoveSchema(dbConnectionInfo, "u1325")
			Expect(err).To(BeNil())
		})

		It("Should create a new schema", func() {
			err := CreateSchema(dbConnectionInfo, "public")
			Expect(err).To(BeNil())
		})
	})

	Context("When I restore a database", func() {

		It("Should restore a dump file", func() {

			// dbConnectionInfo.Database = "sebastian"

			tempDBName := createTempDBName()
			var err error
			err = CreateNewDatabase(dbConnectionInfo, tempDBName, nil)

			dbConnectionInfo.Database = tempDBName

			options := RestoreOptions{ExePath: "/usr/local/bin/pg_restore", CustomArgs: []string{"-Fc"}}
			err = RestoreDumpFile(dbConnectionInfo, "/Users/sebastian/tmp/file.dump", options)
			Expect(err).To(BeNil())

			dbConnectionInfo.Database = "postgres"
			err = DropDatabase(dbConnectionInfo, tempDBName)
			Expect(err).To(BeNil())
		})
	})

	Context("When I connect to the database", func() {

		It("Should run a query", func() {
			err := ExecuteQuery(dbConnectionInfo, "select 1;")
			Expect(err).To(BeNil())
		})

		It("Should create the replication function", func() {

			tempDBName := createTempDBName()
			var err error

			// step 1: create a new database
			err = CreateNewDatabase(dbConnectionInfo, tempDBName, nil)
			Expect(err).To(BeNil())

			// step 2: import the functions
			err = ExecuteQuery(dbConnectionInfo, ReplicationLogFunction)
			Expect(err).To(BeNil())

			// step 3: drop the new database
			dbConnectionInfo.Database = "postgres"
			err = DropDatabase(dbConnectionInfo, tempDBName)
			Expect(err).To(BeNil())
		})
	})
})
<commit_msg>doing tests without ssl by default<commit_after>package setup

import (
	"crypto/md5"
	"encoding/hex"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Setup database user and groups", func() {

	var (
		dbConnectionInfo = ConnectionDetails{
			Username: "sebastian",
			Database: "sebastian",
			SslMode:  "disable"}
		sampleConnString = "user=sebastian dbname=sebastian sslmode=disable"

		testUserName  = "dbview"
		wrongUserName = "missing_user_for_this_database"

		createTempDBName = func() string {
			hasher := md5.New()
			hasher.Write([]byte(time.Now().Local().Format(time.UnixDate)))
			return "temp_" + hex.EncodeToString(hasher.Sum(nil))
		}
	)

	Context("When I connect to the database", func() {

		It("Should convert a the connection info to the lib/pq connection", func() {
			Expect(dbConnectionInfo.toString()).To(Equal(sampleConnString))
		})

		It("Should not set a value in the connection string when it is empty", func() {
			Expect(ConnectionDetails{Username: "sebastian", Host: ""}.toString()).To(Equal("user=sebastian"))
		})

		It("Should connect to a database", func() {
			_, err := connect(dbConnectionInfo)

			Expect(err).To(BeNil())
		})

	})

	Context("When I Create a user at the database", func() {

		It("It check if user exists before try to create a new one", func() {
			_, err := checkIfUserExists(dbConnectionInfo, testUserName)
			Expect(err).To(BeNil())
		})

		It("Should create a user in the database", func() {
			err := CreateUser(dbConnectionInfo, testUserName, []string{"PASSWORD 'super_senha'", "SUPERUSER"})
			Expect(err).To(BeNil())
		})

		It("Should grant some roles to a new user", func() {
			err := GrantRolesToUser(dbConnectionInfo, "sebastian", []string{"dbview"})
			Expect(err).To(BeNil())
		})

		It("Should check if user exists before grant a role", func() {
			err := GrantRolesToUser(dbConnectionInfo, wrongUserName, []string{"dbview"})
			Expect(err).To(BeNil())
		})

		It("Should be possible to update the 'search_path' configuration for the user", func() {
			err := SetSearchPathForUser(dbConnectionInfo, "sebastian", []string{"dbview", "public"})
			Expect(err).To(BeNil())
		})

		It("Should check if user exists before update the 'search_path'", func() {
			err := SetSearchPathForUser(dbConnectionInfo, wrongUserName, []string{"dbview", "public"})
			Expect(err).To(BeNil())
		})

		It("Should drop a user if it exists", func() {

			err := DropUser(dbConnectionInfo, wrongUserName)
			Expect(err).To(BeNil())
		})

	})

	Context("When I create a database", func() {
		It("Should create a new database", func() {
			err := CreateNewDatabase(dbConnectionInfo, "dbview", nil)
			Expect(err).To(BeNil())
		})

		It("Should drop a database", func() {
			err := DropDatabase(dbConnectionInfo, "dbview")
			Expect(err).To(BeNil())
		})

		It("Should check if the database exists before create a new one", func() {
			_, err := checkIfDatabaseExists(dbConnectionInfo, "template1")
			Expect(err).To(BeNil())
		})

		It("Should create a some extensions", func() {
			err := CreateExtensionsInDatabase(dbConnectionInfo, []string{"plpgsql"})
			Expect(err).To(BeNil())
		})

		It("Should check if a schema exists", func() {
			_, err := CheckIfSchemaExists(dbConnectionInfo, "public")
			Expect(err).To(BeNil())
		})

		It("Should remove a schema and all of this contents", func() {
			err := RemoveSchema(dbConnectionInfo, "u1325")
			Expect(err).To(BeNil())
		})

		It("Should create a new schema", func() {
			err := CreateSchema(dbConnectionInfo, "public")
			Expect(err).To(BeNil())
		})
	})

	Context("When I restore a database", func() {

		It("Should restore a dump file", func() {

			// dbConnectionInfo.Database = "sebastian"

			//tempDBName := "teste_capiroto"
			tempDBName := createTempDBName()
			var err error
			err = CreateNewDatabase(dbConnectionInfo, tempDBName, nil)

			newConn := dbConnectionInfo
			newConn.Database = tempDBName

			err = CreateExtensionsInDatabase(newConn, []string{"hstore", "dblink", "pg_freespacemap", "postgis", "tablefunc", "unaccent"})
			Expect(err).To(BeNil())

			options := RestoreOptions{
				ExePath:    "/usr/local/bin/pg_restore",
				CustomArgs: []string{"-Fc"}}
			err = RestoreDumpFile(newConn, "/Users/sebastian/tmp/file.dump", options)
			Expect(err).To(BeNil())

			err = DropDatabase(dbConnectionInfo, tempDBName)
			Expect(err).To(BeNil())
		})
	})

	Context("When I connect to the database", func() {

		It("Should run a query", func() {
			err := ExecuteQuery(dbConnectionInfo, "select 1;")
			Expect(err).To(BeNil())
		})

		It("Should create the replication function", func() {

			tempDBName := createTempDBName()
			var err error

			// step 1: create a new database
			err = CreateNewDatabase(dbConnectionInfo, tempDBName, nil)
			Expect(err).To(BeNil())

			// step 2: import the functions
			err = ExecuteQuery(dbConnectionInfo, ReplicationLogFunction)
			Expect(err).To(BeNil())

			// step 3: drop the new database
			dbConnectionInfo.Database = "postgres"
			err = DropDatabase(dbConnectionInfo, tempDBName)
			Expect(err).To(BeNil())
		})
	})
})
<|endoftext|>


<commit_before>// Copyright (c) 2012, Wei guangjing <vcc.163@gmail.com>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package driver

import (
	"database/sql"
	"database/sql/driver"
	"errors"
	"io"
	"odbc"
)

func init() {
	d := &Driver{}
	sql.Register("odbc", d)
}

type Driver struct {
}

func (d *Driver) Open(dsn string) (driver.Conn, error) {
	c, err := odbc.Connect(dsn)
	if err != nil {
		return nil, err
	}
	conn := &conn{c: c}
	return conn, nil
}

func (d *Driver) Close() error {
	return nil
}

type conn struct {
	c *odbc.Connection
	t *tx
}

func (c *conn) Prepare(query string) (driver.Stmt, error) {
	st, err := c.c.Prepare(query)
	if err != nil {
		return nil, err
	}

	stmt := &stmt{st: st}
	return stmt, nil
}

func (c *conn) Begin() (driver.Tx, error) {
	if err := c.c.AutoCommit(false); err != nil {
		return nil, err
	}

	return &tx{c: c}, nil
}

func (c *conn) Close() error {
	if c.c != nil {
		return c.c.Close()
	}
	return nil
}

type tx struct {
	c *conn
}

func (t *tx) Commit() error {
	err := t.c.c.Commit()
	return err
}

func (t *tx) Rollback() error {
	err := t.c.c.Rollback()
	return err
}

type stmt struct {
	st *odbc.Statement
}

func (s *stmt) Exec(args []driver.Value) (driver.Result, error) {
	if err := s.st.Execute2(args); err != nil {
		return nil, err
	}

	rowsAffected, err := s.st.RowsAffected()
	r := &result{rowsAffected: int64(rowsAffected)}
	return r, err
}

func (s *stmt) NumInput() int {
	return s.st.NumParams()
}

func (s *stmt) Query(args []driver.Value) (driver.Rows, error) {
	if err := s.st.Execute2(args); err != nil {
		return nil, err
	}
	rows := &rows{s: s}
	return rows, nil
}

func (s *stmt) Close() error {
	s.st.Close()
	return nil
}

type result struct {
	rowsAffected int64
}

func (r *result) LastInsertId() (int64, error) {
	return 0, errors.New("not supported")
}

func (r *result) RowsAffected() (int64, error) {
	return r.rowsAffected, nil
}

type rows struct {
	s *stmt
}

func (r *rows) Columns() []string {
	c, err := r.s.st.NumFields()
	if err != nil {
		return nil
	}
	columns := make([]string, c)
	for i, _ := range columns {
		f, err := r.s.st.FieldMetadata(i + 1)
		if err != nil {
			return nil
		}
		columns[i] = f.Name
	}
	return columns
}

func (r *rows) Close() error {
	return r.s.Close()
}

func (r *rows) Next(dest []driver.Value) error {
	eof, err := r.s.st.FetchOne2(dest)
	if err != nil {
		return err
	}
	if eof {
		return io.EOF
	}
	return nil
}
<commit_msg>fix import path<commit_after>// Copyright (c) 2012, Wei guangjing <vcc.163@gmail.com>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package driver

import (
	"database/sql"
	"database/sql/driver"
	"errors"
	"github.com/weisd/go-odbc"
	"io"
)

func init() {
	d := &Driver{}
	sql.Register("odbc", d)
}

type Driver struct {
}

func (d *Driver) Open(dsn string) (driver.Conn, error) {
	c, err := odbc.Connect(dsn)
	if err != nil {
		return nil, err
	}
	conn := &conn{c: c}
	return conn, nil
}

func (d *Driver) Close() error {
	return nil
}

type conn struct {
	c *odbc.Connection
	t *tx
}

func (c *conn) Prepare(query string) (driver.Stmt, error) {
	st, err := c.c.Prepare(query)
	if err != nil {
		return nil, err
	}

	stmt := &stmt{st: st}
	return stmt, nil
}

func (c *conn) Begin() (driver.Tx, error) {
	if err := c.c.AutoCommit(false); err != nil {
		return nil, err
	}

	return &tx{c: c}, nil
}

func (c *conn) Close() error {
	if c.c != nil {
		return c.c.Close()
	}
	return nil
}

type tx struct {
	c *conn
}

func (t *tx) Commit() error {
	err := t.c.c.Commit()
	return err
}

func (t *tx) Rollback() error {
	err := t.c.c.Rollback()
	return err
}

type stmt struct {
	st *odbc.Statement
}

func (s *stmt) Exec(args []driver.Value) (driver.Result, error) {
	if err := s.st.Execute2(args); err != nil {
		return nil, err
	}

	rowsAffected, err := s.st.RowsAffected()
	r := &result{rowsAffected: int64(rowsAffected)}
	return r, err
}

func (s *stmt) NumInput() int {
	return s.st.NumParams()
}

func (s *stmt) Query(args []driver.Value) (driver.Rows, error) {
	if err := s.st.Execute2(args); err != nil {
		return nil, err
	}
	rows := &rows{s: s}
	return rows, nil
}

func (s *stmt) Close() error {
	s.st.Close()
	return nil
}

type result struct {
	rowsAffected int64
}

func (r *result) LastInsertId() (int64, error) {
	return 0, errors.New("not supported")
}

func (r *result) RowsAffected() (int64, error) {
	return r.rowsAffected, nil
}

type rows struct {
	s *stmt
}

func (r *rows) Columns() []string {
	c, err := r.s.st.NumFields()
	if err != nil {
		return nil
	}
	columns := make([]string, c)
	for i, _ := range columns {
		f, err := r.s.st.FieldMetadata(i + 1)
		if err != nil {
			return nil
		}
		columns[i] = f.Name
	}
	return columns
}

func (r *rows) Close() error {
	return r.s.Close()
}

func (r *rows) Next(dest []driver.Value) error {
	eof, err := r.s.st.FetchOne2(dest)
	if err != nil {
		return err
	}
	if eof {
		return io.EOF
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017 Zack Guo <zack.y.guo@gmail.com>. All rights reserved.
// Use of this source code is governed by a MIT license that can
// be found in the LICENSE file.

// +build ignore

package main

import (
	"log"

	ui "github.com/gizak/termui"
	"github.com/gizak/termui/widgets"
)

func main() {
	if err := ui.Init(); err != nil {
		log.Fatalf("failed to initialize termui: %v", err)
	}
	defer ui.Close()

	l := widgets.NewList()
	l.Title = "List"
	l.Rows = []string{
		"[0] github.com/gizak/termui",
		"[1] [你好，世界](fg:blue)",
		"[2] [こんにちは世界](fg:red)",
		"[3] [color](fg:white,bg:green) output",
		"[4] output.go",
		"[5] random_out.go",
		"[6] dashboard.go",
		"[7] nsf/termbox-go",
	}
	l.TextStyle = ui.NewStyle(ui.ColorYellow)
	l.WrapText = false
	l.SetRect(0, 0, 25, 5)

	ui.Render(l)

	uiEvents := ui.PollEvents()
	for {
		e := <-uiEvents
		switch e.ID {
		case "q", "<C-c>":
			return
		case "j", "<Down>":
			l.ScrollDown()
			ui.Render(l)
		case "k", "<Up>":
			l.ScrollUp()
			ui.Render(l)
		}
	}
}
<commit_msg>Add list page scrolling to list example<commit_after>// Copyright 2017 Zack Guo <zack.y.guo@gmail.com>. All rights reserved.
// Use of this source code is governed by a MIT license that can
// be found in the LICENSE file.

// +build ignore

package main

import (
	"log"

	ui "github.com/gizak/termui"
	"github.com/gizak/termui/widgets"
)

func main() {
	if err := ui.Init(); err != nil {
		log.Fatalf("failed to initialize termui: %v", err)
	}
	defer ui.Close()

	l := widgets.NewList()
	l.Title = "List"
	l.Rows = []string{
		"[0] github.com/gizak/termui",
		"[1] [你好，世界](fg:blue)",
		"[2] [こんにちは世界](fg:red)",
		"[3] [color](fg:white,bg:green) output",
		"[4] output.go",
		"[5] random_out.go",
		"[6] dashboard.go",
		"[7] nsf/termbox-go",
	}
	l.TextStyle = ui.NewStyle(ui.ColorYellow)
	l.WrapText = false
	l.SetRect(0, 0, 25, 5)

	ui.Render(l)

	uiEvents := ui.PollEvents()
	for {
		e := <-uiEvents
		switch e.ID {
		case "q", "<C-c>":
			return
		case "j", "<Down>":
			l.ScrollDown()
			ui.Render(l)
		case "k", "<Up>":
			l.ScrollUp()
			ui.Render(l)
		case "<C-f>":
			l.PageDown()
			ui.Render(l)
		case "<C-b>":
			l.PageUp()
			ui.Render(l)
		}
	}
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"os"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
)

type ServerOptions struct {
	cpuprofile *string
	v          VolumeServerOptions
}

var (
	serverOptions    ServerOptions
	masterOptions    MasterOptions
	filerOptions     FilerOptions
	s3Options        S3Options
	msgBrokerOptions MessageBrokerOptions
)

func init() {
	cmdServer.Run = runServer // break init cycle
}

var cmdServer = &Command{
	UsageLine: "server -dir=/tmp -volume.max=5 -ip=server_name",
	Short:     "start a master server, a volume server, and optionally a filer and a S3 gateway",
	Long: `start both a volume server to provide storage spaces
  and a master server to provide volume=>location mapping service and sequence number of file ids

  This is provided as a convenient way to start both volume server and master server.
  The servers acts exactly the same as starting them separately.
  So other volume servers can connect to this master server also.

  Optionally, a filer server can be started.
  Also optionally, a S3 gateway can be started.

  `,
}

var (
	serverIp                  = cmdServer.Flag.String("ip", util.DetectedHostAddress(), "ip or server name")
	serverBindIp              = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
	serverTimeout             = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
	serverDataCenter          = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
	serverRack                = cmdServer.Flag.String("rack", "", "current volume server's rack name")
	serverWhiteListOption     = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
	serverDisableHttp         = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
	volumeDataFolders         = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
	volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured.")
	volumeMinFreeSpacePercent = cmdServer.Flag.String("volume.minFreeSpacePercent", "0", "minimum free disk space(in percents). Low disk space will mark all volumes as ReadOnly.")

	// pulseSeconds              = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
	isStartingFiler           = cmdServer.Flag.Bool("filer", false, "whether to start filer")
	isStartingS3              = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway")
	isStartingMsgBroker       = cmdServer.Flag.Bool("msgBroker", false, "whether to start message broker")

	serverWhiteList []string

	False = false
)

func init() {
	serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")

	masterOptions.port = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
	masterOptions.metaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
	masterOptions.peers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
	masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
	masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
	masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "000", "Default replication type if not specified.")
	masterOptions.garbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
	masterOptions.metricsAddress = cmdServer.Flag.String("metrics.address", "", "Prometheus gateway address")
	masterOptions.metricsIntervalSec = cmdServer.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")

	filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
	filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
	filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit")
	filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size")
	filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers")

	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
	serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
	serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
	serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
	serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
	serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory")
	serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
	serverOptions.v.pprof = &False

	s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
	s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name, {bucket}.{domainName}")
	s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
	s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
	s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file")

	msgBrokerOptions.port = cmdServer.Flag.Int("msgBroker.port", 17777, "broker gRPC listen port")

}

func runServer(cmd *Command, args []string) bool {

	util.LoadConfiguration("security", false)
	util.LoadConfiguration("master", false)

	if *serverOptions.cpuprofile != "" {
		f, err := os.Create(*serverOptions.cpuprofile)
		if err != nil {
			glog.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *isStartingS3 {
		*isStartingFiler = true
	}
	if *isStartingMsgBroker {
		*isStartingFiler = true
	}

	_, peerList := checkPeers(*serverIp, *masterOptions.port, *masterOptions.peers)
	peers := strings.Join(peerList, ",")
	masterOptions.peers = &peers

	masterOptions.ip = serverIp
	masterOptions.ipBind = serverBindIp
	filerOptions.masters = &peers
	filerOptions.ip = serverIp
	filerOptions.bindIp = serverBindIp
	serverOptions.v.ip = serverIp
	serverOptions.v.bindIp = serverBindIp
	serverOptions.v.masters = &peers
	serverOptions.v.idleConnectionTimeout = serverTimeout
	serverOptions.v.dataCenter = serverDataCenter
	serverOptions.v.rack = serverRack
	msgBrokerOptions.ip = serverIp

	// serverOptions.v.pulseSeconds = pulseSeconds
	// masterOptions.pulseSeconds = pulseSeconds

	masterOptions.whiteList = serverWhiteListOption

	filerOptions.dataCenter = serverDataCenter
	filerOptions.disableHttp = serverDisableHttp
	masterOptions.disableHttp = serverDisableHttp

	filerAddress := fmt.Sprintf("%s:%d", *serverIp, *filerOptions.port)
	s3Options.filer = &filerAddress
	msgBrokerOptions.filer = &filerAddress

	if *filerOptions.defaultReplicaPlacement == "" {
		*filerOptions.defaultReplicaPlacement = *masterOptions.defaultReplication
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	folders := strings.Split(*volumeDataFolders, ",")

	if *masterOptions.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
		glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
	}

	if *masterOptions.metaFolder == "" {
		*masterOptions.metaFolder = folders[0]
	}
	if err := util.TestFolderWritable(*masterOptions.metaFolder); err != nil {
		glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterOptions.metaFolder, err)
	}
	filerOptions.defaultLevelDbDirectory = masterOptions.metaFolder

	if *serverWhiteListOption != "" {
		serverWhiteList = strings.Split(*serverWhiteListOption, ",")
	}

	if *isStartingFiler {
		go func() {
			time.Sleep(1 * time.Second)

			filerOptions.startFiler()

		}()
	}

	if *isStartingS3 {
		go func() {
			time.Sleep(2 * time.Second)

			s3Options.startS3Server()

		}()
	}

	if *isStartingMsgBroker {
		go func() {
			time.Sleep(2 * time.Second)
			msgBrokerOptions.startQueueServer()
		}()
	}

	// start volume server
	{
		go serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption, *volumeMinFreeSpacePercent)

	}

	startMaster(masterOptions, serverWhiteList)

	return true
}
<commit_msg>default to 1%<commit_after>package command

import (
	"fmt"
	"os"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
)

type ServerOptions struct {
	cpuprofile *string
	v          VolumeServerOptions
}

var (
	serverOptions    ServerOptions
	masterOptions    MasterOptions
	filerOptions     FilerOptions
	s3Options        S3Options
	msgBrokerOptions MessageBrokerOptions
)

func init() {
	cmdServer.Run = runServer // break init cycle
}

var cmdServer = &Command{
	UsageLine: "server -dir=/tmp -volume.max=5 -ip=server_name",
	Short:     "start a master server, a volume server, and optionally a filer and a S3 gateway",
	Long: `start both a volume server to provide storage spaces
  and a master server to provide volume=>location mapping service and sequence number of file ids

  This is provided as a convenient way to start both volume server and master server.
  The servers acts exactly the same as starting them separately.
  So other volume servers can connect to this master server also.

  Optionally, a filer server can be started.
  Also optionally, a S3 gateway can be started.

  `,
}

var (
	serverIp                  = cmdServer.Flag.String("ip", util.DetectedHostAddress(), "ip or server name")
	serverBindIp              = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
	serverTimeout             = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
	serverDataCenter          = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
	serverRack                = cmdServer.Flag.String("rack", "", "current volume server's rack name")
	serverWhiteListOption     = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
	serverDisableHttp         = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
	volumeDataFolders         = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
	volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured.")
	volumeMinFreeSpacePercent = cmdServer.Flag.String("volume.minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly.")

	// pulseSeconds              = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
	isStartingFiler           = cmdServer.Flag.Bool("filer", false, "whether to start filer")
	isStartingS3              = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway")
	isStartingMsgBroker       = cmdServer.Flag.Bool("msgBroker", false, "whether to start message broker")

	serverWhiteList []string

	False = false
)

func init() {
	serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")

	masterOptions.port = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
	masterOptions.metaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
	masterOptions.peers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
	masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
	masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
	masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "000", "Default replication type if not specified.")
	masterOptions.garbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
	masterOptions.metricsAddress = cmdServer.Flag.String("metrics.address", "", "Prometheus gateway address")
	masterOptions.metricsIntervalSec = cmdServer.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")

	filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
	filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
	filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit")
	filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size")
	filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers")

	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
	serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
	serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
	serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
	serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
	serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory")
	serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
	serverOptions.v.pprof = &False

	s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
	s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name, {bucket}.{domainName}")
	s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
	s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
	s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file")

	msgBrokerOptions.port = cmdServer.Flag.Int("msgBroker.port", 17777, "broker gRPC listen port")

}

func runServer(cmd *Command, args []string) bool {

	util.LoadConfiguration("security", false)
	util.LoadConfiguration("master", false)

	if *serverOptions.cpuprofile != "" {
		f, err := os.Create(*serverOptions.cpuprofile)
		if err != nil {
			glog.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *isStartingS3 {
		*isStartingFiler = true
	}
	if *isStartingMsgBroker {
		*isStartingFiler = true
	}

	_, peerList := checkPeers(*serverIp, *masterOptions.port, *masterOptions.peers)
	peers := strings.Join(peerList, ",")
	masterOptions.peers = &peers

	masterOptions.ip = serverIp
	masterOptions.ipBind = serverBindIp
	filerOptions.masters = &peers
	filerOptions.ip = serverIp
	filerOptions.bindIp = serverBindIp
	serverOptions.v.ip = serverIp
	serverOptions.v.bindIp = serverBindIp
	serverOptions.v.masters = &peers
	serverOptions.v.idleConnectionTimeout = serverTimeout
	serverOptions.v.dataCenter = serverDataCenter
	serverOptions.v.rack = serverRack
	msgBrokerOptions.ip = serverIp

	// serverOptions.v.pulseSeconds = pulseSeconds
	// masterOptions.pulseSeconds = pulseSeconds

	masterOptions.whiteList = serverWhiteListOption

	filerOptions.dataCenter = serverDataCenter
	filerOptions.disableHttp = serverDisableHttp
	masterOptions.disableHttp = serverDisableHttp

	filerAddress := fmt.Sprintf("%s:%d", *serverIp, *filerOptions.port)
	s3Options.filer = &filerAddress
	msgBrokerOptions.filer = &filerAddress

	if *filerOptions.defaultReplicaPlacement == "" {
		*filerOptions.defaultReplicaPlacement = *masterOptions.defaultReplication
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	folders := strings.Split(*volumeDataFolders, ",")

	if *masterOptions.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
		glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
	}

	if *masterOptions.metaFolder == "" {
		*masterOptions.metaFolder = folders[0]
	}
	if err := util.TestFolderWritable(*masterOptions.metaFolder); err != nil {
		glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterOptions.metaFolder, err)
	}
	filerOptions.defaultLevelDbDirectory = masterOptions.metaFolder

	if *serverWhiteListOption != "" {
		serverWhiteList = strings.Split(*serverWhiteListOption, ",")
	}

	if *isStartingFiler {
		go func() {
			time.Sleep(1 * time.Second)

			filerOptions.startFiler()

		}()
	}

	if *isStartingS3 {
		go func() {
			time.Sleep(2 * time.Second)

			s3Options.startS3Server()

		}()
	}

	if *isStartingMsgBroker {
		go func() {
			time.Sleep(2 * time.Second)
			msgBrokerOptions.startQueueServer()
		}()
	}

	// start volume server
	{
		go serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption, *volumeMinFreeSpacePercent)

	}

	startMaster(masterOptions, serverWhiteList)

	return true
}
<|endoftext|>


<commit_before>// Copyright (c) 2010 AFP Authors
// This source code is released under the terms of the
// MIT license. Please see the file LICENSE for license details.

package distort

import (
	"afp"
	"afp/flags"
	"os"
)

type DistortFilter struct {
	ctx                  *afp.Context
	gain, clip, hardness float32
	clipper              func(*DistortFilter)
}

func NewFilter() afp.Filter {
	return &DistortFilter{}
}

var clipTypes = map[string]func(*DistortFilter){
	"hard":     hard,
	"variable": variable,
	"cubic":    cubic,
	"foldback": foldback,
}

func (self *DistortFilter) Init(ctx *afp.Context, args []string) os.Error {
	self.ctx = ctx

	fParse := flags.FlagParser(args)
	fParse.Float32Var(&self.gain, "g", 1.0,
		"Signal gain to apply before clipping. Must be > 0.")
	fParse.Float32Var(&self.clip, "c", 1.0,
		"The amplitude at which to clip the signal. Must be in (0,1)")
	fParse.Float32Var(&self.hardness, "k", 10,
		"Clipping 'hardness' for the variable clipping filter. Must be"+
			" in [1,\u221E), where 1 is soft clipping and \u221E is hard clipping.")
	clipType := fParse.String("t",
		"soft", "The type of clipping used: hard, variable, cubic, or foldback."+
			" See the afp(1) manpage for more info")

	fParse.Parse()

	if self.gain <= 0 {
		return os.NewError("Gain must be greater than 0.")
	}

	if self.clip > 1 || self.clip < 0 {
		return os.NewError("Clipping level must be between 0 and 1")
	}

	tempClipper, ok := clipTypes[*clipType]

	if !ok {
		return os.NewError("Clipping type must be one of: hard, soft, overflow, or foldback")
	}

	self.clipper = tempClipper

	if self.clipper != variable && self.hardness < 1 {
		return os.NewError("Hardness must be in [1,\u221E).")
	}

	return nil
}

func (self *DistortFilter) Stop() os.Error {
	return nil
}

func (self *DistortFilter) GetType() int {
	return afp.PIPE_LINK
}

func (self *DistortFilter) Start() {
	self.ctx.HeaderSink <- (<-self.ctx.HeaderSource)
	self.clipper(self)
}

func hard(f *DistortFilter) {
	for frame := range f.ctx.Source {
		for slice := range frame {
			for ch, sample := range frame[slice] {
				frame[slice][ch] = hardMin(f.clip, sample * f.gain)
			}
		}
		f.ctx.Sink <- frame
	}
}

//Min function which knows about hard().
//specifically that clip will always be positive
func hardMin(clip, sprime float32) float32 {
	var t float32

	if sprime < 0 {
		t = -sprime
	} else {
		t = sprime
	}

	if t > clip {
		return clip
	}

	return sprime
}

func foldback(f *DistortFilter) {
	for frame := range f.ctx.Source {
		for slice := range frame {
			for ch, sample := range frame[slice] {
				frame[slice][ch] = fold(sample*f.gain, f.clip)
			}
		}
		f.ctx.Sink <- frame
	}
}

//Helper function for foldback
//Computes the actual value of a sample
func fold(sample, clip float32) float32 {

	//A single fold may cause the signal to exceed the clip level
	//on the other side, so we may need to fold multiple times
	for sample > clip || sample < -clip {
		if sample > clip {
			sample = 2*clip - sample
		} else {
			sample = clip + sample
		}
	}

	return sample
}

func cubic(f *DistortFilter) {
	for frame := range f.ctx.Source {
		for slice := range frame {
			for ch, sample := range frame[slice] {
				frame[slice][ch] = cubicClip(sample*f.gain, f.clip)
			}
		}
		f.ctx.Sink <- frame
	}
}

//This algorithm is an adaptation of the one found at:
//https://ccrma.stanford.edu/~jos/pasp/Soft_Clipping.html#29299
//       { -2/3 * clip           x <= -clip
// out = {  x - x^3/3    -clip < x < clip
//       {  2/3 * clip           x >= clip
func cubicClip(sample, clip float32) float32 {
	if sample >= clip {
		sample = 0.66666666666666666666666 * clip
	} else if sample <= -clip {
		sample = -0.66666666666666666666666 * clip
	} else {
		sample = sample - sample*sample*sample/3
	}
	return sample
}

//Variable distortion via a modification of the formula
//from http://www.musicdsp.org/showone.php?id=104
//by scoofy[AT]inf[DOT]elte[DOT]hu
//For each sample, we evaluate:
// c/atan(s) * atan(x*s), where c is the clip level, s is
// the hardness, and x is the sample data.
func variable(f *DistortFilter) {
	//Precompute what we can..
	hardnessMult := f.clip / atan(f.hardness)

	for frame := range f.ctx.Source {
		for slice := range frame {
			for ch, sample := range frame[slice] {
				frame[slice][ch] = hardnessMult * atan(sample*f.hardness)
			}
		}
		f.ctx.Sink <- frame
	}
}

//Provides a good enough approximation of atan
//in [-2,2].  Thanks to antiprosynthesis[AT]hotmail[DOT]com
//Not used at this time.
func fastAtan(x float32) float32 {
	return x / (1 + .28*x*x)
}
<commit_msg>Automated merge with https://afp.googlecode.com/hg/<commit_after>// Copyright (c) 2010 AFP Authors
// This source code is released under the terms of the
// MIT license. Please see the file LICENSE for license details.

package distort

import (
	"afp"
	"afp/flags"
	"os"
)

type DistortFilter struct {
	ctx                  *afp.Context
	gain, clip, hardness float32
	clipper              func(*DistortFilter)
}

func NewFilter() afp.Filter {
	return &DistortFilter{}
}

var clipTypes = map[string]func(*DistortFilter){
	"hard":     hard,
	"variable": variable,
	"cubic":    cubic,
	"foldback": foldback,
}

func (self *DistortFilter) Init(ctx *afp.Context, args []string) os.Error {
	self.ctx = ctx

	fParse := flags.FlagParser(args)
	fParse.Float32Var(&self.gain, "g", 1.0,
		"Signal gain to apply before clipping. Must be > 0.")
	fParse.Float32Var(&self.clip, "c", 1.0,
		"The amplitude at which to clip the signal. Must be in (0,1)")
	fParse.Float32Var(&self.hardness, "h", 10,
		"Clipping 'hardness' for the variable clipping filter. Must be"+
			" in [1,\u221E), where 1 is soft clipping and \u221E is hard clipping.")
	clipType := fParse.String("t",
		"cubic", "The type of clipping used: hard, variable, cubic, or foldback."+
			" See the afp(1) manpage for more info")

	fParse.Parse()

	if self.gain <= 0 {
		return os.NewError("Gain must be greater than 0.")
	}

	if self.clip > 1 || self.clip < 0 {
		return os.NewError("Clipping level must be between 0 and 1")
	}

	tempClipper, ok := clipTypes[*clipType]

	if !ok {
		return os.NewError("Clipping type must be one of: hard, soft, overflow, or foldback")
	}

	self.clipper = tempClipper

	if self.clipper != variable && self.hardness < 1 {
		return os.NewError("Hardness must be in [1,\u221E).")
	}

	return nil
}

func (self *DistortFilter) Stop() os.Error {
	return nil
}

func (self *DistortFilter) GetType() int {
	return afp.PIPE_LINK
}

func (self *DistortFilter) Start() {
	self.ctx.HeaderSink <- (<-self.ctx.HeaderSource)
	self.clipper(self)
}

func hard(f *DistortFilter) {
	for frame := range f.ctx.Source {
		for slice := range frame {
			for ch, sample := range frame[slice] {
				frame[slice][ch] = hardMin(f.clip, sample * f.gain)
			}
		}
		f.ctx.Sink <- frame
	}
}

//Min function which knows about hard().
//specifically that clip will always be positive
func hardMin(clip, sprime float32) float32 {
	var t float32

	if sprime < 0 {
		t = -sprime
	} else {
		t = sprime
	}

	if t > clip {
		return clip
	}

	return sprime
}

func foldback(f *DistortFilter) {
	for frame := range f.ctx.Source {
		for slice := range frame {
			for ch, sample := range frame[slice] {
				frame[slice][ch] = fold(sample*f.gain, f.clip)
			}
		}
		f.ctx.Sink <- frame
	}
}

//Helper function for foldback
//Computes the actual value of a sample
func fold(sample, clip float32) float32 {

	//A single fold may cause the signal to exceed the clip level
	//on the other side, so we may need to fold multiple times
	for sample > clip || sample < -clip {
		if sample > clip {
			sample = 2*clip - sample
		} else {
			sample = clip + sample
		}
	}

	return sample
}

func cubic(f *DistortFilter) {
	for frame := range f.ctx.Source {
		for slice := range frame {
			for ch, sample := range frame[slice] {
				frame[slice][ch] = cubicClip(sample*f.gain, f.clip)
			}
		}
		f.ctx.Sink <- frame
	}
}

//This algorithm is an adaptation of the one found at:
//https://ccrma.stanford.edu/~jos/pasp/Soft_Clipping.html#29299
//       { -2/3 * clip           x <= -clip
// out = {  x - x^3/3    -clip < x < clip
//       {  2/3 * clip           x >= clip
func cubicClip(sample, clip float32) float32 {
	if sample >= clip {
		sample = 0.66666666666666666666666 * clip
	} else if sample <= -clip {
		sample = -0.66666666666666666666666 * clip
	} else {
		sample = sample - sample*sample*sample/3
	}
	return sample
}

//Variable distortion via a modification of the formula
//from http://www.musicdsp.org/showone.php?id=104
//by scoofy[AT]inf[DOT]elte[DOT]hu
//For each sample, we evaluate:
// c/atan(s) * atan(x*s), where c is the clip level, s is
// the hardness, and x is the sample data.
func variable(f *DistortFilter) {
	//Precompute what we can..
	hardnessMult := f.clip / atan(f.hardness)

	for frame := range f.ctx.Source {
		for slice := range frame {
			for ch, sample := range frame[slice] {
				frame[slice][ch] = hardnessMult * atan(sample*f.hardness)
			}
		}
		f.ctx.Sink <- frame
	}
}

//Provides a good enough approximation of atan
//in [-2,2].  Thanks to antiprosynthesis[AT]hotmail[DOT]com
//Not used at this time.
func fastAtan(x float32) float32 {
	return x / (1 + .28*x*x)
}
<|endoftext|>


<commit_before>package cmac

import (
	"bytes"
	"testing"
)

func TestGenSubkeys(t *testing.T) {
	for i, tv := range nistvectors {
		c, err := tv.cipher(tv.key)
		if err != nil {
			t.Fatal(err)
		}
		k1, k2 := gensubkeys(c)
		if !bytes.Equal(k1, tv.k1) {
			t.Errorf("tv[%d]: expected: %x got %x\n", i, tv.k1, k1)
		}
		if !bytes.Equal(k2, tv.k2) {
			t.Errorf("tv[%d]: expected: %x got %x\n", i, tv.k2, k2)
		}
	}
}

func TestNew(t *testing.T) {
	for i := 0; i < 3; i++ {
		tv := nistvectors[i]

		m, err := New(tv.key)
		if err != nil {
			t.Fatalf("tv[%d]: New() err: %s\n", i, err)
		}
		for j, tc := range tv.cases {
			m.Write(tc.msg)
			mac := m.Sum(nil)
			if !bytes.Equal(mac, tc.mac) {
				t.Errorf("tv[%d,%d]: expected: %x got %x\n", i, j, tc.mac, mac)
			}
			m.Reset()
		}
	}
}

func TestNewWithCipher(t *testing.T) {
	for i, tv := range nistvectors {
		c, err := tv.cipher(tv.key)
		if err != nil {
			t.Fatal(err)
		}
		m, err := NewWithCipher(c)
		if err != nil {
			t.Fatalf("tv[%d]: NewWithCipher() err: %s\n", i, err)
		}
		for j, tc := range tv.cases {
			m.Write(tc.msg)
			mac := m.Sum(nil)
			if !bytes.Equal(mac, tc.mac) {
				t.Errorf("tv[%d,%d]: expected: %x got %x\n", i, j, tc.mac, mac)
			}
			m.Reset()
		}
	}
}

func benchmarkHash(b *testing.B, size int64) {
	buf := make([]byte, size)
	h, _ := New(make([]byte, 128/8))
	sum := make([]byte, 0, h.Size())

	b.ReportAllocs()
	b.SetBytes(size)
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		h.Reset()
		h.Write(buf)
		h.Sum(sum)
	}
}

func BenchmarkHash1K(b *testing.B) { benchmarkHash(b, 1<<10) }
func BenchmarkHash1M(b *testing.B) { benchmarkHash(b, 1<<20) }
<commit_msg>Add chunked write test<commit_after>package cmac

import (
	"bytes"
	"testing"
)

func TestGenSubkeys(t *testing.T) {
	for i, tv := range nistvectors {
		c, err := tv.cipher(tv.key)
		if err != nil {
			t.Fatal(err)
		}
		k1, k2 := gensubkeys(c)
		if !bytes.Equal(k1, tv.k1) {
			t.Errorf("tv[%d]: expected: %x got %x\n", i, tv.k1, k1)
		}
		if !bytes.Equal(k2, tv.k2) {
			t.Errorf("tv[%d]: expected: %x got %x\n", i, tv.k2, k2)
		}
	}
}

func TestNew(t *testing.T) {
	for i := 0; i < 3; i++ {
		tv := nistvectors[i]

		m, err := New(tv.key)
		if err != nil {
			t.Fatalf("tv[%d]: New() err: %s\n", i, err)
		}
		for j, tc := range tv.cases {
			m.Write(tc.msg)
			mac := m.Sum(nil)
			if !bytes.Equal(mac, tc.mac) {
				t.Errorf("tv[%d,%d]: expected: %x got %x\n", i, j, tc.mac, mac)
			}
			m.Reset()
		}
	}
}

func TestNewWithCipher(t *testing.T) {
	for i, tv := range nistvectors {
		c, err := tv.cipher(tv.key)
		if err != nil {
			t.Fatal(err)
		}
		m, err := NewWithCipher(c)
		if err != nil {
			t.Fatalf("tv[%d]: NewWithCipher() err: %s\n", i, err)
		}
		for j, tc := range tv.cases {
			m.Write(tc.msg)
			mac := m.Sum(nil)
			if !bytes.Equal(mac, tc.mac) {
				t.Errorf("tv[%d,%d]: expected: %x got %x\n", i, j, tc.mac, mac)
			}
			m.Reset()
		}
	}
}

func TestChunkedWrites(t *testing.T) {
	key := make([]byte, 16)
	expected := []byte{0xe9, 0x0a, 0xe6, 0xf3, 0x44, 0x73, 0x47, 0xf6, 0x19, 0xcf, 0xf1, 0x6a, 0xb2, 0xa3, 0xd4, 0x9c}
	total := 1 << 20

	h, err := New(key)
	if err != nil {
		t.Fatal(err)
	}

	n := 0
	for c := 0; n < total; c += 1 {
		if n+c > total {
			c = total - n
		}

		chunk := make([]byte, c)
		h.Write(chunk)
		n += len(chunk)
	}
	mac := h.Sum(nil)

	if !bytes.Equal(mac, expected) {
		t.Errorf("expected: %x got %x\n", expected, mac)
	}
}

func benchmarkHash(b *testing.B, size int64) {
	buf := make([]byte, size)
	h, _ := New(make([]byte, 128/8))
	sum := make([]byte, 0, h.Size())

	b.ReportAllocs()
	b.SetBytes(size)
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		h.Reset()
		h.Write(buf)
		h.Sum(sum)
	}
}

func BenchmarkHash1K(b *testing.B) { benchmarkHash(b, 1<<10) }
func BenchmarkHash1M(b *testing.B) { benchmarkHash(b, 1<<20) }
<|endoftext|>


<commit_before>// Copyright (c) 2013-2014 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package main

import (
	`fmt`
	`github.com/michaeldv/donna`
	`time`
)

func repl() {
	var game *donna.Game
	var position *donna.Position

	setup := func() {
		if game == nil || position == nil {
			game = donna.NewGame()
			position = game.CacheSize(64).Start()
			fmt.Printf("%s\n", position)
		}
	}

	think := func() {
		if move := game.Think(11); move != 0 {
			position = position.MakeMove(move)
			fmt.Printf("%s\n", position)
		}
	}

	for command := ``; ; command = `` {
		fmt.Print(`donna> `)
		fmt.Scanf(`%s`, &command)
		switch command {
		case ``:
		case `bench`:
			benchmark()
		case `exit`, `quit`:
			return
		case `go`:
			setup()
			think()
		case `help`, `?`:
			fmt.Println("The commands are:\n\n" +
				"   bench   Run benchmark test\n" +
				"   exit    Exit the program\n" +
				"   go      Take side and make a move\n" +
				"   help    Display this help\n" +
				"   new     Start new game\n" +
				"   perft   Run perft test\n" +
				"   score   Show evaluation summary\n")
		case `new`:
			game, position = nil, nil
			setup()
		case `perft`:
			perft(5)
		case `score`:
			setup()
			_, metrics := position.EvaluateWithTrace()
			donna.Summary(metrics)
		default:
			setup()
			if move := position.NewMoveFromString(command); move != 0 {
				if advance := position.MakeMove(move); advance != nil {
					position = advance
					think()
					continue
				}
			}
			// Invalid move (typo) or non-evasion of check.
			fmt.Printf("%s appears to be an invalid move.\n", command)
		}
	}
}

func main() {
	donna.Settings.Log = false
	donna.Settings.Fancy = true
	repl()
}

func benchmark() {
	// Sergey Kaminer, 1935
	// 1.h8Q+ Kxh8 2.Ng4+ Kg7 3.Qh7+ Kf8 4.Qg8+ Ke7 5.Qe8+ Kd6 6.Qd7+ Kc5 7.Qb5+ Kd6
	// 8.Qb6+ Ke7 9.Qc7+ Kf8 10.Qc8+ Ke7 11.Qd7+ Kf8 12.Qe8+ Kg7 13.Qg8+ Kxg8 14.Kxf6+
	game := donna.NewGame(`Kd1,Qh2,Nh6,a4,g3,h7`, `Kg7,Qe4,Bf6,b7,e6,g6`)
	position := game.Start(donna.White)
	fmt.Printf("%s\n", position)
	game.CacheSize(64).Think(12)

	// Tylkowski vs. Wojciechowski, Poznan 1931
	// 30...Rxb2! 31. Nxb2 c3 32. Rxb6 c4!! 33. Rb4 a5! 34.Na4 axb4
	game = donna.NewGame(`Kg1,Rb7,Na4,a2,b2,f4,g2,g3`, `Kh7,Rd2,Bb6,a7,c5,c4,g7,h6`)
	position = game.Start(donna.Black)
	fmt.Printf("%s\n", position)
	game.CacheSize(64).Think(12)

	// Georg Rotlewi vs. Akiba Rubinstein, Lodz 1907, after 21 moves.
	// http://www.chessgames.com/perl/chessgame?gid=1119679
	game = donna.NewGame(`Kh1,Qe2,Ra1,Rf1,Bb2,Be4,Nc3,a3,b4,e5,f4,g2,h2`, `Kg8,Qe7,Rc8,Rd8,Bb6,Bb7,Ng4,a6,b5,e6,f7,g7,h7`)
	position = game.Start(donna.Black)
	fmt.Printf("%s\n", position)
	game.CacheSize(64).Think(12)

	// Donald Byrne vs. Bobby Fischer, Third Rosenwald Trophy, 1956 after 16 moves.
	// http://www.chessgames.com/perl/chessgame?gid=1008361
	// 16... Rf8-e8 17. Ke1-f1 Bg4-e6!!
	game = donna.NewGame(`Ke1,Qa3,Rd1,Rh1,Bc4,Bc5,Nf3,a2,d4,f2,g2,h2`, `Kg8,Qb6,Ra8,Rf8,Bg4,Bg7,Nc3,a7,b7,c6,f7,g6,h7`)
	position = game.Start(donna.Black)
	fmt.Printf("%s\n", position)
	game.CacheSize(64).Think(12)

	// Bobby Fischer vs. James Sherwin, New Jersey Open 1957, after 16 moves.
	// http://www.chessgames.com/perl/chessgame?gid=1008366
	// Fischer played 17. h2-h4!
	game = donna.NewGame(`Kg1,Qc2,Ra1,Re1,Bc1,Bg2,Ng5,a2,b2,c3,d4,f2,g3,h2`, `Kg8,Qd6,Ra8,Rf8,Bc8,Nd5,Ng6,a7,b6,c4,e6,f7,g7,h7`)
	position = game.Start(donna.White)
	fmt.Printf("%s\n", position)
	game.CacheSize(64).Think(12)

	// Mikhail Botvinnik vs. Jose Raul Capablanca, AVRO 1936, after 29 moves.
	// Botvinnik played 30. Bb2-a3!
	game = donna.NewGame(`Kg1,Qe5,Bb2,Ng3,c3,d4,e6,g2,h2`, `Kg7,Qe7,Nb3,Nf6,a7,b6,c4,d5,g6,h7`)
	position = game.Start(donna.White)
	fmt.Printf("%s\n", position)
	game.CacheSize(64).Think(14)

}

func perft(depth int) {
	position := donna.NewGame().Start()
	start := time.Now()
	total := position.Perft(depth)
	finish := time.Since(start).Seconds()
	fmt.Printf("\n  Nodes: %d\n", total)
	fmt.Printf("Elapsed: %.2fs\n", finish)
	fmt.Printf("Nodes/s: %.2f\n", float64(total)/finish)
}
<commit_msg>Few more CLI benchmarks pending epdtest implementation<commit_after>// Copyright (c) 2013-2014 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package main

import (
	`fmt`
	`github.com/michaeldv/donna`
	`time`
)

func repl() {
	var game *donna.Game
	var position *donna.Position

	setup := func() {
		if game == nil || position == nil {
			game = donna.NewGame()
			position = game.CacheSize(64).Start()
			fmt.Printf("%s\n", position)
		}
	}

	think := func() {
		if move := game.Think(11); move != 0 {
			position = position.MakeMove(move)
			fmt.Printf("%s\n", position)
		}
	}

	for command := ``; ; command = `` {
		fmt.Print(`donna> `)
		fmt.Scanf(`%s`, &command)
		switch command {
		case ``:
		case `bench`:
			benchmark()
		case `exit`, `quit`:
			return
		case `go`:
			setup()
			think()
		case `help`, `?`:
			fmt.Println("The commands are:\n\n" +
				"   bench   Run benchmark test\n" +
				"   exit    Exit the program\n" +
				"   go      Take side and make a move\n" +
				"   help    Display this help\n" +
				"   new     Start new game\n" +
				"   perft   Run perft test\n" +
				"   score   Show evaluation summary\n")
		case `new`:
			game, position = nil, nil
			setup()
		case `perft`:
			perft(5)
		case `score`:
			setup()
			_, metrics := position.EvaluateWithTrace()
			donna.Summary(metrics)
		default:
			setup()
			if move := position.NewMoveFromString(command); move != 0 {
				if advance := position.MakeMove(move); advance != nil {
					position = advance
					think()
					continue
				}
			}
			// Invalid move (typo) or non-evasion of check.
			fmt.Printf("%s appears to be an invalid move.\n", command)
		}
	}
}

func main() {
	donna.Settings.Log = false
	donna.Settings.Fancy = true
	repl()
}

func benchmark() {
	// Sergey Kaminer, 1935
	// 1.h8Q+ Kxh8 2.Ng4+ Kg7 3.Qh7+ Kf8 4.Qg8+ Ke7 5.Qe8+ Kd6 6.Qd7+ Kc5 7.Qb5+ Kd6
	// 8.Qb6+ Ke7 9.Qc7+ Kf8 10.Qc8+ Ke7 11.Qd7+ Kf8 12.Qe8+ Kg7 13.Qg8+ Kxg8 14.Kxf6+
	game := donna.NewGame(`Kd1,Qh2,Nh6,a4,g3,h7`, `Kg7,Qe4,Bf6,b7,e6,g6`)
	position := game.Start(donna.White)
	fmt.Printf("%s\n", position)
	game.CacheSize(64).Think(11)

	// Gurevich-Sherbakov, Eu Club Cup, Neum BIH 2000
	// Nxd6!
	game = donna.NewGame(`r3qrk1/p5b1/1p1p1p2/2pP2pp/1nN1PP1B/2Nn3P/PP2Q1P1/R4RK1 w - -`)
	position = game.Start()
	fmt.Printf("%s\n", position)
	game.CacheSize(64).Think(11)

	// Spassky-Gheorghiu, EU-chT, Bath 1973
	// Be3!
	game = donna.NewGame(`1r1n4/5kp1/p4p1p/2B1p3/P1R5/1PK3P1/5P1P/8 w - -`)
	position = game.Start()
	fmt.Printf("%s\n", position)
	game.CacheSize(64).Think(11)

	// Yermolinsky-Majorovas, URS-ch U26 Jurmala 1983
	// d6!
	game = donna.NewGame(`5rk1/2p5/2N3p1/2pPp1Pp/2P1Pp1P/3Q1P2/r4b2/3K4 w - -`)
	position = game.Start()
	fmt.Printf("%s\n", position)
	game.CacheSize(64).Think(11)

	// Tylkowski vs. Wojciechowski, Poznan 1931
	// 30...Rxb2! 31. Nxb2 c3 32. Rxb6 c4!! 33. Rb4 a5! 34.Na4 axb4
	game = donna.NewGame(`Kg1,Rb7,Na4,a2,b2,f4,g2,g3`, `Kh7,Rd2,Bb6,a7,c5,c4,g7,h6`)
	position = game.Start(donna.Black)
	fmt.Printf("%s\n", position)
	game.CacheSize(64).Think(11)

	// Georg Rotlewi vs. Akiba Rubinstein, Lodz 1907, after 21 moves.
	// http://www.chessgames.com/perl/chessgame?gid=1119679
	game = donna.NewGame(`Kh1,Qe2,Ra1,Rf1,Bb2,Be4,Nc3,a3,b4,e5,f4,g2,h2`, `Kg8,Qe7,Rc8,Rd8,Bb6,Bb7,Ng4,a6,b5,e6,f7,g7,h7`)
	position = game.Start(donna.Black)
	fmt.Printf("%s\n", position)
	game.CacheSize(64).Think(11)

	// Donald Byrne vs. Bobby Fischer, Third Rosenwald Trophy, 1956 after 16 moves.
	// http://www.chessgames.com/perl/chessgame?gid=1008361
	// 16... Rf8-e8 17. Ke1-f1 Bg4-e6!!
	game = donna.NewGame(`Ke1,Qa3,Rd1,Rh1,Bc4,Bc5,Nf3,a2,d4,f2,g2,h2`, `Kg8,Qb6,Ra8,Rf8,Bg4,Bg7,Nc3,a7,b7,c6,f7,g6,h7`)
	position = game.Start(donna.Black)
	fmt.Printf("%s\n", position)
	game.CacheSize(64).Think(11)

	// Bobby Fischer vs. James Sherwin, New Jersey Open 1957, after 16 moves.
	// http://www.chessgames.com/perl/chessgame?gid=1008366
	// Fischer played 17. h2-h4!
	game = donna.NewGame(`Kg1,Qc2,Ra1,Re1,Bc1,Bg2,Ng5,a2,b2,c3,d4,f2,g3,h2`, `Kg8,Qd6,Ra8,Rf8,Bc8,Nd5,Ng6,a7,b6,c4,e6,f7,g7,h7`)
	position = game.Start(donna.White)
	fmt.Printf("%s\n", position)
	game.CacheSize(64).Think(11)

	// Mikhail Botvinnik vs. Jose Raul Capablanca, AVRO 1936, after 29 moves.
	// Botvinnik played 30. Bb2-a3!
	game = donna.NewGame(`Kg1,Qe5,Bb2,Ng3,c3,d4,e6,g2,h2`, `Kg7,Qe7,Nb3,Nf6,a7,b6,c4,d5,g6,h7`)
	position = game.Start(donna.White)
	fmt.Printf("%s\n", position)
	game.CacheSize(64).Think(11)

}

func perft(depth int) {
	position := donna.NewGame().Start()
	start := time.Now()
	total := position.Perft(depth)
	finish := time.Since(start).Seconds()
	fmt.Printf("\n  Nodes: %d\n", total)
	fmt.Printf("Elapsed: %.2fs\n", finish)
	fmt.Printf("Nodes/s: %.2f\n", float64(total)/finish)
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package state

import (
	"github.com/stretchr/testify/require"
	"io/ioutil"
	"os"
	"reflect"
	"strings"
	"testing"

	"k8s.io/kubernetes/pkg/kubelet/checkpointmanager"
	"k8s.io/kubernetes/pkg/kubelet/cm/containermap"
	testutil "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/testing"
	"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
)

const testingCheckpoint = "cpumanager_checkpoint_test"

func TestCheckpointStateRestore(t *testing.T) {
	testCases := []struct {
		description       string
		checkpointContent string
		policyName        string
		initialContainers containermap.ContainerMap
		expectedError     string
		expectedState     *stateMemory
	}{
		{
			"Restore non-existing checkpoint",
			"",
			"none",
			containermap.ContainerMap{},
			"",
			&stateMemory{},
		},
		{
			"Restore default cpu set",
			`{
				"policyName": "none",
				"defaultCPUSet": "4-6",
				"entries": {},
				"checksum": 354655845
			}`,
			"none",
			containermap.ContainerMap{},
			"",
			&stateMemory{
				defaultCPUSet: cpuset.NewCPUSet(4, 5, 6),
			},
		},
		{
			"Restore valid checkpoint",
			`{
				"policyName": "none",
				"defaultCPUSet": "1-3",
				"entries": {
					"pod": {
						"container1": "4-6",
						"container2": "1-3"
					}
				},
				"checksum": 3610638499
			}`,
			"none",
			containermap.ContainerMap{},
			"",
			&stateMemory{
				assignments: ContainerCPUAssignments{
					"pod": map[string]cpuset.CPUSet{
						"container1": cpuset.NewCPUSet(4, 5, 6),
						"container2": cpuset.NewCPUSet(1, 2, 3),
					},
				},
				defaultCPUSet: cpuset.NewCPUSet(1, 2, 3),
			},
		},
		{
			"Restore checkpoint with invalid checksum",
			`{
				"policyName": "none",
				"defaultCPUSet": "4-6",
				"entries": {},
				"checksum": 1337
			}`,
			"none",
			containermap.ContainerMap{},
			"checkpoint is corrupted",
			&stateMemory{},
		},
		{
			"Restore checkpoint with invalid JSON",
			`{`,
			"none",
			containermap.ContainerMap{},
			"unexpected end of JSON input",
			&stateMemory{},
		},
		{
			"Restore checkpoint with invalid policy name",
			`{
				"policyName": "other",
				"defaultCPUSet": "1-3",
				"entries": {},
				"checksum": 1394507217
			}`,
			"none",
			containermap.ContainerMap{},
			`configured policy "none" differs from state checkpoint policy "other"`,
			&stateMemory{},
		},
		{
			"Restore checkpoint with unparsable default cpu set",
			`{
				"policyName": "none",
				"defaultCPUSet": "1.3",
				"entries": {},
				"checksum": 3021697696
			}`,
			"none",
			containermap.ContainerMap{},
			`could not parse default cpu set "1.3": strconv.Atoi: parsing "1.3": invalid syntax`,
			&stateMemory{},
		},
		{
			"Restore checkpoint with unparsable assignment entry",
			`{
				"policyName": "none",
				"defaultCPUSet": "1-3",
				"entries": {
					"pod": {
						"container1": "4-6",
						"container2": "asd"
					}
				},
				"checksum": 962272150
			}`,
			"none",
			containermap.ContainerMap{},
			`could not parse cpuset "asd" for container "container2" in pod "pod": strconv.Atoi: parsing "asd": invalid syntax`,
			&stateMemory{},
		},
		{
			"Restore checkpoint from checkpoint with v1 checksum",
			`{
				"policyName": "none",
				"defaultCPUSet": "1-3",
				"checksum": 1694838852
			}`,
			"none",
			containermap.ContainerMap{},
			"",
			&stateMemory{
				defaultCPUSet: cpuset.NewCPUSet(1, 2, 3),
			},
		},
		{
			"Restore checkpoint with migration",
			`{
				"policyName": "none",
				"defaultCPUSet": "1-3",
				"entries": {
					"containerID1": "4-6",
					"containerID2": "1-3"
				},
				"checksum": 3680390589
			}`,
			"none",
			func() containermap.ContainerMap {
				cm := containermap.NewContainerMap()
				cm.Add("pod", "container1", "containerID1")
				cm.Add("pod", "container2", "containerID2")
				return cm
			}(),
			"",
			&stateMemory{
				assignments: ContainerCPUAssignments{
					"pod": map[string]cpuset.CPUSet{
						"container1": cpuset.NewCPUSet(4, 5, 6),
						"container2": cpuset.NewCPUSet(1, 2, 3),
					},
				},
				defaultCPUSet: cpuset.NewCPUSet(1, 2, 3),
			},
		},
	}

	// create temp dir
	testingDir, err := ioutil.TempDir("", "cpumanager_state_test")
	require.NoError(t, err)
	defer os.RemoveAll(testingDir)
	// create checkpoint manager for testing
	cpm, err := checkpointmanager.NewCheckpointManager(testingDir)
	require.NoErrorf(t, err, "could not create testing checkpoint manager: %v", err)

	for _, tc := range testCases {
		t.Run(tc.description, func(t *testing.T) {
			// ensure there is no previous checkpoint
			cpm.RemoveCheckpoint(testingCheckpoint)

			// prepare checkpoint for testing
			if strings.TrimSpace(tc.checkpointContent) != "" {
				checkpoint := &testutil.MockCheckpoint{Content: tc.checkpointContent}
				err = cpm.CreateCheckpoint(testingCheckpoint, checkpoint)
				require.NoErrorf(t, err, "could not create testing checkpoint: %v", err)
			}

			restoredState, err := NewCheckpointState(testingDir, testingCheckpoint, tc.policyName, tc.initialContainers)
			if strings.TrimSpace(tc.expectedError) == "" {
				require.NoError(t, err)
			} else {
				require.Error(t, err)
				require.Contains(t, err.Error(), "could not restore state from checkpoint")
				require.Contains(t, err.Error(), tc.expectedError)
				return
			}

			// compare state after restoration with the one expected
			AssertStateEqual(t, restoredState, tc.expectedState)
		})
	}
}

func TestCheckpointStateStore(t *testing.T) {
	testCases := []struct {
		description   string
		expectedState *stateMemory
	}{
		{
			"Store default cpu set",
			&stateMemory{defaultCPUSet: cpuset.NewCPUSet(1, 2, 3)},
		},
		{
			"Store assignments",
			&stateMemory{
				assignments: map[string]map[string]cpuset.CPUSet{
					"pod": {
						"container1": cpuset.NewCPUSet(1, 5, 8),
					},
				},
			},
		},
	}

	// create temp dir
	testingDir, err := ioutil.TempDir("", "cpumanager_state_test")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(testingDir)

	cpm, err := checkpointmanager.NewCheckpointManager(testingDir)
	if err != nil {
		t.Fatalf("could not create testing checkpoint manager: %v", err)
	}

	for _, tc := range testCases {
		t.Run(tc.description, func(t *testing.T) {
			// ensure there is no previous checkpoint
			cpm.RemoveCheckpoint(testingCheckpoint)

			cs1, err := NewCheckpointState(testingDir, testingCheckpoint, "none", nil)
			if err != nil {
				t.Fatalf("could not create testing checkpointState instance: %v", err)
			}

			// set values of cs1 instance so they are stored in checkpoint and can be read by cs2
			cs1.SetDefaultCPUSet(tc.expectedState.defaultCPUSet)
			cs1.SetCPUAssignments(tc.expectedState.assignments)

			// restore checkpoint with previously stored values
			cs2, err := NewCheckpointState(testingDir, testingCheckpoint, "none", nil)
			if err != nil {
				t.Fatalf("could not create testing checkpointState instance: %v", err)
			}

			AssertStateEqual(t, cs2, tc.expectedState)
		})
	}
}

func TestCheckpointStateHelpers(t *testing.T) {
	testCases := []struct {
		description   string
		defaultCPUset cpuset.CPUSet
		assignments   map[string]map[string]cpuset.CPUSet
	}{
		{
			description:   "One container",
			defaultCPUset: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7, 8),
			assignments: map[string]map[string]cpuset.CPUSet{
				"pod": {
					"c1": cpuset.NewCPUSet(0, 1),
				},
			},
		},
		{
			description:   "Two containers",
			defaultCPUset: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7, 8),
			assignments: map[string]map[string]cpuset.CPUSet{
				"pod": {
					"c1": cpuset.NewCPUSet(0, 1),
					"c2": cpuset.NewCPUSet(2, 3, 4, 5),
				},
			},
		},
		{
			description:   "Container without assigned cpus",
			defaultCPUset: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7, 8),
			assignments: map[string]map[string]cpuset.CPUSet{
				"pod": {
					"c1": cpuset.NewCPUSet(),
				},
			},
		},
	}

	// create temp dir
	testingDir, err := ioutil.TempDir("", "cpumanager_state_test")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(testingDir)

	cpm, err := checkpointmanager.NewCheckpointManager(testingDir)
	if err != nil {
		t.Fatalf("could not create testing checkpoint manager: %v", err)
	}

	for _, tc := range testCases {
		t.Run(tc.description, func(t *testing.T) {
			// ensure there is no previous checkpoint
			cpm.RemoveCheckpoint(testingCheckpoint)

			state, err := NewCheckpointState(testingDir, testingCheckpoint, "none", nil)
			if err != nil {
				t.Fatalf("could not create testing checkpointState instance: %v", err)
			}
			state.SetDefaultCPUSet(tc.defaultCPUset)

			for pod := range tc.assignments {
				for container, set := range tc.assignments[pod] {
					state.SetCPUSet(pod, container, set)
					if cpus, _ := state.GetCPUSet(pod, container); !cpus.Equals(set) {
						t.Fatalf("state inconsistent, got %q instead of %q", set, cpus)
					}

					state.Delete(pod, container)
					if _, ok := state.GetCPUSet(pod, container); ok {
						t.Fatal("deleted container still existing in state")
					}
				}
			}
		})
	}
}

func TestCheckpointStateClear(t *testing.T) {
	testCases := []struct {
		description   string
		defaultCPUset cpuset.CPUSet
		assignments   map[string]map[string]cpuset.CPUSet
	}{
		{
			"Valid state",
			cpuset.NewCPUSet(1, 5, 10),
			map[string]map[string]cpuset.CPUSet{
				"pod": {
					"container1": cpuset.NewCPUSet(1, 4),
				},
			},
		},
	}

	for _, tc := range testCases {
		t.Run(tc.description, func(t *testing.T) {
			// create temp dir
			testingDir, err := ioutil.TempDir("", "cpumanager_state_test")
			if err != nil {
				t.Fatal(err)
			}
			defer os.RemoveAll(testingDir)

			state, err := NewCheckpointState(testingDir, testingCheckpoint, "none", nil)
			if err != nil {
				t.Fatalf("could not create testing checkpointState instance: %v", err)
			}

			state.SetDefaultCPUSet(tc.defaultCPUset)
			state.SetCPUAssignments(tc.assignments)

			state.ClearState()
			if !cpuset.NewCPUSet().Equals(state.GetDefaultCPUSet()) {
				t.Fatal("cleared state with non-empty default cpu set")
			}
			for pod := range tc.assignments {
				for container := range tc.assignments[pod] {
					if _, ok := state.GetCPUSet(pod, container); ok {
						t.Fatalf("container %q in pod %q with non-default cpu set in cleared state", container, pod)
					}
				}
			}
		})
	}
}

func AssertStateEqual(t *testing.T, sf State, sm State) {
	cpusetSf := sf.GetDefaultCPUSet()
	cpusetSm := sm.GetDefaultCPUSet()
	if !cpusetSf.Equals(cpusetSm) {
		t.Errorf("State CPUSet mismatch. Have %v, want %v", cpusetSf, cpusetSm)
	}

	cpuassignmentSf := sf.GetCPUAssignments()
	cpuassignmentSm := sm.GetCPUAssignments()
	if !reflect.DeepEqual(cpuassignmentSf, cpuassignmentSm) {
		t.Errorf("State CPU assignments mismatch. Have %s, want %s", cpuassignmentSf, cpuassignmentSm)
	}
}
<commit_msg>reorder the import packages<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package state

import (
	"io/ioutil"
	"os"
	"reflect"
	"strings"
	"testing"

	"github.com/stretchr/testify/require"
	"k8s.io/kubernetes/pkg/kubelet/checkpointmanager"
	"k8s.io/kubernetes/pkg/kubelet/cm/containermap"
	testutil "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/testing"
	"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
)

const testingCheckpoint = "cpumanager_checkpoint_test"

func TestCheckpointStateRestore(t *testing.T) {
	testCases := []struct {
		description       string
		checkpointContent string
		policyName        string
		initialContainers containermap.ContainerMap
		expectedError     string
		expectedState     *stateMemory
	}{
		{
			"Restore non-existing checkpoint",
			"",
			"none",
			containermap.ContainerMap{},
			"",
			&stateMemory{},
		},
		{
			"Restore default cpu set",
			`{
				"policyName": "none",
				"defaultCPUSet": "4-6",
				"entries": {},
				"checksum": 354655845
			}`,
			"none",
			containermap.ContainerMap{},
			"",
			&stateMemory{
				defaultCPUSet: cpuset.NewCPUSet(4, 5, 6),
			},
		},
		{
			"Restore valid checkpoint",
			`{
				"policyName": "none",
				"defaultCPUSet": "1-3",
				"entries": {
					"pod": {
						"container1": "4-6",
						"container2": "1-3"
					}
				},
				"checksum": 3610638499
			}`,
			"none",
			containermap.ContainerMap{},
			"",
			&stateMemory{
				assignments: ContainerCPUAssignments{
					"pod": map[string]cpuset.CPUSet{
						"container1": cpuset.NewCPUSet(4, 5, 6),
						"container2": cpuset.NewCPUSet(1, 2, 3),
					},
				},
				defaultCPUSet: cpuset.NewCPUSet(1, 2, 3),
			},
		},
		{
			"Restore checkpoint with invalid checksum",
			`{
				"policyName": "none",
				"defaultCPUSet": "4-6",
				"entries": {},
				"checksum": 1337
			}`,
			"none",
			containermap.ContainerMap{},
			"checkpoint is corrupted",
			&stateMemory{},
		},
		{
			"Restore checkpoint with invalid JSON",
			`{`,
			"none",
			containermap.ContainerMap{},
			"unexpected end of JSON input",
			&stateMemory{},
		},
		{
			"Restore checkpoint with invalid policy name",
			`{
				"policyName": "other",
				"defaultCPUSet": "1-3",
				"entries": {},
				"checksum": 1394507217
			}`,
			"none",
			containermap.ContainerMap{},
			`configured policy "none" differs from state checkpoint policy "other"`,
			&stateMemory{},
		},
		{
			"Restore checkpoint with unparsable default cpu set",
			`{
				"policyName": "none",
				"defaultCPUSet": "1.3",
				"entries": {},
				"checksum": 3021697696
			}`,
			"none",
			containermap.ContainerMap{},
			`could not parse default cpu set "1.3": strconv.Atoi: parsing "1.3": invalid syntax`,
			&stateMemory{},
		},
		{
			"Restore checkpoint with unparsable assignment entry",
			`{
				"policyName": "none",
				"defaultCPUSet": "1-3",
				"entries": {
					"pod": {
						"container1": "4-6",
						"container2": "asd"
					}
				},
				"checksum": 962272150
			}`,
			"none",
			containermap.ContainerMap{},
			`could not parse cpuset "asd" for container "container2" in pod "pod": strconv.Atoi: parsing "asd": invalid syntax`,
			&stateMemory{},
		},
		{
			"Restore checkpoint from checkpoint with v1 checksum",
			`{
				"policyName": "none",
				"defaultCPUSet": "1-3",
				"checksum": 1694838852
			}`,
			"none",
			containermap.ContainerMap{},
			"",
			&stateMemory{
				defaultCPUSet: cpuset.NewCPUSet(1, 2, 3),
			},
		},
		{
			"Restore checkpoint with migration",
			`{
				"policyName": "none",
				"defaultCPUSet": "1-3",
				"entries": {
					"containerID1": "4-6",
					"containerID2": "1-3"
				},
				"checksum": 3680390589
			}`,
			"none",
			func() containermap.ContainerMap {
				cm := containermap.NewContainerMap()
				cm.Add("pod", "container1", "containerID1")
				cm.Add("pod", "container2", "containerID2")
				return cm
			}(),
			"",
			&stateMemory{
				assignments: ContainerCPUAssignments{
					"pod": map[string]cpuset.CPUSet{
						"container1": cpuset.NewCPUSet(4, 5, 6),
						"container2": cpuset.NewCPUSet(1, 2, 3),
					},
				},
				defaultCPUSet: cpuset.NewCPUSet(1, 2, 3),
			},
		},
	}

	// create temp dir
	testingDir, err := ioutil.TempDir("", "cpumanager_state_test")
	require.NoError(t, err)
	defer os.RemoveAll(testingDir)
	// create checkpoint manager for testing
	cpm, err := checkpointmanager.NewCheckpointManager(testingDir)
	require.NoErrorf(t, err, "could not create testing checkpoint manager: %v", err)

	for _, tc := range testCases {
		t.Run(tc.description, func(t *testing.T) {
			// ensure there is no previous checkpoint
			cpm.RemoveCheckpoint(testingCheckpoint)

			// prepare checkpoint for testing
			if strings.TrimSpace(tc.checkpointContent) != "" {
				checkpoint := &testutil.MockCheckpoint{Content: tc.checkpointContent}
				err = cpm.CreateCheckpoint(testingCheckpoint, checkpoint)
				require.NoErrorf(t, err, "could not create testing checkpoint: %v", err)
			}

			restoredState, err := NewCheckpointState(testingDir, testingCheckpoint, tc.policyName, tc.initialContainers)
			if strings.TrimSpace(tc.expectedError) == "" {
				require.NoError(t, err)
			} else {
				require.Error(t, err)
				require.Contains(t, err.Error(), "could not restore state from checkpoint")
				require.Contains(t, err.Error(), tc.expectedError)
				return
			}

			// compare state after restoration with the one expected
			AssertStateEqual(t, restoredState, tc.expectedState)
		})
	}
}

func TestCheckpointStateStore(t *testing.T) {
	testCases := []struct {
		description   string
		expectedState *stateMemory
	}{
		{
			"Store default cpu set",
			&stateMemory{defaultCPUSet: cpuset.NewCPUSet(1, 2, 3)},
		},
		{
			"Store assignments",
			&stateMemory{
				assignments: map[string]map[string]cpuset.CPUSet{
					"pod": {
						"container1": cpuset.NewCPUSet(1, 5, 8),
					},
				},
			},
		},
	}

	// create temp dir
	testingDir, err := ioutil.TempDir("", "cpumanager_state_test")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(testingDir)

	cpm, err := checkpointmanager.NewCheckpointManager(testingDir)
	if err != nil {
		t.Fatalf("could not create testing checkpoint manager: %v", err)
	}

	for _, tc := range testCases {
		t.Run(tc.description, func(t *testing.T) {
			// ensure there is no previous checkpoint
			cpm.RemoveCheckpoint(testingCheckpoint)

			cs1, err := NewCheckpointState(testingDir, testingCheckpoint, "none", nil)
			if err != nil {
				t.Fatalf("could not create testing checkpointState instance: %v", err)
			}

			// set values of cs1 instance so they are stored in checkpoint and can be read by cs2
			cs1.SetDefaultCPUSet(tc.expectedState.defaultCPUSet)
			cs1.SetCPUAssignments(tc.expectedState.assignments)

			// restore checkpoint with previously stored values
			cs2, err := NewCheckpointState(testingDir, testingCheckpoint, "none", nil)
			if err != nil {
				t.Fatalf("could not create testing checkpointState instance: %v", err)
			}

			AssertStateEqual(t, cs2, tc.expectedState)
		})
	}
}

func TestCheckpointStateHelpers(t *testing.T) {
	testCases := []struct {
		description   string
		defaultCPUset cpuset.CPUSet
		assignments   map[string]map[string]cpuset.CPUSet
	}{
		{
			description:   "One container",
			defaultCPUset: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7, 8),
			assignments: map[string]map[string]cpuset.CPUSet{
				"pod": {
					"c1": cpuset.NewCPUSet(0, 1),
				},
			},
		},
		{
			description:   "Two containers",
			defaultCPUset: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7, 8),
			assignments: map[string]map[string]cpuset.CPUSet{
				"pod": {
					"c1": cpuset.NewCPUSet(0, 1),
					"c2": cpuset.NewCPUSet(2, 3, 4, 5),
				},
			},
		},
		{
			description:   "Container without assigned cpus",
			defaultCPUset: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7, 8),
			assignments: map[string]map[string]cpuset.CPUSet{
				"pod": {
					"c1": cpuset.NewCPUSet(),
				},
			},
		},
	}

	// create temp dir
	testingDir, err := ioutil.TempDir("", "cpumanager_state_test")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(testingDir)

	cpm, err := checkpointmanager.NewCheckpointManager(testingDir)
	if err != nil {
		t.Fatalf("could not create testing checkpoint manager: %v", err)
	}

	for _, tc := range testCases {
		t.Run(tc.description, func(t *testing.T) {
			// ensure there is no previous checkpoint
			cpm.RemoveCheckpoint(testingCheckpoint)

			state, err := NewCheckpointState(testingDir, testingCheckpoint, "none", nil)
			if err != nil {
				t.Fatalf("could not create testing checkpointState instance: %v", err)
			}
			state.SetDefaultCPUSet(tc.defaultCPUset)

			for pod := range tc.assignments {
				for container, set := range tc.assignments[pod] {
					state.SetCPUSet(pod, container, set)
					if cpus, _ := state.GetCPUSet(pod, container); !cpus.Equals(set) {
						t.Fatalf("state inconsistent, got %q instead of %q", set, cpus)
					}

					state.Delete(pod, container)
					if _, ok := state.GetCPUSet(pod, container); ok {
						t.Fatal("deleted container still existing in state")
					}
				}
			}
		})
	}
}

func TestCheckpointStateClear(t *testing.T) {
	testCases := []struct {
		description   string
		defaultCPUset cpuset.CPUSet
		assignments   map[string]map[string]cpuset.CPUSet
	}{
		{
			"Valid state",
			cpuset.NewCPUSet(1, 5, 10),
			map[string]map[string]cpuset.CPUSet{
				"pod": {
					"container1": cpuset.NewCPUSet(1, 4),
				},
			},
		},
	}

	for _, tc := range testCases {
		t.Run(tc.description, func(t *testing.T) {
			// create temp dir
			testingDir, err := ioutil.TempDir("", "cpumanager_state_test")
			if err != nil {
				t.Fatal(err)
			}
			defer os.RemoveAll(testingDir)

			state, err := NewCheckpointState(testingDir, testingCheckpoint, "none", nil)
			if err != nil {
				t.Fatalf("could not create testing checkpointState instance: %v", err)
			}

			state.SetDefaultCPUSet(tc.defaultCPUset)
			state.SetCPUAssignments(tc.assignments)

			state.ClearState()
			if !cpuset.NewCPUSet().Equals(state.GetDefaultCPUSet()) {
				t.Fatal("cleared state with non-empty default cpu set")
			}
			for pod := range tc.assignments {
				for container := range tc.assignments[pod] {
					if _, ok := state.GetCPUSet(pod, container); ok {
						t.Fatalf("container %q in pod %q with non-default cpu set in cleared state", container, pod)
					}
				}
			}
		})
	}
}

func AssertStateEqual(t *testing.T, sf State, sm State) {
	cpusetSf := sf.GetDefaultCPUSet()
	cpusetSm := sm.GetDefaultCPUSet()
	if !cpusetSf.Equals(cpusetSm) {
		t.Errorf("State CPUSet mismatch. Have %v, want %v", cpusetSf, cpusetSm)
	}

	cpuassignmentSf := sf.GetCPUAssignments()
	cpuassignmentSm := sm.GetCPUAssignments()
	if !reflect.DeepEqual(cpuassignmentSf, cpuassignmentSm) {
		t.Errorf("State CPU assignments mismatch. Have %s, want %s", cpuassignmentSf, cpuassignmentSm)
	}
}
<|endoftext|>


<commit_before>// Copyright © 2017 Marcus Franke <marcus.franke@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"sync"

	"github.com/julienschmidt/httprouter"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"gopkg.in/russross/blackfriday.v2"
)

var (
	port   string
	cmdOut []byte
	mutex  sync.Mutex
)

// serveCmd represents the serve command
var serveCmd = &cobra.Command{
	Use:   "serve",
	Short: "Starts the yummy webserver",
	Run: func(cmd *cobra.Command, args []string) {

		repoPath := viper.GetString("yum.repopath")

		router := httprouter.New()
		router.Handler("GET", "/", http.FileServer(http.Dir(repoPath)))
		router.GET("/:filename", sendFileHandler)
		router.POST("/api/upload", apiPostUploadHandler)
		//router.PUT("/api/upload/:filename", apiUploadPut)
		router.DELETE("/api/delete/:filename", apiDeleteHandler)

		log.Fatal(http.ListenAndServe(":8080", router))
	},
}

func sendFileHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	// if /help is called, we need to redirect this to helpHandler
	if r.URL.Path == "/help" {
		helpHandler(w, r, ps)
	} else {
		http.ServeFile(w, r, viper.GetString("yum.repopath")+"/"+ps.ByName("filename"))
	}
}

func helpHandler(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	// get helpFile path from configuration
	helpFile := viper.GetString("yum.helpFile")

	// ingest the configured helpFile
	help, err := ioutil.ReadFile(helpFile)
	if err != nil {
		log.Println("Help file could not be read!")
		http.Error(w, "Could not load the help file", http.StatusInternalServerError)
		return
	}

	// render the Markdown file to HTML using the
	// blackfriday library
	output := blackfriday.Run(help)
	log.Println("/help requested!")
	fmt.Fprintf(w, string(output))
}

func apiDeleteHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	fileName := ps.ByName("filename")
	repoPath := viper.GetString("yum.repopath")

	if _, err := os.Stat(repoPath + "/" + fileName); err == nil {
		// requested file exists
		if err := os.Remove(repoPath + "/" + fileName); err != nil {
			errText := fmt.Sprintf("%s - Could not delete file!\n", r.URL)
			log.Printf(errText)
			http.Error(w, errText, http.StatusInternalServerError)
			return
		} else {
			// file deleted
			logText := fmt.Sprintf("%s - File deleted!\n", r.URL.Path)
			log.Printf(logText)
		}
	} else {
		// file does not exists
		errText := fmt.Sprintf("%s - File not found!\n", r.URL.Path)
		log.Println(errText)
		http.Error(w, errText, http.StatusNotFound)
	}
}

func apiPostUploadHandler(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {

	errText := ""
	repoPath := viper.GetString("yum.repopath")
	workers := viper.GetString("yum.workers")
	createrepoBinary := viper.GetString("yum.createrepoBinary")

	file, handler, err := r.FormFile("fileupload")
	if err != nil {
		errText = fmt.Sprintf("%s - incorrect FormFile used, must be fileupload!\n", r.URL)
		log.Println(errText)
		http.Error(w, errText, http.StatusBadRequest)
		return
	}
	defer file.Close()

	if filepath.Ext(handler.Filename) != ".rpm" {
		errText = fmt.Sprintf("%s - %s uploaded, not an rpm package!\n", r.URL, handler.Filename)
		log.Printf(errText)
		http.Error(w, errText, http.StatusUnsupportedMediaType)
		return
	}

	// check if the uploaded file already exists
	// if the repository is configured in protected mode
	// the request will return status 403 (forbidden)
	if _, err := os.Stat(repoPath + "/" + handler.Filename); err == nil {
        if viper.GetBool("yum.protected") {
			errText = fmt.Sprintf("%s - File already exists, forbidden to overwrite!", r.URL)
			log.Println(errText)
			http.Error(w, errText, http.StatusForbidden)
			return
        } else {
			log.Println("File already exists, will overwrite: " + handler.Filename)
		}
	}

	// create file handler to write uploaded file to
	f, err := os.OpenFile(repoPath+"/"+handler.Filename, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		errText = fmt.Sprintf("%s - %s/%s could not be created!\n", r.URL, repoPath, handler.Filename)
		log.Println(errText)
		http.Error(w, errText, http.StatusInternalServerError)
		log.Fatal(err)
	}
	defer f.Close()

	// copy the file buffer into the file handle
	_, err = io.Copy(f, file)
	if err != nil {
		errText = fmt.Sprintf("%s - an error occured copying the uploaded file to servers filesystem!\n",
			r.URL)
		log.Println(errText)
		log.Println(err)
		http.Error(w, errText, http.StatusInternalServerError)
		return
	}

	// process the uploaded file
	mutex.Lock()
	cmdOut, err = exec.Command(createrepoBinary, "--update", "--workers", workers, repoPath).CombinedOutput()
	if err != nil {
		fmt.Fprintln(w, string(cmdOut))
		http.Error(w, "Could not update repository", http.StatusInternalServerError)
		log.Println(err, string(cmdOut))
		mutex.Unlock()
		return
	}
	log.Println(string(cmdOut))
	mutex.Unlock()
}

func init() {
	RootCmd.AddCommand(serveCmd)

	// Flags for the serve command.
	serveCmd.Flags().StringVarP(&port, "port", "p", "8080", "Port to listen on")
}
<commit_msg>[serve] fix code indentation<commit_after>// Copyright © 2017 Marcus Franke <marcus.franke@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"sync"

	"github.com/julienschmidt/httprouter"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"gopkg.in/russross/blackfriday.v2"
)

var (
	port   string
	cmdOut []byte
	mutex  sync.Mutex
)

// serveCmd represents the serve command
var serveCmd = &cobra.Command{
	Use:   "serve",
	Short: "Starts the yummy webserver",
	Run: func(cmd *cobra.Command, args []string) {

		repoPath := viper.GetString("yum.repopath")

		router := httprouter.New()
		router.Handler("GET", "/", http.FileServer(http.Dir(repoPath)))
		router.GET("/:filename", sendFileHandler)
		router.POST("/api/upload", apiPostUploadHandler)
		//router.PUT("/api/upload/:filename", apiUploadPut)
		router.DELETE("/api/delete/:filename", apiDeleteHandler)

		log.Fatal(http.ListenAndServe(":8080", router))
	},
}

func sendFileHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	// if /help is called, we need to redirect this to helpHandler
	if r.URL.Path == "/help" {
		helpHandler(w, r, ps)
	} else {
		http.ServeFile(w, r, viper.GetString("yum.repopath")+"/"+ps.ByName("filename"))
	}
}

func helpHandler(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	// get helpFile path from configuration
	helpFile := viper.GetString("yum.helpFile")

	// ingest the configured helpFile
	help, err := ioutil.ReadFile(helpFile)
	if err != nil {
		log.Println("Help file could not be read!")
		http.Error(w, "Could not load the help file", http.StatusInternalServerError)
		return
	}

	// render the Markdown file to HTML using the
	// blackfriday library
	output := blackfriday.Run(help)
	log.Println("/help requested!")
	fmt.Fprintf(w, string(output))
}

func apiDeleteHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	fileName := ps.ByName("filename")
	repoPath := viper.GetString("yum.repopath")

	if _, err := os.Stat(repoPath + "/" + fileName); err == nil {
		// requested file exists
		if err := os.Remove(repoPath + "/" + fileName); err != nil {
			errText := fmt.Sprintf("%s - Could not delete file!\n", r.URL)
			log.Printf(errText)
			http.Error(w, errText, http.StatusInternalServerError)
			return
		} else {
			// file deleted
			logText := fmt.Sprintf("%s - File deleted!\n", r.URL.Path)
			log.Printf(logText)
		}
	} else {
		// file does not exists
		errText := fmt.Sprintf("%s - File not found!\n", r.URL.Path)
		log.Println(errText)
		http.Error(w, errText, http.StatusNotFound)
	}
}

func apiPostUploadHandler(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {

	errText := ""
	repoPath := viper.GetString("yum.repopath")
	workers := viper.GetString("yum.workers")
	createrepoBinary := viper.GetString("yum.createrepoBinary")

	file, handler, err := r.FormFile("fileupload")
	if err != nil {
		errText = fmt.Sprintf("%s - incorrect FormFile used, must be fileupload!\n", r.URL)
		log.Println(errText)
		http.Error(w, errText, http.StatusBadRequest)
		return
	}
	defer file.Close()

	if filepath.Ext(handler.Filename) != ".rpm" {
		errText = fmt.Sprintf("%s - %s uploaded, not an rpm package!\n", r.URL, handler.Filename)
		log.Printf(errText)
		http.Error(w, errText, http.StatusUnsupportedMediaType)
		return
	}

	// check if the uploaded file already exists
	// if the repository is configured in protected mode
	// the request will return status 403 (forbidden)
	if _, err := os.Stat(repoPath + "/" + handler.Filename); err == nil {
		if viper.GetBool("yum.protected") {
			errText = fmt.Sprintf("%s - File already exists, forbidden to overwrite!", r.URL)
			log.Println(errText)
			http.Error(w, errText, http.StatusForbidden)
			return
		} else {
			log.Println("File already exists, will overwrite: " + handler.Filename)
		}
	}

	// create file handler to write uploaded file to
	f, err := os.OpenFile(repoPath+"/"+handler.Filename, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		errText = fmt.Sprintf("%s - %s/%s could not be created!\n", r.URL, repoPath, handler.Filename)
		log.Println(errText)
		http.Error(w, errText, http.StatusInternalServerError)
		log.Fatal(err)
	}
	defer f.Close()

	// copy the file buffer into the file handle
	_, err = io.Copy(f, file)
	if err != nil {
		errText = fmt.Sprintf("%s - an error occured copying the uploaded file to servers filesystem!\n",
			r.URL)
		log.Println(errText)
		log.Println(err)
		http.Error(w, errText, http.StatusInternalServerError)
		return
	}

	// process the uploaded file
	mutex.Lock()
	cmdOut, err = exec.Command(createrepoBinary, "--update", "--workers", workers, repoPath).CombinedOutput()
	if err != nil {
		fmt.Fprintln(w, string(cmdOut))
		http.Error(w, "Could not update repository", http.StatusInternalServerError)
		log.Println(err, string(cmdOut))
		mutex.Unlock()
		return
	}
	log.Println(string(cmdOut))
	mutex.Unlock()
}

func init() {
	RootCmd.AddCommand(serveCmd)

	// Flags for the serve command.
	serveCmd.Flags().StringVarP(&port, "port", "p", "8080", "Port to listen on")
}
<|endoftext|>