<commit_before>// Copyright 2016 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// +build linux,!darwin

package devicemapper

import (
	"bufio"
	"encoding/xml"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/control-center/serviced/commons/diet"
	"github.com/control-center/serviced/utils"
	"github.com/zenoss/glog"
)

var storageStatsUpdateInterval time.Duration = 300

func SetStorageStatsUpdateInterval(interval int) {
	storageStatsUpdateInterval = time.Duration(interval)
}

// DeviceBlockStats represents a devicemapper device
type DeviceBlockStats struct {
	DeviceID int
	diet     *diet.Diet
}

// RangeMapping represents a range of blocks in a metadata xml file
type RangeMapping struct {
	OriginBegin int `xml:"origin_begin,attr"`
	DataBegin   int `xml:"data_begin,attr"`
	Length      int `xml:"length,attr"`
	Time        int `xml:"time,attr"`
}

// SingleMapping represents a single block in a metadata xml file
type SingleMapping struct {
	OriginBlock int `xml:"origin_block,attr"`
	DataBlock   int `xml:"data_block,attr"`
	Time        int `xml:"time,attr"`
}

// UniqueBlocks returns the number of unique blocks that this device has
// allocated compared to another device
func (d *DeviceBlockStats) UniqueBlocks(other *DeviceBlockStats) uint64 {
	return d.diet.Total() - d.diet.IntersectionAll(other.diet)
}

func hasMetadataSnap(pool string) (bool, error) {
	block, err := getMetadataBlock(pool)
	if err != nil {
		return false, err
	}
	return block != "-", nil
}

func getMetadataBlock(pool string) (string, error) {
	cmd := exec.Command("dmsetup", "status", pool)
	out, err := cmd.Output()
	if err != nil {
		return "", err
	}
	parts := strings.Fields(string(out))
	return strings.TrimSpace(parts[6]), nil
}

// reserveMetadataSnap reserves a userspace snapshot of a thin pool's metadata.
// We shell out to send the message to the device-mapper subsystem, but we lock
// our internal DeviceSet to avoid any complications with creating or removing
// devices simultaneously.
func (d *DeviceMapperDriver) reserveMetadataSnap(pool string) error {
	d.DeviceSet.Lock()
	defer d.DeviceSet.Unlock()
	cmd := exec.Command("dmsetup", "message", pool, "0", "reserve_metadata_snap")
	if err := cmd.Run(); err != nil {
		return err
	}
	return nil
}

// releaseMetadataSnap releases an existing userspace snapshot of a thin pool's
// metadata.  We shell out to send the message to the device-mapper subsystem,
// but we lock our internal DeviceSet to avoid any complications with creating
// or removing devices simultaneously.
func (d *DeviceMapperDriver) releaseMetadataSnap(pool string) error {
	d.DeviceSet.Lock()
	defer d.DeviceSet.Unlock()
	cmd := exec.Command("dmsetup", "message", pool, "0", "release_metadata_snap")
	if err := cmd.Run(); err != nil {
		return err
	}
	return nil
}

// getDevices acts on an existing userspace metadata snapshot, and has no
// potential devicemapper conflicts. It simply dumps the XML and parses it.
func getDevices(pool, block, metadatadev string) (map[int]*DeviceBlockStats, error) {
	cmd := exec.Command("thin_dump", "-f", "xml", metadatadev, "-m", block)
	stdout, err := cmd.StdoutPipe()
	cmd.Stderr = os.Stderr
	if err != nil {
		return nil, err
	}
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	devices, err := parseMetadata(stdout)
	if err != nil {
		return nil, err
	}
	if err := cmd.Wait(); err != nil {
		return nil, err
	}
	return devices, nil
}

func parseMetadata(r io.Reader) (map[int]*DeviceBlockStats, error) {
	devices := make(map[int]*DeviceBlockStats)
	decoder := xml.NewDecoder(r)
	var current *DeviceBlockStats
	for {
		t, err := decoder.Token()
		if t == nil || err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		switch se := t.(type) {
		case xml.EndElement:
			if se.Name.Local == "device" {
				if current == nil {
					// current should never be nil in this scenario, but if it
					// is, let's assume there's a good reason and just move on
					continue
				}
				// Balance the tree (necessary for performance, since the data
				// is ordered when we get it, so the tree is pretty much just
				// a linked list at this point)
				current.diet.Balance()
				devices[current.DeviceID], current = current, nil
			}
		case xml.StartElement:
			if se.Name.Local == "device" {
				for _, attr := range se.Attr {
					if attr.Name.Local == "dev_id" {
						id, err := strconv.Atoi(attr.Value)
						if err != nil {
							return nil, err
						}
						current = &DeviceBlockStats{id, diet.NewDiet()}
						break
					}
				}
				continue
			}
			if se.Name.Local == "range_mapping" {
				var m RangeMapping
				if err := decoder.DecodeElement(&m, &se); err != nil {
					return nil, err
				}
				current.diet.Insert(uint64(m.DataBegin), uint64(m.DataBegin+m.Length-1))
			}
			if se.Name.Local == "single_mapping" {
				var m SingleMapping
				if err := decoder.DecodeElement(&m, &se); err != nil {
					return nil, err
				}
				current.diet.Insert(uint64(m.DataBlock), uint64(m.DataBlock))
			}
		}
	}
	return devices, nil
}

// getDeviceSize calls lsblk in a subprocess to retrieve the size in bytes of
// a given device.
func getDeviceSize(dev string) (uint64, error) {
	cmd := exec.Command("lsblk", "-dbno", "SIZE", dev)
	out, err := cmd.Output()
	if err != nil {
		return 0, err
	}
	var size uint64
	if size, err = strconv.ParseUint(strings.TrimSpace(string(out)), 10, 64); err != nil {
		return 0, err
	}
	return size, nil
}

// getFilesystemStats calls dumpe2fs in a subprocess to retrieve filesystem
// statistics. Currently, only ext2/3/4 filesystems are supported by this call;
// XFS filesystems appear to require mounting in order to retrieve the same
// information.
func getFilesystemStats(dev string) (uint64, uint64, error) {
	cmd := exec.Command("dumpe2fs", dev)
	out, err := cmd.StdoutPipe()
	if err != nil {
		return 0, 0, err
	}
	if err = cmd.Start(); err != nil {
		return 0, 0, err
	}
	defer cmd.Wait()
	defer out.Close()
	scanner := bufio.NewScanner(out)
	var blocksize, totalblocks, freeblocks uint64
	for scanner.Scan() {
		line := scanner.Text()
		if strings.TrimSpace(line) == "" {
			break
		}
		if strings.HasPrefix(line, "Block count:") {
			n := strings.Fields(line)[2]
			if totalblocks, err = strconv.ParseUint(n, 10, 64); err != nil {
				return 0, 0, err
			}
		} else if strings.HasPrefix(line, "Free blocks:") {
			n := strings.Fields(line)[2]
			if freeblocks, err = strconv.ParseUint(n, 10, 64); err != nil {
				return 0, 0, err
			}
		} else if strings.HasPrefix(line, "Block size:") {
			n := strings.Fields(line)[2]
			if blocksize, err = strconv.ParseUint(n, 10, 64); err != nil {
				return 0, 0, err
			}
		}
	}
	return totalblocks * blocksize, freeblocks * blocksize, nil
}

var statscache = statCache{make(map[string]*cachedStat), utils.NewMutexMap()}

type statCache struct {
	cache map[string]*cachedStat
	locks *utils.MutexMap
}

type cachedStat struct {
	expiry time.Time
	pool   string
	value  map[int]*DeviceBlockStats
}

func (d *DeviceMapperDriver) getDeviceBlockStats(pool, metadatadev string) (map[int]*DeviceBlockStats, error) {
	var value map[int]*DeviceBlockStats
	pool = fmt.Sprintf("/dev/mapper/%s", strings.TrimPrefix(pool, "/dev/mapper/"))
	statscache.locks.LockKey(pool)
	defer statscache.locks.UnlockKey(pool)
	val, ok := statscache.cache[pool]
	if !ok || time.Now().After(val.expiry) {
		glog.Infof("Refreshing storage stats cache from thin pool metadata")
		// Check for an existing snapshot of the metadata device.
		hasSnap, err := hasMetadataSnap(pool)
		if err != nil {
			return nil, err
		}
		// TODO: Implement file locking to avoid stomping on metadata snaps by out
		// of band processes
		if hasSnap {
			// Release the existing snapshot.
			if err := d.releaseMetadataSnap(pool); err != nil {
				return nil, err
			}
		}
		// Take a userspace-accessible snapshot of the metadata device.
		if err := d.reserveMetadataSnap(pool); err != nil {
			return nil, err
		}
		defer d.releaseMetadataSnap(pool)
		// Ask for the block at which the metadata snap is accessible
		block, err := getMetadataBlock(pool)
		if err != nil {
			return nil, err
		}
		// Dump the metadata from the snapshot to XML, parse it, and return the
		// resulting DeviceBlockStats objects.
		value, err = getDevices(pool, block, metadatadev)
		if err != nil {
			return nil, err
		}
		// Don't cache if there aren't any devices besides the base device yet
		// (e.g., initial deployment)
		if len(value) > 1 {
			statscache.cache[pool] = &cachedStat{
				expiry: time.Now().Add(storageStatsUpdateInterval * time.Second),
				pool:   pool,
				value:  value,
			}
		}
	} else {
		value = val.value
	}
	return value, nil
}
<commit_msg>code review comments<commit_after>// Copyright 2016 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// +build linux,!darwin

package devicemapper

import (
	"bufio"
	"encoding/xml"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/control-center/serviced/commons/diet"
	"github.com/control-center/serviced/utils"
	"github.com/zenoss/glog"
)

var storageStatsUpdateInterval time.Duration = 300

func SetStorageStatsUpdateInterval(interval int) {
	storageStatsUpdateInterval = time.Duration(interval)
}

// DeviceBlockStats represents a devicemapper device
type DeviceBlockStats struct {
	DeviceID int
	diet     *diet.Diet
}

// RangeMapping represents a range of blocks in a metadata xml file
type RangeMapping struct {
	OriginBegin int `xml:"origin_begin,attr"`
	DataBegin   int `xml:"data_begin,attr"`
	Length      int `xml:"length,attr"`
	Time        int `xml:"time,attr"`
}

// SingleMapping represents a single block in a metadata xml file
type SingleMapping struct {
	OriginBlock int `xml:"origin_block,attr"`
	DataBlock   int `xml:"data_block,attr"`
	Time        int `xml:"time,attr"`
}

// UniqueBlocks returns the number of unique blocks that this device has
// allocated compared to another device
func (d *DeviceBlockStats) UniqueBlocks(other *DeviceBlockStats) uint64 {
	return d.diet.Total() - d.diet.IntersectionAll(other.diet)
}

func hasMetadataSnap(pool string) (bool, error) {
	block, err := getMetadataBlock(pool)
	if err != nil {
		return false, err
	}
	return block != "-", nil
}

func getMetadataBlock(pool string) (string, error) {
	cmd := exec.Command("dmsetup", "status", pool)
	out, err := cmd.Output()
	if err != nil {
		return "", err
	}
	parts := strings.Fields(string(out))
	return strings.TrimSpace(parts[6]), nil
}

// reserveMetadataSnap reserves a userspace snapshot of a thin pool's metadata.
// We shell out to send the message to the device-mapper subsystem, but we lock
// our internal DeviceSet to avoid any complications with creating or removing
// devices simultaneously.
func (d *DeviceMapperDriver) reserveMetadataSnap(pool string) error {
	d.DeviceSet.Lock()
	defer d.DeviceSet.Unlock()
	cmd := exec.Command("dmsetup", "message", pool, "0", "reserve_metadata_snap")
	if err := cmd.Run(); err != nil {
		return err
	}
	return nil
}

// releaseMetadataSnap releases an existing userspace snapshot of a thin pool's
// metadata.  We shell out to send the message to the device-mapper subsystem,
// but we lock our internal DeviceSet to avoid any complications with creating
// or removing devices simultaneously.
func (d *DeviceMapperDriver) releaseMetadataSnap(pool string) error {
	d.DeviceSet.Lock()
	defer d.DeviceSet.Unlock()
	cmd := exec.Command("dmsetup", "message", pool, "0", "release_metadata_snap")
	if err := cmd.Run(); err != nil {
		return err
	}
	return nil
}

// getDevices acts on an existing userspace metadata snapshot, and has no
// potential devicemapper conflicts. It simply dumps the XML and parses it.
func getDevices(pool, block, metadatadev string) (map[int]*DeviceBlockStats, error) {
	cmd := exec.Command("thin_dump", "-f", "xml", metadatadev, "-m", block)
	stdout, err := cmd.StdoutPipe()
	cmd.Stderr = os.Stderr
	if err != nil {
		return nil, err
	}
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	devices, err := parseMetadata(stdout)
	if err != nil {
		return nil, err
	}
	if err := cmd.Wait(); err != nil {
		return nil, err
	}
	return devices, nil
}

func parseMetadata(r io.Reader) (map[int]*DeviceBlockStats, error) {
	devices := make(map[int]*DeviceBlockStats)
	decoder := xml.NewDecoder(r)
	var current *DeviceBlockStats
	for {
		t, err := decoder.Token()
		if t == nil || err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		switch se := t.(type) {
		case xml.EndElement:
			if se.Name.Local == "device" {
				if current == nil {
					// current should never be nil in this scenario, but if it
					// is, let's assume there's a good reason and just move on
					continue
				}
				// Balance the tree (necessary for performance, since the data
				// is ordered when we get it, so the tree is pretty much just
				// a linked list at this point)
				current.diet.Balance()
				devices[current.DeviceID], current = current, nil
			}
		case xml.StartElement:
			if se.Name.Local == "device" {
				for _, attr := range se.Attr {
					if attr.Name.Local == "dev_id" {
						id, err := strconv.Atoi(attr.Value)
						if err != nil {
							return nil, err
						}
						current = &DeviceBlockStats{id, diet.NewDiet()}
						break
					}
				}
				continue
			}
			if se.Name.Local == "range_mapping" {
				var m RangeMapping
				if err := decoder.DecodeElement(&m, &se); err != nil {
					return nil, err
				}
				current.diet.Insert(uint64(m.DataBegin), uint64(m.DataBegin+m.Length-1))
			}
			if se.Name.Local == "single_mapping" {
				var m SingleMapping
				if err := decoder.DecodeElement(&m, &se); err != nil {
					return nil, err
				}
				current.diet.Insert(uint64(m.DataBlock), uint64(m.DataBlock))
			}
		}
	}
	return devices, nil
}

// getDeviceSize calls lsblk in a subprocess to retrieve the size in bytes of
// a given device.
func getDeviceSize(dev string) (uint64, error) {
	cmd := exec.Command("lsblk", "-dbno", "SIZE", dev)
	out, err := cmd.Output()
	if err != nil {
		return 0, err
	}
	var size uint64
	if size, err = strconv.ParseUint(strings.TrimSpace(string(out)), 10, 64); err != nil {
		return 0, err
	}
	return size, nil
}

// getFilesystemStats calls dumpe2fs in a subprocess to retrieve filesystem
// statistics. Currently, only ext2/3/4 filesystems are supported by this call;
// XFS filesystems appear to require mounting in order to retrieve the same
// information.
func getFilesystemStats(dev string) (uint64, uint64, error) {
	cmd := exec.Command("dumpe2fs", dev)
	defer cmd.Wait()
	out, err := cmd.StdoutPipe()
	if err != nil {
		return 0, 0, err
	}
	defer out.Close()
	if err = cmd.Start(); err != nil {
		return 0, 0, err
	}
	scanner := bufio.NewScanner(out)
	var blocksize, totalblocks, freeblocks uint64
	for scanner.Scan() {
		line := scanner.Text()
		if strings.TrimSpace(line) == "" {
			break
		}
		if strings.HasPrefix(line, "Block count:") {
			n := strings.Fields(line)[2]
			if totalblocks, err = strconv.ParseUint(n, 10, 64); err != nil {
				return 0, 0, err
			}
		} else if strings.HasPrefix(line, "Free blocks:") {
			n := strings.Fields(line)[2]
			if freeblocks, err = strconv.ParseUint(n, 10, 64); err != nil {
				return 0, 0, err
			}
		} else if strings.HasPrefix(line, "Block size:") {
			n := strings.Fields(line)[2]
			if blocksize, err = strconv.ParseUint(n, 10, 64); err != nil {
				return 0, 0, err
			}
		}
	}
	return totalblocks * blocksize, freeblocks * blocksize, nil
}

var statscache = statCache{make(map[string]*cachedStat), utils.NewMutexMap()}

type statCache struct {
	cache map[string]*cachedStat
	locks *utils.MutexMap
}

type cachedStat struct {
	expiry time.Time
	pool   string
	value  map[int]*DeviceBlockStats
}

func (d *DeviceMapperDriver) getDeviceBlockStats(pool, metadatadev string) (map[int]*DeviceBlockStats, error) {
	var value map[int]*DeviceBlockStats
	pool = fmt.Sprintf("/dev/mapper/%s", strings.TrimPrefix(pool, "/dev/mapper/"))
	statscache.locks.LockKey(pool)
	defer statscache.locks.UnlockKey(pool)
	val, ok := statscache.cache[pool]
	if !ok || time.Now().After(val.expiry) {
		glog.Infof("Refreshing storage stats cache from thin pool metadata")
		// Check for an existing snapshot of the metadata device.
		hasSnap, err := hasMetadataSnap(pool)
		if err != nil {
			return nil, err
		}
		// TODO: Implement file locking to avoid stomping on metadata snaps by out
		// of band processes
		if hasSnap {
			// Release the existing snapshot.
			if err := d.releaseMetadataSnap(pool); err != nil {
				return nil, err
			}
		}
		// Take a userspace-accessible snapshot of the metadata device.
		if err := d.reserveMetadataSnap(pool); err != nil {
			return nil, err
		}
		defer d.releaseMetadataSnap(pool)
		// Ask for the block at which the metadata snap is accessible
		block, err := getMetadataBlock(pool)
		if err != nil {
			return nil, err
		}
		// Dump the metadata from the snapshot to XML, parse it, and return the
		// resulting DeviceBlockStats objects.
		value, err = getDevices(pool, block, metadatadev)
		if err != nil {
			return nil, err
		}
		// Don't cache if there aren't any devices besides the base device yet
		// (e.g., initial deployment)
		if len(value) > 1 {
			statscache.cache[pool] = &cachedStat{
				expiry: time.Now().Add(storageStatsUpdateInterval * time.Second),
				pool:   pool,
				value:  value,
			}
		}
	} else {
		value = val.value
	}
	return value, nil
}
<|endoftext|>


<commit_before>package server

import (
	"net/http"

	"github.com/influxdata/chronograf"
)

// AuthRoute are the routes for each type of OAuth2 provider
type AuthRoute struct {
	Name     string `json:"name"`     // Name uniquely identifies the provider
	Label    string `json:"label"`    // Label is a user-facing string to present in the UI
	Login    string `json:"login"`    // Login is the route to the login redirect path
	Logout   string `json:"logout"`   // Logout is the route to the logout redirect path
	Callback string `json:"callback"` // Callback is the route the provider calls to exchange the code/state
}

// AuthRoutes contains all OAuth2 provider routes.
type AuthRoutes []AuthRoute

// Lookup searches all the routes for a specific provider
func (r *AuthRoutes) Lookup(provider string) (AuthRoute, bool) {
	for _, route := range *r {
		if route.Name == provider {
			return route, true
		}
	}
	return AuthRoute{}, false
}

type getRoutesResponse struct {
	Layouts    string      `json:"layouts"`    // Location of the layouts endpoint
	Mappings   string      `json:"mappings"`   // Location of the application mappings endpoint
	Sources    string      `json:"sources"`    // Location of the sources endpoint
	Me         string      `json:"me"`         // Location of the me endpoint
	Dashboards string      `json:"dashboards"` // Location of the dashboards endpoint
	Auth       []AuthRoute `json:"auth"`       // Location of all auth routes.
}

// AllRoutes returns all top level routes within chronograf
func AllRoutes(authRoutes []AuthRoute, logger chronograf.Logger) http.HandlerFunc {
	routes := getRoutesResponse{
		Sources:    "/chronograf/v1/sources",
		Layouts:    "/chronograf/v1/layouts",
		Me:         "/chronograf/v1/me",
		Mappings:   "/chronograf/v1/mappings",
		Dashboards: "/chronograf/v1/dashboards",
		Auth:       make([]AuthRoute, len(authRoutes)),
	}

	for i, route := range authRoutes {
		routes.Auth[i] = route
	}

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		encodeJSON(w, http.StatusOK, routes, logger)
		return
	})
}
<commit_msg>add kapacitors to AllRoutes<commit_after>package server

import (
	"net/http"

	"github.com/influxdata/chronograf"
)

// AuthRoute are the routes for each type of OAuth2 provider
type AuthRoute struct {
	Name     string `json:"name"`     // Name uniquely identifies the provider
	Label    string `json:"label"`    // Label is a user-facing string to present in the UI
	Login    string `json:"login"`    // Login is the route to the login redirect path
	Logout   string `json:"logout"`   // Logout is the route to the logout redirect path
	Callback string `json:"callback"` // Callback is the route the provider calls to exchange the code/state
}

// AuthRoutes contains all OAuth2 provider routes.
type AuthRoutes []AuthRoute

// Lookup searches all the routes for a specific provider
func (r *AuthRoutes) Lookup(provider string) (AuthRoute, bool) {
	for _, route := range *r {
		if route.Name == provider {
			return route, true
		}
	}
	return AuthRoute{}, false
}

type getRoutesResponse struct {
	Layouts    string      `json:"layouts"`    // Location of the layouts endpoint
	Mappings   string      `json:"mappings"`   // Location of the application mappings endpoint
	Sources    string      `json:"sources"`    // Location of the sources endpoint
	Me         string      `json:"me"`         // Location of the me endpoint
	Dashboards string      `json:"dashboards"` // Location of the dashboards endpoint
	Kapacitors string      `json:"kapacitors"` // Location of the kapacitors endpoint
	Auth       []AuthRoute `json:"auth"`       // Location of all auth routes.
}

// AllRoutes returns all top level routes within chronograf
func AllRoutes(authRoutes []AuthRoute, logger chronograf.Logger) http.HandlerFunc {
	routes := getRoutesResponse{
		Sources:    "/chronograf/v1/sources",
		Layouts:    "/chronograf/v1/layouts",
		Me:         "/chronograf/v1/me",
		Mappings:   "/chronograf/v1/mappings",
		Dashboards: "/chronograf/v1/dashboards",
		Kapacitors: "/chronograf/v1/kapacitors",
		Auth:       make([]AuthRoute, len(authRoutes)),
	}

	for i, route := range authRoutes {
		routes.Auth[i] = route
	}

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		encodeJSON(w, http.StatusOK, routes, logger)
		return
	})
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"net"
	"net/http"
	"os"
	"sync"
	"time"

	"code.google.com/p/gogoprotobuf/proto"
	"github.com/cloudfoundry-incubator/garden/routes"
	"github.com/cloudfoundry-incubator/garden/server/bomberman"
	"github.com/cloudfoundry-incubator/garden/warden"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/rata"
)

type WardenServer struct {
	logger lager.Logger

	server        http.Server
	listenNetwork string
	listenAddr    string

	containerGraceTime time.Duration
	backend            warden.Backend

	listener net.Listener
	handling *sync.WaitGroup

	stopping chan bool

	bomberman *bomberman.Bomberman

	conns map[net.Conn]net.Conn
	mu    sync.Mutex
}

type UnhandledRequestError struct {
	Request proto.Message
}

func (e UnhandledRequestError) Error() string {
	return fmt.Sprintf("unhandled request type: %T", e.Request)
}

func New(
	listenNetwork, listenAddr string,
	containerGraceTime time.Duration,
	backend warden.Backend,
	logger lager.Logger,
) *WardenServer {
	s := &WardenServer{
		logger: logger.Session("garden-server"),

		listenNetwork: listenNetwork,
		listenAddr:    listenAddr,

		containerGraceTime: containerGraceTime,
		backend:            backend,

		stopping: make(chan bool),

		handling: new(sync.WaitGroup),
		conns:    make(map[net.Conn]net.Conn),
	}

	handlers := map[string]http.Handler{
		routes.Ping:                   http.HandlerFunc(s.handlePing),
		routes.Capacity:               http.HandlerFunc(s.handleCapacity),
		routes.Create:                 http.HandlerFunc(s.handleCreate),
		routes.Destroy:                http.HandlerFunc(s.handleDestroy),
		routes.List:                   http.HandlerFunc(s.handleList),
		routes.Stop:                   http.HandlerFunc(s.handleStop),
		routes.StreamIn:               http.HandlerFunc(s.handleStreamIn),
		routes.StreamOut:              http.HandlerFunc(s.handleStreamOut),
		routes.LimitBandwidth:         http.HandlerFunc(s.handleLimitBandwidth),
		routes.CurrentBandwidthLimits: http.HandlerFunc(s.handleCurrentBandwidthLimits),
		routes.LimitCPU:               http.HandlerFunc(s.handleLimitCPU),
		routes.CurrentCPULimits:       http.HandlerFunc(s.handleCurrentCPULimits),
		routes.LimitDisk:              http.HandlerFunc(s.handleLimitDisk),
		routes.CurrentDiskLimits:      http.HandlerFunc(s.handleCurrentDiskLimits),
		routes.LimitMemory:            http.HandlerFunc(s.handleLimitMemory),
		routes.CurrentMemoryLimits:    http.HandlerFunc(s.handleCurrentMemoryLimits),
		routes.NetIn:                  http.HandlerFunc(s.handleNetIn),
		routes.NetOut:                 http.HandlerFunc(s.handleNetOut),
		routes.Info:                   http.HandlerFunc(s.handleInfo),
		routes.Run:                    http.HandlerFunc(s.handleRun),
		routes.Attach:                 http.HandlerFunc(s.handleAttach),
	}

	mux, err := rata.NewRouter(routes.Routes, handlers)
	if err != nil {
		logger.Fatal("failed-to-initialize-rata", err)
	}

	conLogger := logger.Session("connection")

	s.server = http.Server{
		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			mux.ServeHTTP(w, r)
		}),

		ConnState: func(conn net.Conn, state http.ConnState) {
			switch state {
			case http.StateNew:
				conLogger.Info("open", lager.Data{"local_addr": conn.LocalAddr(), "remote_addr": conn.RemoteAddr()})
				s.handling.Add(1)
			case http.StateActive:
				s.mu.Lock()
				delete(s.conns, conn)
				s.mu.Unlock()
			case http.StateIdle:
				select {
				case <-s.stopping:
					conn.Close()
				default:
					s.mu.Lock()
					s.conns[conn] = conn
					s.mu.Unlock()
				}
			case http.StateHijacked, http.StateClosed:
				s.mu.Lock()
				delete(s.conns, conn)
				s.mu.Unlock()
				conLogger.Info("closed", lager.Data{"local_addr": conn.LocalAddr(), "remote_addr": conn.RemoteAddr()})
				s.handling.Done()
			}
		},
	}

	return s
}

func (s *WardenServer) Start() error {
	err := s.removeExistingSocket()
	if err != nil {
		return err
	}

	err = s.backend.Start()
	if err != nil {
		return err
	}

	listener, err := net.Listen(s.listenNetwork, s.listenAddr)
	if err != nil {
		return err
	}

	s.listener = listener

	if s.listenNetwork == "unix" {
		os.Chmod(s.listenAddr, 0777)
	}

	containers, err := s.backend.Containers(nil)
	if err != nil {
		return err
	}

	s.bomberman = bomberman.New(s.backend, s.reapContainer)

	for _, container := range containers {
		s.bomberman.Strap(container)
	}

	go s.server.Serve(listener)

	return nil
}

func (s *WardenServer) Stop() {
	close(s.stopping)

	s.listener.Close()

	s.mu.Lock()
	conns := s.conns
	s.conns = make(map[net.Conn]net.Conn)
	s.mu.Unlock()

	for _, c := range conns {
		s.logger.Debug("closing-idle", lager.Data{
			"addr": c.RemoteAddr(),
		})

		c.Close()
	}

	s.logger.Info("waiting-for-connections-to-close")
	s.handling.Wait()

	s.logger.Info("stopping-backend")
	s.backend.Stop()

	s.logger.Info("stopped")
}

func (s *WardenServer) removeExistingSocket() error {
	if s.listenNetwork != "unix" {
		return nil
	}

	if _, err := os.Stat(s.listenAddr); os.IsNotExist(err) {
		return nil
	}

	err := os.Remove(s.listenAddr)

	if err != nil {
		return fmt.Errorf("error deleting existing socket: %s", err)
	}

	return nil
}

func (s *WardenServer) reapContainer(container warden.Container) {
	s.logger.Info("reaping", lager.Data{
		"handle":     container.Handle(),
		"grace-time": s.backend.GraceTime(container).String(),
	})

	s.backend.Destroy(container.Handle())
}
<commit_msg>reduce connection log levels to debug<commit_after>package server

import (
	"fmt"
	"net"
	"net/http"
	"os"
	"sync"
	"time"

	"code.google.com/p/gogoprotobuf/proto"
	"github.com/cloudfoundry-incubator/garden/routes"
	"github.com/cloudfoundry-incubator/garden/server/bomberman"
	"github.com/cloudfoundry-incubator/garden/warden"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/rata"
)

type WardenServer struct {
	logger lager.Logger

	server        http.Server
	listenNetwork string
	listenAddr    string

	containerGraceTime time.Duration
	backend            warden.Backend

	listener net.Listener
	handling *sync.WaitGroup

	stopping chan bool

	bomberman *bomberman.Bomberman

	conns map[net.Conn]net.Conn
	mu    sync.Mutex
}

type UnhandledRequestError struct {
	Request proto.Message
}

func (e UnhandledRequestError) Error() string {
	return fmt.Sprintf("unhandled request type: %T", e.Request)
}

func New(
	listenNetwork, listenAddr string,
	containerGraceTime time.Duration,
	backend warden.Backend,
	logger lager.Logger,
) *WardenServer {
	s := &WardenServer{
		logger: logger.Session("garden-server"),

		listenNetwork: listenNetwork,
		listenAddr:    listenAddr,

		containerGraceTime: containerGraceTime,
		backend:            backend,

		stopping: make(chan bool),

		handling: new(sync.WaitGroup),
		conns:    make(map[net.Conn]net.Conn),
	}

	handlers := map[string]http.Handler{
		routes.Ping:                   http.HandlerFunc(s.handlePing),
		routes.Capacity:               http.HandlerFunc(s.handleCapacity),
		routes.Create:                 http.HandlerFunc(s.handleCreate),
		routes.Destroy:                http.HandlerFunc(s.handleDestroy),
		routes.List:                   http.HandlerFunc(s.handleList),
		routes.Stop:                   http.HandlerFunc(s.handleStop),
		routes.StreamIn:               http.HandlerFunc(s.handleStreamIn),
		routes.StreamOut:              http.HandlerFunc(s.handleStreamOut),
		routes.LimitBandwidth:         http.HandlerFunc(s.handleLimitBandwidth),
		routes.CurrentBandwidthLimits: http.HandlerFunc(s.handleCurrentBandwidthLimits),
		routes.LimitCPU:               http.HandlerFunc(s.handleLimitCPU),
		routes.CurrentCPULimits:       http.HandlerFunc(s.handleCurrentCPULimits),
		routes.LimitDisk:              http.HandlerFunc(s.handleLimitDisk),
		routes.CurrentDiskLimits:      http.HandlerFunc(s.handleCurrentDiskLimits),
		routes.LimitMemory:            http.HandlerFunc(s.handleLimitMemory),
		routes.CurrentMemoryLimits:    http.HandlerFunc(s.handleCurrentMemoryLimits),
		routes.NetIn:                  http.HandlerFunc(s.handleNetIn),
		routes.NetOut:                 http.HandlerFunc(s.handleNetOut),
		routes.Info:                   http.HandlerFunc(s.handleInfo),
		routes.Run:                    http.HandlerFunc(s.handleRun),
		routes.Attach:                 http.HandlerFunc(s.handleAttach),
	}

	mux, err := rata.NewRouter(routes.Routes, handlers)
	if err != nil {
		logger.Fatal("failed-to-initialize-rata", err)
	}

	conLogger := logger.Session("connection")

	s.server = http.Server{
		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			mux.ServeHTTP(w, r)
		}),

		ConnState: func(conn net.Conn, state http.ConnState) {
			switch state {
			case http.StateNew:
				conLogger.Debug("open", lager.Data{"local_addr": conn.LocalAddr(), "remote_addr": conn.RemoteAddr()})
				s.handling.Add(1)
			case http.StateActive:
				s.mu.Lock()
				delete(s.conns, conn)
				s.mu.Unlock()
			case http.StateIdle:
				select {
				case <-s.stopping:
					conn.Close()
				default:
					s.mu.Lock()
					s.conns[conn] = conn
					s.mu.Unlock()
				}
			case http.StateHijacked, http.StateClosed:
				s.mu.Lock()
				delete(s.conns, conn)
				s.mu.Unlock()
				conLogger.Debug("closed", lager.Data{"local_addr": conn.LocalAddr(), "remote_addr": conn.RemoteAddr()})
				s.handling.Done()
			}
		},
	}

	return s
}

func (s *WardenServer) Start() error {
	err := s.removeExistingSocket()
	if err != nil {
		return err
	}

	err = s.backend.Start()
	if err != nil {
		return err
	}

	listener, err := net.Listen(s.listenNetwork, s.listenAddr)
	if err != nil {
		return err
	}

	s.listener = listener

	if s.listenNetwork == "unix" {
		os.Chmod(s.listenAddr, 0777)
	}

	containers, err := s.backend.Containers(nil)
	if err != nil {
		return err
	}

	s.bomberman = bomberman.New(s.backend, s.reapContainer)

	for _, container := range containers {
		s.bomberman.Strap(container)
	}

	go s.server.Serve(listener)

	return nil
}

func (s *WardenServer) Stop() {
	close(s.stopping)

	s.listener.Close()

	s.mu.Lock()
	conns := s.conns
	s.conns = make(map[net.Conn]net.Conn)
	s.mu.Unlock()

	for _, c := range conns {
		s.logger.Debug("closing-idle", lager.Data{
			"addr": c.RemoteAddr(),
		})

		c.Close()
	}

	s.logger.Info("waiting-for-connections-to-close")
	s.handling.Wait()

	s.logger.Info("stopping-backend")
	s.backend.Stop()

	s.logger.Info("stopped")
}

func (s *WardenServer) removeExistingSocket() error {
	if s.listenNetwork != "unix" {
		return nil
	}

	if _, err := os.Stat(s.listenAddr); os.IsNotExist(err) {
		return nil
	}

	err := os.Remove(s.listenAddr)

	if err != nil {
		return fmt.Errorf("error deleting existing socket: %s", err)
	}

	return nil
}

func (s *WardenServer) reapContainer(container warden.Container) {
	s.logger.Info("reaping", lager.Data{
		"handle":     container.Handle(),
		"grace-time": s.backend.GraceTime(container).String(),
	})

	s.backend.Destroy(container.Handle())
}
<|endoftext|>


<commit_before>package server

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"time"

	"github.com/99designs/aws-vault/vault"
)

const (
	metadataBind    = "169.254.169.254:80"
	awsTimeFormat   = "2006-01-02T15:04:05Z"
	localServerUrl  = "http://127.0.0.1:9099"
	localServerBind = "127.0.0.1:9099"
)

func StartMetadataServer() error {
	if _, err := installNetworkAlias(); err != nil {
		return err
	}

	router := http.NewServeMux()
	router.HandleFunc("/latest/meta-data/iam/security-credentials/", indexHandler)
	router.HandleFunc("/latest/meta-data/iam/security-credentials/local-credentials", credentialsHandler)
	// The AWS Go SDK checks the instance-id endpoint to validate the existence of EC2 Metadata
	router.HandleFunc("/latest/meta-data/instance-id/", instanceIdHandler)

	l, err := net.Listen("tcp", metadataBind)
	if err != nil {
		return err
	}

	log.Printf("Local instance role server running on %s", l.Addr())
	return http.Serve(l, router)
}

type metadataHandler struct {
	http.Handler
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "local-credentials")
}

func credentialsHandler(w http.ResponseWriter, r *http.Request) {
	resp, err := http.Get(localServerUrl)
	if err != nil {
		http.Error(w, err.Error(), http.StatusGatewayTimeout)
		return
	}
	defer resp.Body.Close()

	log.Printf("Fetched credentials from %s", localServerUrl)

	w.Header().Set("Content-Type", "text/plain")
	w.WriteHeader(http.StatusOK)

	io.Copy(w, resp.Body)
}

func instanceIdHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "aws-vault")
}

func checkServerRunning(bind string) bool {
	_, err := net.DialTimeout("tcp", bind, time.Millisecond*10)
	return err == nil
}

func StartCredentialsServer(creds *vault.VaultCredentials) error {
	if !checkServerRunning(metadataBind) {
		log.Printf("Starting `aws-vault server` as root in the background")
		cmd := exec.Command("sudo", "-b", os.Args[0], "server")
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err != nil {
			return err
		}
	}

	l, err := net.Listen("tcp", localServerBind)
	if err != nil {
		return err
	}

	log.Printf("Local instance role server running on %s", l.Addr())
	go http.Serve(l, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Printf("Credentials.IsExpired() = %#v", creds.IsExpired())

		val, err := creds.Get()
		if err != nil {
			http.Error(w, err.Error(), http.StatusGatewayTimeout)
			return
		}

		log.Printf("Serving credentials via http ****************%s, expiration of %s (%s)",
			val.AccessKeyID[len(val.AccessKeyID)-4:],
			creds.Expires().Format(awsTimeFormat),
			creds.Expires().Sub(time.Now()).String())

		json.NewEncoder(w).Encode(map[string]interface{}{
			"Code":            "Success",
			"LastUpdated":     time.Now().Format(awsTimeFormat),
			"Type":            "AWS-HMAC",
			"AccessKeyId":     val.AccessKeyID,
			"SecretAccessKey": val.SecretAccessKey,
			"Token":           val.SessionToken,
			"Expiration":      creds.Expires().Format(awsTimeFormat),
		})
	}))

	return nil
}
<commit_msg>support for windows to start the server component in the background<commit_after>package server

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"time"

	"github.com/99designs/aws-vault/vault"
)

const (
	metadataBind    = "169.254.169.254:80"
	awsTimeFormat   = "2006-01-02T15:04:05Z"
	localServerUrl  = "http://127.0.0.1:9099"
	localServerBind = "127.0.0.1:9099"
)

func StartMetadataServer() error {
	if _, err := installNetworkAlias(); err != nil {
		return err
	}

	router := http.NewServeMux()
	router.HandleFunc("/latest/meta-data/iam/security-credentials/", indexHandler)
	router.HandleFunc("/latest/meta-data/iam/security-credentials/local-credentials", credentialsHandler)
	// The AWS Go SDK checks the instance-id endpoint to validate the existence of EC2 Metadata
	router.HandleFunc("/latest/meta-data/instance-id/", instanceIdHandler)

	l, err := net.Listen("tcp", metadataBind)
	if err != nil {
		return err
	}

	log.Printf("Local instance role server running on %s", l.Addr())
	return http.Serve(l, router)
}

type metadataHandler struct {
	http.Handler
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "local-credentials")
}

func credentialsHandler(w http.ResponseWriter, r *http.Request) {
	resp, err := http.Get(localServerUrl)
	if err != nil {
		http.Error(w, err.Error(), http.StatusGatewayTimeout)
		return
	}
	defer resp.Body.Close()

	log.Printf("Fetched credentials from %s", localServerUrl)

	w.Header().Set("Content-Type", "text/plain")
	w.WriteHeader(http.StatusOK)

	io.Copy(w, resp.Body)
}

func instanceIdHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "aws-vault")
}

func checkServerRunning(bind string) bool {
	_, err := net.DialTimeout("tcp", bind, time.Millisecond*10)
	return err == nil
}

func StartCredentialsServer(creds *vault.VaultCredentials) error {
	if !checkServerRunning(metadataBind) {
		if runtime.GOOS == "windows" {
			log.Printf("Starting `aws-vault server` in the background")
			cmd := exec.Command(os.Args[0], "server")
			cmd.Stdin = os.Stdin
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			if err := cmd.Start(); err != nil {
				return err
			}
		} else {
			log.Printf("Starting `aws-vault server` as root in the background")
			cmd := exec.Command("sudo", "-b", os.Args[0], "server")
			cmd.Stdin = os.Stdin
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			if err := cmd.Run(); err != nil {
				return err
			}
		}
	}

	l, err := net.Listen("tcp", localServerBind)
	if err != nil {
		return err
	}

	log.Printf("Local instance role server running on %s", l.Addr())
	go http.Serve(l, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Printf("Credentials.IsExpired() = %#v", creds.IsExpired())

		val, err := creds.Get()
		if err != nil {
			http.Error(w, err.Error(), http.StatusGatewayTimeout)
			return
		}

		log.Printf("Serving credentials via http ****************%s, expiration of %s (%s)",
			val.AccessKeyID[len(val.AccessKeyID)-4:],
			creds.Expires().Format(awsTimeFormat),
			creds.Expires().Sub(time.Now()).String())

		json.NewEncoder(w).Encode(map[string]interface{}{
			"Code":            "Success",
			"LastUpdated":     time.Now().Format(awsTimeFormat),
			"Type":            "AWS-HMAC",
			"AccessKeyId":     val.AccessKeyID,
			"SecretAccessKey": val.SecretAccessKey,
			"Token":           val.SessionToken,
			"Expiration":      creds.Expires().Format(awsTimeFormat),
		})
	}))

	return nil
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"github.com/apex/log"
	"github.com/creasty/defaults"
	"github.com/patrickmn/go-cache"
	"github.com/pkg/errors"
	"github.com/pterodactyl/wings/api"
	"github.com/pterodactyl/wings/config"
	"github.com/remeh/sizedwaitgroup"
	"math"
	"os"
	"strings"
	"sync"
	"time"
)

var servers *Collection

func GetServers() *Collection {
	return servers
}

// High level definition for a server instance being controlled by Wings.
type Server struct {
	// The unique identifier for the server that should be used when referencing
	// it against the Panel API (and internally). This will be used when naming
	// docker containers as well as in log output.
	Uuid string `json:"uuid"`

	// Whether or not the server is in a suspended state. Suspended servers cannot
	// be started or modified except in certain scenarios by an admin user.
	Suspended bool `json:"suspended"`

	// The power state of the server.
	State string `default:"offline" json:"state"`

	// The command that should be used when booting up the server instance.
	Invocation string `json:"invocation"`

	// An array of environment variables that should be passed along to the running
	// server process.
	EnvVars map[string]string `json:"environment" yaml:"environment"`

	Archiver       Archiver       `json:"-" yaml:"-"`
	CrashDetection CrashDetection `json:"crash_detection" yaml:"crash_detection"`
	Build          BuildSettings  `json:"build"`
	Allocations    Allocations    `json:"allocations"`
	Environment    Environment    `json:"-" yaml:"-"`
	Filesystem     Filesystem     `json:"-" yaml:"-"`
	Resources      ResourceUsage  `json:"resources" yaml:"-"`

	Container struct {
		// Defines the Docker image that will be used for this server
		Image string `json:"image,omitempty"`
		// If set to true, OOM killer will be disabled on the server's Docker container.
		// If not present (nil) we will default to disabling it.
		OomDisabled bool `default:"true" json:"oom_disabled" yaml:"oom_disabled"`
	} `json:"container,omitempty"`

	// Server cache used to store frequently requested information in memory and make
	// certain long operations return faster. For example, FS disk space usage.
	Cache *cache.Cache `json:"-" yaml:"-"`

	// Events emitted by the server instance.
	emitter *EventBus

	// Defines the process configuration for the server instance. This is dynamically
	// fetched from the Pterodactyl Server instance each time the server process is
	// started, and then cached here.
	processConfiguration *api.ProcessConfiguration

	// Internal mutex used to block actions that need to occur sequentially, such as
	// writing the configuration to the disk.
	sync.RWMutex
}

// The build settings for a given server that impact docker container creation and
// resource limits for a server instance.
type BuildSettings struct {
	// The total amount of memory in megabytes that this server is allowed to
	// use on the host system.
	MemoryLimit int64 `json:"memory_limit" yaml:"memory"`

	// The amount of additional swap space to be provided to a container instance.
	Swap int64 `json:"swap"`

	// The relative weight for IO operations in a container. This is relative to other
	// containers on the system and should be a value between 10 and 1000.
	IoWeight uint16 `json:"io_weight" yaml:"io"`

	// The percentage of CPU that this instance is allowed to consume relative to
	// the host. A value of 200% represents complete utilization of two cores. This
	// should be a value between 1 and THREAD_COUNT * 100.
	CpuLimit int64 `json:"cpu_limit" yaml:"cpu"`

	// The amount of disk space in megabytes that a server is allowed to use.
	DiskSpace int64 `json:"disk_space" yaml:"disk"`

	// Sets which CPU threads can be used by the docker instance.
	Threads string `json:"threads" yaml:"threads"`
}

// Converts the CPU limit for a server build into a number that can be better understood
// by the Docker environment. If there is no limit set, return -1 which will indicate to
// Docker that it has unlimited CPU quota.
func (b *BuildSettings) ConvertedCpuLimit() int64 {
	if b.CpuLimit == 0 {
		return -1
	}

	return b.CpuLimit * 1000
}

// Set the hard limit for memory usage to be 5% more than the amount of memory assigned to
// the server. If the memory limit for the server is < 4G, use 10%, if less than 2G use
// 15%. This avoids unexpected crashes from processes like Java which run over the limit.
func (b *BuildSettings) MemoryOverheadMultiplier() float64 {
	if b.MemoryLimit <= 2048 {
		return 1.15
	} else if b.MemoryLimit <= 4096 {
		return 1.10
	}

	return 1.05
}

func (b *BuildSettings) BoundedMemoryLimit() int64 {
	return int64(math.Round(float64(b.MemoryLimit) * b.MemoryOverheadMultiplier() * 1_000_000))
}

// Returns the amount of swap available as a total in bytes. This is returned as the amount
// of memory available to the server initially, PLUS the amount of additional swap to include
// which is the format used by Docker.
func (b *BuildSettings) ConvertedSwap() int64 {
	if b.Swap < 0 {
		return -1
	}

	return (b.Swap * 1_000_000) + b.BoundedMemoryLimit()
}

// Defines the allocations available for a given server. When using the Docker environment
// driver these correspond to mappings for the container that allow external connections.
type Allocations struct {
	// Defines the default allocation that should be used for this server. This is
	// what will be used for {SERVER_IP} and {SERVER_PORT} when modifying configuration
	// files or the startup arguments for a server.
	DefaultMapping struct {
		Ip   string `json:"ip"`
		Port int    `json:"port"`
	} `json:"default" yaml:"default"`

	// Mappings contains all of the ports that should be assigned to a given server
	// attached to the IP they correspond to.
	Mappings map[string][]int `json:"mappings"`
}

// Iterates over a given directory and loads all of the servers listed before returning
// them to the calling function.
func LoadDirectory() error {
	// We could theoretically use a standard wait group here, however doing
	// that introduces the potential to crash the program due to too many
	// open files. This wouldn't happen on a small setup, but once the daemon is
	// handling many servers you run that risk.
	//
	// For now just process 10 files at a time, that should be plenty fast to
	// read and parse the YAML. We should probably make this configurable down
	// the road to help big instances scale better.
	wg := sizedwaitgroup.New(10)

	configs, rerr, err := api.NewRequester().GetAllServerConfigurations()
	if err != nil || rerr != nil {
		if err != nil {
			return errors.WithStack(err)
		}

		return errors.New(rerr.String())
	}

	states, err := getServerStates()
	if err != nil {
		return errors.WithStack(err)
	}

	servers = NewCollection(nil)

	for uuid, data := range configs {
		wg.Add()

		go func(uuid string, data *api.ServerConfigurationResponse) {
			defer wg.Done()

			s, err := FromConfiguration(data)
			if err != nil {
				log.WithField("server", uuid).WithField("error", err).Error("failed to load server, skipping...")
				return
			}

			if state, exists := states[s.Uuid]; exists {
				s.SetState(state)
				s.Log().WithField("state", s.GetState()).Debug("loaded server state from cache file")
			}

			servers.Add(s)
		}(uuid, data)
	}

	// Wait until we've processed all of the configuration files in the directory
	// before continuing.
	wg.Wait()

	return nil
}

// Initializes a server using a data byte array. This will be marshaled into the
// given struct using a YAML marshaler. This will also configure the given environment
// for a server.
func FromConfiguration(data *api.ServerConfigurationResponse) (*Server, error) {
	s := new(Server)

	if err := defaults.Set(s); err != nil {
		return nil, err
	}

	if err := s.UpdateDataStructure(data.Settings, false); err != nil {
		return nil, err
	}

	s.AddEventListeners()

	// Right now we only support a Docker based environment, so I'm going to hard code
	// this logic in. When we're ready to support other environment we'll need to make
	// some modifications here obviously.
	if err := NewDockerEnvironment(s); err != nil {
		return nil, err
	}

	s.Cache = cache.New(time.Minute*10, time.Minute*15)
	s.Archiver = Archiver{
		Server: s,
	}
	s.Filesystem = Filesystem{
		Configuration: &config.Get().System,
		Server:        s,
	}
	s.Resources = ResourceUsage{}

	// Force the disk usage to become cached to return in a resources response
	// or when connecting to the websocket of an offline server.
	go s.Filesystem.HasSpaceAvailable()

	// Forces the configuration to be synced with the panel.
	if err := s.SyncWithConfiguration(data); err != nil {
		return nil, err
	}

	return s, nil
}

// Returns all of the environment variables that should be assigned to a running
// server instance.
func (s *Server) GetEnvironmentVariables() []string {
	zone, _ := time.Now().In(time.Local).Zone()

	var out = []string{
		fmt.Sprintf("TZ=%s", zone),
		fmt.Sprintf("STARTUP=%s", s.Invocation),
		fmt.Sprintf("SERVER_MEMORY=%d", s.Build.MemoryLimit),
		fmt.Sprintf("SERVER_IP=%s", s.Allocations.DefaultMapping.Ip),
		fmt.Sprintf("SERVER_PORT=%d", s.Allocations.DefaultMapping.Port),
	}

eloop:
	for k, v := range s.EnvVars {
		for _, e := range out {
			if strings.HasPrefix(e, strings.ToUpper(k)) {
				continue eloop
			}
		}

		out = append(out, fmt.Sprintf("%s=%s", strings.ToUpper(k), v))
	}

	return out
}

func (s *Server) Log() *log.Entry {
	return log.WithField("server", s.Uuid)
}

// Syncs the state of the server on the Panel with Wings. This ensures that we're always
// using the state of the server from the Panel and allows us to not require successful
// API calls to Wings to do things.
//
// This also means mass actions can be performed against servers on the Panel and they
// will automatically sync with Wings when the server is started.
func (s *Server) Sync() error {
	cfg, rerr, err := s.GetProcessConfiguration()
	if err != nil || rerr != nil {
		if err != nil {
			return errors.WithStack(err)
		}

		if rerr.Status == "404" {
			return &serverDoesNotExist{}
		}

		return errors.New(rerr.String())
	}

	return s.SyncWithConfiguration(cfg)
}

func (s *Server) SyncWithConfiguration(cfg *api.ServerConfigurationResponse) error {
	// Update the data structure and persist it to the disk.
	if err := s.UpdateDataStructure(cfg.Settings, false); err != nil {
		return errors.WithStack(err)
	}

	s.processConfiguration = cfg.ProcessConfiguration
	return nil
}

// Reads the log file for a server up to a specified number of bytes.
func (s *Server) ReadLogfile(len int64) ([]string, error) {
	return s.Environment.Readlog(len)
}

// Determine if the server is bootable in it's current state or not. This will not
// indicate why a server is not bootable, only if it is.
func (s *Server) IsBootable() bool {
	exists, _ := s.Environment.Exists()

	return exists
}

// Initalizes a server instance. This will run through and ensure that the environment
// for the server is setup, and that all of the necessary files are created.
func (s *Server) CreateEnvironment() error {
	return s.Environment.Create()
}

// Gets the process configuration data for the server.
func (s *Server) GetProcessConfiguration() (*api.ServerConfigurationResponse, *api.RequestError, error) {
	return api.NewRequester().GetServerConfiguration(s.Uuid)
}

// Helper function that can receieve a power action and then process the
// actions that need to occur for it.
func (s *Server) HandlePowerAction(action PowerAction) error {
	switch action.Action {
	case "start":
		return s.Environment.Start()
	case "restart":
		if err := s.Environment.WaitForStop(60, false); err != nil {
			return err
		}

		return s.Environment.Start()
	case "stop":
		return s.Environment.Stop()
	case "kill":
		return s.Environment.Terminate(os.Kill)
	default:
		return errors.New("an invalid power action was provided")
	}
}
<commit_msg>Don't try to send space available when loading from a configuration<commit_after>package server

import (
	"fmt"
	"github.com/apex/log"
	"github.com/creasty/defaults"
	"github.com/patrickmn/go-cache"
	"github.com/pkg/errors"
	"github.com/pterodactyl/wings/api"
	"github.com/pterodactyl/wings/config"
	"github.com/remeh/sizedwaitgroup"
	"math"
	"os"
	"strings"
	"sync"
	"time"
)

var servers *Collection

func GetServers() *Collection {
	return servers
}

// High level definition for a server instance being controlled by Wings.
type Server struct {
	// The unique identifier for the server that should be used when referencing
	// it against the Panel API (and internally). This will be used when naming
	// docker containers as well as in log output.
	Uuid string `json:"uuid"`

	// Whether or not the server is in a suspended state. Suspended servers cannot
	// be started or modified except in certain scenarios by an admin user.
	Suspended bool `json:"suspended"`

	// The power state of the server.
	State string `default:"offline" json:"state"`

	// The command that should be used when booting up the server instance.
	Invocation string `json:"invocation"`

	// An array of environment variables that should be passed along to the running
	// server process.
	EnvVars map[string]string `json:"environment" yaml:"environment"`

	Archiver       Archiver       `json:"-" yaml:"-"`
	CrashDetection CrashDetection `json:"crash_detection" yaml:"crash_detection"`
	Build          BuildSettings  `json:"build"`
	Allocations    Allocations    `json:"allocations"`
	Environment    Environment    `json:"-" yaml:"-"`
	Filesystem     Filesystem     `json:"-" yaml:"-"`
	Resources      ResourceUsage  `json:"resources" yaml:"-"`

	Container struct {
		// Defines the Docker image that will be used for this server
		Image string `json:"image,omitempty"`
		// If set to true, OOM killer will be disabled on the server's Docker container.
		// If not present (nil) we will default to disabling it.
		OomDisabled bool `default:"true" json:"oom_disabled" yaml:"oom_disabled"`
	} `json:"container,omitempty"`

	// Server cache used to store frequently requested information in memory and make
	// certain long operations return faster. For example, FS disk space usage.
	Cache *cache.Cache `json:"-" yaml:"-"`

	// Events emitted by the server instance.
	emitter *EventBus

	// Defines the process configuration for the server instance. This is dynamically
	// fetched from the Pterodactyl Server instance each time the server process is
	// started, and then cached here.
	processConfiguration *api.ProcessConfiguration

	// Internal mutex used to block actions that need to occur sequentially, such as
	// writing the configuration to the disk.
	sync.RWMutex
}

// The build settings for a given server that impact docker container creation and
// resource limits for a server instance.
type BuildSettings struct {
	// The total amount of memory in megabytes that this server is allowed to
	// use on the host system.
	MemoryLimit int64 `json:"memory_limit" yaml:"memory"`

	// The amount of additional swap space to be provided to a container instance.
	Swap int64 `json:"swap"`

	// The relative weight for IO operations in a container. This is relative to other
	// containers on the system and should be a value between 10 and 1000.
	IoWeight uint16 `json:"io_weight" yaml:"io"`

	// The percentage of CPU that this instance is allowed to consume relative to
	// the host. A value of 200% represents complete utilization of two cores. This
	// should be a value between 1 and THREAD_COUNT * 100.
	CpuLimit int64 `json:"cpu_limit" yaml:"cpu"`

	// The amount of disk space in megabytes that a server is allowed to use.
	DiskSpace int64 `json:"disk_space" yaml:"disk"`

	// Sets which CPU threads can be used by the docker instance.
	Threads string `json:"threads" yaml:"threads"`
}

// Converts the CPU limit for a server build into a number that can be better understood
// by the Docker environment. If there is no limit set, return -1 which will indicate to
// Docker that it has unlimited CPU quota.
func (b *BuildSettings) ConvertedCpuLimit() int64 {
	if b.CpuLimit == 0 {
		return -1
	}

	return b.CpuLimit * 1000
}

// Set the hard limit for memory usage to be 5% more than the amount of memory assigned to
// the server. If the memory limit for the server is < 4G, use 10%, if less than 2G use
// 15%. This avoids unexpected crashes from processes like Java which run over the limit.
func (b *BuildSettings) MemoryOverheadMultiplier() float64 {
	if b.MemoryLimit <= 2048 {
		return 1.15
	} else if b.MemoryLimit <= 4096 {
		return 1.10
	}

	return 1.05
}

func (b *BuildSettings) BoundedMemoryLimit() int64 {
	return int64(math.Round(float64(b.MemoryLimit) * b.MemoryOverheadMultiplier() * 1_000_000))
}

// Returns the amount of swap available as a total in bytes. This is returned as the amount
// of memory available to the server initially, PLUS the amount of additional swap to include
// which is the format used by Docker.
func (b *BuildSettings) ConvertedSwap() int64 {
	if b.Swap < 0 {
		return -1
	}

	return (b.Swap * 1_000_000) + b.BoundedMemoryLimit()
}

// Defines the allocations available for a given server. When using the Docker environment
// driver these correspond to mappings for the container that allow external connections.
type Allocations struct {
	// Defines the default allocation that should be used for this server. This is
	// what will be used for {SERVER_IP} and {SERVER_PORT} when modifying configuration
	// files or the startup arguments for a server.
	DefaultMapping struct {
		Ip   string `json:"ip"`
		Port int    `json:"port"`
	} `json:"default" yaml:"default"`

	// Mappings contains all of the ports that should be assigned to a given server
	// attached to the IP they correspond to.
	Mappings map[string][]int `json:"mappings"`
}

// Iterates over a given directory and loads all of the servers listed before returning
// them to the calling function.
func LoadDirectory() error {
	// We could theoretically use a standard wait group here, however doing
	// that introduces the potential to crash the program due to too many
	// open files. This wouldn't happen on a small setup, but once the daemon is
	// handling many servers you run that risk.
	//
	// For now just process 10 files at a time, that should be plenty fast to
	// read and parse the YAML. We should probably make this configurable down
	// the road to help big instances scale better.
	wg := sizedwaitgroup.New(10)

	configs, rerr, err := api.NewRequester().GetAllServerConfigurations()
	if err != nil || rerr != nil {
		if err != nil {
			return errors.WithStack(err)
		}

		return errors.New(rerr.String())
	}

	states, err := getServerStates()
	if err != nil {
		return errors.WithStack(err)
	}

	servers = NewCollection(nil)

	for uuid, data := range configs {
		wg.Add()

		go func(uuid string, data *api.ServerConfigurationResponse) {
			defer wg.Done()

			s, err := FromConfiguration(data)
			if err != nil {
				log.WithField("server", uuid).WithField("error", err).Error("failed to load server, skipping...")
				return
			}

			if state, exists := states[s.Uuid]; exists {
				s.SetState(state)
				s.Log().WithField("state", s.GetState()).Debug("loaded server state from cache file")
			}

			servers.Add(s)
		}(uuid, data)
	}

	// Wait until we've processed all of the configuration files in the directory
	// before continuing.
	wg.Wait()

	return nil
}

// Initializes a server using a data byte array. This will be marshaled into the
// given struct using a YAML marshaler. This will also configure the given environment
// for a server.
func FromConfiguration(data *api.ServerConfigurationResponse) (*Server, error) {
	s := new(Server)

	if err := defaults.Set(s); err != nil {
		return nil, err
	}

	if err := s.UpdateDataStructure(data.Settings, false); err != nil {
		return nil, err
	}

	s.AddEventListeners()

	// Right now we only support a Docker based environment, so I'm going to hard code
	// this logic in. When we're ready to support other environment we'll need to make
	// some modifications here obviously.
	if err := NewDockerEnvironment(s); err != nil {
		return nil, err
	}

	s.Cache = cache.New(time.Minute*10, time.Minute*15)
	s.Archiver = Archiver{
		Server: s,
	}
	s.Filesystem = Filesystem{
		Configuration: &config.Get().System,
		Server:        s,
	}
	s.Resources = ResourceUsage{}

	// Forces the configuration to be synced with the panel.
	if err := s.SyncWithConfiguration(data); err != nil {
		return nil, err
	}

	return s, nil
}

// Returns all of the environment variables that should be assigned to a running
// server instance.
func (s *Server) GetEnvironmentVariables() []string {
	zone, _ := time.Now().In(time.Local).Zone()

	var out = []string{
		fmt.Sprintf("TZ=%s", zone),
		fmt.Sprintf("STARTUP=%s", s.Invocation),
		fmt.Sprintf("SERVER_MEMORY=%d", s.Build.MemoryLimit),
		fmt.Sprintf("SERVER_IP=%s", s.Allocations.DefaultMapping.Ip),
		fmt.Sprintf("SERVER_PORT=%d", s.Allocations.DefaultMapping.Port),
	}

eloop:
	for k, v := range s.EnvVars {
		for _, e := range out {
			if strings.HasPrefix(e, strings.ToUpper(k)) {
				continue eloop
			}
		}

		out = append(out, fmt.Sprintf("%s=%s", strings.ToUpper(k), v))
	}

	return out
}

func (s *Server) Log() *log.Entry {
	return log.WithField("server", s.Uuid)
}

// Syncs the state of the server on the Panel with Wings. This ensures that we're always
// using the state of the server from the Panel and allows us to not require successful
// API calls to Wings to do things.
//
// This also means mass actions can be performed against servers on the Panel and they
// will automatically sync with Wings when the server is started.
func (s *Server) Sync() error {
	cfg, rerr, err := s.GetProcessConfiguration()
	if err != nil || rerr != nil {
		if err != nil {
			return errors.WithStack(err)
		}

		if rerr.Status == "404" {
			return &serverDoesNotExist{}
		}

		return errors.New(rerr.String())
	}

	return s.SyncWithConfiguration(cfg)
}

func (s *Server) SyncWithConfiguration(cfg *api.ServerConfigurationResponse) error {
	// Update the data structure and persist it to the disk.
	if err := s.UpdateDataStructure(cfg.Settings, false); err != nil {
		return errors.WithStack(err)
	}

	s.processConfiguration = cfg.ProcessConfiguration
	return nil
}

// Reads the log file for a server up to a specified number of bytes.
func (s *Server) ReadLogfile(len int64) ([]string, error) {
	return s.Environment.Readlog(len)
}

// Determine if the server is bootable in it's current state or not. This will not
// indicate why a server is not bootable, only if it is.
func (s *Server) IsBootable() bool {
	exists, _ := s.Environment.Exists()

	return exists
}

// Initalizes a server instance. This will run through and ensure that the environment
// for the server is setup, and that all of the necessary files are created.
func (s *Server) CreateEnvironment() error {
	return s.Environment.Create()
}

// Gets the process configuration data for the server.
func (s *Server) GetProcessConfiguration() (*api.ServerConfigurationResponse, *api.RequestError, error) {
	return api.NewRequester().GetServerConfiguration(s.Uuid)
}

// Helper function that can receieve a power action and then process the
// actions that need to occur for it.
func (s *Server) HandlePowerAction(action PowerAction) error {
	switch action.Action {
	case "start":
		return s.Environment.Start()
	case "restart":
		if err := s.Environment.WaitForStop(60, false); err != nil {
			return err
		}

		return s.Environment.Start()
	case "stop":
		return s.Environment.Stop()
	case "kill":
		return s.Environment.Terminate(os.Kill)
	default:
		return errors.New("an invalid power action was provided")
	}
}
<|endoftext|>


<commit_before>package factom

import (
	"bytes"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"fmt"

	ed "github.com/FactomProject/ed25519"
)

// An Identity is an array of names and a hierarchy of keys. It can assign/receive
// Attributes as JSON objects and rotate/replace its currently valid keys.
type Identity struct {
	ChainID string
	Name    []string
	Keys    []*IdentityKey
}

// GetIdentityChainID takes an identity name and returns its corresponding ChainID
func GetIdentityChainID(name []string) string {
	hs := sha256.New()
	for _, part := range name {
		h := sha256.Sum256([]byte(part))
		hs.Write(h[:])
	}
	return hex.EncodeToString(hs.Sum(nil))
}

// NewIdentityChain creates an returns a Chain struct for a new identity. Publish it to
// blockchain using the usual factom.CommitChain(...) and factom.RevealChain(...) calls.
func NewIdentityChain(name []string, keys []*IdentityKey) *Chain {
	e := &Entry{}
	for _, part := range name {
		e.ExtIDs = append(e.ExtIDs, []byte(part))
	}

	var publicKeys []string
	for _, key := range keys {
		publicKeys = append(publicKeys, key.PubString())
	}
	keysMap := map[string][]string{"keys": publicKeys}
	keysJSON, _ := json.Marshal(keysMap)
	e.Content = []byte(keysJSON)
	c := NewChain(e)
	return c
}

// GetKeysAtHeight returns the identity's public keys that were/are valid at the highest saved block height
func (i *Identity) GetKeysAtCurrentHeight() ([]*IdentityKey, error) {
	heights, err := GetHeights()
	if err != nil {
		return nil, err
	}
	return i.GetKeysAtHeight(heights.DirectoryBlockHeight)
}

// GetKeysAtHeight returns the identity's public keys that were valid at the specified block height
func (i *Identity) GetKeysAtHeight(height int64) ([]*IdentityKey, error) {
	entries, err := GetAllChainEntriesAtHeight(i.ChainID, height)
	if err != nil {
		return nil, err
	}

	var initialKeys map[string][]string
	initialKeysJSON := entries[0].Content
	err = json.Unmarshal(initialKeysJSON, &initialKeys)
	if err != nil {
		fmt.Println("Failed to unmarshal json from initial key declaration")
		return nil, err
	}

	var validKeys []*IdentityKey
	for _, pubString := range initialKeys["keys"] {
		pub, err := base64.StdEncoding.DecodeString(pubString)
		if err != nil || len(pub) != 32 {
			return nil, fmt.Errorf("invalid Identity public key string in first entry")
		}
		k := NewIdentityKey()
		copy(k.Pub[:], pub)
		validKeys = append(validKeys, k)
	}

	for _, e := range entries {
		if len(e.ExtIDs) < 4 || bytes.Compare(e.ExtIDs[0], []byte("ReplaceKey")) != 0 {
			continue
		}
		if len(e.ExtIDs[1]) != 32 || len(e.ExtIDs[2]) != 32 || len(e.ExtIDs[3]) != 64 {
			continue
		}

		var oldKey [32]byte
		oldPubString := string(e.ExtIDs[1])
		b, err := base64.StdEncoding.DecodeString(oldPubString)
		if err != nil {
			continue
		}
		copy(oldKey[:], b)

		var newKey [32]byte
		newPubString := string(e.ExtIDs[2])
		b, err = base64.StdEncoding.DecodeString(newPubString)
		if err != nil {
			continue
		}
		copy(newKey[:], b)

		var signature [64]byte
		copy(signature[:], e.ExtIDs[3])

		levelToReplace := -1
		for level, key := range validKeys {
			if bytes.Compare(oldKey[:], key.PubBytes()) == 0 {
				levelToReplace = level
			}
		}
		if levelToReplace == -1 {
			// oldkey not in the set of valid keys when this entry was published
			continue
		}

		var message []byte
		message = append(message, oldKey[:]...)
		message = append(message, newKey[:]...)
		for level, key := range validKeys {
			if level > levelToReplace {
				// low priority key trying to replace high priority key, disregard
				break
			}
			if ed.Verify(key.Pub, message, &signature) {
				validKeys[levelToReplace].Pub = &newKey
				break
			}
		}
	}
	return validKeys, nil
}

// ReplaceKey creates an entry in the given identity's chain saying that from this point (the current block),
// the old key will be considered invalid and the new key will be considered valid.
// In order to be recognized as a valid replacement, the identity key used to authorize the
// action must be of the same or higher priority than the one being replaced.
func (i *Identity) ReplaceKey(oldKey *IdentityKey, newKey *IdentityKey, signerKey *IdentityKey, ec *ECAddress) (string, error) {
	//publicKey := ed.GetPublicKey(privateKey)
	var message []byte
	message = append(message, []byte(oldKey.String())...)
	message = append(message, []byte(newKey.String())...)
	signature := signerKey.Sign(message)
	e := Entry{}
	e.ChainID = i.ChainID
	e.ExtIDs = [][]byte{[]byte("ReplaceKey"), []byte(oldKey.String()), []byte(newKey.String()), signature[:]}

	txID, err := CommitEntry(&e, ec)
	if err != nil {
		return "", err
	}
	_, err = RevealEntry(&e)
	if err != nil {
		return "", err
	}
	return txID, nil
}

// WriteAttribute creates an entry that assigns an attribute JSON object to a given identity, signs it
// an identity key, and appends that entry to the specified destination chain.
func WriteAttribute(receiverChainID string, destinationChainID string, attributesJSON string, privateKey *[64]byte, signerChainID string, ec *ECAddress) (string, []byte, error) {
	message := []byte(receiverChainID + destinationChainID + attributesJSON)
	signature := ed.Sign(privateKey, message)
	publicKey := ed.GetPublicKey(privateKey)

	e := Entry{}
	e.ChainID = destinationChainID
	e.ExtIDs = [][]byte{[]byte("IdentityAttribute"), []byte(receiverChainID), signature[:], publicKey[:], []byte(signerChainID)}
	e.Content = []byte(attributesJSON)

	txID, err := CommitEntry(&e, ec)
	if err != nil {
		return "", nil, err
	}
	_, err = RevealEntry(&e)
	if err != nil {
		return "", nil, err
	}
	return txID, e.Hash(), nil
}

// EndorseIdentityAttribute signs a message using the provided private key saying that the signing identity acknowledges/agrees with
// the attribute entry located at entryHash
func EndorseIdentityAttribute(destinationChainID string, attributeEntryHash string, privateKey *[64]byte, signerChainID string, ec *ECAddress) (string, []byte, error) {
	message := []byte(destinationChainID + attributeEntryHash)
	signature := ed.Sign(privateKey, message)
	publicKey := ed.GetPublicKey(privateKey)

	e := Entry{}
	e.ChainID = destinationChainID
	e.ExtIDs = [][]byte{[]byte("IdentityAttributeEndorsement"), signature[:], publicKey[:], []byte(signerChainID)}
	e.Content = []byte(attributeEntryHash)

	txID, err := CommitEntry(&e, ec)
	if err != nil {
		return "", nil, err
	}
	_, err = RevealEntry(&e)
	if err != nil {
		return "", nil, err
	}
	return txID, e.Hash(), nil
}

// IsValidAttribute returns true if the EntryHash points to a correctly formatted attribute entry with a signature
// that was valid for its signer's identity at the time the attribute was published
func IsValidAttribute(entryHash string) (bool, error) {
	e, err := GetEntry(entryHash)
	if err != nil {
		return false, err
	}

	// Check ExtIDs for valid formatting, then process them
	if len(e.ExtIDs) < 5 || bytes.Compare(e.ExtIDs[0], []byte("IdentityAttribute")) != 0 {
		return false, nil
	}
	if len(e.ExtIDs[1]) != 64 || len(e.ExtIDs[2]) != 64 || len(e.ExtIDs[3]) != 32 || len(e.ExtIDs[4]) != 64 {
		return false, nil
	}
	receiverChainID := e.ExtIDs[1]
	var signature [64]byte
	copy(signature[:], e.ExtIDs[2])
	var pubKey [32]byte
	copy(pubKey[:], e.ExtIDs[3])
	signerChainID := string(e.ExtIDs[4])

	// Message that was signed = ReceiverChainID + DestinationChainID + AttributesJSON
	message := receiverChainID
	message = append(message, []byte(e.ChainID)...)
	message = append(message, e.Content...)
	if !ed.Verify(&pubKey, message, &signature) {
		return false, nil
	}

	// Check that public key was valid for the signer at the time of the attribute being published
	receipt, err := GetReceipt(entryHash)
	if err != nil {
		return false, err
	}
	dblock, err := GetDBlock(receipt.DirectoryBlockKeyMR)
	if err != nil {
		return false, err
	}

	signer := &Identity{}
	signer.ChainID = signerChainID
	validKeys, err := signer.GetKeysAtHeight(dblock.Header.SequenceNumber)
	if err != nil {
		return false, err
	}
	for _, key := range validKeys {
		if bytes.Compare(pubKey[:], key.Pub[:]) == 0 {
			// Found provided key to be valid at time of publishing attribute
			return true, nil
		}
	}
	return false, nil
}

// IsValidEndorsement returns true if the EntryHash points to a correctly formatted endorsement entry with a signature
// that was valid for its signer's identity at the time the attribute was published
func IsValidEndorsement(entryHash string) (bool, error) {
	e, err := GetEntry(entryHash)
	if err != nil {
		return false, err
	}

	// Check ExtIDs for valid formatting, then process them
	if len(e.ExtIDs) < 4 || bytes.Compare(e.ExtIDs[0], []byte("IdentityAttributeEndorsement")) != 0 {
		return false, nil
	}
	if len(e.ExtIDs[1]) != 64 || len(e.ExtIDs[2]) != 32 || len(e.ExtIDs[3]) != 64 {
		return false, nil
	}
	var signature [64]byte
	copy(signature[:], e.ExtIDs[1])
	var pubKey [32]byte
	copy(pubKey[:], e.ExtIDs[2])
	signerChainID := string(e.ExtIDs[3])

	// Message that was signed = DestinationChainID + AttributeEntryHash
	message := []byte(e.ChainID)
	message = append(message, e.Content...)
	if !ed.Verify(&pubKey, message, &signature) {
		return false, nil
	}

	// Check that public key was valid for the signer at the time of the attribute being published
	receipt, err := GetReceipt(entryHash)
	if err != nil {
		return false, err
	}
	dblock, err := GetDBlock(receipt.DirectoryBlockKeyMR)
	if err != nil {
		return false, err
	}

	signer := &Identity{}
	signer.ChainID = signerChainID
	validKeys, err := signer.GetKeysAtHeight(dblock.Header.SequenceNumber)
	if err != nil {
		return false, err
	}
	for _, key := range validKeys {
		if bytes.Compare(pubKey[:], key.Pub[:]) == 0 {
			// Found provided key to be valid at time of publishing attribute
			return true, nil
		}
	}
	return false, nil
}
<commit_msg>Added signerKey to key replacement to conform to spec and improve efficiency<commit_after>package factom

import (
	"bytes"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"fmt"

	ed "github.com/FactomProject/ed25519"
)

// An Identity is an array of names and a hierarchy of keys. It can assign/receive
// Attributes as JSON objects and rotate/replace its currently valid keys.
type Identity struct {
	ChainID string
	Name    []string
	Keys    []*IdentityKey
}

// GetIdentityChainID takes an identity name and returns its corresponding ChainID
func GetIdentityChainID(name []string) string {
	hs := sha256.New()
	for _, part := range name {
		h := sha256.Sum256([]byte(part))
		hs.Write(h[:])
	}
	return hex.EncodeToString(hs.Sum(nil))
}

// NewIdentityChain creates an returns a Chain struct for a new identity. Publish it to
// blockchain using the usual factom.CommitChain(...) and factom.RevealChain(...) calls.
func NewIdentityChain(name []string, keys []*IdentityKey) *Chain {
	e := &Entry{}
	for _, part := range name {
		e.ExtIDs = append(e.ExtIDs, []byte(part))
	}

	var publicKeys []string
	for _, key := range keys {
		publicKeys = append(publicKeys, key.PubString())
	}
	keysMap := map[string][]string{"keys": publicKeys}
	keysJSON, _ := json.Marshal(keysMap)
	e.Content = []byte(keysJSON)
	c := NewChain(e)
	return c
}

// GetKeysAtHeight returns the identity's public keys that were/are valid at the highest saved block height
func (i *Identity) GetKeysAtCurrentHeight() ([]*IdentityKey, error) {
	heights, err := GetHeights()
	if err != nil {
		return nil, err
	}
	return i.GetKeysAtHeight(heights.DirectoryBlockHeight)
}

// GetKeysAtHeight returns the identity's public keys that were valid at the specified block height
func (i *Identity) GetKeysAtHeight(height int64) ([]*IdentityKey, error) {
	entries, err := GetAllChainEntriesAtHeight(i.ChainID, height)
	if err != nil {
		return nil, err
	}

	var initialKeys map[string][]string
	initialKeysJSON := entries[0].Content
	err = json.Unmarshal(initialKeysJSON, &initialKeys)
	if err != nil {
		fmt.Println("Failed to unmarshal json from initial key declaration")
		return nil, err
	}

	var validKeys []*IdentityKey
	for _, pubString := range initialKeys["keys"] {
		pub, err := base64.StdEncoding.DecodeString(pubString)
		if err != nil || len(pub) != 32 {
			return nil, fmt.Errorf("invalid Identity public key string in first entry")
		}
		k := NewIdentityKey()
		copy(k.Pub[:], pub)
		validKeys = append(validKeys, k)
	}

	for _, e := range entries {
		if len(e.ExtIDs) < 5 || bytes.Compare(e.ExtIDs[0], []byte("ReplaceKey")) != 0 {
			continue
		}
		if len(e.ExtIDs[1]) != 44 || len(e.ExtIDs[2]) != 44 || len(e.ExtIDs[3]) != 64 {
			continue
		}

		var oldKey [32]byte
		oldPubString := string(e.ExtIDs[1])
		b, err := base64.StdEncoding.DecodeString(oldPubString)
		if err != nil {
			continue
		}
		copy(oldKey[:], b)

		var newKey [32]byte
		newPubString := string(e.ExtIDs[2])
		b, err = base64.StdEncoding.DecodeString(newPubString)
		if err != nil {
			continue
		}
		copy(newKey[:], b)

		var signature [64]byte
		copy(signature[:], e.ExtIDs[3])
		signerPubString := string(e.ExtIDs[4])

		levelToReplace := -1
		for level, key := range validKeys {
			if bytes.Compare(oldKey[:], key.PubBytes()) == 0 {
				levelToReplace = level
			}
		}
		if levelToReplace == -1 {
			// oldkey not in the set of valid keys when this entry was published
			continue
		}

		var message []byte
		message = append(message, oldPubString...)
		message = append(message, newPubString...)
		for level, key := range validKeys {
			if level > levelToReplace {
				// low priority key trying to replace high priority key, disregard
				break
			}
			if key.PubString() == signerPubString && ed.Verify(key.Pub, message, &signature) {
				validKeys[levelToReplace].Pub = &newKey
				break
			}
		}
	}
	return validKeys, nil
}

// ReplaceKey creates an entry in the given identity's chain saying that from this point (the current block),
// the old key will be considered invalid and the new key will be considered valid.
// In order to be recognized as a valid replacement, the identity key used to authorize the
// action must be of the same or higher priority than the one being replaced.
func (i *Identity) ReplaceKey(oldKey *IdentityKey, newKey *IdentityKey, signerKey *IdentityKey, ec *ECAddress) (string, error) {
	//publicKey := ed.GetPublicKey(privateKey)
	var message []byte
	message = append(message, []byte(oldKey.String())...)
	message = append(message, []byte(newKey.String())...)
	signature := signerKey.Sign(message)
	e := Entry{}
	e.ChainID = i.ChainID
	e.ExtIDs = [][]byte{[]byte("ReplaceKey"), []byte(oldKey.String()), []byte(newKey.String()), signature[:], []byte(signerKey.PubString())}

	txID, err := CommitEntry(&e, ec)
	if err != nil {
		return "", err
	}
	_, err = RevealEntry(&e)
	if err != nil {
		return "", err
	}
	return txID, nil
}

// WriteAttribute creates an entry that assigns an attribute JSON object to a given identity, signs it
// an identity key, and appends that entry to the specified destination chain.
func WriteAttribute(receiverChainID string, destinationChainID string, attributesJSON string, privateKey *[64]byte, signerChainID string, ec *ECAddress) (string, []byte, error) {
	message := []byte(receiverChainID + destinationChainID + attributesJSON)
	signature := ed.Sign(privateKey, message)
	publicKey := ed.GetPublicKey(privateKey)

	e := Entry{}
	e.ChainID = destinationChainID
	e.ExtIDs = [][]byte{[]byte("IdentityAttribute"), []byte(receiverChainID), signature[:], publicKey[:], []byte(signerChainID)}
	e.Content = []byte(attributesJSON)

	txID, err := CommitEntry(&e, ec)
	if err != nil {
		return "", nil, err
	}
	_, err = RevealEntry(&e)
	if err != nil {
		return "", nil, err
	}
	return txID, e.Hash(), nil
}

// EndorseIdentityAttribute signs a message using the provided private key saying that the signing identity acknowledges/agrees with
// the attribute entry located at entryHash
func EndorseIdentityAttribute(destinationChainID string, attributeEntryHash string, privateKey *[64]byte, signerChainID string, ec *ECAddress) (string, []byte, error) {
	message := []byte(destinationChainID + attributeEntryHash)
	signature := ed.Sign(privateKey, message)
	publicKey := ed.GetPublicKey(privateKey)

	e := Entry{}
	e.ChainID = destinationChainID
	e.ExtIDs = [][]byte{[]byte("IdentityAttributeEndorsement"), signature[:], publicKey[:], []byte(signerChainID)}
	e.Content = []byte(attributeEntryHash)

	txID, err := CommitEntry(&e, ec)
	if err != nil {
		return "", nil, err
	}
	_, err = RevealEntry(&e)
	if err != nil {
		return "", nil, err
	}
	return txID, e.Hash(), nil
}

// IsValidAttribute returns true if the EntryHash points to a correctly formatted attribute entry with a signature
// that was valid for its signer's identity at the time the attribute was published
func IsValidAttribute(entryHash string) (bool, error) {
	e, err := GetEntry(entryHash)
	if err != nil {
		return false, err
	}

	// Check ExtIDs for valid formatting, then process them
	if len(e.ExtIDs) < 5 || bytes.Compare(e.ExtIDs[0], []byte("IdentityAttribute")) != 0 {
		return false, nil
	}
	if len(e.ExtIDs[1]) != 64 || len(e.ExtIDs[2]) != 64 || len(e.ExtIDs[3]) != 32 || len(e.ExtIDs[4]) != 64 {
		return false, nil
	}
	receiverChainID := e.ExtIDs[1]
	var signature [64]byte
	copy(signature[:], e.ExtIDs[2])
	var pubKey [32]byte
	copy(pubKey[:], e.ExtIDs[3])
	signerChainID := string(e.ExtIDs[4])

	// Message that was signed = ReceiverChainID + DestinationChainID + AttributesJSON
	message := receiverChainID
	message = append(message, []byte(e.ChainID)...)
	message = append(message, e.Content...)
	if !ed.Verify(&pubKey, message, &signature) {
		return false, nil
	}

	// Check that public key was valid for the signer at the time of the attribute being published
	receipt, err := GetReceipt(entryHash)
	if err != nil {
		return false, err
	}
	dblock, err := GetDBlock(receipt.DirectoryBlockKeyMR)
	if err != nil {
		return false, err
	}

	signer := &Identity{}
	signer.ChainID = signerChainID
	validKeys, err := signer.GetKeysAtHeight(dblock.Header.SequenceNumber)
	if err != nil {
		return false, err
	}
	for _, key := range validKeys {
		if bytes.Compare(pubKey[:], key.Pub[:]) == 0 {
			// Found provided key to be valid at time of publishing attribute
			return true, nil
		}
	}
	return false, nil
}

// IsValidEndorsement returns true if the EntryHash points to a correctly formatted endorsement entry with a signature
// that was valid for its signer's identity at the time the attribute was published
func IsValidEndorsement(entryHash string) (bool, error) {
	e, err := GetEntry(entryHash)
	if err != nil {
		return false, err
	}

	// Check ExtIDs for valid formatting, then process them
	if len(e.ExtIDs) < 4 || bytes.Compare(e.ExtIDs[0], []byte("IdentityAttributeEndorsement")) != 0 {
		return false, nil
	}
	if len(e.ExtIDs[1]) != 64 || len(e.ExtIDs[2]) != 32 || len(e.ExtIDs[3]) != 64 {
		return false, nil
	}
	var signature [64]byte
	copy(signature[:], e.ExtIDs[1])
	var pubKey [32]byte
	copy(pubKey[:], e.ExtIDs[2])
	signerChainID := string(e.ExtIDs[3])

	// Message that was signed = DestinationChainID + AttributeEntryHash
	message := []byte(e.ChainID)
	message = append(message, e.Content...)
	if !ed.Verify(&pubKey, message, &signature) {
		return false, nil
	}

	// Check that public key was valid for the signer at the time of the attribute being published
	receipt, err := GetReceipt(entryHash)
	if err != nil {
		return false, err
	}
	dblock, err := GetDBlock(receipt.DirectoryBlockKeyMR)
	if err != nil {
		return false, err
	}

	signer := &Identity{}
	signer.ChainID = signerChainID
	validKeys, err := signer.GetKeysAtHeight(dblock.Header.SequenceNumber)
	if err != nil {
		return false, err
	}
	for _, key := range validKeys {
		if bytes.Compare(pubKey[:], key.Pub[:]) == 0 {
			// Found provided key to be valid at time of publishing attribute
			return true, nil
		}
	}
	return false, nil
}
<|endoftext|>


<commit_before>package bootkube

import (
	"text/template"
)

var (
	// CVOOverrides is the constant to represent contents of cvo-override.yaml file
	// This is a gate to prevent CVO from installing these operators which is conflicting
	// with already owned resources by tectonic-operators.
	// This files can be dropped when the overrides list becomes empty.
	CVOOverrides = template.Must(template.New("cvo-override.yaml").Parse(`
apiVersion: clusterversion.openshift.io/v1
kind: CVOConfig
metadata:
  namespace: openshift-cluster-version
  name: cluster-version-operator
upstream: http://localhost:8080/graph
channel: fast
clusterID: {{.CVOClusterID}}
overrides:
- kind: Deployment                    # this conflicts with kube-core-operator
  namespace: openshift-cluster-network-operator
  name: cluster-network-operator
  unmanaged: true
- kind: Deployment                    # this conflicts with tectonic-ingress-controller-operator
  namespace: openshift-ingress-operator
  name: ingress-operator
  unmanaged: true
- kind: ServiceAccount                # missing run level 0 on the namespace and has 0000_08
  namespace: openshift-cluster-dns-operator
  name: cluster-dns-operator
  unmanaged: true
- kind: Deployment                    # this conflicts with kube-core-operator
  namespace: openshift-cluster-dns-operator
  name: cluster-dns-operator
  unmanaged: true
- kind: APIService                    # packages.apps.redhat.com fails to start properly
  name: v1alpha1.packages.apps.redhat.com
  unmanaged: true
`))
)
<commit_msg>assets: disable pod-checkpointer-operator during bringup phase<commit_after>package bootkube

import (
	"text/template"
)

var (
	// CVOOverrides is the constant to represent contents of cvo-override.yaml file
	// This is a gate to prevent CVO from installing these operators which is conflicting
	// with already owned resources by tectonic-operators.
	// This files can be dropped when the overrides list becomes empty.
	CVOOverrides = template.Must(template.New("cvo-override.yaml").Parse(`
apiVersion: clusterversion.openshift.io/v1
kind: CVOConfig
metadata:
  namespace: openshift-cluster-version
  name: cluster-version-operator
upstream: http://localhost:8080/graph
channel: fast
clusterID: {{.CVOClusterID}}
overrides:
- kind: Deployment                    # this conflicts with kube-core-operator
  namespace: openshift-cluster-network-operator
  name: cluster-network-operator
  unmanaged: true
- kind: Deployment                    # this conflicts with tectonic-ingress-controller-operator
  namespace: openshift-ingress-operator
  name: ingress-operator
  unmanaged: true
- kind: ServiceAccount                # missing run level 0 on the namespace and has 0000_08
  namespace: openshift-cluster-dns-operator
  name: cluster-dns-operator
  unmanaged: true
- kind: Deployment                    # this conflicts with kube-core-operator
  namespace: openshift-cluster-dns-operator
  name: cluster-dns-operator
  unmanaged: true
- kind: APIService                    # packages.apps.redhat.com fails to start properly
  name: v1alpha1.packages.apps.redhat.com
  unmanaged: true
- kind: Deployment                    # still testing this new component
  namespace: openshift-pod-checkpointer
  name: pod-checkpointer-operator
  unmanaged: true
`))
)
<|endoftext|>