<commit_before>// Copyright 2014 Wandoujia Inc. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package models

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"

	"github.com/wandoulabs/codis/pkg/utils"
	"github.com/wandoulabs/codis/pkg/utils/errors"
	"github.com/wandoulabs/codis/pkg/utils/log"
	"github.com/wandoulabs/zkhelper"
)

const (
	SERVER_TYPE_MASTER  string = "master"
	SERVER_TYPE_SLAVE   string = "slave"
	SERVER_TYPE_OFFLINE string = "offline"
)

// redis server instance
type Server struct {
	Type    string `json:"type"`
	GroupId int    `json:"group_id"`
	Addr    string `json:"addr"`
}

// redis server group
type ServerGroup struct {
	Id          int       `json:"id"`
	ProductName string    `json:"product_name"`
	Servers     []*Server `json:"servers"`
}

func (self *Server) String() string {
	b, _ := json.MarshalIndent(self, "", "  ")
	return string(b)
}

func (self *ServerGroup) String() string {
	b, _ := json.MarshalIndent(self, "", "  ")
	return string(b) + "\n"
}

func GetServer(zkConn zkhelper.Conn, zkPath string) (*Server, error) {
	data, _, err := zkConn.Get(zkPath)
	if err != nil {
		return nil, errors.Trace(err)
	}
	srv := Server{}
	if err := json.Unmarshal(data, &srv); err != nil {
		return nil, errors.Trace(err)
	}
	return &srv, nil
}

func NewServer(serverType string, addr string) *Server {
	return &Server{
		Type:    serverType,
		GroupId: INVALID_ID,
		Addr:    addr,
	}
}

func NewServerGroup(productName string, id int) *ServerGroup {
	return &ServerGroup{
		Id:          id,
		ProductName: productName,
	}
}

func GroupExists(zkConn zkhelper.Conn, productName string, groupId int) (bool, error) {
	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", productName, groupId)
	exists, _, err := zkConn.Exists(zkPath)
	if err != nil {
		return false, errors.Trace(err)
	}
	return exists, nil
}

func GetGroup(zkConn zkhelper.Conn, productName string, groupId int) (*ServerGroup, error) {
	exists, err := GroupExists(zkConn, productName, groupId)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if !exists {
		return nil, errors.Errorf("group %d is not found", groupId)
	}

	group := &ServerGroup{
		ProductName: productName,
		Id:          groupId,
	}

	group.Servers, err = group.GetServers(zkConn)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return group, nil
}

func ServerGroups(zkConn zkhelper.Conn, productName string) ([]*ServerGroup, error) {
	var ret []*ServerGroup
	root := fmt.Sprintf("/zk/codis/db_%s/servers", productName)
	groups, _, err := zkConn.Children(root)
	if err != nil {
		return nil, errors.Trace(err)
	}

	// Buggy :X
	//zkhelper.ChildrenRecursive(*zkConn, root)

	for _, group := range groups {
		// parse group_1 => 1
		groupId, err := strconv.Atoi(strings.Split(group, "_")[1])
		if err != nil {
			return nil, errors.Trace(err)
		}
		g, err := GetGroup(zkConn, productName, groupId)
		if err != nil {
			return nil, errors.Trace(err)
		}
		ret = append(ret, g)
	}
	return ret, nil
}

func (self *ServerGroup) Master(zkConn zkhelper.Conn) (*Server, error) {
	servers, err := self.GetServers(zkConn)
	if err != nil {
		return nil, errors.Trace(err)
	}
	for _, s := range servers {
		// TODO check if there are two masters
		if s.Type == SERVER_TYPE_MASTER {
			return s, nil
		}
	}
	return nil, nil
}

func (self *ServerGroup) Remove(zkConn zkhelper.Conn) error {
	// check if this group is not used by any slot
	slots, err := Slots(zkConn, self.ProductName)
	if err != nil {
		return errors.Trace(err)
	}

	for _, slot := range slots {
		if slot.GroupId == self.Id {
			return errors.Errorf("group %d is using by slot %d", slot.GroupId, slot.Id)
		}
		if (slot.State.Status == SLOT_STATUS_MIGRATE || slot.State.Status == SLOT_STATUS_PRE_MIGRATE) && slot.State.MigrateStatus.From == self.Id {
			return errors.Errorf("slot %d has residual data remain in group %d", slot.Id, self.Id)
		}
	}

	// do delete
	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", self.ProductName, self.Id)
	err = zkhelper.DeleteRecursive(zkConn, zkPath, -1)

	// we know that there's no slots affected, so this action doesn't need proxy confirm
	err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_REMOVE, self, "", false)
	return errors.Trace(err)
}

func (self *ServerGroup) RemoveServer(zkConn zkhelper.Conn, addr string) error {
	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d/%s", self.ProductName, self.Id, addr)
	data, _, err := zkConn.Get(zkPath)
	if err != nil {
		return errors.Trace(err)
	}

	var s Server
	err = json.Unmarshal(data, &s)
	if err != nil {
		return errors.Trace(err)
	}
	log.Info(s)
	if s.Type == SERVER_TYPE_MASTER {
		return errors.Errorf("cannot remove master, use promote first")
	}

	err = zkConn.Delete(zkPath, -1)
	if err != nil {
		return errors.Trace(err)
	}

	// update server list
	for i := 0; i < len(self.Servers); i++ {
		if self.Servers[i].Addr == s.Addr {
			self.Servers = append(self.Servers[:i], self.Servers[i+1:]...)
			break
		}
	}

	// remove slave won't need proxy confirm
	err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, self, "", false)
	return errors.Trace(err)
}

func (self *ServerGroup) Promote(conn zkhelper.Conn, addr, passwd string) error {
	var s *Server
	exists := false
	for i := 0; i < len(self.Servers); i++ {
		if self.Servers[i].Addr == addr {
			s = self.Servers[i]
			exists = true
			break
		}
	}

	if !exists {
		return errors.Errorf("no such addr %s", addr)
	}

	err := utils.SlaveNoOne(s.Addr, passwd)
	if err != nil {
		return errors.Trace(err)
	}

	// set origin master offline
	master, err := self.Master(conn)
	if err != nil {
		return errors.Trace(err)
	}

	// old master may be nil
	if master != nil {
		master.Type = SERVER_TYPE_OFFLINE
		err = self.AddServer(conn, master, passwd)
		if err != nil {
			return errors.Trace(err)
		}
	}

	// promote new server to master
	s.Type = SERVER_TYPE_MASTER
	err = self.AddServer(conn, s, passwd)
	return errors.Trace(err)
}

func (self *ServerGroup) Create(zkConn zkhelper.Conn) error {
	if self.Id < 0 {
		return errors.Errorf("invalid server group id %d", self.Id)
	}
	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", self.ProductName, self.Id)
	_, err := zkhelper.CreateOrUpdate(zkConn, zkPath, "", 0, zkhelper.DefaultDirACLs(), true)
	if err != nil {
		return errors.Trace(err)
	}
	err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, self, "", false)
	if err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (self *ServerGroup) Exists(zkConn zkhelper.Conn) (bool, error) {
	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", self.ProductName, self.Id)
	b, err := zkhelper.NodeExists(zkConn, zkPath)
	if err != nil {
		return false, errors.Trace(err)
	}
	return b, nil
}

var ErrNodeExists = errors.New("node already exists")

func (self *ServerGroup) AddServer(zkConn zkhelper.Conn, s *Server, passwd string) error {
	s.GroupId = self.Id

	servers, err := self.GetServers(zkConn)
	if err != nil {
		return errors.Trace(err)
	}
	var masterAddr string
	for _, server := range servers {
		if server.Type == SERVER_TYPE_MASTER {
			masterAddr = server.Addr
		}
	}

	// make sure there is only one master
	if s.Type == SERVER_TYPE_MASTER && len(masterAddr) > 0 {
		return errors.Trace(ErrNodeExists)
	}

	// if this group has no server. auto promote this server to master
	if len(servers) == 0 {
		s.Type = SERVER_TYPE_MASTER
	}

	val, err := json.Marshal(s)
	if err != nil {
		return errors.Trace(err)
	}

	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d/%s", self.ProductName, self.Id, s.Addr)
	_, err = zkhelper.CreateOrUpdate(zkConn, zkPath, string(val), 0, zkhelper.DefaultFileACLs(), true)

	// update servers
	servers, err = self.GetServers(zkConn)
	if err != nil {
		return errors.Trace(err)
	}
	self.Servers = servers

	if s.Type == SERVER_TYPE_MASTER {
		err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, self, "", true)
		if err != nil {
			return errors.Trace(err)
		}
	} else if s.Type == SERVER_TYPE_SLAVE && len(masterAddr) > 0 {
		// send command slaveof to slave
		err := utils.SlaveOf(s.Addr, passwd, masterAddr)
		if err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}

func (self *ServerGroup) GetServers(zkConn zkhelper.Conn) ([]*Server, error) {
	var ret []*Server
	root := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", self.ProductName, self.Id)
	nodes, _, err := zkConn.Children(root)
	if err != nil {
		return nil, errors.Trace(err)
	}
	for _, node := range nodes {
		nodePath := root + "/" + node
		s, err := GetServer(zkConn, nodePath)
		if err != nil {
			return nil, errors.Trace(err)
		}
		ret = append(ret, s)
	}
	return ret, nil
}
<commit_msg>logger when error occours on ZK<commit_after>// Copyright 2014 Wandoujia Inc. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package models

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"

	"github.com/wandoulabs/codis/pkg/utils"
	"github.com/wandoulabs/codis/pkg/utils/errors"
	"github.com/wandoulabs/codis/pkg/utils/log"
	"github.com/wandoulabs/zkhelper"
)

const (
	SERVER_TYPE_MASTER  string = "master"
	SERVER_TYPE_SLAVE   string = "slave"
	SERVER_TYPE_OFFLINE string = "offline"
)

// redis server instance
type Server struct {
	Type    string `json:"type"`
	GroupId int    `json:"group_id"`
	Addr    string `json:"addr"`
}

// redis server group
type ServerGroup struct {
	Id          int       `json:"id"`
	ProductName string    `json:"product_name"`
	Servers     []*Server `json:"servers"`
}

func (self *Server) String() string {
	b, _ := json.MarshalIndent(self, "", "  ")
	return string(b)
}

func (self *ServerGroup) String() string {
	b, _ := json.MarshalIndent(self, "", "  ")
	return string(b) + "\n"
}

func GetServer(zkConn zkhelper.Conn, zkPath string) (*Server, error) {
	data, _, err := zkConn.Get(zkPath)
	if err != nil {
		return nil, errors.Trace(err)
	}
	srv := Server{}
	if err := json.Unmarshal(data, &srv); err != nil {
		return nil, errors.Trace(err)
	}
	return &srv, nil
}

func NewServer(serverType string, addr string) *Server {
	return &Server{
		Type:    serverType,
		GroupId: INVALID_ID,
		Addr:    addr,
	}
}

func NewServerGroup(productName string, id int) *ServerGroup {
	return &ServerGroup{
		Id:          id,
		ProductName: productName,
	}
}

func GroupExists(zkConn zkhelper.Conn, productName string, groupId int) (bool, error) {
	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", productName, groupId)
	exists, _, err := zkConn.Exists(zkPath)
	if err != nil {
		return false, errors.Trace(err)
	}
	return exists, nil
}

func GetGroup(zkConn zkhelper.Conn, productName string, groupId int) (*ServerGroup, error) {
	exists, err := GroupExists(zkConn, productName, groupId)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if !exists {
		return nil, errors.Errorf("group %d is not found", groupId)
	}

	group := &ServerGroup{
		ProductName: productName,
		Id:          groupId,
	}

	group.Servers, err = group.GetServers(zkConn)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return group, nil
}

func ServerGroups(zkConn zkhelper.Conn, productName string) ([]*ServerGroup, error) {
	var ret []*ServerGroup
	root := fmt.Sprintf("/zk/codis/db_%s/servers", productName)
	groups, _, err := zkConn.Children(root)
	if err != nil {
		return nil, errors.Trace(err)
	}

	// Buggy :X
	//zkhelper.ChildrenRecursive(*zkConn, root)

	for _, group := range groups {
		// parse group_1 => 1
		groupId, err := strconv.Atoi(strings.Split(group, "_")[1])
		if err != nil {
			return nil, errors.Trace(err)
		}
		g, err := GetGroup(zkConn, productName, groupId)
		if err != nil {
			return nil, errors.Trace(err)
		}
		ret = append(ret, g)
	}
	return ret, nil
}

func (self *ServerGroup) Master(zkConn zkhelper.Conn) (*Server, error) {
	servers, err := self.GetServers(zkConn)
	if err != nil {
		return nil, errors.Trace(err)
	}
	for _, s := range servers {
		// TODO check if there are two masters
		if s.Type == SERVER_TYPE_MASTER {
			return s, nil
		}
	}
	return nil, nil
}

func (self *ServerGroup) Remove(zkConn zkhelper.Conn) error {
	// check if this group is not used by any slot
	slots, err := Slots(zkConn, self.ProductName)
	if err != nil {
		return errors.Trace(err)
	}

	for _, slot := range slots {
		if slot.GroupId == self.Id {
			return errors.Errorf("group %d is using by slot %d", slot.GroupId, slot.Id)
		}
		if (slot.State.Status == SLOT_STATUS_MIGRATE || slot.State.Status == SLOT_STATUS_PRE_MIGRATE) && slot.State.MigrateStatus.From == self.Id {
			return errors.Errorf("slot %d has residual data remain in group %d", slot.Id, self.Id)
		}
	}

	// do delete
	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", self.ProductName, self.Id)
	err = zkhelper.DeleteRecursive(zkConn, zkPath, -1)

	// we know that there's no slots affected, so this action doesn't need proxy confirm
	err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_REMOVE, self, "", false)
	return errors.Trace(err)
}

func (self *ServerGroup) RemoveServer(zkConn zkhelper.Conn, addr string) error {
	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d/%s", self.ProductName, self.Id, addr)
	data, _, err := zkConn.Get(zkPath)
	if err != nil {
		return errors.Trace(err)
	}

	var s Server
	err = json.Unmarshal(data, &s)
	if err != nil {
		return errors.Trace(err)
	}
	log.Info(s)
	if s.Type == SERVER_TYPE_MASTER {
		return errors.Errorf("cannot remove master, use promote first")
	}

	err = zkConn.Delete(zkPath, -1)
	if err != nil {
		return errors.Trace(err)
	}

	// update server list
	for i := 0; i < len(self.Servers); i++ {
		if self.Servers[i].Addr == s.Addr {
			self.Servers = append(self.Servers[:i], self.Servers[i+1:]...)
			break
		}
	}

	// remove slave won't need proxy confirm
	err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, self, "", false)
	return errors.Trace(err)
}

func (self *ServerGroup) Promote(conn zkhelper.Conn, addr, passwd string) error {
	var s *Server
	exists := false
	for i := 0; i < len(self.Servers); i++ {
		if self.Servers[i].Addr == addr {
			s = self.Servers[i]
			exists = true
			break
		}
	}

	if !exists {
		return errors.Errorf("no such addr %s", addr)
	}

	err := utils.SlaveNoOne(s.Addr, passwd)
	if err != nil {
		return errors.Trace(err)
	}

	// set origin master offline
	master, err := self.Master(conn)
	if err != nil {
		return errors.Trace(err)
	}

	// old master may be nil
	if master != nil {
		master.Type = SERVER_TYPE_OFFLINE
		err = self.AddServer(conn, master, passwd)
		if err != nil {
			return errors.Trace(err)
		}
	}

	// promote new server to master
	s.Type = SERVER_TYPE_MASTER
	err = self.AddServer(conn, s, passwd)
	return errors.Trace(err)
}

func (self *ServerGroup) Create(zkConn zkhelper.Conn) error {
	if self.Id < 0 {
		return errors.Errorf("invalid server group id %d", self.Id)
	}
	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", self.ProductName, self.Id)
	_, err := zkhelper.CreateOrUpdate(zkConn, zkPath, "", 0, zkhelper.DefaultDirACLs(), true)
	if err != nil {
		return errors.Trace(err)
	}
	err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, self, "", false)
	if err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (self *ServerGroup) Exists(zkConn zkhelper.Conn) (bool, error) {
	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", self.ProductName, self.Id)
	b, err := zkhelper.NodeExists(zkConn, zkPath)
	if err != nil {
		return false, errors.Trace(err)
	}
	return b, nil
}

var ErrNodeExists = errors.New("node already exists")

func (self *ServerGroup) AddServer(zkConn zkhelper.Conn, s *Server, passwd string) error {
	s.GroupId = self.Id

	servers, err := self.GetServers(zkConn)
	if err != nil {
		return errors.Trace(err)
	}
	var masterAddr string
	for _, server := range servers {
		if server.Type == SERVER_TYPE_MASTER {
			masterAddr = server.Addr
		}
	}

	// make sure there is only one master
	if s.Type == SERVER_TYPE_MASTER && len(masterAddr) > 0 {
		return errors.Trace(ErrNodeExists)
	}

	// if this group has no server. auto promote this server to master
	if len(servers) == 0 {
		s.Type = SERVER_TYPE_MASTER
	}

	val, err := json.Marshal(s)
	if err != nil {
		return errors.Trace(err)
	}

	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d/%s", self.ProductName, self.Id, s.Addr)
	_, err = zkhelper.CreateOrUpdate(zkConn, zkPath, string(val), 0, zkhelper.DefaultFileACLs(), true)
	if err != nil {
		return errors.Trace(err)
	}

	// update servers
	servers, err = self.GetServers(zkConn)
	if err != nil {
		return errors.Trace(err)
	}
	self.Servers = servers

	if s.Type == SERVER_TYPE_MASTER {
		err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, self, "", true)
		if err != nil {
			return errors.Trace(err)
		}
	} else if s.Type == SERVER_TYPE_SLAVE && len(masterAddr) > 0 {
		// send command slaveof to slave
		err := utils.SlaveOf(s.Addr, passwd, masterAddr)
		if err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}

func (self *ServerGroup) GetServers(zkConn zkhelper.Conn) ([]*Server, error) {
	var ret []*Server
	root := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", self.ProductName, self.Id)
	nodes, _, err := zkConn.Children(root)
	if err != nil {
		return nil, errors.Trace(err)
	}
	for _, node := range nodes {
		nodePath := root + "/" + node
		s, err := GetServer(zkConn, nodePath)
		if err != nil {
			return nil, errors.Trace(err)
		}
		ret = append(ret, s)
	}
	return ret, nil
}
<|endoftext|>


<commit_before>// Code generated by nimona.io/tools/codegen. DO NOT EDIT.

package peer

import (
	json "encoding/json"

	crypto "nimona.io/pkg/crypto"
	object "nimona.io/pkg/object"
)

// basic structs
type (
	Peer struct {
		Addresses []string          `json:"addresses:as"`
		Signature *crypto.Signature `json:"@signature:o"`
	}
)

func (e *Peer) ContextName() string {
	return "nimona.io/peer"
}

func (e *Peer) GetType() string {
	return "nimona.io/peer/Peer"
}

func (e *Peer) ToObject() object.Object {
	m := map[string]interface{}{
		"@ctx:s":    "nimona.io/peer/Peer",
		"@struct:s": "Peer",
	}
	b, _ := json.Marshal(e)
	json.Unmarshal(b, &m)
	return object.Object(m)
}

func (e *Peer) FromObject(o object.Object) error {
	b, _ := json.Marshal(map[string]interface{}(o))
	return json.Unmarshal(b, e)
}

// domain events
type (
	PeerRequested struct {
		Keys      []string          `json:"keys:as"`
		Signature *crypto.Signature `json:"@signature:o"`
	}
	PeerUpdated struct {
		Addresses []string          `json:"addresses:as"`
		Signature *crypto.Signature `json:"@signature:o"`
	}
)

func (e *PeerRequested) ContextName() string {
	return "nimona.io/peer/Peer"
}

func (e *PeerRequested) EventName() string {
	return "Requested"
}

func (e *PeerRequested) GetType() string {
	return "nimona.io/peer/Peer.Requested"
}

func (e *PeerRequested) ToObject() object.Object {
	m := map[string]interface{}{
		"@ctx:s":    "nimona.io/peer/Peer.Requested",
		"@domain:s": "nimona.io/peer/Peer",
		"@event:s":  "Requested",
	}
	b, _ := json.Marshal(e)
	json.Unmarshal(b, &m)
	return object.Object(m)
}

func (e *PeerRequested) FromObject(o object.Object) error {
	b, _ := json.Marshal(map[string]interface{}(o))
	return json.Unmarshal(b, e)
}

func (e *PeerUpdated) ContextName() string {
	return "nimona.io/peer/Peer"
}

func (e *PeerUpdated) EventName() string {
	return "Updated"
}

func (e *PeerUpdated) GetType() string {
	return "nimona.io/peer/Peer.Updated"
}

func (e *PeerUpdated) ToObject() object.Object {
	m := map[string]interface{}{
		"@ctx:s":    "nimona.io/peer/Peer.Updated",
		"@domain:s": "nimona.io/peer/Peer",
		"@event:s":  "Updated",
	}
	b, _ := json.Marshal(e)
	json.Unmarshal(b, &m)
	return object.Object(m)
}

func (e *PeerUpdated) FromObject(o object.Object) error {
	b, _ := json.Marshal(map[string]interface{}(o))
	return json.Unmarshal(b, e)
}
<commit_msg>refactor(peer): update generated peer<commit_after>// Code generated by nimona.io/tools/codegen. DO NOT EDIT.

package peer

import (
	json "encoding/json"

	crypto "nimona.io/pkg/crypto"
	object "nimona.io/pkg/object"
)

type (
	Peer struct {
		Addresses []string          `json:"addresses:as"`
		Signature *crypto.Signature `json:"@signature:o"`
	}
)

func (e *Peer) ContextName() string {
	return "nimona.io/peer"
}

func (e *Peer) GetType() string {
	return "Peer"
}

func (e *Peer) ToObject() object.Object {
	m := map[string]interface{}{
		"@ctx:s":    "Peer",
		"@struct:s": "Peer",
	}
	b, _ := json.Marshal(e)
	json.Unmarshal(b, &m)
	return object.Object(m)
}

func (e *Peer) FromObject(o object.Object) error {
	b, _ := json.Marshal(map[string]interface{}(o))
	return json.Unmarshal(b, e)
}

type (
	Requested struct {
		Keys      []string            `json:"keys:as"`
		Signature *crypto.Signature   `json:"@signature:o"`
		Authors   []*crypto.PublicKey `json:"@authors:ao"`
	}
	Updated struct {
		Addresses []string            `json:"addresses:as"`
		Signature *crypto.Signature   `json:"@signature:o"`
		Authors   []*crypto.PublicKey `json:"@authors:ao"`
	}
)

func (e *Requested) EventName() string {
	return "Requested"
}

func (e *Requested) GetType() string {
	return "Peer.Requested"
}

func (e *Requested) ToObject() object.Object {
	m := map[string]interface{}{
		"@ctx:s":    "Peer.Requested",
		"@domain:s": "Peer",
		"@event:s":  "Requested",
	}
	b, _ := json.Marshal(e)
	json.Unmarshal(b, &m)
	return object.Object(m)
}

func (e *Requested) FromObject(o object.Object) error {
	b, _ := json.Marshal(map[string]interface{}(o))
	return json.Unmarshal(b, e)
}

func (e *Updated) EventName() string {
	return "Updated"
}

func (e *Updated) GetType() string {
	return "Peer.Updated"
}

func (e *Updated) ToObject() object.Object {
	m := map[string]interface{}{
		"@ctx:s":    "Peer.Updated",
		"@domain:s": "Peer",
		"@event:s":  "Updated",
	}
	b, _ := json.Marshal(e)
	json.Unmarshal(b, &m)
	return object.Object(m)
}

func (e *Updated) FromObject(o object.Object) error {
	b, _ := json.Marshal(map[string]interface{}(o))
	return json.Unmarshal(b, e)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package provision

import (
	"bytes"
	"fmt"
	"path"
	"path/filepath"
	"text/template"
	"time"

	"github.com/docker/machine/libmachine/auth"
	"github.com/docker/machine/libmachine/cert"
	"github.com/docker/machine/libmachine/drivers"
	"github.com/docker/machine/libmachine/engine"
	"github.com/docker/machine/libmachine/log"
	"github.com/docker/machine/libmachine/mcnutils"
	"github.com/docker/machine/libmachine/provision"
	"github.com/docker/machine/libmachine/provision/pkgaction"
	"github.com/docker/machine/libmachine/provision/serviceaction"
	"github.com/docker/machine/libmachine/swarm"
	"github.com/pkg/errors"
	"k8s.io/minikube/pkg/minikube/assets"
	"k8s.io/minikube/pkg/minikube/sshutil"
	"k8s.io/minikube/pkg/util"
)

type BuildrootProvisioner struct {
	provision.SystemdProvisioner
}

func init() {
	provision.Register("Buildroot", &provision.RegisteredProvisioner{
		New: NewBuildrootProvisioner,
	})
}

func NewBuildrootProvisioner(d drivers.Driver) provision.Provisioner {
	return &BuildrootProvisioner{
		provision.NewSystemdProvisioner("buildroot", d),
	}
}

func (p *BuildrootProvisioner) String() string {
	return "buildroot"
}

func (p *BuildrootProvisioner) GenerateDockerOptions(dockerPort int) (*provision.DockerOptions, error) {
	var engineCfg bytes.Buffer

	driverNameLabel := fmt.Sprintf("provider=%s", p.Driver.DriverName())
	p.EngineOptions.Labels = append(p.EngineOptions.Labels, driverNameLabel)

	engineConfigTmpl := `[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target docker.socket
Requires=docker.socket

[Service]
Type=notify

# DOCKER_RAMDISK disables pivot_root in Docker, using MS_MOVE instead.
Environment=DOCKER_RAMDISK=yes
{{range .EngineOptions.Env}}Environment={{.}}
{{end}}

# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
#  Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
ExecStart=
ExecStart=/usr/bin/docker daemon -H tcp://0.0.0.0:{{.DockerPort}} -H unix:///var/run/docker.sock --tlsverify --tlscacert {{.AuthOptions.CaCertRemotePath}} --tlscert {{.AuthOptions.ServerCertRemotePath}} --tlskey {{.AuthOptions.ServerKeyRemotePath}} {{ range .EngineOptions.Labels }}--label {{.}} {{ end }}{{ range .EngineOptions.InsecureRegistry }}--insecure-registry {{.}} {{ end }}{{ range .EngineOptions.RegistryMirror }}--registry-mirror {{.}} {{ end }}{{ range .EngineOptions.ArbitraryFlags }}--{{.}} {{ end }}
ExecReload=/bin/kill -s HUP $MAINPID

# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity

# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0

# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes

# kill only the docker process, not all processes in the cgroup
KillMode=process

[Install]
WantedBy=multi-user.target
`
	t, err := template.New("engineConfig").Parse(engineConfigTmpl)
	if err != nil {
		return nil, err
	}

	engineConfigContext := provision.EngineConfigContext{
		DockerPort:    dockerPort,
		AuthOptions:   p.AuthOptions,
		EngineOptions: p.EngineOptions,
	}

	t.Execute(&engineCfg, engineConfigContext)

	return &provision.DockerOptions{
		EngineOptions:     engineCfg.String(),
		EngineOptionsPath: "/lib/systemd/system/docker.service",
	}, nil
}

func (p *BuildrootProvisioner) Package(name string, action pkgaction.PackageAction) error {
	return nil
}

func (p *BuildrootProvisioner) Provision(swarmOptions swarm.Options, authOptions auth.Options, engineOptions engine.Options) error {
	p.SwarmOptions = swarmOptions
	p.AuthOptions = authOptions
	p.EngineOptions = engineOptions

	log.Debugf("setting hostname %q", p.Driver.GetMachineName())
	if err := p.SetHostname(p.Driver.GetMachineName()); err != nil {
		return err
	}

	p.AuthOptions = setRemoteAuthOptions(p)
	log.Debugf("set auth options %+v", p.AuthOptions)

	log.Debugf("setting up certificates")

	configureAuth := func() error {
		if err := configureAuth(p); err != nil {
			return &util.RetriableError{Err: err}
		}
		return nil
	}

	err := util.RetryAfter(5, configureAuth, time.Second*10)
	if err != nil {
		log.Debugf("Error configuring auth during provisioning %v", err)
		return err
	}

	return nil
}

func setRemoteAuthOptions(p provision.Provisioner) auth.Options {
	dockerDir := p.GetDockerOptionsDir()
	authOptions := p.GetAuthOptions()

	// due to windows clients, we cannot use filepath.Join as the paths
	// will be mucked on the linux hosts
	authOptions.CaCertRemotePath = path.Join(dockerDir, "ca.pem")
	authOptions.ServerCertRemotePath = path.Join(dockerDir, "server.pem")
	authOptions.ServerKeyRemotePath = path.Join(dockerDir, "server-key.pem")

	return authOptions
}

func configureAuth(p *BuildrootProvisioner) error {
	driver := p.GetDriver()
	machineName := driver.GetMachineName()
	authOptions := p.GetAuthOptions()
	org := mcnutils.GetUsername() + "." + machineName
	bits := 2048

	ip, err := driver.GetIP()
	if err != nil {
		return errors.Wrap(err, "error getting ip during provisioning")
	}

	hostCerts := map[string]string{
		authOptions.CaCertPath:     filepath.Join(authOptions.StorePath, "ca.pem"),
		authOptions.ClientCertPath: filepath.Join(authOptions.StorePath, "cert.pem"),
		authOptions.ClientKeyPath:  filepath.Join(authOptions.StorePath, "key.pem"),
	}

	for src, dst := range hostCerts {
		f, err := assets.NewFileAsset(src, filepath.Dir(dst), filepath.Base(dst), "0777")
		if err != nil {
			return errors.Wrapf(err, "open cert file: %s", src)
		}
		if err := assets.CopyFileLocal(f); err != nil {
			return errors.Wrapf(err, "transferring file: %+v", f)
		}
	}

	// The Host IP is always added to the certificate's SANs list
	hosts := append(authOptions.ServerCertSANs, ip, "localhost")
	log.Debugf("generating server cert: %s ca-key=%s private-key=%s org=%s san=%s",
		authOptions.ServerCertPath,
		authOptions.CaCertPath,
		authOptions.CaPrivateKeyPath,
		org,
		hosts,
	)

	err = cert.GenerateCert(&cert.Options{
		Hosts:     hosts,
		CertFile:  authOptions.ServerCertPath,
		KeyFile:   authOptions.ServerKeyPath,
		CAFile:    authOptions.CaCertPath,
		CAKeyFile: authOptions.CaPrivateKeyPath,
		Org:       org,
		Bits:      bits,
	})

	if err != nil {
		return fmt.Errorf("error generating server cert: %s", err)
	}

	remoteCerts := map[string]string{
		authOptions.CaCertPath:     authOptions.CaCertRemotePath,
		authOptions.ServerCertPath: authOptions.ServerCertRemotePath,
		authOptions.ServerKeyPath:  authOptions.ServerKeyRemotePath,
	}

	sshClient, err := sshutil.NewSSHClient(driver)
	if err != nil {
		return errors.Wrap(err, "provisioning: error getting ssh client")
	}

	for src, dst := range remoteCerts {
		f, err := assets.NewFileAsset(src, filepath.Dir(dst), filepath.Base(dst), "0640")
		if err != nil {
			return errors.Wrapf(err, "error copying %s to %s", src, dst)
		}
		if err := sshutil.TransferFile(f, sshClient); err != nil {
			return errors.Wrapf(err, "transfering file to machine %v", f)
		}
	}

	dockerCfg, err := p.GenerateDockerOptions(engine.DefaultPort)
	if err != nil {
		return errors.Wrap(err, "generating docker options")
	}

	log.Info("Setting Docker configuration on the remote daemon...")

	if _, err = p.SSHCommand(fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | sudo tee %s", path.Dir(dockerCfg.EngineOptionsPath), dockerCfg.EngineOptions, dockerCfg.EngineOptionsPath)); err != nil {
		return err
	}

	if err := p.Service("docker", serviceaction.Restart); err != nil {
		return err
	}

	return nil
}
<commit_msg>Move setupCerts to use CommandRunners<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package provision

import (
	"bytes"
	"fmt"
	"path"
	"path/filepath"
	"text/template"
	"time"

	"github.com/docker/machine/libmachine/auth"
	"github.com/docker/machine/libmachine/cert"
	"github.com/docker/machine/libmachine/drivers"
	"github.com/docker/machine/libmachine/engine"
	"github.com/docker/machine/libmachine/log"
	"github.com/docker/machine/libmachine/mcnutils"
	"github.com/docker/machine/libmachine/provision"
	"github.com/docker/machine/libmachine/provision/pkgaction"
	"github.com/docker/machine/libmachine/provision/serviceaction"
	"github.com/docker/machine/libmachine/swarm"
	"github.com/pkg/errors"
	"k8s.io/minikube/pkg/minikube/assets"
	"k8s.io/minikube/pkg/minikube/bootstrapper"
	"k8s.io/minikube/pkg/minikube/sshutil"
	"k8s.io/minikube/pkg/util"
)

type BuildrootProvisioner struct {
	provision.SystemdProvisioner
}

func init() {
	provision.Register("Buildroot", &provision.RegisteredProvisioner{
		New: NewBuildrootProvisioner,
	})
}

func NewBuildrootProvisioner(d drivers.Driver) provision.Provisioner {
	return &BuildrootProvisioner{
		provision.NewSystemdProvisioner("buildroot", d),
	}
}

func (p *BuildrootProvisioner) String() string {
	return "buildroot"
}

func (p *BuildrootProvisioner) GenerateDockerOptions(dockerPort int) (*provision.DockerOptions, error) {
	var engineCfg bytes.Buffer

	driverNameLabel := fmt.Sprintf("provider=%s", p.Driver.DriverName())
	p.EngineOptions.Labels = append(p.EngineOptions.Labels, driverNameLabel)

	engineConfigTmpl := `[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target docker.socket
Requires=docker.socket

[Service]
Type=notify

# DOCKER_RAMDISK disables pivot_root in Docker, using MS_MOVE instead.
Environment=DOCKER_RAMDISK=yes
{{range .EngineOptions.Env}}Environment={{.}}
{{end}}

# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
#  Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
ExecStart=
ExecStart=/usr/bin/docker daemon -H tcp://0.0.0.0:{{.DockerPort}} -H unix:///var/run/docker.sock --tlsverify --tlscacert {{.AuthOptions.CaCertRemotePath}} --tlscert {{.AuthOptions.ServerCertRemotePath}} --tlskey {{.AuthOptions.ServerKeyRemotePath}} {{ range .EngineOptions.Labels }}--label {{.}} {{ end }}{{ range .EngineOptions.InsecureRegistry }}--insecure-registry {{.}} {{ end }}{{ range .EngineOptions.RegistryMirror }}--registry-mirror {{.}} {{ end }}{{ range .EngineOptions.ArbitraryFlags }}--{{.}} {{ end }}
ExecReload=/bin/kill -s HUP $MAINPID

# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity

# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0

# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes

# kill only the docker process, not all processes in the cgroup
KillMode=process

[Install]
WantedBy=multi-user.target
`
	t, err := template.New("engineConfig").Parse(engineConfigTmpl)
	if err != nil {
		return nil, err
	}

	engineConfigContext := provision.EngineConfigContext{
		DockerPort:    dockerPort,
		AuthOptions:   p.AuthOptions,
		EngineOptions: p.EngineOptions,
	}

	t.Execute(&engineCfg, engineConfigContext)

	return &provision.DockerOptions{
		EngineOptions:     engineCfg.String(),
		EngineOptionsPath: "/lib/systemd/system/docker.service",
	}, nil
}

func (p *BuildrootProvisioner) Package(name string, action pkgaction.PackageAction) error {
	return nil
}

func (p *BuildrootProvisioner) Provision(swarmOptions swarm.Options, authOptions auth.Options, engineOptions engine.Options) error {
	p.SwarmOptions = swarmOptions
	p.AuthOptions = authOptions
	p.EngineOptions = engineOptions

	log.Debugf("setting hostname %q", p.Driver.GetMachineName())
	if err := p.SetHostname(p.Driver.GetMachineName()); err != nil {
		return err
	}

	p.AuthOptions = setRemoteAuthOptions(p)
	log.Debugf("set auth options %+v", p.AuthOptions)

	log.Debugf("setting up certificates")

	configureAuth := func() error {
		if err := configureAuth(p); err != nil {
			return &util.RetriableError{Err: err}
		}
		return nil
	}

	err := util.RetryAfter(5, configureAuth, time.Second*10)
	if err != nil {
		log.Debugf("Error configuring auth during provisioning %v", err)
		return err
	}

	return nil
}

func setRemoteAuthOptions(p provision.Provisioner) auth.Options {
	dockerDir := p.GetDockerOptionsDir()
	authOptions := p.GetAuthOptions()

	// due to windows clients, we cannot use filepath.Join as the paths
	// will be mucked on the linux hosts
	authOptions.CaCertRemotePath = path.Join(dockerDir, "ca.pem")
	authOptions.ServerCertRemotePath = path.Join(dockerDir, "server.pem")
	authOptions.ServerKeyRemotePath = path.Join(dockerDir, "server-key.pem")

	return authOptions
}

func configureAuth(p *BuildrootProvisioner) error {
	driver := p.GetDriver()
	machineName := driver.GetMachineName()
	authOptions := p.GetAuthOptions()
	org := mcnutils.GetUsername() + "." + machineName
	bits := 2048

	ip, err := driver.GetIP()
	if err != nil {
		return errors.Wrap(err, "error getting ip during provisioning")
	}

	execRunner := &bootstrapper.ExecRunner{}
	hostCerts := map[string]string{
		authOptions.CaCertPath:     filepath.Join(authOptions.StorePath, "ca.pem"),
		authOptions.ClientCertPath: filepath.Join(authOptions.StorePath, "cert.pem"),
		authOptions.ClientKeyPath:  filepath.Join(authOptions.StorePath, "key.pem"),
	}

	for src, dst := range hostCerts {
		f, err := assets.NewFileAsset(src, filepath.Dir(dst), filepath.Base(dst), "0777")
		if err != nil {
			return errors.Wrapf(err, "open cert file: %s", src)
		}
		if err := execRunner.Copy(f); err != nil {
			return errors.Wrapf(err, "transferring file: %+v", f)
		}
	}

	// The Host IP is always added to the certificate's SANs list
	hosts := append(authOptions.ServerCertSANs, ip, "localhost")
	log.Debugf("generating server cert: %s ca-key=%s private-key=%s org=%s san=%s",
		authOptions.ServerCertPath,
		authOptions.CaCertPath,
		authOptions.CaPrivateKeyPath,
		org,
		hosts,
	)

	err = cert.GenerateCert(&cert.Options{
		Hosts:     hosts,
		CertFile:  authOptions.ServerCertPath,
		KeyFile:   authOptions.ServerKeyPath,
		CAFile:    authOptions.CaCertPath,
		CAKeyFile: authOptions.CaPrivateKeyPath,
		Org:       org,
		Bits:      bits,
	})

	if err != nil {
		return fmt.Errorf("error generating server cert: %s", err)
	}

	remoteCerts := map[string]string{
		authOptions.CaCertPath:     authOptions.CaCertRemotePath,
		authOptions.ServerCertPath: authOptions.ServerCertRemotePath,
		authOptions.ServerKeyPath:  authOptions.ServerKeyRemotePath,
	}

	sshClient, err := sshutil.NewSSHClient(driver)
	if err != nil {
		return errors.Wrap(err, "provisioning: error getting ssh client")
	}
	sshRunner := bootstrapper.NewSSHRunner(sshClient)
	for src, dst := range remoteCerts {
		f, err := assets.NewFileAsset(src, filepath.Dir(dst), filepath.Base(dst), "0640")
		if err != nil {
			return errors.Wrapf(err, "error copying %s to %s", src, dst)
		}
		if err := sshRunner.Copy(f); err != nil {
			return errors.Wrapf(err, "transfering file to machine %v", f)
		}
	}

	dockerCfg, err := p.GenerateDockerOptions(engine.DefaultPort)
	if err != nil {
		return errors.Wrap(err, "generating docker options")
	}

	log.Info("Setting Docker configuration on the remote daemon...")

	if _, err = p.SSHCommand(fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | sudo tee %s", path.Dir(dockerCfg.EngineOptionsPath), dockerCfg.EngineOptions, dockerCfg.EngineOptionsPath)); err != nil {
		return err
	}

	if err := p.Service("docker", serviceaction.Restart); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduler

import (
	"time"

	"github.com/golang/glog"

	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/rest"

	"github.com/kubernetes-incubator/kube-arbitrator/pkg/client"
	schedcache "github.com/kubernetes-incubator/kube-arbitrator/pkg/scheduler/cache"
	"github.com/kubernetes-incubator/kube-arbitrator/pkg/scheduler/framework"
)

type Scheduler struct {
	cache  schedcache.Cache
	config *rest.Config
}

func NewScheduler(config *rest.Config, schedulerName string) (*Scheduler, error) {
	scheduler := &Scheduler{
		config: config,
		cache:  schedcache.New(config, schedulerName),
	}

	return scheduler, nil
}

func (pc *Scheduler) Run(stopCh <-chan struct{}) {
	createSchedulingSpecKind(pc.config)

	// Start cache for policy.
	go pc.cache.Run(stopCh)
	pc.cache.WaitForCacheSync(stopCh)

	go wait.Until(pc.runOnce, 2*time.Second, stopCh)
}

func (pc *Scheduler) runOnce() {
	glog.V(4).Infof("Start scheduling ...")
	defer glog.V(4).Infof("End scheduling ...")

	ssn := framework.OpenSession(pc.cache)

	for _, action := range Actions {
		action.Execute(ssn)
	}

	framework.CloseSession(ssn)
}

func createSchedulingSpecKind(config *rest.Config) error {
	extensionscs, err := apiextensionsclient.NewForConfig(config)
	if err != nil {
		return err
	}
	_, err = client.CreateSchedulingSpecKind(extensionscs)
	if err != nil && !apierrors.IsAlreadyExists(err) {
		return err
	}
	return nil
}
<commit_msg>Used defer to close session.<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduler

import (
	"time"

	"github.com/golang/glog"

	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/rest"

	"github.com/kubernetes-incubator/kube-arbitrator/pkg/client"
	schedcache "github.com/kubernetes-incubator/kube-arbitrator/pkg/scheduler/cache"
	"github.com/kubernetes-incubator/kube-arbitrator/pkg/scheduler/framework"
)

type Scheduler struct {
	cache  schedcache.Cache
	config *rest.Config
}

func NewScheduler(config *rest.Config, schedulerName string) (*Scheduler, error) {
	scheduler := &Scheduler{
		config: config,
		cache:  schedcache.New(config, schedulerName),
	}

	return scheduler, nil
}

func (pc *Scheduler) Run(stopCh <-chan struct{}) {
	createSchedulingSpecKind(pc.config)

	// Start cache for policy.
	go pc.cache.Run(stopCh)
	pc.cache.WaitForCacheSync(stopCh)

	go wait.Until(pc.runOnce, 2*time.Second, stopCh)
}

func (pc *Scheduler) runOnce() {
	glog.V(4).Infof("Start scheduling ...")
	defer glog.V(4).Infof("End scheduling ...")

	ssn := framework.OpenSession(pc.cache)
	defer framework.CloseSession(ssn)

	for _, action := range Actions {
		action.Execute(ssn)
	}

}

func createSchedulingSpecKind(config *rest.Config) error {
	extensionscs, err := apiextensionsclient.NewForConfig(config)
	if err != nil {
		return err
	}
	_, err = client.CreateSchedulingSpecKind(extensionscs)
	if err != nil && !apierrors.IsAlreadyExists(err) {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package client

import (
	"bytes"
	"fmt"
	lang "github.com/Aptomi/aptomi/pkg/slinga/language"
	"github.com/Aptomi/aptomi/pkg/slinga/object"
	"github.com/Aptomi/aptomi/pkg/slinga/object/codec"
	"github.com/Aptomi/aptomi/pkg/slinga/object/codec/yaml"
	"github.com/gosuri/uitable"
	"github.com/mattn/go-zglob"
	"github.com/spf13/viper"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"time"
)

func Apply(config *viper.Viper) error {
	policyPaths := config.GetStringSlice("apply.policyPaths")

	catalog := object.NewObjectCatalog(lang.ServiceObject, lang.ContractObject, lang.ClusterObject, lang.RuleObject, lang.DependencyObject)
	cod := yaml.NewCodec(catalog)

	allObjects, err := readFiles(policyPaths, cod)
	if err != nil {
		return err
	}

	// todo(slukjanov): here we can use some more efficient marshaller like gob, etc
	// prepare single []byte to send to API
	data, err := cod.MarshalMany(allObjects)
	if err != nil {
		return fmt.Errorf("Error while marshaling data for sending to API: %s", err)
	}

	client := &http.Client{
		Timeout: 5 * time.Second,
	}

	host := config.GetString("server.host")
	port := config.GetInt("server.port")

	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://%s:%d/api/v1/revision", host, port), bytes.NewBuffer(data))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/yaml")
	req.Header.Set("User-Agent", "aptomictl")

	fmt.Println("Request:", req)

	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// todo(slujanov): process response - check status and print returned data
	fmt.Println("Response:", resp)

	respData, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		panic(fmt.Sprintf("Error while reading bytes from response Body: %s", err))
	}

	objects, err := cod.UnmarshalOneOrMany(respData)
	if err != nil {
		panic(fmt.Sprintf("Error while unmarshaling response: %s", err))
	}

	// todo(slukjanov): pretty print response
	table := uitable.New()
	table.MaxColWidth = 50

	table.AddRow("#", "Namespace", "Kind", "Name", "Generation", "Object")
	for idx, obj := range objects {
		table.AddRow(idx, obj.GetNamespace(), obj.GetKind(), obj.GetName(), obj.GetGeneration(), obj)
	}
	fmt.Println(table)

	return nil
}

func readFiles(policyPaths []string, codec codec.MarshalUnmarshaler) ([]object.Base, error) {
	files, err := findPolicyFiles(policyPaths)
	if err != nil {
		return nil, fmt.Errorf("Error while searching for policy files: %s", err)
	}

	allObjects := make([]object.Base, 0)
	for _, file := range files {
		data, err := ioutil.ReadFile(file)
		if err != nil {
			return nil, fmt.Errorf("Can't read file %s error: %s", file, err)
		}

		// todo(slukjanov): here we can try multiple marshalers, toml for example
		objects, err := codec.UnmarshalOneOrMany(data)
		if err != nil {
			return nil, fmt.Errorf("Can't unmarshal file %s error: %s", file, err)
		}

		allObjects = append(allObjects, objects...)
	}

	if len(allObjects) == 0 {
		return nil, fmt.Errorf("No objects found in %s", policyPaths)
	}

	return allObjects, nil
}

func findPolicyFiles(policyPaths []string) ([]string, error) {
	allFiles := make([]string, 0, len(policyPaths))

	for _, rawPolicyPath := range policyPaths {
		policyPath, err := filepath.Abs(rawPolicyPath)
		if err != nil {
			return nil, fmt.Errorf("Error reading filepath: %s", err)
		}

		if stat, err := os.Stat(policyPath); err == nil {
			if stat.IsDir() { // if dir provided, use all yaml files from it
				files, err := zglob.Glob(filepath.Join(policyPath, "**", "*.yaml"))
				if err != nil {
					return nil, fmt.Errorf("Error while searching yaml files in directory: %s error: %s", policyPath, err)
				}
				allFiles = append(allFiles, files...)
			} else { // if specific file provided, use it
				allFiles = append(allFiles, policyPath)
			}
		} else if os.IsNotExist(err) {
			return nil, fmt.Errorf("Path doesn't exists: %s error: %s", policyPath, err)
		} else {
			return nil, fmt.Errorf("Error while processing path: %s", err)
		}
	}

	sort.Strings(allFiles) // todo(slukjanov): do we really need to sort files?

	// todo(slukjanov): log list of files from which we're applying policy
	//fmt.Println("Apply policy from following files:")
	//for idx, policyPath := range allFiles {
	//	fmt.Println(idx, "-", policyPath)
	//}

	return allFiles, nil
}

/*
c := &http.Client{
Timeout: 15 * time.Second,
}
resp, err := c.Get("https://blog.filippo.io/")
*/
<commit_msg>Drop unneeded comment<commit_after>package client

import (
	"bytes"
	"fmt"
	lang "github.com/Aptomi/aptomi/pkg/slinga/language"
	"github.com/Aptomi/aptomi/pkg/slinga/object"
	"github.com/Aptomi/aptomi/pkg/slinga/object/codec"
	"github.com/Aptomi/aptomi/pkg/slinga/object/codec/yaml"
	"github.com/gosuri/uitable"
	"github.com/mattn/go-zglob"
	"github.com/spf13/viper"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"time"
)

func Apply(config *viper.Viper) error {
	policyPaths := config.GetStringSlice("apply.policyPaths")

	catalog := object.NewObjectCatalog(lang.ServiceObject, lang.ContractObject, lang.ClusterObject, lang.RuleObject, lang.DependencyObject)
	cod := yaml.NewCodec(catalog)

	allObjects, err := readFiles(policyPaths, cod)
	if err != nil {
		return err
	}

	// todo(slukjanov): here we can use some more efficient marshaller like gob, etc
	// prepare single []byte to send to API
	data, err := cod.MarshalMany(allObjects)
	if err != nil {
		return fmt.Errorf("Error while marshaling data for sending to API: %s", err)
	}

	client := &http.Client{
		Timeout: 5 * time.Second,
	}

	host := config.GetString("server.host")
	port := config.GetInt("server.port")

	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://%s:%d/api/v1/revision", host, port), bytes.NewBuffer(data))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/yaml")
	req.Header.Set("User-Agent", "aptomictl")

	fmt.Println("Request:", req)

	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// todo(slujanov): process response - check status and print returned data
	fmt.Println("Response:", resp)

	respData, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		panic(fmt.Sprintf("Error while reading bytes from response Body: %s", err))
	}

	objects, err := cod.UnmarshalOneOrMany(respData)
	if err != nil {
		panic(fmt.Sprintf("Error while unmarshaling response: %s", err))
	}

	// todo(slukjanov): pretty print response
	table := uitable.New()
	table.MaxColWidth = 50

	table.AddRow("#", "Namespace", "Kind", "Name", "Generation", "Object")
	for idx, obj := range objects {
		table.AddRow(idx, obj.GetNamespace(), obj.GetKind(), obj.GetName(), obj.GetGeneration(), obj)
	}
	fmt.Println(table)

	return nil
}

func readFiles(policyPaths []string, codec codec.MarshalUnmarshaler) ([]object.Base, error) {
	files, err := findPolicyFiles(policyPaths)
	if err != nil {
		return nil, fmt.Errorf("Error while searching for policy files: %s", err)
	}

	allObjects := make([]object.Base, 0)
	for _, file := range files {
		data, err := ioutil.ReadFile(file)
		if err != nil {
			return nil, fmt.Errorf("Can't read file %s error: %s", file, err)
		}

		// todo(slukjanov): here we can try multiple marshalers, toml for example
		objects, err := codec.UnmarshalOneOrMany(data)
		if err != nil {
			return nil, fmt.Errorf("Can't unmarshal file %s error: %s", file, err)
		}

		allObjects = append(allObjects, objects...)
	}

	if len(allObjects) == 0 {
		return nil, fmt.Errorf("No objects found in %s", policyPaths)
	}

	return allObjects, nil
}

func findPolicyFiles(policyPaths []string) ([]string, error) {
	allFiles := make([]string, 0, len(policyPaths))

	for _, rawPolicyPath := range policyPaths {
		policyPath, err := filepath.Abs(rawPolicyPath)
		if err != nil {
			return nil, fmt.Errorf("Error reading filepath: %s", err)
		}

		if stat, err := os.Stat(policyPath); err == nil {
			if stat.IsDir() { // if dir provided, use all yaml files from it
				files, err := zglob.Glob(filepath.Join(policyPath, "**", "*.yaml"))
				if err != nil {
					return nil, fmt.Errorf("Error while searching yaml files in directory: %s error: %s", policyPath, err)
				}
				allFiles = append(allFiles, files...)
			} else { // if specific file provided, use it
				allFiles = append(allFiles, policyPath)
			}
		} else if os.IsNotExist(err) {
			return nil, fmt.Errorf("Path doesn't exists: %s error: %s", policyPath, err)
		} else {
			return nil, fmt.Errorf("Error while processing path: %s", err)
		}
	}

	sort.Strings(allFiles) // todo(slukjanov): do we really need to sort files?

	// todo(slukjanov): log list of files from which we're applying policy
	//fmt.Println("Apply policy from following files:")
	//for idx, policyPath := range allFiles {
	//	fmt.Println(idx, "-", policyPath)
	//}

	return allFiles, nil
}
<|endoftext|>


<commit_before>// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package pretty is a pretty-printer for state streams.
package pretty

import (
	"fmt"
	"io"
	"io/ioutil"
	"reflect"
	"strings"

	"gvisor.dev/gvisor/pkg/state"
	"gvisor.dev/gvisor/pkg/state/wire"
)

func formatRef(x *wire.Ref, graph uint64, html bool) string {
	baseRef := fmt.Sprintf("g%dr%d", graph, x.Root)
	fullRef := baseRef
	if len(x.Dots) > 0 {
		// See wire.Ref; Type valid if Dots non-zero.
		typ, _ := formatType(x.Type, graph, html)
		var buf strings.Builder
		buf.WriteString("(*")
		buf.WriteString(typ)
		buf.WriteString(")(")
		buf.WriteString(baseRef)
		for _, component := range x.Dots {
			switch v := component.(type) {
			case *wire.FieldName:
				buf.WriteString(".")
				buf.WriteString(string(*v))
			case wire.Index:
				buf.WriteString(fmt.Sprintf("[%d]", v))
			default:
				panic(fmt.Sprintf("unreachable: switch should be exhaustive, unhandled case %v", reflect.TypeOf(component)))
			}
		}
		buf.WriteString(")")
		fullRef = buf.String()
	}
	if html {
		return fmt.Sprintf("%s", baseRef, fullRef)
	}
	return fullRef
}

func formatType(t wire.TypeSpec, graph uint64, html bool) (string, bool) {
	switch x := t.(type) {
	case wire.TypeID:
		base := fmt.Sprintf("g%dt%d", graph, x)
		if html {
			return fmt.Sprintf("%s", base, base), true
		}
		return fmt.Sprintf("%s", base), true
	case wire.TypeSpecNil:
		return "", false // Only nil type.
	case *wire.TypeSpecPointer:
		element, _ := formatType(x.Type, graph, html)
		return fmt.Sprintf("(*%s)", element), true
	case *wire.TypeSpecArray:
		element, _ := formatType(x.Type, graph, html)
		return fmt.Sprintf("[%d](%s)", x.Count, element), true
	case *wire.TypeSpecSlice:
		element, _ := formatType(x.Type, graph, html)
		return fmt.Sprintf("([]%s)", element), true
	case *wire.TypeSpecMap:
		key, _ := formatType(x.Key, graph, html)
		value, _ := formatType(x.Value, graph, html)
		return fmt.Sprintf("(map[%s]%s)", key, value), true
	default:
		panic(fmt.Sprintf("unreachable: unknown type %T", t))
	}
}

// format formats a single object, for pretty-printing. It also returns whether
// the value is a non-zero value.
func format(graph uint64, depth int, encoded wire.Object, html bool) (string, bool) {
	switch x := encoded.(type) {
	case wire.Nil:
		return "nil", false
	case *wire.String:
		return fmt.Sprintf("%q", *x), *x != ""
	case *wire.Complex64:
		return fmt.Sprintf("%f+%fi", real(*x), imag(*x)), *x != 0.0
	case *wire.Complex128:
		return fmt.Sprintf("%f+%fi", real(*x), imag(*x)), *x != 0.0
	case *wire.Ref:
		return formatRef(x, graph, html), x.Root != 0
	case *wire.Type:
		tabs := "\n" + strings.Repeat("\t", depth)
		items := make([]string, 0, len(x.Fields)+2)
		items = append(items, fmt.Sprintf("type %s {", x.Name))
		for i := 0; i < len(x.Fields); i++ {
			items = append(items, fmt.Sprintf("\t%d: %s,", i, x.Fields[i]))
		}
		items = append(items, "}")
		return strings.Join(items, tabs), true // No zero value.
	case *wire.Slice:
		return fmt.Sprintf("%s{len:%d,cap:%d}", formatRef(&x.Ref, graph, html), x.Length, x.Capacity), x.Capacity != 0
	case *wire.Array:
		if len(x.Contents) == 0 {
			return "[]", false
		}
		items := make([]string, 0, len(x.Contents)+2)
		zeros := make([]string, 0) // used to eliminate zero entries.
		items = append(items, "[")
		tabs := "\n" + strings.Repeat("\t", depth)
		for i := 0; i < len(x.Contents); i++ {
			item, ok := format(graph, depth+1, x.Contents[i], html)
			if !ok {
				zeros = append(zeros, fmt.Sprintf("\t%s,", item))
				continue
			}
			if len(zeros) > 0 {
				items = append(items, zeros...)
				zeros = nil
			}
			items = append(items, fmt.Sprintf("\t%s,", item))
		}
		if len(zeros) > 0 {
			items = append(items, fmt.Sprintf("\t... (%d zeros),", len(zeros)))
		}
		items = append(items, "]")
		return strings.Join(items, tabs), len(zeros) < len(x.Contents)
	case *wire.Struct:
		typ, _ := formatType(x.TypeID, graph, html)
		if x.Fields() == 0 {
			return fmt.Sprintf("struct[%s]{}", typ), false
		}
		items := make([]string, 0, 2)
		items = append(items, fmt.Sprintf("struct[%s]{", typ))
		tabs := "\n" + strings.Repeat("\t", depth)
		allZero := true
		for i := 0; i < x.Fields(); i++ {
			element, ok := format(graph, depth+1, *x.Field(i), html)
			allZero = allZero && !ok
			items = append(items, fmt.Sprintf("\t%d: %s,", i, element))
		}
		items = append(items, "}")
		return strings.Join(items, tabs), !allZero
	case *wire.Map:
		if len(x.Keys) == 0 {
			return "map{}", false
		}
		items := make([]string, 0, len(x.Keys)+2)
		items = append(items, "map{")
		tabs := "\n" + strings.Repeat("\t", depth)
		for i := 0; i < len(x.Keys); i++ {
			key, _ := format(graph, depth+1, x.Keys[i], html)
			value, _ := format(graph, depth+1, x.Values[i], html)
			items = append(items, fmt.Sprintf("\t%s: %s,", key, value))
		}
		items = append(items, "}")
		return strings.Join(items, tabs), true
	case *wire.Interface:
		typ, typOk := formatType(x.Type, graph, html)
		element, elementOk := format(graph, depth+1, x.Value, html)
		return fmt.Sprintf("interface[%s]{%s}", typ, element), typOk || elementOk
	default:
		// Must be a primitive; use reflection.
		return fmt.Sprintf("%v", encoded), true
	}
}

// printStream is the basic print implementation.
func printStream(w io.Writer, r wire.Reader, html bool) (err error) {
	// current graph ID.
	var graph uint64

	if html {
		fmt.Fprintf(w, "
")
		defer fmt.Fprintf(w, "
")
	}

	defer func() {
		if r := recover(); r != nil {
			if rErr, ok := r.(error); ok {
				err = rErr // Override return.
				return
			}
			panic(r) // Propagate.
		}
	}()

	for {
		// Find the first object to begin generation.
		length, object, err := state.ReadHeader(r)
		if err == io.EOF {
			// Nothing else to do.
			break
		} else if err != nil {
			return err
		}
		if !object {
			graph++ // Increment the graph.
			if length > 0 {
				fmt.Fprintf(w, "(%d bytes non-object data)\n", length)
				io.Copy(ioutil.Discard, &io.LimitedReader{
					R: r,
					N: int64(length),
				})
			}
			continue
		}

		// Read & unmarshal the object.
		//
		// Note that this loop must match the general structure of the
		// loop in decode.go. But we don't register type information,
		// etc. and just print the raw structures.
		var (
			oid uint64 = 1
			tid uint64 = 1
		)
		for oid <= length {
			// Unmarshal the object.
			encoded := wire.Load(r)

			// Is this a type?
			if _, ok := encoded.(*wire.Type); ok {
				str, _ := format(graph, 0, encoded, html)
				tag := fmt.Sprintf("g%dt%d", graph, tid)
				if html {
					// See below.
					tag = fmt.Sprintf("%s⚓", tag, tag, tag)
				}
				if _, err := fmt.Fprintf(w, "%s = %s\n", tag, str); err != nil {
					return err
				}
				tid++
				continue
			}

			// Format the node.
			str, _ := format(graph, 0, encoded, html)
			tag := fmt.Sprintf("g%dr%d", graph, oid)
			if html {
				// Create a little tag with an anchor next to it for linking.
				tag = fmt.Sprintf("%s⚓", tag, tag, tag)
			}
			if _, err := fmt.Fprintf(w, "%s = %s\n", tag, str); err != nil {
				return err
			}
			oid++
		}
	}

	return nil
}

// PrintText reads the stream from r and prints text to w.
func PrintText(w io.Writer, r wire.Reader) error {
	return printStream(w, r, false /* html */)
}

// PrintHTML reads the stream from r and prints html to w.
func PrintHTML(w io.Writer, r wire.Reader) error {
	return printStream(w, r, true /* html */)
}
<commit_msg>stateify: Bring back struct field and type names in pretty print<commit_after>// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package pretty is a pretty-printer for state streams.
package pretty

import (
	"fmt"
	"io"
	"io/ioutil"
	"reflect"
	"strings"

	"gvisor.dev/gvisor/pkg/state"
	"gvisor.dev/gvisor/pkg/state/wire"
)

type printer struct {
	html      bool
	typeSpecs map[string]*wire.Type
}

func (p *printer) formatRef(x *wire.Ref, graph uint64) string {
	baseRef := fmt.Sprintf("g%dr%d", graph, x.Root)
	fullRef := baseRef
	if len(x.Dots) > 0 {
		// See wire.Ref; Type valid if Dots non-zero.
		typ, _ := p.formatType(x.Type, graph)
		var buf strings.Builder
		buf.WriteString("(*")
		buf.WriteString(typ)
		buf.WriteString(")(")
		buf.WriteString(baseRef)
		for _, component := range x.Dots {
			switch v := component.(type) {
			case *wire.FieldName:
				buf.WriteString(".")
				buf.WriteString(string(*v))
			case wire.Index:
				buf.WriteString(fmt.Sprintf("[%d]", v))
			default:
				panic(fmt.Sprintf("unreachable: switch should be exhaustive, unhandled case %v", reflect.TypeOf(component)))
			}
		}
		buf.WriteString(")")
		fullRef = buf.String()
	}
	if p.html {
		return fmt.Sprintf("%s", baseRef, fullRef)
	}
	return fullRef
}

func (p *printer) formatType(t wire.TypeSpec, graph uint64) (string, bool) {
	switch x := t.(type) {
	case wire.TypeID:
		tag := fmt.Sprintf("g%dt%d", graph, x)
		desc := tag
		if spec, ok := p.typeSpecs[tag]; ok {
			desc += fmt.Sprintf("=%s", spec.Name)
		} else {
			desc += "!missing-type-spec"
		}
		if p.html {
			return fmt.Sprintf("%s", tag, desc), true
		}
		return desc, true
	case wire.TypeSpecNil:
		return "", false // Only nil type.
	case *wire.TypeSpecPointer:
		element, _ := p.formatType(x.Type, graph)
		return fmt.Sprintf("(*%s)", element), true
	case *wire.TypeSpecArray:
		element, _ := p.formatType(x.Type, graph)
		return fmt.Sprintf("[%d](%s)", x.Count, element), true
	case *wire.TypeSpecSlice:
		element, _ := p.formatType(x.Type, graph)
		return fmt.Sprintf("([]%s)", element), true
	case *wire.TypeSpecMap:
		key, _ := p.formatType(x.Key, graph)
		value, _ := p.formatType(x.Value, graph)
		return fmt.Sprintf("(map[%s]%s)", key, value), true
	default:
		panic(fmt.Sprintf("unreachable: unknown type %T", t))
	}
}

// format formats a single object, for pretty-printing. It also returns whether
// the value is a non-zero value.
func (p *printer) format(graph uint64, depth int, encoded wire.Object) (string, bool) {
	switch x := encoded.(type) {
	case wire.Nil:
		return "nil", false
	case *wire.String:
		return fmt.Sprintf("%q", *x), *x != ""
	case *wire.Complex64:
		return fmt.Sprintf("%f+%fi", real(*x), imag(*x)), *x != 0.0
	case *wire.Complex128:
		return fmt.Sprintf("%f+%fi", real(*x), imag(*x)), *x != 0.0
	case *wire.Ref:
		return p.formatRef(x, graph), x.Root != 0
	case *wire.Type:
		tabs := "\n" + strings.Repeat("\t", depth)
		items := make([]string, 0, len(x.Fields)+2)
		items = append(items, fmt.Sprintf("type %s {", x.Name))
		for i := 0; i < len(x.Fields); i++ {
			items = append(items, fmt.Sprintf("\t%d: %s,", i, x.Fields[i]))
		}
		items = append(items, "}")
		return strings.Join(items, tabs), true // No zero value.
	case *wire.Slice:
		return fmt.Sprintf("%s{len:%d,cap:%d}", p.formatRef(&x.Ref, graph), x.Length, x.Capacity), x.Capacity != 0
	case *wire.Array:
		if len(x.Contents) == 0 {
			return "[]", false
		}
		items := make([]string, 0, len(x.Contents)+2)
		zeros := make([]string, 0) // used to eliminate zero entries.
		items = append(items, "[")
		tabs := "\n" + strings.Repeat("\t", depth)
		for i := 0; i < len(x.Contents); i++ {
			item, ok := p.format(graph, depth+1, x.Contents[i])
			if !ok {
				zeros = append(zeros, fmt.Sprintf("\t%s,", item))
				continue
			}
			if len(zeros) > 0 {
				items = append(items, zeros...)
				zeros = nil
			}
			items = append(items, fmt.Sprintf("\t%s,", item))
		}
		if len(zeros) > 0 {
			items = append(items, fmt.Sprintf("\t... (%d zeros),", len(zeros)))
		}
		items = append(items, "]")
		return strings.Join(items, tabs), len(zeros) < len(x.Contents)
	case *wire.Struct:
		tag := fmt.Sprintf("g%dt%d", graph, x.TypeID)
		spec, _ := p.typeSpecs[tag]
		typ, _ := p.formatType(x.TypeID, graph)
		if x.Fields() == 0 {
			return fmt.Sprintf("struct[%s]{}", typ), false
		}
		items := make([]string, 0, 2)
		items = append(items, fmt.Sprintf("struct[%s]{", typ))
		tabs := "\n" + strings.Repeat("\t", depth)
		allZero := true
		for i := 0; i < x.Fields(); i++ {
			var name string
			if spec != nil && i < len(spec.Fields) {
				name = spec.Fields[i]
			} else {
				name = fmt.Sprintf("%d", i)
			}
			element, ok := p.format(graph, depth+1, *x.Field(i))
			allZero = allZero && !ok
			items = append(items, fmt.Sprintf("\t%s: %s,", name, element))
		}
		items = append(items, "}")
		return strings.Join(items, tabs), !allZero
	case *wire.Map:
		if len(x.Keys) == 0 {
			return "map{}", false
		}
		items := make([]string, 0, len(x.Keys)+2)
		items = append(items, "map{")
		tabs := "\n" + strings.Repeat("\t", depth)
		for i := 0; i < len(x.Keys); i++ {
			key, _ := p.format(graph, depth+1, x.Keys[i])
			value, _ := p.format(graph, depth+1, x.Values[i])
			items = append(items, fmt.Sprintf("\t%s: %s,", key, value))
		}
		items = append(items, "}")
		return strings.Join(items, tabs), true
	case *wire.Interface:
		typ, typOk := p.formatType(x.Type, graph)
		element, elementOk := p.format(graph, depth+1, x.Value)
		return fmt.Sprintf("interface[%s]{%s}", typ, element), typOk || elementOk
	default:
		// Must be a primitive; use reflection.
		return fmt.Sprintf("%v", encoded), true
	}
}

// printStream is the basic print implementation.
func (p *printer) printStream(w io.Writer, r wire.Reader) (err error) {
	// current graph ID.
	var graph uint64

	if p.html {
		fmt.Fprintf(w, "
")
		defer fmt.Fprintf(w, "
")
	}

	defer func() {
		if r := recover(); r != nil {
			if rErr, ok := r.(error); ok {
				err = rErr // Override return.
				return
			}
			panic(r) // Propagate.
		}
	}()

	p.typeSpecs = make(map[string]*wire.Type)

	for {
		// Find the first object to begin generation.
		length, object, err := state.ReadHeader(r)
		if err == io.EOF {
			// Nothing else to do.
			break
		} else if err != nil {
			return err
		}
		if !object {
			graph++ // Increment the graph.
			if length > 0 {
				fmt.Fprintf(w, "(%d bytes non-object data)\n", length)
				io.Copy(ioutil.Discard, &io.LimitedReader{
					R: r,
					N: int64(length),
				})
			}
			continue
		}

		// Read & unmarshal the object.
		//
		// Note that this loop must match the general structure of the
		// loop in decode.go. But we don't register type information,
		// etc. and just print the raw structures.
		var (
			tid     uint64 = 1
			objects []wire.Object
		)
		for oid := uint64(1); oid <= length; {
			// Unmarshal the object.
			encoded := wire.Load(r)

			// Is this a type?
			if typ, ok := encoded.(*wire.Type); ok {
				str, _ := p.format(graph, 0, encoded)
				tag := fmt.Sprintf("g%dt%d", graph, tid)
				p.typeSpecs[tag] = typ
				if p.html {
					// See below.
					tag = fmt.Sprintf("%s⚓", tag, tag, tag)
				}
				if _, err := fmt.Fprintf(w, "%s = %s\n", tag, str); err != nil {
					return err
				}
				tid++
				continue
			}

			// Otherwise, it is a node.
			objects = append(objects, encoded)
			oid++
		}

		for i, encoded := range objects {
			// oid starts at 1.
			oid := i + 1
			// Format the node.
			str, _ := p.format(graph, 0, encoded)
			tag := fmt.Sprintf("g%dr%d", graph, oid)
			if p.html {
				// Create a little tag with an anchor next to it for linking.
				tag = fmt.Sprintf("%s⚓", tag, tag, tag)
			}
			if _, err := fmt.Fprintf(w, "%s = %s\n", tag, str); err != nil {
				return err
			}
		}
	}

	return nil
}

// PrintText reads the stream from r and prints text to w.
func PrintText(w io.Writer, r wire.Reader) error {
	return (&printer{}).printStream(w, r)
}

// PrintHTML reads the stream from r and prints html to w.
func PrintHTML(w io.Writer, r wire.Reader) error {
	return (&printer{html: true}).printStream(w, r)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/gob"
	"flag"
	"log"
	"os"
	"runtime"

	"github.com/BurntSushi/bcbgo/apps/hhsuite"
	"github.com/BurntSushi/bcbgo/hhfrag"
)

var (
	flagBlits = false
	flagSeqDB = "nr20"
	flagPdbDB = "pdb-select25"
	flagCpu   = runtime.NumCPU()

	seqDB hhsuite.Database
	pdbDB hhfrag.PDBDatabase
)

func init() {
	log.SetFlags(0)

	flag.BoolVar(&flagBlits, "blits", flagBlits,
		"When set, hhblits will be used in lieu of hhsearch.")
	flag.StringVar(&flagSeqDB, "seqdb", flagSeqDB,
		"The sequence database used to generate the query HHM.")
	flag.StringVar(&flagPdbDB, "pdbdb", flagPdbDB,
		"The PDB/HHM database used to assign fragments.")
	flag.IntVar(&flagCpu, "cpu", flagCpu,
		"The max number of CPUs to use.")

	flag.Usage = usage
	flag.Parse()

	seqDB = hhsuite.Database(flagSeqDB)
	pdbDB = hhfrag.PDBDatabase(flagPdbDB)

	runtime.GOMAXPROCS(flagCpu)
}

func usage() {
	log.Printf("Usage: hhfrag-map [flags] target-fasta out-fmap\n")
	flag.PrintDefaults()
}

func main() {
	if flag.NArg() != 2 {
		usage()
	}

	fasInp := flag.Arg(0)
	fmapOut := flag.Arg(1)

	conf := hhfrag.DefaultConfig
	conf.Blits = flagBlits
	fmap, err := conf.MapFromFasta(pdbDB, seqDB, fasInp)
	assert(err)

	out, err := os.Create(fmapOut)
	assert(err)
	w := gob.NewEncoder(out)
	err = w.Encode(fmap)
	assert(err)
}

func assert(err error) {
	if err != nil {
		log.Fatalln(err)
	}
}
<commit_msg>Allow hhfrag window parameters to be specified on the command line.<commit_after>package main

import (
	"encoding/gob"
	"flag"
	"log"
	"os"
	"runtime"

	"github.com/BurntSushi/bcbgo/apps/hhsuite"
	"github.com/BurntSushi/bcbgo/hhfrag"
)

var (
	hhfragConf = hhfrag.DefaultConfig
	flagSeqDB  = "nr20"
	flagPdbDB  = "pdb-select25"
	flagCpu    = runtime.NumCPU()

	seqDB hhsuite.Database
	pdbDB hhfrag.PDBDatabase
)

func init() {
	log.SetFlags(0)

	flag.BoolVar(&hhfragConf.Blits, "blits", hhfragConf.Blits,
		"When set, hhblits will be used in lieu of hhsearch.")
	flag.IntVar(&hhfragConf.WindowMin, "win-min", hhfragConf.WindowMin,
		"The minimum HMM window size for HHfrag.")
	flag.IntVar(&hhfragConf.WindowMax, "win-max", hhfragConf.WindowMax,
		"The maximum HMM window size for HHfrag.")
	flag.IntVar(&hhfragConf.WindowIncrement, "win-inc",
		hhfragConf.WindowIncrement,
		"The sliding window increment for HHfrag.")
	flag.StringVar(&flagSeqDB, "seqdb", flagSeqDB,
		"The sequence database used to generate the query HHM.")
	flag.StringVar(&flagPdbDB, "pdbdb", flagPdbDB,
		"The PDB/HHM database used to assign fragments.")
	flag.IntVar(&flagCpu, "cpu", flagCpu,
		"The max number of CPUs to use.")

	flag.Usage = usage
	flag.Parse()

	seqDB = hhsuite.Database(flagSeqDB)
	pdbDB = hhfrag.PDBDatabase(flagPdbDB)

	runtime.GOMAXPROCS(flagCpu)
}

func usage() {
	log.Printf("Usage: hhfrag-map [flags] target-fasta out-fmap\n")
	flag.PrintDefaults()
}

func main() {
	if flag.NArg() != 2 {
		usage()
	}

	fasInp := flag.Arg(0)
	fmapOut := flag.Arg(1)

	fmap, err := hhfragConf.MapFromFasta(pdbDB, seqDB, fasInp)
	assert(err)

	out, err := os.Create(fmapOut)
	assert(err)
	w := gob.NewEncoder(out)
	err = w.Encode(fmap)
	assert(err)
}

func assert(err error) {
	if err != nil {
		log.Fatalln(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package test

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"encoding/base64"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"math/big"
	"os"
	"path/filepath"
	"time"

	"github.com/matrix-org/dendrite/common/config"
	"github.com/matrix-org/gomatrixserverlib"
	"gopkg.in/yaml.v2"
)

const (
	// ConfigFile is the name of the config file for a server.
	ConfigFile = "dendrite.yaml"
	// ServerKeyFile is the name of the file holding the matrix server private key.
	ServerKeyFile = "server_key.pem"
	// TLSCertFile is the name of the file holding the TLS certificate used for federation.
	TLSCertFile = "tls_cert.pem"
	// TLSKeyFile is the name of the file holding the TLS key used for federation.
	TLSKeyFile = "tls_key.pem"
	// MediaDir is the name of the directory used to store media.
	MediaDir = "media"
)

// MakeConfig makes a config suitable for running integration tests.
// Generates new matrix and TLS keys for the server.
func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*config.Dendrite, int, error) {
	var cfg config.Dendrite

	port := startPort
	assignAddress := func() config.Address {
		result := config.Address(fmt.Sprintf("%s:%d", host, port))
		port++
		return result
	}

	serverKeyPath := filepath.Join(configDir, ServerKeyFile)
	tlsCertPath := filepath.Join(configDir, TLSKeyFile)
	tlsKeyPath := filepath.Join(configDir, TLSCertFile)
	mediaBasePath := filepath.Join(configDir, MediaDir)

	if err := NewMatrixKey(serverKeyPath); err != nil {
		return nil, 0, err
	}

	if err := NewTLSKey(tlsKeyPath, tlsCertPath); err != nil {
		return nil, 0, err
	}

	cfg.Version = config.Version

	cfg.Matrix.ServerName = gomatrixserverlib.ServerName(assignAddress())
	cfg.Matrix.PrivateKeyPath = config.Path(serverKeyPath)
	cfg.Matrix.FederationCertificatePaths = []config.Path{config.Path(tlsCertPath)}

	cfg.Media.BasePath = config.Path(mediaBasePath)

	cfg.Kafka.Addresses = []string{kafkaURI}
	// TODO: Different servers should be using different topics.
	// Make this configurable somehow?
	cfg.Kafka.Topics.OutputRoomEvent = "test.room.output"
	cfg.Kafka.Topics.OutputClientData = "test.clientapi.output"
	cfg.Kafka.Topics.OutputTypingEvent = "test.typing.output"
	cfg.Kafka.Topics.UserUpdates = "test.user.output"

	// TODO: Use different databases for the different schemas.
	// Using the same database for every schema currently works because
	// the table names are globally unique. But we might not want to
	// rely on that in the future.
	cfg.Database.Account = config.DataSource(database)
	cfg.Database.Device = config.DataSource(database)
	cfg.Database.MediaAPI = config.DataSource(database)
	cfg.Database.RoomServer = config.DataSource(database)
	cfg.Database.ServerKey = config.DataSource(database)
	cfg.Database.SyncAPI = config.DataSource(database)
	cfg.Database.PublicRoomsAPI = config.DataSource(database)

	cfg.Listen.ClientAPI = assignAddress()
	cfg.Listen.FederationAPI = assignAddress()
	cfg.Listen.MediaAPI = assignAddress()
	cfg.Listen.RoomServer = assignAddress()
	cfg.Listen.SyncAPI = assignAddress()
	cfg.Listen.PublicRoomsAPI = assignAddress()
	cfg.Listen.TypingServer = assignAddress()

	// Bind to the same address as the listen address
	// All microservices are run on the same host in testing
	cfg.Bind.ClientAPI = cfg.Listen.ClientAPI
	cfg.Bind.FederationAPI = cfg.Listen.FederationAPI
	cfg.Bind.MediaAPI = cfg.Listen.MediaAPI
	cfg.Bind.RoomServer = cfg.Listen.RoomServer
	cfg.Bind.SyncAPI = cfg.Listen.SyncAPI
	cfg.Bind.PublicRoomsAPI = cfg.Listen.PublicRoomsAPI
	cfg.Bind.TypingServer = cfg.Listen.TypingServer

	return &cfg, port, nil
}

// WriteConfig writes the config file to the directory.
func WriteConfig(cfg *config.Dendrite, configDir string) error {
	data, err := yaml.Marshal(cfg)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(filepath.Join(configDir, ConfigFile), data, 0666)
}

// NewMatrixKey generates a new ed25519 matrix server key and writes it to a file.
func NewMatrixKey(matrixKeyPath string) (err error) {
	var data [35]byte
	_, err = rand.Read(data[:])
	if err != nil {
		return err
	}
	keyOut, err := os.OpenFile(matrixKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err
	}

	defer (func() {
		err = keyOut.Close()
	})()

	err = pem.Encode(keyOut, &pem.Block{
		Type: "MATRIX PRIVATE KEY",
		Headers: map[string]string{
			"Key-ID": "ed25519:" + base64.RawStdEncoding.EncodeToString(data[:3]),
		},
		Bytes: data[3:],
	})
	return err
}

const certificateDuration = time.Hour * 24 * 365 * 10

// NewTLSKey generates a new RSA TLS key and certificate and writes it to a file.
func NewTLSKey(tlsKeyPath, tlsCertPath string) error {
	priv, err := rsa.GenerateKey(rand.Reader, 4096)
	if err != nil {
		return err
	}

	notBefore := time.Now()
	notAfter := notBefore.Add(certificateDuration)
	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	if err != nil {
		return err
	}

	template := x509.Certificate{
		SerialNumber:          serialNumber,
		NotBefore:             notBefore,
		NotAfter:              notAfter,
		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
	}
	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
	if err != nil {
		return err
	}
	certOut, err := os.Create(tlsCertPath)
	if err != nil {
		return err
	}
	defer certOut.Close() // nolint: errcheck
	if err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
		return err
	}

	keyOut, err := os.OpenFile(tlsKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err
	}
	defer keyOut.Close() // nolint: errcheck
	err = pem.Encode(keyOut, &pem.Block{
		Type:  "RSA PRIVATE KEY",
		Bytes: x509.MarshalPKCS1PrivateKey(priv),
	})
	return err
}
<commit_msg>Add AppService to test config and bind addrs (#800)<commit_after>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package test

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"encoding/base64"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"math/big"
	"os"
	"path/filepath"
	"time"

	"github.com/matrix-org/dendrite/common/config"
	"github.com/matrix-org/gomatrixserverlib"
	"gopkg.in/yaml.v2"
)

const (
	// ConfigFile is the name of the config file for a server.
	ConfigFile = "dendrite.yaml"
	// ServerKeyFile is the name of the file holding the matrix server private key.
	ServerKeyFile = "server_key.pem"
	// TLSCertFile is the name of the file holding the TLS certificate used for federation.
	TLSCertFile = "tls_cert.pem"
	// TLSKeyFile is the name of the file holding the TLS key used for federation.
	TLSKeyFile = "tls_key.pem"
	// MediaDir is the name of the directory used to store media.
	MediaDir = "media"
)

// MakeConfig makes a config suitable for running integration tests.
// Generates new matrix and TLS keys for the server.
func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*config.Dendrite, int, error) {
	var cfg config.Dendrite

	port := startPort
	assignAddress := func() config.Address {
		result := config.Address(fmt.Sprintf("%s:%d", host, port))
		port++
		return result
	}

	serverKeyPath := filepath.Join(configDir, ServerKeyFile)
	tlsCertPath := filepath.Join(configDir, TLSKeyFile)
	tlsKeyPath := filepath.Join(configDir, TLSCertFile)
	mediaBasePath := filepath.Join(configDir, MediaDir)

	if err := NewMatrixKey(serverKeyPath); err != nil {
		return nil, 0, err
	}

	if err := NewTLSKey(tlsKeyPath, tlsCertPath); err != nil {
		return nil, 0, err
	}

	cfg.Version = config.Version

	cfg.Matrix.ServerName = gomatrixserverlib.ServerName(assignAddress())
	cfg.Matrix.PrivateKeyPath = config.Path(serverKeyPath)
	cfg.Matrix.FederationCertificatePaths = []config.Path{config.Path(tlsCertPath)}

	cfg.Media.BasePath = config.Path(mediaBasePath)

	cfg.Kafka.Addresses = []string{kafkaURI}
	// TODO: Different servers should be using different topics.
	// Make this configurable somehow?
	cfg.Kafka.Topics.OutputRoomEvent = "test.room.output"
	cfg.Kafka.Topics.OutputClientData = "test.clientapi.output"
	cfg.Kafka.Topics.OutputTypingEvent = "test.typing.output"
	cfg.Kafka.Topics.UserUpdates = "test.user.output"

	// TODO: Use different databases for the different schemas.
	// Using the same database for every schema currently works because
	// the table names are globally unique. But we might not want to
	// rely on that in the future.
	cfg.Database.Account = config.DataSource(database)
	cfg.Database.AppService = config.DataSource(database)
	cfg.Database.Device = config.DataSource(database)
	cfg.Database.MediaAPI = config.DataSource(database)
	cfg.Database.RoomServer = config.DataSource(database)
	cfg.Database.ServerKey = config.DataSource(database)
	cfg.Database.SyncAPI = config.DataSource(database)
	cfg.Database.PublicRoomsAPI = config.DataSource(database)

	cfg.Listen.ClientAPI = assignAddress()
	cfg.Listen.AppServiceAPI = assignAddress()
	cfg.Listen.FederationAPI = assignAddress()
	cfg.Listen.MediaAPI = assignAddress()
	cfg.Listen.RoomServer = assignAddress()
	cfg.Listen.SyncAPI = assignAddress()
	cfg.Listen.PublicRoomsAPI = assignAddress()
	cfg.Listen.TypingServer = assignAddress()

	// Bind to the same address as the listen address
	// All microservices are run on the same host in testing
	cfg.Bind.ClientAPI = cfg.Listen.ClientAPI
	cfg.Bind.FederationAPI = cfg.Listen.FederationAPI
	cfg.Bind.MediaAPI = cfg.Listen.MediaAPI
	cfg.Bind.RoomServer = cfg.Listen.RoomServer
	cfg.Bind.SyncAPI = cfg.Listen.SyncAPI
	cfg.Bind.PublicRoomsAPI = cfg.Listen.PublicRoomsAPI
	cfg.Bind.TypingServer = cfg.Listen.TypingServer

	return &cfg, port, nil
}

// WriteConfig writes the config file to the directory.
func WriteConfig(cfg *config.Dendrite, configDir string) error {
	data, err := yaml.Marshal(cfg)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(filepath.Join(configDir, ConfigFile), data, 0666)
}

// NewMatrixKey generates a new ed25519 matrix server key and writes it to a file.
func NewMatrixKey(matrixKeyPath string) (err error) {
	var data [35]byte
	_, err = rand.Read(data[:])
	if err != nil {
		return err
	}
	keyOut, err := os.OpenFile(matrixKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err
	}

	defer (func() {
		err = keyOut.Close()
	})()

	err = pem.Encode(keyOut, &pem.Block{
		Type: "MATRIX PRIVATE KEY",
		Headers: map[string]string{
			"Key-ID": "ed25519:" + base64.RawStdEncoding.EncodeToString(data[:3]),
		},
		Bytes: data[3:],
	})
	return err
}

const certificateDuration = time.Hour * 24 * 365 * 10

// NewTLSKey generates a new RSA TLS key and certificate and writes it to a file.
func NewTLSKey(tlsKeyPath, tlsCertPath string) error {
	priv, err := rsa.GenerateKey(rand.Reader, 4096)
	if err != nil {
		return err
	}

	notBefore := time.Now()
	notAfter := notBefore.Add(certificateDuration)
	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	if err != nil {
		return err
	}

	template := x509.Certificate{
		SerialNumber:          serialNumber,
		NotBefore:             notBefore,
		NotAfter:              notAfter,
		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
	}
	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
	if err != nil {
		return err
	}
	certOut, err := os.Create(tlsCertPath)
	if err != nil {
		return err
	}
	defer certOut.Close() // nolint: errcheck
	if err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
		return err
	}

	keyOut, err := os.OpenFile(tlsKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err
	}
	defer keyOut.Close() // nolint: errcheck
	err = pem.Encode(keyOut, &pem.Block{
		Type:  "RSA PRIVATE KEY",
		Bytes: x509.MarshalPKCS1PrivateKey(priv),
	})
	return err
}
<|endoftext|>


<commit_before>// This code requires a configuration file to initialize the crypto11
// library. Use the following config in crypto11.config:
//      {
//      "Path" : "/opt/cloudhsm/lib/libcloudhsm_pkcs11.so",
//      "TokenLabel": "cavium",
//      "Pin" : "$CRYPTO_USER:$PASSWORD"
//      }
// then invoke the program with:
// !CKNFAST_DEBUG=2 CRYPTO11_CONFIG_PATH=crypto11.config go run crypto11_genrsa.go
package main

import (
	"crypto/elliptic"
	"fmt"
	"log"

	"github.com/ThalesIgnite/crypto11"
)

func main() {
	p11Ctx, err := crypto11.Configure(&crypto11.PKCS11Config{
		Path:       "/usr/lib/softhsm/libsofthsm2.so",
		TokenLabel: "test",
		Pin:        "0000",
	})
	if err != nil {
		log.Fatal(err)
	}
	slots, err := p11Ctx.GetSlotList(true)
	if err != nil {
		log.Fatalf("Failed to list PKCS#11 Slots: %s", err.Error())
	}
	if len(slots) < 1 {
		log.Fatal("No slot found")
	}
	rsakey, err := crypto11.GenerateRSAKeyPairOnSlot(slots[0], []byte("testrsa2048"), []byte("testrsa2048"), 2048)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("RSA Key: %+v\n", rsakey)

	rsakey2, err := crypto11.GenerateRSAKeyPairOnSlot(slots[0], []byte("testrsa2048"), []byte("testrsa4096"), 4096)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("RSA 4096 Key: %+v\n", rsakey2)

	ecdsakey, err := crypto11.GenerateECDSAKeyPairOnSlot(slots[0], []byte("testecdsap384"), []byte("testecdsap384"), elliptic.P384())
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("ECDSA Key: %+v\n", ecdsakey)

	p384key, err := crypto11.GenerateECDSAKeyPairOnSlot(slots[0], []byte("testcsp384"), []byte("testcsp384"), elliptic.P384())
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("P384 Key: %+v\n", p384key)
}
<commit_msg>tools: fix typo in hsm genkeys s/2048/4096/<commit_after>// This code requires a configuration file to initialize the crypto11
// library. Use the following config in crypto11.config:
//      {
//      "Path" : "/opt/cloudhsm/lib/libcloudhsm_pkcs11.so",
//      "TokenLabel": "cavium",
//      "Pin" : "$CRYPTO_USER:$PASSWORD"
//      }
// then invoke the program with:
// !CKNFAST_DEBUG=2 CRYPTO11_CONFIG_PATH=crypto11.config go run crypto11_genrsa.go
package main

import (
	"crypto/elliptic"
	"fmt"
	"log"

	"github.com/ThalesIgnite/crypto11"
)

func main() {
	p11Ctx, err := crypto11.Configure(&crypto11.PKCS11Config{
		Path:       "/usr/lib/softhsm/libsofthsm2.so",
		TokenLabel: "test",
		Pin:        "0000",
	})
	if err != nil {
		log.Fatal(err)
	}
	slots, err := p11Ctx.GetSlotList(true)
	if err != nil {
		log.Fatalf("Failed to list PKCS#11 Slots: %s", err.Error())
	}
	if len(slots) < 1 {
		log.Fatal("No slot found")
	}
	rsakey, err := crypto11.GenerateRSAKeyPairOnSlot(slots[0], []byte("testrsa2048"), []byte("testrsa2048"), 2048)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("RSA Key: %+v\n", rsakey)

	rsakey2, err := crypto11.GenerateRSAKeyPairOnSlot(slots[0], []byte("testrsa4096"), []byte("testrsa4096"), 4096)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("RSA 4096 Key: %+v\n", rsakey2)

	ecdsakey, err := crypto11.GenerateECDSAKeyPairOnSlot(slots[0], []byte("testecdsap384"), []byte("testecdsap384"), elliptic.P384())
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("ECDSA Key: %+v\n", ecdsakey)

	p384key, err := crypto11.GenerateECDSAKeyPairOnSlot(slots[0], []byte("testcsp384"), []byte("testcsp384"), elliptic.P384())
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("P384 Key: %+v\n", p384key)
}
<|endoftext|>


<commit_before>package supervisor

import (
	"bufio"
	"github.com/pborman/uuid"
	"io"
	"os/exec"
	"time"
)

type Operation int

const (
	BACKUP Operation = iota
	RESTORE
)

type Status int

const (
	PENDING Status = iota
	RUNNING
	CANCELED
	DONE
)

type PluginConfig struct {
	Plugin   string
	Endpoint string
}

type Task struct {
	uuid uuid.UUID

	Store  *PluginConfig
	Target *PluginConfig

	Op     Operation
	status Status

	startedAt time.Time
	stoppedAt time.Time

	output []string
}

func drain(io io.Reader, name string, ch chan string) {
	s := bufio.NewScanner(io)
	for s.Scan() {
		ch <- s.Text()
	}
}

func (t *Task) Run(stdout chan string, stderr chan string) error {
	var targetCommand string
	var storeCommand string
	if t.Op == BACKUP {
		targetCommand = "backup"
		storeCommand = "store"

	} else {
		targetCommand = "restore"
		storeCommand = "retrieve"
	}

	targetArgs := []string{
		targetCommand,
		"--endpoint",
		t.Target.Endpoint,
	}
	targetCmd := exec.Command(t.Target.Plugin, targetArgs...)
	storeArgs := []string{
		storeCommand,
		"--endpoint",
		t.Store.Endpoint,
	}
	if t.Op != BACKUP {
		storeArgs = append(storeArgs, "--key", "FIXME")
	}
	storeCmd := exec.Command(t.Store.Plugin, storeArgs...)

	var pstdout io.Reader
	//	pipe := bytes.NewBufferString("")
	if t.Op == BACKUP {
		//		targetCmd.Stdout = pipe
		storeCmd.Stdin, _ = targetCmd.StdoutPipe()
		pstdout, _ = storeCmd.StdoutPipe()
	} else {
		targetCmd.Stdin, _ = storeCmd.StdoutPipe()
		pstdout, _ = targetCmd.StdoutPipe()
	}

	pTargetStderr, _ := targetCmd.StderrPipe()
	pStoreStderr, _ := storeCmd.StderrPipe()

	go drain(pTargetStderr, "stderr", stderr)
	go drain(pStoreStderr, "stderr", stderr)
	go drain(pstdout, "stdout", stdout)

	err := targetCmd.Start()
	if err != nil {
		return err
	}
	err = storeCmd.Start()
	if err != nil {
		return err
	}

	err = storeCmd.Wait()
	if err != nil {
		return err
	}
	err = targetCmd.Wait()
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>Cleanup extra cruft<commit_after>package supervisor

import (
	"bufio"
	"github.com/pborman/uuid"
	"io"
	"os/exec"
	"time"
)

type Operation int

const (
	BACKUP Operation = iota
	RESTORE
)

type Status int

const (
	PENDING Status = iota
	RUNNING
	CANCELED
	DONE
)

type PluginConfig struct {
	Plugin   string
	Endpoint string
}

type Task struct {
	uuid uuid.UUID

	Store  *PluginConfig
	Target *PluginConfig

	Op     Operation
	status Status

	startedAt time.Time
	stoppedAt time.Time

	output []string
}

func drain(io io.Reader, name string, ch chan string) {
	s := bufio.NewScanner(io)
	for s.Scan() {
		ch <- s.Text()
	}
}

func (t *Task) Run(stdout chan string, stderr chan string) error {
	var targetCommand string
	var storeCommand string
	if t.Op == BACKUP {
		targetCommand = "backup"
		storeCommand = "store"

	} else {
		targetCommand = "restore"
		storeCommand = "retrieve"
	}

	targetArgs := []string{
		targetCommand,
		"--endpoint",
		t.Target.Endpoint,
	}
	targetCmd := exec.Command(t.Target.Plugin, targetArgs...)
	storeArgs := []string{
		storeCommand,
		"--endpoint",
		t.Store.Endpoint,
	}
	if t.Op != BACKUP {
		storeArgs = append(storeArgs, "--key", "FIXME")
	}
	storeCmd := exec.Command(t.Store.Plugin, storeArgs...)

	var pstdout io.Reader
	if t.Op == BACKUP {
		storeCmd.Stdin, _ = targetCmd.StdoutPipe()
		pstdout, _ = storeCmd.StdoutPipe()
	} else {
		targetCmd.Stdin, _ = storeCmd.StdoutPipe()
		pstdout, _ = targetCmd.StdoutPipe()
	}

	pTargetStderr, _ := targetCmd.StderrPipe()
	pStoreStderr, _ := storeCmd.StderrPipe()

	go drain(pTargetStderr, "stderr", stderr)
	go drain(pStoreStderr, "stderr", stderr)
	go drain(pstdout, "stdout", stdout)

	err := targetCmd.Start()
	if err != nil {
		return err
	}
	err = storeCmd.Start()
	if err != nil {
		return err
	}

	err = storeCmd.Wait()
	if err != nil {
		return err
	}
	err = targetCmd.Wait()
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before><commit_msg>update plugin<commit_after><|endoftext|>