<commit_before>/* Copyright 2014 Ooyala, Inc. All rights reserved.
 *
 * This file is licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
 * except in compliance with the License. You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is
 * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and limitations under the License.
 */

package monitor

import (
	"atlantis/supervisor/containers/serialize"
	"atlantis/supervisor/rpc/types"
	"fmt"
	"github.com/BurntSushi/toml"
	"github.com/jigish/go-flags"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"time"
)

const (
	OK = iota
	Warning
	Critical
	Uknown
)

type Config struct {
	ContainerFile   string `toml:"container_file"`
	ContainersDir   string `toml:"container_dir"`
	InventoryDir    string `toml:"inventory_dir"`
	SSHIdentity     string `toml:"ssh_identity"`
	SSHUser         string `toml:"ssh_user"`
	CheckName       string `toml:"check_name"`
	CheckDir        string `toml:"check_dir"`
	TimeoutDuration uint   `toml:"timeout_duration"`
	Verbose         bool   `toml:"verbose"`
}

type Opts struct {
	ContainerFile   string `short:"f" long:"container-file" description:"file to get contianers information from"`
	ContainersDir   string `short:"s" long:"containers-dir" description:"directory where configs for each container live"`
	SSHIdentity     string `short:"i" long:"ssh-identity" description:"file containing the SSH key for all containers"`
	SSHUser         string `short:"u" long:"ssh-user" description:"user account to ssh into containers"`
	CheckName       string `short:"n" long:"check-name" description:"service name that will appear in Nagios for the monitor"`
	CheckDir        string `short:"d" long:"check-dir" description:"directory containing all the scripts for the monitoring checks"`
	TimeoutDuration uint   `short:"t" long:"timeout-duration" description:"max number of seconds to wait for a monitoring check to finish"`
	Config          string `short:"c" long:"config-file" default:"/etc/atlantis/supervisor/monitor.toml" description:"the config file to use"`
	Verbose         bool   `short:"v" long:"verbose" default:false description:"print verbose debug information"`
}

type ServiceCheck struct {
	Service  string
	User     string
	Identity string
	Host     string
	Port     uint16
	Script   string
}

//TODO(mchandra):Need defaults defined by constants
var config = &Config{
	ContainerFile:   "/etc/atlantis/supervisor/save/containers",
	ContainersDir:   "/etc/atlantis/containers",
	InventoryDir:    "/etc/atlantis/supervisor/inventory",
	SSHIdentity:     "/opt/atlantis/supervisor/master_id_rsa",
	SSHUser:         "root",
	CheckName:       "ContainerMonitor",
	CheckDir:        "/check_mk_checks",
	TimeoutDuration: 110,
	Verbose:         false,
}

func (s *ServiceCheck) cmd() *exec.Cmd {
	return silentSshCmd(s.User, s.Identity, s.Host, s.Script, s.Port)
}

func (s *ServiceCheck) timeOutMsg() string {
	return fmt.Sprintf("%d %s - Timeout occured during check\n", Critical, s.Service)
}

func (s *ServiceCheck) errMsg(err error) string {
	if err != nil {
		return fmt.Sprintf("%d %s - %s\n", Critical, s.Service, err.Error())
	} else {
		return fmt.Sprintf("%d %s - Error encountered while monitoring the service\n", Critical, s.Service)
	}
}

func (s *ServiceCheck) validate(msg string) string {
	m := strings.SplitN(msg, " ", 4)
	if len(m) > 1 && m[1] == s.Service {
		return msg
	}
	return s.errMsg(nil)
}

func (s *ServiceCheck) runCheck(done chan bool) {
	out, err := s.cmd().Output()
	if err != nil {
		fmt.Print(s.errMsg(err))
	} else {
		fmt.Print(s.validate(string(out)))
	}
	done <- true
}

func (s *ServiceCheck) checkWithTimeout(results chan bool, d time.Duration) {
	done := make(chan bool, 1)
	go s.runCheck(done)
	select {
	case <-done:
		results <- true
	case <-time.After(d):
		fmt.Print(s.timeOutMsg())
		results <- true
	}
}

type ContainerCheck struct {
	Name         string
	User         string
	Identity     string
	Directory    string
	Inventory    string
	ContactGroup string
	container    *types.Container
}

type ContainerConfig struct {
	Dependencies map[string]interface{}
}

func (c *ContainerCheck) getContactGroup() {
	c.ContactGroup = "atlantis_orphan_apps"
	config_file := filepath.Join(config.ContainersDir, c.container.ID, "config.json")
	var cont_config ContainerConfig
	if err := serialize.RetrieveObject(config_file, &cont_config); err != nil {
		fmt.Printf("%d %s - Could not retrieve container config %s: %s\n", Critical, config.CheckName, config_file, err)
	} else {
		dep, ok := cont_config.Dependencies["cmk"]
		if !ok {
			fmt.Printf("%d %s - cmk dep not present, defaulting to atlantis_orphan_apps contact group!\n", OK, config.CheckName)
			return
		}
		cmk_dep, ok := dep.(map[string]interface{})
		if !ok {
			fmt.Printf("%d %s - cmk dep present, but value is not map[string]string!\n", Critical, config.CheckName)
			return
		}
		val, ok := cmk_dep["contact_group"]
		if !ok {
			fmt.Printf("%d %s - cmk dep present, but no contact_group key!\n", Critical, config.CheckName)
			return
		}
		group, ok := val.(string)
		if ok {
			c.ContactGroup = group
		} else {
			fmt.Printf("%d %s - Value for contact_group key of cmk dep is not a string!\n", Critical, config.CheckName)
		}
	}
}

func (c *ContainerCheck) setContactGroup(name string) {
	if len(c.ContactGroup) == 0 {
		c.getContactGroup()
	}
	inventoryPath := path.Join(c.Inventory, name)
	if _, err := os.Stat(inventoryPath); os.IsNotExist(err) {
		output, err := exec.Command("/usr/bin/cmk_admin", "-s", name, "-a", c.ContactGroup).CombinedOutput()
		if err != nil {
			fmt.Printf("%d %s - Failure to update contact group for service %s. Error: %s\n", OK, config.CheckName, name, err.Error())
		} else {
			os.Create(inventoryPath)
		}
		if config.Verbose {
			fmt.Printf("\n/usr/bin/cmk_admin -s %s -a %s\n%s\n\n", name, c.ContactGroup, output)
		}
	}
}

func (c *ContainerCheck) Run(t time.Duration, done chan bool) {
	c.setContactGroup(c.Name)
	defer func() { done <- true }()
	o, err := silentSshCmd(c.User, c.Identity, c.container.Host, "ls "+c.Directory, c.container.SSHPort).Output()
	if err != nil {
		fmt.Printf("%d %s - Error getting checks for container: %s\n", Critical, c.Name, err.Error())
		return
	}
	fmt.Printf("%d %s - Got checks for container\n", OK, c.Name)
	scripts := strings.Split(strings.TrimSpace(string(o)), "\n")
	if len(scripts) == 0 || len(scripts[0]) == 0 {
		// nothing to check on this container, exit
		return
	}
	c.checkAll(scripts, t)
}

func (c *ContainerCheck) checkAll(scripts []string, t time.Duration) {
	results := make(chan bool, len(scripts))
	for _, s := range scripts {
		serviceName := fmt.Sprintf("%s_%s", strings.Split(s, ".")[0], c.container.ID)
		c.setContactGroup(serviceName)
		go c.serviceCheck(s).checkWithTimeout(results, t)
	}
	for _ = range scripts {
		<-results
	}
}

func (c *ContainerCheck) serviceCheck(script string) *ServiceCheck {
	// The full path to the script is required
	command := fmt.Sprintf("%s/%s %d %s", c.Directory, script, c.container.PrimaryPort, c.container.ID)
	// The service name is obtained be removing the file extension from the script and appending the container
	// id
	serviceName := fmt.Sprintf("%s_%s", strings.Split(script, ".")[0], c.container.ID)
	return &ServiceCheck{serviceName, c.User, c.Identity, c.container.Host, c.container.SSHPort, command}
}

func silentSshCmd(user, identity, host, cmd string, port uint16) *exec.Cmd {
	args := []string{"-q", user + "@" + host, "-i", identity, "-p", fmt.Sprintf("%d", port), "-o", "StrictHostKeyChecking=no", cmd}
	return exec.Command("ssh", args...)
}

func overlayConfig() {
	opts := &Opts{}
	flags.Parse(opts)
	if opts.Config != "" {
		_, err := toml.DecodeFile(opts.Config, config)
		if err != nil {
			// no need to panic here. we have reasonable defaults.
		}
	}
	if opts.ContainerFile != "" {
		config.ContainerFile = opts.ContainerFile
	}
	if opts.SSHIdentity != "" {
		config.SSHIdentity = opts.SSHIdentity
	}
	if opts.SSHUser != "" {
		config.SSHUser = opts.SSHUser
	}
	if opts.CheckDir != "" {
		config.CheckDir = opts.CheckDir
	}
	if opts.CheckName != "" {
		config.CheckName = opts.CheckName
	}
	if opts.TimeoutDuration != 0 {
		config.TimeoutDuration = opts.TimeoutDuration
	}
	if opts.Verbose {
		config.Verbose = true
	}
}

//file containing containers and service name to show in Nagios for the monitor itself
func Run() {
	overlayConfig()
	var contMap map[string]*types.Container
	//Check if folder exists
	_, err := os.Stat(config.ContainerFile)
	if os.IsNotExist(err) {
		fmt.Printf("%d %s - Container file does not exists %s. Likely no live containers present.\n", OK, config.CheckName, config.ContainerFile)
		return
	}
	if err := serialize.RetrieveObject(config.ContainerFile, &contMap); err != nil {
		fmt.Printf("%d %s - Error retrieving %s: %s\n", Critical, config.CheckName, config.ContainerFile, err)
		return
	}
	done := make(chan bool, len(contMap))
	config.SSHIdentity = strings.Replace(config.SSHIdentity, "~", os.Getenv("HOME"), 1)
	for _, c := range contMap {
		if c.Host == "" {
			c.Host = "localhost"
		}
		check := &ContainerCheck{config.CheckName + "_" + c.ID, config.SSHUser, config.SSHIdentity, config.CheckDir, config.InventoryDir, "", c}
		go check.Run(time.Duration(config.TimeoutDuration)*time.Second, done)
	}
	for _ = range contMap {
		<-done
	}
	// Clean up inventories from containers that no longer exist
	err = filepath.Walk(config.InventoryDir, func(path string, _ os.FileInfo, _ error) error {
		if path == config.InventoryDir {
			return nil
		}
		var err error
		split := strings.Split(path, "_")
		cont := split[len(split)-1]
		if _, ok := contMap[cont]; !ok {
			err = os.Remove(path)
		}
		return err
	})
	if err != nil {
		fmt.Printf("%d %s - Error iterating over inventory to delete obsolete markers. Error: %s\n", OK, config.CheckName, err.Error())
	}
}
<commit_msg>verify contact group<commit_after>/* Copyright 2014 Ooyala, Inc. All rights reserved.
 *
 * This file is licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
 * except in compliance with the License. You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is
 * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and limitations under the License.
 */

package monitor

import (
	"atlantis/supervisor/containers/serialize"
	"atlantis/supervisor/rpc/types"
	"fmt"
	"github.com/BurntSushi/toml"
	"github.com/jigish/go-flags"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"time"
)

const (
	OK = iota
	Warning
	Critical
	Uknown
)

type Config struct {
	ContainerFile   string `toml:"container_file"`
	ContainersDir   string `toml:"container_dir"`
	InventoryDir    string `toml:"inventory_dir"`
	SSHIdentity     string `toml:"ssh_identity"`
	SSHUser         string `toml:"ssh_user"`
	CheckName       string `toml:"check_name"`
	CheckDir        string `toml:"check_dir"`
	DefaultGroup    string `toml:"default_group"`
	TimeoutDuration uint   `toml:"timeout_duration"`
	Verbose         bool   `toml:"verbose"`
}

type Opts struct {
	ContainerFile   string `short:"f" long:"container-file" description:"file to get contianers information from"`
	ContainersDir   string `short:"s" long:"containers-dir" description:"directory where configs for each container live"`
	SSHIdentity     string `short:"i" long:"ssh-identity" description:"file containing the SSH key for all containers"`
	SSHUser         string `short:"u" long:"ssh-user" description:"user account to ssh into containers"`
	CheckName       string `short:"n" long:"check-name" description:"service name that will appear in Nagios for the monitor"`
	CheckDir        string `short:"d" long:"check-dir" description:"directory containing all the scripts for the monitoring checks"`
	DefaultGroup    string `short:"g" long:"default-group" description:"default contact group to use if there is no valid group provided"`
	Config          string `short:"c" long:"config-file" default:"/etc/atlantis/supervisor/monitor.toml" description:"the config file to use"`
	TimeoutDuration uint   `short:"t" long:"timeout-duration" description:"max number of seconds to wait for a monitoring check to finish"`
	Verbose         bool   `short:"v" long:"verbose" default:false description:"print verbose debug information"`
}

type ServiceCheck struct {
	Service  string
	User     string
	Identity string
	Host     string
	Port     uint16
	Script   string
}

//TODO(mchandra):Need defaults defined by constants
var config = &Config{
	ContainerFile:   "/etc/atlantis/supervisor/save/containers",
	ContainersDir:   "/etc/atlantis/containers",
	InventoryDir:    "/etc/atlantis/supervisor/inventory",
	SSHIdentity:     "/opt/atlantis/supervisor/master_id_rsa",
	SSHUser:         "root",
	CheckName:       "ContainerMonitor",
	CheckDir:        "/check_mk_checks",
	DefaultGroup:    "atlantis_orphan_apps",
	TimeoutDuration: 110,
	Verbose:         false,
}

func (s *ServiceCheck) cmd() *exec.Cmd {
	return silentSshCmd(s.User, s.Identity, s.Host, s.Script, s.Port)
}

func (s *ServiceCheck) timeOutMsg() string {
	return fmt.Sprintf("%d %s - Timeout occured during check\n", Critical, s.Service)
}

func (s *ServiceCheck) errMsg(err error) string {
	if err != nil {
		return fmt.Sprintf("%d %s - %s\n", Critical, s.Service, err.Error())
	} else {
		return fmt.Sprintf("%d %s - Error encountered while monitoring the service\n", Critical, s.Service)
	}
}

func (s *ServiceCheck) validate(msg string) string {
	m := strings.SplitN(msg, " ", 4)
	if len(m) > 1 && m[1] == s.Service {
		return msg
	}
	return s.errMsg(nil)
}

func (s *ServiceCheck) runCheck(done chan bool) {
	out, err := s.cmd().Output()
	if err != nil {
		fmt.Print(s.errMsg(err))
	} else {
		fmt.Print(s.validate(string(out)))
	}
	done <- true
}

func (s *ServiceCheck) checkWithTimeout(results chan bool, d time.Duration) {
	done := make(chan bool, 1)
	go s.runCheck(done)
	select {
	case <-done:
		results <- true
	case <-time.After(d):
		fmt.Print(s.timeOutMsg())
		results <- true
	}
}

type ContainerCheck struct {
	Name         string
	User         string
	Identity     string
	Directory    string
	Inventory    string
	ContactGroup string
	container    *types.Container
}

type ContainerConfig struct {
	Dependencies map[string]interface{}
}

func (c *ContainerCheck) getContactGroup() {
	c.ContactGroup = config.DefaultGroup
	config_file := filepath.Join(config.ContainersDir, c.container.ID, "config.json")
	var cont_config ContainerConfig
	if err := serialize.RetrieveObject(config_file, &cont_config); err != nil {
		fmt.Printf("%d %s - Could not retrieve container config %s: %s\n", Critical, config.CheckName, config_file, err)
	} else {
		dep, ok := cont_config.Dependencies["cmk"]
		if !ok {
			fmt.Printf("%d %s - cmk dep not present, defaulting to %s contact group!\n", OK, config.CheckName, config.DefaultGroup)
			return
		}
		cmk_dep, ok := dep.(map[string]interface{})
		if !ok {
			fmt.Printf("%d %s - cmk dep present, but value is not map[string]string!\n", Critical, config.CheckName)
			return
		}
		val, ok := cmk_dep["contact_group"]
		if !ok {
			fmt.Printf("%d %s - cmk dep present, but no contact_group key!\n", Critical, config.CheckName)
			return
		}
		group, ok := val.(string)
		if ok {
			c.ContactGroup = group
			c.verifyContactGroup()
		} else {
			fmt.Printf("%d %s - Value for contact_group key of cmk dep is not a string!\n", Critical, config.CheckName)
		}
	}
}

func (c *ContainerCheck) setContactGroup(name string) {
	if len(c.ContactGroup) == 0 {
		c.getContactGroup()
	}
	inventoryPath := path.Join(c.Inventory, name)
	if _, err := os.Stat(inventoryPath); os.IsNotExist(err) {
		output, err := exec.Command("/usr/bin/cmk_admin", "-s", name, "-a", c.ContactGroup).CombinedOutput()
		if err != nil {
			fmt.Printf("%d %s - Failure to update contact group for service %s. Error: %s\n", OK, config.CheckName, name, err.Error())
		} else {
			os.Create(inventoryPath)
		}
		if config.Verbose {
			fmt.Printf("\n/usr/bin/cmk_admin -s %s -a %s\n%s\n\n", name, c.ContactGroup, output)
		}
	}
}

func (c *ContainerCheck) verifyContactGroup() {
	var output bytes.Buffer
	var stderr bytes.Buffer
	cmk := exec.Command("/usr/bin/cmk_admin", "-l")
	egrep := exec.Command("egrep", "-x", fmt.Sprintf("[*]?\\s+(%s)", c.ContactGroup))
	var err error
	if egrep.Stdin, err = cmk.StdoutPipe(); err != nil {
		fmt.Printf("%d %s - Error piping output while verifying %s. Error: %s\n", Critical, config.CheckName, c.ContactGroup, err.Error())
		c.ContactGroup = config.DefaultGroup
		return
	}
	cmk.Stderr = &stderr
	egrep.Stdout, egrep.Stderr = &output, &stderr
	cmds := [2]*exec.Cmd{cmk, egrep}
	for _, cmd := range cmds {
		if err := cmd.Start(); err != nil {
			fmt.Printf("%d %s - Error starting %s while verifying %s. Error: %s\n", Critical, config.CheckName, cmd.Path, c.ContactGroup, err.Error())
			c.ContactGroup = config.DefaultGroup
			return
		}
	}
	for _, cmd := range cmds {
		if err := cmd.Wait(); err != nil {
			fmt.Printf("%d %s - Error running %s while verfying %s. Error: %s\n", Critical, config.CheckName, cmd.Path, c.ContactGroup, err.Error())
			c.ContactGroup = config.DefaultGroup
			return
		}
	}
}

func (c *ContainerCheck) Run(t time.Duration, done chan bool) {
	c.setContactGroup(c.Name)
	defer func() { done <- true }()
	o, err := silentSshCmd(c.User, c.Identity, c.container.Host, "ls "+c.Directory, c.container.SSHPort).Output()
	if err != nil {
		fmt.Printf("%d %s - Error getting checks for container: %s\n", Critical, c.Name, err.Error())
		return
	}
	fmt.Printf("%d %s - Got checks for container\n", OK, c.Name)
	scripts := strings.Split(strings.TrimSpace(string(o)), "\n")
	if len(scripts) == 0 || len(scripts[0]) == 0 {
		// nothing to check on this container, exit
		return
	}
	c.checkAll(scripts, t)
}

func (c *ContainerCheck) checkAll(scripts []string, t time.Duration) {
	results := make(chan bool, len(scripts))
	for _, s := range scripts {
		serviceName := fmt.Sprintf("%s_%s", strings.Split(s, ".")[0], c.container.ID)
		c.setContactGroup(serviceName)
		go c.serviceCheck(s).checkWithTimeout(results, t)
	}
	for _ = range scripts {
		<-results
	}
}

func (c *ContainerCheck) serviceCheck(script string) *ServiceCheck {
	// The full path to the script is required
	command := fmt.Sprintf("%s/%s %d %s", c.Directory, script, c.container.PrimaryPort, c.container.ID)
	// The service name is obtained be removing the file extension from the script and appending the container
	// id
	serviceName := fmt.Sprintf("%s_%s", strings.Split(script, ".")[0], c.container.ID)
	return &ServiceCheck{serviceName, c.User, c.Identity, c.container.Host, c.container.SSHPort, command}
}

func silentSshCmd(user, identity, host, cmd string, port uint16) *exec.Cmd {
	args := []string{"-q", user + "@" + host, "-i", identity, "-p", fmt.Sprintf("%d", port), "-o", "StrictHostKeyChecking=no", cmd}
	return exec.Command("ssh", args...)
}

func overlayConfig() {
	opts := &Opts{}
	flags.Parse(opts)
	if opts.Config != "" {
		_, err := toml.DecodeFile(opts.Config, config)
		if err != nil {
			// no need to panic here. we have reasonable defaults.
		}
	}
	if opts.ContainerFile != "" {
		config.ContainerFile = opts.ContainerFile
	}
	if opts.ContainersDir != "" {
		config.ContainersDir = opts.ContainersDir
	}
	if opts.SSHIdentity != "" {
		config.SSHIdentity = opts.SSHIdentity
	}
	if opts.SSHUser != "" {
		config.SSHUser = opts.SSHUser
	}
	if opts.CheckDir != "" {
		config.CheckDir = opts.CheckDir
	}
	if opts.CheckName != "" {
		config.CheckName = opts.CheckName
	}
	if opts.DefaultGroup != "" {
		config.DefaultGroup = opts.DefaultGroup
	}
	if opts.TimeoutDuration != 0 {
		config.TimeoutDuration = opts.TimeoutDuration
	}
	if opts.Verbose {
		config.Verbose = true
	}
}

//file containing containers and service name to show in Nagios for the monitor itself
func Run() {
	overlayConfig()
	var contMap map[string]*types.Container
	//Check if folder exists
	_, err := os.Stat(config.ContainerFile)
	if os.IsNotExist(err) {
		fmt.Printf("%d %s - Container file does not exists %s. Likely no live containers present.\n", OK, config.CheckName, config.ContainerFile)
		return
	}
	if err := serialize.RetrieveObject(config.ContainerFile, &contMap); err != nil {
		fmt.Printf("%d %s - Error retrieving %s: %s\n", Critical, config.CheckName, config.ContainerFile, err)
		return
	}
	done := make(chan bool, len(contMap))
	config.SSHIdentity = strings.Replace(config.SSHIdentity, "~", os.Getenv("HOME"), 1)
	for _, c := range contMap {
		if c.Host == "" {
			c.Host = "localhost"
		}
		check := &ContainerCheck{config.CheckName + "_" + c.ID, config.SSHUser, config.SSHIdentity, config.CheckDir, config.InventoryDir, "", c}
		go check.Run(time.Duration(config.TimeoutDuration)*time.Second, done)
	}
	for _ = range contMap {
		<-done
	}
	// Clean up inventories from containers that no longer exist
	err = filepath.Walk(config.InventoryDir, func(path string, _ os.FileInfo, _ error) error {
		if path == config.InventoryDir {
			return nil
		}
		var err error
		split := strings.Split(path, "_")
		cont := split[len(split)-1]
		if _, ok := contMap[cont]; !ok {
			err = os.Remove(path)
		}
		return err
	})
	if err != nil {
		fmt.Printf("%d %s - Error iterating over inventory to delete obsolete markers. Error: %s\n", OK, config.CheckName, err.Error())
	}
}
<|endoftext|>


<commit_before>/*
	Defines SnipsCollection struct-type and NewSnipsCollection()
	It is used by the example REST server for storing and managing snips
	It has the methods Add, WithId, All and Remove.
*/
package main

import (
	"log"
	"container/vector"
)

// Snip definintion
type Snip struct {
	Id   int
	Body string
}

// Create a new snippet
func newSnip(id int, body string) *Snip {
	log.Println("Creating new Snip:", id, body)
	return &Snip{id, body}
}

// SnipsCollection definition
type SnipsCollection struct {
	v      *vector.Vector
	nextId int
}

// SnipsCollection creation function
func NewSnipsCollection() *SnipsCollection {
	log.Println("Creating new SnipsCollection")
	return &SnipsCollection{v: new(vector.Vector), nextId: 0}
}

// Add a new snippet (snipped with passed text as body)
func (snips *SnipsCollection) Add(body string) int {
	log.Println("Adding Snip:", body)
	id := snips.nextId
	snips.nextId++

	snip := newSnip(id, body)
	snips.v.Push(snip)

	return id
}

// Get a snippet by ID
func (snips *SnipsCollection) WithId(id int) (*Snip, bool) {
	log.Println("Finding Snip with id: ", id)
	all := *snips.v
	for _, v := range all {
		snip, ok := v.(*Snip)
		if ok && snip.Id == id {
			return snip, true
		}
	}
	return nil, false
}

// Get all snippets
func (snips *SnipsCollection) All() []*Snip {
	log.Println("Finding all Snips")
	data := *snips.v
	all := make([]*Snip, len(data))

	for k, v := range data {
		if snip, ok := v.(*Snip); ok {
			all[k] = snip
		}
	}

	return all
}

// Remove a snippet by ID
func (snips *SnipsCollection) Remove(id int) {
	length := snips.v.Len()
	for i := 0; i < length; i++ {
		snip := snips.v.At(i).(*Snip)
		if snip.Id == id {
			snips.v.Delete(i)
			return
		}
	}
}
<commit_msg>use array instead of old, nolonger-existant vector (now Go version 1)<commit_after>/*
	Defines SnipsCollection struct-type and NewSnipsCollection()
	It is used by the example REST server for storing and managing snips
	It has the methods Add, WithId, All and Remove.
*/
package main

import (
	"log"
)

// Snip definintion
type Snip struct {
	Id   int
	Body string
}

// Create a new snippet
func newSnip(id int, body string) *Snip {
	log.Println("Creating new Snip:", id, body)
	return &Snip{id, body}
}

// SnipsCollection definition
type SnipsCollection struct {
	v      []Snip
	nextId int
}

// SnipsCollection creation function
func NewSnipsCollection() *SnipsCollection {
	log.Println("Creating new SnipsCollection")
	return &SnipsCollection{v: make([]Snip, 0), nextId: 0}
}

// Add a new snippet (snipped with passed text as body)
func (snips *SnipsCollection) Add(body string) int {
	log.Println("Adding Snip:", body)
	id := snips.nextId
	snips.nextId++

	snip := newSnip(id, body)
	snips.v = append(snips.v, *snip)

	return id
}

// Get a snippet by ID
func (snips *SnipsCollection) WithId(id int) (*Snip, bool) {
	log.Println("Searching for Snip with id: ", id)
	for _, snip := range snips.v {
		if snip.Id == id {
			return &snip, true
		}
	}
	return nil, false
}

// Get all snippets
func (snips *SnipsCollection) All() []Snip {
	log.Println("Finding all Snips")
	data := snips.v
	all := make([]Snip, len(snips.v))

	for k, snip := range data {
		all[k] = snip
	}

	return all
}

// Remove a snippet by ID
func (snips *SnipsCollection) Remove(id int) {

	newSnips := make([]Snip, len(snips.v))
	for _, snip := range snips.v {
		if snip.Id != id {
			newSnips = append(newSnips, snip)
		}
	}
	snips.v = newSnips
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"expvar"
	"io"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"time"

	"github.com/kr/env"
	"github.com/kr/secureheader"
	"golang.org/x/net/context"

	"chain/api"
	"chain/api/appdb"
	"chain/api/asset"
	"chain/database/pg"
	chainlog "chain/log"
	"chain/log/rotation"
	"chain/log/splunk"
	"chain/metrics"
	"chain/metrics/librato"
	chainhttp "chain/net/http"
	"chain/net/http/gzip"
)

var (
	// config vars
	tlsCrt       = env.String("TLSCRT", "")
	tlsKey       = env.String("TLSKEY", "")
	listenAddr   = env.String("LISTEN", ":8080")
	dbURL        = env.String("DB_URL", "postgres:///api?sslmode=disable")
	target       = env.String("STACK", "sandbox") // TODO(kr): rename STACK to TARGET
	samplePer    = env.Duration("SAMPLEPER", 10*time.Second)
	nouserSecret = env.String("NOUSER_SECRET", "")
	splunkAddr   = os.Getenv("SPLUNKADDR")
	logFile      = os.Getenv("LOGFILE")
	logSize      = env.Int("LOGSIZE", 5e6) // 5MB
	logCount     = env.Int("LOGCOUNT", 9)
	// for config var LIBRATO_URL, see func init below

	// build vars; initialized by the linker
	buildTag    = "dev"
	buildCommit = "?"
	buildDate   = "?"

	race []interface{} // initialized in race.go
)

func init() {
	librato.URL = env.URL("LIBRATO_URL", "")
	librato.Prefix = "chain.api."
	expvar.NewString("buildtag").Set(buildTag)
	expvar.NewString("builddate").Set(buildDate)
	expvar.NewString("buildcommit").Set(buildCommit)
}

func main() {
	env.Parse()
	sql.Register("schemadb", pg.SchemaDriver(buildTag))
	log.SetPrefix("api-" + buildTag + ": ")
	log.SetFlags(log.Lshortfile)
	chainlog.SetPrefix(append([]interface{}{"app", "api", "target", *target, "buildtag", buildTag}, race...)...)
	chainlog.SetOutput(logWriter())

	if librato.URL.Host != "" {
		librato.Source = *target
		go librato.SampleMetrics(*samplePer)
	} else {
		log.Println("no metrics; set LIBRATO_URL for prod")
	}

	db, err := sql.Open("schemadb", *dbURL)
	if err != nil {
		log.Fatal(err)
	}
	db.SetMaxOpenConns(100)
	db.SetMaxIdleConns(100)
	appdb.Init(db)

	var h chainhttp.Handler
	h = api.Handler(*nouserSecret)
	h = metrics.Handler{Handler: h}
	h = gzip.Handler{Handler: h}

	bg := context.Background()
	bg = pg.NewContext(bg, db)
	go asset.MakeBlocks(bg, 1*time.Second)
	http.Handle("/", chainhttp.ContextHandler{Context: bg, Handler: h})
	http.HandleFunc("/health", func(http.ResponseWriter, *http.Request) {})

	secureheader.DefaultConfig.PermitClearLoopback = true

	if *tlsCrt != "" {
		err = chainhttp.ListenAndServeTLS(*listenAddr, *tlsCrt, *tlsKey, secureheader.DefaultConfig)
	} else {
		err = http.ListenAndServe(*listenAddr, secureheader.DefaultConfig)
	}
	if err != nil {
		log.Fatalln("ListenAndServe:", err)
	}
}

func logWriter() io.Writer {
	dropmsg := []byte("\nlog data dropped\n")
	rotation := &errlog{w: rotation.Create(logFile, *logSize, *logCount)}
	splunk := &errlog{w: splunk.New(splunkAddr, dropmsg)}

	switch {
	case logFile != "" && splunkAddr != "":
		return io.MultiWriter(rotation, splunk)
	case logFile != "" && splunkAddr == "":
		return rotation
	case logFile == "" && splunkAddr != "":
		return splunk
	}
	return os.Stdout
}

type errlog struct {
	w io.Writer
	t time.Time // protected by chain/log mutex
}

func (w *errlog) Write(p []byte) (int, error) {
	// We don't want to ruin our performance
	// when there's a persistent error
	// writing to a log sink.
	// Print to stderr at most once per minute.
	_, err := w.w.Write(p)
	if err != nil && time.Since(w.t) > time.Minute {
		log.Println("chain/log:", err)
		w.t = time.Now()
	}
	return len(p), nil // report success for the MultiWriter
}
<commit_msg>cmd/api: use an env var for max db conns<commit_after>package main

import (
	"database/sql"
	"expvar"
	"io"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"time"

	"github.com/kr/env"
	"github.com/kr/secureheader"
	"golang.org/x/net/context"

	"chain/api"
	"chain/api/appdb"
	"chain/api/asset"
	"chain/database/pg"
	chainlog "chain/log"
	"chain/log/rotation"
	"chain/log/splunk"
	"chain/metrics"
	"chain/metrics/librato"
	chainhttp "chain/net/http"
	"chain/net/http/gzip"
)

var (
	// config vars
	tlsCrt       = env.String("TLSCRT", "")
	tlsKey       = env.String("TLSKEY", "")
	listenAddr   = env.String("LISTEN", ":8080")
	dbURL        = env.String("DB_URL", "postgres:///api?sslmode=disable")
	target       = env.String("STACK", "sandbox") // TODO(kr): rename STACK to TARGET
	samplePer    = env.Duration("SAMPLEPER", 10*time.Second)
	nouserSecret = env.String("NOUSER_SECRET", "")
	splunkAddr   = os.Getenv("SPLUNKADDR")
	logFile      = os.Getenv("LOGFILE")
	logSize      = env.Int("LOGSIZE", 5e6) // 5MB
	logCount     = env.Int("LOGCOUNT", 9)
	// for config var LIBRATO_URL, see func init below
	maxDBConns = env.Int("MAXDBCONNS", 10) // set to 100 in prod

	// build vars; initialized by the linker
	buildTag    = "dev"
	buildCommit = "?"
	buildDate   = "?"

	race []interface{} // initialized in race.go
)

func init() {
	librato.URL = env.URL("LIBRATO_URL", "")
	librato.Prefix = "chain.api."
	expvar.NewString("buildtag").Set(buildTag)
	expvar.NewString("builddate").Set(buildDate)
	expvar.NewString("buildcommit").Set(buildCommit)
}

func main() {
	env.Parse()
	sql.Register("schemadb", pg.SchemaDriver(buildTag))
	log.SetPrefix("api-" + buildTag + ": ")
	log.SetFlags(log.Lshortfile)
	chainlog.SetPrefix(append([]interface{}{"app", "api", "target", *target, "buildtag", buildTag}, race...)...)
	chainlog.SetOutput(logWriter())

	if librato.URL.Host != "" {
		librato.Source = *target
		go librato.SampleMetrics(*samplePer)
	} else {
		log.Println("no metrics; set LIBRATO_URL for prod")
	}

	db, err := sql.Open("schemadb", *dbURL)
	if err != nil {
		log.Fatal(err)
	}
	db.SetMaxOpenConns(*maxDBConns)
	db.SetMaxIdleConns(100)
	appdb.Init(db)

	var h chainhttp.Handler
	h = api.Handler(*nouserSecret)
	h = metrics.Handler{Handler: h}
	h = gzip.Handler{Handler: h}

	bg := context.Background()
	bg = pg.NewContext(bg, db)
	go asset.MakeBlocks(bg, 1*time.Second)
	http.Handle("/", chainhttp.ContextHandler{Context: bg, Handler: h})
	http.HandleFunc("/health", func(http.ResponseWriter, *http.Request) {})

	secureheader.DefaultConfig.PermitClearLoopback = true

	if *tlsCrt != "" {
		err = chainhttp.ListenAndServeTLS(*listenAddr, *tlsCrt, *tlsKey, secureheader.DefaultConfig)
	} else {
		err = http.ListenAndServe(*listenAddr, secureheader.DefaultConfig)
	}
	if err != nil {
		log.Fatalln("ListenAndServe:", err)
	}
}

func logWriter() io.Writer {
	dropmsg := []byte("\nlog data dropped\n")
	rotation := &errlog{w: rotation.Create(logFile, *logSize, *logCount)}
	splunk := &errlog{w: splunk.New(splunkAddr, dropmsg)}

	switch {
	case logFile != "" && splunkAddr != "":
		return io.MultiWriter(rotation, splunk)
	case logFile != "" && splunkAddr == "":
		return rotation
	case logFile == "" && splunkAddr != "":
		return splunk
	}
	return os.Stdout
}

type errlog struct {
	w io.Writer
	t time.Time // protected by chain/log mutex
}

func (w *errlog) Write(p []byte) (int, error) {
	// We don't want to ruin our performance
	// when there's a persistent error
	// writing to a log sink.
	// Print to stderr at most once per minute.
	_, err := w.w.Write(p)
	if err != nil && time.Since(w.t) > time.Minute {
		log.Println("chain/log:", err)
		w.t = time.Now()
	}
	return len(p), nil // report success for the MultiWriter
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"

	"github.com/danesparza/centralconfig/datastores"
	"github.com/spf13/cobra"
)

var (
	jsonConfig bool
	yamlConfig bool
	mysqlDDL   bool
	mssqlDDL   bool
)

var yamlDefault = []byte(`
http:
  port: 3000
datastore:
  boltdb:
    database: config.db
`)

var jsonDefault = []byte(`{
  "http" : {
    "port": "3000"
  },
  "datastore" : {
    "boltdb": {
      "database": "config.db"
    }
  }
}`)

// defaultsCmd represents the defaults command
var defaultsCmd = &cobra.Command{
	Use:   "defaults",
	Short: "Prints default server configuration files or DDL",
	Long: `Use this to create a default configuration file for the centralconfig server.

Example:

centralconfig defaults > centralconfig.yaml

You can also use this to print SQL database scripts to create your datastore.  Example:

centralconfig defaults --mysql > centralconfigdb.sql

`,
	Run: func(cmd *cobra.Command, args []string) {

		if jsonConfig {
			fmt.Printf("%s", jsonDefault)
		} else if mysqlDDL {
			fmt.Printf("%s", datastores.GetMysqlCreateDDL())
		} else if mssqlDDL {
			fmt.Printf("%s", datastores.GetMSsqlCreateDDL())
		}
		else if yamlConfig {
			fmt.Printf("%s", yamlDefault)
		}
	},
}

func init() {
	RootCmd.AddCommand(defaultsCmd)

	defaultsCmd.Flags().BoolVarP(&jsonConfig, "json", "j", false, "Create a JSON configuration file")
	defaultsCmd.Flags().BoolVarP(&yamlConfig, "yaml", "y", true, "Create a YAML configuration file")
	defaultsCmd.Flags().BoolVarP(&mysqlDDL, "mysql", "m", false, "Create a MySQL database script")
	defaultsCmd.Flags().BoolVarP(&mssqlDDL, "mssql", "s", false, "Create a MSSQL database script")

}
<commit_msg>Fixing compile error.  That will teach me to commit before compiling<commit_after>package cmd

import (
	"fmt"

	"github.com/danesparza/centralconfig/datastores"
	"github.com/spf13/cobra"
)

var (
	jsonConfig bool
	yamlConfig bool
	mysqlDDL   bool
	mssqlDDL   bool
)

var yamlDefault = []byte(`
http:
  port: 3000
datastore:
  boltdb:
    database: config.db
`)

var jsonDefault = []byte(`{
  "http" : {
    "port": "3000"
  },
  "datastore" : {
    "boltdb": {
      "database": "config.db"
    }
  }
}`)

// defaultsCmd represents the defaults command
var defaultsCmd = &cobra.Command{
	Use:   "defaults",
	Short: "Prints default server configuration files or DDL",
	Long: `Use this to create a default configuration file for the centralconfig server.

Example:

centralconfig defaults > centralconfig.yaml

You can also use this to print SQL database scripts to create your datastore.  Example:

centralconfig defaults --mysql > centralconfigdb.sql

`,
	Run: func(cmd *cobra.Command, args []string) {

		if jsonConfig {
			fmt.Printf("%s", jsonDefault)
		} else if mysqlDDL {
			fmt.Printf("%s", datastores.GetMysqlCreateDDL())
		} else if mssqlDDL {
			fmt.Printf("%s", datastores.GetMSsqlCreateDDL())
		} else if yamlConfig {
			fmt.Printf("%s", yamlDefault)
		}
	},
}

func init() {
	RootCmd.AddCommand(defaultsCmd)

	defaultsCmd.Flags().BoolVarP(&jsonConfig, "json", "j", false, "Create a JSON configuration file")
	defaultsCmd.Flags().BoolVarP(&yamlConfig, "yaml", "y", true, "Create a YAML configuration file")
	defaultsCmd.Flags().BoolVarP(&mysqlDDL, "mysql", "m", false, "Create a MySQL database script")
	defaultsCmd.Flags().BoolVarP(&mssqlDDL, "mssql", "s", false, "Create a MSSQL database script")

}
<|endoftext|>


<commit_before>// Copyright 2010  The "goheader" Authors
//
// Use of this source code is governed by the Simplified BSD License
// that can be found in the LICENSE file.
//
// This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
// OR CONDITIONS OF ANY KIND, either express or implied. See the License
// for more details.

package main

import (
	"bufio"
	//"bytes"
	"container/vector"
	"flag"
	"fmt"
	"os"
	"path"
	"regexp"
	"strings"
)

const COMMENT_C_LINE = "//!!! "

var goHeader = `// {cmd}
// MACHINE GENERATED.

package {pkg}

`

// Flags
var (
	system      = flag.String("s", "", "The operating system")
	pkgName     = flag.String("p", "", "The name of the package")
	listSystems = flag.Bool("l", false, "List of valid systems")
	write       = flag.Bool("w", false,
		"If set, write each input file with its output.")
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: goheader -s system -g package [defs.h...]\n")
	flag.PrintDefaults()
	os.Exit(1)
}

// Translates C type declaration into Go type declaration.
func translateC(cHeader string) os.Error {
	// === Regular expressions
	reSkip := regexp.MustCompile(`^(\n|//)`) // Empty lines and comments.

	reType := regexp.MustCompile(`^(typedef)[ \t]+(.+)[ \t]+(.+)[;](.+)?`)

	reStruct := regexp.MustCompile(`^(struct)[ \t]+(.+)[ \t]*{`)
	reStructField := regexp.MustCompile(`^[ \t]*(.+)[ \t]+(.+)[;](.+)?`)
	reStructFieldName := regexp.MustCompile(`^([^_]*_)?(.+)`)

	reDefine := regexp.MustCompile(`^[ \t]*#[ \t]*(define|DEFINE)[ \t]+([^ \t]+)[ \t]+(.+)`)
	reDefineMacro := regexp.MustCompile(`^.*(\(.*\))`)

	reSingleComment := regexp.MustCompile(`^(.+)?/\*[ \t]*(.+)[ \t]*\*/`)
	reStartMultipleComment := regexp.MustCompile(`^/\*(.+)?`)
	reMiddleMultipleComment := regexp.MustCompile(`^[ \t*]*(.+)`)
	reEndMultipleComment := regexp.MustCompile(`^(.+)?\*/`)
	// ===

	// Check the extension.
	if path.Ext(cHeader) != ".h" {
		return os.NewError("error extension")
	}

	// === File to read
	var isMultipleComment, isDefine, isStruct bool
	var extraType vector.StringVector // Types defined in C header.

	inFile, err := os.Open(cHeader, os.O_RDONLY, 0)
	if err != nil {
		return err
	}
	defer inFile.Close()

	inBuf := bufio.NewReader(inFile)

	// === File to write in the actual directory.
	var outBuf *bufio.Writer

	if *write {
		goHeader := strings.Split(path.Base(cHeader), ".h", 2)[0]
		goHeader = fmt.Sprintf("%s_%s.go", goHeader, *system)

		outFile, err := os.Open(goHeader, os.O_CREATE|os.O_WRONLY, 0644)
		if err != nil {
			return err
		}
		defer outFile.Close()

		outBuf = bufio.NewWriter(outFile)
	} else {
		outBuf = bufio.NewWriter(os.Stdout)
	}

	if _, err := outBuf.WriteString(goHeader); err != nil {
		return err
	}

	for {
		line, err := inBuf.ReadString('\n')
		if err == os.EOF {
			break
		}
		line = strings.TrimSpace(line) + "\n"
		isSingleComment := false

		// === Translate comment of single line.
		if !isMultipleComment {
			if sub := reSingleComment.FindStringSubmatch(line); sub != nil {
				isSingleComment = true
				line = "// " + sub[2] + "\n"

				if sub[1] != "" {
					line = sub[1] + line
				}
			}
		}
		if !isSingleComment && !isMultipleComment && strings.HasPrefix(line, "/*") {
			isMultipleComment = true
		}

		// === Translate comments of multiple line.
		if isMultipleComment {
			if sub := reStartMultipleComment.FindStringSubmatch(line); sub != nil {
				if sub[1] != "\n" {
					line = "// " + sub[1]
					if _, err := outBuf.WriteString(line); err != nil {
						return err
					}
				}
				continue
			}
			if sub := reEndMultipleComment.FindStringSubmatch(line); sub != nil {
				if sub[1] != "" {
					line = "// " + sub[1] + "\n"
					if _, err := outBuf.WriteString(line); err != nil {
						return err
					}
				}
				isMultipleComment = false
				continue
			}
			if sub := reMiddleMultipleComment.FindStringSubmatch(line); sub != nil {
				line = "// " + sub[1]
				if _, err := outBuf.WriteString(line); err != nil {
					return err
				}
				continue
			}
		}

		// === Translate type definitions.
		if sub := reType.FindStringSubmatch(line); sub != nil {
			// Add the new type.
			extraType.Push(sub[3])

			gotype, ok := ctypeTogo(sub[2], &extraType)
			line = fmt.Sprintf("type %s %s", sub[3], gotype)

			if sub[4] != "\n" {
				line += sub[4]
			} else {
				line += "\n"
			}
			if !ok {
				line = COMMENT_C_LINE + line
			}

			if _, err := outBuf.WriteString(line); err != nil {
				return err
			}
			continue
		}

		// === Translate defines.
		if sub := reDefine.FindStringSubmatch(line); sub != nil {
			if !isDefine {
				isDefine = true
				if _, err := outBuf.WriteString("const (\n"); err != nil {
					return err
				}
			}
			line = fmt.Sprintf("%s = %s", sub[2], sub[3])

			// Removes comment (if any) to ckeck if it is a macro.
			lastField := strings.Split(sub[3], "//", -1)[0]
			if reDefineMacro.MatchString(lastField) {
				line = COMMENT_C_LINE + line
			}

			if _, err := outBuf.WriteString(line); err != nil {
				return err
			}
			continue
		}
		if isDefine && line == "\n" {
			if _, err := outBuf.WriteString(")\n\n"); err != nil {
				return err
			}
			isDefine = false
			continue
		}

		// === Translate structs.
		if !isStruct {
			if sub := reStruct.FindStringSubmatch(line); sub != nil {
				isStruct = true

				if isDefine {
					if _, err := outBuf.WriteString(")\n"); err != nil {
						return err
					}
					isDefine = false
				}

				if _, err := outBuf.WriteString(fmt.Sprintf(
					"type %s struct {\n", strings.Title(sub[2]))); err != nil {
					return err
				}
				continue
			}
		} else {
			if sub := reStructField.FindStringSubmatch(line); sub != nil {
				// Translate the field type.
				gotype, ok := ctypeTogo(sub[1], &extraType)

				// === Translate the field name.
				fieldName := reStructFieldName.FindStringSubmatch(sub[2])
				_fieldName := ""

				if fieldName[1] != "" {
					_fieldName = fieldName[2]
				} else {
					_fieldName = fieldName[0]
				}
				// ===

				line = fmt.Sprintf("%s %s %s",
					strings.Title(_fieldName), gotype, sub[3])

				// C type not found.
				if !ok {
					line = COMMENT_C_LINE + line
				}

				if _, err := outBuf.WriteString(line); err != nil {
					return err
				}
				continue
			}
			if strings.HasPrefix(line, "}") {
				if _, err := outBuf.WriteString(strings.Replace(line, ";", "", 1)); err != nil {
					return err
				}
				isStruct = false
				continue
			}
		}

		// Comment another C lines.
		//if line != "\n" && !strings.HasPrefix(line, "//") {
		if !reSkip.MatchString(line) {
			line = COMMENT_C_LINE + line
		}

		if _, err := outBuf.WriteString(line); err != nil {
			return err
		}
	}

	if err := outBuf.Flush(); err != nil {
		return err
	}
	return nil
}

// Translates a C type definition into Go definition. The C header could have
// defined new types so they're checked in the firs place.
func ctypeTogo(ctype string, extraCtype *vector.StringVector) (gotype string, ok bool) {
	for _, v := range *extraCtype {
		if v == ctype {
			return ctype, true
		}
	}

	switch ctype {
	case "char", "signed char", "signed short int", "short int", "short":
		return "int8", true
	case "unsigned char", "unsigned short int", "unsigned short":
		return "uint8", true
	case "int", "signed int":
		return "int16", true
	case "unsigned int", "unsigned":
		return "uint16", true
	case "signed long int", "long int", "long":
		return "int32", true
	case "unsigned long int", "unsigned long":
		return "uint32", true
	case "float":
		return "float32", true
	case "double", "long double":
		return "float64", true
	}
	return ctype, false
}

func main() {
	validSystems := []string{"darwin", "freebsd", "linux", "windows"}
	var isSystem bool

	// === Parse the flags
	flag.Usage = usage
	flag.Parse()

	if *listSystems {
		fmt.Print("  = Systems\n\n  ")
		fmt.Println(validSystems)
		os.Exit(0)
	}
	if len(os.Args) == 1 || *system == "" || *pkgName == "" {
		usage()
	}

	*system = strings.ToLower(*system)

	for _, v := range validSystems {
		if v == *system {
			isSystem = true
			break
		}
	}
	if !isSystem {
		fmt.Fprintf(os.Stderr, "ERROR: System passed in flag 's' is invalid\n")
		os.Exit(1)
	}

	// === Update header
	cmd := strings.Join(os.Args, " ")
	goHeader = strings.Replace(goHeader, "{cmd}", path.Base(cmd), 1)
	goHeader = strings.Replace(goHeader, "{pkg}", *pkgName, 1)

	// Translate all headers passed in command line.
	for _, file := range flag.Args() {
		if err := translateC(file); err != nil {
			fmt.Fprintf(os.Stderr, err.String())
			os.Exit(1)
		}
	}

//	File := "../test/header.h" //!!!
}

<commit_msg>Don't use a block for constants in single lines.<commit_after>// Copyright 2010  The "goheader" Authors
//
// Use of this source code is governed by the Simplified BSD License
// that can be found in the LICENSE file.
//
// This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
// OR CONDITIONS OF ANY KIND, either express or implied. See the License
// for more details.

package main

import (
	"bufio"
	//"bytes"
	"container/vector"
	"flag"
	"fmt"
	"os"
	"path"
	"regexp"
	"strings"
)

const COMMENT_C_LINE = "//!!! "

var goHeader = `// {cmd}
// MACHINE GENERATED.

package {pkg}

`

// Flags
var (
	system      = flag.String("s", "", "The operating system")
	pkgName     = flag.String("p", "", "The name of the package")
	listSystems = flag.Bool("l", false, "List of valid systems")
	write       = flag.Bool("w", false,
		"If set, write each input file with its output.")
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: goheader -s system -g package [defs.h...]\n")
	flag.PrintDefaults()
	os.Exit(1)
}

// Translates C type declaration into Go type declaration.
func translateC(cHeader string) os.Error {
	// === Regular expressions
	reSkip := regexp.MustCompile(`^(\n|//)`) // Empty lines and comments.

	reType := regexp.MustCompile(`^(typedef)[ \t]+(.+)[ \t]+(.+)[;](.+)?`)

	reStruct := regexp.MustCompile(`^(struct)[ \t]+(.+)[ \t]*{`)
	reStructField := regexp.MustCompile(`^[ \t]*(.+)[ \t]+(.+)[;](.+)?`)
	reStructFieldName := regexp.MustCompile(`^([^_]*_)?(.+)`)

	reDefine := regexp.MustCompile(`^[ \t]*#[ \t]*(define|DEFINE)[ \t]+([^ \t]+)[ \t]+(.+)`)
	reDefineMacro := regexp.MustCompile(`^.*(\(.*\))`)

	reSingleComment := regexp.MustCompile(`^(.+)?/\*[ \t]*(.+)[ \t]*\*/`)
	reStartMultipleComment := regexp.MustCompile(`^/\*(.+)?`)
	reMiddleMultipleComment := regexp.MustCompile(`^[ \t*]*(.+)`)
	reEndMultipleComment := regexp.MustCompile(`^(.+)?\*/`)
	// ===

	// Check the extension.
	if path.Ext(cHeader) != ".h" {
		return os.NewError("error extension")
	}

	// === File to read
	var isMultipleComment, isDefine, isStruct bool
	var extraType vector.StringVector // Types defined in C header.

	inFile, err := os.Open(cHeader, os.O_RDONLY, 0)
	if err != nil {
		return err
	}
	defer inFile.Close()

	inBuf := bufio.NewReader(inFile)

	// === File to write in the actual directory.
	var outBuf *bufio.Writer

	if *write {
		goHeader := strings.Split(path.Base(cHeader), ".h", 2)[0]
		goHeader = fmt.Sprintf("%s_%s.go", goHeader, *system)

		outFile, err := os.Open(goHeader, os.O_CREATE|os.O_WRONLY, 0644)
		if err != nil {
			return err
		}
		defer outFile.Close()

		outBuf = bufio.NewWriter(outFile)
	} else {
		outBuf = bufio.NewWriter(os.Stdout)
	}

	if _, err := outBuf.WriteString(goHeader); err != nil {
		return err
	}

	for {
		line, err := inBuf.ReadString('\n')
		if err == os.EOF {
			break
		}
		line = strings.TrimSpace(line) + "\n"
		isSingleComment := false

		// === Translate comment of single line.
		if !isMultipleComment {
			if sub := reSingleComment.FindStringSubmatch(line); sub != nil {
				isSingleComment = true
				line = "// " + sub[2] + "\n"

				if sub[1] != "" {
					line = sub[1] + line
				}
			}
		}
		if !isSingleComment && !isMultipleComment && strings.HasPrefix(line, "/*") {
			isMultipleComment = true
		}

		// === Translate comments of multiple line.
		if isMultipleComment {
			if sub := reStartMultipleComment.FindStringSubmatch(line); sub != nil {
				if sub[1] != "\n" {
					line = "// " + sub[1]
					if _, err := outBuf.WriteString(line); err != nil {
						return err
					}
				}
				continue
			}
			if sub := reEndMultipleComment.FindStringSubmatch(line); sub != nil {
				if sub[1] != "" {
					line = "// " + sub[1] + "\n"
					if _, err := outBuf.WriteString(line); err != nil {
						return err
					}
				}
				isMultipleComment = false
				continue
			}
			if sub := reMiddleMultipleComment.FindStringSubmatch(line); sub != nil {
				line = "// " + sub[1]
				if _, err := outBuf.WriteString(line); err != nil {
					return err
				}
				continue
			}
		}

		// === Translate type definitions.
		if sub := reType.FindStringSubmatch(line); sub != nil {
			// Add the new type.
			extraType.Push(sub[3])

			gotype, ok := ctypeTogo(sub[2], &extraType)
			line = fmt.Sprintf("type %s %s", sub[3], gotype)

			if sub[4] != "\n" {
				line += sub[4]
			} else {
				line += "\n"
			}
			if !ok {
				line = COMMENT_C_LINE + line
			}

			if _, err := outBuf.WriteString(line); err != nil {
				return err
			}
			continue
		}

		// === Translate defines.
		if sub := reDefine.FindStringSubmatch(line); sub != nil {
			line = fmt.Sprintf("%s = %s", sub[2], sub[3])

			if !isDefine {
				// Get characters of next line.
				startNextLine, err := inBuf.Peek(6)
				if err != nil {
					return err
				}

				// Constant in single line.
				if startNextLine[0] == '\n' || string(startNextLine) == "struct" {
					line = "const " + line
				} else {
					isDefine = true
					line = "const (\n" + line
				}
			}

			// Removes comment (if any) to ckeck if it is a macro.
			lastField := strings.Split(sub[3], "//", -1)[0]
			if reDefineMacro.MatchString(lastField) {
				line = COMMENT_C_LINE + line
			}

			if _, err := outBuf.WriteString(line); err != nil {
				return err
			}
			continue
		}

		if isDefine && line == "\n" {
			if _, err := outBuf.WriteString(")\n\n"); err != nil {
				return err
			}
			isDefine = false
			continue
		}

		// === Translate structs.
		if !isStruct {
			if sub := reStruct.FindStringSubmatch(line); sub != nil {
				isStruct = true

				if isDefine {
					if _, err := outBuf.WriteString(")\n"); err != nil {
						return err
					}
					isDefine = false
				}

				if _, err := outBuf.WriteString(fmt.Sprintf(
					"type %s struct {\n", strings.Title(sub[2]))); err != nil {
					return err
				}
				continue
			}
		} else {
			if sub := reStructField.FindStringSubmatch(line); sub != nil {
				// Translate the field type.
				gotype, ok := ctypeTogo(sub[1], &extraType)

				// === Translate the field name.
				fieldName := reStructFieldName.FindStringSubmatch(sub[2])
				_fieldName := ""

				if fieldName[1] != "" {
					_fieldName = fieldName[2]
				} else {
					_fieldName = fieldName[0]
				}
				// ===

				line = fmt.Sprintf("%s %s %s",
					strings.Title(_fieldName), gotype, sub[3])

				// C type not found.
				if !ok {
					line = COMMENT_C_LINE + line
				}

				if _, err := outBuf.WriteString(line); err != nil {
					return err
				}
				continue
			}
			if strings.HasPrefix(line, "}") {
				if _, err := outBuf.WriteString(strings.Replace(line, ";", "", 1)); err != nil {
					return err
				}
				isStruct = false
				continue
			}
		}

		// Comment another C lines.
		//if line != "\n" && !strings.HasPrefix(line, "//") {
		if !reSkip.MatchString(line) {
			line = COMMENT_C_LINE + line
		}

		if _, err := outBuf.WriteString(line); err != nil {
			return err
		}
	}

	if err := outBuf.Flush(); err != nil {
		return err
	}
	return nil
}

// Translates a C type definition into Go definition. The C header could have
// defined new types so they're checked in the firs place.
func ctypeTogo(ctype string, extraCtype *vector.StringVector) (gotype string, ok bool) {
	for _, v := range *extraCtype {
		if v == ctype {
			return ctype, true
		}
	}

	switch ctype {
	case "char", "signed char", "signed short int", "short int", "short":
		return "int8", true
	case "unsigned char", "unsigned short int", "unsigned short":
		return "uint8", true
	case "int", "signed int":
		return "int16", true
	case "unsigned int", "unsigned":
		return "uint16", true
	case "signed long int", "long int", "long":
		return "int32", true
	case "unsigned long int", "unsigned long":
		return "uint32", true
	case "float":
		return "float32", true
	case "double", "long double":
		return "float64", true
	}
	return ctype, false
}

func main() {
	validSystems := []string{"darwin", "freebsd", "linux", "windows"}
	var isSystem bool

	// === Parse the flags
	flag.Usage = usage
	flag.Parse()

	if *listSystems {
		fmt.Print("  = Systems\n\n  ")
		fmt.Println(validSystems)
		os.Exit(0)
	}
	if len(os.Args) == 1 || *system == "" || *pkgName == "" {
		usage()
	}

	*system = strings.ToLower(*system)

	for _, v := range validSystems {
		if v == *system {
			isSystem = true
			break
		}
	}
	if !isSystem {
		fmt.Fprintf(os.Stderr, "ERROR: System passed in flag 's' is invalid\n")
		os.Exit(1)
	}

	// === Update header
	cmd := strings.Join(os.Args, " ")
	goHeader = strings.Replace(goHeader, "{cmd}", path.Base(cmd), 1)
	goHeader = strings.Replace(goHeader, "{pkg}", *pkgName, 1)

	// Translate all headers passed in command line.
	for _, file := range flag.Args() {
		if err := translateC(file); err != nil {
			fmt.Fprintf(os.Stderr, err.String())
			os.Exit(1)
		}
	}

//	File := "../test/header.h" //!!!
}

<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/dustin/go-humanize"

	"github.intel.com/hpdd/ce-tools/pkg/applog"
	"github.intel.com/hpdd/lustre"
	"github.intel.com/hpdd/lustre/fs"
	"github.intel.com/hpdd/lustre/hsm"
)

func init() {
	hsmStateFlags := strings.Join(hsm.GetStateFlagNames(), ",")

	hsmCommand := cli.Command{
		Name:  "hsm",
		Usage: "HSM-related data movement actions",
		Subcommands: []cli.Command{
			{
				Name:      "archive",
				Usage:     "Initiate HSM archive of specified paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.IntFlag{
						Name:  "id, i",
						Usage: "Numeric ID of archive backend",
					},
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmAction,
			},
			{
				Name:      "release",
				Usage:     "Release local data of HSM-archived paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmAction,
			},
			{
				Name:      "restore",
				Usage:     "Explicitly restore local data of HSM-archived paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmAction,
			},
			{
				Name:      "remove",
				Usage:     "Remove HSM-archived data of specified paths (local data is not removed)",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmAction,
			},
			{
				Name:      "cancel",
				Usage:     "Cancel HSM operations being performed on specified paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmAction,
			},
			{
				Name:      "set",
				Usage:     "Set HSM flags or archive ID for specified paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
					cli.IntFlag{
						Name:  "id, i",
						Usage: "Numeric ID of archive backend",
					},
					cli.StringSliceFlag{
						Name:  "flag, f",
						Usage: fmt.Sprintf("HSM flag to set (%s)", hsmStateFlags),
						Value: &cli.StringSlice{},
					},
					cli.StringSliceFlag{
						Name:  "clear, F",
						Usage: fmt.Sprintf("HSM flag to clear (%s)", hsmStateFlags),
						Value: &cli.StringSlice{},
					},
				},
				Action: hsmSetAction,
			},
			{
				Name:      "status",
				Usage:     "Display HSM status for specified paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "action, a",
						Usage: "Include current HSM action",
					},
					cli.BoolFlag{
						Name:  "hide-path, H",
						Usage: "Hide pathname in output",
					},
					cli.BoolFlag{
						Name:  "long, l",
						Usage: "Show long-form states",
					},
					cli.BoolFlag{
						Name:  "progress, p",
						Usage: "Show copy progress for archive/restore actions",
					},
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmStatusAction,
			},
		},
	}
	commands = append(commands, hsmCommand)
}

func getFilePaths(c *cli.Context) ([]string, error) {
	var paths []string

	if c.Bool("null") {
		reader := bufio.NewReader(os.Stdin)
		path, err := reader.ReadBytes('\000')
		for err == nil {
			paths = append(paths, string(path[:len(path)-1]))
			path, err = reader.ReadBytes('\000')
		}
		if err != io.EOF {
			return nil, err
		}
	} else {
		paths = c.Args()
	}

	return paths, nil
}

func getPathStatus(c *cli.Context, filePath string) (string, error) {
	var buf bytes.Buffer

	s, err := hsm.GetFileStatus(filePath)
	if err != nil {
		return "", err
	}

	if !c.Bool("hide-path") {
		fmt.Fprintf(&buf, "%s ", filePath)
	}
	fmt.Fprintf(&buf, hsm.FileStatusString(s, !c.Bool("long")))

	if s.Exists() && c.Bool("action") {
		a, err := hsm.GetFileAction(filePath)
		if err != nil {
			return "", err
		}
		if a.IsNone() {
			fmt.Fprintf(&buf, " -")
		} else {
			fmt.Fprintf(&buf, " [%s:%s]", a.Action(), a.State())
			if c.Bool("progress") && (a.IsArchive() || a.IsRestore()) {
				st, err := os.Stat(filePath)
				if err != nil {
					applog.Fail(err)
				}
				fmt.Fprintf(&buf, "(%s/%s)",
					humanize.IBytes(a.BytesCopied),
					humanize.IBytes(uint64(st.Size())))
			}
		}
	} else {
		fmt.Fprintf(&buf, " -")
	}

	// TODO: Display xattrs, once we've standardized them?

	return buf.String(), nil
}

func hsmSetAction(c *cli.Context) {
	paths, err := getFilePaths(c)
	if err != nil {
		applog.Fail(err)
	}

	if len(paths) < 1 {
		applog.Fail(fmt.Errorf("HSM set request must be made with at least 1 path"))
	}

	setFlags, err := hsm.GetStatusMask(c.StringSlice("flag"))
	if err != nil {
		applog.Fail(err)
	}
	clearFlags, err := hsm.GetStatusMask(c.StringSlice("clear"))
	if err != nil {
		applog.Fail(err)
	}
	archiveID := uint32(c.Int("id"))

	if setFlags == 0 && clearFlags == 0 && archiveID == 0 {
		applog.Fail(fmt.Errorf("HSM set request made with no flags to set or clear, and no new archive ID supplied"))
	}

	// TODO: Parallelize this?
	for _, path := range paths {
		if err := hsm.SetFileStatus(path, setFlags, clearFlags, archiveID); err != nil {
			applog.Fail(err)
		}
	}
}

func hsmStatusAction(c *cli.Context) {
	paths, err := getFilePaths(c)
	if err != nil {
		applog.Fail(err)
	}

	if len(paths) < 1 {
		applog.Fail(fmt.Errorf("HSM status request must be made with at least 1 path"))
	}

	for _, path := range paths {
		status, err := getPathStatus(c, path)
		if err != nil {
			applog.Fail("%s: %v", path, err)
		}
		fmt.Println(status)
	}
}

func hsmAction(c *cli.Context) {
	paths, err := getFilePaths(c)
	if err != nil {
		applog.Fail(err)
	}

	if err := submitHsmRequest(c.Command.Name, uint(c.Int("id")), paths...); err != nil {
		applog.Fail(err)
	}
}

func submitHsmRequest(actionName string, archiveID uint, paths ...string) error {
	var fids []*lustre.Fid

	if len(paths) < 1 {
		return fmt.Errorf("HSM %s request must be made with at least 1 path", actionName)
	}

	fsRoot, err := fs.MountRoot(paths[0])
	if err != nil {
		return fmt.Errorf("Error getting fs root from %s: %s", paths[0], err)
	}

	// TODO: Occurs to me that it might be better to break up a large
	// batch into multiple batches, each serviced by its own goroutine.
	for _, path := range paths {
		absPath, err := filepath.Abs(path)
		if !strings.HasPrefix(absPath, string(fsRoot)) {
			return fmt.Errorf("All files in HSM request must be in the same filesystem (%s is not in %s)", path, fsRoot)
		}

		fid, err := fs.LookupFid(path)
		if err != nil {
			return fmt.Errorf("Cannot resolve Fid for %s: %s", path, err)
		}
		fids = append(fids, fid)
	}

	switch actionName {
	case "archive":
		err = hsm.RequestArchive(fsRoot, archiveID, fids)
	case "release":
		err = hsm.RequestRelease(fsRoot, archiveID, fids)
	case "restore":
		err = hsm.RequestRestore(fsRoot, archiveID, fids)
	case "remove":
		err = hsm.RequestRemove(fsRoot, archiveID, fids)
	case "cancel":
		err = hsm.RequestCancel(fsRoot, archiveID, fids)
	default:
		err = fmt.Errorf("Unhandled HSM action: %s", actionName)
	}

	return err
}
<commit_msg>Use closure for hsm request action<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/dustin/go-humanize"

	"github.intel.com/hpdd/ce-tools/pkg/applog"
	"github.intel.com/hpdd/lustre"
	"github.intel.com/hpdd/lustre/fs"
	"github.intel.com/hpdd/lustre/hsm"
)

func init() {
	hsmStateFlags := strings.Join(hsm.GetStateFlagNames(), ",")

	hsmCommand := cli.Command{
		Name:  "hsm",
		Usage: "HSM-related data movement actions",
		Subcommands: []cli.Command{
			{
				Name:      "archive",
				Usage:     "Initiate HSM archive of specified paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.IntFlag{
						Name:  "id, i",
						Usage: "Numeric ID of archive backend",
					},
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmRequestAction(hsm.RequestArchive),
			},
			{
				Name:      "release",
				Usage:     "Release local data of HSM-archived paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmRequestAction(hsm.RequestRelease),
			},
			{
				Name:      "restore",
				Usage:     "Explicitly restore local data of HSM-archived paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmRequestAction(hsm.RequestRestore),
			},
			{
				Name:      "remove",
				Usage:     "Remove HSM-archived data of specified paths (local data is not removed)",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmRequestAction(hsm.RequestRemove),
			},
			{
				Name:      "cancel",
				Usage:     "Cancel HSM operations being performed on specified paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmRequestAction(hsm.RequestCancel),
			},
			{
				Name:      "set",
				Usage:     "Set HSM flags or archive ID for specified paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
					cli.IntFlag{
						Name:  "id, i",
						Usage: "Numeric ID of archive backend",
					},
					cli.StringSliceFlag{
						Name:  "flag, f",
						Usage: fmt.Sprintf("HSM flag to set (%s)", hsmStateFlags),
						Value: &cli.StringSlice{},
					},
					cli.StringSliceFlag{
						Name:  "clear, F",
						Usage: fmt.Sprintf("HSM flag to clear (%s)", hsmStateFlags),
						Value: &cli.StringSlice{},
					},
				},
				Action: hsmSetAction,
			},
			{
				Name:      "status",
				Usage:     "Display HSM status for specified paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "action, a",
						Usage: "Include current HSM action",
					},
					cli.BoolFlag{
						Name:  "hide-path, H",
						Usage: "Hide pathname in output",
					},
					cli.BoolFlag{
						Name:  "long, l",
						Usage: "Show long-form states",
					},
					cli.BoolFlag{
						Name:  "progress, p",
						Usage: "Show copy progress for archive/restore actions",
					},
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmStatusAction,
			},
		},
	}
	commands = append(commands, hsmCommand)
}

func getFilePaths(c *cli.Context) ([]string, error) {
	var paths []string

	if c.Bool("null") {
		reader := bufio.NewReader(os.Stdin)
		path, err := reader.ReadBytes('\000')
		for err == nil {
			paths = append(paths, string(path[:len(path)-1]))
			path, err = reader.ReadBytes('\000')
		}
		if err != io.EOF {
			return nil, err
		}
	} else {
		paths = c.Args()
	}

	return paths, nil
}

func getPathStatus(c *cli.Context, filePath string) (string, error) {
	var buf bytes.Buffer

	s, err := hsm.GetFileStatus(filePath)
	if err != nil {
		return "", err
	}

	if !c.Bool("hide-path") {
		fmt.Fprintf(&buf, "%s ", filePath)
	}
	fmt.Fprintf(&buf, hsm.FileStatusString(s, !c.Bool("long")))

	if s.Exists() && c.Bool("action") {
		a, err := hsm.GetFileAction(filePath)
		if err != nil {
			return "", err
		}
		if a.IsNone() {
			fmt.Fprintf(&buf, " -")
		} else {
			fmt.Fprintf(&buf, " [%s:%s]", a.Action(), a.State())
			if c.Bool("progress") && (a.IsArchive() || a.IsRestore()) {
				st, err := os.Stat(filePath)
				if err != nil {
					applog.Fail(err)
				}
				fmt.Fprintf(&buf, "(%s/%s)",
					humanize.IBytes(a.BytesCopied),
					humanize.IBytes(uint64(st.Size())))
			}
		}
	} else {
		fmt.Fprintf(&buf, " -")
	}

	// TODO: Display xattrs, once we've standardized them?

	return buf.String(), nil
}

func hsmSetAction(c *cli.Context) {
	paths, err := getFilePaths(c)
	if err != nil {
		applog.Fail(err)
	}

	if len(paths) < 1 {
		applog.Fail(fmt.Errorf("HSM set request must be made with at least 1 path"))
	}

	setFlags, err := hsm.GetStatusMask(c.StringSlice("flag"))
	if err != nil {
		applog.Fail(err)
	}
	clearFlags, err := hsm.GetStatusMask(c.StringSlice("clear"))
	if err != nil {
		applog.Fail(err)
	}
	archiveID := uint32(c.Int("id"))

	if setFlags == 0 && clearFlags == 0 && archiveID == 0 {
		applog.Fail(fmt.Errorf("HSM set request made with no flags to set or clear, and no new archive ID supplied"))
	}

	// TODO: Parallelize this?
	for _, path := range paths {
		if err := hsm.SetFileStatus(path, setFlags, clearFlags, archiveID); err != nil {
			applog.Fail(err)
		}
	}
}

func hsmStatusAction(c *cli.Context) {
	paths, err := getFilePaths(c)
	if err != nil {
		applog.Fail(err)
	}

	if len(paths) < 1 {
		applog.Fail(fmt.Errorf("HSM status request must be made with at least 1 path"))
	}

	for _, path := range paths {
		status, err := getPathStatus(c, path)
		if err != nil {
			applog.Fail("%s: %v", path, err)
		}
		fmt.Println(status)
	}
}

type hsmRequestFn func(fs.RootDir, uint, []*lustre.Fid) error

func hsmRequestAction(requestFn func(fs.RootDir, uint, []*lustre.Fid) error) func(*cli.Context) {
	return func(c *cli.Context) {
		paths, err := getFilePaths(c)
		if err != nil {
			applog.Fail(err)
		}
		err = submitHsmRequest(c.Command.Name, hsmRequestFn(requestFn), uint(c.Int("id")), paths...)
		if err != nil {
			applog.Fail(err)
		}
	}
}

func submitHsmRequest(actionName string, requestFn hsmRequestFn, archiveID uint, paths ...string) error {
	var fids []*lustre.Fid

	if len(paths) < 1 {
		return fmt.Errorf("HSM %s request must be made with at least 1 path", actionName)
	}

	fsRoot, err := fs.MountRoot(paths[0])
	if err != nil {
		return fmt.Errorf("Error getting fs root from %s: %s", paths[0], err)
	}

	// TODO: Occurs to me that it might be better to break up a large
	// batch into multiple batches, each serviced by its own goroutine.
	for _, path := range paths {
		absPath, err := filepath.Abs(path)
		if !strings.HasPrefix(absPath, string(fsRoot)) {
			return fmt.Errorf("All files in HSM request must be in the same filesystem (%s is not in %s)",
				path, fsRoot)
		}

		fid, err := fs.LookupFid(path)
		if err != nil {
			return fmt.Errorf("Cannot resolve Fid for %s: %s", path, err)
		}
		fids = append(fids, fid)
	}

	if requestFn != nil {
		err = requestFn(fsRoot, archiveID, fids)
	} else {
		err = fmt.Errorf("Unhandled HSM action: %s", actionName)
	}

	return err
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"runtime/debug"
	"strings"
	"sync"
	"time"

	"git.apache.org/thrift.git/lib/go/thrift"
	"github.com/daviddengcn/go-colortext"
	"github.com/korovkin/limiter"
	"github.com/korovkin/parallel"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

type logger struct {
	ticket   int
	hostname string
	buf      *bytes.Buffer
}

var (
	loggerMutex     = new(sync.Mutex)
	loggerIndex     = int(0)
	loggerStartTime = time.Now()
	loggerHostname  = ""
)

var loggerColors = []ct.Color{
	ct.Green,
	ct.Cyan,
	ct.Magenta,
	ct.Yellow,
	ct.Blue,
	ct.Red,
}

func (l *logger) Write(p []byte) (int, error) {
	buf := bytes.NewBuffer(p)
	wrote := 0
	for {
		line, err := buf.ReadBytes('\n')
		if len(line) > 1 {
			now := time.Now().Format("15:04:05")
			s := string(line)
			ts := time.Since(loggerStartTime).String()

			loggerMutex.Lock()
			ct.ChangeColor(loggerColors[l.ticket%len(loggerColors)], false, ct.None, false)
			fmt.Printf("[%-14s %s %s %d] ", ts, l.hostname, now, l.ticket)
			ct.ResetColor()

			if l.buf != nil {
				l.buf.Write([]byte(s))
			}

			fmt.Print(s)
			loggerMutex.Unlock()

			wrote += len(line)
		}
		if err != nil {
			break
		}
	}
	if len(p) > 0 && p[len(p)-1] != '\n' {
		fmt.Println()
	}

	return len(p), nil
}

func newLogger(ticket int, collectLines bool) *logger {
	l := &logger{ticket: ticket, buf: nil}
	if collectLines {
		l.buf = &bytes.Buffer{}
	}
	return l
}

func CheckFatal(e error) error {
	if e != nil {
		debug.PrintStack()
		log.Println("CHECK: ERROR:", e)
		panic(e)
	}
	return e
}

func CheckNotFatal(e error) error {
	if e != nil {
		debug.PrintStack()
		log.Println("CHECK: ERROR:", e, e.Error())
	}
	return e
}

func executeCommand(p *Parallel, ticket int, cmdLine string) (*parallel.Output, error) {
	p.StatNumCommandsStart.Inc()
	T_START := time.Now()
	var err error
	output := &parallel.Output{}
	loggerOut := newLogger(ticket, true)
	loggerErr := newLogger(ticket, true)

	defer func() {
		dt := time.Since(T_START)
		fmt.Fprintf(
			loggerOut,
			"execute: done: dt: "+dt.String()+"\n",
		)
		if err == nil {
			p.StatNumCommandsDone.Inc()
			p.StatCommandLatency.Observe(dt.Seconds())
		}
	}()

	// execute remotely:
	if len(p.Slaves) > 0 {
		slave := p.Slaves[ticket%len(p.Slaves)]

		var transport thrift.TTransport
		transport, err = thrift.NewTSocket(slave.Address)
		if err != nil {
			log.Fatalln("failed to dial slave:", err.Error())
		}

		transport, err = p.transportFactory.GetTransport(transport)
		if err != nil {
			log.Fatalln("failed to GetTransport:", err.Error())
		}

		err = transport.Open()
		if err != nil {
			log.Fatalln("failed to open:", err.Error())
		}

		defer transport.Close()
		client := parallel.NewParallelClientFactory(transport, p.protocolFactory)

		cmd := parallel.Cmd{
			CmdLine: cmdLine,
			Ticket:  int64(ticket),
		}

		output, err = client.Execute(&cmd)
		if err != nil {
			log.Fatalln("failed to execute:", err.Error())
		}

		hostname := output.Tags["hostname"]
		loggerOut.hostname = hostname
		loggerErr.hostname = hostname

		fmt.Fprintf(loggerOut,
			"execute: remotely: host: %s stdout: [%s]\n",
			hostname,
			output.Stdout)

		fmt.Fprintf(loggerErr,
			"execute: remotely: host: %s stderr: [%s]\n",
			hostname,
			output.Stderr)

		return output, err
	}

	// execute locally:
	cs := []string{"/bin/sh", "-c", cmdLine}
	cmd := exec.Command(cs[0], cs[1:]...)
	cmd.Stdin = nil
	cmd.Stdout = loggerOut
	cmd.Stderr = loggerErr
	cmd.Env = append(
		os.Environ(),
		fmt.Sprintf("PARALLEL_TICKER=%d", ticket),
	)

	fmt.Fprintf(loggerOut, "run: '"+cmdLine+"'\n")

	err = cmd.Start()
	if err != nil {
		log.Fatalln("failed to start:", err)
		return output, err
	}

	if err == nil {
		err = cmd.Wait()
	}

	output.Tags = map[string]string{"hostname": loggerHostname}

	if loggerOut.buf != nil {
		output.Stdout = string(loggerOut.buf.Bytes())
	}

	if loggerErr.buf != nil {
		output.Stderr = string(loggerErr.buf.Bytes())
	}

	return output, err
}

type Slave struct {
	Address string
}

type Parallel struct {
	jobs   int
	logger *logger
	worker *limiter.ConcurrencyLimiter

	// master / slave
	protocolFactory  thrift.TProtocolFactory
	transportFactory thrift.TTransportFactory

	// master:
	Slaves []*Slave

	// slave:
	handler         *ParallelSlaveHandler
	serverTransport thrift.TServerTransport
	slaveAddress    string

	// stats:
	StatNumCommandsStart prometheus.Counter
	StatNumCommandsDone  prometheus.Counter
	StatCommandLatency   prometheus.Summary
}

func (p *Parallel) Close() {
	p.worker.Wait()
}

func mainMaster(p *Parallel) {
	var err error

	// connect to slaves:
	for _, slave := range p.Slaves {
		var transport thrift.TTransport
		transport, err = thrift.NewTSocket(slave.Address)
		if err != nil {
			log.Fatalln("failed to dial slave:", err.Error())
		}

		transport, err = p.transportFactory.GetTransport(transport)
		if err != nil {
			log.Fatalln("failed to open:", err.Error())
		}

		err = transport.Open()
		if err != nil {
			log.Fatalln("failed to open:", err.Error())
		}

		defer transport.Close()

		client := parallel.NewParallelClientFactory(transport, p.protocolFactory)
		ok, err := client.Ping()
		if err != nil {
			log.Fatalln("failed to ping client:", err.Error())
		}

		fmt.Fprintf(
			p.logger,
			"adding slave: %s ok: %s",
			slave.Address,
			ok)
	}

	r := bufio.NewReaderSize(os.Stdin, 1*1024*1024)
	fmt.Fprintf(p.logger, "reading from stdin...\n")
	for {
		line, err := r.ReadString('\n')
		if err == io.EOF {
			break
		}
		line = strings.TrimSpace(line)

		p.worker.ExecuteWithTicket(func(ticket int) {
			executeCommand(p, ticket, line)
		})
	}
}

type ParallelSlaveHandler struct {
	p *Parallel
}

func NewParallelSlaveHandler() *ParallelSlaveHandler {
	return &ParallelSlaveHandler{}
}

func (p *ParallelSlaveHandler) Execute(command *parallel.Cmd) (output *parallel.Output, err error) {
	output = nil
	err = nil
	output, err = executeCommand(p.p, int(command.Ticket), command.CmdLine)

	// TODO:: recover, handle panics

	return output, err
}

func (p *ParallelSlaveHandler) Ping() (r string, err error) {
	return "ping:ok", nil
}

func mainSlave(p *Parallel) {
	var err error

	p.serverTransport, err = thrift.NewTServerSocket(p.slaveAddress)

	if err != nil {
		log.Fatalln("failed to start server:", err.Error())
		return
	}

	p.handler = NewParallelSlaveHandler()
	p.handler.p = p

	server := thrift.NewTSimpleServer4(
		parallel.NewParallelProcessor(p.handler),
		p.serverTransport,
		p.transportFactory,
		p.protocolFactory)

	err = server.Serve()
	if err != nil {
		log.Fatalln("failed to run slave:", err.Error())
	}
}

func main() {
	T_START := time.Now()
	logger := newLogger(0, false)
	defer func() {
		fmt.Fprintf(logger, "all done: dt: "+time.Since(T_START).String()+"\n")
	}()

	flag_jobs := flag.Int(
		"j",
		2,
		"num of concurrent jobs")

	flag_slave := flag.Bool(
		"slave",
		false,
		"run as slave")

	flag_slaves := flag.String(
		"slaves",
		"",
		"CSV list of slave addresses")

	flag_slave_address := flag.String(
		"address",
		"localhost:9010",
		"slave address")

	flag_slave_metrics_address := flag.String(
		"metrics_address",
		"localhost:9011",
		"slave metric address")

	loggerHostname, _ = os.Hostname()

	flag.Parse()
	fmt.Fprintf(logger, "concurrency limit: %d", *flag_jobs)
	fmt.Fprintf(logger, "slaves: %s", *flag_slaves)

	p := Parallel{}
	p.jobs = *flag_jobs
	p.logger = logger
	p.worker = limiter.NewConcurrencyLimiter(p.jobs)
	p.slaveAddress = *flag_slave_address
	p.Slaves = []*Slave{}
	for _, slaveAddr := range strings.Split(*flag_slaves, ",") {
		if slaveAddr != "" {
			slave := Slave{Address: slaveAddr}
			p.Slaves = append(p.Slaves, &slave)
		}
	}

	defer p.Close()

	// thrift protocol
	p.protocolFactory = thrift.NewTBinaryProtocolFactoryDefault()

	// thrift transport
	p.transportFactory = thrift.NewTTransportFactory()
	p.transportFactory = thrift.NewTFramedTransportFactory(p.transportFactory)

	// stats:
	p.StatNumCommandsStart = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "num_commands_start",
			Help: "num received"})
	err := prometheus.Register(p.StatNumCommandsStart)
	CheckFatal(err)

	p.StatNumCommandsDone = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "num_commands_done",
			Help: "num received"})
	err = prometheus.Register(p.StatNumCommandsDone)
	CheckFatal(err)

	p.StatCommandLatency = prometheus.NewSummary(prometheus.SummaryOpts{
		Name: "commands_latency",
		Help: "commands latency stat",
	})
	err = prometheus.Register(p.StatCommandLatency)
	CheckFatal(err)

	if *flag_slave == false {
		loggerHostname = p.slaveAddress
		logger.hostname = loggerHostname

		fmt.Fprintf(logger, "running as master\n")
		mainMaster(&p)
	} else {
		loggerHostname = p.slaveAddress
		logger.hostname = loggerHostname

		// metrics
		metricsHandler := promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{})
		http.HandleFunc("/metrics", func(c http.ResponseWriter, req *http.Request) {
			metricsHandler.ServeHTTP(c, req)
		})

		http.HandleFunc("/",
			func(c http.ResponseWriter, req *http.Request) {
				io.WriteString(c, fmt.Sprintf("parallel %s", time.Now().String()))
			})

		go http.ListenAndServe(*flag_slave_metrics_address, nil)

		fmt.Fprintf(logger, "running as slave on: %s\n", p.slaveAddress)
		mainSlave(&p)
	}
}
<commit_msg>cleanup<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"runtime/debug"
	"strings"
	"sync"
	"time"

	"git.apache.org/thrift.git/lib/go/thrift"
	"github.com/daviddengcn/go-colortext"
	"github.com/korovkin/limiter"
	"github.com/korovkin/parallel"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

type logger struct {
	ticket   int
	hostname string
	buf      *bytes.Buffer
}

var (
	loggerMutex     = new(sync.Mutex)
	loggerIndex     = int(0)
	loggerStartTime = time.Now()
	loggerHostname  = ""
)

var loggerColors = []ct.Color{
	ct.Green,
	ct.Cyan,
	ct.Magenta,
	ct.Yellow,
	ct.Blue,
	ct.Red,
}

func (l *logger) Write(p []byte) (int, error) {
	buf := bytes.NewBuffer(p)
	wrote := 0
	for {
		line, err := buf.ReadBytes('\n')
		if len(line) > 1 {
			now := time.Now().Format("15:04:05")
			s := string(line)
			ts := time.Since(loggerStartTime).String()

			{
				loggerMutex.Lock()
				ct.ChangeColor(loggerColors[l.ticket%len(loggerColors)], false, ct.None, false)
				fmt.Printf("[%-14s %s %s %d] ", ts, l.hostname, now, l.ticket)
				ct.ResetColor()
				if l.buf != nil {
					l.buf.Write([]byte(s))
				}
				fmt.Print(s)
				loggerMutex.Unlock()
			}

			wrote += len(line)
		}
		if err != nil {
			break
		}
	}
	if len(p) > 0 && p[len(p)-1] != '\n' {
		fmt.Println()
	}

	return len(p), nil
}

func newLogger(ticket int, collectLines bool) *logger {
	l := &logger{ticket: ticket, buf: nil}
	if collectLines {
		l.buf = &bytes.Buffer{}
	}
	return l
}

func CheckFatal(e error) error {
	if e != nil {
		debug.PrintStack()
		log.Println("CHECK: ERROR:", e)
		panic(e)
	}
	return e
}

func CheckNotFatal(e error) error {
	if e != nil {
		debug.PrintStack()
		log.Println("CHECK: ERROR:", e, e.Error())
	}
	return e
}

func executeCommand(p *Parallel, ticket int, cmdLine string) (*parallel.Output, error) {
	p.StatNumCommandsStart.Inc()
	T_START := time.Now()
	var err error
	output := &parallel.Output{}
	loggerOut := newLogger(ticket, true)
	loggerErr := newLogger(ticket, true)

	defer func() {
		dt := time.Since(T_START)
		fmt.Fprintf(
			loggerOut,
			"execute: done: dt: "+dt.String()+"\n",
		)
		if err == nil {
			p.StatNumCommandsDone.Inc()
			p.StatCommandLatency.Observe(dt.Seconds())
		}
	}()

	// execute remotely:
	if len(p.Slaves) > 0 {
		slave := p.Slaves[ticket%len(p.Slaves)]

		var transport thrift.TTransport
		transport, err = thrift.NewTSocket(slave.Address)
		if err != nil {
			log.Fatalln("failed to dial slave:", err.Error())
		}

		transport, err = p.transportFactory.GetTransport(transport)
		if err != nil {
			log.Fatalln("failed to GetTransport:", err.Error())
		}

		err = transport.Open()
		if err != nil {
			log.Fatalln("failed to open:", err.Error())
		}

		defer transport.Close()
		client := parallel.NewParallelClientFactory(transport, p.protocolFactory)

		cmd := parallel.Cmd{
			CmdLine: cmdLine,
			Ticket:  int64(ticket),
		}

		output, err = client.Execute(&cmd)
		if err != nil {
			log.Fatalln("failed to execute:", err.Error())
		}

		hostname := output.Tags["hostname"]
		loggerOut.hostname = hostname
		loggerErr.hostname = hostname

		fmt.Fprintf(loggerOut,
			"execute: remotely: host: %s stdout: [%s]\n",
			hostname,
			output.Stdout)

		fmt.Fprintf(loggerErr,
			"execute: remotely: host: %s stderr: [%s]\n",
			hostname,
			output.Stderr)

		return output, err
	}

	// execute locally:
	cs := []string{"/bin/sh", "-c", cmdLine}
	cmd := exec.Command(cs[0], cs[1:]...)
	cmd.Stdin = nil
	cmd.Stdout = loggerOut
	cmd.Stderr = loggerErr
	cmd.Env = append(
		os.Environ(),
		fmt.Sprintf("PARALLEL_TICKER=%d", ticket),
	)

	fmt.Fprintf(loggerOut, "run: '"+cmdLine+"'\n")

	err = cmd.Start()
	if err != nil {
		log.Fatalln("failed to start:", err)
		return output, err
	}

	if err == nil {
		err = cmd.Wait()
	}

	output.Tags = map[string]string{"hostname": loggerHostname}

	if loggerOut.buf != nil {
		output.Stdout = string(loggerOut.buf.Bytes())
	}

	if loggerErr.buf != nil {
		output.Stderr = string(loggerErr.buf.Bytes())
	}

	return output, err
}

type Slave struct {
	Address string
}

type Parallel struct {
	jobs   int
	logger *logger
	worker *limiter.ConcurrencyLimiter

	// master / slave
	protocolFactory  thrift.TProtocolFactory
	transportFactory thrift.TTransportFactory

	// master:
	Slaves []*Slave

	// slave:
	handler         *ParallelSlaveHandler
	serverTransport thrift.TServerTransport
	slaveAddress    string

	// stats:
	StatNumCommandsStart prometheus.Counter
	StatNumCommandsDone  prometheus.Counter
	StatCommandLatency   prometheus.Summary
}

func (p *Parallel) Close() {
	p.worker.Wait()
}

func mainMaster(p *Parallel) {
	var err error

	// connect to slaves:
	for _, slave := range p.Slaves {
		var transport thrift.TTransport
		transport, err = thrift.NewTSocket(slave.Address)
		if err != nil {
			log.Fatalln("failed to dial slave:", err.Error())
		}

		transport, err = p.transportFactory.GetTransport(transport)
		if err != nil {
			log.Fatalln("failed to open:", err.Error())
		}

		err = transport.Open()
		if err != nil {
			log.Fatalln("failed to open:", err.Error())
		}

		defer transport.Close()

		client := parallel.NewParallelClientFactory(transport, p.protocolFactory)
		ok, err := client.Ping()
		if err != nil {
			log.Fatalln("failed to ping client:", err.Error())
		}

		fmt.Fprintf(
			p.logger,
			"adding slave: %s ok: %s",
			slave.Address,
			ok)
	}

	r := bufio.NewReaderSize(os.Stdin, 1*1024*1024)
	fmt.Fprintf(p.logger, "reading from stdin...\n")
	for {
		line, err := r.ReadString('\n')
		if err == io.EOF {
			break
		}
		line = strings.TrimSpace(line)

		p.worker.ExecuteWithTicket(func(ticket int) {
			executeCommand(p, ticket, line)
		})
	}
}

type ParallelSlaveHandler struct {
	p *Parallel
}

func NewParallelSlaveHandler() *ParallelSlaveHandler {
	return &ParallelSlaveHandler{}
}

func (p *ParallelSlaveHandler) Execute(command *parallel.Cmd) (output *parallel.Output, err error) {
	output = nil
	err = nil
	output, err = executeCommand(p.p, int(command.Ticket), command.CmdLine)

	// TODO:: recover, handle panics

	return output, err
}

func (p *ParallelSlaveHandler) Ping() (r string, err error) {
	return "ping:ok", nil
}

func mainSlave(p *Parallel) {
	var err error

	p.serverTransport, err = thrift.NewTServerSocket(p.slaveAddress)

	if err != nil {
		log.Fatalln("failed to start server:", err.Error())
		return
	}

	p.handler = NewParallelSlaveHandler()
	p.handler.p = p

	server := thrift.NewTSimpleServer4(
		parallel.NewParallelProcessor(p.handler),
		p.serverTransport,
		p.transportFactory,
		p.protocolFactory)

	err = server.Serve()
	if err != nil {
		log.Fatalln("failed to run slave:", err.Error())
	}
}

func main() {
	T_START := time.Now()
	logger := newLogger(0, false)
	defer func() {
		fmt.Fprintf(logger, "all done: dt: "+time.Since(T_START).String()+"\n")
	}()

	flag_jobs := flag.Int(
		"j",
		2,
		"num of concurrent jobs")

	flag_slave := flag.Bool(
		"slave",
		false,
		"run as slave")

	flag_slaves := flag.String(
		"slaves",
		"",
		"CSV list of slave addresses")

	flag_slave_address := flag.String(
		"address",
		"localhost:9010",
		"slave address")

	flag_slave_metrics_address := flag.String(
		"metrics_address",
		"localhost:9011",
		"slave metric address")

	loggerHostname, _ = os.Hostname()

	flag.Parse()
	fmt.Fprintf(logger, "concurrency limit: %d", *flag_jobs)
	fmt.Fprintf(logger, "slaves: %s", *flag_slaves)

	p := Parallel{}
	p.jobs = *flag_jobs
	p.logger = logger
	p.worker = limiter.NewConcurrencyLimiter(p.jobs)
	p.slaveAddress = *flag_slave_address
	p.Slaves = []*Slave{}
	for _, slaveAddr := range strings.Split(*flag_slaves, ",") {
		if slaveAddr != "" {
			slave := Slave{Address: slaveAddr}
			p.Slaves = append(p.Slaves, &slave)
		}
	}

	defer p.Close()

	// thrift protocol
	p.protocolFactory = thrift.NewTBinaryProtocolFactoryDefault()

	// thrift transport
	p.transportFactory = thrift.NewTTransportFactory()
	p.transportFactory = thrift.NewTFramedTransportFactory(p.transportFactory)

	// stats:
	p.StatNumCommandsStart = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "num_commands_start",
			Help: "num received"})
	err := prometheus.Register(p.StatNumCommandsStart)
	CheckFatal(err)

	p.StatNumCommandsDone = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "num_commands_done",
			Help: "num received"})
	err = prometheus.Register(p.StatNumCommandsDone)
	CheckFatal(err)

	p.StatCommandLatency = prometheus.NewSummary(prometheus.SummaryOpts{
		Name: "commands_latency",
		Help: "commands latency stat",
	})
	err = prometheus.Register(p.StatCommandLatency)
	CheckFatal(err)

	if *flag_slave == false {
		loggerHostname = p.slaveAddress
		logger.hostname = loggerHostname

		fmt.Fprintf(logger, "running as master\n")
		mainMaster(&p)
	} else {
		loggerHostname = p.slaveAddress
		logger.hostname = loggerHostname

		// metrics
		metricsHandler := promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{})
		http.HandleFunc("/metrics", func(c http.ResponseWriter, req *http.Request) {
			metricsHandler.ServeHTTP(c, req)
		})

		http.HandleFunc("/",
			func(c http.ResponseWriter, req *http.Request) {
				io.WriteString(c, fmt.Sprintf("parallel %s", time.Now().String()))
			})

		go http.ListenAndServe(*flag_slave_metrics_address, nil)

		fmt.Fprintf(logger, "running as slave on: %s\n", p.slaveAddress)
		mainSlave(&p)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/nesv/pm"
	"github.com/spf13/cobra"
)

var CleanCmd = &cobra.Command{
	Use:   "clean",
	Short: "Clean stuff up",
	Long: `The "clean" command will only clean up the parts of itself that you
tell it to. Without any flags provided, "clean" will do nothing.

To have pm clean its cache, provide the "--cache" flag:

    $ pm clean --cache

To clean unpacked, unlinked archives, provide the "--unlinked" flag:

    $ pm clean --unlinked

For convenience, you can supply the "--all" flag, which is synonymous to
calling:

    $ pm clean --cache --unlinked
`,
	Run: runClean,
}

var (
	CleanAll      bool
	CleanCache    bool
	CleanUnlinked bool
)

func init() {
	CleanCmd.Flags().BoolVarP(&CleanAll, "all", "", false, "Clean everything")
	CleanCmd.Flags().BoolVarP(&CleanCache, "cache", "", false, "Remove all cached files")
	CleanCmd.Flags().BoolVarP(&CleanUnlinked, "unlinked", "", false, "Remove all unpacked, unlinked packages")
}

func runClean(cmd *cobra.Command, args []string) {
	if CleanAll {
		CleanCache = true
		CleanUnlinked = true
	}

	if CleanCache {
		cleanCache()
	}

	if CleanUnlinked {
		cleanUnlinked()
	}
}

func cleanCache() {
	cached, err := pm.ListCachedPackagesSlice(rootCacheDir)
	if err != nil {
		log.Fatalln(err)
	}

	for _, v := range cached {
		if Verbose {
			fmt.Println("removing", v, "from cache")
		}

		cachedFile := filepath.Join(
			rootCacheDir,
			fmt.Sprintf("%s-%s-%s.tar.gz", v, runtime.GOOS, runtime.GOARCH),
		)
		if err := os.Remove(cachedFile); err != nil {
			log.Fatalln(err)
		}
	}
}

func cleanUnlinked() {
	lp, err := pm.ListLinkedPackagesSlice(rootBaseDir, rootBinDir)
	if err != nil {
		log.Fatalln(err)
	}

	var linkedPkgs = make(map[string]struct{})
	for _, v := range lp {
		linkedPkgs[v] = struct{}{}
	}

	unpacked, err := pm.ListUnpackedPackagesSlice(rootBaseDir)
	if err != nil {
		log.Fatalln(err)
	}

	for _, u := range unpacked {
		if _, linked := linkedPkgs[u]; !linked {
			if Verbose {
				fmt.Println("removing", u)
			}

			parts := []string{rootBaseDir}
			parts = append(parts, strings.Split(u, pm.PackageFieldSeparator)...)
			pth := filepath.Join(parts...)
			if err := os.RemoveAll(pth); err != nil {
				log.Fatalln(err)
			}
		}
	}
}
<commit_msg>Added short-hand flags for the --cache and --unlinked flags.<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/nesv/pm"
	"github.com/spf13/cobra"
)

var CleanCmd = &cobra.Command{
	Use:   "clean",
	Short: "Clean stuff up",
	Long: `The "clean" command will only clean up the parts of itself that you
tell it to. Without any flags provided, "clean" will do nothing.

To have pm clean its cache, provide the "--cache" flag:

    $ pm clean --cache

To clean unpacked, unlinked archives, provide the "--unlinked" flag:

    $ pm clean --unlinked

For convenience, you can supply the "--all" flag, which is synonymous to
calling:

    $ pm clean --cache --unlinked
`,
	Run: runClean,
}

var (
	CleanAll      bool
	CleanCache    bool
	CleanUnlinked bool
)

func init() {
	CleanCmd.Flags().BoolVarP(&CleanAll, "all", "", false, "Clean everything")
	CleanCmd.Flags().BoolVarP(&CleanCache, "cache", "c", false, "Remove all cached files")
	CleanCmd.Flags().BoolVarP(&CleanUnlinked, "unlinked", "u", false, "Remove all unpacked, unlinked packages")
}

func runClean(cmd *cobra.Command, args []string) {
	if CleanAll {
		CleanCache = true
		CleanUnlinked = true
	}

	if CleanCache {
		cleanCache()
	}

	if CleanUnlinked {
		cleanUnlinked()
	}
}

func cleanCache() {
	cached, err := pm.ListCachedPackagesSlice(rootCacheDir)
	if err != nil {
		log.Fatalln(err)
	}

	for _, v := range cached {
		if Verbose {
			fmt.Println("removing", v, "from cache")
		}

		cachedFile := filepath.Join(
			rootCacheDir,
			fmt.Sprintf("%s-%s-%s.tar.gz", v, runtime.GOOS, runtime.GOARCH),
		)
		if err := os.Remove(cachedFile); err != nil {
			log.Fatalln(err)
		}
	}
}

func cleanUnlinked() {
	lp, err := pm.ListLinkedPackagesSlice(rootBaseDir, rootBinDir)
	if err != nil {
		log.Fatalln(err)
	}

	var linkedPkgs = make(map[string]struct{})
	for _, v := range lp {
		linkedPkgs[v] = struct{}{}
	}

	unpacked, err := pm.ListUnpackedPackagesSlice(rootBaseDir)
	if err != nil {
		log.Fatalln(err)
	}

	for _, u := range unpacked {
		if _, linked := linkedPkgs[u]; !linked {
			if Verbose {
				fmt.Println("removing", u)
			}

			parts := []string{rootBaseDir}
			parts = append(parts, strings.Split(u, pm.PackageFieldSeparator)...)
			pth := filepath.Join(parts...)
			if err := os.RemoveAll(pth); err != nil {
				log.Fatalln(err)
			}
		}
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"errors"
	"os"

	"github.com/jkp85/cli-tools/api"
	"github.com/jkp85/go-sdk/client/projects"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

func projectsCmd() *cobra.Command {
	var limit, offset, format string
	filters := api.NewFilterVal()
	cmd := &cobra.Command{
		Use:   "projects",
		Short: "Handle projects",
		Long:  ``,
		RunE: func(cmd *cobra.Command, args []string) error {
			cli := api.APIClient()
			params := projects.NewProjectsListParams()
			ns := viper.GetString("namespace")
			if ns == "" {
				return errors.New("You must provide a namespace")
			}
			params.SetNamespace(ns)
			params.SetLimit(&limit)
			params.SetOffset(&offset)
			params.SetPrivate(filters.Get("private"))
			resp, err := cli.Projects.ProjectsList(params)
			if err != nil {
				return err
			}
			if format == "" {
				format = viper.GetString("projects_template")
			}
			r := api.NewRenderer(format, &resp.Payload)
			return r.Render(os.Stdout)
		},
	}
	cmd.Flags().StringVar(&limit, "limit", "10", "Limit list results")
	cmd.Flags().StringVar(&offset, "offset", "0", "Offset list results")
	cmd.Flags().StringVar(&format, "format", "json", "Output format")
	cmd.Flags().Var(filters, "filter", "Filter results")
	return cmd
}

func init() {
	RootCmd.AddCommand(projectsCmd())
}
<commit_msg>Add ordering<commit_after>package cmd

import (
	"errors"
	"os"

	"github.com/jkp85/cli-tools/api"
	"github.com/jkp85/go-sdk/client/projects"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

func projectsCmd() *cobra.Command {
	var limit, offset, format, order string
	filters := api.NewFilterVal()
	cmd := &cobra.Command{
		Use:   "projects",
		Short: "Handle projects",
		Long:  ``,
		RunE: func(cmd *cobra.Command, args []string) error {
			cli := api.APIClient()
			params := projects.NewProjectsListParams()
			ns := viper.GetString("namespace")
			if ns == "" {
				return errors.New("You must provide a namespace")
			}
			params.SetNamespace(ns)
			params.SetLimit(&limit)
			params.SetOffset(&offset)
			params.SetOrdering(&order)
			params.SetPrivate(filters.Get("private"))
			resp, err := cli.Projects.ProjectsList(params)
			if err != nil {
				return err
			}
			if format == "" {
				format = viper.GetString("projects_template")
			}
			r := api.NewRenderer(format, &resp.Payload)
			return r.Render(os.Stdout)
		},
	}
	cmd.Flags().StringVar(&limit, "limit", "10", "Limit list results")
	cmd.Flags().StringVar(&offset, "offset", "0", "Offset list results")
	cmd.Flags().StringVar(&format, "format", "json", "Output format")
	cmd.Flags().StringVar(&order, "order", "", "Output order")
	cmd.Flags().Var(filters, "filter", "Filter results")
	return cmd
}

func init() {
	RootCmd.AddCommand(projectsCmd())
}
<|endoftext|>