<commit_before>package irc

import (
	"fmt"
	"strings"
)

type Command struct {
	Action string
	Args   []string
}

func (s *Server) ping() {
	s.readWriter.WriteString(fmt.Sprintf("PONG %s", s.ServerName))
	s.readWriter.Flush()
}

func (s *Server) pass(password string) {
	if len(password) > 1 {
		s.readWriter.WriteString(fmt.Sprintf("PASS %s\r\n", password))
		s.readWriter.Flush()
	}
}

func (s *Server) user(username string) {
	s.readWriter.WriteString(fmt.Sprintf("NICK %s\r\n", username))
	s.readWriter.WriteString(fmt.Sprintf("USER %s %d * :GoIRC bot\r\n", username, 0))
	s.readWriter.Flush()
}

func (s *Server) join(room ...string) {
	if len(room) < 1 {
		return
	}

	rooms := strings.Join(room, ",")
	fmt.Println("join ", rooms)

	_, e := s.readWriter.WriteString(fmt.Sprintf("JOIN %s\r\n", rooms))
	if e != nil {
		s.errChan <- e
		return
	}

	s.readWriter.Flush()
}

func (s *Server) part(room, message string) {
	_, e := s.readWriter.WriteString(fmt.Sprintf("PART %s :%s\r\n", room, message))
	if e != nil {
		s.errChan <- e
		return
	}

	s.readWriter.Flush()
}

func (s *Server) invite(nick, room string) {
	_, e := s.readWriter.WriteString(fmt.Sprintf("INVITE %s %s\r\n", nick, room))
	if e != nil {
		s.errChan <- e
		return
	}

	s.readWriter.Flush()
}

func (s *Server) kick(user, room, message string) {
	_, e := s.readWriter.WriteString(fmt.Sprintf("KICK %s %s :%s\r\n", room, user, message))
	if e != nil {
		s.errChan <- e
		return
	}

	s.readWriter.Flush()
}

func (s *Server) privMessage(target, message string) {
	_, e := s.readWriter.WriteString(fmt.Sprintf("PRIVMSG %s :%s\r\n", target, message))
	if e != nil {
		s.errChan <- e
	}

	s.readWriter.Flush()
}

func (s *Server) list(scope ...string) {
	if len(scope) < 1 {
		return
	}

	scopes := strings.Join(scope, ",")

	_, e := s.readWriter.WriteString(fmt.Sprintf("LIST %s\r\n", scopes))
	if e != nil {
		s.errChan <- e
	}

	s.readWriter.Flush()
}

func (s *Server) name(scope ...string) {
	if len(scope) < 1 {
		return
	}

	scopes := strings.Join(scope, ",")

	_, e := s.readWriter.WriteString(fmt.Sprintf("NAMES %s\r\n", scopes))
	if e != nil {
		s.errChan <- e
	}

	s.readWriter.Flush()
}
<commit_msg>removing un needed println from command<commit_after>package irc

import (
	"fmt"
	"strings"
)

type Command struct {
	Action string
	Args   []string
}

func (s *Server) ping() {
	s.readWriter.WriteString(fmt.Sprintf("PONG %s", s.ServerName))
	s.readWriter.Flush()
}

func (s *Server) pass(password string) {
	if len(password) > 1 {
		s.readWriter.WriteString(fmt.Sprintf("PASS %s\r\n", password))
		s.readWriter.Flush()
	}
}

func (s *Server) user(username string) {
	s.readWriter.WriteString(fmt.Sprintf("NICK %s\r\n", username))
	s.readWriter.WriteString(fmt.Sprintf("USER %s %d * :GoIRC bot\r\n", username, 0))
	s.readWriter.Flush()
}

func (s *Server) join(room ...string) {
	if len(room) < 1 {
		return
	}

	rooms := strings.Join(room, ",")

	_, e := s.readWriter.WriteString(fmt.Sprintf("JOIN %s\r\n", rooms))
	if e != nil {
		s.errChan <- e
		return
	}

	s.readWriter.Flush()
}

func (s *Server) part(room, message string) {
	_, e := s.readWriter.WriteString(fmt.Sprintf("PART %s :%s\r\n", room, message))
	if e != nil {
		s.errChan <- e
		return
	}

	s.readWriter.Flush()
}

func (s *Server) invite(nick, room string) {
	_, e := s.readWriter.WriteString(fmt.Sprintf("INVITE %s %s\r\n", nick, room))
	if e != nil {
		s.errChan <- e
		return
	}

	s.readWriter.Flush()
}

func (s *Server) kick(user, room, message string) {
	_, e := s.readWriter.WriteString(fmt.Sprintf("KICK %s %s :%s\r\n", room, user, message))
	if e != nil {
		s.errChan <- e
		return
	}

	s.readWriter.Flush()
}

func (s *Server) privMessage(target, message string) {
	_, e := s.readWriter.WriteString(fmt.Sprintf("PRIVMSG %s :%s\r\n", target, message))
	if e != nil {
		s.errChan <- e
	}

	s.readWriter.Flush()
}

func (s *Server) list(scope ...string) {
	if len(scope) < 1 {
		return
	}

	scopes := strings.Join(scope, ",")

	_, e := s.readWriter.WriteString(fmt.Sprintf("LIST %s\r\n", scopes))
	if e != nil {
		s.errChan <- e
	}

	s.readWriter.Flush()
}

func (s *Server) name(scope ...string) {
	if len(scope) < 1 {
		return
	}

	scopes := strings.Join(scope, ",")

	_, e := s.readWriter.WriteString(fmt.Sprintf("NAMES %s\r\n", scopes))
	if e != nil {
		s.errChan <- e
	}

	s.readWriter.Flush()
}
<|endoftext|>


<commit_before>package lang

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/runtime"
	"strings"
	"sync"
)

// Policy describes the entire Aptomi policy.
//
// At the highest level, policy consists of namespaces. Namespaces provide isolation for policy objects and access to
// namespaces can be controlled via ACL rules. Thus, different users can have different access rights to different parts
// of Aptomi policy. Namespaces are useful in environments with many users, multiple teams and projects.
//
// Objects get stored in their corresponding namespaces. Names of objects must be unique within a namespace and a given
// object kind.
//
// Once policy is defined, it can be passed to the engine for policy resolution. Policy resolution translates a given
// policy (intent) into actual state (what services/components need to created/updated/deleted, how and where) and the
// corresponding set of actions.
type Policy struct {
	// Namespace is a map from namespace name into a PolicyNamespace
	Namespace map[string]*PolicyNamespace `validate:"dive"`

	// Access control rules for different policy namespaces
	once        sync.Once
	aclResolver *ACLResolver // lazily initialized value
}

// NewPolicy creates a new Policy
func NewPolicy() *Policy {
	return &Policy{Namespace: make(map[string]*PolicyNamespace)}
}

// View returns a policy view object, which allows to make all policy operations on behalf of a certain user
// Policy view object will enforce all ACLs, allowing the user to only perform actions which he is allowed to perform
// All ACL rules should be loaded and added to the policy before this method gets called
func (policy *Policy) View(user *User) *PolicyView {
	policy.once.Do(func() {
		systemNamespace := policy.Namespace[runtime.SystemNS]
		if systemNamespace != nil {
			policy.aclResolver = NewACLResolver(systemNamespace.ACLRules)
		} else {
			policy.aclResolver = NewACLResolver(NewGlobalRules())
		}
	})
	return NewPolicyView(policy, user)
}

// AddObject adds an object into the policy. When you add objects to the policy, they get added to the corresponding
// Namespace. If error occurs (e.g. object has an unknown kind) then the error will be returned
func (policy *Policy) AddObject(obj Base) error {
	policyNamespace, ok := policy.Namespace[obj.GetNamespace()]
	if !ok {
		policyNamespace = NewPolicyNamespace(obj.GetNamespace())
		policy.Namespace[obj.GetNamespace()] = policyNamespace
	}
	return policyNamespace.addObject(obj)
}

// RemoveObject removes an object into the policy. It returns true is something was removed.
func (policy *Policy) RemoveObject(obj Base) bool {
	policyNamespace, ok := policy.Namespace[obj.GetNamespace()]
	if !ok {
		return false
	}

	return policyNamespace.removeObject(obj)
}

// GetObjectsByKind returns all objects in a policy with a given kind, across all namespaces
func (policy *Policy) GetObjectsByKind(kind string) []Base {
	result := []Base{}
	for _, policyNS := range policy.Namespace {
		result = append(result, policyNS.getObjectsByKind(kind)...)
	}
	return result
}

// GetObject looks up and returns an object from the policy, given its kind, locator ([namespace/]name), and current
// namespace relative to which the call is being made
func (policy *Policy) GetObject(kind string, locator string, currentNs string) (runtime.Object, error) {
	// parse locator: [namespace/]name. we might add [domain/] in the future
	parts := strings.Split(locator, "/")
	var ns, name string
	if len(parts) == 1 {
		ns = currentNs
		name = parts[0]
	} else if len(parts) == 2 {
		ns = parts[0]
		name = parts[1]
	} else {
		return nil, fmt.Errorf("can't parse policy object locator: '%s'", locator)
	}

	policyNS, ok := policy.Namespace[ns]
	if !ok {
		return nil, fmt.Errorf("namespace '%s' doesn't exist, but referenced in locator '%s'", ns, locator)
	}

	return policyNS.getObject(kind, name)
}

// Validate performs validation of the entire policy, making sure that all of its objects are well-formed.
// It also checks that all cross-object references are valid. If policy is malformed, then a list of errors is returned.
// Otherwise, if policy is correctly formed, then nil is returned.
// The resulting error can be caster to (validator.ValidationErrors) and iterated over, to get the full list of errors.
func (policy *Policy) Validate() error {
	return NewPolicyValidator(policy).Validate()
}
<commit_msg>comments for Add/Remove Object<commit_after>package lang

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/runtime"
	"strings"
	"sync"
)

// Policy describes the entire Aptomi policy.
//
// At the highest level, policy consists of namespaces. Namespaces provide isolation for policy objects and access to
// namespaces can be controlled via ACL rules. Thus, different users can have different access rights to different parts
// of Aptomi policy. Namespaces are useful in environments with many users, multiple teams and projects.
//
// Objects get stored in their corresponding namespaces. Names of objects must be unique within a namespace and a given
// object kind.
//
// Once policy is defined, it can be passed to the engine for policy resolution. Policy resolution translates a given
// policy (intent) into actual state (what services/components need to created/updated/deleted, how and where) and the
// corresponding set of actions.
type Policy struct {
	// Namespace is a map from namespace name into a PolicyNamespace
	Namespace map[string]*PolicyNamespace `validate:"dive"`

	// Access control rules for different policy namespaces
	once        sync.Once
	aclResolver *ACLResolver // lazily initialized value
}

// NewPolicy creates a new Policy
func NewPolicy() *Policy {
	return &Policy{Namespace: make(map[string]*PolicyNamespace)}
}

// View returns a policy view object, which allows to make all policy operations on behalf of a certain user
// Policy view object will enforce all ACLs, allowing the user to only perform actions which he is allowed to perform
// All ACL rules should be loaded and added to the policy before this method gets called
func (policy *Policy) View(user *User) *PolicyView {
	policy.once.Do(func() {
		systemNamespace := policy.Namespace[runtime.SystemNS]
		if systemNamespace != nil {
			policy.aclResolver = NewACLResolver(systemNamespace.ACLRules)
		} else {
			policy.aclResolver = NewACLResolver(NewGlobalRules())
		}
	})
	return NewPolicyView(policy, user)
}

// AddObject adds a given object into the policy. When you add objects to the policy, they get added to the corresponding
// Namespace. If error occurs (e.g. object has an unknown kind) then the error will be returned
func (policy *Policy) AddObject(obj Base) error {
	policyNamespace, ok := policy.Namespace[obj.GetNamespace()]
	if !ok {
		policyNamespace = NewPolicyNamespace(obj.GetNamespace())
		policy.Namespace[obj.GetNamespace()] = policyNamespace
	}
	return policyNamespace.addObject(obj)
}

// RemoveObject removes a given object from the policy. Returns true if removed and false if nothing got removed.
func (policy *Policy) RemoveObject(obj Base) bool {
	policyNamespace, ok := policy.Namespace[obj.GetNamespace()]
	if !ok {
		return false
	}

	return policyNamespace.removeObject(obj)
}

// GetObjectsByKind returns all objects in a policy with a given kind, across all namespaces
func (policy *Policy) GetObjectsByKind(kind string) []Base {
	result := []Base{}
	for _, policyNS := range policy.Namespace {
		result = append(result, policyNS.getObjectsByKind(kind)...)
	}
	return result
}

// GetObject looks up and returns an object from the policy, given its kind, locator ([namespace/]name), and current
// namespace relative to which the call is being made
func (policy *Policy) GetObject(kind string, locator string, currentNs string) (runtime.Object, error) {
	// parse locator: [namespace/]name. we might add [domain/] in the future
	parts := strings.Split(locator, "/")
	var ns, name string
	if len(parts) == 1 {
		ns = currentNs
		name = parts[0]
	} else if len(parts) == 2 {
		ns = parts[0]
		name = parts[1]
	} else {
		return nil, fmt.Errorf("can't parse policy object locator: '%s'", locator)
	}

	policyNS, ok := policy.Namespace[ns]
	if !ok {
		return nil, fmt.Errorf("namespace '%s' doesn't exist, but referenced in locator '%s'", ns, locator)
	}

	return policyNS.getObject(kind, name)
}

// Validate performs validation of the entire policy, making sure that all of its objects are well-formed.
// It also checks that all cross-object references are valid. If policy is malformed, then a list of errors is returned.
// Otherwise, if policy is correctly formed, then nil is returned.
// The resulting error can be caster to (validator.ValidationErrors) and iterated over, to get the full list of errors.
func (policy *Policy) Validate() error {
	return NewPolicyValidator(policy).Validate()
}
<|endoftext|>


<commit_before>package store

import (
	"bytes"
	"container/list"
	"gob"
	"junta/util"
	"math"
	"os"
	"regexp"
	"strings"
)

const (
	Clobber = ""
	Missing = "0"
	Dir = "dir"
)

var (
	ErrBadPath = os.NewError("bad path")
	ErrBadMutation = os.NewError("bad mutation")
	ErrBadSnapshot = os.NewError("bad snapshot")
	ErrTooLate = os.NewError("too late")
	ErrCasMismatch = os.NewError("cas mismatch")
)

var waitRegexp = regexp.MustCompile(``)

type Store struct {
	applyCh chan apply
	watchCh chan watch
	watches []watch
	todo map[uint64]apply
	state *state
}

type apply struct {
	seqn uint64
	mutation string
}

type lookup struct {
	k string
	ch chan int
	v []string
	cas string
}

type state struct {
	ver uint64
	root node
}

type watch struct {
	pat string
	in, out chan Event
	re *regexp.Regexp
	stop uint64
}

// Creates a new, empty data store. Mutations will be applied in order,
// starting at number 1 (number 0 can be thought of as the creation of the
// store).
func New() *Store {
	s := &Store{
		applyCh: make(chan apply),
		watchCh: make(chan watch),
		todo: make(map[uint64]apply),
		watches: []watch{},
		state: &state{0, emptyDir},
	}

	go s.process()
	return s
}

func split(path string) []string {
	if path == "/" {
		return []string{}
	}
	return strings.Split(path[1:], "/", -1)
}

func join(parts []string) string {
	return "/" + strings.Join(parts, "/")
}

func checkPath(k string) os.Error {
	switch {
	case len(k) < 1,
	     k[0] != '/',
	     len(k) > 1 && k[len(k) - 1] == '/',
	     strings.Count(k, "=") > 0,
	     strings.Count(k, " ") > 0:
		return ErrBadPath
	}
	return nil
}

// Returns a mutation that can be applied to a `Store`. The mutation will set
// the contents of the file at `path` to `body` iff the CAS token of that file
// matches `cas` at the time of application.
//
// If `path` is not valid, returns `ErrBadPath`.
func EncodeSet(path, body string, cas string) (mutation string, err os.Error) {
	if err = checkPath(path); err != nil {
		return
	}
	return cas + ":" + path + "=" + body, nil
}

// Returns a mutation that can be applied to a `Store`. The mutation will cause
// the file at `path` to be deleted iff the CAS token of that file matches
// `cas` at the time of application.
//
// If `path` is not valid, returns `ErrBadPath`.
func EncodeDel(path string, cas string) (mutation string, err os.Error) {
	if err := checkPath(path); err != nil {
		return
	}
	return cas + ":" + path, nil
}

// MustEncodeSet is like EncodeSet but panics if the mutation cannot be
// encoded. It simplifies safe initialization of global variables holding
// mutations.
func MustEncodeSet(path, body, cas string) (mutation string) {
	m, err := EncodeSet(path, body, cas)
	if err != nil {
		panic(err)
	}
	return m
}

// MustEncodeDel is like EncodeDel but panics if the mutation cannot be
// encoded. It simplifies safe initialization of global variables holding
// mutations.
func MustEncodeDel(path, cas string) (mutation string) {
	m, err := EncodeDel(path, cas)
	if err != nil {
		panic(err)
	}
	return m
}

func decode(mutation string) (path, v, cas string, keep bool, err os.Error) {
	cm := strings.Split(mutation, ":", 2)

	if len(cm) != 2 {
		err = ErrBadMutation
		return
	}

	kv := strings.Split(cm[1], "=", 2)

	if err = checkPath(kv[0]); err != nil {
		return
	}

	switch len(kv) {
	case 1:
		return kv[0], "", cm[0], false, nil
	case 2:
		return kv[0], kv[1], cm[0], true, nil
	}
	panic("unreachable")
}

func (s *Store) notify(e Event) {
	for _, w := range s.watches {
		if w.re.MatchString(e.Path) {
			w.in <- e
		}
	}
}

func append(ws *[]watch, w watch) {
	l := len(*ws)
	if l + 1 > cap(*ws) {
		ns := make([]watch, (l + 1)*2)
		copy(ns, *ws)
		*ws = ns
	}
	*ws = (*ws)[0:l + 1]
	(*ws)[l] = w
}

// Unbounded in-order buffering
func buffer(in, out chan Event) {
	list := list.New()
	for {
		f, e := list.Front(), Event{}
		var ch chan Event
		if f != nil {
			e = f.Value.(Event)
			ch = out
		}
		select {
		case x := <-in:
			if closed(in) {
				return
			}
			list.PushBack(x)
		case ch <- e:
			list.Remove(f)
		}
	}
}

func (s *Store) process() {
	logger := util.NewLogger("store")

	for {
		ver, values := s.state.ver, s.state.root

		// Take any incoming requests and queue them up.
		select {
		case a := <-s.applyCh:
			if a.seqn > ver {
				s.todo[a.seqn] = a
			}
		case w := <-s.watchCh:
			if w.stop > ver {
				append(&s.watches, w)
			} else {
				w.in <- Event{Seqn:w.stop, Err:ErrTooLate}
				close(w.in)
			}
		}

		// If we have any mutations that can be applied, do them.
		for t, ok := s.todo[ver+1]; ok; t, ok = s.todo[ver+1] {
			var ev Event
			values, ev = values.apply(t.seqn, t.mutation)
			logger.Logf("apply %s %v %v %v %v %v", ev.Desc(), ev.Seqn, ev.Path, ev.Body, ev.Cas, ev.Err)
			s.state = &state{ev.Seqn, values}
			s.notify(ev)
			for ver < ev.Seqn {
				ver++
				s.todo[ver] = apply{}, false
			}
		}
	}
}

// Applies `mutation` in sequence at position `seqn`. Any error that occurs
// will be written to `ErrorPath`. If a mutation has already been applied at
// this position, this one is sliently ignored.
//
// If `mutation` is a snapshot, notifications will not be sent.
//
// If `mutation` is `Nop`, no change will be made, but a dummy event will still
// be sent.
func (s *Store) Apply(seqn uint64, mutation string) {
	s.applyCh <- apply{seqn, mutation}
}

// Gets the value stored at `path`, if any.
//
// If no value is stored at `path`, `cas` will be `Missing` and `value` will be
// nil.
//
// if `path` is a directory, `cas` will be `Dir` and `value` will be a list of
// entries.
//
// Otherwise, `cas` is the CAS token and `value[0]` is the body.
func (s *Store) Lookup(path string) (value []string, cas string) {
	// WARNING: Be sure to read the pointer value of s.state only once. If you
	// need multiple accesses, copy the pointer first.
	return s.state.root.getp(path)
}

// Retrieves the body stored at `path` and returns it. If `path` is a directory
// or does not exist, returns an empty string.
//
// Note, with this function it is impossible to distinguish between an empty
// string stored at `path`, a missing entry, and a directory. If you need to
// tell the difference, use `Lookup`.
//
// Also note, this function does not return the CAS token for `path`. If you
// need the CAS token, use `Lookup`.
func (s *Store) LookupString(path string) (body string) {
	v, cas := s.Lookup(path)
	if cas == Missing || cas == Dir {
		return ""
	}
	return v[0]
}

// Returns a list of entries in the directory at `path`. If `path` is not a
// directory, returns an empty slice.
//
// Note, with this function it is impossible to distinguish between a string
// stored at `path` and a missing entry. If you need to tell the difference,
// use `Lookup`.
func (s *Store) LookupDir(path string) (entries []string) {
	v, cas := s.Lookup(path)
	if cas != Dir {
		return nil
	}
	return v
}

// Encodes the entire storage state, including the current sequence number, as
// a mutation. This mutation can be applied to an empty store to reproduce the
// state of `s`.
//
// Returns the sequence number of the snapshot and the mutation itself.
//
// A snapshot must be applied at sequence number 1. Once a snapshot has been
// applied, the store's sequence number will be set to `seqn`.
//
// Note that applying a snapshot does not send notifications.
func (s *Store) Snapshot() (seqn uint64, mutation string) {
	w := new(bytes.Buffer)

	// WARNING: Be sure to read the pointer value of s.state only once. If you
	// need multiple accesses, copy the pointer first.
	ss := s.state

	err := gob.NewEncoder(w).Encode(ss.ver)
	if err != nil {
		panic(err)
	}

	err = gob.NewEncoder(w).Encode(ss.root)
	if err != nil {
		panic(err)
	}

	return ss.ver, w.String()
}

// Subscribes `ch` to receive notifications when mutations are applied to paths
// in the store. One event will be sent for each mutation iff the event's path
// matches `pattern`, a Unix-style glob pattern.
//
// Glob notation:
//  - "?" matches a single char in a single path component
//  - "*" matches zero or more chars in a single path component
//  - "**" matches zero or more chars in zero or more components
//  - any other sequence matches itself
//
// Notifications will not be sent for changes made as the result of applying a
// snapshot.
func (s *Store) Watch(pattern string, ch chan Event) {
	re, _ := compileGlob(pattern)
	in := make(chan Event)
	go buffer(in, ch)
	s.watchCh <- watch{pat:pattern, out:ch, in:in, re:re, stop:math.MaxUint64}
}

// Subscribes `ch` to receive a single event representing the change made at
// position `seqn`.
//
// If `seqn` was applied before the call to `Wait`, a dummy event will be
// sent with its `Err` set to `ErrTooLate`.
func (s *Store) Wait(seqn uint64, ch chan Event) {
	all := make(chan Event)
	s.watchCh <- watch{in:all, re:waitRegexp, stop:seqn}
	go func() {
		for e := range all {
			if e.Seqn == seqn {
				close(all)
				ch <- e
			}
		}
	}()
}

// Ensures that the application of mutation at `seqn` happens before the call
// to `Sync` returns.
func (st *Store) Sync(seqn uint64) {
	ch := make(chan Event)
	st.Wait(seqn, ch)
	<-ch
}
<commit_msg>rip out dead code<commit_after>package store

import (
	"bytes"
	"container/list"
	"gob"
	"junta/util"
	"math"
	"os"
	"regexp"
	"strings"
)

const (
	Clobber = ""
	Missing = "0"
	Dir = "dir"
)

var (
	ErrBadPath = os.NewError("bad path")
	ErrBadMutation = os.NewError("bad mutation")
	ErrBadSnapshot = os.NewError("bad snapshot")
	ErrTooLate = os.NewError("too late")
	ErrCasMismatch = os.NewError("cas mismatch")
)

var waitRegexp = regexp.MustCompile(``)

type Store struct {
	applyCh chan apply
	watchCh chan watch
	watches []watch
	todo map[uint64]apply
	state *state
}

type apply struct {
	seqn uint64
	mutation string
}

type state struct {
	ver uint64
	root node
}

type watch struct {
	pat string
	in, out chan Event
	re *regexp.Regexp
	stop uint64
}

// Creates a new, empty data store. Mutations will be applied in order,
// starting at number 1 (number 0 can be thought of as the creation of the
// store).
func New() *Store {
	s := &Store{
		applyCh: make(chan apply),
		watchCh: make(chan watch),
		todo: make(map[uint64]apply),
		watches: []watch{},
		state: &state{0, emptyDir},
	}

	go s.process()
	return s
}

func split(path string) []string {
	if path == "/" {
		return []string{}
	}
	return strings.Split(path[1:], "/", -1)
}

func join(parts []string) string {
	return "/" + strings.Join(parts, "/")
}

func checkPath(k string) os.Error {
	switch {
	case len(k) < 1,
	     k[0] != '/',
	     len(k) > 1 && k[len(k) - 1] == '/',
	     strings.Count(k, "=") > 0,
	     strings.Count(k, " ") > 0:
		return ErrBadPath
	}
	return nil
}

// Returns a mutation that can be applied to a `Store`. The mutation will set
// the contents of the file at `path` to `body` iff the CAS token of that file
// matches `cas` at the time of application.
//
// If `path` is not valid, returns `ErrBadPath`.
func EncodeSet(path, body string, cas string) (mutation string, err os.Error) {
	if err = checkPath(path); err != nil {
		return
	}
	return cas + ":" + path + "=" + body, nil
}

// Returns a mutation that can be applied to a `Store`. The mutation will cause
// the file at `path` to be deleted iff the CAS token of that file matches
// `cas` at the time of application.
//
// If `path` is not valid, returns `ErrBadPath`.
func EncodeDel(path string, cas string) (mutation string, err os.Error) {
	if err := checkPath(path); err != nil {
		return
	}
	return cas + ":" + path, nil
}

// MustEncodeSet is like EncodeSet but panics if the mutation cannot be
// encoded. It simplifies safe initialization of global variables holding
// mutations.
func MustEncodeSet(path, body, cas string) (mutation string) {
	m, err := EncodeSet(path, body, cas)
	if err != nil {
		panic(err)
	}
	return m
}

// MustEncodeDel is like EncodeDel but panics if the mutation cannot be
// encoded. It simplifies safe initialization of global variables holding
// mutations.
func MustEncodeDel(path, cas string) (mutation string) {
	m, err := EncodeDel(path, cas)
	if err != nil {
		panic(err)
	}
	return m
}

func decode(mutation string) (path, v, cas string, keep bool, err os.Error) {
	cm := strings.Split(mutation, ":", 2)

	if len(cm) != 2 {
		err = ErrBadMutation
		return
	}

	kv := strings.Split(cm[1], "=", 2)

	if err = checkPath(kv[0]); err != nil {
		return
	}

	switch len(kv) {
	case 1:
		return kv[0], "", cm[0], false, nil
	case 2:
		return kv[0], kv[1], cm[0], true, nil
	}
	panic("unreachable")
}

func (s *Store) notify(e Event) {
	for _, w := range s.watches {
		if w.re.MatchString(e.Path) {
			w.in <- e
		}
	}
}

func append(ws *[]watch, w watch) {
	l := len(*ws)
	if l + 1 > cap(*ws) {
		ns := make([]watch, (l + 1)*2)
		copy(ns, *ws)
		*ws = ns
	}
	*ws = (*ws)[0:l + 1]
	(*ws)[l] = w
}

// Unbounded in-order buffering
func buffer(in, out chan Event) {
	list := list.New()
	for {
		f, e := list.Front(), Event{}
		var ch chan Event
		if f != nil {
			e = f.Value.(Event)
			ch = out
		}
		select {
		case x := <-in:
			if closed(in) {
				return
			}
			list.PushBack(x)
		case ch <- e:
			list.Remove(f)
		}
	}
}

func (s *Store) process() {
	logger := util.NewLogger("store")

	for {
		ver, values := s.state.ver, s.state.root

		// Take any incoming requests and queue them up.
		select {
		case a := <-s.applyCh:
			if a.seqn > ver {
				s.todo[a.seqn] = a
			}
		case w := <-s.watchCh:
			if w.stop > ver {
				append(&s.watches, w)
			} else {
				w.in <- Event{Seqn:w.stop, Err:ErrTooLate}
				close(w.in)
			}
		}

		// If we have any mutations that can be applied, do them.
		for t, ok := s.todo[ver+1]; ok; t, ok = s.todo[ver+1] {
			var ev Event
			values, ev = values.apply(t.seqn, t.mutation)
			logger.Logf("apply %s %v %v %v %v %v", ev.Desc(), ev.Seqn, ev.Path, ev.Body, ev.Cas, ev.Err)
			s.state = &state{ev.Seqn, values}
			s.notify(ev)
			for ver < ev.Seqn {
				ver++
				s.todo[ver] = apply{}, false
			}
		}
	}
}

// Applies `mutation` in sequence at position `seqn`. Any error that occurs
// will be written to `ErrorPath`. If a mutation has already been applied at
// this position, this one is sliently ignored.
//
// If `mutation` is a snapshot, notifications will not be sent.
//
// If `mutation` is `Nop`, no change will be made, but a dummy event will still
// be sent.
func (s *Store) Apply(seqn uint64, mutation string) {
	s.applyCh <- apply{seqn, mutation}
}

// Gets the value stored at `path`, if any.
//
// If no value is stored at `path`, `cas` will be `Missing` and `value` will be
// nil.
//
// if `path` is a directory, `cas` will be `Dir` and `value` will be a list of
// entries.
//
// Otherwise, `cas` is the CAS token and `value[0]` is the body.
func (s *Store) Lookup(path string) (value []string, cas string) {
	// WARNING: Be sure to read the pointer value of s.state only once. If you
	// need multiple accesses, copy the pointer first.
	return s.state.root.getp(path)
}

// Retrieves the body stored at `path` and returns it. If `path` is a directory
// or does not exist, returns an empty string.
//
// Note, with this function it is impossible to distinguish between an empty
// string stored at `path`, a missing entry, and a directory. If you need to
// tell the difference, use `Lookup`.
//
// Also note, this function does not return the CAS token for `path`. If you
// need the CAS token, use `Lookup`.
func (s *Store) LookupString(path string) (body string) {
	v, cas := s.Lookup(path)
	if cas == Missing || cas == Dir {
		return ""
	}
	return v[0]
}

// Returns a list of entries in the directory at `path`. If `path` is not a
// directory, returns an empty slice.
//
// Note, with this function it is impossible to distinguish between a string
// stored at `path` and a missing entry. If you need to tell the difference,
// use `Lookup`.
func (s *Store) LookupDir(path string) (entries []string) {
	v, cas := s.Lookup(path)
	if cas != Dir {
		return nil
	}
	return v
}

// Encodes the entire storage state, including the current sequence number, as
// a mutation. This mutation can be applied to an empty store to reproduce the
// state of `s`.
//
// Returns the sequence number of the snapshot and the mutation itself.
//
// A snapshot must be applied at sequence number 1. Once a snapshot has been
// applied, the store's sequence number will be set to `seqn`.
//
// Note that applying a snapshot does not send notifications.
func (s *Store) Snapshot() (seqn uint64, mutation string) {
	w := new(bytes.Buffer)

	// WARNING: Be sure to read the pointer value of s.state only once. If you
	// need multiple accesses, copy the pointer first.
	ss := s.state

	err := gob.NewEncoder(w).Encode(ss.ver)
	if err != nil {
		panic(err)
	}

	err = gob.NewEncoder(w).Encode(ss.root)
	if err != nil {
		panic(err)
	}

	return ss.ver, w.String()
}

// Subscribes `ch` to receive notifications when mutations are applied to paths
// in the store. One event will be sent for each mutation iff the event's path
// matches `pattern`, a Unix-style glob pattern.
//
// Glob notation:
//  - "?" matches a single char in a single path component
//  - "*" matches zero or more chars in a single path component
//  - "**" matches zero or more chars in zero or more components
//  - any other sequence matches itself
//
// Notifications will not be sent for changes made as the result of applying a
// snapshot.
func (s *Store) Watch(pattern string, ch chan Event) {
	re, _ := compileGlob(pattern)
	in := make(chan Event)
	go buffer(in, ch)
	s.watchCh <- watch{pat:pattern, out:ch, in:in, re:re, stop:math.MaxUint64}
}

// Subscribes `ch` to receive a single event representing the change made at
// position `seqn`.
//
// If `seqn` was applied before the call to `Wait`, a dummy event will be
// sent with its `Err` set to `ErrTooLate`.
func (s *Store) Wait(seqn uint64, ch chan Event) {
	all := make(chan Event)
	s.watchCh <- watch{in:all, re:waitRegexp, stop:seqn}
	go func() {
		for e := range all {
			if e.Seqn == seqn {
				close(all)
				ch <- e
			}
		}
	}()
}

// Ensures that the application of mutation at `seqn` happens before the call
// to `Sync` returns.
func (st *Store) Sync(seqn uint64) {
	ch := make(chan Event)
	st.Wait(seqn, ch)
	<-ch
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"bytes"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"io/ioutil"
	"math/big"
	"net"
	"os"
	"path/filepath"
	"time"

	"github.com/pkg/errors"
)

func GenerateCACert(certPath, keyPath string, name string) error {
	priv, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return errors.Wrap(err, "Error generating rsa key")
	}

	template := x509.Certificate{
		SerialNumber: big.NewInt(1),
		Subject: pkix.Name{
			CommonName: name,
		},
		NotBefore: time.Now(),
		NotAfter:  time.Now().Add(time.Hour * 24 * 365 * 10),

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
		IsCA: true,
	}

	return writeCertsAndKeys(&template, certPath, priv, keyPath, &template, priv)
}

// You may also specify additional subject alt names (either ip or dns names) for the certificate
// The certificate will be created with file mode 0644. The key will be created with file mode 0600.
// If the certificate or key files already exist, they will be overwritten.
// Any parent directories of the certPath or keyPath will be created as needed with file mode 0755.
func GenerateSignedCert(certPath, keyPath, cn string, ips []net.IP, alternateDNS []string, signerCertPath, signerKeyPath string) error {
	signerCertBytes, err := ioutil.ReadFile(signerCertPath)
	if err != nil {
		return errors.Wrap(err, "Error reading file: signerCertPath")
	}
	decodedSignerCert, _ := pem.Decode(signerCertBytes)
	if decodedSignerCert == nil {
		return errors.New("Unable to decode certificate.")
	}
	signerCert, err := x509.ParseCertificate(decodedSignerCert.Bytes)
	if err != nil {
		return errors.Wrap(err, "Error parsing certificate: decodedSignerCert.Bytes")
	}
	signerKeyBytes, err := ioutil.ReadFile(signerKeyPath)
	if err != nil {
		return errors.Wrap(err, "Error reading file: signerKeyPath")
	}
	decodedSignerKey, _ := pem.Decode(signerKeyBytes)
	if decodedSignerKey == nil {
		return errors.New("Unable to decode key.")
	}
	signerKey, err := x509.ParsePKCS1PrivateKey(decodedSignerKey.Bytes)
	if err != nil {
		return errors.Wrap(err, "Error parsing prive key: decodedSignerKey.Bytes")
	}

	template := x509.Certificate{
		SerialNumber: big.NewInt(2),
		Subject: pkix.Name{
			CommonName:   cn,
			Organization: []string{"system:masters"},
		},
		NotBefore: time.Now(),
		NotAfter:  time.Now().Add(time.Hour * 24 * 365),

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
		BasicConstraintsValid: true,
	}

	template.IPAddresses = append(template.IPAddresses, ips...)
	template.DNSNames = append(template.DNSNames, alternateDNS...)

	priv, err := loadOrGeneratePrivateKey(keyPath)
	if err != nil {
		return errors.Wrap(err, "Error loading or generating private key: keyPath")
	}

	return writeCertsAndKeys(&template, certPath, priv, keyPath, signerCert, signerKey)
}

func loadOrGeneratePrivateKey(keyPath string) (*rsa.PrivateKey, error) {
	keyBytes, err := ioutil.ReadFile(keyPath)
	if err == nil {
		decodedKey, _ := pem.Decode(keyBytes)
		if decodedKey != nil {
			priv, err := x509.ParsePKCS1PrivateKey(decodedKey.Bytes)
			if err == nil {
				return priv, nil
			}
		}
	}
	priv, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, errors.Wrap(err, "Error generating RSA key")
	}
	return priv, nil
}

func writeCertsAndKeys(template *x509.Certificate, certPath string, signeeKey *rsa.PrivateKey, keyPath string, parent *x509.Certificate, signingKey *rsa.PrivateKey) error {
	derBytes, err := x509.CreateCertificate(rand.Reader, template, parent, &signeeKey.PublicKey, signingKey)
	if err != nil {
		return errors.Wrap(err, "Error creating certificate")
	}

	certBuffer := bytes.Buffer{}
	if err := pem.Encode(&certBuffer, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
		return errors.Wrap(err, "Error encoding certificate")
	}

	keyBuffer := bytes.Buffer{}
	if err := pem.Encode(&keyBuffer, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(signeeKey)}); err != nil {
		return errors.Wrap(err, "Error encoding key")
	}

	if err := os.MkdirAll(filepath.Dir(certPath), os.FileMode(0755)); err != nil {
		return errors.Wrap(err, "Error creating certificate directory")
	}
	if err := ioutil.WriteFile(certPath, certBuffer.Bytes(), os.FileMode(0644)); err != nil {
		return errors.Wrap(err, "Error writing certificate to cert path")
	}

	if err := os.MkdirAll(filepath.Dir(keyPath), os.FileMode(0755)); err != nil {
		return errors.Wrap(err, "Error creating key directory")
	}
	if err := ioutil.WriteFile(keyPath, keyBuffer.Bytes(), os.FileMode(0600)); err != nil {
		return errors.Wrap(err, "Error writing key file")
	}

	return nil
}
<commit_msg>Issue certificates with NotBefore set 1 day prior to avoid drift/timezone issues in VM<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"bytes"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"io/ioutil"
	"math/big"
	"net"
	"os"
	"path/filepath"
	"time"

	"github.com/pkg/errors"
)

func GenerateCACert(certPath, keyPath string, name string) error {
	priv, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return errors.Wrap(err, "Error generating rsa key")
	}

	template := x509.Certificate{
		SerialNumber: big.NewInt(1),
		Subject: pkix.Name{
			CommonName: name,
		},
		NotBefore: time.Now().Add(time.Hour * -24),
		NotAfter:  time.Now().Add(time.Hour * 24 * 365 * 10),

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
		IsCA: true,
	}

	return writeCertsAndKeys(&template, certPath, priv, keyPath, &template, priv)
}

// You may also specify additional subject alt names (either ip or dns names) for the certificate
// The certificate will be created with file mode 0644. The key will be created with file mode 0600.
// If the certificate or key files already exist, they will be overwritten.
// Any parent directories of the certPath or keyPath will be created as needed with file mode 0755.
func GenerateSignedCert(certPath, keyPath, cn string, ips []net.IP, alternateDNS []string, signerCertPath, signerKeyPath string) error {
	signerCertBytes, err := ioutil.ReadFile(signerCertPath)
	if err != nil {
		return errors.Wrap(err, "Error reading file: signerCertPath")
	}
	decodedSignerCert, _ := pem.Decode(signerCertBytes)
	if decodedSignerCert == nil {
		return errors.New("Unable to decode certificate.")
	}
	signerCert, err := x509.ParseCertificate(decodedSignerCert.Bytes)
	if err != nil {
		return errors.Wrap(err, "Error parsing certificate: decodedSignerCert.Bytes")
	}
	signerKeyBytes, err := ioutil.ReadFile(signerKeyPath)
	if err != nil {
		return errors.Wrap(err, "Error reading file: signerKeyPath")
	}
	decodedSignerKey, _ := pem.Decode(signerKeyBytes)
	if decodedSignerKey == nil {
		return errors.New("Unable to decode key.")
	}
	signerKey, err := x509.ParsePKCS1PrivateKey(decodedSignerKey.Bytes)
	if err != nil {
		return errors.Wrap(err, "Error parsing prive key: decodedSignerKey.Bytes")
	}

	template := x509.Certificate{
		SerialNumber: big.NewInt(2),
		Subject: pkix.Name{
			CommonName:   cn,
			Organization: []string{"system:masters"},
		},
		NotBefore: time.Now().Add(time.Hour * -24),
		NotAfter:  time.Now().Add(time.Hour * 24 * 365),

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
		BasicConstraintsValid: true,
	}

	template.IPAddresses = append(template.IPAddresses, ips...)
	template.DNSNames = append(template.DNSNames, alternateDNS...)

	priv, err := loadOrGeneratePrivateKey(keyPath)
	if err != nil {
		return errors.Wrap(err, "Error loading or generating private key: keyPath")
	}

	return writeCertsAndKeys(&template, certPath, priv, keyPath, signerCert, signerKey)
}

func loadOrGeneratePrivateKey(keyPath string) (*rsa.PrivateKey, error) {
	keyBytes, err := ioutil.ReadFile(keyPath)
	if err == nil {
		decodedKey, _ := pem.Decode(keyBytes)
		if decodedKey != nil {
			priv, err := x509.ParsePKCS1PrivateKey(decodedKey.Bytes)
			if err == nil {
				return priv, nil
			}
		}
	}
	priv, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, errors.Wrap(err, "Error generating RSA key")
	}
	return priv, nil
}

func writeCertsAndKeys(template *x509.Certificate, certPath string, signeeKey *rsa.PrivateKey, keyPath string, parent *x509.Certificate, signingKey *rsa.PrivateKey) error {
	derBytes, err := x509.CreateCertificate(rand.Reader, template, parent, &signeeKey.PublicKey, signingKey)
	if err != nil {
		return errors.Wrap(err, "Error creating certificate")
	}

	certBuffer := bytes.Buffer{}
	if err := pem.Encode(&certBuffer, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
		return errors.Wrap(err, "Error encoding certificate")
	}

	keyBuffer := bytes.Buffer{}
	if err := pem.Encode(&keyBuffer, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(signeeKey)}); err != nil {
		return errors.Wrap(err, "Error encoding key")
	}

	if err := os.MkdirAll(filepath.Dir(certPath), os.FileMode(0755)); err != nil {
		return errors.Wrap(err, "Error creating certificate directory")
	}
	if err := ioutil.WriteFile(certPath, certBuffer.Bytes(), os.FileMode(0644)); err != nil {
		return errors.Wrap(err, "Error writing certificate to cert path")
	}

	if err := os.MkdirAll(filepath.Dir(keyPath), os.FileMode(0755)); err != nil {
		return errors.Wrap(err, "Error creating key directory")
	}
	if err := ioutil.WriteFile(keyPath, keyBuffer.Bytes(), os.FileMode(0600)); err != nil {
		return errors.Wrap(err, "Error writing key file")
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017-2020 The Usacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package util

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"os"
)

func MarshalJSONFromPathOrContent(pathOrContent string, destination interface{}) error {
	// 今の所JSONのみ対応
	data, err := BytesFromPathOrContent(pathOrContent)
	if err != nil {
		return err
	}
	return json.Unmarshal(data, destination)
}

func StringFromPathOrContent(pathOrContent string) (string, error) {
	data, err := BytesFromPathOrContent(pathOrContent)
	if err != nil {
		return "", err
	}
	return string(data), nil
}

func BytesFromPathOrContent(pathOrContent string) ([]byte, error) {
	if pathOrContent == "" {
		return nil, errors.New("pathOrContent required")
	}
	file, err := os.Open(pathOrContent)
	if err != nil {
		if os.IsNotExist(err) {
			return []byte(pathOrContent), nil // そのまま返す
		}
		return nil, err
	}
	defer file.Close() // nolint

	data, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, err
	}
	return data, nil
}
<commit_msg>Check if pathOrContent is a valid JSON string first<commit_after>// Copyright 2017-2020 The Usacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package util

import (
	"encoding/json"
	"errors"
	"io/ioutil"
)

func MarshalJSONFromPathOrContent(pathOrContent string, destination interface{}) error {
	// 今の所JSONのみ対応
	data, err := BytesFromPathOrContent(pathOrContent)
	if err != nil {
		return err
	}
	return json.Unmarshal(data, destination)
}

func StringFromPathOrContent(pathOrContent string) (string, error) {
	data, err := BytesFromPathOrContent(pathOrContent)
	if err != nil {
		return "", err
	}
	return string(data), nil
}

func BytesFromPathOrContent(pathOrContent string) ([]byte, error) {
	if pathOrContent == "" {
		return nil, errors.New("pathOrContent required")
	}

	if isJSON(pathOrContent) {
		return []byte(pathOrContent), nil
	}

	data, err := ioutil.ReadFile(pathOrContent)
	if err != nil {
		return nil, err
	}
	return data, nil
}

func isJSON(s string)bool {
	m := make(map[string]interface{})
	return json.Unmarshal([]byte(s), &m) == nil
}<|endoftext|>


<commit_before>package webui

import (
	"encoding/json"
	"github.com/j-keck/lsleases/pkg/cscom"
	"github.com/j-keck/plog"
	"net/http"
	"strings"
)

var log = plog.GlobalLogger()

type endpoint struct {
	path string
	hndl func(http.ResponseWriter, *http.Request)
}

type WebUI struct {
	endpoints []endpoint
}

func NewWebUI() WebUI {
	self := new(WebUI)
	self.registerEndpoints()
	return *self
}

func (self *WebUI) ListenAndServe(addr string) {
	browserAddr := addr
	if strings.HasPrefix(browserAddr, ":") {
		browserAddr = "http://localhost" + browserAddr
	} else {
		browserAddr = "http://" + browserAddr
	}

	log.Infof("startup webui on address: %s - you can open the webui at: %s",
		addr, browserAddr)
	http.ListenAndServe(addr, nil)
}

func (self *WebUI) registerEndpoints() {
	// webui
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(index))
	})

	// version
	http.HandleFunc("/api/version", func(w http.ResponseWriter, r *http.Request) {
		if version, err := cscom.AskServer(cscom.GetVersion); err == nil {
			json, _ := json.Marshal(struct {
				V string `json:"version"`
			}{version.(cscom.Version).String()})
			w.Header().Set("Content-Type", "application/json")
			w.Write(json)
		} else {
			log.Warnf("unable to lookup server version: %v", err)
		}
	})

	// leases listing
	http.HandleFunc("/api/leases", func(w http.ResponseWriter, r *http.Request) {
		var leases cscom.Leases
		var err error
		if since := r.URL.Query().Get("since"); len(since) != 0 {
			var resp cscom.ServerResponse
			resp, err = cscom.AskServerWithPayload(
				cscom.GetLeasesSince,
				since,
			)
			leases = resp.(cscom.Leases)
		} else {
			var resp cscom.ServerResponse
			resp, err = cscom.AskServer(cscom.GetLeases)
			leases = resp.(cscom.Leases)
		}

		if err == nil {
			json, _ := json.Marshal(leases)
			w.Header().Set("Content-Type", "application/json")
			w.Write(json)
		} else {
			log.Warnf("unable to lookup leases: %v", err)
		}
	})

	// clear leases
	http.HandleFunc("/api/clear-leases", func(w http.ResponseWriter, r *http.Request) {
		cscom.TellServer(cscom.ClearLeases)
	})
}

var index = `
<!DOCTYPE html>

    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no">
    <title>lsleases WebUI</title>

Captured	IP	MAC	Hostname

lsleases

`
<commit_msg>log an error if the webui can't be started<commit_after>package webui

import (
	"encoding/json"
	"github.com/j-keck/lsleases/pkg/cscom"
	"github.com/j-keck/plog"
	"net/http"
	"strings"
)

var log = plog.GlobalLogger()

type endpoint struct {
	path string
	hndl func(http.ResponseWriter, *http.Request)
}

type WebUI struct {
	endpoints []endpoint
}

func NewWebUI() WebUI {
	self := new(WebUI)
	self.registerEndpoints()
	return *self
}

func (self *WebUI) ListenAndServe(addr string) {
	browserAddr := addr
	if strings.HasPrefix(browserAddr, ":") {
		browserAddr = "http://localhost" + browserAddr
	} else {
		browserAddr = "http://" + browserAddr
	}

	log.Infof("startup webui on address: %s - you can open the webui at: %s",
		addr, browserAddr)
	if err := http.ListenAndServe(addr, nil); err != nil {
		log.Warnf("unable to start webui: %v", err)
	}
}

func (self *WebUI) registerEndpoints() {
	// webui
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(index))
	})

	// version
	http.HandleFunc("/api/version", func(w http.ResponseWriter, r *http.Request) {
		if version, err := cscom.AskServer(cscom.GetVersion); err == nil {
			json, _ := json.Marshal(struct {
				V string `json:"version"`
			}{version.(cscom.Version).String()})
			w.Header().Set("Content-Type", "application/json")
			w.Write(json)
		} else {
			log.Warnf("unable to lookup server version: %v", err)
		}
	})

	// leases listing
	http.HandleFunc("/api/leases", func(w http.ResponseWriter, r *http.Request) {
		var leases cscom.Leases
		var err error
		if since := r.URL.Query().Get("since"); len(since) != 0 {
			var resp cscom.ServerResponse
			resp, err = cscom.AskServerWithPayload(
				cscom.GetLeasesSince,
				since,
			)
			leases = resp.(cscom.Leases)
		} else {
			var resp cscom.ServerResponse
			resp, err = cscom.AskServer(cscom.GetLeases)
			leases = resp.(cscom.Leases)
		}

		if err == nil {
			json, _ := json.Marshal(leases)
			w.Header().Set("Content-Type", "application/json")
			w.Write(json)
		} else {
			log.Warnf("unable to lookup leases: %v", err)
		}
	})

	// clear leases
	http.HandleFunc("/api/clear-leases", func(w http.ResponseWriter, r *http.Request) {
		cscom.TellServer(cscom.ClearLeases)
	})
}

var index = `
<!DOCTYPE html>

    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no">
    <title>lsleases WebUI</title>

Captured	IP	MAC	Hostname

lsleases

`
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

const (
	HELP = `md2pdf [-h] [-x] [-s] [-i dir] [-o file] file.md
Transform a given Markdown file into PDF.
-h        To print this help page.
-x        Print intermediate XHTML output.
-s        Print stylesheet used for transformation.
-a        Output article (instead of blog entry).
-i dir    To indicate image directory.
-o file   The name of the file to output.
file.md   The markdown file to convert.
Note:
This program calls pandoc, xsltproc and htmldoc that must have been installed.`
	STYLESHEET = `<?xml version="1.0" encoding="utf-8"?>

<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
                version="1.0">

  <xsl:output method="xml" encoding="ISO-8859-1"/>
  <xsl:param name="id">ID</xsl:param>
  <xsl:param name="date">DATE</xsl:param>
  <xsl:param name="title">TITLE</xsl:param>
  <xsl:param name="author">AUTHOR</xsl:param>
  <xsl:param name="email">EMAIL</xsl:param>
  <xsl:param name="lang">fr</xsl:param>
  <xsl:param name="toc">yes</xsl:param>

  <xsl:template match="/xhtml">
    <xsl:apply-templates/>
  </xsl:template>

  <xsl:template match="@*|node()">
    <xsl:copy>
      <xsl:apply-templates select="@*|node()"/>
    </xsl:copy>
  </xsl:template>

  <xsl:template match="table">

      <xsl:apply-templates/>

  </xsl:template>

  <xsl:template match="pre">

		    <xsl:apply-templates/>

  </xsl:template>

</xsl:stylesheet>`
	XHTML_HEADER = "<xhtml>\n\n"
	XHTML_FOOTER = "\n</xhtml>"
)

func processXsl(tmpFile string, data map[string]string) {
	xslFile, err := ioutil.TempFile("/tmp", "md2pdf-")
	if err != nil {
		panic(err)
	}
	err = ioutil.WriteFile(xslFile.Name(), []byte(STYLESHEET), 0644)
	if err != nil {
		panic(err)
	}
	defer os.Remove(xslFile.Name())
	params := make([]string, 0, 2+3*len(data))
	for name, value := range data {
		params = append(params, "--stringparam")
		params = append(params, name)
		params = append(params, value)
	}
	params = append(params, xslFile.Name())
	params = append(params, tmpFile)
	command := exec.Command("xsltproc", params...)
	result, err := command.CombinedOutput()
	if err != nil {
		println(result)
		panic(err)
	}
	err = ioutil.WriteFile(tmpFile, result, 0644)
	if err != nil {
		panic(err)
	}
}

func markdown2xhtml(markdown string) []byte {
	mdFile, err := ioutil.TempFile("/tmp", "md2xsl-")
	if err != nil {
		panic(err)
	}
	defer os.Remove(mdFile.Name())
	ioutil.WriteFile(mdFile.Name(), []byte(markdown), 0644)
	command := exec.Command("pandoc", mdFile.Name(), "-f", "markdown", "-t", "html")
	result, err := command.CombinedOutput()
	if err != nil {
		println(result)
		panic(err)
	}
	return []byte(XHTML_HEADER + string(result) + XHTML_FOOTER)
}

func markdownData(text string) (map[string]string, string) {
	data := make(map[string]string)
	lines := strings.Split(text, "\n")
	var limit int
	for index, line := range lines {
		if strings.HasPrefix(line, "% ") && strings.Index(line, ":") >= 0 {
			name := strings.TrimSpace(line[2:strings.Index(line, ":")])
			value := strings.TrimSpace(line[strings.Index(line, ":")+1 : len(line)])
			data[name] = value
		} else {
			limit = index
			break
		}
	}
	return data, strings.Join(lines[limit:len(lines)], "\n")
}

func imageDir(text, imgDir string) string {
	r := regexp.MustCompile(`!\[(.*?)\]\((.*?/)*(.*?)\)`)
	if len(imgDir) > 0 {
		return r.ReplaceAllString(text, "![$1]("+imgDir+"/$3)")
	} else {
		return r.ReplaceAllString(text, "![$1]($3)")
	}
}

func generatePdf(xhtmlFile, outFile string) {
	params := []string{
		"--outfile", outFile,
		"--size", "A4",
		"--top", "2cm",
		"--bottom", "2cm",
		"--left", "2cm",
		"--right", "2cm",
		"--bodyfont", "Times",
		"--fontsize", "12",
		"--header", "...",
		"--footer", "dt1",
		"--headfootfont", "Courier-Oblique",
		"--headfootsize", "10",
		"--linkcolor", "#0000A0",
		"--linkstyle", "plain",
		"--permissions", "no-modify",
		"--charset", "iso-8859-1",
		"--no-title",
		"--no-toc",
		"--compression=9",
		"--embedfonts",
		"--webpage",
		xhtmlFile,
	}
	command := exec.Command("htmldoc", params...)
	result, err := command.CombinedOutput()
	if err != nil {
		println(string(result))
		panic(err)
	}
}

func processFile(filename string, printXhtml bool, imgDir, outFile string) {
	source, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	data, markdown := markdownData(string(source))
	markdown = imageDir(markdown, imgDir)
	xhtml := markdown2xhtml(markdown)
	if printXhtml {
		fmt.Println(string(xhtml))
		return
	}
	tmpFile, err := ioutil.TempFile("/tmp", "md2pdf-")
	if err != nil {
		panic(err)
	}
	defer os.Remove(tmpFile.Name())
	ioutil.WriteFile(tmpFile.Name(), xhtml, 0644)
	processXsl(tmpFile.Name(), data)
	if len(outFile) == 0 {
		outFile = filename[0:len(filename)-len(filepath.Ext(filename))] + ".pdf"
	}
	generatePdf(tmpFile.Name(), outFile)
}

func main() {
	xhtml := false
	imgDir := ""
	outFile := ""
	file := ""
	if len(os.Args) < 2 {
		fmt.Println(HELP)
		os.Exit(1)
	}
	skip := false
	args := os.Args[1:]
	for i, arg := range args {
		if skip {
			skip = false
			continue
		}
		if arg == "-h" || arg == "--help" {
			fmt.Println(HELP)
			os.Exit(0)
		} else if arg == "-x" || arg == "--xhtml" {
			xhtml = true
		} else if arg == "-s" || arg == "--stylesheet" {
			fmt.Println(STYLESHEET)
			os.Exit(0)
		} else if arg == "-i" || arg == "--image-dir" {
			imgDir = args[i+1]
			skip = true
		} else if arg == "-o" || arg == "--output-file" {
			outFile = args[i+1]
			skip = true
		} else {
			file = arg
		}
	}
	processFile(file, xhtml, imgDir, outFile)
}
<commit_msg>Added title management<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

const (
	HELP = `md2pdf [-h] [-x] [-s] [-t] [-i dir] [-o file] file.md
Transform a given Markdown file into PDF.
-h        To print this help page.
-x        Print intermediate XHTML output.
-s        Print stylesheet used for transformation.
-t        Print html output.
-a        Output article (instead of blog entry).
-i dir    To indicate image directory.
-o file   The name of the file to output.
file.md   The markdown file to convert.
Note:
This program calls pandoc, xsltproc and htmldoc that must have been installed.`
	STYLESHEET = `<?xml version="1.0" encoding="utf-8"?>

<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
                version="1.0">

  <xsl:output method="xml" encoding="ISO-8859-1"/>
  <xsl:param name="id">ID</xsl:param>
  <xsl:param name="date">DATE</xsl:param>
  <xsl:param name="title">TITLE</xsl:param>
  <xsl:param name="author">AUTHOR</xsl:param>
  <xsl:param name="email">EMAIL</xsl:param>
  <xsl:param name="lang">fr</xsl:param>
  <xsl:param name="toc">yes</xsl:param>

  <xsl:template match="/xhtml">
    <xsl:if test="$title">

<xsl:value-of select="$title"/>

	</xsl:if>

      <xsl:if test="$author">
	    <xsl:value-of select="$author"/>

      </xsl:if>
	  <xsl:if test="$email">
		<xsl:value-of select="$email"/>
	  </xsl:if>

    <xsl:apply-templates/>
  </xsl:template>

  <xsl:template match="@*|node()">
    <xsl:copy>
      <xsl:apply-templates select="@*|node()"/>
    </xsl:copy>
  </xsl:template>

  <xsl:template match="table">

      <xsl:apply-templates/>

  </xsl:template>

  <xsl:template match="pre">

		    <xsl:apply-templates/>

  </xsl:template>

  <xsl:template match="div[@class='figure']">

	  <xsl:apply-templates />

  </xsl:template>

</xsl:stylesheet>`
	XHTML_HEADER = "<xhtml>\n\n"
	XHTML_FOOTER = "\n</xhtml>"
)

func processXsl(tmpFile string, data map[string]string) {
	xslFile, err := ioutil.TempFile("/tmp", "md2pdf-")
	if err != nil {
		panic(err)
	}
	err = ioutil.WriteFile(xslFile.Name(), []byte(STYLESHEET), 0644)
	if err != nil {
		panic(err)
	}
	defer os.Remove(xslFile.Name())
	params := make([]string, 0, 2+3*len(data))
	for name, value := range data {
		params = append(params, "--stringparam")
		params = append(params, name)
		params = append(params, value)
	}
	params = append(params, xslFile.Name())
	params = append(params, tmpFile)
	command := exec.Command("xsltproc", params...)
	result, err := command.CombinedOutput()
	if err != nil {
		println(result)
		panic(err)
	}
	err = ioutil.WriteFile(tmpFile, result, 0644)
	if err != nil {
		panic(err)
	}
}

func markdown2xhtml(markdown string) []byte {
	mdFile, err := ioutil.TempFile("/tmp", "md2xsl-")
	if err != nil {
		panic(err)
	}
	defer os.Remove(mdFile.Name())
	ioutil.WriteFile(mdFile.Name(), []byte(markdown), 0644)
	command := exec.Command("pandoc", mdFile.Name(), "-f", "markdown", "-t", "html")
	result, err := command.CombinedOutput()
	if err != nil {
		println(result)
		panic(err)
	}
	return []byte(XHTML_HEADER + string(result) + XHTML_FOOTER)
}

func markdownData(text string) (map[string]string, string) {
	data := make(map[string]string)
	lines := strings.Split(text, "\n")
	var limit int
	for index, line := range lines {
		if strings.HasPrefix(line, "% ") && strings.Index(line, ":") >= 0 {
			name := strings.TrimSpace(line[2:strings.Index(line, ":")])
			value := strings.TrimSpace(line[strings.Index(line, ":")+1 : len(line)])
			data[name] = value
		} else {
			limit = index
			break
		}
	}
	return data, strings.Join(lines[limit:len(lines)], "\n")
}

func imageDir(text, imgDir string) string {
	r := regexp.MustCompile(`!\[(.*?)\]\((.*?/)*(.*?)\)`)
	if len(imgDir) > 0 {
		return r.ReplaceAllString(text, "![$1]("+imgDir+"/$3)")
	} else {
		return r.ReplaceAllString(text, "![$1]($3)")
	}
}

func generatePdf(xhtmlFile, outFile string, data map[string]string) {
	toc := "--no-toc"
	if data["toc"] == "yes" || data["toc"] == "true" || data["toc"] == "1" {
		toc = "--no-toc"
	}
	params := []string{
		"--outfile", outFile,
		"--size", "A4",
		"--top", "2cm",
		"--bottom", "2cm",
		"--left", "2cm",
		"--right", "2cm",
		"--bodyfont", "Times",
		"--fontsize", "12",
		"--header", "...",
		"--footer", "dt1",
		"--headfootfont", "Courier-Oblique",
		"--headfootsize", "10",
		"--linkcolor", "#0000A0",
		"--linkstyle", "plain",
		"--permissions", "no-modify",
		"--charset", "iso-8859-1",
		"--no-title",
		toc,
		"--compression=9",
		"--embedfonts",
		"--webpage",
		xhtmlFile,
	}
	command := exec.Command("htmldoc", params...)
	result, err := command.CombinedOutput()
	if err != nil {
		println(string(result))
		panic(err)
	}
}

func processFile(filename string, printXhtml, printHtml bool, imgDir, outFile string) {
	source, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	data, markdown := markdownData(string(source))
	markdown = imageDir(markdown, imgDir)
	xhtml := markdown2xhtml(markdown)
	if printXhtml {
		fmt.Println(string(xhtml))
		return
	}
	tmpFile, err := ioutil.TempFile("/tmp", "md2pdf-")
	if err != nil {
		panic(err)
	}
	defer os.Remove(tmpFile.Name())
	ioutil.WriteFile(tmpFile.Name(), xhtml, 0644)
	processXsl(tmpFile.Name(), data)
	if printHtml {
		source, err := ioutil.ReadFile(tmpFile.Name())
		if err != nil {
			panic(err)
		}
		fmt.Println(string(source))
		return
	}
	if len(outFile) == 0 {
		outFile = filename[0:len(filename)-len(filepath.Ext(filename))] + ".pdf"
	}
	generatePdf(tmpFile.Name(), outFile, data)
}

func main() {
	xhtml := false
	html := false
	imgDir := ""
	outFile := ""
	file := ""
	if len(os.Args) < 2 {
		fmt.Println(HELP)
		os.Exit(1)
	}
	skip := false
	args := os.Args[1:]
	for i, arg := range args {
		if skip {
			skip = false
			continue
		}
		if arg == "-h" || arg == "--help" {
			fmt.Println(HELP)
			os.Exit(0)
		} else if arg == "-x" || arg == "--xhtml" {
			xhtml = true
		} else if arg == "-s" || arg == "--stylesheet" {
			fmt.Println(STYLESHEET)
			os.Exit(0)
		} else if arg == "-t" || arg == "--html" {
			html = true
		} else if arg == "-i" || arg == "--image-dir" {
			imgDir = args[i+1]
			skip = true
		} else if arg == "-o" || arg == "--output-file" {
			outFile = args[i+1]
			skip = true
		} else {
			file = arg
		}
	}
	processFile(file, xhtml, html, imgDir, outFile)
}
<|endoftext|>


<commit_before>// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package procfs

import (
	"fmt"
	"io/ioutil"
	"regexp"
	"strconv"
	"strings"
)

var (
	statusLineRE   = regexp.MustCompile(`(\d+) blocks .*\[(\d+)/(\d+)\] \[[U_]+\]`)
	recoveryLineRE = regexp.MustCompile(`\((\d+)/\d+\)`)
)

// MDStat holds info parsed from /proc/mdstat.
type MDStat struct {
	// Name of the device.
	Name string
	// activity-state of the device.
	ActivityState string
	// Number of active disks.
	DisksActive int64
	// Total number of disks the device requires.
	DisksTotal int64
	// Number of failed disks.
	DisksFailed int64
	// Spare disks in the device.
	DisksSpare int64
	// Number of blocks the device holds.
	BlocksTotal int64
	// Number of blocks on the device that are in sync.
	BlocksSynced int64
}

// MDStat parses an mdstat-file (/proc/mdstat) and returns a slice of
// structs containing the relevant info.  More information available here:
// https://raid.wiki.kernel.org/index.php/Mdstat
func (fs FS) MDStat() ([]MDStat, error) {
	data, err := ioutil.ReadFile(fs.proc.Path("mdstat"))
	if err != nil {
		return nil, fmt.Errorf("error parsing mdstat %s: %s", fs.proc.Path("mdstat"), err)
	}
	mdstat, err := parseMDStat(data)
	if err != nil {
		return nil, fmt.Errorf("error parsing mdstat %s: %s", fs.proc.Path("mdstat"), err)
	}
	return mdstat, nil
}

// parseMDStat parses data from mdstat file (/proc/mdstat) and returns a slice of
// structs containing the relevant info.
func parseMDStat(mdStatData []byte) ([]MDStat, error) {
	mdStats := []MDStat{}
	lines := strings.Split(string(mdStatData), "\n")

	for i, line := range lines {
		if strings.TrimSpace(line) == "" || line[0] == ' ' ||
			strings.HasPrefix(line, "Personalities") ||
			strings.HasPrefix(line, "unused") {
			continue
		}

		deviceFields := strings.Fields(line)
		if len(deviceFields) < 3 {
			return nil, fmt.Errorf("not enough fields in mdline (expected at least 3): %s", line)
		}
		mdName := deviceFields[0] // mdx
		state := deviceFields[2]  // active or inactive

		if len(lines) <= i+3 {
			return nil, fmt.Errorf(
				"error parsing %s: too few lines for md device",
				mdName,
			)
		}

		// Failed disks have the suffix (F) & Spare disks have the suffix (S).
		fail := int64(strings.Count(line, "(F)"))
		spare := int64(strings.Count(line, "(S)"))
		active, total, size, err := evalStatusLine(lines[i], lines[i+1])

		if err != nil {
			return nil, fmt.Errorf("error parsing md device lines: %s", err)
		}

		syncLineIdx := i + 2
		if strings.Contains(lines[i+2], "bitmap") { // skip bitmap line
			syncLineIdx++
		}

		// If device is syncing at the moment, get the number of currently
		// synced bytes, otherwise that number equals the size of the device.
		syncedBlocks := size
		recovering := strings.Contains(lines[syncLineIdx], "recovery")
		resyncing := strings.Contains(lines[syncLineIdx], "resync")

		// Append recovery and resyncing state info.
		if recovering || resyncing {
			if recovering {
				state = "recovering"
			} else {
				state = "resyncing"
			}

			// Handle case when resync=PENDING or resync=DELAYED.
			if strings.Contains(lines[syncLineIdx], "PENDING") ||
				strings.Contains(lines[syncLineIdx], "DELAYED") {
				syncedBlocks = 0
			} else {
				syncedBlocks, err = evalRecoveryLine(lines[syncLineIdx])
				if err != nil {
					return nil, fmt.Errorf("error parsing sync line in md device %s: %s", mdName, err)
				}
			}
		}

		mdStats = append(mdStats, MDStat{
			Name:          mdName,
			ActivityState: state,
			DisksActive:   active,
			DisksFailed:   fail,
			DisksSpare:    spare,
			DisksTotal:    total,
			BlocksTotal:   size,
			BlocksSynced:  syncedBlocks,
		})
	}

	return mdStats, nil
}

func evalStatusLine(deviceLine, statusLine string) (active, total, size int64, err error) {

	sizeStr := strings.Fields(statusLine)[0]
	size, err = strconv.ParseInt(sizeStr, 10, 64)
	if err != nil {
		return 0, 0, 0, fmt.Errorf("unexpected statusLine %s: %s", statusLine, err)
	}

	if strings.Contains(deviceLine, "raid0") || strings.Contains(deviceLine, "linear") {
		// In the device deviceLine, only disks have a number associated with them in [].
		total = int64(strings.Count(deviceLine, "["))
		return total, total, size, nil
	}

	if strings.Contains(deviceLine, "inactive") {
		return 0, 0, size, nil
	}

	matches := statusLineRE.FindStringSubmatch(statusLine)
	if len(matches) != 4 {
		return 0, 0, 0, fmt.Errorf("couldn't find all the substring matches: %s", statusLine)
	}

	total, err = strconv.ParseInt(matches[2], 10, 64)
	if err != nil {
		return 0, 0, 0, fmt.Errorf("unexpected statusLine %s: %s", statusLine, err)
	}

	active, err = strconv.ParseInt(matches[3], 10, 64)
	if err != nil {
		return 0, 0, 0, fmt.Errorf("unexpected statusLine %s: %s", statusLine, err)
	}

	return active, total, size, nil
}

func evalRecoveryLine(recoveryLine string) (syncedBlocks int64, err error) {
	matches := recoveryLineRE.FindStringSubmatch(recoveryLine)
	if len(matches) != 2 {
		return 0, fmt.Errorf("unexpected recoveryLine: %s", recoveryLine)
	}

	syncedBlocks, err = strconv.ParseInt(matches[1], 10, 64)
	if err != nil {
		return 0, fmt.Errorf("%s in recoveryLine: %s", err, recoveryLine)
	}

	return syncedBlocks, nil
}
<commit_msg>Fix MDStat() error return<commit_after>// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package procfs

import (
	"fmt"
	"io/ioutil"
	"regexp"
	"strconv"
	"strings"
)

var (
	statusLineRE   = regexp.MustCompile(`(\d+) blocks .*\[(\d+)/(\d+)\] \[[U_]+\]`)
	recoveryLineRE = regexp.MustCompile(`\((\d+)/\d+\)`)
)

// MDStat holds info parsed from /proc/mdstat.
type MDStat struct {
	// Name of the device.
	Name string
	// activity-state of the device.
	ActivityState string
	// Number of active disks.
	DisksActive int64
	// Total number of disks the device requires.
	DisksTotal int64
	// Number of failed disks.
	DisksFailed int64
	// Spare disks in the device.
	DisksSpare int64
	// Number of blocks the device holds.
	BlocksTotal int64
	// Number of blocks on the device that are in sync.
	BlocksSynced int64
}

// MDStat parses an mdstat-file (/proc/mdstat) and returns a slice of
// structs containing the relevant info.  More information available here:
// https://raid.wiki.kernel.org/index.php/Mdstat
func (fs FS) MDStat() ([]MDStat, error) {
	data, err := ioutil.ReadFile(fs.proc.Path("mdstat"))
	if err != nil {
		return nil, err
	}
	mdstat, err := parseMDStat(data)
	if err != nil {
		return nil, fmt.Errorf("error parsing mdstat %s: %s", fs.proc.Path("mdstat"), err)
	}
	return mdstat, nil
}

// parseMDStat parses data from mdstat file (/proc/mdstat) and returns a slice of
// structs containing the relevant info.
func parseMDStat(mdStatData []byte) ([]MDStat, error) {
	mdStats := []MDStat{}
	lines := strings.Split(string(mdStatData), "\n")

	for i, line := range lines {
		if strings.TrimSpace(line) == "" || line[0] == ' ' ||
			strings.HasPrefix(line, "Personalities") ||
			strings.HasPrefix(line, "unused") {
			continue
		}

		deviceFields := strings.Fields(line)
		if len(deviceFields) < 3 {
			return nil, fmt.Errorf("not enough fields in mdline (expected at least 3): %s", line)
		}
		mdName := deviceFields[0] // mdx
		state := deviceFields[2]  // active or inactive

		if len(lines) <= i+3 {
			return nil, fmt.Errorf(
				"error parsing %s: too few lines for md device",
				mdName,
			)
		}

		// Failed disks have the suffix (F) & Spare disks have the suffix (S).
		fail := int64(strings.Count(line, "(F)"))
		spare := int64(strings.Count(line, "(S)"))
		active, total, size, err := evalStatusLine(lines[i], lines[i+1])

		if err != nil {
			return nil, fmt.Errorf("error parsing md device lines: %s", err)
		}

		syncLineIdx := i + 2
		if strings.Contains(lines[i+2], "bitmap") { // skip bitmap line
			syncLineIdx++
		}

		// If device is syncing at the moment, get the number of currently
		// synced bytes, otherwise that number equals the size of the device.
		syncedBlocks := size
		recovering := strings.Contains(lines[syncLineIdx], "recovery")
		resyncing := strings.Contains(lines[syncLineIdx], "resync")

		// Append recovery and resyncing state info.
		if recovering || resyncing {
			if recovering {
				state = "recovering"
			} else {
				state = "resyncing"
			}

			// Handle case when resync=PENDING or resync=DELAYED.
			if strings.Contains(lines[syncLineIdx], "PENDING") ||
				strings.Contains(lines[syncLineIdx], "DELAYED") {
				syncedBlocks = 0
			} else {
				syncedBlocks, err = evalRecoveryLine(lines[syncLineIdx])
				if err != nil {
					return nil, fmt.Errorf("error parsing sync line in md device %s: %s", mdName, err)
				}
			}
		}

		mdStats = append(mdStats, MDStat{
			Name:          mdName,
			ActivityState: state,
			DisksActive:   active,
			DisksFailed:   fail,
			DisksSpare:    spare,
			DisksTotal:    total,
			BlocksTotal:   size,
			BlocksSynced:  syncedBlocks,
		})
	}

	return mdStats, nil
}

func evalStatusLine(deviceLine, statusLine string) (active, total, size int64, err error) {

	sizeStr := strings.Fields(statusLine)[0]
	size, err = strconv.ParseInt(sizeStr, 10, 64)
	if err != nil {
		return 0, 0, 0, fmt.Errorf("unexpected statusLine %s: %s", statusLine, err)
	}

	if strings.Contains(deviceLine, "raid0") || strings.Contains(deviceLine, "linear") {
		// In the device deviceLine, only disks have a number associated with them in [].
		total = int64(strings.Count(deviceLine, "["))
		return total, total, size, nil
	}

	if strings.Contains(deviceLine, "inactive") {
		return 0, 0, size, nil
	}

	matches := statusLineRE.FindStringSubmatch(statusLine)
	if len(matches) != 4 {
		return 0, 0, 0, fmt.Errorf("couldn't find all the substring matches: %s", statusLine)
	}

	total, err = strconv.ParseInt(matches[2], 10, 64)
	if err != nil {
		return 0, 0, 0, fmt.Errorf("unexpected statusLine %s: %s", statusLine, err)
	}

	active, err = strconv.ParseInt(matches[3], 10, 64)
	if err != nil {
		return 0, 0, 0, fmt.Errorf("unexpected statusLine %s: %s", statusLine, err)
	}

	return active, total, size, nil
}

func evalRecoveryLine(recoveryLine string) (syncedBlocks int64, err error) {
	matches := recoveryLineRE.FindStringSubmatch(recoveryLine)
	if len(matches) != 2 {
		return 0, fmt.Errorf("unexpected recoveryLine: %s", recoveryLine)
	}

	syncedBlocks, err = strconv.ParseInt(matches[1], 10, 64)
	if err != nil {
		return 0, fmt.Errorf("%s in recoveryLine: %s", err, recoveryLine)
	}

	return syncedBlocks, nil
}
<|endoftext|>


<commit_before>package scanner

import (
	"bufio"
	"encoding/gob"
	"errors"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/fsutil"
	"github.com/Symantec/Dominator/lib/hash"
	"io"
	"os"
	"path"
	"time"
)

type unreferencedObject struct {
	Hash   hash.Hash
	Length uint64
	Age    time.Time
}

type unreferencedObjectsEntry struct {
	object unreferencedObject
	prev   *unreferencedObjectsEntry
	next   *unreferencedObjectsEntry
}

type unreferencedObjectsList struct {
	length              uint64
	totalBytes          uint64
	oldest              *unreferencedObjectsEntry
	newest              *unreferencedObjectsEntry
	hashToEntry         map[hash.Hash]*unreferencedObjectsEntry
	lastRegeneratedTime time.Time
}

func loadUnreferencedObjects(fName string) (*unreferencedObjectsList, error) {
	file, err := os.Open(fName)
	if err != nil {
		if !os.IsNotExist(err) {
			return nil, err
		}
		return &unreferencedObjectsList{
			hashToEntry: make(map[hash.Hash]*unreferencedObjectsEntry),
		}, nil
	}
	defer file.Close()
	reader := fsutil.NewChecksumReader(bufio.NewReader(file))
	decoder := gob.NewDecoder(reader)
	var length uint64
	if err := decoder.Decode(&length); err != nil {
		return nil, errors.New("error decoding list length: " + err.Error())
	}
	list := &unreferencedObjectsList{
		hashToEntry: make(map[hash.Hash]*unreferencedObjectsEntry, length),
	}
	for count := uint64(0); count < length; count++ {
		var object unreferencedObject
		if err := decoder.Decode(&object); err != nil {
			return nil, errors.New("error decoding object: " + err.Error())
		}
		entry := &unreferencedObjectsEntry{object: object}
		list.addEntry(entry)
	}
	if err := reader.VerifyChecksum(); err != nil {
		return nil, err
	}
	return list, nil
}

func (list *unreferencedObjectsList) write(w io.Writer) error {
	writer := fsutil.NewChecksumWriter(w)
	encoder := gob.NewEncoder(writer)
	if err := encoder.Encode(list.length); err != nil {
		return err
	}
	for entry := list.oldest; entry != nil; entry = entry.next {
		if err := encoder.Encode(entry.object); err != nil {
			return err
		}
	}
	return writer.WriteChecksum()
}

func (list *unreferencedObjectsList) addEntry(entry *unreferencedObjectsEntry) {
	entry.prev = list.newest
	if list.oldest == nil {
		list.oldest = entry
	} else {
		list.newest.next = entry
	}
	list.newest = entry
	list.hashToEntry[entry.object.Hash] = entry
	list.length++
	list.totalBytes += entry.object.Length
}

func (list *unreferencedObjectsList) addObject(hashVal hash.Hash,
	length uint64) bool {
	if _, ok := list.hashToEntry[hashVal]; !ok {
		object := unreferencedObject{hashVal, length, time.Now()}
		list.addEntry(&unreferencedObjectsEntry{object: object})
		return true
	}
	return false
}

func (list *unreferencedObjectsList) list() map[hash.Hash]uint64 {
	objectsMap := make(map[hash.Hash]uint64, list.length)
	for entry := list.oldest; entry != nil; entry = entry.next {
		objectsMap[entry.object.Hash] = entry.object.Length
	}
	return objectsMap
}

func (list *unreferencedObjectsList) removeObject(hashVal hash.Hash) bool {
	entry := list.hashToEntry[hashVal]
	if entry == nil {
		return false
	}
	if entry.prev == nil {
		list.oldest = entry.next
	} else {
		entry.prev.next = entry.next
	}
	if entry.next == nil {
		list.newest = entry.prev
	} else {
		entry.next.prev = entry.prev
	}
	list.length--
	list.totalBytes -= entry.object.Length
	return true
}

func (imdb *ImageDataBase) maybeAddToUnreferencedObjectsList(
	inodeTable filesystem.InodeTable) {
	// First get a list of objects in the image being deleted.
	objects := make(map[hash.Hash]uint64)
	for _, inode := range inodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			objects[inode.Hash] = inode.Size
		}
	}
	// Scan all remaining images and remove their objects from the list.
	for _, image := range imdb.imageMap {
		for _, inode := range image.FileSystem.InodeTable {
			if inode, ok := inode.(*filesystem.RegularInode); ok {
				delete(objects, inode.Hash)
			}
		}
	}
	changed := false
	for object, size := range objects {
		if imdb.unreferencedObjects.addObject(object, size) {
			changed = true
		}
	}
	if changed {
		imdb.saveUnreferencedObjectsList()
	}
}

func (imdb *ImageDataBase) removeFromUnreferencedObjectsList(
	inodeTable filesystem.InodeTable) {
	changed := false
	for _, inode := range inodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			if imdb.unreferencedObjects.removeObject(inode.Hash) {
				changed = true
			}
		}
	}
	if changed {
		imdb.saveUnreferencedObjectsList()
	}
}

func (imdb *ImageDataBase) saveUnreferencedObjectsList() {
	if err := imdb.writeUnreferencedObjectsList(); err != nil {
		imdb.logger.Printf("Error writing unreferenced objects list: %s\n",
			err)
	}
}

func (imdb *ImageDataBase) writeUnreferencedObjectsList() error {
	filename := path.Join(imdb.baseDir, unreferencedObjectsFile)
	file, err := fsutil.CreateRenamingWriter(filename, filePerms)
	if err != nil {
		return err
	}
	defer file.Close()
	writer := bufio.NewWriter(file)
	defer writer.Flush()
	return imdb.unreferencedObjects.write(writer)
}

func (imdb *ImageDataBase) garbageCollector(bytesToDelete uint64) (
	uint64, error) {
	if bytesToDelete < 1 {
		return 0, nil
	}
	imdb.Lock()
	firstEntry := imdb.unreferencedObjects.oldest
	entry := imdb.unreferencedObjects.oldest
	var nObjects uint64
	var nBytes uint64
	for ; entry != nil && nBytes < bytesToDelete; entry = entry.next {
		nObjects++
		nBytes += entry.object.Length
	}
	imdb.unreferencedObjects.length -= nBytes
	imdb.unreferencedObjects.totalBytes -= nObjects
	imdb.unreferencedObjects.oldest = entry
	if entry == nil {
		imdb.unreferencedObjects.newest = nil
	} else if entry.prev != nil {
		entry.prev.next = nil
		entry.prev = nil
	}
	imdb.Unlock()
	if firstEntry == nil {
		return 0, nil
	}
	nBytes = 0
	var err error
	for entry := firstEntry; entry != nil; entry = entry.next {
		if e := imdb.objectServer.DeleteObject(entry.object.Hash); e != nil {
			if err == nil {
				err = e
			}
		} else {
			nBytes += entry.object.Length
		}
	}
	imdb.saveUnreferencedObjectsList()
	return nBytes, err
}

// This grabs the lock.
func (imdb *ImageDataBase) maybeRegenerateUnreferencedObjectsList() {
	imdb.RLock()
	lastRegeneratedTime := imdb.unreferencedObjects.lastRegeneratedTime
	imdb.RUnlock()
	lastMutationTime := imdb.objectServer.LastMutationTime()
	if lastMutationTime.After(lastRegeneratedTime) {
		imdb.regenerateUnreferencedObjectsList()
	}
}

// This grabs the lock.
func (imdb *ImageDataBase) regenerateUnreferencedObjectsList() {
	scanTime := time.Now()
	// First generate list of currently unused objects.
	objectsMap := imdb.objectServer.ListObjectSizes()
	imdb.Lock()
	defer imdb.Unlock()
	for _, image := range imdb.imageMap {
		for _, inode := range image.FileSystem.InodeTable {
			if inode, ok := inode.(*filesystem.RegularInode); ok {
				delete(objectsMap, inode.Hash)
			}
		}
	}
	changed := false
	// Now add unused objects to cached list.
	for hashVal, length := range objectsMap {
		if imdb.unreferencedObjects.addObject(hashVal, length) {
			changed = true
		}
	}
	// Finally remove objects from cached list which are no longer unreferenced.
	for entry := imdb.unreferencedObjects.oldest; entry != nil; {
		hashVal := entry.object.Hash
		entry = entry.next
		if _, ok := objectsMap[hashVal]; !ok {
			if imdb.unreferencedObjects.removeObject(hashVal) {
				changed = true
			}
		}
	}
	if changed {
		imdb.saveUnreferencedObjectsList()
	}
	imdb.unreferencedObjects.lastRegeneratedTime = scanTime
}
<commit_msg>Fix accounting bug in imageserver/scanner.ImageDataBase.garbageCollector().<commit_after>package scanner

import (
	"bufio"
	"encoding/gob"
	"errors"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/format"
	"github.com/Symantec/Dominator/lib/fsutil"
	"github.com/Symantec/Dominator/lib/hash"
	"io"
	"os"
	"path"
	"time"
)

type unreferencedObject struct {
	Hash   hash.Hash
	Length uint64
	Age    time.Time
}

type unreferencedObjectsEntry struct {
	object unreferencedObject
	prev   *unreferencedObjectsEntry
	next   *unreferencedObjectsEntry
}

type unreferencedObjectsList struct {
	length              uint64
	totalBytes          uint64
	oldest              *unreferencedObjectsEntry
	newest              *unreferencedObjectsEntry
	hashToEntry         map[hash.Hash]*unreferencedObjectsEntry
	lastRegeneratedTime time.Time
}

func loadUnreferencedObjects(fName string) (*unreferencedObjectsList, error) {
	file, err := os.Open(fName)
	if err != nil {
		if !os.IsNotExist(err) {
			return nil, err
		}
		return &unreferencedObjectsList{
			hashToEntry: make(map[hash.Hash]*unreferencedObjectsEntry),
		}, nil
	}
	defer file.Close()
	reader := fsutil.NewChecksumReader(bufio.NewReader(file))
	decoder := gob.NewDecoder(reader)
	var length uint64
	if err := decoder.Decode(&length); err != nil {
		return nil, errors.New("error decoding list length: " + err.Error())
	}
	list := &unreferencedObjectsList{
		hashToEntry: make(map[hash.Hash]*unreferencedObjectsEntry, length),
	}
	for count := uint64(0); count < length; count++ {
		var object unreferencedObject
		if err := decoder.Decode(&object); err != nil {
			return nil, errors.New("error decoding object: " + err.Error())
		}
		entry := &unreferencedObjectsEntry{object: object}
		list.addEntry(entry)
	}
	if err := reader.VerifyChecksum(); err != nil {
		return nil, err
	}
	return list, nil
}

func (list *unreferencedObjectsList) write(w io.Writer) error {
	writer := fsutil.NewChecksumWriter(w)
	encoder := gob.NewEncoder(writer)
	if err := encoder.Encode(list.length); err != nil {
		return err
	}
	for entry := list.oldest; entry != nil; entry = entry.next {
		if err := encoder.Encode(entry.object); err != nil {
			return err
		}
	}
	return writer.WriteChecksum()
}

func (list *unreferencedObjectsList) addEntry(entry *unreferencedObjectsEntry) {
	entry.prev = list.newest
	if list.oldest == nil {
		list.oldest = entry
	} else {
		list.newest.next = entry
	}
	list.newest = entry
	list.hashToEntry[entry.object.Hash] = entry
	list.length++
	list.totalBytes += entry.object.Length
}

func (list *unreferencedObjectsList) addObject(hashVal hash.Hash,
	length uint64) bool {
	if _, ok := list.hashToEntry[hashVal]; !ok {
		object := unreferencedObject{hashVal, length, time.Now()}
		list.addEntry(&unreferencedObjectsEntry{object: object})
		return true
	}
	return false
}

func (list *unreferencedObjectsList) list() map[hash.Hash]uint64 {
	objectsMap := make(map[hash.Hash]uint64, list.length)
	for entry := list.oldest; entry != nil; entry = entry.next {
		objectsMap[entry.object.Hash] = entry.object.Length
	}
	return objectsMap
}

func (list *unreferencedObjectsList) removeObject(hashVal hash.Hash) bool {
	entry := list.hashToEntry[hashVal]
	if entry == nil {
		return false
	}
	if entry.prev == nil {
		list.oldest = entry.next
	} else {
		entry.prev.next = entry.next
	}
	if entry.next == nil {
		list.newest = entry.prev
	} else {
		entry.next.prev = entry.prev
	}
	list.length--
	list.totalBytes -= entry.object.Length
	return true
}

func (imdb *ImageDataBase) maybeAddToUnreferencedObjectsList(
	inodeTable filesystem.InodeTable) {
	// First get a list of objects in the image being deleted.
	objects := make(map[hash.Hash]uint64)
	for _, inode := range inodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			objects[inode.Hash] = inode.Size
		}
	}
	// Scan all remaining images and remove their objects from the list.
	for _, image := range imdb.imageMap {
		for _, inode := range image.FileSystem.InodeTable {
			if inode, ok := inode.(*filesystem.RegularInode); ok {
				delete(objects, inode.Hash)
			}
		}
	}
	changed := false
	for object, size := range objects {
		if imdb.unreferencedObjects.addObject(object, size) {
			changed = true
		}
	}
	if changed {
		imdb.saveUnreferencedObjectsList()
	}
}

func (imdb *ImageDataBase) removeFromUnreferencedObjectsList(
	inodeTable filesystem.InodeTable) {
	changed := false
	for _, inode := range inodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			if imdb.unreferencedObjects.removeObject(inode.Hash) {
				changed = true
			}
		}
	}
	if changed {
		imdb.saveUnreferencedObjectsList()
	}
}

func (imdb *ImageDataBase) saveUnreferencedObjectsList() {
	if err := imdb.writeUnreferencedObjectsList(); err != nil {
		imdb.logger.Printf("Error writing unreferenced objects list: %s\n",
			err)
	}
}

func (imdb *ImageDataBase) writeUnreferencedObjectsList() error {
	filename := path.Join(imdb.baseDir, unreferencedObjectsFile)
	file, err := fsutil.CreateRenamingWriter(filename, filePerms)
	if err != nil {
		return err
	}
	defer file.Close()
	writer := bufio.NewWriter(file)
	defer writer.Flush()
	return imdb.unreferencedObjects.write(writer)
}

func (imdb *ImageDataBase) garbageCollector(bytesToDelete uint64) (
	uint64, error) {
	if bytesToDelete < 1 {
		return 0, nil
	}
	imdb.Lock()
	firstEntry := imdb.unreferencedObjects.oldest
	entry := imdb.unreferencedObjects.oldest
	var nObjects uint64
	var nBytes uint64
	for ; entry != nil && nBytes < bytesToDelete; entry = entry.next {
		nObjects++
		nBytes += entry.object.Length
	}
	imdb.unreferencedObjects.length -= nObjects
	imdb.unreferencedObjects.totalBytes -= nBytes
	imdb.unreferencedObjects.oldest = entry
	if entry == nil {
		imdb.unreferencedObjects.newest = nil
	} else if entry.prev != nil {
		entry.prev.next = nil
		entry.prev = nil
	}
	imdb.Unlock()
	if firstEntry == nil {
		return 0, nil
	}
	nBytes = 0
	var err error
	for entry := firstEntry; entry != nil; entry = entry.next {
		if e := imdb.objectServer.DeleteObject(entry.object.Hash); e != nil {
			if err == nil {
				err = e
			}
		} else {
			nBytes += entry.object.Length
		}
	}
	imdb.saveUnreferencedObjectsList()
	imdb.logger.Printf("Garbage collector deleted: %s in: %d objects\n",
		format.FormatBytes(nBytes), nObjects)
	return nBytes, err
}

// This grabs the lock.
func (imdb *ImageDataBase) maybeRegenerateUnreferencedObjectsList() {
	imdb.RLock()
	lastRegeneratedTime := imdb.unreferencedObjects.lastRegeneratedTime
	imdb.RUnlock()
	lastMutationTime := imdb.objectServer.LastMutationTime()
	if lastMutationTime.After(lastRegeneratedTime) {
		imdb.regenerateUnreferencedObjectsList()
	}
}

// This grabs the lock.
func (imdb *ImageDataBase) regenerateUnreferencedObjectsList() {
	scanTime := time.Now()
	// First generate list of currently unused objects.
	objectsMap := imdb.objectServer.ListObjectSizes()
	imdb.Lock()
	defer imdb.Unlock()
	for _, image := range imdb.imageMap {
		for _, inode := range image.FileSystem.InodeTable {
			if inode, ok := inode.(*filesystem.RegularInode); ok {
				delete(objectsMap, inode.Hash)
			}
		}
	}
	changed := false
	// Now add unused objects to cached list.
	for hashVal, length := range objectsMap {
		if imdb.unreferencedObjects.addObject(hashVal, length) {
			changed = true
		}
	}
	// Finally remove objects from cached list which are no longer unreferenced.
	for entry := imdb.unreferencedObjects.oldest; entry != nil; {
		hashVal := entry.object.Hash
		entry = entry.next
		if _, ok := objectsMap[hashVal]; !ok {
			if imdb.unreferencedObjects.removeObject(hashVal) {
				changed = true
			}
		}
	}
	if changed {
		imdb.saveUnreferencedObjectsList()
	}
	imdb.unreferencedObjects.lastRegeneratedTime = scanTime
}
<|endoftext|>


<commit_before>/*
*  Copyright (c) WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
*
*  WSO2 Inc. licenses this file to you under the Apache License,
*  Version 2.0 (the "License"); you may not use this file except
*  in compliance with the License.
*  You may obtain a copy of the License at
*
*    http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied.  See the License for the
* specific language governing permissions and limitations
* under the License.
 */

package cmd

import (
	"testing"

	"fmt"
	"github.com/go-resty/resty"
	"github.com/renstrom/dedent"
	"github.com/wso2/product-apim-tooling/import-export-cli/utils"
	"net/http"
	"net/http/httptest"
)

func TestExportAPI(t *testing.T) {
	var server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != http.MethodGet {
			t.Errorf("Expected 'GET', got '%s'\n", r.Method)
		}

		if r.Header.Get(utils.HeaderAccept) != utils.HeaderValueApplicationZip {
			t.Errorf("Expected '"+utils.HeaderValueApplicationZip+"', got '%s'\n",
				r.Header.Get(utils.HeaderContentType))
		}

		w.WriteHeader(http.StatusOK)
		w.Header().Set(utils.HeaderContentType, utils.HeaderValueApplicationJSON)
		w.Header().Set(utils.HeaderContentEncoding, utils.HeaderValueGZIP)
		w.Header().Set(utils.HeaderTransferEncoding, utils.HeaderValueChunked)

		body := dedent.Dedent(`

		`)

		w.Write([]byte(body))
	}))
	defer server.Close()

	resp := ExportAPI("test", "1.0", server.URL, "")
	fmt.Println(resp)
}

func TestWriteToZip(t *testing.T) {
	name := "sampleapi"
	version := "1.0.0"
	environment := "dev"
	response := new(resty.Response)
	exportDirectory := utils.CurrentDir
	WriteToZip(name, version, environment, exportDirectory, response)
}
<commit_msg>test(cmd): delete temporary directory after test completion<commit_after>/*
*  Copyright (c) WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
*
*  WSO2 Inc. licenses this file to you under the Apache License,
*  Version 2.0 (the "License"); you may not use this file except
*  in compliance with the License.
*  You may obtain a copy of the License at
*
*    http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied.  See the License for the
* specific language governing permissions and limitations
* under the License.
 */

package cmd

import (
	"os"
	"path/filepath"
	"testing"

	"fmt"
	"github.com/go-resty/resty"
	"github.com/renstrom/dedent"
	"github.com/wso2/product-apim-tooling/import-export-cli/utils"
	"net/http"
	"net/http/httptest"
)

func TestExportAPI(t *testing.T) {
	var server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != http.MethodGet {
			t.Errorf("Expected 'GET', got '%s'\n", r.Method)
		}

		if r.Header.Get(utils.HeaderAccept) != utils.HeaderValueApplicationZip {
			t.Errorf("Expected '"+utils.HeaderValueApplicationZip+"', got '%s'\n",
				r.Header.Get(utils.HeaderContentType))
		}

		w.WriteHeader(http.StatusOK)
		w.Header().Set(utils.HeaderContentType, utils.HeaderValueApplicationJSON)
		w.Header().Set(utils.HeaderContentEncoding, utils.HeaderValueGZIP)
		w.Header().Set(utils.HeaderTransferEncoding, utils.HeaderValueChunked)

		body := dedent.Dedent(`

		`)

		w.Write([]byte(body))
	}))
	defer server.Close()

	resp := ExportAPI("test", "1.0", server.URL, "")
	fmt.Println(resp)
}

func TestWriteToZip(t *testing.T) {
	name := "sampleapi"
	version := "1.0.0"
	environment := "dev"
	response := new(resty.Response)
	exportDirectory := utils.CurrentDir
	WriteToZip(name, version, environment, exportDirectory, response)
	defer os.RemoveAll(filepath.Join(exportDirectory, "dev"))
}
<|endoftext|>