<commit_before>package client

import (
//"honnef.co/go/js/console"
)

//Collection is a collection models.  Note that the type of the collection
//elements are Model which implies Equaler.
//TODO: Convert to Collection and collectionImpl
type Collection struct {
	coll *AttributeImpl
	join []Joiner
}

//Joiners connect a list to something that will manipulate elements of
//the list. Typically this is used for connecting models to views.
type Joiner interface {
	Add(int, Model)
	Remove(int, Model)
}

//EqList is a convenience for talking about the contents of the collection.
//You can use collection.AttributeImpl.(EqList) to read the contents of
//the list.
type EqList []Model

//Equal handles comparison of our list to another collection. We define
//equality to be is same length and has the same contents. Empty collections
//are not equal to anything.
func (self EqList) Equal(o Equaler) bool {
	if o == nil {
		return false
	}
	other := o.(EqList)
	if len(other) == 0 {
		return false
	}
	curr := []Model(self)
	if len(curr) == 0 {
		return false
	}
	if len(curr) != len(other) {
		return false
	}
	for i, e := range other {
		if !curr[i].Equal(e) {
			return false
		}
	}
	return true
}

//computedEmpty represents just the attribute value for the empty state.
type computedEmpty struct {
	*AttributeImpl
	coll *Collection
}

//empty is private because clients of the List should be using
//EmptyAttribute()
func (self *Collection) empty() Equaler {
	if self.coll.Demand() == nil || len(self.coll.Demand().(EqList)) == 0 {
		return BoolEqualer{true}
	}
	return BoolEqualer{false}
}

func (self *computedEmpty) Set(b bool) {
	panic("can't set the value of a computed attribute (empty of the list)")
}

func (self *computedEmpty) Value() bool {
	self.clean = true //tricky
	return self.coll.empty().(BoolEqualer).B
}

//EmptyAttribute should be used by callers to assess if the list is
//is empty or not. It is a BooleanAttribute so it can be used in
//constraint functions.
func (self *Collection) EmptyAttribute() BooleanAttribute {
	result := &computedEmpty{
		NewAttribute(VALUE_ONLY, self.empty, nil), self,
	}
	//console.Log("emtpy attr is ", result.id())
	newEdge(self.coll, result)
	//console.Log("introducing edge", self.id(), "to", result.id())
	return result
}

//computedEmpty represents just the attribute value for the empty state.
type computedLength struct {
	*AttributeImpl
	coll *Collection
}

func (self *computedLength) Set(i int) {
	panic("can't set the value of a computed attribute (length)")
}

func (self *computedLength) Value() int {
	return self.coll.length().(IntEqualer).I
}

//length is a private function; use LengthAttribute() to access the
//the length.
func (self *Collection) length() Equaler {
	return IntEqualer{len(self.coll.Demand().(EqList))}
}

//LengthAttribute should be used by callers to assess how many items
//are in the list. The return value is an attribute that can be used
//in constraint functions.
func (self *Collection) LengthAttribute() IntegerAttribute {
	result := &computedLength{
		NewAttribute(VALUE_ONLY, self.length, nil), self,
	}
	return result
}

//PushRaw the way to push a Model into the list _without_ having
//any checking done about the value of that model.  Note that most
//callers would probably prefer Add() that checks to see if the item
//is already in the list before doing the addition.  Note this does
//imply updating of the empty and length attributes. If the collection
//has joiners, it is called after the PushRaw has completed.
func (self *Collection) PushRaw(m Model) {
	current := self.coll.Demand()
	var result EqList

	if current == nil {
		result = EqList{m}
	} else {
		result = append(current.(EqList), m)
	}
	self.coll.SetEqualer(result)

	if self.join != nil {
		for _, j := range self.join {
			j.Add(len(result), m)
		}
	}
}

//PopRaw is the way to access the last node of the list without any
//checking.  This will panic if the list is empty.  This implies an
//update to the length and empty attribute.  If there are Joiners
//they are called after the PopRaw completes.
func (self *Collection) PopRaw() Model {
	if self.coll.Demand() == nil || len(self.coll.Demand().(EqList)) == 0 {
		panic("can't pop from a empty ListNode!")
	}

	obj := self.coll.Demand().(EqList)
	self.coll.markDirty()

	result := obj[len(obj)-1]
	if len(obj) == 1 {
		self.coll.SetEqualer(nil)
	} else {
		self.coll.SetEqualer(obj[:len(obj)-1])
	}
	if self.join != nil {
		for _, j := range self.join {
			j.Remove(len(obj)-1, result)
		}
	}

	return result
}

//NewCollection returns an empty Collection.  You should a supply a joiner here
//if you want to run a transform on insert or remove from thelist.  You can
//pass nil if you don't need any joiner at the point you create the
//collection.
func NewList(joiner Joiner) *Collection {
	result := &Collection{
		coll: NewAttribute(VALUE_ONLY, nil, nil),
	}
	if joiner != nil {
		result.join = []Joiner{joiner}
	}
	return result
}

//Add checks to see that the model is not already in the collection then
//adds it if it is not. If the object is already in the collection, this has
//no effect (but does request the value of  all the items in the list).
//If the list has any joiners, they are notified about the new element, but
//after the change has taken place.
func (self *Collection) Add(m Model) {

	//console.Log("adding an attribute %O", m)
	obj := self.coll.Demand().(EqList)
	for _, e := range obj {
		if e.Equal(m) {
			return
		}
	}
	self.PushRaw(m)
}

//Remove checks to see that the model is in the collection and
//then removes it.  If the element is in the collection multiple times, only
//the first one is removed.  Calling this on an empty collection is useless
//but not an error. If the list has joiners, they are notified about
//the removal of the element but after the removal
//has occured. The object that was removed is supplied to the Joiner.
func (self *Collection) Remove(m Model) {
	obj := self.coll.Demand().(EqList)

	for i, e := range obj {
		//console.Log("demand checking %O %O", e, m)
		if e.Equal(m) {
			//last?
			if i == len(obj)-1 {
				self.PopRaw()
			} else {
				copy(obj[i:], obj[i+1:])
				obj[len(obj)-1] = nil
				self.coll.SetEqualer(EqList(obj[:len(obj)-1]))
				if self.join != nil {
					for _, j := range self.join {
						j.Remove(i, m)
					}
				}
			}
			break
		}
	}
}

//All returns all the models in this collection.  This is a copy
//of the objects internal data.
func (self *Collection) All() []Model {
	if self.coll.Demand() == nil {
		return nil
	}
	curr := self.coll.Demand().(EqList)
	result := make([]Model, len(curr))
	for i, c := range curr {
		result[i] = c
	}
	return result
}

//Puller is a function that extracts a particular attribute from a
//a model.
type Puller func(Model) Attribute

//FoldingIterator is a function of the previous value and the next
//input attribute. The latter is extracted via a Puller function, so
//one can think of this second parameter as a model.  This function
//returns two values, the first of which is passed to this function
//again on all but the last iteration.  On the last iteration, the
//2nd value is the final result.
type FoldingIterator func(interface{}, Equaler) (interface{}, Equaler)

//AllFold creates a constraint that depends on the same
//attribute in _every_ model in the collection.   The attribute to be computed
//is the first parameter. The initial value of the iterative folding
//is the second argument, and fed to the Folder on the first iteration.
//The puller is used whenever the models in the collection change to
//extract a particular Attribute that the constraint depends on.
func (self *Collection) AllFold(
	targ Attribute,
	initial interface{},
	puller Puller,
	folder FoldingIterator,
	empty Equaler) Constraint {

	result := &foldedConstraint{
		fn:      folder,
		initial: initial,
		pull:    puller,
		targ:    targ,
		empty:   empty,
	}

	self.join = append(self.join, result)
	targ.Attach(result)
	return result

}

type foldedConstraint struct {
	deps    []Attribute
	fn      FoldingIterator
	pull    Puller
	initial interface{}
	targ    Attribute
	empty   Equaler
}

func (self *foldedConstraint) Inputs() []Attribute {
	return self.deps
}

func (self *foldedConstraint) Fn(in []Equaler) Equaler {
	if len(self.deps) == 0 {
		return self.empty
	}

	//normal case
	prev := self.initial
	var e Equaler
	for _, i := range in {
		prev, e = self.fn(prev, i)
	}
	return e
}

func (self *foldedConstraint) Remove(i int, m Model) {
	dropEdge(self.deps[i].(node), self.targ.(node))
	self.targ.(node).markDirty()

	if len(self.deps)-1 == i {
		self.deps = self.deps[:len(self.deps)-1]
	} else {
		self.deps = append(self.deps[:i], self.deps[i+1:]...)
	}
	DrainEagerQueue()
}

func (self *foldedConstraint) Add(i int, m Model) {
	a := self.pull(m)
	self.deps = append(self.deps, a)
	newEdge(a.(node), self.targ.(node))
	self.targ.(node).markDirty()

	DrainEagerQueue()
}
<commit_msg>make Add() to empty list smarter<commit_after>package client

import (
//"honnef.co/go/js/console"
)

//Collection is a collection models.  Note that the type of the collection
//elements are Model which implies Equaler.
//TODO: Convert to Collection and collectionImpl
type Collection struct {
	coll *AttributeImpl
	join []Joiner
}

//Joiners connect a list to something that will manipulate elements of
//the list. Typically this is used for connecting models to views.
type Joiner interface {
	Add(int, Model)
	Remove(int, Model)
}

//EqList is a convenience for talking about the contents of the collection.
//You can use collection.AttributeImpl.(EqList) to read the contents of
//the list.
type EqList []Model

//Equal handles comparison of our list to another collection. We define
//equality to be is same length and has the same contents. Empty collections
//are not equal to anything.
func (self EqList) Equal(o Equaler) bool {
	if o == nil {
		return false
	}
	other := o.(EqList)
	if len(other) == 0 {
		return false
	}
	curr := []Model(self)
	if len(curr) == 0 {
		return false
	}
	if len(curr) != len(other) {
		return false
	}
	for i, e := range other {
		if !curr[i].Equal(e) {
			return false
		}
	}
	return true
}

//computedEmpty represents just the attribute value for the empty state.
type computedEmpty struct {
	*AttributeImpl
	coll *Collection
}

//empty is private because clients of the List should be using
//EmptyAttribute()
func (self *Collection) empty() Equaler {
	if self.coll.Demand() == nil || len(self.coll.Demand().(EqList)) == 0 {
		return BoolEqualer{true}
	}
	return BoolEqualer{false}
}

func (self *computedEmpty) Set(b bool) {
	panic("can't set the value of a computed attribute (empty of the list)")
}

func (self *computedEmpty) Value() bool {
	self.clean = true //tricky
	return self.coll.empty().(BoolEqualer).B
}

//EmptyAttribute should be used by callers to assess if the list is
//is empty or not. It is a BooleanAttribute so it can be used in
//constraint functions.
func (self *Collection) EmptyAttribute() BooleanAttribute {
	result := &computedEmpty{
		NewAttribute(VALUE_ONLY, self.empty, nil), self,
	}
	//console.Log("emtpy attr is ", result.id())
	newEdge(self.coll, result)
	//console.Log("introducing edge", self.id(), "to", result.id())
	return result
}

//computedEmpty represents just the attribute value for the empty state.
type computedLength struct {
	*AttributeImpl
	coll *Collection
}

func (self *computedLength) Set(i int) {
	panic("can't set the value of a computed attribute (length)")
}

func (self *computedLength) Value() int {
	return self.coll.length().(IntEqualer).I
}

//length is a private function; use LengthAttribute() to access the
//the length.
func (self *Collection) length() Equaler {
	return IntEqualer{len(self.coll.Demand().(EqList))}
}

//LengthAttribute should be used by callers to assess how many items
//are in the list. The return value is an attribute that can be used
//in constraint functions.
func (self *Collection) LengthAttribute() IntegerAttribute {
	result := &computedLength{
		NewAttribute(VALUE_ONLY, self.length, nil), self,
	}
	return result
}

//PushRaw the way to push a Model into the list _without_ having
//any checking done about the value of that model.  Note that most
//callers would probably prefer Add() that checks to see if the item
//is already in the list before doing the addition.  Note this does
//imply updating of the empty and length attributes. If the collection
//has joiners, it is called after the PushRaw has completed.
func (self *Collection) PushRaw(m Model) {
	current := self.coll.Demand()
	var result EqList

	if current == nil {
		result = EqList{m}
	} else {
		result = append(current.(EqList), m)
	}
	self.coll.SetEqualer(result)

	if self.join != nil {
		for _, j := range self.join {
			j.Add(len(result), m)
		}
	}
}

//PopRaw is the way to access the last node of the list without any
//checking.  This will panic if the list is empty.  This implies an
//update to the length and empty attribute.  If there are Joiners
//they are called after the PopRaw completes.
func (self *Collection) PopRaw() Model {
	if self.coll.Demand() == nil || len(self.coll.Demand().(EqList)) == 0 {
		panic("can't pop from a empty ListNode!")
	}

	obj := self.coll.Demand().(EqList)
	self.coll.markDirty()

	result := obj[len(obj)-1]
	if len(obj) == 1 {
		self.coll.SetEqualer(nil)
	} else {
		self.coll.SetEqualer(obj[:len(obj)-1])
	}
	if self.join != nil {
		for _, j := range self.join {
			j.Remove(len(obj)-1, result)
		}
	}

	return result
}

//NewCollection returns an empty Collection.  You should a supply a joiner here
//if you want to run a transform on insert or remove from thelist.  You can
//pass nil if you don't need any joiner at the point you create the
//collection.
func NewList(joiner Joiner) *Collection {
	result := &Collection{
		coll: NewAttribute(VALUE_ONLY, nil, nil),
	}
	if joiner != nil {
		result.join = []Joiner{joiner}
	}
	return result
}

//Add checks to see that the model is not already in the collection then
//adds it if it is not. If the object is already in the collection, this has
//no effect (but does request the value of  all the items in the list).
//If the list has any joiners, they are notified about the new element, but
//after the change has taken place.
func (self *Collection) Add(m Model) {

	//console.Log("adding an attribute %O", m)
	if self.coll.Demand() != nil {
		obj := self.coll.Demand().(EqList)
		for _, e := range obj {
			if e.Equal(m) {
				return
			}
		}
	}
	self.PushRaw(m)
}

//Remove checks to see that the model is in the collection and
//then removes it.  If the element is in the collection multiple times, only
//the first one is removed.  Calling this on an empty collection is useless
//but not an error. If the list has joiners, they are notified about
//the removal of the element but after the removal
//has occured. The object that was removed is supplied to the Joiner.
func (self *Collection) Remove(m Model) {
	obj := self.coll.Demand().(EqList)

	for i, e := range obj {
		//console.Log("demand checking %O %O", e, m)
		if e.Equal(m) {
			//last?
			if i == len(obj)-1 {
				self.PopRaw()
			} else {
				copy(obj[i:], obj[i+1:])
				obj[len(obj)-1] = nil
				self.coll.SetEqualer(EqList(obj[:len(obj)-1]))
				if self.join != nil {
					for _, j := range self.join {
						j.Remove(i, m)
					}
				}
			}
			break
		}
	}
}

//All returns all the models in this collection.  This is a copy
//of the objects internal data.
func (self *Collection) All() []Model {
	if self.coll.Demand() == nil {
		return nil
	}
	curr := self.coll.Demand().(EqList)
	result := make([]Model, len(curr))
	for i, c := range curr {
		result[i] = c
	}
	return result
}

//Puller is a function that extracts a particular attribute from a
//a model.
type Puller func(Model) Attribute

//FoldingIterator is a function of the previous value and the next
//input attribute. The latter is extracted via a Puller function, so
//one can think of this second parameter as a model.  This function
//returns two values, the first of which is passed to this function
//again on all but the last iteration.  On the last iteration, the
//2nd value is the final result.
type FoldingIterator func(interface{}, Equaler) (interface{}, Equaler)

//AllFold creates a constraint that depends on the same
//attribute in _every_ model in the collection.   The attribute to be computed
//is the first parameter. The initial value of the iterative folding
//is the second argument, and fed to the Folder on the first iteration.
//The puller is used whenever the models in the collection change to
//extract a particular Attribute that the constraint depends on.
func (self *Collection) AllFold(
	targ Attribute,
	initial interface{},
	puller Puller,
	folder FoldingIterator,
	empty Equaler) Constraint {

	result := &foldedConstraint{
		fn:      folder,
		initial: initial,
		pull:    puller,
		targ:    targ,
		empty:   empty,
	}

	self.join = append(self.join, result)
	targ.Attach(result)
	return result

}

type foldedConstraint struct {
	deps    []Attribute
	fn      FoldingIterator
	pull    Puller
	initial interface{}
	targ    Attribute
	empty   Equaler
}

func (self *foldedConstraint) Inputs() []Attribute {
	return self.deps
}

func (self *foldedConstraint) Fn(in []Equaler) Equaler {
	if len(self.deps) == 0 {
		return self.empty
	}

	//normal case
	prev := self.initial
	var e Equaler
	for _, i := range in {
		prev, e = self.fn(prev, i)
	}
	return e
}

func (self *foldedConstraint) Remove(i int, m Model) {
	dropEdge(self.deps[i].(node), self.targ.(node))
	self.targ.(node).markDirty()

	if len(self.deps)-1 == i {
		self.deps = self.deps[:len(self.deps)-1]
	} else {
		self.deps = append(self.deps[:i], self.deps[i+1:]...)
	}
	DrainEagerQueue()
}

func (self *foldedConstraint) Add(i int, m Model) {
	a := self.pull(m)
	self.deps = append(self.deps, a)
	newEdge(a.(node), self.targ.(node))
	self.targ.(node).markDirty()

	DrainEagerQueue()
}
<|endoftext|>


<commit_before>package fofiwano

import (
	"log"
	"os"
	"os/signal"
	"syscall"

	"strings"

	"github.com/rjeczalik/notify"
	"golang.org/x/net/context"
)

// WatcherNotify defines a notification type for a watcher
type WatcherNotify struct {
	Notify   string
	Event    string
	Options  map[string]string
	Notifier Notifcation
}

// Watcher defines a watcher configuration
// Target can be a folder or a file, add /... to a folder for recursive watching
// e.g. ./test/...
type Watcher struct {
	Target        string
	Notifications []WatcherNotify
}

// Watch starts the watcher with the given configuration
func Watch(watches []Watcher) {

	ctx, done := context.WithCancel(context.Background())

	signalChannel := make(chan os.Signal, 1)
	signal.Notify(signalChannel, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
	go func() {
		switch <-signalChannel {
		case os.Interrupt:
			done()
		case syscall.SIGTERM:
			done()
		}
	}()

	for _, watcher := range watches {

		stopfunc := func(specWatcher Watcher) func() {
			watcherEvents := make(chan notify.EventInfo, 2)

			// setup all notifiers
			var notifier Notifcation
			var err error
			for i := 0; i < len(specWatcher.Notifications); i++ {
				switch specWatcher.Notifications[i].Notify {
				case "slack":
					notifier, err = NewSlackNotification(specWatcher.Notifications[i].Options)
				case "http":
					notifier, err = NewHTTPNotification(specWatcher.Notifications[i].Options)
				}

				if err != nil {
					log.Println(err)
				} else {
					specWatcher.Notifications[i].Notifier = notifier
				}
			}

			go func() {
				for event := range watcherEvents {
					eventString := strings.ToLower(event.Event().String())
					for _, notification := range specWatcher.Notifications {
						// TODO implement more notification providers
						if strings.ToLower(notification.Event) == "all" || strings.Contains(eventString, strings.ToLower(notification.Event)) {
							if err := notification.Notifier.Notify(eventString, event.Path()); err != nil {
								log.Println(err)
							}
						}
					}
				}
			}()

			log.Printf("Starting watcher for %s\n", watcher.Target)
			if err := notify.Watch(watcher.Target, watcherEvents, notify.All); err != nil {
				log.Fatal(err)
			}

			return func() {
				log.Printf("Stopping watcher for %s\n", watcher.Target)
				notify.Stop(watcherEvents)
			}
		}(watcher)

		defer stopfunc()
	}

	<-ctx.Done()

}
<commit_msg>clearify some comments<commit_after>package fofiwano

import (
	"log"
	"os"
	"os/signal"
	"syscall"

	"strings"

	"github.com/rjeczalik/notify"
	"golang.org/x/net/context"
)

// WatcherNotify defines a notification type for a watcher
type WatcherNotify struct {
	Notify   string
	Event    string
	Options  map[string]string
	Notifier Notifcation
}

// Watcher defines a watcher configuration
// Target can be a folder or a file, add /... to a folder for recursive watching
// e.g. ./test/...
type Watcher struct {
	Target        string
	Notifications []WatcherNotify
}

// Watch starts the watcher with the given configuration
func Watch(watches []Watcher) {

	ctx, done := context.WithCancel(context.Background())

	signalChannel := make(chan os.Signal, 1)
	signal.Notify(signalChannel, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
	go func() {
		switch <-signalChannel {
		case os.Interrupt:
			done()
		case syscall.SIGTERM:
			done()
		}
	}()

	for _, watcher := range watches {

		stopfunc := func(specWatcher Watcher) func() {
			watcherEvents := make(chan notify.EventInfo, 2)

			// setup all notifiers and save them for re-use on events
			var notifier Notifcation
			var err error
			for i := 0; i < len(specWatcher.Notifications); i++ {
				// TODO implement more notification providers
				switch specWatcher.Notifications[i].Notify {
				case "slack":
					notifier, err = NewSlackNotification(specWatcher.Notifications[i].Options)
				case "http":
					notifier, err = NewHTTPNotification(specWatcher.Notifications[i].Options)
				}

				if err != nil {
					log.Println(err)
				} else {
					specWatcher.Notifications[i].Notifier = notifier
				}
			}

			go func() {
				for event := range watcherEvents {
					eventString := strings.ToLower(event.Event().String())
					// loop over notifications and call Notify function of specific notifier
					for _, notification := range specWatcher.Notifications {
						if strings.ToLower(notification.Event) == "all" || strings.Contains(eventString, strings.ToLower(notification.Event)) {
							if err := notification.Notifier.Notify(eventString, event.Path()); err != nil {
								log.Println(err)
							}
						}
					}
				}
			}()

			log.Printf("Starting watcher for %s\n", watcher.Target)
			if err := notify.Watch(watcher.Target, watcherEvents, notify.All); err != nil {
				log.Fatal(err)
			}

			return func() {
				log.Printf("Stopping watcher for %s\n", watcher.Target)
				notify.Stop(watcherEvents)
			}
		}(watcher)

		defer stopfunc()
	}

	<-ctx.Done()

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"os/exec"
	"reflect"
	"regexp"
	"strings"
	"testing"
)

func TestNewMatcher(t *testing.T) {
	expr := "a[bc](def)"
	m, err := NewMatcher(expr)
	if err != nil {
		t.Errorf("NewMatcher(%q) returns %q, want nil",
			expr, err)
	}

	expect, err := regexp.Compile(expr)
	if err != nil {
		t.Errorf("regexp.Compile(%q) returns %q, want nil",
			expr, err)
	}
	actual := m.re
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("got %q, want %q", actual, expect)
	}
}

func TestMatch(t *testing.T) {
	expr := `\d+`
	m, err := NewMatcher(expr)
	if err != nil {
		t.Errorf("NewMatcher(%q) returns %q, want nil",
			expr, err)
	}

	src1 := "xxx123"
	expect1 := true
	actual1 := m.MatchString(src1)
	if expect1 != actual1 {
		t.Errorf("MatchString(%q) = %v, want %v",
			expr, actual1, expect1)
	}

	src2 := "xxxabc"
	expect2 := false
	actual2 := m.MatchString(src2)
	if expect2 != actual2 {
		t.Errorf("MatchString(%q) = %v, want %v",
			expr, actual2, expect2)
	}
}

func TestNewProcessor(t *testing.T) {
	name := "mkdir"
	if _, err := exec.LookPath(name); err != nil {
		t.Skipf("%q: doesn't exist", name)
	}
	p, err := NewProcessor(name)
	if err != nil {
		t.Errorf("NewMatcher(%q) returns %q, want nil",
			name, err)
	}

	expect := exec.Command(name)
	actual := p.cmd
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("NewProcessor(%q).cmd = %v, want %v",
			name, actual, expect)
	}
}

func TestProcess(t *testing.T) {
	name, arg := "sed", "s/false/true/g"
	if _, err := exec.LookPath(name); err != nil {
		t.Skipf("%q: doesn't exist", name)
	}
	p, err := NewProcessor(name, arg)
	if err != nil {
		t.Errorf("NewProcessor(%q, %q) returns %q, want nil",
			name, arg, err)
	}

	expect := []string{"true", "true", "true", "nil"}
	actual := []string{"true", "false", "false", "nil"}
	if err = p.Process(actual); err != nil {
		t.Errorf("NewProcessor(%q, %q).Process(%q) returns %q, want nil",
			name, arg, actual, err)
	}
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("got %q, want %q", actual, expect)
	}
}

func TestNewLines(t *testing.T) {
	expect := &Lines{
		lines:          []string{},
		matchedLines:   []string{},
		matchedIndexes: make(map[int]bool),
	}
	actual := NewLines()
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("got %v, want %v", actual, expect)
	}
}

func TestLoadLines(t *testing.T) {
	expr := `\d+`
	m, err := NewMatcher(expr)
	if err != nil {
		t.Errorf("NewMatcher(%q) returns %q, want nil",
			expr, err)
	}
	src := strings.NewReader(`
abc
123
def
456
789
ghi
jkl
mno
`[1:])
	expect := &Lines{
		lines:          []string{"abc", "123", "def", "456", "789", "ghi", "jkl", "mno"},
		matchedLines:   []string{"123", "456", "789"},
		matchedIndexes: map[int]bool{1: true, 3: true, 4: true},
	}
	actual := NewLines()
	if err = actual.LoadLines(src, m); err != nil {
		t.Errorf("NewLines(%v).LoadLines(%v) returns %q, want nil",
			m, src, err)
	}
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("got %v, want %v", actual, expect)
	}
}

func TestFlush(t *testing.T) {
	name, arg := "sed", "s/.*/true/"
	if _, err := exec.LookPath(name); err != nil {
		t.Skipf("%q: doesn't exist", name)
	}
	p, err := NewProcessor(name, arg)
	if err != nil {
		t.Errorf("NewProcessor(%q, %q) returns %q, want nil",
			name, arg, err)
	}
	l := &Lines{
		lines:          []string{"true", "false", "false", "true", "nil", "true", "false"},
		matchedLines:   []string{"false", "false", "false"},
		matchedIndexes: map[int]bool{1: true, 2: true, 6: true},
	}

	out := bytes.NewBuffer(make([]byte, 0))
	if l.Flush(out, p); err != nil {
	}
	expect := `
true
true
true
true
nil
true
true
`[1:]
	actual := out.String()
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("got %q, want %q", actual, expect)
	}
}
<commit_msg>Add test for ParseOption<commit_after>package main

import (
	"bytes"
	"os/exec"
	"reflect"
	"regexp"
	"strings"
	"testing"
)

func TestNewMatcher(t *testing.T) {
	expr := "a[bc](def)"
	m, err := NewMatcher(expr)
	if err != nil {
		t.Errorf("NewMatcher(%q) returns %q, want nil",
			expr, err)
	}

	expect, err := regexp.Compile(expr)
	if err != nil {
		t.Errorf("regexp.Compile(%q) returns %q, want nil",
			expr, err)
	}
	actual := m.re
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("got %q, want %q", actual, expect)
	}
}

func TestMatch(t *testing.T) {
	expr := `\d+`
	m, err := NewMatcher(expr)
	if err != nil {
		t.Errorf("NewMatcher(%q) returns %q, want nil",
			expr, err)
	}

	src1 := "xxx123"
	expect1 := true
	actual1 := m.MatchString(src1)
	if expect1 != actual1 {
		t.Errorf("MatchString(%q) = %v, want %v",
			expr, actual1, expect1)
	}

	src2 := "xxxabc"
	expect2 := false
	actual2 := m.MatchString(src2)
	if expect2 != actual2 {
		t.Errorf("MatchString(%q) = %v, want %v",
			expr, actual2, expect2)
	}
}

func TestNewProcessor(t *testing.T) {
	name := "mkdir"
	if _, err := exec.LookPath(name); err != nil {
		t.Skipf("%q: doesn't exist", name)
	}
	p, err := NewProcessor(name)
	if err != nil {
		t.Errorf("NewMatcher(%q) returns %q, want nil",
			name, err)
	}

	expect := exec.Command(name)
	actual := p.cmd
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("NewProcessor(%q).cmd = %v, want %v",
			name, actual, expect)
	}
}

func TestProcess(t *testing.T) {
	name, arg := "sed", "s/false/true/g"
	if _, err := exec.LookPath(name); err != nil {
		t.Skipf("%q: doesn't exist", name)
	}
	p, err := NewProcessor(name, arg)
	if err != nil {
		t.Errorf("NewProcessor(%q, %q) returns %q, want nil",
			name, arg, err)
	}

	expect := []string{"true", "true", "true", "nil"}
	actual := []string{"true", "false", "false", "nil"}
	if err = p.Process(actual); err != nil {
		t.Errorf("NewProcessor(%q, %q).Process(%q) returns %q, want nil",
			name, arg, actual, err)
	}
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("got %q, want %q", actual, expect)
	}
}

func TestParseOption(t *testing.T) {
	args := []string{`\d+`, "perl", "-ple", "$_*=2", "--", "foo", "bar"}
	expect := &Option{
		IsHelp:  false,
		Pattern: `\d+`,
		Command: "perl",
		Arg:     []string{"-ple", "$_*=2"},
		Files:   []string{"foo", "bar"},
	}
	actual, err := ParseOption(args)
	if err != nil {
		t.Errorf("ParseOption(%q) returns %q, want nil",
			args, err)
	}
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("got %q, want %q", actual, expect)
	}
}

func TestNewLines(t *testing.T) {
	expect := &Lines{
		lines:          []string{},
		matchedLines:   []string{},
		matchedIndexes: make(map[int]bool),
	}
	actual := NewLines()
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("got %v, want %v", actual, expect)
	}
}

func TestLoadLines(t *testing.T) {
	expr := `\d+`
	m, err := NewMatcher(expr)
	if err != nil {
		t.Errorf("NewMatcher(%q) returns %q, want nil",
			expr, err)
	}
	src := strings.NewReader(`
abc
123
def
456
789
ghi
jkl
mno
`[1:])
	expect := &Lines{
		lines:          []string{"abc", "123", "def", "456", "789", "ghi", "jkl", "mno"},
		matchedLines:   []string{"123", "456", "789"},
		matchedIndexes: map[int]bool{1: true, 3: true, 4: true},
	}
	actual := NewLines()
	if err = actual.LoadLines(src, m); err != nil {
		t.Errorf("NewLines(%v).LoadLines(%v) returns %q, want nil",
			m, src, err)
	}
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("got %v, want %v", actual, expect)
	}
}

func TestFlush(t *testing.T) {
	name, arg := "sed", "s/.*/true/"
	if _, err := exec.LookPath(name); err != nil {
		t.Skipf("%q: doesn't exist", name)
	}
	p, err := NewProcessor(name, arg)
	if err != nil {
		t.Errorf("NewProcessor(%q, %q) returns %q, want nil",
			name, arg, err)
	}
	l := &Lines{
		lines:          []string{"true", "false", "false", "true", "nil", "true", "false"},
		matchedLines:   []string{"false", "false", "false"},
		matchedIndexes: map[int]bool{1: true, 2: true, 6: true},
	}

	out := bytes.NewBuffer(make([]byte, 0))
	if l.Flush(out, p); err != nil {
	}
	expect := `
true
true
true
true
nil
true
true
`[1:]
	actual := out.String()
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("got %q, want %q", actual, expect)
	}
}
<|endoftext|>


<commit_before>package dns

// Everything is assumed in the ClassINET class. If
// you need other classes you are on your own.

// SetReply creates a reply packet from a request message.
func (dns *Msg) SetReply(request *Msg) {
	dns.MsgHdr.Id = request.MsgHdr.Id
	dns.MsgHdr.Authoritative = true
	dns.MsgHdr.Response = true
	dns.MsgHdr.Opcode = OpcodeQuery
	dns.MsgHdr.Rcode = RcodeSuccess
	dns.Question = make([]Question, 1)
	dns.Question[0] = request.Question[0]
}

// SetQuestion creates a question packet.
func (dns *Msg) SetQuestion(z string, t uint16) {
	dns.MsgHdr.Id = Id()
	dns.MsgHdr.RecursionDesired = true
	dns.Question = make([]Question, 1)
	dns.Question[0] = Question{z, t, ClassINET}
}

// SetNotify creates a notify packet.
func (dns *Msg) SetNotify(z string) {
	dns.MsgHdr.Opcode = OpcodeNotify
	dns.MsgHdr.Authoritative = true
	dns.MsgHdr.Id = Id()
	dns.Question = make([]Question, 1)
	dns.Question[0] = Question{z, TypeSOA, ClassINET}
}

// SetRcode creates an error packet.
func (dns *Msg) SetRcode(request *Msg, rcode int) {
	dns.MsgHdr.Rcode = rcode
	dns.MsgHdr.Opcode = OpcodeQuery
	dns.MsgHdr.Response = true
	dns.MsgHdr.Authoritative = false
	dns.MsgHdr.Id = request.MsgHdr.Id
	dns.Question = make([]Question, 1)
	dns.Question[0] = request.Question[0]
}

// SetRcodeFormatError creates a packet with FormError set.
func (dns *Msg) SetRcodeFormatError(request *Msg) {
	dns.MsgHdr.Rcode = RcodeFormatError
	dns.MsgHdr.Opcode = OpcodeQuery
	dns.MsgHdr.Response = true
	dns.MsgHdr.Authoritative = false
	dns.MsgHdr.Id = request.MsgHdr.Id
}

// SetUpdate makes the message a dynamic update packet. It
// sets the ZONE section to: z, TypeSOA, classINET.
func (dns *Msg) SetUpdate(z string) {
	dns.MsgHdr.Id = Id()
	dns.MsgHdr.Opcode = OpcodeUpdate
	dns.Question = make([]Question, 1)
	dns.Question[0] = Question{z, TypeSOA, ClassINET}
}

// SetIxfr creates dns msg suitable for requesting an ixfr.
func (dns *Msg) SetIxfr(z string, serial uint32) {
	dns.MsgHdr.Id = Id()
	dns.Question = make([]Question, 1)
	dns.Ns = make([]RR, 1)
	s := new(RR_SOA)
	s.Hdr = RR_Header{z, TypeSOA, ClassINET, DefaultTtl, 0}
	s.Serial = serial

	dns.Question[0] = Question{z, TypeIXFR, ClassINET}
	dns.Ns[0] = s
}

// SetAxfr creates dns msg suitable for requesting an axfr.
func (dns *Msg) SetAxfr(z string) {
	dns.MsgHdr.Id = Id()
	dns.Question = make([]Question, 1)
	dns.Question[0] = Question{z, TypeAXFR, ClassINET}
}

// SetTsig appends a TSIG RR to the message.
// This is only a skeleton Tsig RR that is added as the last RR in the
// additional section. The caller should then call TsigGenerate,
// to generate the complete TSIG with the secret.
func (dns *Msg) SetTsig(z, algo string, fudge uint16, timesigned uint64) {
	t := new(RR_TSIG)
	t.Hdr = RR_Header{z, TypeTSIG, ClassANY, 0, 0}
	t.Algorithm = algo
	t.Fudge = 300
	t.TimeSigned = timesigned
	dns.Extra = append(dns.Extra, t)
}

// SetEdns0 appends a EDNS0 OPT RR to the message.
// TSIG should always the last RR in a message.
func (dns *Msg) SetEdns0(udpsize uint16, do bool) {
	e := new(RR_OPT)
	e.Hdr.Name = "."
	e.Hdr.Rrtype = TypeOPT
	e.SetUDPSize(udpsize)
	if do {
		e.SetDo()
	}
	dns.Extra = append(dns.Extra, e)
}

// IsRcode checks if the header of the packet has rcode set.
func (dns *Msg) IsRcode(rcode int) (ok bool) {
	if len(dns.Question) == 0 {
		return false
	}
	ok = dns.MsgHdr.Rcode == rcode
	return
}

// IsQuestion returns true if the packet is a question.
func (dns *Msg) IsQuestion() (ok bool) {
	if len(dns.Question) == 0 {
		return false
	}
	ok = dns.MsgHdr.Response == false
	return
}

// IsRcodeFormatError checks if the message has FormErr set.
func (dns *Msg) IsRcodeFormatError() (ok bool) {
	if len(dns.Question) == 0 {
		return false
	}
	ok = dns.MsgHdr.Rcode == RcodeFormatError
	return
}

// IsUpdate checks if the message is a dynamic update packet.
func (dns *Msg) IsUpdate() (ok bool) {
	if len(dns.Question) == 0 {
		return false
	}
	ok = dns.MsgHdr.Opcode == OpcodeUpdate
	ok = ok && dns.Question[0].Qtype == TypeSOA
	return
}

// IsNotify checks if the message is a valid notify packet.
func (dns *Msg) IsNotify() (ok bool) {
	if len(dns.Question) == 0 {
		return false
	}
	ok = dns.MsgHdr.Opcode == OpcodeNotify
	ok = ok && dns.Question[0].Qclass == ClassINET
	ok = ok && dns.Question[0].Qtype == TypeSOA
	return
}

// IsAxfr checks if the message is a valid axfr request packet.
func (dns *Msg) IsAxfr() (ok bool) {
	if len(dns.Question) == 0 {
		return false
	}
	ok = dns.MsgHdr.Opcode == OpcodeQuery
	ok = ok && dns.Question[0].Qclass == ClassINET
	ok = ok && dns.Question[0].Qtype == TypeAXFR
	return
}

// IsIXfr checks if the message is a valid ixfr request packet.
func (dns *Msg) IsIxfr() (ok bool) {
	if len(dns.Question) == 0 {
		return false
	}
	ok = dns.MsgHdr.Opcode == OpcodeQuery
	ok = ok && dns.Question[0].Qclass == ClassINET
	ok = ok && dns.Question[0].Qtype == TypeIXFR
	return
}

// IsTsig checks if the message has a TSIG record as the last record
// in the additional section.
func (dns *Msg) IsTsig() (ok bool) {
	if len(dns.Extra) > 0 {
		return dns.Extra[len(dns.Extra)-1].Header().Rrtype == TypeTSIG
	}
	return
}

// IsEdns0 checks if the message has a Edns0 record, any EDNS0
// record in the additional section will do.
func (dns *Msg) IsEdns0() (ok bool) {
	for _, r := range dns.Extra {
		if r.Header().Rrtype == TypeOPT {
			return true
		}
	}
	return
}

// IsDomainName checks if s is a valid domainname, it returns
// the number of labels, total length and true, when a domain name is valid.
// When false is returned the labelcount and length are not defined.
func IsDomainName(s string) (uint8, uint8, bool) { // copied from net package.
	// See RFC 1035, RFC 3696.
	l := len(s)
	if l == 0 || l > 255 {
		return 0, 0, false
	}
	longer := 0
	// Simplify checking loop: make the name end in a dot.
	// Don't call Fqdn() to save another len(s).
	// Keep in mind that if we do this, we report a longer
	// length
	if s[l-1] != '.' {
		s += "."
		l++
		longer = 1
	}
        // Preloop check for root label
        if s == "." {
                return 0, 1, true
        }

	last := byte('.')
	ok := false // ok once we've seen a letter or digit
	partlen := 0
	labels := uint8(0)
	for i := 0; i < l; i++ {
		c := s[i]
		switch {
		default:
			return 0, uint8(l - longer), false
		case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '_' || c == '*':
			ok = true
			partlen++
		case c == '\\':
			// Ok
		case '0' <= c && c <= '9':
			ok = true
			partlen++
		case c == '-':
			// byte before dash cannot be dot
			if last == '.' {
				return 0, uint8(l - longer), false
			}
			partlen++
		case c == '.':
			// byte before dot cannot be dot, dash
			if last == '.' || last == '-' {
				return 0, uint8(l - longer), false
			}
			if last == '\\' { // Ok, escaped dot.
				partlen++
				break
			}
			if partlen > 63 || partlen == 0 {
				return 0, uint8(l - longer), false
			}
			partlen = 0
			labels++
		}
		last = c
	}
	return labels, uint8(l - longer), ok
}

// IsFqdn checks if a domain name is fully qualified.
func IsFqdn(s string) bool {
	if len(s) == 0 {
		return false // ?
	}
	return s[len(s)-1] == '.'
}

// Fqdns return the fully qualified domain name from s.
// If s is already fully qualified, it behaves as the identity function.
func Fqdn(s string) string {
	if IsFqdn(s) {
		return s
	}
	return s + "."
}
<commit_msg>First stab for IsSubDomain<commit_after>package dns

// Everything is assumed in the ClassINET class. If
// you need other classes you are on your own.

// SetReply creates a reply packet from a request message.
func (dns *Msg) SetReply(request *Msg) {
	dns.MsgHdr.Id = request.MsgHdr.Id
	dns.MsgHdr.Authoritative = true
	dns.MsgHdr.Response = true
	dns.MsgHdr.Opcode = OpcodeQuery
	dns.MsgHdr.Rcode = RcodeSuccess
	dns.Question = make([]Question, 1)
	dns.Question[0] = request.Question[0]
}

// SetQuestion creates a question packet.
func (dns *Msg) SetQuestion(z string, t uint16) {
	dns.MsgHdr.Id = Id()
	dns.MsgHdr.RecursionDesired = true
	dns.Question = make([]Question, 1)
	dns.Question[0] = Question{z, t, ClassINET}
}

// SetNotify creates a notify packet.
func (dns *Msg) SetNotify(z string) {
	dns.MsgHdr.Opcode = OpcodeNotify
	dns.MsgHdr.Authoritative = true
	dns.MsgHdr.Id = Id()
	dns.Question = make([]Question, 1)
	dns.Question[0] = Question{z, TypeSOA, ClassINET}
}

// SetRcode creates an error packet.
func (dns *Msg) SetRcode(request *Msg, rcode int) {
	dns.MsgHdr.Rcode = rcode
	dns.MsgHdr.Opcode = OpcodeQuery
	dns.MsgHdr.Response = true
	dns.MsgHdr.Authoritative = false
	dns.MsgHdr.Id = request.MsgHdr.Id
	dns.Question = make([]Question, 1)
	dns.Question[0] = request.Question[0]
}

// SetRcodeFormatError creates a packet with FormError set.
func (dns *Msg) SetRcodeFormatError(request *Msg) {
	dns.MsgHdr.Rcode = RcodeFormatError
	dns.MsgHdr.Opcode = OpcodeQuery
	dns.MsgHdr.Response = true
	dns.MsgHdr.Authoritative = false
	dns.MsgHdr.Id = request.MsgHdr.Id
}

// SetUpdate makes the message a dynamic update packet. It
// sets the ZONE section to: z, TypeSOA, classINET.
func (dns *Msg) SetUpdate(z string) {
	dns.MsgHdr.Id = Id()
	dns.MsgHdr.Opcode = OpcodeUpdate
	dns.Question = make([]Question, 1)
	dns.Question[0] = Question{z, TypeSOA, ClassINET}
}

// SetIxfr creates dns msg suitable for requesting an ixfr.
func (dns *Msg) SetIxfr(z string, serial uint32) {
	dns.MsgHdr.Id = Id()
	dns.Question = make([]Question, 1)
	dns.Ns = make([]RR, 1)
	s := new(RR_SOA)
	s.Hdr = RR_Header{z, TypeSOA, ClassINET, DefaultTtl, 0}
	s.Serial = serial

	dns.Question[0] = Question{z, TypeIXFR, ClassINET}
	dns.Ns[0] = s
}

// SetAxfr creates dns msg suitable for requesting an axfr.
func (dns *Msg) SetAxfr(z string) {
	dns.MsgHdr.Id = Id()
	dns.Question = make([]Question, 1)
	dns.Question[0] = Question{z, TypeAXFR, ClassINET}
}

// SetTsig appends a TSIG RR to the message.
// This is only a skeleton Tsig RR that is added as the last RR in the
// additional section. The caller should then call TsigGenerate,
// to generate the complete TSIG with the secret.
func (dns *Msg) SetTsig(z, algo string, fudge uint16, timesigned uint64) {
	t := new(RR_TSIG)
	t.Hdr = RR_Header{z, TypeTSIG, ClassANY, 0, 0}
	t.Algorithm = algo
	t.Fudge = 300
	t.TimeSigned = timesigned
	dns.Extra = append(dns.Extra, t)
}

// SetEdns0 appends a EDNS0 OPT RR to the message.
// TSIG should always the last RR in a message.
func (dns *Msg) SetEdns0(udpsize uint16, do bool) {
	e := new(RR_OPT)
	e.Hdr.Name = "."
	e.Hdr.Rrtype = TypeOPT
	e.SetUDPSize(udpsize)
	if do {
		e.SetDo()
	}
	dns.Extra = append(dns.Extra, e)
}

// IsRcode checks if the header of the packet has rcode set.
func (dns *Msg) IsRcode(rcode int) (ok bool) {
	if len(dns.Question) == 0 {
		return false
	}
	ok = dns.MsgHdr.Rcode == rcode
	return
}

// IsQuestion returns true if the packet is a question.
func (dns *Msg) IsQuestion() (ok bool) {
	if len(dns.Question) == 0 {
		return false
	}
	ok = dns.MsgHdr.Response == false
	return
}

// IsRcodeFormatError checks if the message has FormErr set.
func (dns *Msg) IsRcodeFormatError() (ok bool) {
	if len(dns.Question) == 0 {
		return false
	}
	ok = dns.MsgHdr.Rcode == RcodeFormatError
	return
}

// IsUpdate checks if the message is a dynamic update packet.
func (dns *Msg) IsUpdate() (ok bool) {
	if len(dns.Question) == 0 {
		return false
	}
	ok = dns.MsgHdr.Opcode == OpcodeUpdate
	ok = ok && dns.Question[0].Qtype == TypeSOA
	return
}

// IsNotify checks if the message is a valid notify packet.
func (dns *Msg) IsNotify() (ok bool) {
	if len(dns.Question) == 0 {
		return false
	}
	ok = dns.MsgHdr.Opcode == OpcodeNotify
	ok = ok && dns.Question[0].Qclass == ClassINET
	ok = ok && dns.Question[0].Qtype == TypeSOA
	return
}

// IsAxfr checks if the message is a valid axfr request packet.
func (dns *Msg) IsAxfr() (ok bool) {
	if len(dns.Question) == 0 {
		return false
	}
	ok = dns.MsgHdr.Opcode == OpcodeQuery
	ok = ok && dns.Question[0].Qclass == ClassINET
	ok = ok && dns.Question[0].Qtype == TypeAXFR
	return
}

// IsIXfr checks if the message is a valid ixfr request packet.
func (dns *Msg) IsIxfr() (ok bool) {
	if len(dns.Question) == 0 {
		return false
	}
	ok = dns.MsgHdr.Opcode == OpcodeQuery
	ok = ok && dns.Question[0].Qclass == ClassINET
	ok = ok && dns.Question[0].Qtype == TypeIXFR
	return
}

// IsTsig checks if the message has a TSIG record as the last record
// in the additional section.
func (dns *Msg) IsTsig() (ok bool) {
	if len(dns.Extra) > 0 {
		return dns.Extra[len(dns.Extra)-1].Header().Rrtype == TypeTSIG
	}
	return
}

// IsEdns0 checks if the message has a Edns0 record, any EDNS0
// record in the additional section will do.
func (dns *Msg) IsEdns0() (ok bool) {
	for _, r := range dns.Extra {
		if r.Header().Rrtype == TypeOPT {
			return true
		}
	}
	return
}

// IsDomainName checks if s is a valid domainname, it returns
// the number of labels, total length and true, when a domain name is valid.
// When false is returned the labelcount and length are not defined.
func IsDomainName(s string) (uint8, uint8, bool) { // copied from net package.
	// See RFC 1035, RFC 3696.
	l := len(s)
	if l == 0 || l > 255 {
		return 0, 0, false
	}
	longer := 0
	// Simplify checking loop: make the name end in a dot.
	// Don't call Fqdn() to save another len(s).
	// Keep in mind that if we do this, otherwise we report a length+1
	if s[l-1] != '.' {
		s += "."
		l++
		longer = 1
	}
        // Preloop check for root label
        if s == "." {
                return 0, 1, true
        }

	last := byte('.')
	ok := false // ok once we've seen a letter or digit
	partlen := 0
	labels := uint8(0)
	for i := 0; i < l; i++ {
		c := s[i]
		switch {
		default:
			return 0, uint8(l - longer), false
		case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '_' || c == '*':
			ok = true
			partlen++
		case c == '\\':
			// Ok
		case '0' <= c && c <= '9':
			ok = true
			partlen++
		case c == '-':
			// byte before dash cannot be dot
			if last == '.' {
				return 0, uint8(l - longer), false
			}
			partlen++
		case c == '.':
			// byte before dot cannot be dot, dash
			if last == '.' || last == '-' {
				return 0, uint8(l - longer), false
			}
			if last == '\\' { // Ok, escaped dot.
				partlen++
				break
			}
			if partlen > 63 || partlen == 0 {
				return 0, uint8(l - longer), false
			}
			partlen = 0
			labels++
		}
		last = c
	}
	return labels, uint8(l - longer), ok
}

// IsSubDomain checks if child is indeed a child of parent.
// In the DNS this is called in bailiwick.
func IsSubDomain(parent, child string) bool {
        // If the number of labels both domain name have
        // in common equals the number of labels of parent,
        // child is a subdomain of parent.
        plabs := SplitLabels(parent)
        clabs := SplitLabels(child)
        if len(plabs) < len(clabs) {
                // parent is smaller than child
                return false
        }
        // Copied from CompareLabels to prevent another SplitLabels
        n := 0
        p := len(plabs) - 1
        c := len(clabs) - 1
        for {
                if p < 0 || c < 0 {
                        break
                }
                if plabs[p] == clabs[c] {
                        n++
                } else {
                        break
                }
                p--
                c--
        }
        return n == len(plabs)
}

// IsFqdn checks if a domain name is fully qualified.
func IsFqdn(s string) bool {
        l := len(s)
	if l == 0 {
		return false // ?
	}
	return s[l-1] == '.'
}

// Fqdns return the fully qualified domain name from s.
// If s is already fully qualified, it behaves as the identity function.
func Fqdn(s string) string {
	if IsFqdn(s) {
		return s
	}
	return s + "."
}
<|endoftext|>


<commit_before>package libp2p

// This file contains all the default configuration options.

import (
	"crypto/rand"

	crypto "github.com/libp2p/go-libp2p-crypto"
	pstore "github.com/libp2p/go-libp2p-peerstore"
	secio "github.com/libp2p/go-libp2p-secio"
	tcp "github.com/libp2p/go-tcp-transport"
	ws "github.com/libp2p/go-ws-transport"
	multiaddr "github.com/multiformats/go-multiaddr"
	mplex "github.com/whyrusleeping/go-smux-multiplex"
	yamux "github.com/whyrusleeping/go-smux-yamux"
)

// DefaultSecurity is the default security option.
//
// Useful when you want to extend, but not replace, the supported transport
// security protocols.
var DefaultSecurity = Security(secio.ID, secio.New)

// DefaultMuxer configures libp2p to use the stream connection multiplexers.
//
// Use this option when you want to *extend* the set of multiplexers used by
// libp2p instead of replacing them.
var DefaultMuxers = ChainOptions(
	Muxer("/yamux/1.0.0", yamux.DefaultTransport),
	Muxer("/mplex/6.3.0", mplex.DefaultTransport),
)

// DefaultTransports are the default libp2p transports.
//
// Use this option when you want to *extend* the set of multiplexers used by
// libp2p instead of replacing them.
var DefaultTransports = ChainOptions(
	Transport(tcp.NewTCPTransport),
	Transport(ws.New),
)

// DefaultPeerstore configures libp2p to use the default peerstore.
var DefaultPeerstore Option = func(cfg *Config) error {
	return cfg.Apply(Peerstore(pstore.NewPeerstore()))
}

// RandomIdentity generates a random identity (default behaviour)
var RandomIdentity = func(cfg *Config) error {
	priv, _, err := crypto.GenerateKeyPairWithReader(crypto.RSA, 2048, rand.Reader)
	if err != nil {
		return err
	}
	return cfg.Apply(Identity(priv))
}

var DefaultListenAddrs = func(cfg *Config) error {
	defaultListenAddr, err := multiaddr.NewMultiaddr("/ip4/0.0.0.0/tcp/0")
	if err != nil {
		return err
	}
	return cfg.Apply(ListenAddrs(defaultListenAddr))
}

// Complete list of default options and when to fallback on them.
//
// Please *DON'T* specify default options any other way. Putting this all here
// makes tracking defaults *much* easier.
var defaults = []struct {
	fallback func(cfg *Config) bool
	opt      Option
}{
	{
		fallback: func(cfg *Config) bool { return cfg.Transports == nil },
		opt:      DefaultTransports,
	},
	{
		fallback: func(cfg *Config) bool { return cfg.Muxers == nil },
		opt:      DefaultMuxers,
	},
	{
		fallback: func(cfg *Config) bool { return !cfg.Insecure && cfg.SecurityTransports == nil },
		opt:      DefaultSecurity,
	},
	{
		fallback: func(cfg *Config) bool { return cfg.PeerKey == nil },
		opt:      RandomIdentity,
	},
	{
		fallback: func(cfg *Config) bool { return cfg.Peerstore == nil },
		opt:      DefaultPeerstore,
	},
	{
		fallback: func(cfg *Config) bool { return cfg.ListenAddrs == nil },
		opt:	  DefaultListenAddrs,
	},
}

// Defaults configures libp2p to use the default options. Can be combined with
// other options to *extend* the default options.
var Defaults Option = func(cfg *Config) error {
	for _, def := range defaults {
		if err := cfg.Apply(def.opt); err != nil {
			return err
		}
	}
	return nil
}

// FallbackDefaults applies default options to the libp2p node if and only if no
// other relevent options have been applied. will be appended to the options
// passed into New.
var FallbackDefaults Option = func(cfg *Config) error {
	for _, def := range defaults {
		if !def.fallback(cfg) {
			continue
		}
		if err := cfg.Apply(def.opt); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>making go fmt happy<commit_after>package libp2p

// This file contains all the default configuration options.

import (
	"crypto/rand"

	crypto "github.com/libp2p/go-libp2p-crypto"
	pstore "github.com/libp2p/go-libp2p-peerstore"
	secio "github.com/libp2p/go-libp2p-secio"
	tcp "github.com/libp2p/go-tcp-transport"
	ws "github.com/libp2p/go-ws-transport"
	multiaddr "github.com/multiformats/go-multiaddr"
	mplex "github.com/whyrusleeping/go-smux-multiplex"
	yamux "github.com/whyrusleeping/go-smux-yamux"
)

// DefaultSecurity is the default security option.
//
// Useful when you want to extend, but not replace, the supported transport
// security protocols.
var DefaultSecurity = Security(secio.ID, secio.New)

// DefaultMuxers configures libp2p to use the stream connection multiplexers.
//
// Use this option when you want to *extend* the set of multiplexers used by
// libp2p instead of replacing them.
var DefaultMuxers = ChainOptions(
	Muxer("/yamux/1.0.0", yamux.DefaultTransport),
	Muxer("/mplex/6.3.0", mplex.DefaultTransport),
)

// DefaultTransports are the default libp2p transports.
//
// Use this option when you want to *extend* the set of multiplexers used by
// libp2p instead of replacing them.
var DefaultTransports = ChainOptions(
	Transport(tcp.NewTCPTransport),
	Transport(ws.New),
)

// DefaultPeerstore configures libp2p to use the default peerstore.
var DefaultPeerstore Option = func(cfg *Config) error {
	return cfg.Apply(Peerstore(pstore.NewPeerstore()))
}

// RandomIdentity generates a random identity (default behaviour)
var RandomIdentity = func(cfg *Config) error {
	priv, _, err := crypto.GenerateKeyPairWithReader(crypto.RSA, 2048, rand.Reader)
	if err != nil {
		return err
	}
	return cfg.Apply(Identity(priv))
}

// DefaultListenAddrs configures libp2p to use default listen address
var DefaultListenAddrs = func(cfg *Config) error {
	defaultListenAddr, err := multiaddr.NewMultiaddr("/ip4/0.0.0.0/tcp/0")
	if err != nil {
		return err
	}
	return cfg.Apply(ListenAddrs(defaultListenAddr))
}

// Complete list of default options and when to fallback on them.
//
// Please *DON'T* specify default options any other way. Putting this all here
// makes tracking defaults *much* easier.
var defaults = []struct {
	fallback func(cfg *Config) bool
	opt      Option
}{
	{
		fallback: func(cfg *Config) bool { return cfg.Transports == nil },
		opt:      DefaultTransports,
	},
	{
		fallback: func(cfg *Config) bool { return cfg.Muxers == nil },
		opt:      DefaultMuxers,
	},
	{
		fallback: func(cfg *Config) bool { return !cfg.Insecure && cfg.SecurityTransports == nil },
		opt:      DefaultSecurity,
	},
	{
		fallback: func(cfg *Config) bool { return cfg.PeerKey == nil },
		opt:      RandomIdentity,
	},
	{
		fallback: func(cfg *Config) bool { return cfg.Peerstore == nil },
		opt:      DefaultPeerstore,
	},
	{
		fallback: func(cfg *Config) bool { return cfg.ListenAddrs == nil },
		opt:      DefaultListenAddrs,
	},
}

// Defaults configures libp2p to use the default options. Can be combined with
// other options to *extend* the default options.
var Defaults Option = func(cfg *Config) error {
	for _, def := range defaults {
		if err := cfg.Apply(def.opt); err != nil {
			return err
		}
	}
	return nil
}

// FallbackDefaults applies default options to the libp2p node if and only if no
// other relevent options have been applied. will be appended to the options
// passed into New.
var FallbackDefaults Option = func(cfg *Config) error {
	for _, def := range defaults {
		if !def.fallback(cfg) {
			continue
		}
		if err := cfg.Apply(def.opt); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"./src/dendrite"
	"bufio"
	"flag"
	"github.com/fizx/logs"
	"io"
	"log"
	"os"
	"runtime"
	"time"
)

var configFile = flag.String("f", "/etc/dendrite/config.yaml", "location of the config file")
var debug = flag.Bool("d", false, "log at DEBUG")
var logFile = flag.String("l", "/var/log/dendrite.log", "location of the log file")
var cpus = flag.Int("c", runtime.NumCPU(), "number of cpus to possibly use")
var quitAfter = flag.Float64("q", -1, "quit after this many seconds (useful for tests)")

func main() {
	flag.Parse()
	runtime.GOMAXPROCS(*cpus)

	// set the logger path
	handle, err := os.OpenFile(*logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		logs.Warn("Unable to open log file %s, using stderr: %s", *logFile, err)
	} else {
		logs.Logger = log.New(handle, "", log.LstdFlags|log.Lshortfile)
	}

	// Check whether we're in debug mode
	if *debug {
		logs.SetLevel(logs.DEBUG)
		logs.Debug("logging at DEBUG")
	} else {
		logs.SetLevel(logs.INFO)
	}

	// Read the config files
	config, err := dendrite.NewConfig(*configFile)
	if err != nil {
		logs.Fatal("Can't read configuration: %s", err)
	}

	// Link up all of the objects
	ch := make(chan dendrite.Record, 100)
	logs.Debug("original %s", ch)
	dests := config.CreateDestinations()
	groups := config.CreateAllTailGroups(ch)

	// If any of our destinations talk back, log it.
	go func() {
		reader := bufio.NewReader(dests.Reader())
		for {
			str, err := reader.ReadString('\n')
			if err == io.EOF {
				logs.Debug("eof")
				time.Sleep(1 * time.Second)
			} else if err != nil {
				logs.Error("error reading: %s", err)
			} else {
				logs.Info("received: %s", str)
			}
		}
	}()

	// Do the event loop
	finished := make(chan bool, 0)
	go dests.Consume(ch, finished)
	if *quitAfter >= 0 {
		start := time.Now()
		logs.Debug("starting the poll")
		for {
			groups.Poll()
			if time.Now().Sub(start) >= time.Duration((*quitAfter)*float64(time.Second)) {
				break
			}
		}
	} else {
		logs.Debug("starting the loop")
		groups.Loop()
	}
	logs.Info("Closing...")
	close(ch)
	<-finished
	logs.Info("Goodbye!")
}
<commit_msg>make import path absolute<commit_after>package main

import (
	"github.com/onemorecloud/dendrite/src/dendrite"
	"bufio"
	"flag"
	"github.com/fizx/logs"
	"io"
	"log"
	"os"
	"runtime"
	"time"
)

var configFile = flag.String("f", "/etc/dendrite/config.yaml", "location of the config file")
var debug = flag.Bool("d", false, "log at DEBUG")
var logFile = flag.String("l", "/var/log/dendrite.log", "location of the log file")
var cpus = flag.Int("c", runtime.NumCPU(), "number of cpus to possibly use")
var quitAfter = flag.Float64("q", -1, "quit after this many seconds (useful for tests)")

func main() {
	flag.Parse()
	runtime.GOMAXPROCS(*cpus)

	// set the logger path
	handle, err := os.OpenFile(*logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		logs.Warn("Unable to open log file %s, using stderr: %s", *logFile, err)
	} else {
		logs.Logger = log.New(handle, "", log.LstdFlags|log.Lshortfile)
	}

	// Check whether we're in debug mode
	if *debug {
		logs.SetLevel(logs.DEBUG)
		logs.Debug("logging at DEBUG")
	} else {
		logs.SetLevel(logs.INFO)
	}

	// Read the config files
	config, err := dendrite.NewConfig(*configFile)
	if err != nil {
		logs.Fatal("Can't read configuration: %s", err)
	}

	// Link up all of the objects
	ch := make(chan dendrite.Record, 100)
	logs.Debug("original %s", ch)
	dests := config.CreateDestinations()
	groups := config.CreateAllTailGroups(ch)

	// If any of our destinations talk back, log it.
	go func() {
		reader := bufio.NewReader(dests.Reader())
		for {
			str, err := reader.ReadString('\n')
			if err == io.EOF {
				logs.Debug("eof")
				time.Sleep(1 * time.Second)
			} else if err != nil {
				logs.Error("error reading: %s", err)
			} else {
				logs.Info("received: %s", str)
			}
		}
	}()

	// Do the event loop
	finished := make(chan bool, 0)
	go dests.Consume(ch, finished)
	if *quitAfter >= 0 {
		start := time.Now()
		logs.Debug("starting the poll")
		for {
			groups.Poll()
			if time.Now().Sub(start) >= time.Duration((*quitAfter)*float64(time.Second)) {
				break
			}
		}
	} else {
		logs.Debug("starting the loop")
		groups.Loop()
	}
	logs.Info("Closing...")
	close(ch)
	<-finished
	logs.Info("Goodbye!")
}
<|endoftext|>


<commit_before>package api

// ServerEnvironment represents the read-only environment fields of a LXD server
type ServerEnvironment struct {
	Addresses              []string `json:"addresses" yaml:"addresses"`
	Architectures          []string `json:"architectures" yaml:"architectures"`
	Certificate            string   `json:"certificate" yaml:"certificate"`
	CertificateFingerprint string   `json:"certificate_fingerprint" yaml:"certificate_fingerprint"`
	Driver                 string   `json:"driver" yaml:"driver"`
	DriverVersion          string   `json:"driver_version" yaml:"driver_version"`
	Kernel                 string   `json:"kernel" yaml:"kernel"`
	KernelArchitecture     string   `json:"kernel_architecture" yaml:"kernel_architecture"`
	KernelVersion          string   `json:"kernel_version" yaml:"kernel_version"`
	Server                 string   `json:"server" yaml:"server"`
	ServerPid              int      `json:"server_pid" yaml:"server_pid"`
	ServerVersion          string   `json:"server_version" yaml:"server_version"`
	Storage                string   `json:"storage" yaml:"storage"`
	StorageVersion         string   `json:"storage_version" yaml:"storage_version"`

	// API extension: clustering
	ServerClustered bool   `json:"server_clustered" yaml:"server_clustered"`
	ServerName      string `json:"server_name" yaml:"server_name"`

	// API extension: projects
	Project string `json:"project" yaml:"project"`
}

// ServerPut represents the modifiable fields of a LXD server configuration
type ServerPut struct {
	Config map[string]interface{} `json:"config" yaml:"config"`
}

// ServerUntrusted represents a LXD server for an untrusted client
type ServerUntrusted struct {
	APIExtensions []string `json:"api_extensions" yaml:"api_extensions"`
	APIStatus     string   `json:"api_status" yaml:"api_status"`
	APIVersion    string   `json:"api_version" yaml:"api_version"`
	Auth          string   `json:"auth" yaml:"auth"`
	Public        bool     `json:"public" yaml:"public"`

	// API extension: macaroon_authentication
	AuthMethods []string `json:"auth_methods" yaml:"auth_methods"`
}

// Server represents a LXD server
type Server struct {
	ServerPut       `yaml:",inline"`
	ServerUntrusted `yaml:",inline"`

	Environment ServerEnvironment `json:"environment" yaml:"environment"`
}

// Writable converts a full Server struct into a ServerPut struct (filters read-only fields)
func (srv *Server) Writable() ServerPut {
	return srv.ServerPut
}
<commit_msg>shared/api: Add KernelFeatures<commit_after>package api

// ServerEnvironment represents the read-only environment fields of a LXD server
type ServerEnvironment struct {
	Addresses              []string `json:"addresses" yaml:"addresses"`
	Architectures          []string `json:"architectures" yaml:"architectures"`
	Certificate            string   `json:"certificate" yaml:"certificate"`
	CertificateFingerprint string   `json:"certificate_fingerprint" yaml:"certificate_fingerprint"`
	Driver                 string   `json:"driver" yaml:"driver"`
	DriverVersion          string   `json:"driver_version" yaml:"driver_version"`
	Kernel                 string   `json:"kernel" yaml:"kernel"`
	KernelArchitecture     string   `json:"kernel_architecture" yaml:"kernel_architecture"`
	KernelVersion          string   `json:"kernel_version" yaml:"kernel_version"`
	Server                 string   `json:"server" yaml:"server"`
	ServerPid              int      `json:"server_pid" yaml:"server_pid"`
	ServerVersion          string   `json:"server_version" yaml:"server_version"`
	Storage                string   `json:"storage" yaml:"storage"`
	StorageVersion         string   `json:"storage_version" yaml:"storage_version"`

	// API extension: clustering
	ServerClustered bool   `json:"server_clustered" yaml:"server_clustered"`
	ServerName      string `json:"server_name" yaml:"server_name"`

	// API extension: projects
	Project string `json:"project" yaml:"project"`

	// API extension: kernel_features
	KernelFeatures map[string]string `json:"kernel_features" yaml:"kernel_features"`
}

// ServerPut represents the modifiable fields of a LXD server configuration
type ServerPut struct {
	Config map[string]interface{} `json:"config" yaml:"config"`
}

// ServerUntrusted represents a LXD server for an untrusted client
type ServerUntrusted struct {
	APIExtensions []string `json:"api_extensions" yaml:"api_extensions"`
	APIStatus     string   `json:"api_status" yaml:"api_status"`
	APIVersion    string   `json:"api_version" yaml:"api_version"`
	Auth          string   `json:"auth" yaml:"auth"`
	Public        bool     `json:"public" yaml:"public"`

	// API extension: macaroon_authentication
	AuthMethods []string `json:"auth_methods" yaml:"auth_methods"`
}

// Server represents a LXD server
type Server struct {
	ServerPut       `yaml:",inline"`
	ServerUntrusted `yaml:",inline"`

	Environment ServerEnvironment `json:"environment" yaml:"environment"`
}

// Writable converts a full Server struct into a ServerPut struct (filters read-only fields)
func (srv *Server) Writable() ServerPut {
	return srv.ServerPut
}
<|endoftext|>


<commit_before>// +build linux

package shared

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"syscall"
	"unsafe"

	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/shared/units"
)

// --- pure Go functions ---

func Major(dev uint64) int {
	return int(((dev >> 8) & 0xfff) | ((dev >> 32) & (0xfffff000)))
}

func Minor(dev uint64) int {
	return int((dev & 0xff) | ((dev >> 12) & (0xffffff00)))
}

func GetFileStat(p string) (uid int, gid int, major int, minor int,
	inode uint64, nlink int, err error) {
	var stat unix.Stat_t
	err = unix.Lstat(p, &stat)
	if err != nil {
		return
	}
	uid = int(stat.Uid)
	gid = int(stat.Gid)
	inode = uint64(stat.Ino)
	nlink = int(stat.Nlink)
	major = -1
	minor = -1
	if stat.Mode&unix.S_IFBLK != 0 || stat.Mode&unix.S_IFCHR != 0 {
		major = Major(stat.Rdev)
		minor = Minor(stat.Rdev)
	}

	return
}

// GetPathMode returns a os.FileMode for the provided path
func GetPathMode(path string) (os.FileMode, error) {
	fi, err := os.Stat(path)
	if err != nil {
		return os.FileMode(0000), err
	}

	mode, _, _ := GetOwnerMode(fi)
	return mode, nil
}

func parseMountinfo(name string) int {
	// In case someone uses symlinks we need to look for the actual
	// mountpoint.
	actualPath, err := filepath.EvalSymlinks(name)
	if err != nil {
		return -1
	}

	f, err := os.Open("/proc/self/mountinfo")
	if err != nil {
		return -1
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := scanner.Text()
		tokens := strings.Fields(line)
		if len(tokens) < 5 {
			return -1
		}
		cleanPath := filepath.Clean(tokens[4])
		if cleanPath == actualPath {
			return 1
		}
	}

	return 0
}

func IsMountPoint(name string) bool {
	ret := parseMountinfo(name)
	if ret == 1 {
		return true
	}

	stat, err := os.Stat(name)
	if err != nil {
		return false
	}

	rootStat, err := os.Lstat(name + "/..")
	if err != nil {
		return false
	}
	// If the directory has the same device as parent, then it's not a mountpoint.
	return stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev
}

func SetSize(fd int, width int, height int) (err error) {
	var dimensions [4]uint16
	dimensions[0] = uint16(height)
	dimensions[1] = uint16(width)

	if _, _, err := unix.Syscall6(unix.SYS_IOCTL, uintptr(fd), uintptr(unix.TIOCSWINSZ), uintptr(unsafe.Pointer(&dimensions)), 0, 0, 0); err != 0 {
		return err
	}
	return nil
}

// This uses ssize_t llistxattr(const char *path, char *list, size_t size); to
// handle symbolic links (should it in the future be possible to set extended
// attributed on symlinks): If path is a symbolic link the extended attributes
// associated with the link itself are retrieved.
func llistxattr(path string, list []byte) (sz int, err error) {
	var _p0 *byte
	_p0, err = unix.BytePtrFromString(path)
	if err != nil {
		return
	}
	var _p1 unsafe.Pointer
	if len(list) > 0 {
		_p1 = unsafe.Pointer(&list[0])
	} else {
		_p1 = unsafe.Pointer(nil)
	}
	r0, _, e1 := unix.Syscall(unix.SYS_LLISTXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(list)))
	sz = int(r0)
	if e1 != 0 {
		err = e1
	}
	return
}

// GetAllXattr retrieves all extended attributes associated with a file,
// directory or symbolic link.
func GetAllXattr(path string) (xattrs map[string]string, err error) {
	e1 := fmt.Errorf("Extended attributes changed during retrieval")

	// Call llistxattr() twice: First, to determine the size of the buffer
	// we need to allocate to store the extended attributes, second, to
	// actually store the extended attributes in the buffer. Also, check if
	// the size/number of extended attributes hasn't changed between the two
	// calls.
	pre, err := llistxattr(path, nil)
	if err != nil || pre < 0 {
		return nil, err
	}
	if pre == 0 {
		return nil, nil
	}

	dest := make([]byte, pre)

	post, err := llistxattr(path, dest)
	if err != nil || post < 0 {
		return nil, err
	}
	if post != pre {
		return nil, e1
	}

	split := strings.Split(string(dest), "\x00")
	if split == nil {
		return nil, fmt.Errorf("No valid extended attribute key found")
	}
	// *listxattr functions return a list of  names  as  an unordered array
	// of null-terminated character strings (attribute names are separated
	// by null bytes ('\0')), like this: user.name1\0system.name1\0user.name2\0
	// Since we split at the '\0'-byte the last element of the slice will be
	// the empty string. We remove it:
	if split[len(split)-1] == "" {
		split = split[:len(split)-1]
	}

	xattrs = make(map[string]string, len(split))

	for _, x := range split {
		xattr := string(x)
		// Call Getxattr() twice: First, to determine the size of the
		// buffer we need to allocate to store the extended attributes,
		// second, to actually store the extended attributes in the
		// buffer. Also, check if the size of the extended attribute
		// hasn't changed between the two calls.
		pre, err = unix.Getxattr(path, xattr, nil)
		if err != nil || pre < 0 {
			return nil, err
		}

		dest = make([]byte, pre)
		post := 0
		if pre > 0 {
			post, err = unix.Getxattr(path, xattr, dest)
			if err != nil || post < 0 {
				return nil, err
			}
		}

		if post != pre {
			return nil, e1
		}

		xattrs[xattr] = string(dest)
	}

	return xattrs, nil
}

var ObjectFound = fmt.Errorf("Found requested object")

func LookupUUIDByBlockDevPath(diskDevice string) (string, error) {
	uuid := ""
	readUUID := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if (info.Mode() & os.ModeSymlink) == os.ModeSymlink {
			link, err := os.Readlink(path)
			if err != nil {
				return err
			}

			// filepath.Join() will call Clean() on the result and
			// thus resolve those ugly "../../" parts that make it
			// hard to compare the strings.
			absPath := filepath.Join("/dev/disk/by-uuid", link)
			if absPath == diskDevice {
				uuid = path
				// Will allows us to avoid needlessly travers
				// the whole directory.
				return ObjectFound
			}
		}
		return nil
	}

	err := filepath.Walk("/dev/disk/by-uuid", readUUID)
	if err != nil && err != ObjectFound {
		return "", fmt.Errorf("Failed to detect UUID: %s", err)
	}

	if uuid == "" {
		return "", fmt.Errorf("Failed to detect UUID")
	}

	lastSlash := strings.LastIndex(uuid, "/")
	return uuid[lastSlash+1:], nil
}

// Detect whether err is an errno.
func GetErrno(err error) (errno error, iserrno bool) {
	sysErr, ok := err.(*os.SyscallError)
	if ok {
		return sysErr.Err, true
	}

	pathErr, ok := err.(*os.PathError)
	if ok {
		return pathErr.Err, true
	}

	tmpErrno, ok := err.(unix.Errno)
	if ok {
		return tmpErrno, true
	}

	return nil, false
}

// Utsname returns the same info as unix.Utsname, as strings
type Utsname struct {
	Sysname    string
	Nodename   string
	Release    string
	Version    string
	Machine    string
	Domainname string
}

// Uname returns Utsname as strings
func Uname() (*Utsname, error) {
	/*
	 * Based on: https://groups.google.com/forum/#!topic/golang-nuts/Jel8Bb-YwX8
	 * there is really no better way to do this, which is
	 * unfortunate. Also, we ditch the more accepted CharsToString
	 * version in that thread, since it doesn't seem as portable,
	 * viz. github issue #206.
	 */

	uname := unix.Utsname{}
	err := unix.Uname(&uname)
	if err != nil {
		return nil, err
	}

	return &Utsname{
		Sysname:    intArrayToString(uname.Sysname),
		Nodename:   intArrayToString(uname.Nodename),
		Release:    intArrayToString(uname.Release),
		Version:    intArrayToString(uname.Version),
		Machine:    intArrayToString(uname.Machine),
		Domainname: intArrayToString(uname.Domainname),
	}, nil
}

func intArrayToString(arr interface{}) string {
	slice := reflect.ValueOf(arr)
	s := ""
	for i := 0; i < slice.Len(); i++ {
		val := slice.Index(i)
		valInt := int64(-1)

		switch val.Kind() {
		case reflect.Int:
		case reflect.Int8:
			valInt = int64(val.Int())
		case reflect.Uint:
		case reflect.Uint8:
			valInt = int64(val.Uint())
		default:
			continue
		}

		if valInt == 0 {
			break
		}

		s += string(byte(valInt))
	}

	return s
}

func Statvfs(path string) (*unix.Statfs_t, error) {
	var st unix.Statfs_t

	err := unix.Statfs(path, &st)
	if err != nil {
		return nil, err
	}

	return &st, nil
}

func DeviceTotalMemory() (int64, error) {
	// Open /proc/meminfo
	f, err := os.Open("/proc/meminfo")
	if err != nil {
		return -1, err
	}
	defer f.Close()

	// Read it line by line
	scan := bufio.NewScanner(f)
	for scan.Scan() {
		line := scan.Text()

		// We only care about MemTotal
		if !strings.HasPrefix(line, "MemTotal:") {
			continue
		}

		// Extract the before last (value) and last (unit) fields
		fields := strings.Split(line, " ")
		value := fields[len(fields)-2] + fields[len(fields)-1]

		// Feed the result to units.ParseByteSizeString to get an int value
		valueBytes, err := units.ParseByteSizeString(value)
		if err != nil {
			return -1, err
		}

		return valueBytes, nil
	}

	return -1, fmt.Errorf("Couldn't find MemTotal")
}
<commit_msg>shared/util/linux: Removes Major and Minor functions<commit_after>// +build linux

package shared

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"syscall"
	"unsafe"

	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/shared/units"
)

// --- pure Go functions ---

func GetFileStat(p string) (uid int, gid int, major uint32, minor uint32, inode uint64, nlink int, err error) {
	var stat unix.Stat_t
	err = unix.Lstat(p, &stat)
	if err != nil {
		return
	}
	uid = int(stat.Uid)
	gid = int(stat.Gid)
	inode = uint64(stat.Ino)
	nlink = int(stat.Nlink)
	if stat.Mode&unix.S_IFBLK != 0 || stat.Mode&unix.S_IFCHR != 0 {
		major = unix.Major(stat.Rdev)
		minor = unix.Minor(stat.Rdev)
	}

	return
}

// GetPathMode returns a os.FileMode for the provided path
func GetPathMode(path string) (os.FileMode, error) {
	fi, err := os.Stat(path)
	if err != nil {
		return os.FileMode(0000), err
	}

	mode, _, _ := GetOwnerMode(fi)
	return mode, nil
}

func parseMountinfo(name string) int {
	// In case someone uses symlinks we need to look for the actual
	// mountpoint.
	actualPath, err := filepath.EvalSymlinks(name)
	if err != nil {
		return -1
	}

	f, err := os.Open("/proc/self/mountinfo")
	if err != nil {
		return -1
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := scanner.Text()
		tokens := strings.Fields(line)
		if len(tokens) < 5 {
			return -1
		}
		cleanPath := filepath.Clean(tokens[4])
		if cleanPath == actualPath {
			return 1
		}
	}

	return 0
}

func IsMountPoint(name string) bool {
	ret := parseMountinfo(name)
	if ret == 1 {
		return true
	}

	stat, err := os.Stat(name)
	if err != nil {
		return false
	}

	rootStat, err := os.Lstat(name + "/..")
	if err != nil {
		return false
	}
	// If the directory has the same device as parent, then it's not a mountpoint.
	return stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev
}

func SetSize(fd int, width int, height int) (err error) {
	var dimensions [4]uint16
	dimensions[0] = uint16(height)
	dimensions[1] = uint16(width)

	if _, _, err := unix.Syscall6(unix.SYS_IOCTL, uintptr(fd), uintptr(unix.TIOCSWINSZ), uintptr(unsafe.Pointer(&dimensions)), 0, 0, 0); err != 0 {
		return err
	}
	return nil
}

// This uses ssize_t llistxattr(const char *path, char *list, size_t size); to
// handle symbolic links (should it in the future be possible to set extended
// attributed on symlinks): If path is a symbolic link the extended attributes
// associated with the link itself are retrieved.
func llistxattr(path string, list []byte) (sz int, err error) {
	var _p0 *byte
	_p0, err = unix.BytePtrFromString(path)
	if err != nil {
		return
	}
	var _p1 unsafe.Pointer
	if len(list) > 0 {
		_p1 = unsafe.Pointer(&list[0])
	} else {
		_p1 = unsafe.Pointer(nil)
	}
	r0, _, e1 := unix.Syscall(unix.SYS_LLISTXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(list)))
	sz = int(r0)
	if e1 != 0 {
		err = e1
	}
	return
}

// GetAllXattr retrieves all extended attributes associated with a file,
// directory or symbolic link.
func GetAllXattr(path string) (xattrs map[string]string, err error) {
	e1 := fmt.Errorf("Extended attributes changed during retrieval")

	// Call llistxattr() twice: First, to determine the size of the buffer
	// we need to allocate to store the extended attributes, second, to
	// actually store the extended attributes in the buffer. Also, check if
	// the size/number of extended attributes hasn't changed between the two
	// calls.
	pre, err := llistxattr(path, nil)
	if err != nil || pre < 0 {
		return nil, err
	}
	if pre == 0 {
		return nil, nil
	}

	dest := make([]byte, pre)

	post, err := llistxattr(path, dest)
	if err != nil || post < 0 {
		return nil, err
	}
	if post != pre {
		return nil, e1
	}

	split := strings.Split(string(dest), "\x00")
	if split == nil {
		return nil, fmt.Errorf("No valid extended attribute key found")
	}
	// *listxattr functions return a list of  names  as  an unordered array
	// of null-terminated character strings (attribute names are separated
	// by null bytes ('\0')), like this: user.name1\0system.name1\0user.name2\0
	// Since we split at the '\0'-byte the last element of the slice will be
	// the empty string. We remove it:
	if split[len(split)-1] == "" {
		split = split[:len(split)-1]
	}

	xattrs = make(map[string]string, len(split))

	for _, x := range split {
		xattr := string(x)
		// Call Getxattr() twice: First, to determine the size of the
		// buffer we need to allocate to store the extended attributes,
		// second, to actually store the extended attributes in the
		// buffer. Also, check if the size of the extended attribute
		// hasn't changed between the two calls.
		pre, err = unix.Getxattr(path, xattr, nil)
		if err != nil || pre < 0 {
			return nil, err
		}

		dest = make([]byte, pre)
		post := 0
		if pre > 0 {
			post, err = unix.Getxattr(path, xattr, dest)
			if err != nil || post < 0 {
				return nil, err
			}
		}

		if post != pre {
			return nil, e1
		}

		xattrs[xattr] = string(dest)
	}

	return xattrs, nil
}

var ObjectFound = fmt.Errorf("Found requested object")

func LookupUUIDByBlockDevPath(diskDevice string) (string, error) {
	uuid := ""
	readUUID := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if (info.Mode() & os.ModeSymlink) == os.ModeSymlink {
			link, err := os.Readlink(path)
			if err != nil {
				return err
			}

			// filepath.Join() will call Clean() on the result and
			// thus resolve those ugly "../../" parts that make it
			// hard to compare the strings.
			absPath := filepath.Join("/dev/disk/by-uuid", link)
			if absPath == diskDevice {
				uuid = path
				// Will allows us to avoid needlessly travers
				// the whole directory.
				return ObjectFound
			}
		}
		return nil
	}

	err := filepath.Walk("/dev/disk/by-uuid", readUUID)
	if err != nil && err != ObjectFound {
		return "", fmt.Errorf("Failed to detect UUID: %s", err)
	}

	if uuid == "" {
		return "", fmt.Errorf("Failed to detect UUID")
	}

	lastSlash := strings.LastIndex(uuid, "/")
	return uuid[lastSlash+1:], nil
}

// Detect whether err is an errno.
func GetErrno(err error) (errno error, iserrno bool) {
	sysErr, ok := err.(*os.SyscallError)
	if ok {
		return sysErr.Err, true
	}

	pathErr, ok := err.(*os.PathError)
	if ok {
		return pathErr.Err, true
	}

	tmpErrno, ok := err.(unix.Errno)
	if ok {
		return tmpErrno, true
	}

	return nil, false
}

// Utsname returns the same info as unix.Utsname, as strings
type Utsname struct {
	Sysname    string
	Nodename   string
	Release    string
	Version    string
	Machine    string
	Domainname string
}

// Uname returns Utsname as strings
func Uname() (*Utsname, error) {
	/*
	 * Based on: https://groups.google.com/forum/#!topic/golang-nuts/Jel8Bb-YwX8
	 * there is really no better way to do this, which is
	 * unfortunate. Also, we ditch the more accepted CharsToString
	 * version in that thread, since it doesn't seem as portable,
	 * viz. github issue #206.
	 */

	uname := unix.Utsname{}
	err := unix.Uname(&uname)
	if err != nil {
		return nil, err
	}

	return &Utsname{
		Sysname:    intArrayToString(uname.Sysname),
		Nodename:   intArrayToString(uname.Nodename),
		Release:    intArrayToString(uname.Release),
		Version:    intArrayToString(uname.Version),
		Machine:    intArrayToString(uname.Machine),
		Domainname: intArrayToString(uname.Domainname),
	}, nil
}

func intArrayToString(arr interface{}) string {
	slice := reflect.ValueOf(arr)
	s := ""
	for i := 0; i < slice.Len(); i++ {
		val := slice.Index(i)
		valInt := int64(-1)

		switch val.Kind() {
		case reflect.Int:
		case reflect.Int8:
			valInt = int64(val.Int())
		case reflect.Uint:
		case reflect.Uint8:
			valInt = int64(val.Uint())
		default:
			continue
		}

		if valInt == 0 {
			break
		}

		s += string(byte(valInt))
	}

	return s
}

func Statvfs(path string) (*unix.Statfs_t, error) {
	var st unix.Statfs_t

	err := unix.Statfs(path, &st)
	if err != nil {
		return nil, err
	}

	return &st, nil
}

func DeviceTotalMemory() (int64, error) {
	// Open /proc/meminfo
	f, err := os.Open("/proc/meminfo")
	if err != nil {
		return -1, err
	}
	defer f.Close()

	// Read it line by line
	scan := bufio.NewScanner(f)
	for scan.Scan() {
		line := scan.Text()

		// We only care about MemTotal
		if !strings.HasPrefix(line, "MemTotal:") {
			continue
		}

		// Extract the before last (value) and last (unit) fields
		fields := strings.Split(line, " ")
		value := fields[len(fields)-2] + fields[len(fields)-1]

		// Feed the result to units.ParseByteSizeString to get an int value
		valueBytes, err := units.ParseByteSizeString(value)
		if err != nil {
			return -1, err
		}

		return valueBytes, nil
	}

	return -1, fmt.Errorf("Couldn't find MemTotal")
}
<|endoftext|>


<commit_before>// huffman.go
package huffman

import (
	"fmt"
	"os"
	"sort"
)

/*
	问题:生成huffman树的结点数大于256个时，1个byte的索引已经无法表示。
	解决方案1:将索引扩展至2byte,将导致字典体积增大一倍
	解决方案2:压缩时，控制block大小，使得生成的huffman树不多于256个节点
		对于过小的block，采取不压缩的方式
	解决方案3:将绝对索引换成相对索引,但是需证明在最坏的情况下,都可以保证1个byte的相对索引可以访问到全部字典
*/

const block_size uint16 = 0x8000
const eob_mark uint16 = 0x1000

type h_node struct {
	left   *h_node
	right  *h_node
	weight uint16
	value  uint16 // 此处应该能表示任意内容
}
type nodelist []*h_node

//为*h_node添加String()方法，便于输出
func (p *h_node) String() string {
	return fmt.Sprintf("[%p]<-this->[%p],\t[%d],\t[%#02X]", p.left, p.right, p.weight, p.value)
}

//为*h_node添加DotString()方法，输出dot格式字符串
func (p *h_node) DotString() string {
	var str string
	if p.left != nil {
		str += fmt.Sprintf("node_%p[label=\"{%d}\"];\n", p, p.weight)
		str += fmt.Sprintf("node_%p->node_%p[headlabel=\"L\"];\n", p, p.left)
		str += fmt.Sprintf("node_%p->node_%p[headlabel=\"R\"];\n", p, p.right)
	} else {
		str += fmt.Sprintf("node_%p[label=\"%d|%#02X\"];\n", p, p.weight, p.value)
	}
	return str
}

// huffman dict struct
type h_dict struct {
	n    uint16 // content
	l    uint8  // length(bit)
	bits uint32
}

// huffman dict struct
type bits struct {
	l    uint8 // length(bit)
	bits uint32
}

/*
每个字典项占2bytes,分为l和r,
如果l!=r,则表示此项非叶子项,
	其左右子枝分别为第l和r项,下标索引即为2*l和2*r
	如果l的索引是它本身,且r==0,则此项为block结束标志
如果l==r,则表示此项为叶子项,
	l=其编码内容
解码时,从根节点开始跟随bit流中的1或0,进行跳转,直到叶子,得到解码内容
*/
//为*h_node添加Decode_dict()方法，输出以此node为root的解码字典
func (p *h_node) Decode_dict(dict *[]uint8) int {
	pos := len(*dict)
	if p.left != nil {
		*dict = append(*dict, 0, 0) // 占位
		left := p.left.Decode_dict(dict)
		right := p.right.Decode_dict(dict)
		(*dict)[pos] = uint8(left / 2)
		(*dict)[pos+1] = uint8(right / 2)
	} else {
		if p.value == eob_mark {
			*dict = append(*dict, uint8(pos/2), 0)
		} else {
			*dict = append(*dict, uint8(p.value), uint8(p.value))
		}
	}
	return pos
}

//为*h_node添加Encode_dict()方法，输出以此node为root的编码字典
// 1 for left, 0 for right
func (p *h_node) Encode_dict(dict *map[uint16]bits, bin bits) {
	if p.left != nil {
		p.left.Encode_dict(dict, bits{bin.l + 1, bin.bits | (0x80000000 >> bin.l)})
		p.right.Encode_dict(dict, bits{bin.l + 1, bin.bits & (^(0x80000000 >> bin.l))})
		return
	}
	(*dict)[p.value] = bits{bin.l, bin.bits}
}

func traverse(p *h_node, hOutput *os.File) {
	if p.left != nil {
		traverse(p.left, hOutput)
	}
	if p.right != nil {
		traverse(p.right, hOutput)
	}
	hOutput.WriteString(p.DotString())
}

// 输出node树的dot格式字符串
func (p *h_node) Dot(index string) {
	hOutput, _ := os.Create("huffman" + index + ".dot")
	hOutput.WriteString("digraph structs {\nnode[shape=record];\n")
	traverse(p, hOutput)
	hOutput.WriteString("}\n")
	hOutput.Close()
}

func Compress(hFile *os.File, hOutput *os.File) {
	var block_index uint16 = 0
	build_huffman_tree := func(buffer []byte) (leaf nodelist) {
		leaf = make(nodelist, 0, 256)
		// 1.统计
		var table [256]uint16
		for _, v := range buffer {
			table[v]++
		}
		// 2.叶子初始化
		for k, v := range table {
			if v > 0 {
				t_leaf := h_node{nil, nil, v, uint16(k)}
				leaf = append(leaf, &t_leaf)
			}
		}
		// end of block mark
		t_eob := h_node{nil, nil, 1, eob_mark}
		leaf = append(leaf, &t_eob)
		for {
			// 3.排序
			sort.Sort(nodelist(leaf))
			// 4.种树
			if len(leaf) > 1 {
				tleft := leaf[0]
				tright := leaf[1]
				root := h_node{tleft, tright, tleft.weight + tright.weight, 0}
				leaf = leaf[1:]
				leaf[0] = &root
			} else {
				leaf[0].Dot(fmt.Sprintf("%d", block_index))
				block_index += 1
				break
			}
		}
		return leaf
	}
	huffman_compress := func(hOutput *os.File, r_buf []byte, root *nodelist) {
		// byte拼接
		// n 表示最后一个元素占用的bit数
		bits2byte := func(arr []byte, n uint8, bin bits) ([]byte, uint8) {
			for bin.l > 0 {
				if n == 0 {
					arr = append(arr, 0)
				}
				arr[len(arr)-1] |= byte(bin.bits >> uint(n+24))
				temp := 8 - n
				if temp > bin.l {
					n += bin.l
				} else {
					n = 0
				}
				if bin.l > temp {
					bin.l -= temp
				} else {
					bin.l = 0
				}
				bin.bits <<= temp
			}
			ret := arr
			return ret, n
		}
		// 生成编码字典
		dict_e := make(map[uint16]bits)
		(*root)[0].Encode_dict(&dict_e, bits{0, 0})
		// 生成解码字典
		dict_d := make([]uint8, 0, 256)
		(*root)[0].Decode_dict(&dict_d)
		// 写入解码字典
		hOutput.Write([]byte{byte(len(dict_d) / 2)})
		hOutput.Write(dict_d)
		var n uint8 = 0
		var t_bytes []byte // temp byte slice
		for _, v := range r_buf {
			bin := dict_e[uint16(v)]
			t_bytes, n = bits2byte(t_bytes, n, bin)
		}
		t_bytes, n = bits2byte(t_bytes, n, dict_e[eob_mark])
		hOutput.Write(t_bytes)
	}
	hFile.Seek(0, os.SEEK_SET)
	hOutput.Seek(0, os.SEEK_SET)
	buf := make([]byte, block_size)
	for {
		n, err := hFile.Read(buf)
		if n > 0 {
			tree := build_huffman_tree(buf[:n])
			huffman_compress(hOutput, buf[:n], &tree)
		}
		if err != nil {
			if err.Error() == "EOF" {
				hOutput.Close()
				return
			} else {
				panic(err)
			}
		}
	}
}

func Decompress(hFile *os.File, hOutput *os.File) {
	hFile.Seek(0, os.SEEK_SET)
	hOutput.Seek(0, os.SEEK_SET)
	var decode_ptr uint16 = 0 // 指向第n个节点(索引为2*n)
	var debug_cnt uint32 = 0
	huffman_decode := func(bytes byte, dict []byte) bool {
		var i uint8
		for i = 0; i < 8; i++ {
			if bytes&(0x80>>i) > 0 {
				decode_ptr = uint16(dict[decode_ptr*2])
			} else {
				decode_ptr = uint16(dict[decode_ptr*2+1])
			}

			if dict[decode_ptr*2] == dict[decode_ptr*2+1] { // decode
				hOutput.Write([]byte{dict[decode_ptr*2]})
				debug_cnt++
				decode_ptr = 0
			} else if uint16(dict[decode_ptr*2]) == decode_ptr && dict[decode_ptr*2+1] == 0 { // block end
				decode_ptr = 0
				return true
			}
		}
		return false
	}
	for {
		dictLength := make([]byte, 1)
		// 读取字典长度
		_, err := hFile.Read(dictLength)
		if err != nil {
			if err.Error() == "EOF" {
				hOutput.Close()
				return
			} else {
				panic(err)
			}
		}
		// 读取字典
		dict := make([]byte, uint16(dictLength[0])*2)
		_, err = hFile.Read(dict)
		// 解码
		buffer := make([]byte, 0x1)
		for {
			_, err = hFile.Read(buffer)
			if err != nil {
				panic(err)
			}
			if huffman_decode(buffer[0], dict) {
				break
			}
		}
	}
}

func (I nodelist) Len() int {
	return len(I)
}
func (I nodelist) Less(i, j int) bool {
	return I[i].weight < I[j].weight
}
func (I nodelist) Swap(i, j int) {
	I[i], I[j] = I[j], I[i]
}
<commit_msg>暂时采用解决方案1<commit_after>// huffman.go
package huffman

import (
	"fmt"
	"os"
	"sort"
)

/*
	问题:生成huffman树的结点数大于256个时，1个byte的索引已经无法表示。
	解决方案1:将索引扩展至2byte,将导致字典体积增大一倍
	解决方案2:压缩时，控制block大小，使得生成的huffman树不多于256个节点
		对于过小的block，采取不压缩的方式
	解决方案3:将绝对索引换成相对索引,但是需证明在最坏的情况下,都可以保证1个byte的相对索引可以访问到全部字典
*/

/*
	暂时采用方案1，
	1. 使用12bit表示字典长度(结点数)(每个结点3bytes)
	2. 每个结点均使用三个字节，左右索引分别占12bits
	3. 第1,3字节分别表示左右索引的低8位，第2字节表示左右索引的高4位。
*/

// 解码字典结点size
const dict_unit_size uint8 = 3

var block_size uint16 = 0xFFFF

const eob_mark uint16 = 0x1000

type h_node struct {
	left   *h_node
	right  *h_node
	weight uint16
	value  uint16 // 此处应该能表示任意内容
}
type nodelist []*h_node

//为*h_node添加String()方法，便于输出
func (p *h_node) String() string {
	return fmt.Sprintf("[%p]<-this->[%p],\t[%d],\t[%#02X]", p.left, p.right, p.weight, p.value)
}

//为*h_node添加DotString()方法，输出dot格式字符串
func (p *h_node) DotString() string {
	var str string
	if p.left != nil {
		str += fmt.Sprintf("node_%p[label=\"{%d}\"];\n", p, p.weight)
		str += fmt.Sprintf("node_%p->node_%p[headlabel=\"L\"];\n", p, p.left)
		str += fmt.Sprintf("node_%p->node_%p[headlabel=\"R\"];\n", p, p.right)
	} else {
		str += fmt.Sprintf("node_%p[label=\"%d|%#02X\"];\n", p, p.weight, p.value)
	}
	return str
}

// huffman dict struct
type h_dict struct {
	n    uint16 // content
	l    uint8  // length(bit)
	bits uint32
}

// huffman dict struct
type bits struct {
	l    uint8 // length(bit)
	bits uint32
}

/*
每个字典项占4bytes,分为l和r,
如果l!=r,则表示此项非叶子项,
	其左右子枝分别为偏移l和r项,下标索引即为this+4*l和this+4*r
	如果l的索引是0x7FFF,且r==0,则此项为block结束标志
如果l==r,则表示此项为叶子项,
	l=其编码内容
解码时,从根节点开始跟随bit流中的1或0,进行跳转,直到叶子,得到解码内容
*/
//为*h_node添加Decode_dict()方法，输出以此node为root的解码字典
func (p *h_node) Decode_dict(dict *[]uint8) int {
	pos := len(*dict)
	if p.left != nil {
		*dict = append(*dict, 0, 0, 0) // 占位
		left := p.left.Decode_dict(dict)
		right := p.right.Decode_dict(dict)
		(*dict)[pos] = uint8((left / 3) & 0xFF)
		(*dict)[pos+1] = uint8((((left / 3) >> 4) & 0xF0) | (((right / 3) >> 8) & 0x0F))
		(*dict)[pos+2] = uint8((right / 3) & 0xFF)
	} else {
		if p.value == eob_mark {
			*dict = append(*dict, uint8(pos/3), uint8(((pos/3)>>4)&0xF0), 0)
		} else {
			*dict = append(*dict, uint8(p.value), 0, uint8(p.value))
		}
	}
	return pos
}

//为*h_node添加Encode_dict()方法，输出以此node为root的编码字典
// 1 for left, 0 for right
func (p *h_node) Encode_dict(dict *map[uint16]bits, bin bits) {
	if p.left != nil {
		p.left.Encode_dict(dict, bits{bin.l + 1, bin.bits | (0x80000000 >> bin.l)})
		p.right.Encode_dict(dict, bits{bin.l + 1, bin.bits & (^(0x80000000 >> bin.l))})
		return
	}
	(*dict)[p.value] = bits{bin.l, bin.bits}
}

func traverse(p *h_node, hOutput *os.File) {
	if p.left != nil {
		traverse(p.left, hOutput)
	}
	if p.right != nil {
		traverse(p.right, hOutput)
	}
	hOutput.WriteString(p.DotString())
}

// 输出node树的dot格式字符串
func (p *h_node) Dot(index string) {
	hOutput, _ := os.Create("huffman" + index + ".dot")
	hOutput.WriteString("digraph structs {\nnode[shape=record];\n")
	traverse(p, hOutput)
	hOutput.WriteString("}\n")
	hOutput.Close()
}

func Compress(hFile *os.File, hOutput *os.File) {
	var block_index uint16 = 0
	build_huffman_tree := func(buffer []byte) (leaf nodelist) {
		leaf = make(nodelist, 0, 256)
		// 1.统计
		var table [256]uint16
		for _, v := range buffer {
			table[v]++
		}
		// 2.叶子初始化
		for k, v := range table {
			if v > 0 {
				t_leaf := h_node{nil, nil, v, uint16(k)}
				leaf = append(leaf, &t_leaf)
			}
		}
		// end of block mark
		t_eob := h_node{nil, nil, 1, eob_mark}
		leaf = append(leaf, &t_eob)
		for {
			// 3.排序
			sort.Sort(nodelist(leaf))
			// 4.种树
			if len(leaf) > 1 {
				tleft := leaf[0]
				tright := leaf[1]
				root := h_node{tleft, tright, tleft.weight + tright.weight, 0}
				leaf = leaf[1:]
				leaf[0] = &root
			} else {
				// uncomment this line to output the dot file of huffman tree
				// leaf[0].Dot(fmt.Sprintf("%d", block_index))
				block_index += 1
				break
			}
		}
		fmt.Println("len(leaf) =", len(leaf))
		return leaf
	}
	huffman_compress := func(hOutput *os.File, r_buf []byte, root *nodelist) {
		// byte拼接
		// n 表示最后一个元素占用的bit数
		bits2byte := func(arr []byte, n uint8, bin bits) ([]byte, uint8) {
			for bin.l > 0 {
				if n == 0 {
					arr = append(arr, 0)
				}
				arr[len(arr)-1] |= byte(bin.bits >> uint(n+24))
				temp := 8 - n
				if temp > bin.l {
					n += bin.l
				} else {
					n = 0
				}
				if bin.l > temp {
					bin.l -= temp
				} else {
					bin.l = 0
				}
				bin.bits <<= temp
			}
			ret := arr
			return ret, n
		}
		// 生成编码字典
		dict_e := make(map[uint16]bits)
		(*root)[0].Encode_dict(&dict_e, bits{0, 0})
		// 生成解码字典
		dict_d := make([]uint8, 0, 256)
		(*root)[0].Decode_dict(&dict_d)
		// 写入解码字典(2bytes LE)
		hOutput.Write([]byte{byte((len(dict_d) / 3) & 0xFF), byte(((len(dict_d) / 3) >> 8) & 0xFF)})
		hOutput.Write(dict_d)
		var n uint8 = 0
		var t_bytes []byte // temp byte slice
		for _, v := range r_buf {
			bin := dict_e[uint16(v)]
			t_bytes, n = bits2byte(t_bytes, n, bin)
		}
		t_bytes, n = bits2byte(t_bytes, n, dict_e[eob_mark])
		hOutput.Write(t_bytes)
	}
	hFile.Seek(0, os.SEEK_SET)
	hOutput.Seek(0, os.SEEK_SET)
	buf := make([]byte, block_size)
	//t_block_size := block_size
	for {
		n, err := hFile.Read(buf)
		if n > 0 {
			tree := build_huffman_tree(buf[:n])
			huffman_compress(hOutput, buf[:n], &tree)
		}
		if err != nil {
			if err.Error() == "EOF" {
				hOutput.Close()
				return
			} else {
				panic(err)
			}
		}
	}
}

func Decompress(hFile *os.File, hOutput *os.File) {
	hFile.Seek(0, os.SEEK_SET)
	hOutput.Seek(0, os.SEEK_SET)
	var decode_ptr uint16 = 0 // 指向第n个节点(索引为3*n)
	var debug_cnt uint32 = 0
	get_decode_lr := func(ptr uint16, flag bool, dict []byte) uint16 {
		// flag: 1 for left, 0 for right
		if flag {
			return uint16(dict[ptr*3]) + (uint16(dict[ptr*3+1])<<4)&0xF00
		} else {
			return uint16(dict[ptr*3+2]) + (uint16(dict[ptr*3+1])<<8)&0xF00
		}
	}
	huffman_decode := func(bytes byte, dict []byte) bool {
		var i uint8
		for i = 0; i < 8; i++ {
			if bytes&(0x80>>i) > 0 {
				decode_ptr = get_decode_lr(decode_ptr, true, dict)
			} else {
				decode_ptr = get_decode_lr(decode_ptr, false, dict)
			}
			if dict[decode_ptr*3] == dict[decode_ptr*3+2] && dict[decode_ptr*3+1] == 0 { // decode
				hOutput.Write([]byte{dict[decode_ptr*3]})
				debug_cnt++
				decode_ptr = 0
			} else if uint16(dict[decode_ptr*3])+(uint16(dict[decode_ptr*3+1])<<4)&0xF00 == decode_ptr && dict[decode_ptr*3+2] == 0 { // block end
				decode_ptr = 0
				return true
			}
		}
		return false
	}
	for {
		dictLength := make([]byte, 2)
		// 读取字典长度
		_, err := hFile.Read(dictLength)
		if err != nil {
			if err.Error() == "EOF" {
				hOutput.Close()
				return
			} else {
				panic(err)
			}
		}
		// 读取字典
		dict := make([]byte, (uint16(dictLength[0])|(uint16(dictLength[1])<<8))*3)
		_, err = hFile.Read(dict)
		// 解码
		buffer := make([]byte, 0x1)
		for {
			_, err = hFile.Read(buffer)
			if err != nil {
				panic(err)
			}
			if huffman_decode(buffer[0], dict) {
				fmt.Println("EOB")
				break
			}
		}
	}
}

func (I nodelist) Len() int {
	return len(I)
}
func (I nodelist) Less(i, j int) bool {
	return I[i].weight < I[j].weight
}
func (I nodelist) Swap(i, j int) {
	I[i], I[j] = I[j], I[i]
}
<|endoftext|>