<commit_before>package main

import (
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"time"
	"strings"
)

var promotions = []string{"", "1STPI", "2STPI",
    "3EII", "3GM", "3GCU", "3GMA", "3INFO", "3SGM", "3SRC",
    "4EII", "4GM", "4GCU", "4GMA", "4INFO", "4SGM", "4SRC",
    "5EII", "5GM", "5GCU", "5GMA", "5INFO", "5SGM", "5SRC",
    "Personnel/Enseignant"}

// User defines how to model a User
type User struct {
	ID          bson.ObjectId   `bson:"_id,omitempty"`
	Name        string          `json:"name"`
	Username    string          `json:"username"`
	Description string          `json:"description"`
	Email       string          `json:"email"`
	EmailPublic bool            `json:"emailpublic"`
	Promotion   string          `json:"promotion"`
	Gender 			string					`json:"gender"`
	Events      []bson.ObjectId `json:"events"`
	PostsLiked  []bson.ObjectId `json:"postsliked"`
}

// Users is an array of User
type Users []User

// AddUser will add the given user from JSON body to the database
func AddUser(user User) User {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	user.Username = strings.ToLower(user.Username)
	db.Insert(user)
	var result User
	db.Find(bson.M{"username": strings.ToLower(user.Username) }).One(&result)
	return result
}

// UpdateUser will update the user link to the given ID,
// with the field of the given user, in the database
func UpdateUser(id bson.ObjectId, user User) User {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	promotion := ""
	for _, promo := range promotions {
		if user.Promotion == promo {
			promotion = user.Promotion
			break
		}
	}
	userID := bson.M{"_id": id}
	change := bson.M{"$set": bson.M{
		"name":        user.Name,
		"description": user.Description,
		"email": 			 user.Email,
		"emailpublic": user.EmailPublic,
		"promotion":   promotion,
		"gender"	:		 user.Gender,
	}}
	db.Update(userID, change)
	var result User
	db.Find(bson.M{"_id": id}).One(&result)
	return result
}

// DeleteUser will delete the given user from the database
func DeleteUser(user User) User {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	DeleteCredentalsForUser(user.ID)
	DeleteNotificationsForUser(user.ID)
	DeleteNotificationTokenForUser(user.ID)
	for _, eventId := range user.Events{
		RemoveParticipant(eventId, user.ID)
	}
	for _, postId := range user.PostsLiked{
		DislikePostWithUser(postId, user.ID)
	}
	DeleteTagsForUser(user.ID)
	DeleteCommentsForUser(user.ID)
	db.RemoveId(user.ID)
	var result User
	db.FindId(user.ID).One(result)
	return result
}

// GetUser will return an User object from the given ID
func GetAllUser() Users {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	var result Users
	db.Find(bson.M{}).All(&result)
	return result
}

// GetUser will return an User object from the given ID
func GetUser(id bson.ObjectId) User {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	var result User
	db.FindId(id).One(&result)
	return result
}

// LikePost will add the postID to the list of liked post
// of the user linked to the given id
func LikePost(id bson.ObjectId, postID bson.ObjectId) User {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	userID := bson.M{"_id": id}
	change := bson.M{"$addToSet": bson.M{
		"postsliked": postID,
	}}
	db.Update(userID, change)
	var result User
	db.Find(bson.M{"_id": id}).One(&result)
	return result
}

// DislikePost will remove the postID from the list of liked
// post of the user linked to the given id
func DislikePost(id bson.ObjectId, postID bson.ObjectId) User {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	userID := bson.M{"_id": id}
	change := bson.M{"$pull": bson.M{
		"postsliked": postID,
	}}
	db.Update(userID, change)
	var result User
	db.Find(bson.M{"_id": id}).One(&result)
	return result
}

// AddEventToUser will add the eventID to the list
// of the user's event linked to the given id
func AddEventToUser(id bson.ObjectId, eventID bson.ObjectId) User {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	userID := bson.M{"_id": id}
	change := bson.M{"$addToSet": bson.M{
		"events": eventID,
	}}
	db.Update(userID, change)
	var result User
	db.Find(bson.M{"_id": id}).One(&result)
	return result
}

// RemoveEventFromUser will remove the eventID from the list
// of the user's event linked to the given id
func RemoveEventFromUser(id bson.ObjectId, eventID bson.ObjectId) User {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	userID := bson.M{"_id": id}
	change := bson.M{"$pull": bson.M{
		"events": eventID,
	}}
	db.Update(userID, change)
	var result User
	db.Find(bson.M{"_id": id}).One(&result)
	return result
}

func SearchUser(username string) Users {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	var result Users
	db.Find(bson.M{"$or" : []interface{}{
		bson.M{"username" : bson.M{ "$regex" : bson.RegEx{`^.*` + username + `.*`, "i"}}}, bson.M{"name" : bson.M{ "$regex" : bson.RegEx{`^.*` + username + `.*`, "i"}}}}}).All(&result)
	return result
}

func ReportUser(id bson.ObjectId, reporterID bson.ObjectId) {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	var user User
	db.Find(bson.M{"_id": id}).One(&user)
	var reporter User
	db.Find(bson.M{"_id": reporterID}).One(&reporter)
	SendEmail("aeir@insa-rennes.fr", "Un utilisateur a été reporté sur Insapp",
		"Cet utilisateur a été reporté le " + time.Now().String() +
		"\n\n" + reporter.ID.Hex() + "\n" + reporter.Username + "\n" + reporter.Name +
		"\n\n" + user.ID.Hex() + "\n" + user.Username + "\n" + user.Name + "\n" + user.Description)
}
<commit_msg>update report system<commit_after>package main

import (
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"time"
	"strings"
)

var promotions = []string{"", "1STPI", "2STPI",
    "3EII", "3GM", "3GCU", "3GMA", "3INFO", "3SGM", "3SRC",
    "4EII", "4GM", "4GCU", "4GMA", "4INFO", "4SGM", "4SRC",
    "5EII", "5GM", "5GCU", "5GMA", "5INFO", "5SGM", "5SRC",
    "Personnel/Enseignant"}

// User defines how to model a User
type User struct {
	ID          bson.ObjectId   `bson:"_id,omitempty"`
	Name        string          `json:"name"`
	Username    string          `json:"username"`
	Description string          `json:"description"`
	Email       string          `json:"email"`
	EmailPublic bool            `json:"emailpublic"`
	Promotion   string          `json:"promotion"`
	Gender 			string					`json:"gender"`
	Events      []bson.ObjectId `json:"events"`
	PostsLiked  []bson.ObjectId `json:"postsliked"`
}

// Users is an array of User
type Users []User

// AddUser will add the given user from JSON body to the database
func AddUser(user User) User {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	user.Username = strings.ToLower(user.Username)
	db.Insert(user)
	var result User
	db.Find(bson.M{"username": strings.ToLower(user.Username) }).One(&result)
	return result
}

// UpdateUser will update the user link to the given ID,
// with the field of the given user, in the database
func UpdateUser(id bson.ObjectId, user User) User {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	promotion := ""
	for _, promo := range promotions {
		if user.Promotion == promo {
			promotion = user.Promotion
			break
		}
	}
	userID := bson.M{"_id": id}
	change := bson.M{"$set": bson.M{
		"name":        user.Name,
		"description": user.Description,
		"email": 			 user.Email,
		"emailpublic": user.EmailPublic,
		"promotion":   promotion,
		"gender"	:		 user.Gender,
	}}
	db.Update(userID, change)
	var result User
	db.Find(bson.M{"_id": id}).One(&result)
	return result
}

// DeleteUser will delete the given user from the database
func DeleteUser(user User) User {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	DeleteCredentalsForUser(user.ID)
	DeleteNotificationsForUser(user.ID)
	DeleteNotificationTokenForUser(user.ID)
	for _, eventId := range user.Events{
		RemoveParticipant(eventId, user.ID)
	}
	for _, postId := range user.PostsLiked{
		DislikePostWithUser(postId, user.ID)
	}
	DeleteTagsForUser(user.ID)
	DeleteCommentsForUser(user.ID)
	db.RemoveId(user.ID)
	var result User
	db.FindId(user.ID).One(result)
	return result
}

// GetUser will return an User object from the given ID
func GetAllUser() Users {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	var result Users
	db.Find(bson.M{}).All(&result)
	return result
}

// GetUser will return an User object from the given ID
func GetUser(id bson.ObjectId) User {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	var result User
	db.FindId(id).One(&result)
	return result
}

// LikePost will add the postID to the list of liked post
// of the user linked to the given id
func LikePost(id bson.ObjectId, postID bson.ObjectId) User {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	userID := bson.M{"_id": id}
	change := bson.M{"$addToSet": bson.M{
		"postsliked": postID,
	}}
	db.Update(userID, change)
	var result User
	db.Find(bson.M{"_id": id}).One(&result)
	return result
}

// DislikePost will remove the postID from the list of liked
// post of the user linked to the given id
func DislikePost(id bson.ObjectId, postID bson.ObjectId) User {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	userID := bson.M{"_id": id}
	change := bson.M{"$pull": bson.M{
		"postsliked": postID,
	}}
	db.Update(userID, change)
	var result User
	db.Find(bson.M{"_id": id}).One(&result)
	return result
}

// AddEventToUser will add the eventID to the list
// of the user's event linked to the given id
func AddEventToUser(id bson.ObjectId, eventID bson.ObjectId) User {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	userID := bson.M{"_id": id}
	change := bson.M{"$addToSet": bson.M{
		"events": eventID,
	}}
	db.Update(userID, change)
	var result User
	db.Find(bson.M{"_id": id}).One(&result)
	return result
}

// RemoveEventFromUser will remove the eventID from the list
// of the user's event linked to the given id
func RemoveEventFromUser(id bson.ObjectId, eventID bson.ObjectId) User {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	userID := bson.M{"_id": id}
	change := bson.M{"$pull": bson.M{
		"events": eventID,
	}}
	db.Update(userID, change)
	var result User
	db.Find(bson.M{"_id": id}).One(&result)
	return result
}

func SearchUser(username string) Users {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	var result Users
	db.Find(bson.M{"$or" : []interface{}{
		bson.M{"username" : bson.M{ "$regex" : bson.RegEx{`^.*` + username + `.*`, "i"}}}, bson.M{"name" : bson.M{ "$regex" : bson.RegEx{`^.*` + username + `.*`, "i"}}}}}).All(&result)
	return result
}

func ReportUser(id bson.ObjectId, reporterID bson.ObjectId) {
	session, _ := mgo.Dial("127.0.0.1")
	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	db := session.DB("insapp").C("user")
	var user User
	db.Find(bson.M{"_id": id}).One(&user)
	var reporter User
	db.Find(bson.M{"_id": reporterID}).One(&reporter)
	SendEmail("aeir@insa-rennes.fr", "Un utilisateur a été reporté sur Insapp",
		"Cet utilisateur a été reporté le " + time.Now().String() +
		"\n\nReporteur:\n" + reporter.ID.Hex() + "\n" + reporter.Username + "\n" + reporter.Name +
		"\n\nSignaler:\n" + user.ID.Hex() + "\n" + user.Username + "\n" + user.Name + "\n" + user.Description)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/url"
	"os"

	"github.com/urfave/cli"
)

func main() {
	app := cli.NewApp()
	app.Name = "yuri"
	app.Usage = "parse the urlz!"
	app.Action = func(c *cli.Context) error {
		if len(c.Args()) < 1 {
			log.Fatal("No arguments. Usage: yuri <URI>")
		}

		if len(c.Args()) > 1 {
			log.Fatal("More than 1 argument. Usage: yuri <URI>")
		}

		uri := c.Args().First()
		parsedURI, err := url.Parse(uri)
		if err != nil {
			log.Fatal(err)
		}

		var m map[string]string
		m = make(map[string]string)

		m["scheme"] = parsedURI.Scheme
		m["opaque"] = parsedURI.Opaque
		m["host"] = parsedURI.Host
		m["path"] = parsedURI.Path
		m["rawpath"] = parsedURI.RawPath
		m["rawquery"] = parsedURI.RawQuery
		m["username"] = parsedURI.User.Username()
		m["password"], _ = parsedURI.User.Password()

		b, err := json.Marshal(m)
		if err != nil {
			fmt.Println("error:", err)
		}
		os.Stdout.Write(b)

		return nil
	}

	app.Run(os.Args)
}
<commit_msg>Fix panic if `ParsedURI.User` is nil<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/url"
	"os"

	"github.com/urfave/cli"
)

func main() {
	app := cli.NewApp()
	app.Name = "yuri"
	app.Usage = "parse the urlz!"
	app.Action = func(c *cli.Context) error {
		if len(c.Args()) < 1 {
			log.Fatal("No arguments. Usage: yuri <URI>")
		}

		if len(c.Args()) > 1 {
			log.Fatal("More than 1 argument. Usage: yuri <URI>")
		}

		uri := c.Args().First()
		parsedURI, err := url.Parse(uri)
		if err != nil {
			log.Fatal(err)
		}

		var m map[string]string
		m = make(map[string]string)

		// TODO: Factor out into function for testing
		m["scheme"] = parsedURI.Scheme
		m["opaque"] = parsedURI.Opaque
		m["host"] = parsedURI.Host
		m["path"] = parsedURI.Path
		m["rawpath"] = parsedURI.RawPath
		m["rawquery"] = parsedURI.RawQuery

		if parsedURI.User == nil {
			m["username"] = ""
			m["password"] = ""
		} else {
			m["username"] = parsedURI.User.Username()
			m["password"], _ = parsedURI.User.Password()
		}

		b, err := json.Marshal(m)
		if err != nil {
			fmt.Println("error:", err)
		}
		os.Stdout.Write(b)

		return nil
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package flux

import (
	"io"
	slog "log"
	"sync/atomic"

	"github.com/op/go-logging"
)

type (

	//HalfStackable defines a stackable function type that can return a value
	HalfStackable func(interface{})

	//Stackable defines a stackable function type that can return a value
	Stackable func(interface{}, Stacks) interface{}

	//Stacks provides the finite definition of function stacks
	Stacks interface {
		Stack(Stackable, bool) Stacks
		Listen(HalfStackable) Stacks
		Unstack()

		//Isolate applies to this stack only and does not propagate the value to other stacks
		Isolate(interface{}) interface{}

		//Identity returns the value apply to it as the return value
		Identity(interface{}) interface{}

		//Call runs the value, returning the return value of just this particular stacks return value
		Call(interface{}) interface{}

		//Apply applies the value, returning the values of either this stack or the child stacks if any exists
		Apply(interface{}) interface{}

		//Lift lifts this value up the root stack for use with the root .Apply() function(allows you to call a root .Apply() method from anylevel of the stack)
		Lift(interface{}) interface{}

		//LiftApply calls the stack from bottom up and takes the result from a lower stack to apply to its parent stack
		LiftApply(interface{}) interface{}
		Levitate(interface{}) interface{}

		getWrap() *StackWrap
		Close() error
		LockClose(Stacks)
	}

	//StackWrap provides a single-layer two function binder
	StackWrap struct {
		Fn      Stackable
		next    *StackWrap
		owner   Stacks
		closefx func()
	}

	//Stack is the concrete definition of Stacks
	Stack struct {
		wrap   *StackWrap
		root   Stacks
		active int64
	}

	//StackStreamers provides a streaming function for stacks member rules
	StackStreamers interface {
		Stacks
		StreamWriter(io.Writer) Stacks
		Write([]byte) (int, error)
		Read([]byte) (int, error)
		Closed() Stacks
	}

	//StackStream is the concrete implementation for stacks
	StackStream struct {
		Stacks
		closeNotifier Stacks
	}

	//Reports provides a abstraction of report
	Reports struct {
		Tag  string
		Meta map[string]interface{}
	}

	//StackReport provides a internal bank of reports
	StackReport struct {
		stacks chan *Reports
		ended  int64
	}
)

var (
	//StackableIdentity provides an identity function for stacks
	StackableIdentity = func(b interface{}, _ Stacks) interface{} {
		return b
	}
)

//ReportCard returns a new StackReport instance, generally ensure its a good max so you can get reports without over-filling the stack,if the stack is full then old reports would be discard
func ReportCard(max int) *StackReport {
	return &StackReport{
		stacks: make(chan *Reports, max),
	}
}

//Checkout returns a Stack which gets filled with the current and possible future reports
func (r *StackReport) Checkout() StackStreamers {
	cs := NewIdentityStream()
	GoDefer("CheckoutReport", func() {
		for {

			state := atomic.LoadInt64(&r.ended)

			if state > 0 {
				break
			}

			select {
			case v, ok := <-r.stacks:
				if !ok {
					return
				}
				cs.Call(v)
			}
		}
	})
	return cs
}

//Report adds a new report to the reportstack
func (r *StackReport) Report(tag string, meta map[string]interface{}) {
	state := atomic.LoadInt64(&r.ended)

	if state > 0 {
		return
	}

	curcap := cap(r.stacks)
	curlen := len(r.stacks)

	if curlen >= curcap {
		GoDefer("DiscardTopReport", func() {
			<-r.stacks
		})
	}

	r.stacks <- &Reports{tag, meta}
}

//Close ends the report card operations
func (r *StackReport) Close() {
	state := atomic.LoadInt64(&r.ended)
	if state > 0 {
		return
	}

	atomic.StoreInt64(&r.ended, 1)
	close(r.stacks)
}

//NewIdentityStream  returns new StackStream
func NewIdentityStream() StackStreamers {
	return NewStackStream(StackableIdentity)
}

//StreamReader streams from a reader,hence all values are replaced by the data in the io.Reader
func StreamReader(w io.Reader) (s StackStreamers) {
	s = NewStackStream(StackableIdentity)
	go func() {
		_, _ = io.Copy(s, w)
	}()
	return
}

//Read reads into a byte but its empty
func (s *StackStream) Read(bu []byte) (int, error) {
	return 0, io.ErrNoProgress
}

//Write writes into the stream
func (s *StackStream) Write(bu []byte) (int, error) {
	s.Call(bu)
	return len(bu), nil
}

//StreamWriter streams to a writer
func (s *StackStream) StreamWriter(w io.Writer) Stacks {
	return s.Stack(func(data interface{}, _ Stacks) interface{} {
		buff, ok := data.([]byte)

		if !ok {

			str, ok := data.(string)

			if ok {
				_, _ = w.Write([]byte(str))
			}

		}
		_, _ = w.Write(buff)
		return data
	}, true)
}

//Closed returns a stack that is emitted when closed
func (s *StackStream) Closed() Stacks {
	return s.closeNotifier
}

//Close ends this stacks connections
func (s *StackStream) Close() error {
	defer s.closeNotifier.Close()
	s.closeNotifier.Call(true)
	s.Stacks.Close()
	return nil
}

//NewStackStream returns a new stackstream
func NewStackStream(mx Stackable) StackStreamers {
	return &StackStream{
		Stacks:        NewStack(mx),
		closeNotifier: NewStack(StackableIdentity),
	}
}

//Close destroys the stackwrap
func (s *StackWrap) Close() {
	if s.next != nil {
		if s.next.owner != nil {
			s.next.owner.Close()
		}
	}
	s.Fn = nil
	s.owner = nil
	if s.closefx != nil {
		defer func() { s.closefx = nil }()
		s.closefx()
	}
}

//Unwrap unwraps a Stackwrap for release
func (s *StackWrap) Unwrap() *StackWrap {
	nx := s.next
	s.next = nil
	return nx
}

//Rewrap wraps a new Stackwrap aw next
func (s *StackWrap) Rewrap(sc *StackWrap) *StackWrap {
	nx := s.Unwrap()
	s.next = sc
	return nx
}

//NewStackWrap returns a new Stackwrap
func NewStackWrap(fn Stackable, own Stacks, fx func()) *StackWrap {
	return &StackWrap{Fn: fn, owner: own, next: nil, closefx: fx}
}

//NewStack returns a new stack
func NewStack(fn Stackable) (s *Stack) {
	s = &Stack{
		active: 1,
	}

	s.wrap = NewStackWrap(fn, s, nil)
	return
}

//Isolate applies to this stack only and does not propagate the value to other stacks
func (s *Stack) Isolate(b interface{}) interface{} {
	if b == nil {
		return nil
	}
	if s.wrap == nil {
		return nil
	}

	res := b
	state := atomic.LoadInt64(&s.active)
	if state > 0 {
		res = s.wrap.Fn(b, s)
	}
	return res
}

//Identity sends a data into the stack and returns the value supplied,like apply a matrix op V * I = V
func (s *Stack) Identity(b interface{}) interface{} {
	if b == nil {
		return nil
	}
	if s.wrap == nil {
		return nil
	}

	// res := b
	state := atomic.LoadInt64(&s.active)
	if state > 0 {
		_ = s.wrap.Fn(b, s)
		if s.wrap != nil {
			if s.wrap.next != nil {
				if s.wrap.next.owner != nil {
					_ = s.wrap.next.owner.Identity(b)
				}
			}
		}
	}

	return b
}

//Call runs the value returning the return value of just this particular stacks return value
func (s *Stack) Call(b interface{}) interface{} {
	if b == nil {
		return nil
	}
	if s.wrap == nil {
		return nil
	}

	res := b
	state := atomic.LoadInt64(&s.active)
	if state > 0 {
		res = s.wrap.Fn(b, s)
		if s.wrap != nil {
			if s.wrap.next != nil {
				if s.wrap.next.owner != nil {
					_ = s.wrap.next.owner.Call(res)
				}
			}
		}
	}
	return res
}

//Apply sends a data into the stack and returns this stack
//returned value
func (s *Stack) Apply(b interface{}) interface{} {
	if b == nil {
		return nil
	}
	if s.wrap == nil {
		return nil
	}

	res := b
	state := atomic.LoadInt64(&s.active)
	if state > 0 {
		res = s.wrap.Fn(b, s)
		if s.wrap != nil {
			if s.wrap.next != nil {
				if s.wrap.next.owner != nil {
					return s.wrap.next.owner.Apply(res)
				}
			}
		}
	}
	return res
}

//Lift allows the passing of a value to the root of a tree before executing an emit downwards
func (s *Stack) Lift(b interface{}) interface{} {
	if b == nil {
		return nil
	}
	if s.wrap == nil {
		return nil
	}

	if s.root != nil {
		return s.root.Lift(b)
	}

	return s.Apply(b)
}

//Levitate takes a value,mutates it then passes up to the root, moving from bottom-up and returns the roots returned value
func (s *Stack) Levitate(b interface{}) interface{} {
	if b == nil {
		return nil
	}
	if s.wrap == nil {
		return nil
	}

	res := s.wrap.Fn(b, s)

	if s.root != nil {
		s.root.Levitate(res)
		// return s.root.LiftApply(res)
	}

	return res
}

//LiftApply takes a value,mutates it then pass that to the root. moving from bottom-to-top
func (s *Stack) LiftApply(b interface{}) interface{} {
	if b == nil {
		return nil
	}
	if s.wrap == nil {
		return nil
	}

	res := s.wrap.Fn(b, s)

	if s.root != nil {
		return s.root.LiftApply(res)
	}

	return res
}

//Close destroys the stack and any other chain it has
func (s *Stack) Close() error {
	if s.wrap == nil {
		return nil
	}

	defer func() { s.wrap = nil }()
	s.Unstack()
	s.wrap.Close()
	return nil
}

//Unstack disables and nullifies this stack
func (s *Stack) Unstack() {
	if s.wrap == nil || s.root == nil {
		return
	}

	root := s.root
	nxt := s.wrap.next

	atomic.StoreInt64(&s.active, 0)
	root.getWrap().Rewrap(nxt)
	s.root = nil
}

//getWrap returns the stacks wrapper
func (s *Stack) getWrap() *StackWrap {
	return s.wrap
}

//Listen builds a new stack from this previous stack
func (s *Stack) Listen(fn HalfStackable) Stacks {
	return s.Stack(func(b interface{}, _ Stacks) interface{} {
		fn(b)
		return b
	}, true)
}

//LockClose locks the stacks to the close of this
func (s *Stack) LockClose(m Stacks) {
	cx := s.getWrap().closefx
	s.getWrap().closefx = func() {
		m.Close()
		if cx != nil {
			cx()
		}
	}
}

//Stack builds a new stack from this previous stack, the connectClose defines whether you want the new Stack closed when its parent is closed,if false then we dont close it when the root closes
func (s *Stack) Stack(fn Stackable, connectClose bool) Stacks {
	if s.wrap == nil {
		return nil
	}

	if s.wrap.next != nil {
		if s.wrap.next.owner != nil {
			return s.wrap.next.owner.Stack(fn, connectClose)
		}
	}

	m := NewStack(fn)
	m.root = s

	s.wrap.next = m.wrap

	if connectClose {
		s.LockClose(m)
	}

	return m
}

//LogStack takes a stack and logs all input coming from it
func LogStack(s Stacks) Stacks {
	return s.Stack(func(data interface{}, _ Stacks) interface{} {
		log.Info("LogStack: %+s", data)
		return data
	}, true)
}

//LogHeader takes a stack and logs all input from it
func LogHeader(s Stacks, header string) Stacks {
	return s.Stack(func(data interface{}, _ Stacks) interface{} {
		log.Info(header, data)
		return data
	}, true)
}

//LogStackWith logs all input using a custom logger
func LogStackWith(s Stacks, l *slog.Logger) Stacks {
	return s.Stack(func(data interface{}, _ Stacks) interface{} {
		l.Printf("LogStack: %+s", data)
		return data
	}, true)
}

//CustomLogStackWith logs all input using a custom logger
func CustomLogStackWith(s Stacks, l *logging.Logger) Stacks {
	return s.Stack(func(data interface{}, _ Stacks) interface{} {
		l.Info("LogStack: %+s", data)
		return data
	}, true)
}
<commit_msg>adding stack bool returner<commit_after>package flux

import (
	"io"
	slog "log"
	"sync/atomic"

	"github.com/op/go-logging"
)

type (

	//HalfStackable defines a stackable function type that can return a value
	HalfStackable func(interface{})

	//Stackable defines a stackable function type that can return a value
	Stackable func(interface{}, Stacks) interface{}

	//Stacks provides the finite definition of function stacks
	Stacks interface {
		Stack(Stackable, bool) Stacks
		Listen(HalfStackable) Stacks
		Unstack()

		//Isolate applies to this stack only and does not propagate the value to other stacks
		Isolate(interface{}) interface{}

		//Identity returns the value apply to it as the return value
		Identity(interface{}) interface{}

		//Call runs the value, returning the return value of just this particular stacks return value
		Call(interface{}) interface{}

		//Apply applies the value, returning the values of either this stack or the child stacks if any exists
		Apply(interface{}) interface{}

		//Lift lifts this value up the root stack for use with the root .Apply() function(allows you to call a root .Apply() method from anylevel of the stack)
		Lift(interface{}) interface{}

		//LiftApply calls the stack from bottom up and takes the result from a lower stack to apply to its parent stack and returns the root return value
		LiftApply(interface{}) interface{}

		//Levitate calls the stack from bottom up feeding the root with the returned value from the child but returns the child returned value
		Levitate(interface{}) interface{}

		getWrap() *StackWrap
		Close() error
		LockClose(Stacks)
	}

	//StackWrap provides a single-layer two function binder
	StackWrap struct {
		Fn      Stackable
		next    *StackWrap
		owner   Stacks
		closefx func()
	}

	//Stack is the concrete definition of Stacks
	Stack struct {
		wrap   *StackWrap
		root   Stacks
		active int64
	}

	//StackStreamers provides a streaming function for stacks member rules
	StackStreamers interface {
		Stacks
		StreamWriter(io.Writer) Stacks
		Write([]byte) (int, error)
		Read([]byte) (int, error)
		Closed() Stacks
	}

	//StackStream is the concrete implementation for stacks
	StackStream struct {
		Stacks
		closeNotifier Stacks
	}

	//Reports provides a abstraction of report
	Reports struct {
		Tag  string
		Meta map[string]interface{}
	}

	//StackReport provides a internal bank of reports
	StackReport struct {
		stacks chan *Reports
		ended  int64
	}
)

var (
	//StackableIdentity provides an identity function for stacks
	StackableIdentity = func(b interface{}, _ Stacks) interface{} {
		return b
	}
)

//ReportCard returns a new StackReport instance, generally ensure its a good max so you can get reports without over-filling the stack,if the stack is full then old reports would be discard
func ReportCard(max int) *StackReport {
	return &StackReport{
		stacks: make(chan *Reports, max),
	}
}

//Checkout returns a Stack which gets filled with the current and possible future reports
func (r *StackReport) Checkout() StackStreamers {
	cs := NewIdentityStream()
	GoDefer("CheckoutReport", func() {
		for {

			state := atomic.LoadInt64(&r.ended)

			if state > 0 {
				break
			}

			select {
			case v, ok := <-r.stacks:
				if !ok {
					return
				}
				cs.Call(v)
			}
		}
	})
	return cs
}

//Report adds a new report to the reportstack
func (r *StackReport) Report(tag string, meta map[string]interface{}) {
	state := atomic.LoadInt64(&r.ended)

	if state > 0 {
		return
	}

	curcap := cap(r.stacks)
	curlen := len(r.stacks)

	if curlen >= curcap {
		GoDefer("DiscardTopReport", func() {
			<-r.stacks
		})
	}

	r.stacks <- &Reports{tag, meta}
}

//Close ends the report card operations
func (r *StackReport) Close() {
	state := atomic.LoadInt64(&r.ended)
	if state > 0 {
		return
	}

	atomic.StoreInt64(&r.ended, 1)
	close(r.stacks)
}

//NewIdentityStream  returns new StackStream
func NewIdentityStream() StackStreamers {
	return NewStackStream(StackableIdentity)
}

//StreamReader streams from a reader,hence all values are replaced by the data in the io.Reader
func StreamReader(w io.Reader) (s StackStreamers) {
	s = NewStackStream(StackableIdentity)
	go func() {
		_, _ = io.Copy(s, w)
	}()
	return
}

//Read reads into a byte but its empty
func (s *StackStream) Read(bu []byte) (int, error) {
	return 0, io.ErrNoProgress
}

//Write writes into the stream
func (s *StackStream) Write(bu []byte) (int, error) {
	s.Call(bu)
	return len(bu), nil
}

//StreamWriter streams to a writer
func (s *StackStream) StreamWriter(w io.Writer) Stacks {
	return s.Stack(func(data interface{}, _ Stacks) interface{} {
		buff, ok := data.([]byte)

		if !ok {

			str, ok := data.(string)

			if ok {
				_, _ = w.Write([]byte(str))
			}

		}
		_, _ = w.Write(buff)
		return data
	}, true)
}

//Closed returns a stack that is emitted when closed
func (s *StackStream) Closed() Stacks {
	return s.closeNotifier
}

//Close ends this stacks connections
func (s *StackStream) Close() error {
	defer s.closeNotifier.Close()
	s.closeNotifier.Call(true)
	s.Stacks.Close()
	return nil
}

//NewStackStream returns a new stackstream
func NewStackStream(mx Stackable) StackStreamers {
	return &StackStream{
		Stacks:        NewStack(mx),
		closeNotifier: NewStack(StackableIdentity),
	}
}

//Close destroys the stackwrap
func (s *StackWrap) Close() {
	if s.next != nil {
		if s.next.owner != nil {
			s.next.owner.Close()
		}
	}
	s.Fn = nil
	s.owner = nil
	if s.closefx != nil {
		defer func() { s.closefx = nil }()
		s.closefx()
	}
}

//Unwrap unwraps a Stackwrap for release
func (s *StackWrap) Unwrap() *StackWrap {
	nx := s.next
	s.next = nil
	return nx
}

//Rewrap wraps a new Stackwrap aw next
func (s *StackWrap) Rewrap(sc *StackWrap) *StackWrap {
	nx := s.Unwrap()
	s.next = sc
	return nx
}

//NewStackWrap returns a new Stackwrap
func NewStackWrap(fn Stackable, own Stacks, fx func()) *StackWrap {
	return &StackWrap{Fn: fn, owner: own, next: nil, closefx: fx}
}

//NewStack returns a new stack
func NewStack(fn Stackable) (s *Stack) {
	s = &Stack{
		active: 1,
	}

	s.wrap = NewStackWrap(fn, s, nil)
	return
}

//Isolate applies to this stack only and does not propagate the value to other stacks
func (s *Stack) Isolate(b interface{}) interface{} {
	if b == nil {
		return nil
	}
	if s.wrap == nil {
		return nil
	}

	res := b
	state := atomic.LoadInt64(&s.active)
	if state > 0 {
		res = s.wrap.Fn(b, s)
	}
	return res
}

//Identity sends a data into the stack and returns the value supplied,like apply a matrix op V * I = V
func (s *Stack) Identity(b interface{}) interface{} {
	if b == nil {
		return nil
	}
	if s.wrap == nil {
		return nil
	}

	// res := b
	state := atomic.LoadInt64(&s.active)
	if state > 0 {
		_ = s.wrap.Fn(b, s)
		if s.wrap != nil {
			if s.wrap.next != nil {
				if s.wrap.next.owner != nil {
					_ = s.wrap.next.owner.Identity(b)
				}
			}
		}
	}

	return b
}

//Call runs the value returning the return value of just this particular stacks return value
func (s *Stack) Call(b interface{}) interface{} {
	if b == nil {
		return nil
	}
	if s.wrap == nil {
		return nil
	}

	res := b
	state := atomic.LoadInt64(&s.active)
	if state > 0 {
		res = s.wrap.Fn(b, s)
		if s.wrap != nil {
			if s.wrap.next != nil {
				if s.wrap.next.owner != nil {
					_ = s.wrap.next.owner.Call(res)
				}
			}
		}
	}
	return res
}

//Apply sends a data into the stack and returns this stack
//returned value
func (s *Stack) Apply(b interface{}) interface{} {
	if b == nil {
		return nil
	}
	if s.wrap == nil {
		return nil
	}

	res := b
	state := atomic.LoadInt64(&s.active)
	if state > 0 {
		res = s.wrap.Fn(b, s)
		if s.wrap != nil {
			if s.wrap.next != nil {
				if s.wrap.next.owner != nil {
					return s.wrap.next.owner.Apply(res)
				}
			}
		}
	}
	return res
}

//Lift allows the passing of a value to the root of a tree before executing an emit downwards
func (s *Stack) Lift(b interface{}) interface{} {
	if b == nil {
		return nil
	}
	if s.wrap == nil {
		return nil
	}

	if s.root != nil {
		return s.root.Lift(b)
	}

	return s.Apply(b)
}

//Levitate takes a value,mutates it then passes up to the root, moving from bottom-up and returns the roots returned value
func (s *Stack) Levitate(b interface{}) interface{} {
	if b == nil {
		return nil
	}
	if s.wrap == nil {
		return nil
	}

	res := s.wrap.Fn(b, s)

	if s.root != nil {
		s.root.Levitate(res)
		// return s.root.LiftApply(res)
	}

	return res
}

//LiftApply takes a value,mutates it then pass that to the root. moving from bottom-to-top
func (s *Stack) LiftApply(b interface{}) interface{} {
	if b == nil {
		return nil
	}
	if s.wrap == nil {
		return nil
	}

	res := s.wrap.Fn(b, s)

	if s.root != nil {
		return s.root.LiftApply(res)
	}

	return res
}

//Close destroys the stack and any other chain it has
func (s *Stack) Close() error {
	if s.wrap == nil {
		return nil
	}

	defer func() { s.wrap = nil }()
	s.Unstack()
	s.wrap.Close()
	return nil
}

//Unstack disables and nullifies this stack
func (s *Stack) Unstack() {
	if s.wrap == nil || s.root == nil {
		return
	}

	root := s.root
	nxt := s.wrap.next

	atomic.StoreInt64(&s.active, 0)
	root.getWrap().Rewrap(nxt)
	s.root = nil
}

//getWrap returns the stacks wrapper
func (s *Stack) getWrap() *StackWrap {
	return s.wrap
}

//Listen builds a new stack from this previous stack
func (s *Stack) Listen(fn HalfStackable) Stacks {
	return s.Stack(func(b interface{}, _ Stacks) interface{} {
		fn(b)
		return b
	}, true)
}

//LockClose locks the stacks to the close of this
func (s *Stack) LockClose(m Stacks) {
	cx := s.getWrap().closefx
	s.getWrap().closefx = func() {
		m.Close()
		if cx != nil {
			cx()
		}
	}
}

//Stack builds a new stack from this previous stack, the connectClose defines whether you want the new Stack closed when its parent is closed,if false then we dont close it when the root closes
func (s *Stack) Stack(fn Stackable, connectClose bool) Stacks {
	if s.wrap == nil {
		return nil
	}

	if s.wrap.next != nil {
		if s.wrap.next.owner != nil {
			return s.wrap.next.owner.Stack(fn, connectClose)
		}
	}

	m := NewStack(fn)
	m.root = s

	s.wrap.next = m.wrap

	if connectClose {
		s.LockClose(m)
	}

	return m
}

//LogStack takes a stack and logs all input coming from it
func LogStack(s Stacks) Stacks {
	return s.Stack(func(data interface{}, _ Stacks) interface{} {
		log.Info("LogStack: %+s", data)
		return data
	}, true)
}

//LogHeader takes a stack and logs all input from it
func LogHeader(s Stacks, header string) Stacks {
	return s.Stack(func(data interface{}, _ Stacks) interface{} {
		log.Info(header, data)
		return data
	}, true)
}

//LogStackWith logs all input using a custom logger
func LogStackWith(s Stacks, l *slog.Logger) Stacks {
	return s.Stack(func(data interface{}, _ Stacks) interface{} {
		l.Printf("LogStack: %+s", data)
		return data
	}, true)
}

//CustomLogStackWith logs all input using a custom logger
func CustomLogStackWith(s Stacks, l *logging.Logger) Stacks {
	return s.Stack(func(data interface{}, _ Stacks) interface{} {
		l.Info("LogStack: %+s", data)
		return data
	}, true)
}

//BoolOnly ensures only boolean value pass through
func BoolOnly(s Stacks) Stacks {
	sm := NewIdentityStream()

	s.Listen(func(data interface{}) {
		bl, ok := data.(bool)
		if ok {
			sm.Apply(bl)
		}
	})

	s.LockClose(sm)

	return sm
}
<|endoftext|>


<commit_before>package formats

import (
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

// Layout represents a parsed file structure layout as a flat list
type Layout struct {
	Offset int64
	Length byte
	Type   DataType
	Info   string
}

// DataType ...
type DataType int

func (dt DataType) String() string {

	m := map[DataType]string{
		Int8:     "int8",
		Uint8:    "uint8",
		Int16le:  "int16-le",
		Uint16le: "uint16-le",
		Int32le:  "int32-le",
		Uint32le: "uint32-le",
		ASCII:    "ASCII",
		ASCIIZ:   "ASCIIZ",
	}

	if val, ok := m[dt]; ok {
		return val
	}

	// NOTE should only be able to panic during dev (as in:
	// adding a new datatype and forgetting to add it to the map)
	panic(dt)
}

// ParsedLayout ...
type ParsedLayout struct {
	FormatName string
	FileSize   int64
	Layout     []Layout
}

// ...
const (
	Int8 DataType = 1 + iota
	Uint8
	Int16le
	Uint16le
	Int32le
	Uint32le
	ASCII
	ASCIIZ
)

func fileExt(file *os.File) string {

	ext := filepath.Ext(file.Name())
	if len(ext) > 0 {
		// strip leading dot
		ext = ext[1:]
	}
	return ext
}

// ParseLayout returns a ParsedLayout for the file
func ParseLayout(file *os.File) (*ParsedLayout, error) {

	parsed, err := parseFileByDescription(file, fileExt(file))
	if parsed == nil {
		fmt.Println(err)
		panic("XXX if find by extension fails, search all for magic id")
	}

	return parsed, err
}

func getFileSize(file *os.File) int64 {

	fi, err := file.Stat()
	if err != nil {
		log.Fatal(err)
	}
	return fi.Size()
}

func parseFileByDescription(
	file *os.File, formatName string) (*ParsedLayout, error) {

	format, err := ReadFormatDescription(formatName)
	if err != nil {
		return nil, err
	}

	res := ParsedLayout{
		FormatName: formatName,
		FileSize:   getFileSize(file),
	}

	for _, step := range format.Details {

		layout, err := res.intoLayout(file, step)
		if err != nil {
			fmt.Println("trouble parsing:", err)
		}

		res.Layout = append(res.Layout, *layout)
	}

	return &res, nil
}

func (l *Layout) parseByteN(reader io.Reader, expectedLen int64) ([]byte, error) {

	buf := make([]byte, expectedLen)

	readLen, err := reader.Read(buf)
	if err != nil {
		return nil, err
	}

	if int64(readLen) != expectedLen {
		return nil, fmt.Errorf("Expected %d bytes, got %d", expectedLen, readLen)
	}
	return buf, nil
}

// transforms a part of file into a Layout, according to `step`
func (pl *ParsedLayout) intoLayout(file *os.File, step string) (*Layout, error) {

	reader := io.Reader(file)

	// params: name | data type and size | type-dependant
	params := strings.Split(step, "|")

	layout := Layout{}

	layout.Offset, _ = file.Seek(0, os.SEEK_CUR)
	layout.Info = params[0]

	param1 := ""
	param2 := ""
	if len(params) > 1 {
		param1 = params[1]
	}
	if len(params) > 2 {
		param2 = params[2]
	}

	if expectedLen, err := parseExpectedBytes(&layout, reader, param1, param2); err == nil {
		layout.Length = byte(expectedLen)
		layout.Type = ASCII
	} else if _, err := parseExpectedByte(reader, param1, param2); err == nil {
		layout.Length = 1
		layout.Type = Uint8
	} else if _, err := parseExpectedUint16le(reader, param1, param2); err == nil {
		layout.Length = 2
		layout.Type = Uint16le
	} else {
		return nil, fmt.Errorf("dunno how to handle %s", param1)
	}

	return &layout, nil
}

func parseExpectedUint16le(reader io.Reader, param1 string, param2 string) (uint16, error) {

	if param1 != "uint16le" {
		return 0, fmt.Errorf("wrong type")
	}
	var b uint16
	err := binary.Read(reader, binary.LittleEndian, &b);
	return b, err
}

func parseExpectedByte(reader io.Reader, param1 string, param2 string) (byte, error) {

	if param1 != "uint8" && param1 != "byte" {
		return 0, fmt.Errorf("wrong type")
	}
	// XXX "byte", params[2] describes a bit field
	var b byte
	err := binary.Read(reader, binary.LittleEndian, &b);
	return b, err
}

func parseExpectedBytes(layout *Layout, reader io.Reader, param1 string, param2 string) (int64, error) {

	p1 := strings.Split(param1, ":")

	if p1[0] != "byte" || len(p1) != 2 {
		return 0, fmt.Errorf("wrong type")
	}

	expectedLen, err := parseExpectedLen(p1[1])
	if err != nil {
		return 0,err
	}

	// "byte:3", params[2] holds the bytes
	buf, err := layout.parseByteN(reader, expectedLen)
	if err != nil {
		return 0, err
	}

	// split expected forms on comma
	expectedForms := strings.Split(param2, ",")
	found := false
	for _, expectedForm := range expectedForms {

		expectedBytes := []byte(expectedForm)

		if int64(len(expectedForm)) == 2*expectedLen {
			// guess it's hex
			bytes, err := hex.DecodeString(expectedForm)
			if err == nil && byteSliceEquals(buf, bytes) {
				found = true
			}
		}

		if !found && string(buf) == string(expectedBytes) {
			found = true
		}
		if found {
			break
		}
	}
	if !found {
		return 0, fmt.Errorf("didnt find expected bytes %s", param2)
	}

	return expectedLen, nil
}

func parseExpectedLen(s string) (int64, error) {
	expectedLen, err := strconv.ParseInt(s, 10, 64)
	if err != nil {
		return 0, err
	}
	if expectedLen > 255 {
		return 0, fmt.Errorf("len too big (max 255)")
	}
	if expectedLen <= 0 {
		return 0, fmt.Errorf("len too small (min 1)")
	}
	return expectedLen, nil
}

// PrettyHexView ...
func (pl *ParsedLayout) PrettyHexView(file *os.File) string {

	ofsFmt := "%08x"
	if pl.FileSize <= 0xffff {
		ofsFmt = "%04x"
	} else if pl.FileSize <= 0xffffff {
		ofsFmt = "%06x"
	}

	hex := ""

	base := HexView.StartingRow * int64(HexView.RowWidth)
	ceil := base + int64(HexView.VisibleRows*HexView.RowWidth)

	for i := base; i < ceil; i += int64(HexView.RowWidth) {

		ofs, err := file.Seek(i, os.SEEK_SET)
		if i != ofs {
			log.Fatalf("err: unexpected offset %04x, expected %04x\n", ofs, i)
		}
		line, err := pl.GetHex(file)

		ofsText := fmt.Sprintf(ofsFmt, i)

		hex += fmt.Sprintf("[[%s]](fg-yellow) %s\n", ofsText, line)
		if err != nil {
			fmt.Println("got err", err)
			break
		}
	}

	return hex
}

func (pl *ParsedLayout) isOffsetKnown(ofs int64) bool {

	for _, layout := range pl.Layout {
		if ofs >= layout.Offset && ofs < layout.Offset+int64(layout.Length) {
			return true
		}
	}
	return false
}

// GetHex dumps a row of hex from io.Reader
func (pl *ParsedLayout) GetHex(file *os.File) (string, error) {

	layout := pl.Layout[HexView.CurrentField]

	reader := io.Reader(file)

	symbols := []string{}

	base, err := file.Seek(0, os.SEEK_CUR)
	if err != nil {
		return "", err
	}

	for w := int64(0); w < 16; w++ {
		var b byte
		if err = binary.Read(reader, binary.LittleEndian, &b); err != nil {
			if err == io.EOF {
				return combineHexRow(symbols), nil
			}
			return "", err
		}

		ceil := base + w

		colorName := "fg-white"
		if !pl.isOffsetKnown(base + w) {
			colorName = "fg-red"
		}
		if ceil >= layout.Offset && ceil < layout.Offset+int64(layout.Length) {
			colorName = "fg-blue"
		}

		group := fmt.Sprintf("[%02x](%s)", b, colorName)
		symbols = append(symbols, group)
	}

	return combineHexRow(symbols), nil
}
<commit_msg>simplify<commit_after>package formats

import (
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

// Layout represents a parsed file structure layout as a flat list
type Layout struct {
	Offset int64
	Length byte
	Type   DataType
	Info   string
}

// DataType ...
type DataType int

func (dt DataType) String() string {

	m := map[DataType]string{
		Int8:     "int8",
		Uint8:    "uint8",
		Int16le:  "int16-le",
		Uint16le: "uint16-le",
		Int32le:  "int32-le",
		Uint32le: "uint32-le",
		ASCII:    "ASCII",
		ASCIIZ:   "ASCIIZ",
	}

	if val, ok := m[dt]; ok {
		return val
	}

	// NOTE should only be able to panic during dev (as in:
	// adding a new datatype and forgetting to add it to the map)
	panic(dt)
}

// ParsedLayout ...
type ParsedLayout struct {
	FormatName string
	FileSize   int64
	Layout     []Layout
}

// ...
const (
	Int8 DataType = 1 + iota
	Uint8
	Int16le
	Uint16le
	Int32le
	Uint32le
	ASCII
	ASCIIZ
)

func fileExt(file *os.File) string {

	ext := filepath.Ext(file.Name())
	if len(ext) > 0 {
		// strip leading dot
		ext = ext[1:]
	}
	return ext
}

// ParseLayout returns a ParsedLayout for the file
func ParseLayout(file *os.File) (*ParsedLayout, error) {

	parsed, err := parseFileByDescription(file, fileExt(file))
	if parsed == nil {
		fmt.Println(err)
		panic("XXX if find by extension fails, search all for magic id")
	}

	return parsed, err
}

func getFileSize(file *os.File) int64 {

	fi, err := file.Stat()
	if err != nil {
		log.Fatal(err)
	}
	return fi.Size()
}

func parseFileByDescription(
	file *os.File, formatName string) (*ParsedLayout, error) {

	format, err := ReadFormatDescription(formatName)
	if err != nil {
		return nil, err
	}

	res := ParsedLayout{
		FormatName: formatName,
		FileSize:   getFileSize(file),
	}

	for _, step := range format.Details {

		layout, err := res.intoLayout(file, step)
		if err != nil {
			fmt.Println("trouble parsing:", err)
		}

		res.Layout = append(res.Layout, *layout)
	}

	return &res, nil
}

func (l *Layout) parseByteN(reader io.Reader, expectedLen int64) ([]byte, error) {

	buf := make([]byte, expectedLen)

	readLen, err := reader.Read(buf)
	if err != nil {
		return nil, err
	}

	if int64(readLen) != expectedLen {
		return nil, fmt.Errorf("Expected %d bytes, got %d", expectedLen, readLen)
	}
	return buf, nil
}

// transforms a part of file into a Layout, according to `step`
func (pl *ParsedLayout) intoLayout(file *os.File, step string) (*Layout, error) {

	reader := io.Reader(file)

	// params: name | data type and size | type-dependant
	params := strings.Split(step, "|")

	layout := Layout{}

	layout.Offset, _ = file.Seek(0, os.SEEK_CUR)
	layout.Info = params[0]

	param1 := ""
	param2 := ""
	if len(params) > 1 {
		param1 = params[1]
	}
	if len(params) > 2 {
		param2 = params[2]
	}

	if expectedLen, err := parseExpectedBytes(&layout, reader, param1, param2); err == nil {
		layout.Length = byte(expectedLen)
		layout.Type = ASCII
	} else if _, err := parseExpectedByte(reader, param1, param2); err == nil {
		layout.Length = 1
		layout.Type = Uint8
	} else if _, err := parseExpectedUint16le(reader, param1, param2); err == nil {
		layout.Length = 2
		layout.Type = Uint16le
	} else {
		return nil, fmt.Errorf("dunno how to handle %s", param1)
	}

	return &layout, nil
}

func parseExpectedUint16le(reader io.Reader, param1 string, param2 string) (uint16, error) {

	if param1 != "uint16le" {
		return 0, fmt.Errorf("wrong type")
	}
	var b uint16
	err := binary.Read(reader, binary.LittleEndian, &b);
	return b, err
}

func parseExpectedByte(reader io.Reader, param1 string, param2 string) (byte, error) {

	if param1 != "uint8" && param1 != "byte" {
		return 0, fmt.Errorf("wrong type")
	}
	// XXX "byte", params[2] describes a bit field
	var b byte
	err := binary.Read(reader, binary.LittleEndian, &b);
	return b, err
}

func parseExpectedBytes(layout *Layout, reader io.Reader, param1 string, param2 string) (int64, error) {

	p1 := strings.Split(param1, ":")

	if p1[0] != "byte" || len(p1) != 2 {
		return 0, fmt.Errorf("wrong type")
	}

	expectedLen, err := parseExpectedLen(p1[1])
	if err != nil {
		return 0, err
	}

	// "byte:3", params[2] holds the bytes
	buf, err := layout.parseByteN(reader, expectedLen)
	if err != nil {
		return 0, err
	}

	// split expected forms on comma
	expectedForms := strings.Split(param2, ",")
	for _, expectedForm := range expectedForms {

		expectedBytes := []byte(expectedForm)
		if int64(len(expectedForm)) == 2*expectedLen {
			// hex string?
			bytes, err := hex.DecodeString(expectedForm)
			if err == nil && byteSliceEquals(buf, bytes) {
				return expectedLen, nil
			}
		}
		if string(buf) == string(expectedBytes) {
			return expectedLen, nil
		}
	}

	return 0, fmt.Errorf("didnt find expected bytes %s", param2)
}

func parseExpectedLen(s string) (int64, error) {
	expectedLen, err := strconv.ParseInt(s, 10, 64)
	if err != nil {
		return 0, err
	}
	if expectedLen > 255 {
		return 0, fmt.Errorf("len too big (max 255)")
	}
	if expectedLen <= 0 {
		return 0, fmt.Errorf("len too small (min 1)")
	}
	return expectedLen, nil
}

// PrettyHexView ...
func (pl *ParsedLayout) PrettyHexView(file *os.File) string {

	ofsFmt := "%08x"
	if pl.FileSize <= 0xffff {
		ofsFmt = "%04x"
	} else if pl.FileSize <= 0xffffff {
		ofsFmt = "%06x"
	}

	hex := ""

	base := HexView.StartingRow * int64(HexView.RowWidth)
	ceil := base + int64(HexView.VisibleRows*HexView.RowWidth)

	for i := base; i < ceil; i += int64(HexView.RowWidth) {

		ofs, err := file.Seek(i, os.SEEK_SET)
		if i != ofs {
			log.Fatalf("err: unexpected offset %04x, expected %04x\n", ofs, i)
		}
		line, err := pl.GetHex(file)

		ofsText := fmt.Sprintf(ofsFmt, i)

		hex += fmt.Sprintf("[[%s]](fg-yellow) %s\n", ofsText, line)
		if err != nil {
			fmt.Println("got err", err)
			break
		}
	}

	return hex
}

func (pl *ParsedLayout) isOffsetKnown(ofs int64) bool {

	for _, layout := range pl.Layout {
		if ofs >= layout.Offset && ofs < layout.Offset+int64(layout.Length) {
			return true
		}
	}
	return false
}

// GetHex dumps a row of hex from io.Reader
func (pl *ParsedLayout) GetHex(file *os.File) (string, error) {

	layout := pl.Layout[HexView.CurrentField]

	reader := io.Reader(file)

	symbols := []string{}

	base, err := file.Seek(0, os.SEEK_CUR)
	if err != nil {
		return "", err
	}

	for w := int64(0); w < 16; w++ {
		var b byte
		if err = binary.Read(reader, binary.LittleEndian, &b); err != nil {
			if err == io.EOF {
				return combineHexRow(symbols), nil
			}
			return "", err
		}

		ceil := base + w

		colorName := "fg-white"
		if !pl.isOffsetKnown(base + w) {
			colorName = "fg-red"
		}
		if ceil >= layout.Offset && ceil < layout.Offset+int64(layout.Length) {
			colorName = "fg-blue"
		}

		group := fmt.Sprintf("[%02x](%s)", b, colorName)
		symbols = append(symbols, group)
	}

	return combineHexRow(symbols), nil
}
<|endoftext|>


<commit_before>package agents

import (
	"github.com/michenriksen/aquatone/core"
)

type URLPublisher struct {
	session *core.Session
}

func NewURLPublisher() *URLPublisher {
	return &URLPublisher{}
}

func (d *URLPublisher) ID() string {
	return "agent:url_publisher"
}

func (a *URLPublisher) Register(s *core.Session) error {
	s.EventBus.SubscribeAsync(core.TCPPort, a.OnTCPPort, false)
	a.session = s
	return nil
}

func (a *URLPublisher) OnTCPPort(port int, host string) {
	a.session.Out.Debug("[%s] Received new open port on %s: %d\n", a.ID(), host, port)
	url := HostAndPortToURL(host, port, "")
	a.session.EventBus.Publish(core.URL, url)
}
<commit_msg>Attempt to detect TLS on non-standard ports in URL Publisher agent<commit_after>package agents

import (
	"crypto/tls"
	"fmt"
	"net"
	"time"

	"github.com/michenriksen/aquatone/core"
)

type URLPublisher struct {
	session *core.Session
}

func NewURLPublisher() *URLPublisher {
	return &URLPublisher{}
}

func (d *URLPublisher) ID() string {
	return "agent:url_publisher"
}

func (a *URLPublisher) Register(s *core.Session) error {
	s.EventBus.SubscribeAsync(core.TCPPort, a.OnTCPPort, false)
	a.session = s
	return nil
}

func (a *URLPublisher) OnTCPPort(port int, host string) {
	a.session.Out.Debug("[%s] Received new open port on %s: %d\n", a.ID(), host, port)
	var url string
	if a.isTLS(port, host) {
		url = HostAndPortToURL(host, port, "https")
	} else {
		url = HostAndPortToURL(host, port, "http")
	}
	a.session.EventBus.Publish(core.URL, url)
}

func (a *URLPublisher) isTLS(port int, host string) bool {
	if port == 80 {
		return false
	}

	if port == 443 {
		return true
	}

	dialer := &net.Dialer{Timeout: time.Duration(*a.session.Options.HTTPTimeout) * time.Millisecond}
	conf := &tls.Config{
		InsecureSkipVerify: true,
	}
	conn, err := tls.DialWithDialer(dialer, "tcp", fmt.Sprintf("%s:%d", host, port), conf)
	if err != nil {
		return false
	}
	conn.Close()
	return true
}
<|endoftext|>


<commit_before>package auth2

import (
	"database/sql"
	"path/filepath"
	"testing"

	"github.com/BurntSushi/toml"
	"github.com/dfreire/fservices/mailer"
	"github.com/dfreire/fservices/util"
	"github.com/stretchr/testify/assert"
)

func TestSignup(t *testing.T) {
	auth, db := createAuthService()
	defer dropSchema(db)

	assert.Nil(t, auth.Signup("myapp", "dario.freire@gmail.com", "123", "en_US"))
}

func createAuthService() (Auth, *sql.DB) {
	var authConfig AuthConfig
	_, err := toml.DecodeFile("auth_test.toml", &authConfig)
	util.PanicIfNotNil(err)

	db, err := sql.Open("postgres", "postgres://drome:@localhost/fservices_test?sslmode=disable")
	util.PanicIfNotNil(err)
	storePg, err := NewStorePg(db)
	util.PanicIfNotNil(err)

	var smtpConfig mailer.SmtpConfig
	_, err = toml.DecodeFile(filepath.Join("..", "mailer", "mailer_test.secret.toml"), &smtpConfig)
	util.PanicIfNotNil(err)
	mailer := mailer.NewMailer(smtpConfig)

	return NewAuth(authConfig, storePg, mailer), db
}

func dropSchema(db *sql.DB) {
	_, err := db.Exec("DROP SCHEMA auth CASCADE;")
	util.PanicIfNotNil(err)
}
<commit_msg>update<commit_after>package auth2

import (
	"database/sql"
	"testing"

	"github.com/BurntSushi/toml"
	mailermock "github.com/dfreire/fservices/mailer/mock"
	"github.com/dfreire/fservices/util"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/mock"
)

func TestSignup(t *testing.T) {
	auth, _, mailerMock, db := createAuthService()
	defer dropSchema(db)

	// request := mailer.SendMailRequest{
	// 	From:    cfg.FromEmail,
	// 	To:      []string{"dario.freire@gmail.com"},
	// 	Subject: cfg.ConfirmationEmail["en_US"].Subject,
	// 	Body:    util.MustRenderTemplate(cfg.ConfirmationEmail[lang].Body, templateValues)
	// }
	// mailerMock.On("Send", request).Return(nil)
	mailerMock.On("Send", mock.AnythingOfTypeArgument("mailer.SendMailRequest")).Return(nil)
	assert.Nil(t, auth.Signup("myapp", "dario.freire@gmail.com", "123", "en_US"))
	mailerMock.AssertNumberOfCalls(t, "Send", 1)
	mailerMock.AssertExpectations(t)
}

func createAuthService() (Auth, AuthConfig, *mailermock.MailerMock, *sql.DB) {
	var authConfig AuthConfig
	_, err := toml.DecodeFile("auth_test.toml", &authConfig)
	util.PanicIfNotNil(err)

	db, err := sql.Open("postgres", "postgres://drome:@localhost/fservices_test?sslmode=disable")
	util.PanicIfNotNil(err)
	storePg, err := NewStorePg(db)
	util.PanicIfNotNil(err)

	// var smtpConfig mailer.SmtpConfig
	// _, err = toml.DecodeFile(filepath.Join("..", "mailer", "mailer_test.secret.toml"), &smtpConfig)
	// util.PanicIfNotNil(err)
	// mailer := mailer.NewMailer(smtpConfig)
	mailer := new(mailermock.MailerMock)

	return NewAuth(authConfig, storePg, mailer), authConfig, mailer, db
}

func dropSchema(db *sql.DB) {
	_, err := db.Exec("DROP SCHEMA auth CASCADE;")
	util.PanicIfNotNil(err)
}
<|endoftext|>


<commit_before>package search

import (
	"fmt"
	"net/http"

	"github.com/algolia/algoliasearch-client-go/algolia/call"
	iopt "github.com/algolia/algoliasearch-client-go/algolia/internal/opt"
	"github.com/algolia/algoliasearch-client-go/algolia/transport"
)

type Index struct {
	appID        string
	name         string
	maxBatchSize int
	client       *Client
	transport    *transport.Transport
}

func newIndex(client *Client, name string) *Index {
	return &Index{
		client:       client,
		name:         name,
		maxBatchSize: client.maxBatchSize,
		transport:    client.transport,
	}
}

func (i *Index) path(format string, a ...interface{}) string {
	prefix := fmt.Sprintf("/1/indexes/%s", i.name)
	suffix := fmt.Sprintf(format, a...)
	return prefix + suffix
}

func (i *Index) WaitTask(taskID int) error {
	return waitWithRetry(func() (bool, error) {
		res, err := i.GetStatus(taskID)
		if err != nil {
			return true, err
		}
		if res.Status == "published" {
			return true, nil
		}
		return false, nil
	})
}

func (i *Index) operation(destination, op string, opts ...interface{}) (res UpdateTaskRes, err error) {
	var scopes []string
	if opt := iopt.ExtractScopes(opts...); opt != nil {
		scopes = opt.Get()
	}
	req := IndexOperation{
		Destination: destination,
		Operation:   op,
		Scopes:      scopes,
	}
	path := i.path("/operation")
	err = i.transport.Request(&res, http.MethodPost, path, req, call.Write, opts...)
	res.wait = i.WaitTask
	return
}

func (i *Index) GetAppID() string {
	return i.appID
}

func (i *Index) Clear(opts ...interface{}) (res UpdateTaskRes, err error) {
	path := i.path("/clear")
	err = i.transport.Request(&res, http.MethodPost, path, nil, call.Write, opts...)
	res.wait = i.WaitTask
	return
}

func (i *Index) Delete(opts ...interface{}) (res DeleteTaskRes, err error) {
	path := i.path("")
	err = i.transport.Request(&res, http.MethodDelete, path, nil, call.Write, opts...)
	res.wait = i.WaitTask
	return
}

func (i *Index) GetStatus(taskID int) (res TaskStatusRes, err error) {
	path := i.path("/task/%d", taskID)
	err = i.transport.Request(&res, http.MethodGet, path, nil, call.Read)
	return
}
<commit_msg>fixed: correctly set the appID private field of Index instances<commit_after>package search

import (
	"fmt"
	"net/http"

	"github.com/algolia/algoliasearch-client-go/algolia/call"
	iopt "github.com/algolia/algoliasearch-client-go/algolia/internal/opt"
	"github.com/algolia/algoliasearch-client-go/algolia/transport"
)

type Index struct {
	appID        string
	name         string
	maxBatchSize int
	client       *Client
	transport    *transport.Transport
}

func newIndex(client *Client, name string) *Index {
	return &Index{
		appID:        client.appID,
		client:       client,
		name:         name,
		maxBatchSize: client.maxBatchSize,
		transport:    client.transport,
	}
}

func (i *Index) path(format string, a ...interface{}) string {
	prefix := fmt.Sprintf("/1/indexes/%s", i.name)
	suffix := fmt.Sprintf(format, a...)
	return prefix + suffix
}

func (i *Index) WaitTask(taskID int) error {
	return waitWithRetry(func() (bool, error) {
		res, err := i.GetStatus(taskID)
		if err != nil {
			return true, err
		}
		if res.Status == "published" {
			return true, nil
		}
		return false, nil
	})
}

func (i *Index) operation(destination, op string, opts ...interface{}) (res UpdateTaskRes, err error) {
	var scopes []string
	if opt := iopt.ExtractScopes(opts...); opt != nil {
		scopes = opt.Get()
	}
	req := IndexOperation{
		Destination: destination,
		Operation:   op,
		Scopes:      scopes,
	}
	path := i.path("/operation")
	err = i.transport.Request(&res, http.MethodPost, path, req, call.Write, opts...)
	res.wait = i.WaitTask
	return
}

func (i *Index) GetAppID() string {
	return i.appID
}

func (i *Index) Clear(opts ...interface{}) (res UpdateTaskRes, err error) {
	path := i.path("/clear")
	err = i.transport.Request(&res, http.MethodPost, path, nil, call.Write, opts...)
	res.wait = i.WaitTask
	return
}

func (i *Index) Delete(opts ...interface{}) (res DeleteTaskRes, err error) {
	path := i.path("")
	err = i.transport.Request(&res, http.MethodDelete, path, nil, call.Write, opts...)
	res.wait = i.WaitTask
	return
}

func (i *Index) GetStatus(taskID int) (res TaskStatusRes, err error) {
	path := i.path("/task/%d", taskID)
	err = i.transport.Request(&res, http.MethodGet, path, nil, call.Read)
	return
}
<|endoftext|>


<commit_before>package dpn

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/nsqio/go-nsq"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
)

// copier.go copies tarred bags from other nodes via rsync.
// This is used when replicating content from other nodes.
// For putting together DPN bags from APTrust files, see fetcher.go.

type Copier struct {
	LookupChannel       chan *DPNResult
	CopyChannel         chan *DPNResult
	PostProcessChannel  chan *DPNResult
	DPNConfig           *DPNConfig
	ProcUtil            *bagman.ProcessUtil
	LocalClient         *DPNRestClient
	RemoteClients       map[string]*DPNRestClient
	// WaitGroup is for running local tests only.
	WaitGroup           sync.WaitGroup
}

type CopyResult struct {
	LocalPath       string
	ErrorMessage    string
	RsyncStdout     string
	RsyncStderr     string
	InfoMessage     string
	BagWasCopied    bool
}

func NewCopier(procUtil *bagman.ProcessUtil, dpnConfig *DPNConfig) (*Copier, error) {
	localClient, err := NewDPNRestClient(
		dpnConfig.RestClient.LocalServiceURL,
		dpnConfig.RestClient.LocalAPIRoot,
		dpnConfig.RestClient.LocalAuthToken,
		dpnConfig.LocalNode,
		dpnConfig,
		procUtil.MessageLog)
	if err != nil {
		return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	remoteClients, err := GetRemoteClients(localClient, dpnConfig,
		procUtil.MessageLog)
	if err != nil {
		return nil, err
	}
	copier := &Copier {
		DPNConfig: dpnConfig,
		ProcUtil: procUtil,
		LocalClient: localClient,
		RemoteClients: remoteClients,
	}
	workerBufferSize := procUtil.Config.DPNPackageWorker.Workers * 4
	copier.LookupChannel = make(chan *DPNResult, workerBufferSize)
	copier.CopyChannel = make(chan *DPNResult, workerBufferSize)
	copier.PostProcessChannel = make(chan *DPNResult, workerBufferSize)
	for i := 0; i < procUtil.Config.DPNPackageWorker.Workers; i++ {
		go copier.doLookup()
		go copier.doCopy()
		go copier.postProcess()
	}
	return copier, nil
}

func (copier *Copier) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()
	dpnResult := &DPNResult{}
	err := json.Unmarshal(message.Body, dpnResult)
	if err != nil {
		detailedError := fmt.Errorf("Could not unmarshal JSON data from nsq. " +
			"Error is: %v    JSON is: %s", err.Error(), string(message.Body))
		copier.ProcUtil.MessageLog.Error(detailedError.Error())
		message.Finish()
		return detailedError
	}

	// Set up the copy result
	dpnResult.CopyResult = &CopyResult{
		BagWasCopied: false,
	}
	dpnResult.NsqMessage = message

	// Start processing.
	dpnResult.Stage = STAGE_COPY
	copier.LookupChannel <- dpnResult
	copier.ProcUtil.MessageLog.Info("Put %s from %s into copy channel",
		dpnResult.DPNBag.UUID, dpnResult.DPNBag.AdminNode)
	return nil
}

// Look up the DPN bag on the admin node. Although we already have the
// bag object as bart of the DPNResult object, this request may have been
// sitting in the queue for many hours, and the replication request may
// have been fulfilled or cancelled in that time. So check the status on
// the authoritative node to avoid unnecessarily processing what might
// be hundreds of gigs of data.
func (copier *Copier) doLookup() {
	for result := range copier.LookupChannel {
		// Get a client to talk to the FromNode
		remoteClient := copier.RemoteClients[result.TransferRequest.FromNode]

		copier.ProcUtil.MessageLog.Debug(
			"Looking up ReplicationId %s, bag %s, on node %s ",
				result.TransferRequest.ReplicationId,
				result.TransferRequest.BagId,
				result.TransferRequest.FromNode)

		// If we can find out for sure that this replication request should
		// not be processed, then don't process it...
		xfer, _ := remoteClient.ReplicationTransferGet(
			result.TransferRequest.ReplicationId)
		if xfer != nil && xfer.Status != "requested" {
			message := fmt.Sprintf(
				"Cancelling copy of ReplicationId %s (bag %s) because " +
					"replication status on %s is %s",
				result.TransferRequest.ReplicationId,
				result.TransferRequest.BagId,
				result.TransferRequest.FromNode,
				xfer.Status)
			copier.ProcUtil.MessageLog.Info(message)
			result.CopyResult.InfoMessage = message
			result.Retry = false
			result.TransferRequest = xfer
			copier.PostProcessChannel <- result
			continue
		}
		// ...otherwise, proceed with processing.
		copier.CopyChannel <- result
	}
}

// Copy the file from the remote node to our local staging area.
// Calculate checksums.
func (copier *Copier) doCopy() {
	for result := range copier.CopyChannel {

		// Make sure we have enough room on the volume
		// to download and unpack this bag.
		err := copier.ProcUtil.Volume.Reserve(uint64(float64(result.DPNBag.Size) * float64(2.1)))
		if err != nil {
			// Not enough room on disk
			copier.ProcUtil.MessageLog.Warning(
				"Requeueing %s from %s (%d bytes) - not enough disk space",
				result.TransferRequest.ReplicationId,
				result.TransferRequest.FromNode,
				result.DPNBag.Size)
			result.CopyResult.ErrorMessage = err.Error()
			result.ErrorMessage = err.Error()
			result.Retry = true
			copier.PostProcessChannel <- result
			continue
		}

		localPath := filepath.Join(
			copier.ProcUtil.Config.DPNStagingDirectory,
			fmt.Sprintf("%s.tar", result.TransferRequest.BagId))

		if !bagman.FileExists(copier.ProcUtil.Config.DPNStagingDirectory) {
			os.MkdirAll(copier.ProcUtil.Config.DPNStagingDirectory, 0755)
		}

		// DEBUG - use for tracing 'file not found'
		// fmt.Printf("Rsync link is %s\n", result.TransferRequest.Link)

		copier.ProcUtil.MessageLog.Info("Rsync link is %s", result.TransferRequest.Link)
		rsyncCommand := GetRsyncCommand(result.TransferRequest.Link,
			localPath, copier.DPNConfig.UseSSHWithRsync)

		// Touch message on both sides of rsync, so NSQ doesn't time out.
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}
		output, err := rsyncCommand.CombinedOutput()
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}
		if err != nil {
			result.CopyResult.ErrorMessage = fmt.Sprintf("%s: %s",
				err.Error(), string(output))
		} else {
			result.LocalPath = localPath
			result.CopyResult.LocalPath = localPath
			result.CopyResult.BagWasCopied = true
			// TODO: This is not necessary. We just need to calculate the checksum
			// of the SHA-256 manifest
			fileDigest, err := bagman.CalculateDigests(localPath)
			if result.NsqMessage != nil {
				result.NsqMessage.Touch()
			}
			if err != nil {
				result.ErrorMessage = fmt.Sprintf("Could not calculate checksums on '%s': %v",
					result.PackageResult.TarFilePath, err)
				copier.PostProcessChannel <- result
				continue
			}
			result.BagMd5Digest = fileDigest.Md5Digest
			result.BagSha256Digest = fileDigest.Sha256Digest
			result.BagSize = fileDigest.Size
		}
		copier.PostProcessChannel <- result
	}
}

func (copier *Copier) postProcess() {
	// On success, send to validation queue.
	// Otherwise, send to trouble queue.
	for result := range copier.PostProcessChannel {
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}
		result.ErrorMessage = result.CopyResult.ErrorMessage

		// On error, log and send to trouble queue if the error is fatal
		if result.ErrorMessage != "" {
			copier.ProcUtil.MessageLog.Error(result.ErrorMessage)
			copier.ProcUtil.IncrementFailed()
			if result.Retry == false {
				SendToTroubleQueue(result, copier.ProcUtil)
			}
			if bagman.FileExists(result.CopyResult.LocalPath) {
				os.Remove(result.CopyResult.LocalPath)
				copier.ProcUtil.MessageLog.Debug(
					"Deleting bag file %s", result.CopyResult.LocalPath)
			}
		} else if result.CopyResult.BagWasCopied == false {
			// We didn't copy the bag, but there was no error.
			// This happens when the transfer request is marked
			// as completed or cancelled on the remote node.
			// Count this as success, because we did what we're
			// supposed to do in this case, which is nothing.
			copier.ProcUtil.IncrementSucceeded()
		} else {
			// We successfully copied the bag. Send it on to
			// the validation queue.
			copier.ProcUtil.IncrementSucceeded()
			SendToValidationQueue(result, copier.ProcUtil)
		}

		if result.NsqMessage == nil {
			// This is a test message, running outside production.
			copier.WaitGroup.Done()
		} else {
			result.NsqMessage.Finish()
		}
		copier.ProcUtil.LogStats()

	}
}

func (copier *Copier) RunTest(dpnResult *DPNResult) {
	copier.WaitGroup.Add(1)
	copier.ProcUtil.MessageLog.Info("Putting %s into lookup channel",
		dpnResult.BagIdentifier)
	copier.CopyChannel <- dpnResult
	copier.WaitGroup.Wait()
}

// Returns a command object for copying from the remote location to
// the local filesystem. The copy is done via rsync over ssh, and
// the command will capture stdout and stderr. The copyFrom param
// should be a valid scp target in this format:
//
// remoteuser@remotehost:/remote/dir/bag.tar
//
// The copyTo param should be an absolute path on a locally-accessible
// file system, such as:
//
// /mnt/dpn/data/bag.tar
//
// Using this assumes a few things:
//
// 1. You have rsync installed.
// 2. You have an ssh client installed.
// 3. You have an entry in your ~/.ssh/config file specifying
//    connection and key information for the remote host.
//
// Usage:
//
// command := GetRsyncCommand("aptrust@tdr:bag.tar", "/mnt/dpn/bag.tar")
// err := command.Run()
// if err != nil {
//    ... do something ...
// }
//
// -- OR --
//
// output, err := command.CombinedOutput()
// if err != nil {
//    fmt.Println(err.Error())
//    fmt.Println(string(output))
// }

//
func GetRsyncCommand(copyFrom, copyTo string, useSSH bool) (*exec.Cmd) {
//	rsync -avz -e ssh remoteuser@remotehost:/remote/dir /this/dir/
	if useSSH {
		return exec.Command("rsync", "-avz", "-e",  "ssh", copyFrom, copyTo)
	}
	return exec.Command("rsync", "-avz", copyFrom, copyTo)
}
<commit_msg>Hack/fix - monitor DPN volume for DPN ingest<commit_after>package dpn

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/nsqio/go-nsq"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
)

// copier.go copies tarred bags from other nodes via rsync.
// This is used when replicating content from other nodes.
// For putting together DPN bags from APTrust files, see fetcher.go.

type Copier struct {
	LookupChannel       chan *DPNResult
	CopyChannel         chan *DPNResult
	PostProcessChannel  chan *DPNResult
	DPNConfig           *DPNConfig
	ProcUtil            *bagman.ProcessUtil
	LocalClient         *DPNRestClient
	RemoteClients       map[string]*DPNRestClient
	// WaitGroup is for running local tests only.
	WaitGroup           sync.WaitGroup
}

type CopyResult struct {
	LocalPath       string
	ErrorMessage    string
	RsyncStdout     string
	RsyncStderr     string
	InfoMessage     string
	BagWasCopied    bool
}

func NewCopier(procUtil *bagman.ProcessUtil, dpnConfig *DPNConfig) (*Copier, error) {
	localClient, err := NewDPNRestClient(
		dpnConfig.RestClient.LocalServiceURL,
		dpnConfig.RestClient.LocalAPIRoot,
		dpnConfig.RestClient.LocalAuthToken,
		dpnConfig.LocalNode,
		dpnConfig,
		procUtil.MessageLog)

	// HACK: ProcUtil assumes the volume it should be managing is
	// the APTrust volume. In this case, it's the DPN volume.
	// DPNStagingDirectory
	volume, err := bagman.NewVolume(procUtil.Config.DPNStagingDirectory, procUtil.MessageLog)
	if err != nil {
		message := fmt.Sprintf("Exiting. Cannot init Volume object: %v", err)
		fmt.Fprintln(os.Stderr, message)
		procUtil.MessageLog.Fatal(message)
	}
	procUtil.Volume = volume

	if err != nil {
		return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	remoteClients, err := GetRemoteClients(localClient, dpnConfig,
		procUtil.MessageLog)
	if err != nil {
		return nil, err
	}
	copier := &Copier {
		DPNConfig: dpnConfig,
		ProcUtil: procUtil,
		LocalClient: localClient,
		RemoteClients: remoteClients,
	}
	workerBufferSize := procUtil.Config.DPNPackageWorker.Workers * 4
	copier.LookupChannel = make(chan *DPNResult, workerBufferSize)
	copier.CopyChannel = make(chan *DPNResult, workerBufferSize)
	copier.PostProcessChannel = make(chan *DPNResult, workerBufferSize)
	for i := 0; i < procUtil.Config.DPNPackageWorker.Workers; i++ {
		go copier.doLookup()
		go copier.doCopy()
		go copier.postProcess()
	}
	return copier, nil
}

func (copier *Copier) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()
	dpnResult := &DPNResult{}
	err := json.Unmarshal(message.Body, dpnResult)
	if err != nil {
		detailedError := fmt.Errorf("Could not unmarshal JSON data from nsq. " +
			"Error is: %v    JSON is: %s", err.Error(), string(message.Body))
		copier.ProcUtil.MessageLog.Error(detailedError.Error())
		message.Finish()
		return detailedError
	}

	// Set up the copy result
	dpnResult.CopyResult = &CopyResult{
		BagWasCopied: false,
	}
	dpnResult.NsqMessage = message

	// Start processing.
	dpnResult.Stage = STAGE_COPY
	copier.LookupChannel <- dpnResult
	copier.ProcUtil.MessageLog.Info("Put %s from %s into copy channel",
		dpnResult.DPNBag.UUID, dpnResult.DPNBag.AdminNode)
	return nil
}

// Look up the DPN bag on the admin node. Although we already have the
// bag object as bart of the DPNResult object, this request may have been
// sitting in the queue for many hours, and the replication request may
// have been fulfilled or cancelled in that time. So check the status on
// the authoritative node to avoid unnecessarily processing what might
// be hundreds of gigs of data.
func (copier *Copier) doLookup() {
	for result := range copier.LookupChannel {
		// Get a client to talk to the FromNode
		remoteClient := copier.RemoteClients[result.TransferRequest.FromNode]

		copier.ProcUtil.MessageLog.Debug(
			"Looking up ReplicationId %s, bag %s, on node %s ",
				result.TransferRequest.ReplicationId,
				result.TransferRequest.BagId,
				result.TransferRequest.FromNode)

		// If we can find out for sure that this replication request should
		// not be processed, then don't process it...
		xfer, _ := remoteClient.ReplicationTransferGet(
			result.TransferRequest.ReplicationId)
		if xfer != nil && xfer.Status != "requested" {
			message := fmt.Sprintf(
				"Cancelling copy of ReplicationId %s (bag %s) because " +
					"replication status on %s is %s",
				result.TransferRequest.ReplicationId,
				result.TransferRequest.BagId,
				result.TransferRequest.FromNode,
				xfer.Status)
			copier.ProcUtil.MessageLog.Info(message)
			result.CopyResult.InfoMessage = message
			result.Retry = false
			result.TransferRequest = xfer
			copier.PostProcessChannel <- result
			continue
		}
		// ...otherwise, proceed with processing.
		copier.CopyChannel <- result
	}
}

// Copy the file from the remote node to our local staging area.
// Calculate checksums.
func (copier *Copier) doCopy() {
	for result := range copier.CopyChannel {

		// Make sure we have enough room on the volume
		// to download and unpack this bag.
		err := copier.ProcUtil.Volume.Reserve(uint64(float64(result.DPNBag.Size) * float64(2.1)))
		if err != nil {
			// Not enough room on disk
			copier.ProcUtil.MessageLog.Warning(
				"Requeueing %s from %s (%d bytes) - not enough disk space",
				result.TransferRequest.ReplicationId,
				result.TransferRequest.FromNode,
				result.DPNBag.Size)
			result.CopyResult.ErrorMessage = err.Error()
			result.ErrorMessage = err.Error()
			result.Retry = true
			copier.PostProcessChannel <- result
			continue
		}

		localPath := filepath.Join(
			copier.ProcUtil.Config.DPNStagingDirectory,
			fmt.Sprintf("%s.tar", result.TransferRequest.BagId))

		if !bagman.FileExists(copier.ProcUtil.Config.DPNStagingDirectory) {
			os.MkdirAll(copier.ProcUtil.Config.DPNStagingDirectory, 0755)
		}

		// DEBUG - use for tracing 'file not found'
		// fmt.Printf("Rsync link is %s\n", result.TransferRequest.Link)

		copier.ProcUtil.MessageLog.Info("Rsync link is %s", result.TransferRequest.Link)
		rsyncCommand := GetRsyncCommand(result.TransferRequest.Link,
			localPath, copier.DPNConfig.UseSSHWithRsync)

		// Touch message on both sides of rsync, so NSQ doesn't time out.
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}
		output, err := rsyncCommand.CombinedOutput()
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}
		if err != nil {
			result.CopyResult.ErrorMessage = fmt.Sprintf("%s: %s",
				err.Error(), string(output))
		} else {
			result.LocalPath = localPath
			result.CopyResult.LocalPath = localPath
			result.CopyResult.BagWasCopied = true
			// TODO: This is not necessary. We just need to calculate the checksum
			// of the SHA-256 manifest
			fileDigest, err := bagman.CalculateDigests(localPath)
			if result.NsqMessage != nil {
				result.NsqMessage.Touch()
			}
			if err != nil {
				result.ErrorMessage = fmt.Sprintf("Could not calculate checksums on '%s': %v",
					result.PackageResult.TarFilePath, err)
				copier.PostProcessChannel <- result
				continue
			}
			result.BagMd5Digest = fileDigest.Md5Digest
			result.BagSha256Digest = fileDigest.Sha256Digest
			result.BagSize = fileDigest.Size
		}
		copier.PostProcessChannel <- result
	}
}

func (copier *Copier) postProcess() {
	// On success, send to validation queue.
	// Otherwise, send to trouble queue.
	for result := range copier.PostProcessChannel {
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}
		result.ErrorMessage = result.CopyResult.ErrorMessage

		// On error, log and send to trouble queue if the error is fatal
		if result.ErrorMessage != "" {
			copier.ProcUtil.MessageLog.Error(result.ErrorMessage)
			copier.ProcUtil.IncrementFailed()
			if result.Retry == false {
				SendToTroubleQueue(result, copier.ProcUtil)
			}
			if bagman.FileExists(result.CopyResult.LocalPath) {
				os.Remove(result.CopyResult.LocalPath)
				copier.ProcUtil.MessageLog.Debug(
					"Deleting bag file %s", result.CopyResult.LocalPath)
			}
		} else if result.CopyResult.BagWasCopied == false {
			// We didn't copy the bag, but there was no error.
			// This happens when the transfer request is marked
			// as completed or cancelled on the remote node.
			// Count this as success, because we did what we're
			// supposed to do in this case, which is nothing.
			copier.ProcUtil.IncrementSucceeded()
		} else {
			// We successfully copied the bag. Send it on to
			// the validation queue.
			copier.ProcUtil.IncrementSucceeded()
			SendToValidationQueue(result, copier.ProcUtil)
		}

		if result.NsqMessage == nil {
			// This is a test message, running outside production.
			copier.WaitGroup.Done()
		} else {
			result.NsqMessage.Finish()
		}
		copier.ProcUtil.LogStats()

	}
}

func (copier *Copier) RunTest(dpnResult *DPNResult) {
	copier.WaitGroup.Add(1)
	copier.ProcUtil.MessageLog.Info("Putting %s into lookup channel",
		dpnResult.BagIdentifier)
	copier.CopyChannel <- dpnResult
	copier.WaitGroup.Wait()
}

// Returns a command object for copying from the remote location to
// the local filesystem. The copy is done via rsync over ssh, and
// the command will capture stdout and stderr. The copyFrom param
// should be a valid scp target in this format:
//
// remoteuser@remotehost:/remote/dir/bag.tar
//
// The copyTo param should be an absolute path on a locally-accessible
// file system, such as:
//
// /mnt/dpn/data/bag.tar
//
// Using this assumes a few things:
//
// 1. You have rsync installed.
// 2. You have an ssh client installed.
// 3. You have an entry in your ~/.ssh/config file specifying
//    connection and key information for the remote host.
//
// Usage:
//
// command := GetRsyncCommand("aptrust@tdr:bag.tar", "/mnt/dpn/bag.tar")
// err := command.Run()
// if err != nil {
//    ... do something ...
// }
//
// -- OR --
//
// output, err := command.CombinedOutput()
// if err != nil {
//    fmt.Println(err.Error())
//    fmt.Println(string(output))
// }

//
func GetRsyncCommand(copyFrom, copyTo string, useSSH bool) (*exec.Cmd) {
//	rsync -avz -e ssh remoteuser@remotehost:/remote/dir /this/dir/
	if useSSH {
		return exec.Command("rsync", "-avz", "-e",  "ssh", copyFrom, copyTo)
	}
	return exec.Command("rsync", "-avz", copyFrom, copyTo)
}
<|endoftext|>


<commit_before>package draw

// TODO
<commit_msg>goplan9/draw: implement ReadSnarf and WriteSnarf It would be preferable if the RPC used bytes rather than strings, but I didn't want to mess with that.<commit_after>package draw

import (
	"fmt"
	"os"
)

func (d *Display) ReadSnarf() (string, error) {
	str, err := d.conn.ReadSnarf()
	if err != nil {
		fmt.Fprintf(os.Stderr, "ReadSnarf: %v\n", err)
		return "", err
	}
	return str, nil
}

func (d *Display) WriteSnarf(str string) error {
	err := d.conn.WriteSnarf(str)
	if err != nil {
		fmt.Fprintf(os.Stderr, "WriteSnarf: %v\n", err)
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package aliyunecs

const autoFdiskScript = `#/bin/bash
#fdisk ,formating and create the file system on /dev/xvdb
fdisk_fun()
{
fdisk -S 56 /dev/xvdb << EOF
n
p
1

wq
EOF

sleep 5
mkfs.ext4 /dev/xvdb1
}

#config /etc/fstab and mount device
main()
{
  fdisk_fun
  mkdir /var/lib/docker
  echo "/dev/xvdb1    /var/lib/docker     ext4    defaults        0 0" >>/etc/fstab
  mount -a
}

main
df -h

`
<commit_msg>support iooptimized disk mount<commit_after>package aliyunecs

const autoFdiskScript = `#/bin/bash
#fdisk ,formating and create the file system on /dev/xvdb or /dev/vdb
DISK_ATTACH_POINT="/dev/xvdb"
fdisk_fun()
{
fdisk -S 56 \$DISK_ATTACH_POINT << EOF
n
p
1

wq
EOF

sleep 5
mkfs.ext4 \${DISK_ATTACH_POINT}1
}

#config /etc/fstab and mount device
main()
{
  if [ -b "/dev/vdb" ]; then
    DISK_ATTACH_POINT="/dev/vdb"
  fi

  fdisk_fun
  flag=0
  if [ -d "/var/lib/docker" ];then
    flag=1
    service docker stop
    rm -fr /var/lib/docker
  fi
  mkdir /var/lib/docker
  echo "\${DISK_ATTACH_POINT}1    /var/lib/docker     ext4    defaults        0 0" >>/etc/fstab
  mount -a

  if [ \$flag==1 ]; then
    service docker start
  fi
}

main
df -h

`
<|endoftext|>