<commit_before>package web

import (
	"fmt"
	"html/template"
	"log"
	"net/http"
	"net/url"
	"os"
	"psychic-rat/mdl"
	"psychic-rat/sess"
	"psychic-rat/types"
	"strconv"

	"github.com/gorilla/sessions"
)

type (
	Auth0 struct {
		Auth0ClientId    string
		Auth0CallbackURL string
		Auth0Domain      string
	}

	pageVariables struct {
		Auth0
		Items     []types.Item
		User      mdl.User
		NewItems  []types.NewItem
		Companies []types.Company
	}

	renderFunc     func(writer http.ResponseWriter, templateName string, vars *pageVariables)
	handlerFunc    func(http.ResponseWriter, *http.Request)
	methodSelector map[string]handlerFunc
)

var (
	renderPage     = renderPageUsingTemplate
	isUserLoggedIn = isUserLoggedInSession
	auth0Store     = sessions.NewCookieStore([]byte("something-very-secret"))
	logDbg         = log.New(os.Stderr, "DBG:", 0).Print
	logDbgf        = log.New(os.Stderr, "DBG:", 0).Printf
)

func (pv *pageVariables) withSessionVars(r *http.Request) *pageVariables {
	// TODO: nil is a smell. StoreReader/Writer interfaces.
	s := sess.NewSessionStore(r, nil)
	user, err := s.Get()
	if err != nil {
		// todo - return error?
		log.Fatal(err)
	}
	if user != nil {
		pv.User = *user
	}
	return pv
}

func (pv *pageVariables) withAuth0Vars() *pageVariables {
	pv.Auth0Domain = os.Getenv("AUTH0_DOMAIN")
	pv.Auth0CallbackURL = os.Getenv("AUTH0_CALLBACK_URL")
	pv.Auth0ClientId = os.Getenv("AUTH0_CLIENT_ID")
	return pv
}

func renderPageUsingTemplate(writer http.ResponseWriter, templateName string, variables *pageVariables) {
	tpt := template.Must(template.New(templateName).ParseFiles(templateName, "header.html.tmpl", "footer.html.tmpl", "navi.html.tmpl"))
	tpt.Execute(writer, variables)
}

func HomePageHandler(writer http.ResponseWriter, request *http.Request) {
	selector := methodSelector{
		"GET": func(writer http.ResponseWriter, request *http.Request) {
			vars := (&pageVariables{}).withSessionVars(request)
			renderPage(writer, "home.html.tmpl", vars)
		},
	}
	execHandlerForMethod(selector, writer, request)
}

func execHandlerForMethod(selector methodSelector, writer http.ResponseWriter, request *http.Request) {
	f, exists := selector[request.Method]
	if !exists {
		http.Error(writer, "", http.StatusMethodNotAllowed)
		return
	}
	f(writer, request)
}

func SignInPageHandler(writer http.ResponseWriter, request *http.Request) {
	method := signInSimple
	if flags.enableAuth0 {
		method = signInAuth0
	}
	execHandlerForMethod(methodSelector{"GET": method}, writer, request)
}

func signInSimple(writer http.ResponseWriter, request *http.Request) {
	s := sess.NewSessionStore(request, writer)
	user, err := s.Get()
	if err != nil {
		http.Error(writer, err.Error(), http.StatusInternalServerError)
		return
	}
	if user == nil {
		logDbg("no user, attempting auth")
		if err := authUser(request, s); err != nil {
			log.Print(err)
			http.Error(writer, "authentication failed", http.StatusForbidden)
			return
		}
	}

	vars := (&pageVariables{}).withSessionVars(request)
	renderPage(writer, "signin.html.tmpl", vars)
}

func authUser(request *http.Request, session *sess.SessionStore) error {
	if err := request.ParseForm(); err != nil {
		return err
	}

	userId := request.FormValue("u")
	if userId == "" {
		return fmt.Errorf("userId not specified")
	}

	user, err := apis.User.GetUser(userId)
	if err != nil {
		return fmt.Errorf("can't get user by id %v : %v", userId, err)
	}
	return session.Save(*user)
}

func signInAuth0(writer http.ResponseWriter, request *http.Request) {
	vars := (&pageVariables{}).withAuth0Vars()
	log.Printf("vars = %+v\n", vars)
	renderPage(writer, "signin-auth0.html.tmpl", vars)
}

func userLoginRequired(h handlerFunc) handlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if !isUserLoggedIn(r) {
			log.Print("user not logged in")
			http.Error(w, "", http.StatusForbidden)
			return
		}
		h(w, r)
	}
}

func adminLoginRequired(h handlerFunc) handlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if !isUserAdmin(r) {
			log.Print("user not admin")
			http.Error(w, "", http.StatusForbidden)
			return
		}
		h(w, r)
	}
}

func PledgePageHandler(writer http.ResponseWriter, request *http.Request) {
	selector := methodSelector{
		"GET":  userLoginRequired(pledgeGetHandler),
		"POST": userLoginRequired(pledgePostHandler),
	}
	execHandlerForMethod(selector, writer, request)
}

func isUserLoggedInSession(request *http.Request) bool {
	// TODO: nil is a smell. StoreReader/Writer interfaces.
	s := sess.NewSessionStore(request, nil)
	user, err := s.Get()
	if err != nil {
		log.Print(err)
		return false
	}
	return user != nil
}

func isUserAdmin(request *http.Request) bool {
	s := sess.NewSessionStore(request, nil)
	user, err := s.Get()
	if err != nil {
		log.Print(err)
		return false
	}
	return user != nil && user.IsAdmin
}

func pledgeGetHandler(writer http.ResponseWriter, request *http.Request) {
	report, err := apis.Item.ListItems()
	if err != nil {
		log.Print(err)
		http.Error(writer, "", http.StatusInternalServerError)
		return
	}
	vars := &pageVariables{Items: report}
	vars = vars.withSessionVars(request)
	renderPage(writer, "pledge.html.tmpl", vars)
}

func pledgePostHandler(writer http.ResponseWriter, request *http.Request) {
	if err := request.ParseForm(); err != nil {
		log.Print(err)
		http.Error(writer, "", http.StatusInternalServerError)
		return
	}

	itemId64, err := strconv.ParseInt(request.FormValue("item"), 10, 32)
	if err != nil {
		http.Error(writer, "", http.StatusBadRequest)
		return
	}
	itemId := int(itemId64)

	item, err := apis.Item.GetItem(itemId)
	if err != nil {
		log.Printf("error looking up item %v : %v", itemId, err)
		http.Error(writer, "", http.StatusBadRequest)
		return
	}

	// TODO ignoring a couple of errors
	s := sess.NewSessionStore(request, writer)
	user, _ := s.Get()
	userId := user.Id

	log.Printf("pledge item %v from user %v", itemId, userId)
	plId, err := apis.Pledge.NewPledge(itemId, userId)
	if err != nil {
		log.Print("unable to pledge : ", err)
		return
	}
	log.Printf("pledge %v created", plId)

	vars := &pageVariables{
		User:  *user,
		Items: []types.Item{item},
	}
	renderPage(writer, "pledge-post.html.tmpl", vars)
}

func ThanksPageHandler(writer http.ResponseWriter, request *http.Request) {
	selector := methodSelector{
		"GET": userLoginRequired(func(writer http.ResponseWriter, request *http.Request) {
			vars := (&pageVariables{}).withSessionVars(request)
			renderPage(writer, "thanks.html.tmpl", vars)
		}),
	}
	execHandlerForMethod(selector, writer, request)
}

func NewItemHandler(w http.ResponseWriter, r *http.Request) {
	selector := methodSelector{
		"GET":  userLoginRequired(newItemListHandler),
		"POST": userLoginRequired(newItemPostHandler),
	}
	execHandlerForMethod(selector, w, r)
}

func newItemListHandler(w http.ResponseWriter, r *http.Request) {

}

func newItemPostHandler(w http.ResponseWriter, r *http.Request) {
	if err := r.ParseForm(); err != nil {
		log.Print(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}
	// TODO ignoring a couple of errors
	s := sess.NewSessionStore(r, w)
	user, _ := s.Get()
	userId := user.Id

	company := r.FormValue("company")
	make := r.FormValue("make")
	model := r.FormValue("model")
	if company == "" || model == "" || make == "" {
		http.Error(w, "", http.StatusBadRequest)
		return
	}
	//value, err := strconv.ParseFloat(r.FormValue("value"), 10)
	//if err != nil {
	//	log.Print(err)
	//	http.Error(w, "", http.StatusBadRequest)
	//	return
	//}

	newItem := types.NewItem{UserID: userId, IsPledge: true, Make: make, Model: model, Company: company}
	_, err := apis.NewItem.AddNewItem(newItem)
	if err != nil {
		log.Printf("unable to add new item %v:", newItem, err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	item := types.Item{Make: newItem.Make, Model: newItem.Model, Company: types.Company{Name: company}}
	vars := &pageVariables{
		User:  *user,
		Items: []types.Item{item},
	}
	renderPage(w, "pledge-post-new-item.html.tmpl", vars)
}

func AdminItemHandler(w http.ResponseWriter, r *http.Request) {
	selector := methodSelector{
		"GET":  adminLoginRequired(listNewItems),
		"POST": adminLoginRequired(approveNewItems),
	}
	execHandlerForMethod(selector, w, r)
}

func listNewItems(w http.ResponseWriter, r *http.Request) {
	newItems, err := apis.NewItem.ListNewItems()
	if err != nil {
		log.Printf("unable to retrieve new items: %v", err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}
	items, err := apis.Item.ListItems()
	if err != nil {
		log.Printf("unable to retrieve items: %v", err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}
	companies, err := apis.Company.GetCompanies()
	if err != nil {
		log.Printf("unable to retrieve companies: %v", err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	vars := &pageVariables{Items: items, NewItems: newItems, Companies: companies}
	renderPage(w, "admin-new-items.html.tmpl", vars)
}

type formReader struct {
	form url.Values
	row  int
	err  []error
}

func (f *formReader) getString(field string) string {
	v, ok := f.form[field]
	if !ok || !(f.row < len(v)) {
		f.err = append(f.err, fmt.Errorf("%s not found in form (looking up row %d in %d items)", field, f.row, len(v)))
		return ""
	}
	return v[f.row]
}

func (f *formReader) getInt(field string) int {
	val := f.getString(field)
	i, err := strconv.ParseInt(val, 10, 32)
	if err != nil {
		f.err = append(f.err, fmt.Errorf("error parsing field %s = %s into int: %v", field, val, err))
		return 0
	}
	return int(i)
}

func approveNewItems(w http.ResponseWriter, r *http.Request) {
	err := r.ParseForm()
	if err != nil {
		log.Printf("unable to parse form: %v", err)
		http.Error(w, "", http.StatusBadRequest)
		return
	}
	addItems, ok := r.Form["add[]"]
	if !ok {
		log.Printf("no items to add\n")
		log.Printf("r.Form = %+v\n", r.Form)
		return
	}
	for _, rowString := range addItems {
		rowID, err := strconv.ParseInt(rowString, 10, 32)
		if err != nil {
			log.Printf("unable to parse row Id: %v", err)
			http.Error(w, "", http.StatusBadRequest)
			return
		}
		reader := formReader{r.Form, int(rowID), []error{}}
		itemID := reader.getInt("item[]")
		companyID := reader.getInt("company[]")
		userCompany := reader.getString("usercompany[]")
		userMake := reader.getString("usermake[]")
		userModel := reader.getString("usermodel[]")
		isPledge := reader.getString("isPledge[]")
		newItemID := reader.getInt("id[]")
		userID := reader.getString("userID[]")
		if len(reader.err) > 0 {
			log.Printf("errors while parsing form line %d: %v", rowID, err)
			http.Error(w, "", http.StatusBadRequest)
			return
		}

		var newItem *types.Item
		if itemID == 0 {
			// Adding a new item, figure out the company
			var company *types.Company
			if companyID == 0 {
				company, _ = apis.Company.NewCompany(types.Company{Name: userCompany})
			}
			ni := types.Item{Company: *company, Make: userMake, Model: userModel}
			newItem, _ = apis.Item.AddItem(ni)
		}

		if isPledge == "1" {
			apis.Pledge.NewPledge(newItem.Id, userID)
		}

		err = apis.NewItem.DeleteNewItem(int(newItemID))
		if err != nil {
			log.Printf("unable to delete new item %d:  %v", itemID, err)
			http.Error(w, "", http.StatusBadRequest)
			return
		}

	}
}
<commit_msg>added transaction to handle errors<commit_after>package web

import (
	"fmt"
	"html/template"
	"log"
	"net/http"
	"net/url"
	"os"
	"psychic-rat/mdl"
	"psychic-rat/sess"
	"psychic-rat/types"
	"strconv"

	"github.com/gorilla/sessions"
)

type (
	Auth0 struct {
		Auth0ClientId    string
		Auth0CallbackURL string
		Auth0Domain      string
	}

	pageVariables struct {
		Auth0
		Items     []types.Item
		User      mdl.User
		NewItems  []types.NewItem
		Companies []types.Company
	}

	renderFunc     func(writer http.ResponseWriter, templateName string, vars *pageVariables)
	handlerFunc    func(http.ResponseWriter, *http.Request)
	methodSelector map[string]handlerFunc
)

var (
	renderPage     = renderPageUsingTemplate
	isUserLoggedIn = isUserLoggedInSession
	auth0Store     = sessions.NewCookieStore([]byte("something-very-secret"))
	logDbg         = log.New(os.Stderr, "DBG:", 0).Print
	logDbgf        = log.New(os.Stderr, "DBG:", 0).Printf
)

func (pv *pageVariables) withSessionVars(r *http.Request) *pageVariables {
	// TODO: nil is a smell. StoreReader/Writer interfaces.
	s := sess.NewSessionStore(r, nil)
	user, err := s.Get()
	if err != nil {
		// todo - return error?
		log.Fatal(err)
	}
	if user != nil {
		pv.User = *user
	}
	return pv
}

func (pv *pageVariables) withAuth0Vars() *pageVariables {
	pv.Auth0Domain = os.Getenv("AUTH0_DOMAIN")
	pv.Auth0CallbackURL = os.Getenv("AUTH0_CALLBACK_URL")
	pv.Auth0ClientId = os.Getenv("AUTH0_CLIENT_ID")
	return pv
}

func renderPageUsingTemplate(writer http.ResponseWriter, templateName string, variables *pageVariables) {
	tpt := template.Must(template.New(templateName).ParseFiles(templateName, "header.html.tmpl", "footer.html.tmpl", "navi.html.tmpl"))
	tpt.Execute(writer, variables)
}

func HomePageHandler(writer http.ResponseWriter, request *http.Request) {
	selector := methodSelector{
		"GET": func(writer http.ResponseWriter, request *http.Request) {
			vars := (&pageVariables{}).withSessionVars(request)
			renderPage(writer, "home.html.tmpl", vars)
		},
	}
	execHandlerForMethod(selector, writer, request)
}

func execHandlerForMethod(selector methodSelector, writer http.ResponseWriter, request *http.Request) {
	f, exists := selector[request.Method]
	if !exists {
		http.Error(writer, "", http.StatusMethodNotAllowed)
		return
	}
	f(writer, request)
}

func SignInPageHandler(writer http.ResponseWriter, request *http.Request) {
	method := signInSimple
	if flags.enableAuth0 {
		method = signInAuth0
	}
	execHandlerForMethod(methodSelector{"GET": method}, writer, request)
}

func signInSimple(writer http.ResponseWriter, request *http.Request) {
	s := sess.NewSessionStore(request, writer)
	user, err := s.Get()
	if err != nil {
		http.Error(writer, err.Error(), http.StatusInternalServerError)
		return
	}
	if user == nil {
		logDbg("no user, attempting auth")
		if err := authUser(request, s); err != nil {
			log.Print(err)
			http.Error(writer, "authentication failed", http.StatusForbidden)
			return
		}
	}

	vars := (&pageVariables{}).withSessionVars(request)
	renderPage(writer, "signin.html.tmpl", vars)
}

func authUser(request *http.Request, session *sess.SessionStore) error {
	if err := request.ParseForm(); err != nil {
		return err
	}

	userId := request.FormValue("u")
	if userId == "" {
		return fmt.Errorf("userId not specified")
	}

	user, err := apis.User.GetUser(userId)
	if err != nil {
		return fmt.Errorf("can't get user by id %v : %v", userId, err)
	}
	return session.Save(*user)
}

func signInAuth0(writer http.ResponseWriter, request *http.Request) {
	vars := (&pageVariables{}).withAuth0Vars()
	log.Printf("vars = %+v\n", vars)
	renderPage(writer, "signin-auth0.html.tmpl", vars)
}

func userLoginRequired(h handlerFunc) handlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if !isUserLoggedIn(r) {
			log.Print("user not logged in")
			http.Error(w, "", http.StatusForbidden)
			return
		}
		h(w, r)
	}
}

func adminLoginRequired(h handlerFunc) handlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if !isUserAdmin(r) {
			log.Print("user not admin")
			http.Error(w, "", http.StatusForbidden)
			return
		}
		h(w, r)
	}
}

func PledgePageHandler(writer http.ResponseWriter, request *http.Request) {
	selector := methodSelector{
		"GET":  userLoginRequired(pledgeGetHandler),
		"POST": userLoginRequired(pledgePostHandler),
	}
	execHandlerForMethod(selector, writer, request)
}

func isUserLoggedInSession(request *http.Request) bool {
	// TODO: nil is a smell. StoreReader/Writer interfaces.
	s := sess.NewSessionStore(request, nil)
	user, err := s.Get()
	if err != nil {
		log.Print(err)
		return false
	}
	return user != nil
}

func isUserAdmin(request *http.Request) bool {
	s := sess.NewSessionStore(request, nil)
	user, err := s.Get()
	if err != nil {
		log.Print(err)
		return false
	}
	return user != nil && user.IsAdmin
}

func pledgeGetHandler(writer http.ResponseWriter, request *http.Request) {
	report, err := apis.Item.ListItems()
	if err != nil {
		log.Print(err)
		http.Error(writer, "", http.StatusInternalServerError)
		return
	}
	vars := &pageVariables{Items: report}
	vars = vars.withSessionVars(request)
	renderPage(writer, "pledge.html.tmpl", vars)
}

func pledgePostHandler(writer http.ResponseWriter, request *http.Request) {
	if err := request.ParseForm(); err != nil {
		log.Print(err)
		http.Error(writer, "", http.StatusInternalServerError)
		return
	}

	itemId64, err := strconv.ParseInt(request.FormValue("item"), 10, 32)
	if err != nil {
		http.Error(writer, "", http.StatusBadRequest)
		return
	}
	itemId := int(itemId64)

	item, err := apis.Item.GetItem(itemId)
	if err != nil {
		log.Printf("error looking up item %v : %v", itemId, err)
		http.Error(writer, "", http.StatusBadRequest)
		return
	}

	// TODO ignoring a couple of errors
	s := sess.NewSessionStore(request, writer)
	user, _ := s.Get()
	userId := user.Id

	log.Printf("pledge item %v from user %v", itemId, userId)
	plId, err := apis.Pledge.NewPledge(itemId, userId)
	if err != nil {
		log.Print("unable to pledge : ", err)
		return
	}
	log.Printf("pledge %v created", plId)

	vars := &pageVariables{
		User:  *user,
		Items: []types.Item{item},
	}
	renderPage(writer, "pledge-post.html.tmpl", vars)
}

func ThanksPageHandler(writer http.ResponseWriter, request *http.Request) {
	selector := methodSelector{
		"GET": userLoginRequired(func(writer http.ResponseWriter, request *http.Request) {
			vars := (&pageVariables{}).withSessionVars(request)
			renderPage(writer, "thanks.html.tmpl", vars)
		}),
	}
	execHandlerForMethod(selector, writer, request)
}

func NewItemHandler(w http.ResponseWriter, r *http.Request) {
	selector := methodSelector{
		"GET":  userLoginRequired(newItemListHandler),
		"POST": userLoginRequired(newItemPostHandler),
	}
	execHandlerForMethod(selector, w, r)
}

func newItemListHandler(w http.ResponseWriter, r *http.Request) {

}

func newItemPostHandler(w http.ResponseWriter, r *http.Request) {
	if err := r.ParseForm(); err != nil {
		log.Print(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}
	// TODO ignoring a couple of errors
	s := sess.NewSessionStore(r, w)
	user, _ := s.Get()
	userId := user.Id

	company := r.FormValue("company")
	make := r.FormValue("make")
	model := r.FormValue("model")
	if company == "" || model == "" || make == "" {
		http.Error(w, "", http.StatusBadRequest)
		return
	}
	//value, err := strconv.ParseFloat(r.FormValue("value"), 10)
	//if err != nil {
	//	log.Print(err)
	//	http.Error(w, "", http.StatusBadRequest)
	//	return
	//}

	newItem := types.NewItem{UserID: userId, IsPledge: true, Make: make, Model: model, Company: company}
	_, err := apis.NewItem.AddNewItem(newItem)
	if err != nil {
		log.Printf("unable to add new item %v:", newItem, err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	item := types.Item{Make: newItem.Make, Model: newItem.Model, Company: types.Company{Name: company}}
	vars := &pageVariables{
		User:  *user,
		Items: []types.Item{item},
	}
	renderPage(w, "pledge-post-new-item.html.tmpl", vars)
}

func AdminItemHandler(w http.ResponseWriter, r *http.Request) {
	selector := methodSelector{
		"GET":  adminLoginRequired(listNewItems),
		"POST": adminLoginRequired(approveNewItems),
	}
	execHandlerForMethod(selector, w, r)
}

func listNewItems(w http.ResponseWriter, r *http.Request) {
	newItems, err := apis.NewItem.ListNewItems()
	if err != nil {
		log.Printf("unable to retrieve new items: %v", err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}
	items, err := apis.Item.ListItems()
	if err != nil {
		log.Printf("unable to retrieve items: %v", err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}
	companies, err := apis.Company.GetCompanies()
	if err != nil {
		log.Printf("unable to retrieve companies: %v", err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	vars := &pageVariables{Items: items, NewItems: newItems, Companies: companies}
	renderPage(w, "admin-new-items.html.tmpl", vars)
}

type formReader struct {
	form url.Values
	row  int
	err  []error
}

func (f *formReader) getString(field string) string {
	v, ok := f.form[field]
	if !ok || !(f.row < len(v)) {
		f.err = append(f.err, fmt.Errorf("%s not found in form (looking up row %d in %d items)", field, f.row, len(v)))
		return ""
	}
	return v[f.row]
}

func (f *formReader) getInt(field string) int {
	val := f.getString(field)
	i, err := strconv.ParseInt(val, 10, 32)
	if err != nil {
		f.err = append(f.err, fmt.Errorf("error parsing field %s = %s into int: %v", field, val, err))
		return 0
	}
	return int(i)
}

func approveNewItems(w http.ResponseWriter, r *http.Request) {
	err := r.ParseForm()
	if err != nil {
		log.Printf("unable to parse form: %v", err)
		http.Error(w, "", http.StatusBadRequest)
		return
	}
	addItems, ok := r.Form["add[]"]
	if !ok {
		log.Printf("no items to add\n")
		log.Printf("r.Form = %+v\n", r.Form)
		return
	}
	for _, rowString := range addItems {
		rowID, err := strconv.ParseInt(rowString, 10, 32)
		if err != nil {
			log.Printf("unable to parse row Id: %v", err)
			http.Error(w, "", http.StatusBadRequest)
			return
		}
		reader := formReader{r.Form, int(rowID), []error{}}
		newItemID := reader.getInt("id[]")
		userID := reader.getString("userID[]")
		itemID := reader.getInt("item[]")
		companyID := reader.getInt("company[]")
		userCompany := reader.getString("usercompany[]")
		userMake := reader.getString("usermake[]")
		userModel := reader.getString("usermodel[]")
		isPledge := reader.getString("isPledge[]")
		if len(reader.err) > 0 {
			log.Printf("errors while parsing form line %d: %v", rowID, err)
			http.Error(w, "", http.StatusBadRequest)
			return
		}

		txn := apiTxn{nil, apis}
		var newItem *types.Item
		if itemID == 0 {
			var company *types.Company
			if companyID == 0 {
				company = txn.newCompany(types.Company{Name: userCompany})
			}
			ni := types.Item{Company: *company, Make: userMake, Model: userModel}
			newItem = txn.addItem(ni)
		}

		if isPledge == "1" {
			txn.addPledge(newItem.Id, userID)
		}
		txn.deleteNewItem(int(newItemID))
		if txn.err != nil {
			log.Printf("unable to complete transaction for new item %d:  %v", itemID, err)
			http.Error(w, "", http.StatusBadRequest)
			return
		}

	}
}

type apiTxn struct {
	err  error
	apis API
}

func (a *apiTxn) newCompany(co types.Company) (c *types.Company) {
	if a.err != nil {
		return &co
	}
	c, a.err = a.apis.Company.NewCompany(co)
	return c
}

func (a *apiTxn) addItem(item types.Item) (i *types.Item) {
	if a.err != nil {
		return &item
	}
	i, a.err = a.apis.Item.AddItem(item)
	return i
}

func (a *apiTxn) addPledge(itemID int, userID string) (p int) {
	if a.err != nil {
		return 0
	}
	p, a.err = a.apis.Pledge.NewPledge(itemID, userID)
	return p
}

func (a *apiTxn) deleteNewItem(id int) {
	if a.err != nil {
		return
	}
	a.err = apis.NewItem.DeleteNewItem(id)
}
<|endoftext|>


<commit_before>package web

import (
	"context"
	"encoding/json"
	"fmt"
	"math"
	"net/http"
	"net/url"
	"sort"
	"sync"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/getlantern/zenodb/core"
	"github.com/getlantern/zenodb/encoding"
	"github.com/gorilla/mux"
	"github.com/retailnext/hllpp"
)

const (
	nanosPerMilli = 1000000

	pauseTime    = 250 * time.Millisecond
	shortTimeout = 5 * time.Second
	longTimeout  = 1000 * time.Hour
)

type QueryResult struct {
	SQL                string
	Permalink          string
	TS                 int64
	TSCardinality      uint64
	Fields             []string
	FieldCardinalities []uint64
	Dims               []string
	DimCardinalities   []uint64
	Rows               []*ResultRow
}

type ResultRow struct {
	TS   int64
	Key  map[string]interface{}
	Vals []float64
}

func (h *handler) runQuery(resp http.ResponseWriter, req *http.Request) {
	h.sqlQuery(resp, req, longTimeout)
}

func (h *handler) asyncQuery(resp http.ResponseWriter, req *http.Request) {
	h.sqlQuery(resp, req, shortTimeout)
}

func (h *handler) cachedQuery(resp http.ResponseWriter, req *http.Request) {
	if !h.authenticate(resp, req) {
		resp.WriteHeader(http.StatusForbidden)
		return
	}

	permalink := mux.Vars(req)["permalink"]
	ce, err := h.cache.getByPermalink(permalink)
	if ce == nil {
		http.NotFound(resp, req)
		return
	}
	h.respondWithCacheEntry(resp, req, ce, err, shortTimeout)
}

func (h *handler) sqlQuery(resp http.ResponseWriter, req *http.Request, timeout time.Duration) {
	if !h.authenticate(resp, req) {
		resp.WriteHeader(http.StatusForbidden)
		return
	}

	sqlString, _ := url.QueryUnescape(req.URL.RawQuery)

	ce, err := h.query(req, sqlString)
	h.respondWithCacheEntry(resp, req, ce, err, timeout)
}

func (h *handler) respondWithCacheEntry(resp http.ResponseWriter, req *http.Request, ce cacheEntry, err error, timeout time.Duration) {
	limit := int(timeout / pauseTime)
	for i := 0; i < limit; i++ {
		if err != nil {
			log.Error(err)
			resp.WriteHeader(http.StatusInternalServerError)
			fmt.Fprint(resp, err.Error())
			return
		}
		switch ce.status() {
		case statusSuccess:
			h.respondSuccess(resp, req, ce)
			return
		case statusError:
			h.respondError(resp, req, ce)
			return
		case statusPending:
			// Pause a little bit and try again
			time.Sleep(pauseTime)
			ce, err = h.cache.getByPermalink(ce.permalink())
		}
	}
	// Let the client know that we're still working on it
	resp.WriteHeader(http.StatusAccepted)
	fmt.Fprintf(resp, "/cached/%v", ce.permalink())
}

func (h *handler) respondSuccess(resp http.ResponseWriter, req *http.Request, ce cacheEntry) {
	resp.Header().Set("Content-Type", "application/json")
	resp.Header().Set("Cache-control", fmt.Sprintf("max-age=%d", int64(h.CacheTTL.Seconds())))
	resp.WriteHeader(http.StatusOK)
	resp.Write(ce.data())
}

func (h *handler) respondError(resp http.ResponseWriter, req *http.Request, ce cacheEntry) {
	resp.WriteHeader(http.StatusInternalServerError)
	resp.Write(ce.error())
}

func (h *handler) query(req *http.Request, sqlString string) (ce cacheEntry, err error) {
	if req.Header.Get("Cache-control") == "no-cache" {
		ce, err = h.cache.begin(sqlString)
		if err != nil {
			return
		}
	} else {
		var created bool
		ce, created, err = h.cache.getOrBegin(sqlString)
		if err != nil || !created {
			return
		}
		if ce.status() != statusPending {
			log.Debugf("Found results for %v in cache", sqlString)
			return
		}
	}

	// Run the query in the background
	go func() {
		var result *QueryResult
		result, err = h.doQuery(sqlString, ce.permalink())
		if err != nil {
			err = fmt.Errorf("Unable to query: %v", err)
			log.Error(err)
			ce = ce.fail(err)
		} else {
			resultBytes, err := json.Marshal(result)
			if err != nil {
				err = fmt.Errorf("Unable to marshal result: %v", err)
				log.Error(err)
				ce = ce.fail(err)
			} else if len(resultBytes) > h.MaxResponseBytes {
				err = fmt.Errorf("Size of query result exceeded %v", humanize.Bytes(uint64(len(resultBytes))))
				log.Error(err)
				ce = ce.fail(err)
			} else {
				ce = ce.succeed(resultBytes)
			}
		}
		h.cache.put(sqlString, ce)
		log.Debugf("Cached results for %v", sqlString)
	}()

	return
}

func (h *handler) doQuery(sqlString string, permalink string) (*QueryResult, error) {
	rs, err := h.db.Query(sqlString, false, nil, false)
	if err != nil {
		return nil, err
	}

	var addDim func(dim string)

	result := &QueryResult{
		SQL:       sqlString,
		Permalink: permalink,
		TS:        time.Now().UnixNano() / nanosPerMilli,
	}
	groupBy := rs.GetGroupBy()
	if len(groupBy) > 0 {
		addDim = func(dim string) {
			// noop
		}
		for _, gb := range groupBy {
			result.Dims = append(result.Dims, gb.Name)
		}
	} else {
		addDim = func(dim string) {
			found := false
			for _, existing := range result.Dims {
				if existing == dim {
					found = true
					break
				}
			}
			if !found {
				result.Dims = append(result.Dims, dim)
			}
		}
	}

	var fields core.Fields
	var fieldCardinalities []*hllpp.HLLPP
	dimCardinalities := make(map[string]*hllpp.HLLPP)
	tsCardinality := hllpp.New()
	cbytes := make([]byte, 8)

	var mx sync.Mutex
	ctx, cancel := context.WithTimeout(context.Background(), h.QueryTimeout)
	defer cancel()
	rs.Iterate(ctx, func(inFields core.Fields) error {
		fields = inFields
		for _, field := range fields {
			result.Fields = append(result.Fields, field.Name)
			fieldCardinalities = append(fieldCardinalities, hllpp.New())
		}
		return nil
	}, func(row *core.FlatRow) (bool, error) {
		mx.Lock()
		key := make(map[string]interface{}, 10)
		row.Key.Iterate(true, true, func(dim string, value interface{}, valueBytes []byte) bool {
			key[dim] = value
			addDim(dim)
			hlp := dimCardinalities[dim]
			if hlp == nil {
				hlp = hllpp.New()
				dimCardinalities[dim] = hlp
			}
			hlp.Add(valueBytes)
			return true
		})

		encoding.Binary.PutUint64(cbytes, uint64(row.TS))
		tsCardinality.Add(cbytes)

		resultRow := &ResultRow{
			TS:   row.TS / nanosPerMilli,
			Key:  key,
			Vals: make([]float64, 0, len(row.Values)),
		}

		for i, value := range row.Values {
			resultRow.Vals = append(resultRow.Vals, value)
			encoding.Binary.PutUint64(cbytes, math.Float64bits(value))
			fieldCardinalities[i].Add(cbytes)
		}
		result.Rows = append(result.Rows, resultRow)
		mx.Unlock()
		return true, nil
	})

	result.TSCardinality = tsCardinality.Count()
	result.Dims = make([]string, 0, len(dimCardinalities))
	for dim := range dimCardinalities {
		result.Dims = append(result.Dims, dim)
	}
	sort.Strings(result.Dims)
	for _, dim := range result.Dims {
		result.DimCardinalities = append(result.DimCardinalities, dimCardinalities[dim].Count())
	}

	result.FieldCardinalities = make([]uint64, 0, len(fieldCardinalities))
	for _, fieldCardinality := range fieldCardinalities {
		result.FieldCardinalities = append(result.FieldCardinalities, fieldCardinality.Count())
	}

	return result, nil
}

func intToBytes(i uint64) []byte {
	b := make([]byte, 8)
	encoding.Binary.PutUint64(b, i)
	return b
}
<commit_msg>Checking result size while accumulating results<commit_after>package web

import (
	"context"
	"encoding/json"
	"fmt"
	"math"
	"net/http"
	"net/url"
	"sort"
	"sync"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/getlantern/zenodb/core"
	"github.com/getlantern/zenodb/encoding"
	"github.com/gorilla/mux"
	"github.com/retailnext/hllpp"
)

const (
	nanosPerMilli = 1000000

	pauseTime    = 250 * time.Millisecond
	shortTimeout = 5 * time.Second
	longTimeout  = 1000 * time.Hour
)

type QueryResult struct {
	SQL                string
	Permalink          string
	TS                 int64
	TSCardinality      uint64
	Fields             []string
	FieldCardinalities []uint64
	Dims               []string
	DimCardinalities   []uint64
	Rows               []*ResultRow
}

type ResultRow struct {
	TS   int64
	Key  map[string]interface{}
	Vals []float64
}

func (h *handler) runQuery(resp http.ResponseWriter, req *http.Request) {
	h.sqlQuery(resp, req, longTimeout)
}

func (h *handler) asyncQuery(resp http.ResponseWriter, req *http.Request) {
	h.sqlQuery(resp, req, shortTimeout)
}

func (h *handler) cachedQuery(resp http.ResponseWriter, req *http.Request) {
	if !h.authenticate(resp, req) {
		resp.WriteHeader(http.StatusForbidden)
		return
	}

	permalink := mux.Vars(req)["permalink"]
	ce, err := h.cache.getByPermalink(permalink)
	if ce == nil {
		http.NotFound(resp, req)
		return
	}
	h.respondWithCacheEntry(resp, req, ce, err, shortTimeout)
}

func (h *handler) sqlQuery(resp http.ResponseWriter, req *http.Request, timeout time.Duration) {
	if !h.authenticate(resp, req) {
		resp.WriteHeader(http.StatusForbidden)
		return
	}

	sqlString, _ := url.QueryUnescape(req.URL.RawQuery)

	ce, err := h.query(req, sqlString)
	h.respondWithCacheEntry(resp, req, ce, err, timeout)
}

func (h *handler) respondWithCacheEntry(resp http.ResponseWriter, req *http.Request, ce cacheEntry, err error, timeout time.Duration) {
	limit := int(timeout / pauseTime)
	for i := 0; i < limit; i++ {
		if err != nil {
			log.Error(err)
			resp.WriteHeader(http.StatusInternalServerError)
			fmt.Fprint(resp, err.Error())
			return
		}
		switch ce.status() {
		case statusSuccess:
			h.respondSuccess(resp, req, ce)
			return
		case statusError:
			h.respondError(resp, req, ce)
			return
		case statusPending:
			// Pause a little bit and try again
			time.Sleep(pauseTime)
			ce, err = h.cache.getByPermalink(ce.permalink())
		}
	}
	// Let the client know that we're still working on it
	resp.WriteHeader(http.StatusAccepted)
	fmt.Fprintf(resp, "/cached/%v", ce.permalink())
}

func (h *handler) respondSuccess(resp http.ResponseWriter, req *http.Request, ce cacheEntry) {
	resp.Header().Set("Content-Type", "application/json")
	resp.Header().Set("Cache-control", fmt.Sprintf("max-age=%d", int64(h.CacheTTL.Seconds())))
	resp.WriteHeader(http.StatusOK)
	resp.Write(ce.data())
}

func (h *handler) respondError(resp http.ResponseWriter, req *http.Request, ce cacheEntry) {
	resp.WriteHeader(http.StatusInternalServerError)
	resp.Write(ce.error())
}

func (h *handler) query(req *http.Request, sqlString string) (ce cacheEntry, err error) {
	if req.Header.Get("Cache-control") == "no-cache" {
		ce, err = h.cache.begin(sqlString)
		if err != nil {
			return
		}
	} else {
		var created bool
		ce, created, err = h.cache.getOrBegin(sqlString)
		if err != nil || !created {
			return
		}
		if ce.status() != statusPending {
			log.Debugf("Found results for %v in cache", sqlString)
			return
		}
	}

	// Run the query in the background
	go func() {
		var result *QueryResult
		result, err = h.doQuery(sqlString, ce.permalink())
		if err != nil {
			err = fmt.Errorf("Unable to query: %v", err)
			log.Error(err)
			ce = ce.fail(err)
		} else {
			resultBytes, err := json.Marshal(result)
			if err != nil {
				err = fmt.Errorf("Unable to marshal result: %v", err)
				log.Error(err)
				ce = ce.fail(err)
			} else if len(resultBytes) > h.MaxResponseBytes {
				err = fmt.Errorf("Query result size %v exceeded limit of %v", humanize.Bytes(uint64(len(resultBytes))), humanize.Bytes(uint64(h.MaxResponseBytes)))
				log.Error(err)
				ce = ce.fail(err)
			} else {
				ce = ce.succeed(resultBytes)
			}
		}
		h.cache.put(sqlString, ce)
		log.Debugf("Cached results for %v", sqlString)
	}()

	return
}

func (h *handler) doQuery(sqlString string, permalink string) (*QueryResult, error) {
	rs, err := h.db.Query(sqlString, false, nil, false)
	if err != nil {
		return nil, err
	}

	var addDim func(dim string)

	result := &QueryResult{
		SQL:       sqlString,
		Permalink: permalink,
		TS:        time.Now().UnixNano() / nanosPerMilli,
	}
	groupBy := rs.GetGroupBy()
	if len(groupBy) > 0 {
		addDim = func(dim string) {
			// noop
		}
		for _, gb := range groupBy {
			result.Dims = append(result.Dims, gb.Name)
		}
	} else {
		addDim = func(dim string) {
			found := false
			for _, existing := range result.Dims {
				if existing == dim {
					found = true
					break
				}
			}
			if !found {
				result.Dims = append(result.Dims, dim)
			}
		}
	}

	var fields core.Fields
	var fieldCardinalities []*hllpp.HLLPP
	dimCardinalities := make(map[string]*hllpp.HLLPP)
	tsCardinality := hllpp.New()
	cbytes := make([]byte, 8)

	estimatedResultBytes := 0
	var mx sync.Mutex
	ctx, cancel := context.WithTimeout(context.Background(), h.QueryTimeout)
	defer cancel()
	rs.Iterate(ctx, func(inFields core.Fields) error {
		fields = inFields
		for _, field := range fields {
			result.Fields = append(result.Fields, field.Name)
			fieldCardinalities = append(fieldCardinalities, hllpp.New())
		}
		return nil
	}, func(row *core.FlatRow) (bool, error) {
		mx.Lock()
		key := make(map[string]interface{}, 10)
		row.Key.Iterate(true, true, func(dim string, value interface{}, valueBytes []byte) bool {
			key[dim] = value
			addDim(dim)
			hlp := dimCardinalities[dim]
			if hlp == nil {
				hlp = hllpp.New()
				dimCardinalities[dim] = hlp
			}
			hlp.Add(valueBytes)
			estimatedResultBytes += len(dim) + len(valueBytes)
			return true
		})

		estimatedResultBytes += 8 * len(row.Values)
		if estimatedResultBytes > h.MaxResponseBytes {
			mx.Unlock()
			// Note - the estimated size here is always an underestimate of the final
			// JSON size, so this is a conservative way to check. The final check
			// after generating the JSON may sometimes catch things that slipped
			// through here.
			return false, fmt.Errorf("Estimated query result size %v exceeded limit of %v", humanize.Bytes(uint64(estimatedResultBytes)), humanize.Bytes(uint64(h.MaxResponseBytes)))
		}

		encoding.Binary.PutUint64(cbytes, uint64(row.TS))
		tsCardinality.Add(cbytes)

		resultRow := &ResultRow{
			TS:   row.TS / nanosPerMilli,
			Key:  key,
			Vals: make([]float64, 0, len(row.Values)),
		}

		for i, value := range row.Values {
			resultRow.Vals = append(resultRow.Vals, value)
			encoding.Binary.PutUint64(cbytes, math.Float64bits(value))
			fieldCardinalities[i].Add(cbytes)
		}
		result.Rows = append(result.Rows, resultRow)
		mx.Unlock()
		return true, nil
	})

	result.TSCardinality = tsCardinality.Count()
	result.Dims = make([]string, 0, len(dimCardinalities))
	for dim := range dimCardinalities {
		result.Dims = append(result.Dims, dim)
	}
	sort.Strings(result.Dims)
	for _, dim := range result.Dims {
		result.DimCardinalities = append(result.DimCardinalities, dimCardinalities[dim].Count())
	}

	result.FieldCardinalities = make([]uint64, 0, len(fieldCardinalities))
	for _, fieldCardinality := range fieldCardinalities {
		result.FieldCardinalities = append(result.FieldCardinalities, fieldCardinality.Count())
	}

	return result, nil
}

func intToBytes(i uint64) []byte {
	b := make([]byte, 8)
	encoding.Binary.PutUint64(b, i)
	return b
}
<|endoftext|>


<commit_before>package filesys

import (
	"context"

	"github.com/seaweedfs/fuse"
	"github.com/seaweedfs/fuse/fs"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirectory fs.Node) error {

	newDir := newDirectory.(*Dir)

	newPath := util.NewFullPath(newDir.FullPath(), req.NewName)
	oldPath := util.NewFullPath(dir.FullPath(), req.OldName)

	glog.V(4).Infof("dir Rename %s => %s", oldPath, newPath)

	// find local old entry
	oldEntry, err := dir.wfs.metaCache.FindEntry(context.Background(), oldPath)
	if err != nil {
		glog.V(0).Infof("dir Rename can not find source %s : %v", oldPath, err)
		return fuse.ENOENT
	}

	// update remote filer
	err = dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.AtomicRenameEntryRequest{
			OldDirectory: dir.FullPath(),
			OldName:      req.OldName,
			NewDirectory: newDir.FullPath(),
			NewName:      req.NewName,
		}

		_, err := client.AtomicRenameEntry(context.Background(), request)
		if err != nil {
			return fuse.EIO
		}

		return nil

	})
	if err != nil {
		glog.V(0).Infof("dir Rename %s => %s : %v", oldPath, newPath, err)
		return fuse.EIO
	}

	// TODO: replicate renaming logic on filer
	if err := dir.wfs.metaCache.DeleteEntry(context.Background(), oldPath); err != nil {
		glog.V(0).Infof("dir Rename delete local %s => %s : %v", oldPath, newPath, err)
		return fuse.EIO
	}
	oldEntry.FullPath = newPath
	if err := dir.wfs.metaCache.InsertEntry(context.Background(), oldEntry); err != nil {
		glog.V(0).Infof("dir Rename insert local %s => %s : %v", oldPath, newPath, err)
		return fuse.EIO
	}

	// fmt.Printf("rename path: %v => %v\n", oldPath, newPath)
	dir.wfs.fsNodeCache.Move(oldPath, newPath)

	wfs.handlesLock.Lock()
	defer wfs.handlesLock.Unlock()
	delete(dir.wfs.handles, oldPath.AsInode())

	return err
}
<commit_msg>fix compilation<commit_after>package filesys

import (
	"context"

	"github.com/seaweedfs/fuse"
	"github.com/seaweedfs/fuse/fs"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirectory fs.Node) error {

	newDir := newDirectory.(*Dir)

	newPath := util.NewFullPath(newDir.FullPath(), req.NewName)
	oldPath := util.NewFullPath(dir.FullPath(), req.OldName)

	glog.V(4).Infof("dir Rename %s => %s", oldPath, newPath)

	// find local old entry
	oldEntry, err := dir.wfs.metaCache.FindEntry(context.Background(), oldPath)
	if err != nil {
		glog.V(0).Infof("dir Rename can not find source %s : %v", oldPath, err)
		return fuse.ENOENT
	}

	// update remote filer
	err = dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.AtomicRenameEntryRequest{
			OldDirectory: dir.FullPath(),
			OldName:      req.OldName,
			NewDirectory: newDir.FullPath(),
			NewName:      req.NewName,
		}

		_, err := client.AtomicRenameEntry(context.Background(), request)
		if err != nil {
			return fuse.EIO
		}

		return nil

	})
	if err != nil {
		glog.V(0).Infof("dir Rename %s => %s : %v", oldPath, newPath, err)
		return fuse.EIO
	}

	// TODO: replicate renaming logic on filer
	if err := dir.wfs.metaCache.DeleteEntry(context.Background(), oldPath); err != nil {
		glog.V(0).Infof("dir Rename delete local %s => %s : %v", oldPath, newPath, err)
		return fuse.EIO
	}
	oldEntry.FullPath = newPath
	if err := dir.wfs.metaCache.InsertEntry(context.Background(), oldEntry); err != nil {
		glog.V(0).Infof("dir Rename insert local %s => %s : %v", oldPath, newPath, err)
		return fuse.EIO
	}

	// fmt.Printf("rename path: %v => %v\n", oldPath, newPath)
	dir.wfs.fsNodeCache.Move(oldPath, newPath)

	dir.wfs.handlesLock.Lock()
	defer dir.wfs.handlesLock.Unlock()
	delete(dir.wfs.handles, oldPath.AsInode())

	return err
}
<|endoftext|>


<commit_before>package filesys

import (
	"context"
	"fmt"
	"mime"
	"path"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer2"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/gabriel-vasile/mimetype"
	"github.com/seaweedfs/fuse"
	"github.com/seaweedfs/fuse/fs"
)

type FileHandle struct {
	// cache file has been written to
	dirtyPages    *ContinuousDirtyPages
	contentType   string
	dirtyMetadata bool
	handle        uint64

	f         *File
	RequestId fuse.RequestID // unique ID for request
	NodeId    fuse.NodeID    // file or directory the request is about
	Uid       uint32         // user ID of process making request
	Gid       uint32         // group ID of process making request
}

func newFileHandle(file *File, uid, gid uint32) *FileHandle {
	return &FileHandle{
		f:          file,
		dirtyPages: newDirtyPages(file),
		Uid:        uid,
		Gid:        gid,
	}
}

var _ = fs.Handle(&FileHandle{})

// var _ = fs.HandleReadAller(&FileHandle{})
var _ = fs.HandleReader(&FileHandle{})
var _ = fs.HandleFlusher(&FileHandle{})
var _ = fs.HandleWriter(&FileHandle{})
var _ = fs.HandleReleaser(&FileHandle{})

func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {

	glog.V(4).Infof("%s read fh %d: [%d,%d)", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size))

	// this value should come from the filer instead of the old f
	if len(fh.f.entry.Chunks) == 0 {
		glog.V(1).Infof("empty fh %v/%v", fh.f.dir.Path, fh.f.Name)
		return nil
	}

	buff := make([]byte, req.Size)

	if fh.f.entryViewCache == nil {
		fh.f.entryViewCache = filer2.NonOverlappingVisibleIntervals(fh.f.entry.Chunks)
	}

	chunkViews := filer2.ViewFromVisibleIntervals(fh.f.entryViewCache, req.Offset, req.Size)

	totalRead, err := filer2.ReadIntoBuffer(ctx, fh.f.wfs, fh.f.fullpath(), buff, chunkViews, req.Offset)

	resp.Data = buff[:totalRead]

	return err
}

// Write to the file handle
func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error {

	// write the request to volume servers

	glog.V(4).Infof("%+v/%v write fh %d: [%d,%d)", fh.f.dir.Path, fh.f.Name, fh.handle, req.Offset, req.Offset+int64(len(req.Data)))

	chunks, err := fh.dirtyPages.AddPage(ctx, req.Offset, req.Data)
	if err != nil {
		glog.Errorf("%+v/%v write fh %d: [%d,%d): %v", fh.f.dir.Path, fh.f.Name, fh.handle, req.Offset, req.Offset+int64(len(req.Data)), err)
		return fmt.Errorf("write %s/%s at [%d,%d): %v", fh.f.dir.Path, fh.f.Name, req.Offset, req.Offset+int64(len(req.Data)), err)
	}

	resp.Size = len(req.Data)

	if req.Offset == 0 {
		// detect mime type
		var possibleExt string
		fh.contentType, possibleExt = mimetype.Detect(req.Data)
		if ext := path.Ext(fh.f.Name); ext != possibleExt {
			fh.contentType = mime.TypeByExtension(ext)
		}

		fh.dirtyMetadata = true
	}

	fh.f.addChunks(chunks)

	if len(chunks) > 0 {
		fh.dirtyMetadata = true
	}

	return nil
}

func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error {

	glog.V(4).Infof("%v release fh %d", fh.f.fullpath(), fh.handle)

	fh.dirtyPages.releaseResource()

	fh.f.wfs.ReleaseHandle(fh.f.fullpath(), fuse.HandleID(fh.handle))

	fh.f.isOpen = false

	return nil
}

func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
	// fflush works at fh level
	// send the data to the OS
	glog.V(4).Infof("%s fh %d flush %v", fh.f.fullpath(), fh.handle, req)

	chunk, err := fh.dirtyPages.FlushToStorage(ctx)
	if err != nil {
		glog.Errorf("flush %s/%s: %v", fh.f.dir.Path, fh.f.Name, err)
		return fmt.Errorf("flush %s/%s: %v", fh.f.dir.Path, fh.f.Name, err)
	}

	fh.f.addChunk(chunk)

	if !fh.dirtyMetadata {
		return nil
	}

	return fh.f.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {

		if fh.f.entry.Attributes != nil {
			fh.f.entry.Attributes.Mime = fh.contentType
			fh.f.entry.Attributes.Uid = req.Uid
			fh.f.entry.Attributes.Gid = req.Gid
			fh.f.entry.Attributes.Mtime = time.Now().Unix()
			fh.f.entry.Attributes.Crtime = time.Now().Unix()
			fh.f.entry.Attributes.FileMode = uint32(0770)
		}

		request := &filer_pb.CreateEntryRequest{
			Directory: fh.f.dir.Path,
			Entry:     fh.f.entry,
		}

		//glog.V(1).Infof("%s/%s set chunks: %v", fh.f.dir.Path, fh.f.Name, len(fh.f.entry.Chunks))
		//for i, chunk := range fh.f.entry.Chunks {
		//	glog.V(4).Infof("%s/%s chunks %d: %v [%d,%d)", fh.f.dir.Path, fh.f.Name, i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
		//}

		chunks, garbages := filer2.CompactFileChunks(fh.f.entry.Chunks)
		fh.f.entry.Chunks = chunks
		// fh.f.entryViewCache = nil
		fh.f.wfs.deleteFileChunks(ctx, garbages)

		if _, err := client.CreateEntry(ctx, request); err != nil {
			return fmt.Errorf("update fh: %v", err)
		}

		return nil
	})
}

<commit_msg>delete chunks only when file writing is successful<commit_after>package filesys

import (
	"context"
	"fmt"
	"mime"
	"path"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer2"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/gabriel-vasile/mimetype"
	"github.com/seaweedfs/fuse"
	"github.com/seaweedfs/fuse/fs"
)

type FileHandle struct {
	// cache file has been written to
	dirtyPages    *ContinuousDirtyPages
	contentType   string
	dirtyMetadata bool
	handle        uint64

	f         *File
	RequestId fuse.RequestID // unique ID for request
	NodeId    fuse.NodeID    // file or directory the request is about
	Uid       uint32         // user ID of process making request
	Gid       uint32         // group ID of process making request
}

func newFileHandle(file *File, uid, gid uint32) *FileHandle {
	return &FileHandle{
		f:          file,
		dirtyPages: newDirtyPages(file),
		Uid:        uid,
		Gid:        gid,
	}
}

var _ = fs.Handle(&FileHandle{})

// var _ = fs.HandleReadAller(&FileHandle{})
var _ = fs.HandleReader(&FileHandle{})
var _ = fs.HandleFlusher(&FileHandle{})
var _ = fs.HandleWriter(&FileHandle{})
var _ = fs.HandleReleaser(&FileHandle{})

func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {

	glog.V(4).Infof("%s read fh %d: [%d,%d)", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size))

	// this value should come from the filer instead of the old f
	if len(fh.f.entry.Chunks) == 0 {
		glog.V(1).Infof("empty fh %v/%v", fh.f.dir.Path, fh.f.Name)
		return nil
	}

	buff := make([]byte, req.Size)

	if fh.f.entryViewCache == nil {
		fh.f.entryViewCache = filer2.NonOverlappingVisibleIntervals(fh.f.entry.Chunks)
	}

	chunkViews := filer2.ViewFromVisibleIntervals(fh.f.entryViewCache, req.Offset, req.Size)

	totalRead, err := filer2.ReadIntoBuffer(ctx, fh.f.wfs, fh.f.fullpath(), buff, chunkViews, req.Offset)

	resp.Data = buff[:totalRead]

	return err
}

// Write to the file handle
func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error {

	// write the request to volume servers

	glog.V(4).Infof("%+v/%v write fh %d: [%d,%d)", fh.f.dir.Path, fh.f.Name, fh.handle, req.Offset, req.Offset+int64(len(req.Data)))

	chunks, err := fh.dirtyPages.AddPage(ctx, req.Offset, req.Data)
	if err != nil {
		glog.Errorf("%+v/%v write fh %d: [%d,%d): %v", fh.f.dir.Path, fh.f.Name, fh.handle, req.Offset, req.Offset+int64(len(req.Data)), err)
		return fmt.Errorf("write %s/%s at [%d,%d): %v", fh.f.dir.Path, fh.f.Name, req.Offset, req.Offset+int64(len(req.Data)), err)
	}

	resp.Size = len(req.Data)

	if req.Offset == 0 {
		// detect mime type
		var possibleExt string
		fh.contentType, possibleExt = mimetype.Detect(req.Data)
		if ext := path.Ext(fh.f.Name); ext != possibleExt {
			fh.contentType = mime.TypeByExtension(ext)
		}

		fh.dirtyMetadata = true
	}

	fh.f.addChunks(chunks)

	if len(chunks) > 0 {
		fh.dirtyMetadata = true
	}

	return nil
}

func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error {

	glog.V(4).Infof("%v release fh %d", fh.f.fullpath(), fh.handle)

	fh.dirtyPages.releaseResource()

	fh.f.wfs.ReleaseHandle(fh.f.fullpath(), fuse.HandleID(fh.handle))

	fh.f.isOpen = false

	return nil
}

func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
	// fflush works at fh level
	// send the data to the OS
	glog.V(4).Infof("%s fh %d flush %v", fh.f.fullpath(), fh.handle, req)

	chunk, err := fh.dirtyPages.FlushToStorage(ctx)
	if err != nil {
		glog.Errorf("flush %s/%s: %v", fh.f.dir.Path, fh.f.Name, err)
		return fmt.Errorf("flush %s/%s: %v", fh.f.dir.Path, fh.f.Name, err)
	}

	fh.f.addChunk(chunk)

	if !fh.dirtyMetadata {
		return nil
	}

	return fh.f.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {

		if fh.f.entry.Attributes != nil {
			fh.f.entry.Attributes.Mime = fh.contentType
			fh.f.entry.Attributes.Uid = req.Uid
			fh.f.entry.Attributes.Gid = req.Gid
			fh.f.entry.Attributes.Mtime = time.Now().Unix()
			fh.f.entry.Attributes.Crtime = time.Now().Unix()
			fh.f.entry.Attributes.FileMode = uint32(0770)
		}

		request := &filer_pb.CreateEntryRequest{
			Directory: fh.f.dir.Path,
			Entry:     fh.f.entry,
		}

		//glog.V(1).Infof("%s/%s set chunks: %v", fh.f.dir.Path, fh.f.Name, len(fh.f.entry.Chunks))
		//for i, chunk := range fh.f.entry.Chunks {
		//	glog.V(4).Infof("%s/%s chunks %d: %v [%d,%d)", fh.f.dir.Path, fh.f.Name, i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
		//}

		chunks, garbages := filer2.CompactFileChunks(fh.f.entry.Chunks)
		fh.f.entry.Chunks = chunks
		// fh.f.entryViewCache = nil

		if _, err := client.CreateEntry(ctx, request); err != nil {
			return fmt.Errorf("update fh: %v", err)
		}

		fh.f.wfs.deleteFileChunks(ctx, garbages)

		return nil
	})
}

<|endoftext|>


<commit_before>package main

// This is a simple quick script to take a goship config file and put into ETCD. Note: It does not wipe out your
// existing etcd setup.

import (
	"flag"
	"fmt"
	"github.com/coreos/go-etcd/etcd"
	"github.com/kylelemons/go-gypsy/yaml"
	"log"
	"strings"
)

var (
	ConfigFile = flag.String("c", "config.yml", "Path to data directory (default config.yml)")
	ETCDServer = flag.String("e", "http://127.0.0.1:4001", "Etcd Server (default http://127.0.0.1:4001")
)

// Host stores information on a host, such as URI and the latest commit revision.
type Host struct {
	URI             string
	LatestCommit    string
	GitHubCommitURL string
	GitHubDiffURL   string
	ShortCommitHash string
}

// Environment stores information about an individual environment, such as its name and whether it is deployable.
type Environment struct {
	Name               string
	Deploy             string
	RepoPath           string
	Hosts              []Host
	Branch             string
	LatestGitHubCommit string
	IsDeployable       bool
}

// Project stores information about a GitHub project, such as its GitHub URL and repo name.
type Project struct {
	Name         string
	GitHubURL    string
	RepoName     string
	RepoOwner    string
	Environments []Environment
}

type PivotalConfiguration struct {
	project string
	token   string
}

// config contains the information from config.yml.
type config struct {
	Projects   []Project
	DeployUser string
	Notify     string
	Pivotal    *PivotalConfiguration
}

// getYAMLString is a helper function for extracting strings from a yaml.Node.
func getYAMLString(n yaml.Node, key string) string {
	s, ok := n.(yaml.Map)[key].(yaml.Scalar)
	if !ok {
		return ""
	}
	return strings.TrimSpace(s.String())
}

// Send etcd Data and output.
func setETCD(client *etcd.Client, full_key, value string) {
	log.Printf("Setting %s => %s \n", full_key, value)
	client.Create(full_key, value, 0)
}

// parseYAMLEnvironment populates an Environment given a yaml.Node and returns the Environment.
func YAMLtoETCDEnvironment(m yaml.Node, client *etcd.Client, projPath string) {

	for k, v := range m.(yaml.Map) {
		projPath = projPath + "environments/" + k + "/"

		log.Printf("Setting env name=> %s \n", projPath)
		client.CreateDir(projPath, 0)

		branch := getYAMLString(v, "branch")
		setETCD(client, projPath+"branch", branch)

		repoPath := getYAMLString(v, "repo_path")
		setETCD(client, projPath+"repo_path", repoPath)

		deploy := getYAMLString(v, "deploy")
		setETCD(client, projPath+"deploy", deploy)

		log.Printf("Creating Host Directory => %s \n", projPath+"hosts/")
		client.CreateDir(projPath+"hosts/", 0)

		for _, host := range v.(yaml.Map)["hosts"].(yaml.List) {
			h := Host{URI: host.(yaml.Scalar).String()}
			projPath = projPath + "/hosts"
			log.Printf("Setting Hosts => %s \n", projPath+h.URI)
			client.CreateDir(projPath+h.URI, 0)
		}
	}
}

// parseYAML parses the config.yml file and returns the appropriate structs and strings.
func YAMLtoETCD(client *etcd.Client) (c config, err error) {
	config, err := yaml.ReadFile(*ConfigFile)
	if err != nil {
		return c, err
	}
	log.Printf("Setting project root => /projects")
	client.CreateDir("/projects", 0)
	configRoot, _ := config.Root.(yaml.Map)
	projects, _ := configRoot["projects"].(yaml.List)
	for _, p := range projects {
		for k, v := range p.(yaml.Map) {

			log.Printf("Setting project => %s \n", k)
			client.CreateDir(k, 0)

			projectPath := "/projects/" + k + "/"

			name := getYAMLString(v, "project_name")
			setETCD(client, projectPath+"project_name", name)

			repoOwner := getYAMLString(v, "repo_owner")
			setETCD(client, projectPath+"repo_owner", repoOwner)

			repoName := getYAMLString(v, "repo_name")
			setETCD(client, projectPath+"repo_name", repoName)

			for _, v := range v.(yaml.Map)["environments"].(yaml.List) {
				YAMLtoETCDEnvironment(v, client, projectPath)
			}

		}
	}

	piv_project, _ := config.Get("pivotal_project")
	setETCD(client, "pivotal_project", piv_project)

	piv_token, _ := config.Get("pivotal_token")
	setETCD(client, "pivotal_token", piv_token)

	notify, _ := config.Get("notify")
	setETCD(client, "notify", notify)

	return c, err
}

func main() {
	flag.Parse()
	log.Printf("Reading Config file: %s Connecting to ETCD server: %s", *ConfigFile, *ETCDServer)
	// Note the ETCD client library swallows errors connecting to etcd (worry)
	a := etcd.NewClient([]string{*ETCDServer})
	_, err := YAMLtoETCD(a)
	if err != nil {
		fmt.Printf("Failed to Parse Yaml and Add to ETCD [%s]\n", err)
	}
}
<commit_msg>Tweaking path.<commit_after>package main

// This is a simple quick script to take a goship config file and put into ETCD. Note: It does not wipe out your
// existing etcd setup.

import (
	"flag"
	"fmt"
	"github.com/coreos/go-etcd/etcd"
	"github.com/kylelemons/go-gypsy/yaml"
	"log"
	"strings"
)

var (
	ConfigFile = flag.String("c", "config.yml", "Path to data directory (default config.yml)")
	ETCDServer = flag.String("e", "http://127.0.0.1:4001", "Etcd Server (default http://127.0.0.1:4001")
)

// Host stores information on a host, such as URI and the latest commit revision.
type Host struct {
	URI             string
	LatestCommit    string
	GitHubCommitURL string
	GitHubDiffURL   string
	ShortCommitHash string
}

// Environment stores information about an individual environment, such as its name and whether it is deployable.
type Environment struct {
	Name               string
	Deploy             string
	RepoPath           string
	Hosts              []Host
	Branch             string
	LatestGitHubCommit string
	IsDeployable       bool
}

// Project stores information about a GitHub project, such as its GitHub URL and repo name.
type Project struct {
	Name         string
	GitHubURL    string
	RepoName     string
	RepoOwner    string
	Environments []Environment
}

type PivotalConfiguration struct {
	project string
	token   string
}

// config contains the information from config.yml.
type config struct {
	Projects   []Project
	DeployUser string
	Notify     string
	Pivotal    *PivotalConfiguration
}

// getYAMLString is a helper function for extracting strings from a yaml.Node.
func getYAMLString(n yaml.Node, key string) string {
	s, ok := n.(yaml.Map)[key].(yaml.Scalar)
	if !ok {
		return ""
	}
	return strings.TrimSpace(s.String())
}

// Send etcd Data and output.
func setETCD(client *etcd.Client, full_key, value string) {
	log.Printf("Setting %s => %s \n", full_key, value)
	client.Create(full_key, value, 0)
}

// parseYAMLEnvironment populates an Environment given a yaml.Node and returns the Environment.
func YAMLtoETCDEnvironment(m yaml.Node, client *etcd.Client, projPath string) {

	for k, v := range m.(yaml.Map) {
		projPath = projPath + "environments/" + k + "/"

		log.Printf("Setting env name=> %s \n", projPath)
		client.CreateDir(projPath, 0)

		branch := getYAMLString(v, "branch")
		setETCD(client, projPath+"branch", branch)

		repoPath := getYAMLString(v, "repo_path")
		setETCD(client, projPath+"repo_path", repoPath)

		deploy := getYAMLString(v, "deploy")
		setETCD(client, projPath+"deploy", deploy)

		projPath = projPath + "hosts/"
		log.Printf("Creating Host Directory => %s \n", projPath+"hosts/")
		client.CreateDir(projPath, 0)

		for _, host := range v.(yaml.Map)["hosts"].(yaml.List) {
			h := Host{URI: host.(yaml.Scalar).String()}
			log.Printf("Setting Hosts => %s \n", projPath+h.URI)
			client.CreateDir(projPath+h.URI, 0)
		}
	}
}

// parseYAML parses the config.yml file and returns the appropriate structs and strings.
func YAMLtoETCD(client *etcd.Client) (c config, err error) {
	config, err := yaml.ReadFile(*ConfigFile)
	if err != nil {
		return c, err
	}
	log.Printf("Setting project root => /projects")
	client.CreateDir("/projects", 0)
	configRoot, _ := config.Root.(yaml.Map)
	projects, _ := configRoot["projects"].(yaml.List)
	for _, p := range projects {
		for k, v := range p.(yaml.Map) {

			log.Printf("Setting project => %s \n", k)
			client.CreateDir(k, 0)

			projectPath := "/projects/" + k + "/"

			name := getYAMLString(v, "project_name")
			setETCD(client, projectPath+"project_name", name)

			repoOwner := getYAMLString(v, "repo_owner")
			setETCD(client, projectPath+"repo_owner", repoOwner)

			repoName := getYAMLString(v, "repo_name")
			setETCD(client, projectPath+"repo_name", repoName)

			for _, v := range v.(yaml.Map)["environments"].(yaml.List) {
				YAMLtoETCDEnvironment(v, client, projectPath)
			}

		}
	}

	piv_project, _ := config.Get("pivotal_project")
	setETCD(client, "pivotal_project", piv_project)

	piv_token, _ := config.Get("pivotal_token")
	setETCD(client, "pivotal_token", piv_token)

	notify, _ := config.Get("notify")
	setETCD(client, "notify", notify)

	return c, err
}

func main() {
	flag.Parse()
	log.Printf("Reading Config file: %s Connecting to ETCD server: %s", *ConfigFile, *ETCDServer)
	// Note the ETCD client library swallows errors connecting to etcd (worry)
	a := etcd.NewClient([]string{*ETCDServer})
	_, err := YAMLtoETCD(a)
	if err != nil {
		fmt.Printf("Failed to Parse Yaml and Add to ETCD [%s]\n", err)
	}
}
<|endoftext|>


<commit_before>package whirlytubes

<commit_msg>first draft of code<commit_after>package whirlytubes

import (
	"net"
	"regexp"
)

type Address interface {
	Send(string) error
	Receive() (string, error)
	Verify() (bool, error)
}

type TcpAddress net.Conn

func (addr TcpAddress) Verify() (b bool, err error) {
	b, err = regexp.MatchString(".*:.*", addr) //fix connection for address, store connection in address
	return
}

func (addr TcpAddress) Send(msg string) (err error) {
}
<|endoftext|>


<commit_before>package core

import (
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"strings"
	"sync"

	"github.com/tinzenite/channel"
	"github.com/tinzenite/model"
	"github.com/tinzenite/shared"
)

/*
Tinzenite is the struct on which all important operations should be called.
*/
type Tinzenite struct {
	Path           string
	auth           *Authentication
	selfpeer       *shared.Peer
	channel        *channel.Channel
	cInterface     *chaninterface
	allPeers       []*shared.Peer
	model          *model.Model
	sendChannel    chan shared.UpdateMessage
	stop           chan bool
	wg             sync.WaitGroup
	peerValidation PeerValidation
}

/*
Sync the entire file system model of the directory, first locally and then
remotely if other peers are connected. NOTE: All Sync{|Local|Remote} methods can
block for a potentially long time, especially when first run!
*/
func (t *Tinzenite) Sync() error {
	// first ensure that local model is up to date
	err := t.SyncLocal()
	if err != nil {
		return err
	}
	return t.SyncRemote()
}

/*
SyncLocal changes. Will send updates to connected peers but not synchronize with
other peers.
*/
func (t *Tinzenite) SyncLocal() error {
	// update peers
	err := t.applyPeers()
	if err != nil {
		return err
	}
	// update model
	return t.model.Update()
}

/*
SyncRemote changes. Will request the models of connected peers and merge them
to the local peer.

TODO: fetches model from other peers and syncs (this is for manual sync)
*/
func (t *Tinzenite) SyncRemote() error {
	// iterate over all known peers
	//the following can be parallelized!
	msg := shared.CreateRequestMessage(shared.ReModel, "TODO")
	t.sendAll(msg.String())
	/*TODO implement model detection on receive? Not here, I guess?*/
	return nil
}

/*
Address of this Tinzenite peer that can be used to connect to.
*/
func (t *Tinzenite) Address() (string, error) {
	return t.channel.ConnectionAddress()
}

/*
Close cleanly stores everything and shuts Tinzenite down.
*/
func (t *Tinzenite) Close() {
	// send stop signal
	t.stop <- false
	// wait for it to close
	t.wg.Wait()
	// store all information
	t.Store()
	// FINALLY close (afterwards because I still need info from channel for store!)
	t.channel.Close()
}

/*
Store the tinzenite directory structure to disk. Will resolve all important
objects and store them so that it can later be reloaded. NOTE: Will not update
the full model, so be sure to have called Update() to guarantee an up to date
save.
*/
func (t *Tinzenite) Store() error {
	err := t.makeDotTinzenite()
	if err != nil {
		return err
	}
	// write all peers to files
	for _, peer := range t.allPeers {
		err := peer.Store(t.Path)
		if err != nil {
			return err
		}
	}
	// store local peer info with toxdata
	toxData, err := t.channel.ToxData()
	if err != nil {
		return err
	}
	toxPeerDump := &shared.ToxPeerDump{
		SelfPeer: t.selfpeer,
		ToxData:  toxData}
	err = toxPeerDump.Store(t.Path)
	if err != nil {
		return err
	}
	// store auth file
	err = t.auth.Store(t.Path)
	if err != nil {
		return err
	}
	// store bootstrap
	err = t.cInterface.Store(t.Path)
	if err != nil {
		return err
	}
	// update model for tinzenite dir to catch above stores
	err = t.model.PartialUpdate(t.Path + "/" + shared.TINZENITEDIR)
	if err != nil {
		return err
	}
	// finally store model (last because peers and org stuff is included!)
	return t.model.Store()
}

/*
Connect this tinzenite to another peer, beginning the bootstrap process.
*/
func (t *Tinzenite) Connect(address string) error {
	return t.cInterface.Connect(address)
}

/*
applyPeers loads all peers and readies the communication channel accordingly.
*/
func (t *Tinzenite) applyPeers() error {
	peers, err := shared.LoadPeers(t.Path)
	if err != nil {
		return err
	}
	// make sure they are all tox ready
	for _, peer := range peers {
		// tox will return an error if the address has already been added, so we just ignore it
		_ = t.channel.AcceptConnection(peer.Address)
	}
	// finally apply
	t.allPeers = peers
	return nil
}

/*
Send the given message string to all online and connected peers.
*/
func (t *Tinzenite) sendAll(msg string) {
	for _, peer := range t.allPeers {
		if strings.EqualFold(peer.Address, t.selfpeer.Address) {
			continue
		}
		online, err := t.channel.IsOnline(peer.Address)
		if err != nil {
			log.Println(err)
			continue
		}
		if online {
			/*TODO - also make this concurrent?*/
			err := t.channel.Send(peer.Address, msg)
			if err != nil {
				log.Println(err.Error(), peer.Address)
			}
		}
		// if online -> continue
		// if not init -> init
		// sync
		/*TODO must store init state in allPeers too, also in future encryption*/
	}
}

/*
Send the given message to the peer with the address if online.
*/
func (t *Tinzenite) send(address, msg string) error {
	return t.channel.Send(address, msg)
}

/*
Merge an update message to the local model.

TODO: with move implemented one of the merged file copies can be kept and simply
renamed, also solving the ID problem. Look into this!
*/
func (t *Tinzenite) merge(msg *shared.UpdateMessage) error {
	relPath := shared.CreatePath(t.Path, msg.Object.Path)
	// first: apply local changes to model (this is why writing PartialUpdate was no waste of time, isn't this cool?! :D)
	err := t.model.PartialUpdate(relPath.FullPath())
	if err != nil {
		return err
	}
	// check if content is same, no need for merge then (except for version)
	stin, err := t.model.GetInfo(relPath)
	if err != nil {
		log.Println("Core:", "Can not check if content is same!")
	} else {
		if stin.Content == msg.Object.Content {
			log.Println("Core:", "Merge not required as updates are in sync!")
			// so all we need to do is apply the version update
			/*TODO: we need applymodify WITHOUT the fetching of the file...*/
			t.model.ApplyModify(relPath, &msg.Object)
			return nil
		}
	}
	// second: move to new name
	err = os.Rename(relPath.FullPath(), relPath.FullPath()+LOCAL)
	if err != nil {
		log.Println("Original can not be found!")
		return err
	}
	err = t.model.PartialUpdate(relPath.FullPath() + LOCAL)
	if err != nil {
		return err
	}
	// third: remove original
	err = t.model.ApplyRemove(relPath, nil)
	if err != nil {
		return err
	}
	// fourth: change path and apply remote as create
	msg.Operation = shared.OpCreate
	msg.Object.Path = relPath.SubPath() + REMOTE
	msg.Object.Name = relPath.LastElement() + REMOTE
	/*TODO what of the id? For now to be sure: new one.*/
	oldID := msg.Object.Identification
	msg.Object.Identification, err = shared.NewIdentifier()
	if err != nil {
		return err
	}
	// new id --> rename temp file
	tempPath := t.Path + "/" + shared.TINZENITEDIR + "/" + shared.TEMPDIR
	err = os.Rename(tempPath+"/"+oldID, tempPath+"/"+msg.Object.Identification)
	if err != nil {
		log.Println("Updating remote object file failed!")
		return err
	}
	// fifth: create remote file
	err = t.model.ApplyCreate(relPath.Apply(relPath.FullPath()+REMOTE), &msg.Object)
	if err != nil {
		return err
	}
	return nil
}

/*
storeGlobalConfig stores the path value into the user's home directory so that clients
can locate it.
*/
func (t *Tinzenite) storeGlobalConfig() error {
	// ready outside data
	user, err := user.Current()
	if err != nil {
		return err
	}
	path := user.HomeDir + "/.config/tinzenite"
	err = shared.MakeDirectory(path)
	if err != nil {
		return err
	}
	path += "/" + shared.DIRECTORYLIST
	file, err := os.OpenFile(path, shared.FILEFLAGCREATEAPPEND, shared.FILEPERMISSIONMODE)
	if err != nil {
		return err
	}
	defer file.Close()
	// write path to file
	_, err = file.WriteString(t.Path + "\n")
	if err != nil {
		return err
	}
	// ensure that the file is valid
	return shared.PrettifyDirectoryList()
}

/*
makeDotTinzenite creates the directory structure for the .tinzenite directory
including the .tinignore file required for it.
*/
func (t *Tinzenite) makeDotTinzenite() error {
	root := t.Path + "/" + shared.TINZENITEDIR
	// build directory structure
	err := shared.MakeDirectories(root, shared.ORGDIR+"/"+shared.PEERSDIR, shared.TEMPDIR, shared.REMOVEDIR, shared.LOCALDIR, shared.RECEIVINGDIR)
	if err != nil {
		return err
	}
	// write required .tinignore file
	return ioutil.WriteFile(root+"/"+shared.TINIGNORE, []byte(TINDIRIGNORE), shared.FILEPERMISSIONMODE)
}

/*
initialize the background process.
*/
func (t *Tinzenite) initialize() {
	// prepare send channel that will distribute updates
	t.wg.Add(1)
	t.stop = make(chan bool, 1)
	t.sendChannel = make(chan shared.UpdateMessage, 1)
	go t.background()
	t.model.Register(t.sendChannel)
}

/*
background function that handles all async stuff that needs to be done.
*/
func (t *Tinzenite) background() {
	for {
		select {
		case <-t.stop:
			t.wg.Done()
			return
		case msg := <-t.sendChannel:
			t.sendAll(msg.String())
		} // select
	} // for
}
<commit_msg>better sendAll method<commit_after>package core

import (
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"sync"

	"github.com/tinzenite/channel"
	"github.com/tinzenite/model"
	"github.com/tinzenite/shared"
)

/*
Tinzenite is the struct on which all important operations should be called.
*/
type Tinzenite struct {
	Path           string
	auth           *Authentication
	selfpeer       *shared.Peer
	channel        *channel.Channel
	cInterface     *chaninterface
	allPeers       []*shared.Peer
	model          *model.Model
	sendChannel    chan shared.UpdateMessage
	stop           chan bool
	wg             sync.WaitGroup
	peerValidation PeerValidation
}

/*
Sync the entire file system model of the directory, first locally and then
remotely if other peers are connected. NOTE: All Sync{|Local|Remote} methods can
block for a potentially long time, especially when first run!
*/
func (t *Tinzenite) Sync() error {
	// first ensure that local model is up to date
	err := t.SyncLocal()
	if err != nil {
		return err
	}
	return t.SyncRemote()
}

/*
SyncLocal changes. Will send updates to connected peers but not synchronize with
other peers.
*/
func (t *Tinzenite) SyncLocal() error {
	// update peers
	err := t.applyPeers()
	if err != nil {
		return err
	}
	// update model
	return t.model.Update()
}

/*
SyncRemote changes. Will request the models of connected peers and merge them
to the local peer.

TODO: fetches model from other peers and syncs (this is for manual sync)
*/
func (t *Tinzenite) SyncRemote() error {
	// iterate over all known peers
	//the following can be parallelized!
	msg := shared.CreateRequestMessage(shared.ReModel, "TODO")
	t.sendAll(msg.String())
	/*TODO implement model detection on receive? Not here, I guess?*/
	return nil
}

/*
Address of this Tinzenite peer that can be used to connect to.
*/
func (t *Tinzenite) Address() (string, error) {
	return t.channel.ConnectionAddress()
}

/*
Close cleanly stores everything and shuts Tinzenite down.
*/
func (t *Tinzenite) Close() {
	// send stop signal
	t.stop <- false
	// wait for it to close
	t.wg.Wait()
	// store all information
	t.Store()
	// FINALLY close (afterwards because I still need info from channel for store!)
	t.channel.Close()
}

/*
Store the tinzenite directory structure to disk. Will resolve all important
objects and store them so that it can later be reloaded. NOTE: Will not update
the full model, so be sure to have called Update() to guarantee an up to date
save.
*/
func (t *Tinzenite) Store() error {
	err := t.makeDotTinzenite()
	if err != nil {
		return err
	}
	// write all peers to files
	for _, peer := range t.allPeers {
		err := peer.Store(t.Path)
		if err != nil {
			return err
		}
	}
	// store local peer info with toxdata
	toxData, err := t.channel.ToxData()
	if err != nil {
		return err
	}
	toxPeerDump := &shared.ToxPeerDump{
		SelfPeer: t.selfpeer,
		ToxData:  toxData}
	err = toxPeerDump.Store(t.Path)
	if err != nil {
		return err
	}
	// store auth file
	err = t.auth.Store(t.Path)
	if err != nil {
		return err
	}
	// store bootstrap
	err = t.cInterface.Store(t.Path)
	if err != nil {
		return err
	}
	// update model for tinzenite dir to catch above stores
	err = t.model.PartialUpdate(t.Path + "/" + shared.TINZENITEDIR)
	if err != nil {
		return err
	}
	// finally store model (last because peers and org stuff is included!)
	return t.model.Store()
}

/*
Connect this tinzenite to another peer, beginning the bootstrap process.
*/
func (t *Tinzenite) Connect(address string) error {
	return t.cInterface.Connect(address)
}

/*
applyPeers loads all peers and readies the communication channel accordingly.
*/
func (t *Tinzenite) applyPeers() error {
	peers, err := shared.LoadPeers(t.Path)
	if err != nil {
		return err
	}
	// make sure they are all tox ready
	for _, peer := range peers {
		// tox will return an error if the address has already been added, so we just ignore it
		_ = t.channel.AcceptConnection(peer.Address)
	}
	// finally apply
	t.allPeers = peers
	return nil
}

/*
Send the given message string to all online and connected peers.
*/
func (t *Tinzenite) sendAll(msg string) {
	online, err := t.channel.OnlineAddresses()
	if err != nil {
		log.Println("sendAll:", err)
		return
	}
	for _, address := range online {
		/*TODO - also make this concurrent?*/
		err := t.channel.Send(address, msg)
		if err != nil {
			log.Println(err.Error(), address)
		}
		// if online -> continue
		// if not init -> init
		// sync
		/*TODO must store init state in allPeers too, also in future encryption*/
	}
}

/*
Send the given message to the peer with the address if online.
*/
func (t *Tinzenite) send(address, msg string) error {
	return t.channel.Send(address, msg)
}

/*
Merge an update message to the local model.

TODO: with move implemented one of the merged file copies can be kept and simply
renamed, also solving the ID problem. Look into this!
*/
func (t *Tinzenite) merge(msg *shared.UpdateMessage) error {
	relPath := shared.CreatePath(t.Path, msg.Object.Path)
	// first: apply local changes to model (this is why writing PartialUpdate was no waste of time, isn't this cool?! :D)
	err := t.model.PartialUpdate(relPath.FullPath())
	if err != nil {
		return err
	}
	// check if content is same, no need for merge then (except for version)
	stin, err := t.model.GetInfo(relPath)
	if err != nil {
		log.Println("Core:", "Can not check if content is same!")
	} else {
		if stin.Content == msg.Object.Content {
			log.Println("Core:", "Merge not required as updates are in sync!")
			// so all we need to do is apply the version update
			/*TODO: we need applymodify WITHOUT the fetching of the file...*/
			t.model.ApplyModify(relPath, &msg.Object)
			return nil
		}
	}
	// second: move to new name
	err = os.Rename(relPath.FullPath(), relPath.FullPath()+LOCAL)
	if err != nil {
		log.Println("Original can not be found!")
		return err
	}
	err = t.model.PartialUpdate(relPath.FullPath() + LOCAL)
	if err != nil {
		return err
	}
	// third: remove original
	err = t.model.ApplyRemove(relPath, nil)
	if err != nil {
		return err
	}
	// fourth: change path and apply remote as create
	msg.Operation = shared.OpCreate
	msg.Object.Path = relPath.SubPath() + REMOTE
	msg.Object.Name = relPath.LastElement() + REMOTE
	/*TODO what of the id? For now to be sure: new one.*/
	oldID := msg.Object.Identification
	msg.Object.Identification, err = shared.NewIdentifier()
	if err != nil {
		return err
	}
	// new id --> rename temp file
	tempPath := t.Path + "/" + shared.TINZENITEDIR + "/" + shared.TEMPDIR
	err = os.Rename(tempPath+"/"+oldID, tempPath+"/"+msg.Object.Identification)
	if err != nil {
		log.Println("Updating remote object file failed!")
		return err
	}
	// fifth: create remote file
	err = t.model.ApplyCreate(relPath.Apply(relPath.FullPath()+REMOTE), &msg.Object)
	if err != nil {
		return err
	}
	return nil
}

/*
storeGlobalConfig stores the path value into the user's home directory so that clients
can locate it.
*/
func (t *Tinzenite) storeGlobalConfig() error {
	// ready outside data
	user, err := user.Current()
	if err != nil {
		return err
	}
	path := user.HomeDir + "/.config/tinzenite"
	err = shared.MakeDirectory(path)
	if err != nil {
		return err
	}
	path += "/" + shared.DIRECTORYLIST
	file, err := os.OpenFile(path, shared.FILEFLAGCREATEAPPEND, shared.FILEPERMISSIONMODE)
	if err != nil {
		return err
	}
	defer file.Close()
	// write path to file
	_, err = file.WriteString(t.Path + "\n")
	if err != nil {
		return err
	}
	// ensure that the file is valid
	return shared.PrettifyDirectoryList()
}

/*
makeDotTinzenite creates the directory structure for the .tinzenite directory
including the .tinignore file required for it.
*/
func (t *Tinzenite) makeDotTinzenite() error {
	root := t.Path + "/" + shared.TINZENITEDIR
	// build directory structure
	err := shared.MakeDirectories(root, shared.ORGDIR+"/"+shared.PEERSDIR, shared.TEMPDIR, shared.REMOVEDIR, shared.LOCALDIR, shared.RECEIVINGDIR)
	if err != nil {
		return err
	}
	// write required .tinignore file
	return ioutil.WriteFile(root+"/"+shared.TINIGNORE, []byte(TINDIRIGNORE), shared.FILEPERMISSIONMODE)
}

/*
initialize the background process.
*/
func (t *Tinzenite) initialize() {
	// prepare send channel that will distribute updates
	t.wg.Add(1)
	t.stop = make(chan bool, 1)
	t.sendChannel = make(chan shared.UpdateMessage, 1)
	go t.background()
	t.model.Register(t.sendChannel)
}

/*
background function that handles all async stuff that needs to be done.
*/
func (t *Tinzenite) background() {
	for {
		select {
		case <-t.stop:
			t.wg.Done()
			return
		case msg := <-t.sendChannel:
			t.sendAll(msg.String())
		} // select
	} // for
}
<|endoftext|>


<commit_before>package tmsh

import (
	"fmt"
	"reflect"
	"testing"
)

type TestSSHConn struct {
	validCmd bool
}

func (conn *TestSSHConn) Send(cmd string) (int, error) {
	conn.validCmd = (cmd == "show sys clock")
	return 0, nil
}

func (conn *TestSSHConn) Recv(suffix string) ([]byte, error) {
	var ret []byte
	var err error

	if conn.validCmd {
		ret = []byte("Last login: Mon Jul  1 10:20:34 2017 from 192.0.2.10\nadmin.prd@(LB000)(cfg-sync In Sync)(/S1-green-P:Active)(/admin.prd)(tmos)#\nshow sys clock\n------------------------\nSys::Clock\n------------------------\nMon Jul 03 14:25:02 JST 2017\n\nn.prd)(tmos)# ve)(/admi\n")
		err = nil
	} else {
		ret = nil
		err = fmt.Errorf("Syntax Error: unexpected argument \"foo\"")
	}

	return ret, err
}

func (conn *TestSSHConn) Close() error {
	return nil
}

func TestExecuteCommand(t *testing.T) {
	bigip := &BigIP{
		host:    "example.com",
		user:    "admin.prd",
		sshconn: &TestSSHConn{},
	}

	expect := "------------------------\nSys::Clock\n------------------------\nMon Jul 03 14:25:02 JST 2017\n"
	actual, err := bigip.ExecuteCommand("show sys clock")
	if err != nil {
		t.Errorf("%v", err)
	}

	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("got %v\nwant %v", actual, expect)
	}

	expectErr := fmt.Errorf("Syntax Error: unexpected argument \"foo\"")
	_, actualErr := bigip.ExecuteCommand("show foo")

	if !reflect.DeepEqual(actualErr, expectErr) {
		t.Errorf("got %v\nwant %v", actualErr, expectErr)
	}
}
<commit_msg>Modify tmsh test<commit_after>package tmsh

import (
	"fmt"
	"reflect"
	"testing"
)

var validCmds = []string{
	"show sys clock",
}

type TestSSHConnection struct {
	validCmd bool
	ret      []byte
}

func (c *TestSSHConnection) Send(cmd string) (int, error) {
	for _, vc := range validCmds {
		if vc == cmd {
			c.validCmd = true
			return 0, nil
		}
	}
	c.validCmd = false
	return 1, nil
}

func (c *TestSSHConnection) Recv(suffix string) ([]byte, error) {
	if !c.validCmd {
		return nil, fmt.Errorf("Syntax Error: unexpected argument \"foo\"")
	}
	return c.ret, nil
}

func (c *TestSSHConnection) Close() error {
	return nil
}

func TestExecuteCommand(t *testing.T) {
	bigip := &BigIP{
		host: "example.com",
		user: "admin.prd",
		sshconn: &TestSSHConnection{
			ret: []byte("Last login: Mon Jul  1 10:20:34 2017 from 192.0.2.10\nadmin.prd@(LB000)(cfg-sync In Sync)(/S1-green-P:Active)(/admin.prd)(tmos)#\nshow sys clock\n------------------------\nSys::Clock\n------------------------\nMon Jul 03 14:25:02 JST 2017\n\nn.prd)(tmos)# ve)(/admi\n"),
		},
	}

	expect := "------------------------\nSys::Clock\n------------------------\nMon Jul 03 14:25:02 JST 2017\n"
	actual, err := bigip.ExecuteCommand("show sys clock")
	if err != nil {
		t.Errorf("%v", err)
	}

	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("got %v\nwant %v", actual, expect)
	}

	expectErr := fmt.Errorf("Syntax Error: unexpected argument \"foo\"")
	ret, actualErr := bigip.ExecuteCommand("show foo")

	if ret != "" {
		t.Errorf("got %v\nwant %v", ret, "")
	}

	if !reflect.DeepEqual(actualErr, expectErr) {
		t.Errorf("got %v\nwant %v", actualErr, expectErr)
	}
}
<|endoftext|>


<commit_before>package logger

import (
    "fmt"
    "log"
    "os"
)

type Flags int

const (
    USUAL Flags = log.Ldate | log.Ltime | log.Lshortfile
)

type Level int

const (
    DEBUG Level = iota
    INFO
    WARN
    ERROR
    FATAL
)

type LevelLogger struct {
    logger log.Logger
    level Level
}

var logs LevelLogger

func init() {
    logs.level = DEBUG
}

func SetupLogger(lev Level, flags Flags) {
    logs.logger.SetFlags(int(flags))
    logs.level = lev
}

func logIfLevelf(level Level, format string, v ...interface{}) {
    if (int(level) >= int(logs.level)) {
        log.Printf(format, v)
    }
}

func Debugf(format string, v ...interface{}) {
    formatted := fmt.Sprintf(format, v)
    logIfLevelf(DEBUG, "[DEBUG] %v", formatted)
}

func Debug(v ...interface{}) {
    Debugf("%v", v)
}

func Infof(format string, v ...interface{}) {
    formatted := fmt.Sprintf(format, v)
    logIfLevelf(INFO, "[INFO] %v", formatted)
}

func Info(v ...interface{}) {
    Infof("%v", v)
}

func Warnf(format string, v ...interface{}) {
    formatted := fmt.Sprintf(format, v)
    logIfLevelf(WARN, "[WARN] %v", formatted)
}

func Warn(v ...interface{}) {
    Warnf("%v", v)
}

func Errorf(format string, v ...interface{}) {
    formatted := fmt.Sprintf(format, v)
    logIfLevelf(ERROR, "[ERROR] %v", formatted)
}

func Error(v ...interface{}) {
    Errorf("%v", v)
}

func Fatalf(format string, v ...interface{}) {
    formatted := fmt.Sprintf(format, v)
    logIfLevelf(FATAL, "[FATAL] %v", formatted)
    os.Exit(1)
}

func Fatal(v ...interface{}) {
    Fatalf("%v", v)
}
<commit_msg>Fix logger so output is properly formatted<commit_after>package logger

import (
    "fmt"
    "log"
    "os"
)

type Flags int

const (
    USUAL Flags = log.Ldate | log.Ltime | log.Lshortfile
)

type Level int

const (
    DEBUG Level = iota
    INFO
    WARN
    ERROR
    FATAL
)

type LevelLogger struct {
    logger log.Logger
    level Level
}

var logs LevelLogger

func init() {
    logs.level = DEBUG
}

func SetupLogger(lev Level, flags Flags) {
    logs.logger.SetFlags(int(flags))
    logs.level = lev
}

func logIfLevelf(level Level, format string, v ...interface{}) {
    if (int(level) >= int(logs.level)) {
        log.Printf(format, v...)
    }
}

func Debugf(format string, v ...interface{}) {
    formatted := fmt.Sprintf(format, v...)
    logIfLevelf(DEBUG, "[DEBUG] %v", formatted)
}

func Debug(v ...interface{}) {
    Debugf("%v", v...)
}

func Infof(format string, v ...interface{}) {
    formatted := fmt.Sprintf(format, v...)
    logIfLevelf(INFO, "[INFO] %v", formatted)
}

func Info(v ...interface{}) {
    Infof("%v", v...)
}

func Warnf(format string, v ...interface{}) {
    formatted := fmt.Sprintf(format, v...)
    logIfLevelf(WARN, "[WARN] %v", formatted)
}

func Warn(v ...interface{}) {
    Warnf("%v", v...)
}

func Errorf(format string, v ...interface{}) {
    formatted := fmt.Sprintf(format, v...)
    logIfLevelf(ERROR, "[ERROR] %v", formatted)
}

func Error(v ...interface{}) {
    Errorf("%v", v...)
}

func Fatalf(format string, v ...interface{}) {
    formatted := fmt.Sprintf(format, v...)
    logIfLevelf(FATAL, "[FATAL] %v", formatted)
    os.Exit(1)
}

func Fatal(v ...interface{}) {
    Fatalf("%v", v...)
}
<|endoftext|>


<commit_before>// Package gzip_file_server provides a http.Handler that serves the given virtual file system with gzip compression,
// without special handling of index.html, and detailed HTTP error messages.
package gzip_file_server

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"html"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
	"sort"
	"strings"
	"time"
)

type gzipFileServer struct {
	root http.FileSystem
}

// New returns a gzip file server, that serves the given virtual file system without special handling of index.html.
func New(root http.FileSystem) http.Handler {
	return &gzipFileServer{root: root}
}

func (f *gzipFileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if !strings.HasPrefix(r.URL.Path, "/") {
		r.URL.Path = "/" + r.URL.Path
	}
	f.serveFile(w, r, path.Clean(r.URL.Path))
}

// name is '/'-separated, not filepath.Separator.
func (fs *gzipFileServer) serveFile(w http.ResponseWriter, req *http.Request, name string) {
	f, err := fs.root.Open(name)
	if err != nil {
		msg, code := toHTTPError(err)
		http.Error(w, msg, code)
		return
	}
	defer f.Close()

	d, err := f.Stat()
	if err != nil {
		msg, code := toHTTPError(err)
		http.Error(w, msg, code)
		return
	}

	// redirect to canonical path: / at end of directory url
	// req.URL.Path always begins with /
	url := req.URL.Path
	if d.IsDir() {
		if url[len(url)-1] != '/' {
			localRedirect(w, req, path.Base(url)+"/")
			return
		}
	} else {
		if url[len(url)-1] == '/' && url != "/" {
			localRedirect(w, req, "../"+path.Base(url))
			return
		}
	}

	// A directory?
	if d.IsDir() {
		if checkLastModified(w, req, d.ModTime()) {
			return
		}
		dirList(w, f, name)
		return
	}

	/*if _, plain := req.URL.Query()["plain"]; plain {
		w.Header().Set("Content-Type", "text/plain")
	}*/

	// If client doesn't accept gzip encoding, serve without compression.
	if !isGzipEncodingAccepted(req) {
		http.ServeContent(w, req, d.Name(), d.ModTime(), f)
		return
	}

	// If the file is not worth gzip compressing, serve it as is.
	type notWorthGzipCompressing interface {
		NotWorthGzipCompressing()
	}
	if _, ok := f.(notWorthGzipCompressing); ok {
		http.ServeContent(w, req, d.Name(), d.ModTime(), f)
		return
	}

	// If there are gzip encoded bytes available, use them directly.
	type gzipByter interface {
		GzipBytes() []byte
	}
	if gzipFile, ok := f.(gzipByter); ok {
		w.Header().Set("Content-Encoding", "gzip")
		http.ServeContent(w, req, d.Name(), d.ModTime(), bytes.NewReader(gzipFile.GzipBytes()))
		return
	}

	// Perform compression and serve gzip compressed bytes.
	rs, err := gzipCompress(f)
	if err != nil {
		msg, code := toHTTPError(err)
		http.Error(w, msg, code)
		return
	}
	w.Header().Set("Content-Encoding", "gzip")
	http.ServeContent(w, req, d.Name(), d.ModTime(), rs)
}

func gzipCompress(r io.Reader) (io.ReadSeeker, error) {
	var buf bytes.Buffer
	gw := gzip.NewWriter(&buf)
	_, err := io.Copy(gw, r)
	if err != nil {
		return nil, err
	}
	err = gw.Close()
	if err != nil {
		return nil, err
	}
	return bytes.NewReader(buf.Bytes()), nil
}

func dirList(w http.ResponseWriter, f http.File, name string) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	fmt.Fprintf(w, "
\n")
	switch name {
	case "/":
		fmt.Fprintf(w, "%s\n", "/", ".")
	default:
		fmt.Fprintf(w, "%s\n", path.Clean(name+"/.."), "..")
	}
	for {
		dirs, err := f.Readdir(100)
		if err != nil || len(dirs) == 0 {
			break
		}
		sort.Sort(byName(dirs))
		for _, d := range dirs {
			name := d.Name()
			if d.IsDir() {
				name += "/"
			}
			// name may contain '?' or '#', which must be escaped to remain
			// part of the URL path, and not indicate the start of a query
			// string or fragment.
			url := url.URL{Path: name}
			fmt.Fprintf(w, "%s\n", url.String(), html.EscapeString(name))
		}
	}
	fmt.Fprintf(w, "
\n")
}

// localRedirect gives a Moved Permanently response.
// It does not convert relative paths to absolute paths like Redirect does.
func localRedirect(w http.ResponseWriter, r *http.Request, newPath string) {
	if q := r.URL.RawQuery; q != "" {
		newPath += "?" + q
	}
	w.Header().Set("Location", newPath)
	w.WriteHeader(http.StatusMovedPermanently)
}

// modtime is the modification time of the resource to be served, or IsZero().
// return value is whether this request is now complete.
func checkLastModified(w http.ResponseWriter, r *http.Request, modtime time.Time) bool {
	if modtime.IsZero() {
		return false
	}

	// The Date-Modified header truncates sub-second precision, so
	// use mtime < t+1s instead of mtime <= t to check for unmodified.
	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.Before(t.Add(1*time.Second)) {
		h := w.Header()
		delete(h, "Content-Type")
		delete(h, "Content-Length")
		w.WriteHeader(http.StatusNotModified)
		return true
	}
	w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
	return false
}

// byName implements sort.Interface.
type byName []os.FileInfo

func (f byName) Len() int           { return len(f) }
func (f byName) Less(i, j int) bool { return f[i].Name() < f[j].Name() }
func (f byName) Swap(i, j int)      { f[i], f[j] = f[j], f[i] }

// toHTTPError returns a detailed HTTP error message and status code
// for a given non-nil error value.
func toHTTPError(err error) (msg string, httpStatus int) {
	switch {
	case os.IsNotExist(err):
		return fmt.Sprintf("404 Page Not Found\n\n%v", err), http.StatusNotFound
	case os.IsPermission(err):
		return fmt.Sprintf("403 Forbidden\n\n%v", err), http.StatusForbidden
	default:
		return fmt.Sprintf("500 Internal Server Error\n\n%v", err), http.StatusInternalServerError
	}
}

// isGzipEncodingAccepted returns true if the request includes "gzip" under Accept-Encoding header.
func isGzipEncodingAccepted(req *http.Request) bool {
	for _, v := range strings.Split(req.Header.Get("Accept-Encoding"), ",") {
		if strings.TrimSpace(v) == "gzip" {
			return true
		}
	}
	return false
}
<commit_msg>gzip_file_server: Don't serve dynamically compressed content if original version is smaller.<commit_after>// Package gzip_file_server provides a http.Handler that serves the given virtual file system with gzip compression,
// without special handling of index.html, and detailed HTTP error messages.
package gzip_file_server

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"html"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
	"sort"
	"strings"
	"time"
)

type gzipFileServer struct {
	root http.FileSystem
}

// New returns a gzip file server, that serves the given virtual file system without special handling of index.html.
func New(root http.FileSystem) http.Handler {
	return &gzipFileServer{root: root}
}

func (f *gzipFileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if !strings.HasPrefix(r.URL.Path, "/") {
		r.URL.Path = "/" + r.URL.Path
	}
	f.serveFile(w, r, path.Clean(r.URL.Path))
}

// name is '/'-separated, not filepath.Separator.
func (fs *gzipFileServer) serveFile(w http.ResponseWriter, req *http.Request, name string) {
	f, err := fs.root.Open(name)
	if err != nil {
		msg, code := toHTTPError(err)
		http.Error(w, msg, code)
		return
	}
	defer f.Close()

	d, err := f.Stat()
	if err != nil {
		msg, code := toHTTPError(err)
		http.Error(w, msg, code)
		return
	}

	// redirect to canonical path: / at end of directory url
	// req.URL.Path always begins with /
	url := req.URL.Path
	if d.IsDir() {
		if url[len(url)-1] != '/' {
			localRedirect(w, req, path.Base(url)+"/")
			return
		}
	} else {
		if url[len(url)-1] == '/' && url != "/" {
			localRedirect(w, req, "../"+path.Base(url))
			return
		}
	}

	// A directory?
	if d.IsDir() {
		if checkLastModified(w, req, d.ModTime()) {
			return
		}
		dirList(w, f, name)
		return
	}

	/*if _, plain := req.URL.Query()["plain"]; plain {
		w.Header().Set("Content-Type", "text/plain")
	}*/

	// If client doesn't accept gzip encoding, serve without compression.
	if !isGzipEncodingAccepted(req) {
		http.ServeContent(w, req, d.Name(), d.ModTime(), f)
		return
	}

	// If the file is not worth gzip compressing, serve it as is.
	type notWorthGzipCompressing interface {
		NotWorthGzipCompressing()
	}
	if _, ok := f.(notWorthGzipCompressing); ok {
		http.ServeContent(w, req, d.Name(), d.ModTime(), f)
		return
	}

	// If there are gzip encoded bytes available, use them directly.
	type gzipByter interface {
		GzipBytes() []byte
	}
	if gzipFile, ok := f.(gzipByter); ok {
		w.Header().Set("Content-Encoding", "gzip")
		http.ServeContent(w, req, d.Name(), d.ModTime(), bytes.NewReader(gzipFile.GzipBytes()))
		return
	}

	// Perform compression and serve gzip compressed bytes (if it's worth it).
	if rs, err := gzipCompress(f); err == nil {
		w.Header().Set("Content-Encoding", "gzip")
		http.ServeContent(w, req, d.Name(), d.ModTime(), rs)
		return
	}

	// Serve as is.
	http.ServeContent(w, req, d.Name(), d.ModTime(), f)
}

// gzipCompress compresses input from r and returns it as an io.ReadSeeker.
// It returns an error if compressed size is not smaller than uncompressed.
func gzipCompress(r io.Reader) (io.ReadSeeker, error) {
	var buf bytes.Buffer
	gw := gzip.NewWriter(&buf)
	n, err := io.Copy(gw, r)
	if err != nil {
		return nil, err
	}
	err = gw.Close()
	if err != nil {
		return nil, err
	}
	if int64(buf.Len()) >= n {
		return nil, fmt.Errorf("not worth gzip compressing: original size %v, compressed size %v", n, buf.Len())
	}
	return bytes.NewReader(buf.Bytes()), nil
}

func dirList(w http.ResponseWriter, f http.File, name string) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	fmt.Fprintf(w, "
\n")
	switch name {
	case "/":
		fmt.Fprintf(w, "%s\n", "/", ".")
	default:
		fmt.Fprintf(w, "%s\n", path.Clean(name+"/.."), "..")
	}
	for {
		dirs, err := f.Readdir(100)
		if err != nil || len(dirs) == 0 {
			break
		}
		sort.Sort(byName(dirs))
		for _, d := range dirs {
			name := d.Name()
			if d.IsDir() {
				name += "/"
			}
			// name may contain '?' or '#', which must be escaped to remain
			// part of the URL path, and not indicate the start of a query
			// string or fragment.
			url := url.URL{Path: name}
			fmt.Fprintf(w, "%s\n", url.String(), html.EscapeString(name))
		}
	}
	fmt.Fprintf(w, "
\n")
}

// localRedirect gives a Moved Permanently response.
// It does not convert relative paths to absolute paths like Redirect does.
func localRedirect(w http.ResponseWriter, r *http.Request, newPath string) {
	if q := r.URL.RawQuery; q != "" {
		newPath += "?" + q
	}
	w.Header().Set("Location", newPath)
	w.WriteHeader(http.StatusMovedPermanently)
}

// modtime is the modification time of the resource to be served, or IsZero().
// return value is whether this request is now complete.
func checkLastModified(w http.ResponseWriter, r *http.Request, modtime time.Time) bool {
	if modtime.IsZero() {
		return false
	}

	// The Date-Modified header truncates sub-second precision, so
	// use mtime < t+1s instead of mtime <= t to check for unmodified.
	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.Before(t.Add(1*time.Second)) {
		h := w.Header()
		delete(h, "Content-Type")
		delete(h, "Content-Length")
		w.WriteHeader(http.StatusNotModified)
		return true
	}
	w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
	return false
}

// byName implements sort.Interface.
type byName []os.FileInfo

func (f byName) Len() int           { return len(f) }
func (f byName) Less(i, j int) bool { return f[i].Name() < f[j].Name() }
func (f byName) Swap(i, j int)      { f[i], f[j] = f[j], f[i] }

// toHTTPError returns a detailed HTTP error message and status code
// for a given non-nil error value.
func toHTTPError(err error) (msg string, httpStatus int) {
	switch {
	case os.IsNotExist(err):
		return fmt.Sprintf("404 Page Not Found\n\n%v", err), http.StatusNotFound
	case os.IsPermission(err):
		return fmt.Sprintf("403 Forbidden\n\n%v", err), http.StatusForbidden
	default:
		return fmt.Sprintf("500 Internal Server Error\n\n%v", err), http.StatusInternalServerError
	}
}

// isGzipEncodingAccepted returns true if the request includes "gzip" under Accept-Encoding header.
func isGzipEncodingAccepted(req *http.Request) bool {
	for _, v := range strings.Split(req.Header.Get("Accept-Encoding"), ",") {
		if strings.TrimSpace(v) == "gzip" {
			return true
		}
	}
	return false
}
<|endoftext|>