<commit_before>package main

import (
	"crypto/tls"
	"errors"
	"io"
	"log"
	"math/rand"
	"net"
	"net/http"
	"net/http/httputil"
	"path"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/coreos/go-etcd/etcd"
	"github.com/flynn/go-discoverd"
)

type HTTPFrontend struct {
	Addr      string
	TLSAddr   string
	TLSConfig *tls.Config

	mtx      sync.RWMutex
	domains  map[string]*httpServer
	services map[string]*httpServer

	etcdPrefix string

	etcd      *etcd.Client
	discoverd *discoverd.Client
}

func NewHTTPFrontend(addr string) (*HTTPFrontend, error) {
	f := &HTTPFrontend{
		Addr:       addr,
		etcd:       etcd.NewClient(nil),
		etcdPrefix: "/strowger/http/",
		domains:    make(map[string]*httpServer),
		services:   make(map[string]*httpServer),
	}
	var err error
	f.discoverd, err = discoverd.NewClient()
	return f, err
}

func (s *HTTPFrontend) AddHTTPDomain(domain string, service string, certs [][]byte, key []byte) error {
	return s.addDomain(domain, service, true)
}

var ErrDomainExists = errors.New("strowger: domain exists with different service")

func (s *HTTPFrontend) addDomain(domain string, service string, persist bool) error {
	s.mtx.Lock()
	defer s.mtx.Unlock()

	if server, ok := s.domains[domain]; ok {
		if server.name != service {
			return ErrDomainExists
		}
		return nil
	}

	server := s.services[service]
	if server == nil {
		services, err := s.discoverd.NewServiceSet(service)
		if err != nil {
			return err
		}
		server = &httpServer{name: service, services: services}
	}
	if persist {
		if _, err := s.etcd.Create(s.etcdPrefix+domain+"/service", service, 0); err != nil {
			return err
		}
	}
	// TODO: set cert/key data if provided

	server.refs++
	s.domains[domain] = server
	s.services[service] = server
	// TODO: TLS config

	log.Println("Add service", service, "to domain", domain)

	return nil
}

func (s *HTTPFrontend) RemoveHTTPDomain(domain string) {
	s.mtx.Lock()
	defer s.mtx.Unlock()
	server := s.domains[domain]
	if server == nil {
		return
	}
	delete(s.domains, domain)
	server.refs++
	if server.refs <= 0 {
		// TODO: close service set stream
		delete(s.services, server.name)
	}
	// TODO: persist
}

func (s *HTTPFrontend) syncDatabase() {
	var since uint64
	data, err := s.etcd.Get(s.etcdPrefix, false, true)
	if e, ok := err.(etcd.EtcdError); ok && e.ErrorCode == 100 {
		// key not found, ignore
		goto watch
	}
	if err != nil {
		log.Fatal(err)
		return
	}
	since = data.EtcdIndex
	for _, node := range data.Node.Nodes {
		if !node.Dir {
			continue
		}
		domain := path.Base(node.Key)
		serviceRes, err := s.etcd.Get(node.Key+"/service", false, false)
		if err != nil {
			log.Fatal(err)
		}
		if err := s.addDomain(domain, serviceRes.Node.Value, false); err != nil {
			log.Fatal(err)
		}
	}

watch:
	stream := make(chan *etcd.Response)
	stop := make(chan bool)
	// TODO: store stop
	go s.etcd.Watch(s.etcdPrefix, since, false, stream, stop)
	for res := range stream {
		if res.Node.Dir || path.Base(res.Node.Key) == "service" {
			continue
		}
		domain := path.Base(path.Dir(res.Node.Key))
		s.mtx.Lock()
		_, exists := s.domains[domain]
		s.mtx.Unlock()
		if !exists {
			if err := s.addDomain(domain, res.Node.Value, false); err != nil {
				// TODO: log error
			}
		}
	}
	// TODO: handle delete
	log.Println("done watching etcd")
}

func (s *HTTPFrontend) serve() {
	l, err := net.Listen("tcp", s.Addr)
	if err != nil {
		// TODO: log error
		return
	}
	for {
		conn, err := l.Accept()
		if err != nil {
			// TODO: log error
			break
		}
		go s.handle(conn)
	}
}

func (s *HTTPFrontend) serveTLS() {
	l, err := net.Listen("tcp", s.TLSAddr)
	if err != nil {
		// TODO: log error
		return
	}
	for {
		conn, err := l.Accept()
		if err != nil {
			// TODO: log error
			break
		}
		go s.handle(conn)
	}
}

func (s *HTTPFrontend) handle(conn net.Conn) {
	defer conn.Close()
	sc := httputil.NewServerConn(conn, nil)
	req, err := sc.Read()
	if err != nil {
		if err != httputil.ErrPersistEOF {
			// TODO: log error
		}
		return
	}

	s.mtx.RLock()
	// TODO: handle wildcard domains
	backend := s.domains[req.Host]
	s.mtx.RUnlock()
	log.Println(req, backend)
	if backend == nil {
		// TODO: return 404
		return
	}
	_, tls := conn.(*tls.Conn)
	backend.handle(req, sc, tls)
}

type httpServer struct {
	name     string
	services *discoverd.ServiceSet
	refs     int
}

func (s *httpServer) getBackend() *httputil.ClientConn {
	for _, addr := range shuffle(s.services.Addrs()) {
		// TODO: set connection timeout
		backend, err := net.Dial("tcp", addr)
		if err != nil {
			// TODO: log error
			// TODO: limit number of backends tried
			// TODO: temporarily quarantine failing backends
			log.Println("backend error", err)
			continue
		}
		return httputil.NewClientConn(backend, nil)
	}
	// TODO: log no backends found error
	return nil
}

func (s *httpServer) handle(req *http.Request, sc *httputil.ServerConn, tls bool) {
	req.Header.Set("X-Request-Start", strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10))
	backend := s.getBackend()
	if backend == nil {
		// TODO: Return 503
		log.Println("no backend found")
		return
	}
	defer backend.Close()

	for {
		if req.Method != "GET" && req.Method != "POST" && req.Method != "HEAD" &&
			req.Method != "OPTIONS" && req.Method != "PUT" && req.Method != "DELETE" && req.Method != "TRACE" {
			// TODO: return 405
		}

		req.Proto = "HTTP/1.1"
		req.ProtoMajor = 1
		req.ProtoMinor = 1
		delete(req.Header, "Te")
		delete(req.Header, "Transfer-Encoding")

		if clientIP, _, err := net.SplitHostPort(req.RemoteAddr); err == nil {
			// If we aren't the first proxy retain prior
			// X-Forwarded-For information as a comma+space
			// separated list and fold multiple headers into one.
			if prior, ok := req.Header["X-Forwarded-For"]; ok {
				clientIP = strings.Join(prior, ", ") + ", " + clientIP
			}
			req.Header.Set("X-Forwarded-For", clientIP)
		}
		if tls {
			req.Header.Set("X-Forwarded-Proto", "https")
		} else {
			req.Header.Set("X-Forwarded-Proto", "http")
		}
		// TODO: Set X-Forwarded-Port

		if err := backend.Write(req); err != nil {
			log.Println("server write err:", err)
			return
		}
		res, err := backend.Read(req)
		if res != nil {
			if err := sc.Write(req, res); err != nil {
				if err != io.EOF && err != httputil.ErrPersistEOF {
					log.Println("client write err:", err)
					// TODO: log error
				}
				return
			}
		}
		if err != nil {
			if err != io.EOF && err != httputil.ErrPersistEOF {
				log.Println("server read err:", err)
				// TODO: log error
				// TODO: Return 502
			}
			return
		}

		// TODO: Proxy HTTP CONNECT? (example: Go RPC over HTTP)
		if res.StatusCode == http.StatusSwitchingProtocols {
			serverW, serverR := backend.Hijack()
			clientW, clientR := sc.Hijack()
			defer serverW.Close()
			done := make(chan struct{})
			go func() {
				serverR.WriteTo(clientW)
				close(done)
			}()
			clientR.WriteTo(serverW)
			<-done
			return
		}

		// TODO: http pipelining
		req, err = sc.Read()
		if err != nil {
			if err != io.EOF && err != httputil.ErrPersistEOF {
				log.Println("client read err:", err)
			}
			return
		}
		req.Header.Set("X-Request-Start", strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10))
	}
}

func shuffle(s []string) []string {
	for i := len(s) - 1; i > 0; i-- {
		j := rand.Intn(i + 1)
		s[i], s[j] = s[j], s[i]
	}
	return s
}

func init() {
	rand.Seed(time.Now().UnixNano())
}
<commit_msg>router: Use interface for etcd client in frontend<commit_after>package main

import (
	"crypto/tls"
	"errors"
	"io"
	"log"
	"math/rand"
	"net"
	"net/http"
	"net/http/httputil"
	"path"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/coreos/go-etcd/etcd"
	"github.com/flynn/go-discoverd"
)

type HTTPFrontend struct {
	Addr      string
	TLSAddr   string
	TLSConfig *tls.Config

	mtx      sync.RWMutex
	domains  map[string]*httpServer
	services map[string]*httpServer

	etcdPrefix string

	etcd      httpFrontendEtcdClient
	discoverd *discoverd.Client
}

type httpFrontendEtcdClient interface {
	Create(key string, value string, ttl uint64) (*etcd.Response, error)
	Get(key string, sort, recursive bool) (*etcd.Response, error)
	Watch(prefix string, waitIndex uint64, recursive bool, receiver chan *etcd.Response, stop chan bool) (*etcd.Response, error)
}

func NewHTTPFrontend(addr string) (*HTTPFrontend, error) {
	f := &HTTPFrontend{
		Addr:       addr,
		etcd:       etcd.NewClient(nil),
		etcdPrefix: "/strowger/http/",
		domains:    make(map[string]*httpServer),
		services:   make(map[string]*httpServer),
	}
	var err error
	f.discoverd, err = discoverd.NewClient()
	return f, err
}

func (s *HTTPFrontend) AddHTTPDomain(domain string, service string, certs [][]byte, key []byte) error {
	return s.addDomain(domain, service, true)
}

var ErrDomainExists = errors.New("strowger: domain exists with different service")

func (s *HTTPFrontend) addDomain(domain string, service string, persist bool) error {
	s.mtx.Lock()
	defer s.mtx.Unlock()

	if server, ok := s.domains[domain]; ok {
		if server.name != service {
			return ErrDomainExists
		}
		return nil
	}

	server := s.services[service]
	if server == nil {
		services, err := s.discoverd.NewServiceSet(service)
		if err != nil {
			return err
		}
		server = &httpServer{name: service, services: services}
	}
	if persist {
		if _, err := s.etcd.Create(s.etcdPrefix+domain+"/service", service, 0); err != nil {
			return err
		}
	}
	// TODO: set cert/key data if provided

	server.refs++
	s.domains[domain] = server
	s.services[service] = server
	// TODO: TLS config

	log.Println("Add service", service, "to domain", domain)

	return nil
}

func (s *HTTPFrontend) RemoveHTTPDomain(domain string) {
	s.mtx.Lock()
	defer s.mtx.Unlock()
	server := s.domains[domain]
	if server == nil {
		return
	}
	delete(s.domains, domain)
	server.refs++
	if server.refs <= 0 {
		// TODO: close service set stream
		delete(s.services, server.name)
	}
	// TODO: persist
}

func (s *HTTPFrontend) syncDatabase() {
	var since uint64
	data, err := s.etcd.Get(s.etcdPrefix, false, true)
	if e, ok := err.(etcd.EtcdError); ok && e.ErrorCode == 100 {
		// key not found, ignore
		goto watch
	}
	if err != nil {
		log.Fatal(err)
		return
	}
	since = data.EtcdIndex
	for _, node := range data.Node.Nodes {
		if !node.Dir {
			continue
		}
		domain := path.Base(node.Key)
		serviceRes, err := s.etcd.Get(node.Key+"/service", false, false)
		if err != nil {
			log.Fatal(err)
		}
		if err := s.addDomain(domain, serviceRes.Node.Value, false); err != nil {
			log.Fatal(err)
		}
	}

watch:
	stream := make(chan *etcd.Response)
	stop := make(chan bool)
	// TODO: store stop
	go s.etcd.Watch(s.etcdPrefix, since, false, stream, stop)
	for res := range stream {
		if res.Node.Dir || path.Base(res.Node.Key) == "service" {
			continue
		}
		domain := path.Base(path.Dir(res.Node.Key))
		s.mtx.Lock()
		_, exists := s.domains[domain]
		s.mtx.Unlock()
		if !exists {
			if err := s.addDomain(domain, res.Node.Value, false); err != nil {
				// TODO: log error
			}
		}
	}
	// TODO: handle delete
	log.Println("done watching etcd")
}

func (s *HTTPFrontend) serve() {
	l, err := net.Listen("tcp", s.Addr)
	if err != nil {
		// TODO: log error
		return
	}
	for {
		conn, err := l.Accept()
		if err != nil {
			// TODO: log error
			break
		}
		go s.handle(conn)
	}
}

func (s *HTTPFrontend) serveTLS() {
	l, err := net.Listen("tcp", s.TLSAddr)
	if err != nil {
		// TODO: log error
		return
	}
	for {
		conn, err := l.Accept()
		if err != nil {
			// TODO: log error
			break
		}
		go s.handle(conn)
	}
}

func (s *HTTPFrontend) handle(conn net.Conn) {
	defer conn.Close()
	sc := httputil.NewServerConn(conn, nil)
	req, err := sc.Read()
	if err != nil {
		if err != httputil.ErrPersistEOF {
			// TODO: log error
		}
		return
	}

	s.mtx.RLock()
	// TODO: handle wildcard domains
	backend := s.domains[req.Host]
	s.mtx.RUnlock()
	log.Println(req, backend)
	if backend == nil {
		// TODO: return 404
		return
	}
	_, tls := conn.(*tls.Conn)
	backend.handle(req, sc, tls)
}

type httpServer struct {
	name     string
	services *discoverd.ServiceSet
	refs     int
}

func (s *httpServer) getBackend() *httputil.ClientConn {
	for _, addr := range shuffle(s.services.Addrs()) {
		// TODO: set connection timeout
		backend, err := net.Dial("tcp", addr)
		if err != nil {
			// TODO: log error
			// TODO: limit number of backends tried
			// TODO: temporarily quarantine failing backends
			log.Println("backend error", err)
			continue
		}
		return httputil.NewClientConn(backend, nil)
	}
	// TODO: log no backends found error
	return nil
}

func (s *httpServer) handle(req *http.Request, sc *httputil.ServerConn, tls bool) {
	req.Header.Set("X-Request-Start", strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10))
	backend := s.getBackend()
	if backend == nil {
		// TODO: Return 503
		log.Println("no backend found")
		return
	}
	defer backend.Close()

	for {
		if req.Method != "GET" && req.Method != "POST" && req.Method != "HEAD" &&
			req.Method != "OPTIONS" && req.Method != "PUT" && req.Method != "DELETE" && req.Method != "TRACE" {
			// TODO: return 405
		}

		req.Proto = "HTTP/1.1"
		req.ProtoMajor = 1
		req.ProtoMinor = 1
		delete(req.Header, "Te")
		delete(req.Header, "Transfer-Encoding")

		if clientIP, _, err := net.SplitHostPort(req.RemoteAddr); err == nil {
			// If we aren't the first proxy retain prior
			// X-Forwarded-For information as a comma+space
			// separated list and fold multiple headers into one.
			if prior, ok := req.Header["X-Forwarded-For"]; ok {
				clientIP = strings.Join(prior, ", ") + ", " + clientIP
			}
			req.Header.Set("X-Forwarded-For", clientIP)
		}
		if tls {
			req.Header.Set("X-Forwarded-Proto", "https")
		} else {
			req.Header.Set("X-Forwarded-Proto", "http")
		}
		// TODO: Set X-Forwarded-Port

		if err := backend.Write(req); err != nil {
			log.Println("server write err:", err)
			return
		}
		res, err := backend.Read(req)
		if res != nil {
			if err := sc.Write(req, res); err != nil {
				if err != io.EOF && err != httputil.ErrPersistEOF {
					log.Println("client write err:", err)
					// TODO: log error
				}
				return
			}
		}
		if err != nil {
			if err != io.EOF && err != httputil.ErrPersistEOF {
				log.Println("server read err:", err)
				// TODO: log error
				// TODO: Return 502
			}
			return
		}

		// TODO: Proxy HTTP CONNECT? (example: Go RPC over HTTP)
		if res.StatusCode == http.StatusSwitchingProtocols {
			serverW, serverR := backend.Hijack()
			clientW, clientR := sc.Hijack()
			defer serverW.Close()
			done := make(chan struct{})
			go func() {
				serverR.WriteTo(clientW)
				close(done)
			}()
			clientR.WriteTo(serverW)
			<-done
			return
		}

		// TODO: http pipelining
		req, err = sc.Read()
		if err != nil {
			if err != io.EOF && err != httputil.ErrPersistEOF {
				log.Println("client read err:", err)
			}
			return
		}
		req.Header.Set("X-Request-Start", strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10))
	}
}

func shuffle(s []string) []string {
	for i := len(s) - 1; i > 0; i-- {
		j := rand.Intn(i + 1)
		s[i], s[j] = s[j], s[i]
	}
	return s
}

func init() {
	rand.Seed(time.Now().UnixNano())
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"strings"

	"github.com/pocke/gfm-viewer/env"
	"github.com/pocke/hlog.go"
	"github.com/yosssi/ace"
)

type Server struct {
	storage *Storage
}

func NewServer() *Server {
	s := &Server{
		storage: NewStorage(),
	}

	go func() {
		f := func(w http.ResponseWriter, r *http.Request) {
			path := r.URL.Path
			if path == "/" || path == "/index.html" {
				if s.storage.token.hasToken() {
					s.indexHandler(w, r)
				} else {
					s.beforeAuthHandler(w, r)
				}
			} else if path == "/auth" {
				s.authHandler(w, r)
			} else if strings.HasPrefix(path, "/files") {
				s.ServeFile(w, r)
			} else if path == "/css/github-markdown.css" {
				s.serveCSS(w, r)
			} else {
				http.Error(w, "404 Not Found", http.StatusNotFound)
				return
			}
		}
		if env.DEBUG {
			f = hlog.Wrap(f)
		}
		http.HandleFunc("/", f)
		// TODO: port
		http.ListenAndServe(":1124", nil)
	}()

	return s
}

func (s *Server) ServeFile(w http.ResponseWriter, r *http.Request) {
	path := strings.TrimLeft(r.URL.Path, "/files")
	html, ok := s.storage.Get(path)
	if !ok {
		http.Error(w, fmt.Sprintf("%s page not found", path), http.StatusNotFound)
		return
	}
	w.Write([]byte(html))
}

func (s *Server) authHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	v := r.PostForm
	user := v.Get("username")
	pass := v.Get("password")

	err := s.storage.token.Init(user, pass)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	s.storage.UpdateAll()
	http.Redirect(w, r, "/", http.StatusFound)
}

func (s *Server) indexHandler(w http.ResponseWriter, r *http.Request) {
	loadAce(w, "index", s.storage.Index())
}

func (s *Server) beforeAuthHandler(w http.ResponseWriter, r *http.Request) {
	loadAce(w, "before_auth", nil)
}

func loadAce(w http.ResponseWriter, action string, data interface{}) {
	tpl, err := ace.Load("assets/"+action, "", &ace.Options{
		DynamicReload: env.DEBUG,
		Asset:         Asset,
	})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	err = tpl.Execute(w, data)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

func (s *Server) serveCSS(w http.ResponseWriter, r *http.Request) {
	file, err := Asset("assets/github-markdown.css")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "text/css")
	w.Write(file)
}
<commit_msg>Use denco as HTTP router. Close #9<commit_after>package main

import (
	"fmt"
	"net/http"

	"github.com/naoina/denco"
	"github.com/pocke/gfm-viewer/env"
	"github.com/pocke/hlog.go"
	"github.com/yosssi/ace"
)

type Server struct {
	storage *Storage
}

func NewServer() *Server {
	s := &Server{
		storage: NewStorage(),
	}

	go func() {
		mux := denco.NewMux()
		f, err := mux.Build([]denco.Handler{
			mux.GET("/", s.indexHandler),
			mux.POST("/auth", s.authHandler),
			mux.GET("/files/*path", s.ServeFile),
			mux.GET("/css/github-markdown.css", s.serveCSS),
		})
		if err != nil {
			panic(err)
		}
		handler := f.ServeHTTP
		if env.DEBUG {
			handler = hlog.Wrap(f.ServeHTTP)
		}
		http.HandleFunc("/", handler)
		// TODO: port
		http.ListenAndServe(":1124", nil)
	}()

	return s
}

func (s *Server) ServeFile(w http.ResponseWriter, r *http.Request, p denco.Params) {
	path := p.Get("path")
	html, ok := s.storage.Get(path)
	if !ok {
		http.Error(w, fmt.Sprintf("%s page not found", path), http.StatusNotFound)
		return
	}
	w.Write([]byte(html))
}

func (s *Server) authHandler(w http.ResponseWriter, r *http.Request, _ denco.Params) {
	r.ParseForm()
	v := r.PostForm
	user := v.Get("username")
	pass := v.Get("password")

	err := s.storage.token.Init(user, pass)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	s.storage.UpdateAll()
	http.Redirect(w, r, "/", http.StatusFound)
}

func (s *Server) indexHandler(w http.ResponseWriter, r *http.Request, _ denco.Params) {
	if s.storage.token.hasToken() {

		loadAce(w, "index", s.storage.Index())
	} else {

		loadAce(w, "before_auth", nil)
	}
}

func loadAce(w http.ResponseWriter, action string, data interface{}) {
	tpl, err := ace.Load("assets/"+action, "", &ace.Options{
		DynamicReload: env.DEBUG,
		Asset:         Asset,
	})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	err = tpl.Execute(w, data)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

func (s *Server) serveCSS(w http.ResponseWriter, r *http.Request, _ denco.Params) {
	file, err := Asset("assets/github-markdown.css")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "text/css")
	w.Write(file)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"time"
	"strings"
	"net/http"
	"github.com/stretchr/goweb"
	"github.com/stretchr/goweb/context"
)

type HTTP struct {
	Ses *http.Server
	Addr string
	ReadTimeout time.Duration
	WriteTimeout time.Duration
	MaxHeaderBytes int
	W chan *MPLX
}

func (M *Main) HTTP_Init() {
	DebugLn("HTTP_Init():Initialized")

	M.H = new(HTTP)
	M.HTTP_Defaults()
	M.HTTP_Setup()
	M.HTTP_Routes()

	go HUB.WS_Run()
	go M.HTTP_Serve()
}

func (M *Main) HTTP_Defaults() {
/* FIXME - only allow localhost */
// H.Addr = "127.0.0.1:8080
	M.H.Addr = fmt.Sprintf("%s:%s",M.Startup_Config.Http.Host, M.Startup_Config.Http.Port) //"0.0.0.0:911"
	M.H.ReadTimeout = time.Duration(2*time.Second)
	M.H.WriteTimeout = time.Duration(2*time.Second)
	M.H.MaxHeaderBytes = 1 << 20
	return
}

func (M *Main) HTTP_Setup() {

	M.H.Ses = &http.Server{
		Addr : M.H.Addr,
		ReadTimeout : M.H.ReadTimeout,
		WriteTimeout : M.H.WriteTimeout,
		MaxHeaderBytes : M.H.MaxHeaderBytes,
		Handler : goweb.DefaultHttpHandler(),
	}

	goweb.MapStaticFile("/", "pub/indeH.html")
	goweb.MapStatic("/pub", "pub")

	goweb.MapBefore(func(c context.Context) (error) {
		req := c.HttpRequest()
		Debug("LOGGER: %v %v\n", req.Method, req.URL)
		return nil
	})

	goweb.MapAfter(func(c context.Context) (error) {
		req := c.HttpRequest()
		Debug("LOGGER: %v %v\n", req.Method, req.URL)
		return nil
	})

 return
}

func (M *Main) HTTP_Routes() {

	M.HTTP_WS_Init()
	M.HTTP_Ping_Init()
	M.HTTP_Policies_Init()
	M.HTTP_Accounts_Init()
	M.HTTP_Sessions_Init()
	M.HTTP_State_Init()
	M.HTTP_Query_Init()
	M.HTTP_Help_Init()

}

func (M *Main) HTTP_Serve() {
	err := M.H.Ses.ListenAndServe()
	if err != nil {
		log.Fatal("HTTP_Serve:ListenAndServe()", err)
	}
	return
}

func HTTP_Filter(c context.Context, path string) ([]string, error) {
	list := c.PathValue(path)

	if list == "" {
		return nil, nil
	}
	list_array := strings.Split(string(list),":")

	return list_array, nil
}
<commit_msg>allow nil to return empty set in HTTP_Filter<commit_after>package main

import (
	"fmt"
	"log"
	"time"
	"strings"
	"net/http"
	"github.com/stretchr/goweb"
	"github.com/stretchr/goweb/context"
)

type HTTP struct {
	Ses *http.Server
	Addr string
	ReadTimeout time.Duration
	WriteTimeout time.Duration
	MaxHeaderBytes int
	W chan *MPLX
}

func (M *Main) HTTP_Init() {
	DebugLn("HTTP_Init():Initialized")

	M.H = new(HTTP)
	M.HTTP_Defaults()
	M.HTTP_Setup()
	M.HTTP_Routes()

	go HUB.WS_Run()
	go M.HTTP_Serve()
}

func (M *Main) HTTP_Defaults() {
/* FIXME - only allow localhost */
// H.Addr = "127.0.0.1:8080
	M.H.Addr = fmt.Sprintf("%s:%s",M.Startup_Config.Http.Host, M.Startup_Config.Http.Port) //"0.0.0.0:911"
	M.H.ReadTimeout = time.Duration(2*time.Second)
	M.H.WriteTimeout = time.Duration(2*time.Second)
	M.H.MaxHeaderBytes = 1 << 20
	return
}

func (M *Main) HTTP_Setup() {

	M.H.Ses = &http.Server{
		Addr : M.H.Addr,
		ReadTimeout : M.H.ReadTimeout,
		WriteTimeout : M.H.WriteTimeout,
		MaxHeaderBytes : M.H.MaxHeaderBytes,
		Handler : goweb.DefaultHttpHandler(),
	}

	goweb.MapStaticFile("/", "pub/indeH.html")
	goweb.MapStatic("/pub", "pub")

	goweb.MapBefore(func(c context.Context) (error) {
		req := c.HttpRequest()
		Debug("LOGGER: %v %v\n", req.Method, req.URL)
		return nil
	})

	goweb.MapAfter(func(c context.Context) (error) {
		req := c.HttpRequest()
		Debug("LOGGER: %v %v\n", req.Method, req.URL)
		return nil
	})

 return
}

func (M *Main) HTTP_Routes() {

	M.HTTP_WS_Init()
	M.HTTP_Ping_Init()
	M.HTTP_Policies_Init()
	M.HTTP_Accounts_Init()
	M.HTTP_Sessions_Init()
	M.HTTP_State_Init()
	M.HTTP_Query_Init()
	M.HTTP_Help_Init()

}

func (M *Main) HTTP_Serve() {
	err := M.H.Ses.ListenAndServe()
	if err != nil {
		log.Fatal("HTTP_Serve:ListenAndServe()", err)
	}
	return
}

func HTTP_Filter(c context.Context, path string) ([]string, error) {
	list := c.PathValue(path)

	if list == "" {
		return nil, nil
	}

	if list == "nil" {
		return []string{}, nil
	}

	list_array := strings.Split(string(list),":")

	return list_array, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/dustin/gomemcached/client"
)

type BlobOwnership struct {
	OID    string               `json:"oid"`
	Length int64                `json:"length"`
	Nodes  map[string]time.Time `json:"nodes"`
	Type   string               `json:"type"`
}

func recordBlobOwnership(h string, l int64) error {
	sid := serverIdentifier()

	k := "/" + h
	return couchbase.Do(k, func(mc *memcached.Client, vb uint16) error {
		_, err := mc.CAS(vb, k, func(in []byte) []byte {
			ownership := BlobOwnership{}
			err := json.Unmarshal(in, &ownership)
			if err == nil {
				ownership.Nodes[sid] = time.Now().UTC()
			} else {
				ownership.Nodes = map[string]time.Time{
					sid: time.Now().UTC(),
				}
			}
			ownership.OID = h
			ownership.Length = l
			ownership.Type = "blob"

			rv, err := json.Marshal(&ownership)
			if err != nil {
				log.Fatalf("Error marshaling blob ownership: %v", err)
			}
			return rv
		}, 0)
		return err
	})
}

func recordBlobAccess(h string) {
	_, err := couchbase.Incr("/"+h+"/r", 1, 1, 0)
	if err != nil {
		log.Printf("Error incrementing counter for %v: %v", h, err)
	}

	_, err = couchbase.Incr("/"+serverIdentifier()+"/r", 1, 1, 0)
	if err != nil {
		log.Printf("Error incrementing node identifier: %v", h, err)
	}
}

func putUserFile(w http.ResponseWriter, req *http.Request) {
	f, err := NewHashRecord(*root, "")
	if err != nil {
		log.Printf("Error writing tmp file: %v", err)
		w.WriteHeader(500)
		return
	}
	defer f.Close()

	h, length, err := f.Process(req.Body)
	if err != nil {
		log.Printf("Error completing blob write: %v", err)
	}

	log.Printf("Wrote %v -> %v (%#v)", req.URL.Path, h, req.Header)

	fm := fileMeta{
		req.Header,
		h,
		length,
	}

	err = storeMeta(req.URL.Path[1:], fm)
	if err != nil {
		log.Printf("Error storing file meta: %v", err)
		w.WriteHeader(500)
		fmt.Fprintf(w, "Error recording blob ownership: %v", err)
		return
	}

	err = recordBlobOwnership(h, length)
	if err != nil {
		log.Printf("Error storing blob ownership: %v", err)
		w.WriteHeader(500)
		fmt.Fprintf(w, "Error recording blob ownership: %v", err)
		return
	}

	w.WriteHeader(204)
}

func putRawHash(w http.ResponseWriter, req *http.Request) {
	inputhash := ""
	form, err := url.ParseQuery(req.URL.RawQuery)
	if err == nil {
		inputhash = form.Get("oid")
	}

	if inputhash == "" {
		w.WriteHeader(400)
		w.Write([]byte("No oid specified"))
		return
	}

	f, err := NewHashRecord(*root, inputhash)
	if err != nil {
		log.Printf("Error writing tmp file: %v", err)
		w.WriteHeader(500)
		return
	}
	defer f.Close()

	_, length, err := f.Process(req.Body)
	if err != nil {
		log.Printf("Error linking in raw hash: %v", err)
		w.WriteHeader(500)
		w.Write([]byte(err.Error()))
		return
	}

	err = recordBlobOwnership(inputhash, length)
	if err != nil {
		log.Printf("Error recording blob ownership: %v", err)
		w.WriteHeader(500)
		fmt.Fprintf(w, "Error recording blob ownership: %v", err)
		return
	}

	w.WriteHeader(204)
}

func doPut(w http.ResponseWriter, req *http.Request) {
	if req.URL.Path == "/" {
		putRawHash(w, req)
	} else {
		putUserFile(w, req)
	}
}

func isResponseHeader(s string) bool {
	switch s {
	case "Content-Type", "Content-Length":
		return true
	}
	return false
}

func doGetUserDoc(w http.ResponseWriter, req *http.Request) {
	path := req.URL.Path
	if path == "/" {
		path = *defaultPath
	}

	if len(path) > 0 && path[0] == '/' {
		path = path[1:]
	}

	got := fileMeta{}
	err := couchbase.Get(path, &got)
	if err != nil {
		log.Printf("Error getting file %#v: %v", path, err)
		w.WriteHeader(404)
		return
	}

	f, err := os.Open(hashFilename(got.OID))
	if err != nil {
		getBlobFromRemote(w, got)
		return
	}
	defer f.Close()

	for k, v := range got.Headers {
		if isResponseHeader(k) {
			w.Header()[k] = v
		}
	}
	w.WriteHeader(200)

	_, err = io.Copy(w, f)
	if err != nil {
		log.Printf("Failed to write file: %v", err)
	}
	go recordBlobAccess(got.OID)
}

func getBlobFromRemote(w http.ResponseWriter, meta fileMeta) {

	// Find the owners of this blob
	ownership := BlobOwnership{}
	oidkey := "/" + meta.OID
	err := couchbase.Get(oidkey, &ownership)
	if err != nil {
		log.Printf("Missing ownership record for OID: %v", meta.OID)
		// Not sure 404 is the right response here
		w.WriteHeader(404)
		return
	}

	// Loop through the nodes that claim to own this blob
	// If we encounter any errors along the way, try the next node
	for sid := range ownership.Nodes {
		// Skip myself
		if sid == serverIdentifier() {
			continue
		}

		log.Printf("Trying to get %s from %s", meta.OID, sid)
		sidaddr, err := getNodeAddress(sid)
		if err != nil {
			log.Printf("Missing node record for %s", sid)
			continue
		}

		remoteOidURL := fmt.Sprintf("http://%s/?oid=%s", sidaddr, meta.OID)
		resp, err := http.Get(remoteOidURL)
		if err != nil {
			log.Printf("Error reading oid %s from node %s", meta.OID, sid)
			continue
		}

		if resp.StatusCode != 200 {
			log.Printf("Error response code %d from node %s", resp.StatusCode, sid)
			continue
		}

		// Found one, set the headers and send it.  Keep a
		// local copy for good luck.

		for k, v := range meta.Headers {
			if isResponseHeader(k) {
				w.Header()[k] = v
			}
		}
		w.WriteHeader(200)

		writeTo := io.Writer(w)
		hw, err := NewHashRecord(*root, meta.OID)
		if err == nil {
			writeTo = io.MultiWriter(hw, w)
		} else {
			hw = nil
		}

		length, err := io.Copy(writeTo, resp.Body)

		if err != nil {
			log.Printf("Failed to write from remote stream %v", err)
		} else {
			// A successful copy with a working hash
			// record means we should link in and record
			// our copy of this file.
			if hw != nil {
				_, err = hw.Finish()
				if err == nil {
					go recordBlobOwnership(meta.OID, length)
				}
			}
		}

		return
	}

	//if we got to this point, no node in the list actually had it
	log.Printf("Don't have hash file: %v and no remote nodes could help",
		meta.OID)
	w.WriteHeader(500)
	return
}

func doList(w http.ResponseWriter, req *http.Request) {
	w.WriteHeader(200)
	explen := getHash().Size() * 2
	filepath.Walk(*root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() && !strings.HasPrefix(info.Name(), "tmp") &&
			len(info.Name()) == explen {
			_, e := w.Write([]byte(info.Name() + "\n"))
			return e
		}
		return nil
	})
}

func doGet(w http.ResponseWriter, req *http.Request) {
	switch {
	case req.URL.Path == "/" && req.FormValue("oid") != "":
		http.ServeFile(w, req, hashFilename(req.FormValue("oid")))
	case req.URL.Path == "/" && req.FormValue("list") != "":
		doList(w, req)
	default:
		doGetUserDoc(w, req)
	}
}

func httpHandler(w http.ResponseWriter, req *http.Request) {
	defer req.Body.Close()
	switch req.Method {
	case "PUT":
		doPut(w, req)
	case "GET":
		doGet(w, req)
	default:
		w.WriteHeader(http.StatusMethodNotAllowed)
	}
}
<commit_msg>DELETE implementation.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/dustin/gomemcached/client"
)

type BlobOwnership struct {
	OID    string               `json:"oid"`
	Length int64                `json:"length"`
	Nodes  map[string]time.Time `json:"nodes"`
	Type   string               `json:"type"`
}

func recordBlobOwnership(h string, l int64) error {
	sid := serverIdentifier()

	k := "/" + h
	return couchbase.Do(k, func(mc *memcached.Client, vb uint16) error {
		_, err := mc.CAS(vb, k, func(in []byte) []byte {
			ownership := BlobOwnership{}
			err := json.Unmarshal(in, &ownership)
			if err == nil {
				ownership.Nodes[sid] = time.Now().UTC()
			} else {
				ownership.Nodes = map[string]time.Time{
					sid: time.Now().UTC(),
				}
			}
			ownership.OID = h
			ownership.Length = l
			ownership.Type = "blob"

			rv, err := json.Marshal(&ownership)
			if err != nil {
				log.Fatalf("Error marshaling blob ownership: %v", err)
			}
			return rv
		}, 0)
		return err
	})
}

func recordBlobAccess(h string) {
	_, err := couchbase.Incr("/"+h+"/r", 1, 1, 0)
	if err != nil {
		log.Printf("Error incrementing counter for %v: %v", h, err)
	}

	_, err = couchbase.Incr("/"+serverIdentifier()+"/r", 1, 1, 0)
	if err != nil {
		log.Printf("Error incrementing node identifier: %v", h, err)
	}
}

func putUserFile(w http.ResponseWriter, req *http.Request) {
	f, err := NewHashRecord(*root, "")
	if err != nil {
		log.Printf("Error writing tmp file: %v", err)
		w.WriteHeader(500)
		return
	}
	defer f.Close()

	h, length, err := f.Process(req.Body)
	if err != nil {
		log.Printf("Error completing blob write: %v", err)
	}

	log.Printf("Wrote %v -> %v (%#v)", req.URL.Path, h, req.Header)

	fm := fileMeta{
		req.Header,
		h,
		length,
	}

	err = storeMeta(req.URL.Path[1:], fm)
	if err != nil {
		log.Printf("Error storing file meta: %v", err)
		w.WriteHeader(500)
		fmt.Fprintf(w, "Error recording blob ownership: %v", err)
		return
	}

	err = recordBlobOwnership(h, length)
	if err != nil {
		log.Printf("Error storing blob ownership: %v", err)
		w.WriteHeader(500)
		fmt.Fprintf(w, "Error recording blob ownership: %v", err)
		return
	}

	w.WriteHeader(204)
}

func putRawHash(w http.ResponseWriter, req *http.Request) {
	inputhash := ""
	form, err := url.ParseQuery(req.URL.RawQuery)
	if err == nil {
		inputhash = form.Get("oid")
	}

	if inputhash == "" {
		w.WriteHeader(400)
		w.Write([]byte("No oid specified"))
		return
	}

	f, err := NewHashRecord(*root, inputhash)
	if err != nil {
		log.Printf("Error writing tmp file: %v", err)
		w.WriteHeader(500)
		return
	}
	defer f.Close()

	_, length, err := f.Process(req.Body)
	if err != nil {
		log.Printf("Error linking in raw hash: %v", err)
		w.WriteHeader(500)
		w.Write([]byte(err.Error()))
		return
	}

	err = recordBlobOwnership(inputhash, length)
	if err != nil {
		log.Printf("Error recording blob ownership: %v", err)
		w.WriteHeader(500)
		fmt.Fprintf(w, "Error recording blob ownership: %v", err)
		return
	}

	w.WriteHeader(204)
}

func doPut(w http.ResponseWriter, req *http.Request) {
	if req.URL.Path == "/" {
		putRawHash(w, req)
	} else {
		putUserFile(w, req)
	}
}

func isResponseHeader(s string) bool {
	switch s {
	case "Content-Type", "Content-Length":
		return true
	}
	return false
}

func doGetUserDoc(w http.ResponseWriter, req *http.Request) {
	path := req.URL.Path
	if path == "/" {
		path = *defaultPath
	}

	if len(path) > 0 && path[0] == '/' {
		path = path[1:]
	}

	got := fileMeta{}
	err := couchbase.Get(path, &got)
	if err != nil {
		log.Printf("Error getting file %#v: %v", path, err)
		w.WriteHeader(404)
		return
	}

	f, err := os.Open(hashFilename(got.OID))
	if err != nil {
		getBlobFromRemote(w, got)
		return
	}
	defer f.Close()

	for k, v := range got.Headers {
		if isResponseHeader(k) {
			w.Header()[k] = v
		}
	}
	w.WriteHeader(200)

	_, err = io.Copy(w, f)
	if err != nil {
		log.Printf("Failed to write file: %v", err)
	}
	go recordBlobAccess(got.OID)
}

func getBlobFromRemote(w http.ResponseWriter, meta fileMeta) {

	// Find the owners of this blob
	ownership := BlobOwnership{}
	oidkey := "/" + meta.OID
	err := couchbase.Get(oidkey, &ownership)
	if err != nil {
		log.Printf("Missing ownership record for OID: %v", meta.OID)
		// Not sure 404 is the right response here
		w.WriteHeader(404)
		return
	}

	// Loop through the nodes that claim to own this blob
	// If we encounter any errors along the way, try the next node
	for sid := range ownership.Nodes {
		// Skip myself
		if sid == serverIdentifier() {
			continue
		}

		log.Printf("Trying to get %s from %s", meta.OID, sid)
		sidaddr, err := getNodeAddress(sid)
		if err != nil {
			log.Printf("Missing node record for %s", sid)
			continue
		}

		remoteOidURL := fmt.Sprintf("http://%s/?oid=%s", sidaddr, meta.OID)
		resp, err := http.Get(remoteOidURL)
		if err != nil {
			log.Printf("Error reading oid %s from node %s", meta.OID, sid)
			continue
		}

		if resp.StatusCode != 200 {
			log.Printf("Error response code %d from node %s", resp.StatusCode, sid)
			continue
		}

		// Found one, set the headers and send it.  Keep a
		// local copy for good luck.

		for k, v := range meta.Headers {
			if isResponseHeader(k) {
				w.Header()[k] = v
			}
		}
		w.WriteHeader(200)

		writeTo := io.Writer(w)
		hw, err := NewHashRecord(*root, meta.OID)
		if err == nil {
			writeTo = io.MultiWriter(hw, w)
		} else {
			hw = nil
		}

		length, err := io.Copy(writeTo, resp.Body)

		if err != nil {
			log.Printf("Failed to write from remote stream %v", err)
		} else {
			// A successful copy with a working hash
			// record means we should link in and record
			// our copy of this file.
			if hw != nil {
				_, err = hw.Finish()
				if err == nil {
					go recordBlobOwnership(meta.OID, length)
				}
			}
		}

		return
	}

	//if we got to this point, no node in the list actually had it
	log.Printf("Don't have hash file: %v and no remote nodes could help",
		meta.OID)
	w.WriteHeader(500)
	return
}

func doList(w http.ResponseWriter, req *http.Request) {
	w.WriteHeader(200)
	explen := getHash().Size() * 2
	filepath.Walk(*root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() && !strings.HasPrefix(info.Name(), "tmp") &&
			len(info.Name()) == explen {
			_, e := w.Write([]byte(info.Name() + "\n"))
			return e
		}
		return nil
	})
}

func doGet(w http.ResponseWriter, req *http.Request) {
	switch {
	case req.URL.Path == "/" && req.FormValue("oid") != "":
		http.ServeFile(w, req, hashFilename(req.FormValue("oid")))
	case req.URL.Path == "/" && req.FormValue("list") != "":
		doList(w, req)
	default:
		doGetUserDoc(w, req)
	}
}

func doDeleteOID(w http.ResponseWriter, req *http.Request) {
	oid := req.FormValue("oid")
	err := os.Remove(hashFilename(oid))
	if err == nil {
		w.WriteHeader(201)
	} else {
		w.WriteHeader(404)
		w.Write([]byte(err.Error()))
	}
}

func doDeleteUserDoc(w http.ResponseWriter, req *http.Request) {
	err := couchbase.Delete(req.URL.Path[1:])
	if err == nil {
		w.WriteHeader(201)
	} else {
		w.WriteHeader(404)
		w.Write([]byte(err.Error()))
	}
}

func doDelete(w http.ResponseWriter, req *http.Request) {
	switch {
	case req.URL.Path == "/" && req.FormValue("oid") != "":
		doDeleteOID(w, req)
	default:
		doDeleteUserDoc(w, req)
	}
}

func httpHandler(w http.ResponseWriter, req *http.Request) {
	defer req.Body.Close()
	switch req.Method {
	case "PUT":
		doPut(w, req)
	case "GET":
		doGet(w, req)
	case "DELETE":
		doDelete(w, req)
	default:
		w.WriteHeader(http.StatusMethodNotAllowed)
	}
}
<|endoftext|>


<commit_before>package git

// This file implements HTTP (both "smart" and "dumb") transport for the
// Git protocol.
import (
	"http"
	"regexp"
	"strings"
)

// HttpHandler provides an interface to both smart and dumb HTTP protocols.
type HttpHandler struct {
	Repo *Repo // the git repository to operate on
	// AllowDumb specifies whether or not to allow the "dumb" HTTP protocol.
	// This version of the protocol supports older clients, but can use
	// much more bandwidth than the "smart" version.
	AllowDumb bool
}

var routes = []struct {
	pattern *regexp.Regexp
	handler func(*Repo, http.ResponseWriter, *http.Request)
}{
	// POST-only
	{regexp.MustCompile("/git-upload-pack$"), uploadPack},
	{regexp.MustCompile("/git-receive-pack$"), receivePack},
	// GET-only
	{regexp.MustCompile("/info/refs$"), serveRefs},
	{regexp.MustCompile("/HEAD$"), serveText},
	{regexp.MustCompile("/objects/info/alternates$"), serveText},
	{regexp.MustCompile("/objects/info/http-alternates$"), serveText},
	{regexp.MustCompile("/objects/info/packs$"), servePacks},
	{regexp.MustCompile("/objects/info/[^/]*$"), serveText},
	// this last bracket expression should really be matched 38 times
	{regexp.MustCompile("/objects/info/[0-9a-f][0-9a-f]/[0-9a-f]+$"), serveLoose},
	// this one should be matched 40 times
	{regexp.MustCompile("/objects/pack/pack-[0-9a-f]+\\.pack$"), servePackFile},
	// same here
	{regexp.MustCompile("/objects/pack/pack-[0-9a-f]+\\.idx$"), serveIndex},
}

func (h *HttpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	for _, route := range routes {
		if route.pattern.MatchString(r.RawURL) {
			route.handler(h.Repo, w, r)
			break
		}
	}
}

func uploadPack(repo *Repo, w http.ResponseWriter, r *http.Request) {
}

func receivePack(repo *Repo, w http.ResponseWriter, r *http.Request) {
}

func serveRefs(repo *Repo, w http.ResponseWriter, r *http.Request) {
	service := r.FormValue("service")
	if service == "" {
		serveText(repo, w, r)
		return
	}

	if !strings.HasPrefix(service, "git-") || r.Method != "GET" {
		// forbidden
		return
	}

	w.Header().Set("Content-Type", "application/x-" + service + "-advertisement")
	repo.advertiseRefs(w)
}

func serveText(repo *Repo, w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	serveFile(repo, w, r)
}

func servePacks(repo *Repo, w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	serveFile(repo, w, r)
}

func serveLoose(repo *Repo, w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/x-git-loose-object")
	serveFile(repo, w, r)
}

func servePackFile(repo *Repo, w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/x-git-packed-objects")
	serveFile(repo, w, r)
}

func serveIndex(repo *Repo, w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/x-git-packed-objects-toc")
	serveFile(repo, w, r)
}

func serveFile(repo *Repo, w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		// forbidden?
	}

}
<commit_msg>Some transport fixes; git now waits for a pack<commit_after>package git

// This file implements HTTP (both "smart" and "dumb") transport for the
// Git protocol.
import (
	"http"
	"io"
	"os"
	"regexp"
	"strings"
)

// HttpHandler provides an interface to both smart and dumb HTTP protocols.
type HttpHandler struct {
	Repo *Repo // the git repository to operate on
	// AllowDumb specifies whether or not to allow the "dumb" HTTP protocol.
	// This version of the protocol supports older clients, but can use
	// much more bandwidth than the "smart" version.
	AllowDumb bool
}

var routes = []struct {
	pattern *regexp.Regexp
	handler func(*Repo, http.ResponseWriter, *http.Request)
}{
	// POST-only
	{regexp.MustCompile("/git-upload-pack$"), uploadPack},
	{regexp.MustCompile("/git-receive-pack$"), receivePack},
	// GET-only
	{regexp.MustCompile("/info/refs(\\?.*)?$"), serveRefs},
	{regexp.MustCompile("/HEAD$"), serveText},
	{regexp.MustCompile("/objects/info/alternates$"), serveText},
	{regexp.MustCompile("/objects/info/http-alternates$"), serveText},
	{regexp.MustCompile("/objects/info/packs$"), servePacks},
	{regexp.MustCompile("/objects/info/[^/]*$"), serveText},
	// this last bracket expression should really be matched 38 times
	{regexp.MustCompile("/objects/info/[0-9a-f][0-9a-f]/[0-9a-f]+$"), serveLoose},
	// this one should be matched 40 times
	{regexp.MustCompile("/objects/pack/pack-[0-9a-f]+\\.pack$"), servePackFile},
	// same here
	{regexp.MustCompile("/objects/pack/pack-[0-9a-f]+\\.idx$"), serveIndex},
}

func (h *HttpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	for _, route := range routes {
		if route.pattern.MatchString(r.RawURL) {
			route.handler(h.Repo, w, r)
			break
		}
	}
}

func uploadPack(repo *Repo, w http.ResponseWriter, r *http.Request) {
}

func receivePack(repo *Repo, w http.ResponseWriter, r *http.Request) {
}

func serveRefs(repo *Repo, w http.ResponseWriter, r *http.Request) {
	service := r.FormValue("service")
	if service == "" {
		serveText(repo, w, r)
		return
	}

	if !strings.HasPrefix(service, "git-") || r.Method != "GET" {
		// forbidden
		return
	}

	w.Header().Set("Content-Type", "application/x-" + service + "-advertisement")
	writePacket(w, []byte("# service=" + service))
	flush(w)
	repo.advertiseRefs(w)
}

func serveText(repo *Repo, w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	serveFile(repo, w, r)
}

func servePacks(repo *Repo, w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	serveFile(repo, w, r)
}

func serveLoose(repo *Repo, w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/x-git-loose-object")
	serveFile(repo, w, r)
}

func servePackFile(repo *Repo, w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/x-git-packed-objects")
	serveFile(repo, w, r)
}

func serveIndex(repo *Repo, w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/x-git-packed-objects-toc")
	serveFile(repo, w, r)
}

// http.ServeFile won't let us set our own Content-Type
func serveFile(repo *Repo, w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		// forbidden?
		return
	}
	name := repo.file(r.URL.Path)
	f, err := os.Open(name, os.O_RDONLY, 0)
	if err != nil {
		http.NotFound(w, r)
		return
	}
	defer f.Close()
	io.Copy(w, f)
}
<|endoftext|>


<commit_before>//go:generate go get github.com/jteeuwen/go-bindata
//go:generate go-bindata -nomemcopy -pkg hugo -prefix "assets" -o binary.go assets/...

// Package hugo makes the bridge between the static website generator Hugo
// and the webserver Caddy, also providing an administrative user interface.
package hugo

import (
	"bytes"
	"io/ioutil"
	"log"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"time"

	"github.com/hacdias/caddy-filemanager"
	"github.com/hacdias/caddy-filemanager/assets"
	"github.com/hacdias/caddy-filemanager/frontmatter"
	"github.com/hacdias/caddy-filemanager/utils/variables"
	"github.com/hacdias/caddy-hugo/utils/commands"
	"github.com/mholt/caddy/caddyhttp/httpserver"
	"github.com/robfig/cron"
	"github.com/spf13/hugo/parser"
)

// Hugo is hugo
type Hugo struct {
	Next        httpserver.Handler
	Config      *Config
	FileManager *filemanager.FileManager
}

// ServeHTTP determines if the request is for this plugin, and if all prerequisites are met.
func (h Hugo) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
	// If the site matches the baseURL
	if httpserver.Path(r.URL.Path).Matches(h.Config.BaseURL) {
		// Serve the hugo assets
		if httpserver.Path(r.URL.Path).Matches(h.Config.BaseURL + AssetsURL) {
			return serveAssets(w, r, h.Config)
		}

		// Serve the filemanager assets
		if httpserver.Path(r.URL.Path).Matches(h.Config.BaseURL + assets.BaseURL) {
			return h.FileManager.ServeHTTP(w, r)
		}

		// If the url matches exactly with /{admin}/settings/ serve that page
		// page variable isn't used here to avoid people using URLs like
		// "/{admin}/settings/something".
		if r.URL.Path == h.Config.BaseURL+"/settings/" || r.URL.Path == h.Config.BaseURL+"/settings" {
			var frontmatter string
			var err error

			if _, err = os.Stat(h.Config.Root + "config.yaml"); err == nil {
				frontmatter = "yaml"
			}

			if _, err = os.Stat(h.Config.Root + "config.json"); err == nil {
				frontmatter = "json"
			}

			if _, err = os.Stat(h.Config.Root + "config.toml"); err == nil {
				frontmatter = "toml"
			}

			http.Redirect(w, r, h.FileManager.Configs[0].AbsoluteURL()+"/config."+frontmatter, http.StatusTemporaryRedirect)
			return 0, nil
		}

		if r.Method == http.MethodPut && r.Header.Get("archetype") != "" {
			filename := r.Header.Get("Filename")
			archetype := r.Header.Get("archetype")

			if !strings.HasSuffix(filename, ".md") && !strings.HasSuffix(filename, ".markdown") {
				return h.FileManager.ServeHTTP(w, r)
			}

			filename = strings.Replace(r.URL.Path, h.Config.BaseURL+h.Config.WebDavURL+"/content/", "", 1)
			filename = filepath.Clean(filename)

			args := []string{"new", filename, "--kind", archetype}

			if err := commands.Run(h.Config.Hugo, args, h.Config.Root); err != nil {
				return http.StatusInternalServerError, err
			}

			return http.StatusCreated, nil
		}

		if canBeEdited(r.URL.Path) && r.Method == http.MethodPut {
			code, err := h.FileManager.ServeHTTP(w, r)

			if err != nil {
				return code, err
			}

			if r.Header.Get("Regenerate") == "true" {
				if err = h.Config.BeforePublish(
					r,
					&h.FileManager.Configs[0],
					h.FileManager.Configs[0].User,
				); err != nil {
					return http.StatusInternalServerError, err
				}

				RunHugo(h.Config, false)

				if err = h.Config.AfterPublish(
					r,
					&h.FileManager.Configs[0],
					h.FileManager.Configs[0].User,
				); err != nil {
					return http.StatusInternalServerError, err
				}
			}

			if r.Header.Get("Schedule") != "" {
				code, err = h.Schedule(w, r)
			}

			return code, err
		}

		return h.FileManager.ServeHTTP(w, r)
	}

	return h.Next.ServeHTTP(w, r)
}

// RunHugo is used to run the static website generator
func RunHugo(c *Config, force bool) {
	os.RemoveAll(c.Root + "public")

	// Prevent running if watching is enabled
	if b, pos := variables.StringInSlice("--watch", c.Args); b && !force {
		if len(c.Args) > pos && c.Args[pos+1] != "false" {
			return
		}

		if len(c.Args) == pos+1 {
			return
		}
	}

	if err := commands.Run(c.Hugo, c.Args, c.Root); err != nil {
		log.Println(err)
	}
}

// Schedule schedules a post to be published later
func (h Hugo) Schedule(w http.ResponseWriter, r *http.Request) (int, error) {
	t, err := time.Parse("2006-01-02T15:04", r.Header.Get("Schedule"))

	if err != nil {
		return http.StatusInternalServerError, err
	}

	format := rune(r.Header.Get("Rune")[0])
	scheduler := cron.New()
	scheduler.AddFunc(t.Format("05 04 15 02 01 *"), func() {
		filename := r.URL.Path
		filename = strings.Replace(filename, h.FileManager.Configs[0].WebDavURL, h.Config.Root, 1)
		filename = filepath.Clean(filename)

		raw, err := ioutil.ReadFile(filename)
		if err != nil {
			log.Println(err)
			return
		}

		buffer := bytes.NewBuffer(raw)
		page, err := parser.ReadFrom(buffer)

		if err != nil {
			log.Println(err)
			return
		}

		content := strings.TrimSpace(string(page.Content()))
		front, err := frontmatter.Unmarshal(page.FrontMatter())

		if err != nil {
			log.Println(err)
			return
		}

		kind := reflect.TypeOf(front)

		if kind == reflect.TypeOf(map[interface{}]interface{}{}) {
			delete(front.(map[interface{}]interface{}), "draft")
			delete(front.(map[interface{}]interface{}), "Draft")
		} else {
			delete(front.(map[string]interface{}), "draft")
			delete(front.(map[string]interface{}), "Draft")
		}

		fm, err := frontmatter.Marshal(front, format)

		if err != nil {
			log.Println(err)
			return
		}

		f := new(bytes.Buffer)
		f.Write(fm)
		f.Write([]byte(content))
		file := f.Bytes()

		if err = ioutil.WriteFile(filename, file, 0666); err != nil {
			return
		}

		RunHugo(h.Config, false)
	})
	scheduler.Start()

	return http.StatusOK, nil
}

func canBeEdited(name string) bool {
	extensions := [...]string{
		".md", ".markdown", ".mdown", ".mmark",
		".asciidoc", ".adoc", ".ad",
		".rst",
		".json", ".toml", ".yaml", ".csv", ".xml", ".rss", ".conf", ".ini",
		".tex", ".sty",
		".css", ".sass", ".scss",
		".js",
		".html",
		".txt", ".rtf",
		".sh", ".bash", ".ps1", ".bat", ".cmd",
		".php", ".pl", ".py",
		"Caddyfile",
		".c", ".cc", ".h", ".hh", ".cpp", ".hpp", ".f90",
		".f", ".bas", ".d", ".ada", ".nim", ".cr", ".java", ".cs", ".vala", ".vapi",
	}

	for _, extension := range extensions {
		if strings.HasSuffix(name, extension) {
			return true
		}
	}

	return false
}

// serveAssets provides the needed assets for the front-end
func serveAssets(w http.ResponseWriter, r *http.Request, c *Config) (int, error) {
	// gets the filename to be used with Assets function
	filename := strings.Replace(r.URL.Path, c.BaseURL+AssetsURL, "public", 1)
	file, err := Asset(filename)
	if err != nil {
		return http.StatusNotFound, nil
	}

	// Get the file extension and its mimetype
	extension := filepath.Ext(filename)
	mediatype := mime.TypeByExtension(extension)

	// Write the header with the Content-Type and write the file
	// content to the buffer
	w.Header().Set("Content-Type", mediatype)
	w.Write(file)
	return 200, nil
}
<commit_msg>update generate get<commit_after>//go:generate go get github.com/jteeuwen/go-bindata/go-bindata
//go:generate go-bindata -nomemcopy -pkg hugo -prefix "assets" -o binary.go assets/...

// Package hugo makes the bridge between the static website generator Hugo
// and the webserver Caddy, also providing an administrative user interface.
package hugo

import (
	"bytes"
	"io/ioutil"
	"log"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"time"

	"github.com/hacdias/caddy-filemanager"
	"github.com/hacdias/caddy-filemanager/assets"
	"github.com/hacdias/caddy-filemanager/frontmatter"
	"github.com/hacdias/caddy-filemanager/utils/variables"
	"github.com/hacdias/caddy-hugo/utils/commands"
	"github.com/mholt/caddy/caddyhttp/httpserver"
	"github.com/robfig/cron"
	"github.com/spf13/hugo/parser"
)

// Hugo is hugo
type Hugo struct {
	Next        httpserver.Handler
	Config      *Config
	FileManager *filemanager.FileManager
}

// ServeHTTP determines if the request is for this plugin, and if all prerequisites are met.
func (h Hugo) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
	// If the site matches the baseURL
	if httpserver.Path(r.URL.Path).Matches(h.Config.BaseURL) {
		// Serve the hugo assets
		if httpserver.Path(r.URL.Path).Matches(h.Config.BaseURL + AssetsURL) {
			return serveAssets(w, r, h.Config)
		}

		// Serve the filemanager assets
		if httpserver.Path(r.URL.Path).Matches(h.Config.BaseURL + assets.BaseURL) {
			return h.FileManager.ServeHTTP(w, r)
		}

		// If the url matches exactly with /{admin}/settings/ serve that page
		// page variable isn't used here to avoid people using URLs like
		// "/{admin}/settings/something".
		if r.URL.Path == h.Config.BaseURL+"/settings/" || r.URL.Path == h.Config.BaseURL+"/settings" {
			var frontmatter string
			var err error

			if _, err = os.Stat(h.Config.Root + "config.yaml"); err == nil {
				frontmatter = "yaml"
			}

			if _, err = os.Stat(h.Config.Root + "config.json"); err == nil {
				frontmatter = "json"
			}

			if _, err = os.Stat(h.Config.Root + "config.toml"); err == nil {
				frontmatter = "toml"
			}

			http.Redirect(w, r, h.FileManager.Configs[0].AbsoluteURL()+"/config."+frontmatter, http.StatusTemporaryRedirect)
			return 0, nil
		}

		if r.Method == http.MethodPut && r.Header.Get("archetype") != "" {
			filename := r.Header.Get("Filename")
			archetype := r.Header.Get("archetype")

			if !strings.HasSuffix(filename, ".md") && !strings.HasSuffix(filename, ".markdown") {
				return h.FileManager.ServeHTTP(w, r)
			}

			filename = strings.Replace(r.URL.Path, h.Config.BaseURL+h.Config.WebDavURL+"/content/", "", 1)
			filename = filepath.Clean(filename)

			args := []string{"new", filename, "--kind", archetype}

			if err := commands.Run(h.Config.Hugo, args, h.Config.Root); err != nil {
				return http.StatusInternalServerError, err
			}

			return http.StatusCreated, nil
		}

		if canBeEdited(r.URL.Path) && r.Method == http.MethodPut {
			code, err := h.FileManager.ServeHTTP(w, r)

			if err != nil {
				return code, err
			}

			if r.Header.Get("Regenerate") == "true" {
				if err = h.Config.BeforePublish(
					r,
					&h.FileManager.Configs[0],
					h.FileManager.Configs[0].User,
				); err != nil {
					return http.StatusInternalServerError, err
				}

				RunHugo(h.Config, false)

				if err = h.Config.AfterPublish(
					r,
					&h.FileManager.Configs[0],
					h.FileManager.Configs[0].User,
				); err != nil {
					return http.StatusInternalServerError, err
				}
			}

			if r.Header.Get("Schedule") != "" {
				code, err = h.Schedule(w, r)
			}

			return code, err
		}

		return h.FileManager.ServeHTTP(w, r)
	}

	return h.Next.ServeHTTP(w, r)
}

// RunHugo is used to run the static website generator
func RunHugo(c *Config, force bool) {
	os.RemoveAll(c.Root + "public")

	// Prevent running if watching is enabled
	if b, pos := variables.StringInSlice("--watch", c.Args); b && !force {
		if len(c.Args) > pos && c.Args[pos+1] != "false" {
			return
		}

		if len(c.Args) == pos+1 {
			return
		}
	}

	if err := commands.Run(c.Hugo, c.Args, c.Root); err != nil {
		log.Println(err)
	}
}

// Schedule schedules a post to be published later
func (h Hugo) Schedule(w http.ResponseWriter, r *http.Request) (int, error) {
	t, err := time.Parse("2006-01-02T15:04", r.Header.Get("Schedule"))

	if err != nil {
		return http.StatusInternalServerError, err
	}

	format := rune(r.Header.Get("Rune")[0])
	scheduler := cron.New()
	scheduler.AddFunc(t.Format("05 04 15 02 01 *"), func() {
		filename := r.URL.Path
		filename = strings.Replace(filename, h.FileManager.Configs[0].WebDavURL, h.Config.Root, 1)
		filename = filepath.Clean(filename)

		raw, err := ioutil.ReadFile(filename)
		if err != nil {
			log.Println(err)
			return
		}

		buffer := bytes.NewBuffer(raw)
		page, err := parser.ReadFrom(buffer)

		if err != nil {
			log.Println(err)
			return
		}

		content := strings.TrimSpace(string(page.Content()))
		front, err := frontmatter.Unmarshal(page.FrontMatter())

		if err != nil {
			log.Println(err)
			return
		}

		kind := reflect.TypeOf(front)

		if kind == reflect.TypeOf(map[interface{}]interface{}{}) {
			delete(front.(map[interface{}]interface{}), "draft")
			delete(front.(map[interface{}]interface{}), "Draft")
		} else {
			delete(front.(map[string]interface{}), "draft")
			delete(front.(map[string]interface{}), "Draft")
		}

		fm, err := frontmatter.Marshal(front, format)

		if err != nil {
			log.Println(err)
			return
		}

		f := new(bytes.Buffer)
		f.Write(fm)
		f.Write([]byte(content))
		file := f.Bytes()

		if err = ioutil.WriteFile(filename, file, 0666); err != nil {
			return
		}

		RunHugo(h.Config, false)
	})
	scheduler.Start()

	return http.StatusOK, nil
}

func canBeEdited(name string) bool {
	extensions := [...]string{
		".md", ".markdown", ".mdown", ".mmark",
		".asciidoc", ".adoc", ".ad",
		".rst",
		".json", ".toml", ".yaml", ".csv", ".xml", ".rss", ".conf", ".ini",
		".tex", ".sty",
		".css", ".sass", ".scss",
		".js",
		".html",
		".txt", ".rtf",
		".sh", ".bash", ".ps1", ".bat", ".cmd",
		".php", ".pl", ".py",
		"Caddyfile",
		".c", ".cc", ".h", ".hh", ".cpp", ".hpp", ".f90",
		".f", ".bas", ".d", ".ada", ".nim", ".cr", ".java", ".cs", ".vala", ".vapi",
	}

	for _, extension := range extensions {
		if strings.HasSuffix(name, extension) {
			return true
		}
	}

	return false
}

// serveAssets provides the needed assets for the front-end
func serveAssets(w http.ResponseWriter, r *http.Request, c *Config) (int, error) {
	// gets the filename to be used with Assets function
	filename := strings.Replace(r.URL.Path, c.BaseURL+AssetsURL, "public", 1)
	file, err := Asset(filename)
	if err != nil {
		return http.StatusNotFound, nil
	}

	// Get the file extension and its mimetype
	extension := filepath.Ext(filename)
	mediatype := mime.TypeByExtension(extension)

	// Write the header with the Content-Type and write the file
	// content to the buffer
	w.Header().Set("Content-Type", mediatype)
	w.Write(file)
	return 200, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"syscall"
	"time"

	"github.com/sevlyar/go-daemon"
)

var (
	signal = flag.String("z", "", "\n  zdaemon -z... reload -- restart the configuration\n\t\t  quit -- graceful shutdown\n")
)

var (
	done = make(chan struct{})
	quit = make(chan struct{})
	reload = make(chan struct{})
)

func main() {
	flag.Parse()
	daemon.AddCommand(daemon.StringFlag(signal, "quit"), syscall.SIGQUIT, TermHandler)
	daemon.AddCommand(daemon.StringFlag(signal, "reload"), syscall.SIGHUP, ReloadHandler)

	cntxt := &daemon.Context{
		PidFileName: "/var/run/zdaemon.pid",
		PidFilePerm: 0644,
		WorkDir:     "./",
		Umask:       027,
		Args:        []string{"[zdaemon]"},
	}
	if len(daemon.ActiveFlags()) > 0 {
		d, _ := cntxt.Search()
		if err != nil {
			fmt.Println("[WARNING] unable send signal to the daemon!")
		}
		daemon.SendCommands(d)
		return
	}
	d, err := cntxt.Reborn()
	if d != nil {
		return
	}
	if err != nil {
		os.Exit(1)
	}
	defer cntxt.Release()

	// Start daemon
	ticker := time.NewTicker(time.Minute)
	zeplic := true
	for zeplic {
		select {
		case <- ticker.C:
			go Writer()
		case <- quit:
			// Got a quit signal, stopping
			done <- struct{}{}
			zeplic = false
			ticker.Stop()
			return
/*		case <- reload:
			// Got a reload signal, reloading
			done <- struct{}{}
			zeplic = true
			return*/
		default:
			// No stop signal, continuing loop
		}
	}
}

// Writer writes a string into a file
func Writer() error {
	read, _ := ioutil.ReadFile("/root/test")
	contents := string(read)

	fileHandle, _ := os.Create("/root/stest")
	writer := bufio.NewWriter(fileHandle)
	defer fileHandle.Close()
	fmt.Fprintln(writer, contents)
	writer.Flush()

	return nil
}

// TermHandler is the struct for quit signal
func TermHandler(sig os.Signal) error {
	quit <- struct{}{}
	if sig == syscall.SIGQUIT {
		<-done
	}
	return nil
}

// ReloadHandler is the struct for reload signal
func ReloadHandler(sig os.Signal) error {
/*	reload <- struct{}{}
	if sig == syscall.SIGHUP {
		<-done
	}*/
	return nil
}
<commit_msg>Update zdaemon<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"syscall"
	"time"

	"github.com/sevlyar/go-daemon"
)

var (
	signal = flag.String("z", "", "\n  zdaemon -z... reload -- restart the configuration\n\t\t  quit -- graceful shutdown\n")
)

var (
	done = make(chan struct{})
	quit = make(chan struct{})
	reload = make(chan struct{})
)

func main() {
	flag.Parse()
	daemon.AddCommand(daemon.StringFlag(signal, "quit"), syscall.SIGQUIT, TermHandler)
	daemon.AddCommand(daemon.StringFlag(signal, "reload"), syscall.SIGHUP, ReloadHandler)

	cntxt := &daemon.Context{
		PidFileName: "/var/run/zdaemon.pid",
		PidFilePerm: 0644,
		WorkDir:     "./",
		Umask:       027,
		Args:        []string{"[zdaemon]"},
	}
	if len(daemon.ActiveFlags()) > 0 {
		d, err := cntxt.Search()
		if err != nil {
			fmt.Println("[WARNING] unable send signal to the daemon!")
		}
		daemon.SendCommands(d)
		return
	}
	d, err := cntxt.Reborn()
	if d != nil {
		return
	}
	if err != nil {
		os.Exit(1)
	}
	defer cntxt.Release()

	// Start daemon
	ticker := time.NewTicker(time.Minute)
	zeplic := true
	for zeplic {
		select {
		case <- ticker.C:
			go Writer()
		case <- quit:
			// Got a quit signal, stopping
			done <- struct{}{}
			zeplic = false
			ticker.Stop()
			return
/*		case <- reload:
			// Got a reload signal, reloading
			done <- struct{}{}
			zeplic = true
			return*/
		default:
			// No stop signal, continuing loop
		}
	}
}

// Writer writes a string into a file
func Writer() error {
	read, _ := ioutil.ReadFile("/root/test")
	contents := string(read)

	fileHandle, _ := os.Create("/root/stest")
	writer := bufio.NewWriter(fileHandle)
	defer fileHandle.Close()
	fmt.Fprintln(writer, contents)
	writer.Flush()

	return nil
}

// TermHandler is the struct for quit signal
func TermHandler(sig os.Signal) error {
	quit <- struct{}{}
	if sig == syscall.SIGQUIT {
		<-done
	}
	return nil
}

// ReloadHandler is the struct for reload signal
func ReloadHandler(sig os.Signal) error {
/*	reload <- struct{}{}
	if sig == syscall.SIGHUP {
		<-done
	}*/
	return nil
}
<|endoftext|>


<commit_before>package mount

import (
	"context"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/hanwen/go-fuse/v2/fuse"
	"math"
	"os"
	"sync"
)

type DirectoryHandleId uint64

type DirectoryHandle struct {
	isFinished    bool
	lastEntryName string
}

type DirectoryHandleToInode struct {
	// shares the file handle id sequencer with FileHandleToInode{nextFh}
	sync.Mutex
	dir2inode map[DirectoryHandleId]*DirectoryHandle
}

func NewDirectoryHandleToInode() *DirectoryHandleToInode {
	return &DirectoryHandleToInode{
		dir2inode: make(map[DirectoryHandleId]*DirectoryHandle),
	}
}

func (wfs *WFS) AcquireDirectoryHandle() (DirectoryHandleId, *DirectoryHandle) {
	wfs.fhmap.Lock()
	fh := wfs.fhmap.nextFh
	wfs.fhmap.nextFh++
	wfs.fhmap.Unlock()

	wfs.dhmap.Lock()
	defer wfs.dhmap.Unlock()
	dh := &DirectoryHandle{
		isFinished:    false,
		lastEntryName: "",
	}
	wfs.dhmap.dir2inode[DirectoryHandleId(fh)] = dh
	return DirectoryHandleId(fh), dh
}

func (wfs *WFS) GetDirectoryHandle(dhid DirectoryHandleId) *DirectoryHandle {
	wfs.dhmap.Lock()
	defer wfs.dhmap.Unlock()
	if dh, found := wfs.dhmap.dir2inode[dhid]; found {
		return dh
	}
	dh := &DirectoryHandle{
		isFinished:    false,
		lastEntryName: "",
	}

	wfs.dhmap.dir2inode[dhid] = dh
	return dh
}

func (wfs *WFS) ReleaseDirectoryHandle(dhid DirectoryHandleId) {
	wfs.dhmap.Lock()
	defer wfs.dhmap.Unlock()
	delete(wfs.dhmap.dir2inode, dhid)
}

// Directory handling

/** Open directory
 *
 * Unless the 'default_permissions' mount option is given,
 * this method should check if opendir is permitted for this
 * directory. Optionally opendir may also return an arbitrary
 * filehandle in the fuse_file_info structure, which will be
 * passed to readdir, releasedir and fsyncdir.
 */
func (wfs *WFS) OpenDir(cancel <-chan struct{}, input *fuse.OpenIn, out *fuse.OpenOut) (code fuse.Status) {
	if !wfs.inodeToPath.HasInode(input.NodeId) {
		return fuse.ENOENT
	}
	dhid, _ := wfs.AcquireDirectoryHandle()
	out.Fh = uint64(dhid)
	return fuse.OK
}

/** Release directory
 *
 * If the directory has been removed after the call to opendir, the
 * path parameter will be NULL.
 */
func (wfs *WFS) ReleaseDir(input *fuse.ReleaseIn) {
	wfs.ReleaseDirectoryHandle(DirectoryHandleId(input.Fh))
}

/** Synchronize directory contents
 *
 * If the directory has been removed after the call to opendir, the
 * path parameter will be NULL.
 *
 * If the datasync parameter is non-zero, then only the user data
 * should be flushed, not the meta data
 */
func (wfs *WFS) FsyncDir(cancel <-chan struct{}, input *fuse.FsyncIn) (code fuse.Status) {
	return fuse.OK
}

/** Read directory
 *
 * The filesystem may choose between two modes of operation:
 *
 * 1) The readdir implementation ignores the offset parameter, and
 * passes zero to the filler function's offset.  The filler
 * function will not return '1' (unless an error happens), so the
 * whole directory is read in a single readdir operation.
 *
 * 2) The readdir implementation keeps track of the offsets of the
 * directory entries.  It uses the offset parameter and always
 * passes non-zero offset to the filler function.  When the buffer
 * is full (or an error happens) the filler function will return
 * '1'.
 */
func (wfs *WFS) ReadDir(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) {
	return wfs.doReadDirectory(input, out, false)
}

func (wfs *WFS) ReadDirPlus(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) {
	return wfs.doReadDirectory(input, out, true)
}

func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPlusMode bool) fuse.Status {

	dh := wfs.GetDirectoryHandle(DirectoryHandleId(input.Fh))
	if dh.isFinished {
		if input.Offset == 0 {
			dh.isFinished = false
			dh.lastEntryName = ""
		} else {
			return fuse.OK
		}
	}

	isEarlyTerminated := false
	dirPath := wfs.inodeToPath.GetPath(input.NodeId)

	var dirEntry fuse.DirEntry
	if input.Offset == 0 && !isPlusMode {
		dirEntry.Ino = input.NodeId
		dirEntry.Name = "."
		dirEntry.Mode = toSystemMode(os.ModeDir)
		out.AddDirEntry(dirEntry)

		parentDir, _ := dirPath.DirAndName()
		parentInode := wfs.inodeToPath.GetInode(util.FullPath(parentDir))
		dirEntry.Ino = parentInode
		dirEntry.Name = ".."
		dirEntry.Mode = toSystemMode(os.ModeDir)
		out.AddDirEntry(dirEntry)

	}

	processEachEntryFn := func(entry *filer.Entry, isLast bool) bool {
		dirEntry.Name = entry.Name()
		inode := wfs.inodeToPath.GetInode(dirPath.Child(dirEntry.Name))
		dirEntry.Ino = inode
		dirEntry.Mode = toSystemMode(entry.Mode)
		if !isPlusMode {
			if !out.AddDirEntry(dirEntry) {
				isEarlyTerminated = true
				return false
			}
		} else {
			entryOut := out.AddDirLookupEntry(dirEntry)
			if entryOut == nil {
				isEarlyTerminated = true
				return false
			}
			wfs.outputFilerEntry(entryOut, inode, entry)
		}
		dh.lastEntryName = entry.Name()
		return true
	}

	if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath); err != nil {
		glog.Errorf("dir ReadDirAll %s: %v", dirPath, err)
		return fuse.EIO
	}
	listErr := wfs.metaCache.ListDirectoryEntries(context.Background(), dirPath, dh.lastEntryName, false, int64(math.MaxInt32), func(entry *filer.Entry) bool {
		return processEachEntryFn(entry, false)
	})
	if listErr != nil {
		glog.Errorf("list meta cache: %v", listErr)
		return fuse.EIO
	}
	if !isEarlyTerminated {
		dh.isFinished = true
	}

	return fuse.OK
}
<commit_msg>fix listing with correct inode<commit_after>package mount

import (
	"context"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
	"github.com/hanwen/go-fuse/v2/fuse"
	"math"
	"sync"
)

type DirectoryHandleId uint64

type DirectoryHandle struct {
	isFinished    bool
	lastEntryName string
}

type DirectoryHandleToInode struct {
	// shares the file handle id sequencer with FileHandleToInode{nextFh}
	sync.Mutex
	dir2inode map[DirectoryHandleId]*DirectoryHandle
}

func NewDirectoryHandleToInode() *DirectoryHandleToInode {
	return &DirectoryHandleToInode{
		dir2inode: make(map[DirectoryHandleId]*DirectoryHandle),
	}
}

func (wfs *WFS) AcquireDirectoryHandle() (DirectoryHandleId, *DirectoryHandle) {
	wfs.fhmap.Lock()
	fh := wfs.fhmap.nextFh
	wfs.fhmap.nextFh++
	wfs.fhmap.Unlock()

	wfs.dhmap.Lock()
	defer wfs.dhmap.Unlock()
	dh := &DirectoryHandle{
		isFinished:    false,
		lastEntryName: "",
	}
	wfs.dhmap.dir2inode[DirectoryHandleId(fh)] = dh
	return DirectoryHandleId(fh), dh
}

func (wfs *WFS) GetDirectoryHandle(dhid DirectoryHandleId) *DirectoryHandle {
	wfs.dhmap.Lock()
	defer wfs.dhmap.Unlock()
	if dh, found := wfs.dhmap.dir2inode[dhid]; found {
		return dh
	}
	dh := &DirectoryHandle{
		isFinished:    false,
		lastEntryName: "",
	}

	wfs.dhmap.dir2inode[dhid] = dh
	return dh
}

func (wfs *WFS) ReleaseDirectoryHandle(dhid DirectoryHandleId) {
	wfs.dhmap.Lock()
	defer wfs.dhmap.Unlock()
	delete(wfs.dhmap.dir2inode, dhid)
}

// Directory handling

/** Open directory
 *
 * Unless the 'default_permissions' mount option is given,
 * this method should check if opendir is permitted for this
 * directory. Optionally opendir may also return an arbitrary
 * filehandle in the fuse_file_info structure, which will be
 * passed to readdir, releasedir and fsyncdir.
 */
func (wfs *WFS) OpenDir(cancel <-chan struct{}, input *fuse.OpenIn, out *fuse.OpenOut) (code fuse.Status) {
	if !wfs.inodeToPath.HasInode(input.NodeId) {
		return fuse.ENOENT
	}
	dhid, _ := wfs.AcquireDirectoryHandle()
	out.Fh = uint64(dhid)
	return fuse.OK
}

/** Release directory
 *
 * If the directory has been removed after the call to opendir, the
 * path parameter will be NULL.
 */
func (wfs *WFS) ReleaseDir(input *fuse.ReleaseIn) {
	wfs.ReleaseDirectoryHandle(DirectoryHandleId(input.Fh))
}

/** Synchronize directory contents
 *
 * If the directory has been removed after the call to opendir, the
 * path parameter will be NULL.
 *
 * If the datasync parameter is non-zero, then only the user data
 * should be flushed, not the meta data
 */
func (wfs *WFS) FsyncDir(cancel <-chan struct{}, input *fuse.FsyncIn) (code fuse.Status) {
	return fuse.OK
}

/** Read directory
 *
 * The filesystem may choose between two modes of operation:
 *
 * 1) The readdir implementation ignores the offset parameter, and
 * passes zero to the filler function's offset.  The filler
 * function will not return '1' (unless an error happens), so the
 * whole directory is read in a single readdir operation.
 *
 * 2) The readdir implementation keeps track of the offsets of the
 * directory entries.  It uses the offset parameter and always
 * passes non-zero offset to the filler function.  When the buffer
 * is full (or an error happens) the filler function will return
 * '1'.
 */
func (wfs *WFS) ReadDir(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) {
	return wfs.doReadDirectory(input, out, false)
}

func (wfs *WFS) ReadDirPlus(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) {
	return wfs.doReadDirectory(input, out, true)
}

func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPlusMode bool) fuse.Status {

	dh := wfs.GetDirectoryHandle(DirectoryHandleId(input.Fh))
	if dh.isFinished {
		if input.Offset == 0 {
			dh.isFinished = false
			dh.lastEntryName = ""
		} else {
			return fuse.OK
		}
	}

	isEarlyTerminated := false
	dirPath := wfs.inodeToPath.GetPath(input.NodeId)

	var dirEntry fuse.DirEntry
	if input.Offset == 0 {
		if !isPlusMode {
			out.AddDirEntry(fuse.DirEntry{Mode: fuse.S_IFDIR, Name: "."})
			out.AddDirEntry(fuse.DirEntry{Mode: fuse.S_IFDIR, Name: ".."})
		} else {
			out.AddDirLookupEntry(fuse.DirEntry{Mode: fuse.S_IFDIR, Name: "."})
			out.AddDirLookupEntry(fuse.DirEntry{Mode: fuse.S_IFDIR, Name: ".."})
		}
	}

	processEachEntryFn := func(entry *filer.Entry, isLast bool) bool {
		dirEntry.Name = entry.Name()
		dirEntry.Mode = toSystemMode(entry.Mode)
		if !isPlusMode {
			inode := wfs.inodeToPath.Lookup(dirPath.Child(dirEntry.Name), entry.IsDirectory(), false)
			dirEntry.Ino = inode
			if !out.AddDirEntry(dirEntry) {
				isEarlyTerminated = true
				return false
			}
		} else {
			inode := wfs.inodeToPath.Lookup(dirPath.Child(dirEntry.Name), entry.IsDirectory(), true)
			dirEntry.Ino = inode
			entryOut := out.AddDirLookupEntry(dirEntry)
			if entryOut == nil {
				isEarlyTerminated = true
				return false
			}
			wfs.outputFilerEntry(entryOut, inode, entry)
		}
		dh.lastEntryName = entry.Name()
		return true
	}

	if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath); err != nil {
		glog.Errorf("dir ReadDirAll %s: %v", dirPath, err)
		return fuse.EIO
	}
	listErr := wfs.metaCache.ListDirectoryEntries(context.Background(), dirPath, dh.lastEntryName, false, int64(math.MaxInt32), func(entry *filer.Entry) bool {
		return processEachEntryFn(entry, false)
	})
	if listErr != nil {
		glog.Errorf("list meta cache: %v", listErr)
		return fuse.EIO
	}
	if !isEarlyTerminated {
		dh.isFinished = true
	}

	return fuse.OK
}
<|endoftext|>


<commit_before>package storage

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/stats"
	"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/google/uuid"
)

type DiskLocation struct {
	Directory              string
	DirectoryUuid          string
	IdxDirectory           string
	DiskType               types.DiskType
	MaxVolumeCount         int
	OriginalMaxVolumeCount int
	MinFreeSpace           util.MinFreeSpace
	volumes                map[needle.VolumeId]*Volume
	volumesLock            sync.RWMutex

	// erasure coding
	ecVolumes     map[needle.VolumeId]*erasure_coding.EcVolume
	ecVolumesLock sync.RWMutex

	isDiskSpaceLow bool
}

func GenerateDirUuid(dir string) (dirUuidString string, err error) {
	glog.V(1).Infof("Getting uuid of volume directory:%s", dir)
	dirUuidString = ""
	fileName := dir + "/vol_dir.uuid"
	if !util.FileExists(fileName) {
		dirUuid, _ := uuid.NewRandom()
		dirUuidString = dirUuid.String()
		writeErr := util.WriteFile(fileName, []byte(dirUuidString), 0644)
		if writeErr != nil {
			glog.Warningf("failed to write uuid to %s : %v", fileName, writeErr)
			return "", fmt.Errorf("failed to write uuid to %s : %v", fileName, writeErr)
		}
	} else {
		uuidData, readErr := os.ReadFile(fileName)
		if readErr != nil {
			glog.Warningf("failed to read uuid from %s : %v", fileName, readErr)
			return "", fmt.Errorf("failed to read uuid from %s : %v", fileName, readErr)
		}
		dirUuidString = string(uuidData)
	}
	return dirUuidString, nil
}

func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpace util.MinFreeSpace, idxDir string, diskType types.DiskType) *DiskLocation {
	dir = util.ResolvePath(dir)
	if idxDir == "" {
		idxDir = dir
	} else {
		idxDir = util.ResolvePath(idxDir)
	}
	dirUuid, _ := GenerateDirUuid(dir)
	location := &DiskLocation{
		Directory:              dir,
		DirectoryUuid:          dirUuid,
		IdxDirectory:           idxDir,
		DiskType:               diskType,
		MaxVolumeCount:         maxVolumeCount,
		OriginalMaxVolumeCount: maxVolumeCount,
		MinFreeSpace:           minFreeSpace,
	}
	location.volumes = make(map[needle.VolumeId]*Volume)
	location.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume)
	go location.CheckDiskSpace()
	return location
}

func volumeIdFromFileName(filename string) (needle.VolumeId, string, error) {
	if isValidVolume(filename) {
		base := filename[:len(filename)-4]
		collection, volumeId, err := parseCollectionVolumeId(base)
		return volumeId, collection, err
	}

	return 0, "", fmt.Errorf("file is not a volume: %s", filename)
}

func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeId, err error) {
	i := strings.LastIndex(base, "_")
	if i > 0 {
		collection, base = base[0:i], base[i+1:]
	}
	vol, err := needle.NewVolumeId(base)
	return collection, vol, err
}

func isValidVolume(basename string) bool {
	return strings.HasSuffix(basename, ".idx") || strings.HasSuffix(basename, ".vif")
}

func getValidVolumeName(basename string) string {
	if isValidVolume(basename) {
		return basename[:len(basename)-4]
	}
	return ""
}

func (l *DiskLocation) loadExistingVolume(dirEntry os.DirEntry, needleMapKind NeedleMapKind, skipIfEcVolumesExists bool) bool {
	basename := dirEntry.Name()
	if dirEntry.IsDir() {
		return false
	}
	volumeName := getValidVolumeName(basename)
	if volumeName == "" {
		return false
	}

	// skip if ec volumes exists
	if skipIfEcVolumesExists {
		if util.FileExists(l.Directory + "/" + volumeName + ".ecx") {
			return false
		}
	}

	// check for incomplete volume
	noteFile := l.Directory + "/" + volumeName + ".note"
	if util.FileExists(noteFile) {
		note, _ := os.ReadFile(noteFile)
		glog.Warningf("volume %s was not completed: %s", volumeName, string(note))
		removeVolumeFiles(l.Directory + "/" + volumeName)
		removeVolumeFiles(l.IdxDirectory + "/" + volumeName)
		return false
	}

	// parse out collection, volume id
	vid, collection, err := volumeIdFromFileName(basename)
	if err != nil {
		glog.Warningf("get volume id failed, %s, err : %s", volumeName, err)
		return false
	}

	// avoid loading one volume more than once
	l.volumesLock.RLock()
	_, found := l.volumes[vid]
	l.volumesLock.RUnlock()
	if found {
		glog.V(1).Infof("loaded volume, %v", vid)
		return true
	}

	// load the volume
	v, e := NewVolume(l.Directory, l.IdxDirectory, collection, vid, needleMapKind, nil, nil, 0, 0)
	if e != nil {
		glog.V(0).Infof("new volume %s error %s", volumeName, e)
		return false
	}

	l.SetVolume(vid, v)

	size, _, _ := v.FileStat()
	glog.V(0).Infof("data file %s, replication=%s v=%d size=%d ttl=%s",
		l.Directory+"/"+volumeName+".dat", v.ReplicaPlacement, v.Version(), size, v.Ttl.String())
	return true
}

func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapKind, concurrency int) {

	task_queue := make(chan os.DirEntry, 10*concurrency)
	go func() {
		foundVolumeNames := make(map[string]bool)
		if dirEntries, err := os.ReadDir(l.Directory); err == nil {
			for _, entry := range dirEntries {
				volumeName := getValidVolumeName(entry.Name())
				if volumeName == "" {
					continue
				}
				if _, found := foundVolumeNames[volumeName]; !found {
					foundVolumeNames[volumeName] = true
					task_queue <- entry
				}
			}
		}
		close(task_queue)
	}()

	var wg sync.WaitGroup
	for workerNum := 0; workerNum < concurrency; workerNum++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for fi := range task_queue {
				_ = l.loadExistingVolume(fi, needleMapKind, true)
			}
		}()
	}
	wg.Wait()

}

func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapKind) {

	l.concurrentLoadingVolumes(needleMapKind, 10)
	glog.V(0).Infof("Store started on dir: %s with %d volumes max %d", l.Directory, len(l.volumes), l.MaxVolumeCount)

	l.loadAllEcShards()
	glog.V(0).Infof("Store started on dir: %s with %d ec shards", l.Directory, len(l.ecVolumes))

}

func (l *DiskLocation) DeleteCollectionFromDiskLocation(collection string) (e error) {

	l.volumesLock.Lock()
	delVolsMap := l.unmountVolumeByCollection(collection)
	l.volumesLock.Unlock()

	l.ecVolumesLock.Lock()
	delEcVolsMap := l.unmountEcVolumeByCollection(collection)
	l.ecVolumesLock.Unlock()

	errChain := make(chan error, 2)
	var wg sync.WaitGroup
	wg.Add(2)
	go func() {
		for _, v := range delVolsMap {
			if err := v.Destroy(); err != nil {
				errChain <- err
			}
		}
		wg.Done()
	}()

	go func() {
		for _, v := range delEcVolsMap {
			v.Destroy()
		}
		wg.Done()
	}()

	go func() {
		wg.Wait()
		close(errChain)
	}()

	errBuilder := strings.Builder{}
	for err := range errChain {
		errBuilder.WriteString(err.Error())
		errBuilder.WriteString("; ")
	}
	if errBuilder.Len() > 0 {
		e = fmt.Errorf(errBuilder.String())
	}

	return
}

func (l *DiskLocation) deleteVolumeById(vid needle.VolumeId) (found bool, e error) {
	v, ok := l.volumes[vid]
	if !ok {
		return
	}
	e = v.Destroy()
	if e != nil {
		return
	}
	found = true
	delete(l.volumes, vid)
	return
}

func (l *DiskLocation) LoadVolume(vid needle.VolumeId, needleMapKind NeedleMapKind) bool {
	if fileInfo, found := l.LocateVolume(vid); found {
		return l.loadExistingVolume(fileInfo, needleMapKind, false)
	}
	return false
}

var ErrVolumeNotFound = fmt.Errorf("volume not found")

func (l *DiskLocation) DeleteVolume(vid needle.VolumeId) error {
	l.volumesLock.Lock()
	defer l.volumesLock.Unlock()

	_, ok := l.volumes[vid]
	if !ok {
		return ErrVolumeNotFound
	}
	_, err := l.deleteVolumeById(vid)
	return err
}

func (l *DiskLocation) UnloadVolume(vid needle.VolumeId) error {
	l.volumesLock.Lock()
	defer l.volumesLock.Unlock()

	v, ok := l.volumes[vid]
	if !ok {
		return ErrVolumeNotFound
	}
	v.Close()
	delete(l.volumes, vid)
	return nil
}

func (l *DiskLocation) unmountVolumeByCollection(collectionName string) map[needle.VolumeId]*Volume {
	deltaVols := make(map[needle.VolumeId]*Volume, 0)
	for k, v := range l.volumes {
		if v.Collection == collectionName && !v.isCompacting && !v.isCommitCompacting {
			deltaVols[k] = v
		}
	}

	for k := range deltaVols {
		delete(l.volumes, k)
	}
	return deltaVols
}

func (l *DiskLocation) SetVolume(vid needle.VolumeId, volume *Volume) {
	l.volumesLock.Lock()
	defer l.volumesLock.Unlock()

	l.volumes[vid] = volume
	volume.location = l
}

func (l *DiskLocation) FindVolume(vid needle.VolumeId) (*Volume, bool) {
	l.volumesLock.RLock()
	defer l.volumesLock.RUnlock()

	v, ok := l.volumes[vid]
	return v, ok
}

func (l *DiskLocation) VolumesLen() int {
	l.volumesLock.RLock()
	defer l.volumesLock.RUnlock()

	return len(l.volumes)
}

func (l *DiskLocation) SetStopping() {
	l.volumesLock.Lock()
	for _, v := range l.volumes {
		v.SetStopping()
	}
	l.volumesLock.Unlock()

	return
}

func (l *DiskLocation) Close() {
	l.volumesLock.Lock()
	for _, v := range l.volumes {
		v.Close()
	}
	l.volumesLock.Unlock()

	l.ecVolumesLock.Lock()
	for _, ecVolume := range l.ecVolumes {
		ecVolume.Close()
	}
	l.ecVolumesLock.Unlock()

	return
}

func (l *DiskLocation) LocateVolume(vid needle.VolumeId) (os.DirEntry, bool) {
	// println("LocateVolume", vid, "on", l.Directory)
	if dirEntries, err := os.ReadDir(l.Directory); err == nil {
		for _, entry := range dirEntries {
			// println("checking", entry.Name(), "...")
			volId, _, err := volumeIdFromFileName(entry.Name())
			// println("volId", volId, "err", err)
			if vid == volId && err == nil {
				return entry, true
			}
		}
	}

	return nil, false
}

func (l *DiskLocation) UnUsedSpace(volumeSizeLimit uint64) (unUsedSpace uint64) {

	l.volumesLock.RLock()
	defer l.volumesLock.RUnlock()

	for _, vol := range l.volumes {
		if vol.IsReadOnly() {
			continue
		}
		datSize, idxSize, _ := vol.FileStat()
		unUsedSpace += volumeSizeLimit - (datSize + idxSize)
	}

	return
}

func (l *DiskLocation) CheckDiskSpace() {
	for {
		if dir, e := filepath.Abs(l.Directory); e == nil {
			s := stats.NewDiskStatus(dir)
			stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "all").Set(float64(s.All))
			stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "used").Set(float64(s.Used))
			stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "free").Set(float64(s.Free))

			isLow, desc := l.MinFreeSpace.IsLow(s.Free, s.PercentFree)
			if isLow != l.isDiskSpaceLow {
				l.isDiskSpaceLow = !l.isDiskSpaceLow
			}

			logLevel := glog.Level(4)
			if l.isDiskSpaceLow {
				logLevel = glog.Level(0)
			}

			glog.V(logLevel).Infof("dir %s %s", dir, desc)
		}
		time.Sleep(time.Minute)
	}

}
<commit_msg>fix 3238: handle errors for GenerateDirUuid method<commit_after>package storage

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/stats"
	"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/google/uuid"
)

type DiskLocation struct {
	Directory              string
	DirectoryUuid          string
	IdxDirectory           string
	DiskType               types.DiskType
	MaxVolumeCount         int
	OriginalMaxVolumeCount int
	MinFreeSpace           util.MinFreeSpace
	volumes                map[needle.VolumeId]*Volume
	volumesLock            sync.RWMutex

	// erasure coding
	ecVolumes     map[needle.VolumeId]*erasure_coding.EcVolume
	ecVolumesLock sync.RWMutex

	isDiskSpaceLow bool
}

func GenerateDirUuid(dir string) (dirUuidString string, err error) {
	glog.V(1).Infof("Getting uuid of volume directory:%s", dir)
	dirUuidString = ""
	fileName := dir + "/vol_dir.uuid"
	if !util.FileExists(fileName) {
		dirUuid, _ := uuid.NewRandom()
		dirUuidString = dirUuid.String()
		writeErr := util.WriteFile(fileName, []byte(dirUuidString), 0644)
		if writeErr != nil {
			return "", fmt.Errorf("failed to write uuid to %s : %v", fileName, writeErr)
		}
	} else {
		uuidData, readErr := os.ReadFile(fileName)
		if readErr != nil {
			return "", fmt.Errorf("failed to read uuid from %s : %v", fileName, readErr)
		}
		dirUuidString = string(uuidData)
	}
	return dirUuidString, nil
}

func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpace util.MinFreeSpace, idxDir string, diskType types.DiskType) *DiskLocation {
	dir = util.ResolvePath(dir)
	if idxDir == "" {
		idxDir = dir
	} else {
		idxDir = util.ResolvePath(idxDir)
	}
	dirUuid, err := GenerateDirUuid(dir)
	if err != nil {
		glog.Fatalf("cannot generate uuid of dir %s: %v", dir, err)
	}
	location := &DiskLocation{
		Directory:              dir,
		DirectoryUuid:          dirUuid,
		IdxDirectory:           idxDir,
		DiskType:               diskType,
		MaxVolumeCount:         maxVolumeCount,
		OriginalMaxVolumeCount: maxVolumeCount,
		MinFreeSpace:           minFreeSpace,
	}
	location.volumes = make(map[needle.VolumeId]*Volume)
	location.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume)
	go location.CheckDiskSpace()
	return location
}

func volumeIdFromFileName(filename string) (needle.VolumeId, string, error) {
	if isValidVolume(filename) {
		base := filename[:len(filename)-4]
		collection, volumeId, err := parseCollectionVolumeId(base)
		return volumeId, collection, err
	}

	return 0, "", fmt.Errorf("file is not a volume: %s", filename)
}

func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeId, err error) {
	i := strings.LastIndex(base, "_")
	if i > 0 {
		collection, base = base[0:i], base[i+1:]
	}
	vol, err := needle.NewVolumeId(base)
	return collection, vol, err
}

func isValidVolume(basename string) bool {
	return strings.HasSuffix(basename, ".idx") || strings.HasSuffix(basename, ".vif")
}

func getValidVolumeName(basename string) string {
	if isValidVolume(basename) {
		return basename[:len(basename)-4]
	}
	return ""
}

func (l *DiskLocation) loadExistingVolume(dirEntry os.DirEntry, needleMapKind NeedleMapKind, skipIfEcVolumesExists bool) bool {
	basename := dirEntry.Name()
	if dirEntry.IsDir() {
		return false
	}
	volumeName := getValidVolumeName(basename)
	if volumeName == "" {
		return false
	}

	// skip if ec volumes exists
	if skipIfEcVolumesExists {
		if util.FileExists(l.Directory + "/" + volumeName + ".ecx") {
			return false
		}
	}

	// check for incomplete volume
	noteFile := l.Directory + "/" + volumeName + ".note"
	if util.FileExists(noteFile) {
		note, _ := os.ReadFile(noteFile)
		glog.Warningf("volume %s was not completed: %s", volumeName, string(note))
		removeVolumeFiles(l.Directory + "/" + volumeName)
		removeVolumeFiles(l.IdxDirectory + "/" + volumeName)
		return false
	}

	// parse out collection, volume id
	vid, collection, err := volumeIdFromFileName(basename)
	if err != nil {
		glog.Warningf("get volume id failed, %s, err : %s", volumeName, err)
		return false
	}

	// avoid loading one volume more than once
	l.volumesLock.RLock()
	_, found := l.volumes[vid]
	l.volumesLock.RUnlock()
	if found {
		glog.V(1).Infof("loaded volume, %v", vid)
		return true
	}

	// load the volume
	v, e := NewVolume(l.Directory, l.IdxDirectory, collection, vid, needleMapKind, nil, nil, 0, 0)
	if e != nil {
		glog.V(0).Infof("new volume %s error %s", volumeName, e)
		return false
	}

	l.SetVolume(vid, v)

	size, _, _ := v.FileStat()
	glog.V(0).Infof("data file %s, replication=%s v=%d size=%d ttl=%s",
		l.Directory+"/"+volumeName+".dat", v.ReplicaPlacement, v.Version(), size, v.Ttl.String())
	return true
}

func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapKind, concurrency int) {

	task_queue := make(chan os.DirEntry, 10*concurrency)
	go func() {
		foundVolumeNames := make(map[string]bool)
		if dirEntries, err := os.ReadDir(l.Directory); err == nil {
			for _, entry := range dirEntries {
				volumeName := getValidVolumeName(entry.Name())
				if volumeName == "" {
					continue
				}
				if _, found := foundVolumeNames[volumeName]; !found {
					foundVolumeNames[volumeName] = true
					task_queue <- entry
				}
			}
		}
		close(task_queue)
	}()

	var wg sync.WaitGroup
	for workerNum := 0; workerNum < concurrency; workerNum++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for fi := range task_queue {
				_ = l.loadExistingVolume(fi, needleMapKind, true)
			}
		}()
	}
	wg.Wait()

}

func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapKind) {

	l.concurrentLoadingVolumes(needleMapKind, 10)
	glog.V(0).Infof("Store started on dir: %s with %d volumes max %d", l.Directory, len(l.volumes), l.MaxVolumeCount)

	l.loadAllEcShards()
	glog.V(0).Infof("Store started on dir: %s with %d ec shards", l.Directory, len(l.ecVolumes))

}

func (l *DiskLocation) DeleteCollectionFromDiskLocation(collection string) (e error) {

	l.volumesLock.Lock()
	delVolsMap := l.unmountVolumeByCollection(collection)
	l.volumesLock.Unlock()

	l.ecVolumesLock.Lock()
	delEcVolsMap := l.unmountEcVolumeByCollection(collection)
	l.ecVolumesLock.Unlock()

	errChain := make(chan error, 2)
	var wg sync.WaitGroup
	wg.Add(2)
	go func() {
		for _, v := range delVolsMap {
			if err := v.Destroy(); err != nil {
				errChain <- err
			}
		}
		wg.Done()
	}()

	go func() {
		for _, v := range delEcVolsMap {
			v.Destroy()
		}
		wg.Done()
	}()

	go func() {
		wg.Wait()
		close(errChain)
	}()

	errBuilder := strings.Builder{}
	for err := range errChain {
		errBuilder.WriteString(err.Error())
		errBuilder.WriteString("; ")
	}
	if errBuilder.Len() > 0 {
		e = fmt.Errorf(errBuilder.String())
	}

	return
}

func (l *DiskLocation) deleteVolumeById(vid needle.VolumeId) (found bool, e error) {
	v, ok := l.volumes[vid]
	if !ok {
		return
	}
	e = v.Destroy()
	if e != nil {
		return
	}
	found = true
	delete(l.volumes, vid)
	return
}

func (l *DiskLocation) LoadVolume(vid needle.VolumeId, needleMapKind NeedleMapKind) bool {
	if fileInfo, found := l.LocateVolume(vid); found {
		return l.loadExistingVolume(fileInfo, needleMapKind, false)
	}
	return false
}

var ErrVolumeNotFound = fmt.Errorf("volume not found")

func (l *DiskLocation) DeleteVolume(vid needle.VolumeId) error {
	l.volumesLock.Lock()
	defer l.volumesLock.Unlock()

	_, ok := l.volumes[vid]
	if !ok {
		return ErrVolumeNotFound
	}
	_, err := l.deleteVolumeById(vid)
	return err
}

func (l *DiskLocation) UnloadVolume(vid needle.VolumeId) error {
	l.volumesLock.Lock()
	defer l.volumesLock.Unlock()

	v, ok := l.volumes[vid]
	if !ok {
		return ErrVolumeNotFound
	}
	v.Close()
	delete(l.volumes, vid)
	return nil
}

func (l *DiskLocation) unmountVolumeByCollection(collectionName string) map[needle.VolumeId]*Volume {
	deltaVols := make(map[needle.VolumeId]*Volume, 0)
	for k, v := range l.volumes {
		if v.Collection == collectionName && !v.isCompacting && !v.isCommitCompacting {
			deltaVols[k] = v
		}
	}

	for k := range deltaVols {
		delete(l.volumes, k)
	}
	return deltaVols
}

func (l *DiskLocation) SetVolume(vid needle.VolumeId, volume *Volume) {
	l.volumesLock.Lock()
	defer l.volumesLock.Unlock()

	l.volumes[vid] = volume
	volume.location = l
}

func (l *DiskLocation) FindVolume(vid needle.VolumeId) (*Volume, bool) {
	l.volumesLock.RLock()
	defer l.volumesLock.RUnlock()

	v, ok := l.volumes[vid]
	return v, ok
}

func (l *DiskLocation) VolumesLen() int {
	l.volumesLock.RLock()
	defer l.volumesLock.RUnlock()

	return len(l.volumes)
}

func (l *DiskLocation) SetStopping() {
	l.volumesLock.Lock()
	for _, v := range l.volumes {
		v.SetStopping()
	}
	l.volumesLock.Unlock()

	return
}

func (l *DiskLocation) Close() {
	l.volumesLock.Lock()
	for _, v := range l.volumes {
		v.Close()
	}
	l.volumesLock.Unlock()

	l.ecVolumesLock.Lock()
	for _, ecVolume := range l.ecVolumes {
		ecVolume.Close()
	}
	l.ecVolumesLock.Unlock()

	return
}

func (l *DiskLocation) LocateVolume(vid needle.VolumeId) (os.DirEntry, bool) {
	// println("LocateVolume", vid, "on", l.Directory)
	if dirEntries, err := os.ReadDir(l.Directory); err == nil {
		for _, entry := range dirEntries {
			// println("checking", entry.Name(), "...")
			volId, _, err := volumeIdFromFileName(entry.Name())
			// println("volId", volId, "err", err)
			if vid == volId && err == nil {
				return entry, true
			}
		}
	}

	return nil, false
}

func (l *DiskLocation) UnUsedSpace(volumeSizeLimit uint64) (unUsedSpace uint64) {

	l.volumesLock.RLock()
	defer l.volumesLock.RUnlock()

	for _, vol := range l.volumes {
		if vol.IsReadOnly() {
			continue
		}
		datSize, idxSize, _ := vol.FileStat()
		unUsedSpace += volumeSizeLimit - (datSize + idxSize)
	}

	return
}

func (l *DiskLocation) CheckDiskSpace() {
	for {
		if dir, e := filepath.Abs(l.Directory); e == nil {
			s := stats.NewDiskStatus(dir)
			stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "all").Set(float64(s.All))
			stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "used").Set(float64(s.Used))
			stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "free").Set(float64(s.Free))

			isLow, desc := l.MinFreeSpace.IsLow(s.Free, s.PercentFree)
			if isLow != l.isDiskSpaceLow {
				l.isDiskSpaceLow = !l.isDiskSpaceLow
			}

			logLevel := glog.Level(4)
			if l.isDiskSpaceLow {
				logLevel = glog.Level(0)
			}

			glog.V(logLevel).Infof("dir %s %s", dir, desc)
		}
		time.Sleep(time.Minute)
	}

}
<|endoftext|>