<commit_before>package gocache

import(
    "fmt"
    "net/http"
    "errors"
    "os"
    "strings"
    "log"
    "encoding/json"
    "bytes"
    "io/ioutil"
    "path"
    "io"
)

type Entry struct{
    Response *http.Response
    Content string
}

func main(){
    fmt.Println("Testing web cache");
    url := "http://blog.golang.org/error-handling-and-go"
    Get(url)
}

func Get(url string) (resp *http.Response, err error) {

    resp, err = getFromCache(url)

    if err != nil{
        //log.Println(url,"could not be served out of cache.", err)
        return getFromWeb(url)
    }

    //log.Print("Request to ", url, " served out of cache")
    return
}

// Fetch contents of url from web and write to cache
func getFromWeb(url string) (resp *http.Response, err error){
    resp, err = http.Get(url)

    if err != nil{
        return
    }
    var resp2 http.Response
    resp2 = *resp

    // Draining the body

    body, err := ioutil.ReadAll(resp.Body)

    if err != nil {
        log.Print("Reading response body went bad. ", err);

    }

    writeToCache(url, body, &resp2)

    resp.Body = nopCloser{bytes.NewBufferString(string(body))}

    return
}

func writeToCache(url string, body []byte, resp *http.Response){

    cacheEntry := generateCacheEntry(resp, body)

    b, err := json.Marshal(cacheEntry)

    if err != nil {
        log.Println("Could not marshal response ", err)
        return
    }

    filename := getFilePath(url)

    // Set up any directory needed to write the file.
    // e.g. vg.no/first/second/file will be stored as
    // cache/first/second/file

    err = createDirectories(filename)
    if err != nil{
        pe, _ := err.(*os.PathError)

        if ! strings.Contains(pe.Error(),"file exists") {
            log.Println("Could not create directories ", err)
            return
        }
    }

    // Create file
    file, err := os.Create(filename)
    if err != nil {
        log.Println("could not create cache file ", err)
        return
    }

    // Close and check for error on exit
    defer func() {
        if err := file.Close(); err != nil {

            log.Println("Could not close file ", err)
        }
    }()

    // Write file
    _, err = file.Write(b)
    if err != nil{
        log.Println("Could not write to cache file ", err)
    }

    //log.Println(url, "successfully written to cache");
}

func generateCacheEntry(resp *http.Response, body []byte ) Entry {

    /*
    body, err := ioutil.ReadAll(resp.Body)

    if err != nil {
        log.Print("Reading response body went bad. ", err);

    }
    */

    Response := resp
    Content := string(body)
    entry := Entry{Response, Content}

    // Cannot marshal the body from get go
    entry.Response.Body = nil

    return entry

}

// Try to fetch contents of url from cache
func getFromCache(url string) (resp *http.Response, err error) {

    filename := getFilePath(url)

    file, err := os.Open(filename)
    if err != nil{
        err = errors.New("File '"+filename+"' not Found")
        return
    }

    entry, err := readFromFile(file)

    if err != nil{
        return
    }

    resp = entry.GenerateHttpResponse()

    return

}

// Read cache entry from file
func readFromFile(file *os.File) (entry *Entry, err error) {
    fileinfo, err := file.Stat()
    var size int
    size = int(fileinfo.Size())

    buf := make([]byte, size)

    _, err = file.Read(buf)

    if err != nil{
        //log.Println("Reading file got:", err, "after",n,"bytes");
        return entry, errors.New("Reading file went wrong")
    }

    // Must trim buffer before unmarshaling it. This is because of
    // the unmarshaling failing if entire buffer is returned
    buf = bytes.Trim(buf[0:], string(0))

    entry = new(Entry)
    err = json.Unmarshal(buf, entry)

    if err != nil {
        log.Print("Unmarshaling gone wrong: ", err)
        return entry, errors.New("Unmarshaling gone wrong")
    }

    return
}

func (entry *Entry) Print () {

    //n := bytes.Index(entry.Content, []byte{0})
    //content := string(entry.Content[:n])
    log.Print("Content: ", entry.Content)

}

func (entry *Entry) GenerateHttpResponse() (resp *http.Response){

    resp = entry.Response
    resp.Body = nopCloser{bytes.NewBufferString(entry.Content)}

    return resp

}

func getFilePath(url string) (dir string) {
	urlTokens := strings.Split(url, "/")
    // 2 because we need to strip away 'http:', ' '
    strippedUrl := urlTokens[2:]
	dir = "cache/"+strings.Join(strippedUrl,"/")
	return
}

func createDirectories(filename string) error {

    filepath := path.Dir(filename)
    directories := strings.Split(filepath, "/")

    p := ""
    for i := range directories {
        p += directories[i] + "/"
        err := os.Mkdir(p, 0755)

        if err != nil {
            pe, _ := err.(*os.PathError)

            // if folder exists, continue to the next one
            if ! strings.Contains(pe.Error(),"file exists") {
                log.Println("Mkdir failed miserably: ", err)
                return err
            }
        }
    }

    return nil
}

/* Below are from:
    https://groups.google.com/forum/#!topic/golang-nuts/J-Y4LtdGNSw
*/

type nopCloser struct {
    io.Reader
}

func (nopCloser) Close() error {
    return nil
}
<commit_msg>got to close them bodies<commit_after>package gocache

import(
    "fmt"
    "net/http"
    "errors"
    "os"
    "strings"
    "log"
    "encoding/json"
    "bytes"
    "io/ioutil"
    "path"
    "io"
)

type Entry struct{
    Response *http.Response
    Content string
}

func main(){
    fmt.Println("Testing web cache");
    url := "http://blog.golang.org/error-handling-and-go"
    Get(url)
}

func Get(url string) (resp *http.Response, err error) {

    resp, err = getFromCache(url)

    if err != nil{
        //log.Println(url,"could not be served out of cache.", err)
        return getFromWeb(url)
    }

    //log.Print("Request to ", url, " served out of cache")
    return
}

// Fetch contents of url from web and write to cache
func getFromWeb(url string) (resp *http.Response, err error){
    resp, err = http.Get(url)

    if err != nil{
        return
    }
    var resp2 http.Response
    resp2 = *resp

    defer resp.Body.Close()

    // Draining the body

    body, err := ioutil.ReadAll(resp.Body)

    if err != nil {
        log.Print("Reading response body went bad. ", err);

    }

    writeToCache(url, body, &resp2)

    resp.Body = nopCloser{bytes.NewBufferString(string(body))}

    return
}

func writeToCache(url string, body []byte, resp *http.Response){

    cacheEntry := generateCacheEntry(resp, body)

    b, err := json.Marshal(cacheEntry)

    if err != nil {
        log.Println("Could not marshal response ", err)
        return
    }

    filename := getFilePath(url)

    // Set up any directory needed to write the file.
    // e.g. vg.no/first/second/file will be stored as
    // cache/first/second/file

    err = createDirectories(filename)
    if err != nil{
        pe, _ := err.(*os.PathError)

        if ! strings.Contains(pe.Error(),"file exists") {
            log.Println("Could not create directories ", err)
            return
        }
    }

    // Create file
    file, err := os.Create(filename)
    if err != nil {
        log.Println("could not create cache file ", err)
        return
    }

    // Close and check for error on exit
    defer func() {
        if err := file.Close(); err != nil {
            log.Println("Could not close file ", err)
        }
    }()

    // Write file
    _, err = file.Write(b)
    if err != nil{
        log.Println("Could not write to cache file ", err)
    }

    //log.Println(url, "successfully written to cache");
}

func generateCacheEntry(resp *http.Response, body []byte ) Entry {
    Response := resp
    Content := string(body)
    entry := Entry{Response, Content}

    // Cannot marshal the body from get go
    entry.Response.Body = nil

    return entry

}

// Try to fetch contents of url from cache
func getFromCache(url string) (resp *http.Response, err error) {

    filename := getFilePath(url)

    file, err := os.Open(filename)
    if err != nil{
        err = errors.New("File '"+filename+"' not Found")
        return
    }

    entry, err := readFromFile(file)

    if err != nil{
        return
    }

    resp = entry.GenerateHttpResponse()

    return

}

// Read cache entry from file
func readFromFile(file *os.File) (entry *Entry, err error) {
    fileinfo, err := file.Stat()
    var size int
    size = int(fileinfo.Size())

    buf := make([]byte, size)

    _, err = file.Read(buf)

    if err != nil{
        //log.Println("Reading file got:", err, "after",n,"bytes");
        return entry, errors.New("Reading file went wrong")
    }

    // Must trim buffer before unmarshaling it. This is because of
    // the unmarshaling failing if entire buffer is returned
    buf = bytes.Trim(buf[0:], string(0))

    entry = new(Entry)
    err = json.Unmarshal(buf, entry)

    if err != nil {
        log.Print("Unmarshaling gone wrong: ", err)
        return entry, errors.New("Unmarshaling gone wrong")
    }

    return
}

func (entry *Entry) Print () {

    //n := bytes.Index(entry.Content, []byte{0})
    //content := string(entry.Content[:n])
    log.Print("Content: ", entry.Content)

}

func (entry *Entry) GenerateHttpResponse() (resp *http.Response){

    resp = entry.Response
    resp.Body = nopCloser{bytes.NewBufferString(entry.Content)}

    return resp

}

func getFilePath(url string) (dir string) {
	urlTokens := strings.Split(url, "/")
    // 2 because we need to strip away 'http:', ' '
    strippedUrl := urlTokens[2:]
	dir = "cache/"+strings.Join(strippedUrl,"/")
	return
}

func createDirectories(filename string) error {

    filepath := path.Dir(filename)
    directories := strings.Split(filepath, "/")

    p := ""
    for i := range directories {
        p += directories[i] + "/"
        err := os.Mkdir(p, 0755)

        if err != nil {
            pe, _ := err.(*os.PathError)

            // if folder exists, continue to the next one
            if ! strings.Contains(pe.Error(),"file exists") {
                log.Println("Mkdir failed miserably: ", err)
                return err
            }
        }
    }

    return nil
}

/* Below are from:
    https://groups.google.com/forum/#!topic/golang-nuts/J-Y4LtdGNSw
*/

type nopCloser struct {
    io.Reader
}

func (nopCloser) Close() error {
    return nil
}
<|endoftext|>


<commit_before>package gog

import (
	"bytes"
	"go/ast"
	"go/build"
	"go/doc"
	"go/parser"
	"go/token"
	"io"
	"os"
	"path/filepath"
	"sort"

	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
)

type Doc struct {
	*DefKey

	Format string
	Data   string

	File string    `json:",omitempty"`
	Span [2]uint32 `json:",omitempty"`
}

func parseFiles(fset *token.FileSet, filenames []string) (map[string]*ast.File, error) {
	files := make(map[string]*ast.File)
	for _, path := range filenames {
		// read file contents using go/build context so we use our vfs if
		// present
		var f io.ReadCloser
		var err error
		if build.Default.OpenFile != nil {
			f, err = build.Default.OpenFile(path)
		} else {
			f, err = os.Open(path)
		}
		if err != nil {
			return nil, err
		}
		defer f.Close()

		file, err := parser.ParseFile(fset, path, f, parser.ParseComments)
		if err != nil {
			return nil, err
		}
		files[path] = file
	}
	return files, nil
}

func (g *Grapher) emitDocs(pkgInfo *loader.PackageInfo) error {
	objOf := make(map[token.Position]types.Object, len(pkgInfo.Defs))
	for ident, obj := range pkgInfo.Defs {
		objOf[g.program.Fset.Position(ident.Pos())] = obj
	}

	var filenames []string
	for _, f := range pkgInfo.Files {
		name := g.program.Fset.Position(f.Name.Pos()).Filename
		if filepath.Base(name) == "C" {
			// skip cgo-generated file
			continue
		}
		filenames = append(filenames, name)
	}
	sort.Strings(filenames)
	files, err := parseFiles(g.program.Fset, filenames)
	if err != nil {
		return err
	}

	// First we collect all of the Doc comments from the files,
	// which will make up the Doc for the package. If more than
	// one file has a doc associated with it, append them
	// together.
	pkgDoc := ""
	for _, f := range files {
		if f.Doc == nil {
			continue
		}
		if pkgDoc == "" {
			pkgDoc = f.Doc.Text()
			continue
		}
		pkgDoc += "\n" + f.Doc.Text()
	}
	g.emitDoc(types.NewPkgName(0, pkgInfo.Pkg, pkgInfo.Pkg.Path(), pkgInfo.Pkg), nil, pkgDoc, "")

	// We walk the AST for comments attached to nodes.
	for filename, f := range files {
		if f.Doc != nil {
			g.emitDoc(nil, f.Doc, f.Doc.Text(), filename)
		}
		// docSeen is a map from the starting byte of a doc to
		// an empty struct.
		docSeen := make(map[token.Pos]struct{})
		ast.Inspect(f, func(node ast.Node) bool {
			switch n := node.(type) {
			case *ast.Field:
				if n.Doc == nil {
					return true
				}
				for _, i := range n.Names {
					if g.emitDoc(objOf[g.program.Fset.Position(i.Pos())], n.Doc, n.Doc.Text(), filename) {
						docSeen[n.Doc.Pos()] = struct{}{}
					}
				}
			case *ast.FuncDecl:
				if n.Doc == nil || n.Name == nil {
					return true
				}
				if g.emitDoc(objOf[g.program.Fset.Position(n.Name.Pos())], n.Doc, n.Doc.Text(), filename) {
					docSeen[n.Doc.Pos()] = struct{}{}
				}
			case *ast.GenDecl:
				for _, spec := range n.Specs {
					switch spec := spec.(type) {
					case *ast.ValueSpec:
						for _, name := range spec.Names {
							c := firstNonNil(spec.Doc, spec.Comment, n.Doc)
							if g.emitDoc(objOf[g.program.Fset.Position(name.Pos())], c, c.Text(), filename) {
								docSeen[c.Pos()] = struct{}{}
							}
						}
					case *ast.TypeSpec:
						c := firstNonNil(spec.Doc, spec.Comment, n.Doc)
						if g.emitDoc(objOf[g.program.Fset.Position(spec.Name.Pos())], c, c.Text(), filename) {
							docSeen[c.Pos()] = struct{}{}
						}
					}
				}
			case *ast.ImportSpec:
				if n.Doc == nil || n.Name == nil {
					return true
				}
				if g.emitDoc(objOf[g.program.Fset.Position(n.Name.Pos())], n.Doc, n.Doc.Text(), filename) {
					docSeen[n.Doc.Pos()] = struct{}{}
				}
			case *ast.TypeSpec:
				if n.Doc == nil || n.Name == nil {
					return true
				}
				if g.emitDoc(objOf[g.program.Fset.Position(n.Name.Pos())], n.Doc, n.Doc.Text(), filename) {
					docSeen[n.Doc.Pos()] = struct{}{}
				}
			case *ast.ValueSpec:
				if n.Doc == nil {
					return true
				}
				for _, i := range n.Names {
					if g.emitDoc(objOf[g.program.Fset.Position(i.Pos())], n.Doc, n.Doc.Text(), filename) {
						docSeen[n.Doc.Pos()] = struct{}{}
					}
				}
			}
			return true
		})
		// Add comments that haven't already been seen.
		for _, c := range f.Comments {
			if _, seen := docSeen[c.Pos()]; !seen {
				g.emitDoc(nil, c, c.Text(), filename)
			}
		}
	}
	return nil
}

func firstNonNil(comments ...*ast.CommentGroup) *ast.CommentGroup {
	for _, c := range comments {
		if c != nil {
			return c
		}
	}
	return nil
}

func (g *Grapher) emitDoc(obj types.Object, dc *ast.CommentGroup, docstring, filename string) bool {
	if docstring == "" {
		return false
	}
	if obj == nil {
		var htmlBuf bytes.Buffer
		doc.ToHTML(&htmlBuf, docstring, nil)
		var span [2]uint32
		if dc != nil {
			span = makeSpan(g.program.Fset, dc)
		}
		g.addDoc(&Doc{
			DefKey: nil,
			Format: "text/html",
			Data:   htmlBuf.String(),
			File:   filename,
			Span:   span,
		})
		g.addDoc(&Doc{
			DefKey: nil,
			Format: "text/plain",
			Data:   docstring,
			File:   filename,
			Span:   span,
		})
		return true
	}

	if g.seenDocObjs == nil {
		g.seenDocObjs = make(map[types.Object]struct{})
	}
	if _, seen := g.seenDocObjs[obj]; seen {
		return false
	}
	g.seenDocObjs[obj] = struct{}{}

	key, _, err := g.defInfo(obj)
	if err != nil {
		return false
	}

	if g.seenDocKeys == nil {
		g.seenDocKeys = make(map[string]struct{})
	}
	if _, seen := g.seenDocKeys[key.String()]; seen {
		return false
	}
	g.seenDocKeys[key.String()] = struct{}{}

	var htmlBuf bytes.Buffer
	doc.ToHTML(&htmlBuf, docstring, nil)

	var span [2]uint32
	if dc != nil {
		span = makeSpan(g.program.Fset, dc)
	}

	g.addDoc(&Doc{
		DefKey: key,
		Format: "text/html",
		Data:   htmlBuf.String(),
		File:   filename,
		Span:   span,
	})
	g.addDoc(&Doc{
		DefKey: key,
		Format: "text/plain",
		Data:   docstring,
		File:   filename,
		Span:   span,
	})

	return true
}

func (g *Grapher) addDoc(doc *Doc) {
	g.Docs = append(g.Docs, doc)
}
<commit_msg>only emit file doc once<commit_after>package gog

import (
	"bytes"
	"go/ast"
	"go/build"
	"go/doc"
	"go/parser"
	"go/token"
	"io"
	"os"
	"path/filepath"
	"sort"

	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
)

type Doc struct {
	*DefKey

	Format string
	Data   string

	File string    `json:",omitempty"`
	Span [2]uint32 `json:",omitempty"`
}

func parseFiles(fset *token.FileSet, filenames []string) (map[string]*ast.File, error) {
	files := make(map[string]*ast.File)
	for _, path := range filenames {
		// read file contents using go/build context so we use our vfs if
		// present
		var f io.ReadCloser
		var err error
		if build.Default.OpenFile != nil {
			f, err = build.Default.OpenFile(path)
		} else {
			f, err = os.Open(path)
		}
		if err != nil {
			return nil, err
		}
		defer f.Close()

		file, err := parser.ParseFile(fset, path, f, parser.ParseComments)
		if err != nil {
			return nil, err
		}
		files[path] = file
	}
	return files, nil
}

func (g *Grapher) emitDocs(pkgInfo *loader.PackageInfo) error {
	objOf := make(map[token.Position]types.Object, len(pkgInfo.Defs))
	for ident, obj := range pkgInfo.Defs {
		objOf[g.program.Fset.Position(ident.Pos())] = obj
	}

	var filenames []string
	for _, f := range pkgInfo.Files {
		name := g.program.Fset.Position(f.Name.Pos()).Filename
		if filepath.Base(name) == "C" {
			// skip cgo-generated file
			continue
		}
		filenames = append(filenames, name)
	}
	sort.Strings(filenames)
	files, err := parseFiles(g.program.Fset, filenames)
	if err != nil {
		return err
	}

	// First we collect all of the Doc comments from the files,
	// which will make up the Doc for the package. If more than
	// one file has a doc associated with it, append them
	// together.
	pkgDoc := ""
	for _, f := range files {
		if f.Doc == nil {
			continue
		}
		if pkgDoc == "" {
			pkgDoc = f.Doc.Text()
			continue
		}
		pkgDoc += "\n" + f.Doc.Text()
	}
	g.emitDoc(types.NewPkgName(0, pkgInfo.Pkg, pkgInfo.Pkg.Path(), pkgInfo.Pkg), nil, pkgDoc, "")

	// We walk the AST for comments attached to nodes.
	for filename, f := range files {
		// docSeen is a map from the starting byte of a doc to
		// an empty struct.
		docSeen := make(map[token.Pos]struct{})
		ast.Inspect(f, func(node ast.Node) bool {
			switch n := node.(type) {
			case *ast.Field:
				if n.Doc == nil {
					return true
				}
				for _, i := range n.Names {
					if g.emitDoc(objOf[g.program.Fset.Position(i.Pos())], n.Doc, n.Doc.Text(), filename) {
						docSeen[n.Doc.Pos()] = struct{}{}
					}
				}
			case *ast.FuncDecl:
				if n.Doc == nil || n.Name == nil {
					return true
				}
				if g.emitDoc(objOf[g.program.Fset.Position(n.Name.Pos())], n.Doc, n.Doc.Text(), filename) {
					docSeen[n.Doc.Pos()] = struct{}{}
				}
			case *ast.GenDecl:
				for _, spec := range n.Specs {
					switch spec := spec.(type) {
					case *ast.ValueSpec:
						for _, name := range spec.Names {
							c := firstNonNil(spec.Doc, spec.Comment, n.Doc)
							if g.emitDoc(objOf[g.program.Fset.Position(name.Pos())], c, c.Text(), filename) {
								docSeen[c.Pos()] = struct{}{}
							}
						}
					case *ast.TypeSpec:
						c := firstNonNil(spec.Doc, spec.Comment, n.Doc)
						if g.emitDoc(objOf[g.program.Fset.Position(spec.Name.Pos())], c, c.Text(), filename) {
							docSeen[c.Pos()] = struct{}{}
						}
					}
				}
			case *ast.ImportSpec:
				if n.Doc == nil || n.Name == nil {
					return true
				}
				if g.emitDoc(objOf[g.program.Fset.Position(n.Name.Pos())], n.Doc, n.Doc.Text(), filename) {
					docSeen[n.Doc.Pos()] = struct{}{}
				}
			case *ast.TypeSpec:
				if n.Doc == nil || n.Name == nil {
					return true
				}
				if g.emitDoc(objOf[g.program.Fset.Position(n.Name.Pos())], n.Doc, n.Doc.Text(), filename) {
					docSeen[n.Doc.Pos()] = struct{}{}
				}
			case *ast.ValueSpec:
				if n.Doc == nil {
					return true
				}
				for _, i := range n.Names {
					if g.emitDoc(objOf[g.program.Fset.Position(i.Pos())], n.Doc, n.Doc.Text(), filename) {
						docSeen[n.Doc.Pos()] = struct{}{}
					}
				}
			}
			return true
		})
		// Add comments that haven't already been seen.
		for _, c := range f.Comments {
			if _, seen := docSeen[c.Pos()]; !seen {
				g.emitDoc(nil, c, c.Text(), filename)
			}
		}
	}
	return nil
}

func firstNonNil(comments ...*ast.CommentGroup) *ast.CommentGroup {
	for _, c := range comments {
		if c != nil {
			return c
		}
	}
	return nil
}

func (g *Grapher) emitDoc(obj types.Object, dc *ast.CommentGroup, docstring, filename string) bool {
	if docstring == "" {
		return false
	}
	if obj == nil {
		var htmlBuf bytes.Buffer
		doc.ToHTML(&htmlBuf, docstring, nil)
		var span [2]uint32
		if dc != nil {
			span = makeSpan(g.program.Fset, dc)
		}
		g.addDoc(&Doc{
			DefKey: nil,
			Format: "text/html",
			Data:   htmlBuf.String(),
			File:   filename,
			Span:   span,
		})
		g.addDoc(&Doc{
			DefKey: nil,
			Format: "text/plain",
			Data:   docstring,
			File:   filename,
			Span:   span,
		})
		return true
	}

	if g.seenDocObjs == nil {
		g.seenDocObjs = make(map[types.Object]struct{})
	}
	if _, seen := g.seenDocObjs[obj]; seen {
		return false
	}
	g.seenDocObjs[obj] = struct{}{}

	key, _, err := g.defInfo(obj)
	if err != nil {
		return false
	}

	if g.seenDocKeys == nil {
		g.seenDocKeys = make(map[string]struct{})
	}
	if _, seen := g.seenDocKeys[key.String()]; seen {
		return false
	}
	g.seenDocKeys[key.String()] = struct{}{}

	var htmlBuf bytes.Buffer
	doc.ToHTML(&htmlBuf, docstring, nil)

	var span [2]uint32
	if dc != nil {
		span = makeSpan(g.program.Fset, dc)
	}

	g.addDoc(&Doc{
		DefKey: key,
		Format: "text/html",
		Data:   htmlBuf.String(),
		File:   filename,
		Span:   span,
	})
	g.addDoc(&Doc{
		DefKey: key,
		Format: "text/plain",
		Data:   docstring,
		File:   filename,
		Span:   span,
	})

	return true
}

func (g *Grapher) addDoc(doc *Doc) {
	g.Docs = append(g.Docs, doc)
}
<|endoftext|>


<commit_before>package croc

import (
	"net"
	"time"

	log "github.com/cihub/seelog"
	"github.com/pkg/errors"
	"github.com/schollz/peerdiscovery"
)

func init() {
	SetLogLevel("debug")
}

// Relay initiates a relay
func (c *Croc) Relay() error {
	// start relay
	go c.startRelay()

	// start server
	return c.startServer()
}

// Send will take an existing file or folder and send it through the croc relay
func (c *Croc) Send(fname string, codePhrase string) (err error) {
	// prepare code phrase
	defer c.cleanup()
	c.cs.Lock()
	c.cs.channel.codePhrase = codePhrase
	if len(codePhrase) == 0 {
		// generate code phrase
		codePhrase = getRandomName()
	}
	if len(codePhrase) < 4 {
		err = errors.New("code phrase must be more than 4 characters")
		c.cs.Unlock()
		return
	}
	c.cs.channel.codePhrase = codePhrase
	c.cs.channel.Channel = codePhrase[:3]
	c.cs.channel.passPhrase = codePhrase[3:]
	log.Debugf("codephrase: '%s'", codePhrase)
	log.Debugf("channel: '%s'", c.cs.channel.Channel)
	log.Debugf("passPhrase: '%s'", c.cs.channel.passPhrase)
	channel := c.cs.channel.Channel
	c.cs.Unlock()

	// start peer discovery
	go func() {
		if c.NoLocal {
			return
		}
		log.Debug("listening for local croc relay...")
		go peerdiscovery.Discover(peerdiscovery.Settings{
			Limit:     1,
			TimeLimit: 600 * time.Second,
			Delay:     50 * time.Millisecond,
			Payload:   []byte(codePhrase),
		})
	}()

	if len(fname) == 0 {
		err = errors.New("must include filename")
		return
	}
	err = c.processFile(fname)
	if err != nil {
		return
	}

	// start relay for listening
	type runInfo struct {
		err           error
		bothConnected bool
	}
	runClientError := make(chan runInfo, 2)
	go func() {
		if c.NoLocal {
			return
		}
		d := Init()
		d.ServerPort = "8140"
		d.TcpPorts = []string{"27140", "27141"}
		go d.startRelay()
		go d.startServer()
		time.Sleep(100 * time.Millisecond)
		ce := Init()
		ce.WebsocketAddress = "ws://127.0.0.1:8140"
		// copy over the information
		c.cs.Lock()
		ce.cs.Lock()
		ce.cs.channel.codePhrase = codePhrase
		ce.cs.channel.Channel = codePhrase[:3]
		ce.cs.channel.passPhrase = codePhrase[3:]
		ce.cs.channel.fileMetaData = c.cs.channel.fileMetaData
		ce.crocFile = c.crocFile
		ce.crocFileEncrypted = ce.crocFileEncrypted
		ce.isLocal = true
		ce.cs.Unlock()
		c.cs.Unlock()
		defer func() {
			// delete croc files
			ce.cleanup()
		}()
		var ri runInfo
		ri.err = ce.client(0, channel)
		ri.bothConnected = ce.bothConnected
		runClientError <- ri
	}()

	// start main client
	go func() {
		if c.LocalOnly {
			return
		}
		var ri runInfo
		ri.err = c.client(0, channel)
		ri.bothConnected = c.bothConnected
		runClientError <- ri
	}()

	var ri runInfo
	ri = <-runClientError
	if ri.bothConnected || c.LocalOnly || c.NoLocal {
		return ri.err
	}
	ri = <-runClientError
	return ri.err
}

// Receive will receive something through the croc relay
func (c *Croc) Receive(codePhrase string) (err error) {
	defer c.cleanup()
	if !c.NoLocal {
		// try to discovery codephrase and server through peer network
		discovered, errDiscover := peerdiscovery.Discover(peerdiscovery.Settings{
			Limit:     1,
			TimeLimit: 1 * time.Second,
			Delay:     50 * time.Millisecond,
			Payload:   []byte(codePhrase),
		})
		if errDiscover != nil {
			log.Debug(errDiscover)
		}
		if len(discovered) > 0 {
			log.Debugf("discovered %s on %s", discovered[0].Payload, discovered[0].Address)
			_, connectTimeout := net.DialTimeout("tcp", discovered[0].Address+":27140", 1*time.Second)
			if connectTimeout == nil {
				log.Debug("connected")
				c.WebsocketAddress = "ws://" + discovered[0].Address + ":8140"
				c.isLocal = true
				log.Debug(discovered[0].Address)
				// codePhrase = string(discovered[0].Payload)
			} else {
				log.Debug("but could not connect to ports")
			}
		} else {
			log.Debug("discovered no peers")
		}
	}

	// prepare codephrase
	c.cs.Lock()
	if len(codePhrase) == 0 {
		// prompt codephrase
		codePhrase = promptCodePhrase()
	}
	if len(codePhrase) < 4 {
		err = errors.New("code phrase must be more than 4 characters")
		c.cs.Unlock()
		return
	}
	c.cs.channel.codePhrase = codePhrase
	c.cs.channel.Channel = codePhrase[:3]
	c.cs.channel.passPhrase = codePhrase[3:]
	log.Debugf("codephrase: '%s'", codePhrase)
	log.Debugf("channel: '%s'", c.cs.channel.Channel)
	log.Debugf("passPhrase: '%s'", c.cs.channel.passPhrase)
	channel := c.cs.channel.Channel
	c.cs.Unlock()

	return c.client(1, channel)
}
<commit_msg>don't broadcast channel<commit_after>package croc

import (
	"net"
	"time"

	log "github.com/cihub/seelog"
	"github.com/pkg/errors"
	"github.com/schollz/peerdiscovery"
)

func init() {
	SetLogLevel("debug")
}

// Relay initiates a relay
func (c *Croc) Relay() error {
	// start relay
	go c.startRelay()

	// start server
	return c.startServer()
}

// Send will take an existing file or folder and send it through the croc relay
func (c *Croc) Send(fname string, codePhrase string) (err error) {
	// prepare code phrase
	defer c.cleanup()
	c.cs.Lock()
	c.cs.channel.codePhrase = codePhrase
	if len(codePhrase) == 0 {
		// generate code phrase
		codePhrase = getRandomName()
	}
	if len(codePhrase) < 4 {
		err = errors.New("code phrase must be more than 4 characters")
		c.cs.Unlock()
		return
	}
	c.cs.channel.codePhrase = codePhrase
	c.cs.channel.Channel = codePhrase[:3]
	c.cs.channel.passPhrase = codePhrase[3:]
	log.Debugf("codephrase: '%s'", codePhrase)
	log.Debugf("channel: '%s'", c.cs.channel.Channel)
	log.Debugf("passPhrase: '%s'", c.cs.channel.passPhrase)
	channel := c.cs.channel.Channel
	c.cs.Unlock()

	// start peer discovery
	go func() {
		if c.NoLocal {
			return
		}
		log.Debug("listening for local croc relay...")
		go peerdiscovery.Discover(peerdiscovery.Settings{
			Limit:     1,
			TimeLimit: 600 * time.Second,
			Delay:     50 * time.Millisecond,
			Payload:   []byte(codePhrase[:3]),
		})
	}()

	if len(fname) == 0 {
		err = errors.New("must include filename")
		return
	}
	err = c.processFile(fname)
	if err != nil {
		return
	}

	// start relay for listening
	type runInfo struct {
		err           error
		bothConnected bool
	}
	runClientError := make(chan runInfo, 2)
	go func() {
		if c.NoLocal {
			return
		}
		d := Init()
		d.ServerPort = "8140"
		d.TcpPorts = []string{"27140", "27141"}
		go d.startRelay()
		go d.startServer()
		time.Sleep(100 * time.Millisecond)
		ce := Init()
		ce.WebsocketAddress = "ws://127.0.0.1:8140"
		// copy over the information
		c.cs.Lock()
		ce.cs.Lock()
		ce.cs.channel.codePhrase = codePhrase
		ce.cs.channel.Channel = codePhrase[:3]
		ce.cs.channel.passPhrase = codePhrase[3:]
		ce.cs.channel.fileMetaData = c.cs.channel.fileMetaData
		ce.crocFile = c.crocFile
		ce.crocFileEncrypted = ce.crocFileEncrypted
		ce.isLocal = true
		ce.cs.Unlock()
		c.cs.Unlock()
		defer func() {
			// delete croc files
			ce.cleanup()
		}()
		var ri runInfo
		ri.err = ce.client(0, channel)
		ri.bothConnected = ce.bothConnected
		runClientError <- ri
	}()

	// start main client
	go func() {
		if c.LocalOnly {
			return
		}
		var ri runInfo
		ri.err = c.client(0, channel)
		ri.bothConnected = c.bothConnected
		runClientError <- ri
	}()

	var ri runInfo
	ri = <-runClientError
	if ri.bothConnected || c.LocalOnly || c.NoLocal {
		return ri.err
	}
	ri = <-runClientError
	return ri.err
}

// Receive will receive something through the croc relay
func (c *Croc) Receive(codePhrase string) (err error) {
	defer c.cleanup()
	if !c.NoLocal {
		// try to discovery codephrase and server through peer network
		discovered, errDiscover := peerdiscovery.Discover(peerdiscovery.Settings{
			Limit:     1,
			TimeLimit: 1 * time.Second,
			Delay:     50 * time.Millisecond,
			Payload:   []byte(codePhrase[:3]),
		})
		if errDiscover != nil {
			log.Debug(errDiscover)
		}
		if len(discovered) > 0 {
			log.Debugf("discovered %s on %s", discovered[0].Payload, discovered[0].Address)
			_, connectTimeout := net.DialTimeout("tcp", discovered[0].Address+":27140", 1*time.Second)
			if connectTimeout == nil {
				log.Debug("connected")
				c.WebsocketAddress = "ws://" + discovered[0].Address + ":8140"
				c.isLocal = true
				log.Debug(discovered[0].Address)
				// codePhrase = string(discovered[0].Payload)
			} else {
				log.Debug("but could not connect to ports")
			}
		} else {
			log.Debug("discovered no peers")
		}
	}

	// prepare codephrase
	c.cs.Lock()
	if len(codePhrase) == 0 {
		// prompt codephrase
		codePhrase = promptCodePhrase()
	}
	if len(codePhrase) < 4 {
		err = errors.New("code phrase must be more than 4 characters")
		c.cs.Unlock()
		return
	}
	c.cs.channel.codePhrase = codePhrase
	c.cs.channel.Channel = codePhrase[:3]
	c.cs.channel.passPhrase = codePhrase[3:]
	log.Debugf("codephrase: '%s'", codePhrase)
	log.Debugf("channel: '%s'", c.cs.channel.Channel)
	log.Debugf("passPhrase: '%s'", c.cs.channel.passPhrase)
	channel := c.cs.channel.Channel
	c.cs.Unlock()

	return c.client(1, channel)
}
<|endoftext|>


<commit_before>package raft

import (
	"fmt"
	"log"
	"math/rand"
	"sync"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/grpc"

	"github.com/relab/gorums/idutil"
	"github.com/relab/raft/debug"
	"github.com/relab/raft/proto/gorums"
)

// State represents one of the Raft server states.
type State int

// Server states.
// TODO: Generator?
const (
	FOLLOWER State = iota
	CANDIDATE
	LEADER
)

// Timeouts in milliseconds.
const (
	HEARTBEAT = 50
	ELECTION  = 150
)

// NONE represents no server.
// This must be a value which cannot be returned from idutil.IDFromAddress(address).
const NONE = 0

func min(a, b uint64) uint64 {
	if a < b {
		return a
	}

	return b
}

func max(a, b uint64) uint64 {
	if a > b {
		return a
	}

	return b
}

func randomTimeout() time.Duration {
	return time.Duration(ELECTION+rand.Intn(ELECTION*2-ELECTION)) * time.Millisecond
}

// Replica represents a Raft server
type Replica struct {
	// Must be acquired before mutating Replica state.
	sync.Mutex

	id       uint32
	leader   uint32
	votedFor uint32

	state State

	conf *gorums.Configuration

	nodes map[uint32]*gorums.Node

	currentTerm uint64

	log []*gorums.Entry

	commitIndex int

	nextIndex  map[uint32]int
	matchIndex map[uint32]int

	electionTimeout  time.Duration
	heartbeatTimeout time.Duration

	election  Timer
	heartbeat Timer
}

func (r *Replica) logTerm(index int) uint64 {
	if index < 1 || index > len(r.log) {
		return 0
	}

	return r.log[index-1].Term
}

// Init initializes a Replica.
// This must always be run before Run.
func (r *Replica) Init(this string, nodes []string) error {
	mgr, err := gorums.NewManager(nodes,
		gorums.WithGrpcDialOptions(
			grpc.WithBlock(),
			grpc.WithInsecure(),
			grpc.WithTimeout(time.Second*10)))

	if err != nil {
		return err
	}

	n := len(nodes) + 1

	qspec := &QuorumSpec{
		N: n,
		Q: n / 2,
	}

	conf, err := mgr.NewConfiguration(mgr.NodeIDs(), qspec, time.Second)

	if err != nil {
		return err
	}

	r.conf = conf

	r.id, err = idutil.IDFromAddress(this)

	if err != nil {
		return err
	}

	r.nodes = make(map[uint32]*gorums.Node, n-1)

	for _, node := range mgr.Nodes(false) {
		r.nodes[node.ID()] = node
	}

	r.electionTimeout = randomTimeout()
	r.heartbeatTimeout = HEARTBEAT * time.Millisecond

	debug.Debugln(r.id, ":: TIMEOUT SET,", r.electionTimeout)

	r.election = NewTimer(r.electionTimeout)
	r.heartbeat = NewTimer(0)
	r.heartbeat.Stop()

	r.votedFor = NONE

	r.nextIndex = make(map[uint32]int, n-1)
	r.matchIndex = make(map[uint32]int, n-1)

	for id := range r.nodes {
		// Initialized to leader last log index + 1.
		r.nextIndex[id] = 1
		r.matchIndex[id] = 0
	}

	r.Unlock()

	return nil
}

// Run handles timeouts.
// Always call Init before this method.
// All RPCs are handled by Gorums.
func (r *Replica) Run() {
	for {
		select {
		case <-r.election.C:
			// #F2 If election timeout elapses without receiving AppendEntries RPC from current leader
			// or granting vote to candidate: convert to candidate.
			r.startElection()

		case <-r.heartbeat.C:
			r.sendAppendEntries()
		}
	}
}

// RequestVote handles a RequestVoteRequest which is invoked by candidates to gather votes.
// See Raft paper § 5.2.
func (r *Replica) RequestVote(ctx context.Context, request *gorums.RequestVoteRequest) (*gorums.RequestVoteResponse, error) {
	r.Lock()
	defer r.Unlock()

	debug.Debugln(r.id, ":: VOTE REQUESTED, from", request.CandidateID, "for term", request.Term)

	// #RV1 Reply false if term < currentTerm.
	if request.Term < r.currentTerm {
		return &gorums.RequestVoteResponse{Term: r.currentTerm, RequestTerm: request.Term}, nil
	}

	// #A2 If RPC request or response contains term T > currentTerm: set currentTerm = T, convert to follower.
	if request.Term > r.currentTerm {
		r.becomeFollower(request.Term)
	}

	// #RV2 If votedFor is null or candidateId, and candidate's log is at least as up-to-date as receiver's log, grant vote.
	if (r.votedFor == NONE || r.votedFor == request.CandidateID) &&
		(request.LastLogTerm > r.logTerm(len(r.log)) ||
			(request.LastLogTerm == r.logTerm(len(r.log)) && request.LastLogIndex >= uint64(len(r.log)))) {
		debug.Debugln(r.id, ":: VOTE GRANTED, to", request.CandidateID, "for term", request.Term)

		r.votedFor = request.CandidateID

		// #F2 If election timeout elapses without receiving AppendEntries RPC from current leader or granting a vote to candidate: convert to candidate.
		// Here we are granting a vote to a candidate so we reset the election timeout.
		r.election.Reset(r.electionTimeout)

		return &gorums.RequestVoteResponse{VoteGranted: true, Term: r.currentTerm, RequestTerm: request.Term}, nil
	}

	// #RV2 The candidate's log was not up-to-date
	return &gorums.RequestVoteResponse{Term: r.currentTerm, RequestTerm: request.Term}, nil
}

// AppendEntries invoked by leader to replicate log entries, also used as a heartbeat.
// See Raft paper § 5.3 and § 5.2.
func (r *Replica) AppendEntries(ctx context.Context, request *gorums.AppendEntriesRequest) (*gorums.AppendEntriesResponse, error) {
	r.Lock()
	defer r.Unlock()

	debug.Traceln(r.id, "::APPENDENTRIES,", request)

	// #AE1 Reply false if term < currentTerm.
	if request.Term < r.currentTerm {
		return &gorums.AppendEntriesResponse{FollowerID: r.id, Success: false, Term: r.currentTerm}, nil
	}

	success := request.PrevLogIndex == 0 || (request.PrevLogIndex-1 < uint64(len(r.log)) && r.log[request.PrevLogIndex-1].Term == request.PrevLogTerm)

	// #A2 If RPC request or response contains term T > currentTerm: set currentTerm = T, convert to follower.
	if request.Term > r.currentTerm {
		r.becomeFollower(request.Term)
	} else if r.id != request.LeaderID {
		r.becomeFollower(r.currentTerm)
	}

	if success {
		debug.Debugln(r.id, ":: OK", r.currentTerm)

		r.leader = request.LeaderID

		index := int(request.PrevLogIndex)

		for _, entry := range request.Entries {
			index++

			if index == len(r.log) || r.logTerm(index) != entry.Term {
				r.log = r.log[:index-1] // Remove excessive log entries.
				r.log = append(r.log, entry)

				log := ""

				for _, entry := range r.log {
					log += string(entry.Data) + " "
				}

				debug.Debugln(r.id, ":: LOG, len:", len(r.log), "data:", log)
			}

			r.commitIndex = int(min(request.CommitIndex, uint64(index)))
		}
	}

	return &gorums.AppendEntriesResponse{FollowerID: r.id, Term: r.currentTerm, MatchIndex: uint64(len(r.log)), Success: success}, nil
}

func (r *Replica) startElection() {
	r.Lock()
	defer r.Unlock()

	r.state = CANDIDATE
	r.electionTimeout = randomTimeout()

	// We are now a candidate. See Raft Paper Figure 2 -> Rules for Servers -> Candidates.
	// #C1 Increment currentTerm.
	r.currentTerm++

	// #C2 Vote for self.
	r.votedFor = r.id

	debug.Debugln(r.id, ":: ELECTION STARTED, for term", r.currentTerm)

	// #C3 Reset election timer.
	r.election.Reset(r.electionTimeout)

	// #C4 Send RequestVote RPCs to all other servers.
	req := r.conf.RequestVoteFuture(&gorums.RequestVoteRequest{CandidateID: r.id, Term: r.currentTerm, LastLogIndex: r.logTerm(len(r.log)), LastLogTerm: uint64(len(r.log))})

	go func() {
		reply, err := req.Get()

		if reply.Reply != nil {
			r.handleRequestVoteResponse(reply.Reply)
		} else {
			log.Println("Got no replies:", err)
		}
	}()

	// Election is now started. Election will be continued in handleRequestVote when a response from Gorums is received.
	// See RequestVoteQF for the quorum function creating the response.
}

func (r *Replica) handleRequestVoteResponse(response *gorums.RequestVoteResponse) {
	r.Lock()
	defer r.Unlock()

	// #A2 If RPC request or response contains term T > currentTerm: set currentTerm = T, convert to follower.
	if response.Term > r.currentTerm {
		r.becomeFollower(response.Term)

		return
	}

	// Ignore late response
	if response.Term < r.currentTerm {
		return
	}

	// Cont. from startElection(). We have now received a response from Gorums.

	// #C5 If votes received from majority of server: become leader.
	// Make sure we have not stepped down while waiting for replies.
	if r.state == CANDIDATE && response.VoteGranted {
		// We have received at least a quorum of votes.
		// We are the leader for this term. See Raft Paper Figure 2 -> Rules for Servers -> Leaders.

		debug.Debugln(r.id, ":: ELECTED LEADER, for term", r.currentTerm)

		r.state = LEADER
		r.leader = r.id

		for id := range r.nextIndex {
			r.nextIndex[id] = len(r.log) + 1
		}

		// #L1 Upon election: send initial empty AppendEntries RPCs (heartbeat) to each server;
		r.heartbeat.Reset(0)

		r.election.Stop()

		return
	}

	// TODO: We didn't win the election. We should continue sending AppendEntries RPCs until the election runs out.

	// #C7 If election timeout elapses: start new election.
	// This will happened if we don't receive enough replies in time. Or we lose the election but don't see a higher term number.
}

func (r *Replica) sendAppendEntries() {
	r.Lock()
	defer r.Unlock()

	debug.Debugln(r.id, ":: APPENDENTRIES, for term", r.currentTerm)

	n := rand.Intn(100)

	if n > 90 {
		debug.Debugln(r.id, ":: APPENDENTRIES, with log entry")
		r.log = append(r.log, &gorums.Entry{Term: r.currentTerm, Data: []byte(fmt.Sprintf("%d", r.currentTerm))})
	}

	// #L1
	for id, node := range r.nodes {
		entries := []*gorums.Entry{}

		nextIndex := r.nextIndex[id] - 1

		if len(r.log) > nextIndex {
			entries = r.log[nextIndex : nextIndex+1]
		}

		req := &gorums.AppendEntriesRequest{
			LeaderID:     r.id,
			Term:         r.currentTerm,
			PrevLogIndex: uint64(nextIndex),
			PrevLogTerm:  r.logTerm(nextIndex),
			Entries:      entries,
		}

		go func(node *gorums.Node, req *gorums.AppendEntriesRequest) {
			// TOOD: It's unclear at the moment which context I should use.
			// I'm assuming I need some form of timeout?
			resp, err := node.RaftClient.AppendEntries(context.TODO(), req)

			if err != nil {
				log.Println(err)
			} else {
				r.handleAppendEntriesResponse(resp)
			}
		}(node, req)
	}

	r.heartbeat.Reset(r.heartbeatTimeout)
}

func (r *Replica) handleAppendEntriesResponse(response *gorums.AppendEntriesResponse) {
	r.Lock()
	defer r.Unlock()

	// #A2 If RPC request or response contains term T > currentTerm: set currentTerm = T, convert to follower.
	if response.Term > r.currentTerm {
		r.becomeFollower(response.Term)

		return
	}

	// Ignore late response
	if response.Term < r.currentTerm {
		return
	}

	if r.state == LEADER {
		if response.Success {
			r.matchIndex[response.FollowerID] = int(response.MatchIndex)
			r.nextIndex[response.FollowerID] = r.matchIndex[response.FollowerID] + 1

			return
		}

		r.nextIndex[response.FollowerID] = int(max(0, uint64(r.nextIndex[response.FollowerID]-1)))
	}
}

func (r *Replica) becomeFollower(term uint64) {
	debug.Debugln(r.id, ":: STEPDOWN,", r.currentTerm, "->", term)

	r.state = FOLLOWER
	r.currentTerm = term
	r.votedFor = NONE

	r.election.Reset(r.electionTimeout)
	r.heartbeat.Stop()
}
<commit_msg>Add timeout to AppendEntries call using context.Context<commit_after>package raft

import (
	"fmt"
	"log"
	"math/rand"
	"sync"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/grpc"

	"github.com/relab/gorums/idutil"
	"github.com/relab/raft/debug"
	"github.com/relab/raft/proto/gorums"
)

// State represents one of the Raft server states.
type State int

// Server states.
// TODO: Generator?
const (
	FOLLOWER State = iota
	CANDIDATE
	LEADER
)

// Timeouts in milliseconds.
const (
	HEARTBEAT     = 50
	ELECTION      = 150
	APPENDENTRIES = 100
)

// NONE represents no server.
// This must be a value which cannot be returned from idutil.IDFromAddress(address).
const NONE = 0

func min(a, b uint64) uint64 {
	if a < b {
		return a
	}

	return b
}

func max(a, b uint64) uint64 {
	if a > b {
		return a
	}

	return b
}

func randomTimeout() time.Duration {
	return time.Duration(ELECTION+rand.Intn(ELECTION*2-ELECTION)) * time.Millisecond
}

// Replica represents a Raft server
type Replica struct {
	// Must be acquired before mutating Replica state.
	sync.Mutex

	id       uint32
	leader   uint32
	votedFor uint32

	state State

	conf *gorums.Configuration

	nodes map[uint32]*gorums.Node

	currentTerm uint64

	log []*gorums.Entry

	commitIndex int

	nextIndex  map[uint32]int
	matchIndex map[uint32]int

	electionTimeout  time.Duration
	heartbeatTimeout time.Duration

	election  Timer
	heartbeat Timer
}

func (r *Replica) logTerm(index int) uint64 {
	if index < 1 || index > len(r.log) {
		return 0
	}

	return r.log[index-1].Term
}

// Init initializes a Replica.
// This must always be run before Run.
func (r *Replica) Init(this string, nodes []string) error {
	mgr, err := gorums.NewManager(nodes,
		gorums.WithGrpcDialOptions(
			grpc.WithBlock(),
			grpc.WithInsecure(),
			grpc.WithTimeout(time.Second*10)))

	if err != nil {
		return err
	}

	n := len(nodes) + 1

	qspec := &QuorumSpec{
		N: n,
		Q: n / 2,
	}

	conf, err := mgr.NewConfiguration(mgr.NodeIDs(), qspec, time.Second)

	if err != nil {
		return err
	}

	r.conf = conf

	r.id, err = idutil.IDFromAddress(this)

	if err != nil {
		return err
	}

	r.nodes = make(map[uint32]*gorums.Node, n-1)

	for _, node := range mgr.Nodes(false) {
		r.nodes[node.ID()] = node
	}

	r.electionTimeout = randomTimeout()
	r.heartbeatTimeout = HEARTBEAT * time.Millisecond

	debug.Debugln(r.id, ":: TIMEOUT SET,", r.electionTimeout)

	r.election = NewTimer(r.electionTimeout)
	r.heartbeat = NewTimer(0)
	r.heartbeat.Stop()

	r.votedFor = NONE

	r.nextIndex = make(map[uint32]int, n-1)
	r.matchIndex = make(map[uint32]int, n-1)

	for id := range r.nodes {
		// Initialized to leader last log index + 1.
		r.nextIndex[id] = 1
		r.matchIndex[id] = 0
	}

	r.Unlock()

	return nil
}

// Run handles timeouts.
// Always call Init before this method.
// All RPCs are handled by Gorums.
func (r *Replica) Run() {
	for {
		select {
		case <-r.election.C:
			// #F2 If election timeout elapses without receiving AppendEntries RPC from current leader
			// or granting vote to candidate: convert to candidate.
			r.startElection()

		case <-r.heartbeat.C:
			r.sendAppendEntries()
		}
	}
}

// RequestVote handles a RequestVoteRequest which is invoked by candidates to gather votes.
// See Raft paper § 5.2.
func (r *Replica) RequestVote(ctx context.Context, request *gorums.RequestVoteRequest) (*gorums.RequestVoteResponse, error) {
	r.Lock()
	defer r.Unlock()

	debug.Debugln(r.id, ":: VOTE REQUESTED, from", request.CandidateID, "for term", request.Term)

	// #RV1 Reply false if term < currentTerm.
	if request.Term < r.currentTerm {
		return &gorums.RequestVoteResponse{Term: r.currentTerm, RequestTerm: request.Term}, nil
	}

	// #A2 If RPC request or response contains term T > currentTerm: set currentTerm = T, convert to follower.
	if request.Term > r.currentTerm {
		r.becomeFollower(request.Term)
	}

	// #RV2 If votedFor is null or candidateId, and candidate's log is at least as up-to-date as receiver's log, grant vote.
	if (r.votedFor == NONE || r.votedFor == request.CandidateID) &&
		(request.LastLogTerm > r.logTerm(len(r.log)) ||
			(request.LastLogTerm == r.logTerm(len(r.log)) && request.LastLogIndex >= uint64(len(r.log)))) {
		debug.Debugln(r.id, ":: VOTE GRANTED, to", request.CandidateID, "for term", request.Term)

		r.votedFor = request.CandidateID

		// #F2 If election timeout elapses without receiving AppendEntries RPC from current leader or granting a vote to candidate: convert to candidate.
		// Here we are granting a vote to a candidate so we reset the election timeout.
		r.election.Reset(r.electionTimeout)

		return &gorums.RequestVoteResponse{VoteGranted: true, Term: r.currentTerm, RequestTerm: request.Term}, nil
	}

	// #RV2 The candidate's log was not up-to-date
	return &gorums.RequestVoteResponse{Term: r.currentTerm, RequestTerm: request.Term}, nil
}

// AppendEntries invoked by leader to replicate log entries, also used as a heartbeat.
// See Raft paper § 5.3 and § 5.2.
func (r *Replica) AppendEntries(ctx context.Context, request *gorums.AppendEntriesRequest) (*gorums.AppendEntriesResponse, error) {
	r.Lock()
	defer r.Unlock()

	debug.Traceln(r.id, "::APPENDENTRIES,", request)

	// #AE1 Reply false if term < currentTerm.
	if request.Term < r.currentTerm {
		return &gorums.AppendEntriesResponse{FollowerID: r.id, Success: false, Term: r.currentTerm}, nil
	}

	success := request.PrevLogIndex == 0 || (request.PrevLogIndex-1 < uint64(len(r.log)) && r.log[request.PrevLogIndex-1].Term == request.PrevLogTerm)

	// #A2 If RPC request or response contains term T > currentTerm: set currentTerm = T, convert to follower.
	if request.Term > r.currentTerm {
		r.becomeFollower(request.Term)
	} else if r.id != request.LeaderID {
		r.becomeFollower(r.currentTerm)
	}

	if success {
		debug.Debugln(r.id, ":: OK", r.currentTerm)

		r.leader = request.LeaderID

		index := int(request.PrevLogIndex)

		for _, entry := range request.Entries {
			index++

			if index == len(r.log) || r.logTerm(index) != entry.Term {
				r.log = r.log[:index-1] // Remove excessive log entries.
				r.log = append(r.log, entry)

				log := ""

				for _, entry := range r.log {
					log += string(entry.Data) + " "
				}

				debug.Debugln(r.id, ":: LOG, len:", len(r.log), "data:", log)
			}

			r.commitIndex = int(min(request.CommitIndex, uint64(index)))
		}
	}

	return &gorums.AppendEntriesResponse{FollowerID: r.id, Term: r.currentTerm, MatchIndex: uint64(len(r.log)), Success: success}, nil
}

func (r *Replica) startElection() {
	r.Lock()
	defer r.Unlock()

	r.state = CANDIDATE
	r.electionTimeout = randomTimeout()

	// We are now a candidate. See Raft Paper Figure 2 -> Rules for Servers -> Candidates.
	// #C1 Increment currentTerm.
	r.currentTerm++

	// #C2 Vote for self.
	r.votedFor = r.id

	debug.Debugln(r.id, ":: ELECTION STARTED, for term", r.currentTerm)

	// #C3 Reset election timer.
	r.election.Reset(r.electionTimeout)

	// #C4 Send RequestVote RPCs to all other servers.
	req := r.conf.RequestVoteFuture(&gorums.RequestVoteRequest{CandidateID: r.id, Term: r.currentTerm, LastLogIndex: r.logTerm(len(r.log)), LastLogTerm: uint64(len(r.log))})

	go func() {
		reply, err := req.Get()

		if reply.Reply != nil {
			r.handleRequestVoteResponse(reply.Reply)
		} else {
			log.Println("Got no replies:", err)
		}
	}()

	// Election is now started. Election will be continued in handleRequestVote when a response from Gorums is received.
	// See RequestVoteQF for the quorum function creating the response.
}

func (r *Replica) handleRequestVoteResponse(response *gorums.RequestVoteResponse) {
	r.Lock()
	defer r.Unlock()

	// #A2 If RPC request or response contains term T > currentTerm: set currentTerm = T, convert to follower.
	if response.Term > r.currentTerm {
		r.becomeFollower(response.Term)

		return
	}

	// Ignore late response
	if response.Term < r.currentTerm {
		return
	}

	// Cont. from startElection(). We have now received a response from Gorums.

	// #C5 If votes received from majority of server: become leader.
	// Make sure we have not stepped down while waiting for replies.
	if r.state == CANDIDATE && response.VoteGranted {
		// We have received at least a quorum of votes.
		// We are the leader for this term. See Raft Paper Figure 2 -> Rules for Servers -> Leaders.

		debug.Debugln(r.id, ":: ELECTED LEADER, for term", r.currentTerm)

		r.state = LEADER
		r.leader = r.id

		for id := range r.nextIndex {
			r.nextIndex[id] = len(r.log) + 1
		}

		// #L1 Upon election: send initial empty AppendEntries RPCs (heartbeat) to each server;
		r.heartbeat.Reset(0)

		r.election.Stop()

		return
	}

	// TODO: We didn't win the election. We should continue sending AppendEntries RPCs until the election runs out.

	// #C7 If election timeout elapses: start new election.
	// This will happened if we don't receive enough replies in time. Or we lose the election but don't see a higher term number.
}

func (r *Replica) sendAppendEntries() {
	r.Lock()
	defer r.Unlock()

	debug.Debugln(r.id, ":: APPENDENTRIES, for term", r.currentTerm)

	n := rand.Intn(100)

	if n > 90 {
		debug.Debugln(r.id, ":: APPENDENTRIES, with log entry")
		r.log = append(r.log, &gorums.Entry{Term: r.currentTerm, Data: []byte(fmt.Sprintf("%d", r.currentTerm))})
	}

	// #L1
	for id, node := range r.nodes {
		entries := []*gorums.Entry{}

		nextIndex := r.nextIndex[id] - 1

		if len(r.log) > nextIndex {
			entries = r.log[nextIndex : nextIndex+1]
		}

		req := &gorums.AppendEntriesRequest{
			LeaderID:     r.id,
			Term:         r.currentTerm,
			PrevLogIndex: uint64(nextIndex),
			PrevLogTerm:  r.logTerm(nextIndex),
			Entries:      entries,
		}

		go func(node *gorums.Node, req *gorums.AppendEntriesRequest) {
			ctx, cancel := context.WithTimeout(context.Background(), APPENDENTRIES)
			defer cancel()
			resp, err := node.RaftClient.AppendEntries(ctx, req)

			if err != nil {
				log.Println(err)
			} else {
				r.handleAppendEntriesResponse(resp)
			}
		}(node, req)
	}

	r.heartbeat.Reset(r.heartbeatTimeout)
}

func (r *Replica) handleAppendEntriesResponse(response *gorums.AppendEntriesResponse) {
	r.Lock()
	defer r.Unlock()

	// #A2 If RPC request or response contains term T > currentTerm: set currentTerm = T, convert to follower.
	if response.Term > r.currentTerm {
		r.becomeFollower(response.Term)

		return
	}

	// Ignore late response
	if response.Term < r.currentTerm {
		return
	}

	if r.state == LEADER {
		if response.Success {
			r.matchIndex[response.FollowerID] = int(response.MatchIndex)
			r.nextIndex[response.FollowerID] = r.matchIndex[response.FollowerID] + 1

			return
		}

		r.nextIndex[response.FollowerID] = int(max(0, uint64(r.nextIndex[response.FollowerID]-1)))
	}
}

func (r *Replica) becomeFollower(term uint64) {
	debug.Debugln(r.id, ":: STEPDOWN,", r.currentTerm, "->", term)

	r.state = FOLLOWER
	r.currentTerm = term
	r.votedFor = NONE

	r.election.Reset(r.electionTimeout)
	r.heartbeat.Stop()
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net/http"

	"github.com/austinkelmore/catarang/catarang"
	"github.com/austinkelmore/catarang/web"
)

func main() {
	log.Println("Running Catarang!")
	// if len(os.Args) >= 2 && os.Args[1] != "" {
	// 	greeter.ClientPluginStart()
	// 	return
	// }

	log.SetFlags(log.LstdFlags | log.Lshortfile)
	catarang.ReadInConfig()

	catarang.AddJob("ping_test", "../ping_test")

	r := web.CreateRoutes()
	http.Handle("/", r)

	// greeter.ServerStart(os.Args)

	log.Fatal(http.ListenAndServe(":8080", nil))
}
<commit_msg>Removing the quick test I was using to add my ping_test, since that shouldn't have been committed.<commit_after>package main

import (
	"log"
	"net/http"

	"github.com/austinkelmore/catarang/catarang"
	"github.com/austinkelmore/catarang/web"
)

func main() {
	log.Println("Running Catarang!")
	// if len(os.Args) >= 2 && os.Args[1] != "" {
	// 	greeter.ClientPluginStart()
	// 	return
	// }

	log.SetFlags(log.LstdFlags | log.Lshortfile)
	catarang.ReadInConfig()

	r := web.CreateRoutes()
	http.Handle("/", r)

	// greeter.ServerStart(os.Args)

	log.Fatal(http.ListenAndServe(":8080", nil))
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Command server launches a stand-alone inverting proxy.
//
// Example usage:
//   go build -o ~/bin/inverting-proxy ./server/server.go
//   ~/bin/inverting-proxy --port 8081
package main

import (
	"bufio"
	"context"
	"crypto/sha256"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"math/rand"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/google/inverting-proxy/agent/utils"
)

var (
	port = flag.Int("port", 0, "Port on which to listen")
)

// pendingRequest represents a frontend request
type pendingRequest struct {
	startTime time.Time
	req       *http.Request
	respChan  chan *http.Response
}

func newPendingRequest(r *http.Request) *pendingRequest {
	return &pendingRequest{
		startTime: time.Now(),
		req:       r,
		respChan:  make(chan *http.Response),
	}
}

type proxy struct {
	requestIDs    chan string
	randGenerator *rand.Rand

	// protects the map below
	sync.Mutex
	requests map[string]*pendingRequest
}

func newProxy() *proxy {
	return &proxy{
		requestIDs:    make(chan string),
		randGenerator: rand.New(rand.NewSource(time.Now().UnixNano())),
		requests:      make(map[string]*pendingRequest),
	}
}

func (p *proxy) handleAgentPostResponse(w http.ResponseWriter, r *http.Request, requestID string) {
	p.Lock()
	pending, ok := p.requests[requestID]
	p.Unlock()
	if !ok {
		log.Printf("Could not find pending request: %q", requestID)
		http.NotFound(w, r)
		return
	}
	resp, err := http.ReadResponse(bufio.NewReader(r.Body), pending.req)
	if err != nil {
		log.Printf("Could not parse response to request %q: %v", requestID, err)
		http.Error(w, "Failure parsing request body", http.StatusBadRequest)
		return
	}
	// We want to track whether or not the body has finished being read so that we can
	// make sure that this method does not return until after that. However, we do not
	// want to block the sending of the response to the client while it is being read.
	//
	// To accommodate both goals, we replace the response body with a pipereader, start
	// forwarding the response immediately, and then copy the original body to the
	// corresponding pipewriter.
	respBody := resp.Body
	defer respBody.Close()

	pr, pw := io.Pipe()
	defer pw.Close()

	resp.Body = pr
	select {
	case <-r.Context().Done():
		return
	case pending.respChan <- resp:
	}
	if _, err := io.Copy(pw, respBody); err != nil {
		log.Printf("Could not read response to request %q: %v", requestID, err)
		http.Error(w, "Failure reading request body", http.StatusInternalServerError)
	}
}

func (p *proxy) handleAgentGetRequest(w http.ResponseWriter, r *http.Request, requestID string) {
	p.Lock()
	pending, ok := p.requests[requestID]
	p.Unlock()
	if !ok {
		log.Printf("Could not find pending request: %q", requestID)
		http.NotFound(w, r)
		return
	}
	log.Printf("Returning pending request: %q", requestID)
	w.Header().Set(utils.HeaderRequestStartTime, pending.startTime.Format(time.RFC3339Nano))
	w.WriteHeader(http.StatusOK)
	pending.req.Write(w)
}

// waitForRequestIDs blocks until at least one request ID is available, and then returns
// a slice of all of the IDs available at that time.
//
// Note that any IDs returned by this method will never be returned again.
func (p *proxy) waitForRequestIDs(ctx context.Context) []string {
	var requestIDs []string
	select {
	case <-ctx.Done():
		return nil
	case <-time.After(30 * time.Second):
		return nil
	case id := <-p.requestIDs:
		requestIDs = append(requestIDs, id)
	}
	for {
		select {
		case id := <-p.requestIDs:
			requestIDs = append(requestIDs, id)
		default:
			return requestIDs
		}
	}
}

func (p *proxy) handleAgentListRequests(w http.ResponseWriter, r *http.Request) {
	requestIDs := p.waitForRequestIDs(r.Context())
	respJSON, err := json.Marshal(requestIDs)
	if err != nil {
		http.Error(w, fmt.Sprintf("Failure serializing the request IDs: %v", err), http.StatusInternalServerError)
		return
	}
	log.Printf("Reporting pending requests: %s", respJSON)
	w.WriteHeader(http.StatusOK)
	w.Write(respJSON)
}

func (p *proxy) handleAgentRequest(w http.ResponseWriter, r *http.Request, backendID string) {
	requestID := r.Header.Get(utils.HeaderRequestID)
	if requestID == "" {
		log.Printf("Received new backend list request from %q", backendID)
		p.handleAgentListRequests(w, r)
		return
	}
	if r.Method == http.MethodPost {
		log.Printf("Received new backend post request from %q", backendID)
		p.handleAgentPostResponse(w, r, requestID)
		return
	}
	log.Printf("Received new backend get request from %q", backendID)
	p.handleAgentGetRequest(w, r, requestID)
}

func (p *proxy) newID() string {
	sum := sha256.Sum256([]byte(fmt.Sprintf("%d", p.randGenerator.Int63())))
	return fmt.Sprintf("%x", sum)
}

// isHopByHopHeader determines whether or not the given header name represents
// a header that is specific to a single network hop and thus should not be
// retransmitted by a proxy.
//
// See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers#hbh
func isHopByHopHeader(name string) bool {
	switch strings.ToLower(name) {
	case "connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailer", "transfer-encoding", "upgrade":
		return true
	default:
		return false
	}
}

func (p *proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if backendID := r.Header.Get(utils.HeaderBackendID); backendID != "" {
		p.handleAgentRequest(w, r, backendID)
		return
	}
	id := p.newID()
	log.Printf("Received new frontend request %q", id)
	// Filter out hop-by-hop headers from the request
	for name := range r.Header {
		if isHopByHopHeader(name) {
			r.Header.Del(name)
		}
	}
	pending := newPendingRequest(r)
	p.Lock()
	p.requests[id] = pending
	p.Unlock()

	// Enqueue the request
	select {
	case <-r.Context().Done():
		// The client request was cancelled
		log.Printf("Timeout waiting to enqueue the request ID for %q", id)
		return
	case p.requestIDs <- id:
	}
	log.Printf("Request %q enqueued after %s", id, time.Since(pending.startTime))

	// Pull out and copy the response
	select {
	case <-r.Context().Done():
		// The client request was cancelled
		log.Printf("Timeout waiting for the response to %q", id)
		return
	case resp := <-pending.respChan:
		defer resp.Body.Close()

		// Copy all of the non-hop-by-hop headers to the proxied response
		for name, vals := range resp.Header {
			if !isHopByHopHeader(name) {
				w.Header()[name] = vals
			}
		}
		w.WriteHeader(resp.StatusCode)
		io.Copy(w, resp.Body)
		return
	}
	log.Printf("Response for %q received after %s", id, time.Since(pending.startTime))
}

func main() {
	flag.Parse()
	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
	if err != nil {
		log.Fatalf("Failed to create the TCP listener for port %d: %v", *port, err)
	}
	log.Printf("Listening on %s", listener.Addr())
	log.Fatal(http.Serve(listener, newProxy()))
}
<commit_msg>Revert additional whitespace<commit_after>/*
Copyright 2018 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Command server launches a stand-alone inverting proxy.
//
// Example usage:
//   go build -o ~/bin/inverting-proxy ./server/server.go
//   ~/bin/inverting-proxy --port 8081
package main

import (
	"bufio"
	"context"
	"crypto/sha256"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"math/rand"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/google/inverting-proxy/agent/utils"
)

var (
	port = flag.Int("port", 0, "Port on which to listen")
)

// pendingRequest represents a frontend request
type pendingRequest struct {
	startTime time.Time
	req       *http.Request
	respChan  chan *http.Response
}

func newPendingRequest(r *http.Request) *pendingRequest {
	return &pendingRequest{
		startTime: time.Now(),
		req:       r,
		respChan:  make(chan *http.Response),
	}
}

type proxy struct {
	requestIDs    chan string
	randGenerator *rand.Rand

	// protects the map below
	sync.Mutex
	requests map[string]*pendingRequest
}

func newProxy() *proxy {
	return &proxy{
		requestIDs:    make(chan string),
		randGenerator: rand.New(rand.NewSource(time.Now().UnixNano())),
		requests:      make(map[string]*pendingRequest),
	}
}

func (p *proxy) handleAgentPostResponse(w http.ResponseWriter, r *http.Request, requestID string) {
	p.Lock()
	pending, ok := p.requests[requestID]
	p.Unlock()
	if !ok {
		log.Printf("Could not find pending request: %q", requestID)
		http.NotFound(w, r)
		return
	}
	resp, err := http.ReadResponse(bufio.NewReader(r.Body), pending.req)
	if err != nil {
		log.Printf("Could not parse response to request %q: %v", requestID, err)
		http.Error(w, "Failure parsing request body", http.StatusBadRequest)
		return
	}
	// We want to track whether or not the body has finished being read so that we can
	// make sure that this method does not return until after that. However, we do not
	// want to block the sending of the response to the client while it is being read.
	//
	// To accommodate both goals, we replace the response body with a pipereader, start
	// forwarding the response immediately, and then copy the original body to the
	// corresponding pipewriter.
	respBody := resp.Body
	defer respBody.Close()

	pr, pw := io.Pipe()
	defer pw.Close()

	resp.Body = pr
	select {
	case <-r.Context().Done():
		return
	case pending.respChan <- resp:
	}
	if _, err := io.Copy(pw, respBody); err != nil {
		log.Printf("Could not read response to request %q: %v", requestID, err)
		http.Error(w, "Failure reading request body", http.StatusInternalServerError)
	}
}

func (p *proxy) handleAgentGetRequest(w http.ResponseWriter, r *http.Request, requestID string) {
	p.Lock()
	pending, ok := p.requests[requestID]
	p.Unlock()
	if !ok {
		log.Printf("Could not find pending request: %q", requestID)
		http.NotFound(w, r)
		return
	}
	log.Printf("Returning pending request: %q", requestID)
	w.Header().Set(utils.HeaderRequestStartTime, pending.startTime.Format(time.RFC3339Nano))
	w.WriteHeader(http.StatusOK)
	pending.req.Write(w)
}

// waitForRequestIDs blocks until at least one request ID is available, and then returns
// a slice of all of the IDs available at that time.
//
// Note that any IDs returned by this method will never be returned again.
func (p *proxy) waitForRequestIDs(ctx context.Context) []string {
	var requestIDs []string
	select {
	case <-ctx.Done():
		return nil
	case <-time.After(30 * time.Second):
		return nil
	case id := <-p.requestIDs:
		requestIDs = append(requestIDs, id)
	}
	for {
		select {
		case id := <-p.requestIDs:
			requestIDs = append(requestIDs, id)
		default:
			return requestIDs
		}
	}
}

func (p *proxy) handleAgentListRequests(w http.ResponseWriter, r *http.Request) {
	requestIDs := p.waitForRequestIDs(r.Context())
	respJSON, err := json.Marshal(requestIDs)
	if err != nil {
		http.Error(w, fmt.Sprintf("Failure serializing the request IDs: %v", err), http.StatusInternalServerError)
		return
	}
	log.Printf("Reporting pending requests: %s", respJSON)
	w.WriteHeader(http.StatusOK)
	w.Write(respJSON)
}

func (p *proxy) handleAgentRequest(w http.ResponseWriter, r *http.Request, backendID string) {
	requestID := r.Header.Get(utils.HeaderRequestID)
	if requestID == "" {
		log.Printf("Received new backend list request from %q", backendID)
		p.handleAgentListRequests(w, r)
		return
	}
	if r.Method == http.MethodPost {
		log.Printf("Received new backend post request from %q", backendID)
		p.handleAgentPostResponse(w, r, requestID)
		return
	}
	log.Printf("Received new backend get request from %q", backendID)
	p.handleAgentGetRequest(w, r, requestID)
}

func (p *proxy) newID() string {
	sum := sha256.Sum256([]byte(fmt.Sprintf("%d", p.randGenerator.Int63())))
	return fmt.Sprintf("%x", sum)
}

// isHopByHopHeader determines whether or not the given header name represents
// a header that is specific to a single network hop and thus should not be
// retransmitted by a proxy.
//
// See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers#hbh
func isHopByHopHeader(name string) bool {
	switch strings.ToLower(name) {
	case "connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailer", "transfer-encoding", "upgrade":
		return true
	default:
		return false
	}
}

func (p *proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if backendID := r.Header.Get(utils.HeaderBackendID); backendID != "" {
		p.handleAgentRequest(w, r, backendID)
		return
	}
	id := p.newID()
	log.Printf("Received new frontend request %q", id)
	// Filter out hop-by-hop headers from the request
	for name := range r.Header {
		if isHopByHopHeader(name) {
			r.Header.Del(name)
		}
	}
	pending := newPendingRequest(r)
	p.Lock()
	p.requests[id] = pending
	p.Unlock()

	// Enqueue the request
	select {
	case <-r.Context().Done():
		// The client request was cancelled
		log.Printf("Timeout waiting to enqueue the request ID for %q", id)
		return
	case p.requestIDs <- id:
	}
	log.Printf("Request %q enqueued after %s", id, time.Since(pending.startTime))

	// Pull out and copy the response
	select {
	case <-r.Context().Done():
		// The client request was cancelled
		log.Printf("Timeout waiting for the response to %q", id)
		return
	case resp := <-pending.respChan:
		defer resp.Body.Close()
		// Copy all of the non-hop-by-hop headers to the proxied response
		for name, vals := range resp.Header {
			if !isHopByHopHeader(name) {
				w.Header()[name] = vals
			}
		}
		w.WriteHeader(resp.StatusCode)
		io.Copy(w, resp.Body)
		return
	}
	log.Printf("Response for %q received after %s", id, time.Since(pending.startTime))
}

func main() {
	flag.Parse()
	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
	if err != nil {
		log.Fatalf("Failed to create the TCP listener for port %d: %v", *port, err)
	}
	log.Printf("Listening on %s", listener.Addr())
	log.Fatal(http.Serve(listener, newProxy()))
}
<|endoftext|>


<commit_before>package server

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"

	"bazil.org/bazil/cas/chunks/kvchunks"
	"bazil.org/bazil/db"
	"bazil.org/bazil/fs"
	"bazil.org/bazil/kv"
	"bazil.org/bazil/kv/kvfiles"
	"bazil.org/bazil/kv/kvmulti"
	"bazil.org/bazil/kv/kvpeer"
	"bazil.org/bazil/kv/untrusted"
	"bazil.org/bazil/peer"
	"bazil.org/bazil/tokens"
	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
	"github.com/boltdb/bolt"
)

type mountState struct {
	// closed after the serve loop has exited
	unmounted chan struct{}
}

type App struct {
	DataDir  string
	lockFile *os.File
	DB       *db.DB
	mounts   struct {
		sync.Mutex
		open map[db.VolumeID]*mountState
	}
	Keys *CryptoKeys
	tls  struct {
		config atomic.Value
		gen    sync.Mutex
	}
}

func New(dataDir string) (app *App, err error) {
	err = os.Mkdir(dataDir, 0700)
	if err != nil && !os.IsExist(err) {
		return nil, err
	}

	lockPath := filepath.Join(dataDir, "lock")
	lockFile, err := lock(lockPath)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			// if we're reporting an error, also unlock
			_ = lockFile.Close()
		}
	}()

	kvpath := filepath.Join(dataDir, "chunks")
	err = kvfiles.Create(kvpath)
	if err != nil {
		return nil, err
	}

	dbpath := filepath.Join(dataDir, "bazil.bolt")
	database, err := db.Open(dbpath, 0600, nil)
	if err != nil {
		return nil, err
	}

	err = database.DB.Update(func(tx *bolt.Tx) error {
		if _, err := tx.CreateBucketIfNotExists([]byte(tokens.BucketBazil)); err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		database.Close()
		return nil, err
	}

	keys, err := loadOrGenerateKeys(database.DB)
	if err != nil {
		return nil, err
	}

	app = &App{
		DataDir:  dataDir,
		lockFile: lockFile,
		DB:       database,
		Keys:     keys,
	}
	app.mounts.open = make(map[db.VolumeID]*mountState)
	return app, nil
}

func (app *App) Close() {
	app.DB.Close()
	app.lockFile.Close()
}

// TODO this function smells
func (app *App) serveMount(vol *fs.Volume, id *db.VolumeID, mountpoint string) error {
	conn, err := fuse.Mount(mountpoint,
		fuse.MaxReadahead(32*1024*1024),
		fuse.AsyncRead(),
	)
	if err != nil {
		// remove map entry if the mount never took place
		app.mounts.Lock()
		delete(app.mounts.open, *id)
		app.mounts.Unlock()
		return fmt.Errorf("mount fail: %v", err)
	}

	serveErr := make(chan error, 1)
	go func() {
		defer func() {
			// remove map entry on unmount or failed mount
			app.mounts.Lock()
			delete(app.mounts.open, *id)
			app.mounts.Unlock()
		}()
		defer conn.Close()
		serveErr <- fusefs.Serve(conn, vol)
	}()

	select {
	case <-conn.Ready:
		if conn.MountError != nil {
			return fmt.Errorf("mount fail (delayed): %v", err)
		}
		return nil
	case err = <-serveErr:
		// Serve quit early
		if err != nil {
			return fmt.Errorf("filesystem failure: %v", err)
		}
		return errors.New("Serve exited early")
	}
}

type MountInfo struct {
	VolumeID db.VolumeID
}

func (app *App) openKV(tx *db.Tx, storage *db.VolumeStorage) (kv.KV, error) {
	var kvstores []kv.KV

	c := storage.Cursor()
	for item := c.First(); item != nil; item = c.Next() {
		backend, err := item.Backend()
		if err != nil {
			return nil, err
		}
		s, err := app.openStorage(backend)
		if err != nil {
			return nil, err
		}

		sharingKeyName, err := item.SharingKeyName()
		if err != nil {
			return nil, err
		}
		sharingKey, err := tx.SharingKeys().Get(sharingKeyName)
		if err != nil {
			return nil, fmt.Errorf("getting sharing key %q: %v", sharingKeyName, err)
		}
		var secret [32]byte
		sharingKey.Secret(&secret)
		s = untrusted.New(s, &secret)

		kvstores = append(kvstores, s)
	}

	return kvmulti.New(kvstores...), nil
}

func (app *App) openStorage(backend string) (kv.KV, error) {
	switch backend {
	case "local":
		kvpath := filepath.Join(app.DataDir, "chunks")
		return kvfiles.Open(kvpath)
	}
	if backend != "" && backend[0] == '/' {
		return kvfiles.Open(backend)
	}
	idx := strings.IndexByte(backend, ':')
	if idx != -1 {
		scheme, rest := backend[:idx], backend[idx+1:]
		switch scheme {
		case "peerkey":
			var key peer.PublicKey
			if err := key.Set(rest); err != nil {
				return nil, err
			}
			p, err := app.DialPeer(&key)
			if err != nil {
				return nil, err
			}
			// TODO Close
			return kvpeer.Open(p)
		}
	}
	return nil, errors.New("unknown storage backend")
}

func (app *App) ValidateKV(backend string) error {
	// TODO opening a KV just to validate the input string would be a
	// bad idea if a backend was costly to open; then again, current
	// API doesn't support a Close anyway.
	_, err := app.openStorage(backend)
	return err
}

// Mount makes the contents of the volume visible at the given
// mountpoint. If Mount returns with a nil error, the mount has
// occurred.
func (app *App) Mount(volumeName string, mountpoint string) (*MountInfo, error) {
	// TODO obey `bazil -debug server run`

	var vol *fs.Volume
	info := &MountInfo{}
	var ready = make(chan error, 1)
	app.mounts.Lock()
	err := app.DB.View(func(tx *db.Tx) error {
		v, err := tx.Volumes().GetByName(volumeName)
		if err != nil {
			return err
		}
		v.VolumeID(&info.VolumeID)

		if _, ok := app.mounts.open[info.VolumeID]; ok {
			return errors.New("volume already mounted")
		}

		kvstore, err := app.openKV(tx, v.Storage())
		if err != nil {
			return err
		}

		chunkStore := kvchunks.New(kvstore)
		vol, err = fs.Open(app.DB, chunkStore, &info.VolumeID, (*peer.PublicKey)(app.Keys.Sign.Pub))
		if err != nil {
			return err
		}
		mnt := &mountState{
			unmounted: make(chan struct{}),
		}
		go func() {
			defer close(mnt.unmounted)
			ready <- app.serveMount(vol, &info.VolumeID, mountpoint)
		}()
		app.mounts.open[info.VolumeID] = mnt
		return nil
	})
	app.mounts.Unlock()
	if err != nil {
		return nil, err
	}
	err = <-ready
	if err != nil {
		return nil, err
	}
	return info, nil
}

var ErrNotMounted = errors.New("not currently mounted")

func (app *App) WaitForUnmount(volumeID *db.VolumeID) error {
	app.mounts.Lock()
	// we hold onto mnt after releasing the lock, but it's safe in
	// this case; gc keeps it pinned, and we don't look at mutable
	// data
	mnt, ok := app.mounts.open[*volumeID]
	app.mounts.Unlock()
	if !ok {
		return ErrNotMounted
	}
	<-mnt.unmounted
	return nil
}
<commit_msg>server: Refactor to call fuse.Mount earlier<commit_after>package server

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"

	"bazil.org/bazil/cas/chunks/kvchunks"
	"bazil.org/bazil/db"
	"bazil.org/bazil/fs"
	"bazil.org/bazil/kv"
	"bazil.org/bazil/kv/kvfiles"
	"bazil.org/bazil/kv/kvmulti"
	"bazil.org/bazil/kv/kvpeer"
	"bazil.org/bazil/kv/untrusted"
	"bazil.org/bazil/peer"
	"bazil.org/bazil/tokens"
	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
	"github.com/boltdb/bolt"
)

type mountState struct {
	// closed after the serve loop has exited
	unmounted chan struct{}
}

type App struct {
	DataDir  string
	lockFile *os.File
	DB       *db.DB
	mounts   struct {
		sync.Mutex
		open map[db.VolumeID]*mountState
	}
	Keys *CryptoKeys
	tls  struct {
		config atomic.Value
		gen    sync.Mutex
	}
}

func New(dataDir string) (app *App, err error) {
	err = os.Mkdir(dataDir, 0700)
	if err != nil && !os.IsExist(err) {
		return nil, err
	}

	lockPath := filepath.Join(dataDir, "lock")
	lockFile, err := lock(lockPath)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			// if we're reporting an error, also unlock
			_ = lockFile.Close()
		}
	}()

	kvpath := filepath.Join(dataDir, "chunks")
	err = kvfiles.Create(kvpath)
	if err != nil {
		return nil, err
	}

	dbpath := filepath.Join(dataDir, "bazil.bolt")
	database, err := db.Open(dbpath, 0600, nil)
	if err != nil {
		return nil, err
	}

	err = database.DB.Update(func(tx *bolt.Tx) error {
		if _, err := tx.CreateBucketIfNotExists([]byte(tokens.BucketBazil)); err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		database.Close()
		return nil, err
	}

	keys, err := loadOrGenerateKeys(database.DB)
	if err != nil {
		return nil, err
	}

	app = &App{
		DataDir:  dataDir,
		lockFile: lockFile,
		DB:       database,
		Keys:     keys,
	}
	app.mounts.open = make(map[db.VolumeID]*mountState)
	return app, nil
}

func (app *App) Close() {
	app.DB.Close()
	app.lockFile.Close()
}

// TODO this function smells
func (app *App) serveMount(vol *fs.Volume, id *db.VolumeID, mountpoint string, conn *fuse.Conn) error {
	serveErr := make(chan error, 1)
	go func() {
		defer func() {
			// remove map entry on unmount or failed mount
			app.mounts.Lock()
			delete(app.mounts.open, *id)
			app.mounts.Unlock()
		}()
		defer conn.Close()
		serveErr <- fusefs.Serve(conn, vol)
	}()

	select {
	case <-conn.Ready:
		if err := conn.MountError; err != nil {
			return fmt.Errorf("mount fail (delayed): %v", err)
		}
		return nil
	case err := <-serveErr:
		// Serve quit early
		if err != nil {
			return fmt.Errorf("filesystem failure: %v", err)
		}
		return errors.New("Serve exited early")
	}
}

type MountInfo struct {
	VolumeID db.VolumeID
}

func (app *App) openKV(tx *db.Tx, storage *db.VolumeStorage) (kv.KV, error) {
	var kvstores []kv.KV

	c := storage.Cursor()
	for item := c.First(); item != nil; item = c.Next() {
		backend, err := item.Backend()
		if err != nil {
			return nil, err
		}
		s, err := app.openStorage(backend)
		if err != nil {
			return nil, err
		}

		sharingKeyName, err := item.SharingKeyName()
		if err != nil {
			return nil, err
		}
		sharingKey, err := tx.SharingKeys().Get(sharingKeyName)
		if err != nil {
			return nil, fmt.Errorf("getting sharing key %q: %v", sharingKeyName, err)
		}
		var secret [32]byte
		sharingKey.Secret(&secret)
		s = untrusted.New(s, &secret)

		kvstores = append(kvstores, s)
	}

	return kvmulti.New(kvstores...), nil
}

func (app *App) openStorage(backend string) (kv.KV, error) {
	switch backend {
	case "local":
		kvpath := filepath.Join(app.DataDir, "chunks")
		return kvfiles.Open(kvpath)
	}
	if backend != "" && backend[0] == '/' {
		return kvfiles.Open(backend)
	}
	idx := strings.IndexByte(backend, ':')
	if idx != -1 {
		scheme, rest := backend[:idx], backend[idx+1:]
		switch scheme {
		case "peerkey":
			var key peer.PublicKey
			if err := key.Set(rest); err != nil {
				return nil, err
			}
			p, err := app.DialPeer(&key)
			if err != nil {
				return nil, err
			}
			// TODO Close
			return kvpeer.Open(p)
		}
	}
	return nil, errors.New("unknown storage backend")
}

func (app *App) ValidateKV(backend string) error {
	// TODO opening a KV just to validate the input string would be a
	// bad idea if a backend was costly to open; then again, current
	// API doesn't support a Close anyway.
	_, err := app.openStorage(backend)
	return err
}

// Mount makes the contents of the volume visible at the given
// mountpoint. If Mount returns with a nil error, the mount has
// occurred.
func (app *App) Mount(volumeName string, mountpoint string) (*MountInfo, error) {
	// TODO obey `bazil -debug server run`

	info := &MountInfo{}
	var ready = make(chan error, 1)
	app.mounts.Lock()
	err := app.DB.View(func(tx *db.Tx) error {
		v, err := tx.Volumes().GetByName(volumeName)
		if err != nil {
			return err
		}
		v.VolumeID(&info.VolumeID)

		if _, ok := app.mounts.open[info.VolumeID]; ok {
			return errors.New("volume already mounted")
		}

		conn, err := fuse.Mount(mountpoint,
			fuse.MaxReadahead(32*1024*1024),
			fuse.AsyncRead(),
		)
		if err != nil {
			return fmt.Errorf("mount fail: %v", err)
		}

		kvstore, err := app.openKV(tx, v.Storage())
		if err != nil {
			return err
		}

		chunkStore := kvchunks.New(kvstore)
		vol, err := fs.Open(app.DB, chunkStore, &info.VolumeID, (*peer.PublicKey)(app.Keys.Sign.Pub))
		if err != nil {
			return err
		}
		mnt := &mountState{
			unmounted: make(chan struct{}),
		}
		go func() {
			defer close(mnt.unmounted)
			ready <- app.serveMount(vol, &info.VolumeID, mountpoint, conn)
		}()
		app.mounts.open[info.VolumeID] = mnt
		return nil
	})
	app.mounts.Unlock()
	if err != nil {
		return nil, err
	}
	err = <-ready
	if err != nil {
		return nil, err
	}
	return info, nil
}

var ErrNotMounted = errors.New("not currently mounted")

func (app *App) WaitForUnmount(volumeID *db.VolumeID) error {
	app.mounts.Lock()
	// we hold onto mnt after releasing the lock, but it's safe in
	// this case; gc keeps it pinned, and we don't look at mutable
	// data
	mnt, ok := app.mounts.open[*volumeID]
	app.mounts.Unlock()
	if !ok {
		return ErrNotMounted
	}
	<-mnt.unmounted
	return nil
}
<|endoftext|>


<commit_before>package server

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/micro/cli/v2"
	"github.com/micro/go-micro/v2"
	"github.com/micro/go-micro/v2/client"
	"github.com/micro/go-micro/v2/cmd"
	log "github.com/micro/go-micro/v2/logger"
	gorun "github.com/micro/go-micro/v2/runtime"
	"github.com/micro/go-micro/v2/util/file"
	ccli "github.com/micro/micro/v2/client/cli"
	"github.com/micro/micro/v2/internal/update"
	"github.com/micro/micro/v2/service/runtime/profile"
)

var (
	// list of services managed
	services = []string{
		// runtime services
		"config",   // ????
		"auth",     // :8010
		"network",  // :8085
		"runtime",  // :8088
		"registry", // :8000
		"broker",   // :8001
		"store",    // :8002
		"debug",    // :????
		"proxy",    // :8081
		"api",      // :8080
		"web",      // :8082
	}
)

var (
	// Name of the server microservice
	Name = "go.micro.server"
	// Address is the router microservice bind address
	Address = ":10001"
)

// upload is used for file uploads to the server
func upload(ctx *cli.Context, args []string) ([]byte, error) {
	if ctx.Args().Len() == 0 {
		return nil, errors.New("Required filename to upload")
	}

	filename := ctx.Args().Get(0)
	localfile := ctx.Args().Get(1)

	fileClient := file.New("go.micro.server", client.DefaultClient)
	return nil, fileClient.Upload(filename, localfile)
}

func Commands(options ...micro.Option) []*cli.Command {
	command := &cli.Command{
		Name:  "server",
		Usage: "Run the micro server",
		Description: `Launching the micro server ('micro server') will enable one to connect to it by
		setting the appropriate Micro environment (see 'micro env' && 'micro env --help') commands.`,
		Flags: []cli.Flag{
			&cli.StringFlag{
				Name:    "address",
				Usage:   "Set the micro server address :10001",
				EnvVars: []string{"MICRO_SERVER_ADDRESS"},
			},
			&cli.BoolFlag{
				Name:  "peer",
				Usage: "Peer with the global network to share services",
			},
			&cli.StringFlag{
				Name:    "profile",
				Usage:   "Set the runtime profile to use for services e.g local, kubernetes, platform",
				EnvVars: []string{"MICRO_RUNTIME_PROFILE"},
			},
		},
		Action: func(ctx *cli.Context) error {
			Run(ctx)
			return nil
		},
		Subcommands: []*cli.Command{{
			Name:  "file",
			Usage: "Move files between your local machine and the server",
			Subcommands: []*cli.Command{
				{
					Name:   "upload",
					Action: ccli.Print(upload),
				},
			},
		}},
	}

	for _, p := range Plugins() {
		if cmds := p.Commands(); len(cmds) > 0 {
			command.Subcommands = append(command.Subcommands, cmds...)
		}

		if flags := p.Flags(); len(flags) > 0 {
			command.Flags = append(command.Flags, flags...)
		}
	}

	return []*cli.Command{command}
}

// Run runs the entire platform
func Run(context *cli.Context) error {
	log.Init(log.WithFields(map[string]interface{}{"service": "micro"}))

	if context.Args().Len() > 0 {
		cli.ShowSubcommandHelp(context)
		os.Exit(1)
	}

	// get the network flag
	peer := context.Bool("peer")

	// pass through the environment
	// By default we want a file store when we run micro server.
	// This will get overridden if user has set their own MICRO_STORE env var or passed in --store
	env := []string{"MICRO_STORE=file"}
	env = append(env, os.Environ()...)

	// connect to the network if specified
	if peer {
		log.Info("Setting global network")

		if v := os.Getenv("MICRO_NETWORK_NODES"); len(v) == 0 {
			// set the resolver to use https://micro.mu/network
			env = append(env, "MICRO_NETWORK_NODES=network.micro.mu")
			log.Info("Setting default network micro.mu")
		}
		if v := os.Getenv("MICRO_NETWORK_TOKEN"); len(v) == 0 {
			// set the network token
			env = append(env, "MICRO_NETWORK_TOKEN=micro.mu")
			log.Info("Setting default network token")
		}
	}

	log.Info("Loading core services")

	// create new micro runtime
	muRuntime := cmd.DefaultCmd.Options().Runtime

	// Use default update notifier
	if context.Bool("auto_update") {
		updateURL := context.String("update_url")
		if len(updateURL) == 0 {
			updateURL = update.DefaultURL
		}

		options := []gorun.Option{
			gorun.WithScheduler(update.NewScheduler(updateURL, fmt.Sprintf("%d", time.Now().Unix()))),
		}
		(*muRuntime).Init(options...)
	}

	for _, service := range services {
		name := service

		if namespace := context.String("namespace"); len(namespace) > 0 {
			name = fmt.Sprintf("%s.%s", namespace, service)
		}

		log.Infof("Registering %s", name)
		args := []string{}
		envs := []string{}
		switch service {
		case "proxy", "web", "api", "bot", "cli":
			envs = profile.Server()
		default:
			// run non-clients as "micro service [name]"
			args = append(args, "service")
			envs = env
		}

		// pass the profile for the runtime
		if service == "runtime" {
			if p := context.String("profile"); len(p) > 0 {
				env = append(env, "MICRO_RUNTIME_PROFILE="+p)
			} else {
				env = append(env, "MICRO_RUNTIME_PROFILE=server")
			}
		}

		// we want to pass through the global args so go up one level in the context lineage
		if len(context.Lineage()) > 1 {
			globCtx := context.Lineage()[1]
			for _, f := range globCtx.FlagNames() {
				args = append(args, "--"+f, context.String(f))
			}
		}
		args = append(args, service)

		// runtime based on environment we run the service in
		opts := []gorun.CreateOption{
			gorun.WithCommand(os.Args[0]),
			gorun.WithArgs(args...),
			gorun.WithEnv(envs),
			gorun.WithOutput(os.Stdout),
			gorun.WithRetries(10),
			gorun.CreateImage("micro/micro"),
		}

		// NOTE: we use Version right now to check for the latest release
		muService := &gorun.Service{Name: name, Version: fmt.Sprintf("%d", time.Now().Unix())}
		if err := (*muRuntime).Create(muService, opts...); err != nil {
			log.Errorf("Failed to create runtime environment: %v", err)
			return err
		}
	}

	log.Info("Starting service runtime")

	// start the runtime
	if err := (*muRuntime).Start(); err != nil {
		log.Fatal(err)
		return err
	}

	log.Info("Service runtime started")

	// TODO: should we launch the console?
	// start the console
	// cli.Init(context)

	server := micro.NewService(
		micro.Name(Name),
		micro.Address(Address),
	)

	// @todo make this configurable
	uploadDir := filepath.Join(os.TempDir(), "micro", "uploads")
	os.MkdirAll(uploadDir, 0777)
	file.RegisterHandler(server.Server(), uploadDir)
	// start the server
	server.Run()

	log.Info("Stopping service runtime")

	// stop all the things
	if err := (*muRuntime).Stop(); err != nil {
		log.Fatal(err)
		return err
	}

	log.Info("Service runtime shutdown")

	// exit success
	os.Exit(0)
	return nil
}
<commit_msg>revert server changes<commit_after>package server

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/micro/cli/v2"
	"github.com/micro/go-micro/v2"
	"github.com/micro/go-micro/v2/client"
	"github.com/micro/go-micro/v2/cmd"
	log "github.com/micro/go-micro/v2/logger"
	gorun "github.com/micro/go-micro/v2/runtime"
	"github.com/micro/go-micro/v2/util/file"
	ccli "github.com/micro/micro/v2/client/cli"
	"github.com/micro/micro/v2/internal/update"
)

var (
	// list of services managed
	services = []string{
		// runtime services
		"config",   // ????
		"auth",     // :8010
		"network",  // :8085
		"runtime",  // :8088
		"registry", // :8000
		"broker",   // :8001
		"store",    // :8002
		"debug",    // :????
		"proxy",    // :8081
		"api",      // :8080
		"web",      // :8082
	}
)

var (
	// Name of the server microservice
	Name = "go.micro.server"
	// Address is the router microservice bind address
	Address = ":10001"
)

// upload is used for file uploads to the server
func upload(ctx *cli.Context, args []string) ([]byte, error) {
	if ctx.Args().Len() == 0 {
		return nil, errors.New("Required filename to upload")
	}

	filename := ctx.Args().Get(0)
	localfile := ctx.Args().Get(1)

	fileClient := file.New("go.micro.server", client.DefaultClient)
	return nil, fileClient.Upload(filename, localfile)
}

func Commands(options ...micro.Option) []*cli.Command {
	command := &cli.Command{
		Name:  "server",
		Usage: "Run the micro server",
		Description: `Launching the micro server ('micro server') will enable one to connect to it by
		setting the appropriate Micro environment (see 'micro env' && 'micro env --help') commands.`,
		Flags: []cli.Flag{
			&cli.StringFlag{
				Name:    "address",
				Usage:   "Set the micro server address :10001",
				EnvVars: []string{"MICRO_SERVER_ADDRESS"},
			},
			&cli.BoolFlag{
				Name:  "peer",
				Usage: "Peer with the global network to share services",
			},
			&cli.StringFlag{
				Name:    "profile",
				Usage:   "Set the runtime profile to use for services e.g local, kubernetes, platform",
				EnvVars: []string{"MICRO_RUNTIME_PROFILE"},
			},
		},
		Action: func(ctx *cli.Context) error {
			Run(ctx)
			return nil
		},
		Subcommands: []*cli.Command{{
			Name:  "file",
			Usage: "Move files between your local machine and the server",
			Subcommands: []*cli.Command{
				{
					Name:   "upload",
					Action: ccli.Print(upload),
				},
			},
		}},
	}

	for _, p := range Plugins() {
		if cmds := p.Commands(); len(cmds) > 0 {
			command.Subcommands = append(command.Subcommands, cmds...)
		}

		if flags := p.Flags(); len(flags) > 0 {
			command.Flags = append(command.Flags, flags...)
		}
	}

	return []*cli.Command{command}
}

// Run runs the entire platform
func Run(context *cli.Context) error {
	log.Init(log.WithFields(map[string]interface{}{"service": "micro"}))

	if context.Args().Len() > 0 {
		cli.ShowSubcommandHelp(context)
		os.Exit(1)
	}

	// get the network flag
	peer := context.Bool("peer")

	// pass through the environment
	// By default we want a file store when we run micro server.
	// This will get overridden if user has set their own MICRO_STORE env var or passed in --store
	env := []string{"MICRO_STORE=file"}
	profile := context.String("profile")
	if len(profile) == 0 {
		profile = "server"
	}

	env = append(env, "MICRO_RUNTIME_PROFILE="+profile)
	env = append(env, os.Environ()...)

	// connect to the network if specified
	if peer {
		log.Info("Setting global network")

		if v := os.Getenv("MICRO_NETWORK_NODES"); len(v) == 0 {
			// set the resolver to use https://micro.mu/network
			env = append(env, "MICRO_NETWORK_NODES=network.micro.mu")
			log.Info("Setting default network micro.mu")
		}
		if v := os.Getenv("MICRO_NETWORK_TOKEN"); len(v) == 0 {
			// set the network token
			env = append(env, "MICRO_NETWORK_TOKEN=micro.mu")
			log.Info("Setting default network token")
		}
	}

	log.Info("Loading core services")

	// create new micro runtime
	muRuntime := cmd.DefaultCmd.Options().Runtime

	// Use default update notifier
	if context.Bool("auto_update") {
		updateURL := context.String("update_url")
		if len(updateURL) == 0 {
			updateURL = update.DefaultURL
		}

		options := []gorun.Option{
			gorun.WithScheduler(update.NewScheduler(updateURL, fmt.Sprintf("%d", time.Now().Unix()))),
		}
		(*muRuntime).Init(options...)
	}

	for _, service := range services {
		name := service

		if namespace := context.String("namespace"); len(namespace) > 0 {
			name = fmt.Sprintf("%s.%s", namespace, service)
		}

		log.Infof("Registering %s", name)
		// @todo this is a hack
		envs := env
		cmdArgs := []string{}

		switch service {
		case "proxy", "web", "api", "bot", "cli":
			envs = append(envs, "MICRO_AUTH=service")
			envs = append(envs, "MICRO_REGISTRY=service")
		default:
			// run server as "micro service [cmd]"
			cmdArgs = append(cmdArgs, "service")
		}

		// we want to pass through the global args so go up one level in the context lineage
		if len(context.Lineage()) > 1 {
			globCtx := context.Lineage()[1]
			for _, f := range globCtx.FlagNames() {
				cmdArgs = append(cmdArgs, "--"+f, context.String(f))
			}
		}
		cmdArgs = append(cmdArgs, service)

		// runtime based on environment we run the service in
		args := []gorun.CreateOption{
			gorun.WithCommand(os.Args[0]),
			gorun.WithArgs(cmdArgs...),
			gorun.WithEnv(envs),
			gorun.WithOutput(os.Stdout),
			gorun.WithRetries(10),
			gorun.CreateImage("micro/micro"),
		}

		// NOTE: we use Version right now to check for the latest release
		muService := &gorun.Service{Name: name, Version: fmt.Sprintf("%d", time.Now().Unix())}
		if err := (*muRuntime).Create(muService, args...); err != nil {
			log.Errorf("Failed to create runtime environment: %v", err)
			return err
		}
	}

	log.Info("Starting service runtime")

	// start the runtime
	if err := (*muRuntime).Start(); err != nil {
		log.Fatal(err)
		return err
	}

	log.Info("Service runtime started")

	// TODO: should we launch the console?
	// start the console
	// cli.Init(context)

	server := micro.NewService(
		micro.Name(Name),
		micro.Address(Address),
	)

	// @todo make this configurable
	uploadDir := filepath.Join(os.TempDir(), "micro", "uploads")
	os.MkdirAll(uploadDir, 0777)
	file.RegisterHandler(server.Server(), uploadDir)
	// start the server
	server.Run()

	log.Info("Stopping service runtime")

	// stop all the things
	if err := (*muRuntime).Stop(); err != nil {
		log.Fatal(err)
		return err
	}

	log.Info("Service runtime shutdown")

	// exit success
	os.Exit(0)
	return nil
}
<|endoftext|>


<commit_before>// Gophers runs scripts or commands concurrently and kills them when gophers was killed.
package main

import (
	"fmt"
	"os"
	"os/exec"
	"sync"
)

func main() {
	if len(os.Args) == 1 || os.Args[0] == "-h" || os.Args[0] == "-help" {
		fmt.Print(`Usage of gophers:
	gophers command1 command2 ...
Gophers runs scripts or commands concurrently and kills them when gophers was killed.
`)
		os.Exit(0)
	}
	cmds := os.Args[1:]
	var wg sync.WaitGroup
	for _, cmd := range cmds {
		wg.Add(1)
		go func(cmd string) {
			ecmd := exec.Command("sh", "-c", cmd)
			ecmd.Stdout = os.Stdout
			ecmd.Stdin = os.Stdin
			ecmd.Stderr = os.Stderr
			if err := ecmd.Run(); err != nil {
				fmt.Fprintf(os.Stderr, "error (sh -c %s): %s", cmd, err)
			}
			wg.Done()
		}(cmd)
	}
	wg.Wait()
}
<commit_msg>Update docs<commit_after>// Gophers runs scripts or commands concurrently and kills them when gophers was killed.
//
// Usage
//  	gophers gulp "find . -iname '*.go' | entr -rd grm"
//
// What is grm: https://github.com/bom-d-van/bin/blob/master/grm
package main

import (
	"fmt"
	"os"
	"os/exec"
	"sync"
)

func main() {
	if len(os.Args) == 1 || os.Args[0] == "-h" || os.Args[0] == "-help" {
		fmt.Print(`Usage of gophers:
	gophers command1 command2 ...
Gophers runs scripts or commands concurrently and kills them when gophers was killed.
`)
		os.Exit(0)
	}
	cmds := os.Args[1:]
	var wg sync.WaitGroup
	for _, cmd := range cmds {
		wg.Add(1)
		go func(cmd string) {
			ecmd := exec.Command("sh", "-c", cmd)
			ecmd.Stdout = os.Stdout
			ecmd.Stdin = os.Stdin
			ecmd.Stderr = os.Stderr
			if err := ecmd.Run(); err != nil {
				fmt.Fprintf(os.Stderr, "error (sh -c %s): %s", cmd, err)
			}
			wg.Done()
		}(cmd)
	}
	wg.Wait()
}
<|endoftext|>