<commit_before>package s3

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"fmt"
	"io"
	"net/http"
	"strings"
	stdlog "log"

	"github.com/gorilla/mux"
	"github.com/gogo/protobuf/types"
	log "github.com/sirupsen/logrus"
	logutil "github.com/pachyderm/pachyderm/src/server/pkg/log"

	"github.com/pachyderm/pachyderm/src/client"
)

const locationResponse = `<?xml version="1.0" encoding="UTF-8"?>
<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">PACHYDERM</LocationConstraint>`

func writeOK(w http.ResponseWriter) {
	w.Write([]byte("OK"))
}

func writeBadRequest(w http.ResponseWriter, err error) {
	http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
}

func writeMaybeNotFound(w http.ResponseWriter, err error) {
	if strings.Contains(err.Error(), "not found") {
		// This error message matches what the mux router returns when it 404s
		http.Error(w, "404 page not found", http.StatusNotFound)
	} else {
		writeServerError(w, err)
	}
}

func writeServerError(w http.ResponseWriter, err error) {
	http.Error(w, fmt.Sprintf("%v", err), http.StatusInternalServerError)
}

type handler struct {
	pc     *client.APIClient
}

func (h handler) ping(w http.ResponseWriter, r *http.Request) {
	writeOK(w)
}

func (h handler) repo(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	repo := vars["repo"]

	if err := r.ParseForm(); err != nil {
		writeBadRequest(w, err)
		return
	}

	if _, err := h.pc.InspectRepo(repo); err != nil {
		writeMaybeNotFound(w, err)
		return
	}

	if _, ok := r.Form["location"]; ok {
		w.Header().Set("Content-Type", "application/xml")
		w.Write([]byte(locationResponse))
	} else {
		writeOK(w)
	}
}

func (h handler) getObject(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	repo := vars["repo"]
	branch := vars["branch"]
	file := vars["file"]

	fileInfo, err := h.pc.InspectFile(repo, branch, file)
	if err != nil {
		writeMaybeNotFound(w, err)
		return
	}

	timestamp, err := types.TimestampFromProto(fileInfo.Committed)
	if err != nil {
		writeServerError(w, err)
		return
	}

	reader, err := h.pc.GetFileReadSeeker(repo, branch, file)
	if err != nil {
		writeServerError(w, err)
		return
	}

	http.ServeContent(w, r, "", timestamp, reader)
}

func (h handler) putObject(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	repo := vars["repo"]
	branch := vars["branch"]
	file := vars["file"]

	expectedHash := r.Header.Get("Content-MD5")

	if expectedHash != "" {
		expectedHashBytes, err := base64.StdEncoding.DecodeString(expectedHash)

		if err != nil {
			writeBadRequest(w, fmt.Errorf("could not decode `Content-MD5`, as it is not base64-encoded"))
			return
		}

		h.putObjectVerifying(w, r, repo, branch, file, expectedHashBytes)
		return
	}

	h.putObjectUnverified(w, r, repo, branch, file)
}

func (h handler) putObjectVerifying(w http.ResponseWriter, r *http.Request, repo, branch, file string, expectedHash []byte) {
	hasher := md5.New()
	reader := io.TeeReader(r.Body, hasher)

	_, err := h.pc.PutFileOverwrite(repo, branch, file, reader, 0)
	if err != nil {
		// the error may be because the repo or branch does not exist -
		// double-check that by inspecting the branch, so we can serve a 404
		// instead
		_, inspectError := h.pc.InspectBranch(repo, branch)
		writeMaybeNotFound(w, inspectError)
		return
	}

	actualHash := hasher.Sum(nil)

	if !bytes.Equal(expectedHash, actualHash) {
		err = fmt.Errorf("content checksums differ; expected=%x, actual=%x", expectedHash, actualHash)
		writeServerError(w, err)
		return
	}

	writeOK(w)
}

func (h handler) putObjectUnverified(w http.ResponseWriter, r *http.Request, repo, branch, file string) {
	_, err := h.pc.PutFileOverwrite(repo, branch, file, r.Body, 0)
	if err != nil {
		// the error may be because the repo or branch does not exist -
		// double-check that by inspecting the branch, so we can serve a 404
		// instead
		_, inspectError := h.pc.InspectBranch(repo, branch)
		writeMaybeNotFound(w, inspectError)
		return
	}

	writeOK(w)
}

// Server runs an HTTP server with an S3-like API for PFS. This allows you to
// use s3 clients to acccess PFS contents.
//
// This returns an `http.Server` instance. It is the responsibility of the
// caller to start the server. This also makes it possible for the caller to
// enable graceful shutdown if desired; see the `http` package for details.
//
// Bucket names correspond to repo names, and files are accessible via the s3
// key pattern "<branch>/<filepath>". For example, to get the file "a/b/c.txt"
// on the "foo" repo's "master" branch, you'd making an s3 get request with
// bucket = "foo", key = "master/a/b/c.txt".
//
// Note: in s3, bucket names are constrained by IETF RFC 1123, (and its
// predecessor RFC 952) but pachyderm's repo naming constraints are slightly
// more liberal. If the s3 client does any kind of bucket name validation
// (this includes minio), repos whose names do not comply with RFC 1123 will
// not be accessible.
func Server(pc *client.APIClient, port uint16) *http.Server {
	handler := handler{pc: pc}

	router := mux.NewRouter()
	router.HandleFunc("/_ping", handler.ping).Methods("GET")
	router.HandleFunc("/{repo}/", handler.repo).Methods("GET")
	router.HandleFunc("/{repo}/{branch}/{file:.+}", handler.getObject).Methods("GET")
	router.HandleFunc("/{repo}/{branch}/{file:.+}", handler.putObject).Methods("PUT")

	writer := logutil.NewGRPCLogWriter(log.StandardLogger(), "s3gateway")
	logger := stdlog.New(writer, "", 0)

	return &http.Server{
		Addr:     fmt.Sprintf(":%d", port),
		Handler:  http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			log.Debugf("s3 gateway request: %s %s", r.Method, r.RequestURI)
			router.ServeHTTP(w, r)
		}),
		ErrorLog: logger,
	}
}
<commit_msg>Support for listing buckets<commit_after>package s3

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"fmt"
	"io"
	"net/http"
	"strings"
	"text/template"

	"github.com/gogo/protobuf/types"
	"github.com/gorilla/mux"
	log "github.com/sirupsen/logrus"

	"github.com/pachyderm/pachyderm/src/client"
)

const locationResponse = `<?xml version="1.0" encoding="UTF-8"?>
<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">PACHYDERM</LocationConstraint>`

const listBucketSource = `<?xml version="1.0" encoding="UTF-8"?>
<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
    <Owner>
    	<ID>000000000000000000000000000000</ID>
    	<DisplayName>pachyderm</DisplayName>
    </Owner>
    <Buckets>
        {{ range . }}
            <Bucket>
                <Name>{{ .Repo.Name }}</Name>
                <CreationDate>{{ formatTime .Created }}</CreationDate>
            </Bucket>
        {{ end }}
    </Buckets>
</ListAllMyBucketsResult>`

func writeOK(w http.ResponseWriter) {
	w.Write([]byte("OK"))
}

func writeBadRequest(w http.ResponseWriter, err error) {
	http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
}

func writeMaybeNotFound(w http.ResponseWriter, err error) {
	if strings.Contains(err.Error(), "not found") {
		// This error message matches what the mux router returns when it 404s
		http.Error(w, "404 page not found", http.StatusNotFound)
	} else {
		writeServerError(w, err)
	}
}

func writeServerError(w http.ResponseWriter, err error) {
	http.Error(w, fmt.Sprintf("%v", err), http.StatusInternalServerError)
}

type handler struct {
	pc                  *client.APIClient
	listBucketsTemplate *template.Template
}

func newHandler(pc *client.APIClient) handler {
	funcMap := template.FuncMap{
		"formatTime": func(timestamp *types.Timestamp) string {
			return timestamp.String()
		},
	}

	listBucketsTemplate := template.Must(template.New("list-buckets").
		Funcs(funcMap).
		Parse(listBucketSource))

	return handler{
		pc:                  pc,
		listBucketsTemplate: listBucketsTemplate,
	}
}

func (h handler) ping(w http.ResponseWriter, r *http.Request) {
	writeOK(w)
}

func (h handler) root(w http.ResponseWriter, r *http.Request) {
	buckets, err := h.pc.ListRepo()
	if err != nil {
		writeServerError(w, err)
		return
	}

	if err = h.listBucketsTemplate.Execute(w, buckets); err != nil {
		writeServerError(w, err)
		return
	}

	w.Header().Set("Content-Type", "application/xml")
}

func (h handler) repo(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	repo := vars["repo"]

	if err := r.ParseForm(); err != nil {
		writeBadRequest(w, err)
		return
	}

	if _, err := h.pc.InspectRepo(repo); err != nil {
		writeMaybeNotFound(w, err)
		return
	}

	if _, ok := r.Form["location"]; ok {
		w.Header().Set("Content-Type", "application/xml")
		w.Write([]byte(locationResponse))
	} else {
		writeOK(w)
	}
}

func (h handler) getObject(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	repo := vars["repo"]
	branch := vars["branch"]
	file := vars["file"]

	fileInfo, err := h.pc.InspectFile(repo, branch, file)
	if err != nil {
		writeMaybeNotFound(w, err)
		return
	}

	timestamp, err := types.TimestampFromProto(fileInfo.Committed)
	if err != nil {
		writeServerError(w, err)
		return
	}

	reader, err := h.pc.GetFileReadSeeker(repo, branch, file)
	if err != nil {
		writeServerError(w, err)
		return
	}

	http.ServeContent(w, r, "", timestamp, reader)
}

func (h handler) putObject(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	repo := vars["repo"]
	branch := vars["branch"]
	file := vars["file"]

	expectedHash := r.Header.Get("Content-MD5")

	if expectedHash != "" {
		expectedHashBytes, err := base64.StdEncoding.DecodeString(expectedHash)

		if err != nil {
			writeBadRequest(w, fmt.Errorf("could not decode `Content-MD5`, as it is not base64-encoded"))
			return
		}

		h.putObjectVerifying(w, r, repo, branch, file, expectedHashBytes)
		return
	}

	h.putObjectUnverified(w, r, repo, branch, file)
}

func (h handler) putObjectVerifying(w http.ResponseWriter, r *http.Request, repo, branch, file string, expectedHash []byte) {
	hasher := md5.New()
	reader := io.TeeReader(r.Body, hasher)

	_, err := h.pc.PutFileOverwrite(repo, branch, file, reader, 0)
	if err != nil {
		// the error may be because the repo or branch does not exist -
		// double-check that by inspecting the branch, so we can serve a 404
		// instead
		_, inspectError := h.pc.InspectBranch(repo, branch)
		writeMaybeNotFound(w, inspectError)
		return
	}

	actualHash := hasher.Sum(nil)

	if !bytes.Equal(expectedHash, actualHash) {
		err = fmt.Errorf("content checksums differ; expected=%x, actual=%x", expectedHash, actualHash)
		writeServerError(w, err)
		return
	}

	writeOK(w)
}

func (h handler) putObjectUnverified(w http.ResponseWriter, r *http.Request, repo, branch, file string) {
	_, err := h.pc.PutFileOverwrite(repo, branch, file, r.Body, 0)
	if err != nil {
		// the error may be because the repo or branch does not exist -
		// double-check that by inspecting the branch, so we can serve a 404
		// instead
		_, inspectError := h.pc.InspectBranch(repo, branch)
		writeMaybeNotFound(w, inspectError)
		return
	}

	writeOK(w)
}

// Server runs an HTTP server with an S3-like API for PFS. This allows you to
// use s3 clients to acccess PFS contents.
//
// This returns an `http.Server` instance. It is the responsibility of the
// caller to start the server. This also makes it possible for the caller to
// enable graceful shutdown if desired; see the `http` package for details.
//
// Bucket names correspond to repo names, and files are accessible via the s3
// key pattern "<branch>/<filepath>". For example, to get the file "a/b/c.txt"
// on the "foo" repo's "master" branch, you'd making an s3 get request with
// bucket = "foo", key = "master/a/b/c.txt".
//
// Note: in s3, bucket names are constrained by IETF RFC 1123, (and its
// predecessor RFC 952) but pachyderm's repo naming constraints are slightly
// more liberal. If the s3 client does any kind of bucket name validation
// (this includes minio), repos whose names do not comply with RFC 1123 will
// not be accessible.
func Server(pc *client.APIClient, port uint16) *http.Server {
	handler := newHandler(pc)

	router := mux.NewRouter()
	router.HandleFunc("/", handler.root).Methods("GET")
	router.HandleFunc("/{repo}/", handler.repo).Methods("GET")
	router.HandleFunc("/{repo}/{branch}/{file:.+}", handler.getObject).Methods("GET")
	router.HandleFunc("/{repo}/{branch}/{file:.+}", handler.putObject).Methods("PUT")
	router.HandleFunc("/_ping", handler.ping).Methods("GET")

	// Note: error log is not customized on this `http.Server`, which means
	// it'll default to using the stdlib logger and produce log messages that
	// don't look like the ones produced elsewhere, and aren't configured
	// properly. In testing, this didn't seem to be a big deal because it's
	// rather hard to trigger it anyways, but if we find a reliable way to
	// create error logs, it might be worthwhile to fix.
	return &http.Server{
		Addr: fmt.Sprintf(":%d", port),
		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			log.Debugf("s3 gateway request: %s %s", r.Method, r.RequestURI)
			router.ServeHTTP(w, r)
		}),
	}
}
<|endoftext|>


<commit_before>// Wrapper for kasia.go templates designed for easy writing web applications.
package kview

import (
	"fmt"
	"github.com/ziutek/kasia.go"
	"io"
	"log"
	"path"
	"reflect"
	"strings"
)

var (
	// You can modify this, if you store templates in a different directory.
	TemplatesDir = "templates"

	// You can modify this, if want a different error handling.
	ErrorHandler = func(name string, err error) {
		log.Printf("%%View '%s' error. %s\n", name, err.Error())
	}
)

type View interface {
	Copy() View
	Strict(bool)
	Div(string, View)
	Exec(io.Writer, ...interface{})
	Render(...interface{}) *kasia.NestedTemplate
}

// View definition
type KView struct {
	name    string
	tpl     *kasia.Template
	globals map[string]interface{}
}

// Returns a pointer to a page
func New(name string, globals ...map[string]interface{}) *KView {
	var (
		pg  KView
		err error
	)
	pg.name = name
	pg.tpl, err = kasia.ParseFile(path.Join(TemplatesDir, name))
	if err != nil {
		ErrorHandler(name, err)
	}
	pg.globals = make(map[string]interface{})
	// First some default globals
	for k, v := range Globals {
		pg.globals[k] = v
	}
	// globals may default
	for _, g := range globals {
		for k, v := range g {
			pg.globals[k] = v
		}
	}
	return &pg
}

// Returns a pointer to a copy of the page
func (pg *KView) Copy() View {
	new_pg := *pg
	// Make a copy of globals
	new_pg.globals = make(map[string]interface{})
	for k, v := range pg.globals {
		new_pg.globals[k] = v
	}
	return &new_pg
}

// Set strig render flag
func (pg *KView) Strict(strict bool) {
	pg.tpl.Strict = strict
}

// Add subview
func (pg *KView) Div(name string, view View) {
	pg.globals[name] = view
}

func prepend(slice []interface{}, pre ...interface{}) (ret []interface{}) {
	ret = make([]interface{}, len(slice)+len(pre))
	copy(ret, pre)
	copy(ret[len(pre):], slice)
	return
}

// Render view to wr with data
func (pg *KView) Exec(wr io.Writer, ctx ...interface{}) {
	// Add globals to the bottom of the context stack
	ctx = prepend(ctx, pg.globals)
	err := pg.tpl.Run(wr, ctx...)
	if err != nil {
		ErrorHandler(pg.name, err)
	}
}

// Use this method in template text to render page inside other page.
func (pg *KView) Render(ctx ...interface{}) *kasia.NestedTemplate {
	if len(ctx) > 0 {
		// Check if render was called with full template context as first arg
		if ci, ok := ctx[0].(kasia.ContextItself); ok {
			// Rearange context, remove old globals
			ctx = append(ci[1:], ctx[1:])
		}
	}
	// Add globals to the bottom of the context stack
	ctx = prepend(ctx, pg.globals)
	return pg.tpl.Nested(ctx...)
}

// Some useful functions for globals.
// You can add there your functions/variables which will be visable in any
// view. See also globals parameter in New function.
var Globals = map[string]interface{}{
	"len": func(a interface{}) int {
		v := reflect.ValueOf(a)
		if v.Kind() == reflect.Array || v.Kind() == reflect.Slice || v.Kind() == reflect.String {
			return v.Len()
		}
		return -1
	},
	"fmt": fmt.Sprintf,
	"join" : strings.Join,
	"add": func(a, b int) int {
		return a + b
	},
	"sub": func(a, b int) int {
		return a - b
	},
	"mul": func(a, b int) int {
		return a * b
	},
	"div": func(a, b int) int {
		return a / b
	},
	"mod": func(a, b int) int {
		return a % b
	},
	"and": func(a, b bool) bool {
		return a && b
	},
	"or": func(a, b bool) bool {
		return a || b
	},
	"not": func(a bool) bool {
		return !a
	},
}
<commit_msg>Some typos fixed<commit_after>// Wrapper for kasia.go templates designed for easy writing web applications.
package kview

import (
	"fmt"
	"github.com/ziutek/kasia.go"
	"io"
	"log"
	"path"
	"reflect"
	"strings"
)

var (
	// You can modify this, if you store templates in a different directory.
	TemplatesDir = "templates"

	// You can modify this, if you want a different error handling.
	ErrorHandler = func(name string, err error) {
		log.Printf("%%View '%s' error. %s\n", name, err.Error())
	}
)

type View interface {
	Copy() View
	Strict(bool)
	Div(string, View)
	Exec(io.Writer, ...interface{})
	Render(...interface{}) *kasia.NestedTemplate
}

// View definition
type KView struct {
	name    string
	tpl     *kasia.Template
	globals map[string]interface{}
}

// Returns a pointer to the page
func New(name string, globals ...map[string]interface{}) *KView {
	var (
		pg  KView
		err error
	)
	pg.name = name
	pg.tpl, err = kasia.ParseFile(path.Join(TemplatesDir, name))
	if err != nil {
		ErrorHandler(name, err)
	}
	pg.globals = make(map[string]interface{})
	// First some default globals
	for k, v := range Globals {
		pg.globals[k] = v
	}
	// globals may default
	for _, g := range globals {
		for k, v := range g {
			pg.globals[k] = v
		}
	}
	return &pg
}

// Returns a copy of the page
func (pg *KView) Copy() View {
	new_pg := *pg
	// Make a copy of globals
	new_pg.globals = make(map[string]interface{})
	for k, v := range pg.globals {
		new_pg.globals[k] = v
	}
	return &new_pg
}

// Set the strict render flag
func (pg *KView) Strict(strict bool) {
	pg.tpl.Strict = strict
}

// Add subview
func (pg *KView) Div(name string, view View) {
	pg.globals[name] = view
}

func prepend(slice []interface{}, pre ...interface{}) (ret []interface{}) {
	ret = make([]interface{}, len(slice)+len(pre))
	copy(ret, pre)
	copy(ret[len(pre):], slice)
	return
}

// Render view to wr with data
func (pg *KView) Exec(wr io.Writer, ctx ...interface{}) {
	// Add globals to the bottom of the context stack
	ctx = prepend(ctx, pg.globals)
	err := pg.tpl.Run(wr, ctx...)
	if err != nil {
		ErrorHandler(pg.name, err)
	}
}

// Use this method in template text to render page inside other page.
func (pg *KView) Render(ctx ...interface{}) *kasia.NestedTemplate {
	if len(ctx) > 0 {
		// Check if render was called with full template context as first arg
		if ci, ok := ctx[0].(kasia.ContextItself); ok {
			// Rearange context, remove old globals
			ctx = append(ci[1:], ctx[1:])
		}
	}
	// Add globals to the bottom of the context stack
	ctx = prepend(ctx, pg.globals)
	return pg.tpl.Nested(ctx...)
}

// Some useful functions for globals.
// You can add there your functions/variables which will be visable in any
// view. See also globals parameter in New function.
var Globals = map[string]interface{}{
	"len": func(a interface{}) int {
		v := reflect.ValueOf(a)
		if v.Kind() == reflect.Array || v.Kind() == reflect.Slice || v.Kind() == reflect.String {
			return v.Len()
		}
		return -1
	},
	"fmt": fmt.Sprintf,
	"join" : strings.Join,
	"add": func(a, b int) int {
		return a + b
	},
	"sub": func(a, b int) int {
		return a - b
	},
	"mul": func(a, b int) int {
		return a * b
	},
	"div": func(a, b int) int {
		return a / b
	},
	"mod": func(a, b int) int {
		return a % b
	},
	"and": func(a, b bool) bool {
		return a && b
	},
	"or": func(a, b bool) bool {
		return a || b
	},
	"not": func(a bool) bool {
		return !a
	},
}
<|endoftext|>


<commit_before>package lclip

import (
	"os"
	"path/filepath"

	_ "github.com/mattn/go-sqlite3"
	"github.com/mitchellh/go-homedir"
	"github.com/naoina/genmai"
)

func exists(path string) bool {
	_, err := os.Stat(path)
	return err == nil
}

func DefaultPath() (string, error) {
	h, err := homedir.Dir()
	if err != nil {
		return "", err
	}
	return filepath.Join(h, ".lclip.json"), nil
}

type Variable struct {
	Label string `db:"pk"`
	Data  []byte
}

type Clipboard struct {
	path string
	db   *genmai.DB
}

func NewClipboard(path string) (*Clipboard, error) {
	db, err := genmai.New(&genmai.SQLite3Dialect{}, path)
	if err != nil {
		return nil, err
	}
	if err = db.CreateTableIfNotExists(&Variable{}); err != nil {
		return nil, err
	}
	return &Clipboard{
		path: path,
		db:   db,
	}, nil
}

func NewClipboardWithDefaultPath() (*Clipboard, error) {
	path, err := DefaultPath()
	if err != nil {
		return nil, err
	}
	return NewClipboard(path)
}

func (c *Clipboard) Path() string {
	return c.path
}

func (c *Clipboard) searchVariable(label string) (*Variable, error) {
	res := make([]Variable, 0, 1)
	err := c.db.Select(&res, c.db.Where("label", "=", label))
	if err != nil {
		return nil, err
	}
	if len(res) < 1 {
		return nil, nil
	}
	return &res[0], nil
}

func (c *Clipboard) Get(label string) ([]byte, error) {
	v, err := c.searchVariable(label)
	if err != nil {
		return nil, err
	}
	if v == nil {
		return []byte(``), nil
	}
	return v.Data, nil
}

func (c *Clipboard) Set(label string, data []byte) error {
	v, err := c.searchVariable(label)
	if err != nil {
		return err
	}
	if v == nil {
		_, err = c.db.Insert(&Variable{Label: label, Data: data})
		return err
	}
	_, err = c.db.Update(&Variable{Label: v.Label, Data: data})
	return err
}

func (c *Clipboard) Labels() ([]string, error) {
	res := make([]Variable, 0)
	if err := c.db.Select(&res); err != nil {
		return nil, err
	}
	labels := make([]string, len(res))
	for i := 0; i < len(res); i++ {
		labels[i] = res[i].Label
	}
	return labels, nil
}

func (c *Clipboard) Delete(label string) error {
	v, err := c.searchVariable(label)
	if err != nil {
		return err
	}
	if v == nil {
		return nil
	}
	_, err = c.db.Delete(v)
	return nil
}

func (c *Clipboard) Close() error {
	return c.db.Close()
}
<commit_msg>Make Lables doesn't return err<commit_after>package lclip

import (
	"os"
	"path/filepath"

	_ "github.com/mattn/go-sqlite3"
	"github.com/mitchellh/go-homedir"
	"github.com/naoina/genmai"
)

func exists(path string) bool {
	_, err := os.Stat(path)
	return err == nil
}

func DefaultPath() (string, error) {
	h, err := homedir.Dir()
	if err != nil {
		return "", err
	}
	return filepath.Join(h, ".lclip.json"), nil
}

type Variable struct {
	Label string `db:"pk"`
	Data  []byte
}

type Clipboard struct {
	path string
	db   *genmai.DB
}

func NewClipboard(path string) (*Clipboard, error) {
	db, err := genmai.New(&genmai.SQLite3Dialect{}, path)
	if err != nil {
		return nil, err
	}
	if err = db.CreateTableIfNotExists(&Variable{}); err != nil {
		return nil, err
	}
	return &Clipboard{
		path: path,
		db:   db,
	}, nil
}

func NewClipboardWithDefaultPath() (*Clipboard, error) {
	path, err := DefaultPath()
	if err != nil {
		return nil, err
	}
	return NewClipboard(path)
}

func (c *Clipboard) Path() string {
	return c.path
}

func (c *Clipboard) searchVariable(label string) (*Variable, error) {
	res := make([]Variable, 0, 1)
	err := c.db.Select(&res, c.db.Where("label", "=", label))
	if err != nil {
		return nil, err
	}
	if len(res) < 1 {
		return nil, nil
	}
	return &res[0], nil
}

func (c *Clipboard) Get(label string) ([]byte, error) {
	v, err := c.searchVariable(label)
	if err != nil {
		return nil, err
	}
	if v == nil {
		return []byte(``), nil
	}
	return v.Data, nil
}

func (c *Clipboard) Set(label string, data []byte) error {
	v, err := c.searchVariable(label)
	if err != nil {
		return err
	}
	if v == nil {
		_, err = c.db.Insert(&Variable{Label: label, Data: data})
		return err
	}
	_, err = c.db.Update(&Variable{Label: v.Label, Data: data})
	return err
}

func (c *Clipboard) Labels() []string {
	res := make([]Variable, 0)
	if err := c.db.Select(&res); err != nil {
		return nil
	}
	labels := make([]string, len(res))
	for i := 0; i < len(res); i++ {
		labels[i] = res[i].Label
	}
	return labels
}

func (c *Clipboard) Delete(label string) error {
	v, err := c.searchVariable(label)
	if err != nil {
		return err
	}
	if v == nil {
		return nil
	}
	_, err = c.db.Delete(v)
	return nil
}

func (c *Clipboard) Close() error {
	return c.db.Close()
}
<|endoftext|>


<commit_before>package sse

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/calvernaz/things/encode"
)

type Broker struct {

	// Create a map of clients, the keys of the map are the channels
	// over which we can push messages to attached clients.  (The values
	// are just booleans and are meaningless.)
	//
	clients map[chan string]bool

	// Channel into which new clients can be pushed
	//
	newClients chan chan string

	// Channel into which disconnected clients should be pushed
	//
	defunctClients chan chan string

	// Channel into which messages are pushed to be broadcast out
	// to attached clients.
	//
	messages chan string
}

func (b *Broker) Start() {

	go func() {
		for {
			select {
			case s := <-b.newClients:
				// There is a new client attached and we
				// want to start sending them messages.
				b.clients[s] = true
				log.Println("Added new client")
			case s := <-b.defunctClients:
				// A client has dettached and we want to
				// stop sending them messages.
				delete(b.clients, s)
				close(s)
				log.Println("Removed client")
			case msg := <-b.messages:
				// There is a new message to send.  For each
				// attached client, push the new message
				// into the client's message channel.
				for s := range b.clients {
					s <- msg
				}
				log.Printf("Broadcast message to %d clients", len(b.clients))
			}
		}
	}()
}

func (b *Broker) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	// Make sure that the writer supports flushing.
	//
	f, ok := w.(http.Flusher)
	if !ok {
		http.Error(w, "Streaming unsupported!", http.StatusInternalServerError)
		return
	}

	// Create a new channel, over which the broker can
	// send this client messages.
	messageChan := make(chan string)

	// Add this client to the map of those that should
	// receive updates
	b.newClients <- messageChan

	// Set the headers related to event streaming.
	w.Header().Set("Content-Type", "text/event-stream")
	w.Header().Set("Cache-Control", "no-cache")
	w.Header().Set("Connection", "keep-alive")

	// Listen to the closing of the http connection via the CloseNotifier
	cn, ok := w.(http.CloseNotifier)
	if !ok {
		http.Error(w, "cannot stream", http.StatusInternalServerError)
		return
	}

	for {
		select {
		case <-cn.CloseNotify():
			b.defunctClients <- messageChan
			log.Println("HTTP connection just closed.")
			return
		case msg := <-messageChan:
			fmt.Fprint(w, "event: temp\n")
			fmt.Fprintf(w, "data: %s\n", msg)
			f.Flush()
		}
	}

	// Done.
	log.Println("Finished HTTP request at ", r.URL.Path)
}

func Main(sseChannel chan string) {
	log.Println("Starting work goroutine: Main")
	b := &Broker{
		make(map[chan string]bool),
		make(chan (chan string)),
		make(chan (chan string)),
		make(chan string),
	}

	// Start processing events
	b.Start()

	// Make b the HTTP handler for "/events/".  It can do
	// this because it has a ServeHTTP method.  That method
	// is called in a separate goroutine for each
	// request to "/events/".
	http.Handle("/events/", b)

	go func() {
		for {
			select {
			case msg := <-sseChannel:
				log.Println("Pushing messages to clients")
				js, _ := json.Marshal(encode.Encode(msg, time.Now()))
				b.messages <- string(js)
			default:

			}
		}
	}()

	// When we get a request at "/", call `MainPageHandler`
	// in a new goroutine.
	http.Handle("/", http.FileServer(http.Dir("./html")))

	// Start the server and listen forever on port 8000.
	fmt.Println("Serving at :8000")
	http.ListenAndServeTLS("0.0.0.0:8000", "./server.crt", "./server.key", nil)
}
<commit_msg>Add log<commit_after>package sse

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/calvernaz/things/encode"
)

type Broker struct {

	// Create a map of clients, the keys of the map are the channels
	// over which we can push messages to attached clients.  (The values
	// are just booleans and are meaningless.)
	//
	clients map[chan string]bool

	// Channel into which new clients can be pushed
	//
	newClients chan chan string

	// Channel into which disconnected clients should be pushed
	//
	defunctClients chan chan string

	// Channel into which messages are pushed to be broadcast out
	// to attached clients.
	//
	messages chan string
}

func (b *Broker) Start() {

	go func() {
		for {
			select {
			case s := <-b.newClients:
				// There is a new client attached and we
				// want to start sending them messages.
				b.clients[s] = true
				log.Println("Added new client")
			case s := <-b.defunctClients:
				// A client has dettached and we want to
				// stop sending them messages.
				delete(b.clients, s)
				close(s)
				log.Println("Removed client")
			case msg := <-b.messages:
				// There is a new message to send.  For each
				// attached client, push the new message
				// into the client's message channel.
				for s := range b.clients {
					s <- msg
				}
				log.Printf("Broadcast message to %d clients", len(b.clients))
			}
		}
	}()
}

func (b *Broker) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	// Make sure that the writer supports flushing.
	//
	f, ok := w.(http.Flusher)
	if !ok {
		http.Error(w, "Streaming unsupported!", http.StatusInternalServerError)
		return
	}

	// Create a new channel, over which the broker can
	// send this client messages.
	messageChan := make(chan string)

	// Add this client to the map of those that should
	// receive updates
	b.newClients <- messageChan

	// Set the headers related to event streaming.
	w.Header().Set("Content-Type", "text/event-stream")
	w.Header().Set("Cache-Control", "no-cache")
	w.Header().Set("Connection", "keep-alive")

	// Listen to the closing of the http connection via the CloseNotifier
	cn, ok := w.(http.CloseNotifier)
	if !ok {
		http.Error(w, "cannot stream", http.StatusInternalServerError)
		return
	}

	for {
		select {
		case <-cn.CloseNotify():
			b.defunctClients <- messageChan
			log.Println("HTTP connection just closed.")
			return
		case msg := <-messageChan:
			fmt.Fprint(w, "event: temp\n")
			fmt.Fprintf(w, "data: %s\n", msg)
			f.Flush()
		}
	}

	// Done.
	log.Println("Finished HTTP request at ", r.URL.Path)
}

func Main(sseChannel chan string) {
	log.Println("Starting work goroutine: Main")
	b := &Broker{
		make(map[chan string]bool),
		make(chan (chan string)),
		make(chan (chan string)),
		make(chan string),
	}

	// Start processing events
	b.Start()

	// Make b the HTTP handler for "/events/".  It can do
	// this because it has a ServeHTTP method.  That method
	// is called in a separate goroutine for each
	// request to "/events/".
	http.Handle("/events/", b)

	go func() {
		for {
			select {
			case msg := <-sseChannel:
				js, _ := json.Marshal(encode.Encode(msg, time.Now()))
				log.Printf("Pushing messages to clients: %v", string(js))
				b.messages <- string(js)
			}
		}
	}()

	// When we get a request at "/", call `MainPageHandler`
	// in a new goroutine.
	http.Handle("/", http.FileServer(http.Dir("./html")))

	// Start the server and listen forever on port 8000.
	fmt.Println("Serving at :8000")
	http.ListenAndServeTLS("0.0.0.0:8000", "./server.crt", "./server.key", nil)
}
<|endoftext|>


<commit_before>package eparser

import (
	"fmt"
	"unicode"
)

const eol rune = -1

// Lexer holds the lexer's state while scanning an expression. If an error is
// encountered it's appended to errors and the error count gets increased. If
// there are 5 errors already it will stop emitting them to prevent spam.
type Lexer struct {
	expr   []rune   // the input expression
	ch     rune     // current character
	pos    int      // current character position
	start  int      // current read offset
	tokens []*Token // tokenized lexemes

	errors     []error // errors
	ErrorCount int     // error count
}

func isIdent(c rune) bool {
	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= 0x80 && unicode.IsLetter(c))
}

func isNumber(c rune) bool {
	return (c >= '0' && c <= '9') || c == '.'
}

func (l *Lexer) error(msg string) {
	if l.ErrorCount > 5 {
		// At this point we're just spamming output
		return
	}
	l.ErrorCount++
	l.errors = append(l.errors, fmt.Errorf("Syntax Error: %s at position %d", msg, l.start+1))
}

// Lex starts lexing an expression. We keep reading until EOL is found, which
// we add because we need a padding of 1 to always be able to peek().
//
// Returns the generated tokens and any error found.
func Lex(expr string) ([]*Token, []error) {
	l := &Lexer{
		expr:  append([]rune(expr), eol), // add eol as padding
		pos:   0,
		start: 0,

		errors:     nil,
		ErrorCount: 0,
	}

	return l.lex()
}

func (l *Lexer) lex() ([]*Token, []error) {
	for l.ch != eol {
		l.start = l.pos

		l.eat()

		switch {
		case isIdent(l.ch):
			l.readIdent()
		case isNumber(l.ch):
			l.readNumber()
		default:
			switch l.ch {
			case '+':
				l.switchEq(ADD, ADD_EQ)
			case '-':
				l.switchEq(SUB, SUB_EQ)
			case '/':
				l.switchEq(DIV, DIV_EQ)
			case '*':
				if l.peek() == '*' {
					l.eat()
					l.switchEq(POW, POW_EQ)
				} else {
					l.switchEq(MUL, MUL_EQ)
				}
			case '%':
				l.switchEq(REM, REM_EQ)
			case '&':
				l.switchEq(AND, AND_EQ)
			case '|':
				l.switchEq(OR, OR_EQ)
			case '^':
				l.switchEq(XOR, XOR_EQ)
			case '<':
				if l.peek() == '<' {
					l.eat()
					l.switchEq(LSH, LSH_EQ)
				} else {
					l.emit(ILLEGAL)
					l.eat()
					l.error("expected <<, got <")
				}
			case '>':
				if l.peek() == '>' {
					l.eat()
					l.switchEq(RSH, RSH_EQ)
				} else {
					l.emit(ILLEGAL)
					l.eat()
					l.error("expected >>, got >")
				}
			case '~':
				l.emit(NOT)
			case '=':
				l.emit(EQ)
			case '(':
				l.emit(LPAREN)
			case ')':
				l.emit(RPAREN)
			case '\r', ' ', '\t':
				l.skipWhitespace()
			case eol:
				l.emit(EOL)
			default:
				l.emit(ILLEGAL)
				l.error("unexpected token " + string(l.ch))
			}
		}
	}

	return l.tokens, l.errors
}

func (l *Lexer) peek() rune {
	return l.expr[l.pos]
}

func (l *Lexer) eat() rune {
	l.ch = l.peek()
	l.pos++
	return l.ch
}

func (l *Lexer) emit(toktype tokenType) {
	l.tokens = append(l.tokens, newToken(toktype, string(l.expr[l.start:l.pos]), l.start))
}

func (l *Lexer) skipWhitespace() {
	for l.peek() == '\t' || l.peek() == ' ' || l.peek() == '\r' {
		l.eat()
	}
}

func (l *Lexer) readIdent() {
	for isIdent(l.peek()) || isNumber(l.peek()) {
		l.eat()
	}

	l.emit(IDENT)
}

func (l *Lexer) readNumber() {
	toktype := INT
	for isNumber(l.peek()) {
		if l.ch == '.' {
			toktype = FLOAT
		}
		l.eat()
	}

	l.emit(toktype)
}

func (l *Lexer) switchEq(tokA, tokB tokenType) {
	if l.peek() == '=' {
		l.eat()
		l.emit(tokB)
	} else {
		l.emit(tokA)
	}
}
<commit_msg>Correctly limit error messages in the lexer to 5<commit_after>package eparser

import (
	"fmt"
	"unicode"
)

const eol rune = -1

// Lexer holds the lexer's state while scanning an expression. If an error is
// encountered it's appended to errors and the error count gets increased. If
// there are 5 errors already it will stop emitting them to prevent spam.
type Lexer struct {
	expr   []rune   // the input expression
	ch     rune     // current character
	pos    int      // current character position
	start  int      // current read offset
	tokens []*Token // tokenized lexemes

	errors     []error // errors
	ErrorCount int     // error count
}

func isIdent(c rune) bool {
	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= 0x80 && unicode.IsLetter(c))
}

func isNumber(c rune) bool {
	return (c >= '0' && c <= '9') || c == '.'
}

func (l *Lexer) error(msg string) {
	if l.ErrorCount >= 5 {
		// At this point we're just spamming output
		return
	}
	l.ErrorCount++
	l.errors = append(l.errors, fmt.Errorf("Syntax Error: %s at position %d", msg, l.start+1))
}

// Lex starts lexing an expression. We keep reading until EOL is found, which
// we add because we need a padding of 1 to always be able to peek().
//
// Returns the generated tokens and any error found.
func Lex(expr string) ([]*Token, []error) {
	l := &Lexer{
		expr:  append([]rune(expr), eol), // add eol as padding
		pos:   0,
		start: 0,

		errors:     nil,
		ErrorCount: 0,
	}

	return l.lex()
}

func (l *Lexer) lex() ([]*Token, []error) {
	for l.ch != eol {
		l.start = l.pos

		l.eat()

		switch {
		case isIdent(l.ch):
			l.readIdent()
		case isNumber(l.ch):
			l.readNumber()
		default:
			switch l.ch {
			case '+':
				l.switchEq(ADD, ADD_EQ)
			case '-':
				l.switchEq(SUB, SUB_EQ)
			case '/':
				l.switchEq(DIV, DIV_EQ)
			case '*':
				if l.peek() == '*' {
					l.eat()
					l.switchEq(POW, POW_EQ)
				} else {
					l.switchEq(MUL, MUL_EQ)
				}
			case '%':
				l.switchEq(REM, REM_EQ)
			case '&':
				l.switchEq(AND, AND_EQ)
			case '|':
				l.switchEq(OR, OR_EQ)
			case '^':
				l.switchEq(XOR, XOR_EQ)
			case '<':
				if l.peek() == '<' {
					l.eat()
					l.switchEq(LSH, LSH_EQ)
				} else {
					l.emit(ILLEGAL)
					l.eat()
					l.error("expected <<, got <")
				}
			case '>':
				if l.peek() == '>' {
					l.eat()
					l.switchEq(RSH, RSH_EQ)
				} else {
					l.emit(ILLEGAL)
					l.eat()
					l.error("expected >>, got >")
				}
			case '~':
				l.emit(NOT)
			case '=':
				l.emit(EQ)
			case '(':
				l.emit(LPAREN)
			case ')':
				l.emit(RPAREN)
			case '\r', ' ', '\t':
				l.skipWhitespace()
			case eol:
				l.emit(EOL)
			default:
				l.emit(ILLEGAL)
				l.error("unexpected token " + string(l.ch))
			}
		}
	}

	return l.tokens, l.errors
}

func (l *Lexer) peek() rune {
	return l.expr[l.pos]
}

func (l *Lexer) eat() rune {
	l.ch = l.peek()
	l.pos++
	return l.ch
}

func (l *Lexer) emit(toktype tokenType) {
	l.tokens = append(l.tokens, newToken(toktype, string(l.expr[l.start:l.pos]), l.start))
}

func (l *Lexer) skipWhitespace() {
	for l.peek() == '\t' || l.peek() == ' ' || l.peek() == '\r' {
		l.eat()
	}
}

func (l *Lexer) readIdent() {
	for isIdent(l.peek()) || isNumber(l.peek()) {
		l.eat()
	}

	l.emit(IDENT)
}

func (l *Lexer) readNumber() {
	toktype := INT
	for isNumber(l.peek()) {
		if l.ch == '.' {
			toktype = FLOAT
		}
		l.eat()
	}

	l.emit(toktype)
}

func (l *Lexer) switchEq(tokA, tokB tokenType) {
	if l.peek() == '=' {
		l.eat()
		l.emit(tokB)
	} else {
		l.emit(tokA)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
	"testing"
	"time"
)

const requestTimeout = time.Second * 5

var (
	edgeHost    = flag.String("edgeHost", "www.gov.uk", "Hostname of edge")
	originPort  = flag.Int("originPort", 8080, "Origin port to listen on for requests")
	insecureTLS = flag.Bool("insecureTLS", false, "Whether to check server certificates")

	client       *http.Transport
	originServer *CDNServeMux
)

// Setup clients and servers.
func init() {

	flag.Parse()

	tlsOptions := &tls.Config{}
	if *insecureTLS {
		tlsOptions.InsecureSkipVerify = true
	}

	client = &http.Transport{
		ResponseHeaderTimeout: requestTimeout,
		TLSClientConfig:       tlsOptions,
	}
	originServer = StartServer(*originPort)

	log.Println("Confirming that CDN has successfully probed Origin")
	err := confirmOriginIsEnabled(originServer, *edgeHost)
	if err != nil {
		log.Fatal(err)
	}
}

func TestHelpers(t *testing.T) {
	testHelpersCDNServeMuxHandlers(t, originServer)
	testHelpersCDNServeMuxProbes(t, originServer)
}

// Should redirect from HTTP to HTTPS without hitting origin.
func TestProtocolRedirect(t *testing.T) {
	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		t.Error("Request should not have made it to origin")
	})

	sourceUrl := fmt.Sprintf("http://%s/foo/bar", *edgeHost)
	destUrl := fmt.Sprintf("https://%s/foo/bar", *edgeHost)

	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp, err := client.RoundTrip(req)

	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != 301 {
		t.Errorf("Status code expected 301, got %d", resp.StatusCode)
	}
	if d := resp.Header.Get("Location"); d != destUrl {
		t.Errorf("Location header expected %s, got %s", destUrl, d)
	}
}

// Should send request to origin by default
func TestRequestsGoToOriginByDefault(t *testing.T) {
	uuid := NewUUID()
	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "GET" && r.URL.Path == fmt.Sprintf("/%s", uuid) {
			w.Header().Set("EnsureOriginServed", uuid)
		}
	})

	sourceUrl := fmt.Sprintf("https://%s/%s", *edgeHost, uuid)

	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp, err := client.RoundTrip(req)

	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != 200 {
		t.Errorf("Status code expected 200, got %d", resp.StatusCode)
	}
	if d := resp.Header.Get("EnsureOriginServed"); d != uuid {
		t.Errorf("EnsureOriginServed header has not come from Origin: expected %q, got %q", uuid, d)
	}

}

// Should cache first response and return it on second request without
// hitting origin again.
func TestFirstResponseCached(t *testing.T) {
	const bodyExpected = "first request"
	const requestsExpectedCount = 1
	requestsReceivedCount := 0

	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		if requestsReceivedCount == 0 {
			w.Write([]byte(bodyExpected))
		} else {
			w.Write([]byte("subsequent request"))
		}

		requestsReceivedCount++
	})

	url := fmt.Sprintf("https://%s/%s", *edgeHost, NewUUID())
	req, _ := http.NewRequest("GET", url, nil)

	for i := 0; i < 2; i++ {
		resp, err := client.RoundTrip(req)
		if err != nil {
			t.Fatal(err)
		}

		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			t.Fatal(err)
		}
		if string(body) != bodyExpected {
			t.Errorf("Incorrect response body. Expected %q, got %q", bodyExpected, body)
		}
	}

	if requestsReceivedCount > requestsExpectedCount {
		t.Errorf("originServer got too many requests. Expected %d requests, got %d", requestsExpectedCount, requestsReceivedCount)
	}
}

// Should return 403 for PURGE requests from IPs not in the whitelist.
func TestRestrictPurgeRequests(t *testing.T) {
	t.Error("Not implemented")
}

// Should create an X-Forwarded-For header containing the client's IP.
func TestHeaderCreateXFF(t *testing.T) {
	t.Error("Not implemented")
}

// Should append client's IP to existing X-Forwarded-For header.
func TestHeaderAppendXFF(t *testing.T) {
	t.Error("Not implemented")
}

// Should create a True-Client-IP header containing the client's IP
// address, discarding the value provided in the original request.
func TestHeaderUnspoofableClientIP(t *testing.T) {
	t.Error("Not implemented")
}

// Should not modify Host header from original request.
func TestHeaderHostUnmodified(t *testing.T) {
	t.Error("Not implemented")
}

// Should set a default TTL if the response doesn't set one.
func TestDefaultTTL(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve stale object and not hit mirror(s) if origin is down and
// object is beyond TTL but still in cache.
func TestFailoverOriginDownServeStale(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve stale object and not hit mirror(s) if origin returns a 5xx
// response and object is beyond TTL but still in cache.
func TestFailoverOrigin5xxServeStale(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to first mirror if origin is down and object is not in
// cache (active or stale).
func TestFailoverOriginDownUseFirstMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to first mirror if origin returns 5xx response and object
// is not in cache (active or stale).
func TestFailoverOrigin5xxUseFirstMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to second mirror if both origin and first mirror are
// down.
func TestFailoverOriginDownFirstMirrorDownUseSecondMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to second mirror if both origin and first mirror return
// 5xx responses.
func TestFailoverOrigin5xxFirstMirror5xxUseSecondMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should not fallback to mirror if origin returns a 5xx response with a
// No-Fallback header.
func TestFailoverNoFallbackHeader(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve a known static error page if cannot serve a page
// from origin, stale or any mirror.
// NB: ideally this should be a page that we control that has a mechanism
//     to alert us that it has been served.
func TestErrorPageIsServedWhenNoBackendAvailable(t *testing.T) {
	t.Error("Not implemented")
}

// Should not cache a response with a Set-Cookie a header.
func TestNoCacheHeaderSetCookie(t *testing.T) {
	t.Error("Not implemented")
}

// Should not cache a response with a Cache-Control: private header.
func TestNoCacheHeaderCacheControlPrivate(t *testing.T) {
	t.Error("Not implemented")
}

// ---------------------------------------------------------
// Test that useful common cache-related parameters are sent to the
// client by this CDN provider.

// Should set an Age header itself rather than passing the Age header from origin.
func TestAgeHeaderIsSetByProviderNotOrigin(t *testing.T) {
	t.Error("Not implemented")
}

// Should set an X-Cache header containing HIT/MISS from 'origin, itself'
func TestXCacheHeaderContainsHitMissFromBothProviderAndOrigin(t *testing.T) {
	t.Error("Not implemented")
}

// Should set an X-Served-By header giving information on the node and location served from.
func TestXServedByHeaderContainsANodeIdAndLocation(t *testing.T) {

	uuid := NewUUID()
	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "GET" && r.URL.Path == fmt.Sprintf("/%s", uuid) {
			w.WriteHeader(200)
		}
	})

	sourceUrl := fmt.Sprintf("https://%s/%s", *edgeHost, uuid)

	// Get first request, will come from origin. Edge Hit Count 0
	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp, err := client.RoundTrip(req)
	if err != nil {
		t.Fatal(err)
	}

	actualHeader := resp.Header.Get("X-Served-By")
	if actualHeader == "" {
		t.Error("X-Served-By header has not been set by Edge")
	}

	re := regexp.MustCompile("^cache-[a-z0-9]+-[A-Z]{3}$")
	if re.FindString(actualHeader) != actualHeader {
		t.Errorf("X-Served-By is not as expected: got %q", actualHeader)
	}

}

// Should set an X-Cache-Hits header containing hit count for this object,
// from the provider not origin
func TestXCacheHitsContainsProviderHitCountForThisObject(t *testing.T) {
	t.Error("Not implemented")
}
<commit_msg>Rename Test and regexp to highlight Fastly-specific<commit_after>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
	"testing"
	"time"
)

const requestTimeout = time.Second * 5

var (
	edgeHost    = flag.String("edgeHost", "www.gov.uk", "Hostname of edge")
	originPort  = flag.Int("originPort", 8080, "Origin port to listen on for requests")
	insecureTLS = flag.Bool("insecureTLS", false, "Whether to check server certificates")

	client       *http.Transport
	originServer *CDNServeMux
)

// Setup clients and servers.
func init() {

	flag.Parse()

	tlsOptions := &tls.Config{}
	if *insecureTLS {
		tlsOptions.InsecureSkipVerify = true
	}

	client = &http.Transport{
		ResponseHeaderTimeout: requestTimeout,
		TLSClientConfig:       tlsOptions,
	}
	originServer = StartServer(*originPort)

	log.Println("Confirming that CDN has successfully probed Origin")
	err := confirmOriginIsEnabled(originServer, *edgeHost)
	if err != nil {
		log.Fatal(err)
	}
}

func TestHelpers(t *testing.T) {
	testHelpersCDNServeMuxHandlers(t, originServer)
	testHelpersCDNServeMuxProbes(t, originServer)
}

// Should redirect from HTTP to HTTPS without hitting origin.
func TestProtocolRedirect(t *testing.T) {
	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		t.Error("Request should not have made it to origin")
	})

	sourceUrl := fmt.Sprintf("http://%s/foo/bar", *edgeHost)
	destUrl := fmt.Sprintf("https://%s/foo/bar", *edgeHost)

	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp, err := client.RoundTrip(req)

	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != 301 {
		t.Errorf("Status code expected 301, got %d", resp.StatusCode)
	}
	if d := resp.Header.Get("Location"); d != destUrl {
		t.Errorf("Location header expected %s, got %s", destUrl, d)
	}
}

// Should send request to origin by default
func TestRequestsGoToOriginByDefault(t *testing.T) {
	uuid := NewUUID()
	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "GET" && r.URL.Path == fmt.Sprintf("/%s", uuid) {
			w.Header().Set("EnsureOriginServed", uuid)
		}
	})

	sourceUrl := fmt.Sprintf("https://%s/%s", *edgeHost, uuid)

	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp, err := client.RoundTrip(req)

	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != 200 {
		t.Errorf("Status code expected 200, got %d", resp.StatusCode)
	}
	if d := resp.Header.Get("EnsureOriginServed"); d != uuid {
		t.Errorf("EnsureOriginServed header has not come from Origin: expected %q, got %q", uuid, d)
	}

}

// Should cache first response and return it on second request without
// hitting origin again.
func TestFirstResponseCached(t *testing.T) {
	const bodyExpected = "first request"
	const requestsExpectedCount = 1
	requestsReceivedCount := 0

	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		if requestsReceivedCount == 0 {
			w.Write([]byte(bodyExpected))
		} else {
			w.Write([]byte("subsequent request"))
		}

		requestsReceivedCount++
	})

	url := fmt.Sprintf("https://%s/%s", *edgeHost, NewUUID())
	req, _ := http.NewRequest("GET", url, nil)

	for i := 0; i < 2; i++ {
		resp, err := client.RoundTrip(req)
		if err != nil {
			t.Fatal(err)
		}

		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			t.Fatal(err)
		}
		if string(body) != bodyExpected {
			t.Errorf("Incorrect response body. Expected %q, got %q", bodyExpected, body)
		}
	}

	if requestsReceivedCount > requestsExpectedCount {
		t.Errorf("originServer got too many requests. Expected %d requests, got %d", requestsExpectedCount, requestsReceivedCount)
	}
}

// Should return 403 for PURGE requests from IPs not in the whitelist.
func TestRestrictPurgeRequests(t *testing.T) {
	t.Error("Not implemented")
}

// Should create an X-Forwarded-For header containing the client's IP.
func TestHeaderCreateXFF(t *testing.T) {
	t.Error("Not implemented")
}

// Should append client's IP to existing X-Forwarded-For header.
func TestHeaderAppendXFF(t *testing.T) {
	t.Error("Not implemented")
}

// Should create a True-Client-IP header containing the client's IP
// address, discarding the value provided in the original request.
func TestHeaderUnspoofableClientIP(t *testing.T) {
	t.Error("Not implemented")
}

// Should not modify Host header from original request.
func TestHeaderHostUnmodified(t *testing.T) {
	t.Error("Not implemented")
}

// Should set a default TTL if the response doesn't set one.
func TestDefaultTTL(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve stale object and not hit mirror(s) if origin is down and
// object is beyond TTL but still in cache.
func TestFailoverOriginDownServeStale(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve stale object and not hit mirror(s) if origin returns a 5xx
// response and object is beyond TTL but still in cache.
func TestFailoverOrigin5xxServeStale(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to first mirror if origin is down and object is not in
// cache (active or stale).
func TestFailoverOriginDownUseFirstMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to first mirror if origin returns 5xx response and object
// is not in cache (active or stale).
func TestFailoverOrigin5xxUseFirstMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to second mirror if both origin and first mirror are
// down.
func TestFailoverOriginDownFirstMirrorDownUseSecondMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to second mirror if both origin and first mirror return
// 5xx responses.
func TestFailoverOrigin5xxFirstMirror5xxUseSecondMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should not fallback to mirror if origin returns a 5xx response with a
// No-Fallback header.
func TestFailoverNoFallbackHeader(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve a known static error page if cannot serve a page
// from origin, stale or any mirror.
// NB: ideally this should be a page that we control that has a mechanism
//     to alert us that it has been served.
func TestErrorPageIsServedWhenNoBackendAvailable(t *testing.T) {
	t.Error("Not implemented")
}

// Should not cache a response with a Set-Cookie a header.
func TestNoCacheHeaderSetCookie(t *testing.T) {
	t.Error("Not implemented")
}

// Should not cache a response with a Cache-Control: private header.
func TestNoCacheHeaderCacheControlPrivate(t *testing.T) {
	t.Error("Not implemented")
}

// ---------------------------------------------------------
// Test that useful common cache-related parameters are sent to the
// client by this CDN provider.

// Should set an Age header itself rather than passing the Age header from origin.
func TestAgeHeaderIsSetByProviderNotOrigin(t *testing.T) {
	t.Error("Not implemented")
}

// Should set an X-Cache header containing HIT/MISS from 'origin, itself'
func TestXCacheHeaderContainsHitMissFromBothProviderAndOrigin(t *testing.T) {
	t.Error("Not implemented")
}

// Should set an X-Served-By header giving information on the (Fastly) node and location served from.
func TestXServedByHeaderContainsFastlyNodeIdAndLocation(t *testing.T) {

	expectedFastlyXServedByRegexp := regexp.MustCompile("^cache-[a-z0-9]+-[A-Z]{3}$")

	uuid := NewUUID()
	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "GET" && r.URL.Path == fmt.Sprintf("/%s", uuid) {
			w.WriteHeader(200)
		}
	})

	sourceUrl := fmt.Sprintf("https://%s/%s", *edgeHost, uuid)

	// Get first request, will come from origin. Edge Hit Count 0
	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp, err := client.RoundTrip(req)
	if err != nil {
		t.Fatal(err)
	}

	actualHeader := resp.Header.Get("X-Served-By")
	if actualHeader == "" {
		t.Error("X-Served-By header has not been set by Edge")
	}

	if expectedFastlyXServedByRegexp.FindString(actualHeader) != actualHeader {
		t.Errorf("X-Served-By is not as expected: got %q", actualHeader)
	}

}

// Should set an X-Cache-Hits header containing hit count for this object,
// from the provider not origin
func TestXCacheHitsContainsProviderHitCountForThisObject(t *testing.T) {
	t.Error("Not implemented")
}
<|endoftext|>


<commit_before>package certificates

import (
	"database/sql"
	"io"
	"log"

	"github.com/lflux/eve-sdeloader/utils"
)

var skillLevelMap = map[string]int{
	"basic":    0,
	"standard": 1,
	"improved": 2,
	"advanced": 3,
	"elite":    4,
}

type Certificate struct {
	Description    string
	GroupID        int64 `yaml:"groupID"`
	Name           string
	RecommendedFor []int64                     `yaml:"recommendedFor"`
	SkillTypes     map[string]map[string]int64 `yaml:"skillTypes"`
}

func InsertCertCertsStmt(tx *sql.Tx) (*sql.Stmt, error) {
	return tx.Prepare(`INSERT INTO certcerts (certid, description, groupid, name) VALUES ($1, $2, $3, $4)`)
}

func InsertCertSkillsStmt(tx *sql.Tx) (*sql.Stmt, error) {
	return tx.Prepare(`INSERT INTO certskills (certid, skillid, certlevelint, certleveltext, skilllevel) VALUES ($1, $2, $3, $4, $5)`)
}

func Import(db *sql.DB, r io.Reader) error {
	entries := make(map[string]*Certificate)

	err := utils.LoadFromReader(r, entries)
	if err != nil {
		return err
	}

	tx, err := db.Begin()
	if err != nil {
		return err
	}

	insertCertCerts, err := InsertCertCertsStmt(tx)
	if err != nil {
		return err
	}

	insertCertSkills, err := InsertCertSkillsStmt(tx)
	if err != nil {
		return err
	}

	for certID, cert := range entries {
		log.Println(certID)
		_, err = insertCertCerts.Exec(certID, cert.Description, cert.GroupID, cert.Name)
		if err != nil {
			return err
		}

		for skillID, skillType := range cert.SkillTypes {
			for skillLevel, value := range skillType {
				log.Println(certID, skillID, skillLevelMap[skillLevel], skillLevel, value)
				_, err = insertCertSkills.Exec(certID, skillID, skillLevelMap[skillLevel], skillLevel, value)
			}
		}
	}

	return tx.Commit()
}
<commit_msg>Remove log lines<commit_after>package certificates

import (
	"database/sql"
	"io"

	"github.com/lflux/eve-sdeloader/utils"
)

var skillLevelMap = map[string]int{
	"basic":    0,
	"standard": 1,
	"improved": 2,
	"advanced": 3,
	"elite":    4,
}

type Certificate struct {
	Description    string
	GroupID        int64 `yaml:"groupID"`
	Name           string
	RecommendedFor []int64                     `yaml:"recommendedFor"`
	SkillTypes     map[string]map[string]int64 `yaml:"skillTypes"`
}

func InsertCertCertsStmt(tx *sql.Tx) (*sql.Stmt, error) {
	return tx.Prepare(`INSERT INTO certcerts (certid, description, groupid, name) VALUES ($1, $2, $3, $4)`)
}

func InsertCertSkillsStmt(tx *sql.Tx) (*sql.Stmt, error) {
	return tx.Prepare(`INSERT INTO certskills (certid, skillid, certlevelint, certleveltext, skilllevel) VALUES ($1, $2, $3, $4, $5)`)
}

func Import(db *sql.DB, r io.Reader) error {
	entries := make(map[string]*Certificate)

	err := utils.LoadFromReader(r, entries)
	if err != nil {
		return err
	}

	tx, err := db.Begin()
	if err != nil {
		return err
	}

	insertCertCerts, err := InsertCertCertsStmt(tx)
	if err != nil {
		return err
	}

	insertCertSkills, err := InsertCertSkillsStmt(tx)
	if err != nil {
		return err
	}

	for certID, cert := range entries {
		_, err = insertCertCerts.Exec(certID, cert.Description, cert.GroupID, cert.Name)
		if err != nil {
			return err
		}

		for skillID, skillType := range cert.SkillTypes {
			for skillLevel, value := range skillType {
				_, err = insertCertSkills.Exec(certID, skillID, skillLevelMap[skillLevel], skillLevel, value)
			}
		}
	}

	return tx.Commit()
}
<|endoftext|>


<commit_before>//Client to the ATM server.

/*
	TODO:
	Read configurations from file.
	Make contact with server.
*/

package main

import (
	"code.google.com/p/gcfg"
	"flag"
	"fmt"
	"net"
	"os"
	"os/signal"
)

//Configuration stuff.
var (
	configPath string
	prompt     = "Unicorn@ATM>"
	version    = 1.0
	author     = "Christopher Lillthors. Unicorn INC"
)

//Struct to hold all the configurations.
type Config struct {
	Client struct {
		Address string
		Port    string
	}
}

type client struct {
	Config
	conn net.Conn
}

func init() {
	//For configurations.
	flag.StringVar(&configPath, "config", "client.gcfg", "Path to config file")

	//For UNIX signal handling.
	c := make(chan os.Signal, 1)   //A channel to listen on keyboard events.
	signal.Notify(c, os.Interrupt) //If user pressed CTRL - C.

	//A goroutine to check for keyboard events.
	go func() {
		<-c //blocking.
		//inform server that I will quit.
		fmt.Fprintln(os.Stderr, "Bye")
		os.Exit(1) //will just quit client if user pressed CTRL - C
	}() //Execute goroutine
}

func main() {
	//create a new client.
	client := new(client)
}
<commit_msg>Added configuration<commit_after>// The MIT License (MIT)

// Copyright (c) 2014 Christopher Lillthors

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

/*
	TODO:
	Read configurations from file.
	Make contact with server.
*/

package main

import (
	"code.google.com/p/gcfg"
	"flag"
	"fmt"
	"net"
	"os"
	"os/signal"
)

//Configuration stuff.
var (
	configPath string
	prompt     = "Unicorn@ATM>"
	version    = 1.0
	author     = "Christopher Lillthors. Unicorn INC"
)

//Struct to hold all the configurations.
type Config struct {
	Client struct {
		Address string
		Port    string
	}
}

type client struct {
	Config
	conn net.Conn
}

func init() {
	//For configurations.
	flag.StringVar(&configPath, "config", "client.gcfg", "Path to config file")

	//For UNIX signal handling.
	c := make(chan os.Signal, 1)   //A channel to listen on keyboard events.
	signal.Notify(c, os.Interrupt) //If user pressed CTRL - C.

	//A goroutine to check for keyboard events.
	go func() {
		<-c //blocking.
		//inform server that I will quit.
		fmt.Fprintln(os.Stderr, "Bye")
		os.Exit(1) //will just quit client if user pressed CTRL - C
	}() //Execute goroutine
}

func main() {
	//create a new client.
	client := new(client)
}
<|endoftext|>


<commit_before>package TrieTree

type WordCount struct {
	Word  string
	Count int
}

type topN struct {
	top []*WordCount
	min int
}

func (this *topN) insert(r []rune, count int) {
	wc := &WordCount{string(r), count}
	for i := 0; i < len(this.top) && wc != nil; i++ {
		if this.top[i] != nil && this.top[i].Count == count && this.top[i].Word == wc.Word {
			return
		}
		if this.top[i] == nil || wc.Count > this.top[i].Count {
			this.top[i], wc = wc, this.top[i]
		}
	}
	if wc != nil {
		this.min = this.top[len(this.top)-1].Count
	}
}

func (this *topN) compact() []*WordCount {
	res := make([]*WordCount, 0, len(this.top))
	for _, v := range this.top {
		if v != nil {
			res = append(res, v)
		}
	}
	return res
}

type Node struct {
	Count    int
	Children map[rune]*Node
}

func NewTrieTree() *Node {
	return new(Node)
}

func (this *Node) add(seg []rune, index, count, incr int) int {
	if index >= len(seg) {
		if count > 0 {
			this.Count = count
		} else {
			this.Count += incr
		}
		return this.Count
	}

	if this.Children == nil {
		this.Children = make(map[rune]*Node, 1)
	}

	value := seg[index]
	if child, ok := this.Children[value]; !ok || child == nil {
		this.Children[value] = new(Node)
	}

	return this.Children[value].add(seg, index+1, count, incr)
}

func (this *Node) Add(str string, count, incr int) int {
	return this.add([]rune(str), 0, count, incr)
}

func (this *Node) all(seg []rune, top *topN) {
	if this.Count > top.min {
		top.insert(seg, this.Count)
	}

	for r, n := range this.Children {
		n.all(append(seg, r), top)
	}
}

func (this *Node) find(seg []rune) *Node {
	node := this
	for _, v := range seg {
		if child, ok := node.Children[v]; ok && child != nil {
			node = child
		} else {
			return nil
		}
	}
	return node
}

func (this *Node) PrefixSearch(prefix string, topCount int) []*WordCount {
	seg, top := []rune(prefix), topN{make([]*WordCount, topCount), 0}
	if node := this.find(seg); node != nil {
		node.all(seg, &top)
	}
	return top.compact()
}

func (this *Node) substr(root *Node, pre, seg []rune, top *topN) {
	rp := append(pre, seg...)
	if node := root.find(rp); node != nil {
		node.all(rp, top)
	}

	for r, c := range this.Children {
		c.substr(root, append(pre, r), seg, top)
	}
}

func (this *Node) SubstrSearch(sub string, topCount int) []*WordCount {
	seg, top := []rune(sub), topN{make([]*WordCount, topCount), 0}
	this.substr(this, nil, seg, &top)
	return top.compact()
}

func (this *Node) fuzzy(pre, seg []rune, index int, top *topN) {
	if index >= len(seg) {
		this.all(pre, top)
		for r, c := range this.Children {
			c.all(append(pre, r), top)
		}
		return
	}

	for r, c := range this.Children {
		if r == seg[index] {
			c.fuzzy(append(pre, r), seg, index+1, top)
		} else {
			c.fuzzy(append(pre, r), seg, index, top)
		}
	}
}

func (this *Node) FuzzySearch(fuzzy string, topCount int) []*WordCount {
	seg, top := []rune(fuzzy), topN{make([]*WordCount, topCount), 0}
	this.fuzzy(nil, seg, 0, &top)
	return top.compact()
}
<commit_msg>add SearchResult<commit_after>package TrieTree

import "sort"

type WordCount struct {
	Word  string
	Count int
}

type SearchResult []*WordCount

func (this SearchResult) Sort()         { sort.Sort(this) }
func (this SearchResult) Len() int      { return len(this) }
func (this SearchResult) Swap(i, j int) { this[i], this[j] = this[j], this[i] }
func (this SearchResult) Less(i, j int) bool {
	return this[i].Count > this[j].Count || this[i].Word > this[j].Word
}

type topN struct {
	top []*WordCount
	min int
}

func (this *topN) insert(r []rune, count int) {
	wc := &WordCount{string(r), count}
	for i := 0; i < len(this.top) && wc != nil; i++ {
		if this.top[i] != nil && this.top[i].Count == count && this.top[i].Word == wc.Word {
			return
		}
		if this.top[i] == nil || wc.Count > this.top[i].Count {
			this.top[i], wc = wc, this.top[i]
		}
	}
	if wc != nil {
		this.min = this.top[len(this.top)-1].Count
	}
}

func (this *topN) compact() []*WordCount {
	res := make([]*WordCount, 0, len(this.top))
	for _, v := range this.top {
		if v != nil {
			res = append(res, v)
		}
	}
	return res
}

type Node struct {
	Count    int
	Children map[rune]*Node
}

func NewTrieTree() *Node {
	return new(Node)
}

func (this *Node) add(seg []rune, index, count, incr int) int {
	if index >= len(seg) {
		if count > 0 {
			this.Count = count
		} else {
			this.Count += incr
		}
		return this.Count
	}

	if this.Children == nil {
		this.Children = make(map[rune]*Node, 1)
	}

	value := seg[index]
	if child, ok := this.Children[value]; !ok || child == nil {
		this.Children[value] = new(Node)
	}

	return this.Children[value].add(seg, index+1, count, incr)
}

func (this *Node) Add(str string, count, incr int) int {
	return this.add([]rune(str), 0, count, incr)
}

func (this *Node) all(seg []rune, top *topN) {
	if this.Count > top.min {
		top.insert(seg, this.Count)
	}

	for r, n := range this.Children {
		n.all(append(seg, r), top)
	}
}

func (this *Node) find(seg []rune) *Node {
	node := this
	for _, v := range seg {
		if child, ok := node.Children[v]; ok && child != nil {
			node = child
		} else {
			return nil
		}
	}
	return node
}

func (this *Node) PrefixSearch(prefix string, topCount int) SearchResult {
	seg, top := []rune(prefix), topN{make([]*WordCount, topCount), 0}
	if node := this.find(seg); node != nil {
		node.all(seg, &top)
	}
	return SearchResult(top.compact())
}

func (this *Node) substr(root *Node, pre, seg []rune, top *topN) {
	rp := append(pre, seg...)
	if node := root.find(rp); node != nil {
		node.all(rp, top)
	}

	for r, c := range this.Children {
		c.substr(root, append(pre, r), seg, top)
	}
}

func (this *Node) SubstrSearch(sub string, topCount int) SearchResult {
	seg, top := []rune(sub), topN{make([]*WordCount, topCount), 0}
	this.substr(this, nil, seg, &top)
	return SearchResult(top.compact())
}

func (this *Node) fuzzy(pre, seg []rune, index int, top *topN) {
	if index >= len(seg) {
		this.all(pre, top)
		for r, c := range this.Children {
			c.all(append(pre, r), top)
		}
		return
	}

	for r, c := range this.Children {
		if r == seg[index] {
			c.fuzzy(append(pre, r), seg, index+1, top)
		} else {
			c.fuzzy(append(pre, r), seg, index, top)
		}
	}
}

func (this *Node) FuzzySearch(fuzzy string, topCount int) SearchResult {
	seg, top := []rune(fuzzy), topN{make([]*WordCount, topCount), 0}
	this.fuzzy(nil, seg, 0, &top)
	return SearchResult(top.compact())
}
<|endoftext|>


<commit_before>/**
  Steam Library For Go
  Copyright (C) 2016 Ahmed Samy <f.fallen45@gmail.com>
  Copyright (C) 2016 Mark Samman <mark.samman@gmail.com>

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/
package steam

import (
	"crypto/md5"
	"crypto/rand"
	"crypto/rsa"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"math/big"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"strconv"
	"time"
)

type LoginResponse struct {
	Success      bool   `json:"success"`
	PublicKeyMod string `json:"publickey_mod"`
	PublicKeyExp string `json:"publickey_exp"`
	Timestamp    string
	TokenGID     string
}

type OAuth struct {
	SteamID       SteamID `json:"steamid,string"`
	Token         string  `json:"oauth_token"`
	WGToken       string  `json:"wgtoken"`
	WGTokenSecure string  `json:"wgtoken_secure"`
	WebCookie     string  `json:"webcookie"`
}

type LoginSession struct {
	Success           bool   `json:"success"`
	LoginComplete     bool   `json:"login_complete"`
	RequiresTwoFactor bool   `json:"requires_twofactor"`
	Message           string `json:"message"`
	RedirectURI       string `json:"redirect_uri"`
	OAuthInfo         string `json:"oauth"`
}

type Community struct {
	client    *http.Client
	oauth     OAuth
	sessionID string
	apiKey    string
	deviceID  string
}

const (
	deviceIDCookieName = "steamMachineAuth"

	httpXRequestedWithValue = "com.valvesoftware.android.steam.community"
	httpUserAgentValue      = "Mozilla/5.0 (Linux; U; Android 4.1.1; en-us; Google Nexus 4 - 4.1.1 - API 16 - 768x1280 Build/JRO03S) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30"
	httpAcceptValue         = "text/javascript, text/html, application/xml, text/xml, */*"
)

var (
	ErrUnableToLogin             = errors.New("unable to login")
	ErrInvalidUsername           = errors.New("invalid username")
	ErrNeedTwoFactor             = errors.New("invalid twofactor code")
	ErrMachineAuthCookieNotFound = errors.New("machine auth cookie not found")
)

func (community *Community) proceedDirectLogin(response *LoginResponse, accountName, password, sharedSecret string) error {
	n := &big.Int{}
	n.SetString(response.PublicKeyMod, 16)

	exp, err := strconv.ParseInt(response.PublicKeyExp, 16, 32)
	if err != nil {
		return err
	}

	pub := &rsa.PublicKey{N: n, E: int(exp)}
	rsaOut, err := rsa.EncryptPKCS1v15(rand.Reader, pub, []byte(password))
	if err != nil {
		return err
	}

	var twoFactorCode string
	if sharedSecret != "" {
		if twoFactorCode, err = GenerateTwoFactorCode(sharedSecret); err != nil {
			return err
		}
	}

	params := url.Values{
		"captcha_text":      {""},
		"captchagid":        {"-1"},
		"emailauth":         {""},
		"emailsteamid":      {""},
		"password":          {base64.StdEncoding.EncodeToString(rsaOut)},
		"remember_login":    {"true"},
		"rsatimestamp":      {response.Timestamp},
		"twofactorcode":     {twoFactorCode},
		"username":          {accountName},
		"oauth_client_id":   {"DE45CD61"},
		"oauth_scope":       {"read_profile write_profile read_client write_client"},
		"loginfriendlyname": {"#login_emailauth_friendlyname_mobile"},
		"donotcache":        {strconv.FormatInt(time.Now().Unix()*1000, 10)},
	}

	req, err := http.NewRequest(http.MethodPost, "https://steamcommunity.com/login/dologin/?"+params.Encode(), nil)
	if err != nil {
		return err
	}

	req.Header.Add("X-Requested-With", httpXRequestedWithValue)
	req.Header.Add("Referer", "https://steamcommunity.com/mobilelogin?oauth_client_id=DE45CD61&oauth_scope=read_profile%20write_profile%20read_client%20write_client")
	req.Header.Add("User-Agent", httpUserAgentValue)
	req.Header.Add("Accept", httpAcceptValue)

	resp, err := community.client.Do(req)
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return err
	}

	var session LoginSession
	if err := json.NewDecoder(resp.Body).Decode(&session); err != nil {
		return err
	}

	if !session.Success {
		if session.RequiresTwoFactor {
			return ErrNeedTwoFactor
		}

		return ErrUnableToLogin
	}

	randomBytes := make([]byte, 6)
	if _, err := rand.Read(randomBytes); err != nil {
		return err
	}

	sessionID := make([]byte, hex.EncodedLen(len(randomBytes)))
	hex.Encode(sessionID, randomBytes)
	community.sessionID = string(sessionID)

	url, _ := url.Parse("https://steamcommunity.com")
	cookies := community.client.Jar.Cookies(url)
	for _, cookie := range cookies {
		if cookie.Name == "mobileClient" || cookie.Name == "mobileClientVersion" {
			// remove by setting max age -1
			cookie.MaxAge = -1
		}
	}

	if sharedSecret != "" {
		sum := md5.Sum([]byte(sharedSecret))
		community.deviceID = fmt.Sprintf(
			"android:%x-%x-%x-%x-%x",
			sum[:2], sum[2:4], sum[4:6], sum[6:8], sum[8:10],
		)
	}

	community.client.Jar.SetCookies(
		url,
		append(cookies, &http.Cookie{
			Name:  "sessionid",
			Value: community.sessionID,
		}),
	)

	return json.Unmarshal([]byte(session.OAuthInfo), &community.oauth)
}

func (community *Community) Login(accountName, password, sharedSecret string) error {
	req, err := http.NewRequest(http.MethodPost, "https://steamcommunity.com/login/getrsakey?username="+accountName, nil)
	if err != nil {
		return err
	}

	jar, err := cookiejar.New(nil)
	if err != nil {
		return err
	}

	req.Header.Add("X-Requested-With", httpXRequestedWithValue)
	req.Header.Add("Referer", "https://steamcommunity.com/mobilelogin?oauth_client_id=DE45CD61&oauth_scope=read_profile%20write_profile%20read_client%20write_client")
	req.Header.Add("User-Agent", httpUserAgentValue)
	req.Header.Add("Accept", httpAcceptValue)

	cookies := []*http.Cookie{
		&http.Cookie{Name: "mobileClientVersion", Value: "0 (2.1.3)"},
		&http.Cookie{Name: "mobileClient", Value: "android"},
		&http.Cookie{Name: "Steam_Language", Value: "english"},
		&http.Cookie{Name: "timezoneOffset", Value: "0,0"},
	}
	url, _ := url.Parse("https://steamcommunity.com")
	jar.SetCookies(url, cookies)

	// Construct the client
	community.client = &http.Client{Jar: jar}

	resp, err := community.client.Do(req)
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return err
	}

	var response LoginResponse
	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return err
	}

	if !response.Success {
		return ErrInvalidUsername
	}

	return community.proceedDirectLogin(&response, accountName, password, sharedSecret)
}

func (community *Community) GetSteamID() SteamID {
	return community.oauth.SteamID
}
<commit_msg>Remove unused error<commit_after>/**
  Steam Library For Go
  Copyright (C) 2016 Ahmed Samy <f.fallen45@gmail.com>
  Copyright (C) 2016 Mark Samman <mark.samman@gmail.com>

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/
package steam

import (
	"crypto/md5"
	"crypto/rand"
	"crypto/rsa"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"math/big"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"strconv"
	"time"
)

type LoginResponse struct {
	Success      bool   `json:"success"`
	PublicKeyMod string `json:"publickey_mod"`
	PublicKeyExp string `json:"publickey_exp"`
	Timestamp    string
	TokenGID     string
}

type OAuth struct {
	SteamID       SteamID `json:"steamid,string"`
	Token         string  `json:"oauth_token"`
	WGToken       string  `json:"wgtoken"`
	WGTokenSecure string  `json:"wgtoken_secure"`
	WebCookie     string  `json:"webcookie"`
}

type LoginSession struct {
	Success           bool   `json:"success"`
	LoginComplete     bool   `json:"login_complete"`
	RequiresTwoFactor bool   `json:"requires_twofactor"`
	Message           string `json:"message"`
	RedirectURI       string `json:"redirect_uri"`
	OAuthInfo         string `json:"oauth"`
}

type Community struct {
	client    *http.Client
	oauth     OAuth
	sessionID string
	apiKey    string
	deviceID  string
}

const (
	deviceIDCookieName = "steamMachineAuth"

	httpXRequestedWithValue = "com.valvesoftware.android.steam.community"
	httpUserAgentValue      = "Mozilla/5.0 (Linux; U; Android 4.1.1; en-us; Google Nexus 4 - 4.1.1 - API 16 - 768x1280 Build/JRO03S) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30"
	httpAcceptValue         = "text/javascript, text/html, application/xml, text/xml, */*"
)

var (
	ErrUnableToLogin   = errors.New("unable to login")
	ErrInvalidUsername = errors.New("invalid username")
	ErrNeedTwoFactor   = errors.New("invalid twofactor code")
)

func (community *Community) proceedDirectLogin(response *LoginResponse, accountName, password, sharedSecret string) error {
	n := &big.Int{}
	n.SetString(response.PublicKeyMod, 16)

	exp, err := strconv.ParseInt(response.PublicKeyExp, 16, 32)
	if err != nil {
		return err
	}

	pub := &rsa.PublicKey{N: n, E: int(exp)}
	rsaOut, err := rsa.EncryptPKCS1v15(rand.Reader, pub, []byte(password))
	if err != nil {
		return err
	}

	var twoFactorCode string
	if sharedSecret != "" {
		if twoFactorCode, err = GenerateTwoFactorCode(sharedSecret); err != nil {
			return err
		}
	}

	params := url.Values{
		"captcha_text":      {""},
		"captchagid":        {"-1"},
		"emailauth":         {""},
		"emailsteamid":      {""},
		"password":          {base64.StdEncoding.EncodeToString(rsaOut)},
		"remember_login":    {"true"},
		"rsatimestamp":      {response.Timestamp},
		"twofactorcode":     {twoFactorCode},
		"username":          {accountName},
		"oauth_client_id":   {"DE45CD61"},
		"oauth_scope":       {"read_profile write_profile read_client write_client"},
		"loginfriendlyname": {"#login_emailauth_friendlyname_mobile"},
		"donotcache":        {strconv.FormatInt(time.Now().Unix()*1000, 10)},
	}

	req, err := http.NewRequest(http.MethodPost, "https://steamcommunity.com/login/dologin/?"+params.Encode(), nil)
	if err != nil {
		return err
	}

	req.Header.Add("X-Requested-With", httpXRequestedWithValue)
	req.Header.Add("Referer", "https://steamcommunity.com/mobilelogin?oauth_client_id=DE45CD61&oauth_scope=read_profile%20write_profile%20read_client%20write_client")
	req.Header.Add("User-Agent", httpUserAgentValue)
	req.Header.Add("Accept", httpAcceptValue)

	resp, err := community.client.Do(req)
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return err
	}

	var session LoginSession
	if err := json.NewDecoder(resp.Body).Decode(&session); err != nil {
		return err
	}

	if !session.Success {
		if session.RequiresTwoFactor {
			return ErrNeedTwoFactor
		}

		return ErrUnableToLogin
	}

	randomBytes := make([]byte, 6)
	if _, err := rand.Read(randomBytes); err != nil {
		return err
	}

	sessionID := make([]byte, hex.EncodedLen(len(randomBytes)))
	hex.Encode(sessionID, randomBytes)
	community.sessionID = string(sessionID)

	url, _ := url.Parse("https://steamcommunity.com")
	cookies := community.client.Jar.Cookies(url)
	for _, cookie := range cookies {
		if cookie.Name == "mobileClient" || cookie.Name == "mobileClientVersion" {
			// remove by setting max age -1
			cookie.MaxAge = -1
		}
	}

	if sharedSecret != "" {
		sum := md5.Sum([]byte(sharedSecret))
		community.deviceID = fmt.Sprintf(
			"android:%x-%x-%x-%x-%x",
			sum[:2], sum[2:4], sum[4:6], sum[6:8], sum[8:10],
		)
	}

	community.client.Jar.SetCookies(
		url,
		append(cookies, &http.Cookie{
			Name:  "sessionid",
			Value: community.sessionID,
		}),
	)

	return json.Unmarshal([]byte(session.OAuthInfo), &community.oauth)
}

func (community *Community) Login(accountName, password, sharedSecret string) error {
	req, err := http.NewRequest(http.MethodPost, "https://steamcommunity.com/login/getrsakey?username="+accountName, nil)
	if err != nil {
		return err
	}

	jar, err := cookiejar.New(nil)
	if err != nil {
		return err
	}

	req.Header.Add("X-Requested-With", httpXRequestedWithValue)
	req.Header.Add("Referer", "https://steamcommunity.com/mobilelogin?oauth_client_id=DE45CD61&oauth_scope=read_profile%20write_profile%20read_client%20write_client")
	req.Header.Add("User-Agent", httpUserAgentValue)
	req.Header.Add("Accept", httpAcceptValue)

	cookies := []*http.Cookie{
		&http.Cookie{Name: "mobileClientVersion", Value: "0 (2.1.3)"},
		&http.Cookie{Name: "mobileClient", Value: "android"},
		&http.Cookie{Name: "Steam_Language", Value: "english"},
		&http.Cookie{Name: "timezoneOffset", Value: "0,0"},
	}
	url, _ := url.Parse("https://steamcommunity.com")
	jar.SetCookies(url, cookies)

	// Construct the client
	community.client = &http.Client{Jar: jar}

	resp, err := community.client.Do(req)
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return err
	}

	var response LoginResponse
	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return err
	}

	if !response.Success {
		return ErrInvalidUsername
	}

	return community.proceedDirectLogin(&response, accountName, password, sharedSecret)
}

func (community *Community) GetSteamID() SteamID {
	return community.oauth.SteamID
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/cripplet/clicker/clicker-rest/lib"
	"net/http"
)

func main() {
	http.HandleFunc("/", cc_rest_lib.GameRouter)
	http.ListenAndServe(":8080", nil)
}
<commit_msg>Add rate limiter to REST server<commit_after>package main

import (
	"github.com/cripplet/clicker/clicker-rest/lib"
	"github.com/didip/tollbooth"
	"github.com/didip/tollbooth/limiter"
	"net/http"
	"time"
)

var rateLimiter *limiter.Limiter = tollbooth.NewLimiter(2, time.Second, nil)

func main() {
	http.Handle("/", tollbooth.LimitFuncHandler(rateLimiter, cc_rest_lib.GameRouter))
	http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>package driver

import (
	"encoding/json"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/go-plugin"
	"github.com/hashicorp/nomad/client/config"
	"github.com/hashicorp/nomad/client/driver/executor"
	cstructs "github.com/hashicorp/nomad/client/driver/structs"
	"github.com/hashicorp/nomad/helper/discover"
	"github.com/hashicorp/nomad/nomad/structs"
)

// cgroupsMounted returns true if the cgroups are mounted on a system otherwise
// returns false
func cgroupsMounted(node *structs.Node) bool {
	_, ok := node.Attributes["unique.cgroup.mountpoint"]
	return ok
}

// createExecutor launches an executor plugin and returns an instance of the
// Executor interface
func createExecutor(w io.Writer, clientConfig *config.Config,
	executorConfig *cstructs.ExecutorConfig) (executor.Executor, *plugin.Client, error) {

	c, err := json.Marshal(executorConfig)
	if err != nil {
		return nil, nil, fmt.Errorf("unable to create executor config: %v", err)
	}
	bin, err := discover.NomadExecutable()
	if err != nil {
		return nil, nil, fmt.Errorf("unable to find the nomad binary: %v", err)
	}

	config := &plugin.ClientConfig{
		Cmd: exec.Command(bin, "executor", string(c)),
	}
	config.HandshakeConfig = HandshakeConfig
	config.Plugins = GetPluginMap(w, clientConfig.LogLevel)
	config.MaxPort = clientConfig.ClientMaxPort
	config.MinPort = clientConfig.ClientMinPort

	// setting the setsid of the plugin process so that it doesn't get signals sent to
	// the nomad client.
	if config.Cmd != nil {
		isolateCommand(config.Cmd)
	}

	executorClient := plugin.NewClient(config)
	rpcClient, err := executorClient.Client()
	if err != nil {
		return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
	}

	raw, err := rpcClient.Dispense("executor")
	if err != nil {
		return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
	}
	executorPlugin := raw.(executor.Executor)
	return executorPlugin, executorClient, nil
}

func createExecutorWithConfig(config *plugin.ClientConfig, w io.Writer) (executor.Executor, *plugin.Client, error) {
	config.HandshakeConfig = HandshakeConfig

	// Setting this to DEBUG since the log level at the executor server process
	// is already set, and this effects only the executor client.
	config.Plugins = GetPluginMap(w, "DEBUG")

	executorClient := plugin.NewClient(config)
	rpcClient, err := executorClient.Client()
	if err != nil {
		return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
	}

	raw, err := rpcClient.Dispense("executor")
	if err != nil {
		return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
	}
	executorPlugin, ok := raw.(*ExecutorRPC)
	if !ok {
		return nil, nil, fmt.Errorf("unexpected executor rpc type: %T", raw)
	}
	// 0.6 Upgrade path: Deregister services from the executor as the Nomad
	// client agent now handles all Consul interactions.
	if err := executorPlugin.DeregisterServices(); err != nil {
		return nil, nil, err
	}
	return executorPlugin, executorClient, nil
}

// killProcess kills a process with the given pid
func killProcess(pid int) error {
	proc, err := os.FindProcess(pid)
	if err != nil {
		return err
	}
	return proc.Kill()
}

// destroyPlugin kills the plugin with the given pid and also kills the user
// process
func destroyPlugin(pluginPid int, userPid int) error {
	var merr error
	if err := killProcess(pluginPid); err != nil {
		merr = multierror.Append(merr, err)
	}

	if err := killProcess(userPid); err != nil {
		merr = multierror.Append(merr, err)
	}
	return merr
}

// validateCommand validates that the command only has a single value and
// returns a user friendly error message telling them to use the passed
// argField.
func validateCommand(command, argField string) error {
	trimmed := strings.TrimSpace(command)
	if len(trimmed) == 0 {
		return fmt.Errorf("command empty: %q", command)
	}

	if len(trimmed) != len(command) {
		return fmt.Errorf("command contains extra white space: %q", command)
	}

	return nil
}

// GetKillTimeout returns the kill timeout to use given the tasks desired kill
// timeout and the operator configured max kill timeout.
func GetKillTimeout(desired, max time.Duration) time.Duration {
	maxNanos := max.Nanoseconds()
	desiredNanos := desired.Nanoseconds()

	// Make the minimum time between signal and kill, 1 second.
	if desiredNanos <= 0 {
		desiredNanos = (1 * time.Second).Nanoseconds()
	}

	// Protect against max not being set properly.
	if maxNanos <= 0 {
		maxNanos = (10 * time.Second).Nanoseconds()
	}

	if desiredNanos < maxNanos {
		return time.Duration(desiredNanos)
	}

	return max
}

// GetAbsolutePath returns the absolute path of the passed binary by resolving
// it in the path and following symlinks.
func GetAbsolutePath(bin string) (string, error) {
	lp, err := exec.LookPath(bin)
	if err != nil {
		return "", fmt.Errorf("failed to resolve path to %q executable: %v", bin, err)
	}

	return filepath.EvalSymlinks(lp)
}

// getExecutorUser returns the user of the task, defaulting to
// cstructs.DefaultUnprivilegedUser if none was given.
func getExecutorUser(task *structs.Task) string {
	if task.User == "" {
		return cstructs.DefaultUnpriviledgedUser
	}
	return task.User
}
<commit_msg>Ignore Consul deregister errors on executors<commit_after>package driver

import (
	"encoding/json"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/go-plugin"
	"github.com/hashicorp/nomad/client/config"
	"github.com/hashicorp/nomad/client/driver/executor"
	cstructs "github.com/hashicorp/nomad/client/driver/structs"
	"github.com/hashicorp/nomad/helper/discover"
	"github.com/hashicorp/nomad/nomad/structs"
)

// cgroupsMounted returns true if the cgroups are mounted on a system otherwise
// returns false
func cgroupsMounted(node *structs.Node) bool {
	_, ok := node.Attributes["unique.cgroup.mountpoint"]
	return ok
}

// createExecutor launches an executor plugin and returns an instance of the
// Executor interface
func createExecutor(w io.Writer, clientConfig *config.Config,
	executorConfig *cstructs.ExecutorConfig) (executor.Executor, *plugin.Client, error) {

	c, err := json.Marshal(executorConfig)
	if err != nil {
		return nil, nil, fmt.Errorf("unable to create executor config: %v", err)
	}
	bin, err := discover.NomadExecutable()
	if err != nil {
		return nil, nil, fmt.Errorf("unable to find the nomad binary: %v", err)
	}

	config := &plugin.ClientConfig{
		Cmd: exec.Command(bin, "executor", string(c)),
	}
	config.HandshakeConfig = HandshakeConfig
	config.Plugins = GetPluginMap(w, clientConfig.LogLevel)
	config.MaxPort = clientConfig.ClientMaxPort
	config.MinPort = clientConfig.ClientMinPort

	// setting the setsid of the plugin process so that it doesn't get signals sent to
	// the nomad client.
	if config.Cmd != nil {
		isolateCommand(config.Cmd)
	}

	executorClient := plugin.NewClient(config)
	rpcClient, err := executorClient.Client()
	if err != nil {
		return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
	}

	raw, err := rpcClient.Dispense("executor")
	if err != nil {
		return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
	}
	executorPlugin := raw.(executor.Executor)
	return executorPlugin, executorClient, nil
}

func createExecutorWithConfig(config *plugin.ClientConfig, w io.Writer) (executor.Executor, *plugin.Client, error) {
	config.HandshakeConfig = HandshakeConfig

	// Setting this to DEBUG since the log level at the executor server process
	// is already set, and this effects only the executor client.
	config.Plugins = GetPluginMap(w, "DEBUG")

	executorClient := plugin.NewClient(config)
	rpcClient, err := executorClient.Client()
	if err != nil {
		return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
	}

	raw, err := rpcClient.Dispense("executor")
	if err != nil {
		return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
	}
	executorPlugin, ok := raw.(*ExecutorRPC)
	if !ok {
		return nil, nil, fmt.Errorf("unexpected executor rpc type: %T", raw)
	}
	// 0.6 Upgrade path: Deregister services from the executor as the Nomad
	// client agent now handles all Consul interactions. Ignore errors as
	// this shouldn't cause the alloc to fail and there's nothing useful to
	// do with them.
	executorPlugin.DeregisterServices()
	return executorPlugin, executorClient, nil
}

// killProcess kills a process with the given pid
func killProcess(pid int) error {
	proc, err := os.FindProcess(pid)
	if err != nil {
		return err
	}
	return proc.Kill()
}

// destroyPlugin kills the plugin with the given pid and also kills the user
// process
func destroyPlugin(pluginPid int, userPid int) error {
	var merr error
	if err := killProcess(pluginPid); err != nil {
		merr = multierror.Append(merr, err)
	}

	if err := killProcess(userPid); err != nil {
		merr = multierror.Append(merr, err)
	}
	return merr
}

// validateCommand validates that the command only has a single value and
// returns a user friendly error message telling them to use the passed
// argField.
func validateCommand(command, argField string) error {
	trimmed := strings.TrimSpace(command)
	if len(trimmed) == 0 {
		return fmt.Errorf("command empty: %q", command)
	}

	if len(trimmed) != len(command) {
		return fmt.Errorf("command contains extra white space: %q", command)
	}

	return nil
}

// GetKillTimeout returns the kill timeout to use given the tasks desired kill
// timeout and the operator configured max kill timeout.
func GetKillTimeout(desired, max time.Duration) time.Duration {
	maxNanos := max.Nanoseconds()
	desiredNanos := desired.Nanoseconds()

	// Make the minimum time between signal and kill, 1 second.
	if desiredNanos <= 0 {
		desiredNanos = (1 * time.Second).Nanoseconds()
	}

	// Protect against max not being set properly.
	if maxNanos <= 0 {
		maxNanos = (10 * time.Second).Nanoseconds()
	}

	if desiredNanos < maxNanos {
		return time.Duration(desiredNanos)
	}

	return max
}

// GetAbsolutePath returns the absolute path of the passed binary by resolving
// it in the path and following symlinks.
func GetAbsolutePath(bin string) (string, error) {
	lp, err := exec.LookPath(bin)
	if err != nil {
		return "", fmt.Errorf("failed to resolve path to %q executable: %v", bin, err)
	}

	return filepath.EvalSymlinks(lp)
}

// getExecutorUser returns the user of the task, defaulting to
// cstructs.DefaultUnprivilegedUser if none was given.
func getExecutorUser(task *structs.Task) string {
	if task.User == "" {
		return cstructs.DefaultUnpriviledgedUser
	}
	return task.User
}
<|endoftext|>