<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package format implements standard formatting of Go source.
package format

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/parser"
	"go/printer"
	"go/token"
	"io"
)

var config = printer.Config{Mode: printer.UseSpaces | printer.TabIndent, Tabwidth: 8}

const parserMode = parser.ParseComments

// Node formats node in canonical gofmt style and writes the result to dst.
//
// The node type must be *ast.File, *printer.CommentedNode, []ast.Decl,
// []ast.Stmt, or assignment-compatible to ast.Expr, ast.Decl, ast.Spec,
// or ast.Stmt. Node does not modify node. Imports are not sorted for
// nodes representing partial source files (i.e., if the node is not an
// *ast.File or a *printer.CommentedNode not wrapping an *ast.File).
//
// The function may return early (before the entire result is written)
// and return a formatting error, for instance due to an incorrect AST.
//
func Node(dst io.Writer, fset *token.FileSet, node interface{}) error {
	// Determine if we have a complete source file (file != nil).
	var file *ast.File
	var cnode *printer.CommentedNode
	switch n := node.(type) {
	case *ast.File:
		file = n
	case *printer.CommentedNode:
		if f, ok := n.Node.(*ast.File); ok {
			file = f
			cnode = n
		}
	}

	// Sort imports if necessary.
	if file != nil && hasUnsortedImports(file) {
		// Make a copy of the AST because ast.SortImports is destructive.
		// TODO(gri) Do this more efficiently.
		var buf bytes.Buffer
		err := config.Fprint(&buf, fset, file)
		if err != nil {
			return err
		}
		file, err = parser.ParseFile(fset, "", buf.Bytes(), parserMode)
		if err != nil {
			// We should never get here. If we do, provide good diagnostic.
			return fmt.Errorf("format.Node internal error (%s)", err)
		}
		ast.SortImports(fset, file)

		// Use new file with sorted imports.
		node = file
		if cnode != nil {
			node = &printer.CommentedNode{Node: file, Comments: cnode.Comments}
		}
	}

	return config.Fprint(dst, fset, node)
}

// Source formats src in canonical gofmt style and returns the result
// or an (I/O or syntax) error. src is expected to be a syntactically
// correct Go source file, or a list of Go declarations or statements.
//
// If src is a partial source file, the leading and trailing space of src
// is applied to the result (such that it has the same leading and trailing
// space as src), and the result is indented by the same amount as the first
// line of src containing code. Imports are not sorted for partial source files.
//
func Source(src []byte) ([]byte, error) {
	fset := token.NewFileSet()
	file, sourceAdj, indentAdj, err := parse(fset, "", src, true)
	if err != nil {
		return nil, err
	}

	if sourceAdj == nil {
		// Complete source file.
		// TODO(gri) consider doing this always.
		ast.SortImports(fset, file)
	}

	return format(fset, file, sourceAdj, indentAdj, src, config)
}

func hasUnsortedImports(file *ast.File) bool {
	for _, d := range file.Decls {
		d, ok := d.(*ast.GenDecl)
		if !ok || d.Tok != token.IMPORT {
			// Not an import declaration, so we're done.
			// Imports are always first.
			return false
		}
		if d.Lparen.IsValid() {
			// For now assume all grouped imports are unsorted.
			// TODO(gri) Should check if they are sorted already.
			return true
		}
		// Ungrouped imports are sorted by default.
	}
	return false
}
<commit_msg>go/format: document use of Source better<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package format implements standard formatting of Go source.
package format

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/parser"
	"go/printer"
	"go/token"
	"io"
)

var config = printer.Config{Mode: printer.UseSpaces | printer.TabIndent, Tabwidth: 8}

const parserMode = parser.ParseComments

// Node formats node in canonical gofmt style and writes the result to dst.
//
// The node type must be *ast.File, *printer.CommentedNode, []ast.Decl,
// []ast.Stmt, or assignment-compatible to ast.Expr, ast.Decl, ast.Spec,
// or ast.Stmt. Node does not modify node. Imports are not sorted for
// nodes representing partial source files (for instance, if the node is
// not an *ast.File or a *printer.CommentedNode not wrapping an *ast.File).
//
// The function may return early (before the entire result is written)
// and return a formatting error, for instance due to an incorrect AST.
//
func Node(dst io.Writer, fset *token.FileSet, node interface{}) error {
	// Determine if we have a complete source file (file != nil).
	var file *ast.File
	var cnode *printer.CommentedNode
	switch n := node.(type) {
	case *ast.File:
		file = n
	case *printer.CommentedNode:
		if f, ok := n.Node.(*ast.File); ok {
			file = f
			cnode = n
		}
	}

	// Sort imports if necessary.
	if file != nil && hasUnsortedImports(file) {
		// Make a copy of the AST because ast.SortImports is destructive.
		// TODO(gri) Do this more efficiently.
		var buf bytes.Buffer
		err := config.Fprint(&buf, fset, file)
		if err != nil {
			return err
		}
		file, err = parser.ParseFile(fset, "", buf.Bytes(), parserMode)
		if err != nil {
			// We should never get here. If we do, provide good diagnostic.
			return fmt.Errorf("format.Node internal error (%s)", err)
		}
		ast.SortImports(fset, file)

		// Use new file with sorted imports.
		node = file
		if cnode != nil {
			node = &printer.CommentedNode{Node: file, Comments: cnode.Comments}
		}
	}

	return config.Fprint(dst, fset, node)
}

// Source formats src in canonical gofmt style and returns the result
// or an (I/O or syntax) error. src is expected to be a syntactically
// correct Go source file, or a list of Go declarations or statements.
//
// If src is a partial source file, the leading and trailing space of src
// is applied to the result (such that it has the same leading and trailing
// space as src), and the result is indented by the same amount as the first
// line of src containing code. Imports are not sorted for partial source files.
//
// Caution: Tools relying on consistent formatting based on the installed
// version of gofmt (for instance, such as for presubmit checks) should
// execute that gofmt binary instead of calling Source.
//
func Source(src []byte) ([]byte, error) {
	fset := token.NewFileSet()
	file, sourceAdj, indentAdj, err := parse(fset, "", src, true)
	if err != nil {
		return nil, err
	}

	if sourceAdj == nil {
		// Complete source file.
		// TODO(gri) consider doing this always.
		ast.SortImports(fset, file)
	}

	return format(fset, file, sourceAdj, indentAdj, src, config)
}

func hasUnsortedImports(file *ast.File) bool {
	for _, d := range file.Decls {
		d, ok := d.(*ast.GenDecl)
		if !ok || d.Tok != token.IMPORT {
			// Not an import declaration, so we're done.
			// Imports are always first.
			return false
		}
		if d.Lparen.IsValid() {
			// For now assume all grouped imports are unsorted.
			// TODO(gri) Should check if they are sorted already.
			return true
		}
		// Ungrouped imports are sorted by default.
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"html"
	"log"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"
)

//import "github.com/JamesDunne/go-util/base"
import "github.com/JamesDunne/go-util/web"

type ImageViewModel struct {
	ID             int64   `json:"id"`
	Base62ID       string  `json:"base62id"`
	Title          string  `json:"title"`
	Kind           string  `json:"kind"`
	ImageURL       string  `json:"imageURL"`
	ThumbURL       string  `json:"thumbURL"`
	Submitter      string  `json:"submitter,omitempty"`
	CollectionName string  `json:"collectionName,omitempty"`
	SourceURL      *string `json:"sourceURL,omitempty"`
	RedirectToID   *int64  `json:"redirectToID,omitempty"`
	IsClean        bool    `json:"isClean"`
}

func queryBit() (list []*ImageViewModel, err error) {
	// Query i.bittwiddlers.org
	breq, err := http.NewRequest("GET", "https://i.bittwiddlers.org/api/v1/all", nil)
	if err != nil {
		return nil, err
	}

	breq.Header.Set("Authorization", env["BIT_AUTH"])

	// Skip bad self-signed x509 certificate:
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	client := &http.Client{Transport: tr}

	brsp, err := client.Do(breq)
	if err != nil {
		log.Printf("ERROR: %s\n", err)
		return nil, err
	}

	list = make([]*ImageViewModel, 0, 1000)
	json.NewDecoder(brsp.Body).Decode(
		&struct {
			Result struct {
				List *[]*ImageViewModel `json:"list"`
			} `json:"result"`
		}{
			Result: struct {
				List *[]*ImageViewModel `json:"list"`
			}{
				List: &list,
			},
		},
	)

	return list, nil
}

func jsonReplyText(text string) interface{} {
	return &struct {
		Text string `json:"text"`
	}{
		Text: text,
	}
}

func handleChatMessage(formValues map[string]string) (jsonResponse interface{}, werr *web.Error) {

	// NOTE(jsd): "@bitsbot" does not trigger with outgoing webhooks via trigger words.
	// Strip "bitsbot" prefix off text:
	text := strings.Trim(formValues["text"], " \t\n")
	if strings.HasPrefix(text, "bitsbot") {
		text = strings.TrimLeft(text[len("bitsbot"):], " :\t\n")
	}

	// Text is HTML encoded otherwise.

	// Debug tool for user "jdunne":
	if strings.HasPrefix(text, "json=") && formValues["user_id"] == "U03PV154T" {
		// Remove angle brackets around URLs:
		text = strings.Replace(text, "<", "", -1)
		text = strings.Replace(text, ">", "", -1)

		// Decode HTML:
		text = html.UnescapeString(text)

		// Unmarshal JSON:
		o := make(map[string]interface{})
		err := json.Unmarshal([]byte(text[len("json="):]), &o)
		if err != nil {
			log.Printf("ERROR: %s\n", err)
			goto otherwise
		}

		// Echo incoming JSON data as our response:
		return o, nil
	}

otherwise:
	// Query i.bittwiddlers.org for the list of images:
	list, err := queryBit()
	if err != nil {
		log.Printf("ERROR: %s\n", err)
		return nil, nil
	}

	// -list prefix will list best matches instead of randomly selecting one:
	do_list := false
	if strings.HasPrefix(text, "-list") {
		do_list = true
		text = strings.TrimLeft(text[len("-list"):], " :\t\n")

		// Shortcut to list all images:
		if text == "" {
			out := new(bytes.Buffer)
			if len(list) == 0 {
				fmt.Fprintf(out, "No images!\n")
			} else {
				fmt.Fprintf(out, "All images:\n")
				for _, img := range list {
					if img.Kind != "gif" && img.Kind != "jpeg" && img.Kind != "png" {
						continue
					}

					fmt.Fprintf(out, " * <http://i.bittwiddlers.org/b/%s|%s>\n", img.Base62ID, img.Title)
				}
			}

			return jsonReplyText(out.String()), nil
		}
	}

	// Runes used to split words:
	const wordSplitters = " \n\t:,;.-+=[]!?()$%^&*<>\"`"

	// Search by keyword:
	keywords := strings.FieldsFunc(
		strings.ToLower(text),
		func(c rune) bool { return strings.ContainsRune(wordSplitters, c) },
	)
	log.Printf("  keywords: %v\n", keywords)

	highest := -1
	highest_idxs := make([]int, 0, 20)
	for idx, img := range list {
		if img.Kind != "gif" && img.Kind != "jpeg" && img.Kind != "png" {
			continue
		}

		titleLower := strings.ToLower(img.Title)

		words := strings.FieldsFunc(
			titleLower,
			func(c rune) bool { return strings.ContainsRune(wordSplitters, c) },
		)

		h := -2

		// Add points for each keyword match:
		last_word_idx := -1
		// TODO(jsd): Don't count single-word matches on useless filler words like articles; only count them if in a phrase.
		// TODO(jsd): Prefer to match all keywords.
		for _, keyword := range keywords {
			for word_idx, word := range words {
				if word == keyword {
					if last_word_idx > -1 {
						if word_idx > last_word_idx+1 {
							// Penalize distance (word count) from last word found (helps phrases match better):
							h -= ((word_idx - last_word_idx) + 1)
						}
					}

					h += 10
					h = (h * 20) / 16
					last_word_idx = word_idx

					// Only trigger once per keyword:
					break
				}
			}
		}

		if h > -2 {
			log.Printf("  %4d %s\n", h, img.Title)

			if h > highest {
				highest = h
				// Build a new list of winners at this index:
				highest_idxs = highest_idxs[:0]
				highest_idxs = append(highest_idxs, idx)
			} else if h == highest {
				// Add to the winning pool:
				highest_idxs = append(highest_idxs, idx)
			}
		}
	}

	if do_list {
		out := new(bytes.Buffer)
		if len(highest_idxs) == 0 {
			fmt.Fprintf(out, "No matches for '%s'.\n", text)
		} else {
			fmt.Fprintf(out, "Best matches for '%s':\n", text)
			for _, idx := range highest_idxs {
				img := list[idx]
				fmt.Fprintf(out, " * <http://i.bittwiddlers.org/b/%s|%s>\n", img.Base62ID, img.Title)
			}
		}
		return jsonReplyText(out.String()), nil
	}

	winning_idx := -1
	if len(highest_idxs) == 0 {
		log.Printf("  No match!\n")
	} else if len(highest_idxs) == 1 {
		log.Printf("  Single match!\n")
		winning_idx = highest_idxs[0]
	} else {
		log.Printf("  %d winners at score %d; randomly selecting a winner\n", len(highest_idxs), highest)
		for _, idx := range highest_idxs {
			img := list[idx]
			log.Printf("    %s: %s\n", img.Base62ID, img.Title)
		}

		// Initialize a pseudo-random source:
		timestamp := int64(0)
		timestamp_float, err := strconv.ParseFloat(formValues["timestamp"], 64)
		if err != nil {
			timestamp = time.Now().UnixNano()
		} else {
			timestamp = int64(timestamp_float)
		}

		// Select a random winner:
		r := rand.New(rand.NewSource(timestamp))
		winning_idx = highest_idxs[r.Intn(len(highest_idxs))]
	}

	if winning_idx == -1 {
		return jsonReplyText(fmt.Sprintf("Sorry, %s, no match for '%s'.", formValues["user_name"], text)), nil
	}

	img := list[winning_idx]
	log.Printf("  winner: %s: %s\n", img.Base62ID, img.Title)

	return &struct {
		Text        string `json:"text"`
		Attachments []struct {
			Fallback string `json:"fallback"`
			ImageURL string `json:"image_url"`
		} `json:"attachments"`
	}{
		Text: img.Title,
		Attachments: []struct {
			Fallback string `json:"fallback"`
			ImageURL string `json:"image_url"`
		}{
			struct {
				Fallback string `json:"fallback"`
				ImageURL string `json:"image_url"`
			}{
				Fallback: img.Title,
				ImageURL: "http://i.bittwiddlers.org" + img.ImageURL,
			},
		},
	}, nil
}

func processRequest(rsp http.ResponseWriter, req *http.Request) *web.Error {
	if err := req.ParseForm(); err != nil {
		log.Printf("Could not parse form: %s\n", err)
		return web.AsError(err, http.StatusBadRequest)
	}

	formValues := make(map[string]string)
	for key, values := range req.PostForm {
		formValues[key] = strings.Join(values, " ")
	}

	if formValues["token"] != env["SLACK_TOKEN"] {
		// Not meant for us.
		rsp.WriteHeader(http.StatusOK)
		return nil
	}

	// Don't accept messages not intended for us:
	if formValues["trigger_word"] != "bitsbot" {
		rsp.WriteHeader(http.StatusOK)
		return nil
	}

	// Prevent infinite echos:
	user_name := formValues["user_name"]
	if user_name == "slackbot" {
		rsp.WriteHeader(http.StatusOK)
		return nil
	}

	// Log incoming text:
	log.Printf(
		"#%s <%s (%s)>: %s\n",
		formValues["channel_name"],
		user_name,
		formValues["user_id"],
		formValues["text"],
	)

	jsonResponse, werr := handleChatMessage(formValues)
	if werr != nil {
		return werr
	}

	// Write JSON response with attached image:
	rsp.Header().Set("Content-Type", "application/json")
	rsp.WriteHeader(http.StatusOK)

	o, err := json.Marshal(jsonResponse)
	if err != nil {
		log.Printf("ERROR: %s\n", err)
		return nil
	}

	//log.Printf("%s\n", string(o))
	rsp.Write(o)

	return nil
}
<commit_msg>Moved keyword search out to separate function.<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"html"
	"log"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"
)

//import "github.com/JamesDunne/go-util/base"
import "github.com/JamesDunne/go-util/web"

type ImageViewModel struct {
	ID             int64   `json:"id"`
	Base62ID       string  `json:"base62id"`
	Title          string  `json:"title"`
	Kind           string  `json:"kind"`
	ImageURL       string  `json:"imageURL"`
	ThumbURL       string  `json:"thumbURL"`
	Submitter      string  `json:"submitter,omitempty"`
	CollectionName string  `json:"collectionName,omitempty"`
	SourceURL      *string `json:"sourceURL,omitempty"`
	RedirectToID   *int64  `json:"redirectToID,omitempty"`
	IsClean        bool    `json:"isClean"`
}

func queryBit() (list []*ImageViewModel, err error) {
	// Query i.bittwiddlers.org
	breq, err := http.NewRequest("GET", "https://i.bittwiddlers.org/api/v1/all", nil)
	if err != nil {
		return nil, err
	}

	breq.Header.Set("Authorization", env["BIT_AUTH"])

	// Skip bad self-signed x509 certificate:
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	client := &http.Client{Transport: tr}

	brsp, err := client.Do(breq)
	if err != nil {
		log.Printf("ERROR: %s\n", err)
		return nil, err
	}

	list = make([]*ImageViewModel, 0, 1000)
	json.NewDecoder(brsp.Body).Decode(
		&struct {
			Result struct {
				List *[]*ImageViewModel `json:"list"`
			} `json:"result"`
		}{
			Result: struct {
				List *[]*ImageViewModel `json:"list"`
			}{
				List: &list,
			},
		},
	)

	return list, nil
}

func jsonReplyText(text string) interface{} {
	return &struct {
		Text string `json:"text"`
	}{
		Text: text,
	}
}

func keywordMatch(text string, list []*ImageViewModel) (winners []*ImageViewModel) {
	// Runes used to split words:
	const wordSplitters = " \n\t:,;.-+=[]!?()$%^&*<>\"`"

	// Search by keyword:
	keywords := strings.FieldsFunc(
		strings.ToLower(text),
		func(c rune) bool { return strings.ContainsRune(wordSplitters, c) },
	)
	log.Printf("  keywords: %v\n", keywords)

	highest := -1
	highest_idxs := make([]int, 0, 20)
	for idx, img := range list {
		if img.Kind != "gif" && img.Kind != "jpeg" && img.Kind != "png" {
			continue
		}

		titleLower := strings.ToLower(img.Title)

		words := strings.FieldsFunc(
			titleLower,
			func(c rune) bool { return strings.ContainsRune(wordSplitters, c) },
		)

		h := -2

		// Add points for each keyword match:
		last_word_idx := -1
		// TODO(jsd): Don't count single-word matches on useless filler words like articles; only count them if in a phrase.
		// TODO(jsd): Prefer to match all keywords.
		for _, keyword := range keywords {
			for word_idx, word := range words {
				if word == keyword {
					if last_word_idx > -1 {
						if word_idx > last_word_idx+1 {
							// Penalize distance (word count) from last word found (helps phrases match better):
							h -= ((word_idx - last_word_idx) + 1)
						}
					}

					h += 10
					h = (h * 20) / 16
					last_word_idx = word_idx

					// Only trigger once per keyword:
					break
				}
			}
		}

		if h > -2 {
			log.Printf("  %4d %s\n", h, img.Title)

			if h > highest {
				highest = h
				// Build a new list of winners at this index:
				highest_idxs = highest_idxs[:0]
				highest_idxs = append(highest_idxs, idx)
			} else if h == highest {
				// Add to the winning pool:
				highest_idxs = append(highest_idxs, idx)
			}
		}
	}

	winners = make([]*ImageViewModel, 0, len(highest_idxs))
	for _, idx := range highest_idxs {
		winners = append(winners, list[idx])
	}
	return
}

func handleChatMessage(formValues map[string]string) (jsonResponse interface{}, werr *web.Error) {
	// NOTE(jsd): "@bitsbot" does not trigger with outgoing webhooks via trigger words.
	// Strip "bitsbot" prefix off text:
	text := strings.Trim(formValues["text"], " \t\n")
	if strings.HasPrefix(text, "bitsbot") {
		text = strings.TrimLeft(text[len("bitsbot"):], " :\t\n")
	}

	// Text is HTML encoded otherwise.

	// Debug tool for user "jdunne":
	if strings.HasPrefix(text, "json=") && formValues["user_id"] == "U03PV154T" {
		// Remove angle brackets around URLs:
		text = strings.Replace(text, "<", "", -1)
		text = strings.Replace(text, ">", "", -1)

		// Decode HTML:
		text = html.UnescapeString(text)

		// Unmarshal JSON:
		o := make(map[string]interface{})
		err := json.Unmarshal([]byte(text[len("json="):]), &o)
		if err != nil {
			log.Printf("ERROR: %s\n", err)
			goto otherwise
		}

		// Echo incoming JSON data as our response:
		return o, nil
	}

otherwise:
	// Query i.bittwiddlers.org for the list of images:
	list, err := queryBit()
	if err != nil {
		log.Printf("ERROR: %s\n", err)
		return nil, nil
	}

	// -list prefix will list best matches instead of randomly selecting one:
	do_list := false
	if strings.HasPrefix(text, "-list") {
		do_list = true
		text = strings.TrimLeft(text[len("-list"):], " :\t\n")

		// Shortcut to list all images:
		if text == "" {
			out := new(bytes.Buffer)
			if len(list) == 0 {
				fmt.Fprintf(out, "No images!\n")
			} else {
				fmt.Fprintf(out, "All images:\n")
				for _, img := range list {
					if img.Kind != "gif" && img.Kind != "jpeg" && img.Kind != "png" {
						continue
					}

					fmt.Fprintf(out, " * <http://i.bittwiddlers.org/b/%s|%s>\n", img.Base62ID, img.Title)
				}
			}

			return jsonReplyText(out.String()), nil
		}
	}

	// Keyword search through image titles and find best matches:
	winners := keywordMatch(text, list)

	if do_list {
		out := new(bytes.Buffer)
		if len(winners) == 0 {
			fmt.Fprintf(out, "No matches for '%s'.\n", text)
		} else {
			fmt.Fprintf(out, "Best matches for '%s':\n", text)
			for _, img := range winners {
				fmt.Fprintf(out, " * <http://i.bittwiddlers.org/b/%s|%s>\n", img.Base62ID, img.Title)
			}
		}
		return jsonReplyText(out.String()), nil
	}

	img := (*ImageViewModel)(nil)
	if len(winners) == 0 {
		log.Printf("  No match!\n")
	} else if len(winners) == 1 {
		log.Printf("  Single match!\n")
		img = winners[0]
	} else {
		log.Printf("  %d winners at score %d; randomly selecting a winner\n", len(winners))
		for _, img := range winners {
			log.Printf("    %s: %s\n", img.Base62ID, img.Title)
		}

		// Initialize a pseudo-random source:
		timestamp := int64(0)
		timestamp_float, err := strconv.ParseFloat(formValues["timestamp"], 64)
		if err != nil {
			timestamp = time.Now().UnixNano()
		} else {
			timestamp = int64(timestamp_float)
		}

		// Select a random winner:
		r := rand.New(rand.NewSource(timestamp))
		img = winners[r.Intn(len(winners))]
	}

	if img == nil {
		return jsonReplyText(fmt.Sprintf("Sorry, %s, no match for '%s'.", formValues["user_name"], text)), nil
	}

	log.Printf("  winner: %s: %s\n", img.Base62ID, img.Title)

	return &struct {
		Text        string `json:"text"`
		Attachments []struct {
			Fallback string `json:"fallback"`
			ImageURL string `json:"image_url"`
		} `json:"attachments"`
	}{
		Text: img.Title,
		Attachments: []struct {
			Fallback string `json:"fallback"`
			ImageURL string `json:"image_url"`
		}{
			struct {
				Fallback string `json:"fallback"`
				ImageURL string `json:"image_url"`
			}{
				Fallback: img.Title,
				ImageURL: "http://i.bittwiddlers.org" + img.ImageURL,
			},
		},
	}, nil
}

func processRequest(rsp http.ResponseWriter, req *http.Request) *web.Error {
	if err := req.ParseForm(); err != nil {
		log.Printf("Could not parse form: %s\n", err)
		return web.AsError(err, http.StatusBadRequest)
	}

	formValues := make(map[string]string)
	for key, values := range req.PostForm {
		formValues[key] = strings.Join(values, " ")
	}

	if formValues["token"] != env["SLACK_TOKEN"] {
		// Not meant for us.
		rsp.WriteHeader(http.StatusOK)
		return nil
	}

	// Don't accept messages not intended for us:
	if formValues["trigger_word"] != "bitsbot" {
		rsp.WriteHeader(http.StatusOK)
		return nil
	}

	// Prevent infinite echos:
	user_name := formValues["user_name"]
	if user_name == "slackbot" {
		rsp.WriteHeader(http.StatusOK)
		return nil
	}

	// Log incoming text:
	log.Printf(
		"#%s <%s (%s)>: %s\n",
		formValues["channel_name"],
		user_name,
		formValues["user_id"],
		formValues["text"],
	)

	jsonResponse, werr := handleChatMessage(formValues)
	if werr != nil {
		return werr
	}

	// Write JSON response with attached image:
	rsp.Header().Set("Content-Type", "application/json")
	rsp.WriteHeader(http.StatusOK)

	o, err := json.Marshal(jsonResponse)
	if err != nil {
		log.Printf("ERROR: %s\n", err)
		return nil
	}

	//log.Printf("%s\n", string(o))
	rsp.Write(o)

	return nil
}
<|endoftext|>


<commit_before>package gowrapp

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"runtime/debug"
	"time"

	"github.com/Sirupsen/logrus"
)

type loggedResponse struct {
	w       http.ResponseWriter
	started time.Time
	status  int
	size    int
}

func (l *loggedResponse) Flush() {
	if wf, ok := l.w.(http.Flusher); ok {
		wf.Flush()
	}
}

func (l *loggedResponse) Header() http.Header { return l.w.Header() }

func (l *loggedResponse) Write(b []byte) (int, error) {
	if l.status == 0 {
		// The status will be StatusOK if WriteHeader has not been called yet
		l.status = http.StatusOK
	}
	size, err := l.w.Write(b)
	l.size += size
	return size, err
}

func (l *loggedResponse) WriteHeader(status int) {
	l.w.WriteHeader(status)
	l.status = status
}

// Recover is a middleware that recovers a handler from an error and logs the traceback
func Recover(handler http.Handler, log *logrus.Logger) http.Handler {
	return http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			defer func() {
				if rec := recover(); rec != nil {
					log.WithFields(logrus.Fields{
						"traceback": string(debug.Stack()),
					}).Error("Unhandled panic")
					w.WriteHeader(http.StatusInternalServerError)
				}
			}()
			handler.ServeHTTP(w, r)
		})
}

// LogRequest is a middleware that logs a request
func LogRequest(handler http.Handler, log *logrus.Logger) http.Handler {
	return http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {

			lw := loggedResponse{w: w, started: time.Now()}
			handler.ServeHTTP(&lw, r)

			lm := log.WithFields(logrus.Fields{
				"status": lw.status,
				"remote": r.RemoteAddr,
				"method": r.Method,
				"proto":  r.Proto,
				"uri":    r.RequestURI,
				"took":   time.Now().Sub(lw.started),
				"size":   lw.size,
			})
			byteBodyMessage, _ := ioutil.ReadAll(r.Body)
			bodyMessage := string(byteBodyMessage[:])
			switch {
			case lw.status < 400:
				lm.Info(http.StatusText(lw.status) + ": " + bodyMessage)
			default:
				lm.Error(http.StatusText(lw.status) + ": " + bodyMessage)
			}
		})
}

// RunHTTP starts a webserver with Wrapp logging and panic recovery
// The port number is fetched from the environment variable SERVICE_PORT
func RunHTTP(serviceName string, log *logrus.Logger, h http.Handler) {
	servicePort := GetenvDefault("SERVICE_PORT", "8080")
	log.Info(fmt.Sprintf("Starting %s on port %s", serviceName, servicePort))
	log.Fatal(http.ListenAndServe(":"+servicePort, LogRequest(Recover(h, log), log)))
}
<commit_msg>wrapping http.Error<commit_after>package gowrapp

import (
	"fmt"
	"net/http"
	"runtime/debug"
	"time"

	"github.com/Sirupsen/logrus"
)

type loggedResponse struct {
	w       http.ResponseWriter
	started time.Time
	status  int
	size    int
}

func (l *loggedResponse) Flush() {
	if wf, ok := l.w.(http.Flusher); ok {
		wf.Flush()
	}
}

func (l *loggedResponse) Header() http.Header { return l.w.Header() }

func (l *loggedResponse) Write(b []byte) (int, error) {
	if l.status == 0 {
		// The status will be StatusOK if WriteHeader has not been called yet
		l.status = http.StatusOK
	}
	size, err := l.w.Write(b)
	l.size += size
	return size, err
}

func (l *loggedResponse) WriteHeader(status int) {
	l.w.WriteHeader(status)
	l.status = status
}

// Recover is a middleware that recovers a handler from an error and logs the traceback
func Recover(handler http.Handler, log *logrus.Logger) http.Handler {
	return http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			defer func() {
				if rec := recover(); rec != nil {
					log.WithFields(logrus.Fields{
						"traceback": string(debug.Stack()),
					}).Error("Unhandled panic")
					w.WriteHeader(http.StatusInternalServerError)
				}
			}()
			handler.ServeHTTP(w, r)
		})
}

// LogRequest is a middleware that logs a request
func LogRequest(handler http.Handler, log *logrus.Logger) http.Handler {
	return http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {

			lw := loggedResponse{w: w, started: time.Now()}
			handler.ServeHTTP(&lw, r)

			lm := log.WithFields(logrus.Fields{
				"status": lw.status,
				"remote": r.RemoteAddr,
				"method": r.Method,
				"proto":  r.Proto,
				"uri":    r.RequestURI,
				"took":   time.Now().Sub(lw.started),
				"size":   lw.size,
			})
			switch {
			case lw.status < 400:
				lm.Info(http.StatusText(lw.status))
			default:
				lm.Error(http.StatusText(lw.status))
			}
		})
}

//wraps http.Error so we get the error message we return logged in the system
func Error(w http.ResponseWriter, error string, code int, log *logrus.Logger) {
	http.Error(w, error, code)
	log.Error(error)
}

// RunHTTP starts a webserver with Wrapp logging and panic recovery
// The port number is fetched from the environment variable SERVICE_PORT
func RunHTTP(serviceName string, log *logrus.Logger, h http.Handler) {
	servicePort := GetenvDefault("SERVICE_PORT", "8080")
	log.Info(fmt.Sprintf("Starting %s on port %s", serviceName, servicePort))
	log.Fatal(http.ListenAndServe(":"+servicePort, LogRequest(Recover(h, log), log)))
}
<|endoftext|>


<commit_before>package web

import (
	"encoding/json"
	"encoding/xml"
	"net/http"
	"net/url"
)

const (
	pbkdf2Iterations = 64000
	keySize          = 32
)

type Handler func(*Context)

type Param struct {
	Key   string
	Value string
}

type Params []Param

func (params Params) Get(name string) string {
	for i := range params {
		if params[i].Key == name {
			return params[i].Value
		}
	}
	return ""
}

type Context struct {
	Params  *Params
	Query   url.Values
	Payload url.Values
	Server  *Server
	Request *http.Request
	http.ResponseWriter
}

// Get value from Params by key
func (ctx *Context) Val(key string) string {
	return ctx.Params.Get(key)
}

// Get value from url Query by key
func (ctx *Context) Get(key string) string {

	if ctx.Query == nil {
		ctx.Query = ctx.Request.URL.Query()
	}

	return ctx.Query.Get(key)
}

// Get value from post Form by key
func (ctx *Context) Post(key string) string {

	if ctx.Payload == nil {
		ctx.Request.ParseForm()
		ctx.Payload = ctx.Request.PostForm
	}

	return ctx.Payload.Get(key)
}

func (ctx *Context) WriteString(text string) (int, error) {
	return ctx.ResponseWriter.Write([]byte(text))
}

func (ctx *Context) WriteJson(v interface{}) (int, error) {
	b, err := json.Marshal(v)

	if err != nil {
		return 0, err
	}

	return ctx.ResponseWriter.Write(b)
}

func (ctx *Context) WriteXml(v interface{}) (int, error) {
	b, err := xml.Marshal(v)

	if err != nil {
		return 0, err
	}

	return ctx.ResponseWriter.Write(b)
}

func (ctx *Context) SetHeader(key string, value string, unique bool) {
	if unique {
		ctx.Header().Set(key, value)
	} else {
		ctx.Header().Add(key, value)
	}
}

func (ctx *Context) SetContentType(val string) {
	ctx.Header().Set("Content-Type", contentType(val))
}

func (ctx *Context) SetCookie(cookie *http.Cookie) {
	ctx.SetHeader("Set-Cookie", cookie.String(), false)
}

func (ctx *Context) SetSecureCookie(name string, val string, age int64) error {
	// server := ctx.Server
	// if len(server.Config.CookieSecret) == 0 {
	// 	return ErrMissingCookieSecret
	// }
	// if len(server.encKey) == 0 || len(server.signKey) == 0 {
	// 	return ErrInvalidKey
	// }
	// ciphertext, err := encrypt([]byte(val), server.encKey)
	// if err != nil {
	// 	return err
	// }
	// sig := sign(ciphertext, server.signKey)
	// data := base64.StdEncoding.EncodeToString(ciphertext) + "|" + base64.StdEncoding.EncodeToString(sig)
	// ctx.SetCookie(newCookie(name, data, age))
	return nil
}

func (ctx *Context) GetSecureCookie(name string) (string, bool) {
	// for _, cookie := range ctx.Request.Cookies() {
	// 	if cookie.Name != name {
	// 		continue
	// 	}
	// 	parts := strings.SplitN(cookie.Value, "|", 2)
	// 	if len(parts) != 2 {
	// 		return "", false
	// 	}
	// 	ciphertext, err := base64.StdEncoding.DecodeString(parts[0])
	// 	if err != nil {
	// 		return "", false
	// 	}
	// 	sig, err := base64.StdEncoding.DecodeString(parts[1])
	// 	if err != nil {
	// 		return "", false
	// 	}
	// 	expectedSig := sign([]byte(ciphertext), ctx.Server.signKey)
	// 	if !bytes.Equal(expectedSig, sig) {
	// 		return "", false
	// 	}
	// 	plaintext, err := decrypt(ciphertext, ctx.Server.encKey)
	// 	if err != nil {
	// 		return "", false
	// 	}
	// 	return string(plaintext), true
	// }
	return "", false
}
<commit_msg>Change WriteJSON and WriteXML<commit_after>package web

import (
	"encoding/json"
	"encoding/xml"
	"net/http"
	"net/url"
)

const (
	pbkdf2Iterations = 64000
	keySize          = 32
)

type Handler func(*Context)

type Param struct {
	Key   string
	Value string
}

type Params []Param

func (params Params) Get(name string) string {
	for i := range params {
		if params[i].Key == name {
			return params[i].Value
		}
	}
	return ""
}

type Context struct {
	Params  *Params
	Query   url.Values
	Payload url.Values
	Server  *Server
	Request *http.Request
	http.ResponseWriter
}

// Get value from Params by key
func (ctx *Context) Val(key string) string {
	return ctx.Params.Get(key)
}

// Get value from url Query by key
func (ctx *Context) Get(key string) string {

	if ctx.Query == nil {
		ctx.Query = ctx.Request.URL.Query()
	}

	return ctx.Query.Get(key)
}

// Get value from post Form by key
func (ctx *Context) Post(key string) string {

	if ctx.Payload == nil {
		ctx.Request.ParseForm()
		ctx.Payload = ctx.Request.PostForm
	}

	return ctx.Payload.Get(key)
}

func (ctx *Context) WriteString(text string) (int, error) {
	return ctx.ResponseWriter.Write([]byte(text))
}

func (ctx *Context) WriteJSON(v interface{}) error {
	return json.NewEncoder(ctx.ResponseWriter).Encode(v)
}

func (ctx *Context) WriteXML(v interface{}) error {
	return xml.NewEncoder(ctx.ResponseWriter).Encode(v)
}

func (ctx *Context) SetHeader(key string, value string, unique bool) {
	if unique {
		ctx.Header().Set(key, value)
	} else {
		ctx.Header().Add(key, value)
	}
}

func (ctx *Context) SetContentType(val string) {
	ctx.Header().Set("Content-Type", contentType(val))
}

func (ctx *Context) SetCookie(cookie *http.Cookie) {
	ctx.SetHeader("Set-Cookie", cookie.String(), false)
}

func (ctx *Context) SetSecureCookie(name string, val string, age int64) error {
	// server := ctx.Server
	// if len(server.Config.CookieSecret) == 0 {
	// 	return ErrMissingCookieSecret
	// }
	// if len(server.encKey) == 0 || len(server.signKey) == 0 {
	// 	return ErrInvalidKey
	// }
	// ciphertext, err := encrypt([]byte(val), server.encKey)
	// if err != nil {
	// 	return err
	// }
	// sig := sign(ciphertext, server.signKey)
	// data := base64.StdEncoding.EncodeToString(ciphertext) + "|" + base64.StdEncoding.EncodeToString(sig)
	// ctx.SetCookie(newCookie(name, data, age))
	return nil
}

func (ctx *Context) GetSecureCookie(name string) (string, bool) {
	// for _, cookie := range ctx.Request.Cookies() {
	// 	if cookie.Name != name {
	// 		continue
	// 	}
	// 	parts := strings.SplitN(cookie.Value, "|", 2)
	// 	if len(parts) != 2 {
	// 		return "", false
	// 	}
	// 	ciphertext, err := base64.StdEncoding.DecodeString(parts[0])
	// 	if err != nil {
	// 		return "", false
	// 	}
	// 	sig, err := base64.StdEncoding.DecodeString(parts[1])
	// 	if err != nil {
	// 		return "", false
	// 	}
	// 	expectedSig := sign([]byte(ciphertext), ctx.Server.signKey)
	// 	if !bytes.Equal(expectedSig, sig) {
	// 		return "", false
	// 	}
	// 	plaintext, err := decrypt(ciphertext, ctx.Server.encKey)
	// 	if err != nil {
	// 		return "", false
	// 	}
	// 	return string(plaintext), true
	// }
	return "", false
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"regexp"
	"time"
)

var idChars = []rune("0123456789abcdef")
var idLength = 32

func randId() string {
	b := make([]rune, idLength)
	for i := range b {
		b[i] = idChars[rand.Intn(len(idChars))]
	}
	return string(b)
}

type Clients struct {
	subscriptions  map[string][]Client
	newClients     chan Client
	defunctClients chan string
	calls          chan rpcCall
}

type Client struct {
	id      string
	session string
	channel chan string
}

type rpcCall struct {
	id   string
	body string
}

func (c *Clients) subscribe(id string, client Client) {
	list, set := c.subscriptions[id]
	if !set {
		c.subscriptions[id] = []Client{}
	}
	c.subscriptions[id] = append(list, client)
}

func (c *Clients) unsubscribe(id string, client Client) {
	list := c.subscriptions[id]
	for i, other := range list {
		if other == client {
			c.subscriptions[id] = append(list[:i], list[i+1:]...)
			return
		}
	}
}

func (c *Clients) Start(conn net.Conn) {
	for {
		select {
		case client := <-c.newClients:
			c.subscriptions[client.id] = []Client{client}
			c.subscribe(client.session, client)
			c.subscribe("world", client)
			fmt.Fprintf(conn, "connected %s %s\n", client.id, client.session)
			fmt.Printf("-> connected %s %s\n", client.id, client.session)
		case id := <-c.defunctClients:
			client := c.subscriptions[id][0]
			delete(c.subscriptions, id)
			c.unsubscribe(client.session, client)
			c.unsubscribe("world", client)
			fmt.Fprintf(conn, "disconnected %s %d\n", id, time.Now().Unix())
			fmt.Printf("-> disconnected %s %d\n", id, time.Now().Unix())
		case call := <-c.calls:
			fmt.Fprintf(conn, "call %s %s\n", call.id, call.body)
			fmt.Printf("-> call %s %s\n", call.id, call.body)
		}
	}
}

func (c *Clients) processCall(w http.ResponseWriter, r *http.Request) {
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, "Error reading body.", http.StatusInternalServerError)
		fmt.Println(err)
		return
	}
	c.calls <- rpcCall{r.URL.Path, string(body)}
}

func (c *Clients) processStream(w http.ResponseWriter, r *http.Request) {
	cookie, err := r.Cookie("session")
	var sessionId string
	if err == http.ErrNoCookie {
		sessionId = "session" + randId()
		http.SetCookie(w, &http.Cookie{Name: "session",
			Value:    sessionId,
			Path:     "/",
			Expires:  time.Now().AddDate(1, 0, 0),
			MaxAge:   60 * 60 * 24 * 365,
			Secure:   true,
			HttpOnly: true})
	} else {
		sessionId = cookie.Value
	}

	f, ok := w.(http.Flusher)
	if !ok {
		http.Error(w, "Streaming unsupported!", http.StatusInternalServerError)
		return
	}

	id := r.URL.Path
	messageChan := make(chan string)
	c.newClients <- Client{id, sessionId, messageChan}

	notify := w.(http.CloseNotifier).CloseNotify()
	go func() {
		<-notify
		c.defunctClients <- id
	}()

	w.Header().Set("Content-Type", "text/event-stream")
	w.Header().Set("Cache-Control", "no-cache")
	w.Header().Set("Connection", "keep-alive")

	// Tell nginx to not buffer. Without this it may take up to a minute
	// for events to arrive at the client.
	w.Header().Set("X-Accel-Buffering", "no")

	// Send something to force the headers to flush.
	fmt.Fprintf(w, "\n\n")
	f.Flush()

	for {
		select {
		case msg := <-messageChan:
			fmt.Fprintf(w, "data: %s\n\n", msg)
		case <-time.After(30 * time.Second):
			fmt.Fprintf(w, ":heartbeat\n\n")
		}
		f.Flush()
	}
}

var pattern = regexp.MustCompile(`(\S+) (\S+) (.+)`)

func ReadCommands(conn net.Conn, clients *Clients) {
	reader := bufio.NewReader(conn)
	for {
		line, err := reader.ReadString('\n')
		if err == io.EOF {
			// If the socket is closed we will be having connection errors
			// everywhere when we try to report events. Best to close
			// everything.
			panic("Command center client disconnected.")
		}
		if err != nil {
			panic(err)
		}
		parts := pattern.FindStringSubmatch(line)
		if len(parts) == 0 {
			fmt.Println("Invalid command format. Expected 'command id params'.\n")
			continue
		}
		command := parts[1]
		id := parts[2]
		params := parts[3]
        fmt.Println("<- " + line)
		switch command {
		case "send":
			for _, client := range clients.subscriptions[id] {
				client.channel <- params
			}
		default:
			fmt.Println("Invalid command " + command + "\n")
		}
	}
}

func waitForClient() net.Conn {
	fmt.Println("Listening for command center client at :8001.")

	ln, err := net.Listen("tcp", ":8001")
	if err != nil {
		panic(err.Error())
	}

	conn, err := ln.Accept()
	if err != nil {
		panic(err.Error())
	}

	fmt.Println("Command center client connected.")

	return conn
}

func main() {
	rand.Seed(time.Now().UTC().UnixNano())

	conn := waitForClient()

	clients := &Clients{
		make(map[string][]Client),
		make(chan Client),
		make(chan string),
		make(chan rpcCall),
	}

	go clients.Start(conn)

	go ReadCommands(conn, clients)

	http.Handle("/call/", http.StripPrefix("/call/", http.HandlerFunc(clients.processCall)))
	http.Handle("/events/", http.StripPrefix("/events/", http.HandlerFunc(clients.processStream)))

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "index.html")
	})
	http.HandleFunc("/sse.js", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "sse.js")
	})
	panic(http.ListenAndServe(":8000", nil))
}
<commit_msg>Use set of clients instead of list for removal performance<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"regexp"
	"time"
)

var idChars = []rune("0123456789abcdef")
var idLength = 32

func randId() string {
	b := make([]rune, idLength)
	for i := range b {
		b[i] = idChars[rand.Intn(len(idChars))]
	}
	return string(b)
}

type Clients struct {
	clientsById    map[string]Client
	subscriptions  map[string]map[Client]bool
	newClients     chan Client
	defunctClients chan string
	calls          chan rpcCall
}

type Client struct {
	id      string
	session string
	channel chan string
}

type rpcCall struct {
	id   string
	body string
}

func (c *Clients) subscribe(id string, client Client) {
	_, set := c.subscriptions[id]
	if !set {
		c.subscriptions[id] = map[Client]bool{client: true}
	} else {
		c.subscriptions[id][client] = true
	}
}

func (c *Clients) unsubscribe(id string, client Client) {
	delete(c.subscriptions[id], client)
	if len(c.subscriptions[id]) == 0 {
		delete(c.subscriptions, id)
	}
}

func (c *Clients) Start(conn net.Conn) {
	for {
		select {
		case client := <-c.newClients:
			c.clientsById[client.id] = client
			c.subscribe(client.id, client)
			c.subscribe(client.session, client)
			c.subscribe("world", client)
			fmt.Fprintf(conn, "connected %s %s\n", client.id, client.session)
			fmt.Printf("-> connected %s %s\n", client.id, client.session)
		case id := <-c.defunctClients:
			client := c.clientsById[id]
			c.unsubscribe(client.id, client)
			c.unsubscribe(client.session, client)
			c.unsubscribe("world", client)
			delete(c.clientsById, id)
			fmt.Fprintf(conn, "disconnected %s %s\n", id, client.session)
			fmt.Printf("-> disconnected %s %s\n", id, client.session)
		case call := <-c.calls:
			fmt.Fprintf(conn, "call %s %s\n", call.id, call.body)
			fmt.Printf("-> call %s %s\n", call.id, call.body)
		}
	}
}

func (c *Clients) processCall(w http.ResponseWriter, r *http.Request) {
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, "Error reading body.", http.StatusInternalServerError)
		fmt.Println(err)
		return
	}
	c.calls <- rpcCall{r.URL.Path, string(body)}
}

func (c *Clients) processStream(w http.ResponseWriter, r *http.Request) {
	cookie, err := r.Cookie("session")
	var sessionId string
	if err == http.ErrNoCookie {
		sessionId = "session" + randId()
		http.SetCookie(w, &http.Cookie{Name: "session",
			Value:    sessionId,
			Path:     "/",
			Expires:  time.Now().AddDate(1, 0, 0),
			MaxAge:   60 * 60 * 24 * 365,
			Secure:   true,
			HttpOnly: true})
	} else {
		sessionId = cookie.Value
	}

	f, ok := w.(http.Flusher)
	if !ok {
		http.Error(w, "Streaming unsupported!", http.StatusInternalServerError)
		return
	}

	id := r.URL.Path
	messageChan := make(chan string)
	c.newClients <- Client{id, sessionId, messageChan}

	notify := w.(http.CloseNotifier).CloseNotify()
	go func() {
		<-notify
		c.defunctClients <- id
	}()

	w.Header().Set("Content-Type", "text/event-stream")
	w.Header().Set("Cache-Control", "no-cache")
	w.Header().Set("Connection", "keep-alive")

	// Tell nginx to not buffer. Without this it may take up to a minute
	// for events to arrive at the client.
	w.Header().Set("X-Accel-Buffering", "no")

	// Send something to force the headers to flush.
	fmt.Fprintf(w, "\n\n")
	f.Flush()

	for {
		select {
		case msg := <-messageChan:
			fmt.Fprintf(w, "data: %s\n\n", msg)
		case <-time.After(30 * time.Second):
			fmt.Fprintf(w, ":heartbeat\n\n")
		}
		f.Flush()
	}
}

var pattern = regexp.MustCompile(`(\S+) (\S+) (.+)`)

func ReadCommands(conn net.Conn, clients *Clients) {
	reader := bufio.NewReader(conn)
	for {
		line, err := reader.ReadString('\n')
		if err == io.EOF {
			// If the socket is closed we will be having connection errors
			// everywhere when we try to report events. Best to close
			// everything.
			panic("Command center client disconnected.")
		}
		if err != nil {
			panic(err)
		}
		parts := pattern.FindStringSubmatch(line)
		if len(parts) == 0 {
			fmt.Println("Invalid command format. Expected 'command id params'.\n")
			continue
		}
		command := parts[1]
		id := parts[2]
		params := parts[3]
		fmt.Println("<- " + line)
		switch command {
		case "send":
			for client, _ := range clients.subscriptions[id] {
				client.channel <- params
			}
		default:
			fmt.Println("Invalid command " + command + "\n")
		}
	}
}

func waitForClient() net.Conn {
	fmt.Println("Listening for command center client at :8001.")

	ln, err := net.Listen("tcp", ":8001")
	if err != nil {
		panic(err.Error())
	}

	conn, err := ln.Accept()
	if err != nil {
		panic(err.Error())
	}

	fmt.Println("Command center client connected.")

	return conn
}

func main() {
	rand.Seed(time.Now().UTC().UnixNano())

	conn := waitForClient()

	clients := &Clients{
		make(map[string]Client),
		make(map[string]map[Client]bool),
		make(chan Client),
		make(chan string),
		make(chan rpcCall),
	}

	go clients.Start(conn)

	go ReadCommands(conn, clients)

	http.Handle("/call/", http.StripPrefix("/call/", http.HandlerFunc(clients.processCall)))
	http.Handle("/events/", http.StripPrefix("/events/", http.HandlerFunc(clients.processStream)))

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "index.html")
	})
	http.HandleFunc("/sse.js", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "sse.js")
	})
	panic(http.ListenAndServe(":8000", nil))
}
<|endoftext|>


<commit_before>package associations

import (
	"reflect"

	"github.com/markbates/pop/columns"
)

// Association represents a definition of a model association
// field. It can represent a association of the type has_many
// belongs_to or has_one, and other customized types.
type Association interface {
	Kind() reflect.Kind
	Interface() interface{}
	Constraint() (string, []interface{})
}

// AssociationSortable a type to be sortable.
type AssociationSortable interface {
	OrderBy() string
	Association
}

// Associations a group of model associations.
type Associations []Association

// associationParams a wrapper for associations definition
// and creation.
type associationParams struct {
	field      reflect.StructField // an association field defined in model.
	modelType  reflect.Type        // the model type where this field is defined.
	modelValue reflect.Value       // the model value where this field is defined.
	popTags    columns.Tags        // the tags defined in this association field.
	model      interface{}         // the model, owner of the association.
}

// associationBuilder is a type representing an association builder implementation.
// see the builder defined in ./has_many_association.go as a guide of how to use it.
type associationBuilder func(associationParams) (Association, error)

// tableNameable represents a model with a customized table name.
type tableNameable interface {
	TableName() string
}
<commit_msg>remove tableNameble interface since it is not used<commit_after>package associations

import (
	"reflect"

	"github.com/markbates/pop/columns"
)

// Association represents a definition of a model association
// field. It can represent a association of the type has_many
// belongs_to or has_one, and other customized types.
type Association interface {
	Kind() reflect.Kind
	Interface() interface{}
	Constraint() (string, []interface{})
}

// AssociationSortable a type to be sortable.
type AssociationSortable interface {
	OrderBy() string
	Association
}

// Associations a group of model associations.
type Associations []Association

// associationParams a wrapper for associations definition
// and creation.
type associationParams struct {
	field      reflect.StructField // an association field defined in model.
	modelType  reflect.Type        // the model type where this field is defined.
	modelValue reflect.Value       // the model value where this field is defined.
	popTags    columns.Tags        // the tags defined in this association field.
	model      interface{}         // the model, owner of the association.
}

// associationBuilder is a type representing an association builder implementation.
// see the builder defined in ./has_many_association.go as a guide of how to use it.
type associationBuilder func(associationParams) (Association, error)
<|endoftext|>


<commit_before>package redisq

import (
	"time"
)

// Killmail from a kill
type Killmail struct {
	ID            int        `json:"killmail_id"`
	Hash          string     `json:"killmail_hash"`
	SolarSystemID int        `json:"solar_system_id"`
	Timestamp     time.Time  `json:"killmail_time"`
	Victim        Victim     `json:"victim"`
	Attackers     []Attacker `json:"attackers"`
	Zkb           Zkb        `json:"zkb"`
}

<<<<<<< HEAD
=======
// UnmarshalJSON parses the timestamp from CREST in to Go's time.Time type.
func (t *KillTime) UnmarshalJSON(b []byte) (err error) {
	t.Time, err = time.Parse("2006.01.02 15:04:05", strings.Replace(string(b), "\"", "", 2))
	return err
}

>>>>>>> eba3022d757d2b6a7e0d4b73564781e12c333720
// Attacker in a killmail
type Attacker struct {
	CharacterID    int     `json:"character_id"`
	CorporationID  int     `json:"corporation_id"`
	AllianceID     int     `json:"alliance_id"`
	ShipTypeID     int     `json:"ship_type_id"`
	WeaponTypeID   int     `json:"weapon_type_id"`
	DamageDone     int     `json:"damage_done"`
	FinalBlow      bool    `json:"final_blow"`
	SecurityStatus float32 `json:"security_status"`
}

// Victim in a killmail
type Victim struct {
	CharacterID   int    `json:"character_id"`
	CorporationID int    `json:"corporation_id"`
	AllianceID    int    `json:"alliance_id"`
	ShipTypeID    int    `json:"ship_type_id"`
	DamageTaken   int    `json:"damage_taken"`
	Items         []Item `json:"items"`
	Position      struct {
		X float64 `json:"x"`
		Y float64 `json:"y"`
		Z float64 `json:"z"`
	} `json:"position"`
}

// Item dropped/destroyed in a killmail
type Item struct {
	ItemTypeID        int `json:"item_type_id"`
	Flag              int `json:"flag"`
	Singleton         int `json:"singleton"`
	QuantityDropped   int `json:"quantity_dropped"`
	QuantityDestroyed int `json:"quantity_destroyed"`
}

// Zkb is the meta data returned from ZKillboard.
type Zkb struct {
	Hash        string  `json:"hash"`
	FittedValue float32 `json:"fittedValue"`
	TotalValue  float32 `json:"totalValue"`
	Points      int     `json:"points"`
	NPC         bool    `json:"npc"`
	Solo        bool    `json:"solo"`
	AWOX        bool    `json:"awox"`
	Href        string  `json:"href"`
}
<commit_msg>fixes bad merge<commit_after>package redisq

import (
	"time"
)

// Killmail from a kill
type Killmail struct {
	ID            int        `json:"killmail_id"`
	Hash          string     `json:"killmail_hash"`
	SolarSystemID int        `json:"solar_system_id"`
	Timestamp     time.Time  `json:"killmail_time"`
	Victim        Victim     `json:"victim"`
	Attackers     []Attacker `json:"attackers"`
	Zkb           Zkb        `json:"zkb"`
}

// Attacker in a killmail
type Attacker struct {
	CharacterID    int     `json:"character_id"`
	CorporationID  int     `json:"corporation_id"`
	AllianceID     int     `json:"alliance_id"`
	ShipTypeID     int     `json:"ship_type_id"`
	WeaponTypeID   int     `json:"weapon_type_id"`
	DamageDone     int     `json:"damage_done"`
	FinalBlow      bool    `json:"final_blow"`
	SecurityStatus float32 `json:"security_status"`
}

// Victim in a killmail
type Victim struct {
	CharacterID   int    `json:"character_id"`
	CorporationID int    `json:"corporation_id"`
	AllianceID    int    `json:"alliance_id"`
	ShipTypeID    int    `json:"ship_type_id"`
	DamageTaken   int    `json:"damage_taken"`
	Items         []Item `json:"items"`
	Position      struct {
		X float64 `json:"x"`
		Y float64 `json:"y"`
		Z float64 `json:"z"`
	} `json:"position"`
}

// Item dropped/destroyed in a killmail
type Item struct {
	ItemTypeID        int `json:"item_type_id"`
	Flag              int `json:"flag"`
	Singleton         int `json:"singleton"`
	QuantityDropped   int `json:"quantity_dropped"`
	QuantityDestroyed int `json:"quantity_destroyed"`
}

// Zkb is the meta data returned from ZKillboard.
type Zkb struct {
	Hash        string  `json:"hash"`
	FittedValue float32 `json:"fittedValue"`
	TotalValue  float32 `json:"totalValue"`
	Points      int     `json:"points"`
	NPC         bool    `json:"npc"`
	Solo        bool    `json:"solo"`
	AWOX        bool    `json:"awox"`
	Href        string  `json:"href"`
}
<|endoftext|>


<commit_before>package commands

import (
	"context"
	"fmt"
	"strconv"

	"golang.org/x/sys/windows"

	"github.com/dustin/go-humanize"

	"github.com/zetamatta/nyagos/dos"
)

func driveType(rootPathName string) string {
	_rootPathName, err := windows.UTF16PtrFromString(rootPathName)
	if err != nil {
		return "UNKNOWN"
	}

	t := windows.GetDriveType(_rootPathName)
	switch t {
	case windows.DRIVE_REMOVABLE:
		return "REMOVABLE"
	case windows.DRIVE_FIXED:
		return "FIXED"
	case windows.DRIVE_REMOTE:
		return "REMOTE"
	case windows.DRIVE_CDROM:
		return "CDROM"
	case windows.DRIVE_RAMDISK:
		return "RAMDISK"
	default:
		return "UNKNOWN"
	}
}

func df(rootPathName string) ([]string, error) {
	label, fs, err := dos.VolumeName(rootPathName)
	if err != nil {
		return nil, fmt.Errorf("%s: %s", rootPathName, err)
	}
	free, total, totalFree, err := dos.GetDiskFreeSpace(rootPathName)
	if err != nil {
		return nil, fmt.Errorf("%s: %s", rootPathName, err)
	}
	return []string{
		rootPathName,
		humanize.Comma(int64(free)),
		humanize.Comma(int64(total)),
		humanize.Comma(int64(totalFree)),
		strconv.FormatUint(100*(total-free)/total, 10),
		fs,
		driveType(rootPathName),
		label,
	}, nil
}

func cmdDiskFree(_ context.Context, cmd Param) (int, error) {
	bits, err := windows.GetLogicalDrives()
	if err != nil {
		return 0, err
	}
	dfs := [][]string{
		[]string{"", "Available", "Total", "TotalFree", "Use%"},
	}

	for _, arg1 := range cmd.Args()[1:] {
		if df1, err := df(arg1); err != nil {
			fmt.Fprintln(cmd.Err(), err)
		} else {
			dfs = append(dfs, df1)
		}
	}
	if len(dfs) <= 1 {
		for d := 'A'; d <= 'Z'; d++ {
			if (bits & 1) != 0 {
				rootPathName := fmt.Sprintf("%c:\\", d)
				if df1, err := df(rootPathName); err != nil {
					fmt.Fprintln(cmd.Err(), err)
				} else {
					dfs = append(dfs, df1)
				}
			}
			bits >>= 1
		}
	}

	colsiz := []int{}
	for _, df1 := range dfs {
		for i, s := range df1 {
			if i >= len(colsiz) {
				colsiz = append(colsiz, len(s))
			} else if len(s) > colsiz[i] {
				colsiz[i] = len(s)
			}
		}
	}
	for _, df1 := range dfs {
		for i, s := range df1 {
			if i > 0 {
				cmd.Out().Write([]byte{' '})
			}
			if i >= 5 {
				fmt.Fprintf(cmd.Out(), "%-*s", colsiz[i], s)
			} else {
				fmt.Fprintf(cmd.Out(), "%*s", colsiz[i], s)
			}
		}
		fmt.Fprintln(cmd.Out())
	}

	return 0, nil
}
<commit_msg>diskfree: trim spaces from the end of line<commit_after>package commands

import (
	"context"
	"fmt"
	"strconv"

	"golang.org/x/sys/windows"

	"github.com/dustin/go-humanize"

	"github.com/zetamatta/nyagos/dos"
)

func driveType(rootPathName string) string {
	_rootPathName, err := windows.UTF16PtrFromString(rootPathName)
	if err != nil {
		return "UNKNOWN"
	}

	t := windows.GetDriveType(_rootPathName)
	switch t {
	case windows.DRIVE_REMOVABLE:
		return "REMOVABLE"
	case windows.DRIVE_FIXED:
		return "FIXED"
	case windows.DRIVE_REMOTE:
		return "REMOTE"
	case windows.DRIVE_CDROM:
		return "CDROM"
	case windows.DRIVE_RAMDISK:
		return "RAMDISK"
	default:
		return "UNKNOWN"
	}
}

func df(rootPathName string) ([]string, error) {
	label, fs, err := dos.VolumeName(rootPathName)
	if err != nil {
		return nil, fmt.Errorf("%s: %s", rootPathName, err)
	}
	free, total, totalFree, err := dos.GetDiskFreeSpace(rootPathName)
	if err != nil {
		return nil, fmt.Errorf("%s: %s", rootPathName, err)
	}
	return []string{
		rootPathName,
		humanize.Comma(int64(free)),
		humanize.Comma(int64(total)),
		humanize.Comma(int64(totalFree)),
		strconv.FormatUint(100*(total-free)/total, 10),
		fs,
		driveType(rootPathName),
		label,
	}, nil
}

func cmdDiskFree(_ context.Context, cmd Param) (int, error) {
	bits, err := windows.GetLogicalDrives()
	if err != nil {
		return 0, err
	}
	dfs := [][]string{
		[]string{"", "Available", "Total", "TotalFree", "Use%"},
	}

	for _, arg1 := range cmd.Args()[1:] {
		if df1, err := df(arg1); err != nil {
			fmt.Fprintln(cmd.Err(), err)
		} else {
			dfs = append(dfs, df1)
		}
	}
	if len(dfs) <= 1 {
		for d := 'A'; d <= 'Z'; d++ {
			if (bits & 1) != 0 {
				rootPathName := fmt.Sprintf("%c:\\", d)
				if df1, err := df(rootPathName); err != nil {
					fmt.Fprintln(cmd.Err(), err)
				} else {
					dfs = append(dfs, df1)
				}
			}
			bits >>= 1
		}
	}

	colsiz := []int{}
	for _, df1 := range dfs {
		for i, s := range df1 {
			if i >= len(colsiz) {
				colsiz = append(colsiz, len(s))
			} else if len(s) > colsiz[i] {
				colsiz[i] = len(s)
			}
		}
	}
	for _, df1 := range dfs {
		for i, s := range df1 {
			if i > 0 {
				cmd.Out().Write([]byte{' '})
			}
			if i == len(df1)-1 {
				fmt.Fprintln(cmd.Out(), s)
			} else if i >= 5 {
				fmt.Fprintf(cmd.Out(), "%-*s", colsiz[i], s)
			} else {
				fmt.Fprintf(cmd.Out(), "%*s", colsiz[i], s)
			}
		}
	}

	return 0, nil
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"

	"google.golang.org/api/iterator"

	"cloud.google.com/go/datastore"
)

// StorageService using Google Cloud Datastore
type googleCDSStore struct {
	client *datastore.Client
}

func makeGoogleCDSStore(ctx context.Context, projectID string) (StorageService, error) {
	client, err := datastore.NewClient(ctx, projectID)
	if err != nil {
		return nil, fmt.Errorf("Could not create datastore client: %s", err)
	}
	return &googleCDSStore{client: client}, nil
}

func getTeamKey(teamID string) *datastore.Key {
	return datastore.NameKey("Team", teamID, nil)
}

func getPeriodKey(teamKey *datastore.Key, periodID string) *datastore.Key {
	return datastore.NameKey("Period", periodID, teamKey)
}

func (s *googleCDSStore) GetAllTeams(ctx context.Context) ([]Team, error) {
	query := datastore.NewQuery("Team").Order("DisplayName")
	iter := s.client.Run(ctx, query)
	result := []Team{}
	for {
		var t Team
		_, err := iter.Next(&t)
		if err == iterator.Done {
			break
		}
		if err != nil {
			return result, err
		}
		result = append(result, t)
	}
	return result, nil
}

func (s *googleCDSStore) GetTeam(ctx context.Context, teamID string) (Team, bool, error) {
	key := getTeamKey(teamID)
	var team Team
	err := s.client.Get(ctx, key, &team)
	if err == datastore.ErrNoSuchEntity {
		return team, false, nil
	}
	if err != nil {
		return team, true, err
	}
	return team, true, nil
}

func (s *googleCDSStore) CreateTeam(ctx context.Context, team Team) error {
	key := getTeamKey(team.ID)
	_, err := s.client.RunInTransaction(ctx, func(tx *datastore.Transaction) error {
		var empty Team
		if err := tx.Get(key, &empty); err != datastore.ErrNoSuchEntity {
			return fmt.Errorf("Expected no existing team '%s', found: %s", team.ID, err)
		}
		_, err := tx.Put(key, &team)
		return err
	})
	return err
}

func (s *googleCDSStore) UpdateTeam(ctx context.Context, team Team) error {
	key := getTeamKey(team.ID)
	_, err := s.client.RunInTransaction(ctx, func(tx *datastore.Transaction) error {
		var ignored Team
		if err := tx.Get(key, &ignored); err != nil {
			return fmt.Errorf("Could not retrieve team '%s': %s", team.ID, err)
		}
		_, err := tx.Put(key, &team)
		return err
	})
	return err
}

func (s *googleCDSStore) GetAllPeriods(ctx context.Context, teamID string) ([]Period, bool, error) {
	if _, ok, err := s.GetTeam(ctx, teamID); !ok || err != nil {
		return nil, false, err
	}
	teamKey := getTeamKey(teamID)
	query := datastore.NewQuery("Period").Ancestor(teamKey)
	iter := s.client.Run(ctx, query)
	result := []Period{}
	for {
		var p Period
		_, err := iter.Next(&p)
		if err == iterator.Done {
			break
		}
		if err != nil {
			return result, true, err
		}
		scrubLoadedPeriod(&p)
		result = append(result, p)
	}
	return result, true, nil
}

func (s *googleCDSStore) GetPeriod(ctx context.Context, teamID, periodID string) (Period, bool, error) {
	teamKey := getTeamKey(teamID)
	periodKey := getPeriodKey(teamKey, periodID)
	var period Period
	err := s.client.Get(ctx, periodKey, &period)
	if err == datastore.ErrNoSuchEntity {
		return period, false, nil
	}
	scrubLoadedPeriod(&period)
	return period, true, err
}

func (s *googleCDSStore) CreatePeriod(ctx context.Context, teamID string, period Period) error {
	teamKey := getTeamKey(teamID)
	periodKey := getPeriodKey(teamKey, period.ID)
	_, err := s.client.RunInTransaction(ctx, func(tx *datastore.Transaction) error {
		var empty Period
		if err := tx.Get(periodKey, &empty); err != datastore.ErrNoSuchEntity {
			return fmt.Errorf("Expected no period '%s' for team '%s': %s", period.ID, teamID, err)
		}
		_, err := tx.Put(periodKey, &period)
		return err
	})
	return err
}

func (s *googleCDSStore) UpdatePeriod(ctx context.Context, teamID string, period Period) error {
	teamKey := getTeamKey(teamID)
	periodKey := getPeriodKey(teamKey, period.ID)
	_, err := s.client.RunInTransaction(ctx, func(tx *datastore.Transaction) error {
		var ignored Period
		if err := tx.Get(periodKey, &ignored); err != nil {
			return fmt.Errorf("Could not retrieve period '%s' for team '%s': %s", period.ID, teamID, err)
		}
		_, err := tx.Put(periodKey, &period)
		return err
	})
	return err
}

func (s *googleCDSStore) Close() error {
	return s.client.Close()
}

// Cloud Datastore does not save zero-length slices, so when retrieving entities with slice members,
// they may be nil. This function is to avoid clients having to deal with this.
func scrubLoadedPeriod(period *Period) {
	if period.People == nil {
		period.People = []Person{}
	}
	if period.Buckets == nil {
		period.Buckets = []Bucket{}
	}
	for i := range period.Buckets {
		if period.Buckets[i].Objectives == nil {
			period.Buckets[i].Objectives = []Objective{}
		}
		for j := range period.Buckets[i].Objectives {
			if period.Buckets[i].Objectives[j].Assignments == nil {
				period.Buckets[i].Objectives[j].Assignments = []Assignment{}
			}
		}
	}
}
<commit_msg>Extract constants for kind names<commit_after>package main

import (
	"context"
	"fmt"

	"google.golang.org/api/iterator"

	"cloud.google.com/go/datastore"
)

const (
	// TeamKind - Datastore kind name for teams
	TeamKind = "Team"
	// PeriodKind - Datastore kind name for periods
	PeriodKind = "Period"
)

// StorageService using Google Cloud Datastore
type googleCDSStore struct {
	client *datastore.Client
}

func makeGoogleCDSStore(ctx context.Context, projectID string) (StorageService, error) {
	client, err := datastore.NewClient(ctx, projectID)
	if err != nil {
		return nil, fmt.Errorf("Could not create datastore client: %s", err)
	}
	return &googleCDSStore{client: client}, nil
}

func getTeamKey(teamID string) *datastore.Key {
	return datastore.NameKey(TeamKind, teamID, nil)
}

func getPeriodKey(teamKey *datastore.Key, periodID string) *datastore.Key {
	return datastore.NameKey(PeriodKind, periodID, teamKey)
}

func (s *googleCDSStore) GetAllTeams(ctx context.Context) ([]Team, error) {
	query := datastore.NewQuery(TeamKind).Order("DisplayName")
	iter := s.client.Run(ctx, query)
	result := []Team{}
	for {
		var t Team
		_, err := iter.Next(&t)
		if err == iterator.Done {
			break
		}
		if err != nil {
			return result, err
		}
		result = append(result, t)
	}
	return result, nil
}

func (s *googleCDSStore) GetTeam(ctx context.Context, teamID string) (Team, bool, error) {
	key := getTeamKey(teamID)
	var team Team
	err := s.client.Get(ctx, key, &team)
	if err == datastore.ErrNoSuchEntity {
		return team, false, nil
	}
	if err != nil {
		return team, true, err
	}
	return team, true, nil
}

func (s *googleCDSStore) CreateTeam(ctx context.Context, team Team) error {
	key := getTeamKey(team.ID)
	_, err := s.client.RunInTransaction(ctx, func(tx *datastore.Transaction) error {
		var empty Team
		if err := tx.Get(key, &empty); err != datastore.ErrNoSuchEntity {
			return fmt.Errorf("Expected no existing team '%s', found: %s", team.ID, err)
		}
		_, err := tx.Put(key, &team)
		return err
	})
	return err
}

func (s *googleCDSStore) UpdateTeam(ctx context.Context, team Team) error {
	key := getTeamKey(team.ID)
	_, err := s.client.RunInTransaction(ctx, func(tx *datastore.Transaction) error {
		var ignored Team
		if err := tx.Get(key, &ignored); err != nil {
			return fmt.Errorf("Could not retrieve team '%s': %s", team.ID, err)
		}
		_, err := tx.Put(key, &team)
		return err
	})
	return err
}

func (s *googleCDSStore) GetAllPeriods(ctx context.Context, teamID string) ([]Period, bool, error) {
	if _, ok, err := s.GetTeam(ctx, teamID); !ok || err != nil {
		return nil, false, err
	}
	teamKey := getTeamKey(teamID)
	query := datastore.NewQuery(PeriodKind).Ancestor(teamKey)
	iter := s.client.Run(ctx, query)
	result := []Period{}
	for {
		var p Period
		_, err := iter.Next(&p)
		if err == iterator.Done {
			break
		}
		if err != nil {
			return result, true, err
		}
		scrubLoadedPeriod(&p)
		result = append(result, p)
	}
	return result, true, nil
}

func (s *googleCDSStore) GetPeriod(ctx context.Context, teamID, periodID string) (Period, bool, error) {
	teamKey := getTeamKey(teamID)
	periodKey := getPeriodKey(teamKey, periodID)
	var period Period
	err := s.client.Get(ctx, periodKey, &period)
	if err == datastore.ErrNoSuchEntity {
		return period, false, nil
	}
	scrubLoadedPeriod(&period)
	return period, true, err
}

func (s *googleCDSStore) CreatePeriod(ctx context.Context, teamID string, period Period) error {
	teamKey := getTeamKey(teamID)
	periodKey := getPeriodKey(teamKey, period.ID)
	_, err := s.client.RunInTransaction(ctx, func(tx *datastore.Transaction) error {
		var empty Period
		if err := tx.Get(periodKey, &empty); err != datastore.ErrNoSuchEntity {
			return fmt.Errorf("Expected no period '%s' for team '%s': %s", period.ID, teamID, err)
		}
		_, err := tx.Put(periodKey, &period)
		return err
	})
	return err
}

func (s *googleCDSStore) UpdatePeriod(ctx context.Context, teamID string, period Period) error {
	teamKey := getTeamKey(teamID)
	periodKey := getPeriodKey(teamKey, period.ID)
	_, err := s.client.RunInTransaction(ctx, func(tx *datastore.Transaction) error {
		var ignored Period
		if err := tx.Get(periodKey, &ignored); err != nil {
			return fmt.Errorf("Could not retrieve period '%s' for team '%s': %s", period.ID, teamID, err)
		}
		_, err := tx.Put(periodKey, &period)
		return err
	})
	return err
}

func (s *googleCDSStore) Close() error {
	return s.client.Close()
}

// Cloud Datastore does not save zero-length slices, so when retrieving entities with slice members,
// they may be nil. This function is to avoid clients having to deal with this.
func scrubLoadedPeriod(period *Period) {
	if period.People == nil {
		period.People = []Person{}
	}
	if period.Buckets == nil {
		period.Buckets = []Bucket{}
	}
	for i := range period.Buckets {
		if period.Buckets[i].Objectives == nil {
			period.Buckets[i].Objectives = []Objective{}
		}
		for j := range period.Buckets[i].Objectives {
			if period.Buckets[i].Objectives[j].Assignments == nil {
				period.Buckets[i].Objectives[j].Assignments = []Assignment{}
			}
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>Update hello-secret.go<commit_after><|endoftext|>


<commit_before>package docker

import (
	"fmt"
	"time"
)

// Basic image information as returned by docker.
type Image struct {
	Repository  string   `json:"Repository"`
	Tags        []string `json:"RepoTags"`
	Id          string   `json:"Id"`
	created     int
	Size        int
	VirtualSize int
}

// Image details as returned by docker.
type ImageDetails struct {
	Id              string           `json:"id"`
	Parent          string           `json:"parent"`
	Created         string           `json:"created"`
	Container       string           `json:"container"`
	Size            int              `json:"size"`
	Architecture    string           `json:"architecture"`
	DockerVersion   string           `json:"docker_version"`
	ContainerConfig *ContainerConfig `json:"container_config"`
	Config          *ContainerConfig `json:"config"`
}

// Image history entries as returned by docker.
type ImageHistory []struct {
	Id        string
	Tags      []string
	created   int
	CreatedBy string
}

func (self *Image) CreatedAt() time.Time {
	return time.Unix(int64(self.created), 0)
}

func (image *ImageDetails) CreatedAt() (time.Time, error) {
	t, e := time.Parse("2006-01-02T15:04:05.999999999Z", image.Created)
	if e != nil {
		t, e = time.Parse("2006-01-02T15:04:05.999999999-07:00", image.Created)
	}
	return t, e
}

func (img *Image) String() string {
	return fmt.Sprintf("%s:%s (%s)", img.Repository, img.Tags, img.CreatedAt().UTC())
}
<commit_msg>fix mapping for Image<commit_after>package docker

import (
	"fmt"
	"time"
)

// Basic image information as returned by docker.
type Image struct {
	Repository  string   `json:"Repository"`
	Tags        []string `json:"RepoTags"`
	Id          string   `json:"Id"`
	Created     int
	Size        int
	VirtualSize int
}

// Image details as returned by docker.
type ImageDetails struct {
	Id              string           `json:"id"`
	Parent          string           `json:"parent"`
	Created         string           `json:"created"`
	Container       string           `json:"container"`
	Size            int              `json:"size"`
	Architecture    string           `json:"architecture"`
	DockerVersion   string           `json:"docker_version"`
	ContainerConfig *ContainerConfig `json:"container_config"`
	Config          *ContainerConfig `json:"config"`
}

// Image history entries as returned by docker.
type ImageHistory []struct {
	Id        string
	Tags      []string
	created   int
	CreatedBy string
}

func (self *Image) CreatedAt() time.Time {
	return time.Unix(int64(self.Created), 0)
}

func (image *ImageDetails) CreatedAt() (time.Time, error) {
	t, e := time.Parse("2006-01-02T15:04:05.999999999Z", image.Created)
	if e != nil {
		t, e = time.Parse("2006-01-02T15:04:05.999999999-07:00", image.Created)
	}
	return t, e
}

func (img *Image) String() string {
	return fmt.Sprintf("%s:%s (%s)", img.Repository, img.Tags, img.CreatedAt().UTC())
}
<|endoftext|>


<commit_before>package common

import (
	"sync"
	"time"
)

// Used by RepeatTimer the first time,
// and every time it's Reset() after Stop().
type TickerMaker func(dur time.Duration) Ticker

// Ticker is a basic ticker interface.
type Ticker interface {

	// Never changes, never closes.
	Chan() <-chan time.Time

	// Stopping a stopped Ticker will panic.
	Stop()
}

//----------------------------------------
// defaultTickerMaker

func defaultTickerMaker(dur time.Duration) Ticker {
	ticker := time.NewTicker(dur)
	return (*defaultTicker)(ticker)
}

type defaultTicker time.Ticker

// Implements Ticker
func (t *defaultTicker) Chan() <-chan time.Time {
	return t.C
}

// Implements Ticker
func (t *defaultTicker) Stop() {
	((*time.Ticker)(t)).Stop()
}

//----------------------------------------
// LogicalTickerMaker

// Construct a TickerMaker that always uses `source`.
// It's useful for simulating a deterministic clock.
func NewLogicalTickerMaker(source chan time.Time) TickerMaker {
	return func(dur time.Duration) Ticker {
		return newLogicalTicker(source, dur)
	}
}

type logicalTicker struct {
	source <-chan time.Time
	ch     chan time.Time
	quit   chan struct{}
}

func newLogicalTicker(source <-chan time.Time, interval time.Duration) Ticker {
	lt := &logicalTicker{
		source: source,
		ch:     make(chan time.Time),
		quit:   make(chan struct{}),
	}
	go lt.fireRoutine(interval)
	return lt
}

// We need a goroutine to read times from t.source
// and fire on t.Chan() when `interval` has passed.
func (t *logicalTicker) fireRoutine(interval time.Duration) {
	source := t.source

	// Init `lasttime`
	lasttime := time.Time{}
	select {
	case lasttime = <-source:
	case <-t.quit:
		return
	}
	// Init `lasttime` end

	for {
		select {
		case newtime := <-source:
			elapsed := newtime.Sub(lasttime)
			if interval <= elapsed {
				// Block for determinism until the ticker is stopped.
				select {
				case t.ch <- newtime:
				case <-t.quit:
					return
				}
				// Reset timeleft.
				// Don't try to "catch up" by sending more.
				// "Ticker adjusts the intervals or drops ticks to make up for
				// slow receivers" - https://golang.org/pkg/time/#Ticker
				lasttime = newtime
			}
		case <-t.quit:
			return // done
		}
	}
}

// Implements Ticker
func (t *logicalTicker) Chan() <-chan time.Time {
	return t.ch // immutable
}

// Implements Ticker
func (t *logicalTicker) Stop() {
	close(t.quit) // it *should* panic when stopped twice.
}

//---------------------------------------------------------------------

/*
	RepeatTimer repeatedly sends a struct{}{} to `.Chan()` after each `dur`
	period. (It's good for keeping connections alive.)
	A RepeatTimer must be stopped, or it will keep a goroutine alive.
*/
type RepeatTimer struct {
	name string
	ch   chan time.Time
	tm   TickerMaker

	mtx    sync.Mutex
	dur    time.Duration
	ticker Ticker
	quit   chan struct{}
}

// NewRepeatTimer returns a RepeatTimer with a defaultTicker.
func NewRepeatTimer(name string, dur time.Duration) *RepeatTimer {
	return NewRepeatTimerWithTickerMaker(name, dur, defaultTickerMaker)
}

// NewRepeatTimerWithTicker returns a RepeatTimer with the given ticker
// maker.
func NewRepeatTimerWithTickerMaker(name string, dur time.Duration, tm TickerMaker) *RepeatTimer {
	var t = &RepeatTimer{
		name:   name,
		ch:     make(chan time.Time),
		tm:     tm,
		dur:    dur,
		ticker: nil,
		quit:   nil,
	}
	t.reset()
	return t
}

func (t *RepeatTimer) fireRoutine(ch <-chan time.Time, quit <-chan struct{}) {
	for {
		select {
		case t_ := <-ch:
			select {
			case t.ch <- t_:
			case <-quit:
				return
			}
		case <-quit: // NOTE: `t.quit` races.
			return
		}
	}
}

func (t *RepeatTimer) Chan() <-chan time.Time {
	return t.ch
}

func (t *RepeatTimer) Stop() {
	t.mtx.Lock()
	defer t.mtx.Unlock()

	t.stop()
}

// Wait the duration again before firing.
func (t *RepeatTimer) Reset() {
	t.mtx.Lock()
	defer t.mtx.Unlock()

	t.reset()
}

//----------------------------------------
// Misc.

// CONTRACT: (non-constructor) caller should hold t.mtx.
func (t *RepeatTimer) reset() {
	if t.ticker != nil {
		t.stop()
	}
	t.ticker = t.tm(t.dur)
	t.quit = make(chan struct{})
	go t.fireRoutine(t.ticker.Chan(), t.quit)
}

// CONTRACT: caller should hold t.mtx.
func (t *RepeatTimer) stop() {
	if t.ticker == nil {
		/*
			Similar to the case of closing channels twice:
			https://groups.google.com/forum/#!topic/golang-nuts/rhxMiNmRAPk
			Stopping a RepeatTimer twice implies that you do
			not know whether you are done or not.
			If you're calling stop on a stopped RepeatTimer,
			you probably have race conditions.
		*/
		panic("Tried to stop a stopped RepeatTimer")
	}
	t.ticker.Stop()
	t.ticker = nil
	/*
		From https://golang.org/pkg/time/#Ticker:
		"Stop the ticker to release associated resources"
		"After Stop, no more ticks will be sent"
		So we shouldn't have to do the below.

		select {
		case <-t.ch:
			// read off channel if there's anything there
		default:
		}
	*/
	close(t.quit)
}
<commit_msg>minor nit<commit_after>package common

import (
	"sync"
	"time"
)

// Used by RepeatTimer the first time,
// and every time it's Reset() after Stop().
type TickerMaker func(dur time.Duration) Ticker

// Ticker is a basic ticker interface.
type Ticker interface {

	// Never changes, never closes.
	Chan() <-chan time.Time

	// Stopping a stopped Ticker will panic.
	Stop()
}

//----------------------------------------
// defaultTicker

var _ Ticker = (*defaultTicker)(nil)

type defaultTicker time.Ticker

func defaultTickerMaker(dur time.Duration) Ticker {
	ticker := time.NewTicker(dur)
	return (*defaultTicker)(ticker)
}

// Implements Ticker
func (t *defaultTicker) Chan() <-chan time.Time {
	return t.C
}

// Implements Ticker
func (t *defaultTicker) Stop() {
	((*time.Ticker)(t)).Stop()
}

//----------------------------------------
// LogicalTickerMaker

// Construct a TickerMaker that always uses `source`.
// It's useful for simulating a deterministic clock.
func NewLogicalTickerMaker(source chan time.Time) TickerMaker {
	return func(dur time.Duration) Ticker {
		return newLogicalTicker(source, dur)
	}
}

type logicalTicker struct {
	source <-chan time.Time
	ch     chan time.Time
	quit   chan struct{}
}

func newLogicalTicker(source <-chan time.Time, interval time.Duration) Ticker {
	lt := &logicalTicker{
		source: source,
		ch:     make(chan time.Time),
		quit:   make(chan struct{}),
	}
	go lt.fireRoutine(interval)
	return lt
}

// We need a goroutine to read times from t.source
// and fire on t.Chan() when `interval` has passed.
func (t *logicalTicker) fireRoutine(interval time.Duration) {
	source := t.source

	// Init `lasttime`
	lasttime := time.Time{}
	select {
	case lasttime = <-source:
	case <-t.quit:
		return
	}
	// Init `lasttime` end

	for {
		select {
		case newtime := <-source:
			elapsed := newtime.Sub(lasttime)
			if interval <= elapsed {
				// Block for determinism until the ticker is stopped.
				select {
				case t.ch <- newtime:
				case <-t.quit:
					return
				}
				// Reset timeleft.
				// Don't try to "catch up" by sending more.
				// "Ticker adjusts the intervals or drops ticks to make up for
				// slow receivers" - https://golang.org/pkg/time/#Ticker
				lasttime = newtime
			}
		case <-t.quit:
			return // done
		}
	}
}

// Implements Ticker
func (t *logicalTicker) Chan() <-chan time.Time {
	return t.ch // immutable
}

// Implements Ticker
func (t *logicalTicker) Stop() {
	close(t.quit) // it *should* panic when stopped twice.
}

//---------------------------------------------------------------------

/*
	RepeatTimer repeatedly sends a struct{}{} to `.Chan()` after each `dur`
	period. (It's good for keeping connections alive.)
	A RepeatTimer must be stopped, or it will keep a goroutine alive.
*/
type RepeatTimer struct {
	name string
	ch   chan time.Time
	tm   TickerMaker

	mtx    sync.Mutex
	dur    time.Duration
	ticker Ticker
	quit   chan struct{}
}

// NewRepeatTimer returns a RepeatTimer with a defaultTicker.
func NewRepeatTimer(name string, dur time.Duration) *RepeatTimer {
	return NewRepeatTimerWithTickerMaker(name, dur, defaultTickerMaker)
}

// NewRepeatTimerWithTicker returns a RepeatTimer with the given ticker
// maker.
func NewRepeatTimerWithTickerMaker(name string, dur time.Duration, tm TickerMaker) *RepeatTimer {
	var t = &RepeatTimer{
		name:   name,
		ch:     make(chan time.Time),
		tm:     tm,
		dur:    dur,
		ticker: nil,
		quit:   nil,
	}
	t.reset()
	return t
}

// receive ticks on ch, send out on t.ch
func (t *RepeatTimer) fireRoutine(ch <-chan time.Time, quit <-chan struct{}) {
	for {
		select {
		case tick := <-ch:
			select {
			case t.ch <- tick:
			case <-quit:
				return
			}
		case <-quit: // NOTE: `t.quit` races.
			return
		}
	}
}

func (t *RepeatTimer) Chan() <-chan time.Time {
	return t.ch
}

func (t *RepeatTimer) Stop() {
	t.mtx.Lock()
	defer t.mtx.Unlock()

	t.stop()
}

// Wait the duration again before firing.
func (t *RepeatTimer) Reset() {
	t.mtx.Lock()
	defer t.mtx.Unlock()

	t.reset()
}

//----------------------------------------
// Misc.

// CONTRACT: (non-constructor) caller should hold t.mtx.
func (t *RepeatTimer) reset() {
	if t.ticker != nil {
		t.stop()
	}
	t.ticker = t.tm(t.dur)
	t.quit = make(chan struct{})
	go t.fireRoutine(t.ticker.Chan(), t.quit)
}

// CONTRACT: caller should hold t.mtx.
func (t *RepeatTimer) stop() {
	if t.ticker == nil {
		/*
			Similar to the case of closing channels twice:
			https://groups.google.com/forum/#!topic/golang-nuts/rhxMiNmRAPk
			Stopping a RepeatTimer twice implies that you do
			not know whether you are done or not.
			If you're calling stop on a stopped RepeatTimer,
			you probably have race conditions.
		*/
		panic("Tried to stop a stopped RepeatTimer")
	}
	t.ticker.Stop()
	t.ticker = nil
	/*
		From https://golang.org/pkg/time/#Ticker:
		"Stop the ticker to release associated resources"
		"After Stop, no more ticks will be sent"
		So we shouldn't have to do the below.

		select {
		case <-t.ch:
			// read off channel if there's anything there
		default:
		}
	*/
	close(t.quit)
}
<|endoftext|>


<commit_before><commit_msg>Fix stats Hz to kHz conversion for Concentratord backend.<commit_after><|endoftext|>


<commit_before>package tcp

import (
	"encoding/binary"
	"errors"
	"io"
	"log"
	"net"
	"os"
	"sync"

	"github.com/influxdb/influxdb/data"
)

var (
	// ErrBindAddressRequired is returned when starting the Server
	// without a TCP or UDP listening address.
	ErrBindAddressRequired = errors.New("bind address required")

	// ErrServerClosed return when closing an already closed graphite server.
	ErrServerClosed = errors.New("server already closed")
)

// Server processes data received over raw TCP connections.
type Server struct {
	writer   data.ShardWriter
	listener *net.Listener

	wg sync.WaitGroup

	Logger *log.Logger

	shutdown chan struct{}
}

// NewServer returns a new instance of a Server.
func NewServer(w data.ShardWriter) *Server {
	return &Server{
		writer:   w,
		Logger:   log.New(os.Stderr, "[tcp] ", log.LstdFlags),
		shutdown: make(chan struct{}),
	}
}

// ListenAndServe instructs the Server to start processing connections
// on the given interface. iface must be in the form host:port
// If successful, it returns the host as the first argument
func (s *Server) ListenAndServe(laddr string) (string, error) {
	if laddr == "" { // Make sure we have an laddr
		return "", ErrBindAddressRequired
	}

	ln, err := net.Listen("tcp", laddr)
	if err != nil {
		return "", err
	}
	s.listener = &ln

	s.Logger.Println("listening on TCP connection", ln.Addr().String())
	s.wg.Add(1)
	go func() {
		defer s.wg.Done()

		for {
			// Are we shutting down? If so, exit
			select {
			case <-s.shutdown:
				return
			default:
			}

			conn, err := ln.Accept()
			if opErr, ok := err.(*net.OpError); ok && !opErr.Temporary() {
				s.Logger.Println("error temporarily accepting TCP connection", err.Error())
				continue
			}
			if err != nil {
				s.Logger.Println("TCP listener closed")
				return
			}

			s.wg.Add(1)
			go s.handleConnection(conn)
		}
	}()

	// Return the host we started up on. Mostly needed for testing
	return ln.Addr().String(), nil
}

// Close will close the listener
func (s *Server) Close() error {
	// Stop accepting client connections
	if s.listener != nil {
		err := (*s.listener).Close()
		if err != nil {
			return err
		}
	} else {
		return ErrServerClosed
	}
	// Shut down all handlers
	close(s.shutdown)
	s.wg.Wait()
	s.listener = nil

	return nil
}

// handleConnection services an individual TCP connection.
func (s *Server) handleConnection(conn net.Conn) {
	defer func() {
		conn.Close()
		s.wg.Done()
	}()

	messageChannel := make(chan byte)

	// Start our reader up in a go routine so we don't block checking our close channel
	go func() {
		var messageType byte
		err := binary.Read(conn, binary.LittleEndian, &messageType)
		if err != nil {
			s.Logger.Printf("unable to read message type %s", err)
			return
		}
		messageChannel <- messageType
	}()

	for {
		select {
		case <-s.shutdown:
			// Are we shutting down? If so, exit
			return
		case messageType := <-messageChannel:
			switch messageType {
			case writeShardRequestMessage:
				if err := s.WriteShardRequest(conn); err != nil {
					s.WriteShardResponse(conn, err)
					return
				}
				s.WriteShardResponse(conn, nil)
			}
		default:
		}
	}

}

func (s *Server) WriteShardRequest(conn net.Conn) error {
	messageChannel := make(chan data.WriteShardRequest)
	errChan := make(chan error)

	go func() {
		var size int64
		if err := binary.Read(conn, binary.LittleEndian, &size); err != nil {
			errChan <- err
			return
		}

		message := make([]byte, size)

		reader := io.LimitReader(conn, size)
		_, err := reader.Read(message)
		if err != nil {
			errChan <- err
			return
		}
		var wsr data.WriteShardRequest
		if err := wsr.UnmarshalBinary(message); err != nil {
			errChan <- err
			return
		}
		messageChannel <- wsr
	}()

	for {
		select {
		case <-s.shutdown:
			// Are we shutting down? If so, exit
			return nil
		case e := <-errChan:
			return e
		case wsr := <-messageChannel:
			if _, err := s.writer.WriteShard(wsr.ShardID(), wsr.Points()); err != nil {
				return err
			}
			return nil
		default:
		}
	}
}

func (s *Server) WriteShardResponse(conn net.Conn, e error) {
	s.Logger.Println("writing shard response")
	var mt byte = writeShardResponseMessage
	if err := binary.Write(conn, binary.LittleEndian, &mt); err != nil {
		s.Logger.Printf("error writing shard response message type: %s", err)
		return
	}

	var wsr data.WriteShardResponse
	if e != nil {
		wsr.SetCode(1)
		wsr.SetMessage(e.Error())
	} else {
		wsr.SetCode(0)
	}

	b, err := wsr.MarshalBinary()
	if err != nil {
		s.Logger.Printf("error marshalling shard response: %s", err)
		return
	}

	size := int64(len(b))

	if err := binary.Write(conn, binary.LittleEndian, &size); err != nil {
		s.Logger.Printf("error writing shard response length: %s", err)
		return
	}

	if _, err := conn.Write(b); err != nil {
		s.Logger.Printf("error writing shard response: %s", err)
		return
	}
}
<commit_msg>clarify error comment<commit_after>package tcp

import (
	"encoding/binary"
	"errors"
	"io"
	"log"
	"net"
	"os"
	"sync"

	"github.com/influxdb/influxdb/data"
)

var (
	// ErrBindAddressRequired is returned when starting the Server
	// without providing a bind address
	ErrBindAddressRequired = errors.New("bind address required")

	// ErrServerClosed return when closing an already closed graphite server.
	ErrServerClosed = errors.New("server already closed")
)

// Server processes data received over raw TCP connections.
type Server struct {
	writer   data.ShardWriter
	listener *net.Listener

	wg sync.WaitGroup

	Logger *log.Logger

	shutdown chan struct{}
}

// NewServer returns a new instance of a Server.
func NewServer(w data.ShardWriter) *Server {
	return &Server{
		writer:   w,
		Logger:   log.New(os.Stderr, "[tcp] ", log.LstdFlags),
		shutdown: make(chan struct{}),
	}
}

// ListenAndServe instructs the Server to start processing connections
// on the given interface. iface must be in the form host:port
// If successful, it returns the host as the first argument
func (s *Server) ListenAndServe(laddr string) (string, error) {
	if laddr == "" { // Make sure we have an laddr
		return "", ErrBindAddressRequired
	}

	ln, err := net.Listen("tcp", laddr)
	if err != nil {
		return "", err
	}
	s.listener = &ln

	s.Logger.Println("listening on TCP connection", ln.Addr().String())
	s.wg.Add(1)
	go func() {
		defer s.wg.Done()

		for {
			// Are we shutting down? If so, exit
			select {
			case <-s.shutdown:
				return
			default:
			}

			conn, err := ln.Accept()
			if opErr, ok := err.(*net.OpError); ok && !opErr.Temporary() {
				s.Logger.Println("error temporarily accepting TCP connection", err.Error())
				continue
			}
			if err != nil {
				s.Logger.Println("TCP listener closed")
				return
			}

			s.wg.Add(1)
			go s.handleConnection(conn)
		}
	}()

	// Return the host we started up on. Mostly needed for testing
	return ln.Addr().String(), nil
}

// Close will close the listener
func (s *Server) Close() error {
	// Stop accepting client connections
	if s.listener != nil {
		err := (*s.listener).Close()
		if err != nil {
			return err
		}
	} else {
		return ErrServerClosed
	}
	// Shut down all handlers
	close(s.shutdown)
	s.wg.Wait()
	s.listener = nil

	return nil
}

// handleConnection services an individual TCP connection.
func (s *Server) handleConnection(conn net.Conn) {
	defer func() {
		conn.Close()
		s.wg.Done()
	}()

	messageChannel := make(chan byte)

	// Start our reader up in a go routine so we don't block checking our close channel
	go func() {
		var messageType byte
		err := binary.Read(conn, binary.LittleEndian, &messageType)
		if err != nil {
			s.Logger.Printf("unable to read message type %s", err)
			return
		}
		messageChannel <- messageType
	}()

	for {
		select {
		case <-s.shutdown:
			// Are we shutting down? If so, exit
			return
		case messageType := <-messageChannel:
			switch messageType {
			case writeShardRequestMessage:
				if err := s.WriteShardRequest(conn); err != nil {
					s.WriteShardResponse(conn, err)
					return
				}
				s.WriteShardResponse(conn, nil)
			}
		default:
		}
	}

}

func (s *Server) WriteShardRequest(conn net.Conn) error {
	messageChannel := make(chan data.WriteShardRequest)
	errChan := make(chan error)

	go func() {
		var size int64
		if err := binary.Read(conn, binary.LittleEndian, &size); err != nil {
			errChan <- err
			return
		}

		message := make([]byte, size)

		reader := io.LimitReader(conn, size)
		_, err := reader.Read(message)
		if err != nil {
			errChan <- err
			return
		}
		var wsr data.WriteShardRequest
		if err := wsr.UnmarshalBinary(message); err != nil {
			errChan <- err
			return
		}
		messageChannel <- wsr
	}()

	for {
		select {
		case <-s.shutdown:
			// Are we shutting down? If so, exit
			return nil
		case e := <-errChan:
			return e
		case wsr := <-messageChannel:
			if _, err := s.writer.WriteShard(wsr.ShardID(), wsr.Points()); err != nil {
				return err
			}
			return nil
		default:
		}
	}
}

func (s *Server) WriteShardResponse(conn net.Conn, e error) {
	s.Logger.Println("writing shard response")
	var mt byte = writeShardResponseMessage
	if err := binary.Write(conn, binary.LittleEndian, &mt); err != nil {
		s.Logger.Printf("error writing shard response message type: %s", err)
		return
	}

	var wsr data.WriteShardResponse
	if e != nil {
		wsr.SetCode(1)
		wsr.SetMessage(e.Error())
	} else {
		wsr.SetCode(0)
	}

	b, err := wsr.MarshalBinary()
	if err != nil {
		s.Logger.Printf("error marshalling shard response: %s", err)
		return
	}

	size := int64(len(b))

	if err := binary.Write(conn, binary.LittleEndian, &size); err != nil {
		s.Logger.Printf("error writing shard response length: %s", err)
		return
	}

	if _, err := conn.Write(b); err != nil {
		s.Logger.Printf("error writing shard response: %s", err)
		return
	}
}
<|endoftext|>


<commit_before>package termite

import (
	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/unionfs"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

type WorkerFuseFs struct {
	rwDir  string
	tmpDir string
	mount  string
	*fuse.MountState
	fsConnector *fuse.FileSystemConnector
	unionFs     *unionfs.UnionFs
}

func (me *WorkerFuseFs) Stop() {
	err := me.MountState.Unmount()
	if err != nil {
		// TODO - Should be fatal?
		log.Println("Unmount fail:", err)
	} else {
		// If the unmount fails, the RemoveAll will stat all
		// of the FUSE file system.
		os.RemoveAll(me.tmpDir)
	}
}

func (me *Mirror) returnFuse(wfs *WorkerFuseFs) {
	me.fuseFileSystemsMutex.Lock()
	defer me.fuseFileSystemsMutex.Unlock()

	if me.shuttingDown {
		wfs.Stop()
	} else {
		me.fuseFileSystems = append(me.fuseFileSystems, wfs)
	}
	me.workingFileSystems[wfs] = "", false
	me.cond.Signal()
}

func newWorkerFuseFs(tmpDir string, rpcFs fuse.FileSystem, writableRoot string) (*WorkerFuseFs, os.Error) {
	tmpDir, err := ioutil.TempDir(tmpDir, "termite-task")
	if err != nil {
		return nil, err
	}
	w := WorkerFuseFs{
		tmpDir: tmpDir,
	}

	type dirInit struct {
		dst *string
		val string
	}

	for _, v := range []dirInit{
		dirInit{&w.rwDir, "rw"},
		dirInit{&w.mount, "mnt"},
	} {
		*v.dst = filepath.Join(w.tmpDir, v.val)
		err = os.Mkdir(*v.dst, 0700)
		if err != nil {
			return nil, err
		}
	}

	tmpBacking := filepath.Join(w.tmpDir, "tmp-backingstore")
	if err := os.Mkdir(tmpBacking, 0700); err != nil {
		return nil, err
	}

	rwFs := fuse.NewLoopbackFileSystem(w.rwDir)

	ttl := 30.0
	opts := unionfs.UnionFsOptions{
		BranchCacheTTLSecs:   ttl,
		DeletionCacheTTLSecs: ttl,
		DeletionDirName:      _DELETIONS,
	}
	mOpts := fuse.FileSystemOptions{
		EntryTimeout:    ttl,
		AttrTimeout:     ttl,
		NegativeTimeout: ttl,
	}

	tmpFs := fuse.NewLoopbackFileSystem(tmpBacking)

	w.unionFs = unionfs.NewUnionFs([]fuse.FileSystem{rwFs, rpcFs}, opts)
	swFs := []fuse.SwitchedFileSystem{
		{"dev", &DevNullFs{}, true},
		{"", rpcFs, false},
		{"tmp", tmpFs, true},
		{"var/tmp", tmpFs, true},

		// TODO - configurable.
		{writableRoot, w.unionFs, false},
	}

	w.fsConnector = fuse.NewFileSystemConnector(fuse.NewSwitchFileSystem(swFs), &mOpts)
	w.MountState = fuse.NewMountState(w.fsConnector)

	fuseOpts := fuse.MountOptions{
		// Compilers are not that highly parallel.  A lower
		// number also helps stacktrace be less overwhelming.
		MaxBackground: 4,
	}
	if os.Geteuid() == 0 {
		fuseOpts.AllowOther = true
	}

	err = w.MountState.Mount(w.mount, &fuseOpts)
	if err != nil {
		return nil, err
	}

	go w.MountState.Loop(true)

	return &w, nil
}

func (me *WorkerFuseFs) update(attrs []FileAttr) {
	paths := []string{}
	for _, attr := range attrs {
		path := strings.TrimLeft(attr.Path, "/")
		paths = append(paths, path)

		// We only invalidate files.  This is a kludge - we
		// should skip invalidations on dirs that we know that
		// they held files.
		if attr.FileInfo != nil && !attr.FileInfo.IsDirectory() {
			// TODO - should have bulk interface?
			dir, base := filepath.Split(path)
			dir = filepath.Clean(dir)

			me.fsConnector.EntryNotify(dir, base)
		}
	}
	me.unionFs.DropBranchCache(paths)
	me.unionFs.DropDeletionCache()
}
<commit_msg>Use smart notify to minimize entry notifies.<commit_after>package termite

import (
	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/unionfs"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

type WorkerFuseFs struct {
	rwDir  string
	tmpDir string
	mount  string
	*fuse.MountState
	fsConnector *fuse.FileSystemConnector
	unionFs     *unionfs.UnionFs
}

func (me *WorkerFuseFs) Stop() {
	err := me.MountState.Unmount()
	if err != nil {
		// TODO - Should be fatal?
		log.Println("Unmount fail:", err)
	} else {
		// If the unmount fails, the RemoveAll will stat all
		// of the FUSE file system.
		os.RemoveAll(me.tmpDir)
	}
}

func (me *Mirror) returnFuse(wfs *WorkerFuseFs) {
	me.fuseFileSystemsMutex.Lock()
	defer me.fuseFileSystemsMutex.Unlock()

	if me.shuttingDown {
		wfs.Stop()
	} else {
		me.fuseFileSystems = append(me.fuseFileSystems, wfs)
	}
	me.workingFileSystems[wfs] = "", false
	me.cond.Signal()
}

func newWorkerFuseFs(tmpDir string, rpcFs fuse.FileSystem, writableRoot string) (*WorkerFuseFs, os.Error) {
	tmpDir, err := ioutil.TempDir(tmpDir, "termite-task")
	if err != nil {
		return nil, err
	}
	w := WorkerFuseFs{
		tmpDir: tmpDir,
	}

	type dirInit struct {
		dst *string
		val string
	}

	for _, v := range []dirInit{
		dirInit{&w.rwDir, "rw"},
		dirInit{&w.mount, "mnt"},
	} {
		*v.dst = filepath.Join(w.tmpDir, v.val)
		err = os.Mkdir(*v.dst, 0700)
		if err != nil {
			return nil, err
		}
	}

	tmpBacking := filepath.Join(w.tmpDir, "tmp-backingstore")
	if err := os.Mkdir(tmpBacking, 0700); err != nil {
		return nil, err
	}

	rwFs := fuse.NewLoopbackFileSystem(w.rwDir)

	ttl := 30.0
	opts := unionfs.UnionFsOptions{
		BranchCacheTTLSecs:   ttl,
		DeletionCacheTTLSecs: ttl,
		DeletionDirName:      _DELETIONS,
	}
	mOpts := fuse.FileSystemOptions{
		EntryTimeout:    ttl,
		AttrTimeout:     ttl,
		NegativeTimeout: ttl,
	}

	tmpFs := fuse.NewLoopbackFileSystem(tmpBacking)

	w.unionFs = unionfs.NewUnionFs([]fuse.FileSystem{rwFs, rpcFs}, opts)
	swFs := []fuse.SwitchedFileSystem{
		{"dev", &DevNullFs{}, true},
		{"", rpcFs, false},
		{"tmp", tmpFs, true},
		{"var/tmp", tmpFs, true},

		// TODO - configurable.
		{writableRoot, w.unionFs, false},
	}

	w.fsConnector = fuse.NewFileSystemConnector(fuse.NewSwitchFileSystem(swFs), &mOpts)
	w.MountState = fuse.NewMountState(w.fsConnector)

	fuseOpts := fuse.MountOptions{
		// Compilers are not that highly parallel.  A lower
		// number also helps stacktrace be less overwhelming.
		MaxBackground: 4,
	}
	if os.Geteuid() == 0 {
		fuseOpts.AllowOther = true
	}

	err = w.MountState.Mount(w.mount, &fuseOpts)
	if err != nil {
		return nil, err
	}

	go w.MountState.Loop(true)

	return &w, nil
}

func (me *WorkerFuseFs) update(attrs []FileAttr) {
	paths := []string{}
	for _, attr := range attrs {
		path := strings.TrimLeft(attr.Path, "/")
		paths = append(paths, path)

		me.fsConnector.Notify(path)
	}
	me.unionFs.DropBranchCache(paths)
	me.unionFs.DropDeletionCache()
}
<|endoftext|>