<commit_before>package goka

import (
	"errors"
	"fmt"
	"sync"

	"github.com/lovoo/goka/kafka"
	"github.com/lovoo/goka/logger"
	"github.com/lovoo/goka/multierr"
	"github.com/lovoo/goka/storage"
)

// Getter functions return a value for a key or an error. If no value exists for the key, nil is returned without errors.
type Getter func(string) (interface{}, error)

// View is a materialized (i.e. persistent) cache of a group table.
type View struct {
	topic      string
	opts       *voptions
	partitions []*partition
	consumer   kafka.Consumer
	done       chan bool
	dead       chan bool

	errors   multierr.Errors
	stopOnce sync.Once
}

// NewView creates a new View object from a group.
func NewView(brokers []string, topic Table, codec Codec, options ...ViewOption) (*View, error) {
	options = append(
		// default options comes first
		[]ViewOption{
			WithViewLogger(logger.Default()),
			WithViewCallback(DefaultUpdate),
			WithViewPartitionChannelSize(defaultPartitionChannelSize),
			WithViewStorageBuilder(DefaultStorageBuilder(DefaultViewStoragePath())),
		},

		// then the user passed options
		options...,
	)

	// figure out how many partitions the group has
	opts := new(voptions)
	err := opts.applyOptions(topic, options...)
	if err != nil {
		return nil, fmt.Errorf("Error applying user-defined options: %v", err)
	}

	opts.tableCodec = codec

	consumer, err := opts.builders.consumer(brokers, "goka-view", opts.clientID)
	if err != nil {
		return nil, fmt.Errorf("view: cannot create Kafka consumer: %v", err)
	}

	v := &View{
		topic:    string(topic),
		opts:     opts,
		consumer: consumer,
		done:     make(chan bool),
		dead:     make(chan bool),
	}

	if err = v.createPartitions(brokers); err != nil {
		return nil, err
	}

	return v, err
}

func (v *View) createPartitions(brokers []string) (err error) {
	tm, err := v.opts.builders.topicmgr(brokers)
	if err != nil {
		return fmt.Errorf("Error creating topic manager: %v", err)
	}
	defer func() {
		e := tm.Close()
		if e != nil && err == nil {
			err = fmt.Errorf("Error closing topic manager: %v", e)
		}
	}()

	partitions, err := tm.Partitions(v.topic)
	if err != nil {
		return fmt.Errorf("Error getting partitions for topic %s: %v", v.topic, err)
	}

	// check assumption that partitions are gap-less
	for i, p := range partitions {
		if i != int(p) {
			return fmt.Errorf("Partition numbers are not sequential for topic %s", v.topic)
		}
	}

	v.opts.log.Printf("Table %s has %d partitions", v.topic, len(partitions))
	for _, p := range partitions {
		st, err := v.opts.builders.storage(v.topic, p)
		if err != nil {
			// TODO(diogo): gracefully terminate all partitions
			return fmt.Errorf("Error creating local storage for partition %d: %v", p, err)
		}

		po := newPartition(v.opts.log, v.topic, nil,
			&storageProxy{Storage: st, partition: p, update: v.opts.updateCallback},
			&proxy{p, v.consumer},
			v.opts.partitionChannelSize,
		)
		v.partitions = append(v.partitions, po)
	}

	return nil
}

// Start starts consuming the view's topic.
func (v *View) Start() error {
	go v.run()

	var wg sync.WaitGroup
	wg.Add(len(v.partitions))
	for id, p := range v.partitions {
		go func(id int32, p *partition) {
			defer wg.Done()
			err := p.startCatchup()
			if err != nil {
				v.fail(fmt.Errorf("view: error opening partition %d: %v", id, err))
			}
		}(int32(id), p)
	}
	wg.Wait()

	<-v.dead
	if v.errors.HasErrors() {
		return &v.errors
	}
	return nil
}

func (v *View) fail(err error) {
	v.opts.log.Printf("View: failing: %v", err)
	v.errors.Collect(err)
	go v.stop()
}

func (v *View) stop() {
	v.stopOnce.Do(func() {
		defer close(v.dead)
		// stop consumer
		if err := v.consumer.Close(); err != nil {
			err = fmt.Errorf("failed to close consumer on stopping the view: %v", err)
			v.opts.log.Printf("error: %v", err)
			v.errors.Collect(err)
		}
		<-v.done
		v.opts.log.Printf("View: stopping partitions")

		var wg sync.WaitGroup
		for i, par := range v.partitions {
			wg.Add(1)
			go func(pid int, p *partition) {
				p.stop()
				wg.Done()
				v.opts.log.Printf("View: partition %d stopped", pid)
			}(i, par)
		}
		wg.Wait()
		v.opts.log.Printf("View: shutdown complete")
	})
}

// Stop stops the view, frees any resources + connections to kafka
func (v *View) Stop() {
	v.opts.log.Printf("View: stopping")
	v.stop()
}

func (v *View) hash(key string) (int32, error) {
	// create a new hasher every time. Alternative would be to store the hash in
	// view and every time reset the hasher (ie, hasher.Reset()). But that would
	// also require us to protect the access of the hasher with a mutex.
	hasher := v.opts.hasher()

	_, err := hasher.Write([]byte(key))
	if err != nil {
		return -1, err
	}
	hash := int32(hasher.Sum32())
	if hash < 0 {
		hash = -hash
	}
	if len(v.partitions) == 0 {
		return 0, errors.New("No partitions found.")
	}
	return hash % int32(len(v.partitions)), nil
}

func (v *View) find(key string) (storage.Storage, error) {
	h, err := v.hash(key)
	if err != nil {
		return nil, err
	}
	return v.partitions[h].st, nil
}

// Topic returns  the view's topic
func (v *View) Topic() string {
	return v.topic
}

// Get returns the value for the key in the view, if exists. Nil if it doesn't.
// Get can be called by multiple goroutines concurrently.
// Get can only be called after Recovered returns true.
func (v *View) Get(key string) (interface{}, error) {
	// find partition where key is located
	s, err := v.find(key)
	if err != nil {
		return nil, err
	}

	// get key and return
	data, err := s.Get(key)
	if err != nil {
		return nil, fmt.Errorf("error getting value (key %s): err", key, err)
	} else if data == nil {
		return nil, nil
	}

	// decode value
	value, err := v.opts.tableCodec.Decode(data)
	if err != nil {
		return nil, fmt.Errorf("error decoding value (key %s): err", key, err)
	}

	// if the key does not exist the return value is nil
	return value, nil
}

// Has checks whether a value for passed key exists in the view.
func (v *View) Has(key string) (bool, error) {
	// find partition where key is located
	s, err := v.find(key)
	if err != nil {
		return false, err
	}

	return s.Has(key)
}

// Iterator returns an iterator that iterates over the state of the View.
func (v *View) Iterator() (Iterator, error) {
	iters := make([]storage.Iterator, 0, len(v.partitions))
	for i := range v.partitions {
		iter, err := v.partitions[i].st.Iterator()
		if err != nil {
			// release already opened iterators
			for i := range iters {
				iters[i].Release()
			}

			return nil, fmt.Errorf("error opening partition iterator: %v", err)
		}

		iters = append(iters, iter)
	}

	return &iterator{
		iter:  storage.NewMultiIterator(iters),
		codec: v.opts.tableCodec,
	}, nil
}

// IteratorWithRange returns an iterator that iterates over the state of the View. This iterator is build using the range.
func (v *View) IteratorWithRange(start, limit []byte) (Iterator, error) {
	iters := make([]storage.Iterator, 0, len(v.partitions))
	for i := range v.partitions {
		iter, err := v.partitions[i].st.IteratorWithRange(start, limit)
		if err != nil {
			// release already opened iterators
			for i := range iters {
				iters[i].Release()
			}

			return nil, fmt.Errorf("error opening partition iterator: %v", err)
		}

		iters = append(iters, iter)
	}

	return &iterator{
		iter:  storage.NewMultiIterator(iters),
		codec: v.opts.tableCodec,
	}, nil
}

// Evict removes the given key only from the local cache. In order to delete a
// key from Kafka and other Views, context.Delete should be used on a Processor.
func (v *View) Evict(key string) error {
	s, err := v.find(key)
	if err != nil {
		return err
	}

	return s.Delete(key)
}

func (v *View) run() {
	defer close(v.done)
	v.opts.log.Printf("View: started")
	defer v.opts.log.Printf("View: stopped")

	for ev := range v.consumer.Events() {
		switch ev := ev.(type) {
		case *kafka.Message:
			partition := v.partitions[int(ev.Partition)]
			partition.ch <- ev
		case *kafka.BOF:
			partition := v.partitions[int(ev.Partition)]
			partition.ch <- ev
		case *kafka.EOF:
			partition := v.partitions[int(ev.Partition)]
			partition.ch <- ev
		case *kafka.Error:
			v.fail(fmt.Errorf("view: error from kafka consumer: %v", ev))
			return
		default:
			v.fail(fmt.Errorf("view: cannot handle %T = %v", ev, ev))
			return
		}
	}
}

// Recovered returns true when the view has caught up with events from kafka.
func (v *View) Recovered() bool {
	for _, p := range v.partitions {
		if !p.recovered() {
			return false
		}
	}

	return true
}

func (v *View) Stats() *ViewStats {
	var (
		m     sync.Mutex
		wg    sync.WaitGroup
		stats = newViewStats()
	)

	wg.Add(len(v.partitions))
	for i, p := range v.partitions {
		go func(pid int32, par *partition) {
			s := par.fetchStats()
			m.Lock()
			stats.Partitions[pid] = s
			m.Unlock()
			wg.Done()
		}(int32(i), p)
	}
	wg.Wait()
	return stats
}
<commit_msg>Update view.go<commit_after>package goka

import (
	"errors"
	"fmt"
	"sync"

	"github.com/lovoo/goka/kafka"
	"github.com/lovoo/goka/logger"
	"github.com/lovoo/goka/multierr"
	"github.com/lovoo/goka/storage"
)

// Getter functions return a value for a key or an error. If no value exists for the key, nil is returned without errors.
type Getter func(string) (interface{}, error)

// View is a materialized (i.e. persistent) cache of a group table.
type View struct {
	topic      string
	opts       *voptions
	partitions []*partition
	consumer   kafka.Consumer
	done       chan bool
	dead       chan bool

	errors   multierr.Errors
	stopOnce sync.Once
}

// NewView creates a new View object from a group.
func NewView(brokers []string, topic Table, codec Codec, options ...ViewOption) (*View, error) {
	options = append(
		// default options comes first
		[]ViewOption{
			WithViewLogger(logger.Default()),
			WithViewCallback(DefaultUpdate),
			WithViewPartitionChannelSize(defaultPartitionChannelSize),
			WithViewStorageBuilder(DefaultStorageBuilder(DefaultViewStoragePath())),
		},

		// then the user passed options
		options...,
	)

	// figure out how many partitions the group has
	opts := new(voptions)
	err := opts.applyOptions(topic, options...)
	if err != nil {
		return nil, fmt.Errorf("Error applying user-defined options: %v", err)
	}

	opts.tableCodec = codec

	consumer, err := opts.builders.consumer(brokers, "goka-view", opts.clientID)
	if err != nil {
		return nil, fmt.Errorf("view: cannot create Kafka consumer: %v", err)
	}

	v := &View{
		topic:    string(topic),
		opts:     opts,
		consumer: consumer,
		done:     make(chan bool),
		dead:     make(chan bool),
	}

	if err = v.createPartitions(brokers); err != nil {
		return nil, err
	}

	return v, err
}

func (v *View) createPartitions(brokers []string) (err error) {
	tm, err := v.opts.builders.topicmgr(brokers)
	if err != nil {
		return fmt.Errorf("Error creating topic manager: %v", err)
	}
	defer func() {
		e := tm.Close()
		if e != nil && err == nil {
			err = fmt.Errorf("Error closing topic manager: %v", e)
		}
	}()

	partitions, err := tm.Partitions(v.topic)
	if err != nil {
		return fmt.Errorf("Error getting partitions for topic %s: %v", v.topic, err)
	}

	// check assumption that partitions are gap-less
	for i, p := range partitions {
		if i != int(p) {
			return fmt.Errorf("Partition numbers are not sequential for topic %s", v.topic)
		}
	}

	v.opts.log.Printf("Table %s has %d partitions", v.topic, len(partitions))
	for _, p := range partitions {
		st, err := v.opts.builders.storage(v.topic, p)
		if err != nil {
			// TODO(diogo): gracefully terminate all partitions
			return fmt.Errorf("Error creating local storage for partition %d: %v", p, err)
		}

		po := newPartition(v.opts.log, v.topic, nil,
			&storageProxy{Storage: st, partition: p, update: v.opts.updateCallback},
			&proxy{p, v.consumer},
			v.opts.partitionChannelSize,
		)
		v.partitions = append(v.partitions, po)
	}

	return nil
}

// Start starts consuming the view's topic.
func (v *View) Start() error {
	go v.run()

	var wg sync.WaitGroup
	wg.Add(len(v.partitions))
	for id, p := range v.partitions {
		go func(id int32, p *partition) {
			defer wg.Done()
			err := p.startCatchup()
			if err != nil {
				v.fail(fmt.Errorf("view: error opening partition %d: %v", id, err))
			}
		}(int32(id), p)
	}
	wg.Wait()

	<-v.dead
	if v.errors.HasErrors() {
		return &v.errors
	}
	return nil
}

func (v *View) fail(err error) {
	v.opts.log.Printf("View: failing: %v", err)
	v.errors.Collect(err)
	go v.stop()
}

func (v *View) stop() {
	v.stopOnce.Do(func() {
		defer close(v.dead)
		// stop consumer
		if err := v.consumer.Close(); err != nil {
			err = fmt.Errorf("failed to close consumer on stopping the view: %v", err)
			v.opts.log.Printf("error: %v", err)
			v.errors.Collect(err)
		}
		<-v.done
		v.opts.log.Printf("View: stopping partitions")

		var wg sync.WaitGroup
		for i, par := range v.partitions {
			wg.Add(1)
			go func(pid int, p *partition) {
				p.stop()
				wg.Done()
				v.opts.log.Printf("View: partition %d stopped", pid)
			}(i, par)
		}
		wg.Wait()
		v.opts.log.Printf("View: shutdown complete")
	})
}

// Stop stops the view, frees any resources + connections to kafka
func (v *View) Stop() {
	v.opts.log.Printf("View: stopping")
	v.stop()
}

func (v *View) hash(key string) (int32, error) {
	// create a new hasher every time. Alternative would be to store the hash in
	// view and every time reset the hasher (ie, hasher.Reset()). But that would
	// also require us to protect the access of the hasher with a mutex.
	hasher := v.opts.hasher()

	_, err := hasher.Write([]byte(key))
	if err != nil {
		return -1, err
	}
	hash := int32(hasher.Sum32())
	if hash < 0 {
		hash = -hash
	}
	if len(v.partitions) == 0 {
		return 0, errors.New("No partitions found.")
	}
	return hash % int32(len(v.partitions)), nil
}

func (v *View) find(key string) (storage.Storage, error) {
	h, err := v.hash(key)
	if err != nil {
		return nil, err
	}
	return v.partitions[h].st, nil
}

// Topic returns  the view's topic
func (v *View) Topic() string {
	return v.topic
}

// Get returns the value for the key in the view, if exists. Nil if it doesn't.
// Get can be called by multiple goroutines concurrently.
// Get can only be called after Recovered returns true.
func (v *View) Get(key string) (interface{}, error) {
	// find partition where key is located
	s, err := v.find(key)
	if err != nil {
		return nil, err
	}

	// get key and return
	data, err := s.Get(key)
	if err != nil {
		return nil, fmt.Errorf("error getting value (key %s): err", key, err)
	} else if data == nil {
		return nil, nil
	}

	// decode value
	value, err := v.opts.tableCodec.Decode(data)
	if err != nil {
		return nil, fmt.Errorf("error decoding value (key %s): err", key, err)
	}

	// if the key does not exist the return value is nil
	return value, nil
}

// Has checks whether a value for passed key exists in the view.
func (v *View) Has(key string) (bool, error) {
	// find partition where key is located
	s, err := v.find(key)
	if err != nil {
		return false, err
	}

	return s.Has(key)
}

// Iterator returns an iterator that iterates over the state of the View.
func (v *View) Iterator() (Iterator, error) {
	iters := make([]storage.Iterator, 0, len(v.partitions))
	for i := range v.partitions {
		iter, err := v.partitions[i].st.Iterator()
		if err != nil {
			// release already opened iterators
			for i := range iters {
				iters[i].Release()
			}

			return nil, fmt.Errorf("error opening partition iterator: %v", err)
		}

		iters = append(iters, iter)
	}

	return &iterator{
		iter:  storage.NewMultiIterator(iters),
		codec: v.opts.tableCodec,
	}, nil
}

// IteratorWithRange returns an iterator that iterates over the state of the View. This iterator is build using the range.
func (v *View) IteratorWithRange(start, limit string) (Iterator, error) {
	iters := make([]storage.Iterator, 0, len(v.partitions))
	for i := range v.partitions {
		iter, err := v.partitions[i].st.IteratorWithRange([]byte(start), []byte(limit))
		if err != nil {
			// release already opened iterators
			for i := range iters {
				iters[i].Release()
			}

			return nil, fmt.Errorf("error opening partition iterator: %v", err)
		}

		iters = append(iters, iter)
	}

	return &iterator{
		iter:  storage.NewMultiIterator(iters),
		codec: v.opts.tableCodec,
	}, nil
}

// Evict removes the given key only from the local cache. In order to delete a
// key from Kafka and other Views, context.Delete should be used on a Processor.
func (v *View) Evict(key string) error {
	s, err := v.find(key)
	if err != nil {
		return err
	}

	return s.Delete(key)
}

func (v *View) run() {
	defer close(v.done)
	v.opts.log.Printf("View: started")
	defer v.opts.log.Printf("View: stopped")

	for ev := range v.consumer.Events() {
		switch ev := ev.(type) {
		case *kafka.Message:
			partition := v.partitions[int(ev.Partition)]
			partition.ch <- ev
		case *kafka.BOF:
			partition := v.partitions[int(ev.Partition)]
			partition.ch <- ev
		case *kafka.EOF:
			partition := v.partitions[int(ev.Partition)]
			partition.ch <- ev
		case *kafka.Error:
			v.fail(fmt.Errorf("view: error from kafka consumer: %v", ev))
			return
		default:
			v.fail(fmt.Errorf("view: cannot handle %T = %v", ev, ev))
			return
		}
	}
}

// Recovered returns true when the view has caught up with events from kafka.
func (v *View) Recovered() bool {
	for _, p := range v.partitions {
		if !p.recovered() {
			return false
		}
	}

	return true
}

func (v *View) Stats() *ViewStats {
	var (
		m     sync.Mutex
		wg    sync.WaitGroup
		stats = newViewStats()
	)

	wg.Add(len(v.partitions))
	for i, p := range v.partitions {
		go func(pid int32, par *partition) {
			s := par.fetchStats()
			m.Lock()
			stats.Partitions[pid] = s
			m.Unlock()
			wg.Done()
		}(int32(i), p)
	}
	wg.Wait()
	return stats
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"html/template"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"

	"github.com/russross/blackfriday"
)

type Page struct {
	Title string
	Body  template.HTML
}

var pagesDir string = "pages"

func init() {
	if _, err := os.Stat(pagesDir); os.IsNotExist(err) {
		if err := os.Mkdir(pagesDir, 0700); err != nil {
			panic(err)
		}
	}
}

func (p *Page) save() error {
	filename := filepath.Join(pagesDir, p.Title+".txt")
	return ioutil.WriteFile(filename, []byte(p.Body), 0600)
}

func loadPage(title string) (*Page, error) {
	filename := filepath.Join(pagesDir, title+".txt")
	body, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	return &Page{Title: title, Body: template.HTML(body)}, nil
}

func viewHandler(w http.ResponseWriter, r *http.Request, title string) {
	p, err := loadPage(title)
	if err != nil {
		http.Redirect(w, r, "/edit/"+title, http.StatusFound)
		return
	}

	markdownBody := blackfriday.MarkdownCommon([]byte(p.Body))
	wikiMarkdown := convertWikiMarkup(markdownBody)
	p.Body = template.HTML(wikiMarkdown)
	renderTemplate(w, "view", p)
}

func editHandler(w http.ResponseWriter, r *http.Request, title string) {
	p, err := loadPage(title)
	if err != nil {
		p = &Page{Title: title}
	}
	renderTemplate(w, "edit", p)
}

func saveHandler(w http.ResponseWriter, r *http.Request, title string) {
	body := r.FormValue("body")
	p := &Page{Title: title, Body: template.HTML(body)}
	err := p.save()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	http.Redirect(w, r, "/view/"+title, http.StatusFound)
}

var templateDir string = "templates"
var templateFiles []string = []string{
	filepath.Join(templateDir, "edit.html"),
	filepath.Join(templateDir, "view.html")}

var templates = template.Must(template.ParseFiles(templateFiles...))

func renderTemplate(w http.ResponseWriter, tmpl string, p *Page) {
	err := templates.ExecuteTemplate(w, tmpl+".html", p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

var validPath = regexp.MustCompile("^/(edit|save|view)/([a-zA-Z0-9]+)$")

func makeHandler(fn func(http.ResponseWriter, *http.Request, string)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		m := validPath.FindStringSubmatch(r.URL.Path)
		if m == nil {
			http.NotFound(w, r)
			return
		}
		fn(w, r, m[2])
	}
}

// a wikiLink looks like [[Words]]
var wikiLink = regexp.MustCompile("\\[\\[(Home)\\]\\]")

func convertWikiMarkup(text []byte) []byte {
	var resultText = wikiLink.ReplaceAll(text, []byte("$1"))
	return resultText
}

func main() {
	var server = "localhost:8080"

	var browser *exec.Cmd
	var url string = "http://" + server + "/view/Home"

	switch runtime.GOOS {
	case "windows":
		browser = exec.Command(`C:\Windows\System32\rundll32.exe`, "url.dll,FileProtocolHandler", url)
	case "darwin":
		browser = exec.Command("open", url)
	default:
		browser = exec.Command("xdg-open", url)
	}
	if err := browser.Start(); err != nil {
		panic(err)
	}

	http.HandleFunc("/view/", makeHandler(viewHandler))
	http.HandleFunc("/edit/", makeHandler(editHandler))
	http.HandleFunc("/save/", makeHandler(saveHandler))
	http.Handle("/resources/", http.StripPrefix("/resources/", http.FileServer(http.Dir("resources"))))
	http.ListenAndServe(server, nil)
}
<commit_msg>Added some comments.<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"html/template"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"

	"github.com/russross/blackfriday"
)

// Page represents a single page in the wiki.
type Page struct {
	Title string
	Body  template.HTML
}

// save writes the page out to disk.
func (p *Page) save() error {
	filename := filepath.Join(pagesDir, p.Title+".txt")
	return ioutil.WriteFile(filename, []byte(p.Body), 0600)
}

// loadPage reads a page from disk.
func loadPage(title string) (*Page, error) {
	filename := filepath.Join(pagesDir, title+".txt")
	body, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	return &Page{Title: title, Body: template.HTML(body)}, nil
}

// viewHandler prepares the page to be rendered by passing it through the
// markdown and wikiMarkup filters.
func viewHandler(w http.ResponseWriter, r *http.Request, title string) {
	p, err := loadPage(title)
	if err != nil {
		http.Redirect(w, r, "/edit/"+title, http.StatusFound)
		return
	}

	markdownBody := blackfriday.MarkdownCommon([]byte(p.Body))
	wikiMarkdown := convertWikiMarkup(markdownBody)
	p.Body = template.HTML(wikiMarkdown)
	renderTemplate(w, "view", p)
}

// editHandler loads an existing page from disk or creates a new empty page to
// be rendered.
func editHandler(w http.ResponseWriter, r *http.Request, title string) {
	p, err := loadPage(title)
	if err != nil {
		p = &Page{Title: title}
	}
	renderTemplate(w, "edit", p)
}

// saveHandler saves the changes and redirects back to the page's view.
func saveHandler(w http.ResponseWriter, r *http.Request, title string) {
	body := r.FormValue("body")
	p := &Page{Title: title, Body: template.HTML(body)}
	err := p.save()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	http.Redirect(w, r, "/view/"+title, http.StatusFound)
}

// Parse the templates.
var templateDir string = "templates"
var templateFiles []string = []string{
	filepath.Join(templateDir, "edit.html"),
	filepath.Join(templateDir, "view.html")}

var templates = template.Must(template.ParseFiles(templateFiles...))

// renderTemplate takes the renders the html for the given template.
func renderTemplate(w http.ResponseWriter, tmpl string, p *Page) {
	err := templates.ExecuteTemplate(w, tmpl+".html", p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

// Defines the set of valid URLs to expect.
var validPath = regexp.MustCompile("^/(edit|save|view)/([a-zA-Z0-9]+)$")

func makeHandler(fn func(http.ResponseWriter, *http.Request, string)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		m := validPath.FindStringSubmatch(r.URL.Path)
		if m == nil {
			http.NotFound(w, r)
			return
		}
		fn(w, r, m[2])
	}
}

// a wikiLink looks like [[Words]]
var wikiLink = regexp.MustCompile("\\[\\[(Home)\\]\\]")

// convertWikiMarkup replaces wiki syntax with equivalent html.
func convertWikiMarkup(text []byte) []byte {
	var resultText = wikiLink.ReplaceAll(text, []byte("$1"))
	return resultText
}

// Ensure the pages directory exists before the program gets going.
var pagesDir string = "pages"

func init() {
	if _, err := os.Stat(pagesDir); os.IsNotExist(err) {
		if err := os.Mkdir(pagesDir, 0700); err != nil {
			panic(err)
		}
	}
}

func main() {
	var server = "localhost:8080"

	// open the default browser to the view/Home endpoint.
	var browser *exec.Cmd
	var url string = "http://" + server + "/view/Home"

	switch runtime.GOOS {
	case "windows":
		browser = exec.Command(`C:\Windows\System32\rundll32.exe`, "url.dll,FileProtocolHandler", url)
	case "darwin":
		browser = exec.Command("open", url)
	default:
		browser = exec.Command("xdg-open", url)
	}
	if err := browser.Start(); err != nil {
		panic(err)
	}

	// register the handlers and start the server.
	http.HandleFunc("/view/", makeHandler(viewHandler))
	http.HandleFunc("/edit/", makeHandler(editHandler))
	http.HandleFunc("/save/", makeHandler(saveHandler))
	http.Handle("/resources/", http.StripPrefix("/resources/", http.FileServer(http.Dir("resources"))))
	http.ListenAndServe(server, nil)
}
<|endoftext|>


<commit_before>// Package implements creation of XLSX simple spreadsheet files

package xlsx

import (
	"archive/zip"
	"bufio"
	"bytes"
	"fmt"
	"html"
	"io"
	"os"
	"strconv"
	"time"
)

type CellType uint

// Basic spreadsheet cell types
const (
	CellTypeNumber CellType = iota
	CellTypeString
	CellTypeDatetime
)

// XLSX Spreadsheet Cell
type Cell struct {
	Type  CellType
	Value string
}

// XLSX Spreadsheet Row
type Row struct {
	Cells []Cell
}

// XLSX Spreadsheet Column
type Column struct {
	Name  string
	Width uint64
}

// XLSX Spreadsheet Document Properties
type DocumentInfo struct {
	CreatedBy  string
	ModifiedBy string
	CreatedAt  time.Time
	ModifiedAt time.Time
}

// XLSX Spreadsheet
type Sheet struct {
	Title           string
	columns         []Column
	rows            []Row
	sharedStringMap map[string]int
	sharedStrings   []string
	DocumentInfo    DocumentInfo
}

// Create a sheet with no dimensions
func NewSheet() Sheet {
	c := make([]Column, 0)
	r := make([]Row, 0)
	ssm := make(map[string]int)
	sst := make([]string, 0)

	s := Sheet{
		Title:           "Data",
		columns:         c,
		rows:            r,
		sharedStringMap: ssm,
		sharedStrings:   sst,
	}

	return s
}

// Create a sheet with dimensions derived from the given columns
func NewSheetWithColumns(c []Column) Sheet {
	r := make([]Row, 0)
	ssm := make(map[string]int)
	sst := make([]string, 0)

	s := Sheet{
		Title:           "Data",
		columns:         c,
		rows:            r,
		sharedStringMap: ssm,
		sharedStrings:   sst,
	}

	s.DocumentInfo.CreatedBy = "xlsx.go"
	s.DocumentInfo.CreatedAt = time.Now()

	s.DocumentInfo.ModifiedBy = s.DocumentInfo.CreatedBy
	s.DocumentInfo.ModifiedAt = s.DocumentInfo.CreatedAt

	return s
}

// Create a new row with a length caculated by the sheets known column count
func (s *Sheet) NewRow() Row {
	c := make([]Cell, len(s.columns))
	r := Row{
		Cells: c,
	}
	return r
}

// Append a row to the sheet
func (s *Sheet) AppendRow(r Row) error {
	if len(r.Cells) != len(s.columns) {
		return fmt.Errorf("the given row has %d cells and %d were expected", len(r.Cells), len(s.columns))
	}

	cells := make([]Cell, len(s.columns))

	for n, c := range r.Cells {
		cells[n].Type = c.Type
		cells[n].Value = c.Value

		if cells[n].Type == CellTypeString {
			// calculate string reference
			cells[n].Value = html.EscapeString(cells[n].Value)
			i, exists := s.sharedStringMap[cells[n].Value]
			if !exists {
				i = len(s.sharedStrings)
				s.sharedStringMap[cells[n].Value] = i
				s.sharedStrings = append(s.sharedStrings, cells[n].Value)
			}
			cells[n].Value = strconv.Itoa(i)
		} else if cells[n].Type == CellTypeDatetime {
			d, err := time.Parse(time.RFC3339, cells[n].Value)
			if err == nil {
				cells[n].Value = OADate(d)
			}
		}
	}

	row := s.NewRow()
	row.Cells = cells

	s.rows = append(s.rows, row)

	return nil
}

// Get the Shared Strings in the order they were added to the map
func (s *Sheet) SharedStrings() []string {
	return s.sharedStrings
}

// Given zero-based array indices output the Excel cell reference. For
// example (0,0) => "A1"; (2,2) => "C3"; (26,45) => "AA46"
func CellIndex(x, y uint64) string {
	return fmt.Sprintf("%s%d", colName(x), y+1)
}

// From a zero-based column number return the Excel column name.
// For example: 0 => "A"; 2 => "C"; 26 => "AA"
func colName(n uint64) string {
	var s string
	n += 1

	for n > 0 {
		n -= 1
		s = fmt.Sprintf("%s%s", string(65+(n%26)), s)
		n /= 26
	}

	return s
}

// Convert time to the OLE Automation format.
func OADate(d time.Time) string {
	epoch := time.Date(1899, 12, 30, 0, 0, 0, 0, time.UTC)
	nsPerDay := 24 * time.Hour

	v := -1 * float64(epoch.Sub(d)) / float64(nsPerDay)

	// TODO: deal with dates before epoch
	// e.g. http://stackoverflow.com/questions/15549823/oadate-to-milliseconds-timestamp-in-javascript/15550284#15550284

	if d.Hour() == 0 && d.Minute() == 0 && d.Second() == 0 {
		return fmt.Sprintf("%d", int64(v))
	} else {
		return fmt.Sprintf("%f", v)
	}
}

// Create filename and save the XLSX file
func (s *Sheet) SaveToFile(filename string) error {
	outputfile, err := os.Create(filename)
	if err != nil {
		return err
	}
	w := bufio.NewWriter(outputfile)
	err = s.SaveToWriter(w)
	defer w.Flush()
	return err
}

func (sw *SheetWriter) WriteRows(rows []Row) error {

	var err error

	for i, r := range rows {
		rb := &bytes.Buffer{}

		if sw.maxNCols < uint64(len(r.Cells)) {
			sw.maxNCols = uint64(len(r.Cells))
		}

		for j, c := range r.Cells {

			cell := struct {
				CellIndex string
				Value     string
			}{
				CellIndex: CellIndex(uint64(j), uint64(i)+sw.currentIndex),
				Value:     c.Value,
			}

			switch c.Type {
			case CellTypeString:
				err = TemplateCellString.Execute(rb, cell)
			case CellTypeNumber:
				err = TemplateCellNumber.Execute(rb, cell)
			case CellTypeDatetime:
				err = TemplateCellDateTime.Execute(rb, cell)
			}

			if err != nil {
				return err
			}
		}

		rowString := fmt.Sprintf(`<row r="%d">%s</row>`, uint64(i)+sw.currentIndex+1, rb.String())

		_, err = io.WriteString(sw.f, rowString)
		if err != nil {
			return err
		}

	}

	sw.currentIndex += uint64(len(rows))

	return nil
}

// Save the XLSX file to the given writer
func (s *Sheet) SaveToWriter(w io.Writer) error {

	ww := NewWorkbookWriter(w)

	err := ww.WriteHeader(s)
	if err != nil {
		return err
	}

	sw := ww.NewSheetWriter(s.Title)

	sw.WriteHeader(s)
	sw.WriteRows(s.rows)
	sw.Close()

	err = ww.Close()
	if err != nil {
		return err
	}

	return nil
}

func (ww *WorkbookWriter) WriteHeader(s *Sheet) error {

	z := ww.zipWriter

	f, err := z.Create("[Content_Types].xml")
	err = TemplateContentTypes.Execute(f, nil)
	if err != nil {
		return err
	}

	f, err = z.Create("docProps/app.xml")
	err = TemplateApp.Execute(f, s)
	if err != nil {
		return err
	}

	f, err = z.Create("docProps/core.xml")
	err = TemplateCore.Execute(f, s.DocumentInfo)
	if err != nil {
		return err
	}

	f, err = z.Create("_rels/.rels")
	err = TemplateRelationships.Execute(f, nil)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/workbook.xml")
	err = TemplateWorkbook.Execute(f, s)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/_rels/workbook.xml.rels")
	err = TemplateWorkbookRelationships.Execute(f, nil)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/styles.xml")
	err = TemplateStyles.Execute(f, nil)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/sharedStrings.xml")
	err = TemplateStringLookups.Execute(f, s.SharedStrings())
	if err != nil {
		return err
	}

	return nil
}

type WorkbookWriter struct {
	zipWriter *zip.Writer
}

func NewWorkbookWriter(w io.Writer) *WorkbookWriter {
	return &WorkbookWriter{zip.NewWriter(w)}
}

func (ww *WorkbookWriter) Close() error {
	return ww.zipWriter.Close()
}

func (ww *WorkbookWriter) NewSheetWriter(title string) *SheetWriter {
	f, err := ww.zipWriter.Create("xl/worksheets/" + "sheet1" + ".xml")
	return &SheetWriter{f, err, 0, 0}
}

type SheetWriter struct {
	f            io.Writer
	err          error
	currentIndex uint64
	maxNCols     uint64
}

func (sw *SheetWriter) Close() error {
	sheet := struct {
		Start string
		End   string
	}{
		Start: "A1",
		End:   CellIndex(sw.maxNCols-1, sw.currentIndex-1),
	}

	err := TemplateSheetEnd.Execute(sw.f, sheet)
	return err
}

func (sw *SheetWriter) WriteHeader(s *Sheet) error {
	sheet := struct {
		Cols []Column
	}{
		Cols: s.columns,
	}

	err := TemplateSheetStart.Execute(sw.f, sheet)
	return err
}
<commit_msg>Automatically close SheetWriters when they are done with<commit_after>// Package implements creation of XLSX simple spreadsheet files

package xlsx

import (
	"archive/zip"
	"bufio"
	"bytes"
	"fmt"
	"html"
	"io"
	"os"
	"strconv"
	"time"
)

type CellType uint

// Basic spreadsheet cell types
const (
	CellTypeNumber CellType = iota
	CellTypeString
	CellTypeDatetime
)

// XLSX Spreadsheet Cell
type Cell struct {
	Type  CellType
	Value string
}

// XLSX Spreadsheet Row
type Row struct {
	Cells []Cell
}

// XLSX Spreadsheet Column
type Column struct {
	Name  string
	Width uint64
}

// XLSX Spreadsheet Document Properties
type DocumentInfo struct {
	CreatedBy  string
	ModifiedBy string
	CreatedAt  time.Time
	ModifiedAt time.Time
}

// XLSX Spreadsheet
type Sheet struct {
	Title           string
	columns         []Column
	rows            []Row
	sharedStringMap map[string]int
	sharedStrings   []string
	DocumentInfo    DocumentInfo
}

// Create a sheet with no dimensions
func NewSheet() Sheet {
	c := make([]Column, 0)
	r := make([]Row, 0)
	ssm := make(map[string]int)
	sst := make([]string, 0)

	s := Sheet{
		Title:           "Data",
		columns:         c,
		rows:            r,
		sharedStringMap: ssm,
		sharedStrings:   sst,
	}

	return s
}

// Create a sheet with dimensions derived from the given columns
func NewSheetWithColumns(c []Column) Sheet {
	r := make([]Row, 0)
	ssm := make(map[string]int)
	sst := make([]string, 0)

	s := Sheet{
		Title:           "Data",
		columns:         c,
		rows:            r,
		sharedStringMap: ssm,
		sharedStrings:   sst,
	}

	s.DocumentInfo.CreatedBy = "xlsx.go"
	s.DocumentInfo.CreatedAt = time.Now()

	s.DocumentInfo.ModifiedBy = s.DocumentInfo.CreatedBy
	s.DocumentInfo.ModifiedAt = s.DocumentInfo.CreatedAt

	return s
}

// Create a new row with a length caculated by the sheets known column count
func (s *Sheet) NewRow() Row {
	c := make([]Cell, len(s.columns))
	r := Row{
		Cells: c,
	}
	return r
}

// Append a row to the sheet
func (s *Sheet) AppendRow(r Row) error {
	if len(r.Cells) != len(s.columns) {
		return fmt.Errorf("the given row has %d cells and %d were expected", len(r.Cells), len(s.columns))
	}

	cells := make([]Cell, len(s.columns))

	for n, c := range r.Cells {
		cells[n].Type = c.Type
		cells[n].Value = c.Value

		if cells[n].Type == CellTypeString {
			// calculate string reference
			cells[n].Value = html.EscapeString(cells[n].Value)
			i, exists := s.sharedStringMap[cells[n].Value]
			if !exists {
				i = len(s.sharedStrings)
				s.sharedStringMap[cells[n].Value] = i
				s.sharedStrings = append(s.sharedStrings, cells[n].Value)
			}
			cells[n].Value = strconv.Itoa(i)
		} else if cells[n].Type == CellTypeDatetime {
			d, err := time.Parse(time.RFC3339, cells[n].Value)
			if err == nil {
				cells[n].Value = OADate(d)
			}
		}
	}

	row := s.NewRow()
	row.Cells = cells

	s.rows = append(s.rows, row)

	return nil
}

// Get the Shared Strings in the order they were added to the map
func (s *Sheet) SharedStrings() []string {
	return s.sharedStrings
}

// Given zero-based array indices output the Excel cell reference. For
// example (0,0) => "A1"; (2,2) => "C3"; (26,45) => "AA46"
func CellIndex(x, y uint64) string {
	return fmt.Sprintf("%s%d", colName(x), y+1)
}

// From a zero-based column number return the Excel column name.
// For example: 0 => "A"; 2 => "C"; 26 => "AA"
func colName(n uint64) string {
	var s string
	n += 1

	for n > 0 {
		n -= 1
		s = fmt.Sprintf("%s%s", string(65+(n%26)), s)
		n /= 26
	}

	return s
}

// Convert time to the OLE Automation format.
func OADate(d time.Time) string {
	epoch := time.Date(1899, 12, 30, 0, 0, 0, 0, time.UTC)
	nsPerDay := 24 * time.Hour

	v := -1 * float64(epoch.Sub(d)) / float64(nsPerDay)

	// TODO: deal with dates before epoch
	// e.g. http://stackoverflow.com/questions/15549823/oadate-to-milliseconds-timestamp-in-javascript/15550284#15550284

	if d.Hour() == 0 && d.Minute() == 0 && d.Second() == 0 {
		return fmt.Sprintf("%d", int64(v))
	} else {
		return fmt.Sprintf("%f", v)
	}
}

// Create filename and save the XLSX file
func (s *Sheet) SaveToFile(filename string) error {
	outputfile, err := os.Create(filename)
	if err != nil {
		return err
	}
	w := bufio.NewWriter(outputfile)
	err = s.SaveToWriter(w)
	defer w.Flush()
	return err
}

func (sw *SheetWriter) WriteRows(rows []Row) error {

	var err error

	for i, r := range rows {
		rb := &bytes.Buffer{}

		if sw.maxNCols < uint64(len(r.Cells)) {
			sw.maxNCols = uint64(len(r.Cells))
		}

		for j, c := range r.Cells {

			cell := struct {
				CellIndex string
				Value     string
			}{
				CellIndex: CellIndex(uint64(j), uint64(i)+sw.currentIndex),
				Value:     c.Value,
			}

			switch c.Type {
			case CellTypeString:
				err = TemplateCellString.Execute(rb, cell)
			case CellTypeNumber:
				err = TemplateCellNumber.Execute(rb, cell)
			case CellTypeDatetime:
				err = TemplateCellDateTime.Execute(rb, cell)
			}

			if err != nil {
				return err
			}
		}

		rowString := fmt.Sprintf(`<row r="%d">%s</row>`, uint64(i)+sw.currentIndex+1, rb.String())

		_, err = io.WriteString(sw.f, rowString)
		if err != nil {
			return err
		}

	}

	sw.currentIndex += uint64(len(rows))

	return nil
}

// Save the XLSX file to the given writer
func (s *Sheet) SaveToWriter(w io.Writer) error {

	ww := NewWorkbookWriter(w)

	err := ww.WriteHeader(s)
	if err != nil {
		return err
	}

	sw := ww.NewSheetWriter(s.Title)

	sw.WriteHeader(s)
	sw.WriteRows(s.rows)

	err = ww.Close()
	if err != nil {
		return err
	}

	return nil
}

func (ww *WorkbookWriter) WriteHeader(s *Sheet) error {

	z := ww.zipWriter

	f, err := z.Create("[Content_Types].xml")
	err = TemplateContentTypes.Execute(f, nil)
	if err != nil {
		return err
	}

	f, err = z.Create("docProps/app.xml")
	err = TemplateApp.Execute(f, s)
	if err != nil {
		return err
	}

	f, err = z.Create("docProps/core.xml")
	err = TemplateCore.Execute(f, s.DocumentInfo)
	if err != nil {
		return err
	}

	f, err = z.Create("_rels/.rels")
	err = TemplateRelationships.Execute(f, nil)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/workbook.xml")
	err = TemplateWorkbook.Execute(f, s)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/_rels/workbook.xml.rels")
	err = TemplateWorkbookRelationships.Execute(f, nil)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/styles.xml")
	err = TemplateStyles.Execute(f, nil)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/sharedStrings.xml")
	err = TemplateStringLookups.Execute(f, s.SharedStrings())
	if err != nil {
		return err
	}

	return nil
}

type WorkbookWriter struct {
	zipWriter   *zip.Writer
	sheetWriter *SheetWriter
}

func NewWorkbookWriter(w io.Writer) *WorkbookWriter {
	return &WorkbookWriter{zip.NewWriter(w), nil}
}

func (ww *WorkbookWriter) Close() error {
	if ww.sheetWriter != nil {
		ww.sheetWriter.Close()
	}
	return ww.zipWriter.Close()
}

func (ww *WorkbookWriter) NewSheetWriter(title string) *SheetWriter {
	f, err := ww.zipWriter.Create("xl/worksheets/" + "sheet1" + ".xml")
	sw := &SheetWriter{f, err, 0, 0}

	if ww.sheetWriter != nil {
		ww.sheetWriter.Close()
	}

	ww.sheetWriter = sw

	return sw
}

type SheetWriter struct {
	f            io.Writer
	err          error
	currentIndex uint64
	maxNCols     uint64
}

func (sw *SheetWriter) Close() error {
	sheet := struct {
		Start string
		End   string
	}{
		Start: "A1",
		End:   CellIndex(sw.maxNCols-1, sw.currentIndex-1),
	}

	err := TemplateSheetEnd.Execute(sw.f, sheet)
	return err
}

func (sw *SheetWriter) WriteHeader(s *Sheet) error {
	sheet := struct {
		Cols []Column
	}{
		Cols: s.columns,
	}

	err := TemplateSheetStart.Execute(sw.f, sheet)
	return err
}
<|endoftext|>


<commit_before>package sms

import (
	"crypto/rand"
	"encoding/base32"
	"encoding/xml"
	"fmt"
	"log"
	"os"
	"strings"

	xco "github.com/mndrix/go-xco"
)

// xmppProcess is the piece which interacts with the XMPP network and
// converts those communications into values which the rest of
// sms-over-xmpp can understand.
type xmppProcess struct {
	// where to connect to the XMPP server
	host string
	port int

	// credentials for XMPP auth
	name   string
	secret string

	// channel for sending XMPP stanzas to server
	tx chan<- interface{}

	// channels for communicating with the Gateway process
	gatewayRx chan<- *xco.Message
	gatewayTx <-chan *xco.Message
}

// runXmppComponent creates a goroutine for sending and receiving XMPP
// stanzas.  it returns a channel for monitoring the goroutine's
// health.  if that channel closes, the XMPP process has died.
func (x *xmppProcess) run() <-chan struct{} {
	opts := xco.Options{
		Name:         x.name,
		SharedSecret: x.secret,
		Address:      fmt.Sprintf("%s:%d", x.host, x.port),
		Logger:       log.New(os.Stderr, "", log.LstdFlags),
	}

	healthCh := make(chan struct{})
	go x.loop(opts, healthCh)
	return healthCh
}

func (x *xmppProcess) loop(opts xco.Options, healthCh chan<- struct{}) {
	defer func() { close(healthCh) }()

	c, err := xco.NewComponent(opts)
	if err != nil {
		log.Printf("can't create internal XMPP component: %s", err)
		return
	}

	tx, rx, errx := c.RunAsync()
	x.tx = tx
	for {
		select {
		case stanza := <-rx:
			switch st := stanza.(type) {
			case *xco.Message:
				log.Printf("Message: %+v", st)
				if st.Body == "" {
					log.Printf("  ignoring message with empty body")
					break
				}
				go func() { x.gatewayRx <- st }()
			case *xco.Presence:
				log.Printf("Presence: %+v", st)
			case *xco.Iq:
				if st.IsDiscoInfo() {
					var ids []xco.DiscoIdentity
					var features []xco.DiscoFeature
					ids, features, err = x.onDiscoInfo(st)
					if err == nil {
						st, err = st.DiscoInfoReply(ids, features)
						if err == nil {
							go func() { tx <- st }()
						}
					}
				} else {
					log.Printf("Iq: %+v", st)
				}
			case *xml.StartElement:
				log.Printf("Unknown: %+v", st)
			default:
				panic(fmt.Sprintf("Unexpected stanza type: %#v", stanza))
			}
		case stanza := <-x.gatewayTx:
			go func() { tx <- stanza }()
		case err = <-errx:
		}

		if err != nil {
			break
		}
	}

	log.Printf("lost XMPP connection: %s", err)
}

func (x *xmppProcess) onDiscoInfo(iq *xco.Iq) ([]xco.DiscoIdentity, []xco.DiscoFeature, error) {
	log.Printf("Disco: %+v", iq)
	ids := []xco.DiscoIdentity{
		{
			Category: "gateway",
			Type:     "sms",
			Name:     "SMS over XMPP",
		},
	}
	features := []xco.DiscoFeature{
		{
			Var: "urn:xmpp:receipts",
		},
	}
	return ids, features, nil
}

// NewId generates a random string which is suitable as an XMPP stanza
// ID.  The string contains enough entropy to be universally unique.
func NewId() string {
	// generate 128 random bits (6 more than standard UUID)
	bytes := make([]byte, 16)
	_, err := rand.Read(bytes)
	if err != nil {
		panic(err)
	}

	// convert them to base 32 encoding
	s := base32.StdEncoding.EncodeToString(bytes)
	return strings.ToLower(strings.TrimRight(s, "="))
}
<commit_msg>Remove unused XMPP channel<commit_after>package sms

import (
	"crypto/rand"
	"encoding/base32"
	"encoding/xml"
	"fmt"
	"log"
	"os"
	"strings"

	xco "github.com/mndrix/go-xco"
)

// xmppProcess is the piece which interacts with the XMPP network and
// converts those communications into values which the rest of
// sms-over-xmpp can understand.
type xmppProcess struct {
	// where to connect to the XMPP server
	host string
	port int

	// credentials for XMPP auth
	name   string
	secret string

	// channels for communicating with the Gateway process
	gatewayRx chan<- *xco.Message
	gatewayTx <-chan *xco.Message
}

// runXmppComponent creates a goroutine for sending and receiving XMPP
// stanzas.  it returns a channel for monitoring the goroutine's
// health.  if that channel closes, the XMPP process has died.
func (x *xmppProcess) run() <-chan struct{} {
	opts := xco.Options{
		Name:         x.name,
		SharedSecret: x.secret,
		Address:      fmt.Sprintf("%s:%d", x.host, x.port),
		Logger:       log.New(os.Stderr, "", log.LstdFlags),
	}

	healthCh := make(chan struct{})
	go x.loop(opts, healthCh)
	return healthCh
}

func (x *xmppProcess) loop(opts xco.Options, healthCh chan<- struct{}) {
	defer func() { close(healthCh) }()

	c, err := xco.NewComponent(opts)
	if err != nil {
		log.Printf("can't create internal XMPP component: %s", err)
		return
	}

	tx, rx, errx := c.RunAsync()
	for {
		select {
		case stanza := <-rx:
			switch st := stanza.(type) {
			case *xco.Message:
				log.Printf("Message: %+v", st)
				if st.Body == "" {
					log.Printf("  ignoring message with empty body")
					break
				}
				go func() { x.gatewayRx <- st }()
			case *xco.Presence:
				log.Printf("Presence: %+v", st)
			case *xco.Iq:
				if st.IsDiscoInfo() {
					var ids []xco.DiscoIdentity
					var features []xco.DiscoFeature
					ids, features, err = x.onDiscoInfo(st)
					if err == nil {
						st, err = st.DiscoInfoReply(ids, features)
						if err == nil {
							go func() { tx <- st }()
						}
					}
				} else {
					log.Printf("Iq: %+v", st)
				}
			case *xml.StartElement:
				log.Printf("Unknown: %+v", st)
			default:
				panic(fmt.Sprintf("Unexpected stanza type: %#v", stanza))
			}
		case stanza := <-x.gatewayTx:
			go func() { tx <- stanza }()
		case err = <-errx:
		}

		if err != nil {
			break
		}
	}

	log.Printf("lost XMPP connection: %s", err)
}

func (x *xmppProcess) onDiscoInfo(iq *xco.Iq) ([]xco.DiscoIdentity, []xco.DiscoFeature, error) {
	log.Printf("Disco: %+v", iq)
	ids := []xco.DiscoIdentity{
		{
			Category: "gateway",
			Type:     "sms",
			Name:     "SMS over XMPP",
		},
	}
	features := []xco.DiscoFeature{
		{
			Var: "urn:xmpp:receipts",
		},
	}
	return ids, features, nil
}

// NewId generates a random string which is suitable as an XMPP stanza
// ID.  The string contains enough entropy to be universally unique.
func NewId() string {
	// generate 128 random bits (6 more than standard UUID)
	bytes := make([]byte, 16)
	_, err := rand.Read(bytes)
	if err != nil {
		panic(err)
	}

	// convert them to base 32 encoding
	s := base32.StdEncoding.EncodeToString(bytes)
	return strings.ToLower(strings.TrimRight(s, "="))
}
<|endoftext|>


<commit_before>package handlers

import (
	"context"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strings"
	"time"

	"github.com/alphagov/router/logger"
)

var TLSSkipVerify bool

func NewBackendHandler(backendURL *url.URL, connectTimeout, headerTimeout time.Duration, logger logger.Logger) http.Handler {
	proxy := httputil.NewSingleHostReverseProxy(backendURL)
	proxy.Transport = newBackendTransport(connectTimeout, headerTimeout, logger)

	defaultDirector := proxy.Director
	proxy.Director = func(req *http.Request) {
		defaultDirector(req)

		// Set the Host header to match the backend hostname instead of the one from the incoming request.
		req.Host = backendURL.Host

		// Setting a blank User-Agent causes the http lib not to output one, whereas if there
		// is no header, it will output a default one.
		// See: https://github.com/golang/go/blob/release-branch.go1.5/src/net/http/request.go#L419
		if _, present := req.Header["User-Agent"]; !present {
			req.Header.Set("User-Agent", "")
		}

		populateViaHeader(req.Header, fmt.Sprintf("%d.%d", req.ProtoMajor, req.ProtoMinor))
	}

	return proxy
}

func populateViaHeader(header http.Header, httpVersion string) {
	via := httpVersion + " router"
	if prior, ok := header["Via"]; ok {
		via = strings.Join(prior, ", ") + ", " + via
	}
	header.Set("Via", via)
}

type backendTransport struct {
	wrapped *http.Transport
	logger  logger.Logger
}

// Construct a backendTransport that wraps an http.Transport and implements http.RoundTripper.
// This allows us to intercept the response from the backend and modify it before it's copied
// back to the client.
func newBackendTransport(connectTimeout, headerTimeout time.Duration, logger logger.Logger) (transport *backendTransport) {
	transport = &backendTransport{&http.Transport{}, logger}

	transport.wrapped.DialContext = func(_ context.Context, network, address string) (net.Conn, error) {
		return net.DialTimeout(network, address, connectTimeout)
	}
	// Allow the proxy to keep more than the default (2) keepalive connections
	// per upstream.
	transport.wrapped.MaxIdleConnsPerHost = 20
	transport.wrapped.ResponseHeaderTimeout = headerTimeout

	if TLSSkipVerify {
		transport.wrapped.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	}

	return
}

func closeBody(resp *http.Response) {
	if resp != nil && resp.Body != nil {
		resp.Body.Close()
	}
}

func (bt *backendTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	resp, err = bt.wrapped.RoundTrip(req)
	if err == nil {
		populateViaHeader(resp.Header, fmt.Sprintf("%d.%d", resp.ProtoMajor, resp.ProtoMinor))
	} else {
		// Log the error (deferred to allow special case error handling to add/change details)
		logDetails := map[string]interface{}{"error": err.Error(), "status": 500}
		defer bt.logger.LogFromBackendRequest(logDetails, req)
		defer logger.NotifySentry(logger.ReportableError{ Error: err, Request: req, Response: resp })
		defer closeBody(resp)

		// Intercept some specific errors and generate an appropriate HTTP error response
		if netErr, ok := err.(net.Error); ok {
			if netErr.Timeout() {
				logDetails["status"] = 504
				return newErrorResponse(504), nil
			}
		}
		if strings.Contains(err.Error(), "connection refused") {
			logDetails["status"] = 502
			return newErrorResponse(502), nil
		}

		// 500 for all other errors
		return newErrorResponse(500), nil
	}
	return
}

func newErrorResponse(status int) (resp *http.Response) {
	resp = &http.Response{StatusCode: status}
	resp.Body = ioutil.NopCloser(strings.NewReader(""))
	return
}
<commit_msg>Configure backend like http.DefaultTransport<commit_after>package handlers

import (
	"context"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strings"
	"time"

	"github.com/alphagov/router/logger"
)

var TLSSkipVerify bool

func NewBackendHandler(backendURL *url.URL, connectTimeout, headerTimeout time.Duration, logger logger.Logger) http.Handler {
	proxy := httputil.NewSingleHostReverseProxy(backendURL)
	proxy.Transport = newBackendTransport(connectTimeout, headerTimeout, logger)

	defaultDirector := proxy.Director
	proxy.Director = func(req *http.Request) {
		defaultDirector(req)

		// Set the Host header to match the backend hostname instead of the one from the incoming request.
		req.Host = backendURL.Host

		// Setting a blank User-Agent causes the http lib not to output one, whereas if there
		// is no header, it will output a default one.
		// See: https://github.com/golang/go/blob/release-branch.go1.5/src/net/http/request.go#L419
		if _, present := req.Header["User-Agent"]; !present {
			req.Header.Set("User-Agent", "")
		}

		populateViaHeader(req.Header, fmt.Sprintf("%d.%d", req.ProtoMajor, req.ProtoMinor))
	}

	return proxy
}

func populateViaHeader(header http.Header, httpVersion string) {
	via := httpVersion + " router"
	if prior, ok := header["Via"]; ok {
		via = strings.Join(prior, ", ") + ", " + via
	}
	header.Set("Via", via)
}

type backendTransport struct {
	wrapped *http.Transport
	logger  logger.Logger
}

// Construct a backendTransport that wraps an http.Transport and implements http.RoundTripper.
// This allows us to intercept the response from the backend and modify it before it's copied
// back to the client.
func newBackendTransport(connectTimeout, headerTimeout time.Duration, logger logger.Logger) *backendTransport {
	transport := http.Transport{}

	transport.DialContext = func(_ context.Context, network, address string) (net.Conn, error) {
		return net.DialTimeout(network, address, connectTimeout)
	}

	// Remember, we have one transport per backend
	//
	// Using the below settings, and (for example) we have 25 backends
	//   25 * 60 = 1500
	// we will have a maximum of 1500 open idle connections
	//
	// The Go http.DefaultTransport sets this to 100,
	// we set to 60 because of potential file handle limits
	// because we have multiple backends
	transport.MaxIdleConns = 60
	// This is an arbitrarily selected number that is less than 60
	transport.MaxIdleConnsPerHost = 20
	// By default, idle connections do not expire,
	// unless they are closed by the other end of the connection,
	// and sometimes the other end will silently close the connection.
	// We should expire idle connections after a while.
	//
	// We arbitrarily chose 10 minutes
	transport.IdleConnTimeout = 10 * time.Minute

	// If we do not configure the timeouts, then connections will hang
	//
	// Configured by the caller
	transport.ResponseHeaderTimeout = headerTimeout
	//
	// Same values as http.DefaultTransport
	transport.TLSHandshakeTimeout = 10 * time.Second
	transport.ExpectContinueTimeout = 1 * time.Second

	if TLSSkipVerify {
		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	}

	return &backendTransport{&transport, logger}
}

func closeBody(resp *http.Response) {
	if resp != nil && resp.Body != nil {
		resp.Body.Close()
	}
}

func (bt *backendTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	resp, err = bt.wrapped.RoundTrip(req)
	if err == nil {
		populateViaHeader(resp.Header, fmt.Sprintf("%d.%d", resp.ProtoMajor, resp.ProtoMinor))
	} else {
		// Log the error (deferred to allow special case error handling to add/change details)
		logDetails := map[string]interface{}{"error": err.Error(), "status": 500}
		defer bt.logger.LogFromBackendRequest(logDetails, req)
		defer logger.NotifySentry(logger.ReportableError{ Error: err, Request: req, Response: resp })
		defer closeBody(resp)

		// Intercept some specific errors and generate an appropriate HTTP error response
		if netErr, ok := err.(net.Error); ok {
			if netErr.Timeout() {
				logDetails["status"] = 504
				return newErrorResponse(504), nil
			}
		}
		if strings.Contains(err.Error(), "connection refused") {
			logDetails["status"] = 502
			return newErrorResponse(502), nil
		}

		// 500 for all other errors
		return newErrorResponse(500), nil
	}
	return
}

func newErrorResponse(status int) (resp *http.Response) {
	resp = &http.Response{StatusCode: status}
	resp.Body = ioutil.NopCloser(strings.NewReader(""))
	return
}
<|endoftext|>


<commit_before>package params

import (
	"sync/atomic"
)

// A pool of Params
type Pool struct {
	misses int64
	size   int
	list   chan Params
}

// Create a new pool of count params.
// Each ArrayParam has a maximum size of size.
// If the size exceeds this value, the ArrayParam is converted to a MapParams
func NewPool(size, count int) *Pool {
	pool := &Pool{
		size: size,
		list: make(chan Params, count),
	}
	for i := 0; i < count; i++ {
		pool.list <- pooled(pool, size)
	}
	return pool
}

// The number of times we tried to checkout but had no available
// params in our pool
func (p *Pool) Misses() int64 {
	return atomic.SwapInt64(&p.depleted, 0)
}

// Get a param from the pool or create a new one if the pool is empty
func (p *Pool) Checkout() Params {
	select {
	case item := <-p.list:
		return item
	default:
		atomic.AddInt64(&p.misses, 1)
		return New(p.size)
	}
}
<commit_msg>wow..ok...<commit_after>package params

import (
	"sync/atomic"
)

// A pool of Params
type Pool struct {
	misses int64
	size   int
	list   chan Params
}

// Create a new pool of count params.
// Each ArrayParam has a maximum size of size.
// If the size exceeds this value, the ArrayParam is converted to a MapParams
func NewPool(size, count int) *Pool {
	pool := &Pool{
		size: size,
		list: make(chan Params, count),
	}
	for i := 0; i < count; i++ {
		pool.list <- pooled(pool, size)
	}
	return pool
}

// The number of times we tried to checkout but had no available
// params in our pool
func (p *Pool) Misses() int64 {
	return atomic.SwapInt64(&p.misses, 0)
}

// Get a param from the pool or create a new one if the pool is empty
func (p *Pool) Checkout() Params {
	select {
	case item := <-p.list:
		return item
	default:
		atomic.AddInt64(&p.misses, 1)
		return New(p.size)
	}
}
<|endoftext|>


<commit_before>// gPool is a utility for pooling workers
package gpool

import (
	"errors"
	"sync"
	"time"
)

// ErrClosedPool indicates that a send was attempted on a pool which has already been closed.
var ErrClosedPool = errors.New("send on closed pool")

// ErrKilled indicates that the pool was killed by a call to Kill()
var ErrKilled = errors.New("pool killed by signal")

// Pool is the main pool struct containing a bus and workers.
// Pool should always be invoked via NewPool().
type Pool struct {
	// Workers
	wQ chan jobRequest // Queue
	wR chan JobResult  // Return
	wC chan struct{}   // Cancel
	wD chan bool       // Done

	fJ []JobResult
	iJ int // Job id

	tQ chan ticket // Send ticket requests

	s   *stateManager
	err error

	wg *sync.WaitGroup
	// Hooks are optional functions that are executed during different stages of a Job.
	// They are invoked by the worker and thus are called concurrently.
	// The implementor should consider any race conditions for hook callbacks.
	Hook struct {
		Start HookStart
		Stop  HookStop
	}

	closed bool
	killed bool
	done   bool
}

// NewPool creates a new Pool with the given worker count.
// Workers in the pool are started automatically.
func NewPool(Workers int) *Pool {
	p := &Pool{
		tQ: make(chan ticket),
		wQ: make(chan jobRequest),
		wC: make(chan struct{}),
		wR: make(chan JobResult),
		wD: make(chan bool),
		wg: &sync.WaitGroup{},
		s:  newStateManager(Workers),
	}
	p.start()
	return p
}

// start starts Pool workers and Pool bus
func (p *Pool) start() {
	for range make([]int, p.s.GetTarget()) {
		p.startWorker()
	}
	go func() {
		p.wg.Wait()
		p.wD <- true
	}()
	go p.bus()
}

// ticket request types
const (
	tReqJob int = 1 << iota
	tReqClose
	tReqKill
	tReqWait
	tReqGrow
	tReqGetOpen
	tReqGetError
)

// A ticket is a request for input in the queue.
// This prevents direct access to queue channels which reduces the risk of bad things happening.
type ticket struct {
	t    int         // Ticket type
	data interface{} // Ticket request data
	r    chan error  // Return channel
}

// Kill sends a kill request to the pool bus.
// When sent, any currently running jobs have Cancel() called.
// If the pool has already been killed or closed ErrClosedPool is returned.
func (p *Pool) Kill() error {
	t := ticket{
		tReqKill, nil, make(chan error),
	}
	p.tQ <- t
	return <-t.r
}

// Close sends a graceful close request to the pool bus.
// Workers will finish after the last submitted job is complete.
// Close does not return an error if the pool is already closed.
func (p *Pool) Close() error {
	t := ticket{
		tReqClose, nil, make(chan error),
	}
	p.tQ <- t
	return <-t.r
}

// Wait waits for the pool worker group to finish and then returns all jobs completed during execution.
// If the pool has an error it is returned here.
// Wait will block until the pool is marked as done (via call to Pool.Close) or has an error.
// As such it is important that the caller either implements a timeout around Wait,
// or ensures a call to Pool.Close will be made.
// Wait requests are stacked until they can be resolved in the next bus cycle.
func (p *Pool) Wait() ([]JobResult, error) {
	t := ticket{
		tReqWait, nil, make(chan error),
	}
	p.tQ <- t
	return p.fJ, <-t.r
}

// Send sends the given PoolJob as a request to the pool bus.
// If the pool is closed the error ErrClosedPool is returned.
// No error is returned if the Send() was successful.
// A call to Send is blocked until a worker accepts the Job.
func (p *Pool) Send(job Job) error {
	t := ticket{
		tReqJob, job, make(chan error),
	}
	p.tQ <- t
	return <-t.r
}

// IsOpen returns true if the pool is currently open.
func (p *Pool) IsOpen() bool {
	t := ticket{
		tReqGetOpen, nil, make(chan error),
	}
	p.tQ <- t
	e := <-t.r
	if e == nil {
		return true
	}
	return false
}

// Error returns the current error present in the pool.
func (p *Pool) Error() error {
	t := ticket{
		tReqGetError, nil, make(chan error),
	}
	p.tQ <- t
	return <-t.r
}

// Grow grows the amount of workers running in the pool by the requested amount.
// If the pool is closed ErrClosedPool is return.
func (p *Pool) Grow(Req int) error {
	if Req == 0 {
		return nil
	}
	t := ticket{
		tReqGrow, Req, make(chan error),
	}
	p.tQ <- t
	return <-t.r
}

// WorkerState returns the current amount of running workers and the amount of requested workers.
func (p *Pool) WorkerState() (c int, t int) {
	return p.s.WorkerState()
}

// JobState returns the amount of currently executing jobs in the pool
// and the amount of executed jobs in the pool (including failed).
func (p *Pool) JobState() (c int, f int) {
	return p.s.JobState()
}

type jobRequest struct {
	Job Job
	ID  int
}

// close closes the pool if not already closed.
// Returns true if the pool was closed.
func (p *Pool) close(kill bool) bool {
	if !p.killed && kill {
		p.killed = true
		if p.err == nil {
			p.err = ErrKilled
		}
		close(p.wC)
	}
	if !p.closed {
		p.closed = true
		close(p.wQ)
		return true
	}
	return false
}

// res receives a JobResult and processes it.
func (p *Pool) res(resp JobResult) {
	if resp.Error != nil {
		p.err = resp.Error
		p.close(true)
		return
	}
	p.fJ = append(p.fJ, resp)
}

// doSend tries to send a jobRequest to the worker pool.
// If the pool is full a deadlock is possible unless we also collect the return message from the workers
func (p *Pool) doSend(r jobRequest) {
	for {
		select {
		case p.wQ <- r:
			// Request successfully sent
			return
		case resp := <-p.wR:
			p.res(resp)
		}
	}
}

func (p *Pool) unhealthy() bool {
	return p.done || p.err != nil || p.closed || p.killed
}

// bus is the central communication bus for the pool.
// All pool inputs are collected here as tickets and then actioned on depending on the ticket type.
// If the ticket is a tReqWait request they are added to the slice of pending tickets and upon pool closure.
// A central bus is used to prevent concurrent access to any property of the pool.
func (p *Pool) bus() {
	// Pending tReqWaits
	pendWait := []ticket{}
	for {
		select {
		case <-p.wD:
			p.close(false)
			p.done = true
		// Receive worker response
		case resp := <-p.wR:
			p.res(resp)
		// Receive ticket request
		case t := <-p.tQ:
			switch t.t {
			// New Job request
			case tReqJob:
				if p.unhealthy() {
					t.r <- ErrClosedPool
					continue
				}
				p.iJ++
				p.doSend(jobRequest{
					ID:  p.iJ,
					Job: t.data.(Job),
				})
				t.r <- nil
			// Pool close request
			case tReqClose:
				if p.done {
					t.r <- ErrClosedPool
					continue
				}
				p.close(false)
				t.r <- nil
			// Pool kill request
			case tReqKill:
				if p.done {
					t.r <- ErrClosedPool
					continue
				}
				if ok := p.close(true); ok {
					t.r <- nil
				} else {
					t.r <- ErrClosedPool
				}
			// Pool wait request
			case tReqWait:
				pendWait = append(pendWait, t)
			case tReqGrow:
				if p.unhealthy() {
					t.r <- ErrClosedPool
				}
				for range make([]int, t.data.(int)) {
					p.s.IncrTarget()
					p.startWorker()
				}
				t.r <- nil
			// Pool is open request
			case tReqGetOpen:
				if p.unhealthy() {
					t.r <- ErrClosedPool
					continue
				}
				t.r <- nil
			case tReqGetError:
				t.r <- p.err
			default:
				panic("unknown ticket type")
			}
		// Default case resolves any pending tickets
		default:
			if p.done && len(pendWait) != 0 {
				// Pop every item in pending and send return signal to ticket
				for {
					if len(pendWait) == 0 {
						// No more pending message, continue with next cycle
						break
					}
					var t ticket
					t, pendWait = pendWait[len(pendWait)-1], pendWait[:len(pendWait)-1]
					t.r <- p.err
				}
			}

		}

	}
}

// startWorker starts a worker and waits for it to complete starting before return.
// This prevents a race condition on call to Pool.Running() where the worker may not have completed it's state registration.
func (p *Pool) startWorker() {
	ok := make(chan bool)
	p.wg.Add(1)
	go p.worker(ok)
	<-ok
}

// worker is the Pool worker routine that receives jobRequests from the pool worker queue until the queue is closed.
// It executes the job and returns the result to the worker return channel as a JobResult.
func (p *Pool) worker(started chan bool) {
	defer p.wg.Done()
	p.s.AddWorker()
	defer p.s.RemoveWorker()
	// Acknowledge start
	close(started)
	for {
		select {
		case req, ok := <-p.wQ:
			// Worker queue has been closed
			if !ok {
				return
			}

			p.s.AddJob()

			if p.Hook.Start != nil {
				p.Hook.Start(req.ID, req.Job)
			}

			// Cancel wrapper
			d := make(chan struct{})
			go func() {
				select {
				// Job done
				case <-d:
					return
				// Worker cancel signal received
				case <-p.wC:
					req.Job.Cancel()
				}
			}()

			s := time.Now()

			// Execute Job
			e := req.Job.Run()

			// Close cancel wrapping
			close(d)

			if e != nil {
				e = newPoolError(&req, e)
			}
			j := JobResult{
				Job:      req.Job,
				ID:       req.ID,
				Duration: time.Since(s),
				Error:    e,
			}
			// Send Job result to return queue
			p.wR <- j

			if p.Hook.Stop != nil {
				p.Hook.Stop(req.ID, j)
			}

			p.s.RemoveJob()

		case <-p.wC:
			return
		}
	}
}
<commit_msg>golint complete, check for nil input job<commit_after>// Package gpool is a utility for executing jobs in a pool of workers
package gpool

import (
	"errors"
	"sync"
	"time"
)

// ErrClosedPool indicates that a send was attempted on a pool which has already been closed.
var ErrClosedPool = errors.New("send on closed pool")

// ErrKilled indicates that the pool was killed by a call to Kill()
var ErrKilled = errors.New("pool killed by signal")

// Pool is the main pool struct containing a bus and workers.
// Pool should always be invoked via NewPool().
type Pool struct {
	// Workers
	wQ chan jobRequest // Queue
	wR chan JobResult  // Return
	wC chan struct{}   // Cancel
	wD chan bool       // Done

	fJ []JobResult
	iJ int // Job id

	tQ chan ticket // Send ticket requests

	s   *stateManager
	err error

	wg *sync.WaitGroup
	// Hooks are optional functions that are executed during different stages of a Job.
	// They are invoked by the worker and thus are called concurrently.
	// The implementor should consider any race conditions for hook callbacks.
	Hook struct {
		Start HookStart
		Stop  HookStop
	}

	closed bool
	killed bool
	done   bool
}

// NewPool creates a new Pool with the given worker count.
// Workers in the pool are started automatically.
func NewPool(Workers int) *Pool {
	p := &Pool{
		tQ: make(chan ticket),
		wQ: make(chan jobRequest),
		wC: make(chan struct{}),
		wR: make(chan JobResult),
		wD: make(chan bool),
		wg: &sync.WaitGroup{},
		s:  newStateManager(Workers),
	}
	p.start()
	return p
}

// start starts Pool workers and Pool bus
func (p *Pool) start() {
	for range make([]int, p.s.GetTarget()) {
		p.startWorker()
	}
	go func() {
		p.wg.Wait()
		p.wD <- true
	}()
	go p.bus()
}

// ticket request types
const (
	tReqJob int = 1 << iota
	tReqClose
	tReqKill
	tReqWait
	tReqGrow
	tReqGetOpen
	tReqGetError
)

// A ticket is a request for input in the queue.
// This prevents direct access to queue channels which reduces the risk of bad things happening.
type ticket struct {
	t    int         // Ticket type
	data interface{} // Ticket request data
	r    chan error  // Return channel
}

// Kill sends a kill request to the pool bus.
// When sent, any currently running jobs have Cancel() called.
// If the pool has already been killed or closed ErrClosedPool is returned.
func (p *Pool) Kill() error {
	t := ticket{
		tReqKill, nil, make(chan error),
	}
	p.tQ <- t
	return <-t.r
}

// Close sends a graceful close request to the pool bus.
// Workers will finish after the last submitted job is complete.
// Close does not return an error if the pool is already closed.
func (p *Pool) Close() error {
	t := ticket{
		tReqClose, nil, make(chan error),
	}
	p.tQ <- t
	return <-t.r
}

// Wait waits for the pool worker group to finish and then returns all jobs completed during execution.
// If the pool has an error it is returned here.
// Wait will block until the pool is marked as done (via call to Pool.Close) or has an error.
// As such it is important that the caller either implements a timeout around Wait,
// or ensures a call to Pool.Close will be made.
// Wait requests are stacked until they can be resolved in the next bus cycle.
func (p *Pool) Wait() ([]JobResult, error) {
	t := ticket{
		tReqWait, nil, make(chan error),
	}
	p.tQ <- t
	return p.fJ, <-t.r
}

// Send sends the given PoolJob as a request to the pool bus.
// If the pool is closed the error ErrClosedPool is returned.
// No error is returned if the Send() was successful.
// A call to Send is blocked until a worker accepts the Job.
func (p *Pool) Send(job Job) error {
	if job == nil {
		panic("send of nil job")
	}
	t := ticket{
		tReqJob, job, make(chan error),
	}
	p.tQ <- t
	return <-t.r
}

// IsOpen returns true if the pool is currently open.
func (p *Pool) IsOpen() bool {
	t := ticket{
		tReqGetOpen, nil, make(chan error),
	}
	p.tQ <- t
	e := <-t.r
	if e == nil {
		return true
	}
	return false
}

// Error returns the current error present in the pool.
func (p *Pool) Error() error {
	t := ticket{
		tReqGetError, nil, make(chan error),
	}
	p.tQ <- t
	return <-t.r
}

// Grow grows the amount of workers running in the pool by the requested amount.
// If the pool is closed ErrClosedPool is return.
func (p *Pool) Grow(Req int) error {
	if Req == 0 {
		return nil
	}
	t := ticket{
		tReqGrow, Req, make(chan error),
	}
	p.tQ <- t
	return <-t.r
}

// WorkerState returns the current amount of running workers and the amount of requested workers.
func (p *Pool) WorkerState() (c int, t int) {
	return p.s.WorkerState()
}

// JobState returns the amount of currently executing jobs in the pool
// and the amount of executed jobs in the pool (including failed).
func (p *Pool) JobState() (c int, f int) {
	return p.s.JobState()
}

type jobRequest struct {
	Job Job
	ID  int
}

// close closes the pool if not already closed.
// Returns true if the pool was closed.
func (p *Pool) close(kill bool) bool {
	if !p.killed && kill {
		p.killed = true
		if p.err == nil {
			p.err = ErrKilled
		}
		close(p.wC)
	}
	if !p.closed {
		p.closed = true
		close(p.wQ)
		return true
	}
	return false
}

// res receives a JobResult and processes it.
func (p *Pool) res(resp JobResult) {
	if resp.Error != nil {
		p.err = resp.Error
		p.close(true)
		return
	}
	p.fJ = append(p.fJ, resp)
}

// doSend tries to send a jobRequest to the worker pool.
// If the pool is full a deadlock is possible unless we also collect the return message from the workers
func (p *Pool) doSend(r jobRequest) {
	for {
		select {
		case p.wQ <- r:
			// Request successfully sent
			return
		case resp := <-p.wR:
			p.res(resp)
		}
	}
}

func (p *Pool) unhealthy() bool {
	return p.done || p.err != nil || p.closed || p.killed
}

// bus is the central communication bus for the pool.
// All pool inputs are collected here as tickets and then actioned on depending on the ticket type.
// If the ticket is a tReqWait request they are added to the slice of pending tickets and upon pool closure.
// A central bus is used to prevent concurrent access to any property of the pool.
func (p *Pool) bus() {
	// Pending tReqWaits
	pendWait := []ticket{}
	for {
		select {
		case <-p.wD:
			p.close(false)
			p.done = true
		// Receive worker response
		case resp := <-p.wR:
			p.res(resp)
		// Receive ticket request
		case t := <-p.tQ:
			switch t.t {
			// New Job request
			case tReqJob:
				if p.unhealthy() {
					t.r <- ErrClosedPool
					continue
				}
				p.iJ++
				p.doSend(jobRequest{
					ID:  p.iJ,
					Job: t.data.(Job),
				})
				t.r <- nil
			// Pool close request
			case tReqClose:
				if p.done {
					t.r <- ErrClosedPool
					continue
				}
				p.close(false)
				t.r <- nil
			// Pool kill request
			case tReqKill:
				if p.done {
					t.r <- ErrClosedPool
					continue
				}
				if ok := p.close(true); ok {
					t.r <- nil
				} else {
					t.r <- ErrClosedPool
				}
			// Pool wait request
			case tReqWait:
				pendWait = append(pendWait, t)
			case tReqGrow:
				if p.unhealthy() {
					t.r <- ErrClosedPool
				}
				for range make([]int, t.data.(int)) {
					p.s.IncrTarget()
					p.startWorker()
				}
				t.r <- nil
			// Pool is open request
			case tReqGetOpen:
				if p.unhealthy() {
					t.r <- ErrClosedPool
					continue
				}
				t.r <- nil
			case tReqGetError:
				t.r <- p.err
			default:
				panic("unknown ticket type")
			}
		// Default case resolves any pending tickets
		default:
			if p.done && len(pendWait) != 0 {
				// Pop every item in pending and send return signal to ticket
				for {
					if len(pendWait) == 0 {
						// No more pending message, continue with next cycle
						break
					}
					var t ticket
					t, pendWait = pendWait[len(pendWait)-1], pendWait[:len(pendWait)-1]
					t.r <- p.err
				}
			}

		}

	}
}

// startWorker starts a worker and waits for it to complete starting before return.
// This prevents a race condition on call to Pool.Running() where the worker may not have completed it's state registration.
func (p *Pool) startWorker() {
	ok := make(chan bool)
	p.wg.Add(1)
	go p.worker(ok)
	<-ok
}

// worker is the Pool worker routine that receives jobRequests from the pool worker queue until the queue is closed.
// It executes the job and returns the result to the worker return channel as a JobResult.
func (p *Pool) worker(started chan bool) {
	defer p.wg.Done()
	p.s.AddWorker()
	defer p.s.RemoveWorker()
	// Acknowledge start
	close(started)
	for {
		select {
		case req, ok := <-p.wQ:
			// Worker queue has been closed
			if !ok {
				return
			}

			p.s.AddJob()

			if p.Hook.Start != nil {
				p.Hook.Start(req.ID, req.Job)
			}

			// Cancel wrapper
			d := make(chan struct{})
			go func() {
				select {
				// Job done
				case <-d:
					return
				// Worker cancel signal received
				case <-p.wC:
					req.Job.Cancel()
				}
			}()

			s := time.Now()

			// Execute Job
			e := req.Job.Run()

			// Close cancel wrapping
			close(d)

			if e != nil {
				e = newPoolError(&req, e)
			}
			j := JobResult{
				Job:      req.Job,
				ID:       req.ID,
				Duration: time.Since(s),
				Error:    e,
			}
			// Send Job result to return queue
			p.wR <- j

			if p.Hook.Stop != nil {
				p.Hook.Stop(req.ID, j)
			}

			p.s.RemoveJob()

		case <-p.wC:
			return
		}
	}
}
<|endoftext|>


<commit_before>package pb

import (
	"fmt"
	"os"
	"time"
)

type Pool struct {
	RefreshRate time.Duration
	bars        []*ProgressBar
	process     *os.Process
	isFinished  bool
}

func (p *Pool) Add(pbs ...*ProgressBar) {
	for _, bar := range pbs {
		bar.ManualUpdate = true
		bar.NotPrint = true
		bar.Start()
		p.bars = append(p.bars, bar)
	}
}

func (p *Pool) Start() (err error) {
	p.RefreshRate = DefaultRefreshRate
	quit, err := lockEcho()
	if err != nil {
		return
	}
	go p.writer(quit)
	return
}

func (p *Pool) writer(finish chan int) {
	var first = true
	var out string

	for {
		if first {
			first = false
		} else {
			out = fmt.Sprintf("\033[%dA", len(p.bars))
		}
		isFinished := true
		for _, bar := range p.bars {
			bar.Update()
			out += fmt.Sprintf("\r%s\n", bar.String())
			if !bar.isFinish {
				isFinished = false
			}
		}
		fmt.Print(out)
		if isFinished {
			p.isFinished = true
			finish <- 1
			return
		}
		time.Sleep(p.RefreshRate)
	}
}
<commit_msg>cleanup<commit_after>package pb

import (
	"fmt"
	"time"
)

type Pool struct {
	RefreshRate time.Duration
	bars        []*ProgressBar
	isFinished  bool
}

func (p *Pool) Add(pbs ...*ProgressBar) {
	for _, bar := range pbs {
		bar.ManualUpdate = true
		bar.NotPrint = true
		bar.Start()
		p.bars = append(p.bars, bar)
	}
}

func (p *Pool) Start() (err error) {
	p.RefreshRate = DefaultRefreshRate
	quit, err := lockEcho()
	if err != nil {
		return
	}
	go p.writer(quit)
	return
}

func (p *Pool) writer(finish chan int) {
	var first = true
	var out string

	for {
		if first {
			first = false
		} else {
			out = fmt.Sprintf("\033[%dA", len(p.bars))
		}
		isFinished := true
		for _, bar := range p.bars {
			bar.Update()
			out += fmt.Sprintf("\r%s\n", bar.String())
			if !bar.isFinish {
				isFinished = false
			}
		}
		fmt.Print(out)
		if isFinished {
			p.isFinished = true
			finish <- 1
			return
		}
		time.Sleep(p.RefreshRate)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/minio/minio-go"
	"github.com/minio/minio-go/pkg/policy"
	"github.com/pborman/uuid"
	"github.com/rcrowley/go-metrics"
	"io"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"path"
	"strconv"
	"syscall"
)

var (
	rootDir    string
	listenAddr string
	s3         S3
)

func init() {
	flag.StringVar(&listenAddr, "listen", "", "address to listen to (default: :8080)")
	flag.StringVar(&s3.endpoint, "s3-endpoint", "", "s3 endpoint")
	flag.StringVar(&s3.region, "s3-region", "", "s3 region")
	flag.StringVar(&s3.bucket, "s3-bucket", "", "s3 bucket")
	flag.StringVar(&s3.accessKey, "s3-access-key", "", "s3 access key")
	flag.StringVar(&s3.secretKey, "s3-secret-key", "", "s3 secret key")
}

func main() {
	flag.Parse()

	if len(listenAddr) == 0 {
		listenAddr = os.Getenv("PUT_LISTEN")
	}

	if len(listenAddr) == 0 {
		listenAddr = ":8080"
	}

	if len(s3.endpoint) == 0 {
		s3.endpoint = os.Getenv("PUT_S3_ENDPOINT")
	}

	if len(s3.region) == 0 {
		s3.region = os.Getenv("PUT_S3_REGION")
	}

	if len(s3.bucket) == 0 {
		s3.bucket = os.Getenv("PUT_S3_BUCKET")
	}

	if len(s3.accessKey) == 0 {
		s3.accessKey = os.Getenv("PUT_S3_ACCESS_KEY")
	}

	if len(s3.secretKey) == 0 {
		s3.secretKey = os.Getenv("PUT_S3_SECRET_KEY")
	}
	l, err := listen(listenAddr)

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	defer l.Close()

	metrics := NewMetrics()

	mux := http.NewServeMux()
	mux.Handle("/", NewRequestHandler(&s3, metrics))
	mux.Handle("/health", NewHealthHandler(&s3))
	mux.Handle("/metrics", NewMetricsHandler(metrics, &s3))

	go http.Serve(l, mux)

	sig := make(chan os.Signal)
	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL)
	<-sig
}

func listen(addr string) (net.Listener, error) {
	if len(addr) > 0 && addr[0] != '/' {
		return net.Listen("tcp", addr)
	}
	return net.Listen("unix", addr)
}

type S3 struct {
	minio     *minio.Client
	url       *url.URL
	endpoint  string
	bucket    string
	region    string
	secretKey string
	accessKey string
}

func (s3 *S3) Put(name, contentType string, body io.Reader) (string, error) {
	client, err := s3.client()

	if err != nil {
		return "", err
	}
	meta := minio.PutObjectOptions{ContentType: contentType}

	if _, err := client.PutObject(s3.bucket, name, body, -1, meta); err != nil {
		return "", err
	}
	s3.url.Path = path.Join(s3.bucket, name)
	return s3.url.String(), nil
}

func (s3 *S3) HealthCheck() error {
	client, err := s3.client()

	if err != nil {
		return err
	}
	ok, err := client.BucketExists(s3.bucket)

	if err != nil {
		return err
	}

	if !ok {
		return fmt.Errorf("bucket unavailable: %s", s3.bucket)
	}
	return nil
}

func (s3 *S3) Metrics() (count int64, size int64, err error) {
	client, err := s3.client()

	if err != nil {
		return count, size, err
	}

	for obj := range client.ListObjects(s3.bucket, "", false, make(chan struct{})) {
		count += 1
		size += obj.Size
	}
	return count, size, err
}

func (s3 *S3) client() (client *minio.Client, err error) {
	if s3.minio != nil {
		return s3.minio, nil
	}
	s3.url, err = url.Parse(s3.endpoint)

	if err != nil {
		return nil, err
	}
	client, err = minio.New(s3.url.Host, s3.accessKey, s3.secretKey, s3.url.Scheme == "https")

	if err != nil {
		return nil, err
	}
	ok, err := client.BucketExists(s3.bucket)

	if err != nil {
		return nil, err
	}

	if !ok {
		if err := client.MakeBucket(s3.bucket, s3.region); err != nil {
			return nil, err
		}
	}

	if err := client.SetBucketPolicy(s3.bucket, policy.BucketPolicyReadOnly); err != nil {
		return nil, err
	}

	s3.minio = client
	return client, nil
}

type RequestHandler struct {
	s3      *S3
	metrics Metrics
}

func NewRequestHandler(s3 *S3, metrics Metrics) *RequestHandler {
	return &RequestHandler{
		s3:      s3,
		metrics: metrics,
	}
}

func (rh *RequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		rh.ServeIndex(w)
		return
	}
	rh.metrics.Total.Inc(1)

	if r.Method != "POST" && r.Method != "PUT" {
		rh.metrics.Invalid.Inc(1)
		w.Header().Set("Allow", "GET, POST, PUT")
		http.Error(w, "method not allowed", 405)
		return
	}
	objectUrl, err := rh.s3.Put(uuid.New(), r.Header.Get("Content-Type"), r.Body)

	if err != nil {
		rh.metrics.Failed.Inc(1)
		fmt.Println("could not store object:", err)
		http.Error(w, "internal server error", 500)
		return
	}

	if _, err := fmt.Fprintln(w, objectUrl); err != nil {
		rh.metrics.Failed.Inc(1)
		fmt.Println("could not respond with object url:", err)
		return
	}
	rh.metrics.Success.Inc(1)
}

func (rh *RequestHandler) ServeIndex(w http.ResponseWriter) {
	asset, err := indexHtml()

	if err != nil {
		fmt.Println("could not fetch index.html:", err)
		http.Error(w, "internal server error", 500)
	}
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	w.Header().Set("Content-Length", strconv.FormatInt(asset.info.Size(), 10))
	w.Header().Set("Last-Modified", asset.info.ModTime().Format(http.TimeFormat))

	if _, err = w.Write(asset.bytes); err != nil {
		fmt.Println("could not send index.html:", err)
	}
}

type HealthHandler struct {
	s3 *S3
}

type HealthPayload struct {
	Status string `json:"status"`
}

func NewHealthHandler(s3 *S3) *HealthHandler {
	return &HealthHandler{s3: s3}
}

func (hh *HealthHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	payload := &HealthPayload{Status: "ok"}

	if err := hh.s3.HealthCheck(); err != nil {
		payload.Status = fmt.Sprintf("critical: %s", err)
	}
	w.Header().Set("Content-Type", "application/json")

	if err := json.NewEncoder(w).Encode(payload); err != nil {
		fmt.Println("could not respond with health payload:", err)
	}
}

type Metrics struct {
	Total   metrics.Counter
	Invalid metrics.Counter
	Failed  metrics.Counter
	Success metrics.Counter
}

func NewMetrics() Metrics {
	return Metrics{
		Total:   metrics.NewCounter(),
		Invalid: metrics.NewCounter(),
		Failed:  metrics.NewCounter(),
		Success: metrics.NewCounter(),
	}
}

type MetricsHandler struct {
	metrics Metrics
	s3      *S3
}

type MetricsPayload struct {
	RequestsTotal   int64 `json:"requests_total"`
	RequestsInvalid int64 `json:"requests_invalid"`
	RequestsFailed  int64 `json:"requests_failed"`
	RequestsSuccess int64 `json:"requests_success"`
	S3Objects       int64 `json:"s3_objects"`
	S3Usage         int64 `json:"s3_usage"`
}

func NewMetricsHandler(metrics Metrics, s3 *S3) *MetricsHandler {
	return &MetricsHandler{metrics: metrics, s3: s3}
}

func (mh *MetricsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	s3Objects, s3Usage, _ := mh.s3.Metrics()

	payload := MetricsPayload{
		RequestsTotal:   mh.metrics.Total.Count(),
		RequestsInvalid: mh.metrics.Invalid.Count(),
		RequestsFailed:  mh.metrics.Failed.Count(),
		RequestsSuccess: mh.metrics.Success.Count(),
		S3Objects:       s3Objects,
		S3Usage:         s3Usage,
	}
	w.Header().Set("Content-Type", "application/json")

	if err := json.NewEncoder(w).Encode(payload); err != nil {
		fmt.Println("could not respond with metrics payload:", err)
	}
}
<commit_msg>do not setup bucket<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/minio/minio-go"
	"github.com/pborman/uuid"
	"github.com/rcrowley/go-metrics"
	"io"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"path"
	"strconv"
	"syscall"
)

var (
	listenAddr  string
	s3Endpoint  string
	s3Bucket    string
	s3AccessKey string
	s3SecretKey string
)

func init() {
	flag.StringVar(&listenAddr, "listen", "", "address to listen to (default: :8080)")
	flag.StringVar(&s3Endpoint, "s3-endpoint", "", "s3 endpoint")
	flag.StringVar(&s3Bucket, "s3-bucket", "", "s3 bucket")
	flag.StringVar(&s3AccessKey, "s3-access-key", "", "s3 access key")
	flag.StringVar(&s3SecretKey, "s3-secret-key", "", "s3 secret key")
}

func main() {
	flag.Parse()

	if len(listenAddr) == 0 {
		listenAddr = os.Getenv("PUT_LISTEN")
	}

	if len(listenAddr) == 0 {
		listenAddr = ":8080"
	}

	if len(s3Endpoint) == 0 {
		s3Endpoint = os.Getenv("PUT_S3_ENDPOINT")
	}

	if len(s3Bucket) == 0 {
		s3Bucket = os.Getenv("PUT_S3_BUCKET")
	}

	if len(s3AccessKey) == 0 {
		s3AccessKey = os.Getenv("PUT_S3_ACCESS_KEY")
	}

	if len(s3SecretKey) == 0 {
		s3SecretKey = os.Getenv("PUT_S3_SECRET_KEY")
	}
	url, err := url.Parse(s3Endpoint)

	if err != nil {
		fmt.Println("could not parse s3 endpoint:", err)
		os.Exit(1)
	}
	s3Client, err := minio.New(url.Host, s3AccessKey, s3SecretKey, url.Scheme == "https")

	if err != nil {
		fmt.Println("could not set up s3 client:", err)
		os.Exit(1)
	}
	l, err := listen(listenAddr)

	if err != nil {
		fmt.Println("could not set up listener:", err)
		os.Exit(1)
	}
	defer l.Close()

	s3 := NewS3(s3Client, s3Endpoint, s3Bucket)
	metrics := NewMetrics()

	mux := http.NewServeMux()
	mux.Handle("/", NewRequestHandler(s3, metrics))
	mux.Handle("/health", NewHealthHandler(s3))
	mux.Handle("/metrics", NewMetricsHandler(metrics, s3))

	go http.Serve(l, mux)

	sig := make(chan os.Signal)
	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL)
	<-sig
}

func listen(addr string) (net.Listener, error) {
	if len(addr) > 0 && addr[0] != '/' {
		return net.Listen("tcp", addr)
	}
	return net.Listen("unix", addr)
}

type S3 struct {
	client   *minio.Client
	endpoint string
	bucket   string
}

func NewS3(client *minio.Client, endpoint, bucket string) *S3 {
	return &S3{client: client, endpoint: endpoint, bucket: bucket}
}

func (s3 *S3) Put(name, contentType string, body io.Reader) (string, error) {
	url, err := url.Parse(s3.endpoint)

	if err != nil {
		return "", err
	}
	meta := minio.PutObjectOptions{ContentType: contentType}

	if _, err := s3.client.PutObject(s3.bucket, name, body, -1, meta); err != nil {
		return "", err
	}
	url.Path = path.Join(s3.bucket, name)
	return url.String(), nil
}

func (s3 *S3) HealthCheck() error {
	ok, err := s3.client.BucketExists(s3.bucket)

	if err != nil {
		return err
	}

	if !ok {
		return fmt.Errorf("bucket unavailable: %s", s3.bucket)
	}
	return nil
}

func (s3 *S3) Metrics() (count int64, size int64, err error) {
	for obj := range s3.client.ListObjects(s3.bucket, "", false, make(chan struct{})) {
		count += 1
		size += obj.Size
	}
	return count, size, err
}

type RequestHandler struct {
	s3      *S3
	metrics Metrics
}

func NewRequestHandler(s3 *S3, metrics Metrics) *RequestHandler {
	return &RequestHandler{
		s3:      s3,
		metrics: metrics,
	}
}

func (rh *RequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		rh.ServeIndex(w)
		return
	}
	rh.metrics.Total.Inc(1)

	if r.Method != "POST" && r.Method != "PUT" {
		rh.metrics.Invalid.Inc(1)
		w.Header().Set("Allow", "GET, POST, PUT")
		http.Error(w, "method not allowed", 405)
		return
	}
	objectUrl, err := rh.s3.Put(uuid.New(), r.Header.Get("Content-Type"), r.Body)

	if err != nil {
		rh.metrics.Failed.Inc(1)
		fmt.Println("could not store object:", err)
		http.Error(w, "internal server error", 500)
		return
	}

	if _, err := fmt.Fprintln(w, objectUrl); err != nil {
		rh.metrics.Failed.Inc(1)
		fmt.Println("could not respond with object url:", err)
		return
	}
	rh.metrics.Success.Inc(1)
}

func (rh *RequestHandler) ServeIndex(w http.ResponseWriter) {
	asset, err := indexHtml()

	if err != nil {
		fmt.Println("could not fetch index.html:", err)
		http.Error(w, "internal server error", 500)
	}
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	w.Header().Set("Content-Length", strconv.FormatInt(asset.info.Size(), 10))
	w.Header().Set("Last-Modified", asset.info.ModTime().Format(http.TimeFormat))

	if _, err = w.Write(asset.bytes); err != nil {
		fmt.Println("could not send index.html:", err)
	}
}

type HealthHandler struct {
	s3 *S3
}

type HealthPayload struct {
	Status string `json:"status"`
}

func NewHealthHandler(s3 *S3) *HealthHandler {
	return &HealthHandler{s3: s3}
}

func (hh *HealthHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	payload := &HealthPayload{Status: "ok"}

	if err := hh.s3.HealthCheck(); err != nil {
		payload.Status = fmt.Sprintf("critical: %s", err)
	}
	w.Header().Set("Content-Type", "application/json")

	if err := json.NewEncoder(w).Encode(payload); err != nil {
		fmt.Println("could not respond with health payload:", err)
	}
}

type Metrics struct {
	Total   metrics.Counter
	Invalid metrics.Counter
	Failed  metrics.Counter
	Success metrics.Counter
}

func NewMetrics() Metrics {
	return Metrics{
		Total:   metrics.NewCounter(),
		Invalid: metrics.NewCounter(),
		Failed:  metrics.NewCounter(),
		Success: metrics.NewCounter(),
	}
}

type MetricsHandler struct {
	metrics Metrics
	s3      *S3
}

type MetricsPayload struct {
	RequestsTotal   int64 `json:"requests_total"`
	RequestsInvalid int64 `json:"requests_invalid"`
	RequestsFailed  int64 `json:"requests_failed"`
	RequestsSuccess int64 `json:"requests_success"`
	S3Objects       int64 `json:"s3_objects"`
	S3Usage         int64 `json:"s3_usage"`
}

func NewMetricsHandler(metrics Metrics, s3 *S3) *MetricsHandler {
	return &MetricsHandler{metrics: metrics, s3: s3}
}

func (mh *MetricsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	s3Objects, s3Usage, _ := mh.s3.Metrics()

	payload := MetricsPayload{
		RequestsTotal:   mh.metrics.Total.Count(),
		RequestsInvalid: mh.metrics.Invalid.Count(),
		RequestsFailed:  mh.metrics.Failed.Count(),
		RequestsSuccess: mh.metrics.Success.Count(),
		S3Objects:       s3Objects,
		S3Usage:         s3Usage,
	}
	w.Header().Set("Content-Type", "application/json")

	if err := json.NewEncoder(w).Encode(payload); err != nil {
		fmt.Println("could not respond with metrics payload:", err)
	}
}
<|endoftext|>


<commit_before>package raft

type RaftState uint8

const (
	Follower RaftState = iota
	Candidate
	Leader
)

type Raft struct {
	// Configuration
	conf *Config

	// Current state
	state RaftState

	// stable is a StableStore implementation for durable state
	stable StableStore

	// logs is a LogStore implementation to keep our logs
	logs LogStore

	// Highest commited log entry
	commitIndex uint64

	// Last applied log to the FSM
	lastApplied uint64

	// If we are the leader, we have extra state
	leader *LeaderState
}
<commit_msg>Filling in raft<commit_after>package raft

import (
	"sync"
)

type RaftState uint8

const (
	Follower RaftState = iota
	Candidate
	Leader
)

type Raft struct {
	// Configuration
	conf *Config

	// Current state
	state RaftState

	// stable is a StableStore implementation for durable state
	stable StableStore

	// logs is a LogStore implementation to keep our logs
	logs LogStore

	// Highest commited log entry
	commitIndex uint64

	// Last applied log to the FSM
	lastApplied uint64

	// FSM is the state machine that can handle the logs
	fsm FSM

	// The transport layer we use
	trans Transport

	// If we are the leader, we have extra state
	leader *LeaderState

	// Shutdown channel to exit
	shutdownCh   chan struct{}
	shutdownLock sync.Mutex
}

// NewRaft is used to construct a new Raft node
func NewRaft(conf *Config, stable StableStore, logs LogStore, fsm FSM, trans Transport) (*Raft, error) {
	r := &Raft{
		conf:        conf,
		state:       Follower,
		stable:      stable,
		logs:        logs,
		commitIndex: 0,
		lastApplied: 0,
		fsm:         fsm,
		trans:       trans,
		shutdownCh:  make(chan struct{}),
	}

	go r.run()
	return r, nil
}

// run is a long running goroutine that runs the Raft FSM
func (r *Raft) run() {
	for {
		// Check if we are doing a shutdown
		select {
		case <-r.shutdownCh:
			return
		default:
		}

		// Enter into a sub-FSM
		switch r.state {
		case Follower:
			r.runFollower()
		case Candidate:
			r.runCandidate()
		case Leader:
			r.runLeader()
		}
	}
}

// runFollower runs the FSM for a follower
func (r *Raft) runFollower() {
	for {
		select {
		case <-r.shutdownCh:
			return
		}
	}
}

// runCandidate runs the FSM for a candidate
func (r *Raft) runCandidate() {
	for {
		select {
		case <-r.shutdownCh:
			return
		}
	}
}

// runLeader runs the FSM for a leader
func (r *Raft) runLeader() {
	for {
		select {
		case <-r.shutdownCh:
			return
		}
	}
}

// Shutdown is used to stop the Raft background routines.
// This is not a graceful operation.
func (r *Raft) Shutdown() {
	r.shutdownLock.Lock()
	defer r.shutdownLock.Lock()

	if r.shutdownCh != nil {
		close(r.shutdownCh)
		r.shutdownCh = nil
	}
}
<|endoftext|>