<commit_before>package main

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"

	"golang.org/x/net/html"
)

func getAttr(t html.Token, name string) (ok bool, href string) {
	for _, a := range t.Attr {
		if a.Key == name {
			href = a.Val
			ok = true
		}
	}

	return
}

type UrlResponse struct {
	from string
	url  string
	code int
	err  error
}

type NewUrl struct {
	from string
	url  string
}

var skipUrls = map[string]int{
	"https://build.opensuse.org/project/show/Virtualization:containers": 1,
	"http://10.0.0.2:5000":                                              1,
	"https://linux.oracle.com":                                          1,
	"http://supervisord.org/":                                           1,
	"http://goo.gl/HSz8UT":                                              1,
}

func crawl(chWork chan NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	for true {
		new := <-chWork
		crawlOne(new, ch, chFinished)
	}
}

// Extract all http** links from a given webpage
func crawlOne(req NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	base, err := url.Parse(req.url)
	reply := UrlResponse{
		url:  req.url,
		from: req.from,
		code: 999,
	}
	if _, ok := skipUrls[req.url]; ok {
		fmt.Printf("Skipping: %s\n", req.url)
		reply.code = 299
		chFinished <- reply
		return
	}
	fmt.Printf("Crawling: %s\n", req.url)
	if err != nil {
		fmt.Println("ERROR: failed to Parse \"" + req.url + "\"")
		reply.err = err
		chFinished <- reply
		return
	}
	switch base.Scheme {
	case "mailto", "irc":
		reply.err = fmt.Errorf("%s on page %s", base.Scheme, req.from)
		reply.code = 900
		chFinished <- reply
		return
	}
	resp, err := http.Get(req.url)
	if err != nil {
		fmt.Println("ERROR: Failed to crawl \"" + req.url + "\"  " + err.Error())
		reply.err = err
		chFinished <- reply
		return
	}
	defer func() {
		// Notify that we're done after this function
		reply.code = resp.StatusCode
		chFinished <- reply
	}()

	loc, err := resp.Location()
	if err == nil && req.url != loc.String() {
		fmt.Printf("\t crawled \"%s\"", req.url)
		fmt.Printf("\t\t to \"%s\"", loc)
	}

	b := resp.Body
	defer b.Close() // close Body when the function returns

	// only parse if this page is on the original site
	// if we moved this check back to the main loop, we'd parse more sites
	if !strings.HasPrefix(req.url, seedUrl) {
		return
	}

	// don't parse js files..
	if strings.HasSuffix(req.url, ".js") || strings.HasSuffix(req.url, ".js") {
		return
	}

	// TODO: it seems to sucessfully parse non-html (like js/css)
	z := html.NewTokenizer(b)

	for {
		tt := z.Next()

		switch tt {
		case html.ErrorToken:
			// End of the document, we're done
			return
		case html.StartTagToken, html.SelfClosingTagToken:
			t := z.Token()
			var ok bool
			var newUrl string

			switch t.Data {
			case "base":
				// use the actual baseUrl set in the html file
				ok, baseUrl := getAttr(t, "href")
				if !ok {
					continue
				}

				newBase, err := url.Parse(baseUrl)
				if err != nil {
					continue
				}
				base = base.ResolveReference(newBase)
				continue
			case "a", "link":
				ok, newUrl = getAttr(t, "href")
				if !ok {
					continue
				}
			case "img", "script":
				ok, newUrl = getAttr(t, "src")
				if !ok {
					continue
				}
			default:
				continue
			}

			u, e := url.Parse(newUrl)
			if e != nil {
				fmt.Println("ERROR: failed to Parse \"" + newUrl + "\"")
				continue
			}
			new := NewUrl{
				from: req.url,
				url:  base.ResolveReference(u).String(),
			}
			ch <- new
		}
	}
}

var seedUrl string

type FoundUrls struct {
	response   int
	usageCount int
	err        error
	from       map[string]int
}

func main() {
	if len(os.Args[1:]) == 0 {
		seedUrl = os.Getenv("CHECKURL")
		if seedUrl == "" {
			fmt.Println("Please specify a URL to check")
			os.Exit(-1)
		}
	} else {
		seedUrl = os.Args[1]
	}

	// Channels
	chUrls := make(chan NewUrl, 1000)
	chWork := make(chan NewUrl, 3000)
	chFinished := make(chan UrlResponse)

	var foundUrls = make(map[string]FoundUrls)

	for w := 1; w <= 20; w++ {
		go crawl(chWork, chUrls, chFinished)
	}

	new := NewUrl{
		from: "",
		url:  seedUrl,
	}
	chUrls <- new

	// Subscribe to both channels
	count := 0
	for len(chUrls) > 0 || count > 0 {
		select {
		case foundUrl := <-chUrls:
			// don't need to check err - its already been checked before its put in the chUrls que
			u, _ := url.Parse(foundUrl.url)
			// TODO: need a different pipeline for ensuring anchor fragments exist
			// TODO: consider only removing the query/fragment for docs urls
			u.RawQuery = ""
			u.Fragment = ""
			resourceUrl := u.String()

			f, ok := foundUrls[resourceUrl]
			if !ok {
				count++
				f.usageCount = 0
				f.response = 0
				f.from = make(map[string]int)
				f.from[foundUrl.from] = 1
				chWork <- NewUrl{
					from: foundUrl.from,
					url:  resourceUrl,
				}
			}
			f.usageCount++
			f.from[foundUrl.from]++
			foundUrls[resourceUrl] = f

		case ret := <-chFinished:
			count--
			info := foundUrls[ret.url]
			//info.from[ret.from]++
			info.response = ret.code
			info.err = ret.err
			foundUrls[ret.url] = info
		}
		// fmt.Printf("(w%d, u%d, c%d)", len(chWork), len(chUrls), count)
	}

	// We're done! Print the results...
	fmt.Println("\nDone.")
	summary := make(map[int]int)
	for url, info := range foundUrls {
		summary[info.response]++
		if info.response != 200 && info.response != 900 {
			fmt.Printf(" - %d (%d): %s\n", info.response, info.usageCount, url)
			if info.err != nil {
				fmt.Printf("\t%s\n", info.err)
			}
			for from, count := range info.from {
				fmt.Printf("\t\t%d times from %s\n", count, from)
			}
		}
	}
	fmt.Println("\nFound", len(foundUrls), "unique urls\n")
	errorCount := 0
	explain := map[int]string{
		900: "mailto or irc",
		299: "url skipped",
		200: "ok",
		404: "forbidden",
		403: "forbidden",
	}
	for code, count := range summary {
		reason, ok := explain[code]
		if !ok {
			// TODO: I presume go has a text error mapping
			reason = "HTTP code"
		}
		fmt.Printf("\t\tStatus %d : %d - %s\n", code, count, reason)
		if code != 200 && code != 299 && code != 900 {
			errorCount += count
		}
	}
	fmt.Println("\nError Count:", errorCount)

	close(chUrls)

	// return the number of 404's to show that there are things to be fixed
	os.Exit(errorCount)
}
<commit_msg>Add some more external links to skip<commit_after>package main

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"

	"golang.org/x/net/html"
)

func getAttr(t html.Token, name string) (ok bool, href string) {
	for _, a := range t.Attr {
		if a.Key == name {
			href = a.Val
			ok = true
		}
	}

	return
}

type UrlResponse struct {
	from string
	url  string
	code int
	err  error
}

type NewUrl struct {
	from string
	url  string
}

var skipUrls = map[string]int{
	"https://build.opensuse.org/project/show/Virtualization:containers": 1,
	"https://build.opensuse.org/":                                       1,
	"http://10.0.0.2:5000":                                              1,
	"https://linux.oracle.com":                                          1,
	"http://supervisord.org/":                                           1,
	"http://goo.gl/HSz8UT":                                              1,
	"https://www.linkedin.com/company/docker":                           1,
}

func crawl(chWork chan NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	for true {
		new := <-chWork
		crawlOne(new, ch, chFinished)
	}
}

// Extract all http** links from a given webpage
func crawlOne(req NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	base, err := url.Parse(req.url)
	reply := UrlResponse{
		url:  req.url,
		from: req.from,
		code: 999,
	}
	if _, ok := skipUrls[req.url]; ok {
		fmt.Printf("Skipping: %s\n", req.url)
		reply.code = 299
		chFinished <- reply
		return
	}
	fmt.Printf("Crawling: %s\n", req.url)
	if err != nil {
		fmt.Println("ERROR: failed to Parse \"" + req.url + "\"")
		reply.err = err
		chFinished <- reply
		return
	}
	switch base.Scheme {
	case "mailto", "irc":
		reply.err = fmt.Errorf("%s on page %s", base.Scheme, req.from)
		reply.code = 900
		chFinished <- reply
		return
	}
	resp, err := http.Get(req.url)
	if err != nil {
		fmt.Println("ERROR: Failed to crawl \"" + req.url + "\"  " + err.Error())
		reply.err = err
		chFinished <- reply
		return
	}
	defer func() {
		// Notify that we're done after this function
		reply.code = resp.StatusCode
		chFinished <- reply
	}()

	loc, err := resp.Location()
	if err == nil && req.url != loc.String() {
		fmt.Printf("\t crawled \"%s\"", req.url)
		fmt.Printf("\t\t to \"%s\"", loc)
	}

	b := resp.Body
	defer b.Close() // close Body when the function returns

	// only parse if this page is on the original site
	// if we moved this check back to the main loop, we'd parse more sites
	if !strings.HasPrefix(req.url, seedUrl) {
		return
	}

	// don't parse js files..
	if strings.HasSuffix(req.url, ".js") || strings.HasSuffix(req.url, ".js") {
		return
	}

	// TODO: it seems to sucessfully parse non-html (like js/css)
	z := html.NewTokenizer(b)

	for {
		tt := z.Next()

		switch tt {
		case html.ErrorToken:
			// End of the document, we're done
			return
		case html.StartTagToken, html.SelfClosingTagToken:
			t := z.Token()
			var ok bool
			var newUrl string

			switch t.Data {
			case "base":
				// use the actual baseUrl set in the html file
				ok, baseUrl := getAttr(t, "href")
				if !ok {
					continue
				}

				newBase, err := url.Parse(baseUrl)
				if err != nil {
					continue
				}
				base = base.ResolveReference(newBase)
				continue
			case "a", "link":
				ok, newUrl = getAttr(t, "href")
				if !ok {
					continue
				}
			case "img", "script":
				ok, newUrl = getAttr(t, "src")
				if !ok {
					continue
				}
			default:
				continue
			}

			u, e := url.Parse(newUrl)
			if e != nil {
				fmt.Println("ERROR: failed to Parse \"" + newUrl + "\"")
				continue
			}
			new := NewUrl{
				from: req.url,
				url:  base.ResolveReference(u).String(),
			}
			ch <- new
		}
	}
}

var seedUrl string

type FoundUrls struct {
	response   int
	usageCount int
	err        error
	from       map[string]int
}

func main() {
	if len(os.Args[1:]) == 0 {
		seedUrl = os.Getenv("CHECKURL")
		if seedUrl == "" {
			fmt.Println("Please specify a URL to check")
			os.Exit(-1)
		}
	} else {
		seedUrl = os.Args[1]
	}

	// Channels
	chUrls := make(chan NewUrl, 1000)
	chWork := make(chan NewUrl, 3000)
	chFinished := make(chan UrlResponse)

	var foundUrls = make(map[string]FoundUrls)

	for w := 1; w <= 20; w++ {
		go crawl(chWork, chUrls, chFinished)
	}

	new := NewUrl{
		from: "",
		url:  seedUrl,
	}
	chUrls <- new

	// Subscribe to both channels
	count := 0
	for len(chUrls) > 0 || count > 0 {
		select {
		case foundUrl := <-chUrls:
			// don't need to check err - its already been checked before its put in the chUrls que
			u, _ := url.Parse(foundUrl.url)
			// TODO: need a different pipeline for ensuring anchor fragments exist
			// TODO: consider only removing the query/fragment for docs urls
			u.RawQuery = ""
			u.Fragment = ""
			resourceUrl := u.String()

			f, ok := foundUrls[resourceUrl]
			if !ok {
				count++
				f.usageCount = 0
				f.response = 0
				f.from = make(map[string]int)
				f.from[foundUrl.from] = 1
				chWork <- NewUrl{
					from: foundUrl.from,
					url:  resourceUrl,
				}
			}
			f.usageCount++
			f.from[foundUrl.from]++
			foundUrls[resourceUrl] = f

		case ret := <-chFinished:
			count--
			info := foundUrls[ret.url]
			//info.from[ret.from]++
			info.response = ret.code
			info.err = ret.err
			foundUrls[ret.url] = info
		}
		// fmt.Printf("(w%d, u%d, c%d)", len(chWork), len(chUrls), count)
	}

	// We're done! Print the results...
	fmt.Println("\nDone.")
	summary := make(map[int]int)
	for url, info := range foundUrls {
		summary[info.response]++
		if info.response != 200 && info.response != 900 {
			fmt.Printf(" - %d (%d): %s\n", info.response, info.usageCount, url)
			if info.err != nil {
				fmt.Printf("\t%s\n", info.err)
			}
			for from, count := range info.from {
				fmt.Printf("\t\t%d times from %s\n", count, from)
			}
		}
	}
	fmt.Println("\nFound", len(foundUrls), "unique urls\n")
	errorCount := 0
	explain := map[int]string{
		900: "mailto or irc",
		299: "url skipped",
		200: "ok",
		404: "forbidden",
		403: "forbidden",
	}
	for code, count := range summary {
		reason, ok := explain[code]
		if !ok {
			// TODO: I presume go has a text error mapping
			reason = "HTTP code"
		}
		fmt.Printf("\t\tStatus %d : %d - %s\n", code, count, reason)
		if code != 200 && code != 299 && code != 900 {
			errorCount += count
		}
	}
	fmt.Println("\nError Count:", errorCount)

	close(chUrls)

	// return the number of 404's to show that there are things to be fixed
	os.Exit(errorCount)
}
<|endoftext|>


<commit_before>package logrus_papertrail

import (
	"fmt"
	"net"
	"os"
	"time"

	"github.com/Sirupsen/logrus"
)

const (
	format = "Jan 2 15:04:05"
)

// PapertrailHook to send logs to a logging service compatible with the Papertrail API.
type PapertrailHook struct {
	Host    string
	Port    int
	AppName string
	UDPConn net.Conn
}

// NewPapertrailHook creates a hook to be added to an instance of logger.
func NewPapertrailHook(host string, port int, appName string) (*PapertrailHook, error) {
	conn, err := net.Dial("udp", fmt.Sprintf("%s:%d", host, port))
	return &PapertrailHook{host, port, appName, conn}, err
}

// Fire is called when a log event is fired.
func (hook *PapertrailHook) Fire(entry *logrus.Entry) error {
	defer hook.UDPConn.Close()
	date := time.Now().Format(format)
	payload := fmt.Sprintf("<22> %s %s: [%s] %s", date, hook.AppName, entry.Data["level"], entry.Message)

	line, err := entry.String()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to read entry, %v", err)
		return err
	}

	payload := fmt.Sprintf("<22> %s %s: [%s] %s", date, hook.AppName, entry.Data["level"], line)

	_, err = hook.UDPConn.Write([]byte(payload))
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to send log line to Papertrail via UDP, %v", err)
		return err
	}

	return nil
}

// Levels returns the available logging levels.
func (hook *PapertrailHook) Levels() []logrus.Level {
	return []logrus.Level{
		logrus.Panic,
		logrus.Fatal,
		logrus.Error,
		logrus.Warn,
		logrus.Info,
	}
}
<commit_msg>Include number of bytes written as debugging output.<commit_after>package logrus_papertrail

import (
	"fmt"
	"net"
	"os"
	"time"

	"github.com/Sirupsen/logrus"
)

const (
	format = "Jan 2 15:04:05"
)

// PapertrailHook to send logs to a logging service compatible with the Papertrail API.
type PapertrailHook struct {
	Host    string
	Port    int
	AppName string
	UDPConn net.Conn
}

// NewPapertrailHook creates a hook to be added to an instance of logger.
func NewPapertrailHook(host string, port int, appName string) (*PapertrailHook, error) {
	conn, err := net.Dial("udp", fmt.Sprintf("%s:%d", host, port))
	return &PapertrailHook{host, port, appName, conn}, err
}

// Fire is called when a log event is fired.
func (hook *PapertrailHook) Fire(entry *logrus.Entry) error {
	defer hook.UDPConn.Close()
	date := time.Now().Format(format)
	payload := fmt.Sprintf("<22> %s %s: [%s] %s", date, hook.AppName, entry.Data["level"], entry.Message)

	line, err := entry.String()
	bytesWritten, err := hook.UDPConn.Write([]byte(payload))
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to read entry, %v", err)
		return err
	}

	payload := fmt.Sprintf("<22> %s %s: [%s] %s", date, hook.AppName, entry.Data["level"], line)

	_, err = hook.UDPConn.Write([]byte(payload))
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to send log line to Papertrail via UDP, %v", err)
		fmt.Fprintf(os.Stderr, "Unable to send log line to Papertrail via UDP. Wrote %d bytes before error: %v", bytesWritten, err)
		return err
	}

	return nil
}

// Levels returns the available logging levels.
func (hook *PapertrailHook) Levels() []logrus.Level {
	return []logrus.Level{
		logrus.Panic,
		logrus.Fatal,
		logrus.Error,
		logrus.Warn,
		logrus.Info,
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Daniel Theophanes (kardianos@gmail.com)

package cli

import (
	"flag"

	"github.com/cockroachdb/cockroach/server"
)

// initFlags sets the server.Context values to flag values.
// Keep in sync with "server/context.go". Values in Context should be
// settable here.
func initFlags(ctx *server.Context) {
	// Server flags.
	flag.StringVar(&ctx.Addr, "addr", ctx.Addr, "when run as the server the host:port to bind for "+
		"HTTP/RPC traffic; when run as the client the address for connection to the cockroach cluster.")

	flag.BoolVar(&ctx.Insecure, "insecure", ctx.Insecure, "run over plain HTTP. WARNING: "+
		"this is strongly discouraged.")

	flag.StringVar(&ctx.Certs, "certs", ctx.Certs, "directory containing RSA key and x509 certs.")

	flag.StringVar(&ctx.Stores, "stores", ctx.Stores, "specify a comma-separated list of stores, "+
		"specified by a colon-separated list of device attributes followed by '=' and "+
		"either a filepath for a persistent store or an integer size in bytes for an "+
		"in-memory store. Device attributes typically include whether the store is "+
		"flash (ssd), spinny disk (hdd), fusion-io (fio), in-memory (mem); device "+
		"attributes might also include speeds and other specs (7200rpm, 200kiops, etc.). "+
		"For example, -store=hdd:7200rpm=/mnt/hda1,ssd=/mnt/ssd01,ssd=/mnt/ssd02,mem=1073741824.")

	flag.StringVar(&ctx.Attrs, "attrs", ctx.Attrs, "specify an ordered, colon-separated list of node "+
		"attributes. Attributes are arbitrary strings specifying topography or "+
		"machine capabilities. Topography might include datacenter designation "+
		"(e.g. \"us-west-1a\", \"us-west-1b\", \"us-east-1c\"). Machine capabilities "+
		"might include specialized hardware or number of cores (e.g. \"gpu\", "+
		"\"x16c\"). "+
		"The relative geographic proximity of two nodes is inferred from the "+
		"common prefix of the attributes list, so topographic attributes should be "+
		"specified first and in the same order for all nodes. "+
		"For example: -attrs=us-west-1b,gpu.")

	flag.DurationVar(&ctx.MaxOffset, "max-offset", ctx.MaxOffset, "specify "+
		"the maximum clock offset for the cluster. Clock offset is measured on all "+
		"node-to-node links and if any node notices it has clock offset in excess "+
		"of -max-offset, it will commit suicide. Setting this value too high may "+
		"decrease transaction performance in the presence of contention.")

	// Gossip flags.
	flag.StringVar(&ctx.GossipBootstrap, "gossip", ctx.GossipBootstrap, "specify a "+
		"comma-separated list of gossip addresses or resolvers for gossip bootstrap. "+
		"Each item in the list has an optional type: [type=]<address>. "+
		"Unspecified type means ip address or dns. Type can also be a load balancer (\"lb\"), "+
		"a unix socket (\"unix\") or, for single-node systems, \"self\".")

	flag.DurationVar(&ctx.GossipInterval, "gossip-interval", ctx.GossipInterval,
		"approximate interval (time.Duration) for gossiping new information to peers.")

	// KV flags.

	flag.BoolVar(&ctx.Linearizable, "linearizable", ctx.Linearizable, "enables linearizable behaviour "+
		"of operations on this node by making sure that no commit timestamp is reported "+
		"back to the client until all other node clocks have necessarily passed it.")

	// Engine flags.

	flag.Int64Var(&ctx.CacheSize, "cache-size", ctx.CacheSize, "total size in bytes for "+
		"caches, shared evenly if there are multiple storage devices.")

	flag.DurationVar(&ctx.ScanInterval, "scan-interval", ctx.ScanInterval, "specify "+
		"--scan_interval to adjust the target for the duration of a single scan "+
		"through a store's ranges. The scan is slowed as necessary to approximately"+
		"achieve this duration.")
}

func init() {
	initFlags(Context)
}
<commit_msg>Clarify -certs flag<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Daniel Theophanes (kardianos@gmail.com)

package cli

import (
	"flag"

	"github.com/cockroachdb/cockroach/server"
)

// initFlags sets the server.Context values to flag values.
// Keep in sync with "server/context.go". Values in Context should be
// settable here.
func initFlags(ctx *server.Context) {
	// Server flags.
	flag.StringVar(&ctx.Addr, "addr", ctx.Addr, "when run as the server the host:port to bind for "+
		"HTTP/RPC traffic; when run as the client the address for connection to the cockroach cluster.")

	flag.BoolVar(&ctx.Insecure, "insecure", ctx.Insecure, "run over plain HTTP. WARNING: "+
		"this is strongly discouraged.")

	flag.StringVar(&ctx.Certs, "certs", ctx.Certs, "directory containing RSA key and x509 certs. "+
		"This flag is required if -insecure=false.")

	flag.StringVar(&ctx.Stores, "stores", ctx.Stores, "specify a comma-separated list of stores, "+
		"specified by a colon-separated list of device attributes followed by '=' and "+
		"either a filepath for a persistent store or an integer size in bytes for an "+
		"in-memory store. Device attributes typically include whether the store is "+
		"flash (ssd), spinny disk (hdd), fusion-io (fio), in-memory (mem); device "+
		"attributes might also include speeds and other specs (7200rpm, 200kiops, etc.). "+
		"For example, -store=hdd:7200rpm=/mnt/hda1,ssd=/mnt/ssd01,ssd=/mnt/ssd02,mem=1073741824.")

	flag.StringVar(&ctx.Attrs, "attrs", ctx.Attrs, "specify an ordered, colon-separated list of node "+
		"attributes. Attributes are arbitrary strings specifying topography or "+
		"machine capabilities. Topography might include datacenter designation "+
		"(e.g. \"us-west-1a\", \"us-west-1b\", \"us-east-1c\"). Machine capabilities "+
		"might include specialized hardware or number of cores (e.g. \"gpu\", "+
		"\"x16c\"). "+
		"The relative geographic proximity of two nodes is inferred from the "+
		"common prefix of the attributes list, so topographic attributes should be "+
		"specified first and in the same order for all nodes. "+
		"For example: -attrs=us-west-1b,gpu.")

	flag.DurationVar(&ctx.MaxOffset, "max-offset", ctx.MaxOffset, "specify "+
		"the maximum clock offset for the cluster. Clock offset is measured on all "+
		"node-to-node links and if any node notices it has clock offset in excess "+
		"of -max-offset, it will commit suicide. Setting this value too high may "+
		"decrease transaction performance in the presence of contention.")

	// Gossip flags.
	flag.StringVar(&ctx.GossipBootstrap, "gossip", ctx.GossipBootstrap, "specify a "+
		"comma-separated list of gossip addresses or resolvers for gossip bootstrap. "+
		"Each item in the list has an optional type: [type=]<address>. "+
		"Unspecified type means ip address or dns. Type can also be a load balancer (\"lb\"), "+
		"a unix socket (\"unix\") or, for single-node systems, \"self\".")

	flag.DurationVar(&ctx.GossipInterval, "gossip-interval", ctx.GossipInterval,
		"approximate interval (time.Duration) for gossiping new information to peers.")

	// KV flags.

	flag.BoolVar(&ctx.Linearizable, "linearizable", ctx.Linearizable, "enables linearizable behaviour "+
		"of operations on this node by making sure that no commit timestamp is reported "+
		"back to the client until all other node clocks have necessarily passed it.")

	// Engine flags.

	flag.Int64Var(&ctx.CacheSize, "cache-size", ctx.CacheSize, "total size in bytes for "+
		"caches, shared evenly if there are multiple storage devices.")

	flag.DurationVar(&ctx.ScanInterval, "scan-interval", ctx.ScanInterval, "specify "+
		"--scan_interval to adjust the target for the duration of a single scan "+
		"through a store's ranges. The scan is slowed as necessary to approximately"+
		"achieve this duration.")
}

func init() {
	initFlags(Context)
}
<|endoftext|>


<commit_before>package main

import (
	"time"
	"fmt"
	"golang.org/x/tour/tree"
)

func say(s string) {
	for i := 0; i < 5; i++ {
		time.Sleep(100 * time.Millisecond)
		fmt.Println(s)
	}
}

// Note: Channels can be used as function parameters / arguments
func sum(s []int, c chan int) {
	sum := 0
	for _, v := range s {
		sum += v
	}
	c <- sum // send sum to c
}

func channelExamples() {
	s := []int{7, 2, 8, -9, 4, 0}

	c := make(chan int)
	// Split the work between two goroutines - slicing the array to achieve the portioning of the work
	go sum(s[:len(s)/2], c)
	go sum(s[len(s)/2:], c)
	x, y := <-c, <-c // receive from c

	fmt.Println(x, y, x+y)
}

func bufferedChannelExamples() {
	ch := make(chan int, 2)
	ch <- 1
	ch <- 2
	// ch <- 3 - would cause the code to block here (fatal error: all goroutines are asleep - deadlock!)
	fmt.Println(<-ch)
	fmt.Println(<-ch)
}

func fibonacciChannels(n int, c chan int) {
	x, y := 0, 1
	for i := 0; i < n; i++ {
		c <- x
		x, y = y, x+y
	}
	// Note: Only the sender should close a channel, never the receiver. Sending on a closed channel will cause a panic.
	// Note: Channels aren't like files; you don't usually need to close them. Closing is only necessary when the receiver must be told there are no more values coming, such as to terminate a range loop.
	close(c)
}

func controllingChannelsExample() {
	c := make(chan int, 10)
	go fibonacciChannels(cap(c), c) // Also works if it is not run in a separate goroutine
	// The loop for i := range c receives values from the channel repeatedly until it is closed.
	for i := range c {
		fmt.Println(i)
	}
}

func fibonacciSelect(c, quit chan int) {
	x, y := 0, 1
	for {
		select {
		case c <- x:
			x, y = y, x+y
		case <-quit:
			fmt.Println("quit")
			return
		}
	}
}

func selectExample() {
	c := make(chan int)
	quit := make(chan int)
	go func() {
		for i := 0; i < 10; i++ {
			fmt.Println(<-c)
		}
		quit <- 0
	}()
	fibonacciSelect(c, quit)

	// The default case in a select is run if no other case is ready.
	// Use a default case to try a send or receive without blocking:
	tick := time.Tick(100 * time.Millisecond)
	boom := time.After(500 * time.Millisecond)
	for {
		select {
		case <-tick:
			fmt.Println("tick.")
		case <-boom:
			fmt.Println("BOOM!")
			return // Breaks out of the for-loop
		default:
			fmt.Println("    .")
			time.Sleep(50 * time.Millisecond)
		}
	}
}

// Walk walks the tree t sending all values
// from the tree to the channel ch.
func Walk(t *tree.Tree, ch chan int) {
	fmt.Println(t.String())
	walkSubTree(t, ch)
	// Need to close the channel here
	close(ch)
}

func walkSubTree(t *tree.Tree, ch chan int, ) {
	if t == nil {
		return
	}
	walkSubTree(t.Left, ch)
	ch <- t.Value
	walkSubTree(t.Right, ch)
}

func WalkQuit(t *tree.Tree, ch, quit chan int) {
	walkSubTreeQuit(t, ch, quit)
	close(ch)
}

func walkSubTreeQuit(t *tree.Tree, ch, quit chan int) {
	if t == nil {
		return
	}
	walkSubTreeQuit(t.Left, ch, quit)
	select {
	case ch <- t.Value:
		// Value successfully sent.
	case <-quit:
		return
	}
	walkSubTreeQuit(t.Right, ch, quit)
}

// Same determines whether the trees
// t1 and t2 contain the same values.
func Same(t1, t2 *tree.Tree) bool {
	w1, w2 := make(chan int), make(chan int)

	go Walk(t1, w1)
	go Walk(t2, w2)

	for {
		v1, ok1 := <-w1
		v2, ok2 := <-w2
		if !ok1 || !ok2 {
			return ok1 == ok2
		}
		if v1 != v2 {
			return false
		}
	}
}

func printContentsOfChannel(ch chan int) {
	for i := range ch {
		fmt.Println(i)
	}
}
func channelExercise() {
	ch := make(chan int, 10)
	fmt.Println("Walk tree(1)")
	go Walk(tree.New(1), ch)
	printContentsOfChannel(ch)
	fmt.Println("Walk tree(2)")
	ch = make(chan int, 10)
	go Walk(tree.New(2), ch)
	printContentsOfChannel(ch)
	fmt.Println("Walk tree quit(3)")
	ch = make(chan int, 10)
	quit := make(chan int)
	defer close(quit)
	go WalkQuit(tree.New(3), ch, quit)
	printContentsOfChannel(ch)
	fmt.Print("tree.New(1) == tree.New(1): ")
	if Same(tree.New(1), tree.New(1)) {
		fmt.Println("PASSED")
	} else {
		fmt.Println("FAILED")
	}

	fmt.Print("tree.New(1) != tree.New(2): ")
	if !Same(tree.New(1), tree.New(2)) {
		fmt.Println("PASSED")
	} else {
		fmt.Println("FAILED")
	}
}

func main() {
	// GOROUTINE
	// A goroutine is a lightweight thread managed by the Go runtime.
	// go <func> starts a new goroutine running
	fmt.Println("calling the goroutine statements")
	go say("world")
	say("hello")

	// Goroutines run in the same address space, so access to shared memory must be synchronized.
	// The sync package provides useful primitives, although you won't need them much in Go as there are other primitives...

	// CHANNELS
	// Channels are a typed conduit through which you can send and receive values with the channel operator, <-. (The data flows in the direction of the arrow.)
	// e.g.
	// ch <- v    // Send v to channel ch.
	// v := <-ch  // Receive from ch, and assign value to v.

	// Like maps and slices, channels must be created before use:
	// ch := make(chan int)
	// NOTE: By default, sends and receives block until the other side is ready. This allows goroutines to synchronize without explicit locks or condition variables.
	fmt.Println("calling the channel statements")
	channelExamples()

	// BUFFERED CHANNELS
	// Channels can be buffered. Provide the buffer length as the second argument to make to initialize a buffered channel:
	// ch := make(chan int, 100)
	// Sends to a buffered channel block only when the buffer is full. Receives block when the buffer is empty.
	fmt.Println("calling the buffered channel statements")
	bufferedChannelExamples()

	// CONTROLLING CHANNELS
	// A sender can close a channel to indicate that no more values will be sent. Receivers can test whether a channel has been closed by assigning a second parameter to the receive expression:
	// v, ok := <-ch
	// ok is false if there are no more values to receive and the channel is closed.
	fmt.Println("calling the controlling channels statements")
	controllingChannelsExample()

	// SELECT
	// The select statement lets a goroutine wait on multiple communication operations.
	// A select blocks until one of its cases can run, then it executes that case. It chooses one at random if multiple are ready.
	fmt.Println("calling the select statements")
	selectExample()

	// CHANNEL EXERCISE
	// Requires "go get golang.org/x/tour/tree" to be run first
	fmt.Println("calling the channel exercise")
	channelExercise()
}
<commit_msg>Adding lessons and exercise for locking<commit_after>package main

import (
	"time"
	"fmt"
	"golang.org/x/tour/tree"
	"sync"
	"errors"
)

func say(s string) {
	for i := 0; i < 5; i++ {
		time.Sleep(100 * time.Millisecond)
		fmt.Println(s)
	}
}

// Note: Channels can be used as function parameters / arguments
func sum(s []int, c chan int) {
	sum := 0
	for _, v := range s {
		sum += v
	}
	c <- sum // send sum to c
}

func channelExamples() {
	s := []int{7, 2, 8, -9, 4, 0}

	c := make(chan int)
	// Split the work between two goroutines - slicing the array to achieve the portioning of the work
	go sum(s[:len(s)/2], c)
	go sum(s[len(s)/2:], c)
	x, y := <-c, <-c // receive from c

	fmt.Println(x, y, x+y)
}

func bufferedChannelExamples() {
	ch := make(chan int, 2)
	ch <- 1
	ch <- 2
	// ch <- 3 - would cause the code to block here (fatal error: all goroutines are asleep - deadlock!)
	fmt.Println(<-ch)
	fmt.Println(<-ch)
}

func fibonacciChannels(n int, c chan int) {
	x, y := 0, 1
	for i := 0; i < n; i++ {
		c <- x
		x, y = y, x+y
	}
	// Note: Only the sender should close a channel, never the receiver. Sending on a closed channel will cause a panic.
	// Note: Channels aren't like files; you don't usually need to close them. Closing is only necessary when the receiver must be told there are no more values coming, such as to terminate a range loop.
	close(c)
}

func controllingChannelsExample() {
	c := make(chan int, 10)
	go fibonacciChannels(cap(c), c) // Also works if it is not run in a separate goroutine
	// The loop for i := range c receives values from the channel repeatedly until it is closed.
	for i := range c {
		fmt.Println(i)
	}
}

func fibonacciSelect(c, quit chan int) {
	x, y := 0, 1
	for {
		select {
		case c <- x:
			x, y = y, x+y
		case <-quit:
			fmt.Println("quit")
			return
		}
	}
}

func selectExample() {
	c := make(chan int)
	quit := make(chan int)
	go func() {
		for i := 0; i < 10; i++ {
			fmt.Println(<-c)
		}
		quit <- 0
	}()
	fibonacciSelect(c, quit)

	// The default case in a select is run if no other case is ready.
	// Use a default case to try a send or receive without blocking:
	tick := time.Tick(100 * time.Millisecond)
	boom := time.After(500 * time.Millisecond)
	for {
		select {
		case <-tick:
			fmt.Println("tick.")
		case <-boom:
			fmt.Println("BOOM!")
			return // Breaks out of the for-loop
		default:
			fmt.Println("    .")
			time.Sleep(50 * time.Millisecond)
		}
	}
}

// Walk walks the tree t sending all values
// from the tree to the channel ch.
func Walk(t *tree.Tree, ch chan int) {
	fmt.Println(t.String())
	walkSubTree(t, ch)
	// Need to close the channel here
	close(ch)
}

func walkSubTree(t *tree.Tree, ch chan int, ) {
	if t == nil {
		return
	}
	walkSubTree(t.Left, ch)
	ch <- t.Value
	walkSubTree(t.Right, ch)
}

func WalkQuit(t *tree.Tree, ch, quit chan int) {
	walkSubTreeQuit(t, ch, quit)
	close(ch)
}

func walkSubTreeQuit(t *tree.Tree, ch, quit chan int) {
	if t == nil {
		return
	}
	walkSubTreeQuit(t.Left, ch, quit)
	select {
	case ch <- t.Value:
		// Value successfully sent.
	case <-quit:
		return
	}
	walkSubTreeQuit(t.Right, ch, quit)
}

// Same determines whether the trees
// t1 and t2 contain the same values.
func Same(t1, t2 *tree.Tree) bool {
	w1, w2 := make(chan int), make(chan int)

	go Walk(t1, w1)
	go Walk(t2, w2)

	for {
		v1, ok1 := <-w1
		v2, ok2 := <-w2
		if !ok1 || !ok2 {
			return ok1 == ok2
		}
		if v1 != v2 {
			return false
		}
	}
}

func printContentsOfChannel(ch chan int) {
	for i := range ch {
		fmt.Println(i)
	}
}
func channelExercise() {
	ch := make(chan int, 10)
	fmt.Println("Walk tree(1)")
	go Walk(tree.New(1), ch)
	printContentsOfChannel(ch)
	fmt.Println("Walk tree(2)")
	ch = make(chan int, 10)
	go Walk(tree.New(2), ch)
	printContentsOfChannel(ch)
	fmt.Println("Walk tree quit(3)")
	ch = make(chan int, 10)
	quit := make(chan int)
	defer close(quit)
	go WalkQuit(tree.New(3), ch, quit)
	printContentsOfChannel(ch)
	fmt.Print("tree.New(1) == tree.New(1): ")
	if Same(tree.New(1), tree.New(1)) {
		fmt.Println("PASSED")
	} else {
		fmt.Println("FAILED")
	}

	fmt.Print("tree.New(1) != tree.New(2): ")
	if !Same(tree.New(1), tree.New(2)) {
		fmt.Println("PASSED")
	} else {
		fmt.Println("FAILED")
	}
}

// SafeCounter is safe to use concurrently.
type SafeCounter struct {
	v   map[string]int
	mux sync.Mutex
}

// Inc increments the counter for the given key.
// We can define a block of code to be executed in mutual exclusion by surrounding it with a call to Lock and Unlock
func (c *SafeCounter) Inc(key string) {
	c.mux.Lock()
	// Lock so only one goroutine at a time can access the map c.v.
	c.v[key]++
	c.mux.Unlock()
}

// Value returns the current value of the counter for the given key.
func (c *SafeCounter) Value(key string) int {
	c.mux.Lock()
	// Lock so only one goroutine at a time can access the map c.v.
	// We can also use defer to ensure the mutex will be unlocked as in the Value method.
	defer c.mux.Unlock()
	return c.v[key]
}

func mutexExample() {
	c := SafeCounter{v: make(map[string]int)}
	for i := 0; i < 1000; i++ {
		go c.Inc("somekey")
	}

	time.Sleep(time.Second)
	fmt.Println(c.Value("somekey"))
}

//
func mutexExercise() {
	Crawl("http://golang.org/", 4, fetcher)
}

type Fetcher interface {
	// Fetch returns the body of URL and a slice of URLs found on that page.
	Fetch(url string) (body string, urls []string, err error)
}

var fetched = struct {
	m map[string]error
	sync.Mutex
}{m: make(map[string]error)}

var loading = errors.New("url load in progress") // sentinel value

// Crawl uses fetcher to recursively crawl
// pages starting with url, to a maximum of depth.
func Crawl(url string, depth int, fetcher Fetcher) {
	if depth <= 0 {
		fmt.Printf("<- Done with %v, depth 0.\n", url)
		return
	}

	fetched.Lock()
	if _, ok := fetched.m[url]; ok {
		fetched.Unlock()
		fmt.Printf("<- Done with %v, already fetched.\n", url)
		return
	}
	// We mark the url to be loading to avoid others reloading it at the same time.
	fetched.m[url] = loading
	fetched.Unlock()

	// We load it concurrently.
	body, urls, err := fetcher.Fetch(url)

	// And update the status in a synced zone.
	fetched.Lock()
	fetched.m[url] = err
	fetched.Unlock()

	if err != nil {
		fmt.Printf("<- Error on %v: %v\n", url, err)
		return
	}
	fmt.Printf("Found: %s %q\n", url, body)
	done := make(chan bool)
	for i, u := range urls {
		fmt.Printf("-> Crawling child %v/%v of %v : %v.\n", i, len(urls), url, u)
		go func(url string) {
			Crawl(url, depth-1, fetcher)
			done <- true
		}(u)
	}
	for i, u := range urls {
		fmt.Printf("<- [%v] %v/%v Waiting for child %v.\n", url, i, len(urls), u)
		<-done
	}
	fmt.Printf("<- Done with %v\n", url)
}

// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult

type fakeResult struct {
	body string
	urls []string
}

func (f fakeFetcher) Fetch(url string) (string, []string, error) {
	if res, ok := f[url]; ok {
		return res.body, res.urls, nil
	}
	return "", nil, fmt.Errorf("not found: %s", url)
}

// fetcher is a populated fakeFetcher.
var fetcher = fakeFetcher{
	"http://golang.org/": &fakeResult{
		"The Go Programming Language",
		[]string{
			"http://golang.org/pkg/",
			"http://golang.org/cmd/",
		},
	},
	"http://golang.org/pkg/": &fakeResult{
		"Packages",
		[]string{
			"http://golang.org/",
			"http://golang.org/cmd/",
			"http://golang.org/pkg/fmt/",
			"http://golang.org/pkg/os/",
		},
	},
	"http://golang.org/pkg/fmt/": &fakeResult{
		"Package fmt",
		[]string{
			"http://golang.org/",
			"http://golang.org/pkg/",
		},
	},
	"http://golang.org/pkg/os/": &fakeResult{
		"Package os",
		[]string{
			"http://golang.org/",
			"http://golang.org/pkg/",
		},
	},
}

func main() {
	// GOROUTINE
	// A goroutine is a lightweight thread managed by the Go runtime.
	// go <func> starts a new goroutine running
	fmt.Println("calling the goroutine statements")
	go say("world")
	say("hello")

	// Goroutines run in the same address space, so access to shared memory must be synchronized.
	// The sync package provides useful primitives, although you won't need them much in Go as there are other primitives...

	// CHANNELS
	// Channels are a typed conduit through which you can send and receive values with the channel operator, <-. (The data flows in the direction of the arrow.)
	// e.g.
	// ch <- v    // Send v to channel ch.
	// v := <-ch  // Receive from ch, and assign value to v.

	// Like maps and slices, channels must be created before use:
	// ch := make(chan int)
	// NOTE: By default, sends and receives block until the other side is ready. This allows goroutines to synchronize without explicit locks or condition variables.
	fmt.Println("calling the channel statements")
	channelExamples()

	// BUFFERED CHANNELS
	// Channels can be buffered. Provide the buffer length as the second argument to make to initialize a buffered channel:
	// ch := make(chan int, 100)
	// Sends to a buffered channel block only when the buffer is full. Receives block when the buffer is empty.
	fmt.Println("calling the buffered channel statements")
	bufferedChannelExamples()

	// CONTROLLING CHANNELS
	// A sender can close a channel to indicate that no more values will be sent. Receivers can test whether a channel has been closed by assigning a second parameter to the receive expression:
	// v, ok := <-ch
	// ok is false if there are no more values to receive and the channel is closed.
	fmt.Println("calling the controlling channels statements")
	controllingChannelsExample()

	// SELECT
	// The select statement lets a goroutine wait on multiple communication operations.
	// A select blocks until one of its cases can run, then it executes that case. It chooses one at random if multiple are ready.
	fmt.Println("calling the select statements")
	selectExample()

	// CHANNEL EXERCISE
	// Requires "go get golang.org/x/tour/tree" to be run first
	fmt.Println("calling the channel exercise")
	channelExercise()

	// SYNC.MUTEX
	// Go's standard library provides mutual exclusion with sync.Mutex and its two methods: Lock and Unlock
	fmt.Println("calling the mutex statements")
	mutexExample()

	// CHANNEL EXERCISE
	// Use Go's concurrency features to parallelise a web crawler.
	fmt.Println("calling the mutex exercise")
	mutexExercise()
}
<|endoftext|>


<commit_before>// package http implements an http server that serves static content from ipfs
package http

import (
	"fmt"
	"io"
	"net/http"
	"strings"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gorilla/mux"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
	manet "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/net"
	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"

	core "github.com/jbenet/go-ipfs/core"
)

type objectHandler struct {
	ipfs
}

// Serve starts the http server
func Serve(address ma.Multiaddr, node *core.IpfsNode) error {
	r := mux.NewRouter()
	objectHandler := &objectHandler{&ipfsHandler{node}}
	apiHandler := &apiHandler{}

	r.HandleFunc("/ipfs/", objectHandler.postHandler).Methods("POST")
	r.PathPrefix("/ipfs/").Handler(objectHandler).Methods("GET")

	http.Handle("/", r)

	_, host, err := manet.DialArgs(address)
	if err != nil {
		return err
	}

	return http.ListenAndServe(host, nil)
}

func (i *objectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	path := r.URL.Path[5:]

	nd, err := i.ResolvePath(path)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Println(err)
		return
	}

	dr, err := i.NewDagReader(nd)
	if err != nil {
		// TODO: return json object containing the tree data if it's a directory (err == ErrIsDir)
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Println(err)
		return
	}

	io.Copy(w, dr)
}

func (i *objectHandler) postHandler(w http.ResponseWriter, r *http.Request) {
	nd, err := i.NewDagFromReader(r.Body)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Println(err)
		return
	}

	k, err := i.AddNodeToDAG(nd)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Println(err)
		return
	}

	//TODO: return json representation of list instead
	w.WriteHeader(http.StatusCreated)
	w.Write([]byte(mh.Multihash(k).B58String()))
}
<commit_msg>server/http: Fixed build error<commit_after>// package http implements an http server that serves static content from ipfs
package http

import (
	"fmt"
	"io"
	"net/http"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gorilla/mux"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
	manet "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/net"
	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"

	core "github.com/jbenet/go-ipfs/core"
)

type handler struct {
	ipfs
}

// Serve starts the http server
func Serve(address ma.Multiaddr, node *core.IpfsNode) error {
	r := mux.NewRouter()
	handler := &handler{&ipfsHandler{node}}

	r.HandleFunc("/ipfs/", handler.postHandler).Methods("POST")
	r.PathPrefix("/ipfs/").Handler(handler).Methods("GET")

	http.Handle("/", r)

	_, host, err := manet.DialArgs(address)
	if err != nil {
		return err
	}

	return http.ListenAndServe(host, nil)
}

func (i *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	path := r.URL.Path[5:]

	nd, err := i.ResolvePath(path)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Println(err)
		return
	}

	dr, err := i.NewDagReader(nd)
	if err != nil {
		// TODO: return json object containing the tree data if it's a directory (err == ErrIsDir)
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Println(err)
		return
	}

	io.Copy(w, dr)
}

func (i *handler) postHandler(w http.ResponseWriter, r *http.Request) {
	nd, err := i.NewDagFromReader(r.Body)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Println(err)
		return
	}

	k, err := i.AddNodeToDAG(nd)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Println(err)
		return
	}

	//TODO: return json representation of list instead
	w.WriteHeader(http.StatusCreated)
	w.Write([]byte(mh.Multihash(k).B58String()))
}
<|endoftext|>


<commit_before>package tasky

import (
	"crypto/rand"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"sync"

	"github.com/gorilla/mux"
)

const (
	Enabled  = "Enabled"
	Disabled = "Disabled"
)

type Action uint64

const (
	Cancel Action = iota
	Pause
	Resume
	Restart
)

//WorkerDetails is an object that describes the worker
type WorkerDetails struct {
	// Name of the worker
	Name string `json:"name"`

	// human readable description of the worker
	Description string `json:"description"`

	//an config struct for all config values
	Config interface{} `json:"config_objects,omitempty"`
}

type Worker interface {
	// Details provides all metadata info about the worker and it's config
	Details() *WorkerDetails

	// Worker name
	Name() string

	// Description of the worker and it's usage
	Usage() string

	// Execute the task
	Perform([]byte, chan []byte, chan error, chan bool)

	// Worker status
	Status() string

	// Action to be taken on ongoing task
	Signal(Action) bool

	// Maximum number of simultaneous tasks allowed
	MaxNumTasks() uint64
}

type TaskyError struct {
	Error string
}

var (
	wMut    sync.RWMutex
	workers map[string]Worker

	tMut  sync.RWMutex
	tasks map[string]*taskyTask

	apiBase string
)

func init() {
	workers = make(map[string]Worker)
	tasks = make(map[string]*taskyTask)
	apiBase = "/tasky/v1"
}

func NewWorker(w Worker) (Worker, error) {
	tw := &taskyWorker{}
	tw.w = w

	name := strings.ToLower(w.Name())

	wMut.Lock()
	workers[name] = tw
	wMut.Unlock()

	return tw, nil
}

func uuid() string {
	b := make([]byte, 16)
	_, err := io.ReadFull(rand.Reader, b)
	if err != nil {
		log.Fatal(err)
	}
	b[6] = (b[6] & 0x0F) | 0x40
	b[8] = (b[8] &^ 0x40) | 0x80
	return fmt.Sprintf("%x%x%x%x%x", b[:4], b[4:6], b[6:8], b[8:10], b[10:])
}

type taskid struct {
	TaskId string
}

type tstat struct {
	TaskId   string
	Status   string
	Duration string `json:"Duration,omitempty"`
}

func handlerGetTaskOutput(rw http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["id"]
	log.Println("id: ", id)

	tMut.RLock()
	t, ok := tasks[id]
	tMut.RUnlock()

	if !ok {
		e := TaskyError{"Could not found a task with given id"}
		estr, _ := json.Marshal(e)
		log.Println("estr: ", estr)
		fmt.Fprintf(rw, "%s\n", estr)
		rw.WriteHeader(http.StatusNotFound)
		return
	}

	output := string(t.result())

	if len(output) > 0 {
		js := make(map[string]interface{})

		js["TaskId"] = id
		js["Output"] = output

		jsonStr, err := json.Marshal(js)
		if err != nil {
			e := TaskyError{err.Error()}
			estr, _ := json.Marshal(e)
			log.Println("estr: ", estr)
			fmt.Fprintf(rw, "%s\n", estr)
			rw.WriteHeader(http.StatusInternalServerError)
			return
		}

		fmt.Fprintf(rw, "%s\n", string(jsonStr))
	}
}

func handlerCancelTask(rw http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["id"]
	log.Println("id: ", id)

	tMut.RLock()
	t, ok := tasks[id]
	tMut.RUnlock()

	if !ok {
		e := TaskyError{"Could not found a task with given id"}
		estr, _ := json.Marshal(e)
		log.Println("estr: ", estr)
		fmt.Fprintf(rw, "%s\n", estr)
		rw.WriteHeader(http.StatusNotFound)
		return
	}

	t.cancel()
}

func handlerGetTaskStatus(rw http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["id"]
	log.Println("id: ", id)

	tMut.RLock()
	t, ok := tasks[id]
	tMut.RUnlock()

	if !ok {
		e := TaskyError{"Could not found a task with given id"}
		estr, _ := json.Marshal(e)
		log.Println("estr: ", estr)
		fmt.Fprintf(rw, "%s\n", estr)
		rw.WriteHeader(http.StatusNotFound)
		return
	}

	s := t.status()

	var durStr string

	if s != Running {
		d := t.duration()
		if d > 0 {
			durStr = fmt.Sprintf("%v", d)
		}
	}

	ts := tstat{id, s, durStr}
	log.Println("ts: ", ts)
	jsonStr, err := json.Marshal(ts)
	if err != nil {
		e := TaskyError{err.Error()}
		estr, _ := json.Marshal(e)
		log.Println("estr: ", estr)
		fmt.Fprintf(rw, "%s\n", estr)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	fmt.Fprintf(rw, "%s\n", string(jsonStr))
}

func newTask(w Worker, job []byte) taskid {
	id := uuid()

	t := &taskyTask{}
	t.new(w)

	tMut.Lock()
	tasks[id] = t
	tMut.Unlock()

	go t.run(job)

	return taskid{id}
}

type ts struct {
	Tasks []tstat
}

func listTasks() ts {
	t := ts{}

	tMut.RLock()
	for k, v := range tasks {
		s := v.status()
		log.Println("s: ", s)

		durStr := ""

		if s != Running {
			d := v.duration()
			log.Println("d: ", d)
			if d > 0 {
				durStr = fmt.Sprintf("%v", d)
			}
		}

		if len(t.Tasks) <= 0 {
			t.Tasks = make([]tstat, 1)
			t.Tasks[0] = tstat{k, s, durStr}
		} else {
			t.Tasks = append(t.Tasks, tstat{k, s, durStr})
		}
	}
	tMut.RUnlock()

	return t
}

func handlerListTasks(rw http.ResponseWriter, r *http.Request) {
	t := listTasks()
	log.Println("tasks: ", t)

	jsonStr, err := json.Marshal(t)
	if err != nil {
		e := TaskyError{err.Error()}
		estr, _ := json.Marshal(e)
		log.Println("estr: ", estr)
		fmt.Fprintf(rw, "%s\n", estr)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	fmt.Fprintf(rw, "%s\n", jsonStr)
}

func handlerNewTask(rw http.ResponseWriter, r *http.Request) {
	job, err := ioutil.ReadAll(r.Body)
	if err != nil {
		e := TaskyError{err.Error()}
		estr, _ := json.Marshal(e)
		log.Println("estr: ", estr)
		fmt.Fprintf(rw, "%s\n", estr)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	log.Println("job: ", job)

	name := mux.Vars(r)["name"]
	log.Println("name: ", name)

	wMut.RLock()
	w, ok := workers[name]
	wMut.RUnlock()

	if !ok {
		e := TaskyError{"Could not found worker with given name"}
		estr, _ := json.Marshal(e)
		log.Println("estr: ", estr)
		fmt.Fprintf(rw, "%s\n", estr)
		rw.WriteHeader(http.StatusNotFound)
		return
	}

	t := newTask(w, job)
	jsonStr, err := json.Marshal(t)
	if err != nil {
		e := TaskyError{err.Error()}
		estr, _ := json.Marshal(e)
		log.Println("estr: ", estr)
		fmt.Fprintf(rw, "%s\n", estr)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	fmt.Fprintf(rw, "%s\n", string(jsonStr))
}

func handlerListWorkers(rw http.ResponseWriter, r *http.Request) {
	// jsonStr, _ := listWorkers()

	listWorkers, err := listWorkerDetails()
	if err != nil {
		rw.WriteHeader(http.StatusNoContent)
		return
	}
	rw.Header().Set("Content-Type", "application/json")
	RespondJSON(rw, r, listWorkers)
	return

	// fmt.Fprintf(rw, "%s\n", jsonStr)
}

func RegisterTaskyHandlers(r *mux.Router) {
	r.StrictSlash(true) //enables matching a route with or without a trailing slash
	//Handles /tasky/v1 routes. Create new subrouters off this
	tr := r.PathPrefix(apiBase).Subrouter()

	workersRtr := tr.PathPrefix("/workers").Subrouter()
	workersRtr.HandleFunc("/", handlerListWorkers).Methods("GET")
	workersRtr.HandleFunc("/{name}", handlerNewTask).Methods("POST")

	tasksRtr := tr.PathPrefix("/tasks").Subrouter()
	tasksRtr.HandleFunc("/", handlerListTasks).Methods("GET")
	// tasksRtr.HandleFunc("/{id:[0-9a-f]+}", handlerGetTaskInfo).Methods("GET")
	tasksRtr.HandleFunc("/{id:[0-9a-f]+}/status", handlerGetTaskStatus).Methods("GET")
	tasksRtr.HandleFunc("/{id:[0-9a-f]+}/cancel", handlerCancelTask).Methods("POST")
	tasksRtr.HandleFunc("/{id:[0-9a-f]+}/result", handlerGetTaskOutput).Methods("GET")
}

func RespondJSON(w http.ResponseWriter, req *http.Request, v interface{}, code ...int) error {
	if code != nil {
		w.WriteHeader(code[0])
	}
	err := json.NewEncoder(w).Encode(v)
	if err != nil {
		//encoding failed
		panic(err)
	}
	return err
}
<commit_msg>removed debugging dependencies<commit_after>package tasky

import (
	"crypto/rand"
	"encoding/json"
	"fmt"
	// "github.com/davecgh/go-spew/spew"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"sync"

	"github.com/gorilla/mux"
)

const (
	Enabled  = "Enabled"
	Disabled = "Disabled"
)

type Action uint64

const (
	Cancel Action = iota
	Pause
	Resume
	Restart
)

//WorkerDetails is an object that describes the worker
type WorkerDetails struct {
	// Name of the worker
	Name string `json:"name"`

	// human readable description of the worker
	Description string `json:"description"`

	//an config struct for all config values
	Config interface{} `json:"config_objects,omitempty"`
}

type Worker interface {
	// Details provides all metadata info about the worker and it's config
	Details() *WorkerDetails

	// Worker name
	Name() string

	// Description of the worker and it's usage
	Usage() string

	// Execute the task
	Perform([]byte, chan []byte, chan error, chan bool)

	// Worker status
	Status() string

	// Action to be taken on ongoing task
	Signal(Action) bool

	// Maximum number of simultaneous tasks allowed
	MaxNumTasks() uint64
}

type TaskyError struct {
	Error string
}

var (
	wMut    sync.RWMutex
	workers map[string]Worker

	tMut  sync.RWMutex
	tasks map[string]*taskyTask

	apiBase string
)

func init() {
	workers = make(map[string]Worker)
	tasks = make(map[string]*taskyTask)
	apiBase = "/tasky/v1"
}

func NewWorker(w Worker) (Worker, error) {
	tw := &taskyWorker{}
	tw.w = w

	name := strings.ToLower(w.Name())

	wMut.Lock()
	workers[name] = tw
	wMut.Unlock()

	return tw, nil
}

func uuid() string {
	b := make([]byte, 16)
	_, err := io.ReadFull(rand.Reader, b)
	if err != nil {
		log.Fatal(err)
	}
	b[6] = (b[6] & 0x0F) | 0x40
	b[8] = (b[8] &^ 0x40) | 0x80
	return fmt.Sprintf("%x%x%x%x%x", b[:4], b[4:6], b[6:8], b[8:10], b[10:])
}

type taskid struct {
	TaskId string
}

type tstat struct {
	TaskId   string
	Status   string
	Duration string `json:"Duration,omitempty"`
}

func handlerGetTaskOutput(rw http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["id"]
	log.Println("id: ", id)

	tMut.RLock()
	t, ok := tasks[id]
	tMut.RUnlock()

	if !ok {
		e := TaskyError{"Could not found a task with given id"}
		estr, _ := json.Marshal(e)
		log.Println("estr: ", estr)
		fmt.Fprintf(rw, "%s\n", estr)
		rw.WriteHeader(http.StatusNotFound)
		return
	}

	output := string(t.result())

	if len(output) > 0 {
		js := make(map[string]interface{})

		js["TaskId"] = id
		js["Output"] = output

		jsonStr, err := json.Marshal(js)
		if err != nil {
			e := TaskyError{err.Error()}
			estr, _ := json.Marshal(e)
			log.Println("estr: ", estr)
			fmt.Fprintf(rw, "%s\n", estr)
			rw.WriteHeader(http.StatusInternalServerError)
			return
		}

		fmt.Fprintf(rw, "%s\n", string(jsonStr))
	}
}

func handlerCancelTask(rw http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["id"]
	log.Println("id: ", id)

	tMut.RLock()
	t, ok := tasks[id]
	tMut.RUnlock()

	if !ok {
		e := TaskyError{"Could not found a task with given id"}
		estr, _ := json.Marshal(e)
		log.Println("estr: ", estr)
		fmt.Fprintf(rw, "%s\n", estr)
		rw.WriteHeader(http.StatusNotFound)
		return
	}

	t.cancel()
}

func handlerGetTaskStatus(rw http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["id"]
	log.Println("id: ", id)

	tMut.RLock()
	t, ok := tasks[id]
	tMut.RUnlock()

	if !ok {
		e := TaskyError{"Could not found a task with given id"}
		estr, _ := json.Marshal(e)
		log.Println("estr: ", estr)
		fmt.Fprintf(rw, "%s\n", estr)
		rw.WriteHeader(http.StatusNotFound)
		return
	}

	s := t.status()

	var durStr string

	if s != Running {
		d := t.duration()
		if d > 0 {
			durStr = fmt.Sprintf("%v", d)
		}
	}

	ts := tstat{id, s, durStr}
	log.Println("ts: ", ts)
	jsonStr, err := json.Marshal(ts)
	if err != nil {
		e := TaskyError{err.Error()}
		estr, _ := json.Marshal(e)
		log.Println("estr: ", estr)
		fmt.Fprintf(rw, "%s\n", estr)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	fmt.Fprintf(rw, "%s\n", string(jsonStr))
}

func newTask(w Worker, job []byte) taskid {
	id := uuid()

	t := &taskyTask{}
	t.new(w)

	tMut.Lock()
	tasks[id] = t
	tMut.Unlock()

	go t.run(job)

	return taskid{id}
}

type ts struct {
	Tasks []tstat
}

func listTasks() ts {
	t := ts{}

	tMut.RLock()
	for k, v := range tasks {
		s := v.status()
		log.Println("s: ", s)

		durStr := ""

		if s != Running {
			d := v.duration()
			log.Println("d: ", d)
			if d > 0 {
				durStr = fmt.Sprintf("%v", d)
			}
		}

		if len(t.Tasks) <= 0 {
			t.Tasks = make([]tstat, 1)
			t.Tasks[0] = tstat{k, s, durStr}
		} else {
			t.Tasks = append(t.Tasks, tstat{k, s, durStr})
		}
	}
	tMut.RUnlock()

	return t
}

func handlerListTasks(rw http.ResponseWriter, r *http.Request) {
	t := listTasks()
	log.Println("tasks: ", t)

	jsonStr, err := json.Marshal(t)
	if err != nil {
		e := TaskyError{err.Error()}
		estr, _ := json.Marshal(e)
		log.Println("estr: ", estr)
		fmt.Fprintf(rw, "%s\n", estr)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	fmt.Fprintf(rw, "%s\n", jsonStr)
}

func handlerNewTask(rw http.ResponseWriter, r *http.Request) {
	job, err := ioutil.ReadAll(r.Body)
	if err != nil {
		e := TaskyError{err.Error()}
		estr, _ := json.Marshal(e)
		log.Println("estr: ", estr)
		fmt.Fprintf(rw, "%s\n", estr)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	log.Println("job: ", job)

	name := mux.Vars(r)["name"]
	log.Println("name: ", name)

	wMut.RLock()
	w, ok := workers[name]
	wMut.RUnlock()

	if !ok {
		e := TaskyError{"Could not found worker with given name"}
		estr, _ := json.Marshal(e)
		log.Println("estr: ", estr)
		fmt.Fprintf(rw, "%s\n", estr)
		rw.WriteHeader(http.StatusNotFound)
		return
	}

	t := newTask(w, job)
	jsonStr, err := json.Marshal(t)
	if err != nil {
		e := TaskyError{err.Error()}
		estr, _ := json.Marshal(e)
		log.Println("estr: ", estr)
		fmt.Fprintf(rw, "%s\n", estr)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	fmt.Fprintf(rw, "%s\n", string(jsonStr))
}

func handlerListWorkers(rw http.ResponseWriter, r *http.Request) {
	// jsonStr, _ := listWorkers()

	listWorkers, err := listWorkerDetails()
	if err != nil {
		rw.WriteHeader(http.StatusNoContent)
		return
	}
	rw.Header().Set("Content-Type", "application/json")
	RespondJSON(rw, r, listWorkers)
	return

	// fmt.Fprintf(rw, "%s\n", jsonStr)
}

func RegisterTaskyHandlers(r *mux.Router) {
	r.StrictSlash(true) //enables matching a route with or without a trailing slash
	//Handles /tasky/v1 routes. Create new subrouters off this
	tr := r.PathPrefix(apiBase).Subrouter()

	workersRtr := tr.PathPrefix("/workers").Subrouter()
	workersRtr.HandleFunc("/", handlerListWorkers).Methods("GET")
	workersRtr.HandleFunc("/{name}", handlerNewTask).Methods("POST")

	tasksRtr := tr.PathPrefix("/tasks").Subrouter()
	tasksRtr.HandleFunc("/", handlerListTasks).Methods("GET")
	// tasksRtr.HandleFunc("/{id:[0-9a-f]+}", handlerGetTaskInfo).Methods("GET")
	tasksRtr.HandleFunc("/{id:[0-9a-f]+}/status", handlerGetTaskStatus).Methods("GET")
	tasksRtr.HandleFunc("/{id:[0-9a-f]+}/cancel", handlerCancelTask).Methods("POST")
	tasksRtr.HandleFunc("/{id:[0-9a-f]+}/result", handlerGetTaskOutput).Methods("GET")
}

func RespondJSON(w http.ResponseWriter, req *http.Request, v interface{}, code ...int) error {
	if code != nil {
		w.WriteHeader(code[0])
	}
	err := json.NewEncoder(w).Encode(v)
	if err != nil {
		//encoding failed
		panic(err)
	}
	return err
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"
	"sync"
)

// A local-only, memory-only implementation of Cfg interface, useful
// for development and testing.

type CfgMem struct {
	m       sync.Mutex
	casNext uint64
	entries map[string]*CfgMemEntry
}

type CfgMemEntry struct {
	cas uint64
	val []byte
}

func NewCfgMem() Cfg {
	return &CfgMem{
		casNext: 1,
		entries: make(map[string]*CfgMemEntry),
	}
}

func (c *CfgMem) Get(key string, cas uint64) (
	[]byte, uint64, error) {
	c.m.Lock()
	defer c.m.Unlock()

	entry, exists := c.entries[key]
	if !exists {
		return nil, 0, nil
	}
	if cas != 0 && cas != entry.cas {
		return nil, 0, fmt.Errorf("error: mismatched Cfg CAS")
	}
	val := make([]byte, len(entry.val))
	copy(val, entry.val)
	return val, entry.cas, nil
}

func (c *CfgMem) Set(key string, val []byte, cas uint64) (
	uint64, error) {
	c.m.Lock()
	defer c.m.Unlock()

	prevEntry, exists := c.entries[key]
	if cas == 0 {
		if exists {
			return 0, fmt.Errorf("error: entry already exists, key: %s", key)
		}
	} else { // cas != 0
		if !exists {
			return 0, fmt.Errorf("error: no entry, key: %s", key)
		}
		if cas != prevEntry.cas {
			return 0, fmt.Errorf("error: mismatched cas, key: %s", key)
		}
	}
	nextEntry := &CfgMemEntry{
		cas: c.casNext,
		val: make([]byte, len(val)),
	}
	copy(nextEntry.val, val)
	c.entries[key] = nextEntry
	c.casNext += 1
	return nextEntry.cas, nil
}

func (c *CfgMem) Del(key string, cas uint64) error {
	c.m.Lock()
	defer c.m.Unlock()

	if cas != 0 {
		entry, exists := c.entries[key]
		if !exists || cas != entry.cas {
			return fmt.Errorf("error: mismatched Cfg CAS")
		}
	}
	delete(c.entries, key)
	return nil
}
<commit_msg>made CfgMem fields public for JSON'ability<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"
	"sync"
)

// A local-only, memory-only implementation of Cfg interface, useful
// for development and testing.

type CfgMem struct {
	m       sync.Mutex
	CASNext uint64
	Entries map[string]*CfgMemEntry
}

type CfgMemEntry struct {
	CAS uint64
	Val []byte
}

func NewCfgMem() Cfg {
	return &CfgMem{
		CASNext: 1,
		Entries: make(map[string]*CfgMemEntry),
	}
}

func (c *CfgMem) Get(key string, cas uint64) (
	[]byte, uint64, error) {
	c.m.Lock()
	defer c.m.Unlock()

	entry, exists := c.Entries[key]
	if !exists {
		return nil, 0, nil
	}
	if cas != 0 && cas != entry.CAS {
		return nil, 0, fmt.Errorf("error: mismatched Cfg CAS")
	}
	val := make([]byte, len(entry.Val))
	copy(val, entry.Val)
	return val, entry.CAS, nil
}

func (c *CfgMem) Set(key string, val []byte, cas uint64) (
	uint64, error) {
	c.m.Lock()
	defer c.m.Unlock()

	prevEntry, exists := c.Entries[key]
	if cas == 0 {
		if exists {
			return 0, fmt.Errorf("error: entry already exists, key: %s", key)
		}
	} else { // cas != 0
		if !exists {
			return 0, fmt.Errorf("error: no entry, key: %s", key)
		}
		if cas != prevEntry.CAS {
			return 0, fmt.Errorf("error: mismatched cas, key: %s", key)
		}
	}
	nextEntry := &CfgMemEntry{
		CAS: c.CASNext,
		Val: make([]byte, len(val)),
	}
	copy(nextEntry.Val, val)
	c.Entries[key] = nextEntry
	c.CASNext += 1
	return nextEntry.CAS, nil
}

func (c *CfgMem) Del(key string, cas uint64) error {
	c.m.Lock()
	defer c.m.Unlock()

	if cas != 0 {
		entry, exists := c.Entries[key]
		if !exists || cas != entry.CAS {
			return fmt.Errorf("error: mismatched Cfg CAS")
		}
	}
	delete(c.Entries, key)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net"
	"os"
	"unicode/utf8"
)

// http://semver.org/
const version = "0.1.0"

// Defines a single panel
type Panel struct {
	timeout int64
	panel   string
}

// Do some basic logging for any error
func checkErr(error err) bool {
	if err != nil {
		log.Fatalln("Error encountered:", err)
		return true
	}
	return false
}

// Load a specified file into the panels slice
func loadFile(string fileName, queue chan *stringOut) {
	file, err := os.Open(filename)
	if checkErr(err) {
		return
	}

	stringOut <- "END"

}

func load(string fileName) {
	log.Println("Loading", fileName)
	lines := make(chan string, 100)

	go loadFile(fileName, lines)

	for {
		line := <-lines
		if line != "END" {
			runeValue, width := utf8.DecodeRuneInString(line[0:])

		} else {
			break
		}
	}

}

func main() {
	fmt.Println("Go ASCII Service version", version)

	go load("sw1.txt")

	sock, err := net.Listen("tcp", "21")

	if checkErr(err) {
		return
	}

	for {

	}
}
<commit_msg>semi-workable<commit_after>package main

import (
	"bufio"
	"fmt"
	"log"
	"net"
	"os"
	"strconv"
	"time"
)

// http://semver.org/
const version = "0.1.0"

var globalPanels []PanelGroup

// Defines a single panel
type Panel struct {
	Timeout int64
	Panel   string
}

type PanelGroup struct {
	V4Panel []Panel
	V6Panel []Panel
}

// Do some basic logging for any error
func checkErr(err error) bool {
	if err != nil {
		log.Fatalln("Error encountered:", err)
		return true
	}
	return false
}

// Load a specified file into the panels slice
func loadFile(fileName string, stringOut chan<- Panel) {
	fileh, err := os.Open(fileName)

	if checkErr(err) {
		return
	}

	scanner := bufio.NewScanner(fileh)

	thisPanel := Panel{
		Timeout: 0,
		Panel:   "START",
	}

	for scanner.Scan() {
		line := scanner.Text()

		if len(line) >= 1 {
			if line[0] > '0' && line[0] <= '9' {
				if thisPanel.Panel != "START" {
					stringOut <- thisPanel
				}
				thisPanel = Panel{}
				uhold, _ := strconv.ParseInt(line, 10, 64)
				thisPanel.Timeout = uhold
				// fmt.Println("start a new panel, wait time", thisPanel.Timeout)
			} else {
				thisPanel.Panel = fmt.Sprintln(thisPanel.Panel, line)
			}
		} else {
			thisPanel.Panel = fmt.Sprintln(thisPanel.Panel, line)
		}

	}

	stringOut <- thisPanel

	close(stringOut)

}

// Generates a slice of panels from a filename
// calls loadFile()
func load(fileName string) []Panel {
	log.Println("Loading", fileName)
	panels := make(chan Panel, 100)

	go loadFile(fileName, panels)

	outPanel := make([]Panel, 0)

	for range panels {
		panel := <-panels
		outPanel = append(outPanel, panel)
	}

	return outPanel

}

func PlayPanel(con net.Conn, frames []Panel) {
	for _, frame := range frames {
		fmt.Fprintf(con, "\x1b[2J%s", frame.Panel)
		time.Sleep(time.Duration(frame.Timeout) * time.Second)
	}
}

func main() {
	fmt.Println("Go ASCII Service version", version)

	globalPanels = make([]PanelGroup, 0)

	sw1 := PanelGroup{}
	sem := make(chan string, 2)

	go func(fileName string) {
		log.Println("File:", fileName, "has begun loading")
		sw1.V4Panel = load(fileName)
		sem <- fileName
	}("swv4.txt")

	go func(fileName string) {
		log.Println("File:", fileName, "has begun loading")
		sw1.V6Panel = load(fileName)
		sem <- fileName
	}("swv6.txt")

	log.Println("File:", <-sem, "has finished loading")
	log.Println("File:", <-sem, "has finished loading")

	close(sem)

	globalPanels = append(globalPanels, sw1)

	sock, err := net.Listen("tcp", ":21")

	if checkErr(err) {
		return
	}

	log.Println("Now listening on port 21")

	for {
		conn, err := sock.Accept()
		if checkErr(err) {
			return
		}
		test, err := net.ResolveTCPAddr("tcp", sock.Addr().String())

		if test.IP.To4() != nil {
			log.Println("New v4 User: ", sock.Addr().String())
			go PlayPanel(conn, globalPanels[0].V4Panel)
		} else {
			log.Println("New v6 User: ", sock.Addr().String())
			go PlayPanel(conn, globalPanels[0].V6Panel)
		}
	}
}
<|endoftext|>


<commit_before>package tempCrit

import (
	"math"
)
import (
	"../bzone"
	"../tempAll"
	vec "../vector"
)

// Evaluate the retarded pair Green's function Pi_R(q, omega)_{xx, xy, yy}.
// q must be a two-dimensional vector.
func Pi(env *tempAll.Environment, q vec.Vector, omega float64) vec.Vector {
	var piInner func(k vec.Vector, out *vec.Vector)
	if env.F0 == 0.0 {
		piInner = func(k vec.Vector, out *vec.Vector) {
			// do vector operations on out to avoid allocation:
			// out = k + q/2
			(*out)[0] = k[0] + q[0]/2.0
			(*out)[1] = k[1] + q[1]/2.0
			xp := env.Xi_h(*out)
			// out = k - q/2
			(*out)[0] = k[0] - q[0]/2.0
			(*out)[1] = k[1] - q[1]/2.0
			xm := env.Xi_h(*out)

			tp := math.Tanh(env.Beta * xp / 2.0)
			tm := math.Tanh(env.Beta * xm / 2.0)
			common := -(tp + tm) / (omega - xp - xm)
			sx := math.Sin(k[0])
			sy := math.Sin(k[1])
			// out = result
			(*out)[0] = sx * sx * common
			(*out)[1] = sx * sy * common
			(*out)[2] = sy * sy * common
		}
	} else {
		piInner = func(k vec.Vector, out *vec.Vector) {
			// out = k + q/2
			(*out)[0] = k[0] + q[0]/2.0
			(*out)[1] = k[1] + q[1]/2.0
			x1 := env.Xi_h(*out)
			E1 := env.BogoEnergy(*out)
			// out = k - q/2
			(*out)[0] = k[0] - q[0]/2.0
			(*out)[1] = k[1] - q[1]/2.0
			x2 := env.Xi_h(*out)
			E2 := env.BogoEnergy(*out)

			A1 := 0.5 * (1.0 + x1/E1)
			A2 := 0.5 * (1.0 + x2/E2)
			B1 := 0.5 * (1.0 - x1/E1)
			B2 := 0.5 * (1.0 - x2/E2)
			t1 := math.Tanh(env.Beta * E1 / 2.0)
			t2 := math.Tanh(env.Beta * E2 / 2.0)
			common := -(t1+t2)*(A1*A2/(omega-E1-E2)-B1*B2/(omega+E1+E2)) - (t1-t2)*(A1*B2/(omega-E1+E2)-B1*A2/(omega+E1-E2))
			sx := math.Sin(k[0])
			sy := math.Sin(k[1])
			// out = result
			(*out)[0] = sx * sx * common
			(*out)[1] = sx * sy * common
			(*out)[2] = sy * sy * common
		}
	}
	return bzone.VectorAvg(env.PointsPerSide, 2, 3, piInner)
}
<commit_msg>Make notation in pi.go match notation in notes.<commit_after>package tempCrit

import (
	"math"
)
import (
	"../bzone"
	"../tempAll"
	vec "../vector"
)

// Evaluate the retarded pair Green's function Pi_R(k, omega)_{xx, xy, yy}.
// k must be a two-dimensional vector.
func Pi(env *tempAll.Environment, k vec.Vector, omega float64) vec.Vector {
	var piInner func(k vec.Vector, out *vec.Vector)
	// TODO: should this comparison be math.Abs(env.F0)? Not using that to
	// avoid going to finite F0 procedure when F0 < 0 (since F0 is
	// positive by choice of gauge). Also - would it be better to just
	// test if F0 == 0.0? Would prefer to avoid equality comparison
	// on float.
	if env.F0 < 1e-9 {
		piInner = func(q vec.Vector, out *vec.Vector) {
			// do vector operations on out to avoid allocation:
			// out = k/2 + q
			(*out)[0] = k[0]/2.0 + q[0]
			(*out)[1] = k[1]/2.0 + q[1]
			xp := env.Xi_h(*out)
			// out = k/2 - q
			(*out)[0] = k[0]/2.0 - q[0]
			(*out)[1] = k[1]/2.0 - q[1]
			xm := env.Xi_h(*out)

			tp := math.Tanh(env.Beta * xp / 2.0)
			tm := math.Tanh(env.Beta * xm / 2.0)
			common := -(tp + tm) / (omega - xp - xm)
			sx := math.Sin(q[0])
			sy := math.Sin(q[1])
			// out = result
			(*out)[0] = sx * sx * common
			(*out)[1] = sx * sy * common
			(*out)[2] = sy * sy * common
		}
	} else {
		piInner = func(q vec.Vector, out *vec.Vector) {
			// out = k/2 + q
			(*out)[0] = k[0]/2.0 + q[0]
			(*out)[1] = k[1]/2.0 + q[1]
			xi1 := env.Xi_h(*out)
			E1 := env.BogoEnergy(*out)
			// out = k/2 - q
			(*out)[0] = k[0]/2.0 - q[0]
			(*out)[1] = k[1]/2.0 - q[1]
			xi2 := env.Xi_h(*out)
			E2 := env.BogoEnergy(*out)

			A1 := 0.5 * (1.0 + xi1/E1)
			A2 := 0.5 * (1.0 + xi2/E2)
			B1 := 0.5 * (1.0 - xi1/E1)
			B2 := 0.5 * (1.0 - xi2/E2)
			t1 := math.Tanh(env.Beta * E1 / 2.0)
			t2 := math.Tanh(env.Beta * E2 / 2.0)
			common := -(t1+t2)*(A1*A2/(omega-E1-E2)-B1*B2/(omega+E1+E2)) - (t1-t2)*(A1*B2/(omega-E1+E2)-B1*A2/(omega+E1-E2))
			sx := math.Sin(q[0])
			sy := math.Sin(q[1])
			// out = result
			(*out)[0] = sx * sx * common
			(*out)[1] = sx * sy * common
			(*out)[2] = sy * sy * common
		}
	}
	return bzone.VectorAvg(env.PointsPerSide, 2, 3, piInner)
}
<|endoftext|>


<commit_before>package nagiosplugin

import (
	"fmt"
	"os"
	"strings"
)

const (
	MessageSeparator = ", "
)

// Standalone Exit function for simple checks without multiple results
// or perfdata.
func Exit(status Status, message string) {
	fmt.Printf("%v: %s\n", status, message)
	os.Exit(int(status))
}

// Represents the state of a Nagios check.
type Check struct {
	results []Result
	status  Status
}

// AddResult adds a check result. This will not terminate the check. If
// status is the highest yet reported, this will update the check's
// final return status.
func (c Check) AddResult(status Status, message string) {
	var result Result
	result.status = status
	result.message = message
	c.results = append(c.results, result)
	if result.status > c.status {
		c.status = result.status
	}
}

// exitInfoText returns the most important result text, formatted for
// the first line of plugin output.
//
// Returns joined string of (MessageSeparator-separated) info text from
// results which have a status of at least c.status.
func (c Check) exitInfoText() string {
	importantMessages := make([]string, 0)
	for _, result := range c.results {
		if result.status >= c.status {
			importantMessages = append(importantMessages, result.message)
		}
	}
	return strings.Join(importantMessages, MessageSeparator)
}

// String representation of the check results, suitable for output and
// parsing by Nagios.
func (c Check) String() string {
	return fmt.Sprintf("%v: %s", c.status, c.exitInfoText())
}

// Finish ends the check, prints its output (to stdout), and exits with
// the correct status.
func (c Check) Finish() {
	fmt.Println(c)
	os.Exit(int(c.status))
}
<commit_msg>Add perfdata slice to Check<commit_after>package nagiosplugin

import (
	"fmt"
	"os"
	"strings"
)

const (
	MessageSeparator = ", "
)

// Standalone Exit function for simple checks without multiple results
// or perfdata.
func Exit(status Status, message string) {
	fmt.Printf("%v: %s\n", status, message)
	os.Exit(int(status))
}

// Represents the state of a Nagios check.
type Check struct {
	results []Result
	perfdata []PerfDatum
	status  Status
}

// AddResult adds a check result. This will not terminate the check. If
// status is the highest yet reported, this will update the check's
// final return status.
func (c Check) AddResult(status Status, message string) {
	var result Result
	result.status = status
	result.message = message
	c.results = append(c.results, result)
	if result.status > c.status {
		c.status = result.status
	}
}

// AddPerfDatum adds a metric to the set output by the check. unit must
// be a valid Nagios unit of measurement (UOM): "us", "ms", "s",
// "%", "b", "kb", "mb", "gb", "tb", "c", or the empty string. Zero or
// more of the thresholds min, max, warn and crit may be supplied; these
// must be of the same UOM as the value.
//
// A threshold may be positive or negative infinity, in which case it
// will be omitted in the check output. A value may not be either
// infinity.
//
// Returns error on invalid parameters.
func (c Check) AddPerfDatum(label, unit string, value float64, thresholds ...float64) error {
	datum, err := NewPerfDatum(label, unit, value, thresholds...)
	if err != nil {
		return err
	}
	c.perfdata = append(c.perfdata, *datum)
	return nil
}

// exitInfoText returns the most important result text, formatted for
// the first line of plugin output.
//
// Returns joined string of (MessageSeparator-separated) info text from
// results which have a status of at least c.status.
func (c Check) exitInfoText() string {
	importantMessages := make([]string, 0)
	for _, result := range c.results {
		if result.status >= c.status {
			importantMessages = append(importantMessages, result.message)
		}
	}
	return strings.Join(importantMessages, MessageSeparator)
}

// String representation of the check results, suitable for output and
// parsing by Nagios.
func (c Check) String() string {
	return fmt.Sprintf("%v: %s", c.status, c.exitInfoText())
}

// Finish ends the check, prints its output (to stdout), and exits with
// the correct status.
func (c Check) Finish() {
	fmt.Println(c)
	os.Exit(int(c.status))
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package embed

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"path"

	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/etcdserver/api/v2http"
	"github.com/coreos/etcd/pkg/cors"
	runtimeutil "github.com/coreos/etcd/pkg/runtime"
	"github.com/coreos/etcd/pkg/transport"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/rafthttp"
	"github.com/coreos/pkg/capnslog"
)

var plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "embed")

const (
	// internal fd usage includes disk usage and transport usage.
	// To read/write snapshot, snap pkg needs 1. In normal case, wal pkg needs
	// at most 2 to read/lock/write WALs. One case that it needs to 2 is to
	// read all logs after some snapshot index, which locates at the end of
	// the second last and the head of the last. For purging, it needs to read
	// directory, so it needs 1. For fd monitor, it needs 1.
	// For transport, rafthttp builds two long-polling connections and at most
	// four temporary connections with each member. There are at most 9 members
	// in a cluster, so it should reserve 96.
	// For the safety, we set the total reserved number to 150.
	reservedInternalFDNum = 150
)

// Etcd contains a running etcd server and its listeners.
type Etcd struct {
	Peers   []net.Listener
	Clients []net.Listener
	Server  *etcdserver.EtcdServer

	cfg   Config
	errc  chan error
	sctxs map[string]*serveCtx
}

// StartEtcd launches the etcd server and HTTP handlers for client/server communication.
func StartEtcd(inCfg *Config) (e *Etcd, err error) {
	if err = inCfg.Validate(); err != nil {
		return nil, err
	}
	e = &Etcd{cfg: *inCfg}
	cfg := &e.cfg
	defer func() {
		if err != nil {
			e.Close()
			e = nil
		}
	}()

	if e.Peers, err = startPeerListeners(cfg); err != nil {
		return
	}
	if e.sctxs, err = startClientListeners(cfg); err != nil {
		return
	}
	for _, sctx := range e.sctxs {
		e.Clients = append(e.Clients, sctx.l)
	}

	var (
		urlsmap types.URLsMap
		token   string
	)

	if !isMemberInitialized(cfg) {
		urlsmap, token, err = cfg.PeerURLsMapAndToken("etcd")
		if err != nil {
			return nil, fmt.Errorf("error setting up initial cluster: %v", err)
		}
	}

	srvcfg := &etcdserver.ServerConfig{
		Name:                    cfg.Name,
		ClientURLs:              cfg.ACUrls,
		PeerURLs:                cfg.APUrls,
		DataDir:                 cfg.Dir,
		DedicatedWALDir:         cfg.WalDir,
		SnapCount:               cfg.SnapCount,
		MaxSnapFiles:            cfg.MaxSnapFiles,
		MaxWALFiles:             cfg.MaxWalFiles,
		InitialPeerURLsMap:      urlsmap,
		InitialClusterToken:     token,
		DiscoveryURL:            cfg.Durl,
		DiscoveryProxy:          cfg.Dproxy,
		NewCluster:              cfg.IsNewCluster(),
		ForceNewCluster:         cfg.ForceNewCluster,
		PeerTLSInfo:             cfg.PeerTLSInfo,
		TickMs:                  cfg.TickMs,
		ElectionTicks:           cfg.ElectionTicks(),
		AutoCompactionRetention: cfg.AutoCompactionRetention,
		QuotaBackendBytes:       cfg.QuotaBackendBytes,
		StrictReconfigCheck:     cfg.StrictReconfigCheck,
		EnablePprof:             cfg.EnablePprof,
	}

	if e.Server, err = etcdserver.NewServer(srvcfg); err != nil {
		return
	}

	// buffer channel so goroutines on closed connections won't wait forever
	e.errc = make(chan error, len(e.Peers)+len(e.Clients)+2*len(e.sctxs))

	e.Server.Start()
	e.serve()
	<-e.Server.ReadyNotify()
	return
}

func (e *Etcd) Close() {
	for _, sctx := range e.sctxs {
		sctx.cancel()
	}
	for i := range e.Peers {
		if e.Peers[i] != nil {
			e.Peers[i].Close()
		}
	}
	for i := range e.Clients {
		if e.Clients[i] != nil {
			e.Clients[i].Close()
		}
	}
	if e.Server != nil {
		e.Server.Stop()
	}
}

func (e *Etcd) Err() <-chan error { return e.errc }

func startPeerListeners(cfg *Config) (plns []net.Listener, err error) {
	if cfg.PeerAutoTLS && cfg.PeerTLSInfo.Empty() {
		phosts := make([]string, len(cfg.LPUrls))
		for i, u := range cfg.LPUrls {
			phosts[i] = u.Host
		}
		cfg.PeerTLSInfo, err = transport.SelfCert(path.Join(cfg.Dir, "fixtures/peer"), phosts)
		if err != nil {
			plog.Fatalf("could not get certs (%v)", err)
		}
	} else if cfg.PeerAutoTLS {
		plog.Warningf("ignoring peer auto TLS since certs given")
	}

	if !cfg.PeerTLSInfo.Empty() {
		plog.Infof("peerTLS: %s", cfg.PeerTLSInfo)
	}

	plns = make([]net.Listener, len(cfg.LPUrls))
	defer func() {
		if err == nil {
			return
		}
		for i := range plns {
			if plns[i] == nil {
				continue
			}
			plns[i].Close()
			plog.Info("stopping listening for peers on ", cfg.LPUrls[i].String())
		}
	}()

	for i, u := range cfg.LPUrls {
		var tlscfg *tls.Config
		if u.Scheme == "http" {
			if !cfg.PeerTLSInfo.Empty() {
				plog.Warningf("The scheme of peer url %s is HTTP while peer key/cert files are presented. Ignored peer key/cert files.", u.String())
			}
			if cfg.PeerTLSInfo.ClientCertAuth {
				plog.Warningf("The scheme of peer url %s is HTTP while client cert auth (--peer-client-cert-auth) is enabled. Ignored client cert auth for this url.", u.String())
			}
		}
		if !cfg.PeerTLSInfo.Empty() {
			if tlscfg, err = cfg.PeerTLSInfo.ServerConfig(); err != nil {
				return nil, err
			}
		}
		if plns[i], err = rafthttp.NewListener(u, tlscfg); err != nil {
			return nil, err
		}
		plog.Info("listening for peers on ", u.String())
	}
	return plns, nil
}

func startClientListeners(cfg *Config) (sctxs map[string]*serveCtx, err error) {
	if cfg.ClientAutoTLS && cfg.ClientTLSInfo.Empty() {
		chosts := make([]string, len(cfg.LCUrls))
		for i, u := range cfg.LCUrls {
			chosts[i] = u.Host
		}
		cfg.ClientTLSInfo, err = transport.SelfCert(path.Join(cfg.Dir, "fixtures/client"), chosts)
		if err != nil {
			plog.Fatalf("could not get certs (%v)", err)
		}
	} else if cfg.ClientAutoTLS {
		plog.Warningf("ignoring client auto TLS since certs given")
	}

	sctxs = make(map[string]*serveCtx)
	for _, u := range cfg.LCUrls {
		sctx := newServeCtx()

		if u.Scheme == "http" {
			if !cfg.ClientTLSInfo.Empty() {
				plog.Warningf("The scheme of client url %s is HTTP while peer key/cert files are presented. Ignored key/cert files.", u.String())
			}
			if cfg.ClientTLSInfo.ClientCertAuth {
				plog.Warningf("The scheme of client url %s is HTTP while client cert auth (--client-cert-auth) is enabled. Ignored client cert auth for this url.", u.String())
			}
		}
		if u.Scheme == "https" && cfg.ClientTLSInfo.Empty() {
			return nil, fmt.Errorf("TLS key/cert (--cert-file, --key-file) must be provided for client url %s with HTTPs scheme", u.String())
		}

		sctx.secure = u.Scheme == "https"
		sctx.insecure = !sctx.secure
		if oldctx := sctxs[u.Host]; oldctx != nil {
			oldctx.secure = oldctx.secure || sctx.secure
			oldctx.insecure = oldctx.insecure || sctx.insecure
			continue
		}

		if sctx.l, err = net.Listen("tcp", u.Host); err != nil {
			return nil, err
		}

		if fdLimit, fderr := runtimeutil.FDLimit(); fderr == nil {
			if fdLimit <= reservedInternalFDNum {
				plog.Fatalf("file descriptor limit[%d] of etcd process is too low, and should be set higher than %d to ensure internal usage", fdLimit, reservedInternalFDNum)
			}
			sctx.l = transport.LimitListener(sctx.l, int(fdLimit-reservedInternalFDNum))
		}

		if sctx.l, err = transport.NewKeepAliveListener(sctx.l, "tcp", nil); err != nil {
			return nil, err
		}

		plog.Info("listening for client requests on ", u.Host)
		defer func() {
			if err != nil {
				sctx.l.Close()
				plog.Info("stopping listening for client requests on ", u.Host)
			}
		}()
		sctxs[u.Host] = sctx
	}
	return sctxs, nil
}

func (e *Etcd) serve() (err error) {
	var ctlscfg *tls.Config
	if !e.cfg.ClientTLSInfo.Empty() {
		plog.Infof("ClientTLS: %s", e.cfg.ClientTLSInfo)
		if ctlscfg, err = e.cfg.ClientTLSInfo.ServerConfig(); err != nil {
			return err
		}
	}

	if e.cfg.CorsInfo.String() != "" {
		plog.Infof("cors = %s", e.cfg.CorsInfo)
	}

	// Start the peer server in a goroutine
	ph := v2http.NewPeerHandler(e.Server)
	for _, l := range e.Peers {
		go func(l net.Listener) {
			e.errc <- servePeerHTTP(l, ph)
		}(l)
	}

	// Start a client server goroutine for each listen address
	ch := http.Handler(&cors.CORSHandler{
		Handler: v2http.NewClientHandler(e.Server, e.Server.Cfg.ReqTimeout()),
		Info:    e.cfg.CorsInfo,
	})
	for _, sctx := range e.sctxs {
		// read timeout does not work with http close notify
		// TODO: https://github.com/golang/go/issues/9524
		go func(s *serveCtx) {
			e.errc <- s.serve(e.Server, ctlscfg, ch, e.errc)
		}(sctx)
	}
	return nil
}
<commit_msg>embed: fix nil dereference on error to set up initial cluster<commit_after>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package embed

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"path"

	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/etcdserver/api/v2http"
	"github.com/coreos/etcd/pkg/cors"
	runtimeutil "github.com/coreos/etcd/pkg/runtime"
	"github.com/coreos/etcd/pkg/transport"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/rafthttp"
	"github.com/coreos/pkg/capnslog"
)

var plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "embed")

const (
	// internal fd usage includes disk usage and transport usage.
	// To read/write snapshot, snap pkg needs 1. In normal case, wal pkg needs
	// at most 2 to read/lock/write WALs. One case that it needs to 2 is to
	// read all logs after some snapshot index, which locates at the end of
	// the second last and the head of the last. For purging, it needs to read
	// directory, so it needs 1. For fd monitor, it needs 1.
	// For transport, rafthttp builds two long-polling connections and at most
	// four temporary connections with each member. There are at most 9 members
	// in a cluster, so it should reserve 96.
	// For the safety, we set the total reserved number to 150.
	reservedInternalFDNum = 150
)

// Etcd contains a running etcd server and its listeners.
type Etcd struct {
	Peers   []net.Listener
	Clients []net.Listener
	Server  *etcdserver.EtcdServer

	cfg   Config
	errc  chan error
	sctxs map[string]*serveCtx
}

// StartEtcd launches the etcd server and HTTP handlers for client/server communication.
func StartEtcd(inCfg *Config) (e *Etcd, err error) {
	if err = inCfg.Validate(); err != nil {
		return nil, err
	}
	e = &Etcd{cfg: *inCfg}
	cfg := &e.cfg
	defer func() {
		if e != nil && err != nil {
			e.Close()
			e = nil
		}
	}()

	if e.Peers, err = startPeerListeners(cfg); err != nil {
		return
	}
	if e.sctxs, err = startClientListeners(cfg); err != nil {
		return
	}
	for _, sctx := range e.sctxs {
		e.Clients = append(e.Clients, sctx.l)
	}

	var (
		urlsmap types.URLsMap
		token   string
	)

	if !isMemberInitialized(cfg) {
		urlsmap, token, err = cfg.PeerURLsMapAndToken("etcd")
		if err != nil {
			return e, fmt.Errorf("error setting up initial cluster: %v", err)
		}
	}

	srvcfg := &etcdserver.ServerConfig{
		Name:                    cfg.Name,
		ClientURLs:              cfg.ACUrls,
		PeerURLs:                cfg.APUrls,
		DataDir:                 cfg.Dir,
		DedicatedWALDir:         cfg.WalDir,
		SnapCount:               cfg.SnapCount,
		MaxSnapFiles:            cfg.MaxSnapFiles,
		MaxWALFiles:             cfg.MaxWalFiles,
		InitialPeerURLsMap:      urlsmap,
		InitialClusterToken:     token,
		DiscoveryURL:            cfg.Durl,
		DiscoveryProxy:          cfg.Dproxy,
		NewCluster:              cfg.IsNewCluster(),
		ForceNewCluster:         cfg.ForceNewCluster,
		PeerTLSInfo:             cfg.PeerTLSInfo,
		TickMs:                  cfg.TickMs,
		ElectionTicks:           cfg.ElectionTicks(),
		AutoCompactionRetention: cfg.AutoCompactionRetention,
		QuotaBackendBytes:       cfg.QuotaBackendBytes,
		StrictReconfigCheck:     cfg.StrictReconfigCheck,
		EnablePprof:             cfg.EnablePprof,
	}

	if e.Server, err = etcdserver.NewServer(srvcfg); err != nil {
		return
	}

	// buffer channel so goroutines on closed connections won't wait forever
	e.errc = make(chan error, len(e.Peers)+len(e.Clients)+2*len(e.sctxs))

	e.Server.Start()
	e.serve()
	<-e.Server.ReadyNotify()
	return
}

func (e *Etcd) Close() {
	for _, sctx := range e.sctxs {
		sctx.cancel()
	}
	for i := range e.Peers {
		if e.Peers[i] != nil {
			e.Peers[i].Close()
		}
	}
	for i := range e.Clients {
		if e.Clients[i] != nil {
			e.Clients[i].Close()
		}
	}
	if e.Server != nil {
		e.Server.Stop()
	}
}

func (e *Etcd) Err() <-chan error { return e.errc }

func startPeerListeners(cfg *Config) (plns []net.Listener, err error) {
	if cfg.PeerAutoTLS && cfg.PeerTLSInfo.Empty() {
		phosts := make([]string, len(cfg.LPUrls))
		for i, u := range cfg.LPUrls {
			phosts[i] = u.Host
		}
		cfg.PeerTLSInfo, err = transport.SelfCert(path.Join(cfg.Dir, "fixtures/peer"), phosts)
		if err != nil {
			plog.Fatalf("could not get certs (%v)", err)
		}
	} else if cfg.PeerAutoTLS {
		plog.Warningf("ignoring peer auto TLS since certs given")
	}

	if !cfg.PeerTLSInfo.Empty() {
		plog.Infof("peerTLS: %s", cfg.PeerTLSInfo)
	}

	plns = make([]net.Listener, len(cfg.LPUrls))
	defer func() {
		if err == nil {
			return
		}
		for i := range plns {
			if plns[i] == nil {
				continue
			}
			plns[i].Close()
			plog.Info("stopping listening for peers on ", cfg.LPUrls[i].String())
		}
	}()

	for i, u := range cfg.LPUrls {
		var tlscfg *tls.Config
		if u.Scheme == "http" {
			if !cfg.PeerTLSInfo.Empty() {
				plog.Warningf("The scheme of peer url %s is HTTP while peer key/cert files are presented. Ignored peer key/cert files.", u.String())
			}
			if cfg.PeerTLSInfo.ClientCertAuth {
				plog.Warningf("The scheme of peer url %s is HTTP while client cert auth (--peer-client-cert-auth) is enabled. Ignored client cert auth for this url.", u.String())
			}
		}
		if !cfg.PeerTLSInfo.Empty() {
			if tlscfg, err = cfg.PeerTLSInfo.ServerConfig(); err != nil {
				return nil, err
			}
		}
		if plns[i], err = rafthttp.NewListener(u, tlscfg); err != nil {
			return nil, err
		}
		plog.Info("listening for peers on ", u.String())
	}
	return plns, nil
}

func startClientListeners(cfg *Config) (sctxs map[string]*serveCtx, err error) {
	if cfg.ClientAutoTLS && cfg.ClientTLSInfo.Empty() {
		chosts := make([]string, len(cfg.LCUrls))
		for i, u := range cfg.LCUrls {
			chosts[i] = u.Host
		}
		cfg.ClientTLSInfo, err = transport.SelfCert(path.Join(cfg.Dir, "fixtures/client"), chosts)
		if err != nil {
			plog.Fatalf("could not get certs (%v)", err)
		}
	} else if cfg.ClientAutoTLS {
		plog.Warningf("ignoring client auto TLS since certs given")
	}

	sctxs = make(map[string]*serveCtx)
	for _, u := range cfg.LCUrls {
		sctx := newServeCtx()

		if u.Scheme == "http" {
			if !cfg.ClientTLSInfo.Empty() {
				plog.Warningf("The scheme of client url %s is HTTP while peer key/cert files are presented. Ignored key/cert files.", u.String())
			}
			if cfg.ClientTLSInfo.ClientCertAuth {
				plog.Warningf("The scheme of client url %s is HTTP while client cert auth (--client-cert-auth) is enabled. Ignored client cert auth for this url.", u.String())
			}
		}
		if u.Scheme == "https" && cfg.ClientTLSInfo.Empty() {
			return nil, fmt.Errorf("TLS key/cert (--cert-file, --key-file) must be provided for client url %s with HTTPs scheme", u.String())
		}

		sctx.secure = u.Scheme == "https"
		sctx.insecure = !sctx.secure
		if oldctx := sctxs[u.Host]; oldctx != nil {
			oldctx.secure = oldctx.secure || sctx.secure
			oldctx.insecure = oldctx.insecure || sctx.insecure
			continue
		}

		if sctx.l, err = net.Listen("tcp", u.Host); err != nil {
			return nil, err
		}

		if fdLimit, fderr := runtimeutil.FDLimit(); fderr == nil {
			if fdLimit <= reservedInternalFDNum {
				plog.Fatalf("file descriptor limit[%d] of etcd process is too low, and should be set higher than %d to ensure internal usage", fdLimit, reservedInternalFDNum)
			}
			sctx.l = transport.LimitListener(sctx.l, int(fdLimit-reservedInternalFDNum))
		}

		if sctx.l, err = transport.NewKeepAliveListener(sctx.l, "tcp", nil); err != nil {
			return nil, err
		}

		plog.Info("listening for client requests on ", u.Host)
		defer func() {
			if err != nil {
				sctx.l.Close()
				plog.Info("stopping listening for client requests on ", u.Host)
			}
		}()
		sctxs[u.Host] = sctx
	}
	return sctxs, nil
}

func (e *Etcd) serve() (err error) {
	var ctlscfg *tls.Config
	if !e.cfg.ClientTLSInfo.Empty() {
		plog.Infof("ClientTLS: %s", e.cfg.ClientTLSInfo)
		if ctlscfg, err = e.cfg.ClientTLSInfo.ServerConfig(); err != nil {
			return err
		}
	}

	if e.cfg.CorsInfo.String() != "" {
		plog.Infof("cors = %s", e.cfg.CorsInfo)
	}

	// Start the peer server in a goroutine
	ph := v2http.NewPeerHandler(e.Server)
	for _, l := range e.Peers {
		go func(l net.Listener) {
			e.errc <- servePeerHTTP(l, ph)
		}(l)
	}

	// Start a client server goroutine for each listen address
	ch := http.Handler(&cors.CORSHandler{
		Handler: v2http.NewClientHandler(e.Server, e.Server.Cfg.ReqTimeout()),
		Info:    e.cfg.CorsInfo,
	})
	for _, sctx := range e.sctxs {
		// read timeout does not work with http close notify
		// TODO: https://github.com/golang/go/issues/9524
		go func(s *serveCtx) {
			e.errc <- s.serve(e.Server, ctlscfg, ch, e.errc)
		}(sctx)
	}
	return nil
}
<|endoftext|>