<commit_before>package cmd

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"testing"

	"github.com/exercism/cli/config"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
	"github.com/stretchr/testify/assert"
)

func TestDownloadWithoutToken(t *testing.T) {
	cfg := config.Configuration{
		UserViperConfig: viper.New(),
	}

	err := runDownload(cfg, pflag.NewFlagSet("fake", pflag.PanicOnError), []string{})
	if assert.Error(t, err) {
		assert.Regexp(t, "Welcome to Exercism", err.Error())
	}
}

func TestDownloadWithoutFlags(t *testing.T) {
	v := viper.New()
	v.Set("token", "abc123")

	cfg := config.Configuration{
		UserViperConfig: v,
	}

	flags := pflag.NewFlagSet("fake", pflag.PanicOnError)
	setupDownloadFlags(flags)

	err := runDownload(cfg, flags, []string{})
	if assert.Error(t, err) {
		assert.Regexp(t, "need an --exercise name or a solution --uuid", err.Error())
	}
}

func TestDownload(t *testing.T) {
	oldOut := Out
	oldErr := Err
	Out = ioutil.Discard
	Err = ioutil.Discard
	defer func() {
		Out = oldOut
		Err = oldErr
	}()

	tmpDir, err := ioutil.TempDir("", "download-cmd")
	assert.NoError(t, err)

	ts := fakeDownloadServer()
	defer ts.Close()

	v := viper.New()
	v.Set("workspace", tmpDir)
	v.Set("apibaseurl", ts.URL)
	v.Set("token", "abc123")

	cfg := config.Configuration{
		UserViperConfig: v,
	}
	flags := pflag.NewFlagSet("fake", pflag.PanicOnError)
	setupDownloadFlags(flags)
	flags.Set("exercise", "bogus-exercise")

	err = runDownload(cfg, flags, []string{})
	assert.NoError(t, err)

	expectedFiles := []struct {
		desc     string
		path     string
		contents string
	}{
		{
			desc:     "a file in the exercise root directory",
			path:     filepath.Join(tmpDir, "bogus-track", "bogus-exercise", "file-1.txt"),
			contents: "this is file 1",
		},
		{
			desc:     "a file in a subdirectory",
			path:     filepath.Join(tmpDir, "bogus-track", "bogus-exercise", "subdir", "file-2.txt"),
			contents: "this is file 2",
		},
		{
			desc:     "the solution metadata file",
			path:     filepath.Join(tmpDir, "bogus-track", "bogus-exercise", ".solution.json"),
			contents: `{"track":"bogus-track","exercise":"bogus-exercise","id":"bogus-id","url":"","handle":"alice","is_requester":true,"auto_approve":false}`,
		},
	}

	for _, file := range expectedFiles {
		t.Run(file.desc, func(t *testing.T) {
			b, err := ioutil.ReadFile(file.path)
			assert.NoError(t, err)
			assert.Equal(t, file.contents, string(b))
		})
	}

	path := filepath.Join(tmpDir, "bogus-track", "bogus-exercise", "file-3.txt")
	_, err = os.Lstat(path)
	assert.True(t, os.IsNotExist(err), "It should not write the file if empty.")
}

func fakeDownloadServer() *httptest.Server {
	mux := http.NewServeMux()
	server := httptest.NewServer(mux)

	path1 := "file-1.txt"
	mux.HandleFunc("/"+path1, func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, "this is file 1")
	})

	path2 := "subdir/file-2.txt"
	mux.HandleFunc("/"+path2, func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, "this is file 2")
	})

	path3 := "file-3.txt"
	mux.HandleFunc("/"+path3, func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, "")
	})

	payloadBody := fmt.Sprintf(payloadTemplate, server.URL+"/", path1, path2, path3)
	mux.HandleFunc("/solutions/latest", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, payloadBody)
	})

	return server
}

const payloadTemplate = `
{
	"solution": {
		"id": "bogus-id",
		"user": {
			"handle": "alice",
			"is_requester": true
		},
		"exercise": {
			"id": "bogus-exercise",
			"instructions_url": "http://example.com/bogus-exercise",
			"auto_approve": false,
			"track": {
				"id": "bogus-track",
				"language": "Bogus Language"
			}
		},
		"file_download_base_url": "%s",
		"files": [
		"%s",
		"%s",
		"%s"
		],
		"iteration": {
			"submitted_at": "2017-08-21t10:11:12.130z"
		}
	}
}
`
<commit_msg>Extract downloaded files assertion<commit_after>package cmd

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"testing"

	"github.com/exercism/cli/config"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
	"github.com/stretchr/testify/assert"
)

func TestDownloadWithoutToken(t *testing.T) {
	cfg := config.Configuration{
		UserViperConfig: viper.New(),
	}

	err := runDownload(cfg, pflag.NewFlagSet("fake", pflag.PanicOnError), []string{})
	if assert.Error(t, err) {
		assert.Regexp(t, "Welcome to Exercism", err.Error())
	}
}

func TestDownloadWithoutFlags(t *testing.T) {
	v := viper.New()
	v.Set("token", "abc123")

	cfg := config.Configuration{
		UserViperConfig: v,
	}

	flags := pflag.NewFlagSet("fake", pflag.PanicOnError)
	setupDownloadFlags(flags)

	err := runDownload(cfg, flags, []string{})
	if assert.Error(t, err) {
		assert.Regexp(t, "need an --exercise name or a solution --uuid", err.Error())
	}
}

func TestDownload(t *testing.T) {
	oldOut := Out
	oldErr := Err
	Out = ioutil.Discard
	Err = ioutil.Discard
	defer func() {
		Out = oldOut
		Err = oldErr
	}()

	tmpDir, err := ioutil.TempDir("", "download-cmd")
	assert.NoError(t, err)

	ts := fakeDownloadServer(requestorSelf)
	defer ts.Close()

	v := viper.New()
	v.Set("workspace", tmpDir)
	v.Set("apibaseurl", ts.URL)
	v.Set("token", "abc123")

	cfg := config.Configuration{
		UserViperConfig: v,
	}
	flags := pflag.NewFlagSet("fake", pflag.PanicOnError)
	setupDownloadFlags(flags)
	flags.Set("exercise", "bogus-exercise")

	err = runDownload(cfg, flags, []string{})
	assert.NoError(t, err)

	assertDownloadedCorrectFiles(t, tmpDir)
}

func fakeDownloadServer(requestor string) *httptest.Server {
	mux := http.NewServeMux()
	server := httptest.NewServer(mux)

	path1 := "file-1.txt"
	mux.HandleFunc("/"+path1, func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, "this is file 1")
	})

	path2 := "subdir/file-2.txt"
	mux.HandleFunc("/"+path2, func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, "this is file 2")
	})

	path3 := "file-3.txt"
	mux.HandleFunc("/"+path3, func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, "")
	})

	payloadBody := fmt.Sprintf(payloadTemplate, requestor, server.URL+"/", path1, path2, path3)
	mux.HandleFunc("/solutions/latest", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, payloadBody)
	})

	return server
}

func assertDownloadedCorrectFiles(t *testing.T, targetDir string) {
	expectedFiles := []struct {
		desc     string
		path     string
		contents string
	}{
		{
			desc:     "a file in the exercise root directory",
			path:     filepath.Join(targetDir, "bogus-track", "bogus-exercise", "file-1.txt"),
			contents: "this is file 1",
		},
		{
			desc:     "a file in a subdirectory",
			path:     filepath.Join(targetDir, "bogus-track", "bogus-exercise", "subdir", "file-2.txt"),
			contents: "this is file 2",
		},
		{
			desc:     "the solution metadata file",
			path:     filepath.Join(targetDir, "bogus-track", "bogus-exercise", ".solution.json"),
			contents: `{"track":"bogus-track","exercise":"bogus-exercise","id":"bogus-id","url":"","handle":"alice","is_requester":true,"auto_approve":false}`,
		},
	}

	for _, file := range expectedFiles {
		t.Run(file.desc, func(t *testing.T) {
			b, err := ioutil.ReadFile(file.path)
			assert.NoError(t, err)
			assert.Equal(t, file.contents, string(b))
		})
	}

	path := filepath.Join(targetDir, "bogus-track", "bogus-exercise", "file-3.txt")
	_, err := os.Lstat(path)
	assert.True(t, os.IsNotExist(err), "It should not write the file if empty.")
}

const requestorSelf = "true"
const requestorOther = "false"

const payloadTemplate = `
{
	"solution": {
		"id": "bogus-id",
		"user": {
			"handle": "alice",
			"is_requester": %s
		},
		"exercise": {
			"id": "bogus-exercise",
			"instructions_url": "http://example.com/bogus-exercise",
			"auto_approve": false,
			"track": {
				"id": "bogus-track",
				"language": "Bogus Language"
			}
		},
		"file_download_base_url": "%s",
		"files": [
		"%s",
		"%s",
		"%s"
		],
		"iteration": {
			"submitted_at": "2017-08-21t10:11:12.130z"
		}
	}
}
`
<|endoftext|>


<commit_before>/* Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strings"
)

const (
	cmdName = "fdroidcl"

	repoName = "repo"
	repoURL  = "https://f-droid.org/repo"
)

// A Command is an implementation of a go command
// like go build or go fix.
type Command struct {
	// Run runs the command.
	// The args are the arguments after the command name.
	Run func(args []string)

	// UsageLine is the one-line usage message.
	// The first word in the line is taken to be the command name.
	UsageLine string

	// Short is the short description.
	Short string

	// Flag is a set of flags specific to this command.
	Flag flag.FlagSet
}

// Name returns the command's name: the first word in the usage line.
func (c *Command) Name() string {
	name := c.UsageLine
	i := strings.Index(name, " ")
	if i >= 0 {
		name = name[:i]
	}
	return name
}

func (c *Command) Usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s %s\n", cmdName, c.UsageLine)
	anyFlags := false
	c.Flag.VisitAll(func(f *flag.Flag) { anyFlags = true })
	if anyFlags {
		fmt.Fprintf(os.Stderr, "\nAvailable options:\n")
		c.Flag.PrintDefaults()
	}
	os.Exit(2)
}

func init() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: fdroidcl [-h] <command> [<args>]")
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "Available commands:")
		fmt.Fprintln(os.Stderr, "   update              Update the index")
		fmt.Fprintln(os.Stderr, "   list                List all available apps")
		fmt.Fprintln(os.Stderr, "   search <regexp...>  Search available apps")
		fmt.Fprintln(os.Stderr, "   show <appid...>     Show detailed info of an app")
		fmt.Fprintln(os.Stderr, "   devices             List connected devices")
		fmt.Fprintln(os.Stderr, "   installed           List installed apps")
	}
}

// Commands lists the available commands.
var commands = []*Command{
	cmdUpdate,
	cmdList,
	cmdSearch,
	cmdShow,
	cmdDevices,
	cmdInstalled,
}

func main() {
	flag.Parse()
	args := flag.Args()

	if len(args) < 1 {
		flag.Usage()
		os.Exit(2)
	}

	for _, cmd := range commands {
		if cmd.Name() != args[0] {
			continue
		}
		cmd.Flag.Usage = func() { cmd.Usage() }
		cmd.Flag.Parse(args[1:])
		args = cmd.Flag.Args()
		cmd.Run(args)
		os.Exit(0)
	}

	switch args[0] {
	default:
		log.Printf("Unrecognised command '%s'\n\n", args[0])
		flag.Usage()
		os.Exit(2)
	}
}
<commit_msg>Don't hardcode global usage<commit_after>/* Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strings"
)

const (
	cmdName = "fdroidcl"

	repoName = "repo"
	repoURL  = "https://f-droid.org/repo"
)

// A Command is an implementation of a go command
// like go build or go fix.
type Command struct {
	// Run runs the command.
	// The args are the arguments after the command name.
	Run func(args []string)

	// UsageLine is the one-line usage message.
	// The first word in the line is taken to be the command name.
	UsageLine string

	// Short is the short description.
	Short string

	// Flag is a set of flags specific to this command.
	Flag flag.FlagSet
}

// Name returns the command's name: the first word in the usage line.
func (c *Command) Name() string {
	name := c.UsageLine
	i := strings.Index(name, " ")
	if i >= 0 {
		name = name[:i]
	}
	return name
}

func (c *Command) Usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s %s\n", cmdName, c.UsageLine)
	anyFlags := false
	c.Flag.VisitAll(func(f *flag.Flag) { anyFlags = true })
	if anyFlags {
		fmt.Fprintf(os.Stderr, "\nAvailable options:\n")
		c.Flag.PrintDefaults()
	}
	os.Exit(2)
}

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [-h] <command> [<args>]\n\n", cmdName)
		fmt.Fprintf(os.Stderr, "Available commands:\n")
		maxUsageLen := 0
		for _, c := range commands {
			if len(c.UsageLine) > maxUsageLen {
				maxUsageLen = len(c.UsageLine)
			}
		}
		for _, c := range commands {
			fmt.Fprintf(os.Stderr, "   %s%s  %s\n", c.UsageLine,
				strings.Repeat(" ", maxUsageLen-len(c.UsageLine)), c.Short)
		}
	}
}

// Commands lists the available commands.
var commands = []*Command{
	cmdUpdate,
	cmdList,
	cmdSearch,
	cmdShow,
	cmdDevices,
	cmdInstalled,
}

func main() {
	flag.Parse()
	args := flag.Args()

	if len(args) < 1 {
		flag.Usage()
		os.Exit(2)
	}

	for _, cmd := range commands {
		if cmd.Name() != args[0] {
			continue
		}
		cmd.Flag.Usage = func() { cmd.Usage() }
		cmd.Flag.Parse(args[1:])
		args = cmd.Flag.Args()
		cmd.Run(args)
		os.Exit(0)
	}

	switch args[0] {
	default:
		log.Printf("Unrecognised command '%s'\n\n", args[0])
		flag.Usage()
		os.Exit(2)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"html/template"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	g "github.com/gogits/git"
	"github.com/lemmi/ghfs"
	"github.com/lemmi/glubcms"
	"github.com/pkg/errors"
	"github.com/raymondbutcher/tidyhtml"
)

const (
	tmplPath = "templates"
)

type stackTracer interface {
	StackTrace() errors.StackTrace
}

func HttpError(w http.ResponseWriter, code int, logErr error) {
	//switch err := logErr.(type) {
	//case stackTracer:
	//	log.Printf("%+v", err.StackTrace())
	//default:
	//	log.Print(err)
	//}
	log.Print(logErr)
	http.Error(w, http.StatusText(code), code)
}

func parseTemplates(fs http.FileSystem) (*template.Template, error) {
	dir, err := fs.Open(tmplPath)
	if err != nil {
		return nil, errors.Wrapf(err, "Cannot open directory: %q", tmplPath)
	}
	defer dir.Close()
	tmain := template.New("main")
	fis, err := dir.Readdir(-1)
	if err != nil {
		return nil, errors.Wrapf(err, "Cannot read directory: %q", tmplPath)
	}
	for _, fi := range fis {
		if !strings.HasSuffix(fi.Name(), ".tmpl") {
			continue
		}
		fpath := filepath.Join(tmplPath, fi.Name())
		data, err := fs.Open(fpath)
		if err != nil {
			return nil, errors.Wrapf(err, "Cannot open file: %q", fpath)
		}
		databytes, err := ioutil.ReadAll(data)
		data.Close()
		if err != nil {
			return nil, errors.Wrapf(err, "Cannot read file: %q", fpath)
		}

		tname := strings.TrimSuffix(fi.Name(), ".tmpl")
		_, err = tmain.New(tname).Parse(string(databytes))
		if err != nil {
			return nil, errors.Wrapf(err, "Cannot parse template: %q", fpath)
		}
	}

	return tmain, nil
}

// Handling of an article or menu entry
// TODO:
// - use http.FileSystem only

type pageHandler struct {
	c *g.Commit
}

func newPageHandler(c *g.Commit) http.Handler {
	return pageHandler{c}
}
func (h pageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	tmpl := template.Must(parseTemplates(ghfs.FromCommit(h.c)))
	stree, err := h.c.Tree.SubTree("pages")
	if err != nil {
		HttpError(w, http.StatusNotFound, errors.Wrap(err, "Pages"))
		return
	}

	p, err := glubcms.PageFromDir(ghfs.FromCommit(h.c, stree), r.URL.Path)
	if err != nil {
		HttpError(w, http.StatusInternalServerError, errors.Wrapf(err, "page generation failed"))
		return
	}
	buf := bytes.Buffer{}
	if err := tmpl.ExecuteTemplate(&buf, "main", p); err != nil {
		HttpError(w, http.StatusInternalServerError, errors.Wrapf(err, "template execution failed: %q", r.URL.Path))
		return
	}
	tbuf := bytes.Buffer{}
	if err := tidyhtml.Copy(&tbuf, &buf); err != nil {
		HttpError(w, http.StatusInternalServerError, errors.Wrapf(err, "tidyhtml failed: %q", r.URL.Path))
		return
	}
	w.Header().Set("Content-Type", "text/html")
	http.ServeContent(w, r, "", h.c.Author.When, bytes.NewReader(tbuf.Bytes()))
}

// Main handling of the site
// TODO:
// - parse meta entries first
// - choose handler based on that, not by path

type handler struct {
	prefix string
}

func newHandler(prefix string) handler {
	return handler{
		prefix: prefix,
	}
}

func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	path, err := filepath.Abs(h.prefix)
	if err != nil {
		HttpError(w, http.StatusInternalServerError, errors.Wrap(err, "filepath.Abs("+h.prefix+")"))
		return
	}

	repo, err := g.OpenRepository(path)
	if err != nil {
		HttpError(w, http.StatusInternalServerError, errors.Wrap(err, "g.OpenRepository("+path+")"))
		return
	}

	commit, err := repo.GetCommitOfBranch("master")
	if err != nil {
		HttpError(w, http.StatusInternalServerError, errors.Wrap(err, "Can not open master branch"))
		return
	}

	mux := http.NewServeMux()

	staticHandler := glubcms.NewStaticHandler(ghfs.FromCommit(commit))

	mux.Handle("/static/", staticHandler)
	mux.Handle("/robots.txt", staticHandler.Cd("/static"))
	mux.Handle("/favicon.ico", staticHandler.Cd("/static"))
	mux.Handle("/", newPageHandler(commit))
	w.Header().Set("ETag", strings.Trim(commit.Id.String(), "\""))
	w.Header().Set("Cache-Control", "max-age=32")
	mux.ServeHTTP(w, r)
}

func main() {
	prefix := flag.String("prefix", "../example_page", "path to the root dir")
	addr := flag.String("bind", "localhost:8080", "address or path to bind to")
	network := flag.String("net", "tcp", `"tcp", "tcp4", "tcp6", "unix" or "unixpacket"`)
	flag.Parse()
	ln, err := net.Listen(*network, *addr)
	if err != nil {
		panic(err)
	}
	defer ln.Close()
	if strings.HasPrefix(*network, "unix") {
		err = os.Chmod(*addr, 0666)
	}
	if err != nil {
		panic(err)
	}
	log.Println("Starting")
	log.Fatal(http.Serve(ln, newHandler(*prefix)))
}
<commit_msg>gcserver: add filesystem backend, -git switch<commit_after>package main

import (
	"bytes"
	"flag"
	"html/template"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	g "github.com/gogits/git"
	"github.com/lemmi/ghfs"
	"github.com/lemmi/glubcms"
	"github.com/pkg/errors"
	"github.com/raymondbutcher/tidyhtml"
)

const (
	tmplPath = "templates"
)

type stackTracer interface {
	StackTrace() errors.StackTrace
}

func HttpError(w http.ResponseWriter, code int, logErr error) {
	//switch err := logErr.(type) {
	//case stackTracer:
	//	log.Printf("%+v", err.StackTrace())
	//default:
	//	log.Print(err)
	//}
	log.Print(logErr)
	http.Error(w, http.StatusText(code), code)
}

func parseTemplates(fs http.FileSystem) (*template.Template, error) {
	dir, err := fs.Open(tmplPath)
	if err != nil {
		return nil, errors.Wrapf(err, "Cannot open directory: %q", tmplPath)
	}
	defer dir.Close()
	tmain := template.New("main")
	fis, err := dir.Readdir(-1)
	if err != nil {
		return nil, errors.Wrapf(err, "Cannot read directory: %q", tmplPath)
	}
	for _, fi := range fis {
		if !strings.HasSuffix(fi.Name(), ".tmpl") {
			continue
		}
		fpath := filepath.Join(tmplPath, fi.Name())
		data, err := fs.Open(fpath)
		if err != nil {
			return nil, errors.Wrapf(err, "Cannot open file: %q", fpath)
		}
		databytes, err := ioutil.ReadAll(data)
		data.Close()
		if err != nil {
			return nil, errors.Wrapf(err, "Cannot read file: %q", fpath)
		}

		tname := strings.TrimSuffix(fi.Name(), ".tmpl")
		_, err = tmain.New(tname).Parse(string(databytes))
		if err != nil {
			return nil, errors.Wrapf(err, "Cannot parse template: %q", fpath)
		}
	}

	return tmain, nil
}

// Handling of an article or menu entry
// TODO:
// - use http.FileSystem only

type pageHandler struct {
	glubcms.StaticHandler
}

func (h pageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	tmpl := template.Must(parseTemplates(h))
	pages := h.Cd("pages")
	//stree, err := h.c.Tree.SubTree("pages")
	//if err != nil {
	//	HttpError(w, http.StatusNotFound, errors.Wrap(err, "Pages"))
	//	return
	//}

	p, err := glubcms.PageFromDir(pages, r.URL.Path)
	if err != nil {
		HttpError(w, http.StatusInternalServerError, errors.Wrapf(err, "page generation failed"))
		return
	}
	buf := bytes.Buffer{}
	if err := tmpl.ExecuteTemplate(&buf, "main", p); err != nil {
		HttpError(w, http.StatusInternalServerError, errors.Wrapf(err, "template execution failed: %q", r.URL.Path))
		return
	}
	tbuf := bytes.Buffer{}
	if err := tidyhtml.Copy(&tbuf, &buf); err != nil {
		HttpError(w, http.StatusInternalServerError, errors.Wrapf(err, "tidyhtml failed: %q", r.URL.Path))
		return
	}
	w.Header().Set("Content-Type", "text/html")
	http.ServeContent(w, r, "", p.ModTime, bytes.NewReader(tbuf.Bytes()))
}

// Main handling of the site
// TODO:
// - parse meta entries first
// - choose handler based on that, not by path

type handler struct {
	prefix string
	git    bool
}

func newHandler(prefix string, git bool) handler {
	return handler{
		prefix: prefix,
		git:    git,
	}
}

func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	path, err := filepath.Abs(h.prefix)
	if err != nil {
		HttpError(w, http.StatusInternalServerError, errors.Wrap(err, "filepath.Abs("+h.prefix+")"))
		return
	}

	var fs http.FileSystem

	if h.git {
		repo, err := g.OpenRepository(path)
		if err != nil {
			HttpError(w, http.StatusInternalServerError, errors.Wrap(err, "g.OpenRepository("+path+")"))
			return
		}

		commit, err := repo.GetCommitOfBranch("master")
		if err != nil {
			HttpError(w, http.StatusInternalServerError, errors.Wrap(err, "Can not open master branch"))
			return
		}
		fs = ghfs.FromCommit(commit)
		w.Header().Set("ETag", strings.Trim(commit.Id.String(), "\""))
	} else {
		fs = http.Dir(path)
	}

	staticHandler := glubcms.NewStaticHandler(fs)

	mux := http.NewServeMux()
	mux.Handle("/static/", staticHandler)
	mux.Handle("/robots.txt", staticHandler.Cd("/static"))
	mux.Handle("/favicon.ico", staticHandler.Cd("/static"))
	mux.Handle("/", pageHandler{staticHandler})
	w.Header().Set("Cache-Control", "max-age=32")
	mux.ServeHTTP(w, r)
}

func main() {
	prefix := flag.String("prefix", "../example_page", "path to the root dir")
	addr := flag.String("bind", "localhost:8080", "address or path to bind to")
	network := flag.String("net", "tcp", `"tcp", "tcp4", "tcp6", "unix" or "unixpacket"`)
	git := flag.Bool("git", false, "prefix is a git repo")
	flag.Parse()
	ln, err := net.Listen(*network, *addr)
	if err != nil {
		panic(err)
	}
	defer ln.Close()
	if strings.HasPrefix(*network, "unix") {
		err = os.Chmod(*addr, 0666)
	}
	if err != nil {
		panic(err)
	}
	log.Println("Starting")
	log.Fatal(http.Serve(ln, newHandler(*prefix, *git)))
}
<|endoftext|>


<commit_before>// Copyright (C) 2015 Nippon Telegraph and Telephone Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"net"

	"github.com/spf13/cobra"

	api "github.com/osrg/gobgp/api"
	"github.com/osrg/gobgp/internal/pkg/apiutil"
	"github.com/osrg/gobgp/pkg/packet/bgp"
)

func makeMonitorRouteArgs(p *api.Path, showIdentifier bgp.BGPAddPathMode) []interface{} {
	pathStr := make([]interface{}, 0)

	// Title
	title := "ROUTE"
	if p.IsWithdraw {
		title = "DELROUTE"
	}
	pathStr = append(pathStr, title)

	// NLRI
	// If Add-Path required, append Path Identifier.
	nlri, _ := apiutil.GetNativeNlri(p)
	if showIdentifier != bgp.BGP_ADD_PATH_NONE {
		pathStr = append(pathStr, p.GetIdentifier())
	}
	pathStr = append(pathStr, nlri)

	attrs, _ := apiutil.GetNativePathAttributes(p)
	// Next Hop
	nexthop := "fictitious"
	if n := getNextHopFromPathAttributes(attrs); n != nil {
		nexthop = n.String()
	}
	pathStr = append(pathStr, nexthop)

	// AS_PATH
	aspathstr := func() string {
		for _, attr := range attrs {
			switch a := attr.(type) {
			case *bgp.PathAttributeAsPath:
				return bgp.AsPathString(a)
			}
		}
		return ""
	}()
	pathStr = append(pathStr, aspathstr)

	// Path Attributes
	pathStr = append(pathStr, getPathAttributeString(nlri, attrs))

	return pathStr
}

func monitorRoute(pathList []*api.Path, showIdentifier bgp.BGPAddPathMode) {
	var pathStrs [][]interface{}

	for _, p := range pathList {
		pathStrs = append(pathStrs, makeMonitorRouteArgs(p, showIdentifier))
	}

	format := "[%s] %s via %s aspath [%s] attrs %s\n"
	if showIdentifier != bgp.BGP_ADD_PATH_NONE {
		format = "[%s] %d:%s via %s aspath [%s] attrs %s\n"
	}
	for _, pathStr := range pathStrs {
		fmt.Printf(format, pathStr...)
	}
}

func newMonitorCmd() *cobra.Command {

	var current bool

	monitor := func(recver interface {
		Recv() (*api.MonitorTableResponse, error)
	}, showIdentifier bgp.BGPAddPathMode) {
		for {
			r, err := recver.Recv()
			if err == io.EOF {
				break
			} else if err != nil {
				exitWithError(err)
			}
			if globalOpts.Json {
				j, _ := json.Marshal(apiutil.NewDestination(&api.Destination{Paths: []*api.Path{r.Path}}))
				fmt.Println(string(j))
			} else {
				monitorRoute([]*api.Path{r.Path}, bgp.BGP_ADD_PATH_NONE)
			}
		}
	}

	ribCmd := &cobra.Command{
		Use: cmdRib,
		Run: func(cmd *cobra.Command, args []string) {
			family, err := checkAddressFamily(ipv4UC)
			if err != nil {
				exitWithError(err)
			}
			recver, err := client.MonitorTable(ctx, &api.MonitorTableRequest{
				Type:    api.Resource_GLOBAL,
				Family:  family,
				Current: current,
			})
			if err != nil {
				exitWithError(err)
			}
			monitor(recver, bgp.BGP_ADD_PATH_NONE)
		},
	}
	ribCmd.PersistentFlags().StringVarP(&subOpts.AddressFamily, "address-family", "a", "", "address family")

	globalCmd := &cobra.Command{
		Use: cmdGlobal,
	}
	globalCmd.AddCommand(ribCmd)

	neighborCmd := &cobra.Command{
		Use:  fmt.Sprintf("%s [<neighbor address>]", cmdNeighbor),
		Args: cobra.MaximumNArgs(1),
		Run: func(cmd *cobra.Command, args []string) {
			name := ""
			if len(args) > 0 {
				name = args[0]
			}
			stream, err := client.MonitorPeer(ctx, &api.MonitorPeerRequest{
				Address: name,
				Current: current,
			})
			if err != nil {
				exitWithError(err)
			}
			for {
				r, err := stream.Recv()
				if err == io.EOF {
					break
				} else if err != nil {
					exitWithError(err)
				}
				s := r.Peer
				if globalOpts.Json {
					j, _ := json.Marshal(s)
					fmt.Println(string(j))
				} else {
					addr := s.Conf.NeighborAddress
					if s.Conf.NeighborInterface != "" {
						addr = fmt.Sprintf("%s(%s)", addr, s.Conf.NeighborInterface)
					}
					fmt.Printf("[NEIGH] %s fsm: %s admin: %s\n", addr, s.State.SessionState, s.State.AdminState)
				}
			}
		},
	}

	adjInCmd := &cobra.Command{
		Use: cmdAdjIn,
		Run: func(cmd *cobra.Command, args []string) {
			name := ""
			if len(args) > 0 {
				remoteIP := net.ParseIP(args[0])
				if remoteIP == nil {
					exitWithError(fmt.Errorf("invalid ip address: %s", args[0]))
				}
				name = args[0]
			}
			family, err := checkAddressFamily(ipv4UC)
			if err != nil {
				exitWithError(err)
			}
			recver, err := client.MonitorTable(ctx, &api.MonitorTableRequest{
				Type:    api.Resource_ADJ_IN,
				Name:    name,
				Family:  family,
				Current: current,
			})
			if err != nil {
				exitWithError(err)
			}
			monitor(recver, bgp.BGP_ADD_PATH_RECEIVE)
		},
	}
	adjInCmd.PersistentFlags().StringVarP(&subOpts.AddressFamily, "address-family", "a", "", "address family")

	monitorCmd := &cobra.Command{
		Use: cmdMonitor,
	}
	monitorCmd.AddCommand(globalCmd)
	monitorCmd.AddCommand(neighborCmd)
	monitorCmd.AddCommand(adjInCmd)

	monitorCmd.PersistentFlags().BoolVarP(&current, "current", "", false, "dump current contents")

	return monitorCmd
}
<commit_msg>added timestamp<commit_after>// Copyright (C) 2015 Nippon Telegraph and Telephone Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"net"
	"time"

	"github.com/spf13/cobra"

	api "github.com/osrg/gobgp/api"
	"github.com/osrg/gobgp/internal/pkg/apiutil"
	"github.com/osrg/gobgp/pkg/packet/bgp"
)

func makeMonitorRouteArgs(p *api.Path, showIdentifier bgp.BGPAddPathMode) []interface{} {
	pathStr := make([]interface{}, 0)

	// Title
	title := "ROUTE"
	if p.IsWithdraw {
		title = "DELROUTE"
	}
	pathStr = append(pathStr, title)

	// NLRI
	// If Add-Path required, append Path Identifier.
	nlri, _ := apiutil.GetNativeNlri(p)
	if showIdentifier != bgp.BGP_ADD_PATH_NONE {
		pathStr = append(pathStr, p.GetIdentifier())
	}
	pathStr = append(pathStr, nlri)

	attrs, _ := apiutil.GetNativePathAttributes(p)
	// Next Hop
	nexthop := "fictitious"
	if n := getNextHopFromPathAttributes(attrs); n != nil {
		nexthop = n.String()
	}
	pathStr = append(pathStr, nexthop)

	// AS_PATH
	aspathstr := func() string {
		for _, attr := range attrs {
			switch a := attr.(type) {
			case *bgp.PathAttributeAsPath:
				return bgp.AsPathString(a)
			}
		}
		return ""
	}()
	pathStr = append(pathStr, aspathstr)

	// Path Attributes
	pathStr = append(pathStr, getPathAttributeString(nlri, attrs))

	return pathStr
}

func monitorRoute(pathList []*api.Path, showIdentifier bgp.BGPAddPathMode) {
	var pathStrs [][]interface{}

	for _, p := range pathList {
		pathStrs = append(pathStrs, makeMonitorRouteArgs(p, showIdentifier))
	}

	format := time.Now().UTC().Format(time.RFC3339)
	if showIdentifier == bgp.BGP_ADD_PATH_NONE {
		format += " [%s] %s via %s aspath [%s] attrs %s\n"
	} else {
		format += " [%s] %d:%s via %s aspath [%s] attrs %s\n"
	}
	for _, pathStr := range pathStrs {
		fmt.Printf(format, pathStr...)
	}
}

func newMonitorCmd() *cobra.Command {

	var current bool

	monitor := func(recver interface {
		Recv() (*api.MonitorTableResponse, error)
	}, showIdentifier bgp.BGPAddPathMode) {
		for {
			r, err := recver.Recv()
			if err == io.EOF {
				break
			} else if err != nil {
				exitWithError(err)
			}
			if globalOpts.Json {
				j, _ := json.Marshal(apiutil.NewDestination(&api.Destination{Paths: []*api.Path{r.Path}}))
				fmt.Println(string(j))
			} else {
				monitorRoute([]*api.Path{r.Path}, bgp.BGP_ADD_PATH_NONE)
			}
		}
	}

	ribCmd := &cobra.Command{
		Use: cmdRib,
		Run: func(cmd *cobra.Command, args []string) {
			family, err := checkAddressFamily(ipv4UC)
			if err != nil {
				exitWithError(err)
			}
			recver, err := client.MonitorTable(ctx, &api.MonitorTableRequest{
				Type:    api.Resource_GLOBAL,
				Family:  family,
				Current: current,
			})
			if err != nil {
				exitWithError(err)
			}
			monitor(recver, bgp.BGP_ADD_PATH_NONE)
		},
	}
	ribCmd.PersistentFlags().StringVarP(&subOpts.AddressFamily, "address-family", "a", "", "address family")

	globalCmd := &cobra.Command{
		Use: cmdGlobal,
	}
	globalCmd.AddCommand(ribCmd)

	neighborCmd := &cobra.Command{
		Use:  fmt.Sprintf("%s [<neighbor address>]", cmdNeighbor),
		Args: cobra.MaximumNArgs(1),
		Run: func(cmd *cobra.Command, args []string) {
			name := ""
			if len(args) > 0 {
				name = args[0]
			}
			stream, err := client.MonitorPeer(ctx, &api.MonitorPeerRequest{
				Address: name,
				Current: current,
			})
			if err != nil {
				exitWithError(err)
			}
			for {
				r, err := stream.Recv()
				if err == io.EOF {
					break
				} else if err != nil {
					exitWithError(err)
				}
				s := r.Peer
				if globalOpts.Json {
					j, _ := json.Marshal(s)
					fmt.Println(string(j))
				} else {
					addr := s.Conf.NeighborAddress
					if s.Conf.NeighborInterface != "" {
						addr = fmt.Sprintf("%s(%s)", addr, s.Conf.NeighborInterface)
					}
					fmt.Printf("%s [NEIGH] %s fsm: %s admin: %s\n", time.Now().UTC().Format(time.RFC3339), addr, s.State.SessionState, s.State.AdminState)
				}
			}
		},
	}

	adjInCmd := &cobra.Command{
		Use: cmdAdjIn,
		Run: func(cmd *cobra.Command, args []string) {
			name := ""
			if len(args) > 0 {
				remoteIP := net.ParseIP(args[0])
				if remoteIP == nil {
					exitWithError(fmt.Errorf("invalid ip address: %s", args[0]))
				}
				name = args[0]
			}
			family, err := checkAddressFamily(ipv4UC)
			if err != nil {
				exitWithError(err)
			}
			recver, err := client.MonitorTable(ctx, &api.MonitorTableRequest{
				Type:    api.Resource_ADJ_IN,
				Name:    name,
				Family:  family,
				Current: current,
			})
			if err != nil {
				exitWithError(err)
			}
			monitor(recver, bgp.BGP_ADD_PATH_RECEIVE)
		},
	}
	adjInCmd.PersistentFlags().StringVarP(&subOpts.AddressFamily, "address-family", "a", "", "address family")

	monitorCmd := &cobra.Command{
		Use: cmdMonitor,
	}
	monitorCmd.AddCommand(globalCmd)
	monitorCmd.AddCommand(neighborCmd)
	monitorCmd.AddCommand(adjInCmd)

	monitorCmd.PersistentFlags().BoolVarP(&current, "current", "", false, "dump current contents")

	return monitorCmd
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"fmt"
	"os"
	"path/filepath"
	"time"

	"launchpad.net/gnuflag"
	"launchpad.net/loggo"
	"launchpad.net/tomb"

	"launchpad.net/juju-core/agent"
	"launchpad.net/juju-core/charm"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/instance"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/names"
	"launchpad.net/juju-core/provider"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/api"
	"launchpad.net/juju-core/state/api/params"
	"launchpad.net/juju-core/state/apiserver"
	"launchpad.net/juju-core/upstart"
	"launchpad.net/juju-core/worker"
	"launchpad.net/juju-core/worker/addressupdater"
	"launchpad.net/juju-core/worker/cleaner"
	"launchpad.net/juju-core/worker/deployer"
	"launchpad.net/juju-core/worker/firewaller"
	"launchpad.net/juju-core/worker/localstorage"
	"launchpad.net/juju-core/worker/logger"
	"launchpad.net/juju-core/worker/machiner"
	"launchpad.net/juju-core/worker/minunitsworker"
	"launchpad.net/juju-core/worker/provisioner"
	"launchpad.net/juju-core/worker/resumer"
	"launchpad.net/juju-core/worker/upgrader"
)

type workerRunner interface {
	worker.Worker
	StartWorker(id string, startFunc func() (worker.Worker, error)) error
	StopWorker(id string) error
}

var newRunner = func(isFatal func(error) bool, moreImportant func(e0, e1 error) bool) workerRunner {
	return worker.NewRunner(isFatal, moreImportant)
}

const bootstrapMachineId = "0"

var retryDelay = 3 * time.Second

// MachineAgent is a cmd.Command responsible for running a machine agent.
type MachineAgent struct {
	cmd.CommandBase
	tomb      tomb.Tomb
	Conf      AgentConf
	MachineId string
	runner    workerRunner
}

// Info returns usage information for the command.
func (a *MachineAgent) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "machine",
		Purpose: "run a juju machine agent",
	}
}

func (a *MachineAgent) SetFlags(f *gnuflag.FlagSet) {
	a.Conf.addFlags(f)
	f.StringVar(&a.MachineId, "machine-id", "", "id of the machine to run")
}

// Init initializes the command for running.
func (a *MachineAgent) Init(args []string) error {
	if !names.IsMachine(a.MachineId) {
		return fmt.Errorf("--machine-id option must be set, and expects a non-negative integer")
	}
	if err := a.Conf.checkArgs(args); err != nil {
		return err
	}
	a.runner = newRunner(isFatal, moreImportant)
	return nil
}

// Wait waits for the machine agent to finish.
func (a *MachineAgent) Wait() error {
	return a.tomb.Wait()
}

// Stop stops the machine agent.
func (a *MachineAgent) Stop() error {
	a.runner.Kill()
	return a.tomb.Wait()
}

// Run runs a machine agent.
func (a *MachineAgent) Run(_ *cmd.Context) error {
	// Due to changes in the logging, and needing to care about old
	// environments that have been upgraded, we need to explicitly remove the
	// file writer if one has been added, otherwise we will get duplicate
	// lines of all logging in the log file.
	loggo.RemoveWriter("logfile")
	defer a.tomb.Done()
	log.Infof("machine agent %v start", a.Tag())
	if err := a.Conf.read(a.Tag()); err != nil {
		return err
	}
	charm.CacheDir = filepath.Join(a.Conf.dataDir, "charmcache")

	// ensureStateWorker ensures that there is a worker that
	// connects to the state that runs within itself all the workers
	// that need a state connection. Unless we're bootstrapping, we
	// need to connect to the API server to find out if we need to
	// call this, so we make the APIWorker call it when necessary if
	// the machine requires it. Note that ensureStateWorker can be
	// called many times - StartWorker does nothing if there is
	// already a worker started with the given name.
	ensureStateWorker := func() {
		a.runner.StartWorker("state", a.StateWorker)
	}
	// We might be bootstrapping, and the API server is not
	// running yet. If so, make sure we run a state worker instead.
	if a.MachineId == bootstrapMachineId {
		// TODO(rog) When we have HA, we only want to do this
		// when we really are bootstrapping - once other
		// instances of the API server have been started, we
		// should follow the normal course of things and ignore
		// the fact that this was once the bootstrap machine.
		log.Infof("Starting StateWorker for machine-0")
		ensureStateWorker()
	}
	a.runner.StartWorker("api", func() (worker.Worker, error) {
		return a.APIWorker(ensureStateWorker)
	})
	err := a.runner.Wait()
	if err == worker.ErrTerminateAgent {
		err = a.uninstallAgent()
	}
	err = agentDone(err)
	a.tomb.Kill(err)
	return err
}

// APIWorker returns a Worker that connects to the API and starts any
// workers that need an API connection.
//
// If a state worker is necessary, APIWorker calls ensureStateWorker.
func (a *MachineAgent) APIWorker(ensureStateWorker func()) (worker.Worker, error) {
	agentConfig := a.Conf.config
	st, entity, err := openAPIState(agentConfig, a)
	if err != nil {
		return nil, err
	}
	reportOpenedAPI(st)
	for _, job := range entity.Jobs() {
		if job.NeedsState() {
			ensureStateWorker()
			break
		}
	}
	runner := newRunner(connectionIsFatal(st), moreImportant)
	runner.StartWorker("machiner", func() (worker.Worker, error) {
		return machiner.NewMachiner(st.Machiner(), agentConfig), nil
	})
	runner.StartWorker("upgrader", func() (worker.Worker, error) {
		return upgrader.NewUpgrader(st.Upgrader(), agentConfig), nil
	})
	runner.StartWorker("logger", func() (worker.Worker, error) {
		return logger.NewLogger(st.Logger(), agentConfig), nil
	})
	// At this stage, since we don't embed LXC containers, just start an lxc
	// provisioner task for non-lxc containers.  Since we have only LXC
	// containers and normal machines, this effectively means that we only
	// have an LXC provisioner when we have a normally provisioned machine
	// (through the environ-provisioner).  With the upcoming advent of KVM
	// containers, it is likely that we will want an LXC provisioner on a KVM
	// machine, and once we get nested LXC containers, we can remove this
	// check.
	//
	// TODO(dimitern) 2013-09-25 bug #1230289
	// Create jobs for container providers, rather than
	// using the provider and container type like this.
	providerType := agentConfig.Value(agent.ProviderType)
	if providerType != provider.Local && entity.ContainerType() != instance.LXC {
		workerName := fmt.Sprintf("%s-provisioner", provisioner.LXC)
		runner.StartWorker(workerName, func() (worker.Worker, error) {
			return provisioner.NewProvisioner(provisioner.LXC, st.Provisioner(), agentConfig), nil
		})
	}
	for _, job := range entity.Jobs() {
		switch job {
		case params.JobHostUnits:
			runner.StartWorker("deployer", func() (worker.Worker, error) {
				apiDeployer := st.Deployer()
				context := newDeployContext(apiDeployer, agentConfig)
				return deployer.NewDeployer(apiDeployer, context), nil
			})
		case params.JobManageEnviron:
			runner.StartWorker("environ-provisioner", func() (worker.Worker, error) {
				return provisioner.NewProvisioner(provisioner.ENVIRON, st.Provisioner(), agentConfig), nil
			})
			// TODO(dimitern): Add firewaller here, when using the API.
		case params.JobManageState:
			// Not yet implemented with the API.
		default:
			// TODO(dimitern): Once all workers moved over to using
			// the API, report "unknown job type" here.
		}
	}
	return newCloseWorker(runner, st), nil // Note: a worker.Runner is itself a worker.Worker.
}

// StateJobs returns a worker running all the workers that require
// a *state.State cofnnection.
func (a *MachineAgent) StateWorker() (worker.Worker, error) {
	agentConfig := a.Conf.config
	st, entity, err := openState(agentConfig, a)
	if err != nil {
		return nil, err
	}
	reportOpenedState(st)
	m := entity.(*state.Machine)

	runner := newRunner(connectionIsFatal(st), moreImportant)
	// Take advantage of special knowledge here in that we will only ever want
	// the storage provider on one machine, and that is the "bootstrap" node.
	providerType := agentConfig.Value(agent.ProviderType)
	if (providerType == provider.Local || providerType == provider.Null) && m.Id() == bootstrapMachineId {
		runner.StartWorker("local-storage", func() (worker.Worker, error) {
			// TODO(axw) 2013-09-24 bug #1229507
			// Make another job to enable storage.
			// There's nothing special about this.
			return localstorage.NewWorker(agentConfig), nil
		})
	}
	for _, job := range m.Jobs() {
		switch job {
		case state.JobHostUnits:
			// Implemented in APIWorker.
		case state.JobManageEnviron:
			// TODO(axw) 2013-09-24 bug #1229506
			// Make another job to enable the firewaller. Not all environments
			// are capable of managing ports centrally.
			runner.StartWorker("firewaller", func() (worker.Worker, error) {
				return firewaller.NewFirewaller(st), nil
			})
			runner.StartWorker("addressupdater", func() (worker.Worker, error) {
				return addressupdater.NewWorker(st), nil
			})
		case state.JobManageState:
			runner.StartWorker("apiserver", func() (worker.Worker, error) {
				// If the configuration does not have the required information,
				// it is currently not a recoverable error, so we kill the whole
				// agent, potentially enabling human intervention to fix
				// the agent's configuration file. In the future, we may retrieve
				// the state server certificate and key from the state, and
				// this should then change.
				port, cert, key := a.Conf.config.APIServerDetails()
				if len(cert) == 0 || len(key) == 0 {
					return nil, &fatalError{"configuration does not have state server cert/key"}
				}
				return apiserver.NewServer(st, fmt.Sprintf(":%d", port), cert, key)
			})
			runner.StartWorker("cleaner", func() (worker.Worker, error) {
				return cleaner.NewCleaner(st), nil
			})
			runner.StartWorker("resumer", func() (worker.Worker, error) {
				// The action of resumer is so subtle that it is not tested,
				// because we can't figure out how to do so without brutalising
				// the transaction log.
				return resumer.NewResumer(st), nil
			})
			runner.StartWorker("minunitsworker", func() (worker.Worker, error) {
				return minunitsworker.NewMinUnitsWorker(st), nil
			})
		default:
			log.Warningf("ignoring unknown job %q", job)
		}
	}
	return newCloseWorker(runner, st), nil
}

func (a *MachineAgent) Entity(st *state.State) (AgentState, error) {
	m, err := st.Machine(a.MachineId)
	if err != nil {
		return nil, err
	}
	// Check the machine nonce as provisioned matches the agent.Conf value.
	if !m.CheckProvisioned(a.Conf.config.Nonce()) {
		// The agent is running on a different machine to the one it
		// should be according to state. It must stop immediately.
		log.Errorf("running machine %v agent on inappropriate instance", m)
		return nil, worker.ErrTerminateAgent
	}
	return m, nil
}

func (a *MachineAgent) Tag() string {
	return names.MachineTag(a.MachineId)
}

func (m *MachineAgent) uninstallAgent() error {
	// TODO(axw) get this from agent config when it's available
	name := os.Getenv("UPSTART_JOB")
	if name != "" {
		return upstart.NewService(name).Remove()
	}
	return nil
}

// Below pieces are used for testing,to give us access to the *State opened
// by the agent, and allow us to trigger syncs without waiting 5s for them
// to happen automatically.

var stateReporter chan<- *state.State

func reportOpenedState(st *state.State) {
	select {
	case stateReporter <- st:
	default:
	}
}

func sendOpenedStates(dst chan<- *state.State) (undo func()) {
	var original chan<- *state.State
	original, stateReporter = stateReporter, dst
	return func() { stateReporter = original }
}

var apiReporter chan<- *api.State

func reportOpenedAPI(st *api.State) {
	select {
	case apiReporter <- st:
	default:
	}
}
func sendOpenedAPIs(dst chan<- *api.State) (undo func()) {
	var original chan<- *api.State
	original, apiReporter = apiReporter, dst
	return func() { apiReporter = original }
}
<commit_msg>Bump up GOMAXPROCS when we are running the apiserver<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"time"

	"launchpad.net/gnuflag"
	"launchpad.net/loggo"
	"launchpad.net/tomb"

	"launchpad.net/juju-core/agent"
	"launchpad.net/juju-core/charm"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/instance"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/names"
	"launchpad.net/juju-core/provider"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/api"
	"launchpad.net/juju-core/state/api/params"
	"launchpad.net/juju-core/state/apiserver"
	"launchpad.net/juju-core/upstart"
	"launchpad.net/juju-core/worker"
	"launchpad.net/juju-core/worker/addressupdater"
	"launchpad.net/juju-core/worker/cleaner"
	"launchpad.net/juju-core/worker/deployer"
	"launchpad.net/juju-core/worker/firewaller"
	"launchpad.net/juju-core/worker/localstorage"
	"launchpad.net/juju-core/worker/logger"
	"launchpad.net/juju-core/worker/machiner"
	"launchpad.net/juju-core/worker/minunitsworker"
	"launchpad.net/juju-core/worker/provisioner"
	"launchpad.net/juju-core/worker/resumer"
	"launchpad.net/juju-core/worker/upgrader"
)

var machineLogger = loggo.GetLogger("juju.jujud.machine")

type workerRunner interface {
	worker.Worker
	StartWorker(id string, startFunc func() (worker.Worker, error)) error
	StopWorker(id string) error
}

var newRunner = func(isFatal func(error) bool, moreImportant func(e0, e1 error) bool) workerRunner {
	return worker.NewRunner(isFatal, moreImportant)
}

const bootstrapMachineId = "0"

var retryDelay = 3 * time.Second

// MachineAgent is a cmd.Command responsible for running a machine agent.
type MachineAgent struct {
	cmd.CommandBase
	tomb      tomb.Tomb
	Conf      AgentConf
	MachineId string
	runner    workerRunner
}

// Info returns usage information for the command.
func (a *MachineAgent) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "machine",
		Purpose: "run a juju machine agent",
	}
}

func (a *MachineAgent) SetFlags(f *gnuflag.FlagSet) {
	a.Conf.addFlags(f)
	f.StringVar(&a.MachineId, "machine-id", "", "id of the machine to run")
}

// Init initializes the command for running.
func (a *MachineAgent) Init(args []string) error {
	if !names.IsMachine(a.MachineId) {
		return fmt.Errorf("--machine-id option must be set, and expects a non-negative integer")
	}
	if err := a.Conf.checkArgs(args); err != nil {
		return err
	}
	a.runner = newRunner(isFatal, moreImportant)
	return nil
}

// Wait waits for the machine agent to finish.
func (a *MachineAgent) Wait() error {
	return a.tomb.Wait()
}

// Stop stops the machine agent.
func (a *MachineAgent) Stop() error {
	a.runner.Kill()
	return a.tomb.Wait()
}

// Run runs a machine agent.
func (a *MachineAgent) Run(_ *cmd.Context) error {
	// Due to changes in the logging, and needing to care about old
	// environments that have been upgraded, we need to explicitly remove the
	// file writer if one has been added, otherwise we will get duplicate
	// lines of all logging in the log file.
	loggo.RemoveWriter("logfile")
	defer a.tomb.Done()
	log.Infof("machine agent %v start", a.Tag())
	if err := a.Conf.read(a.Tag()); err != nil {
		return err
	}
	charm.CacheDir = filepath.Join(a.Conf.dataDir, "charmcache")

	// ensureStateWorker ensures that there is a worker that
	// connects to the state that runs within itself all the workers
	// that need a state connection. Unless we're bootstrapping, we
	// need to connect to the API server to find out if we need to
	// call this, so we make the APIWorker call it when necessary if
	// the machine requires it. Note that ensureStateWorker can be
	// called many times - StartWorker does nothing if there is
	// already a worker started with the given name.
	ensureStateWorker := func() {
		a.runner.StartWorker("state", a.StateWorker)
	}
	// We might be bootstrapping, and the API server is not
	// running yet. If so, make sure we run a state worker instead.
	if a.MachineId == bootstrapMachineId {
		// TODO(rog) When we have HA, we only want to do this
		// when we really are bootstrapping - once other
		// instances of the API server have been started, we
		// should follow the normal course of things and ignore
		// the fact that this was once the bootstrap machine.
		log.Infof("Starting StateWorker for machine-0")
		ensureStateWorker()
	}
	a.runner.StartWorker("api", func() (worker.Worker, error) {
		return a.APIWorker(ensureStateWorker)
	})
	err := a.runner.Wait()
	if err == worker.ErrTerminateAgent {
		err = a.uninstallAgent()
	}
	err = agentDone(err)
	a.tomb.Kill(err)
	return err
}

// APIWorker returns a Worker that connects to the API and starts any
// workers that need an API connection.
//
// If a state worker is necessary, APIWorker calls ensureStateWorker.
func (a *MachineAgent) APIWorker(ensureStateWorker func()) (worker.Worker, error) {
	agentConfig := a.Conf.config
	st, entity, err := openAPIState(agentConfig, a)
	if err != nil {
		return nil, err
	}
	reportOpenedAPI(st)
	for _, job := range entity.Jobs() {
		if job.NeedsState() {
			ensureStateWorker()
			break
		}
	}
	runner := newRunner(connectionIsFatal(st), moreImportant)
	runner.StartWorker("machiner", func() (worker.Worker, error) {
		return machiner.NewMachiner(st.Machiner(), agentConfig), nil
	})
	runner.StartWorker("upgrader", func() (worker.Worker, error) {
		return upgrader.NewUpgrader(st.Upgrader(), agentConfig), nil
	})
	runner.StartWorker("logger", func() (worker.Worker, error) {
		return logger.NewLogger(st.Logger(), agentConfig), nil
	})
	// At this stage, since we don't embed LXC containers, just start an lxc
	// provisioner task for non-lxc containers.  Since we have only LXC
	// containers and normal machines, this effectively means that we only
	// have an LXC provisioner when we have a normally provisioned machine
	// (through the environ-provisioner).  With the upcoming advent of KVM
	// containers, it is likely that we will want an LXC provisioner on a KVM
	// machine, and once we get nested LXC containers, we can remove this
	// check.
	//
	// TODO(dimitern) 2013-09-25 bug #1230289
	// Create jobs for container providers, rather than
	// using the provider and container type like this.
	providerType := agentConfig.Value(agent.ProviderType)
	if providerType != provider.Local && entity.ContainerType() != instance.LXC {
		workerName := fmt.Sprintf("%s-provisioner", provisioner.LXC)
		runner.StartWorker(workerName, func() (worker.Worker, error) {
			return provisioner.NewProvisioner(provisioner.LXC, st.Provisioner(), agentConfig), nil
		})
	}
	for _, job := range entity.Jobs() {
		switch job {
		case params.JobHostUnits:
			runner.StartWorker("deployer", func() (worker.Worker, error) {
				apiDeployer := st.Deployer()
				context := newDeployContext(apiDeployer, agentConfig)
				return deployer.NewDeployer(apiDeployer, context), nil
			})
		case params.JobManageEnviron:
			runner.StartWorker("environ-provisioner", func() (worker.Worker, error) {
				return provisioner.NewProvisioner(provisioner.ENVIRON, st.Provisioner(), agentConfig), nil
			})
			// TODO(dimitern): Add firewaller here, when using the API.
		case params.JobManageState:
			// Not yet implemented with the API.
		default:
			// TODO(dimitern): Once all workers moved over to using
			// the API, report "unknown job type" here.
		}
	}
	return newCloseWorker(runner, st), nil // Note: a worker.Runner is itself a worker.Worker.
}

// StateJobs returns a worker running all the workers that require
// a *state.State cofnnection.
func (a *MachineAgent) StateWorker() (worker.Worker, error) {
	agentConfig := a.Conf.config
	st, entity, err := openState(agentConfig, a)
	if err != nil {
		return nil, err
	}
	reportOpenedState(st)
	m := entity.(*state.Machine)

	runner := newRunner(connectionIsFatal(st), moreImportant)
	// Take advantage of special knowledge here in that we will only ever want
	// the storage provider on one machine, and that is the "bootstrap" node.
	providerType := agentConfig.Value(agent.ProviderType)
	if (providerType == provider.Local || providerType == provider.Null) && m.Id() == bootstrapMachineId {
		runner.StartWorker("local-storage", func() (worker.Worker, error) {
			// TODO(axw) 2013-09-24 bug #1229507
			// Make another job to enable storage.
			// There's nothing special about this.
			return localstorage.NewWorker(agentConfig), nil
		})
	}
	for _, job := range m.Jobs() {
		switch job {
		case state.JobHostUnits:
			// Implemented in APIWorker.
		case state.JobManageEnviron:
			// TODO(axw) 2013-09-24 bug #1229506
			// Make another job to enable the firewaller. Not all environments
			// are capable of managing ports centrally.
			runner.StartWorker("firewaller", func() (worker.Worker, error) {
				return firewaller.NewFirewaller(st), nil
			})
			runner.StartWorker("addressupdater", func() (worker.Worker, error) {
				return addressupdater.NewWorker(st), nil
			})
		case state.JobManageState:
			machineLogger.Debugf("setting GOMAXPROCS to %d to run the apiserver", runtime.NumCPU())
			runtime.GOMAXPROCS(runtime.NumCPU())
			runner.StartWorker("apiserver", func() (worker.Worker, error) {
				// If the configuration does not have the required information,
				// it is currently not a recoverable error, so we kill the whole
				// agent, potentially enabling human intervention to fix
				// the agent's configuration file. In the future, we may retrieve
				// the state server certificate and key from the state, and
				// this should then change.
				port, cert, key := a.Conf.config.APIServerDetails()
				if len(cert) == 0 || len(key) == 0 {
					return nil, &fatalError{"configuration does not have state server cert/key"}
				}
				return apiserver.NewServer(st, fmt.Sprintf(":%d", port), cert, key)
			})
			runner.StartWorker("cleaner", func() (worker.Worker, error) {
				return cleaner.NewCleaner(st), nil
			})
			runner.StartWorker("resumer", func() (worker.Worker, error) {
				// The action of resumer is so subtle that it is not tested,
				// because we can't figure out how to do so without brutalising
				// the transaction log.
				return resumer.NewResumer(st), nil
			})
			runner.StartWorker("minunitsworker", func() (worker.Worker, error) {
				return minunitsworker.NewMinUnitsWorker(st), nil
			})
		default:
			log.Warningf("ignoring unknown job %q", job)
		}
	}
	return newCloseWorker(runner, st), nil
}

func (a *MachineAgent) Entity(st *state.State) (AgentState, error) {
	m, err := st.Machine(a.MachineId)
	if err != nil {
		return nil, err
	}
	// Check the machine nonce as provisioned matches the agent.Conf value.
	if !m.CheckProvisioned(a.Conf.config.Nonce()) {
		// The agent is running on a different machine to the one it
		// should be according to state. It must stop immediately.
		log.Errorf("running machine %v agent on inappropriate instance", m)
		return nil, worker.ErrTerminateAgent
	}
	return m, nil
}

func (a *MachineAgent) Tag() string {
	return names.MachineTag(a.MachineId)
}

func (m *MachineAgent) uninstallAgent() error {
	// TODO(axw) get this from agent config when it's available
	name := os.Getenv("UPSTART_JOB")
	if name != "" {
		return upstart.NewService(name).Remove()
	}
	return nil
}

// Below pieces are used for testing,to give us access to the *State opened
// by the agent, and allow us to trigger syncs without waiting 5s for them
// to happen automatically.

var stateReporter chan<- *state.State

func reportOpenedState(st *state.State) {
	select {
	case stateReporter <- st:
	default:
	}
}

func sendOpenedStates(dst chan<- *state.State) (undo func()) {
	var original chan<- *state.State
	original, stateReporter = stateReporter, dst
	return func() { stateReporter = original }
}

var apiReporter chan<- *api.State

func reportOpenedAPI(st *api.State) {
	select {
	case apiReporter <- st:
	default:
	}
}
func sendOpenedAPIs(dst chan<- *api.State) (undo func()) {
	var original chan<- *api.State
	original, apiReporter = apiReporter, dst
	return func() { apiReporter = original }
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"strconv"

	"github.com/docker/go-plugins-helpers/volume"

	"github.com/cloudflavor/miniovol/pkg/client"
	"github.com/cloudflavor/miniovol/pkg/driver"
)

const (
	socketAddress = "/run/docker/plugins/miniovol.sock"
	rootID        = 0
)

// Error implements the error interface and uses it to return a generic
// validation error for environment variables that are mandatory.
type Error struct {
	envVariable string
}

func (e Error) Error() string {
	return fmt.Sprintf("environment variable %s cannot be empty", e.envVariable)
}

func newErrEmptyEnvVar(v string) error {
	return Error{
		envVariable: v,
	}
}

func getEnvDetails() (serverURI, accessKeyID, secretAccessKey, bucket string, secure bool, err error) {
	secure, err = strconv.ParseBool(os.Getenv("SECURE"))
	if err != nil {
		return
	}
	serverURI = os.Getenv("MINIO_SERVER")
	if serverURI == "" {
		err = newErrEmptyEnvVar("MINIO_SERVER")
		return
	}

	accessKeyID = os.Getenv("MINIO_ACCESSKEY")
	if accessKeyID == "" {
		err = newErrEmptyEnvVar("MINIO_ACCESSKEY")
		return
	}

	secretAccessKey = os.Getenv("MINIO_SECRETKEY")
	if secretAccessKey == "" {
		err = newErrEmptyEnvVar("MINIO_SECRETKEY")
		return
	}
	bucket = os.Getenv("MINIO_BUCKET")

	return
}

func main() {
	serverURI, accessKeyID, secretAccessKey, bucket, secure, err := getEnvDetails()
	if err != nil {
		log.Fatalf("An error occured while fetching environment settings: %s", err)
	}
	c, err := client.NewMinioClient(serverURI, accessKeyID, secretAccessKey, bucket, secure)
	if err != nil {
		log.Fatalf("An error occured while creating a new Minio client: %s", err)
	}
	d := driver.NewMinioDriver(c, secure)
	h := volume.NewHandler(d)
	h.ServeUnix(socketAddress, rootID)
}
<commit_msg>shorten var<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"strconv"

	"github.com/docker/go-plugins-helpers/volume"

	"github.com/cloudflavor/miniovol/pkg/client"
	"github.com/cloudflavor/miniovol/pkg/driver"
)

const (
	socketAddress = "/run/docker/plugins/miniovol.sock"
	rootID        = 0
)

// Error implements the error interface and uses it to return a generic
// validation error for environment variables that are mandatory.
type Error struct {
	envVar string
}

func (e Error) Error() string {
	return fmt.Sprintf("environment variable %s cannot be empty", e.envVar)
}

func newErrEmptyEnvVar(v string) error {
	return Error{
		envVar: v,
	}
}

func getEnvDetails() (serverURI, accessKeyID, secretAccessKey, bucket string, secure bool, err error) {
	secure, err = strconv.ParseBool(os.Getenv("SECURE"))
	if err != nil {
		return
	}
	serverURI = os.Getenv("MINIO_SERVER")
	if serverURI == "" {
		err = newErrEmptyEnvVar("MINIO_SERVER")
		return
	}
	accessKeyID = os.Getenv("MINIO_ACCESSKEY")
	if accessKeyID == "" {
		err = newErrEmptyEnvVar("MINIO_ACCESSKEY")
		return
	}
	secretAccessKey = os.Getenv("MINIO_SECRETKEY")
	if secretAccessKey == "" {
		err = newErrEmptyEnvVar("MINIO_SECRETKEY")
		return
	}
	// bucket can be empty, since we're gonna generate a new bucket name if there
	// wasn't one specified.
	bucket = os.Getenv("MINIO_BUCKET")

	return
}

func main() {
	serverURI, accessKeyID, secretAccessKey, bucket, secure, err := getEnvDetails()
	if err != nil {
		log.Fatalf("An error occured while fetching environment settings: %s", err)
	}
	c, err := client.NewMinioClient(serverURI, accessKeyID, secretAccessKey, bucket, secure)
	if err != nil {
		log.Fatalf("An error occured while creating a new Minio client: %s", err)
	}
	d := driver.NewMinioDriver(c, secure)
	h := volume.NewHandler(d)
	h.ServeUnix(socketAddress, rootID)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/toomore/gogrs/realtime"
	"github.com/toomore/gogrs/utils"
)

// TaipeiNow show Taipei Now time.
func TaipeiNow() time.Time {
	d := time.Now()
	return time.Date(d.Year(), d.Month(), d.Day(), 0, 0, 0, 0, utils.TaipeiTimeZone)
}

func prettyprint(data realtime.Data) string {
	return fmt.Sprintf("%s(%s) $%.2f(%.2f) %.0f/%.0f [%s] [%s %s]",
		data.Info.Name, data.Info.No,
		data.Price, data.Price-data.Open, data.Volume, data.VolumeAcc,
		data.TradeTime, data.SysInfo["sysDate"], data.SysInfo["sysTime"])
}

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

var twseNo = flag.String("twse", "", "TWSE NO.")
var otcNo = flag.String("otc", "", "OTC NO.")
var index = flag.Bool("index", false, "Show Index.")

func main() {
	flag.Parse()
	queue := []*realtime.StockRealTime{}
	if *twseNo != "" {
		for _, no := range strings.Split(*twseNo, ",") {
			queue = append(queue, realtime.NewTWSE(no, TaipeiNow()))
		}
	}
	if *otcNo != "" {
		for _, no := range strings.Split(*otcNo, ",") {
			queue = append(queue, realtime.NewOTC(no, TaipeiNow()))
		}
	}
	if *index {
		queue = append(queue, []*realtime.StockRealTime{realtime.NewWeight(TaipeiNow()),
			realtime.NewOTCI(TaipeiNow()), realtime.NewFRMSA(TaipeiNow())}...)
	}
	result := make(chan string)
	var wg sync.WaitGroup
	wg.Add(len(queue))
	for _, r := range queue {
		go func(r *realtime.StockRealTime) {
			data, _ := r.Get()
			result <- prettyprint(data)
		}(r)
	}
	go func() {
		for v := range result {
			wg.Done()
			log.Println(v)
		}
	}()
	wg.Wait()
	//flag.PrintDefaults()
}
<commit_msg>Add ./cmd/realtime docs.<commit_after>// 擷取盤中即時資訊與大盤、上櫃、寶島指數
//
/*

Install:

	go install -u github.com/toomore/gogrs/cmd/realtime

Usage:

	realtime [flags]

The flags are:

	-twse
		上市股票代碼，可使用 ',' 分隔多組代碼，例：2618,2329
	-otc
		上櫃股票代碼，可使用 ',' 分隔多組代碼，例：8446,2719
	-index
		顯示大盤、上櫃、寶島指數（default: false）

範例

	/realtime -twse=2618,2329 -otc=8446,2719 -index=true

回傳內容

	2015/04/29 23:43:13 櫃買指數(o00) $144.72(-1.15) 1271/23250 [2015-04-29 13:33:00 +0800 CST] [20150429 23:43:13]
	2015/04/29 23:43:13 發行量加權股價指數(t00) $9853.83(-80.99) 5742/113762 [2015-04-29 13:33:00 +0800 CST] [20150429 23:43:13]
	2015/04/29 23:43:13 寶島股價指數(FRMSA) $11416.04(-93.58) 0/0 [2015-04-29 13:33:00 +0800 CST] [20150429 23:43:13]
	2015/04/29 23:43:13 華研(8446) $124.00(1.00) 6/293 [2015-04-29 14:30:00 +0800 CST] [20150429 23:43:13]
	2015/04/29 23:43:13 長榮航(2618) $24.20(-0.55) 666/15618 [2015-04-29 14:30:00 +0800 CST] [20150429 23:43:13]
	2015/04/29 23:43:13 燦星旅(2719) $24.15(0.25) 6/83 [2015-04-29 14:30:00 +0800 CST] [20150429 23:43:13]
	2015/04/29 23:43:13 華泰(2329) $15.95(0.50) 648/19995 [2015-04-29 14:30:00 +0800 CST] [20150429 23:43:13]

*/
package main

import (
	"flag"
	"fmt"
	"log"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/toomore/gogrs/realtime"
	"github.com/toomore/gogrs/utils"
)

// TaipeiNow show Taipei Now time.
func TaipeiNow() time.Time {
	d := time.Now()
	return time.Date(d.Year(), d.Month(), d.Day(), 0, 0, 0, 0, utils.TaipeiTimeZone)
}

func prettyprint(data realtime.Data) string {
	return fmt.Sprintf("%s(%s) $%.2f(%.2f) %.0f/%.0f [%s] [%s %s]",
		data.Info.Name, data.Info.No,
		data.Price, data.Price-data.Open, data.Volume, data.VolumeAcc,
		data.TradeTime, data.SysInfo["sysDate"], data.SysInfo["sysTime"])
}

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

var twseNo = flag.String("twse", "", "上市股票代碼，可使用 ',' 分隔多組代碼，例：2618,2329")
var otcNo = flag.String("otc", "", "上櫃股票代碼，可使用 ',' 分隔多組代碼，例：8446,2719")
var index = flag.Bool("index", false, "顯示大盤、上櫃、寶島指數（default: false）")

func main() {
	flag.Parse()
	queue := []*realtime.StockRealTime{}
	if *twseNo != "" {
		for _, no := range strings.Split(*twseNo, ",") {
			queue = append(queue, realtime.NewTWSE(no, TaipeiNow()))
		}
	}
	if *otcNo != "" {
		for _, no := range strings.Split(*otcNo, ",") {
			queue = append(queue, realtime.NewOTC(no, TaipeiNow()))
		}
	}
	if *index {
		queue = append(queue, []*realtime.StockRealTime{realtime.NewWeight(TaipeiNow()),
			realtime.NewOTCI(TaipeiNow()), realtime.NewFRMSA(TaipeiNow())}...)
	}
	result := make(chan string)
	var wg sync.WaitGroup
	wg.Add(len(queue))
	for _, r := range queue {
		go func(r *realtime.StockRealTime) {
			data, _ := r.Get()
			result <- prettyprint(data)
		}(r)
	}
	go func() {
		for v := range result {
			wg.Done()
			log.Println(v)
		}
	}()
	wg.Wait()
	if len(queue) == 0 {
		flag.PrintDefaults()
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// TODO: syscall.SIGABRT is not defined for Plan 9 (issue #11975)

// +build !plan9

// The stress utility is intended for catching of episodic failures.
// It runs a given process in parallel in a loop and collects any failures.
// Usage:
// 	$ stress ./fmt.test -test.run=TestSometing -test.cpu=10
// You can also specify a number of parallel processes with -p flag;
// instruct the utility to not kill hanged processes for gdb attach;
// or specify the failure output you are looking for (if you want to
// ignore some other episodic failures).
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"syscall"
	"time"
)

var (
	flagP       = flag.Int("p", runtime.NumCPU(), "run `N` processes in parallel")
	flagTimeout = flag.Duration("timeout", 10*time.Minute, "timeout each process after `duration`")
	flagKill    = flag.Bool("kill", true, "kill timed out processes if true, otherwise just print pid (to attach with gdb)")
	flagFailure = flag.String("failure", "", "fail only if output matches `regexp`")
	flagIgnore  = flag.String("ignore", "", "ignore failure if output matches `regexp`")
)

func main() {
	flag.Parse()
	if *flagP <= 0 || *flagTimeout <= 0 || len(flag.Args()) == 0 {
		flag.Usage()
		os.Exit(1)
	}
	var failureRe, ignoreRe *regexp.Regexp
	if *flagFailure != "" {
		var err error
		if failureRe, err = regexp.Compile(*flagFailure); err != nil {
			fmt.Println("bad failure regexp:", err)
			os.Exit(1)
		}
	}
	if *flagIgnore != "" {
		var err error
		if ignoreRe, err = regexp.Compile(*flagIgnore); err != nil {
			fmt.Println("bad ignore regexp:", err)
			os.Exit(1)
		}
	}
	res := make(chan []byte)
	for i := 0; i < *flagP; i++ {
		go func() {
			for {
				cmd := exec.Command(flag.Args()[0], flag.Args()[1:]...)
				done := make(chan bool)
				if *flagTimeout > 0 {
					go func() {
						select {
						case <-done:
							return
						case <-time.After(*flagTimeout):
						}
						if !*flagKill {
							fmt.Printf("process %v timed out\n", cmd.Process.Pid)
							return
						}
						cmd.Process.Signal(syscall.SIGABRT)
						select {
						case <-done:
							return
						case <-time.After(10 * time.Second):
						}
						cmd.Process.Kill()
					}()
				}
				out, err := cmd.CombinedOutput()
				close(done)
				if err != nil && (failureRe == nil || failureRe.Match(out)) && (ignoreRe == nil || !ignoreRe.Match(out)) {
					out = append(out, fmt.Sprintf("\n\nERROR: %v\n", err)...)
				} else {
					out = []byte{}
				}
				res <- out
			}
		}()
	}
	runs, fails := 0, 0
	ticker := time.NewTicker(5 * time.Second).C
	for {
		select {
		case out := <-res:
			runs++
			if len(out) == 0 {
				continue
			}
			fails++
			f, err := ioutil.TempFile("", "go-stress")
			if err != nil {
				fmt.Printf("failed to create temp file: %v\n", err)
				os.Exit(1)
			}
			f.Write(out)
			f.Close()
			if len(out) > 2<<10 {
				out = out[:2<<10]
			}
			fmt.Printf("\n%s\n%s\n", f.Name(), out)
		case <-ticker:
			fmt.Printf("%v runs so far, %v failures\n", runs, fails)
		}
	}
}
<commit_msg>x/tools/cmd/stress: build on Plan 9<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The stress utility is intended for catching of episodic failures.
// It runs a given process in parallel in a loop and collects any failures.
// Usage:
// 	$ stress ./fmt.test -test.run=TestSometing -test.cpu=10
// You can also specify a number of parallel processes with -p flag;
// instruct the utility to not kill hanged processes for gdb attach;
// or specify the failure output you are looking for (if you want to
// ignore some other episodic failures).
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"syscall"
	"time"
)

var (
	flagP       = flag.Int("p", runtime.NumCPU(), "run `N` processes in parallel")
	flagTimeout = flag.Duration("timeout", 10*time.Minute, "timeout each process after `duration`")
	flagKill    = flag.Bool("kill", true, "kill timed out processes if true, otherwise just print pid (to attach with gdb)")
	flagFailure = flag.String("failure", "", "fail only if output matches `regexp`")
	flagIgnore  = flag.String("ignore", "", "ignore failure if output matches `regexp`")
)

func main() {
	flag.Parse()
	if *flagP <= 0 || *flagTimeout <= 0 || len(flag.Args()) == 0 {
		flag.Usage()
		os.Exit(1)
	}
	var failureRe, ignoreRe *regexp.Regexp
	if *flagFailure != "" {
		var err error
		if failureRe, err = regexp.Compile(*flagFailure); err != nil {
			fmt.Println("bad failure regexp:", err)
			os.Exit(1)
		}
	}
	if *flagIgnore != "" {
		var err error
		if ignoreRe, err = regexp.Compile(*flagIgnore); err != nil {
			fmt.Println("bad ignore regexp:", err)
			os.Exit(1)
		}
	}
	res := make(chan []byte)
	for i := 0; i < *flagP; i++ {
		go func() {
			for {
				cmd := exec.Command(flag.Args()[0], flag.Args()[1:]...)
				done := make(chan bool)
				if *flagTimeout > 0 {
					go func() {
						select {
						case <-done:
							return
						case <-time.After(*flagTimeout):
						}
						if !*flagKill {
							fmt.Printf("process %v timed out\n", cmd.Process.Pid)
							return
						}
						cmd.Process.Signal(syscall.SIGABRT)
						select {
						case <-done:
							return
						case <-time.After(10 * time.Second):
						}
						cmd.Process.Kill()
					}()
				}
				out, err := cmd.CombinedOutput()
				close(done)
				if err != nil && (failureRe == nil || failureRe.Match(out)) && (ignoreRe == nil || !ignoreRe.Match(out)) {
					out = append(out, fmt.Sprintf("\n\nERROR: %v\n", err)...)
				} else {
					out = []byte{}
				}
				res <- out
			}
		}()
	}
	runs, fails := 0, 0
	ticker := time.NewTicker(5 * time.Second).C
	for {
		select {
		case out := <-res:
			runs++
			if len(out) == 0 {
				continue
			}
			fails++
			f, err := ioutil.TempFile("", "go-stress")
			if err != nil {
				fmt.Printf("failed to create temp file: %v\n", err)
				os.Exit(1)
			}
			f.Write(out)
			f.Close()
			if len(out) > 2<<10 {
				out = out[:2<<10]
			}
			fmt.Printf("\n%s\n%s\n", f.Name(), out)
		case <-ticker:
			fmt.Printf("%v runs so far, %v failures\n", runs, fails)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"

	jira "github.com/andygrunwald/go-jira/v2/cloud"
)

func main() {
	jiraURL := "https://go-jira-opensource.atlassian.net/"

	// Jira docs: https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/
	// Create a new API token: https://id.atlassian.com/manage-profile/security/api-tokens
	tp := jira.BasicAuthTransport{
		Username: "<username>",
		APIToken: "<api-token>",
	}
	client, err := jira.NewClient(jiraURL, tp.Client())
	if err != nil {
		panic(err)
	}

	u, _, err := client.User.GetCurrentUser(context.Background())
	if err != nil {
		panic(err)
	}

	fmt.Printf("Email: %v\n", u.EmailAddress)
	fmt.Println("Success!")
}
<commit_msg>Cloud/Examples: Fix "GetSelf" method to get the current user<commit_after>package main

import (
	"context"
	"fmt"

	jira "github.com/andygrunwald/go-jira/v2/cloud"
)

func main() {
	jiraURL := "https://go-jira-opensource.atlassian.net/"

	// Jira docs: https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/
	// Create a new API token: https://id.atlassian.com/manage-profile/security/api-tokens
	tp := jira.BasicAuthTransport{
		Username: "<username>",
		APIToken: "<api-token>",
	}
	client, err := jira.NewClient(jiraURL, tp.Client())
	if err != nil {
		panic(err)
	}

	u, _, err := client.User.GetSelf(context.Background())
	if err != nil {
		panic(err)
	}

	fmt.Printf("Email: %v\n", u.EmailAddress)
	fmt.Println("Success!")
}
<|endoftext|>


<commit_before>package machine_test

import (
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/state/api"
	"launchpad.net/juju-core/state/api/params"
	"launchpad.net/juju-core/state/apiserver/machine"
)

type agentSuite struct {
	commonSuite
	agent *machine.AgentAPI
}

var _ = Suite(&agentSuite{})

func (s *agentSuite) SetUpTest(c *C) {
	s.commonSuite.SetUpTest(c)

	// Create a machiner API for machine 1.
	api, err := machine.NewAgentAPI(
		s.State,
		s.authorizer,
	)
	c.Assert(err, IsNil)
	s.agent = api
}

func (s *agentSuite) TestAgentFailsWithNonMachineAgentUser(c *C) {
	auth := s.authorizer
	auth.machineAgent = false
	api, err := machine.NewAgentAPI(s.State, auth)
	c.Assert(err, NotNil)
	c.Assert(api, IsNil)
	c.Assert(err, ErrorMatches, "permission denied")
}

func (s *agentSuite) TestAgentFailsWhenNotLoggedIn(c *C) {
	auth := s.authorizer
	auth.loggedIn = false
	api, err := machine.NewAgentAPI(s.State, auth)
	c.Assert(err, NotNil)
	c.Assert(api, IsNil)
	c.Assert(err, ErrorMatches, "not logged in")
}

func (s *agentSuite) TestGetMachines(c *C) {
	err := s.machine1.Destroy()
	c.Assert(err, IsNil)
	results, err := s.agent.GetMachines(params.Machines{
		Ids: []string{"1", "0", "42"},
	})
	c.Assert(err, IsNil)
	c.Assert(results, DeepEquals, params.MachineAgentGetMachinesResults{
		Machines: []params.MachineAgentGetMachinesResult{{
			Life: "dying",
			Jobs: []params.MachineJob{params.JobHostUnits},
		}, {
			Error: &params.Error{
				Code:    api.CodeUnauthorized,
				Message: "permission denied",
			},
		}, {
			Error: &params.Error{
				Code:    api.CodeUnauthorized,
				Message: "permission denied",
			},
		}},
	})
}

func (s *agentSuite) TestGetNotFoundMachine(c *C) {
	err := s.machine1.Destroy()
	c.Assert(err, IsNil)
	err = s.machine1.EnsureDead()
	c.Assert(err, IsNil)
	err = s.machine1.Remove()
	c.Assert(err, IsNil)
	results, err := s.agent.GetMachines(params.Machines{
		Ids: []string{"1"},
	})
	c.Assert(err, IsNil)
	c.Assert(results, DeepEquals, params.MachineAgentGetMachinesResults{
		Machines: []params.MachineAgentGetMachinesResult{{
			Error: &params.Error{
				Code:    api.CodeNotFound,
				Message: "machine 1 not found",
			},
		}},
	})
}
<commit_msg>state/apiserver/machine: fix test build<commit_after>package machine_test

import (
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/state/api"
	"launchpad.net/juju-core/state/api/params"
	"launchpad.net/juju-core/state/apiserver/machine"
)

type agentSuite struct {
	commonSuite
	agent *machine.AgentAPI
}

var _ = Suite(&agentSuite{})

func (s *agentSuite) SetUpTest(c *C) {
	s.commonSuite.SetUpTest(c)

	// Create a machiner API for machine 1.
	api, err := machine.NewAgentAPI(
		s.State,
		s.authorizer,
	)
	c.Assert(err, IsNil)
	s.agent = api
}

func (s *agentSuite) TestAgentFailsWithNonMachineAgentUser(c *C) {
	auth := s.authorizer
	auth.machineAgent = false
	api, err := machine.NewAgentAPI(s.State, auth)
	c.Assert(err, NotNil)
	c.Assert(api, IsNil)
	c.Assert(err, ErrorMatches, "permission denied")
}

func (s *agentSuite) TestAgentFailsWhenNotLoggedIn(c *C) {
	auth := s.authorizer
	auth.loggedIn = false
	api, err := machine.NewAgentAPI(s.State, auth)
	c.Assert(err, NotNil)
	c.Assert(api, IsNil)
	c.Assert(err, ErrorMatches, "not logged in")
}

func (s *agentSuite) TestGetMachines(c *C) {
	err := s.machine1.Destroy()
	c.Assert(err, IsNil)
	results := s.agent.GetMachines(params.Machines{
		Ids: []string{"1", "0", "42"},
	})
	c.Assert(err, IsNil)
	c.Assert(results, DeepEquals, params.MachineAgentGetMachinesResults{
		Machines: []params.MachineAgentGetMachinesResult{{
			Life: "dying",
			Jobs: []params.MachineJob{params.JobHostUnits},
		}, {
			Error: &params.Error{
				Code:    api.CodeUnauthorized,
				Message: "permission denied",
			},
		}, {
			Error: &params.Error{
				Code:    api.CodeUnauthorized,
				Message: "permission denied",
			},
		}},
	})
}

func (s *agentSuite) TestGetNotFoundMachine(c *C) {
	err := s.machine1.Destroy()
	c.Assert(err, IsNil)
	err = s.machine1.EnsureDead()
	c.Assert(err, IsNil)
	err = s.machine1.Remove()
	c.Assert(err, IsNil)
	results := s.agent.GetMachines(params.Machines{
		Ids: []string{"1"},
	})
	c.Assert(err, IsNil)
	c.Assert(results, DeepEquals, params.MachineAgentGetMachinesResults{
		Machines: []params.MachineAgentGetMachinesResult{{
			Error: &params.Error{
				Code:    api.CodeNotFound,
				Message: "machine 1 not found",
			},
		}},
	})
}
<|endoftext|>


<commit_before>package statistics

import (
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = FDescribe("AggregatorSnapShot", func() {

	var (
		targetSnapShot  *AggregatorSnapShot
		subjectSnapShot *AggregatorSnapShot
	)

	BeforeEach(func() {
		targetSnapShot = NewAggregatorSnapShot()
		subjectSnapShot = NewAggregatorSnapShot()
	})

	It("Updates an empty snap shot counters", func() {

		key := "counter:1"
		value := int64(101)

		timeStamp := time.Now().UnixNano()
		subjectSnapShot.updateCounter(key, value)
		subjectSnapShot.updateTime(timeStamp)

		targetSnapShot.Update(*subjectSnapShot)

		Expect(len(targetSnapShot.Times)).To(Equal(1))
		Expect(targetSnapShot.Times[0]).To(Equal(subjectSnapShot.Times[0]))
		Expect(targetSnapShot.Counters[key]).ToNot(BeNil())
		Expect(targetSnapShot.Counters[key]).To(Equal(subjectSnapShot.Counters[key]))
	})

	It("Updates an empty snap shot gauges", func() {
		key := "gauge:1"
		value := float64(102.1)

		timeStamp := time.Now().UnixNano()
		subjectSnapShot.updateGauge(key, value)
		subjectSnapShot.updateTime(timeStamp)

		targetSnapShot.Update(*subjectSnapShot)

		Expect(len(targetSnapShot.Times)).To(Equal(1))
		Expect(targetSnapShot.Times[0]).To(Equal(subjectSnapShot.Times[0]))
		Expect(targetSnapShot.Gauges[key]).ToNot(BeNil())
		Expect(targetSnapShot.Gauges[key]).To(Equal(subjectSnapShot.Gauges[key]))
	})

})
<commit_msg>Updates an empty snap shot with histograms<commit_after>package statistics

import (
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = FDescribe("AggregatorSnapShot", func() {

	var (
		targetSnapShot  *AggregatorSnapShot
		subjectSnapShot *AggregatorSnapShot
	)

	BeforeEach(func() {
		targetSnapShot = NewAggregatorSnapShot()
		subjectSnapShot = NewAggregatorSnapShot()
	})

	It("Updates an empty snap shot counters", func() {

		key := "counter:1"
		value := int64(101)

		timeStamp := time.Now().UnixNano()
		subjectSnapShot.updateCounter(key, value)
		subjectSnapShot.updateTime(timeStamp)

		targetSnapShot.Update(*subjectSnapShot)

		Expect(len(targetSnapShot.Times)).To(Equal(1))
		Expect(targetSnapShot.Times[0]).To(Equal(subjectSnapShot.Times[0]))
		Expect(targetSnapShot.Counters[key]).ToNot(BeNil())
		Expect(targetSnapShot.Counters[key]).To(Equal(subjectSnapShot.Counters[key]))
	})

	It("Updates an empty snap shot gauges", func() {
		key := "gauge:1"
		value := float64(102.1)

		timeStamp := time.Now().UnixNano()
		subjectSnapShot.updateGauge(key, value)
		subjectSnapShot.updateTime(timeStamp)

		targetSnapShot.Update(*subjectSnapShot)

		Expect(len(targetSnapShot.Times)).To(Equal(1))
		Expect(targetSnapShot.Times[0]).To(Equal(subjectSnapShot.Times[0]))
		Expect(targetSnapShot.Gauges[key]).ToNot(BeNil())
		Expect(targetSnapShot.Gauges[key]).To(Equal(subjectSnapShot.Gauges[key]))
	})

	It("Updates an empty snap shot histograms", func() {
		key := "h:key"
		subKey := "h:subkey"
		value := float64(103.1)

		timeStamp := time.Now().UnixNano()
		subjectSnapShot.updateHistogram(key, subKey, value)
		subjectSnapShot.updateTime(timeStamp)

		targetSnapShot.Update(*subjectSnapShot)

		Expect(len(targetSnapShot.Times)).To(Equal(1))
		Expect(targetSnapShot.Times[0]).To(Equal(subjectSnapShot.Times[0]))
		Expect(targetSnapShot.Histograms[key]).ToNot(BeNil())
		Expect(targetSnapShot.Histograms[key][subKey]).ToNot(BeNil())
		Expect(targetSnapShot.Histograms[key][subKey]).To(Equal(subjectSnapShot.Histograms[key][subKey]))
	})
})
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/PuerkitoBio/goquery"
)

var urlLock = sync.RWMutex{}
var visitedURLs = make(map[string]url.URL)

type URLCrawlRequest struct {
	ParentURL url.URL
	TargetURL url.URL
}

func crawl(xmlSitemapURL url.URL, options CrawlOptions, stop chan bool) error {

	// read the XML sitemap as a initial source for URLs
	urlsFromXMLSitemap, err := getURLs(xmlSitemapURL)
	if err != nil {
		return err
	}

	// the URL queue
	crawlRequestQueue := make(chan URLCrawlRequest, len(urlsFromXMLSitemap))

	// fill the URL queue with the URLs from the XML sitemap
	for _, xmlSitemapURLEntry := range urlsFromXMLSitemap {
		crawlRequestQueue <- URLCrawlRequest{
			ParentURL: xmlSitemapURL,
			TargetURL: xmlSitemapURLEntry,
		}
	}

	// crawl all URLs in the queue
	allWorkersHaveStopped := StartDispatcher(options.NumberOfConcurrentRequests, stop)

	go func() {
		select {
		case <-allWorkersHaveStopped:
			close(crawlRequestQueue)
			return
		}
	}()

	for urlCrawlRequest := range crawlRequestQueue {

		// skip URLs we have already seen
		urlLock.RLock()
		_, alreadyVisited := visitedURLs[urlCrawlRequest.TargetURL.String()]
		urlLock.RUnlock()

		if alreadyVisited {
			continue
		}

		// mark the URL as visited
		urlLock.Lock()
		visitedURLs[urlCrawlRequest.TargetURL.String()] = urlCrawlRequest.TargetURL
		urlLock.Unlock()

		WorkQueue <- createWorkRequest(urlCrawlRequest, crawlRequestQueue)

	}

	return nil
}

func createWorkRequest(urlToCrawl URLCrawlRequest, newUrls chan URLCrawlRequest) WorkRequest {

	return WorkRequest{
		URL: urlToCrawl.TargetURL,
		Execute: func(workerID, numberOfWorkers int) {

			// read the URL
			response, err := readURL(urlToCrawl.TargetURL)
			if err != nil {
				return
			}

			if response.IsHTML() {

				// get dependent links
				links, err := getDependentRequests(urlToCrawl.TargetURL, bytes.NewReader(response.Body()))
				if err != nil {
					return
				}

				for _, link := range links {
					newUrls <- URLCrawlRequest{
						ParentURL: urlToCrawl.TargetURL,
						TargetURL: link,
					}
				}

			}

			workResult := WorkResult{
				parentURL: urlToCrawl.ParentURL,
				url:       urlToCrawl.TargetURL,

				workerID:        workerID,
				numberOfWorkers: numberOfWorkers,

				responseSize: response.Size(),
				statusCode:   response.StatusCode(),
				startTime:    response.StartTime(),
				endTime:      response.EndTime(),
				contentType:  response.ContentType(),
			}

			updateStatistics(workResult)
		}}

}

func getDependentRequests(baseURL url.URL, input io.Reader) ([]url.URL, error) {

	doc, err := goquery.NewDocumentFromReader(input)
	if err != nil {
		return nil, err
	}

	var urls []url.URL

	// base url
	base, _ := doc.Find("base[href]").Attr("href")
	if base == "" {
		base = baseURL.Scheme + "://" + baseURL.Host
	} else if strings.HasPrefix(base, "/") {
		base = baseURL.Scheme + "://" + baseURL.Host + base
	}

	// get all links
	doc.Find("a[href]").Each(func(i int, s *goquery.Selection) {
		// For each item found, get the band and title
		href, _ := s.Attr("href")

		if strings.HasPrefix(href, "/") {
			href = baseURL.Scheme + "://" + baseURL.Host + href
		} else if !strings.HasPrefix(href, "http://") && !strings.HasPrefix(href, "https://") {
			href = strings.TrimSuffix(base, "/") + "/" + href
		}

		hrefURL, err := url.Parse(href)
		if err != nil {
			return
		}

		// ignore external links
		if hrefURL.Host != baseURL.Host {
			return
		}

		urls = append(urls, *hrefURL)
	})

	return urls, nil
}

func getURLs(xmlSitemapURL url.URL) ([]url.URL, error) {

	var urls []url.URL

	urlsFromIndex, indexError := getURLsFromSitemapIndex(xmlSitemapURL)
	if indexError == nil {
		urls = urlsFromIndex
	}

	urlsFromSitemap, sitemapError := getURLsFromSitemap(xmlSitemapURL)
	if sitemapError == nil {
		urls = append(urls, urlsFromSitemap...)
	}

	if isInvalidSitemapIndexContent(indexError) && isInvalidXMLSitemapContent(sitemapError) {
		return nil, fmt.Errorf("%q is neither a sitemap index nor a XML sitemap", xmlSitemapURL.String())
	}

	return urls, nil

}

func getURLsFromSitemap(xmlSitemapURL url.URL) ([]url.URL, error) {

	var urls []url.URL

	sitemap, xmlSitemapError := getXMLSitemap(xmlSitemapURL)
	if xmlSitemapError != nil {
		return nil, xmlSitemapError
	}

	for _, urlEntry := range sitemap.URLs {

		parsedURL, parseError := url.Parse(urlEntry.Location)
		if parseError != nil {
			return nil, parseError
		}

		urls = append(urls, *parsedURL)
	}

	return urls, nil
}

func getURLsFromSitemapIndex(xmlSitemapURL url.URL) ([]url.URL, error) {

	var urls []url.URL

	sitemapIndex, sitemapIndexError := getSitemapIndex(xmlSitemapURL)
	if sitemapIndexError != nil {
		return nil, sitemapIndexError
	}

	for _, sitemap := range sitemapIndex.Sitemaps {

		locationURL, err := url.Parse(sitemap.Location)
		if err != nil {
			return nil, err
		}

		sitemapUrls, err := getURLsFromSitemap(*locationURL)
		if err != nil {
			return nil, err
		}

		urls = append(urls, sitemapUrls...)
	}

	return urls, nil

}

type CrawlOptions struct {
	NumberOfConcurrentRequests int
	Timeout                    time.Duration
}
<commit_msg>Ignore hashes in URLs<commit_after>package main

import (
	"bytes"
	"fmt"
	"io"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/PuerkitoBio/goquery"
)

var urlLock = sync.RWMutex{}
var visitedURLs = make(map[string]url.URL)

type URLCrawlRequest struct {
	ParentURL url.URL
	TargetURL url.URL
}

func crawl(xmlSitemapURL url.URL, options CrawlOptions, stop chan bool) error {

	// read the XML sitemap as a initial source for URLs
	urlsFromXMLSitemap, err := getURLs(xmlSitemapURL)
	if err != nil {
		return err
	}

	// the URL queue
	crawlRequestQueue := make(chan URLCrawlRequest, len(urlsFromXMLSitemap))

	// fill the URL queue with the URLs from the XML sitemap
	for _, xmlSitemapURLEntry := range urlsFromXMLSitemap {
		crawlRequestQueue <- URLCrawlRequest{
			ParentURL: xmlSitemapURL,
			TargetURL: xmlSitemapURLEntry,
		}
	}

	// crawl all URLs in the queue
	allWorkersHaveStopped := StartDispatcher(options.NumberOfConcurrentRequests, stop)

	go func() {
		select {
		case <-allWorkersHaveStopped:
			close(crawlRequestQueue)
			return
		}
	}()

	for urlCrawlRequest := range crawlRequestQueue {

		// skip URLs we have already seen
		urlLock.RLock()
		_, alreadyVisited := visitedURLs[urlCrawlRequest.TargetURL.String()]
		urlLock.RUnlock()

		if alreadyVisited {
			continue
		}

		// mark the URL as visited
		urlLock.Lock()
		visitedURLs[urlCrawlRequest.TargetURL.String()] = urlCrawlRequest.TargetURL
		urlLock.Unlock()

		WorkQueue <- createWorkRequest(urlCrawlRequest, crawlRequestQueue)

	}

	return nil
}

func createWorkRequest(urlToCrawl URLCrawlRequest, newUrls chan URLCrawlRequest) WorkRequest {

	return WorkRequest{
		URL: urlToCrawl.TargetURL,
		Execute: func(workerID, numberOfWorkers int) {

			// read the URL
			response, err := readURL(urlToCrawl.TargetURL)
			if err != nil {
				return
			}

			if response.IsHTML() {

				// get dependent links
				links, err := getDependentRequests(urlToCrawl.TargetURL, bytes.NewReader(response.Body()))
				if err != nil {
					return
				}

				for _, link := range links {
					newUrls <- URLCrawlRequest{
						ParentURL: urlToCrawl.TargetURL,
						TargetURL: link,
					}
				}

			}

			workResult := WorkResult{
				parentURL: urlToCrawl.ParentURL,
				url:       urlToCrawl.TargetURL,

				workerID:        workerID,
				numberOfWorkers: numberOfWorkers,

				responseSize: response.Size(),
				statusCode:   response.StatusCode(),
				startTime:    response.StartTime(),
				endTime:      response.EndTime(),
				contentType:  response.ContentType(),
			}

			updateStatistics(workResult)
		}}

}

func getDependentRequests(baseURL url.URL, input io.Reader) ([]url.URL, error) {

	doc, err := goquery.NewDocumentFromReader(input)
	if err != nil {
		return nil, err
	}

	var urls []url.URL

	// base url
	base, _ := doc.Find("base[href]").Attr("href")
	if base == "" {
		base = baseURL.Scheme + "://" + baseURL.Host
	} else if strings.HasPrefix(base, "/") {
		base = baseURL.Scheme + "://" + baseURL.Host + base
	}

	// get all links
	doc.Find("a[href]").Each(func(i int, s *goquery.Selection) {
		// For each item found, get the band and title
		href, _ := s.Attr("href")

		if strings.HasPrefix(href, "/") {
			href = baseURL.Scheme + "://" + baseURL.Host + href
		} else if !strings.HasPrefix(href, "http://") && !strings.HasPrefix(href, "https://") {
			href = strings.TrimSuffix(base, "/") + "/" + href
		}

		// cut off hashes
		if strings.Contains(href, "#") {
			hashPosition := strings.Index(href, "#")
			href = href[0 : hashPosition-1]
		}

		hrefURL, err := url.Parse(href)
		if err != nil {
			return
		}

		// ignore external links
		if hrefURL.Host != baseURL.Host {
			return
		}

		urls = append(urls, *hrefURL)
	})

	return urls, nil
}

func getURLs(xmlSitemapURL url.URL) ([]url.URL, error) {

	var urls []url.URL

	urlsFromIndex, indexError := getURLsFromSitemapIndex(xmlSitemapURL)
	if indexError == nil {
		urls = urlsFromIndex
	}

	urlsFromSitemap, sitemapError := getURLsFromSitemap(xmlSitemapURL)
	if sitemapError == nil {
		urls = append(urls, urlsFromSitemap...)
	}

	if isInvalidSitemapIndexContent(indexError) && isInvalidXMLSitemapContent(sitemapError) {
		return nil, fmt.Errorf("%q is neither a sitemap index nor a XML sitemap", xmlSitemapURL.String())
	}

	return urls, nil

}

func getURLsFromSitemap(xmlSitemapURL url.URL) ([]url.URL, error) {

	var urls []url.URL

	sitemap, xmlSitemapError := getXMLSitemap(xmlSitemapURL)
	if xmlSitemapError != nil {
		return nil, xmlSitemapError
	}

	for _, urlEntry := range sitemap.URLs {

		parsedURL, parseError := url.Parse(urlEntry.Location)
		if parseError != nil {
			return nil, parseError
		}

		urls = append(urls, *parsedURL)
	}

	return urls, nil
}

func getURLsFromSitemapIndex(xmlSitemapURL url.URL) ([]url.URL, error) {

	var urls []url.URL

	sitemapIndex, sitemapIndexError := getSitemapIndex(xmlSitemapURL)
	if sitemapIndexError != nil {
		return nil, sitemapIndexError
	}

	for _, sitemap := range sitemapIndex.Sitemaps {

		locationURL, err := url.Parse(sitemap.Location)
		if err != nil {
			return nil, err
		}

		sitemapUrls, err := getURLsFromSitemap(*locationURL)
		if err != nil {
			return nil, err
		}

		urls = append(urls, sitemapUrls...)
	}

	return urls, nil

}

type CrawlOptions struct {
	NumberOfConcurrentRequests int
	Timeout                    time.Duration
}
<|endoftext|>


<commit_before>package crawler
<commit_msg>init crawler<commit_after>package crawler

import (
	"net/url"
)

type Crawler struct {
	limit int
}

func New(limit int) *Crawler {
	return &Crawler{
		limit: limit,
	}
}

func (c *Crawler) Start(u string, ch_url chan string, ch_err chan error, ch_done chan bool) {
	u_obj, _ := url.Parse(u)

	//anchor picker
	picker_anchor := &AnchorPicker{}

	//unique filter
	filter_url := &UrlFilter{}

	//unique filter
	filter_unique := &UniqueFilter{
		[]*url.URL{u_obj},
	}

	//same host filter
	filter_samehost := &SameHostFilter{
		u_obj,
	}

	c.crawl(u, []Picker{picker_anchor}, []Filter{filter_url, filter_unique, filter_samehost}, ch_url, ch_err, ch_done)
}

func (c *Crawler) crawl(u string, pickers []Picker, filters []Filter, ch_url chan string, ch_err chan error, ch_done chan bool) {

	urls, err := Fetch(u, pickers[0]) //hardcode

	if err != nil {
		ch_err <- err
		return
	}

parent_loop:
	for _, child_u_obj := range urls {
		for _, f := range filters {
			if !f.Filter(child_u_obj) {
				continue parent_loop
			}
		}

		//valid url
		c.limit--

		select {
		case <-ch_done:
			return
		default:
		}

		if c.limit < 0 {
			close(ch_done)
			return
		}

		child_url := child_u_obj.String()

		ch_url <- child_url

		go c.crawl(child_url, pickers, filters, ch_url, ch_err, ch_done)
	}
}
<|endoftext|>