<commit_before>package scraping

import (
	"fmt"
	"golang.org/x/net/html"
	"io"
	"log"
	"net/http"
	"strings"
	"sync"
)

const HREF = "href"

func GetHref(t html.Token) (ok bool, href string) {
	for _, a := range t.Attr {
		if a.Key == HREF {
			href = a.Val
			ok = true
		}
	}

	return
}

type Extractable interface {
	Extract()
}

type LinkExtractor struct {
	Extractable
}

func (extractor *LinkExtractor) Extract(r io.ReadCloser, callback func(string)) {
	z := html.NewTokenizer(r)
	defer r.Close()

	for {
		tt := z.Next()

		switch {
		case tt == html.ErrorToken:
			return
		case tt == html.StartTagToken:
			t := z.Token()

			isAnchor := t.Data == "a"
			if !isAnchor {
				continue
			}

			ok, url := GetHref(t)
			if !ok {
				continue
			}
			callback(url)
		}
	}
}

type Runnable interface {
	Run() (err error)
}

type DefaultRunner struct {
	finished    int
	scrapers    []*Scraper
	chDone      chan bool
	chFoundUrls chan string
}

func (runner *DefaultRunner) FinishedIncr() {
	runner.finished += 1

}

func (runner DefaultRunner) Done() bool {
	return len(runner.scrapers) == runner.finished
}

func (runner *DefaultRunner) Run() {
	defer runner.Close()

	for _, scraper := range runner.scrapers {
		log.Printf("Starting: %s", scraper)
		go scraper.Fetch(scraper.baseUrl)
	}

	// main scraping loop
	for {
		select {
		case _, ok := <-runner.chFoundUrls:
			if !ok {
				break
			}
		case _, ok := <-runner.chDone:
			runner.FinishedIncr()
			if !ok {
				break
			}

		}
		if runner.Done() {
			break
		}
	}
}

func (runner *DefaultRunner) Close() {
	close(runner.chDone)
	close(runner.chFoundUrls)
}

func (runner *DefaultRunner) PushScraper(scrapers ...*Scraper) *DefaultRunner {
	for _, scraper := range scrapers {
		scraper.runner = runner
	}
	runner.scrapers = append(runner.scrapers, scrapers...)
	return runner
}

type Scraper struct {
	fetchMutex  *sync.Mutex
	domain      string
	baseUrl     string
	fetchedUrls map[string]bool
	runner      *DefaultRunner
	extractor   *LinkExtractor
}

func (scraper *Scraper) MarkAsFetched(url string) {
	scraper.fetchMutex.Lock()
	scraper.fetchedUrls[url] = true
	scraper.fetchMutex.Unlock()
}

func (scraper *Scraper) CheckIfFetched(url string) (ok bool) {
	scraper.fetchMutex.Lock()
	_, ok = scraper.fetchedUrls[url]
	scraper.fetchMutex.Unlock()
	return
}

func (scraper *Scraper) CheckUrl(sourceUrl string) (ok bool, url string) {
	if strings.Contains(sourceUrl, scraper.domain) && strings.Index(sourceUrl, "http") == 0 {
		url = sourceUrl
		ok = true
	} else if strings.Index(sourceUrl, "/") == 0 {
		url = scraper.baseUrl + sourceUrl
		ok = true
	}
	return
}

func (scraper *Scraper) Fetch(url string) {
	if ok := scraper.CheckIfFetched(url); ok {
		return
	}
	scraper.MarkAsFetched(url)

	log.Printf("Fetching: %s", url)
	resp, err := http.Get(url)

	if err != nil {
		log.Printf("ERROR: Failed to crawl %s", url)
		return
	}

	if scraper.extractor != nil {
		scraper.extractor.Extract(resp.Body, func(url string) {
			ok, url := scraper.CheckUrl(url)

			if ok {
				go scraper.Fetch(url)
			}
		})
	}

	return
}

func (scraper *Scraper) String() (result string) {
	result = fmt.Sprintf("<Scraper: %s>", scraper.domain)
	return
}

func NewRunner() (r *DefaultRunner) {
	r = &DefaultRunner{
		finished:    0,
		chDone:      make(chan bool),
		chFoundUrls: make(chan string),
	}
	return
}

func NewScraper(domain string, url string) (s *Scraper) {
	s = &Scraper{
		domain:      domain,
		baseUrl:     url,
		fetchedUrls: make(map[string]bool),
		fetchMutex:  &sync.Mutex{},
		extractor:   &LinkExtractor{},
	}
	return
}
<commit_msg>Added initial synchronisation mechanism<commit_after>package scraping

import (
	"fmt"
	"golang.org/x/net/html"
	"io"
	"log"
	"net/http"
	"strings"
	"sync"
)

const HREF = "href"
const LIMIT_CRAWL = 10
const LIMIT_FAIL = 1

func GetHref(t html.Token) (ok bool, href string) {
	for _, a := range t.Attr {
		if a.Key == HREF {
			href = a.Val
			ok = true
		}
	}

	return
}

type Extractable interface {
	Extract()
}

type LinkExtractor struct {
	Extractable
}

func (extractor *LinkExtractor) Extract(r io.ReadCloser, callback func(string)) {
	z := html.NewTokenizer(r)
	defer r.Close()

	for {
		tt := z.Next()

		switch {
		case tt == html.ErrorToken:
			return
		case tt == html.StartTagToken:
			t := z.Token()

			isAnchor := t.Data == "a"
			if !isAnchor {
				continue
			}

			ok, url := GetHref(t)
			if !ok {
				continue
			}
			callback(url)
		}
	}
}

type Runnable interface {
	Run() (err error)
}

type DefaultRunner struct {
	finished    int
	scrapers    []*Scraper
	chDone      chan *Scraper
	chFoundUrls chan string
}

func (runner *DefaultRunner) FinishedIncr() {
	runner.finished += 1
}

func (runner DefaultRunner) Done() bool {
	return len(runner.scrapers) == runner.finished
}

func (runner *DefaultRunner) Run() {
	defer runner.Close()

	for _, scraper := range runner.scrapers {
		log.Printf("Starting: %s", scraper)
		go scraper.Start(scraper.baseUrl)
	}

	// main scraping loop
	for {
		select {
		case _, ok := <-runner.chFoundUrls:
			if !ok {
				break
			}
		case scraper, ok := <-runner.chDone:
			log.Printf("%s is done.", scraper)
			runner.FinishedIncr()
			if !ok {
				break
			}

		}
		if runner.Done() {
			break
		}
	}
}

func (runner *DefaultRunner) Close() {
	close(runner.chDone)
	close(runner.chFoundUrls)
}

func (runner *DefaultRunner) PushScraper(scrapers ...*Scraper) *DefaultRunner {
	for _, scraper := range scrapers {
		scraper.runner = runner
	}
	runner.scrapers = append(runner.scrapers, scrapers...)
	return runner
}

type Scraper struct {
	crawled int
	successful int
	failed int
	fetchMutex  *sync.Mutex
	crawledMutex *sync.Mutex
	domain      string
	baseUrl     string
	fetchedUrls map[string]bool
	runner      *DefaultRunner
	extractor   *LinkExtractor
}

func (scraper *Scraper) IncrCounters(isSuccessful bool) {
	scraper.crawledMutex.Lock()
	scraper.crawled += 1
	if (isSuccessful) {
		scraper.successful += 1
	} else {
		scraper.failed += 1
	}
	scraper.crawledMutex.Unlock()
}

func (scraper *Scraper) MarkAsFetched(url string) {
	scraper.fetchMutex.Lock()
	scraper.fetchedUrls[url] = true
	scraper.fetchMutex.Unlock()
}

func (scraper *Scraper) CheckIfShouldStop() (ok bool) {
	scraper.crawledMutex.Lock()
	if scraper.crawled == LIMIT_CRAWL {
		log.Printf("Crawl limit exceeded: %s", scraper)
		ok = true
	} else if scraper.failed == LIMIT_FAIL {
		log.Printf("Fail limit exceeeded: %s", scraper)
		ok = true
	}
	scraper.crawledMutex.Unlock()
	return
}

func (scraper *Scraper) CheckIfFetched(url string) (ok bool) {
	scraper.fetchMutex.Lock()
	_, ok = scraper.fetchedUrls[url]
	scraper.fetchMutex.Unlock()
	return
}

func (scraper *Scraper) CheckUrl(sourceUrl string) (ok bool, url string) {
	if strings.Contains(sourceUrl, scraper.domain) && strings.Index(sourceUrl, "http") == 0 {
		url = sourceUrl
		ok = true
	} else if strings.Index(sourceUrl, "/") == 0 {
		url = scraper.baseUrl + sourceUrl
		ok = true
	}
	return
}

func (scraper *Scraper) RunExtractor(resp *http.Response) {
	scraper.extractor.Extract(resp.Body, func(url string) {
		ok, url := scraper.CheckUrl(url)

		if ok {
			go scraper.Fetch(url, true)
		}

		if scraper.CheckIfShouldStop() {
			scraper.Stop()
		}
	})
}

func (scraper *Scraper) Stop() {
	scraper.runner.chDone <- scraper
}

func (scraper *Scraper) Start(baseUrl string) {
	resp, err :=  scraper.Fetch(baseUrl, false)

	if err != nil {
		log.Printf("Base url is corrupted stopping %s", scraper)
		scraper.Stop()
		return
	}

	scraper.RunExtractor(resp)

	return
}

func (scraper *Scraper) Fetch(url string, extract bool) (resp *http.Response, err error) {
	if ok := scraper.CheckIfFetched(url); ok {
		return
	}
	scraper.MarkAsFetched(url)

	log.Printf("Fetching: %s", url)
	resp, err = http.Get(url)

	scraper.IncrCounters(err == nil)
	if err != nil {
		log.Printf("ERROR: Failed to crawl %s", url)
		return
	}

	if extract {
		scraper.RunExtractor(resp)
	}

	return
}

func (scraper *Scraper) String() (result string) {
	result = fmt.Sprintf("<Scraper: %s>. Crawled: %d, successful: %d failed: %d.", scraper.domain, scraper.crawled, scraper.successful, scraper.failed)
	return
}

func NewRunner() (r *DefaultRunner) {
	r = &DefaultRunner{
		finished:    0,
		chDone:      make(chan *Scraper),
		chFoundUrls: make(chan string),
	}
	return
}

func NewScraper(domain string, url string) (s *Scraper) {
	s = &Scraper{
		crawled: 0,
		successful: 0,
		failed: 0,
		domain:      domain,
		baseUrl:     url,
		fetchedUrls: make(map[string]bool),
		crawledMutex: &sync.Mutex{},
		fetchMutex:  &sync.Mutex{},
		extractor:   &LinkExtractor{},
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2013 Wincent Colaiuta. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice,
//    this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
//    this list of conditions and the following disclaimer in the documentation
//    and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.

package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"os/exec"
)

const (
	PBCOPY = "pbcopy"
)

// flags
var listenAddr string
var listenPort int

func init() {
	const (
		defaultListenAddr = "127.0.0.1"
		listenAddrUsage   = "address to bind to"
		defaultListenPort = 8377
		listenPortUsage   = "port to listen on"
		shorthand         = " (shorthand)"
	)

	flag.StringVar(&listenAddr, "address", defaultListenAddr, listenAddrUsage)
	flag.StringVar(&listenAddr, "a", defaultListenAddr, listenAddrUsage+shorthand)
	flag.IntVar(&listenPort, "port", defaultListenPort, listenPortUsage)
	flag.IntVar(&listenPort, "p", defaultListenPort, listenPortUsage+shorthand)
}

func main() {
	flag.Parse()
	if flag.NArg() != 0 {
		// additional command-line options not supported
		flag.Usage()
		os.Exit(1)
	}

	if _, err := exec.LookPath(PBCOPY); err != nil {
		log.Fatal(err.Error())
	}

	log.Print("Starting the server")
	listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", listenAddr, listenPort))
	if err != nil {
		log.Fatal(err.Error())
	}

	for {
		conn, err := listener.Accept()
		if err != nil {
			log.Print(err.Error())
			return
		}

		go handleConnection(conn)
	}
}

func handleConnection(conn net.Conn) {
	defer log.Print("Connection closed")
	defer conn.Close()

	cmd := exec.Command(PBCOPY)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		log.Print(err.Error())
		return
	}
	if err = cmd.Start(); err != nil {
		log.Print(err.Error())
		return
	}

	if copied, err := io.Copy(stdin, conn); err != nil {
		log.Print(err.Error())
	} else {
		log.Print("Echoed ", copied, " bytes")
	}
	stdin.Close()

	if err = cmd.Wait(); err != nil {
		log.Print(err.Error())
	}
}
<commit_msg>Add -l/--logfile flags<commit_after>// Copyright 2013 Wincent Colaiuta. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice,
//    this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
//    this list of conditions and the following disclaimer in the documentation
//    and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.

package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"os/exec"
	"os/user"
	"strings"
)

const (
	PBCOPY = "pbcopy"
)

// flags
var listenAddr string
var listenPort int
var logFile string

func init() {
	const (
		defaultListenAddr = "127.0.0.1"
		listenAddrUsage   = "address to bind to"
		defaultListenPort = 8377
		listenPortUsage   = "port to listen on"
		defaultLogFile    = "~/Library/Logs/com.wincent.clipper.log"
		logFileUsage      = "path to logfile"
		shorthand         = " (shorthand)"
	)

	flag.StringVar(&listenAddr, "address", defaultListenAddr, listenAddrUsage)
	flag.StringVar(&listenAddr, "a", defaultListenAddr, listenAddrUsage+shorthand)
	flag.IntVar(&listenPort, "port", defaultListenPort, listenPortUsage)
	flag.IntVar(&listenPort, "p", defaultListenPort, listenPortUsage+shorthand)
	flag.StringVar(&logFile, "logfile", defaultLogFile, logFileUsage)
	flag.StringVar(&logFile, "l", defaultLogFile, logFileUsage)
}

func main() {
	flag.Parse()
	if flag.NArg() != 0 {
		// additional command-line options not supported
		flag.Usage()
		os.Exit(1)
	}

	expandedPath := pathByExpandingTildeInPath(logFile)
	outfile, err := os.OpenFile(expandedPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
	if err != nil {
		log.Fatal(err)
	}
	defer outfile.Close()
	log.SetOutput(outfile)
	log.SetPrefix("clipper: ")

	if _, err := exec.LookPath(PBCOPY); err != nil {
		log.Fatal(err)
	}

	log.Print("Starting the server")
	listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", listenAddr, listenPort))
	if err != nil {
		log.Fatal(err)
	}

	for {
		conn, err := listener.Accept()
		if err != nil {
			log.Print(err)
			return
		}

		go handleConnection(conn)
	}
}

func pathByExpandingTildeInPath(path string) string {
	if strings.HasPrefix(path, "~") {
		user, err := user.Current()
		if err != nil {
			log.Fatal(err)
		}
		path = user.HomeDir + path[1:]
	}
	return path
}

func handleConnection(conn net.Conn) {
	defer log.Print("Connection closed")
	defer conn.Close()

	cmd := exec.Command(PBCOPY)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		log.Print(err)
		return
	}
	if err = cmd.Start(); err != nil {
		log.Print(err)
		return
	}

	if copied, err := io.Copy(stdin, conn); err != nil {
		log.Print(err)
	} else {
		log.Print("Echoed ", copied, " bytes")
	}
	stdin.Close()

	if err = cmd.Wait(); err != nil {
		log.Print(err)
	}
}
<|endoftext|>


<commit_before>package libcluster

import (
	"errors"
	"sync"
)

var (
	ErrNodeNotConnected      = errors.New("node is not connected to docker's REST API")
	ErrNodeAlreadyRegistered = errors.New("node was already added to the cluster")
)

type Cluster struct {
	mux   sync.Mutex
	nodes map[string]*Node
}

func NewCluster() *Cluster {
	return &Cluster{
		nodes: make(map[string]*Node),
	}
}

// Register a node within the cluster. The node must have been already
// initialized.
func (c *Cluster) AddNode(n *Node) error {
	if !n.IsConnected() {
		return ErrNodeNotConnected
	}

	c.mux.Lock()
	defer c.mux.Unlock()

	if _, exists := c.nodes[n.ID]; exists {
		return ErrNodeAlreadyRegistered
	}

	c.nodes[n.ID] = n
	return nil
}
<commit_msg>Cluster: Implemented support to return a list of all containers<commit_after>package libcluster

import (
	"errors"
	"sync"
)

var (
	ErrNodeNotConnected      = errors.New("node is not connected to docker's REST API")
	ErrNodeAlreadyRegistered = errors.New("node was already added to the cluster")
)

type Cluster struct {
	mux   sync.Mutex
	nodes map[string]*Node
}

func NewCluster() *Cluster {
	return &Cluster{
		nodes: make(map[string]*Node),
	}
}

// Register a node within the cluster. The node must have been already
// initialized.
func (c *Cluster) AddNode(n *Node) error {
	if !n.IsConnected() {
		return ErrNodeNotConnected
	}

	c.mux.Lock()
	defer c.mux.Unlock()

	if _, exists := c.nodes[n.ID]; exists {
		return ErrNodeAlreadyRegistered
	}

	c.nodes[n.ID] = n
	return nil
}

// Containers returns all the containers running in the cluster.
func (c *Cluster) Containers() []*Container {
	c.mux.Lock()
	defer c.mux.Unlock()

	out := []*Container{}
	for _, n := range c.nodes {
		containers := n.Containers()
		for _, container := range containers {
			out = append(out, container)
		}
	}

	return out
}
<|endoftext|>


<commit_before>package goBoom

import (
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"path/filepath"

	"github.com/cheggaaa/pb"
)

type FilesystemService struct {
	c *Client
}

func newFilesystemService(c *Client) *FilesystemService {
	i := &FilesystemService{}
	if c == nil {
		i.c = NewClient(nil)
	} else {
		i.c = c
	}

	return i
}

// Upload
func (s *FilesystemService) Upload(fname string, input io.Reader, size int64) (resp *http.Response, newInfo *ItemInfo, err error) {

	pipeOut, pipeIn := io.Pipe()
	bar := pb.New(int(size)).SetUnits(pb.U_BYTES)
	bar.ShowSpeed = true

	writer := multipart.NewWriter(pipeIn)

	// do the request concurrently
	// var item ItemInfo
	done := make(chan error)
	go func() {

		// prepare request
		req, err := s.c.NewReaderRequest("POST", "ul", pipeOut, writer.FormDataContentType())
		if err != nil {
			done <- err
			return
		}
		// TODO calculate right amount of overhead
		req.ContentLength = size // filesize
		req.ContentLength += 227 // multipart header exclouding filename
		req.ContentLength += int64(len(fname))
		req.ContentLength -= 19

		// set  token
		q := req.URL.Query()
		q.Set("token", s.c.User.session)
		q.Set("parent", "1")
		req.URL.RawQuery = q.Encode()

		fmt.Println("Created Request")
		bar.Start()

		var fInfo ItemInfo
		// resp, err = s.c.c.Do(req)
		resp, err = s.c.DoJson(req, &fInfo)
		if err != nil {
			done <- err
			return
		}
		newInfo = &fInfo

		done <- nil
	}()

	part, err := writer.CreateFormFile("file", filepath.Base(fname))
	if err != nil {
		return
	}

	out := io.MultiWriter(part, bar)
	_, err = io.Copy(out, input)
	if err != nil {
		fmt.Println("pipe copy error")
		err = <-done
		return
	}

	err = writer.Close()
	if err != nil {
		return
	}

	err = pipeIn.Close()
	if err != nil {
		return
	}

	err = <-done
	bar.Finish()
	if err != nil {
		return
	}

	return
}
<commit_msg>Revert "using io.Pipe for upload, content length still incorrect... :/"<commit_after>package goBoom

import (
	"bytes"
	"io"
	"mime/multipart"
	"path/filepath"
)

type FilesystemService struct {
	c *Client
}

func newFilesystemService(c *Client) *FilesystemService {
	i := &FilesystemService{}
	if c == nil {
		i.c = NewClient(nil)
	} else {
		i.c = c
	}

	return i
}

// Upload
func (s *FilesystemService) Upload(fname string, input io.Reader) (int, *ItemInfo, error) {

	var bodyBuf bytes.Buffer
	writer := multipart.NewWriter(&bodyBuf)

	part, err := writer.CreateFormFile("file", filepath.Base(fname))
	if err != nil {
		return 0, nil, err
	}

	_, err = io.Copy(part, input)
	if err != nil {
		return 0, nil, err
	}

	err = writer.Close()
	if err != nil {
		return 0, nil, err
	}

	// prepare request
	req, err := s.c.NewReaderRequest("POST", "ul", &bodyBuf, writer.FormDataContentType())
	if err != nil {
		return 0, nil, err
	}

	// set  token
	q := req.URL.Query()
	q.Set("token", s.c.User.session)
	q.Set("parent", "1")
	req.URL.RawQuery = q.Encode()

	// do the request
	var item ItemInfo
	resp, err := s.c.DoJson(req, &item)
	if err != nil {
		return resp.StatusCode, nil, err
	}

	return resp.StatusCode, &item, nil
}
<|endoftext|>


<commit_before>package communicator

import (
	"errors"
	"fmt"
	"os"
	"time"

	"github.com/mitchellh/packer/template/interpolate"
)

// Config is the common configuration that communicators allow within
// a builder.
type Config struct {
	Type string `mapstructure:"communicator"`

	// SSH
	SSHHost               string        `mapstructure:"ssh_host"`
	SSHPort               int           `mapstructure:"ssh_port"`
	SSHUsername           string        `mapstructure:"ssh_username"`
	SSHPassword           string        `mapstructure:"ssh_password"`
	SSHPrivateKey         string        `mapstructure:"ssh_private_key_file"`
	SSHPty                bool          `mapstructure:"ssh_pty"`
	SSHTimeout            time.Duration `mapstructure:"ssh_timeout"`
	SSHDisableAgent       bool          `mapstructure:"ssh_disable_agent"`
	SSHHandshakeAttempts  int           `mapstructure:"ssh_handshake_attempts"`
	SSHBastionHost        string        `mapstructure:"ssh_bastion_host"`
	SSHBastionPort        int           `mapstructure:"ssh_bastion_port"`
	SSHBastionUsername    string        `mapstructure:"ssh_bastion_username"`
	SSHBastionPassword    string        `mapstructure:"ssh_bastion_password"`
	SSHBastionPrivateKey  string        `mapstructure:"ssh_bastion_private_key_file"`
	SSHFileTransferMethod string        `mapstructure:"ssh_file_transfer_method"`

	// WinRM
	WinRMUser     string        `mapstructure:"winrm_username"`
	WinRMPassword string        `mapstructure:"winrm_password"`
	WinRMHost     string        `mapstructure:"winrm_host"`
	WinRMPort     int           `mapstructure:"winrm_port"`
	WinRMTimeout  time.Duration `mapstructure:"winrm_timeout"`
	WinRMUseSSL   bool          `mapstructure:"winrm_use_ssl"`
	WinRMInsecure bool          `mapstructure:"winrm_insecure"`
}

// Port returns the port that will be used for access based on config.
func (c *Config) Port() int {
	switch c.Type {
	case "ssh":
		return c.SSHPort
	case "winrm":
		return c.WinRMPort
	default:
		return 0
	}
}

func (c *Config) Prepare(ctx *interpolate.Context) []error {
	if c.Type == "" {
		c.Type = "ssh"
	}

	var errs []error
	switch c.Type {
	case "ssh":
		if es := c.prepareSSH(ctx); len(es) > 0 {
			errs = append(errs, es...)
		}
	case "winrm":
		if es := c.prepareWinRM(ctx); len(es) > 0 {
			errs = append(errs, es...)
		}
	case "docker", "none":
		break
	default:
		return []error{fmt.Errorf("Communicator type %s is invalid", c.Type)}
	}

	return errs
}

func (c *Config) prepareSSH(ctx *interpolate.Context) []error {
	if c.SSHPort == 0 {
		c.SSHPort = 22
	}

	if c.SSHTimeout == 0 {
		c.SSHTimeout = 5 * time.Minute
	}

	if c.SSHHandshakeAttempts == 0 {
		c.SSHHandshakeAttempts = 10
	}

	if c.SSHBastionHost != "" {
		if c.SSHBastionPort == 0 {
			c.SSHBastionPort = 22
		}

		if c.SSHBastionPrivateKey == "" && c.SSHPrivateKey != "" {
			c.SSHBastionPrivateKey = c.SSHPrivateKey
		}
	}

	if c.SSHFileTransferMethod == "" {
		c.SSHFileTransferMethod = "scp"
	}

	// Validation
	var errs []error
	if c.SSHUsername == "" {
		errs = append(errs, errors.New("An ssh_username must be specified"))
	}

	if c.SSHPrivateKey != "" {
		if _, err := os.Stat(c.SSHPrivateKey); err != nil {
			errs = append(errs, fmt.Errorf(
				"ssh_private_key_file is invalid: %s", err))
		} else if _, err := SSHFileSigner(c.SSHPrivateKey); err != nil {
			errs = append(errs, fmt.Errorf(
				"ssh_private_key_file is invalid: %s", err))
		}
	}

	if c.SSHBastionHost != "" {
		if c.SSHBastionPassword == "" && c.SSHBastionPrivateKey == "" {
			errs = append(errs, errors.New(
				"ssh_bastion_password or ssh_bastion_private_key_file must be specified"))
		}
	}

	if c.SSHFileTransferMethod != "scp" && c.SSHFileTransferMethod != "sftp" {
		errs = append(errs, fmt.Errorf(
			"ssh_file_transfer_method ('%s') is invalid, valid methods: sftp, scp",
			c.SSHFileTransferMethod))
	}

	return errs
}

func (c *Config) prepareWinRM(ctx *interpolate.Context) []error {
	if c.WinRMPort == 0 {
		c.WinRMPort = 5985
	}

	if c.WinRMTimeout == 0 {
		c.WinRMTimeout = 30 * time.Minute
	}

	var errs []error
	if c.WinRMUser == "" {
		errs = append(errs, errors.New("winrm_username must be specified."))
	}

	return errs
}
<commit_msg>Make communicator's WinRM defaults more intuitive when using SSL.<commit_after>package communicator

import (
	"errors"
	"fmt"
	"os"
	"time"

	"github.com/mitchellh/packer/template/interpolate"
)

// Config is the common configuration that communicators allow within
// a builder.
type Config struct {
	Type string `mapstructure:"communicator"`

	// SSH
	SSHHost               string        `mapstructure:"ssh_host"`
	SSHPort               int           `mapstructure:"ssh_port"`
	SSHUsername           string        `mapstructure:"ssh_username"`
	SSHPassword           string        `mapstructure:"ssh_password"`
	SSHPrivateKey         string        `mapstructure:"ssh_private_key_file"`
	SSHPty                bool          `mapstructure:"ssh_pty"`
	SSHTimeout            time.Duration `mapstructure:"ssh_timeout"`
	SSHDisableAgent       bool          `mapstructure:"ssh_disable_agent"`
	SSHHandshakeAttempts  int           `mapstructure:"ssh_handshake_attempts"`
	SSHBastionHost        string        `mapstructure:"ssh_bastion_host"`
	SSHBastionPort        int           `mapstructure:"ssh_bastion_port"`
	SSHBastionUsername    string        `mapstructure:"ssh_bastion_username"`
	SSHBastionPassword    string        `mapstructure:"ssh_bastion_password"`
	SSHBastionPrivateKey  string        `mapstructure:"ssh_bastion_private_key_file"`
	SSHFileTransferMethod string        `mapstructure:"ssh_file_transfer_method"`

	// WinRM
	WinRMUser     string        `mapstructure:"winrm_username"`
	WinRMPassword string        `mapstructure:"winrm_password"`
	WinRMHost     string        `mapstructure:"winrm_host"`
	WinRMPort     int           `mapstructure:"winrm_port"`
	WinRMTimeout  time.Duration `mapstructure:"winrm_timeout"`
	WinRMUseSSL   bool          `mapstructure:"winrm_use_ssl"`
	WinRMInsecure bool          `mapstructure:"winrm_insecure"`
}

// Port returns the port that will be used for access based on config.
func (c *Config) Port() int {
	switch c.Type {
	case "ssh":
		return c.SSHPort
	case "winrm":
		return c.WinRMPort
	default:
		return 0
	}
}

func (c *Config) Prepare(ctx *interpolate.Context) []error {
	if c.Type == "" {
		c.Type = "ssh"
	}

	var errs []error
	switch c.Type {
	case "ssh":
		if es := c.prepareSSH(ctx); len(es) > 0 {
			errs = append(errs, es...)
		}
	case "winrm":
		if es := c.prepareWinRM(ctx); len(es) > 0 {
			errs = append(errs, es...)
		}
	case "docker", "none":
		break
	default:
		return []error{fmt.Errorf("Communicator type %s is invalid", c.Type)}
	}

	return errs
}

func (c *Config) prepareSSH(ctx *interpolate.Context) []error {
	if c.SSHPort == 0 {
		c.SSHPort = 22
	}

	if c.SSHTimeout == 0 {
		c.SSHTimeout = 5 * time.Minute
	}

	if c.SSHHandshakeAttempts == 0 {
		c.SSHHandshakeAttempts = 10
	}

	if c.SSHBastionHost != "" {
		if c.SSHBastionPort == 0 {
			c.SSHBastionPort = 22
		}

		if c.SSHBastionPrivateKey == "" && c.SSHPrivateKey != "" {
			c.SSHBastionPrivateKey = c.SSHPrivateKey
		}
	}

	if c.SSHFileTransferMethod == "" {
		c.SSHFileTransferMethod = "scp"
	}

	// Validation
	var errs []error
	if c.SSHUsername == "" {
		errs = append(errs, errors.New("An ssh_username must be specified"))
	}

	if c.SSHPrivateKey != "" {
		if _, err := os.Stat(c.SSHPrivateKey); err != nil {
			errs = append(errs, fmt.Errorf(
				"ssh_private_key_file is invalid: %s", err))
		} else if _, err := SSHFileSigner(c.SSHPrivateKey); err != nil {
			errs = append(errs, fmt.Errorf(
				"ssh_private_key_file is invalid: %s", err))
		}
	}

	if c.SSHBastionHost != "" {
		if c.SSHBastionPassword == "" && c.SSHBastionPrivateKey == "" {
			errs = append(errs, errors.New(
				"ssh_bastion_password or ssh_bastion_private_key_file must be specified"))
		}
	}

	if c.SSHFileTransferMethod != "scp" && c.SSHFileTransferMethod != "sftp" {
		errs = append(errs, fmt.Errorf(
			"ssh_file_transfer_method ('%s') is invalid, valid methods: sftp, scp",
			c.SSHFileTransferMethod))
	}

	return errs
}

func (c *Config) prepareWinRM(ctx *interpolate.Context) []error {
	if c.WinRMPort == 0 && c.WinRMUseSSL {
		c.WinRMPort = 5986
	} else if c.WinRMPort == 0 {
		c.WinRMPort = 5985
	}

	if c.WinRMTimeout == 0 {
		c.WinRMTimeout = 30 * time.Minute
	}

	var errs []error
	if c.WinRMUser == "" {
		errs = append(errs, errors.New("winrm_username must be specified."))
	}

	return errs
}
<|endoftext|>


<commit_before>package sftpd

import (
	"io"
	"log"
	"os"
	"github.com/sandreas/sftp"
)

type vfs struct {
	pathMap PathMapper
}

func VfsHandler(mapper *PathMapper) sftp.Handlers {
	virtualFileSystem := &vfs{
		pathMap: *mapper,
	}
	return sftp.Handlers{
		FileGet: virtualFileSystem,
		FilePut: virtualFileSystem,
		FileCmd: virtualFileSystem,
		FileInfo: virtualFileSystem,
	}
}

func dumpSftpRequest(message string, r sftp.Request) {
	log.Println(message, "Filepath: ", r.Filepath, ", Target: ", r.Target, ", Method: ", r.Method)
}

func (fs *vfs) Fileread(r sftp.Request) (io.ReaderAt, error) {
	dumpSftpRequest("Fileread: ", r)

	filePath, err := fs.pathMap.PathTo(r.Filepath)

	if err == nil {
		f, err := os.Open(filePath)
		if err != nil {
			defer f.Close()
		}
		return f, err
	}
	return nil, err
}

func (fs *vfs) Filewrite(r sftp.Request) (io.WriterAt, error) {
	dumpSftpRequest("Filewrite (disabled): ", r)
	return nil, os.ErrInvalid
}

func (fs *vfs) Filecmd(r sftp.Request) error {
	dumpSftpRequest("Filecmd (disabled): ", r)
	return os.ErrInvalid
}

func (fs *vfs) Fileinfo(r sftp.Request) ([]os.FileInfo, error) {
	dumpSftpRequest("Fileinfo: ", r)
	switch r.Method {
	case "List":
		listing, ok := fs.pathMap.List(r.Filepath)
		if ! ok {
			return nil, os.ErrInvalid
		}

		statList := make([]os.FileInfo, len(listing))
		for i, fileName := range listing {
			stat, err := fs.pathMap.Stat(fileName)
			if err != nil {
				log.Println("Could not stat file", fileName, err)
				continue
			}

			statList[i] = stat
			log.Println("Stat for file "+fileName+": isDir=>", stat.IsDir(), "size=>", stat.Size())
		}
		return statList, nil
	case "Stat":
		stat, err := fs.pathMap.Stat(r.Filepath)
		if err != nil {
			return nil, err
		}
		return []os.FileInfo{stat}, nil
	}
	return nil, os.ErrInvalid
}
<commit_msg>Changed vfshandler to match new sftp.FileList interface<commit_after>package sftpd

import (
	"io"
	"log"
	"os"
	"github.com/sandreas/sftp"
)

type vfs struct {
	pathMap PathMapper
}

func VfsHandler(mapper *PathMapper) sftp.Handlers {
	virtualFileSystem := &vfs{
		pathMap: *mapper,
	}
	return sftp.Handlers{
		FileGet:  virtualFileSystem,
		FilePut:  virtualFileSystem,
		FileCmd:  virtualFileSystem,
		FileList: virtualFileSystem,
	}
}

func dumpSftpRequest(message string, r sftp.Request) {
	log.Println(message, "Filepath: ", r.Filepath, ", Target: ", r.Target, ", Method: ", r.Method)
}

func (fs *vfs) Fileread(r sftp.Request) (io.ReaderAt, error) {
	dumpSftpRequest("Fileread: ", r)

	filePath, err := fs.pathMap.PathTo(r.Filepath)

	if err == nil {
		f, err := os.Open(filePath)
		if err != nil {
			defer f.Close()
		}
		return f, err
	}
	return nil, err
}

func (fs *vfs) Filewrite(r sftp.Request) (io.WriterAt, error) {
	dumpSftpRequest("Filewrite (disabled): ", r)
	return nil, os.ErrInvalid
}

func (fs *vfs) Filecmd(r sftp.Request) error {
	dumpSftpRequest("Filecmd (disabled): ", r)
	return os.ErrInvalid
}

type listerAt []os.FileInfo

// Modeled after strings.Reader's ReadAt() implementation
func (l listerAt) ListAt(ls []os.FileInfo, offset int64) (int, error) {
	var n int
	if offset >= int64(len(l)) {
		return 0, io.EOF
	}
	n = copy(ls, l[offset:])
	if n < len(ls) {
		return n, io.EOF
	}
	return n, nil
}

func (fs *vfs) Filelist(r sftp.Request) (sftp.ListerAt, error) {
	dumpSftpRequest("Fileinfo: ", r)
	switch r.Method {
	case "List":
		listing, ok := fs.pathMap.List(r.Filepath)
		if !ok {
			return nil, os.ErrInvalid
		}

		statList := make([]os.FileInfo, len(listing))
		for i, fileName := range listing {
			stat, err := fs.pathMap.Stat(fileName)
			if err != nil {
				log.Println("Could not stat file", fileName, err)
				continue
			}

			statList[i] = stat
			log.Println("Stat for file "+fileName+": isDir=>", stat.IsDir(), "size=>", stat.Size())
		}
		return listerAt(statList), nil
	case "Stat":
		stat, err := fs.pathMap.Stat(r.Filepath)
		if err != nil {
			return nil, err
		}
		return listerAt([]os.FileInfo{stat}), nil
	}
	return nil, os.ErrInvalid
}
<|endoftext|>


<commit_before>package shared

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	"gopkg.in/robfig/cron.v2"

	"github.com/lxc/lxd/shared/units"
)

type ContainerAction string

const (
	Stop     ContainerAction = "stop"
	Start    ContainerAction = "start"
	Restart  ContainerAction = "restart"
	Freeze   ContainerAction = "freeze"
	Unfreeze ContainerAction = "unfreeze"
)

func IsInt64(value string) error {
	if value == "" {
		return nil
	}

	_, err := strconv.ParseInt(value, 10, 64)
	if err != nil {
		return fmt.Errorf("Invalid value for an integer: %s", value)
	}

	return nil
}

func IsUint8(value string) error {
	if value == "" {
		return nil
	}

	_, err := strconv.ParseUint(value, 10, 8)
	if err != nil {
		return fmt.Errorf("Invalid value for an integer: %s. Must be between 0 and 255", value)
	}

	return nil
}

func IsUint32(value string) error {
	if value == "" {
		return nil
	}

	_, err := strconv.ParseUint(value, 10, 32)
	if err != nil {
		return fmt.Errorf("Invalid value for uint32: %s: %v", value, err)
	}

	return nil
}

func IsPriority(value string) error {
	if value == "" {
		return nil
	}

	valueInt, err := strconv.ParseInt(value, 10, 64)
	if err != nil {
		return fmt.Errorf("Invalid value for an integer: %s", value)
	}

	if valueInt < 0 || valueInt > 10 {
		return fmt.Errorf("Invalid value for a limit '%s'. Must be between 0 and 10", value)
	}

	return nil
}

func IsBool(value string) error {
	if value == "" {
		return nil
	}

	if !StringInSlice(strings.ToLower(value), []string{"true", "false", "yes", "no", "1", "0", "on", "off"}) {
		return fmt.Errorf("Invalid value for a boolean: %s", value)
	}

	return nil
}

func IsOneOf(value string, valid []string) error {
	if value == "" {
		return nil
	}

	if !StringInSlice(value, valid) {
		return fmt.Errorf("Invalid value: %s (not one of %s)", value, valid)
	}

	return nil
}

func IsAny(value string) error {
	return nil
}

// IsRootDiskDevice returns true if the given device representation is
// configured as root disk for a container. It typically get passed a specific
// entry of api.Container.Devices.
func IsRootDiskDevice(device map[string]string) bool {
	if device["type"] == "disk" && device["path"] == "/" && device["source"] == "" {
		return true
	}

	return false
}

// GetRootDiskDevice returns the container device that is configured as root disk
func GetRootDiskDevice(devices map[string]map[string]string) (string, map[string]string, error) {
	var devName string
	var dev map[string]string

	for n, d := range devices {
		if IsRootDiskDevice(d) {
			if devName != "" {
				return "", nil, fmt.Errorf("More than one root device found")
			}

			devName = n
			dev = d
		}
	}

	if devName != "" {
		return devName, dev, nil
	}

	return "", nil, fmt.Errorf("No root device could be found")
}

// KnownContainerConfigKeys maps all fully defined, well-known config keys
// to an appropriate checker function, which validates whether or not a
// given value is syntactically legal.
var KnownContainerConfigKeys = map[string]func(value string) error{
	"boot.autostart":             IsBool,
	"boot.autostart.delay":       IsInt64,
	"boot.autostart.priority":    IsInt64,
	"boot.stop.priority":         IsInt64,
	"boot.host_shutdown_timeout": IsInt64,

	"limits.cpu": func(value string) error {
		if value == "" {
			return nil
		}

		// Validate the character set
		match, _ := regexp.MatchString("^[-,0-9]*$", value)
		if !match {
			return fmt.Errorf("Invalid CPU limit syntax")
		}

		// Validate first character
		if strings.HasPrefix(value, "-") || strings.HasPrefix(value, ",") {
			return fmt.Errorf("CPU limit can't start with a separator")
		}

		// Validate last character
		if strings.HasSuffix(value, "-") || strings.HasSuffix(value, ",") {
			return fmt.Errorf("CPU limit can't end with a separator")
		}

		return nil
	},
	"limits.cpu.allowance": func(value string) error {
		if value == "" {
			return nil
		}

		if strings.HasSuffix(value, "%") {
			// Percentage based allocation
			_, err := strconv.Atoi(strings.TrimSuffix(value, "%"))
			if err != nil {
				return err
			}

			return nil
		}

		// Time based allocation
		fields := strings.SplitN(value, "/", 2)
		if len(fields) != 2 {
			return fmt.Errorf("Invalid allowance: %s", value)
		}

		_, err := strconv.Atoi(strings.TrimSuffix(fields[0], "ms"))
		if err != nil {
			return err
		}

		_, err = strconv.Atoi(strings.TrimSuffix(fields[1], "ms"))
		if err != nil {
			return err
		}

		return nil
	},
	"limits.cpu.priority": IsPriority,

	"limits.disk.priority": IsPriority,

	"limits.memory": func(value string) error {
		if value == "" {
			return nil
		}

		if strings.HasSuffix(value, "%") {
			_, err := strconv.ParseInt(strings.TrimSuffix(value, "%"), 10, 64)
			if err != nil {
				return err
			}

			return nil
		}

		_, err := units.ParseByteSizeString(value)
		if err != nil {
			return err
		}

		return nil
	},
	"limits.memory.enforce": func(value string) error {
		return IsOneOf(value, []string{"soft", "hard"})
	},
	"limits.memory.swap":          IsBool,
	"limits.memory.swap.priority": IsPriority,

	"limits.network.priority": IsPriority,

	"limits.processes": IsInt64,

	"linux.kernel_modules": IsAny,

	"migration.incremental.memory":            IsBool,
	"migration.incremental.memory.iterations": IsUint32,
	"migration.incremental.memory.goal":       IsUint32,

	"nvidia.runtime":             IsBool,
	"nvidia.driver.capabilities": IsAny,
	"nvidia.require.cuda":        IsAny,
	"nvidia.require.driver":      IsAny,

	"security.nesting":       IsBool,
	"security.privileged":    IsBool,
	"security.devlxd":        IsBool,
	"security.devlxd.images": IsBool,

	"security.protection.delete": IsBool,
	"security.protection.shift":  IsBool,

	"security.idmap.base":     IsUint32,
	"security.idmap.isolated": IsBool,
	"security.idmap.size":     IsUint32,

	"security.syscalls.blacklist_default":  IsBool,
	"security.syscalls.blacklist_compat":   IsBool,
	"security.syscalls.blacklist":          IsAny,
	"security.syscalls.intercept.mknod":    IsBool,
	"security.syscalls.intercept.setxattr": IsBool,
	"security.syscalls.whitelist":          IsAny,

	"snapshots.schedule": func(value string) error {
		if value == "" {
			return nil
		}

		if len(strings.Split(value, " ")) != 5 {
			return fmt.Errorf("Schedule must be of the form: <minute> <hour> <day-of-month> <month> <day-of-week>")
		}

		_, err := cron.Parse(fmt.Sprintf("* %s", value))
		if err != nil {
			return errors.Wrap(err, "Error parsing schedule")
		}

		return nil
	},
	"snapshots.schedule.stopped": IsBool,
	"snapshots.pattern":          IsAny,
	"snapshots.expiry": func(value string) error {
		// Validate expression
		_, err := GetSnapshotExpiry(time.Time{}, value)
		return err
	},

	// Caller is responsible for full validation of any raw.* value
	"raw.apparmor": IsAny,
	"raw.lxc":      IsAny,
	"raw.seccomp":  IsAny,
	"raw.idmap":    IsAny,

	"volatile.apply_template":   IsAny,
	"volatile.base_image":       IsAny,
	"volatile.last_state.idmap": IsAny,
	"volatile.last_state.power": IsAny,
	"volatile.idmap.base":       IsAny,
	"volatile.idmap.current":    IsAny,
	"volatile.idmap.next":       IsAny,
	"volatile.apply_quota":      IsAny,
}

// ConfigKeyChecker returns a function that will check whether or not
// a provide value is valid for the associate config key.  Returns an
// error if the key is not known.  The checker function only performs
// syntactic checking of the value, semantic and usage checking must
// be done by the caller.  User defined keys are always considered to
// be valid, e.g. user.* and environment.* keys.
func ConfigKeyChecker(key string) (func(value string) error, error) {
	if f, ok := KnownContainerConfigKeys[key]; ok {
		return f, nil
	}

	if strings.HasPrefix(key, "volatile.") {
		if strings.HasSuffix(key, ".hwaddr") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".name") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".host_name") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".mtu") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".created") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".id") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".vlan") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".spoofcheck") {
			return IsAny, nil
		}
	}

	if strings.HasPrefix(key, "environment.") {
		return IsAny, nil
	}

	if strings.HasPrefix(key, "user.") {
		return IsAny, nil
	}

	if strings.HasPrefix(key, "image.") {
		return IsAny, nil
	}

	if strings.HasPrefix(key, "limits.kernel.") &&
		(len(key) > len("limits.kernel.")) {
		return IsAny, nil
	}

	return nil, fmt.Errorf("Unknown configuration key: %s", key)
}
<commit_msg>share/container: Adds IsNotEmpty to help with validation required fields<commit_after>package shared

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	"gopkg.in/robfig/cron.v2"

	"github.com/lxc/lxd/shared/units"
)

type ContainerAction string

const (
	Stop     ContainerAction = "stop"
	Start    ContainerAction = "start"
	Restart  ContainerAction = "restart"
	Freeze   ContainerAction = "freeze"
	Unfreeze ContainerAction = "unfreeze"
)

func IsInt64(value string) error {
	if value == "" {
		return nil
	}

	_, err := strconv.ParseInt(value, 10, 64)
	if err != nil {
		return fmt.Errorf("Invalid value for an integer: %s", value)
	}

	return nil
}

func IsUint8(value string) error {
	if value == "" {
		return nil
	}

	_, err := strconv.ParseUint(value, 10, 8)
	if err != nil {
		return fmt.Errorf("Invalid value for an integer: %s. Must be between 0 and 255", value)
	}

	return nil
}

func IsUint32(value string) error {
	if value == "" {
		return nil
	}

	_, err := strconv.ParseUint(value, 10, 32)
	if err != nil {
		return fmt.Errorf("Invalid value for uint32: %s: %v", value, err)
	}

	return nil
}

func IsPriority(value string) error {
	if value == "" {
		return nil
	}

	valueInt, err := strconv.ParseInt(value, 10, 64)
	if err != nil {
		return fmt.Errorf("Invalid value for an integer: %s", value)
	}

	if valueInt < 0 || valueInt > 10 {
		return fmt.Errorf("Invalid value for a limit '%s'. Must be between 0 and 10", value)
	}

	return nil
}

func IsBool(value string) error {
	if value == "" {
		return nil
	}

	if !StringInSlice(strings.ToLower(value), []string{"true", "false", "yes", "no", "1", "0", "on", "off"}) {
		return fmt.Errorf("Invalid value for a boolean: %s", value)
	}

	return nil
}

func IsOneOf(value string, valid []string) error {
	if value == "" {
		return nil
	}

	if !StringInSlice(value, valid) {
		return fmt.Errorf("Invalid value: %s (not one of %s)", value, valid)
	}

	return nil
}

func IsAny(value string) error {
	return nil
}

func IsNotEmpty(value string) error {
	if value == "" {
		return fmt.Errorf("Required value")
	}

	return nil
}

// IsRootDiskDevice returns true if the given device representation is
// configured as root disk for a container. It typically get passed a specific
// entry of api.Container.Devices.
func IsRootDiskDevice(device map[string]string) bool {
	if device["type"] == "disk" && device["path"] == "/" && device["source"] == "" {
		return true
	}

	return false
}

// GetRootDiskDevice returns the container device that is configured as root disk
func GetRootDiskDevice(devices map[string]map[string]string) (string, map[string]string, error) {
	var devName string
	var dev map[string]string

	for n, d := range devices {
		if IsRootDiskDevice(d) {
			if devName != "" {
				return "", nil, fmt.Errorf("More than one root device found")
			}

			devName = n
			dev = d
		}
	}

	if devName != "" {
		return devName, dev, nil
	}

	return "", nil, fmt.Errorf("No root device could be found")
}

// KnownContainerConfigKeys maps all fully defined, well-known config keys
// to an appropriate checker function, which validates whether or not a
// given value is syntactically legal.
var KnownContainerConfigKeys = map[string]func(value string) error{
	"boot.autostart":             IsBool,
	"boot.autostart.delay":       IsInt64,
	"boot.autostart.priority":    IsInt64,
	"boot.stop.priority":         IsInt64,
	"boot.host_shutdown_timeout": IsInt64,

	"limits.cpu": func(value string) error {
		if value == "" {
			return nil
		}

		// Validate the character set
		match, _ := regexp.MatchString("^[-,0-9]*$", value)
		if !match {
			return fmt.Errorf("Invalid CPU limit syntax")
		}

		// Validate first character
		if strings.HasPrefix(value, "-") || strings.HasPrefix(value, ",") {
			return fmt.Errorf("CPU limit can't start with a separator")
		}

		// Validate last character
		if strings.HasSuffix(value, "-") || strings.HasSuffix(value, ",") {
			return fmt.Errorf("CPU limit can't end with a separator")
		}

		return nil
	},
	"limits.cpu.allowance": func(value string) error {
		if value == "" {
			return nil
		}

		if strings.HasSuffix(value, "%") {
			// Percentage based allocation
			_, err := strconv.Atoi(strings.TrimSuffix(value, "%"))
			if err != nil {
				return err
			}

			return nil
		}

		// Time based allocation
		fields := strings.SplitN(value, "/", 2)
		if len(fields) != 2 {
			return fmt.Errorf("Invalid allowance: %s", value)
		}

		_, err := strconv.Atoi(strings.TrimSuffix(fields[0], "ms"))
		if err != nil {
			return err
		}

		_, err = strconv.Atoi(strings.TrimSuffix(fields[1], "ms"))
		if err != nil {
			return err
		}

		return nil
	},
	"limits.cpu.priority": IsPriority,

	"limits.disk.priority": IsPriority,

	"limits.memory": func(value string) error {
		if value == "" {
			return nil
		}

		if strings.HasSuffix(value, "%") {
			_, err := strconv.ParseInt(strings.TrimSuffix(value, "%"), 10, 64)
			if err != nil {
				return err
			}

			return nil
		}

		_, err := units.ParseByteSizeString(value)
		if err != nil {
			return err
		}

		return nil
	},
	"limits.memory.enforce": func(value string) error {
		return IsOneOf(value, []string{"soft", "hard"})
	},
	"limits.memory.swap":          IsBool,
	"limits.memory.swap.priority": IsPriority,

	"limits.network.priority": IsPriority,

	"limits.processes": IsInt64,

	"linux.kernel_modules": IsAny,

	"migration.incremental.memory":            IsBool,
	"migration.incremental.memory.iterations": IsUint32,
	"migration.incremental.memory.goal":       IsUint32,

	"nvidia.runtime":             IsBool,
	"nvidia.driver.capabilities": IsAny,
	"nvidia.require.cuda":        IsAny,
	"nvidia.require.driver":      IsAny,

	"security.nesting":       IsBool,
	"security.privileged":    IsBool,
	"security.devlxd":        IsBool,
	"security.devlxd.images": IsBool,

	"security.protection.delete": IsBool,
	"security.protection.shift":  IsBool,

	"security.idmap.base":     IsUint32,
	"security.idmap.isolated": IsBool,
	"security.idmap.size":     IsUint32,

	"security.syscalls.blacklist_default":  IsBool,
	"security.syscalls.blacklist_compat":   IsBool,
	"security.syscalls.blacklist":          IsAny,
	"security.syscalls.intercept.mknod":    IsBool,
	"security.syscalls.intercept.setxattr": IsBool,
	"security.syscalls.whitelist":          IsAny,

	"snapshots.schedule": func(value string) error {
		if value == "" {
			return nil
		}

		if len(strings.Split(value, " ")) != 5 {
			return fmt.Errorf("Schedule must be of the form: <minute> <hour> <day-of-month> <month> <day-of-week>")
		}

		_, err := cron.Parse(fmt.Sprintf("* %s", value))
		if err != nil {
			return errors.Wrap(err, "Error parsing schedule")
		}

		return nil
	},
	"snapshots.schedule.stopped": IsBool,
	"snapshots.pattern":          IsAny,
	"snapshots.expiry": func(value string) error {
		// Validate expression
		_, err := GetSnapshotExpiry(time.Time{}, value)
		return err
	},

	// Caller is responsible for full validation of any raw.* value
	"raw.apparmor": IsAny,
	"raw.lxc":      IsAny,
	"raw.seccomp":  IsAny,
	"raw.idmap":    IsAny,

	"volatile.apply_template":   IsAny,
	"volatile.base_image":       IsAny,
	"volatile.last_state.idmap": IsAny,
	"volatile.last_state.power": IsAny,
	"volatile.idmap.base":       IsAny,
	"volatile.idmap.current":    IsAny,
	"volatile.idmap.next":       IsAny,
	"volatile.apply_quota":      IsAny,
}

// ConfigKeyChecker returns a function that will check whether or not
// a provide value is valid for the associate config key.  Returns an
// error if the key is not known.  The checker function only performs
// syntactic checking of the value, semantic and usage checking must
// be done by the caller.  User defined keys are always considered to
// be valid, e.g. user.* and environment.* keys.
func ConfigKeyChecker(key string) (func(value string) error, error) {
	if f, ok := KnownContainerConfigKeys[key]; ok {
		return f, nil
	}

	if strings.HasPrefix(key, "volatile.") {
		if strings.HasSuffix(key, ".hwaddr") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".name") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".host_name") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".mtu") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".created") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".id") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".vlan") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".spoofcheck") {
			return IsAny, nil
		}
	}

	if strings.HasPrefix(key, "environment.") {
		return IsAny, nil
	}

	if strings.HasPrefix(key, "user.") {
		return IsAny, nil
	}

	if strings.HasPrefix(key, "image.") {
		return IsAny, nil
	}

	if strings.HasPrefix(key, "limits.kernel.") &&
		(len(key) > len("limits.kernel.")) {
		return IsAny, nil
	}

	return nil, fmt.Errorf("Unknown configuration key: %s", key)
}
<|endoftext|>


<commit_before>//  Copyright 2016 Red Hat, Inc.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package cmd

import (
	"bytes"
	"fmt"

	"github.com/ghodss/yaml"
	"github.com/spf13/cobra"

	"k8s.io/client-go/1.5/kubernetes"
	"k8s.io/client-go/1.5/pkg/api"
	"k8s.io/client-go/1.5/pkg/api/v1"
	"k8s.io/client-go/1.5/pkg/apis/extensions/v1beta1"

	"github.com/funktionio/funktion/pkg/funktion"
	"github.com/funktionio/funktion/pkg/spec"
)

type getCmd struct {
	kubeclient     *kubernetes.Clientset
	cmd            *cobra.Command
	kubeConfigPath string

	kind      string
	namespace string
	name      string

	deployments map[string]*v1beta1.Deployment
	services    map[string]*v1.Service
}

func init() {
	RootCmd.AddCommand(newGetCmd())
}

func newGetCmd() *cobra.Command {
	p := &getCmd{}
	cmd := &cobra.Command{
		Use:   "get KIND [NAME] [flags]",
		Short: "gets a list of the resources",
		Long:  `This command will list all of the resources of a given kind`,
		Run: func(cmd *cobra.Command, args []string) {
			p.cmd = cmd
			if len(args) == 0 {
				handleError(fmt.Errorf("No resource kind argument supplied! Possible values ['connector', 'flow', 'function', 'runtime']"))
				return
			}
			p.kind = args[0]
			if len(args) > 1 {
				p.name = args[1]
			}
			err := createKubernetesClient(cmd, p.kubeConfigPath, &p.kubeclient, &p.namespace)
			if err != nil {
				handleError(err)
				return
			}
			handleError(p.run())
		},
	}
	f := cmd.Flags()
	//f.StringVarP(&p.format, "output", "o", "", "The format of the output")
	f.StringVar(&p.kubeConfigPath, "kubeconfig", "", "the directory to look for the kubernetes configuration")
	f.StringVarP(&p.namespace, "namespace", "n", "", "the namespace to query")
	return cmd
}

func (p *getCmd) run() error {
	kind, listOpts, err := listOptsForKind(p.kind)
	if err != nil {
		return err
	}
	kubeclient := p.kubeclient
	cms := kubeclient.ConfigMaps(p.namespace)
	resources, err := cms.List(*listOpts)
	if err != nil {
		return err
	}
	p.deployments = map[string]*v1beta1.Deployment{}
	p.services = map[string]*v1.Service{}
	ds, err := kubeclient.Deployments(p.namespace).List(api.ListOptions{})
	if err != nil {
		return err
	}
	p.deployments = map[string]*v1beta1.Deployment{}
	for _, item := range ds.Items {
		// TODO lets assume the name of the Deployment is the name of the Flow
		// but we may want to use a label instead to link them?
		name := item.Name
		p.deployments[name] = &item
	}
	if kind == functionKind {
		ss, err := kubeclient.Services(p.namespace).List(api.ListOptions{})
		if err != nil {
			return err
		}
		for _, item := range ss.Items {
			// TODO lets assume the name of the Service is the name of the Function
			// but we may want to use a label instead to link them?
			name := item.Name
			p.services[name] = &item
		}
	}
	name := p.name
	if len(name) == 0 {
		p.printHeader(kind)
		for _, resource := range resources.Items {
			p.printResource(&resource, kind)
		}

	} else {
		found := false
		for _, resource := range resources.Items {
			if resource.Name == name {
				p.printHeader(kind)
				p.printResource(&resource, kind)
				found = true
				break
			}
		}
		if !found {
			return fmt.Errorf("%s \"%s\" not found", kind, name)
		}
	}
	return nil
}

func (p *getCmd) printHeader(kind string) {
	switch kind {
	case flowKind:
		printFlowRow("NAME", "PODS", "STEPS")
	case functionKind:
		printFunctionRow("NAME", "PODS", "URL")
	default:
		printRuntimeRow("NAME", "VERSION")
	}
}

func (p *getCmd) printResource(cm *v1.ConfigMap, kind string) {
	switch kind {
	case functionKind:
		printFunctionRow(cm.Name, p.podText(cm), p.functionURLText(cm))
	case flowKind:
		printFlowRow(cm.Name, p.podText(cm), p.flowStepsText(cm))
	default:
		printRuntimeRow(cm.Name, p.runtimeVersion(cm))
	}
}

func printFunctionRow(name string, pod string, flow string) {
	fmt.Printf("%-32s %-9s %s\n", name, pod, flow)
}

func printFlowRow(name string, pod string, flow string) {
	fmt.Printf("%-32s %-9s %s\n", name, pod, flow)
}

func printRuntimeRow(name string, version string) {
	fmt.Printf("%-32s %s\n", name, version)
}

func (p *getCmd) podText(cm *v1.ConfigMap) string {
	name := cm.Name
	deployment := p.deployments[name]
	if deployment == nil {
		return ""
	}
	var status = deployment.Status
	return fmt.Sprintf("%d/%d", status.AvailableReplicas, status.Replicas)
}

func (p *getCmd) functionURLText(cm *v1.ConfigMap) string {
	name := cm.Name
	service := p.services[name]
	if service == nil || service.Annotations == nil {
		return ""
	}
	return service.Annotations[exposeURLAnnotation]
}

func (p *getCmd) runtimeVersion(cm *v1.ConfigMap) string {
	labels := cm.Labels
	if labels != nil {
		return labels[funktion.VersionLabel]
	}
	return ""
}

func (p *getCmd) flowStepsText(cm *v1.ConfigMap) string {
	yamlText := cm.Data[funktion.FunktionYmlProperty]
	if len(yamlText) == 0 {
		return fmt.Sprintf("No `%s` property specified", funktion.FunktionYmlProperty)
	}
	fc := spec.FunkionConfig{}
	err := yaml.Unmarshal([]byte(yamlText), &fc)
	if err != nil {
		return fmt.Sprintf("Failed to parse `%s` YAML: %v", funktion.FunktionYmlProperty, err)
	}
	if len(fc.Flows) == 0 {
		return "No funktion flows"
	}
	rule := fc.Flows[0]
	return stepsText(rule.Steps)
}

func stepsText(steps []spec.FunktionStep) string {
	actionMessage := "No steps!"
	if len(steps) > 0 {
		var buffer bytes.Buffer
		for i, step := range steps {
			if i > 0 {
				buffer.WriteString(" => ")
			}
			kind := step.Kind
			text := kind
			switch kind {
			case spec.EndpointKind:
				text = fmt.Sprintf("%s", step.URI)
			case spec.FunctionKind:
				text = fmt.Sprintf("function %s", step.Name)
			}
			buffer.WriteString(text)
		}
		actionMessage = buffer.String()
	}
	return actionMessage
}
<commit_msg>fixed bug listing functions<commit_after>//  Copyright 2016 Red Hat, Inc.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package cmd

import (
	"bytes"
	"fmt"

	"github.com/ghodss/yaml"
	"github.com/spf13/cobra"

	"k8s.io/client-go/1.5/kubernetes"
	"k8s.io/client-go/1.5/pkg/api"
	"k8s.io/client-go/1.5/pkg/api/v1"
	"k8s.io/client-go/1.5/pkg/apis/extensions/v1beta1"

	"github.com/funktionio/funktion/pkg/funktion"
	"github.com/funktionio/funktion/pkg/spec"
)

type getCmd struct {
	kubeclient     *kubernetes.Clientset
	cmd            *cobra.Command
	kubeConfigPath string

	kind      string
	namespace string
	name      string

	deployments map[string]*v1beta1.Deployment
	services    map[string]*v1.Service
}

func init() {
	RootCmd.AddCommand(newGetCmd())
}

func newGetCmd() *cobra.Command {
	p := &getCmd{}
	cmd := &cobra.Command{
		Use:   "get KIND [NAME] [flags]",
		Short: "gets a list of the resources",
		Long:  `This command will list all of the resources of a given kind`,
		Run: func(cmd *cobra.Command, args []string) {
			p.cmd = cmd
			if len(args) == 0 {
				handleError(fmt.Errorf("No resource kind argument supplied! Possible values ['connector', 'flow', 'function', 'runtime']"))
				return
			}
			p.kind = args[0]
			if len(args) > 1 {
				p.name = args[1]
			}
			err := createKubernetesClient(cmd, p.kubeConfigPath, &p.kubeclient, &p.namespace)
			if err != nil {
				handleError(err)
				return
			}
			handleError(p.run())
		},
	}
	f := cmd.Flags()
	//f.StringVarP(&p.format, "output", "o", "", "The format of the output")
	f.StringVar(&p.kubeConfigPath, "kubeconfig", "", "the directory to look for the kubernetes configuration")
	f.StringVarP(&p.namespace, "namespace", "n", "", "the namespace to query")
	return cmd
}

func (p *getCmd) run() error {
	kind, listOpts, err := listOptsForKind(p.kind)
	if err != nil {
		return err
	}
	kubeclient := p.kubeclient
	cms := kubeclient.ConfigMaps(p.namespace)
	resources, err := cms.List(*listOpts)
	if err != nil {
		return err
	}
	p.deployments = map[string]*v1beta1.Deployment{}
	p.services = map[string]*v1.Service{}
	ds, err := kubeclient.Deployments(p.namespace).List(api.ListOptions{})
	if err != nil {
		return err
	}
	p.deployments = map[string]*v1beta1.Deployment{}
	for _, item := range ds.Items {
		// TODO lets assume the name of the Deployment is the name of the Flow
		// but we may want to use a label instead to link them?
		name := item.Name
		copy := v1beta1.Deployment{}
		copy = item
		p.deployments[name] = &copy
	}
	if kind == functionKind {
		ss, err := kubeclient.Services(p.namespace).List(api.ListOptions{})
		if err != nil {
			return err
		}
		for _, item := range ss.Items {
			// TODO lets assume the name of the Service is the name of the Function
			// but we may want to use a label instead to link them?
			name := item.Name
			copy := v1.Service{}
			copy = item
			p.services[name] = &copy
		}
	}
	name := p.name
	if len(name) == 0 {
		p.printHeader(kind)
		for _, resource := range resources.Items {
			p.printResource(&resource, kind)
		}

	} else {
		found := false
		for _, resource := range resources.Items {
			if resource.Name == name {
				p.printHeader(kind)
				p.printResource(&resource, kind)
				found = true
				break
			}
		}
		if !found {
			return fmt.Errorf("%s \"%s\" not found", kind, name)
		}
	}
	return nil
}

func (p *getCmd) printHeader(kind string) {
	switch kind {
	case flowKind:
		printFlowRow("NAME", "PODS", "STEPS")
	case functionKind:
		printFunctionRow("NAME", "PODS", "URL")
	default:
		printRuntimeRow("NAME", "VERSION")
	}
}

func (p *getCmd) printResource(cm *v1.ConfigMap, kind string) {
	switch kind {
	case functionKind:
		printFunctionRow(cm.Name, p.podText(cm), p.functionURLText(cm))
	case flowKind:
		printFlowRow(cm.Name, p.podText(cm), p.flowStepsText(cm))
	default:
		printRuntimeRow(cm.Name, p.runtimeVersion(cm))
	}
}

func printFunctionRow(name string, pod string, flow string) {
	fmt.Printf("%-32s %-9s %s\n", name, pod, flow)
}

func printFlowRow(name string, pod string, flow string) {
	fmt.Printf("%-32s %-9s %s\n", name, pod, flow)
}

func printRuntimeRow(name string, version string) {
	fmt.Printf("%-32s %s\n", name, version)
}

func (p *getCmd) podText(cm *v1.ConfigMap) string {
	name := cm.Name
	deployment := p.deployments[name]
	if deployment == nil {
		return ""
	}
	var status = deployment.Status
	return fmt.Sprintf("%d/%d", status.AvailableReplicas, status.Replicas)
}

func (p *getCmd) functionURLText(cm *v1.ConfigMap) string {
	name := cm.Name
	service := p.services[name]
	if service == nil || service.Annotations == nil {
		return ""
	}
	return service.Annotations[exposeURLAnnotation]
}

func (p *getCmd) runtimeVersion(cm *v1.ConfigMap) string {
	labels := cm.Labels
	if labels != nil {
		return labels[funktion.VersionLabel]
	}
	return ""
}

func (p *getCmd) flowStepsText(cm *v1.ConfigMap) string {
	yamlText := cm.Data[funktion.FunktionYmlProperty]
	if len(yamlText) == 0 {
		return fmt.Sprintf("No `%s` property specified", funktion.FunktionYmlProperty)
	}
	fc := spec.FunkionConfig{}
	err := yaml.Unmarshal([]byte(yamlText), &fc)
	if err != nil {
		return fmt.Sprintf("Failed to parse `%s` YAML: %v", funktion.FunktionYmlProperty, err)
	}
	if len(fc.Flows) == 0 {
		return "No funktion flows"
	}
	rule := fc.Flows[0]
	return stepsText(rule.Steps)
}

func stepsText(steps []spec.FunktionStep) string {
	actionMessage := "No steps!"
	if len(steps) > 0 {
		var buffer bytes.Buffer
		for i, step := range steps {
			if i > 0 {
				buffer.WriteString(" => ")
			}
			kind := step.Kind
			text := kind
			switch kind {
			case spec.EndpointKind:
				text = fmt.Sprintf("%s", step.URI)
			case spec.FunctionKind:
				text = fmt.Sprintf("function %s", step.Name)
			}
			buffer.WriteString(text)
		}
		actionMessage = buffer.String()
	}
	return actionMessage
}
<|endoftext|>


<commit_before>package parse

import "./comb"

const (
	bracketChars = "()[]{}"
	commentChar  = ';'
	invalidChars = "\x00"
	quoteChar    = '`'
	quoteString  = "quote"
	spaceChars   = " \t\n\r"
	specialChar  = '$'
)

func Parse(source string) []interface{} {
	m, err := newState(source).module()()

	if err != nil {
		panic(err.Error())
	}

	return m.([]interface{})
}

func (s *state) module() comb.Parser {
	return s.Exhaust(s.Wrap(s.blank(), s.expressions(), s.None()))
}

func (s *state) expressions() comb.Parser {
	return s.Lazy(s.strictExpressions)
}

func (s *state) strictExpressions() comb.Parser {
	return s.Many(s.expression())
}

func (s *state) expression() comb.Parser {
	return s.strip(s.Or(
		s.firstOrderExpression(),
		s.Lazy(func() comb.Parser { return s.quote(s.expression()) })))
}

func (s *state) firstOrderExpression() comb.Parser {
	return s.Or(s.atom(), s.list(), s.array(), s.dict())
}

func (s *state) atom() comb.Parser {
	return s.Or(s.stringLiteral(), s.identifier())
}

func (s *state) identifier() comb.Parser {
	return s.stringify(s.Many1(s.NotInString(
		bracketChars + string(commentChar) + invalidChars + string(quoteChar) +
			spaceChars + string(specialChar))))
}

func (s *state) stringLiteral() comb.Parser {
	b := s.blank()
	c := s.Char('"')

	return s.App(prependQuote, s.stringify(s.Wrap(
		c,
		s.Many(s.Or(s.NotInString("\"\\"), s.String("\\\""), s.String("\\\\"))),
		s.And(c, b))))
}

func prependQuote(x interface{}) interface{} {
	return []interface{}{quoteString, x}
}

func (s *state) list() comb.Parser {
	return s.sequence('(', ')')
}

func (s *state) array() comb.Parser {
	return s.sequence('[', ']')
}

func (s *state) dict() comb.Parser {
	return s.sequence('{', '}')
}

func (s *state) sequence(l, r rune) comb.Parser {
	return s.wrapChars(l, s.expressions(), r)
}

func (s *state) comment() comb.Parser {
	return s.Void(s.And(
		s.Char(commentChar),
		s.Many(s.NotChar('\n')), s.Char('\n')))
}

func (s *state) wrapChars(l rune, p comb.Parser, r rune) comb.Parser {
	return s.Wrap(s.strip(s.Char(l)), p, s.strip(s.Char(r)))
}

func (s *state) strip(p comb.Parser) comb.Parser {
	b := s.blank()
	return s.Wrap(s.None(), p, b)
}

func (s *state) blank() comb.Parser {
	return s.Void(s.Many(s.Or(s.space(), s.comment())))
}

func (s *state) space() comb.Parser {
	return s.Void(s.Many1(s.InString(spaceChars)))
}

func (s *state) quote(p comb.Parser) comb.Parser {
	return s.And(s.Replace(quoteString, s.Char(quoteChar)), p)
}

func (s *state) stringify(p comb.Parser) comb.Parser {
	return s.App(func(any interface{}) interface{} {
		xs := any.([]interface{})
		rs := make([]rune, len(xs))

		for i, x := range xs {
			rs[i] = x.(rune)
		}

		return string(rs)
	}, p)
}
<commit_msg>Remove space parser<commit_after>package parse

import "./comb"

const (
	bracketChars = "()[]{}"
	commentChar  = ';'
	invalidChars = "\x00"
	quoteChar    = '`'
	quoteString  = "quote"
	spaceChars   = " \t\n\r"
	specialChar  = '$'
)

func Parse(source string) []interface{} {
	m, err := newState(source).module()()

	if err != nil {
		panic(err.Error())
	}

	return m.([]interface{})
}

func (s *state) module() comb.Parser {
	return s.Exhaust(s.Wrap(s.blank(), s.expressions(), s.None()))
}

func (s *state) expressions() comb.Parser {
	return s.Lazy(s.strictExpressions)
}

func (s *state) strictExpressions() comb.Parser {
	return s.Many(s.expression())
}

func (s *state) expression() comb.Parser {
	return s.strip(s.Or(
		s.firstOrderExpression(),
		s.Lazy(func() comb.Parser { return s.quote(s.expression()) })))
}

func (s *state) firstOrderExpression() comb.Parser {
	return s.Or(s.atom(), s.list(), s.array(), s.dict())
}

func (s *state) atom() comb.Parser {
	return s.Or(s.stringLiteral(), s.identifier())
}

func (s *state) identifier() comb.Parser {
	return s.stringify(s.Many1(s.NotInString(
		bracketChars + string(commentChar) + invalidChars + string(quoteChar) +
			spaceChars + string(specialChar))))
}

func (s *state) stringLiteral() comb.Parser {
	b := s.blank()
	c := s.Char('"')

	return s.App(prependQuote, s.stringify(s.Wrap(
		c,
		s.Many(s.Or(s.NotInString("\"\\"), s.String("\\\""), s.String("\\\\"))),
		s.And(c, b))))
}

func prependQuote(x interface{}) interface{} {
	return []interface{}{quoteString, x}
}

func (s *state) list() comb.Parser {
	return s.sequence('(', ')')
}

func (s *state) array() comb.Parser {
	return s.sequence('[', ']')
}

func (s *state) dict() comb.Parser {
	return s.sequence('{', '}')
}

func (s *state) sequence(l, r rune) comb.Parser {
	return s.wrapChars(l, s.expressions(), r)
}

func (s *state) comment() comb.Parser {
	return s.Void(s.And(
		s.Char(commentChar),
		s.Many(s.NotChar('\n')), s.Char('\n')))
}

func (s *state) wrapChars(l rune, p comb.Parser, r rune) comb.Parser {
	return s.Wrap(s.strip(s.Char(l)), p, s.strip(s.Char(r)))
}

func (s *state) strip(p comb.Parser) comb.Parser {
	b := s.blank()
	return s.Wrap(s.None(), p, b)
}

func (s *state) blank() comb.Parser {
	return s.Void(s.Many(s.Or(s.InString(spaceChars), s.comment())))
}

func (s *state) quote(p comb.Parser) comb.Parser {
	return s.And(s.Replace(quoteString, s.Char(quoteChar)), p)
}

func (s *state) stringify(p comb.Parser) comb.Parser {
	return s.App(func(any interface{}) interface{} {
		xs := any.([]interface{})
		rs := make([]rune, len(xs))

		for i, x := range xs {
			rs[i] = x.(rune)
		}

		return string(rs)
	}, p)
}
<|endoftext|>


<commit_before>/*

   Copyright 2013 Niklas Voss

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

*/

package golem

import (
	"encoding/json"
	"errors"
	"strings"
)

var (
	initialProtocol Protocol = Protocol(&DefaultJSONProtocol{})
)

// Protocol-interface provides the required methods necessary for any
// protocol, that should be used with golem, to implement.
// The evented system of golem needs several steps to process incoming data:
// 1. Unpack
// 2. Unmarshal
// For emitting data the process is reversed:
// 1. Marshal
// 2. Pack
type Protocol interface {
	// Unmarshals leftover data into associated type of callback.
	Unmarshal([]byte, interface{}) error
	// Marshal data into byte array
	Marshal(interface{}) ([]byte, error)
	// Unpack splits/extracts event name from incoming data.
	Unpack([]byte) (string, []byte, error)
	// Pack event name into byte array.
	Pack(string, []byte) ([]byte, error)
}

// SetInitialProtocol sets the initial protocol for router creation. Every router
// created after changing the initial protocol will use the new protocol by default.
func SetInitialProtocol(protocol Protocol) {
	initialProtocol = protocol
}

const (
	protocolSeperator = " "
)

type DefaultJSONProtocol struct{}

func (_ *DefaultJSONProtocol) Unmarshal(data []byte, structPtr interface{}) error {
	return json.Unmarshal(data, structPtr)
}

func (_ *DefaultJSONProtocol) Marshal(structPtr interface{}) ([]byte, error) {
	return json.Marshal(structPtr)
}

func (_ *DefaultJSONProtocol) Unpack(data []byte) (string, []byte, error) {
	result := strings.SplitN(string(data), protocolSeperator, 2)
	if len(result) != 2 {
		return "", nil, errors.New("Unable to extract event name from data.")
	}
	return result[0], []byte(result[1]), nil
}

func (_ *DefaultJSONProtocol) Pack(name string, data []byte) ([]byte, error) {
	result := []byte(name + protocolSeperator)
	result = append(result, data...)
	return result, nil
}
<commit_msg>Finished documentation on DefaultJSONProtocol.<commit_after>/*

   Copyright 2013 Niklas Voss

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

*/

package golem

import (
	"encoding/json"
	"errors"
	"strings"
)

var (
	initialProtocol Protocol = Protocol(&DefaultJSONProtocol{})
)

// Protocol-interface provides the required methods necessary for any
// protocol, that should be used with golem, to implement.
// The evented system of golem needs several steps to process incoming data:
//  1. Unpack
//  2. Unmarshal
// For emitting data the process is reversed:
//  1. Marshal
//  2. Pack
type Protocol interface {
	// Unpack splits/extracts event name from incoming data.
	Unpack([]byte) (string, []byte, error)
	// Unmarshals leftover data into associated type of callback.
	Unmarshal([]byte, interface{}) error
	// Marshal data into byte array
	Marshal(interface{}) ([]byte, error)
	// Pack event name into byte array.
	Pack(string, []byte) ([]byte, error)
}

// SetInitialProtocol sets the initial protocol for router creation. Every router
// created after changing the initial protocol will use the new protocol by default.
func SetInitialProtocol(protocol Protocol) {
	initialProtocol = protocol
}

const (
	protocolSeperator = " "
)

// DefaultJSONProtocol is the initial protocol used by golem.
// (Note: there is an article about this simple protocol in golem's wiki)
type DefaultJSONProtocol struct{}

// Unpack splits the event name from the incoming message.
func (_ *DefaultJSONProtocol) Unpack(data []byte) (string, []byte, error) {
	result := strings.SplitN(string(data), protocolSeperator, 2)
	if len(result) != 2 {
		return "", nil, errors.New("Unable to extract event name from data.")
	}
	return result[0], []byte(result[1]), nil
}

// Unmarshals data into requested structure. If not successful the function return an error.
func (_ *DefaultJSONProtocol) Unmarshal(data []byte, structPtr interface{}) error {
	return json.Unmarshal(data, structPtr)
}

// Marshals structure into JSON. If not successful second return value is an error.
func (_ *DefaultJSONProtocol) Marshal(structPtr interface{}) ([]byte, error) {
	return json.Marshal(structPtr)
}

// Adds the event name to the message.
func (_ *DefaultJSONProtocol) Pack(name string, data []byte) ([]byte, error) {
	result := []byte(name + protocolSeperator)
	result = append(result, data...)
	return result, nil
}
<|endoftext|>