<commit_before>package utility

import (
    "fmt"
    "github.com/mitchellh/go-homedir"
    "gopkg.in/ini.v1"
    "log"
    "os"
    "strings"
)

type Util struct {
    ConfigPath     string
    CustomAppNames string
    Apps           map[string]*[]string
}

func NewUtil() *Util {
    home, err := homedir.Dir()
    if err != nil {
        log.Fatalln(err)
    }
    util := &Util{
        ConfigPath: fmt.Sprintf(
            "%s%s.twhelp.ini",
            home,
            string(os.PathSeparator),
        ),
        Apps: make(map[string]*[]string, 8),
    }
    util.Apps["android"] = &[]string{
        "3nVuSoBZnx6U4vzUxf5w",
        "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys",
    }
    util.Apps["win"] = &[]string{
        "TgHNMa7WZE7Cxi1JbkAMQ",
        "SHy9mBMBPNj3Y17et9BF4g5XeqS4y3vkeW24PttDcY",
    }
    util.Apps["wp"] = &[]string{
        "yN3DUNVO0Me63IAQdhTfCA",
        "c768oTKdzAjIYCmpSNIdZbGaG0t6rOhSFQP0S5uC79g",
    }
    util.Apps["google"] = &[]string{
        "iAtYJ4HpUVfIUoNnif1DA",
        "172fOpzuZoYzNYaU3mMYvE8m8MEyLbztOdbrUolU",
    }
    util.Apps["iphone"] = &[]string{
        "IQKbtAYlXLripLGPWd0HUA",
        "GgDYlkSvaPxGxC4X8liwpUoqKwwr3lCADbz8A7ADU",
    }
    util.Apps["ipad"] = &[]string{
        "CjulERsDeqhhjSme66ECg",
        "IQWdVyqFxghAtURHGeGiWAsmCAGmdW3WmbEx6Hck",
    }
    util.Apps["mac"] = &[]string{
        "3rJOl1ODzm9yZy63FACdg",
        "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8",
    }
    util.Apps["deck"] = &[]string{
        "yT577ApRtZw51q4NPMPPOQ",
        "3neq3XqN5fO3obqwZoajavGFCUrC42ZfbrLXy5sCv8",
    }
    customAppNames := make([]string, 0)
    if _, err := os.Stat(util.ConfigPath); err == nil {
        cfg, err := ini.Load(util.ConfigPath)
        if err != nil {
            log.Fatalln(err)
        }
        for _, name := range cfg.SectionStrings() {
            if name == "DEFAULT" {
                continue
            }
            section := cfg.Section(name)
            ck, ckerr := section.GetKey("consumer_key")
            if ckerr != nil {
                log.Fatalln(fmt.Sprintf(`"consumer_key" for %s does not exist`, name))
            }
            cs, cserr := section.GetKey("consumer_secret")
            if cserr != nil {
                log.Fatalln(fmt.Sprintf(`"consumer_secret" for %s does not exist`, name))
            }
            util.Apps[name] = &[]string{ck.String(), cs.String()}
            customAppNames = append(customAppNames, name)
        }
    }
    if len(customAppNames) > 0 {
        util.CustomAppNames = fmt.Sprintf(
            "Config File: %s\nCustom Apps: %s\n",
            util.ConfigPath,
            strings.Join(customAppNames, ", "),
        )
    } else {
        util.CustomAppNames = fmt.Sprintf(
            `Your own applications also can be defined in %s
Example:

[my_app_01]
consumer_key    = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
consumer_secret = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
`, util.ConfigPath)
    }
    return util
}

func (util *Util) Usage() {
    os.Stderr.WriteString(fmt.Sprintf(`Usage: %s [options]
Options:
  -h, --help          Show help.

[ Output Format ]

  Default             Output line by line.
  -i, --ini           Output as INI.
  -y, --yaml          Output as YAML.
  -j, --json          Output as JSON.
  -a, --array         Output as array that compatible with most languages.
  -A, --assoc         Output as PHP-style associative array.
  -e, --env           Output as environmental uppercase variables.

[ OAuth Process ]

  Default             xAuth manipulation with OAuth scraping.
  -x, --xauth         Pure xAuth. Only available with official keys.
  -o, --oauth         Pure OAuth. You have to authorize via web browser.

[ OAuth Credentials ]

  Insufficient components are required to input via STDIN.
  Password is masked.

  --ck  <value>       Specify consumer_key in advance.
  --cs  <value>       Specify consumer_secret in advance.
  --sn  <value>       Specify screen_name or email in advance.
  --pw  <value>       Specify password in advance. (Not masked, DEPRECATED)
  --app <value>       Speficy consumer_key and consumer_secret with app name.

                      app name | full name
                      ------------------------------------
                      android  | Twitter for Andriod
                      win      | Twitter for Andriod
                      wp       | Twitter for Windows Phone
                      google   | Twitter for Google TV
                      iphone   | Twitter for iPhone
                      ipad     | Twitter for iPad
                      mac      | Twitter for Mac
                      deck     | TweetDeck

%s
`, os.Args[0], util.CustomAppNames))
}
<commit_msg>win is not Android, but Windows<commit_after>package utility

import (
    "fmt"
    "github.com/mitchellh/go-homedir"
    "gopkg.in/ini.v1"
    "log"
    "os"
    "strings"
)

type Util struct {
    ConfigPath     string
    CustomAppNames string
    Apps           map[string]*[]string
}

func NewUtil() *Util {
    home, err := homedir.Dir()
    if err != nil {
        log.Fatalln(err)
    }
    util := &Util{
        ConfigPath: fmt.Sprintf(
            "%s%s.twhelp.ini",
            home,
            string(os.PathSeparator),
        ),
        Apps: make(map[string]*[]string, 8),
    }
    util.Apps["android"] = &[]string{
        "3nVuSoBZnx6U4vzUxf5w",
        "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys",
    }
    util.Apps["win"] = &[]string{
        "TgHNMa7WZE7Cxi1JbkAMQ",
        "SHy9mBMBPNj3Y17et9BF4g5XeqS4y3vkeW24PttDcY",
    }
    util.Apps["wp"] = &[]string{
        "yN3DUNVO0Me63IAQdhTfCA",
        "c768oTKdzAjIYCmpSNIdZbGaG0t6rOhSFQP0S5uC79g",
    }
    util.Apps["google"] = &[]string{
        "iAtYJ4HpUVfIUoNnif1DA",
        "172fOpzuZoYzNYaU3mMYvE8m8MEyLbztOdbrUolU",
    }
    util.Apps["iphone"] = &[]string{
        "IQKbtAYlXLripLGPWd0HUA",
        "GgDYlkSvaPxGxC4X8liwpUoqKwwr3lCADbz8A7ADU",
    }
    util.Apps["ipad"] = &[]string{
        "CjulERsDeqhhjSme66ECg",
        "IQWdVyqFxghAtURHGeGiWAsmCAGmdW3WmbEx6Hck",
    }
    util.Apps["mac"] = &[]string{
        "3rJOl1ODzm9yZy63FACdg",
        "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8",
    }
    util.Apps["deck"] = &[]string{
        "yT577ApRtZw51q4NPMPPOQ",
        "3neq3XqN5fO3obqwZoajavGFCUrC42ZfbrLXy5sCv8",
    }
    customAppNames := make([]string, 0)
    if _, err := os.Stat(util.ConfigPath); err == nil {
        cfg, err := ini.Load(util.ConfigPath)
        if err != nil {
            log.Fatalln(err)
        }
        for _, name := range cfg.SectionStrings() {
            if name == "DEFAULT" {
                continue
            }
            section := cfg.Section(name)
            ck, ckerr := section.GetKey("consumer_key")
            if ckerr != nil {
                log.Fatalln(fmt.Sprintf(`"consumer_key" for %s does not exist`, name))
            }
            cs, cserr := section.GetKey("consumer_secret")
            if cserr != nil {
                log.Fatalln(fmt.Sprintf(`"consumer_secret" for %s does not exist`, name))
            }
            util.Apps[name] = &[]string{ck.String(), cs.String()}
            customAppNames = append(customAppNames, name)
        }
    }
    if len(customAppNames) > 0 {
        util.CustomAppNames = fmt.Sprintf(
            "Config File: %s\nCustom Apps: %s\n",
            util.ConfigPath,
            strings.Join(customAppNames, ", "),
        )
    } else {
        util.CustomAppNames = fmt.Sprintf(
            `Your own applications also can be defined in %s
Example:

[my_app_01]
consumer_key    = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
consumer_secret = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
`, util.ConfigPath)
    }
    return util
}

func (util *Util) Usage() {
    os.Stderr.WriteString(fmt.Sprintf(`Usage: %s [options]
Options:
  -h, --help          Show help.

[ Output Format ]

  Default             Output line by line.
  -i, --ini           Output as INI.
  -y, --yaml          Output as YAML.
  -j, --json          Output as JSON.
  -a, --array         Output as array that compatible with most languages.
  -A, --assoc         Output as PHP-style associative array.
  -e, --env           Output as environmental uppercase variables.

[ OAuth Process ]

  Default             xAuth manipulation with OAuth scraping.
  -x, --xauth         Pure xAuth. Only available with official keys.
  -o, --oauth         Pure OAuth. You have to authorize via web browser.

[ OAuth Credentials ]

  Insufficient components are required to input via STDIN.
  Password is masked.

  --ck  <value>       Specify consumer_key in advance.
  --cs  <value>       Specify consumer_secret in advance.
  --sn  <value>       Specify screen_name or email in advance.
  --pw  <value>       Specify password in advance. (Not masked, DEPRECATED)
  --app <value>       Speficy consumer_key and consumer_secret with app name.

                      app name | full name
                      ------------------------------------
                      android  | Twitter for Andriod
                      win      | Twitter for Windows
                      wp       | Twitter for Windows Phone
                      google   | Twitter for Google TV
                      iphone   | Twitter for iPhone
                      ipad     | Twitter for iPad
                      mac      | Twitter for Mac
                      deck     | TweetDeck

%s
`, os.Args[0], util.CustomAppNames))
}
<|endoftext|>


<commit_before>package cli

import (
	"io/ioutil"
	"os"

	"github.com/cosmos/cosmos-sdk/client/flags"
	"github.com/spf13/cobra"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

const (
	NotFoundOutput = "\"Not Found\""
)

func ReadFromFile(target string) (string, error) {
	if _, err := os.Stat(target); err == nil { // check whether it is a path
		bytes, err := ioutil.ReadFile(target)
		if err != nil {
			return "", err
		}

		return string(bytes), nil
	} else { // else return as is
		return target, nil
	}
}

func IsNotFound(err error) bool {
	if err == nil {
		return false
	}
	s, ok := status.FromError(err)
	if !ok {
		return false
	}
	return s.Code() == codes.NotFound
}

func AddTxFlagsToCmd(cmd *cobra.Command) {

	flags.AddTxFlagsToCmd(cmd)

	// TODO there might be a better way how to filter that
	hiddenFlags := []string{
		flags.FlagFees,
		flags.FlagFeeAccount,
		flags.FlagGasPrices,
		flags.FlagGasAdjustment,
		flags.FlagGas,
		flags.FlagFeeAccount,
	}
	for _, f := range hiddenFlags {
		cmd.Flags().MarkHidden(f)
	}

	// TODO is it possible to update a usage for already added flag ? (the below option will fail)
	// cmd.Flags().Bool(flags.FlagDryRun, false, "perform a simulation of a transaction, but don't broadcast it")
}
<commit_msg>remove dry-run flag from the CLI<commit_after>package cli

import (
	"io/ioutil"
	"os"

	"github.com/cosmos/cosmos-sdk/client/flags"
	"github.com/spf13/cobra"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

const (
	NotFoundOutput = "\"Not Found\""
)

func ReadFromFile(target string) (string, error) {
	if _, err := os.Stat(target); err == nil { // check whether it is a path
		bytes, err := ioutil.ReadFile(target)
		if err != nil {
			return "", err
		}

		return string(bytes), nil
	} else { // else return as is
		return target, nil
	}
}

func IsNotFound(err error) bool {
	if err == nil {
		return false
	}
	s, ok := status.FromError(err)
	if !ok {
		return false
	}
	return s.Code() == codes.NotFound
}

func AddTxFlagsToCmd(cmd *cobra.Command) {

	flags.AddTxFlagsToCmd(cmd)

	// TODO there might be a better way how to filter that
	hiddenFlags := []string{
		flags.FlagFees,
		flags.FlagFeeAccount,
		flags.FlagGasPrices,
		flags.FlagGasAdjustment,
		flags.FlagGas,
		flags.FlagFeeAccount,
		flags.FlagDryRun,
	}
	for _, f := range hiddenFlags {
		cmd.Flags().MarkHidden(f)
	}

}
<|endoftext|>


<commit_before>// Copyright 2017 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/user"
	"path"
	"strings"
	"time"

	"github.com/juju/gnuflag"
	"github.com/juju/loggo"
	"golang.org/x/crypto/ssh"

	"github.com/juju/juju/network"
	jujussh "github.com/juju/juju/network/ssh"
)

func knownHostFilename() string {
	usr, err := user.Current()
	if err != nil {
		panic(fmt.Sprintf("unable to find current user: %v", err))
	}
	return path.Join(usr.HomeDir, ".ssh", "known_hosts")
}

// Juju reports the files in /etc/ssh/ssh_host_key_*_key.pub, so they are all
// in AuthorizedKey format.
func getKnownHostKeys(fname string) []string {
	f, err := os.Open(fname)
	if err != nil {
		panic(fmt.Sprintf("unable to read known-hosts file: %q %v", fname, err))
	}
	defer f.Close()
	content, err := ioutil.ReadAll(f)
	if err != nil {
		panic(fmt.Sprintf("failed while reading known-hosts file: %q %v", fname, err))
	}
	pubKeys := make([]string, 0)
	for len(content) > 0 {
		// marker, hosts, pubkey, comment, rest, err
		_, _, pubkey, _, remaining, err := ssh.ParseKnownHosts(content)
		if err != nil {
			panic(fmt.Sprintf("failed while parsing known hosts: %q %v", fname, err))
		}
		content = remaining
		// We convert the "known_hosts" format into AuthorizedKeys format to
		// match what Juju records.
		pubKeys = append(pubKeys, string(ssh.MarshalAuthorizedKey(pubkey)))
	}
	return pubKeys
}

var logger = loggo.GetLogger("juju.ssh_keyscan")

func main() {
	var verbose bool
	var dialTimeout int = 500
	var waitTimeout int = 5000
	var hostFile string
	gnuflag.BoolVar(&verbose, "v", false, "dump debugging information")
	gnuflag.IntVar(&dialTimeout, "dial-timeout", 500, "time to try a single connection (in milliseconds)")
	gnuflag.IntVar(&waitTimeout, "wait-timeout", 5000, "overall time to wait for answers (in milliseconds)")
	gnuflag.StringVar(&hostFile, "known-hosts", knownHostFilename(), "point to an alternate known-hosts file")
	gnuflag.Parse(true)
	if verbose {
		loggo.ConfigureLoggers(`<root>=DEBUG`)
	}
	args := gnuflag.Args()
	pubKeys := getKnownHostKeys(hostFile)
	hostPorts := make([]network.HostPort, 0, len(args))
	for _, arg := range args {
		if strings.Index(arg, ":") < 0 {
			// Not valid for IPv6, but good enough for testing
			arg = arg + ":22"
		}
		hp, err := network.ParseHostPort(arg)
		if err != nil {
			fmt.Fprintf(os.Stderr, "invalid host:port value: %v\n%v\n", arg, err)
			return
		}
		hostPorts = append(hostPorts, *hp)
	}
	logger.Infof("host ports: %v\n", hostPorts)
	logger.Infof("found %d known hosts\n", len(pubKeys))
	logger.Debugf("known hosts: %v\n", pubKeys)
	dialer := &net.Dialer{Timeout: time.Duration(dialTimeout) * time.Millisecond}
	checker := jujussh.NewReachableChecker(dialer, time.Duration(waitTimeout)* time.Millisecond)
	found, err := checker.FindHost(hostPorts, pubKeys)
	if err != nil {
		fmt.Fprintf(os.Stderr, "could not find valid host: %v\n", err)
		return
	}
	fmt.Printf("%v\n", found)
}
<commit_msg>go fmt<commit_after>// Copyright 2017 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/user"
	"path"
	"strings"
	"time"

	"github.com/juju/gnuflag"
	"github.com/juju/loggo"
	"golang.org/x/crypto/ssh"

	"github.com/juju/juju/network"
	jujussh "github.com/juju/juju/network/ssh"
)

func knownHostFilename() string {
	usr, err := user.Current()
	if err != nil {
		panic(fmt.Sprintf("unable to find current user: %v", err))
	}
	return path.Join(usr.HomeDir, ".ssh", "known_hosts")
}

// Juju reports the files in /etc/ssh/ssh_host_key_*_key.pub, so they are all
// in AuthorizedKey format.
func getKnownHostKeys(fname string) []string {
	f, err := os.Open(fname)
	if err != nil {
		panic(fmt.Sprintf("unable to read known-hosts file: %q %v", fname, err))
	}
	defer f.Close()
	content, err := ioutil.ReadAll(f)
	if err != nil {
		panic(fmt.Sprintf("failed while reading known-hosts file: %q %v", fname, err))
	}
	pubKeys := make([]string, 0)
	for len(content) > 0 {
		// marker, hosts, pubkey, comment, rest, err
		_, _, pubkey, _, remaining, err := ssh.ParseKnownHosts(content)
		if err != nil {
			panic(fmt.Sprintf("failed while parsing known hosts: %q %v", fname, err))
		}
		content = remaining
		// We convert the "known_hosts" format into AuthorizedKeys format to
		// match what Juju records.
		pubKeys = append(pubKeys, string(ssh.MarshalAuthorizedKey(pubkey)))
	}
	return pubKeys
}

var logger = loggo.GetLogger("juju.ssh_keyscan")

func main() {
	var verbose bool
	var dialTimeout int = 500
	var waitTimeout int = 5000
	var hostFile string
	gnuflag.BoolVar(&verbose, "v", false, "dump debugging information")
	gnuflag.IntVar(&dialTimeout, "dial-timeout", 500, "time to try a single connection (in milliseconds)")
	gnuflag.IntVar(&waitTimeout, "wait-timeout", 5000, "overall time to wait for answers (in milliseconds)")
	gnuflag.StringVar(&hostFile, "known-hosts", knownHostFilename(), "point to an alternate known-hosts file")
	gnuflag.Parse(true)
	if verbose {
		loggo.ConfigureLoggers(`<root>=DEBUG`)
	}
	args := gnuflag.Args()
	pubKeys := getKnownHostKeys(hostFile)
	hostPorts := make([]network.HostPort, 0, len(args))
	for _, arg := range args {
		if strings.Index(arg, ":") < 0 {
			// Not valid for IPv6, but good enough for testing
			arg = arg + ":22"
		}
		hp, err := network.ParseHostPort(arg)
		if err != nil {
			fmt.Fprintf(os.Stderr, "invalid host:port value: %v\n%v\n", arg, err)
			return
		}
		hostPorts = append(hostPorts, *hp)
	}
	logger.Infof("host ports: %v\n", hostPorts)
	logger.Infof("found %d known hosts\n", len(pubKeys))
	logger.Debugf("known hosts: %v\n", pubKeys)
	dialer := &net.Dialer{Timeout: time.Duration(dialTimeout) * time.Millisecond}
	checker := jujussh.NewReachableChecker(dialer, time.Duration(waitTimeout)*time.Millisecond)
	found, err := checker.FindHost(hostPorts, pubKeys)
	if err != nil {
		fmt.Fprintf(os.Stderr, "could not find valid host: %v\n", err)
		return
	}
	fmt.Printf("%v\n", found)
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package html implements an HTML5-compliant tokenizer and parser.
INCOMPLETE.

Tokenization is done by creating a Tokenizer for an io.Reader r. It is the
caller's responsibility to ensure that r provides UTF-8 encoded HTML.

	z := html.NewTokenizer(r)

Given a Tokenizer z, the HTML is tokenized by repeatedly calling z.Next(),
which parses the next token and returns its type, or an error:

	for {
		tt := z.Next()
		if tt == html.ErrorToken {
			// ...
			return ...
		}
		// Process the current token.
	}

There are two APIs for retrieving the current token. The high-level API is to
call Token; the low-level API is to call Text or TagName / TagAttr. Both APIs
allow optionally calling Raw after Next but before Token, Text, TagName, or
TagAttr. In EBNF notation, the valid call sequence per token is:

	Next {Raw} [ Token | Text | TagName {TagAttr} ]

Token returns an independent data structure that completely describes a token.
Entities (such as "<") are unescaped, tag names and attribute keys are
lower-cased, and attributes are collected into a []Attribute. For example:

	for {
		if z.Next() == html.ErrorToken {
			// Returning io.EOF indicates success.
			return z.Error()
		}
		emitToken(z.Token())
	}

The low-level API performs fewer allocations and copies, but the contents of
the []byte values returned by Text, TagName and TagAttr may change on the next
call to Next. For example, to extract an HTML page's anchor text:

	depth := 0
	for {
		tt := z.Next()
		switch tt {
		case ErrorToken:
			return z.Error()
		case TextToken:
			if depth > 0 {
				// emitBytes should copy the []byte it receives,
				// if it doesn't process it immediately.
				emitBytes(z.Text())
			}
		case StartTagToken, EndTagToken:
			tn, _ := z.TagName()
			if len(tn) == 1 && tn[0] == 'a' {
				if tt == StartTag {
					depth++
				} else {
					depth--
				}
			}
		}
	}

Parsing is done by calling Parse with an io.Reader, which returns the root of
the parse tree (the document element) as a *Node. It is the caller's
responsibility to ensure that the Reader provides UTF-8 encoded HTML. For
example, to process each anchor node in depth-first order:

	doc, err := html.Parse(r)
	if err != nil {
		// ...
	}
	var f func(*html.Node)
	f = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "a" {
			// Do something with n...
		}
		for _, c := range n.Child {
			f(c)
		}
	}
	f(doc)

The relevant specifications include:
http://www.whatwg.org/specs/web-apps/current-work/multipage/syntax.html and
http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html
*/
package html

// The tokenization algorithm implemented by this package is not a line-by-line
// transliteration of the relatively verbose state-machine in the WHATWG
// specification. A more direct approach is used instead, where the program
// counter implies the state, such as whether it is tokenizing a tag or a text
// node. Specification compliance is verified by checking expected and actual
// outputs over a test suite rather than aiming for algorithmic fidelity.

// TODO(nigeltao): Does a DOM API belong in this package or a separate one?
// TODO(nigeltao): How does parsing interact with a JavaScript engine?
<commit_msg>html: fix typo in package docs.<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package html implements an HTML5-compliant tokenizer and parser.
INCOMPLETE.

Tokenization is done by creating a Tokenizer for an io.Reader r. It is the
caller's responsibility to ensure that r provides UTF-8 encoded HTML.

	z := html.NewTokenizer(r)

Given a Tokenizer z, the HTML is tokenized by repeatedly calling z.Next(),
which parses the next token and returns its type, or an error:

	for {
		tt := z.Next()
		if tt == html.ErrorToken {
			// ...
			return ...
		}
		// Process the current token.
	}

There are two APIs for retrieving the current token. The high-level API is to
call Token; the low-level API is to call Text or TagName / TagAttr. Both APIs
allow optionally calling Raw after Next but before Token, Text, TagName, or
TagAttr. In EBNF notation, the valid call sequence per token is:

	Next {Raw} [ Token | Text | TagName {TagAttr} ]

Token returns an independent data structure that completely describes a token.
Entities (such as "<") are unescaped, tag names and attribute keys are
lower-cased, and attributes are collected into a []Attribute. For example:

	for {
		if z.Next() == html.ErrorToken {
			// Returning io.EOF indicates success.
			return z.Error()
		}
		emitToken(z.Token())
	}

The low-level API performs fewer allocations and copies, but the contents of
the []byte values returned by Text, TagName and TagAttr may change on the next
call to Next. For example, to extract an HTML page's anchor text:

	depth := 0
	for {
		tt := z.Next()
		switch tt {
		case ErrorToken:
			return z.Error()
		case TextToken:
			if depth > 0 {
				// emitBytes should copy the []byte it receives,
				// if it doesn't process it immediately.
				emitBytes(z.Text())
			}
		case StartTagToken, EndTagToken:
			tn, _ := z.TagName()
			if len(tn) == 1 && tn[0] == 'a' {
				if tt == StartTagToken {
					depth++
				} else {
					depth--
				}
			}
		}
	}

Parsing is done by calling Parse with an io.Reader, which returns the root of
the parse tree (the document element) as a *Node. It is the caller's
responsibility to ensure that the Reader provides UTF-8 encoded HTML. For
example, to process each anchor node in depth-first order:

	doc, err := html.Parse(r)
	if err != nil {
		// ...
	}
	var f func(*html.Node)
	f = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "a" {
			// Do something with n...
		}
		for _, c := range n.Child {
			f(c)
		}
	}
	f(doc)

The relevant specifications include:
http://www.whatwg.org/specs/web-apps/current-work/multipage/syntax.html and
http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html
*/
package html

// The tokenization algorithm implemented by this package is not a line-by-line
// transliteration of the relatively verbose state-machine in the WHATWG
// specification. A more direct approach is used instead, where the program
// counter implies the state, such as whether it is tokenizing a tag or a text
// node. Specification compliance is verified by checking expected and actual
// outputs over a test suite rather than aiming for algorithmic fidelity.

// TODO(nigeltao): Does a DOM API belong in this package or a separate one?
// TODO(nigeltao): How does parsing interact with a JavaScript engine?
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2012 Matt Jibson <matt.jibson@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package goon

import (
	"errors"
	"fmt"
	"net/http"
	"reflect"
	"sync"

	"appengine"
	"appengine/datastore"
	"appengine/memcache"
)

var (
	// LogErrors issues appengine.Context.Errorf on any error.
	LogErrors bool = true
)

// Goon holds the app engine context and request memory cache.
type Goon struct {
	context       appengine.Context
	cache         map[string]interface{}
	cacheLock     sync.RWMutex // protect the cache from concurrent goroutines to speed up RPC access
	inTransaction bool
	toSet         map[string]interface{}
	toDelete      map[string]bool
}

func memkey(k *datastore.Key) string {
	return k.Encode()
}

// NewGoon creates a new Goon object from the given request.
func NewGoon(r *http.Request) *Goon {
	return FromContext(appengine.NewContext(r))
}

// FromContext creates a new Goon object from the given appengine Context.
func FromContext(c appengine.Context) *Goon {
	return &Goon{
		context: c,
		cache:   make(map[string]interface{}),
	}
}

func (g *Goon) error(err error) {
	if LogErrors {
		g.context.Errorf("goon: %v", err.Error())
	}
}

func (g *Goon) extractKeys(src interface{}) ([]*datastore.Key, error) {
	v := reflect.Indirect(reflect.ValueOf(src))
	if v.Kind() != reflect.Slice {
		return nil, errors.New("goon: value must be a slice or pointer-to-slice")
	}
	l := v.Len()

	keys := make([]*datastore.Key, l)
	for i := 0; i < l; i++ {
		vi := v.Index(i)
		key, err := g.getStructKey(vi.Interface())
		if err != nil {
			return nil, err
		}
		keys[i] = key
	}
	return keys, nil
}

// Key is the same as KeyError, except nil is returned on error.
func (g *Goon) Key(src interface{}) *datastore.Key {
	if k, err := g.KeyError(src); err == nil {
		return k
	}
	return nil
}

// Key returns the key of src based on its properties.
func (g *Goon) KeyError(src interface{}) (*datastore.Key, error) {
	return g.getStructKey(src)
}

// RunInTransaction runs f in a transaction. It calls f with a transaction
// context tg that f should use for all App Engine operations. Neither cache nor
// memcache are used or set during a transaction.
//
// Otherwise similar to appengine/datastore.RunInTransaction:
// https://developers.google.com/appengine/docs/go/datastore/reference#RunInTransaction
func (g *Goon) RunInTransaction(f func(tg *Goon) error, opts *datastore.TransactionOptions) error {
	var ng *Goon
	err := datastore.RunInTransaction(g.context, func(tc appengine.Context) error {
		ng = &Goon{
			context:       tc,
			inTransaction: true,
			toSet:         make(map[string]interface{}),
			toDelete:      make(map[string]bool),
		}
		return f(ng)
	}, opts)

	if err == nil {
		g.cacheLock.Lock()
		defer g.cacheLock.Unlock()
		for k, v := range ng.toSet {
			g.cache[k] = v
		}

		for k, _ := range ng.toDelete {
			delete(g.cache, k)
		}
	} else {
		g.error(err)
	}

	return err
}

// Put saves the entity src into the datastore based on src's key k. If k
// is an incomplete key, the returned key will be a unique key generated by
// the datastore.
func (g *Goon) Put(src interface{}) (*datastore.Key, error) {
	ks, err := g.PutMulti([]interface{}{src})
	if len(ks) == 1 {
		return ks[0], err
	}
	return nil, err
}

// PutMany is a wrapper around PutMulti.
func (g *Goon) PutMany(srcs ...interface{}) ([]*datastore.Key, error) {
	return g.PutMulti(srcs)
}

const putMultiLimit = 500

// PutMulti is a batch version of Put.
//
// src must satisfy the same conditions as the dst argument to GetMulti.
func (g *Goon) PutMulti(src interface{}) ([]*datastore.Key, error) {
	keys, err := g.extractKeys(src)
	if err != nil {
		return nil, err
	}

	var memkeys []string
	for _, key := range keys {
		if !key.Incomplete() {
			memkeys = append(memkeys, memkey(key))
		}
	}

	// Memcache needs to be updated after the datastore to prevent a common race condition
	defer memcache.DeleteMulti(g.context, memkeys)

	v := reflect.Indirect(reflect.ValueOf(src))
	for i := 0; i <= len(keys)/putMultiLimit; i++ {
		lo := i * putMultiLimit
		hi := (i + 1) * putMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}
		rkeys, err := datastore.PutMulti(g.context, keys[lo:hi], v.Slice(lo, hi).Interface())
		if err != nil {
			g.error(err)
			return nil, err
		}

		for i, key := range keys[lo:hi] {
			vi := v.Index(lo + i).Interface()
			if key.Incomplete() {
				setStructKey(vi, rkeys[i])
				keys[i] = rkeys[i]
			}
			if g.inTransaction {
				mk := memkey(rkeys[i])
				delete(g.toDelete, mk)
				g.toSet[mk] = vi
			}
		}
	}

	if !g.inTransaction {
		g.putMemoryMulti(src)
	}

	return keys, nil
}

// PutComplete is like Put, but errors if a key is incomplete.
func (g *Goon) PutComplete(src interface{}) (*datastore.Key, error) {
	k, err := g.getStructKey(src)
	if err != nil {
		return nil, err
	}
	if k.Incomplete() {
		err := fmt.Errorf("goon: incomplete key: %v", k)
		g.error(err)
		return nil, err
	}
	return g.Put(src)
}

// PutMultiComplete is like PutMulti, but errors if a key is incomplete.
func (g *Goon) PutMultiComplete(src interface{}) ([]*datastore.Key, error) {
	keys, err := g.extractKeys(src)
	if err != nil {
		return nil, err
	}
	for i, k := range keys {
		if k.Incomplete() {
			err := fmt.Errorf("goon: incomplete key (%dth index): %v", i, k)
			g.error(err)
			return nil, err
		}
	}
	return g.PutMulti(src)
}

func (g *Goon) putMemoryMulti(src interface{}) {
	v := reflect.Indirect(reflect.ValueOf(src))
	for i := 0; i < v.Len(); i++ {
		g.putMemory(v.Index(i).Interface())
	}
}

func (g *Goon) putMemory(src interface{}) {
	key, _ := g.getStructKey(src)
	g.cacheLock.Lock()
	defer g.cacheLock.Unlock()
	g.cache[memkey(key)] = src
}

func (g *Goon) putMemcache(srcs []interface{}) error {
	items := make([]*memcache.Item, len(srcs))

	for i, src := range srcs {
		gob, err := toGob(src)
		if err != nil {
			g.error(err)
			return err
		}
		key, err := g.getStructKey(src)

		items[i] = &memcache.Item{
			Key:   memkey(key),
			Value: gob,
		}
	}

	memcache.SetMulti(g.context, items)
	g.putMemoryMulti(srcs)
	return nil
}

// Get loads the entity based on dst's key into dst
// If there is no such entity for the key, Get returns
// datastore.ErrNoSuchEntity.
func (g *Goon) Get(dst interface{}) error {
	set := reflect.ValueOf(dst)
	if set.Kind() != reflect.Ptr {
		return errors.New(fmt.Sprintf("goon: expected pointer to a struct, got %#v", dst))
	}
	dsts := []interface{}{dst}
	if err := g.GetMulti(dsts); err != nil {
		// Look for an embedded error if it's multi
		if me, ok := err.(appengine.MultiError); ok {
			for i, merr := range me {
				if i == 0 {
					return merr
				}
			}
		}
		// Not multi, normal error
		return err
	}
	set.Elem().Set(reflect.ValueOf(dsts[0]).Elem())
	return nil
}

const getMultiLimit = 1000

// GetMulti is a batch version of Get.
//
// dst has similar constraints as datastore.GetMulti.
func (g *Goon) GetMulti(dst interface{}) error {
	keys, err := g.extractKeys(dst)
	if err != nil {
		return err
	}

	if g.inTransaction {
		// todo: support getMultiLimit in transactions
		return datastore.GetMulti(g.context, keys, dst)
	}

	var dskeys []*datastore.Key
	var dsdst []interface{}
	var dixs []int

	var memkeys []string
	var mixs []int

	v := reflect.Indirect(reflect.ValueOf(dst))
	g.cacheLock.RLock()
	for i, key := range keys {
		m := memkey(key)
		if s, present := g.cache[m]; present {
			vi := v.Index(i)
			vi.Set(reflect.ValueOf(s))
		} else {
			memkeys = append(memkeys, m)
			mixs = append(mixs, i)
		}
	}
	g.cacheLock.RUnlock()

	if len(memkeys) == 0 {
		return nil
	}

	memvalues, _ := memcache.GetMulti(g.context, memkeys)
	for i, m := range memkeys {
		d := v.Index(mixs[i]).Interface()
		if s, present := memvalues[m]; present {
			err := fromGob(d, s.Value)
			if err != nil {
				g.error(err)
				return err
			}

			g.putMemory(d)
		} else {
			dskeys = append(dskeys, keys[mixs[i]])
			dsdst = append(dsdst, d)
			dixs = append(dixs, mixs[i])
		}
	}
	if len(dskeys) == 0 {
		return nil
	}

	multiErr := make(appengine.MultiError, len(keys))
	var toCache []interface{}
	var ret error
	for i := 0; i <= len(dskeys)/getMultiLimit; i++ {
		lo := i * getMultiLimit
		hi := (i + 1) * getMultiLimit
		if hi > len(dskeys) {
			hi = len(dskeys)
		}
		gmerr := datastore.GetMulti(g.context, dskeys[lo:hi], dsdst[lo:hi])
		if gmerr != nil {
			merr, ok := gmerr.(appengine.MultiError)
			if !ok {
				g.error(gmerr)
				return gmerr
			}
			for i, idx := range dixs[lo:hi] {
				multiErr[idx] = merr[i]
				if merr[i] == nil {
					toCache = append(toCache, dsdst[lo+i])
				}
			}
			ret = multiErr
		} else {
			toCache = append(toCache, dsdst[lo:hi]...)
		}
	}

	if len(toCache) > 0 {
		if err := g.putMemcache(toCache); err != nil {
			g.error(err)
			return err
		}
	}

	return ret
}

// Delete deletes the entity for the given key.
func (g *Goon) Delete(key *datastore.Key) error {
	keys := []*datastore.Key{key}
	return g.DeleteMulti(keys)
}

const deleteMultiLimit = 500

// DeleteMulti is a batch version of Delete.
func (g *Goon) DeleteMulti(keys []*datastore.Key) error {
	memkeys := make([]string, len(keys))

	g.cacheLock.Lock()
	for i, k := range keys {
		mk := memkey(k)
		memkeys[i] = mk

		if g.inTransaction {
			delete(g.toSet, mk)
			g.toDelete[mk] = true
		} else {
			delete(g.cache, mk)
		}
	}
	g.cacheLock.Unlock()

	// Memcache needs to be updated after the datastore to prevent a common race condition
	defer memcache.DeleteMulti(g.context, memkeys)

	for i := 0; i <= len(keys)/deleteMultiLimit; i++ {
		lo := i * deleteMultiLimit
		hi := (i + 1) * deleteMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}
		if err := datastore.DeleteMulti(g.context, keys[lo:hi]); err != nil {
			return err
		}
	}
	return nil
}

// NotFound returns true if err is an appengine.MultiError and err[idx] is a datastore.ErrNoSuchEntity.
func NotFound(err error, idx int) bool {
	if merr, ok := err.(appengine.MultiError); ok {
		return idx < len(merr) && merr[idx] == datastore.ErrNoSuchEntity
	}
	return false
}
<commit_msg>gofmt -s<commit_after>/*
 * Copyright (c) 2012 Matt Jibson <matt.jibson@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package goon

import (
	"errors"
	"fmt"
	"net/http"
	"reflect"
	"sync"

	"appengine"
	"appengine/datastore"
	"appengine/memcache"
)

var (
	// LogErrors issues appengine.Context.Errorf on any error.
	LogErrors bool = true
)

// Goon holds the app engine context and request memory cache.
type Goon struct {
	context       appengine.Context
	cache         map[string]interface{}
	cacheLock     sync.RWMutex // protect the cache from concurrent goroutines to speed up RPC access
	inTransaction bool
	toSet         map[string]interface{}
	toDelete      map[string]bool
}

func memkey(k *datastore.Key) string {
	return k.Encode()
}

// NewGoon creates a new Goon object from the given request.
func NewGoon(r *http.Request) *Goon {
	return FromContext(appengine.NewContext(r))
}

// FromContext creates a new Goon object from the given appengine Context.
func FromContext(c appengine.Context) *Goon {
	return &Goon{
		context: c,
		cache:   make(map[string]interface{}),
	}
}

func (g *Goon) error(err error) {
	if LogErrors {
		g.context.Errorf("goon: %v", err.Error())
	}
}

func (g *Goon) extractKeys(src interface{}) ([]*datastore.Key, error) {
	v := reflect.Indirect(reflect.ValueOf(src))
	if v.Kind() != reflect.Slice {
		return nil, errors.New("goon: value must be a slice or pointer-to-slice")
	}
	l := v.Len()

	keys := make([]*datastore.Key, l)
	for i := 0; i < l; i++ {
		vi := v.Index(i)
		key, err := g.getStructKey(vi.Interface())
		if err != nil {
			return nil, err
		}
		keys[i] = key
	}
	return keys, nil
}

// Key is the same as KeyError, except nil is returned on error.
func (g *Goon) Key(src interface{}) *datastore.Key {
	if k, err := g.KeyError(src); err == nil {
		return k
	}
	return nil
}

// Key returns the key of src based on its properties.
func (g *Goon) KeyError(src interface{}) (*datastore.Key, error) {
	return g.getStructKey(src)
}

// RunInTransaction runs f in a transaction. It calls f with a transaction
// context tg that f should use for all App Engine operations. Neither cache nor
// memcache are used or set during a transaction.
//
// Otherwise similar to appengine/datastore.RunInTransaction:
// https://developers.google.com/appengine/docs/go/datastore/reference#RunInTransaction
func (g *Goon) RunInTransaction(f func(tg *Goon) error, opts *datastore.TransactionOptions) error {
	var ng *Goon
	err := datastore.RunInTransaction(g.context, func(tc appengine.Context) error {
		ng = &Goon{
			context:       tc,
			inTransaction: true,
			toSet:         make(map[string]interface{}),
			toDelete:      make(map[string]bool),
		}
		return f(ng)
	}, opts)

	if err == nil {
		g.cacheLock.Lock()
		defer g.cacheLock.Unlock()
		for k, v := range ng.toSet {
			g.cache[k] = v
		}

		for k := range ng.toDelete {
			delete(g.cache, k)
		}
	} else {
		g.error(err)
	}

	return err
}

// Put saves the entity src into the datastore based on src's key k. If k
// is an incomplete key, the returned key will be a unique key generated by
// the datastore.
func (g *Goon) Put(src interface{}) (*datastore.Key, error) {
	ks, err := g.PutMulti([]interface{}{src})
	if len(ks) == 1 {
		return ks[0], err
	}
	return nil, err
}

// PutMany is a wrapper around PutMulti.
func (g *Goon) PutMany(srcs ...interface{}) ([]*datastore.Key, error) {
	return g.PutMulti(srcs)
}

const putMultiLimit = 500

// PutMulti is a batch version of Put.
//
// src must satisfy the same conditions as the dst argument to GetMulti.
func (g *Goon) PutMulti(src interface{}) ([]*datastore.Key, error) {
	keys, err := g.extractKeys(src)
	if err != nil {
		return nil, err
	}

	var memkeys []string
	for _, key := range keys {
		if !key.Incomplete() {
			memkeys = append(memkeys, memkey(key))
		}
	}

	// Memcache needs to be updated after the datastore to prevent a common race condition
	defer memcache.DeleteMulti(g.context, memkeys)

	v := reflect.Indirect(reflect.ValueOf(src))
	for i := 0; i <= len(keys)/putMultiLimit; i++ {
		lo := i * putMultiLimit
		hi := (i + 1) * putMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}
		rkeys, err := datastore.PutMulti(g.context, keys[lo:hi], v.Slice(lo, hi).Interface())
		if err != nil {
			g.error(err)
			return nil, err
		}

		for i, key := range keys[lo:hi] {
			vi := v.Index(lo + i).Interface()
			if key.Incomplete() {
				setStructKey(vi, rkeys[i])
				keys[i] = rkeys[i]
			}
			if g.inTransaction {
				mk := memkey(rkeys[i])
				delete(g.toDelete, mk)
				g.toSet[mk] = vi
			}
		}
	}

	if !g.inTransaction {
		g.putMemoryMulti(src)
	}

	return keys, nil
}

// PutComplete is like Put, but errors if a key is incomplete.
func (g *Goon) PutComplete(src interface{}) (*datastore.Key, error) {
	k, err := g.getStructKey(src)
	if err != nil {
		return nil, err
	}
	if k.Incomplete() {
		err := fmt.Errorf("goon: incomplete key: %v", k)
		g.error(err)
		return nil, err
	}
	return g.Put(src)
}

// PutMultiComplete is like PutMulti, but errors if a key is incomplete.
func (g *Goon) PutMultiComplete(src interface{}) ([]*datastore.Key, error) {
	keys, err := g.extractKeys(src)
	if err != nil {
		return nil, err
	}
	for i, k := range keys {
		if k.Incomplete() {
			err := fmt.Errorf("goon: incomplete key (%dth index): %v", i, k)
			g.error(err)
			return nil, err
		}
	}
	return g.PutMulti(src)
}

func (g *Goon) putMemoryMulti(src interface{}) {
	v := reflect.Indirect(reflect.ValueOf(src))
	for i := 0; i < v.Len(); i++ {
		g.putMemory(v.Index(i).Interface())
	}
}

func (g *Goon) putMemory(src interface{}) {
	key, _ := g.getStructKey(src)
	g.cacheLock.Lock()
	defer g.cacheLock.Unlock()
	g.cache[memkey(key)] = src
}

func (g *Goon) putMemcache(srcs []interface{}) error {
	items := make([]*memcache.Item, len(srcs))

	for i, src := range srcs {
		gob, err := toGob(src)
		if err != nil {
			g.error(err)
			return err
		}
		key, err := g.getStructKey(src)

		items[i] = &memcache.Item{
			Key:   memkey(key),
			Value: gob,
		}
	}

	memcache.SetMulti(g.context, items)
	g.putMemoryMulti(srcs)
	return nil
}

// Get loads the entity based on dst's key into dst
// If there is no such entity for the key, Get returns
// datastore.ErrNoSuchEntity.
func (g *Goon) Get(dst interface{}) error {
	set := reflect.ValueOf(dst)
	if set.Kind() != reflect.Ptr {
		return errors.New(fmt.Sprintf("goon: expected pointer to a struct, got %#v", dst))
	}
	dsts := []interface{}{dst}
	if err := g.GetMulti(dsts); err != nil {
		// Look for an embedded error if it's multi
		if me, ok := err.(appengine.MultiError); ok {
			for i, merr := range me {
				if i == 0 {
					return merr
				}
			}
		}
		// Not multi, normal error
		return err
	}
	set.Elem().Set(reflect.ValueOf(dsts[0]).Elem())
	return nil
}

const getMultiLimit = 1000

// GetMulti is a batch version of Get.
//
// dst has similar constraints as datastore.GetMulti.
func (g *Goon) GetMulti(dst interface{}) error {
	keys, err := g.extractKeys(dst)
	if err != nil {
		return err
	}

	if g.inTransaction {
		// todo: support getMultiLimit in transactions
		return datastore.GetMulti(g.context, keys, dst)
	}

	var dskeys []*datastore.Key
	var dsdst []interface{}
	var dixs []int

	var memkeys []string
	var mixs []int

	v := reflect.Indirect(reflect.ValueOf(dst))
	g.cacheLock.RLock()
	for i, key := range keys {
		m := memkey(key)
		if s, present := g.cache[m]; present {
			vi := v.Index(i)
			vi.Set(reflect.ValueOf(s))
		} else {
			memkeys = append(memkeys, m)
			mixs = append(mixs, i)
		}
	}
	g.cacheLock.RUnlock()

	if len(memkeys) == 0 {
		return nil
	}

	memvalues, _ := memcache.GetMulti(g.context, memkeys)
	for i, m := range memkeys {
		d := v.Index(mixs[i]).Interface()
		if s, present := memvalues[m]; present {
			err := fromGob(d, s.Value)
			if err != nil {
				g.error(err)
				return err
			}

			g.putMemory(d)
		} else {
			dskeys = append(dskeys, keys[mixs[i]])
			dsdst = append(dsdst, d)
			dixs = append(dixs, mixs[i])
		}
	}
	if len(dskeys) == 0 {
		return nil
	}

	multiErr := make(appengine.MultiError, len(keys))
	var toCache []interface{}
	var ret error
	for i := 0; i <= len(dskeys)/getMultiLimit; i++ {
		lo := i * getMultiLimit
		hi := (i + 1) * getMultiLimit
		if hi > len(dskeys) {
			hi = len(dskeys)
		}
		gmerr := datastore.GetMulti(g.context, dskeys[lo:hi], dsdst[lo:hi])
		if gmerr != nil {
			merr, ok := gmerr.(appengine.MultiError)
			if !ok {
				g.error(gmerr)
				return gmerr
			}
			for i, idx := range dixs[lo:hi] {
				multiErr[idx] = merr[i]
				if merr[i] == nil {
					toCache = append(toCache, dsdst[lo+i])
				}
			}
			ret = multiErr
		} else {
			toCache = append(toCache, dsdst[lo:hi]...)
		}
	}

	if len(toCache) > 0 {
		if err := g.putMemcache(toCache); err != nil {
			g.error(err)
			return err
		}
	}

	return ret
}

// Delete deletes the entity for the given key.
func (g *Goon) Delete(key *datastore.Key) error {
	keys := []*datastore.Key{key}
	return g.DeleteMulti(keys)
}

const deleteMultiLimit = 500

// DeleteMulti is a batch version of Delete.
func (g *Goon) DeleteMulti(keys []*datastore.Key) error {
	memkeys := make([]string, len(keys))

	g.cacheLock.Lock()
	for i, k := range keys {
		mk := memkey(k)
		memkeys[i] = mk

		if g.inTransaction {
			delete(g.toSet, mk)
			g.toDelete[mk] = true
		} else {
			delete(g.cache, mk)
		}
	}
	g.cacheLock.Unlock()

	// Memcache needs to be updated after the datastore to prevent a common race condition
	defer memcache.DeleteMulti(g.context, memkeys)

	for i := 0; i <= len(keys)/deleteMultiLimit; i++ {
		lo := i * deleteMultiLimit
		hi := (i + 1) * deleteMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}
		if err := datastore.DeleteMulti(g.context, keys[lo:hi]); err != nil {
			return err
		}
	}
	return nil
}

// NotFound returns true if err is an appengine.MultiError and err[idx] is a datastore.ErrNoSuchEntity.
func NotFound(err error, idx int) bool {
	if merr, ok := err.(appengine.MultiError); ok {
		return idx < len(merr) && merr[idx] == datastore.ErrNoSuchEntity
	}
	return false
}
<|endoftext|>


<commit_before>package ascii

import "fmt"

const Ascii = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" +
	"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" +
	` !"#$%&'()*+,-./0123456789:;<=>?` +
	`@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_` +
	"`abcdefghijklmnopqrstuvwxyz{|}~\x7f"

// Funksjon tar en streng med kun ASCII tegn og lager en utskrift på
// følgende format:
// [ascii-kode heksadesimalt med store bokstaver A-F][mellomrom]
// [symbol for ascii-kode][mellomrom][ascii-kode binært][linjeskift]
// Eksempel (bruk denne funksjonen i en main.go fil):
//	…
// 3E > 111110
// 3F ? 111111
// 40 @ 1000000
// ...
func IterateOverASCIIStringLiteral() {
	for i := 0; i < len(Ascii); i++ {
		fmt.Printf("%x ", Ascii[i])
		fmt.Printf("%c ", Ascii[i])
		fmt.Printf("%b\n", Ascii[i])
	}
}

// IterateOverASCIIStringLiteral

// Funksjonen skal generere en utskrift fra en sekvens av bytes,
// dvs. av typen []bytes (det betyr at du må finne den heksadesimale
// eller binære representasjonen av alle tegn i strengen,
// som skal skrives ut (inkludert anførselstegn eller
// “double quotes” på engelsk).
// Funksjonen greetingASCII() returnerer en variabel av typen string,
// som inneholder kun ASCII tegn (ikke utvidet ASCII).
// Gjelder oppgave 1b
func greetingASCII() {
	fmt.Printf("Hello \x3A\x2D\x29")
}
<commit_msg>Hello til Hexa<commit_after>package ascii

import "fmt"

const Ascii = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" +
	"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" +
	` !"#$%&'()*+,-./0123456789:;<=>?` +
	`@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_` +
	"`abcdefghijklmnopqrstuvwxyz{|}~\x7f"

// Funksjon tar en streng med kun ASCII tegn og lager en utskrift på
// følgende format:
// [ascii-kode heksadesimalt med store bokstaver A-F][mellomrom]
// [symbol for ascii-kode][mellomrom][ascii-kode binært][linjeskift]
// Eksempel (bruk denne funksjonen i en main.go fil):
//	…
// 3E > 111110
// 3F ? 111111
// 40 @ 1000000
// ...
func IterateOverASCIIStringLiteral() {
	for i := 0; i < len(Ascii); i++ {
		fmt.Printf("%x ", Ascii[i])
		fmt.Printf("%c ", Ascii[i])
		fmt.Printf("%b\n", Ascii[i])
	}
}

// IterateOverASCIIStringLiteral

// Funksjonen skal generere en utskrift fra en sekvens av bytes,
// dvs. av typen []bytes (det betyr at du må finne den heksadesimale
// eller binære representasjonen av alle tegn i strengen,
// som skal skrives ut (inkludert anførselstegn eller
// “double quotes” på engelsk).
// Funksjonen greetingASCII() returnerer en variabel av typen string,
// som inneholder kun ASCII tegn (ikke utvidet ASCII).
// Gjelder oppgave 1b
func greetingASCII() {
	fmt.Printf("\x48\x65\x6C\x6C\x6F \x3A\x2D\x29")
}
<|endoftext|>


<commit_before>/*

Package GoXp is a powerful package for quickly writing modular web application/services in Golang
independantly but fully inspired from the martini package.

Full guide http://github.com/4ur3l13n/goxp

The idea is also at the end to write a version that use booster from 4ur3l13n and not injector
from github.com/codegangsta/inject

Classic -> Sub

*/

package goxp

import (
	"log"
	"net/http"
	"os"
	"reflect"

	"github.com/codegangsta/inject"
)

// GoXp represents the top level web application, inject.Injector methods can be invoked to map services on global level.
type Goxp struct {
	inject.Injector
	handlers []Handler
	action   Handler
	logger   *log.Logger
}

// New creates a bare bones Goxp instance. Use this method if you want to have full control over the middleware that is used.
func New() *Goxp {
	m := &Goxp{Injector: inject.New(), action: func() {}, logger: log.New(os.Stdout, "[goxp] ", 0)}
	m.Map(m.logger)
	m.Map(defaultReturnHandler())
	return m
}

// Handlers sets the entire middleware stack with the given Handlers. This will clear any current middleware handlers.
// Will panic if any of the handlers is not callable function
func (g *Goxp) Handler(handlers ...Handler) {
	m.handlers = make([]Handler, 0)
	for _, Handler := range Handler {
		m.Use(Handler)
	}
}

// Action sets the handler that will be called after all the middleware has been invoked. This is set to goxp.Router in a goxp.Classic().
func (g *Goxp) Use(handler Handler) {
	validateHandler(handler)
	m.action = handler
}

// Use adds a middleware Handler to the stack. Will panic if the handler is not callable func. Middleware Handler are invoked in the order that they are added.
func (g *Goxp) Use(handler Handler) {
	validateHandler(Handler)

	m.handler = append(m.handlers, handler)
}

// ServerHTTP is the HTTP Entry point for a Goxp instance. Useful if you want to control your own HTTP server.
func (g *Goxp) ServeHTTP(res http.ResponseWriter, req *http.Request) {
	m.createContext(res, req).run()
}

// Run the http server on a given host and port.
func (g *Goxp) RunOnAddr(addr string) {
	// TODO: Should probably be implemented using a new instance of http Server in place of
	// calling http.ListenAndServer directly, so that it could be stored in a goxp struct for later user

	// This would also allow to improve testing when a custom host and port are passed.

	logger := m.Injector.Get(reflect.TypeOf(m.logger)).Interface().(*log.Logger)
	logger.Printf("listening on %s (%s)\n", addr, Env)
	logger.Fatalln(http.ListenAndServe(addr, m))
}

// Run the http server. Listening on os.GetEnv("PORT") or 3000 by default.
func (g *Goxp) Run() {
	port := os.Getenv("PORT")
	if len(port) == 0 {
		port = "3000"
	}

	host := os.Getenv("HOST")

	m.RunOnAddr(host + ":" + port)
}

func (g *Goxp) createContext(res http.ResponseWriter, req *http.Request) *context {
	c := &context{inject.New(), m.handlers, m.action, NewResponseWriter(res), 0}
	c.SetParent(m)
	c.MapTo(c, (*Context)(nil))
	c.MapTo(c.rw, (*http.ResponseWriter)(nil))
	c.Map(req)
	return c
}

// ClassicGoxp represents a Goxp with some reasonable defaults. Embeds the router functions for convenience.
type ClassicGoxp struct {
	*Goxp
	Router
}

// Sub creates a sub Goxp with some basic default middleware - goxp.Logger, goxp.Recovery and goxp.Static
// Sub also maps goxp.Routes as a service.
func Sub() *SubGoxp {
	r := NewRouter()
	m := New()
	m.Use(Logger())
	m.Use(Static("public"))
	m.MapTo(r, (*Routes)(nil))
	m.Action(r.Handle)
	return &SubGoxp{m, r}
}

<commit_msg>+callable function +booster<commit_after>/*

Package GoXp is a powerful package for quickly writing modular web application/services in Golang
independantly but fully inspired from the martini package.

Full guide http://github.com/4ur3l13n/goxp

The idea is also at the end to write a version that use booster from 4ur3l13n and not injector
from github.com/codegangsta/inject

Classic -> Sub

*/

package goxp

import (
	"log"
	"net/http"
	"os"
	"reflect"

	"github.com/codegangsta/inject"
)

// GoXp represents the top level web application, inject.Injector methods can be invoked to map services on global level.
type Goxp struct {
	inject.Injector
	handlers []Handler
	action   Handler
	logger   *log.Logger
}

// New creates a bare bones Goxp instance. Use this method if you want to have full control over the middleware that is used.
func New() *Goxp {
	m := &Goxp{Injector: inject.New(), action: func() {}, logger: log.New(os.Stdout, "[goxp] ", 0)}
	m.Map(m.logger)
	m.Map(defaultReturnHandler())
	return m
}

// Handlers sets the entire middleware stack with the given Handlers. This will clear any current middleware handlers.
// Will panic if any of the handlers is not callable function
func (g *Goxp) Handler(handlers ...Handler) {
	m.handlers = make([]Handler, 0)
	for _, Handler := range Handler {
		m.Use(Handler)
	}
}

// Action sets the handler that will be called after all the middleware has been invoked. This is set to goxp.Router in a goxp.Classic().
func (g *Goxp) Use(handler Handler) {
	validateHandler(handler)
	m.action = handler
}

// Use adds a middleware Handler to the stack. Will panic if the handler is not callable func. Middleware Handler are invoked in the order that they are added.
func (g *Goxp) Use(handler Handler) {
	validateHandler(Handler)

	m.handler = append(m.handlers, handler)
}

// ServerHTTP is the HTTP Entry point for a Goxp instance. Useful if you want to control your own HTTP server.
func (g *Goxp) ServeHTTP(res http.ResponseWriter, req *http.Request) {
	m.createContext(res, req).run()
}

// Run the http server on a given host and port.
func (g *Goxp) RunOnAddr(addr string) {
	// TODO: Should probably be implemented using a new instance of http Server in place of
	// calling http.ListenAndServer directly, so that it could be stored in a goxp struct for later user

	// This would also allow to improve testing when a custom host and port are passed.

	logger := m.Injector.Get(reflect.TypeOf(m.logger)).Interface().(*log.Logger)
	logger.Printf("listening on %s (%s)\n", addr, Env)
	logger.Fatalln(http.ListenAndServe(addr, m))
}

// Run the http server. Listening on os.GetEnv("PORT") or 3000 by default.
func (g *Goxp) Run() {
	port := os.Getenv("PORT")
	if len(port) == 0 {
		port = "3000"
	}

	host := os.Getenv("HOST")

	m.RunOnAddr(host + ":" + port)
}

func (g *Goxp) createContext(res http.ResponseWriter, req *http.Request) *context {
	c := &context{inject.New(), m.handlers, m.action, NewResponseWriter(res), 0}
	c.SetParent(m)
	c.MapTo(c, (*Context)(nil))
	c.MapTo(c.rw, (*http.ResponseWriter)(nil))
	c.Map(req)
	return c
}

// ClassicGoxp represents a Goxp with some reasonable defaults. Embeds the router functions for convenience.
type ClassicGoxp struct {
	*Goxp
	Router
}

// Sub creates a sub Goxp with some basic default middleware - goxp.Logger, goxp.Recovery and goxp.Static
// Sub also maps goxp.Routes as a service.
func Sub() *SubGoxp {
	r := NewRouter()
	m := New()
	m.Use(Logger())
	m.Use(Static("public"))
	m.MapTo(r, (*Routes)(nil))
	m.Action(r.Handle)
	return &SubGoxp{m, r}
}

// Handler can be any callable function. Goxp attemps to boost services into the handler's argument list.
// Goxp will panic if an argument could not be fullfilled via dependency booster.
// Here we will use inject.Injector but we hope to replace it soon by boost.Booster which sounds better :-)
type Handler interface{}

func validateHandler(handler Handler) {
	if reflect.TypeOf(handler).Kind() != reflect.Func {
		panic("goxp handler must be a callable func")
	}
}

// Contect represents a request context. Service can be mapped on the request level from this interface.
type Context interface {
	inject.Injector
	// Next is an optional function that Middleware Handlers can call to yield the until after
	// the other Handlers have been executed. This works really well for any operations that must
	// happen after an http request
	Next()
	// Written returns whether or not the response for this context has been written.
	Written() bool
}

type Context struct {
	inject.Injector
	handler []Handler
	action  Handler
	rw      ResponseWriter
	index   int
}

<|endoftext|>


<commit_before>// +build linux

package resource

import (
	"errors"
	"fmt"

	"github.com/coreos/go-systemd/dbus"
	"github.com/coreos/go-systemd/util"
)

// ErrNoSystemd error is returned when the system is detected to
// have no support for systemd.
var ErrNoSystemd = errors.New("No systemd support found")

// Service type is a resource which manages services on a
// GNU/Linux system running with systemd.
//
// Example:
//   svc = resource.service.new("nginx")
//   svc.state = "running"
//   svc.enable = true
type Service struct {
	Base

	// If true then enable the service during boot-time
	Enable bool `luar:"enable"`

	// Systemd unit name
	unit string `luar:"-"`

	conn *dbus.Conn `luar:"-"`
}

// NewService creates a new resource for managing services
// using systemd on a GNU/Linux system
func NewService(name string) (Resource, error) {
	if !util.IsRunningSystemd() {
		return nil, ErrNoSystemd
	}

	s := &Service{
		Base: Base{
			Name:          name,
			Type:          "service",
			State:         "running",
			Require:       make([]string, 0),
			PresentStates: []string{"present", "running"},
			AbsentStates:  []string{"absent", "stopped"},
			Concurrent:    true,
			Subscribe:     make(TriggerMap),
		},
		Enable: true,
		unit:   fmt.Sprintf("%s.service", name),
	}

	return s, nil
}

// unitIsEnabled checks if the unit is enabled or disabled
func (s *Service) unitIsEnabled() (bool, error) {
	unitState, err := s.conn.GetUnitProperty(s.unit, "UnitFileState")
	if err != nil {
		return false, err
	}

	value := unitState.Value.Value().(string)
	switch value {
	case "enabled", "static", "enabled-runtime", "linked", "linked-runtime":
		return true, nil
	case "disabled", "masked", "masked-runtime":
		return false, nil
	case "invalid":
		fallthrough
	default:
		return false, errors.New("Invalid unit state")
	}
}

// enableUnit enables the service unit during boot-time
func (s *Service) enableUnit() error {
	Log(s, "enabling service\n")

	units := []string{s.unit}
	_, changes, err := s.conn.EnableUnitFiles(units, false, false)
	if err != nil {
		return err
	}

	for _, change := range changes {
		Log(s, "%s %s -> %s\n", change.Type, change.Filename, change.Destination)
	}

	return nil
}

// disableUnit disables the service unit during boot-time
func (s *Service) disableUnit() error {
	Log(s, "disabling service\n")

	units := []string{s.unit}
	changes, err := s.conn.DisableUnitFiles(units, false)
	if err != nil {
		return err
	}

	for _, change := range changes {
		Log(s, "%s %s\n", change.Type, change.Filename)
	}

	return nil
}

// setUnitState enables or disables the unit
func (s *Service) setUnitState() error {
	var action func() error

	switch s.Enable {
	case true:
		action = s.enableUnit
	case false:
		action = s.disableUnit
	}

	if err := action(); err != nil {
		return err
	}

	return s.conn.Reload()
}

// Initialize initializes the service resource by establishing a
// connection the systemd D-BUS API
func (s *Service) Initialize() error {
	conn, err := dbus.New()
	s.conn = conn

	return err
}

// Evaluate evaluates the state of the resource
func (s *Service) Evaluate() (State, error) {
	state := State{
		Current:  "unknown",
		Want:     s.State,
		Outdated: false,
	}

	// Check if the unit is started/stopped
	activeState, err := s.conn.GetUnitProperty(s.unit, "ActiveState")
	if err != nil {
		return state, err
	}

	// TODO: Handle cases where the unit is not found

	value := activeState.Value.Value().(string)
	switch value {
	case "active", "reloading", "activating":
		state.Current = "running"
	case "inactive", "failed", "deactivating":
		state.Current = "stopped"
	}

	enabled, err := s.unitIsEnabled()
	if err != nil {
		return state, err
	}

	if s.Enable != enabled {
		state.Outdated = true
	}

	return state, nil
}

// Create starts the service unit
func (s *Service) Create() error {
	Log(s, "starting service\n")

	ch := make(chan string)
	jobID, err := s.conn.StartUnit(s.unit, "replace", ch)
	if err != nil {
		return err
	}

	result := <-ch
	Log(s, "systemd job id %d result: %s\n", jobID, result)

	return s.setUnitState()
}

// Delete stops the service unit
func (s *Service) Delete() error {
	Log(s, "stopping service\n")

	ch := make(chan string)
	jobID, err := s.conn.StopUnit(s.unit, "replace", ch)
	if err != nil {
		return err
	}

	result := <-ch
	Log(s, "systemd job id %d result: %s\n", jobID, result)

	return s.setUnitState()
}

// Update updates the service unit state
func (s *Service) Update() error {
	return s.setUnitState()
}

// Close closes the connection to the systemd D-BUS API
func (s *Service) Close() error {
	s.conn.Close()

	return nil
}

func init() {
	item := ProviderItem{
		Type:      "service",
		Provider:  NewService,
		Namespace: DefaultResourceNamespace,
	}

	RegisterProvider(item)
}
<commit_msg>resource: Make `enable` a property for Service resource<commit_after>// +build linux

package resource

import (
	"errors"
	"fmt"

	"github.com/coreos/go-systemd/dbus"
	"github.com/coreos/go-systemd/util"
)

// ErrNoSystemd error is returned when the system is detected to
// have no support for systemd.
var ErrNoSystemd = errors.New("No systemd support found")

// Service type is a resource which manages services on a
// GNU/Linux system running with systemd.
//
// Example:
//   svc = resource.service.new("nginx")
//   svc.state = "running"
//   svc.enable = true
type Service struct {
	Base

	// EnableProperty specifies whether to enable or disable the
	// service during boot-time. Defaults to true.
	EnableProperty bool `luar:"enable"`

	// Systemd unit name
	unit string `luar:"-"`

	conn *dbus.Conn `luar:"-"`
}

// NewService creates a new resource for managing services
// using systemd on a GNU/Linux system
func NewService(name string) (Resource, error) {
	if !util.IsRunningSystemd() {
		return nil, ErrNoSystemd
	}

	s := &Service{
		Base: Base{
			Name:          name,
			Type:          "service",
			State:         "running",
			Require:       make([]string, 0),
			PresentStates: []string{"present", "running"},
			AbsentStates:  []string{"absent", "stopped"},
			Concurrent:    true,
			Subscribe:     make(TriggerMap),
		},
		EnableProperty: true,
		unit:           fmt.Sprintf("%s.service", name),
	}

	// Set resource properties
	s.Properties = []Property{
		Property{
			Name:     "enable",
			Set:      s.setEnableProperty,
			IsSynced: s.isEnablePropertySynced,
		},
	}

	return s, nil
}

// Initialize initializes the service resource by establishing a
// connection the systemd D-BUS API
func (s *Service) Initialize() error {
	conn, err := dbus.New()
	s.conn = conn

	return err
}

// Evaluate evaluates the state of the resource
func (s *Service) Evaluate() (State, error) {
	state := State{
		Current: "unknown",
		Want:    s.State,
	}

	// Check if the unit is started/stopped
	activeState, err := s.conn.GetUnitProperty(s.unit, "ActiveState")
	if err != nil {
		return state, err
	}

	// TODO: Handle cases where the unit is not found

	value := activeState.Value.Value().(string)
	switch value {
	case "active", "reloading", "activating":
		state.Current = "running"
	case "inactive", "failed", "deactivating":
		state.Current = "stopped"
	}

	return state, nil
}

// Create starts the service.
func (s *Service) Create() error {
	Log(s, "starting service\n")

	ch := make(chan string)
	jobID, err := s.conn.StartUnit(s.unit, "replace", ch)
	if err != nil {
		return err
	}

	result := <-ch
	Log(s, "systemd job id %d result: %s\n", jobID, result)

	return nil
}

// Delete stops the service.
func (s *Service) Delete() error {
	Log(s, "stopping service\n")

	ch := make(chan string)
	jobID, err := s.conn.StopUnit(s.unit, "replace", ch)
	if err != nil {
		return err
	}

	result := <-ch
	Log(s, "systemd job id %d result: %s\n", jobID, result)

	return nil
}

// Close closes the connection to the systemd D-BUS API
func (s *Service) Close() error {
	s.conn.Close()

	return nil
}

// enableUnit enables the service unit during boot-time
func (s *Service) enableUnit() error {
	Log(s, "enabling service\n")

	units := []string{s.unit}
	_, changes, err := s.conn.EnableUnitFiles(units, false, false)
	if err != nil {
		return err
	}

	for _, change := range changes {
		Log(s, "%s %s -> %s\n", change.Type, change.Filename, change.Destination)
	}

	return nil
}

// disableUnit disables the service unit during boot-time
func (s *Service) disableUnit() error {
	Log(s, "disabling service\n")

	units := []string{s.unit}
	changes, err := s.conn.DisableUnitFiles(units, false)
	if err != nil {
		return err
	}

	for _, change := range changes {
		Log(s, "%s %s\n", change.Type, change.Filename)
	}

	return nil
}

// isEnablePropertySynced determines whether the property is synced.
func (s *Service) isEnablePropertySynced() (bool, error) {
	unitState, err := s.conn.GetUnitProperty(s.unit, "UnitFileState")
	if err != nil {
		return false, err
	}

	var enabled bool
	value := unitState.Value.Value().(string)
	switch value {
	case "enabled", "static", "enabled-runtime", "linked", "linked-runtime":
		enabled = true
	case "disabled", "masked", "masked-runtime":
		enabled = false
	case "invalid":
		fallthrough
	default:
		return false, errors.New("Invalid unit state")
	}

	return s.EnableProperty == enabled, nil
}

// setEnableProperty sets the property to it's desired state.
func (s *Service) setEnableProperty() error {
	var action func() error

	switch s.EnableProperty {
	case true:
		action = s.enableUnit
	case false:
		action = s.disableUnit
	}

	if err := action(); err != nil {
		return err
	}

	return s.conn.Reload()
}

func init() {
	item := ProviderItem{
		Type:      "service",
		Provider:  NewService,
		Namespace: DefaultResourceNamespace,
	}

	RegisterProvider(item)
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package indexers

import (
	"errors"

	"github.com/btcsuite/btcd/blockchain"
	"github.com/btcsuite/btcd/chaincfg"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/database"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcutil/gcs"
	"github.com/btcsuite/btcutil/gcs/builder"
)

const (
	// cfIndexName is the human-readable name for the index.
	cfIndexName = "committed filter index"
)

// Committed filters come in two flavours: basic and extended. They are
// generated and dropped in pairs, and both are indexed by a block's hash.
// Besides holding different content, they also live in different buckets.
var (
	// cfIndexParentBucketKey is the name of the parent bucket used to house
	// the index. The rest of the buckets live below this bucket.
	cfIndexParentBucketKey = []byte("cfindexparentbucket")

	// cfIndexKeys is an array of db bucket names used to house indexes of
	// block hashes to cfilters.
	cfIndexKeys = [][]byte{
		[]byte("cf0byhashidx"),
		[]byte("cf1byhashidx"),
	}

	// cfHeaderKeys is an array of db bucket names used to house indexes of
	// block hashes to cf headers.
	cfHeaderKeys = [][]byte{
		[]byte("cf0headerbyhashidx"),
		[]byte("cf1headerbyhashidx"),
	}

	// cfHashKeys is an array of db bucket names used to house indexes of
	// block hashes to cf hashes.
	cfHashKeys = [][]byte{
		[]byte("cf0hashbyhashidx"),
		[]byte("cf1hashbyhashidx"),
	}

	maxFilterType = uint8(len(cfHeaderKeys) - 1)

	// zeroHash is the chainhash.Hash value of all zero bytes, defined here for
	// convenience.
	zeroHash chainhash.Hash
)

// dbFetchFilterIdxEntry retrieves a data blob from the filter index database.
// An entry's absence is not considered an error.
func dbFetchFilterIdxEntry(dbTx database.Tx, key []byte, h *chainhash.Hash) ([]byte, error) {
	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)
	return idx.Get(h[:]), nil
}

// dbStoreFilterIdxEntry stores a data blob in the filter index database.
func dbStoreFilterIdxEntry(dbTx database.Tx, key []byte, h *chainhash.Hash, f []byte) error {
	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)
	return idx.Put(h[:], f)
}

// dbDeleteFilterIdxEntry deletes a data blob from the filter index database.
func dbDeleteFilterIdxEntry(dbTx database.Tx, key []byte, h *chainhash.Hash) error {
	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)
	return idx.Delete(h[:])
}

// CfIndex implements a committed filter (cf) by hash index.
type CfIndex struct {
	db          database.DB
	chainParams *chaincfg.Params
}

// Ensure the CfIndex type implements the Indexer interface.
var _ Indexer = (*CfIndex)(nil)

// Init initializes the hash-based cf index. This is part of the Indexer
// interface.
func (idx *CfIndex) Init() error {
	return nil // Nothing to do.
}

// Key returns the database key to use for the index as a byte slice. This is
// part of the Indexer interface.
func (idx *CfIndex) Key() []byte {
	return cfIndexParentBucketKey
}

// Name returns the human-readable name of the index. This is part of the
// Indexer interface.
func (idx *CfIndex) Name() string {
	return cfIndexName
}

// Create is invoked when the indexer manager determines the index needs to
// be created for the first time. It creates buckets for the two hash-based cf
// indexes (simple, extended).
func (idx *CfIndex) Create(dbTx database.Tx) error {
	meta := dbTx.Metadata()

	cfIndexParentBucket, err := meta.CreateBucket(cfIndexParentBucketKey)
	if err != nil {
		return err
	}

	for _, bucketName := range cfIndexKeys {
		_, err = cfIndexParentBucket.CreateBucket(bucketName)
		if err != nil {
			return err
		}
	}

	for _, bucketName := range cfHeaderKeys {
		_, err = cfIndexParentBucket.CreateBucket(bucketName)
		if err != nil {
			return err
		}
	}

	for _, bucketName := range cfHashKeys {
		_, err = cfIndexParentBucket.CreateBucket(bucketName)
		if err != nil {
			return err
		}
	}

	return nil
}

// storeFilter stores a given filter, and performs the steps needed to
// generate the filter's header.
func storeFilter(dbTx database.Tx, block *btcutil.Block, f *gcs.Filter,
	filterType wire.FilterType) error {
	if uint8(filterType) > maxFilterType {
		return errors.New("unsupported filter type")
	}

	// Figure out which buckets to use.
	fkey := cfIndexKeys[filterType]
	hkey := cfHeaderKeys[filterType]
	hashkey := cfHashKeys[filterType]

	// Start by storing the filter.
	h := block.Hash()
	filterBytes, err := f.NBytes()
	if err != nil {
		return err
	}
	err = dbStoreFilterIdxEntry(dbTx, fkey, h, filterBytes)
	if err != nil {
		return err
	}

	// Next store the filter hash.
	filterHash, err := builder.GetFilterHash(f)
	if err != nil {
		return err
	}
	err = dbStoreFilterIdxEntry(dbTx, hashkey, h, filterHash[:])
	if err != nil {
		return err
	}

	// Then fetch the previous block's filter header.
	var prevHeader *chainhash.Hash
	ph := &block.MsgBlock().Header.PrevBlock
	if ph.IsEqual(&zeroHash) {
		prevHeader = &zeroHash
	} else {
		pfh, err := dbFetchFilterIdxEntry(dbTx, hkey, ph)
		if err != nil {
			return err
		}

		// Construct the new block's filter header, and store it.
		prevHeader, err = chainhash.NewHash(pfh)
		if err != nil {
			return err
		}
	}

	fh, err := builder.MakeHeaderForFilter(f, *prevHeader)
	if err != nil {
		return err
	}
	return dbStoreFilterIdxEntry(dbTx, hkey, h, fh[:])
}

// ConnectBlock is invoked by the index manager when a new block has been
// connected to the main chain. This indexer adds a hash-to-cf mapping for
// every passed block. This is part of the Indexer interface.
func (idx *CfIndex) ConnectBlock(dbTx database.Tx, block *btcutil.Block,
	stxos []blockchain.SpentTxOut) error {

	f, err := builder.BuildBasicFilter(block.MsgBlock())
	if err != nil {
		return err
	}

	err = storeFilter(dbTx, block, f, wire.GCSFilterRegular)
	if err != nil {
		return err
	}

	f, err = builder.BuildExtFilter(block.MsgBlock())
	if err != nil {
		return err
	}

	return storeFilter(dbTx, block, f, wire.GCSFilterExtended)
}

// DisconnectBlock is invoked by the index manager when a block has been
// disconnected from the main chain.  This indexer removes the hash-to-cf
// mapping for every passed block. This is part of the Indexer interface.
func (idx *CfIndex) DisconnectBlock(dbTx database.Tx, block *btcutil.Block,
	_ []blockchain.SpentTxOut) error {

	for _, key := range cfIndexKeys {
		err := dbDeleteFilterIdxEntry(dbTx, key, block.Hash())
		if err != nil {
			return err
		}
	}

	for _, key := range cfHeaderKeys {
		err := dbDeleteFilterIdxEntry(dbTx, key, block.Hash())
		if err != nil {
			return err
		}
	}

	for _, key := range cfHashKeys {
		err := dbDeleteFilterIdxEntry(dbTx, key, block.Hash())
		if err != nil {
			return err
		}
	}

	return nil
}

// entryByBlockHash fetches a filter index entry of a particular type
// (eg. filter, filter header, etc) for a filter type and block hash.
func (idx *CfIndex) entryByBlockHash(filterTypeKeys [][]byte,
	filterType wire.FilterType, h *chainhash.Hash) ([]byte, error) {

	if uint8(filterType) > maxFilterType {
		return nil, errors.New("unsupported filter type")
	}
	key := filterTypeKeys[filterType]

	var entry []byte
	err := idx.db.View(func(dbTx database.Tx) error {
		var err error
		entry, err = dbFetchFilterIdxEntry(dbTx, key, h)
		return err
	})
	return entry, err
}

// entriesByBlockHashes batch fetches a filter index entry of a particular type
// (eg. filter, filter header, etc) for a filter type and slice of block hashes.
func (idx *CfIndex) entriesByBlockHashes(filterTypeKeys [][]byte,
	filterType wire.FilterType, blockHashes []*chainhash.Hash) ([][]byte, error) {

	if uint8(filterType) > maxFilterType {
		return nil, errors.New("unsupported filter type")
	}
	key := filterTypeKeys[filterType]

	entries := make([][]byte, 0, len(blockHashes))
	err := idx.db.View(func(dbTx database.Tx) error {
		for _, blockHash := range blockHashes {
			entry, err := dbFetchFilterIdxEntry(dbTx, key, blockHash)
			if err != nil {
				return err
			}
			entries = append(entries, entry)
		}
		return nil
	})
	return entries, err
}

// FilterByBlockHash returns the serialized contents of a block's basic or
// extended committed filter.
func (idx *CfIndex) FilterByBlockHash(h *chainhash.Hash,
	filterType wire.FilterType) ([]byte, error) {
	return idx.entryByBlockHash(cfIndexKeys, filterType, h)
}

// FiltersByBlockHashes returns the serialized contents of a block's basic or
// extended committed filter for a set of blocks by hash.
func (idx *CfIndex) FiltersByBlockHashes(blockHashes []*chainhash.Hash,
	filterType wire.FilterType) ([][]byte, error) {
	return idx.entriesByBlockHashes(cfIndexKeys, filterType, blockHashes)
}

// FilterHeaderByBlockHash returns the serialized contents of a block's basic
// or extended committed filter header.
func (idx *CfIndex) FilterHeaderByBlockHash(h *chainhash.Hash,
	filterType wire.FilterType) ([]byte, error) {
	return idx.entryByBlockHash(cfHeaderKeys, filterType, h)
}

// FilterHeadersByBlockHashes returns the serialized contents of a block's basic
// or extended committed filter header for a set of blocks by hash.
func (idx *CfIndex) FilterHeadersByBlockHashes(blockHashes []*chainhash.Hash,
	filterType wire.FilterType) ([][]byte, error) {
	return idx.entriesByBlockHashes(cfHeaderKeys, filterType, blockHashes)
}

// FilterHashByBlockHash returns the serialized contents of a block's basic
// or extended committed filter hash.
func (idx *CfIndex) FilterHashByBlockHash(h *chainhash.Hash,
	filterType wire.FilterType) ([]byte, error) {
	return idx.entryByBlockHash(cfHashKeys, filterType, h)
}

// FilterHashesByBlockHashes returns the serialized contents of a block's basic
// or extended committed filter hash for a set of blocks by hash.
func (idx *CfIndex) FilterHashesByBlockHashes(blockHashes []*chainhash.Hash,
	filterType wire.FilterType) ([][]byte, error) {
	return idx.entriesByBlockHashes(cfHashKeys, filterType, blockHashes)
}

// NewCfIndex returns a new instance of an indexer that is used to create a
// mapping of the hashes of all blocks in the blockchain to their respective
// committed filters.
//
// It implements the Indexer interface which plugs into the IndexManager that
// in turn is used by the blockchain package. This allows the index to be
// seamlessly maintained along with the chain.
func NewCfIndex(db database.DB, chainParams *chaincfg.Params) *CfIndex {
	return &CfIndex{db: db, chainParams: chainParams}
}

// DropCfIndex drops the CF index from the provided database if exists.
func DropCfIndex(db database.DB, interrupt <-chan struct{}) error {
	return dropIndex(db, cfIndexParentBucketKey, cfIndexName, interrupt)
}
<commit_msg>blockchain/indexer: remove extended filter<commit_after>// Copyright (c) 2017 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package indexers

import (
	"errors"

	"github.com/btcsuite/btcd/blockchain"
	"github.com/btcsuite/btcd/chaincfg"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/database"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcutil/gcs"
	"github.com/btcsuite/btcutil/gcs/builder"
)

const (
	// cfIndexName is the human-readable name for the index.
	cfIndexName = "committed filter index"
)

// Committed filters come in one flavor currently: basic. They are generated
// and dropped in pairs, and both are indexed by a block's hash.  Besides
// holding different content, they also live in different buckets.
var (
	// cfIndexParentBucketKey is the name of the parent bucket used to
	// house the index. The rest of the buckets live below this bucket.
	cfIndexParentBucketKey = []byte("cfindexparentbucket")

	// cfIndexKeys is an array of db bucket names used to house indexes of
	// block hashes to cfilters.
	cfIndexKeys = [][]byte{
		[]byte("cf0byhashidx"),
	}

	// cfHeaderKeys is an array of db bucket names used to house indexes of
	// block hashes to cf headers.
	cfHeaderKeys = [][]byte{
		[]byte("cf0headerbyhashidx"),
	}

	// cfHashKeys is an array of db bucket names used to house indexes of
	// block hashes to cf hashes.
	cfHashKeys = [][]byte{
		[]byte("cf0hashbyhashidx"),
	}

	maxFilterType = uint8(len(cfHeaderKeys) - 1)

	// zeroHash is the chainhash.Hash value of all zero bytes, defined here
	// for convenience.
	zeroHash chainhash.Hash
)

// dbFetchFilterIdxEntry retrieves a data blob from the filter index database.
// An entry's absence is not considered an error.
func dbFetchFilterIdxEntry(dbTx database.Tx, key []byte, h *chainhash.Hash) ([]byte, error) {
	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)
	return idx.Get(h[:]), nil
}

// dbStoreFilterIdxEntry stores a data blob in the filter index database.
func dbStoreFilterIdxEntry(dbTx database.Tx, key []byte, h *chainhash.Hash, f []byte) error {
	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)
	return idx.Put(h[:], f)
}

// dbDeleteFilterIdxEntry deletes a data blob from the filter index database.
func dbDeleteFilterIdxEntry(dbTx database.Tx, key []byte, h *chainhash.Hash) error {
	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)
	return idx.Delete(h[:])
}

// CfIndex implements a committed filter (cf) by hash index.
type CfIndex struct {
	db          database.DB
	chainParams *chaincfg.Params
}

// Ensure the CfIndex type implements the Indexer interface.
var _ Indexer = (*CfIndex)(nil)

// Init initializes the hash-based cf index. This is part of the Indexer
// interface.
func (idx *CfIndex) Init() error {
	return nil // Nothing to do.
}

// Key returns the database key to use for the index as a byte slice. This is
// part of the Indexer interface.
func (idx *CfIndex) Key() []byte {
	return cfIndexParentBucketKey
}

// Name returns the human-readable name of the index. This is part of the
// Indexer interface.
func (idx *CfIndex) Name() string {
	return cfIndexName
}

// Create is invoked when the indexer manager determines the index needs to
// be created for the first time. It creates buckets for the two hash-based cf
// indexes (regular only currently).
func (idx *CfIndex) Create(dbTx database.Tx) error {
	meta := dbTx.Metadata()

	cfIndexParentBucket, err := meta.CreateBucket(cfIndexParentBucketKey)
	if err != nil {
		return err
	}

	for _, bucketName := range cfIndexKeys {
		_, err = cfIndexParentBucket.CreateBucket(bucketName)
		if err != nil {
			return err
		}
	}

	for _, bucketName := range cfHeaderKeys {
		_, err = cfIndexParentBucket.CreateBucket(bucketName)
		if err != nil {
			return err
		}
	}

	for _, bucketName := range cfHashKeys {
		_, err = cfIndexParentBucket.CreateBucket(bucketName)
		if err != nil {
			return err
		}
	}

	return nil
}

// storeFilter stores a given filter, and performs the steps needed to
// generate the filter's header.
func storeFilter(dbTx database.Tx, block *btcutil.Block, f *gcs.Filter,
	filterType wire.FilterType) error {
	if uint8(filterType) > maxFilterType {
		return errors.New("unsupported filter type")
	}

	// Figure out which buckets to use.
	fkey := cfIndexKeys[filterType]
	hkey := cfHeaderKeys[filterType]
	hashkey := cfHashKeys[filterType]

	// Start by storing the filter.
	h := block.Hash()
	filterBytes, err := f.NBytes()
	if err != nil {
		return err
	}
	err = dbStoreFilterIdxEntry(dbTx, fkey, h, filterBytes)
	if err != nil {
		return err
	}

	// Next store the filter hash.
	filterHash, err := builder.GetFilterHash(f)
	if err != nil {
		return err
	}
	err = dbStoreFilterIdxEntry(dbTx, hashkey, h, filterHash[:])
	if err != nil {
		return err
	}

	// Then fetch the previous block's filter header.
	var prevHeader *chainhash.Hash
	ph := &block.MsgBlock().Header.PrevBlock
	if ph.IsEqual(&zeroHash) {
		prevHeader = &zeroHash
	} else {
		pfh, err := dbFetchFilterIdxEntry(dbTx, hkey, ph)
		if err != nil {
			return err
		}

		// Construct the new block's filter header, and store it.
		prevHeader, err = chainhash.NewHash(pfh)
		if err != nil {
			return err
		}
	}

	fh, err := builder.MakeHeaderForFilter(f, *prevHeader)
	if err != nil {
		return err
	}
	return dbStoreFilterIdxEntry(dbTx, hkey, h, fh[:])
}

// ConnectBlock is invoked by the index manager when a new block has been
// connected to the main chain. This indexer adds a hash-to-cf mapping for
// every passed block. This is part of the Indexer interface.
func (idx *CfIndex) ConnectBlock(dbTx database.Tx, block *btcutil.Block,
	stxos []blockchain.SpentTxOut) error {

	f, err := builder.BuildBasicFilter(block.MsgBlock())
	if err != nil {
		return err
	}

	return storeFilter(dbTx, block, f, wire.GCSFilterRegular)
}

// DisconnectBlock is invoked by the index manager when a block has been
// disconnected from the main chain.  This indexer removes the hash-to-cf
// mapping for every passed block. This is part of the Indexer interface.
func (idx *CfIndex) DisconnectBlock(dbTx database.Tx, block *btcutil.Block,
	_ []blockchain.SpentTxOut) error {

	for _, key := range cfIndexKeys {
		err := dbDeleteFilterIdxEntry(dbTx, key, block.Hash())
		if err != nil {
			return err
		}
	}

	for _, key := range cfHeaderKeys {
		err := dbDeleteFilterIdxEntry(dbTx, key, block.Hash())
		if err != nil {
			return err
		}
	}

	for _, key := range cfHashKeys {
		err := dbDeleteFilterIdxEntry(dbTx, key, block.Hash())
		if err != nil {
			return err
		}
	}

	return nil
}

// entryByBlockHash fetches a filter index entry of a particular type
// (eg. filter, filter header, etc) for a filter type and block hash.
func (idx *CfIndex) entryByBlockHash(filterTypeKeys [][]byte,
	filterType wire.FilterType, h *chainhash.Hash) ([]byte, error) {

	if uint8(filterType) > maxFilterType {
		return nil, errors.New("unsupported filter type")
	}
	key := filterTypeKeys[filterType]

	var entry []byte
	err := idx.db.View(func(dbTx database.Tx) error {
		var err error
		entry, err = dbFetchFilterIdxEntry(dbTx, key, h)
		return err
	})
	return entry, err
}

// entriesByBlockHashes batch fetches a filter index entry of a particular type
// (eg. filter, filter header, etc) for a filter type and slice of block hashes.
func (idx *CfIndex) entriesByBlockHashes(filterTypeKeys [][]byte,
	filterType wire.FilterType, blockHashes []*chainhash.Hash) ([][]byte, error) {

	if uint8(filterType) > maxFilterType {
		return nil, errors.New("unsupported filter type")
	}
	key := filterTypeKeys[filterType]

	entries := make([][]byte, 0, len(blockHashes))
	err := idx.db.View(func(dbTx database.Tx) error {
		for _, blockHash := range blockHashes {
			entry, err := dbFetchFilterIdxEntry(dbTx, key, blockHash)
			if err != nil {
				return err
			}
			entries = append(entries, entry)
		}
		return nil
	})
	return entries, err
}

// FilterByBlockHash returns the serialized contents of a block's basic or
// committed filter.
func (idx *CfIndex) FilterByBlockHash(h *chainhash.Hash,
	filterType wire.FilterType) ([]byte, error) {
	return idx.entryByBlockHash(cfIndexKeys, filterType, h)
}

// FiltersByBlockHashes returns the serialized contents of a block's basic or
// committed filter for a set of blocks by hash.
func (idx *CfIndex) FiltersByBlockHashes(blockHashes []*chainhash.Hash,
	filterType wire.FilterType) ([][]byte, error) {
	return idx.entriesByBlockHashes(cfIndexKeys, filterType, blockHashes)
}

// FilterHeaderByBlockHash returns the serialized contents of a block's basic
// committed filter header.
func (idx *CfIndex) FilterHeaderByBlockHash(h *chainhash.Hash,
	filterType wire.FilterType) ([]byte, error) {
	return idx.entryByBlockHash(cfHeaderKeys, filterType, h)
}

// FilterHeadersByBlockHashes returns the serialized contents of a block's
// basic committed filter header for a set of blocks by hash.
func (idx *CfIndex) FilterHeadersByBlockHashes(blockHashes []*chainhash.Hash,
	filterType wire.FilterType) ([][]byte, error) {
	return idx.entriesByBlockHashes(cfHeaderKeys, filterType, blockHashes)
}

// FilterHashByBlockHash returns the serialized contents of a block's basic
// committed filter hash.
func (idx *CfIndex) FilterHashByBlockHash(h *chainhash.Hash,
	filterType wire.FilterType) ([]byte, error) {
	return idx.entryByBlockHash(cfHashKeys, filterType, h)
}

// FilterHashesByBlockHashes returns the serialized contents of a block's basic
// committed filter hash for a set of blocks by hash.
func (idx *CfIndex) FilterHashesByBlockHashes(blockHashes []*chainhash.Hash,
	filterType wire.FilterType) ([][]byte, error) {
	return idx.entriesByBlockHashes(cfHashKeys, filterType, blockHashes)
}

// NewCfIndex returns a new instance of an indexer that is used to create a
// mapping of the hashes of all blocks in the blockchain to their respective
// committed filters.
//
// It implements the Indexer interface which plugs into the IndexManager that
// in turn is used by the blockchain package. This allows the index to be
// seamlessly maintained along with the chain.
func NewCfIndex(db database.DB, chainParams *chaincfg.Params) *CfIndex {
	return &CfIndex{db: db, chainParams: chainParams}
}

// DropCfIndex drops the CF index from the provided database if exists.
func DropCfIndex(db database.DB, interrupt <-chan struct{}) error {
	return dropIndex(db, cfIndexParentBucketKey, cfIndexName, interrupt)
}
<|endoftext|>