<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"exec"
	"flag"
	"fmt"
	"go/build"
	"go/token"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
)

func usage() {
	fmt.Fprint(os.Stderr, "usage: goinstall importpath...\n")
	fmt.Fprintf(os.Stderr, "\tgoinstall -a\n")
	flag.PrintDefaults()
	os.Exit(2)
}

const logfile = "goinstall.log"

var (
	fset          = token.NewFileSet()
	argv0         = os.Args[0]
	errors        = false
	parents       = make(map[string]string)
	visit         = make(map[string]status)
	installedPkgs = make(map[string]map[string]bool)
	schemeRe      = regexp.MustCompile(`^[a-z]+://`)

	allpkg            = flag.Bool("a", false, "install all previously installed packages")
	reportToDashboard = flag.Bool("dashboard", true, "report public packages at "+dashboardURL)
	update            = flag.Bool("u", false, "update already-downloaded packages")
	doInstall         = flag.Bool("install", true, "build and install")
	clean             = flag.Bool("clean", false, "clean the package directory before installing")
	nuke              = flag.Bool("nuke", false, "clean the package directory and target before installing")
	useMake           = flag.Bool("make", true, "use make to build and install")
	verbose           = flag.Bool("v", false, "verbose")
)

type status int // status for visited map
const (
	unvisited status = iota
	visiting
	done
)

func logf(format string, args ...interface{}) {
	format = "%s: " + format
	args = append([]interface{}{argv0}, args...)
	fmt.Fprintf(os.Stderr, format, args...)
}

func printf(format string, args ...interface{}) {
	if *verbose {
		logf(format, args...)
	}
}

func errorf(format string, args ...interface{}) {
	errors = true
	logf(format, args...)
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if runtime.GOROOT() == "" {
		fmt.Fprintf(os.Stderr, "%s: no $GOROOT\n", argv0)
		os.Exit(1)
	}
	readPackageList()

	// special case - "unsafe" is already installed
	visit["unsafe"] = done

	args := flag.Args()
	if *allpkg {
		if len(args) != 0 {
			usage() // -a and package list both provided
		}
		// install all packages that were ever installed
		n := 0
		for _, pkgs := range installedPkgs {
			for pkg := range pkgs {
				args = append(args, pkg)
				n++
			}
		}
		if n == 0 {
			logf("no installed packages\n")
			os.Exit(1)
		}
	}
	if len(args) == 0 {
		usage()
	}
	for _, path := range args {
		if s := schemeRe.FindString(path); s != "" {
			errorf("%q used in import path, try %q\n", s, path[len(s):])
			continue
		}

		install(path, "")
	}
	if errors {
		os.Exit(1)
	}
}

// printDeps prints the dependency path that leads to pkg.
func printDeps(pkg string) {
	if pkg == "" {
		return
	}
	if visit[pkg] != done {
		printDeps(parents[pkg])
	}
	fmt.Fprintf(os.Stderr, "\t%s ->\n", pkg)
}

// readPackageList reads the list of installed packages from the
// goinstall.log files in GOROOT and the GOPATHs and initalizes
// the installedPkgs variable.
func readPackageList() {
	for _, t := range build.Path {
		installedPkgs[t.Path] = make(map[string]bool)
		name := filepath.Join(t.Path, logfile)
		pkglistdata, err := ioutil.ReadFile(name)
		if err != nil {
			printf("%s\n", err)
			continue
		}
		pkglist := strings.Fields(string(pkglistdata))
		for _, pkg := range pkglist {
			installedPkgs[t.Path][pkg] = true
		}
	}
}

// logPackage logs the named package as installed in the goinstall.log file
// in the given tree if the package is not already in that file.
func logPackage(pkg string, tree *build.Tree) (logged bool) {
	if installedPkgs[tree.Path][pkg] {
		return false
	}
	name := filepath.Join(tree.Path, logfile)
	fout, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		logf("%s\n", err)
		return false
	}
	fmt.Fprintf(fout, "%s\n", pkg)
	fout.Close()
	return true
}

// install installs the package named by path, which is needed by parent.
func install(pkg, parent string) {
	// Make sure we're not already trying to install pkg.
	switch visit[pkg] {
	case done:
		return
	case visiting:
		fmt.Fprintf(os.Stderr, "%s: package dependency cycle\n", argv0)
		printDeps(parent)
		fmt.Fprintf(os.Stderr, "\t%s\n", pkg)
		os.Exit(2)
	}
	parents[pkg] = parent
	visit[pkg] = visiting
	defer func() {
		visit[pkg] = done
	}()

	// Check whether package is local or remote.
	// If remote, download or update it.
	tree, pkg, err := build.FindTree(pkg)
	// Don't build the standard library.
	if err == nil && tree.Goroot && isStandardPath(pkg) {
		if parent == "" {
			errorf("%s: can not goinstall the standard library\n", pkg)
		} else {
			printf("%s: skipping standard library\n", pkg)
		}
		return
	}
	// Download remote packages if not found or forced with -u flag.
	remote, public := isRemote(pkg), false
	if remote && (err == build.ErrNotFound || (err == nil && *update)) {
		printf("%s: download\n", pkg)
		public, err = download(pkg, tree.SrcDir())
	}
	if err != nil {
		errorf("%s: %v\n", pkg, err)
		return
	}
	dir := filepath.Join(tree.SrcDir(), pkg)

	// Install prerequisites.
	dirInfo, err := build.ScanDir(dir, parent == "")
	if err != nil {
		errorf("%s: %v\n", pkg, err)
		return
	}
	if len(dirInfo.GoFiles)+len(dirInfo.CgoFiles) == 0 {
		errorf("%s: package has no files\n", pkg)
		return
	}
	for _, p := range dirInfo.Imports {
		if p != "C" {
			install(p, pkg)
		}
	}
	if errors {
		return
	}

	// Install this package.
	if *useMake {
		err := domake(dir, pkg, tree, dirInfo.IsCommand())
		if err != nil {
			errorf("%s: install: %v\n", pkg, err)
			return
		}
	} else {
		script, err := build.Build(tree, pkg, dirInfo)
		if err != nil {
			errorf("%s: install: %v\n", pkg, err)
			return
		}
		if *nuke {
			printf("%s: nuke\n", pkg)
			script.Nuke()
		} else if *clean {
			printf("%s: clean\n", pkg)
			script.Clean()
		}
		if *doInstall {
			if script.Stale() {
				printf("%s: install\n", pkg)
				if err := script.Run(); err != nil {
					errorf("%s: install: %v\n", pkg, err)
					return
				}
			} else {
				printf("%s: up-to-date\n", pkg)
			}
		}
	}

	if remote {
		// mark package as installed in goinstall.log
		logged := logPackage(pkg, tree)

		// report installation to the dashboard if this is the first
		// install from a public repository.
		if logged && public {
			maybeReportToDashboard(pkg)
		}
	}
}

// Is this a standard package path?  strings container/vector etc.
// Assume that if the first element has a dot, it's a domain name
// and is not the standard package path.
func isStandardPath(s string) bool {
	dot := strings.Index(s, ".")
	slash := strings.Index(s, "/")
	return dot < 0 || 0 < slash && slash < dot
}

// run runs the command cmd in directory dir with standard input stdin.
// If the command fails, run prints the command and output on standard error
// in addition to returning a non-nil os.Error.
func run(dir string, stdin []byte, cmd ...string) os.Error {
	return genRun(dir, stdin, cmd, false)
}

// quietRun is like run but prints nothing on failure unless -v is used.
func quietRun(dir string, stdin []byte, cmd ...string) os.Error {
	return genRun(dir, stdin, cmd, true)
}

// genRun implements run and quietRun.
func genRun(dir string, stdin []byte, arg []string, quiet bool) os.Error {
	cmd := exec.Command(arg[0], arg[1:]...)
	cmd.Stdin = bytes.NewBuffer(stdin)
	cmd.Dir = dir
	printf("%s: %s %s\n", dir, cmd.Path, strings.Join(arg[1:], " "))
	out, err := cmd.CombinedOutput()
	if err != nil {
		if !quiet || *verbose {
			if dir != "" {
				dir = "cd " + dir + "; "
			}
			fmt.Fprintf(os.Stderr, "%s: === %s%s\n", cmd.Path, dir, strings.Join(cmd.Args, " "))
			os.Stderr.Write(out)
			fmt.Fprintf(os.Stderr, "--- %s\n", err)
		}
		return os.NewError("running " + arg[0] + ": " + err.String())
	}
	return nil
}
<commit_msg>goinstall: report all newly-installed public packages<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"exec"
	"flag"
	"fmt"
	"go/build"
	"go/token"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
)

func usage() {
	fmt.Fprint(os.Stderr, "usage: goinstall importpath...\n")
	fmt.Fprintf(os.Stderr, "\tgoinstall -a\n")
	flag.PrintDefaults()
	os.Exit(2)
}

const logfile = "goinstall.log"

var (
	fset          = token.NewFileSet()
	argv0         = os.Args[0]
	errors        = false
	parents       = make(map[string]string)
	visit         = make(map[string]status)
	installedPkgs = make(map[string]map[string]bool)
	schemeRe      = regexp.MustCompile(`^[a-z]+://`)

	allpkg            = flag.Bool("a", false, "install all previously installed packages")
	reportToDashboard = flag.Bool("dashboard", true, "report public packages at "+dashboardURL)
	update            = flag.Bool("u", false, "update already-downloaded packages")
	doInstall         = flag.Bool("install", true, "build and install")
	clean             = flag.Bool("clean", false, "clean the package directory before installing")
	nuke              = flag.Bool("nuke", false, "clean the package directory and target before installing")
	useMake           = flag.Bool("make", true, "use make to build and install")
	verbose           = flag.Bool("v", false, "verbose")
)

type status int // status for visited map
const (
	unvisited status = iota
	visiting
	done
)

func logf(format string, args ...interface{}) {
	format = "%s: " + format
	args = append([]interface{}{argv0}, args...)
	fmt.Fprintf(os.Stderr, format, args...)
}

func printf(format string, args ...interface{}) {
	if *verbose {
		logf(format, args...)
	}
}

func errorf(format string, args ...interface{}) {
	errors = true
	logf(format, args...)
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if runtime.GOROOT() == "" {
		fmt.Fprintf(os.Stderr, "%s: no $GOROOT\n", argv0)
		os.Exit(1)
	}
	readPackageList()

	// special case - "unsafe" is already installed
	visit["unsafe"] = done

	args := flag.Args()
	if *allpkg {
		if len(args) != 0 {
			usage() // -a and package list both provided
		}
		// install all packages that were ever installed
		n := 0
		for _, pkgs := range installedPkgs {
			for pkg := range pkgs {
				args = append(args, pkg)
				n++
			}
		}
		if n == 0 {
			logf("no installed packages\n")
			os.Exit(1)
		}
	}
	if len(args) == 0 {
		usage()
	}
	for _, path := range args {
		if s := schemeRe.FindString(path); s != "" {
			errorf("%q used in import path, try %q\n", s, path[len(s):])
			continue
		}

		install(path, "")
	}
	if errors {
		os.Exit(1)
	}
}

// printDeps prints the dependency path that leads to pkg.
func printDeps(pkg string) {
	if pkg == "" {
		return
	}
	if visit[pkg] != done {
		printDeps(parents[pkg])
	}
	fmt.Fprintf(os.Stderr, "\t%s ->\n", pkg)
}

// readPackageList reads the list of installed packages from the
// goinstall.log files in GOROOT and the GOPATHs and initalizes
// the installedPkgs variable.
func readPackageList() {
	for _, t := range build.Path {
		installedPkgs[t.Path] = make(map[string]bool)
		name := filepath.Join(t.Path, logfile)
		pkglistdata, err := ioutil.ReadFile(name)
		if err != nil {
			printf("%s\n", err)
			continue
		}
		pkglist := strings.Fields(string(pkglistdata))
		for _, pkg := range pkglist {
			installedPkgs[t.Path][pkg] = true
		}
	}
}

// logPackage logs the named package as installed in the goinstall.log file
// in the given tree if the package is not already in that file.
func logPackage(pkg string, tree *build.Tree) (logged bool) {
	if installedPkgs[tree.Path][pkg] {
		return false
	}
	name := filepath.Join(tree.Path, logfile)
	fout, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		logf("%s\n", err)
		return false
	}
	fmt.Fprintf(fout, "%s\n", pkg)
	fout.Close()
	return true
}

// install installs the package named by path, which is needed by parent.
func install(pkg, parent string) {
	// Make sure we're not already trying to install pkg.
	switch visit[pkg] {
	case done:
		return
	case visiting:
		fmt.Fprintf(os.Stderr, "%s: package dependency cycle\n", argv0)
		printDeps(parent)
		fmt.Fprintf(os.Stderr, "\t%s\n", pkg)
		os.Exit(2)
	}
	parents[pkg] = parent
	visit[pkg] = visiting
	defer func() {
		visit[pkg] = done
	}()

	// Check whether package is local or remote.
	// If remote, download or update it.
	tree, pkg, err := build.FindTree(pkg)
	// Don't build the standard library.
	if err == nil && tree.Goroot && isStandardPath(pkg) {
		if parent == "" {
			errorf("%s: can not goinstall the standard library\n", pkg)
		} else {
			printf("%s: skipping standard library\n", pkg)
		}
		return
	}
	// Download remote packages if not found or forced with -u flag.
	remote, public := isRemote(pkg), false
	if remote {
		if err == build.ErrNotFound || (err == nil && *update) {
			// Download remote package.
			printf("%s: download\n", pkg)
			public, err = download(pkg, tree.SrcDir())
		} else {
			// Test if this is a public repository
			// (for reporting to dashboard).
			m, _ := findPublicRepo(pkg)
			public = m != nil
		}
	}
	if err != nil {
		errorf("%s: %v\n", pkg, err)
		return
	}
	dir := filepath.Join(tree.SrcDir(), pkg)

	// Install prerequisites.
	dirInfo, err := build.ScanDir(dir, parent == "")
	if err != nil {
		errorf("%s: %v\n", pkg, err)
		return
	}
	if len(dirInfo.GoFiles)+len(dirInfo.CgoFiles) == 0 {
		errorf("%s: package has no files\n", pkg)
		return
	}
	for _, p := range dirInfo.Imports {
		if p != "C" {
			install(p, pkg)
		}
	}
	if errors {
		return
	}

	// Install this package.
	if *useMake {
		err := domake(dir, pkg, tree, dirInfo.IsCommand())
		if err != nil {
			errorf("%s: install: %v\n", pkg, err)
			return
		}
	} else {
		script, err := build.Build(tree, pkg, dirInfo)
		if err != nil {
			errorf("%s: install: %v\n", pkg, err)
			return
		}
		if *nuke {
			printf("%s: nuke\n", pkg)
			script.Nuke()
		} else if *clean {
			printf("%s: clean\n", pkg)
			script.Clean()
		}
		if *doInstall {
			if script.Stale() {
				printf("%s: install\n", pkg)
				if err := script.Run(); err != nil {
					errorf("%s: install: %v\n", pkg, err)
					return
				}
			} else {
				printf("%s: up-to-date\n", pkg)
			}
		}
	}

	if remote {
		// mark package as installed in goinstall.log
		logged := logPackage(pkg, tree)

		// report installation to the dashboard if this is the first
		// install from a public repository.
		if logged && public {
			maybeReportToDashboard(pkg)
		}
	}
}

// Is this a standard package path?  strings container/vector etc.
// Assume that if the first element has a dot, it's a domain name
// and is not the standard package path.
func isStandardPath(s string) bool {
	dot := strings.Index(s, ".")
	slash := strings.Index(s, "/")
	return dot < 0 || 0 < slash && slash < dot
}

// run runs the command cmd in directory dir with standard input stdin.
// If the command fails, run prints the command and output on standard error
// in addition to returning a non-nil os.Error.
func run(dir string, stdin []byte, cmd ...string) os.Error {
	return genRun(dir, stdin, cmd, false)
}

// quietRun is like run but prints nothing on failure unless -v is used.
func quietRun(dir string, stdin []byte, cmd ...string) os.Error {
	return genRun(dir, stdin, cmd, true)
}

// genRun implements run and quietRun.
func genRun(dir string, stdin []byte, arg []string, quiet bool) os.Error {
	cmd := exec.Command(arg[0], arg[1:]...)
	cmd.Stdin = bytes.NewBuffer(stdin)
	cmd.Dir = dir
	printf("%s: %s %s\n", dir, cmd.Path, strings.Join(arg[1:], " "))
	out, err := cmd.CombinedOutput()
	if err != nil {
		if !quiet || *verbose {
			if dir != "" {
				dir = "cd " + dir + "; "
			}
			fmt.Fprintf(os.Stderr, "%s: === %s%s\n", cmd.Path, dir, strings.Join(cmd.Args, " "))
			os.Stderr.Write(out)
			fmt.Fprintf(os.Stderr, "--- %s\n", err)
		}
		return os.NewError("running " + arg[0] + ": " + err.String())
	}
	return nil
}
<|endoftext|>


<commit_before>package serf

import (
	"fmt"
	"net"
	"strconv"

	"github.com/hashicorp/memberlist"
)

type MergeDelegate interface {
	NotifyMerge([]*Member) error
}

type mergeDelegate struct {
	serf *Serf
}

func (m *mergeDelegate) NotifyMerge(nodes []*memberlist.Node) error {
	members := make([]*Member, len(nodes))
	for idx, n := range nodes {
		var err error
		members[idx], err = m.nodeToMember(n)
		if err != nil {
			return err
		}
	}
	return m.serf.config.Merge.NotifyMerge(members)
}

func (m *mergeDelegate) NotifyAlive(peer *memberlist.Node) error {
	member, err := m.nodeToMember(peer)
	if err != nil {
		return err
	}
	return m.serf.config.Merge.NotifyMerge([]*Member{member})
}

func (m *mergeDelegate) nodeToMember(n *memberlist.Node) (*Member, error) {
	status := StatusNone
	if n.State == memberlist.StateLeft {
		status = StatusLeft
	}
	if err := m.validiateMemberInfo(n); err != nil {
		return nil, err
	}
	return &Member{
		Name:        n.Name,
		Addr:        net.IP(n.Addr),
		Port:        n.Port,
		Tags:        m.serf.decodeTags(n.Meta),
		Status:      status,
		ProtocolMin: n.PMin,
		ProtocolMax: n.PMax,
		ProtocolCur: n.PCur,
		DelegateMin: n.DMin,
		DelegateMax: n.DMax,
		DelegateCur: n.DCur,
	}, nil
}

// validateMemberInfo checks that the data we are sending is valid
func (m *mergeDelegate) validiateMemberInfo(n *memberlist.Node) error {
	if err := m.serf.ValidateNodeNames(); err != nil {
		return err
	}

	host, port, err := net.SplitHostPort(string(n.Addr))
	if err != nil {
		return err
	}

	ip := net.ParseIP(host)
	if ip == nil || (ip.To4() == nil && ip.To16() == nil) {
		return fmt.Errorf("%v is not a valid IPv4 or IPv6 address\n", ip)
	}

	p, err := strconv.Atoi(port)
	if err != nil {
		return err
	}
	if p < 0 || p > 65535 {
		return fmt.Errorf("invalid port %v , port must be a valid number from 0-65535", p)
	}

	if len(n.Meta) > memberlist.MetaMaxSize {
		return fmt.Errorf("Encoded length of tags exceeds limit of %d bytes",
			memberlist.MetaMaxSize)
	}
	return nil
}
<commit_msg>Fix typo: s/validiate/validate/<commit_after>package serf

import (
	"fmt"
	"net"
	"strconv"

	"github.com/hashicorp/memberlist"
)

type MergeDelegate interface {
	NotifyMerge([]*Member) error
}

type mergeDelegate struct {
	serf *Serf
}

func (m *mergeDelegate) NotifyMerge(nodes []*memberlist.Node) error {
	members := make([]*Member, len(nodes))
	for idx, n := range nodes {
		var err error
		members[idx], err = m.nodeToMember(n)
		if err != nil {
			return err
		}
	}
	return m.serf.config.Merge.NotifyMerge(members)
}

func (m *mergeDelegate) NotifyAlive(peer *memberlist.Node) error {
	member, err := m.nodeToMember(peer)
	if err != nil {
		return err
	}
	return m.serf.config.Merge.NotifyMerge([]*Member{member})
}

func (m *mergeDelegate) nodeToMember(n *memberlist.Node) (*Member, error) {
	status := StatusNone
	if n.State == memberlist.StateLeft {
		status = StatusLeft
	}
	if err := m.validateMemberInfo(n); err != nil {
		return nil, err
	}
	return &Member{
		Name:        n.Name,
		Addr:        net.IP(n.Addr),
		Port:        n.Port,
		Tags:        m.serf.decodeTags(n.Meta),
		Status:      status,
		ProtocolMin: n.PMin,
		ProtocolMax: n.PMax,
		ProtocolCur: n.PCur,
		DelegateMin: n.DMin,
		DelegateMax: n.DMax,
		DelegateCur: n.DCur,
	}, nil
}

// validateMemberInfo checks that the data we are sending is valid
func (m *mergeDelegate) validateMemberInfo(n *memberlist.Node) error {
	if err := m.serf.ValidateNodeNames(); err != nil {
		return err
	}

	host, port, err := net.SplitHostPort(string(n.Addr))
	if err != nil {
		return err
	}

	ip := net.ParseIP(host)
	if ip == nil || (ip.To4() == nil && ip.To16() == nil) {
		return fmt.Errorf("%v is not a valid IPv4 or IPv6 address\n", ip)
	}

	p, err := strconv.Atoi(port)
	if err != nil {
		return err
	}
	if p < 0 || p > 65535 {
		return fmt.Errorf("invalid port %v , port must be a valid number from 0-65535", p)
	}

	if len(n.Meta) > memberlist.MetaMaxSize {
		return fmt.Errorf("Encoded length of tags exceeds limit of %d bytes",
			memberlist.MetaMaxSize)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/cumulodev/goutils/pool"
	"github.com/cumulodev/nimbusec"
)

func main() {
	url := flag.String("url", nimbusec.DefaultAPI, "API Url")
	key := flag.String("key", "abc", "API key for authentication")
	secret := flag.String("secret", "abc", "API secret for authentication")
	file := flag.String("file", "import.csv", "path to import file")
	delete := flag.Bool("delete", false, "delete domains from nimbusec if not provided in the CSV")
	update := flag.Bool("update", false, "updates domain info; false to just insert new domains")
	workers := flag.Int("workers", 1, "number of paralell workers (please do not use too many workers)")
	flag.Parse()

	// creates a new nimbusec API instance
	api, err := nimbusec.NewAPI(*url, *key, *secret)
	if err != nil {
		log.Fatal(err)
	}

	// open csv input file and parse it
	fh, err := os.Open(*file)
	if err != nil {
		log.Fatal(err)
	}

	defer fh.Close()
	reader := csv.NewReader(fh)
	reader.FieldsPerRecord = -1 // see the Reader struct information below
	rows, err := reader.ReadAll()
	if err != nil {
		log.Fatal(err)
	}

	pool := pool.New(*workers)
	pool.Start()

	// keep track of domain names (required for delete step later)
	ref := make(map[string]struct{})

	for _, row := range rows {
		name := row[0]
		scheme := row[2]
		bundle := row[3]

		url := scheme + "://" + name
		if len(row) > 4 {
			url = url + row[4]
		}

		// construct domain
		domain := &nimbusec.Domain{
			Name:      name,
			Bundle:    bundle,
			Scheme:    scheme,
			DeepScan:  url,
			FastScans: []string{url},
		}

		// upsert domain
		ref[name] = struct{}{}
		pool.Add(upsertJob{
			api:    api,
			domain: domain,
			update: *update,
		})
	}

	pool.Wait()

	// sync
	// delete domains not listed in new set
	if *delete {
		// read all domains from api
		domains, err := api.FindDomains(nimbusec.EmptyFilter)
		if err != nil {
			log.Fatal(err)
		}

		// cross reference domains in nimbusec with csv file and delete all
		// domains not present in csv file
		for _, domain := range domains {
			if _, ok := ref[domain.Name]; !ok {
				pool.Add(deleteJob{
					api:    api,
					domain: &domain,
				})
			}
		}

		pool.Wait()
	}
}

type upsertJob struct {
	api    *nimbusec.API
	domain *nimbusec.Domain
	update bool
}

func (this upsertJob) Work() {
	fmt.Printf("upsert domain: %+v\n", this.domain)
	if this.update {
		if _, err := this.api.CreateOrUpdateDomain(this.domain); err != nil {
			log.Fatal(err)
		}
	} else {
		if _, err := this.api.CreateOrGetDomain(this.domain); err != nil {
			log.Fatal(err)
		}
	}
}

func (this upsertJob) Save() {}

type deleteJob struct {
	api    *nimbusec.API
	domain *nimbusec.Domain
}

func (this deleteJob) Work() {
	fmt.Printf("delete domain: %s\n", this.domain.Name)
	this.api.DeleteDomain(this.domain, true)
}

func (this deleteJob) Save() {}
<commit_msg>fixed typo<commit_after>package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/cumulodev/goutils/pool"
	"github.com/cumulodev/nimbusec"
)

func main() {
	url := flag.String("url", nimbusec.DefaultAPI, "API Url")
	key := flag.String("key", "abc", "API key for authentication")
	secret := flag.String("secret", "abc", "API secret for authentication")
	file := flag.String("file", "import.csv", "path to import file")
	delete := flag.Bool("delete", false, "delete domains from nimbusec if not provided in the CSV")
	update := flag.Bool("update", false, "updates domain info; false to just insert new domains")
	workers := flag.Int("workers", 1, "number of parallel workers (please do not use too many workers)")
	flag.Parse()

	// creates a new nimbusec API instance
	api, err := nimbusec.NewAPI(*url, *key, *secret)
	if err != nil {
		log.Fatal(err)
	}

	// open csv input file and parse it
	fh, err := os.Open(*file)
	if err != nil {
		log.Fatal(err)
	}

	defer fh.Close()
	reader := csv.NewReader(fh)
	reader.FieldsPerRecord = -1 // see the Reader struct information below
	rows, err := reader.ReadAll()
	if err != nil {
		log.Fatal(err)
	}

	pool := pool.New(*workers)
	pool.Start()

	// keep track of domain names (required for delete step later)
	ref := make(map[string]struct{})

	for _, row := range rows {
		name := row[0]
		scheme := row[2]
		bundle := row[3]

		url := scheme + "://" + name
		//		if len(row) > 4 {
		//			deeplink := row[4]
		//			if deeplink == "" {
		//				// do nothing
		//			} else if strings.HasPrefix(deeplink, "/") {
		//				url = url + deeplink
		//			} else {
		//				url = deeplink
		//			}
		//		}

		// construct domain
		domain := &nimbusec.Domain{
			Name:      name,
			Bundle:    bundle,
			Scheme:    scheme,
			DeepScan:  url,
			FastScans: []string{url},
		}

		// upsert domain
		ref[name] = struct{}{}
		pool.Add(upsertJob{
			api:    api,
			domain: domain,
			update: *update,
		})
	}

	pool.Wait()

	// sync
	// delete domains not listed in new set
	if *delete {
		// read all domains from api
		domains, err := api.FindDomains(nimbusec.EmptyFilter)
		if err != nil {
			log.Fatal(err)
		}

		// cross reference domains in nimbusec with csv file and delete all
		// domains not present in csv file
		for _, domain := range domains {
			if _, ok := ref[domain.Name]; !ok {
				pool.Add(deleteJob{
					api:    api,
					domain: &domain,
				})
			}
		}

		pool.Wait()
	}
}

type upsertJob struct {
	api    *nimbusec.API
	domain *nimbusec.Domain
	update bool
}

func (this upsertJob) Work() {
	fmt.Printf("upsert domain: %+v\n", this.domain)
	if this.update {
		if _, err := this.api.CreateOrUpdateDomain(this.domain); err != nil {
			log.Fatal(err)
		}
	} else {
		if _, err := this.api.CreateOrGetDomain(this.domain); err != nil {
			log.Fatal(err)
		}
	}
}

func (this upsertJob) Save() {}

type deleteJob struct {
	api    *nimbusec.API
	domain *nimbusec.Domain
}

func (this deleteJob) Work() {
	fmt.Printf("delete domain: %s\n", this.domain.Name)
	this.api.DeleteDomain(this.domain, true)
}

func (this deleteJob) Save() {}
<|endoftext|>


<commit_before>// dupes finds duplicate files in the given root directory
//
// TODO concurrent checksum/compare? library for other programs?
package main

import (
	"bytes"
	"crypto/sha256"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"
)

// bytesize ints represent a size as in "bytes of memory"
type bytesize uint64

// String formats the underlying integer with suitable units
// (KB, MB, .., YB) to keep the number itself small-ish.
func (bs bytesize) String() string {
	units := []string{"bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}
	value := float64(bs)
	unit := 0
	for value > 1024.0 && unit < len(units)-1 {
		value /= 1024.0
		unit++
	}
	return fmt.Sprintf("%.2f %s", value, units[unit])
}

// countin ints represent a count
type countin uint64

// String formats the underlying integer with commas as "thousands separators"
// to make it easier to read.
func (ci countin) String() string {
	str := fmt.Sprintf("%d", ci)
	chunks := splitFromBack(str, 3)
	str = strings.Join(chunks, ",")
	return str
}

// split string s into chunks of at most n characters from the back.
func splitFromBack(s string, n int) []string {
	var chunks []string

	fullChunks := len(s) / n
	restChunk := len(s) % n

	if restChunk > 0 {
		chunks = append(chunks, s[0:restChunk])
	}

	var i = restChunk
	for fullChunks > 0 {
		chunks = append(chunks, s[i:i+n])
		i += n
		fullChunks--
	}
	return chunks
}

var paranoid = flag.Bool("p", false, "paranoid byte-by-byte comparison")
var goroutines = flag.Int("j", runtime.NumCPU(), "number of goroutines")

// hashes maps from digests to paths
var hashes = make(map[string]string)

// sizes maps from sizes to paths
var sizes = make(map[int64]string)

// files counts the number of files examined
var files countin

// dupes counts the number of duplicate files
var dupes countin

// wasted counts the space (in bytes) occupied by duplicates
var wasted bytesize

// identical does a byte-by-byte comparison of the files with the
// given paths
func identical(pa, pb string) (bool, error) {
	bufferSize := os.Getpagesize()

	a, err := os.Open(pa)
	if err != nil {
		return false, err
	}
	defer a.Close()
	b, err := os.Open(pb)
	if err != nil {
		return false, err
	}
	defer b.Close()

	ba := make([]byte, bufferSize)
	bb := make([]byte, bufferSize)

	for {
		la, erra := a.Read(ba)
		lb, errb := b.Read(bb)

		if erra != nil || errb != nil {
			if erra == io.EOF && errb == io.EOF {
				return true, nil
			}
			if erra != nil {
				return false, erra
			}
			if errb != nil {
				return false, errb
			}
		}

		if la != lb { // TODO: short read always at end of file?
			return false, nil
		}

		if !bytes.Equal(ba, bb) {
			return false, nil
		}
	}
}

// checksum calculates a hash digest for the file with the given path
func checksum(path string) (string, error) {
	file, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer file.Close()

	hasher := sha256.New()
	io.Copy(hasher, file)
	sum := fmt.Sprintf("%x", hasher.Sum(nil))

	return sum, nil
}

// check is called for each path we walk. It only examines regular, non-empty
// files. It first rules out duplicates by file size; for files that remain
// it calculates a checksum; if it has seen the same checksum before, it
// signals a duplicate; otherwise it remembers the checksum and the path of
// the original file before moving on; in paranoid mode it follows up with a
// byte-by-byte file comparison.
func check(path string, info os.FileInfo, err error) error {
	if err != nil {
		return err
	}

	size := info.Size()

	if !info.Mode().IsRegular() || size == 0 {
		return nil
	}

	files++

	var dupe string
	var ok bool
	if dupe, ok = sizes[size]; !ok {
		sizes[size] = path
		return nil
	}

	// backpatch new file into hashes
	sum, err := checksum(dupe)
	if err != nil {
		return err
	}
	hashes[sum] = dupe

	sum, err = checksum(path)
	if err != nil {
		return err
	}

	if dupe, ok = hashes[sum]; !ok {
		hashes[sum] = path
		return nil
	}

	if *paranoid {
		same, err := identical(path, dupe)
		if err != nil {
			return err
		}
		if !same {
			fmt.Printf("cool: %s sha256-collides with %s!\n", path, dupe)
			return nil
		}
	}

	fmt.Printf("%s\n%s\n\n", path, dupe)
	dupes++
	wasted += bytesize(size)

	return nil
}

func main() {
	flag.Usage = func() {
		var program = os.Args[0]
		fmt.Fprintf(os.Stderr, "Usage: %s [option...] directory...\n", program)
		flag.PrintDefaults()
	}

	flag.Parse()
	if len(flag.Args()) < 1 {
		flag.Usage()
	}

	for _, root := range flag.Args() {
		filepath.Walk(root, check)
	}

	if len(sizes) > 0 || len(hashes) > 0 {
		fmt.Printf("%v files examined, %v duplicates found, %v wasted\n", files, dupes, wasted)
	}
}
<commit_msg>Move TODO to README, remove concurrency pieces.<commit_after>// dupes finds duplicate files in the given root directory
package main

import (
	"bytes"
	"crypto/sha256"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
)

// bytesize ints represent a size as in "bytes of memory"
type bytesize uint64

// String formats the underlying integer with suitable units
// (KB, MB, .., YB) to keep the number itself small-ish.
func (bs bytesize) String() string {
	units := []string{"bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}
	value := float64(bs)
	unit := 0
	for value > 1024.0 && unit < len(units)-1 {
		value /= 1024.0
		unit++
	}
	return fmt.Sprintf("%.2f %s", value, units[unit])
}

// countin ints represent a count
type countin uint64

// String formats the underlying integer with commas as "thousands separators"
// to make it easier to read.
func (ci countin) String() string {
	str := fmt.Sprintf("%d", ci)
	chunks := splitFromBack(str, 3)
	str = strings.Join(chunks, ",")
	return str
}

// split string s into chunks of at most n characters from the back.
func splitFromBack(s string, n int) []string {
	var chunks []string

	fullChunks := len(s) / n
	restChunk := len(s) % n

	if restChunk > 0 {
		chunks = append(chunks, s[0:restChunk])
	}

	var i = restChunk
	for fullChunks > 0 {
		chunks = append(chunks, s[i:i+n])
		i += n
		fullChunks--
	}
	return chunks
}

var paranoid = flag.Bool("p", false, "paranoid byte-by-byte comparison")

// hashes maps from digests to paths
var hashes = make(map[string]string)

// sizes maps from sizes to paths
var sizes = make(map[int64]string)

// files counts the number of files examined
var files countin

// dupes counts the number of duplicate files
var dupes countin

// wasted counts the space (in bytes) occupied by duplicates
var wasted bytesize

// identical does a byte-by-byte comparison of the files with the
// given paths
func identical(pa, pb string) (bool, error) {
	bufferSize := os.Getpagesize()

	a, err := os.Open(pa)
	if err != nil {
		return false, err
	}
	defer a.Close()
	b, err := os.Open(pb)
	if err != nil {
		return false, err
	}
	defer b.Close()

	ba := make([]byte, bufferSize)
	bb := make([]byte, bufferSize)

	for {
		la, erra := a.Read(ba)
		lb, errb := b.Read(bb)

		if erra != nil || errb != nil {
			if erra == io.EOF && errb == io.EOF {
				return true, nil
			}
			if erra != nil {
				return false, erra
			}
			if errb != nil {
				return false, errb
			}
		}

		if la != lb { // TODO: short read always at end of file?
			return false, nil
		}

		if !bytes.Equal(ba, bb) {
			return false, nil
		}
	}
}

// checksum calculates a hash digest for the file with the given path
func checksum(path string) (string, error) {
	file, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer file.Close()

	hasher := sha256.New()
	io.Copy(hasher, file)
	sum := fmt.Sprintf("%x", hasher.Sum(nil))

	return sum, nil
}

// check is called for each path we walk. It only examines regular, non-empty
// files. It first rules out duplicates by file size; for files that remain
// it calculates a checksum; if it has seen the same checksum before, it
// signals a duplicate; otherwise it remembers the checksum and the path of
// the original file before moving on; in paranoid mode it follows up with a
// byte-by-byte file comparison.
func check(path string, info os.FileInfo, err error) error {
	if err != nil {
		return err
	}

	size := info.Size()

	if !info.Mode().IsRegular() || size == 0 {
		return nil
	}

	files++

	var dupe string
	var ok bool
	if dupe, ok = sizes[size]; !ok {
		sizes[size] = path
		return nil
	}

	// backpatch new file into hashes
	sum, err := checksum(dupe)
	if err != nil {
		return err
	}
	hashes[sum] = dupe

	sum, err = checksum(path)
	if err != nil {
		return err
	}

	if dupe, ok = hashes[sum]; !ok {
		hashes[sum] = path
		return nil
	}

	if *paranoid {
		same, err := identical(path, dupe)
		if err != nil {
			return err
		}
		if !same {
			fmt.Printf("cool: %s sha256-collides with %s!\n", path, dupe)
			return nil
		}
	}

	fmt.Printf("%s\n%s\n\n", path, dupe)
	dupes++
	wasted += bytesize(size)

	return nil
}

func main() {
	flag.Usage = func() {
		var program = os.Args[0]
		fmt.Fprintf(os.Stderr, "Usage: %s [option...] directory...\n", program)
		flag.PrintDefaults()
	}

	flag.Parse()
	if len(flag.Args()) < 1 {
		flag.Usage()
	}

	for _, root := range flag.Args() {
		filepath.Walk(root, check)
	}

	if len(sizes) > 0 || len(hashes) > 0 {
		fmt.Printf("%v files examined, %v duplicates found, %v wasted\n", files, dupes, wasted)
	}
}
<|endoftext|>


<commit_before>package stripe

import "encoding/json"

// ReversalParams is the set of parameters that can be used when reversing a transfer.
type ReversalParams struct {
	Params
	Transfer string
	Amount   uint64
	Fee      bool
}

// ReversalListParams is the set of parameters that can be used when listing reversals.
type ReversalListParams struct {
	ListParams
	Transfer string
}

// Reversal represents a transfer reversal.
type Reversal struct {
	ID       string            `json:"id"`
	Amount   uint64            `json:"amount"`
	Created  int64             `json:"created"`
	Currency Currency          `json:"currency"`
	Transfer string            `json:"transfer"`
	Meta     map[string]string `json:"metadata"`
}

// ReversalList is a list of object for reversals.
type ReversalList struct {
	ListMeta
	Values []*Reversal `json:"data"`
}

// UnmarshalJSON handles deserialization of a Reversal.
// This custom unmarshaling is needed because the resulting
// property may be an id or the full struct if it was expanded.
func (r *Reversal) UnmarshalJSON(data []byte) error {
	type reversal Reversal
	var rr reversal
	err := json.Unmarshal(data, &rr)
	if err == nil {
		*r = Reversal(rr)
	} else {
		// the id is surrounded by "\" characters, so strip them
		r.ID = string(data[1 : len(data)-1])
	}

	return nil
}
<commit_msg>Add balance transaction to reversals<commit_after>package stripe

import "encoding/json"

// ReversalParams is the set of parameters that can be used when reversing a transfer.
type ReversalParams struct {
	Params
	Transfer string
	Amount   uint64
	Fee      bool
}

// ReversalListParams is the set of parameters that can be used when listing reversals.
type ReversalListParams struct {
	ListParams
	Transfer string
}

// Reversal represents a transfer reversal.
type Reversal struct {
	ID       string            `json:"id"`
	Amount   uint64            `json:"amount"`
	Created  int64             `json:"created"`
	Currency Currency          `json:"currency"`
	Transfer string            `json:"transfer"`
	Meta     map[string]string `json:"metadata"`
	Tx       *Transaction      `json:"balance_transaction"`
}

// ReversalList is a list of object for reversals.
type ReversalList struct {
	ListMeta
	Values []*Reversal `json:"data"`
}

// UnmarshalJSON handles deserialization of a Reversal.
// This custom unmarshaling is needed because the resulting
// property may be an id or the full struct if it was expanded.
func (r *Reversal) UnmarshalJSON(data []byte) error {
	type reversal Reversal
	var rr reversal
	err := json.Unmarshal(data, &rr)
	if err == nil {
		*r = Reversal(rr)
	} else {
		// the id is surrounded by "\" characters, so strip them
		r.ID = string(data[1 : len(data)-1])
	}

	return nil
}
<|endoftext|>


<commit_before>package decoders

import (
	"errors"
	"fmt"
	"log"
	"math"
	"regexp"
	"strconv"

	"github.com/seadsystem/Backend/DB/landingzone/constants"
)

type SeadPacket struct {
	Type      byte
	Location  byte
	Timestamp float64
	Period    float64
	Count     int
	Data      float64
	Serial    int
}

var headerRegex *regexp.Regexp
var InvalidHeader = errors.New("Invalid header.")
var InvalidPacket = errors.New("Invalid packet.")
var InvalidTime = errors.New("Invalid time.")

// init sets up stuff we need with proper error handling. If it isn't complicated or doesn't need error handling, it can probably just be assigned directly.
func init() {
	var err error
	headerRegex, err = regexp.Compile(constants.HEADER_REGEX)
	if err != nil {
		log.Panic("Regex compile error:", err)
	}
}

// DecodeHeader verifies that the header is in the correct format and extracts the serial number
func DecodeHeader(packet []byte) (serial int, err error) {
	serialStrings := headerRegex.FindSubmatch(packet)

	if serialStrings == nil || len(serialStrings) != 2 {
		err = InvalidHeader
		return
	}

	log.Printf("Header serial string: %s\n", string(serialStrings[1]))

	serial, err = strconv.Atoi(string(serialStrings[1]))
	return
}

// DecodePacket extracts the data sent from sensor
func DecodePacket(buffer []byte) (packet SeadPacket, err error) {
	for i := 0; i < len(buffer); {
		datatype := buffer[i]
		i++

		// Switch on the type of data sent in the packet
		switch {
		case datatype == 'T':
			// Type
			packet.Type = buffer[i]
			i++
		case datatype == 'l':
			// Location
			packet.Location = buffer[i]
			i++
		case datatype == 't':
			// Timestamp
			packet.Timestamp, err = asciiTimeToDouble(buffer[i : i+14])
			i += 14
		case datatype == 'P':
			// Period separator
			packet.Period, err = asciiTimeToDouble(buffer[i : i+14])
			i += 14
		case datatype == 'C':
			// Count
			packet.Count, err = Binary2int(buffer[i : i+2])
			i += 2
		case datatype == 'D':
			// Data
			// if count isn't set, return error
			// TODO finish parsing data
		case datatype == 'S':
			// Serial
			packet.Serial, err = strconv.Atoi(string(buffer[i : i+6]))
			i += 6
		case datatype == 'X':
			return
		default:
			err = InvalidPacket
		}

		if err != nil {
			return
		}
	}
	err = InvalidPacket
	return
}

func doubleToAsciiTime(double_time float64) string {
	// TODO: Check if this logic is correct or if we need to use http://golang.org/pkg/math/#Mod
	int_time := int(double_time)
	var days = math.Floor(double_time / (60 * 60 * 24))
	var hours = (int_time % (60 * 60 * 24)) / (60 * 60)
	var minutes = (int_time % (60 * 60)) / 60
	var seconds = (int_time % (60)) / 1
	var milliseconds = (int_time * 1000) % 1000
	var clock_time = (int_time * 12000) % 12

	return fmt.Sprintf("%03d%02d%02d%02d%03d%02d", days, hours, minutes, seconds, milliseconds, clock_time)
}

func asciiTimeToDouble(ascii_time []byte) (time float64, err error) {
	// Check time string format
	if len(ascii_time) != 16 {
		err = InvalidTime
	}
	_, err = strconv.Atoi(string(ascii_time))
	if err != nil {
		return
	}

	// Do the conversion now that we know it should work
	var ptr int = 0
	days, err := strconv.Atoi(string(ascii_time[ptr : ptr+3]))
	if err != nil {
		return
	}
	ptr += 3
	time += float64(60 * 60 * 24 * days)
	hours, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(60 * 60 * hours)
	minutes, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(60 * minutes)
	seconds, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(seconds)
	milliseconds, err := strconv.Atoi(string(ascii_time[ptr : ptr+3]))
	if err != nil {
		return
	}
	ptr += 3
	time += float64(milliseconds) / 1000.0
	clock, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(clock) / 12000.0
	return
}

// Every checks if every byte in a slice meets some criteria
func Every(data []byte, check func(byte) bool) bool {
	for _, element := range data {
		if !check(element) {
			return false
		}
	}
	return true
}

// Binary2int converts a byte array containing binary data into an int
func Binary2int(data []byte) (total int) {
	for index, element := range data {
		total += int(element)<<(index * 8)
	}
	return
}
<commit_msg>Fixed type error<commit_after>package decoders

import (
	"errors"
	"fmt"
	"log"
	"math"
	"regexp"
	"strconv"

	"github.com/seadsystem/Backend/DB/landingzone/constants"
)

type SeadPacket struct {
	Type      byte
	Location  byte
	Timestamp float64
	Period    float64
	Count     int
	Data      float64
	Serial    int
}

var headerRegex *regexp.Regexp
var InvalidHeader = errors.New("Invalid header.")
var InvalidPacket = errors.New("Invalid packet.")
var InvalidTime = errors.New("Invalid time.")

// init sets up stuff we need with proper error handling. If it isn't complicated or doesn't need error handling, it can probably just be assigned directly.
func init() {
	var err error
	headerRegex, err = regexp.Compile(constants.HEADER_REGEX)
	if err != nil {
		log.Panic("Regex compile error:", err)
	}
}

// DecodeHeader verifies that the header is in the correct format and extracts the serial number
func DecodeHeader(packet []byte) (serial int, err error) {
	serialStrings := headerRegex.FindSubmatch(packet)

	if serialStrings == nil || len(serialStrings) != 2 {
		err = InvalidHeader
		return
	}

	log.Printf("Header serial string: %s\n", string(serialStrings[1]))

	serial, err = strconv.Atoi(string(serialStrings[1]))
	return
}

// DecodePacket extracts the data sent from sensor
func DecodePacket(buffer []byte) (packet SeadPacket, err error) {
	for i := 0; i < len(buffer); {
		datatype := buffer[i]
		i++

		// Switch on the type of data sent in the packet
		switch {
		case datatype == 'T':
			// Type
			packet.Type = buffer[i]
			i++
		case datatype == 'l':
			// Location
			packet.Location = buffer[i]
			i++
		case datatype == 't':
			// Timestamp
			packet.Timestamp, err = asciiTimeToDouble(buffer[i : i+14])
			i += 14
		case datatype == 'P':
			// Period separator
			packet.Period, err = asciiTimeToDouble(buffer[i : i+14])
			i += 14
		case datatype == 'C':
			// Count
			packet.Count, err = Binary2int(buffer[i : i+2])
			i += 2
		case datatype == 'D':
			// Data
			// if count isn't set, return error
			// TODO finish parsing data
		case datatype == 'S':
			// Serial
			packet.Serial, err = strconv.Atoi(string(buffer[i : i+6]))
			i += 6
		case datatype == 'X':
			return
		default:
			err = InvalidPacket
		}

		if err != nil {
			return
		}
	}
	err = InvalidPacket
	return
}

func doubleToAsciiTime(double_time float64) string {
	// TODO: Check if this logic is correct or if we need to use http://golang.org/pkg/math/#Mod
	int_time := int(double_time)
	var days = math.Floor(double_time / (60 * 60 * 24))
	var hours = (int_time % (60 * 60 * 24)) / (60 * 60)
	var minutes = (int_time % (60 * 60)) / 60
	var seconds = (int_time % (60)) / 1
	var milliseconds = (int_time * 1000) % 1000
	var clock_time = (int_time * 12000) % 12

	return fmt.Sprintf("%03d%02d%02d%02d%03d%02d", days, hours, minutes, seconds, milliseconds, clock_time)
}

func asciiTimeToDouble(ascii_time []byte) (time float64, err error) {
	// Check time string format
	if len(ascii_time) != 16 {
		err = InvalidTime
	}
	_, err = strconv.Atoi(string(ascii_time))
	if err != nil {
		return
	}

	// Do the conversion now that we know it should work
	var ptr int = 0
	days, err := strconv.Atoi(string(ascii_time[ptr : ptr+3]))
	if err != nil {
		return
	}
	ptr += 3
	time += float64(60 * 60 * 24 * days)
	hours, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(60 * 60 * hours)
	minutes, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(60 * minutes)
	seconds, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(seconds)
	milliseconds, err := strconv.Atoi(string(ascii_time[ptr : ptr+3]))
	if err != nil {
		return
	}
	ptr += 3
	time += float64(milliseconds) / 1000.0
	clock, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(clock) / 12000.0
	return
}

// Every checks if every byte in a slice meets some criteria
func Every(data []byte, check func(byte) bool) bool {
	for _, element := range data {
		if !check(element) {
			return false
		}
	}
	return true
}

// Binary2int converts a byte array containing binary data into an int
func Binary2int(data []byte) (total int) {
	for index, element := range data {
		total += int(element)<<uint(index * 8)
	}
	return
}
<|endoftext|>


<commit_before>package notifications

import (
	"encoding/json"
	"fmt"
	"time"
)

type Notification struct {
	ID        int       `json:"id"`
	Data      Data      `json:"notification"`
	Timestamp time.Time `json:"timestamp"`
	Read      bool      `json:"read"`
}

type Data interface {
	// TODO maybe should be made 'real interface', which will allow
	// to use typed channels, type checking and semantic dispatching
	// instead of typecase:

	// Serialize() []byte
	// Describe() (string, string)
}

type notificationWrapper struct {
	Notification Data `json:"notification"`
}

type messageWrapper struct {
	Message Data `json:"message"`
}

type messageReadWrapper struct {
	MessageRead interface{} `json:"messageRead"`
}

type messageTypingWrapper struct {
	MessageRead interface{} `json:"messageTyping"`
}

type orderWrapper struct {
	OrderNotification `json:"order"`
}

type paymentWrapper struct {
	PaymentNotification `json:"payment"`
}

type orderConfirmationWrapper struct {
	OrderConfirmationNotification `json:"orderConfirmation"`
}

type orderCancelWrapper struct {
	OrderCancelNotification `json:"orderConfirmation"`
}

type refundWrapper struct {
	RefundNotification `json:"refund"`
}

type fulfillmentWrapper struct {
	FulfillmentNotification `json:"orderFulfillment"`
}

type completionWrapper struct {
	CompletionNotification `json:"orderCompletion"`
}

type disputeOpenWrapper struct {
	DisputeOpenNotification `json:"disputeOpen"`
}

type disputeUpdateWrapper struct {
	DisputeUpdateNotification `json:"disputeUpdate"`
}

type disputeCloseWrapper struct {
	DisputeCloseNotification `json:"disputeClose"`
}

type OrderNotification struct {
	Title             string `json:"title"`
	BuyerGuid         string `json:"buyerGuid"`
	BuyerBlockchainId string `json:"buyerBlockchainId"`
	Thumbnail         string `json:"thumbnail"`
	Timestamp         int    `json:"timestamp"`
	OrderId           string `json:"orderId"`
}

type PaymentNotification struct {
	OrderId      string `json:"orderId"`
	FundingTotal uint64 `json:"fundingTotal"`
}

type OrderConfirmationNotification struct {
	OrderId string `json:"orderId"`
}

type OrderCancelNotification struct {
	OrderId string `json:"orderId"`
}

type RefundNotification struct {
	OrderId string `json:"orderId"`
}

type FulfillmentNotification struct {
	OrderId string `json:"orderId"`
}

type CompletionNotification struct {
	OrderId string `json:"orderId"`
}

type DisputeOpenNotification struct {
	OrderId string `json:"orderId"`
}

type DisputeUpdateNotification struct {
	OrderId string `json:"orderId"`
}

type DisputeCloseNotification struct {
	OrderId string `json:"orderId"`
}

type FollowNotification struct {
	Follow string `json:"follow"`
}

type UnfollowNotification struct {
	Unfollow string `json:"unfollow"`
}

type StatusNotification struct {
	Status string `json:"status"`
}

type ChatMessage struct {
	MessageId string    `json:"messageId"`
	PeerId    string    `json:"peerId"`
	Subject   string    `json:"subject"`
	Message   string    `json:"message"`
	Timestamp time.Time `json:"timestamp"`
}

type ChatRead struct {
	MessageId string `json:"messageId"`
	PeerId    string `json:"peerId"`
	Subject   string `json:"subject"`
}

type ChatTyping struct {
	PeerId  string `json:"peerId"`
	Subject string `json:"subject"`
}

func Serialize(i interface{}) []byte {
	var n notificationWrapper
	switch i.(type) {
	case OrderNotification:
		n = notificationWrapper{
			orderWrapper{
				OrderNotification: i.(OrderNotification),
			},
		}
	case PaymentNotification:
		n = notificationWrapper{
			paymentWrapper{
				PaymentNotification: i.(PaymentNotification),
			},
		}
	case OrderConfirmationNotification:
		n = notificationWrapper{
			orderConfirmationWrapper{
				OrderConfirmationNotification: i.(OrderConfirmationNotification),
			},
		}
	case OrderCancelNotification:
		n = notificationWrapper{
			orderCancelWrapper{
				OrderCancelNotification: i.(OrderCancelNotification),
			},
		}
	case RefundNotification:
		n = notificationWrapper{
			refundWrapper{
				RefundNotification: i.(RefundNotification),
			},
		}
	case FulfillmentNotification:
		n = notificationWrapper{
			fulfillmentWrapper{
				FulfillmentNotification: i.(FulfillmentNotification),
			},
		}
	case CompletionNotification:
		n = notificationWrapper{
			completionWrapper{
				CompletionNotification: i.(CompletionNotification),
			},
		}
	case DisputeOpenNotification:
		n = notificationWrapper{
			disputeOpenWrapper{
				DisputeOpenNotification: i.(DisputeOpenNotification),
			},
		}
	case DisputeUpdateNotification:
		n = notificationWrapper{
			disputeUpdateWrapper{
				DisputeUpdateNotification: i.(DisputeUpdateNotification),
			},
		}
	case DisputeCloseNotification:
		n = notificationWrapper{
			disputeCloseWrapper{
				DisputeCloseNotification: i.(DisputeCloseNotification),
			},
		}
	case FollowNotification:
		n = notificationWrapper{
			i.(FollowNotification),
		}
	case UnfollowNotification:
		n = notificationWrapper{
			i.(UnfollowNotification),
		}
	case StatusNotification:
		s := i.(StatusNotification)
		b, _ := json.Marshal(s)
		return b
	case ChatMessage:
		m := messageWrapper{
			i.(ChatMessage),
		}
		b, _ := json.MarshalIndent(m, "", "    ")
		return b
	case ChatRead:
		m := messageReadWrapper{
			i.(ChatRead),
		}
		b, _ := json.MarshalIndent(m, "", "    ")
		return b
	case ChatTyping:
		m := messageTypingWrapper{
			i.(ChatTyping),
		}
		b, _ := json.MarshalIndent(m, "", "    ")
		return b
	case []byte:
		return i.([]byte)
	}

	b, _ := json.MarshalIndent(n, "", "    ")
	return b
}

func Describe(i interface{}) (string, string) {
	var head, body string
	switch i.(type) {
	case OrderNotification:
		head = "Order received"

		n := i.(OrderNotification)
		var buyer string
		if n.BuyerBlockchainId != "" {
			buyer = n.BuyerBlockchainId
		} else {
			buyer = n.BuyerGuid
		}
		form := "You received an order \"%s\".\n\nOrder ID: %s\nBuyer: %s\nThumbnail: %s\nTimestamp: %d"
		body = fmt.Sprintf(form, n.Title, n.OrderId, buyer, n.Thumbnail, n.Timestamp)

	case PaymentNotification:
		head = "Payment received"

		n := i.(PaymentNotification)
		form := "Payment for order \"%s\" received (total %d)."
		body = fmt.Sprintf(form, n.OrderId, n.FundingTotal)

	case OrderConfirmationNotification:
		head = "Order confirmed"

		n := i.(OrderConfirmationNotification)
		form := "Order \"%s\" has been confirmed."
		body = fmt.Sprintf(form, n.OrderId)

	case OrderCancelNotification:
		head = "Order cancelled"

		n := i.(OrderCancelNotification)
		form := "Order \"%s\" has been cancelled."
		body = fmt.Sprintf(form, n.OrderId)

	case RefundNotification:
		head = "Payment refunded"

		n := i.(RefundNotification)
		form := "Payment refund for order \"%s\" received."
		body = fmt.Sprintf(form, n.OrderId)

	case FulfillmentNotification:
		head = "Order fulfilled"

		n := i.(FulfillmentNotification)
		form := "Order \"%s\" was marked as fulfilled."
		body = fmt.Sprintf(form, n.OrderId)

	case CompletionNotification:
		head = "Order completed"

		n := i.(CompletionNotification)
		form := "Order \"%s\" was marked as completed."
		body = fmt.Sprintf(form, n.OrderId)

	case DisputeOpenNotification:
		head = "Dispute opened"

		n := i.(DisputeOpenNotification)
		form := "Dispute around order \"%s\" was opened."
		body = fmt.Sprintf(form, n.OrderId)

	case DisputeUpdateNotification:
		head = "Dispute updated"

		n := i.(DisputeUpdateNotification)
		form := "Dispute around order \"%s\" was updated."
		body = fmt.Sprintf(form, n.OrderId)

	case DisputeCloseNotification:
		head = "Dispute closed"

		n := i.(DisputeCloseNotification)
		form := "Dispute around order \"%s\" was closed."
		body = fmt.Sprintf(form, n.OrderId)
	}
	return head, body
}
<commit_msg>Create moderator add and remove notifications<commit_after>package notifications

import (
	"encoding/json"
	"fmt"
	"time"
)

type Notification struct {
	ID        int       `json:"id"`
	Data      Data      `json:"notification"`
	Timestamp time.Time `json:"timestamp"`
	Read      bool      `json:"read"`
}

type Data interface {
	// TODO maybe should be made 'real interface', which will allow
	// to use typed channels, type checking and semantic dispatching
	// instead of typecase:

	// Serialize() []byte
	// Describe() (string, string)
}

type notificationWrapper struct {
	Notification Data `json:"notification"`
}

type messageWrapper struct {
	Message Data `json:"message"`
}

type messageReadWrapper struct {
	MessageRead interface{} `json:"messageRead"`
}

type messageTypingWrapper struct {
	MessageRead interface{} `json:"messageTyping"`
}

type orderWrapper struct {
	OrderNotification `json:"order"`
}

type paymentWrapper struct {
	PaymentNotification `json:"payment"`
}

type orderConfirmationWrapper struct {
	OrderConfirmationNotification `json:"orderConfirmation"`
}

type orderCancelWrapper struct {
	OrderCancelNotification `json:"orderConfirmation"`
}

type refundWrapper struct {
	RefundNotification `json:"refund"`
}

type fulfillmentWrapper struct {
	FulfillmentNotification `json:"orderFulfillment"`
}

type completionWrapper struct {
	CompletionNotification `json:"orderCompletion"`
}

type disputeOpenWrapper struct {
	DisputeOpenNotification `json:"disputeOpen"`
}

type disputeUpdateWrapper struct {
	DisputeUpdateNotification `json:"disputeUpdate"`
}

type disputeCloseWrapper struct {
	DisputeCloseNotification `json:"disputeClose"`
}

type OrderNotification struct {
	Title             string `json:"title"`
	BuyerGuid         string `json:"buyerGuid"`
	BuyerBlockchainId string `json:"buyerBlockchainId"`
	Thumbnail         string `json:"thumbnail"`
	Timestamp         int    `json:"timestamp"`
	OrderId           string `json:"orderId"`
}

type PaymentNotification struct {
	OrderId      string `json:"orderId"`
	FundingTotal uint64 `json:"fundingTotal"`
}

type OrderConfirmationNotification struct {
	OrderId string `json:"orderId"`
}

type OrderCancelNotification struct {
	OrderId string `json:"orderId"`
}

type RefundNotification struct {
	OrderId string `json:"orderId"`
}

type FulfillmentNotification struct {
	OrderId string `json:"orderId"`
}

type CompletionNotification struct {
	OrderId string `json:"orderId"`
}

type DisputeOpenNotification struct {
	OrderId string `json:"orderId"`
}

type DisputeUpdateNotification struct {
	OrderId string `json:"orderId"`
}

type DisputeCloseNotification struct {
	OrderId string `json:"orderId"`
}

type FollowNotification struct {
	Follow string `json:"follow"`
}

type UnfollowNotification struct {
	Unfollow string `json:"unfollow"`
}

type ModeratorAddNotification struct {
	ModeratorAdd string `json:"moderatorAdd"`
}

type ModeratorRemoveNotification struct {
	ModeratorRemove string `json:"moderatorRemove"`
}

type StatusNotification struct {
	Status string `json:"status"`
}

type ChatMessage struct {
	MessageId string    `json:"messageId"`
	PeerId    string    `json:"peerId"`
	Subject   string    `json:"subject"`
	Message   string    `json:"message"`
	Timestamp time.Time `json:"timestamp"`
}

type ChatRead struct {
	MessageId string `json:"messageId"`
	PeerId    string `json:"peerId"`
	Subject   string `json:"subject"`
}

type ChatTyping struct {
	PeerId  string `json:"peerId"`
	Subject string `json:"subject"`
}

func Serialize(i interface{}) []byte {
	var n notificationWrapper
	switch i.(type) {
	case OrderNotification:
		n = notificationWrapper{
			orderWrapper{
				OrderNotification: i.(OrderNotification),
			},
		}
	case PaymentNotification:
		n = notificationWrapper{
			paymentWrapper{
				PaymentNotification: i.(PaymentNotification),
			},
		}
	case OrderConfirmationNotification:
		n = notificationWrapper{
			orderConfirmationWrapper{
				OrderConfirmationNotification: i.(OrderConfirmationNotification),
			},
		}
	case OrderCancelNotification:
		n = notificationWrapper{
			orderCancelWrapper{
				OrderCancelNotification: i.(OrderCancelNotification),
			},
		}
	case RefundNotification:
		n = notificationWrapper{
			refundWrapper{
				RefundNotification: i.(RefundNotification),
			},
		}
	case FulfillmentNotification:
		n = notificationWrapper{
			fulfillmentWrapper{
				FulfillmentNotification: i.(FulfillmentNotification),
			},
		}
	case CompletionNotification:
		n = notificationWrapper{
			completionWrapper{
				CompletionNotification: i.(CompletionNotification),
			},
		}
	case DisputeOpenNotification:
		n = notificationWrapper{
			disputeOpenWrapper{
				DisputeOpenNotification: i.(DisputeOpenNotification),
			},
		}
	case DisputeUpdateNotification:
		n = notificationWrapper{
			disputeUpdateWrapper{
				DisputeUpdateNotification: i.(DisputeUpdateNotification),
			},
		}
	case DisputeCloseNotification:
		n = notificationWrapper{
			disputeCloseWrapper{
				DisputeCloseNotification: i.(DisputeCloseNotification),
			},
		}
	case FollowNotification:
		n = notificationWrapper{
			i.(FollowNotification),
		}
	case UnfollowNotification:
		n = notificationWrapper{
			i.(UnfollowNotification),
		}
	case ModeratorAddNotification:
		n = notificationWrapper{
			i.(ModeratorAddNotification),
		}
	case ModeratorRemoveNotification:
		n = notificationWrapper{
			i.(ModeratorRemoveNotification),
		}
	case StatusNotification:
		s := i.(StatusNotification)
		b, _ := json.Marshal(s)
		return b
	case ChatMessage:
		m := messageWrapper{
			i.(ChatMessage),
		}
		b, _ := json.MarshalIndent(m, "", "    ")
		return b
	case ChatRead:
		m := messageReadWrapper{
			i.(ChatRead),
		}
		b, _ := json.MarshalIndent(m, "", "    ")
		return b
	case ChatTyping:
		m := messageTypingWrapper{
			i.(ChatTyping),
		}
		b, _ := json.MarshalIndent(m, "", "    ")
		return b
	case []byte:
		return i.([]byte)
	}

	b, _ := json.MarshalIndent(n, "", "    ")
	return b
}

func Describe(i interface{}) (string, string) {
	var head, body string
	switch i.(type) {
	case OrderNotification:
		head = "Order received"

		n := i.(OrderNotification)
		var buyer string
		if n.BuyerBlockchainId != "" {
			buyer = n.BuyerBlockchainId
		} else {
			buyer = n.BuyerGuid
		}
		form := "You received an order \"%s\".\n\nOrder ID: %s\nBuyer: %s\nThumbnail: %s\nTimestamp: %d"
		body = fmt.Sprintf(form, n.Title, n.OrderId, buyer, n.Thumbnail, n.Timestamp)

	case PaymentNotification:
		head = "Payment received"

		n := i.(PaymentNotification)
		form := "Payment for order \"%s\" received (total %d)."
		body = fmt.Sprintf(form, n.OrderId, n.FundingTotal)

	case OrderConfirmationNotification:
		head = "Order confirmed"

		n := i.(OrderConfirmationNotification)
		form := "Order \"%s\" has been confirmed."
		body = fmt.Sprintf(form, n.OrderId)

	case OrderCancelNotification:
		head = "Order cancelled"

		n := i.(OrderCancelNotification)
		form := "Order \"%s\" has been cancelled."
		body = fmt.Sprintf(form, n.OrderId)

	case RefundNotification:
		head = "Payment refunded"

		n := i.(RefundNotification)
		form := "Payment refund for order \"%s\" received."
		body = fmt.Sprintf(form, n.OrderId)

	case FulfillmentNotification:
		head = "Order fulfilled"

		n := i.(FulfillmentNotification)
		form := "Order \"%s\" was marked as fulfilled."
		body = fmt.Sprintf(form, n.OrderId)

	case CompletionNotification:
		head = "Order completed"

		n := i.(CompletionNotification)
		form := "Order \"%s\" was marked as completed."
		body = fmt.Sprintf(form, n.OrderId)

	case DisputeOpenNotification:
		head = "Dispute opened"

		n := i.(DisputeOpenNotification)
		form := "Dispute around order \"%s\" was opened."
		body = fmt.Sprintf(form, n.OrderId)

	case DisputeUpdateNotification:
		head = "Dispute updated"

		n := i.(DisputeUpdateNotification)
		form := "Dispute around order \"%s\" was updated."
		body = fmt.Sprintf(form, n.OrderId)

	case DisputeCloseNotification:
		head = "Dispute closed"

		n := i.(DisputeCloseNotification)
		form := "Dispute around order \"%s\" was closed."
		body = fmt.Sprintf(form, n.OrderId)
	}
	return head, body
}
<|endoftext|>


<commit_before>package aws

import (
	"bytes"
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/neptune"
)

func resourceAwsNeptuneParameterGroup() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsNeptuneParameterGroupCreate,
		Read:   resourceAwsNeptuneParameterGroupRead,
		Update: resourceAwsNeptuneParameterGroupUpdate,
		Delete: resourceAwsNeptuneParameterGroupDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},
		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				ForceNew: true,
				Required: true,
				StateFunc: func(val interface{}) string {
					return strings.ToLower(val.(string))
				},
			},
			"family": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"description": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Default:  "Managed by Terraform",
			},
			"parameter": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"name": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
						},
						"value": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
						},
						"apply_method": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
							Default:  "immediate",
							// this parameter is not actually state, but a
							// meta-parameter describing how the RDS API call
							// to modify the parameter group should be made.
							// Future reads of the resource from AWS don't tell
							// us what we used for apply_method previously, so
							// by squashing state to an empty string we avoid
							// needing to do an update for every future run.
							StateFunc: func(interface{}) string { return "" },
						},
					},
				},
				Set: resourceAwsNeptuneParameterHash,
			},
		},
	}
}

func resourceAwsNeptuneParameterGroupCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).neptuneconn

	createOpts := neptune.CreateDBParameterGroupInput{
		DBParameterGroupName:   aws.String(d.Get("name").(string)),
		DBParameterGroupFamily: aws.String(d.Get("family").(string)),
		Description:            aws.String(d.Get("description").(string)),
	}

	log.Printf("[DEBUG] Create Neptune Parameter Group: %#v", createOpts)
	resp, err := conn.CreateDBParameterGroup(&createOpts)
	if err != nil {
		return fmt.Errorf("Error creating Neptune Parameter Group: %s", err)
	}

	d.Partial(true)
	d.SetPartial("name")
	d.SetPartial("family")
	d.SetPartial("description")
	d.Partial(false)

	d.SetId(*resp.DBParameterGroup.DBParameterGroupName)
	log.Printf("[INFO] Neptune Parameter Group ID: %s", d.Id())

	return resourceAwsNeptuneParameterGroupUpdate(d, meta)
}

func resourceAwsNeptuneParameterGroupRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).neptuneconn

	describeOpts := neptune.DescribeDBParameterGroupsInput{
		DBParameterGroupName: aws.String(d.Id()),
	}

	describeResp, err := conn.DescribeDBParameterGroups(&describeOpts)
	if err != nil {
		return err
	}

	if len(describeResp.DBParameterGroups) != 1 ||
		*describeResp.DBParameterGroups[0].DBParameterGroupName != d.Id() {
		return fmt.Errorf("Unable to find Parameter Group: %#v", describeResp.DBParameterGroups)
	}

	d.Set("name", describeResp.DBParameterGroups[0].DBParameterGroupName)
	d.Set("family", describeResp.DBParameterGroups[0].DBParameterGroupFamily)
	d.Set("description", describeResp.DBParameterGroups[0].Description)

	// Only include user customized parameters as there's hundreds of system/default ones
	describeParametersOpts := neptune.DescribeDBParametersInput{
		DBParameterGroupName: aws.String(d.Id()),
		Source:               aws.String("user"),
	}

	describeParametersResp, err := conn.DescribeDBParameters(&describeParametersOpts)
	if err != nil {
		return err
	}

	d.Set("parameter", flattenNeptuneParameters(describeParametersResp.Parameters))

	return nil
}

func resourceAwsNeptuneParameterGroupUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).neptuneconn

	d.Partial(true)

	if d.HasChange("parameter") {
		o, n := d.GetChange("parameter")
		if o == nil {
			o = new(schema.Set)
		}
		if n == nil {
			n = new(schema.Set)
		}

		os := o.(*schema.Set)
		ns := n.(*schema.Set)

		toRemove, err := expandNeptuneParameters(os.Difference(ns).List())
		if err != nil {
			return err
		}

		log.Printf("[DEBUG] Parameters to remove: %#v", toRemove)

		toAdd, err := expandNeptuneParameters(ns.Difference(os).List())
		if err != nil {
			return err
		}

		log.Printf("[DEBUG] Parameters to add: %#v", toAdd)

		// We can only modify 20 parameters at a time, so walk them until
		// we've got them all.
		maxParams := 20

		for len(toRemove) > 0 {
			paramsToModify := make([]*neptune.Parameter, 0)
			if len(toRemove) <= maxParams {
				paramsToModify, toRemove = toRemove[:], nil
			} else {
				paramsToModify, toRemove = toRemove[:maxParams], toRemove[maxParams:]
			}
			resetOpts := neptune.ResetDBParameterGroupInput{
				DBParameterGroupName: aws.String(d.Get("name").(string)),
				Parameters:           paramsToModify,
			}

			log.Printf("[DEBUG] Reset Neptune Parameter Group: %s", resetOpts)
			err := resource.Retry(30*time.Second, func() *resource.RetryError {
				_, err = conn.ResetDBParameterGroup(&resetOpts)
				if err != nil {
					if isAWSErr(err, "InvalidDBParameterGroupState", " has pending changes") {
						return resource.RetryableError(err)
					}
					return resource.NonRetryableError(err)
				}
				return nil
			})
			if err != nil {
				return fmt.Errorf("Error resetting Neptune Parameter Group: %s", err)
			}
		}

		for len(toAdd) > 0 {
			paramsToModify := make([]*neptune.Parameter, 0)
			if len(toAdd) <= maxParams {
				paramsToModify, toAdd = toAdd[:], nil
			} else {
				paramsToModify, toAdd = toAdd[:maxParams], toAdd[maxParams:]
			}
			modifyOpts := neptune.ModifyDBParameterGroupInput{
				DBParameterGroupName: aws.String(d.Get("name").(string)),
				Parameters:           paramsToModify,
			}

			log.Printf("[DEBUG] Modify Neptune Parameter Group: %s", modifyOpts)
			_, err = conn.ModifyDBParameterGroup(&modifyOpts)
			if err != nil {
				return fmt.Errorf("Error modifying Neptune Parameter Group: %s", err)
			}
		}

		d.SetPartial("parameter")
	}

	d.Partial(false)

	return resourceAwsNeptuneParameterGroupRead(d, meta)
}

func resourceAwsNeptuneParameterGroupDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).neptuneconn

	return resource.Retry(3*time.Minute, func() *resource.RetryError {
		deleteOpts := neptune.DeleteDBParameterGroupInput{
			DBParameterGroupName: aws.String(d.Id()),
		}
		_, err := conn.DeleteDBParameterGroup(&deleteOpts)
		if err != nil {
			awsErr, ok := err.(awserr.Error)
			if ok && awsErr.Code() == "DBParameterGroupNotFound" {
				return nil
			}
			if ok && awsErr.Code() == "InvalidDBParameterGroupState" {
				return resource.RetryableError(err)
			}
			return resource.NonRetryableError(err)
		}
		return nil
	})
}

func resourceAwsNeptuneParameterHash(v interface{}) int {
	var buf bytes.Buffer
	m := v.(map[string]interface{})
	buf.WriteString(fmt.Sprintf("%s-", m["name"].(string)))
	// Store the value as a lower case string, to match how we store them in flattenParameters
	buf.WriteString(fmt.Sprintf("%s-", strings.ToLower(m["value"].(string))))

	return hashcode.String(buf.String())
}
<commit_msg>cleaning up schema declarations<commit_after>package aws

import (
	"bytes"
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/neptune"
)

func resourceAwsNeptuneParameterGroup() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsNeptuneParameterGroupCreate,
		Read:   resourceAwsNeptuneParameterGroupRead,
		Update: resourceAwsNeptuneParameterGroupUpdate,
		Delete: resourceAwsNeptuneParameterGroupDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},
		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				ForceNew: true,
				Required: true,
				StateFunc: func(val interface{}) string {
					return strings.ToLower(val.(string))
				},
			},
			"family": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"description": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Default:  "Managed by Terraform",
			},
			"parameter": {
				Type:     schema.TypeSet,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"name": {
							Type:     schema.TypeString,
							Required: true,
						},
						"value": {
							Type:     schema.TypeString,
							Required: true,
						},
						"apply_method": {
							Type:     schema.TypeString,
							Optional: true,
							Default:  "immediate",
							// this parameter is not actually state, but a
							// meta-parameter describing how the RDS API call
							// to modify the parameter group should be made.
							// Future reads of the resource from AWS don't tell
							// us what we used for apply_method previously, so
							// by squashing state to an empty string we avoid
							// needing to do an update for every future run.
							StateFunc: func(interface{}) string { return "" },
						},
					},
				},
				Set: resourceAwsNeptuneParameterHash,
			},
		},
	}
}

func resourceAwsNeptuneParameterGroupCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).neptuneconn

	createOpts := neptune.CreateDBParameterGroupInput{
		DBParameterGroupName:   aws.String(d.Get("name").(string)),
		DBParameterGroupFamily: aws.String(d.Get("family").(string)),
		Description:            aws.String(d.Get("description").(string)),
	}

	log.Printf("[DEBUG] Create Neptune Parameter Group: %#v", createOpts)
	resp, err := conn.CreateDBParameterGroup(&createOpts)
	if err != nil {
		return fmt.Errorf("Error creating Neptune Parameter Group: %s", err)
	}

	d.Partial(true)
	d.SetPartial("name")
	d.SetPartial("family")
	d.SetPartial("description")
	d.Partial(false)

	d.SetId(*resp.DBParameterGroup.DBParameterGroupName)
	log.Printf("[INFO] Neptune Parameter Group ID: %s", d.Id())

	return resourceAwsNeptuneParameterGroupUpdate(d, meta)
}

func resourceAwsNeptuneParameterGroupRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).neptuneconn

	describeOpts := neptune.DescribeDBParameterGroupsInput{
		DBParameterGroupName: aws.String(d.Id()),
	}

	describeResp, err := conn.DescribeDBParameterGroups(&describeOpts)
	if err != nil {
		return err
	}

	if len(describeResp.DBParameterGroups) != 1 ||
		*describeResp.DBParameterGroups[0].DBParameterGroupName != d.Id() {
		return fmt.Errorf("Unable to find Parameter Group: %#v", describeResp.DBParameterGroups)
	}

	d.Set("name", describeResp.DBParameterGroups[0].DBParameterGroupName)
	d.Set("family", describeResp.DBParameterGroups[0].DBParameterGroupFamily)
	d.Set("description", describeResp.DBParameterGroups[0].Description)

	// Only include user customized parameters as there's hundreds of system/default ones
	describeParametersOpts := neptune.DescribeDBParametersInput{
		DBParameterGroupName: aws.String(d.Id()),
		Source:               aws.String("user"),
	}

	describeParametersResp, err := conn.DescribeDBParameters(&describeParametersOpts)
	if err != nil {
		return err
	}

	d.Set("parameter", flattenNeptuneParameters(describeParametersResp.Parameters))

	return nil
}

func resourceAwsNeptuneParameterGroupUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).neptuneconn

	d.Partial(true)

	if d.HasChange("parameter") {
		o, n := d.GetChange("parameter")
		if o == nil {
			o = new(schema.Set)
		}
		if n == nil {
			n = new(schema.Set)
		}

		os := o.(*schema.Set)
		ns := n.(*schema.Set)

		toRemove, err := expandNeptuneParameters(os.Difference(ns).List())
		if err != nil {
			return err
		}

		log.Printf("[DEBUG] Parameters to remove: %#v", toRemove)

		toAdd, err := expandNeptuneParameters(ns.Difference(os).List())
		if err != nil {
			return err
		}

		log.Printf("[DEBUG] Parameters to add: %#v", toAdd)

		// We can only modify 20 parameters at a time, so walk them until
		// we've got them all.
		maxParams := 20

		for len(toRemove) > 0 {
			paramsToModify := make([]*neptune.Parameter, 0)
			if len(toRemove) <= maxParams {
				paramsToModify, toRemove = toRemove[:], nil
			} else {
				paramsToModify, toRemove = toRemove[:maxParams], toRemove[maxParams:]
			}
			resetOpts := neptune.ResetDBParameterGroupInput{
				DBParameterGroupName: aws.String(d.Get("name").(string)),
				Parameters:           paramsToModify,
			}

			log.Printf("[DEBUG] Reset Neptune Parameter Group: %s", resetOpts)
			err := resource.Retry(30*time.Second, func() *resource.RetryError {
				_, err = conn.ResetDBParameterGroup(&resetOpts)
				if err != nil {
					if isAWSErr(err, "InvalidDBParameterGroupState", " has pending changes") {
						return resource.RetryableError(err)
					}
					return resource.NonRetryableError(err)
				}
				return nil
			})
			if err != nil {
				return fmt.Errorf("Error resetting Neptune Parameter Group: %s", err)
			}
		}

		for len(toAdd) > 0 {
			paramsToModify := make([]*neptune.Parameter, 0)
			if len(toAdd) <= maxParams {
				paramsToModify, toAdd = toAdd[:], nil
			} else {
				paramsToModify, toAdd = toAdd[:maxParams], toAdd[maxParams:]
			}
			modifyOpts := neptune.ModifyDBParameterGroupInput{
				DBParameterGroupName: aws.String(d.Get("name").(string)),
				Parameters:           paramsToModify,
			}

			log.Printf("[DEBUG] Modify Neptune Parameter Group: %s", modifyOpts)
			_, err = conn.ModifyDBParameterGroup(&modifyOpts)
			if err != nil {
				return fmt.Errorf("Error modifying Neptune Parameter Group: %s", err)
			}
		}

		d.SetPartial("parameter")
	}

	d.Partial(false)

	return resourceAwsNeptuneParameterGroupRead(d, meta)
}

func resourceAwsNeptuneParameterGroupDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).neptuneconn

	return resource.Retry(3*time.Minute, func() *resource.RetryError {
		deleteOpts := neptune.DeleteDBParameterGroupInput{
			DBParameterGroupName: aws.String(d.Id()),
		}
		_, err := conn.DeleteDBParameterGroup(&deleteOpts)
		if err != nil {
			awsErr, ok := err.(awserr.Error)
			if ok && awsErr.Code() == "DBParameterGroupNotFound" {
				return nil
			}
			if ok && awsErr.Code() == "InvalidDBParameterGroupState" {
				return resource.RetryableError(err)
			}
			return resource.NonRetryableError(err)
		}
		return nil
	})
}

func resourceAwsNeptuneParameterHash(v interface{}) int {
	var buf bytes.Buffer
	m := v.(map[string]interface{})
	buf.WriteString(fmt.Sprintf("%s-", m["name"].(string)))
	// Store the value as a lower case string, to match how we store them in flattenParameters
	buf.WriteString(fmt.Sprintf("%s-", strings.ToLower(m["value"].(string))))

	return hashcode.String(buf.String())
}
<|endoftext|>


<commit_before>package message

import (
	"../queue"
	"../util"
	"errors"
	"log"
	"time"
)

type Channel struct {
	name                string
	addClientChan       chan util.ChanReq
	removeClientChan    chan util.ChanReq
	backend             queue.BackendQueue
	incomingMessageChan chan *Message
	msgChan             chan *Message
	inFlightMessageChan chan *Message
	ackMessageChan      chan util.ChanReq
	requeueMessageChan  chan util.ChanReq
	finishMessageChan   chan util.ChanReq
	exitChan            chan int
	inFlightMessages    map[string]*Message
}

// Channel constructor
func NewChannel(channelName string, inMemSize int) *Channel {
	channel := &Channel{name: channelName,
		addClientChan:       make(chan util.ChanReq),
		removeClientChan:    make(chan util.ChanReq),
		backend:             queue.NewDiskQueue(channelName),
		incomingMessageChan: make(chan *Message, 5),
		msgChan:             make(chan *Message, inMemSize),
		inFlightMessageChan: make(chan *Message),
		ackMessageChan:      make(chan util.ChanReq),
		requeueMessageChan:  make(chan util.ChanReq),
		finishMessageChan:   make(chan util.ChanReq),
		exitChan:            make(chan int),
		inFlightMessages:    make(map[string]*Message)}
	go channel.Router()
	return channel
}

// PutMessage writes to the appropriate incoming
// message channel
func (c *Channel) PutMessage(msg *Message) {
	c.incomingMessageChan <- msg
}

func (c *Channel) AckMessage(uuidStr string) error {
	errChan := make(chan interface{})
	c.ackMessageChan <- util.ChanReq{uuidStr, errChan}
	return (<-errChan).(error)
}

func (c *Channel) FinishMessage(uuidStr string) error {
	errChan := make(chan interface{})
	c.finishMessageChan <- util.ChanReq{uuidStr, errChan}
	return (<-errChan).(error)
}

func (c *Channel) RequeueMessage(uuidStr string) error {
	errChan := make(chan interface{})
	c.requeueMessageChan <- util.ChanReq{uuidStr, errChan}
	return (<-errChan).(error)
}

// Router handles the muxing of Channel messages including
// the addition of a Client to the Channel
func (c *Channel) Router() {
	helperCloseChan := make(chan int)

	go func() {
		for {
			select {
			case msg := <-c.inFlightMessageChan:
				c.pushInFlightMessage(msg)
				go func(msg *Message) {
					select {
					case <-time.After(60 * time.Second):
						log.Printf("CHANNEL(%s): auto requeue of message(%s)", c.name, util.UuidToStr(msg.Uuid()))
					case <-msg.timerChan:
						return
					}
					err := c.RequeueMessage(util.UuidToStr(msg.Uuid()))
					if err != nil {
						log.Printf("ERROR: channel(%s) - %s", c.name, err.Error())
					}
				}(msg)
			case requeueReq := <-c.requeueMessageChan:
				uuidStr := requeueReq.Variable.(string)
				msg, err := c.popInFlightMessage(uuidStr)
				if err != nil {
					log.Printf("ERROR: failed to requeue message(%s) - %s", uuidStr, err.Error())
					continue
				} else {
					go func(msg *Message) {
						c.PutMessage(msg)
					}(msg)
				}
				requeueReq.RetChan <- err
			case finishReq := <-c.finishMessageChan:
				uuidStr := finishReq.Variable.(string)
				_, err := c.popInFlightMessage(uuidStr)
				if err != nil {
					log.Printf("ERROR: failed to finish message(%s) - %s", uuidStr, err.Error())
				}
				finishReq.RetChan <- err
			case ackReq := <-c.ackMessageChan:
				// uuidStr := ackReq.Variable.(string)
				// TODO: do something
				ackReq.RetChan <- nil
			case <-helperCloseChan:
				return
			}
		}
	}()

	for {
		select {
		case msg := <-c.incomingMessageChan:
			select {
			case c.msgChan <- msg:
				log.Printf("CHANNEL(%s): wrote to msgChan", c.name)
			default:
				err := c.backend.Put(msg.Data)
				if err != nil {
					log.Printf("ERROR: t.backend.Put() - %s", err.Error())
					// TODO: requeue?
				}
				log.Printf("CHANNEL(%s): wrote to backend", c.name)
			}
		case <-c.exitChan:
			helperCloseChan <- 1
			return
		}
	}
}

func (c *Channel) pushInFlightMessage(msg *Message) {
	uuidStr := util.UuidToStr(msg.Uuid())
	c.inFlightMessages[uuidStr] = msg
}

func (c *Channel) popInFlightMessage(uuidStr string) (*Message, error) {
	msg, ok := c.inFlightMessages[uuidStr]
	if !ok {
		return nil, errors.New("UUID not in flight")
	}
	delete(c.inFlightMessages, uuidStr)
	msg.EndTimer()
	return msg, nil
}

// GetMessage pulls a single message off the client channel
func (c *Channel) GetMessage(block bool) *Message {
	var msg *Message

	for {
		if block {
			select {
			case msg = <-c.msgChan:
			case <-c.backend.ReadReadyChan():
				buf, err := c.backend.Get()
				if err != nil {
					log.Printf("ERROR: c.backend.Get() - %s", err.Error())
					continue
				}
				msg = NewMessage(buf)
			}
		} else {
			select {
			case msg = <-c.msgChan:
			case <-c.backend.ReadReadyChan():
				buf, err := c.backend.Get()
				if err != nil {
					log.Printf("ERROR: c.backend.Get() - %s", err.Error())
					continue
				}
				msg = NewMessage(buf)
			default:
				msg = nil
			}
		}

		if msg != nil {
			c.inFlightMessageChan <- msg
		}

		break
	}

	return msg
}

func (c *Channel) Close() error {
	var err error

	log.Printf("CHANNEL(%s): closing", c.name)

	c.exitChan <- 1

	err = c.backend.Close()
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>throw away type assertion failures<commit_after>package message

import (
	"../queue"
	"../util"
	"errors"
	"log"
	"time"
)

type Channel struct {
	name                string
	addClientChan       chan util.ChanReq
	removeClientChan    chan util.ChanReq
	backend             queue.BackendQueue
	incomingMessageChan chan *Message
	msgChan             chan *Message
	inFlightMessageChan chan *Message
	ackMessageChan      chan util.ChanReq
	requeueMessageChan  chan util.ChanReq
	finishMessageChan   chan util.ChanReq
	exitChan            chan int
	inFlightMessages    map[string]*Message
}

// Channel constructor
func NewChannel(channelName string, inMemSize int) *Channel {
	channel := &Channel{name: channelName,
		addClientChan:       make(chan util.ChanReq),
		removeClientChan:    make(chan util.ChanReq),
		backend:             queue.NewDiskQueue(channelName),
		incomingMessageChan: make(chan *Message, 5),
		msgChan:             make(chan *Message, inMemSize),
		inFlightMessageChan: make(chan *Message),
		ackMessageChan:      make(chan util.ChanReq),
		requeueMessageChan:  make(chan util.ChanReq),
		finishMessageChan:   make(chan util.ChanReq),
		exitChan:            make(chan int),
		inFlightMessages:    make(map[string]*Message)}
	go channel.Router()
	return channel
}

// PutMessage writes to the appropriate incoming
// message channel
func (c *Channel) PutMessage(msg *Message) {
	c.incomingMessageChan <- msg
}

func (c *Channel) AckMessage(uuidStr string) error {
	errChan := make(chan interface{})
	c.ackMessageChan <- util.ChanReq{uuidStr, errChan}
	err, _ := (<-errChan).(error)
	return err
}

func (c *Channel) FinishMessage(uuidStr string) error {
	errChan := make(chan interface{})
	c.finishMessageChan <- util.ChanReq{uuidStr, errChan}
	err, _ := (<-errChan).(error)
	return err
}

func (c *Channel) RequeueMessage(uuidStr string) error {
	errChan := make(chan interface{})
	c.requeueMessageChan <- util.ChanReq{uuidStr, errChan}
	err, _ := (<-errChan).(error)
	return err
}

// Router handles the muxing of Channel messages including
// the addition of a Client to the Channel
func (c *Channel) Router() {
	helperCloseChan := make(chan int)

	go func() {
		for {
			select {
			case msg := <-c.inFlightMessageChan:
				c.pushInFlightMessage(msg)
				go func(msg *Message) {
					select {
					case <-time.After(60 * time.Second):
						log.Printf("CHANNEL(%s): auto requeue of message(%s)", c.name, util.UuidToStr(msg.Uuid()))
					case <-msg.timerChan:
						return
					}
					err := c.RequeueMessage(util.UuidToStr(msg.Uuid()))
					if err != nil {
						log.Printf("ERROR: channel(%s) - %s", c.name, err.Error())
					}
				}(msg)
			case requeueReq := <-c.requeueMessageChan:
				uuidStr := requeueReq.Variable.(string)
				msg, err := c.popInFlightMessage(uuidStr)
				if err != nil {
					log.Printf("ERROR: failed to requeue message(%s) - %s", uuidStr, err.Error())
					continue
				} else {
					go func(msg *Message) {
						c.PutMessage(msg)
					}(msg)
				}
				requeueReq.RetChan <- err
			case finishReq := <-c.finishMessageChan:
				uuidStr := finishReq.Variable.(string)
				_, err := c.popInFlightMessage(uuidStr)
				if err != nil {
					log.Printf("ERROR: failed to finish message(%s) - %s", uuidStr, err.Error())
				}
				finishReq.RetChan <- err
			case ackReq := <-c.ackMessageChan:
				// uuidStr := ackReq.Variable.(string)
				// TODO: acks are wierd in the sense that they're not technically necessary...
				// it's possible we don't need them at all...
				ackReq.RetChan <- nil
			case <-helperCloseChan:
				return
			}
		}
	}()

	for {
		select {
		case msg := <-c.incomingMessageChan:
			select {
			case c.msgChan <- msg:
				log.Printf("CHANNEL(%s): wrote to msgChan", c.name)
			default:
				err := c.backend.Put(msg.Data)
				if err != nil {
					log.Printf("ERROR: t.backend.Put() - %s", err.Error())
					// TODO: requeue?
				}
				log.Printf("CHANNEL(%s): wrote to backend", c.name)
			}
		case <-c.exitChan:
			helperCloseChan <- 1
			return
		}
	}
}

func (c *Channel) pushInFlightMessage(msg *Message) {
	uuidStr := util.UuidToStr(msg.Uuid())
	c.inFlightMessages[uuidStr] = msg
}

func (c *Channel) popInFlightMessage(uuidStr string) (*Message, error) {
	msg, ok := c.inFlightMessages[uuidStr]
	if !ok {
		return nil, errors.New("UUID not in flight")
	}
	delete(c.inFlightMessages, uuidStr)
	msg.EndTimer()
	return msg, nil
}

// GetMessage pulls a single message off the client channel
func (c *Channel) GetMessage(block bool) *Message {
	var msg *Message

	for {
		if block {
			select {
			case msg = <-c.msgChan:
			case <-c.backend.ReadReadyChan():
				buf, err := c.backend.Get()
				if err != nil {
					log.Printf("ERROR: c.backend.Get() - %s", err.Error())
					continue
				}
				msg = NewMessage(buf)
			}
		} else {
			select {
			case msg = <-c.msgChan:
			case <-c.backend.ReadReadyChan():
				buf, err := c.backend.Get()
				if err != nil {
					log.Printf("ERROR: c.backend.Get() - %s", err.Error())
					continue
				}
				msg = NewMessage(buf)
			default:
				msg = nil
			}
		}

		if msg != nil {
			c.inFlightMessageChan <- msg
		}

		break
	}

	return msg
}

func (c *Channel) Close() error {
	var err error

	log.Printf("CHANNEL(%s): closing", c.name)

	c.exitChan <- 1

	err = c.backend.Close()
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>