<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/golang/glog"
	"github.com/skynetservices/skydns/metrics"
	"github.com/skynetservices/skydns/server"
	"k8s.io/kubernetes/cmd/kube-dns/app/options"
	"k8s.io/kubernetes/pkg/api/unversioned"
	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	"k8s.io/kubernetes/pkg/client/restclient"
	kclientcmd "k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
	kdns "k8s.io/kubernetes/pkg/dns"
)

type KubeDNSServer struct {
	// DNS domain name.
	domain      string
	healthzPort int
	dnsPort     int
	kd          *kdns.KubeDNS
}

func NewKubeDNSServerDefault(config *options.KubeDNSConfig) *KubeDNSServer {
	ks := KubeDNSServer{
		domain: config.ClusterDomain,
	}

	kubeClient, err := newKubeClient(config)
	if err != nil {
		glog.Fatalf("Failed to create a kubernetes client: %v", err)
	}
	ks.healthzPort = config.HealthzPort
	ks.dnsPort = config.DNSPort
	ks.kd, err = kdns.NewKubeDNS(kubeClient, config.ClusterDomain, config.Federations)
	if err != nil {
		glog.Fatalf("Failed to start kubeDNS: %v", err)
	}
	return &ks
}

// TODO: evaluate using pkg/client/clientcmd
func newKubeClient(dnsConfig *options.KubeDNSConfig) (clientset.Interface, error) {
	var (
		config *restclient.Config
		err    error
	)

	if dnsConfig.KubeMasterURL != "" && dnsConfig.KubeConfigFile == "" {
		// Only --kube-master-url was provided.
		config = &restclient.Config{
			Host:          dnsConfig.KubeMasterURL,
			ContentConfig: restclient.ContentConfig{GroupVersion: &unversioned.GroupVersion{Version: "v1"}},
		}
	} else {
		// We either have:
		//  1) --kube-master-url and --kubecfg-file
		//  2) just --kubecfg-file
		//  3) neither flag
		// In any case, the logic is the same.  If (3), this will automatically
		// fall back on the service account token.
		overrides := &kclientcmd.ConfigOverrides{}
		overrides.ClusterInfo.Server = dnsConfig.KubeMasterURL                                // might be "", but that is OK
		rules := &kclientcmd.ClientConfigLoadingRules{ExplicitPath: dnsConfig.KubeConfigFile} // might be "", but that is OK
		if config, err = kclientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides).ClientConfig(); err != nil {
			return nil, err
		}
	}

	glog.Infof("Using %s for kubernetes master", config.Host)
	glog.Infof("Using kubernetes API %v", config.GroupVersion)
	return clientset.NewForConfig(config)
}

func (server *KubeDNSServer) Run() {
	setupSignalHandlers()
	server.startSkyDNSServer()
	server.kd.Start()
	server.setupHealthzHandlers()
	glog.Infof("Setting up Healthz Handler(/readiness, /cache) on port :%d", server.healthzPort)
	glog.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", server.healthzPort), nil))
}

// setupHealthzHandlers sets up a readiness and liveness endpoint for kube2sky.
func (server *KubeDNSServer) setupHealthzHandlers() {
	http.HandleFunc("/readiness", func(w http.ResponseWriter, req *http.Request) {
		fmt.Fprintf(w, "ok\n")
	})
	http.HandleFunc("/cache", func(w http.ResponseWriter, req *http.Request) {
		serializedJSON, err := server.kd.GetCacheAsJSON()
		if err == nil {
			fmt.Fprint(w, serializedJSON)
		} else {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprint(w, err)
		}
	})
}

// setupSignalHandlers runs a goroutine that waits on SIGINT or SIGTERM and logs it
// before exiting.
func setupSignalHandlers() {
	sigChan := make(chan os.Signal)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		glog.Fatalf("Received signal: %s", <-sigChan)
	}()
}

func (d *KubeDNSServer) startSkyDNSServer() {
	glog.Infof("Starting SkyDNS server. Listening on port:%d", d.dnsPort)
	skydnsConfig := &server.Config{Domain: d.domain, DnsAddr: fmt.Sprintf("0.0.0.0:%d", d.dnsPort)}
	server.SetDefaults(skydnsConfig)
	s := server.New(d.kd, skydnsConfig)
	if err := metrics.Metrics(); err != nil {
		glog.Fatalf("skydns: %s", err)
	} else {
		glog.Infof("skydns: metrics enabled on :%s%s", metrics.Port, metrics.Path)
	}
	go s.Run()
}
<commit_msg>Add version and flag logging to kube-dns.<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/golang/glog"
	"github.com/skynetservices/skydns/metrics"
	"github.com/skynetservices/skydns/server"
	"github.com/spf13/pflag"

	"k8s.io/kubernetes/cmd/kube-dns/app/options"
	"k8s.io/kubernetes/pkg/api/unversioned"
	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	"k8s.io/kubernetes/pkg/client/restclient"
	kclientcmd "k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
	kdns "k8s.io/kubernetes/pkg/dns"
	"k8s.io/kubernetes/pkg/version"
)

type KubeDNSServer struct {
	// DNS domain name.
	domain      string
	healthzPort int
	dnsPort     int
	kd          *kdns.KubeDNS
}

func NewKubeDNSServerDefault(config *options.KubeDNSConfig) *KubeDNSServer {
	ks := KubeDNSServer{
		domain: config.ClusterDomain,
	}

	kubeClient, err := newKubeClient(config)
	if err != nil {
		glog.Fatalf("Failed to create a kubernetes client: %v", err)
	}
	ks.healthzPort = config.HealthzPort
	ks.dnsPort = config.DNSPort
	ks.kd, err = kdns.NewKubeDNS(kubeClient, config.ClusterDomain, config.Federations)
	if err != nil {
		glog.Fatalf("Failed to start kubeDNS: %v", err)
	}
	return &ks
}

// TODO: evaluate using pkg/client/clientcmd
func newKubeClient(dnsConfig *options.KubeDNSConfig) (clientset.Interface, error) {
	var (
		config *restclient.Config
		err    error
	)

	if dnsConfig.KubeMasterURL != "" && dnsConfig.KubeConfigFile == "" {
		// Only --kube-master-url was provided.
		config = &restclient.Config{
			Host:          dnsConfig.KubeMasterURL,
			ContentConfig: restclient.ContentConfig{GroupVersion: &unversioned.GroupVersion{Version: "v1"}},
		}
	} else {
		// We either have:
		//  1) --kube-master-url and --kubecfg-file
		//  2) just --kubecfg-file
		//  3) neither flag
		// In any case, the logic is the same.  If (3), this will automatically
		// fall back on the service account token.
		overrides := &kclientcmd.ConfigOverrides{}
		overrides.ClusterInfo.Server = dnsConfig.KubeMasterURL                                // might be "", but that is OK
		rules := &kclientcmd.ClientConfigLoadingRules{ExplicitPath: dnsConfig.KubeConfigFile} // might be "", but that is OK
		if config, err = kclientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides).ClientConfig(); err != nil {
			return nil, err
		}
	}

	glog.Infof("Using %s for kubernetes master", config.Host)
	glog.Infof("Using kubernetes API %v", config.GroupVersion)
	return clientset.NewForConfig(config)
}

func (server *KubeDNSServer) Run() {
	glog.Infof("%+v", version.Get())
	pflag.VisitAll(func(flag *pflag.Flag) {
		glog.Infof("FLAG: --%s=%q", flag.Name, flag.Value)
	})
	setupSignalHandlers()
	server.startSkyDNSServer()
	server.kd.Start()
	server.setupHealthzHandlers()
	glog.Infof("Setting up Healthz Handler(/readiness, /cache) on port :%d", server.healthzPort)
	glog.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", server.healthzPort), nil))
}

// setupHealthzHandlers sets up a readiness and liveness endpoint for kube2sky.
func (server *KubeDNSServer) setupHealthzHandlers() {
	http.HandleFunc("/readiness", func(w http.ResponseWriter, req *http.Request) {
		fmt.Fprintf(w, "ok\n")
	})
	http.HandleFunc("/cache", func(w http.ResponseWriter, req *http.Request) {
		serializedJSON, err := server.kd.GetCacheAsJSON()
		if err == nil {
			fmt.Fprint(w, serializedJSON)
		} else {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprint(w, err)
		}
	})
}

// setupSignalHandlers runs a goroutine that waits on SIGINT or SIGTERM and logs it
// before exiting.
func setupSignalHandlers() {
	sigChan := make(chan os.Signal)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		glog.Fatalf("Received signal: %s", <-sigChan)
	}()
}

func (d *KubeDNSServer) startSkyDNSServer() {
	glog.Infof("Starting SkyDNS server. Listening on port:%d", d.dnsPort)
	skydnsConfig := &server.Config{Domain: d.domain, DnsAddr: fmt.Sprintf("0.0.0.0:%d", d.dnsPort)}
	server.SetDefaults(skydnsConfig)
	s := server.New(d.kd, skydnsConfig)
	if err := metrics.Metrics(); err != nil {
		glog.Fatalf("skydns: %s", err)
	} else {
		glog.Infof("skydns: metrics enabled on :%s%s", metrics.Port, metrics.Path)
	}
	go s.Run()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
	"os"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/etcinit/phabulous/app"
	"github.com/facebookgo/inject"
	"github.com/jacobstr/confer"
)

func main() {
	// Seed rand.
	rand.Seed(time.Now().UTC().UnixNano())

	// Create the configuration
	// In this case, we will be using the environment and some safe defaults
	config := confer.NewConfig()
	config.ReadPaths("config/main.yml", "config/main.production.yml")
	config.AutomaticEnv()

	// Create the logger.
	logger := logrus.New()

	// Next, we setup the dependency graph
	// In this example, the graph won't have many nodes, but on more complex
	// applications it becomes more useful.
	var g inject.Graph
	var phabulous app.Phabulous
	g.Provide(
		&inject.Object{Value: config},
		&inject.Object{Value: &phabulous},
		&inject.Object{Value: logger},
	)
	if err := g.Populate(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	// Setup the command line application
	app := cli.NewApp()
	app.Name = "phabulous"
	app.Usage = "A Phabricator bot in Go"

	// Set version and authorship info
	app.Version = "2.4.0-beta1"
	app.Author = "Eduardo Trujillo <ed@chromabits.com>"

	// Setup the default action. This action will be triggered when no
	// subcommand is provided as an argument
	app.Action = func(c *cli.Context) {
		fmt.Println(
			"Usage: phabulous [global options] command [command options] " +
				"[arguments...]",
		)
	}

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config",
			Usage: "Provide an alternative configuration file",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:    "serve",
			Aliases: []string{"s", "server", "listen"},
			Usage:   "Start the API server",
			Action:  phabulous.Serve.Run,
		},
	}

	// Begin
	app.Run(os.Args)
}
<commit_msg>Fix action signature<commit_after>package main

import (
	"fmt"
	"math/rand"
	"os"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/etcinit/phabulous/app"
	"github.com/facebookgo/inject"
	"github.com/jacobstr/confer"
)

func main() {
	// Seed rand.
	rand.Seed(time.Now().UTC().UnixNano())

	// Create the configuration
	// In this case, we will be using the environment and some safe defaults
	config := confer.NewConfig()
	config.ReadPaths("config/main.yml", "config/main.production.yml")
	config.AutomaticEnv()

	// Create the logger.
	logger := logrus.New()

	// Next, we setup the dependency graph
	// In this example, the graph won't have many nodes, but on more complex
	// applications it becomes more useful.
	var g inject.Graph
	var phabulous app.Phabulous
	g.Provide(
		&inject.Object{Value: config},
		&inject.Object{Value: &phabulous},
		&inject.Object{Value: logger},
	)
	if err := g.Populate(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	// Setup the command line application
	app := cli.NewApp()
	app.Name = "phabulous"
	app.Usage = "A Phabricator bot in Go"

	// Set version and authorship info
	app.Version = "2.4.0-beta1"
	app.Author = "Eduardo Trujillo <ed@chromabits.com>"

	// Setup the default action. This action will be triggered when no
	// subcommand is provided as an argument
	app.Action = func(c *cli.Context) error {
		fmt.Println(
			"Usage: phabulous [global options] command [command options] " +
				"[arguments...]",
		)
		return nil
	}

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config",
			Usage: "Provide an alternative configuration file",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:    "serve",
			Aliases: []string{"s", "server", "listen"},
			Usage:   "Start the API server",
			Action:  phabulous.Serve.Run,
		},
	}

	// Begin
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package cli

import (
  //"fmt"
  "testing"

  "github.com/jbenet/go-ipfs/commands"
)

func TestOptionParsing(t *testing.T) {
  cmd := &commands.Command{
    Options: []commands.Option{
      commands.Option{ []string{"b"}, commands.String },
    },
  }
  cmd.Register("test", &commands.Command{})

  path := []string{"test"}
  options, err := cmd.GetOptions(path)
  if err != nil {
    t.Error(err)
  }

  opts, input, err := parseOptions([]string{ "--beep", "--boop=5 lol", "test2", "-cVb", "beep" }, options)
  /*for k, v := range opts {
    fmt.Printf("%s: %s\n", k, v)
  }
  fmt.Printf("%s\n", input)*/
  if err != nil {
    t.Error("Should have passed")
  }
  if len(opts) != 5 || opts["c"] != "" || opts["V"] != "" || opts["beep"] != "" || opts["boop"] != "5 lol" || opts["b"] != "beep" {
    t.Error("Returned options were defferent than expected: %v", opts)
  }
  if len(input) != 1 || input[0] != "test2" {
    t.Error("Returned input was different than expected: %v", input)
  }

  path, args, err := parsePath([]string{ "test", "beep", "boop" }, cmd)
  if err != nil {
    t.Error("Should have passed")
  }
  if len(path) != 1 || path[0] != "test" {
    t.Error("Returned path was defferent than expected: %v", path)
  }
  if len(args) != 2 || args[0] != "beep" || args[1] != "boop" {
    t.Error("Returned args were different than expected: %v", args)
  }
}
<commit_msg>commands/cli: Fixed test for new parsing<commit_after>package cli

import (
  //"fmt"
  "testing"

  "github.com/jbenet/go-ipfs/commands"
)

func TestOptionParsing(t *testing.T) {
  cmd := &commands.Command{
    Options: []commands.Option{
      commands.Option{ []string{"b"}, commands.String },
    },
  }
  cmd.Register("test", &commands.Command{})

  opts, input, err := parseOptions([]string{ "--beep", "-boop=lol", "test2", "-c", "beep", "--foo=5" })
  /*for k, v := range opts {
    fmt.Printf("%s: %s\n", k, v)
  }
  fmt.Printf("%s\n", input)*/
  if err != nil {
    t.Error("Should have passed")
  }
  if len(opts) != 4 || opts["beep"] != "" || opts["boop"] != "lol" || opts["c"] != "" || opts["foo"] != "5" {
    t.Error("Returned options were defferent than expected: %v", opts)
  }
  if len(input) != 2 || input[0] != "test2" || input[1] != "beep" {
    t.Error("Returned input was different than expected: %v", input)
  }

  path, args, err := parsePath([]string{ "test", "beep", "boop" }, cmd)
  if err != nil {
    t.Error("Should have passed")
  }
  if len(path) != 1 || path[0] != "test" {
    t.Error("Returned path was defferent than expected: %v", path)
  }
  if len(args) != 2 || args[0] != "beep" || args[1] != "boop" {
    t.Error("Returned args were different than expected: %v", args)
  }
}
<|endoftext|>


<commit_before>package commands

import (
	"bytes"
	"fmt"
	"io"
	"os"

	"github.com/github/git-lfs/config"
	"github.com/github/git-lfs/errors"
	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/progress"
	"github.com/spf13/cobra"
)

const (
	// cleanFilterBufferCapacity is the desired capacity of the
	// `*git.PacketWriter`'s internal buffer when the filter protocol
	// dictates the "clean" command
	cleanFilterBufferCapacity = 512

	// smudgeFilterBufferCapacity is the desired capacity of the
	// `*git.PacketWriter`'s internal buffer when the filter protocol
	// dictates the "smudge" command
	smudgeFilterBufferCapacity = git.MaxPacketLength
)

var (
	filterSmudgeSkip = false
)

func clean(to io.Writer, reader io.Reader, fileName string) error {
	var cb progress.CopyCallback
	var file *os.File
	var fileSize int64
	if len(fileName) > 0 {
		stat, err := os.Stat(fileName)
		if err == nil && stat != nil {
			fileSize = stat.Size()

			localCb, localFile, err := lfs.CopyCallbackFile("clean", fileName, 1, 1)
			if err != nil {
				Error(err.Error())
			} else {
				cb = localCb
				file = localFile
			}
		}
	}

	cleaned, err := lfs.PointerClean(reader, fileName, fileSize, cb)
	if file != nil {
		file.Close()
	}

	if cleaned != nil {
		defer cleaned.Teardown()
	}

	if errors.IsCleanPointerError(err) {
		// If the contents read from the working directory was _already_
		// a pointer, we'll get a `CleanPointerError`, with the context
		// containing the bytes that we should write back out to Git.
		_, err = to.Write(errors.GetContext(err, "bytes").([]byte))
		return err
	}

	if err != nil {
		Panic(err, "Error cleaning asset.")
	}

	tmpfile := cleaned.Filename
	mediafile, err := lfs.LocalMediaPath(cleaned.Oid)
	if err != nil {
		Panic(err, "Unable to get local media path.")
	}

	if stat, _ := os.Stat(mediafile); stat != nil {
		if stat.Size() != cleaned.Size && len(cleaned.Pointer.Extensions) == 0 {
			Exit("Files don't match:\n%s\n%s", mediafile, tmpfile)
		}
		Debug("%s exists", mediafile)
	} else {
		if err := os.Rename(tmpfile, mediafile); err != nil {
			Panic(err, "Unable to move %s to %s\n", tmpfile, mediafile)
		}

		Debug("Writing %s", mediafile)
	}

	_, err = cleaned.Pointer.Encode(to)
	return err
}

func smudge(to io.Writer, reader io.Reader, filename string) error {
	var pbuf bytes.Buffer
	reader = io.TeeReader(reader, &pbuf)

	ptr, err := lfs.DecodePointer(reader)
	if err != nil {
		// If we tried to decode a pointer out of the data given to us,
		// and the file was _empty_, write out an empty file in
		// response. This occurs because when the clean filter
		// encounters an empty file, and writes out an empty file,
		// instead of a pointer.
		//
		// TODO(taylor): figure out if there is more data on the reader,
		// and buffer that as well.
		if len(pbuf.Bytes()) == 0 {
			if _, cerr := io.Copy(to, &pbuf); cerr != nil {
				Panic(cerr, "Error writing data to stdout:")
			}
			return nil
		}

		return err
	}

	lfs.LinkOrCopyFromReference(ptr.Oid, ptr.Size)

	cb, file, err := lfs.CopyCallbackFile("smudge", filename, 1, 1)
	if err != nil {
		Error(err.Error())
	}

	cfg := config.Config
	download := lfs.FilenamePassesIncludeExcludeFilter(filename, cfg.FetchIncludePaths(), cfg.FetchExcludePaths())

	if filterSmudgeSkip || cfg.Os.Bool("GIT_LFS_SKIP_SMUDGE", false) {
		download = false
	}

	err = ptr.Smudge(to, filename, download, TransferManifest(), cb)
	if file != nil {
		file.Close()
	}

	if err != nil {
		// Download declined error is ok to skip if we weren't requesting download
		if !(errors.IsDownloadDeclinedError(err) && !download) {
			LoggedError(err, "Error downloading object: %s (%s)", filename, ptr.Oid)
			if !cfg.SkipDownloadErrors() {
				// TODO: What to do best here?
				os.Exit(2)
			}
		}

		_, err = ptr.Encode(to)
		return err
	}

	return nil
}

func filterCommand(cmd *cobra.Command, args []string) {
	requireStdin("This command should be run by the Git filter process")
	lfs.InstallHooks(false)

	s := git.NewObjectScanner(os.Stdin, os.Stdout)
	w := git.NewPacketWriter(os.Stdout, 0)

	if err := s.Init(); err != nil {
		ExitWithError(err)
	}
	if err := s.NegotiateCapabilities(); err != nil {
		ExitWithError(err)
	}

Scan:
	for s.Scan() {
		var err error

		req := s.Request()
		if req == nil {
			break
		}
		s.WriteStatus("success")

		switch req.Header["command"] {
		case "clean":
			err = clean(w, req.Payload, req.Header["pathname"])
		case "smudge":
			err = smudge(w, req.Payload, req.Header["pathname"])
		default:
			fmt.Errorf("Unknown command %s", cmd)
			break Scan
		}

		if err == nil {
			_, err = w.Write(nil)
		}

		var status string
		if err != nil && err != io.EOF {
			status = "error"
		} else {
			status = "success"
		}
		s.WriteStatus(status)
	}

	if err := s.Err(); err != nil && err != io.EOF {
		ExitWithError(err)
	}
}

func init() {
	RegisterCommand("filter", filterCommand, func(cmd *cobra.Command) {
		cmd.Flags().BoolVarP(&filterSmudgeSkip, "skip", "s", false, "")
	})
}
<commit_msg>commands/filter: write the flush before sending status<commit_after>package commands

import (
	"bytes"
	"fmt"
	"io"
	"os"

	"github.com/github/git-lfs/config"
	"github.com/github/git-lfs/errors"
	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/progress"
	"github.com/spf13/cobra"
)

const (
	// cleanFilterBufferCapacity is the desired capacity of the
	// `*git.PacketWriter`'s internal buffer when the filter protocol
	// dictates the "clean" command
	cleanFilterBufferCapacity = 512

	// smudgeFilterBufferCapacity is the desired capacity of the
	// `*git.PacketWriter`'s internal buffer when the filter protocol
	// dictates the "smudge" command
	smudgeFilterBufferCapacity = git.MaxPacketLength
)

var (
	filterSmudgeSkip = false
)

func clean(to io.Writer, reader io.Reader, fileName string) error {
	var cb progress.CopyCallback
	var file *os.File
	var fileSize int64
	if len(fileName) > 0 {
		stat, err := os.Stat(fileName)
		if err == nil && stat != nil {
			fileSize = stat.Size()

			localCb, localFile, err := lfs.CopyCallbackFile("clean", fileName, 1, 1)
			if err != nil {
				Error(err.Error())
			} else {
				cb = localCb
				file = localFile
			}
		}
	}

	cleaned, err := lfs.PointerClean(reader, fileName, fileSize, cb)
	if file != nil {
		file.Close()
	}

	if cleaned != nil {
		defer cleaned.Teardown()
	}

	if errors.IsCleanPointerError(err) {
		// If the contents read from the working directory was _already_
		// a pointer, we'll get a `CleanPointerError`, with the context
		// containing the bytes that we should write back out to Git.
		_, err = to.Write(errors.GetContext(err, "bytes").([]byte))
		return err
	}

	if err != nil {
		Panic(err, "Error cleaning asset.")
	}

	tmpfile := cleaned.Filename
	mediafile, err := lfs.LocalMediaPath(cleaned.Oid)
	if err != nil {
		Panic(err, "Unable to get local media path.")
	}

	if stat, _ := os.Stat(mediafile); stat != nil {
		if stat.Size() != cleaned.Size && len(cleaned.Pointer.Extensions) == 0 {
			Exit("Files don't match:\n%s\n%s", mediafile, tmpfile)
		}
		Debug("%s exists", mediafile)
	} else {
		if err := os.Rename(tmpfile, mediafile); err != nil {
			Panic(err, "Unable to move %s to %s\n", tmpfile, mediafile)
		}

		Debug("Writing %s", mediafile)
	}

	_, err = cleaned.Pointer.Encode(to)
	return err
}

func smudge(to io.Writer, reader io.Reader, filename string) error {
	var pbuf bytes.Buffer
	reader = io.TeeReader(reader, &pbuf)

	ptr, err := lfs.DecodePointer(reader)
	if err != nil {
		// If we tried to decode a pointer out of the data given to us,
		// and the file was _empty_, write out an empty file in
		// response. This occurs because when the clean filter
		// encounters an empty file, and writes out an empty file,
		// instead of a pointer.
		//
		// TODO(taylor): figure out if there is more data on the reader,
		// and buffer that as well.
		if len(pbuf.Bytes()) == 0 {
			if _, cerr := io.Copy(to, &pbuf); cerr != nil {
				Panic(cerr, "Error writing data to stdout:")
			}
			return nil
		}

		return err
	}

	lfs.LinkOrCopyFromReference(ptr.Oid, ptr.Size)

	cb, file, err := lfs.CopyCallbackFile("smudge", filename, 1, 1)
	if err != nil {
		Error(err.Error())
	}

	cfg := config.Config
	download := lfs.FilenamePassesIncludeExcludeFilter(filename, cfg.FetchIncludePaths(), cfg.FetchExcludePaths())

	if filterSmudgeSkip || cfg.Os.Bool("GIT_LFS_SKIP_SMUDGE", false) {
		download = false
	}

	err = ptr.Smudge(to, filename, download, TransferManifest(), cb)
	if file != nil {
		file.Close()
	}

	if err != nil {
		// Download declined error is ok to skip if we weren't requesting download
		if !(errors.IsDownloadDeclinedError(err) && !download) {
			LoggedError(err, "Error downloading object: %s (%s)", filename, ptr.Oid)
			if !cfg.SkipDownloadErrors() {
				// TODO: What to do best here?
				os.Exit(2)
			}
		}

		_, err = ptr.Encode(to)
		return err
	}

	return nil
}

func filterCommand(cmd *cobra.Command, args []string) {
	requireStdin("This command should be run by the Git filter process")
	lfs.InstallHooks(false)

	s := git.NewObjectScanner(os.Stdin, os.Stdout)
	w := git.NewPacketWriter(os.Stdout, 0)

	if err := s.Init(); err != nil {
		ExitWithError(err)
	}
	if err := s.NegotiateCapabilities(); err != nil {
		ExitWithError(err)
	}

Scan:
	for s.Scan() {
		var err error

		req := s.Request()
		if req == nil {
			break
		}
		s.WriteStatus("success")

		switch req.Header["command"] {
		case "clean":
			err = clean(w, req.Payload, req.Header["pathname"])
		case "smudge":
			err = smudge(w, req.Payload, req.Header["pathname"])
		default:
			fmt.Errorf("Unknown command %s", cmd)
			break Scan
		}

		var status string
		if _, ferr := w.Write(nil); ferr != nil {
			status = "error"
		} else {
			if err != nil && err != io.EOF {
				status = "error"
			} else {
				status = "success"
			}
		}
		s.WriteStatus(status)
	}

	if err := s.Err(); err != nil && err != io.EOF {
		ExitWithError(err)
	}
}

func init() {
	RegisterCommand("filter", filterCommand, func(cmd *cobra.Command) {
		cmd.Flags().BoolVarP(&filterSmudgeSkip, "skip", "s", false, "")
	})
}
<|endoftext|>


<commit_before>// Package dht implements a distributed hash table that satisfies the ipfs routing
// interface. This DHT is modeled after kademlia with Coral and S/Kademlia modifications.
package dht

import (
	"bytes"
	"errors"
	"fmt"
	"sync"
	"time"

	key "github.com/ipfs/go-ipfs/blocks/key"
	ci "github.com/ipfs/go-ipfs/p2p/crypto"
	host "github.com/ipfs/go-ipfs/p2p/host"
	peer "github.com/ipfs/go-ipfs/p2p/peer"
	protocol "github.com/ipfs/go-ipfs/p2p/protocol"
	routing "github.com/ipfs/go-ipfs/routing"
	pb "github.com/ipfs/go-ipfs/routing/dht/pb"
	kb "github.com/ipfs/go-ipfs/routing/kbucket"
	record "github.com/ipfs/go-ipfs/routing/record"
	u "github.com/ipfs/go-ipfs/util"
	logging "github.com/ipfs/go-ipfs/vendor/go-log-v1.0.0"

	proto "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/gogo/protobuf/proto"
	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
	goprocess "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess"
	goprocessctx "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess/context"
	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
)

var log = logging.Logger("dht")

var ProtocolDHT protocol.ID = "/ipfs/dht"

// NumBootstrapQueries defines the number of random dht queries to do to
// collect members of the routing table.
const NumBootstrapQueries = 5

// TODO. SEE https://github.com/jbenet/node-ipfs/blob/master/submodules/ipfs-dht/index.js

// IpfsDHT is an implementation of Kademlia with Coral and S/Kademlia modifications.
// It is used to implement the base IpfsRouting module.
type IpfsDHT struct {
	host      host.Host      // the network services we need
	self      peer.ID        // Local peer (yourself)
	peerstore peer.Peerstore // Peer Registry

	datastore ds.ThreadSafeDatastore // Local data

	routingTable *kb.RoutingTable // Array of routing tables for differently distanced nodes
	providers    *ProviderManager

	birth    time.Time  // When this peer started up
	diaglock sync.Mutex // lock to make diagnostics work better

	Validator record.Validator // record validator funcs
	Selector  record.Selector  // record selection funcs

	ctx  context.Context
	proc goprocess.Process
}

// NewDHT creates a new DHT object with the given peer as the 'local' host
func NewDHT(ctx context.Context, h host.Host, dstore ds.ThreadSafeDatastore) *IpfsDHT {
	dht := new(IpfsDHT)
	dht.datastore = dstore
	dht.self = h.ID()
	dht.peerstore = h.Peerstore()
	dht.host = h

	// register for network notifs.
	dht.host.Network().Notify((*netNotifiee)(dht))

	dht.proc = goprocess.WithTeardown(func() error {
		// remove ourselves from network notifs.
		dht.host.Network().StopNotify((*netNotifiee)(dht))
		return nil
	})

	dht.ctx = ctx

	h.SetStreamHandler(ProtocolDHT, dht.handleNewStream)
	dht.providers = NewProviderManager(dht.ctx, dht.self)
	dht.proc.AddChild(dht.providers.proc)
	goprocessctx.CloseAfterContext(dht.proc, ctx)

	dht.routingTable = kb.NewRoutingTable(20, kb.ConvertPeerID(dht.self), time.Minute, dht.peerstore)
	dht.birth = time.Now()

	dht.Validator = make(record.Validator)
	dht.Validator["pk"] = record.PublicKeyValidator

	dht.Selector = make(record.Selector)
	dht.Selector["pk"] = record.PublicKeySelector

	return dht
}

// LocalPeer returns the peer.Peer of the dht.
func (dht *IpfsDHT) LocalPeer() peer.ID {
	return dht.self
}

// log returns the dht's logger
func (dht *IpfsDHT) log() logging.EventLogger {
	return log // TODO rm
}

// putValueToPeer stores the given key/value pair at the peer 'p'
func (dht *IpfsDHT) putValueToPeer(ctx context.Context, p peer.ID,
	key key.Key, rec *pb.Record) error {

	pmes := pb.NewMessage(pb.Message_PUT_VALUE, string(key), 0)
	pmes.Record = rec
	rpmes, err := dht.sendRequest(ctx, p, pmes)
	if err != nil {
		return err
	}

	if !bytes.Equal(rpmes.GetRecord().Value, pmes.GetRecord().Value) {
		return errors.New("value not put correctly")
	}
	return nil
}

// putProvider sends a message to peer 'p' saying that the local node
// can provide the value of 'key'
func (dht *IpfsDHT) putProvider(ctx context.Context, p peer.ID, skey string) error {

	// add self as the provider
	pi := peer.PeerInfo{
		ID:    dht.self,
		Addrs: dht.host.Addrs(),
	}

	// // only share WAN-friendly addresses ??
	// pi.Addrs = addrutil.WANShareableAddrs(pi.Addrs)
	if len(pi.Addrs) < 1 {
		// log.Infof("%s putProvider: %s for %s error: no wan-friendly addresses", dht.self, p, key.Key(key), pi.Addrs)
		return fmt.Errorf("no known addresses for self. cannot put provider.")
	}

	pmes := pb.NewMessage(pb.Message_ADD_PROVIDER, skey, 0)
	pmes.ProviderPeers = pb.RawPeerInfosToPBPeers([]peer.PeerInfo{pi})
	err := dht.sendMessage(ctx, p, pmes)
	if err != nil {
		return err
	}

	log.Debugf("%s putProvider: %s for %s (%s)", dht.self, p, key.Key(skey), pi.Addrs)
	return nil
}

// getValueOrPeers queries a particular peer p for the value for
// key. It returns either the value or a list of closer peers.
// NOTE: it will update the dht's peerstore with any new addresses
// it finds for the given peer.
func (dht *IpfsDHT) getValueOrPeers(ctx context.Context, p peer.ID,
	key key.Key) (*pb.Record, []peer.PeerInfo, error) {

	pmes, err := dht.getValueSingle(ctx, p, key)
	if err != nil {
		return nil, nil, err
	}

	// Perhaps we were given closer peers
	peers := pb.PBPeersToPeerInfos(pmes.GetCloserPeers())

	if record := pmes.GetRecord(); record != nil {
		// Success! We were given the value
		log.Debug("getValueOrPeers: got value")

		// make sure record is valid.
		err = dht.verifyRecordOnline(ctx, record)
		if err != nil {
			log.Info("Received invalid record! (discarded)")
			return nil, nil, err
		}
		return record, peers, nil
	}

	if len(peers) > 0 {
		log.Debug("getValueOrPeers: peers")
		return nil, peers, nil
	}

	log.Warning("getValueOrPeers: routing.ErrNotFound")
	return nil, nil, routing.ErrNotFound
}

// getValueSingle simply performs the get value RPC with the given parameters
func (dht *IpfsDHT) getValueSingle(ctx context.Context, p peer.ID,
	key key.Key) (*pb.Message, error) {
	defer log.EventBegin(ctx, "getValueSingle", p, &key).Done()

	pmes := pb.NewMessage(pb.Message_GET_VALUE, string(key), 0)
	return dht.sendRequest(ctx, p, pmes)
}

// getLocal attempts to retrieve the value from the datastore
func (dht *IpfsDHT) getLocal(key key.Key) (*pb.Record, error) {

	log.Debug("getLocal %s", key)
	v, err := dht.datastore.Get(key.DsKey())
	if err != nil {
		return nil, err
	}
	log.Debug("found in db")

	byt, ok := v.([]byte)
	if !ok {
		return nil, errors.New("value stored in datastore not []byte")
	}
	rec := new(pb.Record)
	err = proto.Unmarshal(byt, rec)
	if err != nil {
		return nil, err
	}

	// TODO: 'if paranoid'
	if u.Debug {
		err = dht.verifyRecordLocally(rec)
		if err != nil {
			log.Debugf("local record verify failed: %s (discarded)", err)
			return nil, err
		}
	}

	return rec, nil
}

// getOwnPrivateKey attempts to load the local peers private
// key from the peerstore.
func (dht *IpfsDHT) getOwnPrivateKey() (ci.PrivKey, error) {
	sk := dht.peerstore.PrivKey(dht.self)
	if sk == nil {
		log.Warningf("%s dht cannot get own private key!", dht.self)
		return nil, fmt.Errorf("cannot get private key to sign record!")
	}
	return sk, nil
}

// putLocal stores the key value pair in the datastore
func (dht *IpfsDHT) putLocal(key key.Key, rec *pb.Record) error {
	data, err := proto.Marshal(rec)
	if err != nil {
		return err
	}

	return dht.datastore.Put(key.DsKey(), data)
}

// Update signals the routingTable to Update its last-seen status
// on the given peer.
func (dht *IpfsDHT) Update(ctx context.Context, p peer.ID) {
	log.Event(ctx, "updatePeer", p)
	dht.routingTable.Update(p)
}

// FindLocal looks for a peer with a given ID connected to this dht and returns the peer and the table it was found in.
func (dht *IpfsDHT) FindLocal(id peer.ID) peer.PeerInfo {
	p := dht.routingTable.Find(id)
	if p != "" {
		return dht.peerstore.PeerInfo(p)
	}
	return peer.PeerInfo{}
}

// findPeerSingle asks peer 'p' if they know where the peer with id 'id' is
func (dht *IpfsDHT) findPeerSingle(ctx context.Context, p peer.ID, id peer.ID) (*pb.Message, error) {
	defer log.EventBegin(ctx, "findPeerSingle", p, id).Done()

	pmes := pb.NewMessage(pb.Message_FIND_NODE, string(id), 0)
	return dht.sendRequest(ctx, p, pmes)
}

func (dht *IpfsDHT) findProvidersSingle(ctx context.Context, p peer.ID, key key.Key) (*pb.Message, error) {
	defer log.EventBegin(ctx, "findProvidersSingle", p, &key).Done()

	pmes := pb.NewMessage(pb.Message_GET_PROVIDERS, string(key), 0)
	return dht.sendRequest(ctx, p, pmes)
}

// nearestPeersToQuery returns the routing tables closest peers.
func (dht *IpfsDHT) nearestPeersToQuery(pmes *pb.Message, count int) []peer.ID {
	key := key.Key(pmes.GetKey())
	closer := dht.routingTable.NearestPeers(kb.ConvertKey(key), count)
	return closer
}

// betterPeerToQuery returns nearestPeersToQuery, but iff closer than self.
func (dht *IpfsDHT) betterPeersToQuery(pmes *pb.Message, p peer.ID, count int) []peer.ID {
	closer := dht.nearestPeersToQuery(pmes, count)

	// no node? nil
	if closer == nil {
		return nil
	}

	// == to self? thats bad
	for _, p := range closer {
		if p == dht.self {
			log.Debug("Attempted to return self! this shouldnt happen...")
			return nil
		}
	}

	var filtered []peer.ID
	for _, clp := range closer {
		// Dont send a peer back themselves
		if p == clp {
			continue
		}

		filtered = append(filtered, clp)
	}

	// ok seems like closer nodes
	return filtered
}

// Context return dht's context
func (dht *IpfsDHT) Context() context.Context {
	return dht.ctx
}

// Process return dht's process
func (dht *IpfsDHT) Process() goprocess.Process {
	return dht.proc
}

// Close calls Process Close
func (dht *IpfsDHT) Close() error {
	return dht.proc.Close()
}
<commit_msg>Implement ipns republisher<commit_after>// Package dht implements a distributed hash table that satisfies the ipfs routing
// interface. This DHT is modeled after kademlia with Coral and S/Kademlia modifications.
package dht

import (
	"bytes"
	"errors"
	"fmt"
	"sync"
	"time"

	key "github.com/ipfs/go-ipfs/blocks/key"
	ci "github.com/ipfs/go-ipfs/p2p/crypto"
	host "github.com/ipfs/go-ipfs/p2p/host"
	peer "github.com/ipfs/go-ipfs/p2p/peer"
	protocol "github.com/ipfs/go-ipfs/p2p/protocol"
	routing "github.com/ipfs/go-ipfs/routing"
	pb "github.com/ipfs/go-ipfs/routing/dht/pb"
	kb "github.com/ipfs/go-ipfs/routing/kbucket"
	record "github.com/ipfs/go-ipfs/routing/record"
	logging "github.com/ipfs/go-ipfs/vendor/go-log-v1.0.0"

	proto "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/gogo/protobuf/proto"
	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
	goprocess "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess"
	goprocessctx "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess/context"
	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
)

var log = logging.Logger("dht")

var ProtocolDHT protocol.ID = "/ipfs/dht"

// NumBootstrapQueries defines the number of random dht queries to do to
// collect members of the routing table.
const NumBootstrapQueries = 5

// TODO. SEE https://github.com/jbenet/node-ipfs/blob/master/submodules/ipfs-dht/index.js

// IpfsDHT is an implementation of Kademlia with Coral and S/Kademlia modifications.
// It is used to implement the base IpfsRouting module.
type IpfsDHT struct {
	host      host.Host      // the network services we need
	self      peer.ID        // Local peer (yourself)
	peerstore peer.Peerstore // Peer Registry

	datastore ds.ThreadSafeDatastore // Local data

	routingTable *kb.RoutingTable // Array of routing tables for differently distanced nodes
	providers    *ProviderManager

	birth    time.Time  // When this peer started up
	diaglock sync.Mutex // lock to make diagnostics work better

	Validator record.Validator // record validator funcs
	Selector  record.Selector  // record selection funcs

	ctx  context.Context
	proc goprocess.Process
}

// NewDHT creates a new DHT object with the given peer as the 'local' host
func NewDHT(ctx context.Context, h host.Host, dstore ds.ThreadSafeDatastore) *IpfsDHT {
	dht := new(IpfsDHT)
	dht.datastore = dstore
	dht.self = h.ID()
	dht.peerstore = h.Peerstore()
	dht.host = h

	// register for network notifs.
	dht.host.Network().Notify((*netNotifiee)(dht))

	dht.proc = goprocess.WithTeardown(func() error {
		// remove ourselves from network notifs.
		dht.host.Network().StopNotify((*netNotifiee)(dht))
		return nil
	})

	dht.ctx = ctx

	h.SetStreamHandler(ProtocolDHT, dht.handleNewStream)
	dht.providers = NewProviderManager(dht.ctx, dht.self)
	dht.proc.AddChild(dht.providers.proc)
	goprocessctx.CloseAfterContext(dht.proc, ctx)

	dht.routingTable = kb.NewRoutingTable(20, kb.ConvertPeerID(dht.self), time.Minute, dht.peerstore)
	dht.birth = time.Now()

	dht.Validator = make(record.Validator)
	dht.Validator["pk"] = record.PublicKeyValidator

	dht.Selector = make(record.Selector)
	dht.Selector["pk"] = record.PublicKeySelector

	return dht
}

// LocalPeer returns the peer.Peer of the dht.
func (dht *IpfsDHT) LocalPeer() peer.ID {
	return dht.self
}

// log returns the dht's logger
func (dht *IpfsDHT) log() logging.EventLogger {
	return log // TODO rm
}

// putValueToPeer stores the given key/value pair at the peer 'p'
func (dht *IpfsDHT) putValueToPeer(ctx context.Context, p peer.ID,
	key key.Key, rec *pb.Record) error {

	pmes := pb.NewMessage(pb.Message_PUT_VALUE, string(key), 0)
	pmes.Record = rec
	rpmes, err := dht.sendRequest(ctx, p, pmes)
	if err != nil {
		return err
	}

	if !bytes.Equal(rpmes.GetRecord().Value, pmes.GetRecord().Value) {
		return errors.New("value not put correctly")
	}
	return nil
}

// putProvider sends a message to peer 'p' saying that the local node
// can provide the value of 'key'
func (dht *IpfsDHT) putProvider(ctx context.Context, p peer.ID, skey string) error {

	// add self as the provider
	pi := peer.PeerInfo{
		ID:    dht.self,
		Addrs: dht.host.Addrs(),
	}

	// // only share WAN-friendly addresses ??
	// pi.Addrs = addrutil.WANShareableAddrs(pi.Addrs)
	if len(pi.Addrs) < 1 {
		// log.Infof("%s putProvider: %s for %s error: no wan-friendly addresses", dht.self, p, key.Key(key), pi.Addrs)
		return fmt.Errorf("no known addresses for self. cannot put provider.")
	}

	pmes := pb.NewMessage(pb.Message_ADD_PROVIDER, skey, 0)
	pmes.ProviderPeers = pb.RawPeerInfosToPBPeers([]peer.PeerInfo{pi})
	err := dht.sendMessage(ctx, p, pmes)
	if err != nil {
		return err
	}

	log.Debugf("%s putProvider: %s for %s (%s)", dht.self, p, key.Key(skey), pi.Addrs)
	return nil
}

// getValueOrPeers queries a particular peer p for the value for
// key. It returns either the value or a list of closer peers.
// NOTE: it will update the dht's peerstore with any new addresses
// it finds for the given peer.
func (dht *IpfsDHT) getValueOrPeers(ctx context.Context, p peer.ID,
	key key.Key) (*pb.Record, []peer.PeerInfo, error) {

	pmes, err := dht.getValueSingle(ctx, p, key)
	if err != nil {
		return nil, nil, err
	}

	// Perhaps we were given closer peers
	peers := pb.PBPeersToPeerInfos(pmes.GetCloserPeers())

	if record := pmes.GetRecord(); record != nil {
		// Success! We were given the value
		log.Debug("getValueOrPeers: got value")

		// make sure record is valid.
		err = dht.verifyRecordOnline(ctx, record)
		if err != nil {
			log.Info("Received invalid record! (discarded)")
			return nil, nil, err
		}
		return record, peers, nil
	}

	if len(peers) > 0 {
		log.Debug("getValueOrPeers: peers")
		return nil, peers, nil
	}

	log.Warning("getValueOrPeers: routing.ErrNotFound")
	return nil, nil, routing.ErrNotFound
}

// getValueSingle simply performs the get value RPC with the given parameters
func (dht *IpfsDHT) getValueSingle(ctx context.Context, p peer.ID,
	key key.Key) (*pb.Message, error) {
	defer log.EventBegin(ctx, "getValueSingle", p, &key).Done()

	pmes := pb.NewMessage(pb.Message_GET_VALUE, string(key), 0)
	return dht.sendRequest(ctx, p, pmes)
}

// getLocal attempts to retrieve the value from the datastore
func (dht *IpfsDHT) getLocal(key key.Key) (*pb.Record, error) {

	log.Debug("getLocal %s", key)
	v, err := dht.datastore.Get(key.DsKey())
	if err != nil {
		return nil, err
	}
	log.Debug("found in db")

	byt, ok := v.([]byte)
	if !ok {
		return nil, errors.New("value stored in datastore not []byte")
	}
	rec := new(pb.Record)
	err = proto.Unmarshal(byt, rec)
	if err != nil {
		return nil, err
	}

	err = dht.verifyRecordLocally(rec)
	if err != nil {
		log.Debugf("local record verify failed: %s (discarded)", err)
		return nil, err
	}

	return rec, nil
}

// getOwnPrivateKey attempts to load the local peers private
// key from the peerstore.
func (dht *IpfsDHT) getOwnPrivateKey() (ci.PrivKey, error) {
	sk := dht.peerstore.PrivKey(dht.self)
	if sk == nil {
		log.Warningf("%s dht cannot get own private key!", dht.self)
		return nil, fmt.Errorf("cannot get private key to sign record!")
	}
	return sk, nil
}

// putLocal stores the key value pair in the datastore
func (dht *IpfsDHT) putLocal(key key.Key, rec *pb.Record) error {
	data, err := proto.Marshal(rec)
	if err != nil {
		return err
	}

	return dht.datastore.Put(key.DsKey(), data)
}

// Update signals the routingTable to Update its last-seen status
// on the given peer.
func (dht *IpfsDHT) Update(ctx context.Context, p peer.ID) {
	log.Event(ctx, "updatePeer", p)
	dht.routingTable.Update(p)
}

// FindLocal looks for a peer with a given ID connected to this dht and returns the peer and the table it was found in.
func (dht *IpfsDHT) FindLocal(id peer.ID) peer.PeerInfo {
	p := dht.routingTable.Find(id)
	if p != "" {
		return dht.peerstore.PeerInfo(p)
	}
	return peer.PeerInfo{}
}

// findPeerSingle asks peer 'p' if they know where the peer with id 'id' is
func (dht *IpfsDHT) findPeerSingle(ctx context.Context, p peer.ID, id peer.ID) (*pb.Message, error) {
	defer log.EventBegin(ctx, "findPeerSingle", p, id).Done()

	pmes := pb.NewMessage(pb.Message_FIND_NODE, string(id), 0)
	return dht.sendRequest(ctx, p, pmes)
}

func (dht *IpfsDHT) findProvidersSingle(ctx context.Context, p peer.ID, key key.Key) (*pb.Message, error) {
	defer log.EventBegin(ctx, "findProvidersSingle", p, &key).Done()

	pmes := pb.NewMessage(pb.Message_GET_PROVIDERS, string(key), 0)
	return dht.sendRequest(ctx, p, pmes)
}

// nearestPeersToQuery returns the routing tables closest peers.
func (dht *IpfsDHT) nearestPeersToQuery(pmes *pb.Message, count int) []peer.ID {
	key := key.Key(pmes.GetKey())
	closer := dht.routingTable.NearestPeers(kb.ConvertKey(key), count)
	return closer
}

// betterPeerToQuery returns nearestPeersToQuery, but iff closer than self.
func (dht *IpfsDHT) betterPeersToQuery(pmes *pb.Message, p peer.ID, count int) []peer.ID {
	closer := dht.nearestPeersToQuery(pmes, count)

	// no node? nil
	if closer == nil {
		return nil
	}

	// == to self? thats bad
	for _, p := range closer {
		if p == dht.self {
			log.Debug("Attempted to return self! this shouldnt happen...")
			return nil
		}
	}

	var filtered []peer.ID
	for _, clp := range closer {
		// Dont send a peer back themselves
		if p == clp {
			continue
		}

		filtered = append(filtered, clp)
	}

	// ok seems like closer nodes
	return filtered
}

// Context return dht's context
func (dht *IpfsDHT) Context() context.Context {
	return dht.ctx
}

// Process return dht's process
func (dht *IpfsDHT) Process() goprocess.Process {
	return dht.proc
}

// Close calls Process Close
func (dht *IpfsDHT) Close() error {
	return dht.proc.Close()
}
<|endoftext|>


<commit_before>package dht

// #include "dht.go.h"
// #cgo LDFLAGS: -lrt
import "C"

import (
	"context"
	"errors"
	"fmt"
	"reflect"
	"time"
	"unsafe"

	"github.com/d2r2/go-shell"
	"github.com/davecgh/go-spew/spew"
)

type SensorType int

// Implement Stringer interface.
func (this SensorType) String() string {
	if this == DHT11 {
		return "DHT11"
	} else if this == DHT22 {
		return "DHT22"
	} else if this == AM2302 {
		return "AM2302"
	} else {
		return "!!! unknown !!!"
	}
}

const (
	// Most populare sensor
	DHT11 SensorType = iota + 1
	// More expensive and precise than DHT11
	DHT22
	// Aka DHT22
	AM2302 = DHT22
)

// Keep pulse state with how long it lasted.
type Pulse struct {
	Value    byte
	Duration time.Duration
}

// Activate sensor and get back bunch of pulses for further decoding.
// C function call wrapper.
func dialDHTxxAndGetResponse(pin int, boostPerfFlag bool) ([]Pulse, error) {
	var arr *C.int32_t
	var arrLen C.int32_t
	var list []int32
	var boost C.int32_t = 0
	var err2 *C.Error
	if boostPerfFlag {
		boost = 1
	}
	// Return array: [pulse, duration, pulse, duration, ...]
	r := C.dial_DHTxx_and_read(C.int32_t(pin), boost, &arr, &arrLen, &err2)
	if r == -1 {
		var err error
		if err2 != nil {
			msg := C.GoString(err2.message)
			err = errors.New(spew.Sprintf("Error during call C.dial_DHTxx_and_read(): %v", msg))
			C.free_error(err2)
		} else {
			err = errors.New(spew.Sprintf("Error during call C.dial_DHTxx_and_read()"))
		}
		return nil, err
	}
	defer C.free(unsafe.Pointer(arr))
	// Convert original C array arr to Go slice list
	h := (*reflect.SliceHeader)(unsafe.Pointer(&list))
	h.Data = uintptr(unsafe.Pointer(arr))
	h.Len = int(arrLen)
	h.Cap = int(arrLen)
	pulses := make([]Pulse, len(list)/2)
	// Convert original int array ([pulse, duration, pulse, duration, ...])
	// to Pulse struct array
	for i := 0; i < len(list)/2; i++ {
		var value byte = 0
		if list[i*2] != 0 {
			value = 1
		}
		pulses[i] = Pulse{Value: value,
			Duration: time.Duration(list[i*2+1]) * time.Microsecond}
	}
	return pulses, nil
}

// TODO write comment to function
func decodeByte(pulses []Pulse, start int) (byte, error) {
	if len(pulses)-start < 16 {
		return 0, fmt.Errorf("Can't decode byte, since range between "+
			"index and array length is less than 16: %d, %d", start, len(pulses))
	}
	var b int = 0
	for i := 0; i < 8; i++ {
		pulseL := pulses[start+i*2]
		pulseH := pulses[start+i*2+1]
		if pulseL.Value != 0 {
			return 0, fmt.Errorf("Low edge value expected at index %d", start+i*2)
		}
		if pulseH.Value == 0 {
			return 0, fmt.Errorf("High edge value expected at index %d", start+i*2+1)
		}
		const HIGH_DUR_MAX = (70 + (70 + 54)) / 2 * time.Microsecond
		// Calc average value between 24us (bit 0) and 70us (bit 1).
		// Everything that less than this param is bit 0, bigger - bit 1.
		const HIGH_DUR_AVG = (24 + (70-24)/2) * time.Microsecond
		if pulseH.Duration > HIGH_DUR_MAX {
			return 0, fmt.Errorf("High edge value duration %v exceed "+
				"expected maximum amount %v", pulseH.Duration, HIGH_DUR_MAX)
		}
		if pulseH.Duration > HIGH_DUR_AVG {
			//fmt.Printf("bit %d is high\n", 7-i)
			b = b | (1 << uint(7-i))
		}
	}
	return byte(b), nil
}

// Decode bunch of pulse read from DHTxx sensors.
// Use pdf specifications from /docs folder to read 5 bytes and
// convert them to temperature and humidity.
func decodeDHTxxPulses(sensorType SensorType, pulses []Pulse) (temperature float32,
	humidity float32, err error) {
	if len(pulses) == 85 {
		pulses = pulses[3:]
	} else if len(pulses) == 84 {
		pulses = pulses[2:]
	} else if len(pulses) == 83 {
		pulses = pulses[1:]
	} else if len(pulses) != 82 {
		printPulseArrayForDebug(pulses)
		return -1, -1, fmt.Errorf("Can't decode pulse array received from "+
			"DHTxx sensor, since incorrect length: %d", len(pulses))
	}
	pulses = pulses[:80]
	// Decode 1st byte
	b0, err := decodeByte(pulses, 0)
	if err != nil {
		return -1, -1, err
	}
	// Decode 2nd byte
	b1, err := decodeByte(pulses, 16)
	if err != nil {
		return -1, -1, err
	}
	// Decode 3rd byte
	b2, err := decodeByte(pulses, 32)
	if err != nil {
		return -1, -1, err
	}
	// Decode 4th byte
	b3, err := decodeByte(pulses, 48)
	if err != nil {
		return -1, -1, err
	}
	// Decode 5th byte: control sum to verify all data received from sensor
	sum, err := decodeByte(pulses, 64)
	if err != nil {
		return -1, -1, err
	}
	// Produce data consistency check
	calcSum := byte(b0 + b1 + b2 + b3)
	if sum != calcSum {
		err := errors.New(spew.Sprintf(
			"CRCs doesn't match: checksum from sensor(%v) != "+
				"calculated checksum(%v=%v+%v+%v+%v)",
			sum, calcSum, b0, b1, b2, b3))
		return -1, -1, err
	} else {
		lg.Debugf("CRCs verified: checksum from sensor(%v) = calculated checksum(%v=%v+%v+%v+%v)",
			sum, calcSum, b0, b1, b2, b3)
	}
	// Debug output for 5 bytes
	lg.Debugf("Decoded from DHTxx sensor: [%d, %d, %d, %d, %d]", b0, b1, b2, b3, sum)
	// Extract temprature and humidity depending on sensor type
	temperature, humidity = 0.0, 0.0
	if sensorType == DHT11 {
		humidity = float32(b0)
		temperature = float32(b2)
	} else if sensorType == DHT22 {
		humidity = (float32(b0)*256 + float32(b1)) / 10.0
		temperature = (float32(b2&0x7F)*256 + float32(b3)) / 10.0
		if b2&0x80 != 0 {
			temperature *= -1.0
		}
	}
	// Additional check for data correctness
	if humidity > 100.0 {
		return -1, -1, fmt.Errorf("Humidity value exceed 100%%: %v", humidity)
	}
	// Success
	return temperature, humidity, nil
}

// Print bunch of pulses for debug purpose.
func printPulseArrayForDebug(pulses []Pulse) {
	// var buf bytes.Buffer
	// for i, pulse := range pulses {
	// 	buf.WriteString(fmt.Sprintf("pulse %3d: %v, %v\n", i,
	// 		pulse.Value, pulse.Duration))
	// }
	// lg.Debugf("Pulse count %d:\n%v", len(pulses), buf.String())
	lg.Debugf("Pulses received from DHTxx sensor: %v", pulses)
}

// Send activation request to DHTxx sensor via specific pin.
// Then decode pulses sent back with asynchronous
// protocol specific for DHTxx sensors.
//
// Input parameters:
// 1) sensor type: DHT11, DHT22 (aka AM2302);
// 2) pin number from GPIO connector to interract with sensor;
// 3) boost GPIO performance flag should be used for old devices
// such as Raspberry PI 1 (this will require root privileges).
//
// Return:
// 1) temperature in Celsius;
// 2) humidity in percent;
// 3) error if present.
func ReadDHTxx(sensorType SensorType, pin int,
	boostPerfFlag bool) (temperature float32, humidity float32, err error) {
	// Activate sensor and read data to pulses array
	pulses, err := dialDHTxxAndGetResponse(pin, boostPerfFlag)
	if err != nil {
		return -1, -1, err
	}
	// Output debug information
	printPulseArrayForDebug(pulses)
	// Decode pulses
	temp, hum, err := decodeDHTxxPulses(sensorType, pulses)
	if err != nil {
		return -1, -1, err
	}
	return temp, hum, nil
}

// Send activation request to DHTxx sensor via specific pin.
// Then decode pulses sent back with asynchronous
// protocol specific for DHTxx sensors. Retry n times in case of failure.
//
// Input parameters:
// 1) sensor type: DHT11, DHT22 (aka AM2302);
// 2) pin number from gadget GPIO to interract with sensor;
// 3) boost GPIO performance flag should be used for old devices
// such as Raspberry PI 1 (this will require root privileges);
// 4) how many times to retry until success either сounter is zeroed.
//
// Return:
// 1) temperature in Celsius;
// 2) humidity in percent;
// 3) number of extra retries data from sensor;
// 4) error if present.
func ReadDHTxxWithRetry(sensorType SensorType, pin int, boostPerfFlag bool,
	retry int) (temperature float32, humidity float32, retried int, err error) {
	// Use done channel as a trigger to exit from goroutine
	// waiting for OS termination events.
	done := make(chan struct{})
	defer close(done)
	// Create context with cancelation possibility.
	ctx, cancel := context.WithCancel(context.Background())
	// Run goroutine waiting for OS termantion events, either Ctrl+C.
	shell.CloseContextOnKillSignal(cancel, done)
	retried = 0
	for {
		temp, hum, err := ReadDHTxx(sensorType, pin, boostPerfFlag)
		if err != nil {
			if retry > 0 {
				lg.Warning(err)
				retry--
				retried++
				select {
				case <-ctx.Done():
					// Interrupt loop, if pending termination
					return -1, -1, retried, errors.New("Termination pending...")
				default:
					// Sleep before new attempt
					time.Sleep(1500 * time.Millisecond)
					continue
				}
			}
			return -1, -1, retried, err
		}
		return temp, hum, retried, nil
	}
}
<commit_msg>Add call with context param<commit_after>package dht

// #include "dht.go.h"
// #cgo LDFLAGS: -lrt
import "C"

import (
	"context"
	"errors"
	"fmt"
	"reflect"
	"time"
	"unsafe"

	"github.com/d2r2/go-shell"
	"github.com/davecgh/go-spew/spew"
)

type SensorType int

// Implement Stringer interface.
func (this SensorType) String() string {
	if this == DHT11 {
		return "DHT11"
	} else if this == DHT22 {
		return "DHT22"
	} else if this == AM2302 {
		return "AM2302"
	} else {
		return "!!! unknown !!!"
	}
}

const (
	// Most populare sensor
	DHT11 SensorType = iota + 1
	// More expensive and precise than DHT11
	DHT22
	// Aka DHT22
	AM2302 = DHT22
)

// Keep pulse state with how long it lasted.
type Pulse struct {
	Value    byte
	Duration time.Duration
}

// Activate sensor and get back bunch of pulses for further decoding.
// C function call wrapper.
func dialDHTxxAndGetResponse(pin int, boostPerfFlag bool) ([]Pulse, error) {
	var arr *C.int32_t
	var arrLen C.int32_t
	var list []int32
	var boost C.int32_t = 0
	var err2 *C.Error
	if boostPerfFlag {
		boost = 1
	}
	// Return array: [pulse, duration, pulse, duration, ...]
	r := C.dial_DHTxx_and_read(C.int32_t(pin), boost, &arr, &arrLen, &err2)
	if r == -1 {
		var err error
		if err2 != nil {
			msg := C.GoString(err2.message)
			err = errors.New(spew.Sprintf("Error during call C.dial_DHTxx_and_read(): %v", msg))
			C.free_error(err2)
		} else {
			err = errors.New(spew.Sprintf("Error during call C.dial_DHTxx_and_read()"))
		}
		return nil, err
	}
	defer C.free(unsafe.Pointer(arr))
	// Convert original C array arr to Go slice list
	h := (*reflect.SliceHeader)(unsafe.Pointer(&list))
	h.Data = uintptr(unsafe.Pointer(arr))
	h.Len = int(arrLen)
	h.Cap = int(arrLen)
	pulses := make([]Pulse, len(list)/2)
	// Convert original int array ([pulse, duration, pulse, duration, ...])
	// to Pulse struct array
	for i := 0; i < len(list)/2; i++ {
		var value byte = 0
		if list[i*2] != 0 {
			value = 1
		}
		pulses[i] = Pulse{Value: value,
			Duration: time.Duration(list[i*2+1]) * time.Microsecond}
	}
	return pulses, nil
}

// TODO write comment to function
func decodeByte(pulses []Pulse, start int) (byte, error) {
	if len(pulses)-start < 16 {
		return 0, fmt.Errorf("Can't decode byte, since range between "+
			"index and array length is less than 16: %d, %d", start, len(pulses))
	}
	var b int = 0
	for i := 0; i < 8; i++ {
		pulseL := pulses[start+i*2]
		pulseH := pulses[start+i*2+1]
		if pulseL.Value != 0 {
			return 0, fmt.Errorf("Low edge value expected at index %d", start+i*2)
		}
		if pulseH.Value == 0 {
			return 0, fmt.Errorf("High edge value expected at index %d", start+i*2+1)
		}
		const HIGH_DUR_MAX = (70 + (70 + 54)) / 2 * time.Microsecond
		// Calc average value between 24us (bit 0) and 70us (bit 1).
		// Everything that less than this param is bit 0, bigger - bit 1.
		const HIGH_DUR_AVG = (24 + (70-24)/2) * time.Microsecond
		if pulseH.Duration > HIGH_DUR_MAX {
			return 0, fmt.Errorf("High edge value duration %v exceed "+
				"expected maximum amount %v", pulseH.Duration, HIGH_DUR_MAX)
		}
		if pulseH.Duration > HIGH_DUR_AVG {
			//fmt.Printf("bit %d is high\n", 7-i)
			b = b | (1 << uint(7-i))
		}
	}
	return byte(b), nil
}

// Decode bunch of pulse read from DHTxx sensors.
// Use pdf specifications from /docs folder to read 5 bytes and
// convert them to temperature and humidity.
func decodeDHTxxPulses(sensorType SensorType, pulses []Pulse) (temperature float32,
	humidity float32, err error) {
	if len(pulses) == 85 {
		pulses = pulses[3:]
	} else if len(pulses) == 84 {
		pulses = pulses[2:]
	} else if len(pulses) == 83 {
		pulses = pulses[1:]
	} else if len(pulses) != 82 {
		printPulseArrayForDebug(pulses)
		return -1, -1, fmt.Errorf("Can't decode pulse array received from "+
			"DHTxx sensor, since incorrect length: %d", len(pulses))
	}
	pulses = pulses[:80]
	// Decode 1st byte
	b0, err := decodeByte(pulses, 0)
	if err != nil {
		return -1, -1, err
	}
	// Decode 2nd byte
	b1, err := decodeByte(pulses, 16)
	if err != nil {
		return -1, -1, err
	}
	// Decode 3rd byte
	b2, err := decodeByte(pulses, 32)
	if err != nil {
		return -1, -1, err
	}
	// Decode 4th byte
	b3, err := decodeByte(pulses, 48)
	if err != nil {
		return -1, -1, err
	}
	// Decode 5th byte: control sum to verify all data received from sensor
	sum, err := decodeByte(pulses, 64)
	if err != nil {
		return -1, -1, err
	}
	// Produce data consistency check
	calcSum := byte(b0 + b1 + b2 + b3)
	if sum != calcSum {
		err := errors.New(spew.Sprintf(
			"CRCs doesn't match: checksum from sensor(%v) != "+
				"calculated checksum(%v=%v+%v+%v+%v)",
			sum, calcSum, b0, b1, b2, b3))
		return -1, -1, err
	} else {
		lg.Debugf("CRCs verified: checksum from sensor(%v) = calculated checksum(%v=%v+%v+%v+%v)",
			sum, calcSum, b0, b1, b2, b3)
	}
	// Debug output for 5 bytes
	lg.Debugf("Decoded from DHTxx sensor: [%d, %d, %d, %d, %d]", b0, b1, b2, b3, sum)
	// Extract temprature and humidity depending on sensor type
	temperature, humidity = 0.0, 0.0
	if sensorType == DHT11 {
		humidity = float32(b0)
		temperature = float32(b2)
	} else if sensorType == DHT22 {
		humidity = (float32(b0)*256 + float32(b1)) / 10.0
		temperature = (float32(b2&0x7F)*256 + float32(b3)) / 10.0
		if b2&0x80 != 0 {
			temperature *= -1.0
		}
	}
	// Additional check for data correctness
	if humidity > 100.0 {
		return -1, -1, fmt.Errorf("Humidity value exceed 100%%: %v", humidity)
	}
	// Success
	return temperature, humidity, nil
}

// Print bunch of pulses for debug purpose.
func printPulseArrayForDebug(pulses []Pulse) {
	// var buf bytes.Buffer
	// for i, pulse := range pulses {
	// 	buf.WriteString(fmt.Sprintf("pulse %3d: %v, %v\n", i,
	// 		pulse.Value, pulse.Duration))
	// }
	// lg.Debugf("Pulse count %d:\n%v", len(pulses), buf.String())
	lg.Debugf("Pulses received from DHTxx sensor: %v", pulses)
}

// Send activation request to DHTxx sensor via specific pin.
// Then decode pulses sent back with asynchronous
// protocol specific for DHTxx sensors.
//
// Input parameters:
// 1) sensor type: DHT11, DHT22 (aka AM2302);
// 2) pin number from GPIO connector to interract with sensor;
// 3) boost GPIO performance flag should be used for old devices
// such as Raspberry PI 1 (this will require root privileges).
//
// Return:
// 1) temperature in Celsius;
// 2) humidity in percent;
// 3) error if present.
func ReadDHTxx(sensorType SensorType, pin int,
	boostPerfFlag bool) (temperature float32, humidity float32, err error) {
	// Activate sensor and read data to pulses array
	pulses, err := dialDHTxxAndGetResponse(pin, boostPerfFlag)
	if err != nil {
		return -1, -1, err
	}
	// Output debug information
	printPulseArrayForDebug(pulses)
	// Decode pulses
	temp, hum, err := decodeDHTxxPulses(sensorType, pulses)
	if err != nil {
		return -1, -1, err
	}
	return temp, hum, nil
}

// Send activation request to DHTxx sensor via specific pin.
// Then decode pulses sent back with asynchronous
// protocol specific for DHTxx sensors. Retry n times in case of failure.
//
// Input parameters:
// 1) sensor type: DHT11, DHT22 (aka AM2302);
// 2) pin number from gadget GPIO to interract with sensor;
// 3) boost GPIO performance flag should be used for old devices
// such as Raspberry PI 1 (this will require root privileges);
// 4) how many times to retry until success either сounter is zeroed.
//
// Return:
// 1) temperature in Celsius;
// 2) humidity in percent;
// 3) number of extra retries data from sensor;
// 4) error if present.
func ReadDHTxxWithRetry(sensorType SensorType, pin int, boostPerfFlag bool,
	retry int) (temperature float32, humidity float32, retried int, err error) {
	// Create default context
	ctx := context.Background()
	// Reroute call
	return ReadDHTxxWithContextAndRetry(ctx, sensorType, pin,
		boostPerfFlag, retry)
}

// Send activation request to DHTxx sensor via specific pin.
// Then decode pulses sent back with asynchronous
// protocol specific for DHTxx sensors. Retry n times in case of failure.
//
// Input parameters:
// 1) parent context; could be used to manage life-cycle
//  of sensor request session from code outside;
// 2) sensor type: DHT11, DHT22 (aka AM2302);
// 3) pin number from gadget GPIO to interract with sensor;
// 4) boost GPIO performance flag should be used for old devices
//  such as Raspberry PI 1 (this will require root privileges);
// 5) how many times to retry until success either сounter is zeroed.
//
// Return:
// 1) temperature in Celsius;
// 2) humidity in percent;
// 3) number of extra retries data from sensor;
// 4) error if present.
func ReadDHTxxWithContextAndRetry(parent context.Context, sensorType SensorType, pin int,
	boostPerfFlag bool, retry int) (temperature float32, humidity float32, retried int, err error) {
	// Use done channel as a trigger to exit from signal waiting goroutine.
	done := make(chan struct{})
	defer close(done)
	// Create context with cancelation possibility.
	ctx, cancel := context.WithCancel(parent)
	// Run goroutine waiting for OS termantion events, including keyboard Ctrl+C.
	shell.CloseContextOnKillSignal(cancel, done)
	retried = 0
	for {
		temp, hum, err := ReadDHTxx(sensorType, pin, boostPerfFlag)
		if err != nil {
			if retry > 0 {
				lg.Warning(err)
				retry--
				retried++
				select {
				// Check for termination request.
				case <-ctx.Done():
					// Interrupt loop, if pending termination.
					return -1, -1, retried, ctx.Err()
				// Sleep before new attempt 1.5 sec.
				case <-time.After(1500 * time.Millisecond):
					continue
				}
			}
			return -1, -1, retried, err
		}
		return temp, hum, retried, nil
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Extended and bugfixes by Miek Gieben

// Package dns implements a full featured interface to the DNS.
// The package allows complete control over what is send out to the DNS.
//
// Resource records are native types. They are not stored in wire format.
// Basic usage pattern for creating a new resource record:
//
//         r := new(RR_TXT)
//         r.Hdr = RR_Header{Name: "a.miek.nl", Rrtype: TypeTXT, Class: ClassINET, Ttl: 3600}
//         r.TXT = "This is the content of the TXT record"
//
// The package dns supports querying, incoming/outgoing Axfr/Ixfr, TSIG, EDNS0,
// dynamic updates, notifies and DNSSEC validation/signing.
//
// Querying the DNS is done by using a Resolver structure. Basic use pattern for creating
// a resolver:
//
//      res := new(Resolver)
//      res.Servers = []string{"127.0.0.1"}
//      m := new(Msg)
//      m.MsgHdr.Recursion_desired = true
//      m.Question = make([]Question, 1)
//      m.Question[0] = Question{"miek.nl", TypeSOA, ClassINET}
//      in, err := res.Query(m)
//
// Server side programming is also supported.
// Basic use pattern for creating an UDP DNS server:
//
//      func handle(d *dns.Conn, i *dns.Msg) { /* handle request */ }
//
//      func listen(addr string, e chan os.Error) {
//            err := dns.ListenAndServeUDP(addr, handle)
//            e <- err
//      }
//      err := make(chan os.Error)
//      go listen("127.0.0.1:8053", err)
//
package dns

import (
        "io"
	"os"
	"net"
	"strconv"
)

const (
	Year68         = 2 << (32 - 1) // For RFC1982 (Serial Arithmetic) calculations in 32 bits.
	DefaultMsgSize = 4096          // A standard default for larger than 512 packets.
	MaxMsgSize     = 65536         // Largest possible DNS packet.
	DefaultTTL     = 3600          // Default Ttl.
)

// Error represents a DNS error
type Error struct {
	Error   string
	Name    string
	Server  net.Addr
	Timeout bool
}

func (e *Error) String() string {
	if e == nil {
		return "<nil>"
	}
	return e.Error
}

// A Conn is the lowest primative in the dns package.
// A Conn holds both the UDP and TCP connection, but only one
// can be active any given time.
type Conn struct {
	// The current UDP connection.
	UDP *net.UDPConn

	// The current TCP connection.
	TCP *net.TCPConn

	// The remote side of open the connection.
	Addr net.Addr

	// The remote port number of open the connection.
	Port int

        // The remote addr which is going to be dialed.
        RemoteAddr string

	// If TSIG is used, this holds all the information.
        // If unused is must be nil.
	Tsig *Tsig

	// Timeout in sec before giving up on a connection.
	Timeout int

	// Number of attempts to try to Read/Write from/to a
	// connection.
	Attempts int
}

// Dial connects to the remote address raddr on the network net.
// If the string laddr is not empty, it is used as the local address
// for the connection. Any errors are return in err otherwise err is nil.
func Dial(n, laddr, raddr string) (*Conn, os.Error) {
        d := new(Conn)
        c, err := net.Dial(n, laddr, raddr)
        if err != nil {
                return nil, err
        }
        switch n {
        case "tcp":
                d.TCP = c.(*net.TCPConn)
                d.Addr = d.TCP.RemoteAddr()
                d.Port = d.TCP.RemoteAddr().(*net.TCPAddr).Port
        case "udp":
                d.UDP = c.(*net.UDPConn)
                d.Addr = d.UDP.RemoteAddr()
                d.Port = d.UDP.RemoteAddr().(*net.UDPAddr).Port
        }
        return d, nil
}

// Fill in a Conn from a TCPConn. If a is nil, the
// remote address in the connection is used.
func (d *Conn) SetTCPConn(l *net.TCPConn, a net.Addr) {
        d.TCP = l
        d.UDP = nil
        if a == nil {
                d.Addr = l.RemoteAddr()
        }
        d.Port = d.Addr.(*net.TCPAddr).Port
}

// Fill in a Conn from a TCPConn. If a is nil, the
// remote address in the connection is used.
func (d *Conn) SetUDPConn(l *net.UDPConn, a net.Addr) {
        d.TCP = nil
        d.UDP = l
        if a == nil {
                d.Addr = l.RemoteAddr()
        }
        d.Port = d.Addr.(*net.UDPAddr).Port
}

// Create a new buffer of the appropiate size. With
// TCP the buffer is 64K, with UDP the returned buffer
// has a length of 4K bytes.
func (d *Conn) NewBuffer() []byte {
	if d.TCP != nil {
		b := make([]byte, MaxMsgSize)
		return b
	}
	if d.UDP != nil {
		b := make([]byte, DefaultMsgSize)
		return b
	}
	return nil
}

// ReadMsg reads a dns message m from d.
// Any errors of the underlaying Read call are returned.
func (d *Conn) ReadMsg(m *Msg) os.Error {
	in := d.NewBuffer()
	n, err := d.Read(in)
	if err != nil {
		return err
	}
	in = in[:n]
	ok := m.Unpack(in)
	if !ok {
                return ErrUnpack
	}
	return nil
}

// WriteMsg writes dns message m to d.
// Any errors of the underlaying Write call are returned.
func (d *Conn) WriteMsg(m *Msg) os.Error {
	out, ok := m.Pack()
	if !ok {
		return ErrPack
	}
	_, err := d.Write(out)
	if err != nil {
		return err
	}
	return nil
}

// Read implements the standard Read interface:
// it reads from d. If there was an error
// reading that error is returned; otherwise err is nil.
func (d *Conn) Read(p []byte) (n int, err os.Error) {
	if d.UDP != nil && d.TCP != nil {
		return 0, ErrConn
	}
	switch {
	case d.UDP != nil:
		var addr net.Addr
		n, addr, err = d.UDP.ReadFromUDP(p)
		if err != nil {
			return n, err
		}
		d.Addr = addr
		d.Port = addr.(*net.UDPAddr).Port
	case d.TCP != nil:
		if len(p) < 1 {
			return 0, io.ErrShortBuffer
		}
		n, err = d.TCP.Read(p[0:2])
		if err != nil || n != 2 {
			return n, err
		}
		d.Addr = d.TCP.RemoteAddr()
		d.Port = d.TCP.RemoteAddr().(*net.TCPAddr).Port
		l, _ := unpackUint16(p[0:2], 0)
		if l == 0 {
			return 0, ErrShortRead
		}
		if int(l) > len(p) {
			return int(l), io.ErrShortBuffer
		}
		n, err = d.TCP.Read(p[:l])
		if err != nil {
			return n, err
		}
		i := n
		for i < int(l) {
			j, err := d.TCP.Read(p[i:int(l)])
			if err != nil {
				return i, err
			}
			i += j
		}
		n = i
	}
	if d.Tsig != nil {
		// Check the TSIG that we should be read
		_, err = d.Tsig.Verify(p)
		if err != nil {
			return
		}
	}
	return
}

// Write implements the standard Write interface:
// It write data to d. If there was an error writing
// that error is returned; otherwise err is nil
func (d *Conn) Write(p []byte) (n int, err os.Error) {
	if d.UDP != nil && d.TCP != nil {
		return 0, ErrConn
	}

	var attempts int
	var q []byte
	if d.Attempts == 0 {
		attempts = 1
	} else {
		attempts = d.Attempts
	}
	d.SetTimeout()
	if d.Tsig != nil {
		// Create a new buffer with the TSIG added.
		q, err = d.Tsig.Generate(p)
		if err != nil {
			return 0, err
		}
	} else {
		q = p
	}

	switch {
	case d.UDP != nil:
		for a := 0; a < attempts; a++ {
			n, err = d.UDP.WriteTo(q, d.Addr)
			if err != nil {
				if e, ok := err.(net.Error); ok && e.Timeout() {
					continue
				}
				return 0, err
			}
		}
	case d.TCP != nil:
		for a := 0; a < attempts; a++ {
			l := make([]byte, 2)
			l[0], l[1] = packUint16(uint16(len(q)))
			n, err = d.TCP.Write(l)
			if err != nil {
				if e, ok := err.(net.Error); ok && e.Timeout() {
					continue
				}
				return n, err
			}
			if n != 2 {
				return n, io.ErrShortWrite
			}
			n, err = d.TCP.Write(q)
			if err != nil {
				if e, ok := err.(net.Error); ok && e.Timeout() {
					continue
				}
				return n, err
			}
			i := n
			if i < len(q) {
				j, err := d.TCP.Write(q[i:len(q)])
				if err != nil {
					if e, ok := err.(net.Error); ok && e.Timeout() {
						// We are half way in our write...
						continue
					}
					return i, err
				}
				i += j
			}
			n = i
		}
	}
	return
}

// Close closes the connection in d. Possible
// errors are returned in err; otherwise it is nil.
func (d *Conn) Close() (err os.Error) {
	if d.UDP != nil && d.TCP != nil {
		return ErrConn
	}
	switch {
	case d.UDP != nil:
		err = d.UDP.Close()
	case d.TCP != nil:
		err = d.TCP.Close()
	}
	return
}

// SetTimeout sets the timeout of the socket
// that is contained in d.
func (d *Conn) SetTimeout() (err os.Error) {
	var sec int64
	if d.UDP != nil && d.TCP != nil {
		return ErrConn
	}
	sec = int64(d.Timeout)
	if sec == 0 {
		sec = 1
	}
	if d.UDP != nil {
		err = d.TCP.SetTimeout(sec * 1e9)
	}
	if d.TCP != nil {
		err = d.TCP.SetTimeout(sec * 1e9)
	}
	return
}

// Exchange combines a Write and a Read.
// First the request is written to d and then it waits
// for a reply with Read.
// If nosend is true, the write is skipped.
func (d *Conn) Exchange(request []byte, nosend bool) (reply []byte, err os.Error) {
	var n int
	if !nosend {
		n, err = d.Write(request)
		if err != nil {
			return nil, err
		}
	}
	reply = d.NewBuffer()
	n, err = d.Read(reply)
	if err != nil {
		return nil, err
	}
	reply = reply[:n]
	return
}

type RR interface {
	Header() *RR_Header
	String() string
}

// An RRset is a slice of RRs.
type RRset []RR

func (r RRset) Len() int           { return len(r) }
func (r RRset) Less(i, j int) bool { return r[i].Header().Name < r[j].Header().Name }
func (r RRset) Swap(i, j int)      { r[i], r[j] = r[j], r[i] }

// Check if the RRset is RFC 2181 compliant
func (r RRset) Ok() bool {
	ttl := r[0].Header().Ttl
	name := r[0].Header().Name
	class := r[0].Header().Class
	for _, rr := range r[1:] {
		if rr.Header().Ttl != ttl {
			return false
		}
		if rr.Header().Name != name {
			return false
		}
		if rr.Header().Class != class {
			return false
		}
	}
	return true
}

// DNS resource records.
// There are many types of messages,
// but they all share the same header.
type RR_Header struct {
	Name     string "domain-name"
	Rrtype   uint16
	Class    uint16
	Ttl      uint32
	Rdlength uint16 // length of data after header
}

func (h *RR_Header) Header() *RR_Header {
	return h
}

func (h *RR_Header) String() string {
	var s string

	if h.Rrtype == TypeOPT {
		s = ";"
		// and maybe other things
	}

	if len(h.Name) == 0 {
		s += ".\t"
	} else {
		s += h.Name + "\t"
	}
	s = s + strconv.Itoa(int(h.Ttl)) + "\t"

	if _, ok := Class_str[h.Class]; ok {
		s += Class_str[h.Class] + "\t"
	} else {
		s += "CLASS" + strconv.Itoa(int(h.Class)) + "\t"
	}

	if _, ok := Rr_str[h.Rrtype]; ok {
		s += Rr_str[h.Rrtype] + "\t"
	} else {
		s += "TYPE" + strconv.Itoa(int(h.Rrtype)) + "\t"
	}
	return s
}

// Return the number of labels in a domain name.
func LabelCount(a string) (c uint8) {
	// walk the string and count the dots
	// except when it is escaped
	esc := false
	for _, v := range a {
		switch v {
		case '.':
			if esc {
				esc = !esc
				continue
			}
			c++
		case '\\':
			esc = true
		}
	}
	return
}
<commit_msg>Add mangle to Conn<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Extended and bugfixes by Miek Gieben

// Package dns implements a full featured interface to the DNS.
// The package allows complete control over what is send out to the DNS.
//
// Resource records are native types. They are not stored in wire format.
// Basic usage pattern for creating a new resource record:
//
//         r := new(RR_TXT)
//         r.Hdr = RR_Header{Name: "a.miek.nl", Rrtype: TypeTXT, Class: ClassINET, Ttl: 3600}
//         r.TXT = "This is the content of the TXT record"
//
// The package dns supports querying, incoming/outgoing Axfr/Ixfr, TSIG, EDNS0,
// dynamic updates, notifies and DNSSEC validation/signing.
//
// Querying the DNS is done by using a Resolver structure. Basic use pattern for creating
// a resolver:
//
//      res := new(Resolver)
//      res.Servers = []string{"127.0.0.1"}
//      m := new(Msg)
//      m.MsgHdr.Recursion_desired = true
//      m.Question = make([]Question, 1)
//      m.Question[0] = Question{"miek.nl", TypeSOA, ClassINET}
//      in, err := res.Query(m)
//
// Server side programming is also supported.
// Basic use pattern for creating an UDP DNS server:
//
//      func handle(d *dns.Conn, i *dns.Msg) { /* handle request */ }
//
//      func listen(addr string, e chan os.Error) {
//            err := dns.ListenAndServeUDP(addr, handle)
//            e <- err
//      }
//      err := make(chan os.Error)
//      go listen("127.0.0.1:8053", err)
//
package dns

import (
        "io"
	"os"
	"net"
	"strconv"
)

const (
	Year68         = 2 << (32 - 1) // For RFC1982 (Serial Arithmetic) calculations in 32 bits.
	DefaultMsgSize = 4096          // A standard default for larger than 512 packets.
	MaxMsgSize     = 65536         // Largest possible DNS packet.
	DefaultTTL     = 3600          // Default Ttl.
)

// Error represents a DNS error
type Error struct {
	Error   string
	Name    string
	Server  net.Addr
	Timeout bool
}

func (e *Error) String() string {
	if e == nil {
		return "<nil>"
	}
	return e.Error
}

// A Conn is the lowest primative in the dns package.
// A Conn holds both the UDP and TCP connection, but only one
// can be active any given time.
type Conn struct {
	// The current UDP connection.
	UDP *net.UDPConn

	// The current TCP connection.
	TCP *net.TCPConn

	// The remote side of open the connection.
	Addr net.Addr

	// The remote port number of open the connection.
	Port int

	// If TSIG is used, this holds all the information.
        // If unused is must be nil.
	Tsig *Tsig

	// Timeout in sec before giving up on a connection.
	Timeout int

	// Number of attempts to try to Read/Write from/to a
	// connection.
	Attempts int

        // The remote addr which is going to be dialed.
        RemoteAddr string

        // Mangle the packet before writing it be feeding
        // it through this function.
        Mangle func([]byte) []byte

        // rtt times?
}

// Dial connects to the remote address raddr on the network net.
// If the string laddr is not empty, it is used as the local address
// for the connection. Any errors are return in err otherwise err is nil.
func Dial(n, laddr, raddr string) (*Conn, os.Error) {
        d := new(Conn)
        c, err := net.Dial(n, laddr, raddr)
        if err != nil {
                return nil, err
        }
        switch n {
        case "tcp":
                d.TCP = c.(*net.TCPConn)
                d.Addr = d.TCP.RemoteAddr()
                d.Port = d.TCP.RemoteAddr().(*net.TCPAddr).Port
        case "udp":
                d.UDP = c.(*net.UDPConn)
                d.Addr = d.UDP.RemoteAddr()
                d.Port = d.UDP.RemoteAddr().(*net.UDPAddr).Port
        }
        return d, nil
}

// Fill in a Conn from a TCPConn. If a is nil, the
// remote address in the connection is used.
func (d *Conn) SetTCPConn(l *net.TCPConn, a net.Addr) {
        d.TCP = l
        d.UDP = nil
        if a == nil {
                d.Addr = l.RemoteAddr()
        }
        d.Port = d.Addr.(*net.TCPAddr).Port
}

// Fill in a Conn from a TCPConn. If a is nil, the
// remote address in the connection is used.
func (d *Conn) SetUDPConn(l *net.UDPConn, a net.Addr) {
        d.TCP = nil
        d.UDP = l
        if a == nil {
                d.Addr = l.RemoteAddr()
        }
        d.Port = d.Addr.(*net.UDPAddr).Port
}

// Create a new buffer of the appropiate size. With
// TCP the buffer is 64K, with UDP the returned buffer
// has a length of 4K bytes.
func (d *Conn) NewBuffer() []byte {
	if d.TCP != nil {
		b := make([]byte, MaxMsgSize)
		return b
	}
	if d.UDP != nil {
		b := make([]byte, DefaultMsgSize)
		return b
	}
	return nil
}

// ReadMsg reads a dns message m from d.
// Any errors of the underlaying Read call are returned.
func (d *Conn) ReadMsg(m *Msg) os.Error {
	in := d.NewBuffer()
	n, err := d.Read(in)
	if err != nil {
		return err
	}
	in = in[:n]
	ok := m.Unpack(in)
	if !ok {
                return ErrUnpack
	}
	return nil
}

// WriteMsg writes dns message m to d.
// Any errors of the underlaying Write call are returned.
func (d *Conn) WriteMsg(m *Msg) os.Error {
	out, ok := m.Pack()
	if !ok {
		return ErrPack
	}
	_, err := d.Write(out)
	if err != nil {
		return err
	}
	return nil
}

// Read implements the standard Read interface:
// it reads from d. If there was an error
// reading that error is returned; otherwise err is nil.
func (d *Conn) Read(p []byte) (n int, err os.Error) {
	if d.UDP != nil && d.TCP != nil {
		return 0, ErrConn
	}
	switch {
	case d.UDP != nil:
		var addr net.Addr
		n, addr, err = d.UDP.ReadFromUDP(p)
		if err != nil {
			return n, err
		}
		d.Addr = addr
		d.Port = addr.(*net.UDPAddr).Port
	case d.TCP != nil:
		if len(p) < 1 {
			return 0, io.ErrShortBuffer
		}
		n, err = d.TCP.Read(p[0:2])
		if err != nil || n != 2 {
			return n, err
		}
		d.Addr = d.TCP.RemoteAddr()
		d.Port = d.TCP.RemoteAddr().(*net.TCPAddr).Port
		l, _ := unpackUint16(p[0:2], 0)
		if l == 0 {
			return 0, ErrShortRead
		}
		if int(l) > len(p) {
			return int(l), io.ErrShortBuffer
		}
		n, err = d.TCP.Read(p[:l])
		if err != nil {
			return n, err
		}
		i := n
		for i < int(l) {
			j, err := d.TCP.Read(p[i:int(l)])
			if err != nil {
				return i, err
			}
			i += j
		}
		n = i
	}
	if d.Tsig != nil {
		// Check the TSIG that we should be read
		_, err = d.Tsig.Verify(p)
		if err != nil {
			return
		}
	}
	return
}

// Write implements the standard Write interface:
// It write data to d. If there was an error writing
// that error is returned; otherwise err is nil
func (d *Conn) Write(p []byte) (n int, err os.Error) {
	if d.UDP != nil && d.TCP != nil {
		return 0, ErrConn
	}

	var attempts int
	var q []byte
	if d.Attempts == 0 {
		attempts = 1
	} else {
		attempts = d.Attempts
	}
        // Mangle before TSIG?
        if d.Mangle != nil {
                p = d.Mangle(p)
        }

	d.SetTimeout()
	if d.Tsig != nil {
		// Create a new buffer with the TSIG added.
		q, err = d.Tsig.Generate(p)
		if err != nil {
			return 0, err
		}
	} else {
		q = p
	}

	switch {
	case d.UDP != nil:
		for a := 0; a < attempts; a++ {
			n, err = d.UDP.WriteTo(q, d.Addr)
			if err != nil {
				if e, ok := err.(net.Error); ok && e.Timeout() {
					continue
				}
				return 0, err
			}
		}
	case d.TCP != nil:
		for a := 0; a < attempts; a++ {
			l := make([]byte, 2)
			l[0], l[1] = packUint16(uint16(len(q)))
			n, err = d.TCP.Write(l)
			if err != nil {
				if e, ok := err.(net.Error); ok && e.Timeout() {
					continue
				}
				return n, err
			}
			if n != 2 {
				return n, io.ErrShortWrite
			}
			n, err = d.TCP.Write(q)
			if err != nil {
				if e, ok := err.(net.Error); ok && e.Timeout() {
					continue
				}
				return n, err
			}
			i := n
			if i < len(q) {
				j, err := d.TCP.Write(q[i:len(q)])
				if err != nil {
					if e, ok := err.(net.Error); ok && e.Timeout() {
						// We are half way in our write...
						continue
					}
					return i, err
				}
				i += j
			}
			n = i
		}
	}
	return
}

// Close closes the connection in d. Possible
// errors are returned in err; otherwise it is nil.
func (d *Conn) Close() (err os.Error) {
	if d.UDP != nil && d.TCP != nil {
		return ErrConn
	}
	switch {
	case d.UDP != nil:
		err = d.UDP.Close()
	case d.TCP != nil:
		err = d.TCP.Close()
	}
	return
}

// SetTimeout sets the timeout of the socket
// that is contained in d.
func (d *Conn) SetTimeout() (err os.Error) {
	var sec int64
	if d.UDP != nil && d.TCP != nil {
		return ErrConn
	}
	sec = int64(d.Timeout)
	if sec == 0 {
		sec = 1
	}
	if d.UDP != nil {
		err = d.TCP.SetTimeout(sec * 1e9)
	}
	if d.TCP != nil {
		err = d.TCP.SetTimeout(sec * 1e9)
	}
	return
}

// Exchange combines a Write and a Read.
// First the request is written to d and then it waits
// for a reply with Read.
// If nosend is true, the write is skipped.
func (d *Conn) Exchange(request []byte, nosend bool) (reply []byte, err os.Error) {
	var n int
	if !nosend {
		n, err = d.Write(request)
		if err != nil {
			return nil, err
		}
	}
	reply = d.NewBuffer()
	n, err = d.Read(reply)
	if err != nil {
		return nil, err
	}
	reply = reply[:n]
	return
}

type RR interface {
	Header() *RR_Header
	String() string
}

// An RRset is a slice of RRs.
type RRset []RR

func (r RRset) Len() int           { return len(r) }
func (r RRset) Less(i, j int) bool { return r[i].Header().Name < r[j].Header().Name }
func (r RRset) Swap(i, j int)      { r[i], r[j] = r[j], r[i] }

// Check if the RRset is RFC 2181 compliant
func (r RRset) Ok() bool {
	ttl := r[0].Header().Ttl
	name := r[0].Header().Name
	class := r[0].Header().Class
	for _, rr := range r[1:] {
		if rr.Header().Ttl != ttl {
			return false
		}
		if rr.Header().Name != name {
			return false
		}
		if rr.Header().Class != class {
			return false
		}
	}
	return true
}

// DNS resource records.
// There are many types of messages,
// but they all share the same header.
type RR_Header struct {
	Name     string "domain-name"
	Rrtype   uint16
	Class    uint16
	Ttl      uint32
	Rdlength uint16 // length of data after header
}

func (h *RR_Header) Header() *RR_Header {
	return h
}

func (h *RR_Header) String() string {
	var s string

	if h.Rrtype == TypeOPT {
		s = ";"
		// and maybe other things
	}

	if len(h.Name) == 0 {
		s += ".\t"
	} else {
		s += h.Name + "\t"
	}
	s = s + strconv.Itoa(int(h.Ttl)) + "\t"

	if _, ok := Class_str[h.Class]; ok {
		s += Class_str[h.Class] + "\t"
	} else {
		s += "CLASS" + strconv.Itoa(int(h.Class)) + "\t"
	}

	if _, ok := Rr_str[h.Rrtype]; ok {
		s += Rr_str[h.Rrtype] + "\t"
	} else {
		s += "TYPE" + strconv.Itoa(int(h.Rrtype)) + "\t"
	}
	return s
}

// Return the number of labels in a domain name.
func LabelCount(a string) (c uint8) {
	// walk the string and count the dots
	// except when it is escaped
	esc := false
	for _, v := range a {
		switch v {
		case '.':
			if esc {
				esc = !esc
				continue
			}
			c++
		case '\\':
			esc = true
		}
	}
	return
}
<|endoftext|>


<commit_before>// Hanayo is the Ripple front-end web server.
package main

// version is the current version of hanayo
const version = "v1.10.3"
<commit_msg>🔼 1.11.0 🔼<commit_after>// Hanayo is the Ripple front-end web server.
package main

// version is the current version of hanayo
const version = "v1.11.0"
<|endoftext|>


<commit_before>// (c) Copyright 2015-2017 JONNALAGADDA Srinivas
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package flow is a tiny workflow engine written in Go (golang).
package flow

import (
	"database/sql"
	"log"
)

const (
	// DefACRoleCount is the default number of roles a group can have
	// in an access context.
	DefACRoleCount = 1
)

var db *sql.DB
var blobsDir string

//

func init() {
	f := log.Flags()
	log.SetFlags(f | log.Llongfile)
}

// RegisterDB provides an already initialised database handle to `flow`.
//
// N.B. This method **MUST** be called before anything else in `flow`.
func RegisterDB(sdb *sql.DB) error {
	if db == nil {
		log.Fatal("given database handle is `nil`")
	}
	db = sdb

	return nil
}

// SetBlobsDir specifies the base directory inside which blob files
// should be stored.
//
// Inside this base directory, 256 subdirectories are created as hex
// `00` through `ff`.  A blob is stored in the subdirectory whose name
// matches the first two hex digits of its SHA1 sum.
//
// N.B. Once set, this MUST NOT change between runs.  Doing so will
// result in loss of all previously stored blobs.  In addition,
// corresponding documents get corrupted.
func SetBlobsDir(base string) error {
	if base == "" {
		log.Fatal("given base directory path is empty")
	}
	blobsDir = base

	return nil
}
<commit_msg>Correct a typing error in `doc.go` to check for registered DB handle<commit_after>// (c) Copyright 2015-2017 JONNALAGADDA Srinivas
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package flow is a tiny workflow engine written in Go (golang).
package flow

import (
	"database/sql"
	"log"
)

const (
	// DefACRoleCount is the default number of roles a group can have
	// in an access context.
	DefACRoleCount = 1
)

var db *sql.DB
var blobsDir string

//

func init() {
	f := log.Flags()
	log.SetFlags(f | log.Llongfile)
}

// RegisterDB provides an already initialised database handle to `flow`.
//
// N.B. This method **MUST** be called before anything else in `flow`.
func RegisterDB(sdb *sql.DB) error {
	if sdb == nil {
		log.Fatal("given database handle is `nil`")
	}
	db = sdb

	return nil
}

// SetBlobsDir specifies the base directory inside which blob files
// should be stored.
//
// Inside this base directory, 256 subdirectories are created as hex
// `00` through `ff`.  A blob is stored in the subdirectory whose name
// matches the first two hex digits of its SHA1 sum.
//
// N.B. Once set, this MUST NOT change between runs.  Doing so will
// result in loss of all previously stored blobs.  In addition,
// corresponding documents get corrupted.
func SetBlobsDir(base string) error {
	if base == "" {
		log.Fatal("given base directory path is empty")
	}
	blobsDir = base

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Eleme Inc. All rights reserved.

/*

Banshee is a real-time anomalies(outliers) detection system for periodic
metrics.

Use Case

We are using it to monitor our website and rpc services intefaces, including
called frequency, response time and exception calls. Our services send statistics
to statsd, statsd aggregates them every 10 seconds and broadcasts the results to
its backends including banshee, banshee analyzes current metrics with history
data, calculates the trending and alerts us if the trending behaves anomalous.

For example, we have an api named get_user, this api's response time (in
milliseconds) is reported to banshee from statsd every 10 seconds:
	20, 21, 21, 22, 23, 19, 18, 21, 22, 20, ..., 300

Banshee will catch the latest metric 300 and report it as an anomaly.

Why don't we just set a fixed threshold instead (i.e. 200ms)? This may also works
but it is boring and hard to maintain a lot of thresholds. Banshee will analyze
metric trendings automatically, it will find the "thresholds" automatically.

Features

1. Designed for periodic metrics. Reality metrics are always with periodicity,
banshee only peeks metrics with the same "phase" to detect.

2. Multiple alerting rule configuration options, to alert via fixed-thresholds or
via anomalous trendings.

3. Coming with anomalies visualization webapp and alerting rules admin panels.

4. Require no extra storage services, banshee handles storage on disk by itself.

Requirements

1. Go >= 1.4 and godep.

2. Node and gulp.

Build

1. Clone the repo.

2. Build binary via `make`.

3. Build static files via `make static`.

Command Line

Usage:
	banshee [-c config] [-d] [-v]

Flags:
	-c config
		Load config from file.
	-d
		Turn on debug mode.
	-v
		Show version.

Configuration

See package config.

Statsd Integration

In order to forward metrics to banshee from statsd, we need to add the
npm module statsd-banshee to statsd's banckends:

1. Install statsd-banshee on your statsd servers:

	$ cd path/to/statsd
	$ npm install statsd-banshee

2. Add module statsd-banshee to statsd's backends in config.js:

	{
	, backends: ['statsd-banshee']
	, bansheeHost: 'localhost'
	, bansheePort: 2015
	}

Migrate from bell

Require bell.js v2.0+ and banshee v0.0.7+:

	./migrate -from bell.db -to banshee.db -with-projs -with-users
	mv banshee.db path/to/storage/admin

Compontents

Banshee have 4 compontents and they are running in the same process:

1. Detector is to detect incoming metrics with history data and store the
results.

2. Webapp is to visualize the detection results and provides panels to manage
alerting rules, projects and users.

3. Alerter is to send sms and emails once anomalies are found.

4. Cleaner is to clean outdated metrics from storage.

Alerting Sender

See package alerter and alerter/exampleCommand.

Deployment

Just use the shell release.sh, it will build the binary, static files and pack them together:

	./release.sh
	scp bansheeA.B.C.linux-amd64.tar.gz remote-host-name:/target/path

You may need a linux host to build the binary, cause: https://github.com/mattn/go-sqlite3/issues/106

Upgrade

Just pull the latest code:

	git remote add origin git@github.com:eleme/banshee.git
	git pull upstream master

Note that the admin storage sqlite3 schema will be auto-migrated.

Implementation Details

1. Detection algorithms, see package detector.

2. Detector input net protocol, see package detector.

3. Storage, see package storage.

4. Filter, see package filter.

License

MIT (c) eleme, inc.

*/
package main
<commit_msg>Minor fix<commit_after>// Copyright 2015 Eleme Inc. All rights reserved.

/*

Banshee is a real-time anomalies(outliers) detection system for periodic
metrics.

Use Case

We are using it to monitor our website and rpc services intefaces, including
called frequency, response time and exception calls. Our services send statistics
to statsd, statsd aggregates them every 10 seconds and broadcasts the results to
its backends including banshee, banshee analyzes current metrics with history
data, calculates the trending and alerts us if the trending behaves anomalous.

For example, we have an api named get_user, this api's response time (in
milliseconds) is reported to banshee from statsd every 10 seconds:
	20, 21, 21, 22, 23, 19, 18, 21, 22, 20, ..., 300

Banshee will catch the latest metric 300 and report it as an anomaly.

Why don't we just set a fixed threshold instead (i.e. 200ms)? This may also works
but it is boring and hard to maintain a lot of thresholds. Banshee will analyze
metric trendings automatically, it will find the "thresholds" automatically.

Features

1. Designed for periodic metrics. Reality metrics are always with periodicity,
banshee only peeks metrics with the same "phase" to detect.

2. Multiple alerting rule configuration options, to alert via fixed-thresholds or
via anomalous trendings.

3. Coming with anomalies visualization webapp and alerting rules admin panels.

4. Require no extra storage services, banshee handles storage on disk by itself.

Requirements

1. Go >= 1.4 and godep.

2. Node and gulp.

Build

1. Clone the repo.

2. Build binary via `make`.

3. Build static files via `make static`.

Command Line

Usage:
	banshee [-c config] [-d] [-v]

Flags:
	-c config
		Load config from file.
	-d
		Turn on debug mode.
	-v
		Show version.

Configuration

See package config.

Statsd Integration

In order to forward metrics to banshee from statsd, we need to add the
npm module statsd-banshee to statsd's banckends:

1. Install statsd-banshee on your statsd servers:

	$ cd path/to/statsd
	$ npm install statsd-banshee

2. Add module statsd-banshee to statsd's backends in config.js:

	{
	, backends: ['statsd-banshee']
	, bansheeHost: 'localhost'
	, bansheePort: 2015
	}

Migrate from bell

Require bell.js v2.0+ and banshee v0.0.7+:

	./migrate -from bell.db -to banshee.db -with-projs -with-users
	mv banshee.db path/to/storage/admin

Compontents

Banshee have 4 compontents and they are running in the same process:

1. Detector is to detect incoming metrics with history data and store the
results.

2. Webapp is to visualize the detection results and provides panels to manage
alerting rules, projects and users.

3. Alerter is to send sms and emails once anomalies are found.

4. Cleaner is to clean outdated metrics from storage.

Alerting Sender

See package alerter and alerter/exampleCommand.

Deployment

Just use the shell release.sh, it will build the binary, static files and pack them together:

	./release.sh
	scp bansheeA.B.C.linux-amd64.tar.gz remote-host-name:/target/path

You may need a linux host to build the binary, cause: https://github.com/mattn/go-sqlite3/issues/106

Upgrade

Just pull the latest code:

	git remote add upstream git@github.com:eleme/banshee.git
	git pull upstream master

Note that the admin storage sqlite3 schema will be auto-migrated.

Implementation Details

1. Detection algorithms, see package detector.

2. Detector input net protocol, see package detector.

3. Storage, see package storage.

4. Filter, see package filter.

License

MIT (c) eleme, inc.

*/
package main
<|endoftext|>