<commit_before>package main

import (
	"fmt"
	"strings"
	"sync"

	"github.com/InVisionApp/rye"
	log "github.com/Sirupsen/logrus"
	"gopkg.in/alecthomas/kingpin.v2"

	"github.com/9corp/9volt/alerter"
	"github.com/9corp/9volt/api"
	"github.com/9corp/9volt/cfgutil"
	"github.com/9corp/9volt/cluster"
	"github.com/9corp/9volt/config"
	"github.com/9corp/9volt/dal"
	"github.com/9corp/9volt/director"
	"github.com/9corp/9volt/event"
	"github.com/9corp/9volt/manager"
	"github.com/9corp/9volt/state"
	"github.com/9corp/9volt/util"
)

var (
	server        = kingpin.Command("server", "9volt server")
	listenAddress = server.Flag("listen", "Address for 9volt's API to listen on").Short('l').Default("0.0.0.0:8080").Envar("NINEV_LISTEN_ADDRESS").String()
	tags          = server.Flag("tags", "Specify one or more member tags this instance has; see MONITOR_CONFIGS.md for details").Short('t').Envar("NINEV_MEMBER_TAGS").String()

	cfg         = kingpin.Command("cfg", "9volt configuration utility")
	dirArg      = cfg.Arg("dir", "Directory to search for 9volt YAML files").Required().String()
	replaceFlag = cfg.Flag("replace", "Do NOT verify if parsed config already exists in etcd (ie. replace everything)").Short('r').Bool()
	nosyncFlag  = cfg.Flag("nosync", "Do NOT remove any entries in etcd that do not have a corresponding local config").Short('n').Bool()
	dryrunFlag  = cfg.Flag("dryrun", "Do NOT push any changes, just show me what you'd do").Bool()

	etcdPrefix  = kingpin.Flag("etcd-prefix", "Prefix that 9volt's configuration is stored under in etcd").Short('p').Default("9volt").Envar("NINEV_ETCD_PREFIX").String()
	etcdMembers = kingpin.Flag("etcd-members", "List of etcd cluster members").Short('e').Default("http://localhost:2379").Envar("NINEV_ETCD_MEMBERS").String()
	debugUI     = kingpin.Flag("debug-ui", "Debug the user interface locally").Short('u').Bool()
	debug       = kingpin.Flag("debug", "Enable debug mode").Short('d').Envar("NINEV_DEBUG").Bool()

	version string
	command string
)

func init() {
	log.SetLevel(log.InfoLevel)

	// Parse CLI stuff
	kingpin.Version(version)
	kingpin.CommandLine.HelpFlag.Short('h')
	kingpin.CommandLine.VersionFlag.Short('v')
	command = kingpin.Parse()

	if *debug {
		log.SetLevel(log.DebugLevel)
	}
}

func runServer() {
	var wg sync.WaitGroup
	wg.Add(1)

	memberID := util.GetMemberID(*listenAddress)

	// kingpin splits on newline (?); split our tags on ',' instead
	memberTags := util.SplitTags(*tags)
	etcdMemberList := strings.Split(*etcdMembers, ",")

	// Create an initial dal client
	dalClient, err := dal.New(*etcdPrefix, etcdMemberList)
	if err != nil {
		log.Fatalf("Unable to start initial etcd client: %v", err.Error())
	}

	// Create and start event queue
	eventQueue := event.NewQueue(memberID, dalClient)
	eqClient := eventQueue.NewClient()

	// Load our configuration
	cfg := config.New(memberID, *listenAddress, *etcdPrefix, etcdMemberList, memberTags, dalClient, eqClient)

	if err := cfg.Load(); err != nil {
		log.Fatalf("Unable to load configuration from etcd: %v", err.Error())
	}

	// Perform etcd layout validation
	if errorList := cfg.ValidateDirs(); len(errorList) != 0 {
		log.Fatalf("Unable to complete etcd layout validation: %v", strings.Join(errorList, "; "))
	}

	// Create necessary channels
	clusterStateChannel := make(chan bool)
	distributeChannel := make(chan bool)
	messageChannel := make(chan *alerter.Message)
	monitorStateChannel := make(chan *state.Message)

	// Start cluster engine
	cluster, err := cluster.New(cfg, clusterStateChannel, distributeChannel)
	if err != nil {
		log.Fatalf("Unable to instantiate cluster engine: %v", err.Error())
	}

	if err := cluster.Start(); err != nil {
		log.Fatalf("Unable to complete cluster engine initialization: %v", err.Error())
	}

	// start director (check distributor)
	director, err := director.New(cfg, clusterStateChannel, distributeChannel)
	if err != nil {
		log.Fatalf("Unable to instantiate director: %v", err.Error())
	}

	if err := director.Start(); err != nil {
		log.Fatalf("Unable to complete director initialization: %v", err.Error())
	}

	// start manager
	manager, err := manager.New(cfg, messageChannel, monitorStateChannel)
	if err != nil {
		log.Fatalf("Unable to instantiate manager: %v", err.Error())
	}

	if err := manager.Start(); err != nil {
		log.Fatalf("Unable to complete manager initialization: %v", err.Error())
	}

	// start the alerter
	alerter := alerter.New(cfg, messageChannel)

	if err := alerter.Start(); err != nil {
		log.Fatalf("Unable to complete alerter initialization: %v", err.Error())
	}

	// start the state dumper
	state := state.New(cfg, monitorStateChannel)

	if err := state.Start(); err != nil {
		log.Fatalf("Unable to complete state initialization: %v", err.Error())
	}

	// Start the event queue
	if err := eventQueue.Start(); err != nil {
		log.Fatalf("Unable to complete event queue initialization: %v", err.Error())
	}

	// create a new middleware handler
	mwHandler := rye.NewMWHandler(rye.Config{})

	// determines whether or not to use statik or debug interactively
	debugUserInterface := false
	if *debugUI {
		debugUserInterface = true
	}

	// start api server
	apiServer := api.New(cfg, mwHandler, version, debugUserInterface)
	go apiServer.Run()

	log.Infof("9volt has started! API address: %v MemberID: %v Tags: %v", "http://"+
		*listenAddress, memberID, strings.Join(memberTags, ", "))

	wg.Wait()
}

func runCfgUtil() {
	etcdMemberList := strings.Split(*etcdMembers, ",")

	etcdClient, err := dal.NewCfgUtil(etcdMemberList, *etcdPrefix, *replaceFlag, *dryrunFlag, *nosyncFlag)
	if err != nil {
		log.Fatalf("Unable to create initial etcd client: %v", err.Error())
	}

	// verify if given dirArg is actually a dir
	cfg, err := cfgutil.New(*dirArg)
	if err != nil {
		log.Fatal(err.Error())
	}

	log.Infof("Fetching all 9volt configuration files in '%v'", *dirArg)

	yamlFiles, err := cfg.Fetch()
	if err != nil {
		log.Fatalf("Unable to fetch config files from dir '%v': %v", *dirArg, err.Error())
	}

	log.Info("Parsing 9volt config files")

	configs, err := cfg.Parse(yamlFiles)
	if err != nil {
		log.Fatalf("Unable to complete config file parsing: %v", err.Error())
	}

	log.Infof("Found %v alerter configs and %v monitor configs", len(configs.AlerterConfigs), len(configs.MonitorConfigs))
	log.Infof("Pushing 9volt configs to etcd hosts: %v", *etcdMembers)

	// push to etcd
	stats, errorList := etcdClient.Push(configs)
	if len(errorList) != 0 {
		log.Errorf("Encountered %v errors: %v", len(errorList), errorList)
	}

	pushedMessage := fmt.Sprintf("pushed %v monitor config(s) and %v alerter config(s)", stats.MonitorAdded, stats.AlerterAdded)
	skippedMessage := fmt.Sprintf("skipped replacing %v monitor config(s) and %v alerter config(s)", stats.MonitorSkipped, stats.AlerterSkipped)
	removedMessage := fmt.Sprintf("removed %v monitor config(s) and %v alerter config(s)", stats.MonitorRemoved, stats.AlerterRemoved)

	if *dryrunFlag {
		pushedMessage = "DRYRUN: Would have " + pushedMessage
		skippedMessage = "DRYRUN: Would have " + skippedMessage
		removedMessage = "DRYRUN: Would have " + removedMessage
	} else {
		pushedMessage = ":party: Successfully " + pushedMessage
		skippedMessage = "Successfully " + skippedMessage
		removedMessage = "Successfully " + removedMessage
	}

	log.Info(pushedMessage)

	if !*replaceFlag {
		log.Info(skippedMessage)
	}

	if !*nosyncFlag {
		log.Info(removedMessage)
	}
}

func main() {
	switch command {
	case "server":
		runServer()
	case "cfg":
		runCfgUtil()
	}
}
<commit_msg>switched to using `util.SplitTags()`<commit_after>package main

import (
	"fmt"
	"strings"
	"sync"

	"github.com/InVisionApp/rye"
	log "github.com/Sirupsen/logrus"
	"gopkg.in/alecthomas/kingpin.v2"

	"github.com/9corp/9volt/alerter"
	"github.com/9corp/9volt/api"
	"github.com/9corp/9volt/cfgutil"
	"github.com/9corp/9volt/cluster"
	"github.com/9corp/9volt/config"
	"github.com/9corp/9volt/dal"
	"github.com/9corp/9volt/director"
	"github.com/9corp/9volt/event"
	"github.com/9corp/9volt/manager"
	"github.com/9corp/9volt/state"
	"github.com/9corp/9volt/util"
)

var (
	server        = kingpin.Command("server", "9volt server")
	listenAddress = server.Flag("listen", "Address for 9volt's API to listen on").Short('l').Default("0.0.0.0:8080").Envar("NINEV_LISTEN_ADDRESS").String()
	tags          = server.Flag("tags", "Specify one or more member tags this instance has; see MONITOR_CONFIGS.md for details").Short('t').Envar("NINEV_MEMBER_TAGS").String()

	cfg         = kingpin.Command("cfg", "9volt configuration utility")
	dirArg      = cfg.Arg("dir", "Directory to search for 9volt YAML files").Required().String()
	replaceFlag = cfg.Flag("replace", "Do NOT verify if parsed config already exists in etcd (ie. replace everything)").Short('r').Bool()
	nosyncFlag  = cfg.Flag("nosync", "Do NOT remove any entries in etcd that do not have a corresponding local config").Short('n').Bool()
	dryrunFlag  = cfg.Flag("dryrun", "Do NOT push any changes, just show me what you'd do").Bool()

	etcdPrefix  = kingpin.Flag("etcd-prefix", "Prefix that 9volt's configuration is stored under in etcd").Short('p').Default("9volt").Envar("NINEV_ETCD_PREFIX").String()
	etcdMembers = kingpin.Flag("etcd-members", "List of etcd cluster members").Short('e').Default("http://localhost:2379").Envar("NINEV_ETCD_MEMBERS").String()
	debugUI     = kingpin.Flag("debug-ui", "Debug the user interface locally").Short('u').Bool()
	debug       = kingpin.Flag("debug", "Enable debug mode").Short('d').Envar("NINEV_DEBUG").Bool()

	version string
	command string
)

func init() {
	log.SetLevel(log.InfoLevel)

	// Parse CLI stuff
	kingpin.Version(version)
	kingpin.CommandLine.HelpFlag.Short('h')
	kingpin.CommandLine.VersionFlag.Short('v')
	command = kingpin.Parse()

	if *debug {
		log.SetLevel(log.DebugLevel)
	}
}

func runServer() {
	var wg sync.WaitGroup
	wg.Add(1)

	memberID := util.GetMemberID(*listenAddress)

	// kingpin splits on newline (?); split our tags on ',' instead
	memberTags := util.SplitTags(*tags)
	etcdMemberList := util.SplitTags(*etcdMembers)

	// Create an initial dal client
	dalClient, err := dal.New(*etcdPrefix, etcdMemberList)
	if err != nil {
		log.Fatalf("Unable to start initial etcd client: %v", err.Error())
	}

	// Create and start event queue
	eventQueue := event.NewQueue(memberID, dalClient)
	eqClient := eventQueue.NewClient()

	// Load our configuration
	cfg := config.New(memberID, *listenAddress, *etcdPrefix, etcdMemberList, memberTags, dalClient, eqClient)

	if err := cfg.Load(); err != nil {
		log.Fatalf("Unable to load configuration from etcd: %v", err.Error())
	}

	// Perform etcd layout validation
	if errorList := cfg.ValidateDirs(); len(errorList) != 0 {
		log.Fatalf("Unable to complete etcd layout validation: %v", strings.Join(errorList, "; "))
	}

	// Create necessary channels
	clusterStateChannel := make(chan bool)
	distributeChannel := make(chan bool)
	messageChannel := make(chan *alerter.Message)
	monitorStateChannel := make(chan *state.Message)

	// Start cluster engine
	cluster, err := cluster.New(cfg, clusterStateChannel, distributeChannel)
	if err != nil {
		log.Fatalf("Unable to instantiate cluster engine: %v", err.Error())
	}

	if err := cluster.Start(); err != nil {
		log.Fatalf("Unable to complete cluster engine initialization: %v", err.Error())
	}

	// start director (check distributor)
	director, err := director.New(cfg, clusterStateChannel, distributeChannel)
	if err != nil {
		log.Fatalf("Unable to instantiate director: %v", err.Error())
	}

	if err := director.Start(); err != nil {
		log.Fatalf("Unable to complete director initialization: %v", err.Error())
	}

	// start manager
	manager, err := manager.New(cfg, messageChannel, monitorStateChannel)
	if err != nil {
		log.Fatalf("Unable to instantiate manager: %v", err.Error())
	}

	if err := manager.Start(); err != nil {
		log.Fatalf("Unable to complete manager initialization: %v", err.Error())
	}

	// start the alerter
	alerter := alerter.New(cfg, messageChannel)

	if err := alerter.Start(); err != nil {
		log.Fatalf("Unable to complete alerter initialization: %v", err.Error())
	}

	// start the state dumper
	state := state.New(cfg, monitorStateChannel)

	if err := state.Start(); err != nil {
		log.Fatalf("Unable to complete state initialization: %v", err.Error())
	}

	// Start the event queue
	if err := eventQueue.Start(); err != nil {
		log.Fatalf("Unable to complete event queue initialization: %v", err.Error())
	}

	// create a new middleware handler
	mwHandler := rye.NewMWHandler(rye.Config{})

	// determines whether or not to use statik or debug interactively
	debugUserInterface := false
	if *debugUI {
		debugUserInterface = true
	}

	// start api server
	apiServer := api.New(cfg, mwHandler, version, debugUserInterface)
	go apiServer.Run()

	log.Infof("9volt has started! API address: %v MemberID: %v Tags: %v", "http://"+
		*listenAddress, memberID, strings.Join(memberTags, ", "))

	wg.Wait()
}

func runCfgUtil() {
	etcdMemberList := util.SplitTags(*etcdMembers)

	etcdClient, err := dal.NewCfgUtil(etcdMemberList, *etcdPrefix, *replaceFlag, *dryrunFlag, *nosyncFlag)
	if err != nil {
		log.Fatalf("Unable to create initial etcd client: %v", err.Error())
	}

	// verify if given dirArg is actually a dir
	cfg, err := cfgutil.New(*dirArg)
	if err != nil {
		log.Fatal(err.Error())
	}

	log.Infof("Fetching all 9volt configuration files in '%v'", *dirArg)

	yamlFiles, err := cfg.Fetch()
	if err != nil {
		log.Fatalf("Unable to fetch config files from dir '%v': %v", *dirArg, err.Error())
	}

	log.Info("Parsing 9volt config files")

	configs, err := cfg.Parse(yamlFiles)
	if err != nil {
		log.Fatalf("Unable to complete config file parsing: %v", err.Error())
	}

	log.Infof("Found %v alerter configs and %v monitor configs", len(configs.AlerterConfigs), len(configs.MonitorConfigs))
	log.Infof("Pushing 9volt configs to etcd hosts: %v", *etcdMembers)

	// push to etcd
	stats, errorList := etcdClient.Push(configs)
	if len(errorList) != 0 {
		log.Errorf("Encountered %v errors: %v", len(errorList), errorList)
	}

	pushedMessage := fmt.Sprintf("pushed %v monitor config(s) and %v alerter config(s)", stats.MonitorAdded, stats.AlerterAdded)
	skippedMessage := fmt.Sprintf("skipped replacing %v monitor config(s) and %v alerter config(s)", stats.MonitorSkipped, stats.AlerterSkipped)
	removedMessage := fmt.Sprintf("removed %v monitor config(s) and %v alerter config(s)", stats.MonitorRemoved, stats.AlerterRemoved)

	if *dryrunFlag {
		pushedMessage = "DRYRUN: Would have " + pushedMessage
		skippedMessage = "DRYRUN: Would have " + skippedMessage
		removedMessage = "DRYRUN: Would have " + removedMessage
	} else {
		pushedMessage = ":party: Successfully " + pushedMessage
		skippedMessage = "Successfully " + skippedMessage
		removedMessage = "Successfully " + removedMessage
	}

	log.Info(pushedMessage)

	if !*replaceFlag {
		log.Info(skippedMessage)
	}

	if !*nosyncFlag {
		log.Info(removedMessage)
	}
}

func main() {
	switch command {
	case "server":
		runServer()
	case "cfg":
		runCfgUtil()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"
	"strconv"

	log "github.com/Sirupsen/logrus"
)

var (
	port           int
	storage        string
	dockerHost     string
	dockerCertPath string
)

func init() {
	log.SetFormatter(&log.JSONFormatter{})
	log.SetOutput(os.Stderr)

	defaultPort, _ := strconv.Atoi(os.Getenv("PORT"))
	flag.IntVar(&port, "port", defaultPort, "port number")
	flag.StringVar(&storage, "storage", "filesystem", "storage type")
	flag.StringVar(&dockerHost, "docker-host", os.Getenv("DOCKER_HOST"), "docker host")
	flag.StringVar(&dockerCertPath, "docker-cert-path", os.Getenv("DOCKER_CERT_PATH"), "docker cert path")
	flag.Parse()
}

func main() {
	err := Run()
	if err != nil {
		log.Error(err)
	}
}
<commit_msg>Added log file option.<commit_after>package main

import (
	"flag"
	"os"
	"strconv"

	log "github.com/Sirupsen/logrus"
)

var (
	port           int
	storage        string
	dockerHost     string
	dockerCertPath string
	logFile        string
)

func init() {
	defaultPort, _ := strconv.Atoi(os.Getenv("PORT"))
	flag.IntVar(&port, "port", defaultPort, "port number")
	flag.StringVar(&storage, "storage", "filesystem", "storage type")
	flag.StringVar(&dockerHost, "docker-host", os.Getenv("DOCKER_HOST"), "docker host")
	flag.StringVar(&dockerCertPath, "docker-cert-path", os.Getenv("DOCKER_CERT_PATH"), "docker cert path")
	flag.StringVar(&logFile, "logfile", "", "log file path")
	flag.Parse()

	log.SetFormatter(&log.JSONFormatter{})

}

func main() {
	if logFile != "" {
		f, err := os.OpenFile(logFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		log.SetOutput(f)
	} else {
		log.SetOutput(os.Stdout)
	}

	err := Run()
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright © 2017 Solf1re2 <jy1v07@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"

<<<<<<< HEAD
	"github.com/solf1re2/config"
=======
	"net/http"

>>>>>>> 10161e626039dbd3284b15cc2ec29d8c97f495ff
	"github.com/solf1re2/gosol/cmd"
)

func main() {
	config := config.LoadConfig("./config.json")
	fmt.Printf("Server port :%v\n", config.Server.Port)
	http.HandleFunc("/", handler)
	http.ListenAndServe(":"+config.Server.Port, nil)
	cmd.Execute()
}

func handler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Hi there, I love %s!", r.URL.Path[1:])
}
<commit_msg>more of fix<commit_after>// Copyright © 2017 Solf1re2 <jy1v07@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"

	"net/http"

	"github.com/solf1re2/config"
	"github.com/solf1re2/gosol/cmd"
)

func main() {
	config := config.LoadConfig("./config.json")
	fmt.Printf("Server port :%v\n", config.Server.Port)
	http.HandleFunc("/", handler)
	http.ListenAndServe(":"+config.Server.Port, nil)
	cmd.Execute()
}

func handler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Hi there, I love %s!", r.URL.Path[1:])
}
<|endoftext|>


<commit_before>package gophermail

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"mime"
	"mime/multipart"
	"net/mail"
	"net/textproto"
	"path/filepath"
	"strings"

	"github.com/sloonz/go-qprintable"
)

// Message Lint: http://tools.ietf.org/tools/msglint/

const crlf = "\r\n"

var ErrMissingRecipient = errors.New("No recipient specified. At one To, Cc, or Bcc recipient is required.")
var ErrMissingFromAddress = errors.New("No from address specified.")

// A Message represents an email message.
// Addresses may be of any form permitted by RFC 5322.
type Message struct {
	// TODO(JPOEHLS): Add support for specifying the Sender header.

	// Technically this could be a list of addresses but we don't support that. See RFC 2822 s3.6.2.
	From mail.Address

	// Technically this could be a list of addresses but we don't support that. See RFC 2822 s3.6.2.
	ReplyTo mail.Address // optional

	To, Cc, Bcc []mail.Address

	Subject string // optional

	Body     string // optional
	HTMLBody string // optional

	Attachments []Attachment // optional

	// Extra mail headers.
	Headers mail.Header
}

// appendMailAddresses parses any number of addresses and appends them to a
// destination slice. If any of the addresses fail to parse, none of them are
// appended.
func appendMailAddresses(dest *[]mail.Address, addresses ...string) error {
	var parsedAddresses []mail.Address
	var err error

	for _, address := range addresses {
		parsed, err := mail.ParseAddress(address)
		if err != nil {
			return err
		}
		parsedAddresses = append(parsedAddresses, *parsed)
	}

	*dest = append(*dest, parsedAddresses...)
	return err
}

// setMailAddress parses an address and sets it to a destination mail address.
func setMailAddress(dest *mail.Address, address string) error {
	parsed, err := mail.ParseAddress(address)
	if err != nil {
		return err
	}
	*dest = *parsed
	return nil
}

// SetFrom creates a mail.Address and assigns it to the message's From
// field.
func (m *Message) SetFrom(address string) error {
	return setMailAddress(&m.From, address)
}

// SetReplyTo creates a mail.Address and assigns it to the message's ReplyTo
// field.
func (m *Message) SetReplyTo(address string) error {
	return setMailAddress(&m.ReplyTo, address)
}

// AddTo creates a mail.Address and adds it to the list of To addresses in the
// message
func (m *Message) AddTo(addresses ...string) error {
	return appendMailAddresses(&m.To, addresses...)
}

// AddCc creates a mail.Address and adds it to the list of Cc addresses in the
// message
func (m *Message) AddCc(addresses ...string) error {
	return appendMailAddresses(&m.Cc, addresses...)
}

// AddBcc creates a mail.Address and adds it to the list of Bcc addresses in the
// message
func (m *Message) AddBcc(addresses ...string) error {
	return appendMailAddresses(&m.Bcc, addresses...)
}

// An Attachment represents an email attachment.
type Attachment struct {
	// Name must be set to a valid file name.
	Name string

	// Optional.
	// Uses mime.TypeByExtension and falls back
	// to application/octet-stream if unknown.
	ContentType string

	Data io.Reader
}

// Bytes gets the encoded MIME message.
func (m *Message) Bytes() ([]byte, error) {
	var buffer = &bytes.Buffer{}
	header := textproto.MIMEHeader{}

	return m.bytes(buffer, header)
}

// bytes gets the encoded MIME message
func (m *Message) bytes(buffer *bytes.Buffer, header textproto.MIMEHeader) ([]byte, error) {
	var err error

	// Require To, Cc, or Bcc
	// We'll parse the slices into a list of addresses
	// and then make sure that list isn't empty.
	toAddrs := getAddressListString(m.To)
	ccAddrs := getAddressListString(m.Cc)
	bccAddrs := getAddressListString(m.Bcc)

	var hasTo = toAddrs != ""
	var hasCc = ccAddrs != ""
	var hasBcc = bccAddrs != ""

	if !hasTo && !hasCc && !hasBcc {
		return nil, ErrMissingRecipient
	}

	if hasTo {
		header.Add("To", toAddrs)
	}
	if hasCc {
		header.Add("Cc", ccAddrs)
	}
	// BCC header is excluded on purpose.
	// BCC recipients aren't included in the message
	// headers and are only used at the SMTP level.

	var emptyAddress mail.Address
	// Require From address
	if m.From == emptyAddress {
		return nil, ErrMissingFromAddress
	}
	header.Add("From", m.From.String())

	// Optional ReplyTo
	if m.ReplyTo != emptyAddress {
		header.Add("Reply-To", m.ReplyTo.String())
	}

	// Optional Subject
	if m.Subject != "" {
		quotedSubject := qEncodeAndWrap(m.Subject, 9 /* len("Subject: ") */)
		if quotedSubject[0] == '"' {
			// qEncode used simple quoting, which adds quote
			// characters to email subjects.
			quotedSubject = quotedSubject[1 : len(quotedSubject)-2]
		}
		header.Add("Subject", quotedSubject)
	}

	for k, v := range m.Headers {
		header[k] = v
	}

	// Top level multipart writer for our `multipart/mixed` body.
	mixedw := multipart.NewWriter(buffer)

	header.Add("MIME-Version", "1.0")
	header.Add("Content-Type", fmt.Sprintf("multipart/mixed;%s boundary=%s", crlf, mixedw.Boundary()))

	err = writeHeader(buffer, header)
	if err != nil {
		return nil, err
	}

	// Write the start of our `multipart/mixed` body.
	_, err = fmt.Fprintf(buffer, "--%s%s", mixedw.Boundary(), crlf)
	if err != nil {
		return nil, err
	}

	// Does the message have a body?
	if m.Body != "" || m.HTMLBody != "" {

		// Nested multipart writer for our `multipart/alternative` body.
		altw := multipart.NewWriter(buffer)

		header = textproto.MIMEHeader{}
		header.Add("Content-Type", fmt.Sprintf("multipart/alternative;%s boundary=%s", crlf, altw.Boundary()))
		err := writeHeader(buffer, header)
		if err != nil {
			return nil, err
		}

		if m.Body != "" {
			header = textproto.MIMEHeader{}
			header.Add("Content-Type", "text/plain; charset=utf-8")
			header.Add("Content-Transfer-Encoding", "quoted-printable")
			//header.Add("Content-Transfer-Encoding", "base64")

			partw, err := altw.CreatePart(header)
			if err != nil {
				return nil, err
			}

			bodyBytes := []byte(m.Body)
			//encoder := NewBase64MimeEncoder(partw)
			encoder := qprintable.NewEncoder(qprintable.DetectEncoding(m.Body), partw)
			_, err = encoder.Write(bodyBytes)
			if err != nil {
				return nil, err
			}
			err = encoder.Close()
			if err != nil {
				return nil, err
			}
		}

		if m.HTMLBody != "" {
			header = textproto.MIMEHeader{}
			header.Add("Content-Type", "text/html; charset=utf-8")
			//header.Add("Content-Transfer-Encoding", "quoted-printable")
			header.Add("Content-Transfer-Encoding", "base64")

			partw, err := altw.CreatePart(header)
			if err != nil {
				return nil, err
			}

			htmlBodyBytes := []byte(m.HTMLBody)
			encoder := NewBase64MimeEncoder(partw)
			//encoder := qprintable.NewEncoder(qprintable.DetectEncoding(m.HTMLBody), partw)
			_, err = encoder.Write(htmlBodyBytes)
			if err != nil {
				return nil, err
			}
			err = encoder.Close()
			if err != nil {
				return nil, err
			}
		}

		altw.Close()
	}

	if m.Attachments != nil && len(m.Attachments) > 0 {

		for _, attachment := range m.Attachments {

			contentType := attachment.ContentType
			if contentType == "" {
				contentType = mime.TypeByExtension(filepath.Ext(attachment.Name))
				if contentType == "" {
					contentType = "application/octet-stream"
				}
			}

			header := textproto.MIMEHeader{}
			header.Add("Content-Type", contentType)
			header.Add("Content-Disposition", fmt.Sprintf(`attachment;%s filename="%s"`, crlf, attachment.Name))
			header.Add("Content-Transfer-Encoding", "base64")

			attachmentPart, err := mixedw.CreatePart(header)
			if err != nil {
				return nil, err
			}

			if attachment.Data != nil {
				encoder := NewBase64MimeEncoder(attachmentPart)
				_, err = io.Copy(encoder, attachment.Data)
				if err != nil {
					return nil, err
				}
				err = encoder.Close()
				if err != nil {
					return nil, err
				}
			}
		}

	}

	mixedw.Close()

	return buffer.Bytes(), nil
}

// writeHeader writes the specified MIMEHeader to the io.Writer.
// Header values will be trimmed but otherwise left alone.
// Headers with multiple values are not supported and will return an error.
func writeHeader(w io.Writer, header textproto.MIMEHeader) error {
	for k, vs := range header {
		_, err := fmt.Fprintf(w, "%s: ", k)
		if err != nil {
			return err
		}

		for i, v := range vs {
			v = textproto.TrimString(v)

			_, err := fmt.Fprintf(w, "%s", v)
			if err != nil {
				return err
			}

			if i < len(vs)-1 {
				return errors.New("Multiple header values are not supported.")
			}
		}

		_, err = fmt.Fprint(w, crlf)
		if err != nil {
			return err
		}
	}

	// Write a blank line as a spacer
	_, err := fmt.Fprint(w, crlf)
	if err != nil {
		return err
	}

	return nil
}

// qEncode encodes a string with Q encoding defined as an 'encoded-word' in RFC 2047.
// The maximum encoded word length of 75 characters is not accounted for.
// Use qEncodeAndWrap if you need that.
//
// Inspired by https://gist.github.com/andelf/5004821
func qEncode(input string) string {
	// use mail's rfc2047 to encode any string
	addr := mail.Address{Name: input, Address: "a@b.c"}
	s := addr.String()
	return s[:len(s)-8]
}

// qEncodeAndWrap encodes the input as potentially multiple 'encoded-words'
// with CRLF SPACE line breaks between them to (as best as possible)
// guarantee that each encoded-word is no more than 75 characters
// and, padding included, each line is no longer than 76 characters.
// See RFC 2047 s2.
func qEncodeAndWrap(input string, padding int) string {

	// Split at any whitespace but prefer "; " or  ", " or " >" or "> " which
	// denotes a clear semantic break.
	// Remember that the qEncoded input isn't guaranteed to have the same
	// length as the unencoded input (obvious). Example: http://play.golang.org/p/dXA5IJnL22

	// Increase the padding to account for
	// the encoded-word 'envelop' tokens.
	// "?" charset (utf-8 is always assumed) "?" encoding "?" encoded-text "?="
	padding += 11

	// Tokenization included, the encoded word must not
	// be longer than 75 characters.
	const maxEncodedWordLength = 75

	var firstTry = qEncode(input)
	if len(firstTry) > maxEncodedWordLength-padding {

		// TODO(JPOEHLS): Implement an algorithm to break the input into multiple encoded-words.

		return firstTry
	} else {
		return firstTry
	}
}

// getAddressListString encodes a slice of email addresses into
// a string value suitable for a MIME header. Each address is
// Q encoded and wrapped onto its own line to help ensure that
// the header line doesn't cross the 78 character maximum.
func getAddressListString(addresses []mail.Address) string {
	var addressStrings []string

	for _, address := range addresses {
		addressStrings = append(addressStrings, address.String())
	}
	return strings.Join(addressStrings, ","+crlf+" ")
}
<commit_msg>Fixed failing test on go1.5.1<commit_after>package gophermail

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"mime"
	"mime/multipart"
	"net/mail"
	"net/textproto"
	"path/filepath"
	"strings"

	"github.com/sloonz/go-qprintable"
)

// Message Lint: http://tools.ietf.org/tools/msglint/

const crlf = "\r\n"

var ErrMissingRecipient = errors.New("No recipient specified. At one To, Cc, or Bcc recipient is required.")
var ErrMissingFromAddress = errors.New("No from address specified.")

// A Message represents an email message.
// Addresses may be of any form permitted by RFC 5322.
type Message struct {
	// TODO(JPOEHLS): Add support for specifying the Sender header.

	// Technically this could be a list of addresses but we don't support that. See RFC 2822 s3.6.2.
	From mail.Address

	// Technically this could be a list of addresses but we don't support that. See RFC 2822 s3.6.2.
	ReplyTo mail.Address // optional

	To, Cc, Bcc []mail.Address

	Subject string // optional

	Body     string // optional
	HTMLBody string // optional

	Attachments []Attachment // optional

	// Extra mail headers.
	Headers mail.Header
}

// appendMailAddresses parses any number of addresses and appends them to a
// destination slice. If any of the addresses fail to parse, none of them are
// appended.
func appendMailAddresses(dest *[]mail.Address, addresses ...string) error {
	var parsedAddresses []mail.Address
	var err error

	for _, address := range addresses {
		parsed, err := mail.ParseAddress(address)
		if err != nil {
			return err
		}
		parsedAddresses = append(parsedAddresses, *parsed)
	}

	*dest = append(*dest, parsedAddresses...)
	return err
}

// setMailAddress parses an address and sets it to a destination mail address.
func setMailAddress(dest *mail.Address, address string) error {
	parsed, err := mail.ParseAddress(address)
	if err != nil {
		return err
	}
	*dest = *parsed
	return nil
}

// SetFrom creates a mail.Address and assigns it to the message's From
// field.
func (m *Message) SetFrom(address string) error {
	return setMailAddress(&m.From, address)
}

// SetReplyTo creates a mail.Address and assigns it to the message's ReplyTo
// field.
func (m *Message) SetReplyTo(address string) error {
	return setMailAddress(&m.ReplyTo, address)
}

// AddTo creates a mail.Address and adds it to the list of To addresses in the
// message
func (m *Message) AddTo(addresses ...string) error {
	return appendMailAddresses(&m.To, addresses...)
}

// AddCc creates a mail.Address and adds it to the list of Cc addresses in the
// message
func (m *Message) AddCc(addresses ...string) error {
	return appendMailAddresses(&m.Cc, addresses...)
}

// AddBcc creates a mail.Address and adds it to the list of Bcc addresses in the
// message
func (m *Message) AddBcc(addresses ...string) error {
	return appendMailAddresses(&m.Bcc, addresses...)
}

// An Attachment represents an email attachment.
type Attachment struct {
	// Name must be set to a valid file name.
	Name string

	// Optional.
	// Uses mime.TypeByExtension and falls back
	// to application/octet-stream if unknown.
	ContentType string

	Data io.Reader
}

// Bytes gets the encoded MIME message.
func (m *Message) Bytes() ([]byte, error) {
	var buffer = &bytes.Buffer{}
	header := textproto.MIMEHeader{}

	return m.bytes(buffer, header)
}

// bytes gets the encoded MIME message
func (m *Message) bytes(buffer *bytes.Buffer, header textproto.MIMEHeader) ([]byte, error) {
	var err error

	// Require To, Cc, or Bcc
	// We'll parse the slices into a list of addresses
	// and then make sure that list isn't empty.
	toAddrs := getAddressListString(m.To)
	ccAddrs := getAddressListString(m.Cc)
	bccAddrs := getAddressListString(m.Bcc)

	var hasTo = toAddrs != ""
	var hasCc = ccAddrs != ""
	var hasBcc = bccAddrs != ""

	if !hasTo && !hasCc && !hasBcc {
		return nil, ErrMissingRecipient
	}

	if hasTo {
		header.Add("To", toAddrs)
	}
	if hasCc {
		header.Add("Cc", ccAddrs)
	}
	// BCC header is excluded on purpose.
	// BCC recipients aren't included in the message
	// headers and are only used at the SMTP level.

	var emptyAddress mail.Address
	// Require From address
	if m.From == emptyAddress {
		return nil, ErrMissingFromAddress
	}
	header.Add("From", m.From.String())

	// Optional ReplyTo
	if m.ReplyTo != emptyAddress {
		header.Add("Reply-To", m.ReplyTo.String())
	}

	// Optional Subject
	if m.Subject != "" {
		quotedSubject := qEncodeAndWrap(m.Subject, 9 /* len("Subject: ") */)
		if quotedSubject[0] == '"' {
			// qEncode used simple quoting, which adds quote
			// characters to email subjects.
			quotedSubject = quotedSubject[1 : len(quotedSubject)-1]
		}
		header.Add("Subject", quotedSubject)
	}

	for k, v := range m.Headers {
		header[k] = v
	}

	// Top level multipart writer for our `multipart/mixed` body.
	mixedw := multipart.NewWriter(buffer)

	header.Add("MIME-Version", "1.0")
	header.Add("Content-Type", fmt.Sprintf("multipart/mixed;%s boundary=%s", crlf, mixedw.Boundary()))

	err = writeHeader(buffer, header)
	if err != nil {
		return nil, err
	}

	// Write the start of our `multipart/mixed` body.
	_, err = fmt.Fprintf(buffer, "--%s%s", mixedw.Boundary(), crlf)
	if err != nil {
		return nil, err
	}

	// Does the message have a body?
	if m.Body != "" || m.HTMLBody != "" {

		// Nested multipart writer for our `multipart/alternative` body.
		altw := multipart.NewWriter(buffer)

		header = textproto.MIMEHeader{}
		header.Add("Content-Type", fmt.Sprintf("multipart/alternative;%s boundary=%s", crlf, altw.Boundary()))
		err := writeHeader(buffer, header)
		if err != nil {
			return nil, err
		}

		if m.Body != "" {
			header = textproto.MIMEHeader{}
			header.Add("Content-Type", "text/plain; charset=utf-8")
			header.Add("Content-Transfer-Encoding", "quoted-printable")
			//header.Add("Content-Transfer-Encoding", "base64")

			partw, err := altw.CreatePart(header)
			if err != nil {
				return nil, err
			}

			bodyBytes := []byte(m.Body)
			//encoder := NewBase64MimeEncoder(partw)
			encoder := qprintable.NewEncoder(qprintable.DetectEncoding(m.Body), partw)
			_, err = encoder.Write(bodyBytes)
			if err != nil {
				return nil, err
			}
			err = encoder.Close()
			if err != nil {
				return nil, err
			}
		}

		if m.HTMLBody != "" {
			header = textproto.MIMEHeader{}
			header.Add("Content-Type", "text/html; charset=utf-8")
			//header.Add("Content-Transfer-Encoding", "quoted-printable")
			header.Add("Content-Transfer-Encoding", "base64")

			partw, err := altw.CreatePart(header)
			if err != nil {
				return nil, err
			}

			htmlBodyBytes := []byte(m.HTMLBody)
			encoder := NewBase64MimeEncoder(partw)
			//encoder := qprintable.NewEncoder(qprintable.DetectEncoding(m.HTMLBody), partw)
			_, err = encoder.Write(htmlBodyBytes)
			if err != nil {
				return nil, err
			}
			err = encoder.Close()
			if err != nil {
				return nil, err
			}
		}

		altw.Close()
	}

	if m.Attachments != nil && len(m.Attachments) > 0 {

		for _, attachment := range m.Attachments {

			contentType := attachment.ContentType
			if contentType == "" {
				contentType = mime.TypeByExtension(filepath.Ext(attachment.Name))
				if contentType == "" {
					contentType = "application/octet-stream"
				}
			}

			header := textproto.MIMEHeader{}
			header.Add("Content-Type", contentType)
			header.Add("Content-Disposition", fmt.Sprintf(`attachment;%s filename="%s"`, crlf, attachment.Name))
			header.Add("Content-Transfer-Encoding", "base64")

			attachmentPart, err := mixedw.CreatePart(header)
			if err != nil {
				return nil, err
			}

			if attachment.Data != nil {
				encoder := NewBase64MimeEncoder(attachmentPart)
				_, err = io.Copy(encoder, attachment.Data)
				if err != nil {
					return nil, err
				}
				err = encoder.Close()
				if err != nil {
					return nil, err
				}
			}
		}

	}

	mixedw.Close()

	return buffer.Bytes(), nil
}

// writeHeader writes the specified MIMEHeader to the io.Writer.
// Header values will be trimmed but otherwise left alone.
// Headers with multiple values are not supported and will return an error.
func writeHeader(w io.Writer, header textproto.MIMEHeader) error {
	for k, vs := range header {
		_, err := fmt.Fprintf(w, "%s: ", k)
		if err != nil {
			return err
		}

		for i, v := range vs {
			v = textproto.TrimString(v)

			_, err := fmt.Fprintf(w, "%s", v)
			if err != nil {
				return err
			}

			if i < len(vs)-1 {
				return errors.New("Multiple header values are not supported.")
			}
		}

		_, err = fmt.Fprint(w, crlf)
		if err != nil {
			return err
		}
	}

	// Write a blank line as a spacer
	_, err := fmt.Fprint(w, crlf)
	if err != nil {
		return err
	}

	return nil
}

// qEncode encodes a string with Q encoding defined as an 'encoded-word' in RFC 2047.
// The maximum encoded word length of 75 characters is not accounted for.
// Use qEncodeAndWrap if you need that.
//
// Inspired by https://gist.github.com/andelf/5004821
func qEncode(input string) string {
	// use mail's rfc2047 to encode any string
	addr := mail.Address{Name: input, Address: "a@b.c"}
	s := addr.String()
	return s[:len(s)-8]
}

// qEncodeAndWrap encodes the input as potentially multiple 'encoded-words'
// with CRLF SPACE line breaks between them to (as best as possible)
// guarantee that each encoded-word is no more than 75 characters
// and, padding included, each line is no longer than 76 characters.
// See RFC 2047 s2.
func qEncodeAndWrap(input string, padding int) string {

	// Split at any whitespace but prefer "; " or  ", " or " >" or "> " which
	// denotes a clear semantic break.
	// Remember that the qEncoded input isn't guaranteed to have the same
	// length as the unencoded input (obvious). Example: http://play.golang.org/p/dXA5IJnL22

	// Increase the padding to account for
	// the encoded-word 'envelop' tokens.
	// "?" charset (utf-8 is always assumed) "?" encoding "?" encoded-text "?="
	padding += 11

	// Tokenization included, the encoded word must not
	// be longer than 75 characters.
	const maxEncodedWordLength = 75

	var firstTry = qEncode(input)
	if len(firstTry) > maxEncodedWordLength-padding {

		// TODO(JPOEHLS): Implement an algorithm to break the input into multiple encoded-words.

		return firstTry
	} else {
		return firstTry
	}
}

// getAddressListString encodes a slice of email addresses into
// a string value suitable for a MIME header. Each address is
// Q encoded and wrapped onto its own line to help ensure that
// the header line doesn't cross the 78 character maximum.
func getAddressListString(addresses []mail.Address) string {
	var addressStrings []string

	for _, address := range addresses {
		addressStrings = append(addressStrings, address.String())
	}
	return strings.Join(addressStrings, ","+crlf+" ")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/binary"
	"encoding/hex"
	"flag"
	"log"
	"net"
	"os"
	"sync"
	"time"

	"github.com/calmh/syncthing/discover"
	"github.com/golang/groupcache/lru"
	"github.com/juju/ratelimit"
)

type Node struct {
	Addresses []Address
	Updated   time.Time
}

type Address struct {
	IP   []byte
	Port uint16
}

var (
	nodes    = make(map[string]Node)
	lock     sync.Mutex
	queries  = 0
	answered = 0
	limited  = 0
	debug    = false
	limiter  = lru.New(1024)
)

func main() {
	var listen string
	var timestamp bool

	flag.StringVar(&listen, "listen", ":22025", "Listen address")
	flag.BoolVar(&debug, "debug", false, "Enable debug output")
	flag.BoolVar(&timestamp, "timestamp", true, "Timestamp the log output")
	flag.Parse()

	log.SetOutput(os.Stdout)
	if !timestamp {
		log.SetFlags(0)
	}

	addr, _ := net.ResolveUDPAddr("udp", listen)
	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		log.Fatal(err)
	}

	go logStats()

	var buf = make([]byte, 1024)
	for {
		buf = buf[:cap(buf)]
		n, addr, err := conn.ReadFromUDP(buf)

		if limit(addr) {
			// Rate limit in effect for source
			continue
		}

		if err != nil {
			log.Fatal(err)
		}

		if n < 4 {
			log.Printf("Received short packet (%d bytes)", n)
			continue
		}

		buf = buf[:n]
		magic := binary.BigEndian.Uint32(buf)

		switch magic {
		case discover.AnnouncementMagicV1:
			handleAnnounceV1(addr, buf)

		case discover.QueryMagicV1:
			handleQueryV1(conn, addr, buf)

		case discover.AnnouncementMagicV2:
			handleAnnounceV2(addr, buf)

		case discover.QueryMagicV2:
			handleQueryV2(conn, addr, buf)
		}
	}
}

func limit(addr *net.UDPAddr) bool {
	key := addr.IP.String()

	lock.Lock()
	defer lock.Unlock()

	bkt, ok := limiter.Get(key)
	if ok {
		bkt := bkt.(*ratelimit.Bucket)
		if bkt.TakeAvailable(1) != 1 {
			// Rate limit exceeded; ignore packet
			if debug {
				log.Println("Rate limit exceeded for", key)
			}
			limited++
			return true
		}
	} else {
		if debug {
			log.Println("New limiter for", key)
		}
		// One packet per ten seconds average rate, burst ten packets
		limiter.Add(key, ratelimit.NewBucket(10*time.Second, 10))
	}

	return false
}

func handleAnnounceV1(addr *net.UDPAddr, buf []byte) {
	var pkt discover.AnnounceV1
	err := pkt.UnmarshalXDR(buf)
	if err != nil {
		log.Println("AnnounceV1 Unmarshal:", err)
		log.Println(hex.Dump(buf))
		return
	}
	if debug {
		log.Printf("<- %v %#v", addr, pkt)
	}

	ip := addr.IP.To4()
	if ip == nil {
		ip = addr.IP.To16()
	}
	node := Node{
		Addresses: []Address{{
			IP:   ip,
			Port: pkt.Port,
		}},
		Updated: time.Now(),
	}

	lock.Lock()
	nodes[pkt.NodeID] = node
	lock.Unlock()
}

func handleQueryV1(conn *net.UDPConn, addr *net.UDPAddr, buf []byte) {
	var pkt discover.QueryV1
	err := pkt.UnmarshalXDR(buf)
	if err != nil {
		log.Println("QueryV1 Unmarshal:", err)
		log.Println(hex.Dump(buf))
		return
	}
	if debug {
		log.Printf("<- %v %#v", addr, pkt)
	}

	lock.Lock()
	node, ok := nodes[pkt.NodeID]
	queries++
	lock.Unlock()

	if ok && len(node.Addresses) > 0 {
		pkt := discover.AnnounceV1{
			Magic:  discover.AnnouncementMagicV1,
			NodeID: pkt.NodeID,
			Port:   node.Addresses[0].Port,
			IP:     node.Addresses[0].IP,
		}
		if debug {
			log.Printf("-> %v %#v", addr, pkt)
		}

		tb := pkt.MarshalXDR()
		_, _, err = conn.WriteMsgUDP(tb, nil, addr)
		if err != nil {
			log.Println("QueryV1 response write:", err)
		}

		lock.Lock()
		answered++
		lock.Unlock()
	}
}

func handleAnnounceV2(addr *net.UDPAddr, buf []byte) {
	var pkt discover.AnnounceV2
	err := pkt.UnmarshalXDR(buf)
	if err != nil {
		log.Println("AnnounceV2 Unmarshal:", err)
		log.Println(hex.Dump(buf))
		return
	}
	if debug {
		log.Printf("<- %v %#v", addr, pkt)
	}

	ip := addr.IP.To4()
	if ip == nil {
		ip = addr.IP.To16()
	}

	var addrs []Address
	for _, addr := range pkt.Addresses {
		tip := addr.IP
		if len(tip) == 0 {
			tip = ip
		}
		addrs = append(addrs, Address{
			IP:   tip,
			Port: addr.Port,
		})
	}

	node := Node{
		Addresses: addrs,
		Updated:   time.Now(),
	}

	lock.Lock()
	nodes[pkt.NodeID] = node
	lock.Unlock()
}

func handleQueryV2(conn *net.UDPConn, addr *net.UDPAddr, buf []byte) {
	var pkt discover.QueryV2
	err := pkt.UnmarshalXDR(buf)
	if err != nil {
		log.Println("QueryV2 Unmarshal:", err)
		log.Println(hex.Dump(buf))
		return
	}
	if debug {
		log.Printf("<- %v %#v", addr, pkt)
	}

	lock.Lock()
	node, ok := nodes[pkt.NodeID]
	queries++
	lock.Unlock()

	if ok && len(node.Addresses) > 0 {
		pkt := discover.AnnounceV2{
			Magic:  discover.AnnouncementMagicV2,
			NodeID: pkt.NodeID,
		}
		for _, addr := range node.Addresses {
			pkt.Addresses = append(pkt.Addresses, discover.Address{IP: addr.IP, Port: addr.Port})
		}
		if debug {
			log.Printf("-> %v %#v", addr, pkt)
		}

		tb := pkt.MarshalXDR()
		_, _, err = conn.WriteMsgUDP(tb, nil, addr)
		if err != nil {
			log.Println("QueryV2 response write:", err)
		}

		lock.Lock()
		answered++
		lock.Unlock()
	}
}

func logStats() {
	for {
		time.Sleep(600 * time.Second)

		lock.Lock()

		var deleted = 0
		for id, node := range nodes {
			if time.Since(node.Updated) > 60*time.Minute {
				delete(nodes, id)
				deleted++
			}
		}

		log.Printf("Expired %d nodes; %d nodes in registry; %d queries (%d answered)", deleted, len(nodes), queries, answered)
		log.Printf("Limited %d queries; %d entries in limiter cache", limited, limiter.Len())
		queries = 0
		answered = 0
		limited = 0

		lock.Unlock()
	}
}
<commit_msg>discosrv: Remove deprecated v1 support<commit_after>package main

import (
	"encoding/binary"
	"encoding/hex"
	"flag"
	"log"
	"net"
	"os"
	"sync"
	"time"

	"github.com/calmh/syncthing/discover"
	"github.com/golang/groupcache/lru"
	"github.com/juju/ratelimit"
)

type Node struct {
	Addresses []Address
	Updated   time.Time
}

type Address struct {
	IP   []byte
	Port uint16
}

var (
	nodes    = make(map[string]Node)
	lock     sync.Mutex
	queries  = 0
	answered = 0
	limited  = 0
	debug    = false
	limiter  = lru.New(1024)
)

func main() {
	var listen string
	var timestamp bool

	flag.StringVar(&listen, "listen", ":22025", "Listen address")
	flag.BoolVar(&debug, "debug", false, "Enable debug output")
	flag.BoolVar(&timestamp, "timestamp", true, "Timestamp the log output")
	flag.Parse()

	log.SetOutput(os.Stdout)
	if !timestamp {
		log.SetFlags(0)
	}

	addr, _ := net.ResolveUDPAddr("udp", listen)
	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		log.Fatal(err)
	}

	go logStats()

	var buf = make([]byte, 1024)
	for {
		buf = buf[:cap(buf)]
		n, addr, err := conn.ReadFromUDP(buf)

		if limit(addr) {
			// Rate limit in effect for source
			continue
		}

		if err != nil {
			log.Fatal(err)
		}

		if n < 4 {
			log.Printf("Received short packet (%d bytes)", n)
			continue
		}

		buf = buf[:n]
		magic := binary.BigEndian.Uint32(buf)

		switch magic {
		case discover.AnnouncementMagicV2:
			handleAnnounceV2(addr, buf)

		case discover.QueryMagicV2:
			handleQueryV2(conn, addr, buf)
		}
	}
}

func limit(addr *net.UDPAddr) bool {
	key := addr.IP.String()

	lock.Lock()
	defer lock.Unlock()

	bkt, ok := limiter.Get(key)
	if ok {
		bkt := bkt.(*ratelimit.Bucket)
		if bkt.TakeAvailable(1) != 1 {
			// Rate limit exceeded; ignore packet
			if debug {
				log.Println("Rate limit exceeded for", key)
			}
			limited++
			return true
		}
	} else {
		if debug {
			log.Println("New limiter for", key)
		}
		// One packet per ten seconds average rate, burst ten packets
		limiter.Add(key, ratelimit.NewBucket(10*time.Second, 10))
	}

	return false
}

func handleAnnounceV2(addr *net.UDPAddr, buf []byte) {
	var pkt discover.AnnounceV2
	err := pkt.UnmarshalXDR(buf)
	if err != nil {
		log.Println("AnnounceV2 Unmarshal:", err)
		log.Println(hex.Dump(buf))
		return
	}
	if debug {
		log.Printf("<- %v %#v", addr, pkt)
	}

	ip := addr.IP.To4()
	if ip == nil {
		ip = addr.IP.To16()
	}

	var addrs []Address
	for _, addr := range pkt.Addresses {
		tip := addr.IP
		if len(tip) == 0 {
			tip = ip
		}
		addrs = append(addrs, Address{
			IP:   tip,
			Port: addr.Port,
		})
	}

	node := Node{
		Addresses: addrs,
		Updated:   time.Now(),
	}

	lock.Lock()
	nodes[pkt.NodeID] = node
	lock.Unlock()
}

func handleQueryV2(conn *net.UDPConn, addr *net.UDPAddr, buf []byte) {
	var pkt discover.QueryV2
	err := pkt.UnmarshalXDR(buf)
	if err != nil {
		log.Println("QueryV2 Unmarshal:", err)
		log.Println(hex.Dump(buf))
		return
	}
	if debug {
		log.Printf("<- %v %#v", addr, pkt)
	}

	lock.Lock()
	node, ok := nodes[pkt.NodeID]
	queries++
	lock.Unlock()

	if ok && len(node.Addresses) > 0 {
		pkt := discover.AnnounceV2{
			Magic:  discover.AnnouncementMagicV2,
			NodeID: pkt.NodeID,
		}
		for _, addr := range node.Addresses {
			pkt.Addresses = append(pkt.Addresses, discover.Address{IP: addr.IP, Port: addr.Port})
		}
		if debug {
			log.Printf("-> %v %#v", addr, pkt)
		}

		tb := pkt.MarshalXDR()
		_, _, err = conn.WriteMsgUDP(tb, nil, addr)
		if err != nil {
			log.Println("QueryV2 response write:", err)
		}

		lock.Lock()
		answered++
		lock.Unlock()
	}
}

func logStats() {
	for {
		time.Sleep(600 * time.Second)

		lock.Lock()

		var deleted = 0
		for id, node := range nodes {
			if time.Since(node.Updated) > 60*time.Minute {
				delete(nodes, id)
				deleted++
			}
		}

		log.Printf("Expired %d nodes; %d nodes in registry; %d queries (%d answered)", deleted, len(nodes), queries, answered)
		log.Printf("Limited %d queries; %d entries in limiter cache", limited, limiter.Len())
		queries = 0
		answered = 0
		limited = 0

		lock.Unlock()
	}
}
<|endoftext|>


<commit_before>// netbackup - Consistent multi-method backup tool
//
// See instructions in the README.md file that accompanies this program.
//
// (C) 2015 by Marco Paganini <paganini AT paganini DOT net>

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/marcopaganini/logger"
	"github.com/marcopaganini/netbackup/config"
	"github.com/marcopaganini/netbackup/execute"
	"github.com/marcopaganini/netbackup/transports"
)

const (
	defaultLogDir = "/tmp/log/netbackup"
	devMapperDir  = "/dev/mapper"

	// Default permissions for log directories and files.
	// The current umask will apply to these.
	defaultLogDirMode  = 0777
	defaultLogFileMode = 0666

	// Return codes
	osSuccess = 0
	osError   = 1

	// External commands.
	mountCmd      = "mount"
	umountCmd     = "umount"
	cryptSetupCmd = "cryptsetup"
	fsckCmd       = "fsck"
	tunefsCmd     = "tune2fs"
	dfCmd         = "df"
)

// Backup contains information for a given backup instance.
type Backup struct {
	log     *logger.Logger
	config  *config.Config
	outLog  *os.File
	verbose int
	dryRun  bool
}

var (
	// Generic logging object
	log *logger.Logger

	// Output Log
	outLog *os.File = os.Stderr
)

// NewBackup creates a new Backup instance.
func NewBackup(log *logger.Logger, config *config.Config, outLog *os.File, verbose int, dryRun bool) *Backup {
	// Create new Backup and execute.
	return &Backup{
		log:     log,
		config:  config,
		outLog:  outLog,
		verbose: verbose,
		dryRun:  opt.dryrun}
}

// mountDev mounts the destination device into a temporary mount point and
// returns the mount point name.
func (b *Backup) mountDev() (string, error) {
	tmpdir, err := ioutil.TempDir("", "netbackup_mount")
	if err != nil {
		return "", fmt.Errorf("unable to create temp directory: %v", err)
	}

	// We use the mount command instead of the mount syscall as it makes
	// simpler to specify defaults in /etc/fstab.
	cmd := mountCmd + " " + b.config.DestDev + " " + tmpdir
	if err := runCommand("MOUNT", cmd, nil); err != nil {
		return "", err
	}

	return tmpdir, nil
}

// umountDev dismounts the destination device specified in config.DestDev.
func (b *Backup) umountDev() error {
	cmd := umountCmd + " " + b.config.DestDev
	return runCommand("UMOUNT", cmd, nil)
}

// openLuks opens the luks destination device into a temporary /dev/mapper
// device file and retuns the /dev/mapper device filename.
func (b *Backup) openLuks() (string, error) {
	// Our temporary dev/mapper device is based on the config name
	devname := "netbackup_" + b.config.Name
	devfile := filepath.Join(devMapperDir, devname)

	// Make sure it doesn't already exist
	if _, err := os.Stat(devfile); err == nil {
		return "", fmt.Errorf("device mapper file %q already exists", devfile)
	}

	// cryptsetup LuksOpen
	cmd := cryptSetupCmd
	if b.config.LuksKeyFile != "" {
		cmd += " --key-file " + b.config.LuksKeyFile
	}
	cmd += " luksOpen " + b.config.LuksDestDev + " " + devname
	if err := runCommand("LUKS_OPEN", cmd, nil); err != nil {
		return "", err
	}

	return devfile, nil
}

// closeLuks closes the current destination device.
func (b *Backup) closeLuks() error {
	// cryptsetup luksClose needs the /dev/mapper device name.
	cmd := cryptSetupCmd + " luksClose " + b.config.DestDev
	return runCommand("LUKS_CLOSE", cmd, nil)
}

// cleanFilesystem runs fsck to make sure the filesystem under config.dest_dev is
// intact, and sets the number of times to check to 0 and the last time
// checked to now. This option should only be used in EXTn filesystems or
// filesystems that support tunefs.
func (b *Backup) cleanFilesystem() error {
	// fsck (read-only check)
	cmd := fsckCmd + " -n " + b.config.DestDev
	if err := runCommand("FS_CLEANUP", cmd, nil); err != nil {
		return fmt.Errorf("error running %q: %v", cmd, err)
	}
	// Tunefs
	cmd = tunefsCmd + " -C 0 -T now " + b.config.DestDev
	return runCommand("FS_CLEANUP", cmd, nil)
}

// Run executes the backup according to the config file and options.
func (b *Backup) Run() error {
	var transp interface {
		Run() error
	}

	if !b.dryRun {
		// Open LUKS device, if needed
		if b.config.LuksDestDev != "" {
			devfile, err := b.openLuks()
			if err != nil {
				return fmt.Errorf("Error opening LUKS device %q: %v", b.config.LuksDestDev, err)
			}
			// Set the destination device to the /dev/mapper device opened by
			// LUKS. This should allow the natural processing to mount and
			// dismount this device.
			b.config.DestDev = devfile

			// close luks device at the end
			defer b.closeLuks()
			defer time.Sleep(2 * time.Second)
		}

		// Run cleanup on fs prior to backup, if requested.
		if b.config.FSCleanup {
			if err := b.cleanFilesystem(); err != nil {
				return fmt.Errorf("Error performing pre-backup cleanup on %q: %v", b.config.DestDev, err)
			}
		}

		// Mount destination device, if needed.
		if b.config.DestDev != "" {
			tmpdir, err := b.mountDev()
			if err != nil {
				return fmt.Errorf("Error opening destination device %q: %v", b.config.DestDev, err)
			}
			// After we mount the destination device, we set Destdir to that location
			// so the backup will proceed seamlessly.
			b.config.DestDir = tmpdir

			// umount destination filesystem and remove temp mount point.
			defer os.Remove(b.config.DestDir)
			defer b.umountDev()
			// For some reason, not having a pause before attempting to unmount
			// can generate a race condition where umount complains that the fs
			// is busy (even though the transport is already down.)
			defer time.Sleep(2 * time.Second)
		}
	}

	var err error

	// Create new transport based on config.Transport
	switch b.config.Transport {
	case "rclone":
		transp, err = transports.NewRcloneTransport(b.config, nil, b.outLog, int(opt.verbose), b.dryRun)
	case "rdiff-backup":
		transp, err = transports.NewRdiffBackupTransport(b.config, nil, b.outLog, int(opt.verbose), b.dryRun)
	case "rsync":
		transp, err = transports.NewRsyncTransport(b.config, nil, b.outLog, int(opt.verbose), b.dryRun)
	default:
		return fmt.Errorf("Unknown transport %q", b.config.Transport)
	}
	if err != nil {
		return fmt.Errorf("Error creating %s transport: %v", b.config.Transport, err)
	}

	// Execute pre-commands, if any.
	if b.config.PreCommand != "" && !b.dryRun {
		if err := runCommand("PRE", b.config.PreCommand, nil); err != nil {
			return fmt.Errorf("Error running pre-command: %v", err)
		}
	}

	// Make it so...
	if err := transp.Run(); err != nil {
		return fmt.Errorf("Error running backup: %v", err)
	}
	fmt.Fprintf(b.outLog, "*** Backup Result: Success\n")

	// Execute post-commands, if any.
	if b.config.PostCommand != "" && !b.dryRun {
		if err := runCommand("POST", b.config.PostCommand, nil); err != nil {
			fmt.Fprintf(b.outLog, "*** Backup Result: Failure (%v)\n", err)
			return fmt.Errorf("Error running post-command: %v", err)
		}
	}

	return nil
}

// usage prints an error message and program usage to stderr, exiting after
// that.
func usage(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v\n\n", err)
	}
	fmt.Fprintf(os.Stderr, "Usage%s:\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(2)
}

// runCommand executes the given command using the shell. A prefix will
// be used to log the commands to the output log. Returns error.
func runCommand(prefix string, cmd string, ex *execute.Execute) error {
	m := fmt.Sprintf("%s Command: %q", prefix, cmd)
	fmt.Fprintf(outLog, "%s\n", m)
	log.Verboseln(int(opt.verbose), m)

	// Create a new execute object, if current is nil
	e := ex
	if e == nil {
		e = execute.New()
	}

	// All streams copied to output log with "PRE:" as a prefix.
	e.SetStdout(func(buf string) error { _, err := fmt.Fprintf(outLog, "%s(stdout): %s\n", prefix, buf); return err })
	e.SetStderr(func(buf string) error { _, err := fmt.Fprintf(outLog, "%s(stderr): %s\n", prefix, buf); return err })

	// Run using shell
	shell := os.Getenv("SHELL")
	if shell == "" {
		shell = "/bin/sh"
	}
	err := e.Exec([]string{shell, "-c", "--", cmd})
	if err != nil {
		errmsg := fmt.Sprintf("%s returned: %v", prefix, err)
		fmt.Fprintf(outLog, "*** %s\n", errmsg)
		return fmt.Errorf(errmsg)
	}
	fmt.Fprintf(outLog, "%s returned: OK\n", prefix)
	return nil
}

// logPath constructs the name for the output log using the the name and
// the current system date.
func logPath(name string, logDir string) string {
	ymd := time.Now().Format("2006-01-02")
	dir := filepath.Join(logDir, name)
	return filepath.Join(dir, name+"-"+ymd+".log")
}

// logOpen opens (for append) or creates (if needed) the specified file.
// If the file doesn't exist, all intermediate directories will be created.
// Returns an *os.File to the just opened file.
func logOpen(path string) (*os.File, error) {
	// Create full directory path if it doesn't exist yet.
	dir := filepath.Dir(path)
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		if err := os.MkdirAll(dir, defaultLogDirMode); err != nil {
			return nil, fmt.Errorf("unable to create dir tree %q: %v", dir, err)
		}
	}

	// Open for append or create if doesn't exist.
	w, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY|os.O_CREATE, defaultLogFileMode)
	if err != nil {
		return nil, fmt.Errorf("unable to open %q: %v", path, err)
	}
	return w, nil
}

// main
func main() {
	log = logger.New("")

	// Parse command line flags and read config file.
	if err := parseFlags(); err != nil {
		log.Printf("Command line error: %v", err)
		os.Exit(osError)
	}

	// Set verbose level
	if opt.verbose > 0 {
		log.SetVerboseLevel(int(opt.verbose))
	}
	if opt.dryrun {
		log.Verbosef(2, "Warning: Dry-Run mode. Won't execute any commands.")
	}

	// Open and parse config file
	cfg, err := os.Open(opt.config)
	if err != nil {
		log.Printf("Unable to open config file: %v", err)
		os.Exit(osError)
	}
	config, err := config.ParseConfig(cfg)
	if err != nil {
		log.Printf("Configuration error in %q: %v", opt.config, err)
		os.Exit(osError)
	}

	// Create output log. Use the name specified in the config, if any,
	// or create a "standard" name using the backup name and date.
	logFilename := config.Logfile
	if logFilename == "" {
		logFilename = logPath(config.Name, defaultLogDir)
	}
	outLog, err := logOpen(logFilename)
	if err != nil {
		log.Printf("Unable to open/create logfile: %v", err)
		os.Exit(osError)
	}
	defer outLog.Close()

	// Create new Backup and execute.
	b := NewBackup(log, config, outLog, int(opt.verbose), opt.dryrun)

	if err = b.Run(); err != nil {
		log.Println(err)
		os.Exit(osError)
	}

	os.Exit(osSuccess)
}
<commit_msg>log now outputs messages to outLog and stderr.<commit_after>// netbackup - Consistent multi-method backup tool
//
// See instructions in the README.md file that accompanies this program.
//
// (C) 2015 by Marco Paganini <paganini AT paganini DOT net>

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/marcopaganini/logger"
	"github.com/marcopaganini/netbackup/config"
	"github.com/marcopaganini/netbackup/execute"
	"github.com/marcopaganini/netbackup/transports"
)

const (
	defaultLogDir = "/tmp/log/netbackup"
	devMapperDir  = "/dev/mapper"

	// Default permissions for log directories and files.
	// The current umask will apply to these.
	defaultLogDirMode  = 0777
	defaultLogFileMode = 0666

	// Return codes
	osSuccess = 0
	osError   = 1

	// External commands.
	mountCmd      = "mount"
	umountCmd     = "umount"
	cryptSetupCmd = "cryptsetup"
	fsckCmd       = "fsck"
	tunefsCmd     = "tune2fs"
	dfCmd         = "df"
)

// Backup contains information for a given backup instance.
type Backup struct {
	log     *logger.Logger
	config  *config.Config
	outLog  *os.File
	verbose int
	dryRun  bool
}

var (
	// Generic logging object
	log *logger.Logger

	// Output Log
	outLog *os.File = os.Stderr
)

// NewBackup creates a new Backup instance.
func NewBackup(log *logger.Logger, config *config.Config, outLog *os.File, verbose int, dryRun bool) *Backup {
	// Create new Backup and execute.
	return &Backup{
		log:     log,
		config:  config,
		outLog:  outLog,
		verbose: verbose,
		dryRun:  opt.dryrun}
}

// mountDev mounts the destination device into a temporary mount point and
// returns the mount point name.
func (b *Backup) mountDev() (string, error) {
	tmpdir, err := ioutil.TempDir("", "netbackup_mount")
	if err != nil {
		return "", fmt.Errorf("unable to create temp directory: %v", err)
	}

	// We use the mount command instead of the mount syscall as it makes
	// simpler to specify defaults in /etc/fstab.
	cmd := mountCmd + " " + b.config.DestDev + " " + tmpdir
	if err := runCommand("MOUNT", cmd, nil); err != nil {
		return "", err
	}

	return tmpdir, nil
}

// umountDev dismounts the destination device specified in config.DestDev.
func (b *Backup) umountDev() error {
	cmd := umountCmd + " " + b.config.DestDev
	return runCommand("UMOUNT", cmd, nil)
}

// openLuks opens the luks destination device into a temporary /dev/mapper
// device file and retuns the /dev/mapper device filename.
func (b *Backup) openLuks() (string, error) {
	// Our temporary dev/mapper device is based on the config name
	devname := "netbackup_" + b.config.Name
	devfile := filepath.Join(devMapperDir, devname)

	// Make sure it doesn't already exist
	if _, err := os.Stat(devfile); err == nil {
		return "", fmt.Errorf("device mapper file %q already exists", devfile)
	}

	// cryptsetup LuksOpen
	cmd := cryptSetupCmd
	if b.config.LuksKeyFile != "" {
		cmd += " --key-file " + b.config.LuksKeyFile
	}
	cmd += " luksOpen " + b.config.LuksDestDev + " " + devname
	if err := runCommand("LUKS_OPEN", cmd, nil); err != nil {
		return "", err
	}

	return devfile, nil
}

// closeLuks closes the current destination device.
func (b *Backup) closeLuks() error {
	// cryptsetup luksClose needs the /dev/mapper device name.
	cmd := cryptSetupCmd + " luksClose " + b.config.DestDev
	return runCommand("LUKS_CLOSE", cmd, nil)
}

// cleanFilesystem runs fsck to make sure the filesystem under config.dest_dev is
// intact, and sets the number of times to check to 0 and the last time
// checked to now. This option should only be used in EXTn filesystems or
// filesystems that support tunefs.
func (b *Backup) cleanFilesystem() error {
	// fsck (read-only check)
	cmd := fsckCmd + " -n " + b.config.DestDev
	if err := runCommand("FS_CLEANUP", cmd, nil); err != nil {
		return fmt.Errorf("error running %q: %v", cmd, err)
	}
	// Tunefs
	cmd = tunefsCmd + " -C 0 -T now " + b.config.DestDev
	return runCommand("FS_CLEANUP", cmd, nil)
}

// Run executes the backup according to the config file and options.
func (b *Backup) Run() error {
	var transp interface {
		Run() error
	}

	if !b.dryRun {
		// Open LUKS device, if needed
		if b.config.LuksDestDev != "" {
			devfile, err := b.openLuks()
			if err != nil {
				return fmt.Errorf("Error opening LUKS device %q: %v", b.config.LuksDestDev, err)
			}
			// Set the destination device to the /dev/mapper device opened by
			// LUKS. This should allow the natural processing to mount and
			// dismount this device.
			b.config.DestDev = devfile

			// close luks device at the end
			defer b.closeLuks()
			defer time.Sleep(2 * time.Second)
		}

		// Run cleanup on fs prior to backup, if requested.
		if b.config.FSCleanup {
			if err := b.cleanFilesystem(); err != nil {
				return fmt.Errorf("Error performing pre-backup cleanup on %q: %v", b.config.DestDev, err)
			}
		}

		// Mount destination device, if needed.
		if b.config.DestDev != "" {
			tmpdir, err := b.mountDev()
			if err != nil {
				return fmt.Errorf("Error opening destination device %q: %v", b.config.DestDev, err)
			}
			// After we mount the destination device, we set Destdir to that location
			// so the backup will proceed seamlessly.
			b.config.DestDir = tmpdir

			// umount destination filesystem and remove temp mount point.
			defer os.Remove(b.config.DestDir)
			defer b.umountDev()
			// For some reason, not having a pause before attempting to unmount
			// can generate a race condition where umount complains that the fs
			// is busy (even though the transport is already down.)
			defer time.Sleep(2 * time.Second)
		}
	}

	var err error

	// Create new transport based on config.Transport
	switch b.config.Transport {
	case "rclone":
		transp, err = transports.NewRcloneTransport(b.config, nil, b.outLog, b.verbose, b.dryRun)
	case "rdiff-backup":
		transp, err = transports.NewRdiffBackupTransport(b.config, nil, b.outLog, b.verbose, b.dryRun)
	case "rsync":
		transp, err = transports.NewRsyncTransport(b.config, nil, b.outLog, b.verbose, b.dryRun)
	default:
		return fmt.Errorf("Unknown transport %q", b.config.Transport)
	}
	if err != nil {
		return fmt.Errorf("Error creating %s transport: %v", b.config.Transport, err)
	}

	// Execute pre-commands, if any.
	if b.config.PreCommand != "" && !b.dryRun {
		if err := runCommand("PRE", b.config.PreCommand, nil); err != nil {
			return fmt.Errorf("Error running pre-command: %v", err)
		}
	}

	// Make it so...
	if err := transp.Run(); err != nil {
		return fmt.Errorf("Error running backup: %v", err)
	}

	// Execute post-commands, if any.
	if b.config.PostCommand != "" && !b.dryRun {
		if err := runCommand("POST", b.config.PostCommand, nil); err != nil {
			return fmt.Errorf("Error running post-command (possible backup failure): %v", err)
		}
	}

	return nil
}

// usage prints an error message and program usage to stderr, exiting after
// that.
func usage(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v\n\n", err)
	}
	fmt.Fprintf(os.Stderr, "Usage%s:\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(2)
}

// runCommand executes the given command using the shell. A prefix will
// be used to log the commands to the output log. Returns error.
func runCommand(prefix string, cmd string, ex *execute.Execute) error {
	m := fmt.Sprintf("%s Command: %q", prefix, cmd)
	log.Verboseln(1, m)

	// Create a new execute object, if current is nil
	e := ex
	if e == nil {
		e = execute.New()
	}

	// All streams copied to output log with "PRE:" as a prefix.
	e.SetStdout(func(buf string) error { _, err := fmt.Fprintf(outLog, "%s(stdout): %s\n", prefix, buf); return err })
	e.SetStderr(func(buf string) error { _, err := fmt.Fprintf(outLog, "%s(stderr): %s\n", prefix, buf); return err })

	// Run using shell
	shell := os.Getenv("SHELL")
	if shell == "" {
		shell = "/bin/sh"
	}
	err := e.Exec([]string{shell, "-c", "--", cmd})
	if err != nil {
		errmsg := fmt.Sprintf("%s returned: %v", prefix, err)
		log.Verbosef(1, "*** %s\n", errmsg)
		return fmt.Errorf(errmsg)
	}
	log.Verbosef(1, "%s returned: OK\n", prefix)
	return nil
}

// logPath constructs the name for the output log using the the name and
// the current system date.
func logPath(name string, logDir string) string {
	ymd := time.Now().Format("2006-01-02")
	dir := filepath.Join(logDir, name)
	return filepath.Join(dir, name+"-"+ymd+".log")
}

// logOpen opens (for append) or creates (if needed) the specified file.
// If the file doesn't exist, all intermediate directories will be created.
// Returns an *os.File to the just opened file.
func logOpen(path string) (*os.File, error) {
	// Create full directory path if it doesn't exist yet.
	dir := filepath.Dir(path)
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		if err := os.MkdirAll(dir, defaultLogDirMode); err != nil {
			return nil, fmt.Errorf("unable to create dir tree %q: %v", dir, err)
		}
	}

	// Open for append or create if doesn't exist.
	w, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY|os.O_CREATE, defaultLogFileMode)
	if err != nil {
		return nil, fmt.Errorf("unable to open %q: %v", path, err)
	}
	return w, nil
}

// main
func main() {
	log = logger.New("")

	// Parse command line flags and read config file.
	if err := parseFlags(); err != nil {
		log.Printf("Command line error: %v", err)
		os.Exit(osError)
	}

	// Set verbose level
	verbose := int(opt.verbose)
	if verbose > 0 {
		log.SetVerboseLevel(verbose)
	}
	if opt.dryrun {
		log.Verbosef(2, "Warning: Dry-Run mode. Won't execute any commands.")
	}

	// Open and parse config file
	cfg, err := os.Open(opt.config)
	if err != nil {
		log.Printf("Unable to open config file: %v", err)
		os.Exit(osError)
	}
	config, err := config.ParseConfig(cfg)
	if err != nil {
		log.Printf("Configuration error in %q: %v", opt.config, err)
		os.Exit(osError)
	}

	// Create output log. Use the name specified in the config, if any,
	// or create a "standard" name using the backup name and date.
	logFilename := config.Logfile
	if logFilename == "" {
		logFilename = logPath(config.Name, defaultLogDir)
	}
	outLog, err := logOpen(logFilename)
	if err != nil {
		log.Printf("Unable to open/create logfile: %v", err)
		os.Exit(osError)
	}
	defer outLog.Close()

	// Configure log to log everything to stderr and outLog
	log.SetOutput([]*os.File{os.Stderr, outLog})

	// Create new Backup and execute.
	b := NewBackup(log, config, outLog, verbose, opt.dryrun)

	if err = b.Run(); err != nil {
		log.Println(err)
		os.Exit(osError)
	}
	log.Verboseln(1, "*** Backup Result: Success")
	os.Exit(osSuccess)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"time"
)

const (
	cookieKey   string        = "dawanda_uo"
	trackingTTL time.Duration = time.Second * 10
	cookieTTL   time.Duration = time.Hour * 24 * 30
)

func httpTrack(w http.ResponseWriter, r *http.Request, tracker *Tracker) {
	cookie, err := r.Cookie(cookieKey)

	if err != nil {
		sid, err := createSessionID()
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "Failed to generate Session ID. %v\n", err)
			return
		}
		http.SetCookie(w, &http.Cookie{
			Name:    cookieKey,
			Value:   sid,
			Expires: time.Now().Add(cookieTTL),
		})
		tracker.Touch(KindNewUsers, sid)
	} else {
		tracker.Touch(KindRecurring, cookie.Value)
	}

	writeEmptyGif(w, r)
}

func httpNotImplemented(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusNotImplemented)
}

func createSessionID() (string, error) {
	b := make([]byte, 16)
	_, err := rand.Read(b)
	if err != nil {
		return "", err
	}

	uuid := fmt.Sprintf("%X-%X-%X-%X-%X", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
	return uuid, nil
}

func writeEmptyGif(w http.ResponseWriter, r *http.Request) {
	const base64GifPixel = "R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs="
	w.Header().Set("expires", time.Now().Add(trackingTTL).Format(http.TimeFormat))
	w.Header().Set("cache-control", fmt.Sprintf("max-age=%v", int(trackingTTL.Seconds())))
	w.Header().Set("content-type", "image/gif")
	output, _ := base64.StdEncoding.DecodeString(base64GifPixel)
	io.WriteString(w, string(output))
}

func main() {
	httpBindAddr := flag.String("http-bind", "0.0.0.0", "HTTP service bind address")
	httpPort := flag.Int("http-port", 8087, "HTTP service port")
	statsdAddr := flag.String("statsd-addr", "127.0.0.1:8125", "Statsd endpoint")
	statsdPrefix := flag.String("statsd-prefix", "tracker.sessions", "Statsd key prefix")
	debug := flag.Bool("debug", false, "Enable debugging messages")
	flag.Parse()

	tracker, err := NewTracker(*statsdAddr, *statsdPrefix, *debug)
	if err != nil {
		log.Fatalf("Failed to create tracker. %v\n", err)
	}

	http.HandleFunc("/uo/trck.gif", func(w http.ResponseWriter, r *http.Request) {
		httpTrack(w, r, tracker)
	})

	http.HandleFunc("/ping", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "useronline\n")
	})

	http.HandleFunc("/uo/newsessions/count", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, tracker.GetCount(KindNewUsers))
	})

	http.HandleFunc("/uo/sessions/count", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, tracker.GetCount(KindRecurring))
	})

	http.HandleFunc("/uo/sessions/today", httpNotImplemented)

	err = http.ListenAndServe(fmt.Sprintf("%v:%v", *httpBindAddr, *httpPort), nil)
	if err != nil {
		log.Fatalf("Failed to start service listener. %v\n", err)
	}
}
<commit_msg>code cleanup<commit_after>package main

import (
	"crypto/rand"
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"time"
)

type Service struct {
	TrackingTTL time.Duration
	CookieTTL   time.Duration
	CookieKey   string
	Tracker     *Tracker
}

func (service *Service) httpTrack(w http.ResponseWriter, r *http.Request) {
	cookie, err := r.Cookie(service.CookieKey)

	if err != nil {
		sid, err := createSessionID()
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "Failed to generate Session ID. %v\n", err)
			return
		}
		http.SetCookie(w, &http.Cookie{
			Name:    service.CookieKey,
			Value:   sid,
			Expires: time.Now().Add(service.CookieTTL),
		})
		service.Tracker.Touch(KindNewUsers, sid)
	} else {
		service.Tracker.Touch(KindRecurring, cookie.Value)
	}

	service.writeEmptyGif(w, r)
}

func (service *Service) httpSessionsCount(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, service.Tracker.GetCount(KindRecurring))
}

func (service *Service) httpNewSessionsCount(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, service.Tracker.GetCount(KindNewUsers))
}

func (service *Service) httpPing(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, "useronline")
}

func (service *Service) httpSessionsToday(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusNotImplemented) // TODO
}

func (service *Service) writeEmptyGif(w http.ResponseWriter, r *http.Request) {
	const base64GifPixel = "R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs="
	w.Header().Set("expires", time.Now().Add(service.TrackingTTL).Format(http.TimeFormat))
	w.Header().Set("cache-control", fmt.Sprintf("max-age=%v", int(service.TrackingTTL.Seconds())))
	w.Header().Set("content-type", "image/gif")
	output, _ := base64.StdEncoding.DecodeString(base64GifPixel)
	io.WriteString(w, string(output))
}

func createSessionID() (string, error) {
	b := make([]byte, 16)
	_, err := rand.Read(b)
	if err != nil {
		return "", err
	}

	uuid := fmt.Sprintf("%X-%X-%X-%X-%X", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
	return uuid, nil
}

func main() {
	var service = Service{
		TrackingTTL: time.Second * 10,
		CookieTTL:   time.Hour * 24 * 30,
		CookieKey:   "dawanda_uo",
	}

	flag.DurationVar(&service.TrackingTTL, "tracking-ttl", service.TrackingTTL, "tracking pixel expiry timespan")
	flag.DurationVar(&service.CookieTTL, "cookie-ttl", service.CookieTTL, "cookie expiry")
	flag.StringVar(&service.CookieKey, "cookie", service.CookieKey, "Name of the cookie, such as browny")

	httpBindAddr := flag.String("http-bind", "0.0.0.0", "HTTP service bind address")
	httpPort := flag.Int("http-port", 8087, "HTTP service port")
	statsdAddr := flag.String("statsd-addr", "127.0.0.1:8125", "Statsd endpoint")
	statsdPrefix := flag.String("statsd-prefix", "tracker.sessions", "Statsd key prefix")
	debug := flag.Bool("debug", false, "Enable debugging messages")
	flag.Parse()

	tracker, err := NewTracker(*statsdAddr, *statsdPrefix, *debug)
	if err != nil {
		log.Fatalf("Failed to create tracker. %v\n", err)
	}
	service.Tracker = tracker

	http.HandleFunc("/uo/trck.gif", service.httpTrack)
	http.HandleFunc("/ping", service.httpPing)
	http.HandleFunc("/uo/newsessions/count", service.httpNewSessionsCount)
	http.HandleFunc("/uo/sessions/count", service.httpSessionsCount)
	http.HandleFunc("/uo/sessions/today", service.httpSessionsToday)

	err = http.ListenAndServe(fmt.Sprintf("%v:%v", *httpBindAddr, *httpPort), nil)
	if err != nil {
		log.Fatalf("Failed to start service listener. %v\n", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"net"
	"os"
	"os/signal"
	"unsafe"

	bpf "github.com/kinvolk/go-ebpf-kprobe-example/bpf"
)

import "C"

type tcpEvent struct {
	// Timestamp must be the first field, the sorting depends on it
	Timestamp uint64

	Cpu   uint64
	Type  [12]C.char
	Pid   uint32
	Comm  [16]C.char
	SAddr uint32
	DAddr uint32
	SPort uint16
	DPort uint16
	NetNS uint32
}

var byteOrder binary.ByteOrder

// In lack of binary.HostEndian ...
func init() {
	var i int32 = 0x01020304
	u := unsafe.Pointer(&i)
	pb := (*byte)(u)
	b := *pb
	if b == 0x04 {
		byteOrder = binary.LittleEndian
	} else {
		byteOrder = binary.BigEndian
	}
}

var lastTimestamp uint64

func tcpEventCb(event tcpEvent) {
	timestamp := uint64(event.Timestamp)
	cpu := event.Cpu
	typ := C.GoString(&event.Type[0])
	pid := event.Pid & 0xffffffff

	saddrbuf := make([]byte, 4)
	daddrbuf := make([]byte, 4)

	binary.LittleEndian.PutUint32(saddrbuf, uint32(event.SAddr))
	binary.LittleEndian.PutUint32(daddrbuf, uint32(event.DAddr))

	sIP := net.IPv4(saddrbuf[0], saddrbuf[1], saddrbuf[2], saddrbuf[3])
	dIP := net.IPv4(daddrbuf[0], daddrbuf[1], daddrbuf[2], daddrbuf[3])

	sport := event.SPort
	dport := event.DPort
	netns := event.NetNS

	fmt.Printf("%v cpu#%d %s %v %v:%v %v:%v %v\n", timestamp, cpu, typ, pid, sIP, sport, dIP, dport, netns)

	if lastTimestamp > timestamp {
		fmt.Printf("WARNING: late event!\n")
		os.Exit(1)
	}

	lastTimestamp = timestamp
}

func main() {
	b := bpf.NewBpfPerfEvent("kernel/trace_output_kern.o")

	err := b.Load()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}

	fmt.Printf("Ready.\n")

	channel := make(chan []byte)

	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt, os.Kill)

	go func() {
		var event tcpEvent
		for {
			data := <-channel
			err := binary.Read(bytes.NewBuffer(data), byteOrder, &event)
			if err != nil {
				fmt.Printf("failed to decode received data: %s\n", err)
				continue
			}
			tcpEventCb(event)
		}
	}()

	b.PollStart("tcp_event", channel)
	<-sig
	b.PollStop("tcp_event")
}
<commit_msg>Rename repository to gobpf-elf-loader<commit_after>package main

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"net"
	"os"
	"os/signal"
	"unsafe"

	bpf "github.com/kinvolk/gobpf-elf-loader/bpf"
)

import "C"

type tcpEvent struct {
	// Timestamp must be the first field, the sorting depends on it
	Timestamp uint64

	Cpu   uint64
	Type  [12]C.char
	Pid   uint32
	Comm  [16]C.char
	SAddr uint32
	DAddr uint32
	SPort uint16
	DPort uint16
	NetNS uint32
}

var byteOrder binary.ByteOrder

// In lack of binary.HostEndian ...
func init() {
	var i int32 = 0x01020304
	u := unsafe.Pointer(&i)
	pb := (*byte)(u)
	b := *pb
	if b == 0x04 {
		byteOrder = binary.LittleEndian
	} else {
		byteOrder = binary.BigEndian
	}
}

var lastTimestamp uint64

func tcpEventCb(event tcpEvent) {
	timestamp := uint64(event.Timestamp)
	cpu := event.Cpu
	typ := C.GoString(&event.Type[0])
	pid := event.Pid & 0xffffffff

	saddrbuf := make([]byte, 4)
	daddrbuf := make([]byte, 4)

	binary.LittleEndian.PutUint32(saddrbuf, uint32(event.SAddr))
	binary.LittleEndian.PutUint32(daddrbuf, uint32(event.DAddr))

	sIP := net.IPv4(saddrbuf[0], saddrbuf[1], saddrbuf[2], saddrbuf[3])
	dIP := net.IPv4(daddrbuf[0], daddrbuf[1], daddrbuf[2], daddrbuf[3])

	sport := event.SPort
	dport := event.DPort
	netns := event.NetNS

	fmt.Printf("%v cpu#%d %s %v %v:%v %v:%v %v\n", timestamp, cpu, typ, pid, sIP, sport, dIP, dport, netns)

	if lastTimestamp > timestamp {
		fmt.Printf("WARNING: late event!\n")
		os.Exit(1)
	}

	lastTimestamp = timestamp
}

func main() {
	b := bpf.NewBpfPerfEvent("kernel/trace_output_kern.o")

	err := b.Load()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}

	fmt.Printf("Ready.\n")

	channel := make(chan []byte)

	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt, os.Kill)

	go func() {
		var event tcpEvent
		for {
			data := <-channel
			err := binary.Read(bytes.NewBuffer(data), byteOrder, &event)
			if err != nil {
				fmt.Printf("failed to decode received data: %s\n", err)
				continue
			}
			tcpEventCb(event)
		}
	}()

	b.PollStart("tcp_event", channel)
	<-sig
	b.PollStop("tcp_event")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/binary"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"

	"github.com/layeh/gopus"
)

var (
	channels    int = 2                   // 1 for mono, 2 for stereo
	frameRate   int = 48000               // audio sampling rate
	frameSize   int = 960                 // uint16 size of each audio frame
	maxBytes    int = (frameSize * 2) * 2 // max size of opus data
	opusEncoder *gopus.Encoder
	run         *exec.Cmd
)

// very simple program that wraps ffmpeg and outputs opus data
func main() {

	var err error

	if len(os.Args) < 2 {
		fmt.Println("Must supply the filename to process.")
		return
	}

	filename := os.Args[1]

	opusEncoder, err = gopus.NewEncoder(frameRate, channels, gopus.Audio)
	if err != nil {
		fmt.Println("NewEncoder Error:", err)
		return
	}

	// Create a shell command "object" to run.
	run = exec.Command("ffmpeg", "-i", filename, "-f", "s16le", "-ar", strconv.Itoa(frameRate), "-ac", strconv.Itoa(channels), "pipe:1")
	stdout, err := run.StdoutPipe()
	if err != nil {
		fmt.Println("StdoutPipe Error:", err)
		return
	}

	// Starts the ffmpeg command
	err = run.Start()
	if err != nil {
		fmt.Println("RunStart Error:", err)
		return
	}

	// buffer used during loop below
	audiobuf := make([]int16, frameSize*channels)

	// "header" :)
	var opuslen uint16

	for {

		// read data from ffmpeg stdout
		err = binary.Read(stdout, binary.LittleEndian, &audiobuf)
		if err == io.EOF || err == io.ErrUnexpectedEOF {
			return
		}
		if err != nil {
			fmt.Println("error reading from ffmpeg stdout :", err)
			return
		}

		// try encoding pcm frame with Opus
		opus, err := opusEncoder.Encode(audiobuf, frameSize, maxBytes)
		if err != nil {
			fmt.Println("Encoding Error:", err)
			return
		}

		// write header
		opuslen = uint16(len(opus))
		err = binary.Write(os.Stdout, binary.LittleEndian, &opuslen)
		if err != nil {
			fmt.Println("error writing output: ", err)
			return
		}

		// write opus data to stdout
		err = binary.Write(os.Stdout, binary.LittleEndian, &opus)
		if err != nil {
			fmt.Println("error writing output: ", err)
			return
		}
	}
}
<commit_msg>Added cmd arguments, stdin pipe support, threading.<commit_after>package main

import (
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"sync"

	"github.com/layeh/gopus"
)

// All global variables used within the program
var (
	Channels  int
	FrameRate int

	FrameSize int = 960                 // uint16 size of each audio frame
	MaxBytes  int = (FrameSize * 2) * 2 // max size of opus data

	OpusEncoder *gopus.Encoder

	InFile string

	OutFile string = "pipe:1"
	OutBuf  []byte

	EncodeChan chan []int16
	OutputChan chan []byte

	err error

	wg sync.WaitGroup
)

// init configures and parses the command line arguments
func init() {

	flag.StringVar(&InFile, "i", "", "infile")
	flag.IntVar(&Channels, "ac", 2, "audio channels")
	flag.IntVar(&FrameRate, "ar", 48000, "audio sampling rate")
	flag.Parse()
}

// very simple program that wraps ffmpeg and outputs raw opus data frames
// with a uint16 header for each frame with the frame length in bytes
func main() {

	//////////////////////////////////////////////////////////////////////////
	// BLOCK : Basic setup and validation
	//////////////////////////////////////////////////////////////////////////

	// figure out if we're reading in from a file or stdin.
	fi, err := os.Stdin.Stat()
	if err != nil {
		fmt.Println(err)
		return
	}
	if (fi.Mode() & os.ModeCharDevice) == 0 {
		InFile = "pipe:0"
	} else {

		if InFile == "" {
			// TODO: possibly remove, here for legacy support
			if len(os.Args) < 2 {
				fmt.Println("Must supply the filename to process.")
				return
			}
			InFile = os.Args[1]
		}

		// make sure infile exists
		if _, err := os.Stat(InFile); os.IsNotExist(err) {
			fmt.Println("infile does not exist")
			return
		}
	}

	//////////////////////////////////////////////////////////////////////////
	// BLOCK : Create chans, buffers, and encoder for use
	//////////////////////////////////////////////////////////////////////////

	// create an opusEncoder to use
	OpusEncoder, err = gopus.NewEncoder(FrameRate, Channels, gopus.Audio)
	if err != nil {
		fmt.Println("NewEncoder Error:", err)
		return
	}

	OutputChan = make(chan []byte, 1)
	EncodeChan = make(chan []int16, 1)

	//////////////////////////////////////////////////////////////////////////
	// BLOCK : Start reader and writer workers
	//////////////////////////////////////////////////////////////////////////

	wg.Add(1)
	go reader()

	wg.Add(1)
	go encoder()

	wg.Add(1)
	go writer()

	// wait for above goroutines to finish, then exit.
	wg.Wait()
}

// reader reads from the input
func reader() {

	defer func() {
		close(EncodeChan)
		wg.Done()
	}()

	InBuf := make([]int16, FrameSize*Channels)

	// read from file
	if InFile != "pipe:0" {

		// Create a shell command "object" to run.
		ffmpeg := exec.Command("ffmpeg", "-i", InFile, "-f", "s16le", "-ar", strconv.Itoa(FrameRate), "-ac", strconv.Itoa(Channels), "pipe:1")
		stdout, err := ffmpeg.StdoutPipe()
		if err != nil {
			fmt.Println("StdoutPipe Error:", err)
			return
		}

		// Starts the ffmpeg command
		err = ffmpeg.Start()
		if err != nil {
			fmt.Println("RunStart Error:", err)
			return
		}

		for {

			// read data from ffmpeg stdout
			err = binary.Read(stdout, binary.LittleEndian, &InBuf)
			if err == io.EOF || err == io.ErrUnexpectedEOF {
				return
			}
			if err != nil {
				fmt.Println("error reading from ffmpeg stdout :", err)
				return
			}

			// write pcm data to the EncodeChan
			EncodeChan <- InBuf

		}
	}

	// read input from stdin pipe
	if InFile == "pipe:0" {
		for {

			// read data from stdin
			err = binary.Read(os.Stdin, binary.LittleEndian, &InBuf)
			if err == io.EOF || err == io.ErrUnexpectedEOF {
				return
			}
			if err != nil {
				fmt.Println("error reading from ffmpeg stdout :", err)
				return
			}

			// write pcm data to the EncodeChan
			EncodeChan <- InBuf
		}
	}

}

// encoder listens on the EncodeChan and encodes provided PCM16 data
// to opus, then sends the encoded data to the OutputChan
func encoder() {

	defer func() {
		close(OutputChan)
		wg.Done()
	}()

	for {
		pcm, ok := <-EncodeChan
		if !ok {
			// if chan closed, exit
			return
		}

		// try encoding pcm frame with Opus
		opus, err := OpusEncoder.Encode(pcm, FrameSize, MaxBytes)
		if err != nil {
			fmt.Println("Encoding Error:", err)
			return
		}

		// write opus data to OutputChan
		OutputChan <- opus
	}
}

// writer listens on the OutputChan and writes the output to stdout pipe
// TODO: Add support for writing directly to a file
func writer() {

	defer wg.Done()

	var opuslen uint16

	for {
		opus, ok := <-OutputChan
		if !ok {
			// if chan closed, exit
			return
		}

		// write header
		opuslen = uint16(len(opus))
		err = binary.Write(os.Stdout, binary.LittleEndian, &opuslen)
		if err != nil {
			fmt.Println("error writing output: ", err)
			return
		}

		// write opus data to stdout
		err = binary.Write(os.Stdout, binary.LittleEndian, &opus)
		if err != nil {
			fmt.Println("error writing output: ", err)
			return
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"howett.net/plist"
	"os"
	"path/filepath"
	"strings"
)

type weblocHeader struct {
	URL string `plist:"URL"`
}

var delete bool
var noop bool

func init() {
	flag.BoolVar(&delete, "delete", false, "delete .webloc files after conversion")
	flag.BoolVar(&noop, "noop", false, "decode urls, but do not change file system")
}

func main() {
	root := flag.Arg(0)

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [options] path\n  path: the path to process\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()

	filepath.Walk(root, walkpath)
}

func walkpath(path string, f os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	matched, err := filepath.Match("*.webloc", f.Name())
	if err != nil {
		return err
	}
	if matched {
		process(path)
	}
	return nil
}

func process(path string) {
	url := decode(path)
	fmt.Println(url)

	if !noop {
		newPath := convertPath(path)
		writeUrl(newPath, url)
		if delete {
			err := os.Remove(path)
			check(err)
		}
	}
}

func decode(path string) string {
	var data weblocHeader

	f, err := os.Open(path)
	check(err)
	defer f.Close()

	decoder := plist.NewDecoder(f)
	check(decoder.Decode(&data))
	return data.URL
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func convertPath(path string) string {
	newPath := path[:len(path)-len(".webloc")] + ".url"
	newPath = strings.Replace(newPath, "|", "-", -1)
	return newPath
}

func writeUrl(path string, url string) {
	f, err := os.Create(path)
	check(err)
	defer f.Close()

	f.WriteString("[InternetShortcut]\nURL=" + url)
}
<commit_msg>print version<commit_after>package main

import (
	"flag"
	"fmt"
	"howett.net/plist"
	"os"
	"path/filepath"
	"strings"
)

const version string = "1.0.0"

type weblocHeader struct {
	URL string `plist:"URL"`
}

var delete bool
var noop bool

func init() {
	flag.BoolVar(&delete, "delete", false, "delete .webloc files after conversion")
	flag.BoolVar(&noop, "noop", false, "decode urls, but do not change file system")
}

func main() {
	showVersion := flag.Bool("version", false, "print version")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [options] path\n  path: the path to process\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()
	if *showVersion {
		fmt.Println("webloc version", version)
		return
	}

	filepath.Walk(flag.Arg(0), walkpath)
}

func walkpath(path string, f os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	matched, err := filepath.Match("*.webloc", f.Name())
	if err != nil {
		return err
	}
	if matched {
		process(path)
	}
	return nil
}

func process(path string) {
	url := decode(path)
	fmt.Println(url)

	if !noop {
		newPath := convertPath(path)
		writeUrl(newPath, url)
		if delete {
			err := os.Remove(path)
			check(err)
		}
	}
}

func decode(path string) string {
	var data weblocHeader

	f, err := os.Open(path)
	check(err)
	defer f.Close()

	decoder := plist.NewDecoder(f)
	check(decoder.Decode(&data))
	return data.URL
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func convertPath(path string) string {
	newPath := path[:len(path)-len(".webloc")] + ".url"
	newPath = strings.Replace(newPath, "|", "-", -1)
	return newPath
}

func writeUrl(path string, url string) {
	f, err := os.Create(path)
	check(err)
	defer f.Close()

	f.WriteString("[InternetShortcut]\nURL=" + url)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"net/http"
	"log"
	"os"
	"flag"
	"path/filepath"
	"github.com/julienschmidt/httprouter"
)

type config struct {
	HTTP string
}

type Task struct {
	User string
	SampleID string
	//TODO
}

func decrypt(encrypted string) (string) {
	decrypted := encrypted
	//TODO
	return decrypted
}

func validate(task string) (error, []Task) {
	var tasks []Task
	err := json.Unmarshal([]byte(task), &tasks)
	if err != nil {
		return err, nil
	}
	//TODO
	return err, tasks
}

func checkACL(task Task) (error){
	//TODO
	return nil
}

func httpRequestIncoming(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	tsk := ps.ByName("name")
	log.Println("httprequest..." + tsk);
	tsk = decrypt(tsk)
	err, tasks := validate(tsk)
	if err != nil {
		log.Println("Error while validating: ", err)
		return
	}
	for i := 0; i < len(tasks); i++ {
		err = checkACL(tasks[i])
		if err != nil {
			log.Println("Error while checking ACL: ", err)
			return
		}
	}
	log.Printf("%+v", tasks)
}

func initHTTP(httpBinding string) {
	router := httprouter.New()
	router.GET("/task/:name", httpRequestIncoming)
	log.Fatal(http.ListenAndServe(httpBinding, router))
}

func main() {
	var confPath string
	flag.StringVar(&confPath, "config", "", "Path to the config file")
	flag.Parse()

	if confPath == "" {
		confPath, _ = filepath.Abs(filepath.Dir(os.Args[0]))
		confPath += "/config.json"
	}

	conf := &config{}
	cfile, _ := os.Open(confPath)
	if err := json.NewDecoder(cfile).Decode(&conf); err != nil {
		log.Fatal("Couldn't read config file! ", err)
	}

	initHTTP(conf.HTTP)
}<commit_msg>added wrappers for decryption<commit_after>package main

import (
	"encoding/json"
	"net/http"
	"log"
	"os"
	"flag"
	"errors"
	//"crypto/aes"
	//"crypto/cipher"
	"path/filepath"
	"github.com/julienschmidt/httprouter"
)

type config struct {
	HTTP string
}

// Tasks are encrypted with a symmetric key (EncryptedKey), which is
// encrypted with the asymmetric key in KeyFingerprint
type EncryptedTask struct {
	KeyFingerprint  string `json:"asymkey"`
	EncryptedKey    []byte `json:"symkey"`
	Encrypted       []byte
}

type Task struct {
	User     string
	SampleID string
	//TODO
}

func aesDecrypt(text []byte, key []byte) (error, []byte) {
	//TODO
	return nil, text
}

func rsaDecrypt(text []byte, key []byte) (error, []byte) {
	//TODO
	return nil, text
}

func decrypt(encrypted string) (error, string) {
	var enc []EncryptedTask
	if err := json.Unmarshal([]byte(encrypted), &enc); err != nil {
		return err, ""
	} else if len(enc) != 1 {
		return errors.New("Only one encrypted task per request!"), ""
	}
	log.Printf("Parsed: %+v\n", enc)
	//TODO: Fetch private key corresponding to enc[0].keyFingerprint (from where?)
	asymKey := []byte(enc[0].KeyFingerprint)

	//TODO: Actually implement decryption-function!
	//      For now: dec(a) = a
	// Decrypt symmetric key using the asymmetric key
	err, symKey := rsaDecrypt(enc[0].EncryptedKey, asymKey)
	if err != nil{
		return err, ""
	}

	//TODO: Actually implement decryption-function!
	//      For now: dec(a) = a
	// Decrypt using the symmetric key
	err, decrypted := aesDecrypt(enc[0].Encrypted, symKey)
	return err, string(decrypted)
}

func validate(task string) (error, []Task) {
	var tasks []Task
	err := json.Unmarshal([]byte(task), &tasks)
	if err != nil {
		return err, nil
	}
	//TODO Check for required fields; Additional checks?
	return err, tasks
}

func checkACL(task Task) (error){
	//TODO: How shall ACL-Check be executed?
	return nil
}

func httpRequestIncoming(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	task := ps.ByName("name")[1:]
	log.Println("New task request:\n" + task);
	err, decTask := decrypt(task)
	if err != nil {
		log.Println("Error while decrypting: ", err)
		return
	}
	log.Println("Decrypted task:", decTask)
	err, tasks := validate(decTask)
	if err != nil {
		log.Println("Error while validating: ", err)
		return
	}
	for i := 0; i < len(tasks); i++ {
		err = checkACL(tasks[i])
		if err != nil {
			log.Println("Error while checking ACL: ", err)
			return
		}
	}
	log.Printf("%+v", tasks)
}

func initHTTP(httpBinding string) {
	router := httprouter.New()
	router.GET("/task/*name", httpRequestIncoming)
	log.Fatal(http.ListenAndServe(httpBinding, router))
}

func main() {
	var confPath string
	flag.StringVar(&confPath, "config", "", "Path to the config file")
	flag.Parse()

	if confPath == "" {
		confPath, _ = filepath.Abs(filepath.Dir(os.Args[0]))
		confPath += "/config.json"
	}

	conf := &config{}
	cfile, _ := os.Open(confPath)
	if err := json.NewDecoder(cfile).Decode(&conf); err != nil {
		log.Fatal("Couldn't read config file! ", err)
	}

	initHTTP(conf.HTTP)
}<|endoftext|>