<commit_before>package main

import (
	"flag"
	"fmt"
	//"github.com/jda/go-nagios/nagios"
	"os"
)

const version = "check_vitelity_balance 0.1"
const api_url = "https://api.vitelity.net/api.php"

func usage() {
	fmt.Fprintf(os.Stderr, "usage: %s [args]\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(2)
}

var verbose int

func main() {

	verbose = 0
	warn_level := 10.00
	crit_level := 5.00
	username := ""
	password := ""

	flag.Usage = usage
	flag.IntVar(&verbose, "verbose", 0, "verbosity level (0-3)")
	flag.Float64Var(&warn_level, "warning", 100.00, "warn when balance is below this value")
	flag.Float64Var(&crit_level, "critical", 25.00, "critical when balance is below this value")
	flag.StringVar(&username, "username", "", "API Username")
	flag.StringVar(&password, "password", "", "API Password")
	flag.Parse()

	if verbose >= 3 {
		fmt.Println("Verbosity level:", verbose)
		fmt.Println("Warning level:", warn_level)
		fmt.Println("Critical level:", crit_level)
	}

	balance, err := get_balance(username, password)
	if err != nil {
		fmt.Println("Balance Unknown:", err)
		os.Exit(3)
	}

	if verbose >= 2 {
		fmt.Println("Balance:", balance)
	}
}
<commit_msg>works<commit_after>package main

import (
	"flag"
	"fmt"
	//"github.com/jda/go-nagios/nagios"
	"os"
)

const version = "check_vitelity_balance 0.1"
const api_url = "https://api.vitelity.net/api.php"

func usage() {
	fmt.Fprintf(os.Stderr, "usage: %s [args]\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(2)
}

var verbose int

func main() {

	verbose = 0
	warn_level := 10.00
	crit_level := 5.00
	username := ""
	password := ""

	flag.Usage = usage
	flag.IntVar(&verbose, "verbose", 0, "verbosity level (0-3)")
	flag.Float64Var(&warn_level, "warning", 100.00, "warn when balance is below this value")
	flag.Float64Var(&crit_level, "critical", 25.00, "critical when balance is below this value")
	flag.StringVar(&username, "username", "", "API Username")
	flag.StringVar(&password, "password", "", "API Password")
	flag.Parse()

	if verbose >= 3 {
		fmt.Println("Verbosity level:", verbose)
		fmt.Println("Warning level:", warn_level)
		fmt.Println("Critical level:", crit_level)
	}

	balance, err := get_balance(username, password)
	if err != nil {
		fmt.Println("Balance Unknown:", err)
		os.Exit(3)
	}

	if verbose >= 2 {
		fmt.Println("Balance:", balance)
	}

	// check if balance is below critical threshold
	if balance <= crit_level {
		fmt.Printf("Balance Critical - $%.2f\n", balance)
		os.Exit(2)
	}

	// check if balance is below warning threshold
	if balance <= warn_level {
		fmt.Printf("Balance Warning - $%.2f\n", balance)
		os.Exit(1)
	}

	// balance must be ok
	fmt.Printf("Balance OK - $%.2f\n", balance)
	os.Exit(0)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/pkg/errors"
	"os"
	"io/ioutil"
	"flag"
	"log"
	"github.com/mgurov/hipchat-delay/delay"
)

func main() {

	command := delay.Message{}

	flag.StringVar(&command.AuthToken, "auth", "", "https://developer.atlassian.com/hipchat/guide/hipchat-rest-api/api-access-tokens#APIaccesstokens-userUsertoken")
	flag.StringVar(&command.Room, "room", "", "room id or name")
	flag.DurationVar(&command.NeedSilence, "after-silence", 0, "Don't post until the silence of the duration")

	flag.Parse()

	if "" == command.Room {
		log.Fatal("Need room")
	}

	if "" == command.AuthToken {
		flag.PrintDefaults()
		log.Fatal("Need authentication token")
		return
	}

	message, err := ioutil.ReadAll(os.Stdin)
	if (err != nil) {
		log.Fatal(errors.Wrap(err, "reading stdin failed"))
	}

	command.Text = string(message)

	err = command.Send()
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>let the default silence be 1 min<commit_after>package main

import (
	"github.com/pkg/errors"
	"os"
	"io/ioutil"
	"flag"
	"log"
	"github.com/mgurov/hipchat-delay/delay"
	"time"
)

func main() {

	command := delay.Message{}

	flag.StringVar(&command.AuthToken, "auth", "", "https://developer.atlassian.com/hipchat/guide/hipchat-rest-api/api-access-tokens#APIaccesstokens-userUsertoken")
	flag.StringVar(&command.Room, "room", "", "room id or name")
	flag.DurationVar(&command.NeedSilence, "after-silence", 1 * time.Minute, "Don't post until the silence of the duration")

	flag.Parse()

	if "" == command.Room {
		log.Fatal("Need room")
	}

	if "" == command.AuthToken {
		flag.PrintDefaults()
		log.Fatal("Need authentication token")
		return
	}

	message, err := ioutil.ReadAll(os.Stdin)
	if (err != nil) {
		log.Fatal(errors.Wrap(err, "reading stdin failed"))
	}

	command.Text = string(message)

	err = command.Send()
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
)

const (
	// NumCheckers contains the number of workers that are used to check URLs.
	NumCheckers = 3
)

var (
	errDone   = errors.New("not following any more redirects")
	protocols = []string{
		"http",
		"https",
	}
)

// Redirector is the function signature for a redirect checker for http requests.
type Redirector func(req *http.Request, via []*http.Request) error

var (
	// FollowAllRedirects follows all redirects, unconditionally.
	FollowAllRedirects = func(req *http.Request, via []*http.Request) error {
		return nil
	}
	// StopOnFirstRedirect stops at the first redirect it encounters..
	StopOnFirstRedirect = func(req *http.Request, via []*http.Request) error {
		return errDone
	}
	// StopOnRedirectToDifferentDomain stops as soon as you are redirected to a different domain.
	StopOnRedirectToDifferentDomain = func(req *http.Request, via []*http.Request) error {
		if req.URL.Host != via[len(via)-1].URL.Host {
			return errDone
		}
		return nil
	}
	// StopOnCyclicRedirect stops only if redirects get into an infinite loop.
	StopOnCyclicRedirect = func(req *http.Request, via []*http.Request) error {
		for _, prev := range via {
			if prev.URL == req.URL {
				return errDone
			}
		}
		return nil
	}
)

// Site contains the request URL, HTTP status code and response URL.
type Site struct {
	RequestURL  url.URL
	StatusCode  int
	ResponseURL *url.URL
}

func (s *Site) String() string {
	var line = fmt.Sprintf("%s,%d,", s.RequestURL.Host, s.StatusCode)
	if s.ResponseURL != nil {
		line += s.ResponseURL.String()
	}
	return line
}

func main() {
	flag.Parse()
	// start result writer
	var writerGroup sync.WaitGroup
	var result = make(chan *Site)
	writerGroup.Add(1)
	go writeWorker(&writerGroup, result)
	// start url reader
	var work = make(chan string)
	go readWorker(work)
	// start checkers
	var checkRedirect = StopOnFirstRedirect
	var workerGroup sync.WaitGroup
	for i := 0; i < NumCheckers; i++ {
		workerGroup.Add(1)
		go checkWorker(&workerGroup, work, result, checkRedirect)
	}
	// wait for all checkers to finish
	workerGroup.Wait()
	close(result)
	// wait for writer to finish
	writerGroup.Wait()
}

func readWorker(work chan<- string) {
	defer close(work)
	var source io.Reader
	if flag.NArg() < 1 {
		source = os.Stdin
	} else {
		var err error
		source, err = os.Open(flag.Arg(0))
		if err != nil {
			os.Stderr.WriteString("error getting URLs: " + err.Error() + "\n")
			return
		}
	}
	lineReader := bufio.NewReader(source)
	for {
		site, err := lineReader.ReadString('\n')
		if len(site) > 0 {
			for _, prot := range protocols {
				work <- formatURL(prot, strings.TrimRight(site, "\n"))
			}
		}
		if err != nil {
			break
		}
	}
}

func checkWorker(wg *sync.WaitGroup, work <-chan string, result chan<- *Site, check Redirector) {
	defer wg.Done()
	var c http.Client
	c.CheckRedirect = check
	for site := range work {
		r, err := testURL(&c, site)
		if err == nil {
			result <- r
		} else {
			os.Stderr.WriteString(err.Error() + "\n")
		}
	}
}

func writeWorker(wg *sync.WaitGroup, result <-chan *Site) {
	defer wg.Done()
	for r := range result {
		os.Stdout.WriteString(r.String() + "\n")
	}
}

func testURL(c *http.Client, site string) (*Site, error) {
	resp, err := c.Get(site)
	if err != nil {
		switch e := err.(type) {
		case *url.Error:
			if e.Err == errDone {
				// just an errDone, continue
				break
			}
			// an unexpected error
			return nil, err
		default:
			// an unexpected error
			return nil, err
		}
	}
	var result Site
	result.RequestURL = *resp.Request.URL
	result.StatusCode = resp.StatusCode
	if loc, err := resp.Location(); err == nil {
		result.ResponseURL = loc
	}
	return &result, nil
}

func formatURL(protocol string, site string) string {
	return protocol + "://" + site + "/"
}
<commit_msg>Trim both front and back and trim CR and white space chars.<commit_after>package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
)

const (
	// NumCheckers contains the number of workers that are used to check URLs.
	NumCheckers = 3
)

var (
	errDone   = errors.New("not following any more redirects")
	protocols = []string{
		"http",
		"https",
	}
)

// Redirector is the function signature for a redirect checker for http requests.
type Redirector func(req *http.Request, via []*http.Request) error

var (
	// FollowAllRedirects follows all redirects, unconditionally.
	FollowAllRedirects = func(req *http.Request, via []*http.Request) error {
		return nil
	}
	// StopOnFirstRedirect stops at the first redirect it encounters..
	StopOnFirstRedirect = func(req *http.Request, via []*http.Request) error {
		return errDone
	}
	// StopOnRedirectToDifferentDomain stops as soon as you are redirected to a different domain.
	StopOnRedirectToDifferentDomain = func(req *http.Request, via []*http.Request) error {
		if req.URL.Host != via[len(via)-1].URL.Host {
			return errDone
		}
		return nil
	}
	// StopOnCyclicRedirect stops only if redirects get into an infinite loop.
	StopOnCyclicRedirect = func(req *http.Request, via []*http.Request) error {
		for _, prev := range via {
			if prev.URL == req.URL {
				return errDone
			}
		}
		return nil
	}
)

// Site contains the request URL, HTTP status code and response URL.
type Site struct {
	RequestURL  url.URL
	StatusCode  int
	ResponseURL *url.URL
}

func (s *Site) String() string {
	var line = fmt.Sprintf("%s,%d,", s.RequestURL.Host, s.StatusCode)
	if s.ResponseURL != nil {
		line += s.ResponseURL.String()
	}
	return line
}

func main() {
	flag.Parse()
	// start result writer
	var writerGroup sync.WaitGroup
	var result = make(chan *Site)
	writerGroup.Add(1)
	go writeWorker(&writerGroup, result)
	// start url reader
	var work = make(chan string)
	go readWorker(work)
	// start checkers
	var checkRedirect = StopOnFirstRedirect
	var workerGroup sync.WaitGroup
	for i := 0; i < NumCheckers; i++ {
		workerGroup.Add(1)
		go checkWorker(&workerGroup, work, result, checkRedirect)
	}
	// wait for all checkers to finish
	workerGroup.Wait()
	close(result)
	// wait for writer to finish
	writerGroup.Wait()
}

func readWorker(work chan<- string) {
	defer close(work)
	var source io.Reader
	if flag.NArg() < 1 {
		source = os.Stdin
	} else {
		var err error
		source, err = os.Open(flag.Arg(0))
		if err != nil {
			os.Stderr.WriteString("error getting URLs: " + err.Error() + "\n")
			return
		}
	}
	lineReader := bufio.NewReader(source)
	for {
		site, err := lineReader.ReadString('\n')
		if len(site) > 0 {
			for _, prot := range protocols {
				work <- formatURL(prot, strings.Trim(site, " \t\r\n"))
			}
		}
		if err != nil {
			break
		}
	}
}

func checkWorker(wg *sync.WaitGroup, work <-chan string, result chan<- *Site, check Redirector) {
	defer wg.Done()
	var c http.Client
	c.CheckRedirect = check
	for site := range work {
		r, err := testURL(&c, site)
		if err == nil {
			result <- r
		} else {
			os.Stderr.WriteString(err.Error() + "\n")
		}
	}
}

func writeWorker(wg *sync.WaitGroup, result <-chan *Site) {
	defer wg.Done()
	for r := range result {
		os.Stdout.WriteString(r.String() + "\n")
	}
}

func testURL(c *http.Client, site string) (*Site, error) {
	resp, err := c.Get(site)
	if err != nil {
		switch e := err.(type) {
		case *url.Error:
			if e.Err == errDone {
				// just an errDone, continue
				break
			}
			// an unexpected error
			return nil, err
		default:
			// an unexpected error
			return nil, err
		}
	}
	var result Site
	result.RequestURL = *resp.Request.URL
	result.StatusCode = resp.StatusCode
	if loc, err := resp.Location(); err == nil {
		result.ResponseURL = loc
	}
	return &result, nil
}

func formatURL(protocol string, site string) string {
	return protocol + "://" + site + "/"
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	stdlog "log"
	"os"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	MQTT "github.com/eclipse/paho.mqtt.golang"
	colorable "github.com/mattn/go-colorable"
	"github.com/urfave/cli"
)

var usage = `
Usage here
`

var version string

func init() {
	log.SetLevel(log.WarnLevel)
	log.SetOutput(colorable.NewColorableStdout())
}

// connects MQTT broker
func connect(c *cli.Context, opts *MQTT.ClientOptions, subscribed map[string]byte) (*MQTTClient, error) {
	willPayload := c.String("will-payload")
	willQoS := c.Int("will-qos")
	willRetain := c.Bool("will-retain")
	willTopic := c.String("will-topic")
	if willPayload != "" && willTopic != "" {
		opts.SetWill(willTopic, willPayload, byte(willQoS), willRetain)
	}

	client := &MQTTClient{Opts: opts}
	client.lock = new(sync.Mutex)
	client.Subscribed = subscribed

	opts.SetOnConnectHandler(client.SubscribeOnConnect)
	opts.SetConnectionLostHandler(client.ConnectionLost)

	_, err := client.Connect()
	if err != nil {
		return nil, err
	}

	return client, nil
}

func pubsub(c *cli.Context) error {
	setDebugLevel(c)
	opts, err := NewOption(c)
	if err != nil {
		log.Error(err)
		os.Exit(1)
	}

	qos := c.Int("q")
	subtopic := c.String("sub")
	if subtopic == "" {
		log.Errorf("Please specify sub topic")
		os.Exit(1)
	}
	log.Infof("Sub Topic: %s", subtopic)
	pubtopic := c.String("pub")
	if pubtopic == "" {
		log.Errorf("Please specify pub topic")
		os.Exit(1)
	}
	log.Infof("Pub Topic: %s", pubtopic)
	retain := c.Bool("r")

	subscribed := map[string]byte{
		subtopic: byte(0),
	}

	client, err := connect(c, opts, subscribed)
	if err != nil {
		log.Error(err)
		os.Exit(1)
	}

	go func() {
		// Read from Stdin and publish
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			err = client.Publish(pubtopic, []byte(scanner.Text()), qos, retain, false)
			if err != nil {
				log.Error(err)
			}
		}
	}()

	// while loop
	for {
		time.Sleep(1 * time.Second)
	}
	return nil
}

func main() {
	app := cli.NewApp()
	app.Name = "mqttcli"
	app.Usage = usage
	app.Version = version

	commonFlags := []cli.Flag{
		cli.StringFlag{
			Name:   "host",
			Value:  "",
			Usage:  "mqtt host to connect to. Defaults to localhost",
			EnvVar: "MQTT_HOST"},
		cli.IntFlag{
			Name:   "p, port",
			Value:  1883,
			Usage:  "network port to connect to. Defaults to 1883",
			EnvVar: "MQTT_PORT"},
		cli.StringFlag{
			Name:   "u,user",
			Value:  "",
			Usage:  "provide a username",
			EnvVar: "MQTT_USERNAME"},
		cli.StringFlag{
			Name:   "P,password",
			Value:  "",
			Usage:  "provide a password",
			EnvVar: "MQTT_PASSWORD"},
		cli.StringFlag{
			Name:  "t",
			Value: "",
			Usage: "mqtt topic to publish to.",
		},
		cli.IntFlag{
			Name:  "q",
			Value: 0,
			Usage: "QoS",
		},
		cli.StringFlag{
			Name:  "cafile",
			Value: "",
			Usage: "CA certificates",
		},
		cli.StringFlag{
			Name:  "cert",
			Value: "",
			Usage: "Client certificates",
		},
		cli.StringFlag{
			Name:  "key",
			Value: "",
			Usage: "Client private key",
		},
		cli.StringFlag{
			Name:  "i",
			Value: "",
			Usage: "ClientiId. Defaults random.",
		},
		cli.StringFlag{
			Name:  "m",
			Value: "test message",
			Usage: "Message body",
		},
		cli.BoolFlag{
			Name:  "r",
			Usage: "message should be retained.",
		},
		cli.BoolFlag{
			Name:  "d",
			Usage: "enable debug messages",
		},
		cli.BoolFlag{
			Name:  "dd",
			Usage: "enable debug messages",
		},
		cli.BoolFlag{
			Name:  "ddd",
			Usage: "enable debug messages",
		},
		cli.BoolFlag{
			Name:  "dddd",
			Usage: "enable debug messages",
		},
		cli.BoolFlag{
			Name:  "ddddd",
			Usage: "enable debug messages",
		},
		cli.BoolFlag{
			Name:  "insecure",
			Usage: "do not check that the server certificate",
		},
		cli.StringFlag{
			Name:   "conf",
			Value:  "~/.mqttcli.cfg",
			Usage:  "config file path",
			EnvVar: "MQTTCLI_CONFPATH"},
		cli.StringFlag{
			Name:  "will-payload",
			Value: "",
			Usage: "payload for the client Will",
		},
		cli.IntFlag{
			Name:  "will-qos",
			Value: 0,
			Usage: "QoS level for the client Will",
		},
		cli.BoolFlag{
			Name:  "will-retain",
			Usage: "if given, make the client Will retained",
		},
		cli.StringFlag{
			Name:  "will-topic",
			Value: "",
			Usage: "the topic on which to publish the client Will",
		},
	}
	pubFlags := append(commonFlags,
		cli.BoolFlag{
			Name:  "s",
			Usage: "read message from stdin, sending line by line as a message",
		},
	)
	subFlags := append(commonFlags,
		cli.BoolFlag{
			Name:  "c",
			Usage: "disable 'clean session'",
		},
	)
	pubsubFlags := append(commonFlags,
		cli.StringFlag{
			Name:  "pub",
			Usage: "publish topic",
		},
		cli.StringFlag{
			Name:  "sub",
			Usage: "subscribe topic",
		},
	)

	app.Commands = []cli.Command{
		{
			Name:   "pub",
			Usage:  "publish",
			Flags:  pubFlags,
			Action: publish,
		},
		{
			Name:   "sub",
			Usage:  "subscribe",
			Flags:  subFlags,
			Action: subscribe,
		},
		{
			Name:   "pubsub",
			Usage:  "subscribe and publish",
			Flags:  pubsubFlags,
			Action: pubsub,
		},
	}
	err := app.Run(os.Args)
	if err != nil {
		log.Error(err)
	}
}

func setDebugLevel(c *cli.Context) {
	if c.Bool("d") {
		log.SetLevel(log.DebugLevel)
	} else if c.Bool("dd") {
		log.SetLevel(log.DebugLevel)
		MQTT.WARN = stdlog.New(os.Stdout, "", stdlog.LstdFlags)
	} else if c.Bool("ddd") {
		log.SetLevel(log.DebugLevel)
		MQTT.DEBUG = stdlog.New(os.Stdout, "", stdlog.LstdFlags)
	}
}
<commit_msg>Setting default for host to localhost<commit_after>package main

import (
	"bufio"
	stdlog "log"
	"os"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	MQTT "github.com/eclipse/paho.mqtt.golang"
	colorable "github.com/mattn/go-colorable"
	"github.com/urfave/cli"
)

var usage = `
Usage here
`

var version string

func init() {
	log.SetLevel(log.WarnLevel)
	log.SetOutput(colorable.NewColorableStdout())
}

// connects MQTT broker
func connect(c *cli.Context, opts *MQTT.ClientOptions, subscribed map[string]byte) (*MQTTClient, error) {
	willPayload := c.String("will-payload")
	willQoS := c.Int("will-qos")
	willRetain := c.Bool("will-retain")
	willTopic := c.String("will-topic")
	if willPayload != "" && willTopic != "" {
		opts.SetWill(willTopic, willPayload, byte(willQoS), willRetain)
	}

	client := &MQTTClient{Opts: opts}
	client.lock = new(sync.Mutex)
	client.Subscribed = subscribed

	opts.SetOnConnectHandler(client.SubscribeOnConnect)
	opts.SetConnectionLostHandler(client.ConnectionLost)

	_, err := client.Connect()
	if err != nil {
		return nil, err
	}

	return client, nil
}

func pubsub(c *cli.Context) error {
	setDebugLevel(c)
	opts, err := NewOption(c)
	if err != nil {
		log.Error(err)
		os.Exit(1)
	}

	qos := c.Int("q")
	subtopic := c.String("sub")
	if subtopic == "" {
		log.Errorf("Please specify sub topic")
		os.Exit(1)
	}
	log.Infof("Sub Topic: %s", subtopic)
	pubtopic := c.String("pub")
	if pubtopic == "" {
		log.Errorf("Please specify pub topic")
		os.Exit(1)
	}
	log.Infof("Pub Topic: %s", pubtopic)
	retain := c.Bool("r")

	subscribed := map[string]byte{
		subtopic: byte(0),
	}

	client, err := connect(c, opts, subscribed)
	if err != nil {
		log.Error(err)
		os.Exit(1)
	}

	go func() {
		// Read from Stdin and publish
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			err = client.Publish(pubtopic, []byte(scanner.Text()), qos, retain, false)
			if err != nil {
				log.Error(err)
			}
		}
	}()

	// while loop
	for {
		time.Sleep(1 * time.Second)
	}
	return nil
}

func main() {
	app := cli.NewApp()
	app.Name = "mqttcli"
	app.Usage = usage
	app.Version = version

	commonFlags := []cli.Flag{
		cli.StringFlag{
			Name:   "host",
			Value:  "localhost",
			Usage:  "mqtt host to connect to. Defaults to localhost",
			EnvVar: "MQTT_HOST"},
		cli.IntFlag{
			Name:   "p, port",
			Value:  1883,
			Usage:  "network port to connect to. Defaults to 1883",
			EnvVar: "MQTT_PORT"},
		cli.StringFlag{
			Name:   "u,user",
			Value:  "",
			Usage:  "provide a username",
			EnvVar: "MQTT_USERNAME"},
		cli.StringFlag{
			Name:   "P,password",
			Value:  "",
			Usage:  "provide a password",
			EnvVar: "MQTT_PASSWORD"},
		cli.StringFlag{
			Name:  "t",
			Value: "",
			Usage: "mqtt topic to publish to.",
		},
		cli.IntFlag{
			Name:  "q",
			Value: 0,
			Usage: "QoS",
		},
		cli.StringFlag{
			Name:  "cafile",
			Value: "",
			Usage: "CA certificates",
		},
		cli.StringFlag{
			Name:  "cert",
			Value: "",
			Usage: "Client certificates",
		},
		cli.StringFlag{
			Name:  "key",
			Value: "",
			Usage: "Client private key",
		},
		cli.StringFlag{
			Name:  "i",
			Value: "",
			Usage: "ClientiId. Defaults random.",
		},
		cli.StringFlag{
			Name:  "m",
			Value: "test message",
			Usage: "Message body",
		},
		cli.BoolFlag{
			Name:  "r",
			Usage: "message should be retained.",
		},
		cli.BoolFlag{
			Name:  "d",
			Usage: "enable debug messages",
		},
		cli.BoolFlag{
			Name:  "dd",
			Usage: "enable debug messages",
		},
		cli.BoolFlag{
			Name:  "ddd",
			Usage: "enable debug messages",
		},
		cli.BoolFlag{
			Name:  "dddd",
			Usage: "enable debug messages",
		},
		cli.BoolFlag{
			Name:  "ddddd",
			Usage: "enable debug messages",
		},
		cli.BoolFlag{
			Name:  "insecure",
			Usage: "do not check that the server certificate",
		},
		cli.StringFlag{
			Name:   "conf",
			Value:  "~/.mqttcli.cfg",
			Usage:  "config file path",
			EnvVar: "MQTTCLI_CONFPATH"},
		cli.StringFlag{
			Name:  "will-payload",
			Value: "",
			Usage: "payload for the client Will",
		},
		cli.IntFlag{
			Name:  "will-qos",
			Value: 0,
			Usage: "QoS level for the client Will",
		},
		cli.BoolFlag{
			Name:  "will-retain",
			Usage: "if given, make the client Will retained",
		},
		cli.StringFlag{
			Name:  "will-topic",
			Value: "",
			Usage: "the topic on which to publish the client Will",
		},
	}
	pubFlags := append(commonFlags,
		cli.BoolFlag{
			Name:  "s",
			Usage: "read message from stdin, sending line by line as a message",
		},
	)
	subFlags := append(commonFlags,
		cli.BoolFlag{
			Name:  "c",
			Usage: "disable 'clean session'",
		},
	)
	pubsubFlags := append(commonFlags,
		cli.StringFlag{
			Name:  "pub",
			Usage: "publish topic",
		},
		cli.StringFlag{
			Name:  "sub",
			Usage: "subscribe topic",
		},
	)

	app.Commands = []cli.Command{
		{
			Name:   "pub",
			Usage:  "publish",
			Flags:  pubFlags,
			Action: publish,
		},
		{
			Name:   "sub",
			Usage:  "subscribe",
			Flags:  subFlags,
			Action: subscribe,
		},
		{
			Name:   "pubsub",
			Usage:  "subscribe and publish",
			Flags:  pubsubFlags,
			Action: pubsub,
		},
	}
	err := app.Run(os.Args)
	if err != nil {
		log.Error(err)
	}
}

func setDebugLevel(c *cli.Context) {
	if c.Bool("d") {
		log.SetLevel(log.DebugLevel)
	} else if c.Bool("dd") {
		log.SetLevel(log.DebugLevel)
		MQTT.WARN = stdlog.New(os.Stdout, "", stdlog.LstdFlags)
	} else if c.Bool("ddd") {
		log.SetLevel(log.DebugLevel)
		MQTT.DEBUG = stdlog.New(os.Stdout, "", stdlog.LstdFlags)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"net"
	"p2p-port-forward/utils"
	"github.com/selvakn/libzt"
	"fmt"
	"github.com/google/logger"
	"os"
	"gopkg.in/alecthomas/kingpin.v2"
)

const INTERNAL_ZT_PORT = 7878

func dialLocalService() (net.Conn, error) {
	return net.Dial("tcp", fmt.Sprintf("localhost:%s", *forwardPort))
}

func dialRemoteThroughTunnel(zt *libzt.ZT) func() (net.Conn, error) {
	return func() (net.Conn, error) {
		conn, err := zt.Connect6(*connectTo, INTERNAL_ZT_PORT)
		conn = (&utils.DataRateLoggingConnection{}).Init(conn)
		return conn, err
	}
}

var (
	network     = kingpin.Flag("network", "zerotier network id").Short('n').Default("8056c2e21c000001").String()
	forwardPort = kingpin.Flag("forward-port", "port to forward (in listen mode)").Short('f').Default("22").String()
	acceptPort  = kingpin.Flag("accept-port", "port to accept (in connect mode)").Short('a').Default("2222").String()

	connectTo = kingpin.Flag("connect-to", "server (zerotier) ip to connect").Short('c').String()
)

func main() {
	logger.Init("p2p-port-forward", false, false, os.Stdout)

	kingpin.Parse()

	zt := libzt.Init(*network, "./zt")

	logger.Infof("ipv4 = %v \n", zt.GetIPv4Address().String())
	logger.Infof("ipv6 = %v \n", zt.GetIPv6Address().String())

	if len(*connectTo) == 0 {
		logger.Info("Waiting for any client to connect")

		listener, _ := zt.Listen6(INTERNAL_ZT_PORT)
		loggingListener := &utils.LoggingListener{Listener: listener}
		dataRageLogginglistener := &utils.DataRateLoggingListener{Listener: loggingListener}

		go utils.Sync(dialLocalService, dataRageLogginglistener.Accept)

		<-utils.SetupCleanUpOnInterrupt(func() {
			listener.Close()
		})

	} else {
		ln, _ := net.Listen("tcp", fmt.Sprintf(":%s", *acceptPort))

		go utils.Sync(ln.Accept, dialRemoteThroughTunnel(zt))

		<-utils.SetupCleanUpOnInterrupt(func() {
			ln.Close()
		})

	}
}
<commit_msg>gofmt<commit_after>package main

import (
	"fmt"
	"github.com/google/logger"
	"github.com/selvakn/libzt"
	"gopkg.in/alecthomas/kingpin.v2"
	"net"
	"os"
	"p2p-port-forward/utils"
)

const INTERNAL_ZT_PORT = 7878

func dialLocalService() (net.Conn, error) {
	return net.Dial("tcp", fmt.Sprintf("localhost:%s", *forwardPort))
}

func dialRemoteThroughTunnel(zt *libzt.ZT) func() (net.Conn, error) {
	return func() (net.Conn, error) {
		conn, err := zt.Connect6(*connectTo, INTERNAL_ZT_PORT)
		conn = (&utils.DataRateLoggingConnection{}).Init(conn)
		return conn, err
	}
}

var (
	network     = kingpin.Flag("network", "zerotier network id").Short('n').Default("8056c2e21c000001").String()
	forwardPort = kingpin.Flag("forward-port", "port to forward (in listen mode)").Short('f').Default("22").String()
	acceptPort  = kingpin.Flag("accept-port", "port to accept (in connect mode)").Short('a').Default("2222").String()

	connectTo = kingpin.Flag("connect-to", "server (zerotier) ip to connect").Short('c').String()
)

func main() {
	logger.Init("p2p-port-forward", false, false, os.Stdout)

	kingpin.Parse()

	zt := libzt.Init(*network, "./zt")

	logger.Infof("ipv4 = %v \n", zt.GetIPv4Address().String())
	logger.Infof("ipv6 = %v \n", zt.GetIPv6Address().String())

	if len(*connectTo) == 0 {
		logger.Info("Waiting for any client to connect")

		listener, _ := zt.Listen6(INTERNAL_ZT_PORT)
		loggingListener := &utils.LoggingListener{Listener: listener}
		dataRageLogginglistener := &utils.DataRateLoggingListener{Listener: loggingListener}

		go utils.Sync(dialLocalService, dataRageLogginglistener.Accept)

		<-utils.SetupCleanUpOnInterrupt(func() {
			listener.Close()
		})

	} else {
		ln, _ := net.Listen("tcp", fmt.Sprintf(":%s", *acceptPort))

		go utils.Sync(ln.Accept, dialRemoteThroughTunnel(zt))

		<-utils.SetupCleanUpOnInterrupt(func() {
			ln.Close()
		})

	}
}
<|endoftext|>


<commit_before>package arg

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"reflect"
	"strings"
)

// Fail prints usage information to stderr and exits with non-zero status
func (p *Parser) Fail(msg string) {
	p.WriteUsage(os.Stderr)
	fmt.Fprintln(os.Stderr, "error:", msg)
	os.Exit(-1)
}

// WriteUsage writes usage information to the given writer
func (p *Parser) WriteUsage(w io.Writer) {
	var positionals, options []*spec
	for _, spec := range p.spec {
		if spec.positional {
			positionals = append(positionals, spec)
		} else {
			options = append(options, spec)
		}
	}

	fmt.Fprintf(w, "usage: %s", filepath.Base(os.Args[0]))

	// write the option component of the usage message
	for _, spec := range options {
		// prefix with a space
		fmt.Fprint(w, " ")
		if !spec.required {
			fmt.Fprint(w, "[")
		}
		fmt.Fprint(w, synopsis(spec, "--"+spec.long))
		if !spec.required {
			fmt.Fprint(w, "]")
		}
	}

	// write the positional component of the usage message
	for _, spec := range positionals {
		// prefix with a space
		fmt.Fprint(w, " ")
		up := strings.ToUpper(spec.long)
		if spec.multiple {
			fmt.Fprintf(w, "[%s [%s ...]]", up, up)
		} else {
			fmt.Fprint(w, up)
		}
	}
	fmt.Fprint(w, "\n")
}

// WriteHelp writes the usage string followed by the full help string for each option
func (p *Parser) WriteHelp(w io.Writer) {
	var positionals, options []*spec
	for _, spec := range p.spec {
		if spec.positional {
			positionals = append(positionals, spec)
		} else {
			options = append(options, spec)
		}
	}

	p.WriteUsage(w)

	// the width of the left column
	const colWidth = 25

	// write the list of positionals
	if len(positionals) > 0 {
		fmt.Fprint(w, "\npositional arguments:\n")
		for _, spec := range positionals {
			left := "  " + spec.long
			fmt.Fprint(w, left)
			if spec.help != "" {
				if len(left)+2 < colWidth {
					fmt.Fprint(w, strings.Repeat(" ", colWidth-len(left)))
				} else {
					fmt.Fprint(w, "\n"+strings.Repeat(" ", colWidth))
				}
				fmt.Fprint(w, spec.help)
			}
			fmt.Fprint(w, "\n")
		}
	}

	// write the list of options
	fmt.Fprint(w, "\noptions:\n")
	for _, spec := range options {
		printOption(w, spec)
	}

	// write the list of built in options
	printOption(w, &spec{isBool: true, long: "help", short: "h", help: "display this help and exit"})
}

func printOption(w io.Writer, spec *spec) {
	const colWidth = 25
	left := "  " + synopsis(spec, "--"+spec.long)
	if spec.short != "" {
		left += ", " + synopsis(spec, "-"+spec.short)
	}
	fmt.Fprint(w, left)
	if spec.help != "" {
		if len(left)+2 < colWidth {
			fmt.Fprint(w, strings.Repeat(" ", colWidth-len(left)))
		} else {
			fmt.Fprint(w, "\n"+strings.Repeat(" ", colWidth))
		}
		fmt.Fprint(w, spec.help)
	}
	// Check if spec.dest is zero value or not
	// If it isn't a default value have been added
	v := spec.dest
	if v.IsValid() {
		z := reflect.Zero(v.Type())
		if v.Type().Comparable() && z.Type().Comparable() && v.Interface() != z.Interface() {
			fmt.Fprintf(w, " [default: %v]", v)
		}
	}
	fmt.Fprint(w, "\n")
}

func synopsis(spec *spec, form string) string {
	if spec.isBool {
		return form
	}
	return form + " " + strings.ToUpper(spec.long)
}
<commit_msg>extract common colWidth constant<commit_after>package arg

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"reflect"
	"strings"
)

// the width of the left column
const colWidth = 25

// Fail prints usage information to stderr and exits with non-zero status
func (p *Parser) Fail(msg string) {
	p.WriteUsage(os.Stderr)
	fmt.Fprintln(os.Stderr, "error:", msg)
	os.Exit(-1)
}

// WriteUsage writes usage information to the given writer
func (p *Parser) WriteUsage(w io.Writer) {
	var positionals, options []*spec
	for _, spec := range p.spec {
		if spec.positional {
			positionals = append(positionals, spec)
		} else {
			options = append(options, spec)
		}
	}

	fmt.Fprintf(w, "usage: %s", filepath.Base(os.Args[0]))

	// write the option component of the usage message
	for _, spec := range options {
		// prefix with a space
		fmt.Fprint(w, " ")
		if !spec.required {
			fmt.Fprint(w, "[")
		}
		fmt.Fprint(w, synopsis(spec, "--"+spec.long))
		if !spec.required {
			fmt.Fprint(w, "]")
		}
	}

	// write the positional component of the usage message
	for _, spec := range positionals {
		// prefix with a space
		fmt.Fprint(w, " ")
		up := strings.ToUpper(spec.long)
		if spec.multiple {
			fmt.Fprintf(w, "[%s [%s ...]]", up, up)
		} else {
			fmt.Fprint(w, up)
		}
	}
	fmt.Fprint(w, "\n")
}

// WriteHelp writes the usage string followed by the full help string for each option
func (p *Parser) WriteHelp(w io.Writer) {
	var positionals, options []*spec
	for _, spec := range p.spec {
		if spec.positional {
			positionals = append(positionals, spec)
		} else {
			options = append(options, spec)
		}
	}

	p.WriteUsage(w)

	// write the list of positionals
	if len(positionals) > 0 {
		fmt.Fprint(w, "\npositional arguments:\n")
		for _, spec := range positionals {
			left := "  " + spec.long
			fmt.Fprint(w, left)
			if spec.help != "" {
				if len(left)+2 < colWidth {
					fmt.Fprint(w, strings.Repeat(" ", colWidth-len(left)))
				} else {
					fmt.Fprint(w, "\n"+strings.Repeat(" ", colWidth))
				}
				fmt.Fprint(w, spec.help)
			}
			fmt.Fprint(w, "\n")
		}
	}

	// write the list of options
	fmt.Fprint(w, "\noptions:\n")
	for _, spec := range options {
		printOption(w, spec)
	}

	// write the list of built in options
	printOption(w, &spec{isBool: true, long: "help", short: "h", help: "display this help and exit"})
}

func printOption(w io.Writer, spec *spec) {
	left := "  " + synopsis(spec, "--"+spec.long)
	if spec.short != "" {
		left += ", " + synopsis(spec, "-"+spec.short)
	}
	fmt.Fprint(w, left)
	if spec.help != "" {
		if len(left)+2 < colWidth {
			fmt.Fprint(w, strings.Repeat(" ", colWidth-len(left)))
		} else {
			fmt.Fprint(w, "\n"+strings.Repeat(" ", colWidth))
		}
		fmt.Fprint(w, spec.help)
	}
	// Check if spec.dest is zero value or not
	// If it isn't a default value have been added
	v := spec.dest
	if v.IsValid() {
		z := reflect.Zero(v.Type())
		if v.Type().Comparable() && z.Type().Comparable() && v.Interface() != z.Interface() {
			fmt.Fprintf(w, " [default: %v]", v)
		}
	}
	fmt.Fprint(w, "\n")
}

func synopsis(spec *spec, form string) string {
	if spec.isBool {
		return form
	}
	return form + " " + strings.ToUpper(spec.long)
}
<|endoftext|>


<commit_before>package main

import (
	"babelweb2/parser"
	"babelweb2/ws"
	"bufio"
	"fmt"
	"log"
	"net"
	"net/http"
	"sync"
	"time"
)

const node = "[::1]:33123"

func Connection(updates chan parser.BabelUpdate, node string) {
	for {
		conn, err := net.Dial("tcp6", node)
		if err != nil {
			log.Println(err)
			time.Sleep(time.Second)
			conn.Close()
			continue
		}
		fmt.Fprintf(conn, "monitor\n")
		r := bufio.NewReader(conn)
		s := parser.NewScanner(r)
		for {
			ws.Db.Bd.Listen(s, updates)
		}
	}
}

func main() {
	var wg sync.WaitGroup
	wg.Add(2)
	updates := make(chan parser.BabelUpdate, ws.ChanelSize)
	ws.Db.Bd = parser.NewBabelDesc()
	go Connection(updates, node)
	log.Println("connection established")
	bcastGrp := ws.NewListenerGroupe()
	go ws.MCUpdates(updates, bcastGrp)
	ws := ws.Handler(bcastGrp)
	log.Println("manager lauched")
	http.Handle("/", http.FileServer(http.Dir("static/")))
	http.Handle("/style.css", http.FileServer(http.Dir("static/css/")))
	http.Handle("/initialize.js", http.FileServer(http.Dir("static/js")))
	http.Handle("/d3/d3.js", http.FileServer(http.Dir("static/js")))
	http.Handle("/ws", ws)
	err := http.ListenAndServe(":8080", nil)
	if err != nil {

		log.Println(err)
		return
	}
	wg.Wait()
	log.Println("ca va mal")
}
<commit_msg>Removing a small mistake in the connection function*<commit_after>package main

import (
	"babelweb2/parser"
	"babelweb2/ws"
	"bufio"
	"fmt"
	"log"
	"net"
	"net/http"
	"sync"
	"time"
)

const node = "[::1]:33123"

func Connection(updates chan parser.BabelUpdate, node string) {

	//TODO remember to close the connection
	for {
		conn, err := net.Dial("tcp6", node)
		if err != nil {
			log.Println(err)
			time.Sleep(time.Second)
			continue
		}
		fmt.Fprintf(conn, "monitor\n")
		r := bufio.NewReader(conn)
		s := parser.NewScanner(r)
		for {
			ws.Db.Bd.Listen(s, updates)
		}
	}

}

func main() {
	var wg sync.WaitGroup
	wg.Add(2)
	updates := make(chan parser.BabelUpdate, ws.ChanelSize)
	ws.Db.Bd = parser.NewBabelDesc()
	go Connection(updates, node)
	log.Println("connection established")
	bcastGrp := ws.NewListenerGroupe()
	go ws.MCUpdates(updates, bcastGrp)
	ws := ws.Handler(bcastGrp)
	log.Println("manager lauched")
	http.Handle("/", http.FileServer(http.Dir("static/")))
	http.Handle("/style.css", http.FileServer(http.Dir("static/css/")))
	http.Handle("/initialize.js", http.FileServer(http.Dir("static/js")))
	http.Handle("/d3/d3.js", http.FileServer(http.Dir("static/js")))
	http.Handle("/ws", ws)
	err := http.ListenAndServe(":8080", nil)
	if err != nil {

		log.Println(err)
		return
	}
	wg.Wait()
	log.Println("ca va mal")
}
<|endoftext|>


<commit_before>package main

import (
	"go/build"
	"log"
	"os"
	"path/filepath"

	"github.com/LK4D4/vndr/godl"
)

const (
	vendorDir  = "vendor"
	configFile = "vndr.cfg"
	tmpDir     = ".vndr-tmp"
)

func main() {
	if os.Getenv("GOPATH") == "" {
		log.Fatal("GOPATH must be set")
	}
	wd, err := os.Getwd()
	if err != nil {
		log.Fatalf("Error getting working directory: %v", err)
	}
	log.Println("Collecting local packages list")
	initPkgs, err := collectPkgs(wd)
	if err != nil {
		log.Fatalf("Error collecting initial packages: %v", err)
	}
	log.Println("Init pkgs:")
	for _, pkg := range initPkgs {
		log.Printf("\t%s", pkg.ImportPath)
	}
	vd := filepath.Join(wd, vendorDir)
	cfgPath := filepath.Join(wd, configFile)
	var pkgs []*build.Package
	if len(os.Args) > 1 {
		if os.Args[1] == "init" {
			if _, err := os.Stat(configFile); !os.IsNotExist(err) {
				if err == nil {
					log.Fatal("You already have vndr.cfg, remove it if you want to reinit")
				}
				log.Fatal(err)
			}
			if _, err := os.Stat(vd); !os.IsNotExist(err) {
				if err == nil {
					log.Fatal("You already have vendor directory, remove it if you want to reinit")
				}
				log.Fatal(err)
			}
			log.Printf("Go get dependencies to %s", vd)
			var dlDeps []*godl.VCS
			dlFunc := func(importPath, dir string) error {
				vcs, err := godl.Download(importPath, dir, "")
				if err != nil {
					return err
				}
				log.Printf("Downloaded to vendor dir %s", vcs.ImportPath)
				dlDeps = append(dlDeps, vcs)
				return nil
			}
			// traverse dependency tree with on-fly downloading
			initPkgs, err := collectAllDeps(wd, dlFunc, initPkgs...)
			if err != nil {
				log.Fatalf("Error on collecting all dependencies: %v", err)
			}
			log.Printf("All dependencies downloaded")
			// cleanup vcs and vendor dirs
			deps, err := cleanDeps(dlDeps)
			if err != nil {
				log.Fatal(err)
			}
			if err := writeConfig(deps, cfgPath); err != nil {
				log.Fatal(err)
			}
			log.Printf("Config written to %s", cfgPath)
			pkgs = initPkgs
		}
	} else {
		cfg, err := os.Open(configFile)
		if err != nil {
			log.Fatalf("Failed to open config file: %v", err)
		}
		deps, err := parseDeps(cfg, vd)
		if err != nil {
			log.Fatalf("Failed to parse config: %v", err)
		}
		log.Println("Removing old vendor directory")
		if err := os.RemoveAll(vd); err != nil {
			log.Fatalf("Remove old vendor dir: %v", err)
		}
		log.Println("Download dependencies")
		if err := cloneAll(vd, deps); err != nil {
			log.Fatal(err)
		}
		log.Println("Dependencies downloaded")
	}
	log.Println("Collecting all dependencies")
	// pkgs != nil if init was used
	if pkgs != nil {
		upkgs, err := collectAllDeps(wd, nil, initPkgs...)
		if err != nil {
			log.Fatalf("Error on collecting all dependencies: %v", err)
		}
		pkgs = upkgs
	}
	log.Println("All dependencies collected")
	log.Println("Clean vendor dir from unused packages")
	if err := cleanVendor(vd, pkgs); err != nil {
		log.Fatal(err)
	}
	log.Println("Success")
}
<commit_msg>Fix deps collecting on update<commit_after>package main

import (
	"go/build"
	"log"
	"os"
	"path/filepath"

	"github.com/LK4D4/vndr/godl"
)

const (
	vendorDir  = "vendor"
	configFile = "vndr.cfg"
	tmpDir     = ".vndr-tmp"
)

func main() {
	if os.Getenv("GOPATH") == "" {
		log.Fatal("GOPATH must be set")
	}
	wd, err := os.Getwd()
	if err != nil {
		log.Fatalf("Error getting working directory: %v", err)
	}
	log.Println("Collecting local packages list")
	initPkgs, err := collectPkgs(wd)
	if err != nil {
		log.Fatalf("Error collecting initial packages: %v", err)
	}
	log.Println("Init pkgs:")
	for _, pkg := range initPkgs {
		log.Printf("\t%s", pkg.ImportPath)
	}
	vd := filepath.Join(wd, vendorDir)
	cfgPath := filepath.Join(wd, configFile)
	var pkgs []*build.Package
	if len(os.Args) > 1 {
		if os.Args[1] == "init" {
			if _, err := os.Stat(configFile); !os.IsNotExist(err) {
				if err == nil {
					log.Fatal("You already have vndr.cfg, remove it if you want to reinit")
				}
				log.Fatal(err)
			}
			if _, err := os.Stat(vd); !os.IsNotExist(err) {
				if err == nil {
					log.Fatal("You already have vendor directory, remove it if you want to reinit")
				}
				log.Fatal(err)
			}
			log.Printf("Go get dependencies to %s", vd)
			var dlDeps []*godl.VCS
			dlFunc := func(importPath, dir string) error {
				vcs, err := godl.Download(importPath, dir, "")
				if err != nil {
					return err
				}
				log.Printf("Downloaded to vendor dir %s", vcs.ImportPath)
				dlDeps = append(dlDeps, vcs)
				return nil
			}
			// traverse dependency tree with on-fly downloading
			initPkgs, err := collectAllDeps(wd, dlFunc, initPkgs...)
			if err != nil {
				log.Fatalf("Error on collecting all dependencies: %v", err)
			}
			log.Printf("All dependencies downloaded")
			// cleanup vcs and vendor dirs
			deps, err := cleanDeps(dlDeps)
			if err != nil {
				log.Fatal(err)
			}
			if err := writeConfig(deps, cfgPath); err != nil {
				log.Fatal(err)
			}
			log.Printf("Config written to %s", cfgPath)
			pkgs = initPkgs
		}
	} else {
		cfg, err := os.Open(configFile)
		if err != nil {
			log.Fatalf("Failed to open config file: %v", err)
		}
		deps, err := parseDeps(cfg, vd)
		if err != nil {
			log.Fatalf("Failed to parse config: %v", err)
		}
		log.Println("Removing old vendor directory")
		if err := os.RemoveAll(vd); err != nil {
			log.Fatalf("Remove old vendor dir: %v", err)
		}
		log.Println("Download dependencies")
		if err := cloneAll(vd, deps); err != nil {
			log.Fatal(err)
		}
		log.Println("Dependencies downloaded")
	}
	log.Println("Collecting all dependencies")
	// pkgs != nil if init was used
	if pkgs == nil {
		upkgs, err := collectAllDeps(wd, nil, initPkgs...)
		if err != nil {
			log.Fatalf("Error on collecting all dependencies: %v", err)
		}
		pkgs = upkgs
	}
	log.Println("All dependencies collected")
	log.Println("Clean vendor dir from unused packages")
	if err := cleanVendor(vd, pkgs); err != nil {
		log.Fatal(err)
	}
	log.Println("Success")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/slyrz/newscat/html"
	"github.com/slyrz/newscat/model"
	"github.com/slyrz/newscat/util"
	"io"
	"net/http"
	"os"
	"strings"
)

var (
	highlight = util.IsTerminal(os.Stdout)
)

func printChunks(chunks []*html.Chunk) {
	var last *html.Chunk = nil
	for _, chunk := range chunks {
		delim, pre, pos := "", "", ""
		// If the last chunk and the current chunk are part of the same HTML block,
		// separate them by a space character. Otherwise use two newline characters
		// to create a new paragraph.
		if last != nil {
			switch {
			case last.Block != chunk.Block:
				delim = "\n\n"
			case last.Block == chunk.Block:
				delim = " "
			}
		}
		if highlight {
			// Print headings and emphasized text bold.
			switch chunk.Base.Data {
			case "h1", "h2", "h3", "h4", "h5", "h6", "em", "strong", "b":
				pre, pos = "\x1b[1m", "\x1b[0m"
			default:
				pre, pos = "", ""
			}
		}
		fmt.Printf("%s%s%s%s", delim, pre, chunk.Text, pos)
		last = chunk
	}
	fmt.Println()
}

func main() {
	inputData := make(chan io.Reader, 4)
	// Open all input (file paths or URLs) or read from stdin and write the
	// corresponding io.Readers to the inputData channel.
	go func() {
		if args := os.Args[1:]; len(args) > 0 {
			for _, arg := range args {
				if strings.HasPrefix(arg, "http://") {
					if resp, err := http.Get(arg); err == nil {
						inputData <- resp.Body
					}
				} else {
					if file, err := os.Open(arg); err == nil {
						inputData <- file
					}
				}
			}
		} else {
			inputData <- os.Stdin
		}
		close(inputData)
	}()

	ext := model.NewExtractor()
	// Read input from inputData channel and perform content extraction.
	for data := range inputData {
		// TODO: Warn if parsing document failed.
		if doc, err := html.NewDocument(data); err == nil {
			// TODO: Print warning if no chunks were extracted.
			if chunks := ext.Extract(doc); len(chunks) > 0 {
				printChunks(chunks)
			}
		}
	}
}
<commit_msg>accept https urls<commit_after>package main

import (
	"fmt"
	"github.com/slyrz/newscat/html"
	"github.com/slyrz/newscat/model"
	"github.com/slyrz/newscat/util"
	"io"
	"net/http"
	"os"
	"strings"
)

var (
	highlight = util.IsTerminal(os.Stdout)
)

func printChunks(chunks []*html.Chunk) {
	var last *html.Chunk = nil
	for _, chunk := range chunks {
		delim, pre, pos := "", "", ""
		// If the last chunk and the current chunk are part of the same HTML block,
		// separate them by a space character. Otherwise use two newline characters
		// to create a new paragraph.
		if last != nil {
			switch {
			case last.Block != chunk.Block:
				delim = "\n\n"
			case last.Block == chunk.Block:
				delim = " "
			}
		}
		if highlight {
			// Print headings and emphasized text bold.
			switch chunk.Base.Data {
			case "h1", "h2", "h3", "h4", "h5", "h6", "em", "strong", "b":
				pre, pos = "\x1b[1m", "\x1b[0m"
			default:
				pre, pos = "", ""
			}
		}
		fmt.Printf("%s%s%s%s", delim, pre, chunk.Text, pos)
		last = chunk
	}
	fmt.Println()
}

func main() {
	inputData := make(chan io.Reader, 4)
	// Open all input (file paths or URLs) or read from stdin and write the
	// corresponding io.Readers to the inputData channel.
	go func() {
		if args := os.Args[1:]; len(args) > 0 {
			for _, arg := range args {
				if strings.HasPrefix(arg, "http://") || strings.HasPrefix(arg, "https://") {
					if resp, err := http.Get(arg); err == nil {
						inputData <- resp.Body
					}
				} else {
					if file, err := os.Open(arg); err == nil {
						inputData <- file
					}
				}
			}
		} else {
			inputData <- os.Stdin
		}
		close(inputData)
	}()

	ext := model.NewExtractor()
	// Read input from inputData channel and perform content extraction.
	for data := range inputData {
		// TODO: Warn if parsing document failed.
		if doc, err := html.NewDocument(data); err == nil {
			// TODO: Print warning if no chunks were extracted.
			if chunks := ext.Extract(doc); len(chunks) > 0 {
				printChunks(chunks)
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"context"
	"crypto/rand"
	"crypto/sha1"
	"encoding/base64"
	"encoding/hex"
	"errors"
	"fmt"
	"image/jpeg"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/jmoiron/sqlx"
	"github.com/joho/godotenv"
	"github.com/kurin/blazer/b2"
	_ "github.com/mattn/go-sqlite3"
	"github.com/nfnt/resize"
	"github.com/rwcarlsen/goexif/exif"
	"golang.org/x/crypto/nacl/secretbox"
)

var createSchema = `
CREATE TABLE IF NOT EXISTS img (
	img_id text PRIMARY KEY,
	created_at DATETIME,
    img_hash text NOT NULL,
	b2_img_name text NOT NULL,
	b2_thumbnail_name text NOT NULL);`

type image struct {
	ImgID           string    `db:"img_id"`
	CreatedAt       time.Time `db:"created_at"`
	ImgHash         string    `db:"img_hash"`
	B2ImgName       string    `db:"b2_img_name"`
	B2ThumbnailName string    `db:"b2_thumbnail_name"`
}

func main() {
	f, err := os.OpenFile("output.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		log.Fatalf("error opening file: %v", err)
	}
	defer f.Close()
	log.SetOutput(f)

	if err := godotenv.Load("settings.env"); err != nil {
		log.Fatal("Error loading .env file")
	}

	encryptionKey := os.Getenv("ENCR_KEY")
	b2id := os.Getenv("B2_ACCOUNT_ID")
	b2key := os.Getenv("B2_ACCOUNT_KEY")
	bucketName := os.Getenv("B2_BUCKET_NAME")

	db := sqlx.MustConnect("sqlite3", "img.db")
	ctx := context.Background()
	b2Client, err := b2.NewClient(ctx, b2id, b2key)
	if err != nil {
		log.Fatal(err)
	}

	bucket, err := b2Client.Bucket(ctx, bucketName)
	if err != nil {
		log.Fatal(err)
	}
	db.MustExec(createSchema)

	imgRootDir := os.Args[1]

	paths := getAllImagesPaths(imgRootDir)
	log.Printf("Found %v images.", len(paths))
	localState := syncLocalImagesWithBackblaze(ctx, paths, bucket, db, encryptionKey)
	log.Println("Sync compleated.")
	for _, imgID := range localState {
		log.Printf("ImgID: %v", imgID)
	}
}

func getAllImagesPaths(rootPath string) []string {
	paths := make([]string, 0)
	var isJpegPredicate = func(path string, f os.FileInfo) bool {
		return !f.IsDir() && (strings.HasSuffix(strings.ToLower(f.Name()), ".jpg") || strings.HasSuffix(strings.ToLower(f.Name()), ".jpeg"))
	}
	err := filepath.Walk(rootPath, func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			log.Printf("Error while walking the directory structure: %v", err.Error())
		}
		if isJpegPredicate(path, fi) {
			paths = append(paths, path)
		}
		return nil
	})
	if err != nil {
		log.Fatalf(err.Error())
	}
	return paths
}

func syncLocalImagesWithBackblaze(ctx context.Context, paths []string, bucket *b2.Bucket, db *sqlx.DB, encryptionKey string) []string {
	imgIds := make([]string, 0)

	for _, p := range paths {
		imgID, err := syncImage(ctx, p, bucket, db, encryptionKey)
		if err != nil {
			continue
		}
		imgIds = append(imgIds, imgID)
	}
	return imgIds
}

func syncImage(ctx context.Context, imgPath string, bucket *b2.Bucket, db *sqlx.DB, encryptionKey string) (string, error) {
	imgID, err := calculateHash(bytes.NewReader([]byte(imgPath)))
	if err != nil {
		log.Printf("Error calculating path's hash - path: %v - err msg: %v", imgPath, err)
		return "", err
	}
	f, err := os.Open(imgPath)
	if err != nil {
		log.Printf("Error opening file %v - err msg: %v", imgPath, err)
		return "", err
	}
	defer f.Close()
	imgContent, err := ioutil.ReadAll(f)
	if err != nil {
		log.Printf("Error reading file %v - err msg: %v", imgPath, err)
		return "", err
	}
	imgContentReader := bytes.NewReader(imgContent)
	imgCreatedAt, thumbnail, err := readExifMetadata(imgPath, imgContentReader)
	if err != nil {
		log.Printf("Error obtaining exif-metadata from file: %v - err msg: %v", imgPath, err)
		return "", err
	}
	if isImgToOld(imgCreatedAt) {
		return "", err
	}
	imgHash, err := calculateHash(imgContentReader)
	if err != nil {
		log.Printf("Error calculating img hash: %v", err)
		return "", err
	}
	encryptedThumbnail, err := encrypt(encryptionKey, thumbnail)
	if err != nil {
		log.Printf("Error encrypting thumbnail: %v - err msg: %v", imgPath, err)
		return "", err
	}
	log.Printf("Thumbnail size: %v", len(encryptedThumbnail))
	encryptedImg, err := encrypt(encryptionKey, imgContent)
	if err != nil {
		log.Printf("Error encrypting image: %v - err msg: %v", imgPath, err)
		return "", err
	}
	log.Printf("Image size: %v", len(encryptedImg))

	var existingImgs = []image{}
	if err := db.Select(&existingImgs, "SELECT img_id, created_at, img_hash, b2_img_name, b2_thumbnail_name FROM img WHERE img_id=$1 LIMIT 1;", imgID); err != nil {
		log.Printf("Error quering existing image %v - err: %v", imgID, err)
		return "", err
	}

	imgExists := len(existingImgs) > 0
	existingImgChanged := imgExists && existingImgs[0].ImgHash != imgHash
	if imgExists && !existingImgChanged {
		log.Print("Existing image unchanged - do nothing and process next img")
		return imgID, nil
	} else if existingImgChanged {
		log.Print("Existing image changed.")
		_, err := db.Exec("DELETE FROM img WHERE img_id=$1", imgID)
		if err != nil {
			log.Printf("Image with the ID = %v could not be deleted - err: %v", imgID, err)
			return "", err
		}
	}

	b2thumbnailName := generateUniqueFileName("thumb", imgPath, imgCreatedAt)
	b2imgName := generateUniqueFileName("orig", imgPath, imgCreatedAt)

	tx := db.MustBegin()

	img := &image{imgID, imgCreatedAt, imgHash, b2imgName, b2thumbnailName}
	if _, err := tx.NamedExec("INSERT INTO img (img_id, created_at, img_hash, b2_img_name, b2_thumbnail_name) VALUES (:img_id, :created_at, :img_hash, :b2_img_name, :b2_thumbnail_name)", img); err != nil {
		log.Printf("Error inserting into DB: %v", err)
		return "", err
	}
	log.Printf("Starting upload to b2 - img %v", imgPath)

	if err := uploadToB2(ctx, bucket, b2thumbnailName, encryptedThumbnail); err != nil {
		log.Printf("Error uploading to b2: %v - img: %v", err, imgPath)
		return "", err
	}
	if err := uploadToB2(ctx, bucket, b2imgName, encryptedImg); err != nil {
		log.Printf("Error uploading to b2: %v - img: %v", err, imgPath)
		return "", err
	}
	log.Printf("Upload compleated for img %v", imgPath)
	err = tx.Commit()
	if err != nil {
		log.Printf("Error commiting to the db: %v", err)
		return "", err
	}

	return imgID, nil
}

func uploadToB2(ctx context.Context, bucket *b2.Bucket, imgFileName string, payload []byte) error {
	b2Writer := createB2Writer(ctx, bucket, imgFileName)
	reader := bytes.NewReader(payload)
	if _, err := io.Copy(b2Writer, reader); err != nil {
		return err
	}
	if err := b2Writer.Close(); err != nil {
		return err
	}
	return nil
}

func createB2Writer(ctx context.Context, bucket *b2.Bucket, imgFileName string) *b2.Writer {
	imgObj := bucket.Object(imgFileName)
	b2Writer := imgObj.NewWriter(ctx)
	return b2Writer
}

func generateUniqueFileName(prefix string, imgPath string, imgCreatedAt time.Time) string {
	nano := strconv.FormatInt(imgCreatedAt.UnixNano(), 10)
	imgFileName := prefix + "_" + nano + "_" + path.Base(imgPath)
	return imgFileName
}

func isImgToOld(createdAt time.Time) bool {
	return createdAt.Year() < time.Now().Add(-1*time.Hour*24*365*10).Year()
}

func readExifMetadata(imgPath string, r io.ReadSeeker) (time.Time, []byte, error) {
	x, err := exif.Decode(r)
	if err != nil {
		return time.Time{}, nil, err
	}
	imgCreatedAt, err := x.DateTime()
	if err != nil {
		imgCreatedAt, err = findNeighborImgCreatedAt(imgPath)
		if err != nil {
			return time.Time{}, nil, err
		}
	}
	thumbnail, err := x.JpegThumbnail()
	if err != nil {
		thumbnail, err = resizeImg(r)
		if err != nil {
			return time.Time{}, nil, err
		}
	}
	return imgCreatedAt, thumbnail, nil
}

func resizeImg(r io.ReadSeeker) ([]byte, error) {
	r.Seek(0, 0)
	img, err := jpeg.Decode(r)
	if err != nil {
		return nil, err
	}
	m := resize.Thumbnail(200, 200, img, resize.NearestNeighbor)
	var b bytes.Buffer
	writer := bufio.NewWriter(&b)
	if err := jpeg.Encode(writer, m, nil); err != nil {
		return nil, err
	}
	if err := writer.Flush(); err != nil {
		return nil, err
	}
	return b.Bytes(), nil
}

func findNeighborImgCreatedAt(imgPath string) (time.Time, error) {
	var imgCreatedAt time.Time
	containingdDir := filepath.Dir(imgPath)
	matches, _ := filepath.Glob(filepath.Join(containingdDir, "*.jpg"))
	for _, imgfile := range matches {
		imgCreatedAt = func(f string) time.Time {
			if f == imgPath {
				return time.Time{}
			}
			reader, err := os.Open(f)
			if err != nil {
				return time.Time{}
			}
			defer reader.Close()
			other, err := exif.Decode(reader)
			if err != nil {
				return time.Time{}
			}
			imgCreatedAt, err = other.DateTime()
			if err != nil {
				return time.Time{}
			}
			return imgCreatedAt.Add(time.Millisecond * time.Duration(1))
		}(imgfile)
		foundCreatedAt := imgCreatedAt != time.Time{}
		if foundCreatedAt {
			return imgCreatedAt, nil
		}
	}
	return time.Time{}, fmt.Errorf("Coldn't extract CreatedAt for file %v", imgPath)
}

func calculateHash(r io.Reader) (string, error) {
	h := sha1.New()
	if _, err := io.Copy(h, r); err != nil {
		return "", err
	}
	hash := base64.StdEncoding.EncodeToString(h.Sum(nil))
	return hash, nil
}

func encrypt(encryptionKey string, data []byte) ([]byte, error) {
	secretKeyBytes, err := hex.DecodeString(encryptionKey)
	if err != nil {
		return nil, err
	}

	var secretKey [32]byte
	copy(secretKey[:], secretKeyBytes)

	var nonce [24]byte
	if _, err := io.ReadFull(rand.Reader, nonce[:]); err != nil {
		return nil, err
	}

	encrypted := secretbox.Seal(nonce[:], data, &nonce, &secretKey)
	return encrypted, nil
}

func decrypt(encryptionKey string, encryptedData []byte) ([]byte, error) {
	secretKeyBytes, err := hex.DecodeString(encryptionKey)
	if err != nil {
		return nil, err
	}

	var secretKey [32]byte
	copy(secretKey[:], secretKeyBytes)
	var decryptNonce [24]byte
	copy(decryptNonce[:], encryptedData[:24])
	decrypted, ok := secretbox.Open(nil, encryptedData[24:], &decryptNonce, &secretKey)
	if !ok {
		return nil, errors.New("Could not decrypt data")
	}

	return decrypted, nil
}
<commit_msg>refactor<commit_after>package main

import (
	"bufio"
	"bytes"
	"context"
	"crypto/rand"
	"crypto/sha1"
	"encoding/base64"
	"encoding/hex"
	"errors"
	"flag"
	"fmt"
	"image/jpeg"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/jmoiron/sqlx"
	"github.com/joho/godotenv"
	"github.com/kurin/blazer/b2"
	_ "github.com/mattn/go-sqlite3"
	"github.com/nfnt/resize"
	"github.com/rwcarlsen/goexif/exif"
	"golang.org/x/crypto/nacl/secretbox"
)

var createSchema = `
CREATE TABLE IF NOT EXISTS img (
	img_id text PRIMARY KEY,
	created_at DATETIME,
    img_hash text NOT NULL,
	b2_img_name text NOT NULL,
	b2_thumbnail_name text NOT NULL);`

type image struct {
	ImgID           string    `db:"img_id"`
	CreatedAt       time.Time `db:"created_at"`
	ImgHash         string    `db:"img_hash"`
	B2ImgName       string    `db:"b2_img_name"`
	B2ThumbnailName string    `db:"b2_thumbnail_name"`
}

type fileStore interface {
	uploadToB2(imgFileName string, payload []byte) error
	downloadFile(encryptionKey, src, dst string)
}

type b2Store struct {
	bucket *b2.Bucket
	ctx    context.Context
}

func newb2Store(ctx context.Context, b2id string, b2key string, bucketName string) *b2Store {
	b2Client, err := b2.NewClient(ctx, b2id, b2key)
	if err != nil {
		log.Fatal(err)
	}

	bucket, err := b2Client.Bucket(ctx, bucketName)
	if err != nil {
		log.Fatal(err)
	}
	return &b2Store{bucket: bucket, ctx: ctx}
}

func (b2 *b2Store) downloadFile(encryptionKey, src, dst string) {
	r := b2.bucket.Object(src).NewReader(ctx)
	defer r.Close()

	var b bytes.Buffer
	writer := bufio.NewWriter(&b)
	if _, err := io.Copy(writer, r); err != nil {
		log.Fatal("Booom!!!")
	}
	writer.Flush()
	encryptedData := b.Bytes()
	decryptedData, _ := decrypt(encryptionKey, encryptedData)
	f, err := os.Create(dst)
	if err != nil {
		log.Fatal("Booom!!!")
	}
	//r.ConcurrentDownloads = downloads
	if _, err := io.Copy(f, bytes.NewReader(decryptedData)); err != nil {
		log.Fatal("Booom!!!")
	}
}

func (b2 *b2Store) uploadToB2(imgFileName string, payload []byte) error {
	imgObj := b2.bucket.Object(imgFileName)
	b2Writer := imgObj.NewWriter(b2.ctx)
	reader := bytes.NewReader(payload)
	if _, err := io.Copy(b2Writer, reader); err != nil {
		return err
	}
	if err := b2Writer.Close(); err != nil {
		return err
	}
	return nil
}

type imgFileDto struct {
	Path      string
	CreatedAt time.Time
}
type action func(tx *sqlx.Tx) error
type datastore interface {
	executeInsideTran(actionFn action) error
	getImgByID(imgID string) ([]image, error)
	insertImg(tx *sqlx.Tx, imgEntity *image) error
	updateExisting(imgID string, imgContentHash string) error
}

type db struct {
	db *sqlx.DB
}

func newDB(dbName string) *db {
	dbInstance := sqlx.MustConnect("sqlite3", dbName)
	dbInstance.MustExec(createSchema)
	return &db{db: dbInstance}
}

func (datastore *db) executeInsideTran(actionFn action) error {
	tx := datastore.db.MustBegin()
	if err := actionFn(tx); err != nil {
		return err
	}

	if err := tx.Commit(); err != nil {
		return err
	}
	return nil
}

func (datastore *db) getImgByID(imgID string) ([]image, error) {
	var existingImgs = []image{}
	if err := datastore.db.Select(&existingImgs, "SELECT img_id, created_at, img_hash, b2_img_name, b2_thumbnail_name FROM img WHERE img_id=$1 LIMIT 1;", imgID); err != nil {
		log.Printf("Error quering existing image %v - err: %v", imgID, err)
		return nil, err
	}
	return existingImgs, nil
}

func (datastore *db) insertImg(tx *sqlx.Tx, imgEntity *image) error {
	if _, err := tx.NamedExec("INSERT INTO img (img_id, created_at, img_hash, b2_img_name, b2_thumbnail_name) VALUES (:img_id, :created_at, :img_hash, :b2_img_name, :b2_thumbnail_name)", imgEntity); err != nil {
		log.Printf("Error inserting into DB: %v", err)
		return err
	}
	return nil
}

func (datastore *db) updateExisting(imgID string, imgContentHash string) error {
	existingImgs, err := datastore.getImgByID(imgID)
	if err != nil {
		log.Printf("Error quering existing image %v - err: %v", imgID, err)
		return err
	}
	imgExists := len(existingImgs) > 0
	existingImgChanged := imgExists && existingImgs[0].ImgHash != imgContentHash
	if imgExists && !existingImgChanged {
		log.Print("Existing image unchanged - do nothing and process next img")
		return nil
	} else if existingImgChanged {
		log.Print("Existing image changed.")
		_, err := datastore.db.Exec("DELETE FROM img WHERE img_id=$1", imgID)
		if err != nil {
			log.Printf("Image with the ID = %v could not be deleted - err: %v", imgID, err)
			return err
		}
	}
	return nil
}

func main() {
	f, err := os.OpenFile("output.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		log.Fatalf("error opening file: %v", err)
	}
	defer f.Close()
	log.SetOutput(f)

	if err := godotenv.Load("settings.env"); err != nil {
		log.Fatal("Error loading .env file")
	}

	encryptionKey := os.Getenv("ENCR_KEY")
	b2id := os.Getenv("B2_ACCOUNT_ID")
	b2key := os.Getenv("B2_ACCOUNT_KEY")
	bucketName := os.Getenv("B2_BUCKET_NAME")

	ctx := context.Background()

	imgStore := newb2Store(ctx, b2id, b2key, bucketName)
	db := newDB("img.db")

	imgRootDir := flag.String("syncdir", "", "Abs path to the directory containing pictures")
	imgName := flag.String("download", "", "File to download and decrypt")
	flag.Parse()

	if *imgRootDir != "" {
		imgs := getImages(*imgRootDir)
		log.Printf("Found %v images.", len(imgs))
		localState := syncLocalImagesWithBackblaze(ctx, imgs, imgStore, db, encryptionKey)
		log.Println("Sync compleated.")
		for _, imgID := range localState {
			log.Printf("ImgID: %v", imgID)
		}
	} else if *imgName != "" {
		imgStore.downloadFile(encryptionKey, *imgName, "/home/piotr/Documents/imgtest/decrypted.jpg")
	}
}

func getImages(rootPath string) []*imgFileDto {
	imgFiles := make([]*imgFileDto, 0)

	var isJpegPredicate = func(path string, f os.FileInfo) bool {
		return !f.IsDir() && (strings.HasSuffix(strings.ToLower(f.Name()), ".jpg") || strings.HasSuffix(strings.ToLower(f.Name()), ".jpeg"))
	}
	var isImgToOldPredicate = func(createdAt time.Time) bool {
		return createdAt.Year() < time.Now().Add(-1*time.Hour*24*365*10).Year()
	}
	err := filepath.Walk(rootPath, func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			log.Printf("Error while walking the directory structure: %v", err.Error())
		}
		if isJpegPredicate(path, fi) {
			f, err := os.Open(path)
			if err != nil {
				log.Printf("Error opening file %v - err msg: %v", path, err)
				return err
			}
			defer f.Close()
			imgCreatedAt, err := extractCreatedAt(path, f)
			if err != nil {
				log.Printf("Error obtaining exif-metadata from file: %v - err msg: %v", path, err)
				return err
			}
			if !isImgToOldPredicate(imgCreatedAt) {
				imgFiles = append(imgFiles, &imgFileDto{Path: path, CreatedAt: imgCreatedAt})
			}
		}
		return nil
	})
	if err != nil {
		log.Fatalf(err.Error())
	}
	return imgFiles
}

func syncLocalImagesWithBackblaze(ctx context.Context, images []*imgFileDto, imgStore fileStore, db datastore, encryptionKey string) []string {
	imgIds := make([]string, 0)

	for _, img := range images {
		imgID, err := syncImage(ctx, img, imgStore, db, encryptionKey)
		if err != nil {
			continue
		}
		imgIds = append(imgIds, imgID)
	}
	return imgIds
}

func syncImage(ctx context.Context, img *imgFileDto, imgStore fileStore, db datastore, encryptionKey string) (string, error) {
	imgID, err := calculateHash(bytes.NewReader([]byte(img.Path)))
	if err != nil {
		log.Printf("Error calculating path's hash - path: %v - err msg: %v", img.Path, err)
		return "", err
	}
	f, err := os.Open(img.Path)
	if err != nil {
		log.Printf("Error opening file %v - err msg: %v", img.Path, err)
		return "", err
	}
	defer f.Close()
	imgContent, err := ioutil.ReadAll(f)
	if err != nil {
		log.Printf("Error reading file %v - err msg: %v", img.Path, err)
		return "", err
	}
	imgContentReader := bytes.NewReader(imgContent)
	thumbnail, err := extractThumbnail(img.Path, imgContentReader)
	if err != nil {
		log.Printf("Error obtaining exif-metadata from file: %v - err msg: %v", img.Path, err)
		return "", err
	}

	imgHash, err := calculateHash(imgContentReader)
	if err != nil {
		log.Printf("Error calculating img hash: %v", err)
		return "", err
	}
	encryptedThumbnail, err := encrypt(encryptionKey, thumbnail)
	if err != nil {
		log.Printf("Error encrypting thumbnail: %v - err msg: %v", img.Path, err)
		return "", err
	}
	log.Printf("Thumbnail size: %v", len(encryptedThumbnail))
	encryptedImg, err := encrypt(encryptionKey, imgContent)
	if err != nil {
		log.Printf("Error encrypting image: %v - err msg: %v", img.Path, err)
		return "", err
	}

	if err := db.updateExisting(imgID, imgHash); err != nil {
		return "", err
	}

	b2thumbnailName := generateUniqueFileName("thumb", img.Path, img.CreatedAt)
	b2imgName := generateUniqueFileName("orig", img.Path, img.CreatedAt)
	err = db.executeInsideTran(func(tx *sqlx.Tx) error {
		imgEntity := &image{imgID, img.CreatedAt, imgHash, b2imgName, b2thumbnailName}
		if err := db.insertImg(tx, imgEntity); err != nil {
			log.Printf("Error inserting into DB: %v", err)
			return err
		}
		log.Printf("Starting upload to b2 - img %v", img.Path)

		if err := imgStore.uploadToB2(b2thumbnailName, encryptedThumbnail); err != nil {
			log.Printf("Error uploading to b2: %v - img: %v", err, img.Path)
			return err
		}
		if err := imgStore.uploadToB2(b2imgName, encryptedImg); err != nil {
			log.Printf("Error uploading to b2: %v - img: %v", err, img.Path)
			return err
		}
		log.Printf("Upload compleated for img %v", img.Path)
		return nil
	})
	if err != nil {
		return "", err
	}

	return imgID, nil
}

func generateUniqueFileName(prefix string, imgPath string, imgCreatedAt time.Time) string {
	nano := strconv.FormatInt(imgCreatedAt.UnixNano(), 10)
	imgFileName := prefix + "_" + nano + "_" + path.Base(imgPath)
	return imgFileName
}

func extractCreatedAt(imgPath string, r *os.File) (time.Time, error) {
	x, err := exif.Decode(r)
	if err != nil {
		return time.Time{}, err
	}
	defer r.Close()
	imgCreatedAt, err := x.DateTime()
	if err != nil {
		imgCreatedAt, err = findNeighborImgCreatedAt(imgPath)
		if err != nil {
			return time.Time{}, err
		}
	}
	return imgCreatedAt, nil
}

func extractThumbnail(imgPath string, r io.ReadSeeker) ([]byte, error) {
	x, err := exif.Decode(r)
	if err != nil {
		return nil, err
	}

	thumbnail, err := x.JpegThumbnail()
	if err != nil {
		thumbnail, err = resizeImg(r)
		if err != nil {
			return nil, err
		}
	}
	return thumbnail, nil
}

func resizeImg(r io.ReadSeeker) ([]byte, error) {
	r.Seek(0, 0)
	img, err := jpeg.Decode(r)
	if err != nil {
		return nil, err
	}
	m := resize.Thumbnail(200, 200, img, resize.NearestNeighbor)
	var b bytes.Buffer
	writer := bufio.NewWriter(&b)
	if err := jpeg.Encode(writer, m, nil); err != nil {
		return nil, err
	}
	if err := writer.Flush(); err != nil {
		return nil, err
	}
	return b.Bytes(), nil
}

func findNeighborImgCreatedAt(imgPath string) (time.Time, error) {
	var imgCreatedAt time.Time
	containingdDir := filepath.Dir(imgPath)
	matches, _ := filepath.Glob(filepath.Join(containingdDir, "*.jpg"))
	for _, imgfile := range matches {
		imgCreatedAt = func(f string) time.Time {
			if f == imgPath {
				return time.Time{}
			}
			reader, err := os.Open(f)
			if err != nil {
				return time.Time{}
			}
			defer reader.Close()
			other, err := exif.Decode(reader)
			if err != nil {
				return time.Time{}
			}
			imgCreatedAt, err = other.DateTime()
			if err != nil {
				return time.Time{}
			}
			return imgCreatedAt.Add(time.Millisecond * time.Duration(1))
		}(imgfile)
		foundCreatedAt := imgCreatedAt != time.Time{}
		if foundCreatedAt {
			return imgCreatedAt, nil
		}
	}
	return time.Time{}, fmt.Errorf("Coldn't extract CreatedAt for file %v", imgPath)
}

func calculateHash(r io.Reader) (string, error) {
	h := sha1.New()
	if _, err := io.Copy(h, r); err != nil {
		return "", err
	}
	hash := base64.StdEncoding.EncodeToString(h.Sum(nil))
	return hash, nil
}

func encrypt(encryptionKey string, data []byte) ([]byte, error) {
	secretKeyBytes, err := hex.DecodeString(encryptionKey)
	if err != nil {
		return nil, err
	}

	var secretKey [32]byte
	copy(secretKey[:], secretKeyBytes)

	var nonce [24]byte
	if _, err := io.ReadFull(rand.Reader, nonce[:]); err != nil {
		return nil, err
	}

	encrypted := secretbox.Seal(nonce[:], data, &nonce, &secretKey)
	return encrypted, nil
}

func decrypt(encryptionKey string, encryptedData []byte) ([]byte, error) {
	secretKeyBytes, err := hex.DecodeString(encryptionKey)
	if err != nil {
		return nil, err
	}

	var secretKey [32]byte
	copy(secretKey[:], secretKeyBytes)
	var decryptNonce [24]byte
	copy(decryptNonce[:], encryptedData[:24])
	decrypted, ok := secretbox.Open(nil, encryptedData[24:], &decryptNonce, &secretKey)
	if !ok {
		return nil, errors.New("Could not decrypt data")
	}

	return decrypted, nil
}
<|endoftext|>


<commit_before>// TODO:
//
// Fork and merge filter sequences.
package main

import (
	"bufio"
	"crypto/sha1"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"sync"
)

type Arg struct {
	in  <-chan string
	out chan<- string
	// TODO: add an error channel here?
}

// Filter reads a sequence of strings from a channel and produces a
// sequence on another channel.  Many implementations of Filter are
// provided.
type Filter func(Arg)

// Each() returns a channel that contains all output emitted by a
// sequence of filters. The sequence of filters is fed an empty stream
// as the input.
func Each(filters ...Filter) <-chan string {
	in := make(chan string, 0)
	close(in)
	out := make(chan string, 10000)
	go runAndClose(Sequence(filters...), Arg{in, out})
	return out
}

// Sequence returns a filter that is the concatenation of all filter arguments.
func Sequence(filters ...Filter) Filter {
	return func(arg Arg) {
		in := arg.in
		for _, f := range filters {
			c := make(chan string, 10000)
			go runAndClose(f, Arg{in, c})
			in = c
		}
		copydata(Arg{in, arg.out})
	}
}

// Print prints all output emitted by a sequence of filters. The
// sequence of filters is fed an empty stream as the input.
func Print(filters ...Filter) {
	for s := range Each(filters...) {
		fmt.Println(s)
	}
}

func runAndClose(f Filter, arg Arg) {
	f(arg)
	close(arg.out)
}

// copydata copies all items read from in to out.
func copydata(arg Arg) {
	for s := range arg.in {
		arg.out <- s
	}
}

// Echo copies its input and then emits item.
func Echo(items ...string) Filter {
	return func(arg Arg) {
		copydata(arg)
		for _, s := range items {
			arg.out <- s
		}
	}
}

// Numbers copies its input and then emits the integers x..y
func Numbers(x, y int) Filter {
	return func(arg Arg) {
		copydata(arg)
		for i := x; i <= y; i++ {
			arg.out <- fmt.Sprintf("%d", i)
		}
	}
}

// If emits every input x for which fn(x) is true.
func If(fn func(string) bool) Filter {
	return func(arg Arg) {
		for s := range arg.in {
			if fn(s) {
				arg.out <- s
			}
		}
	}
}

// Grep emits every input x that matches the regular expression r.
func Grep(r string) Filter {
	re := regexp.MustCompile(r)
	return If(re.MatchString)
}

// GrepNot emits every input x that does not match the regular expression r.
func GrepNot(r string) Filter {
	re := regexp.MustCompile(r)
	return If(func(s string) bool { return !re.MatchString(s) })
}

// Uniq squashes adjacent identical items in in into a single output.
func Uniq(arg Arg) {
	first := true
	last := ""
	for s := range arg.in {
		if first || last != s {
			arg.out <- s
		}
		last = s
		first = false
	}
}

// UniqWithCount squashes adjacent identical items in in into a single
// output prefixed with the count of identical items.
func UniqWithCount(arg Arg) {
	current := ""
	count := 0
	for s := range arg.in {
		if s != current {
			if count > 0 {
				arg.out <- fmt.Sprintf("%d %s", count, current)
			}
			count = 0
			current = s
		}
		count++
	}
	if count > 0 {
		arg.out <- fmt.Sprintf("%d %s", count, current)
	}
}

// Apply calls fn(x, out) in order for every item x.
func Apply(fn func(string, chan<- string)) Filter {
	return func(arg Arg) {
		for s := range arg.in {
			fn(s, arg.out)
		}
	}
}

// ApplyParallel calls fn(x, out) for every item x in a pool of n goroutines.
func ApplyParallel(n int, fn func(string, chan<- string)) Filter {
	// TODO: Maintain input order?
	// (a) Input goroutine generates <index, str> pairs
	// (b) n appliers read pairs and produce <index, fn(str)> pairs
	// (c) Output goroutine reads and emits in order
	return func(arg Arg) {
		wg := &sync.WaitGroup{}
		wg.Add(n)
		for i := 0; i < n; i++ {
			go func() {
				for s := range arg.in {
					fn(s, arg.out)
				}
				wg.Done()
			}()
		}
		wg.Wait()
	}
}

func ReplaceMatch(r, replacement string) Filter {
	re := regexp.MustCompile(r)
	return Apply(func(s string, out chan<- string) {
		out <- re.ReplaceAllString(s, replacement)
	})
}

func DeleteMatch(r string) Filter {
	return ReplaceMatch(r, "")
}

func Sort(arg Arg) {
	var data []string
	for s := range arg.in {
		data = append(data, s)
	}
	sort.Strings(data)
	for _, s := range data {
		arg.out <- s
	}
}

type fnStringSlice struct {
	sort.StringSlice
	fn func(string, string) bool
}

func (fs fnStringSlice) Less(i, j int) bool {
	return fs.fn(fs.StringSlice[i], fs.StringSlice[j])
}

func SortBy(fn func(string, string) bool) Filter {
	return func(arg Arg) {
		var data fnStringSlice
		data.fn = fn
		for s := range arg.in {
			data.StringSlice = append(data.StringSlice, s)
		}
		sort.Sort(data)
		for _, s := range data.StringSlice {
			arg.out <- s
		}
	}
}

func SortNumeric(column int) Filter {
	toNum := func(s string) (int, int64) {
		r := regexp.MustCompile(`\s+`).Split(s, -1)
		for len(r) > 0 && r[0] == "" {
			r = r[1:]
		}
		for len(r) > 0 && r[len(r)-1] == "" {
			r = r[:len(r)-1]
		}
		if len(r) <= column {
			// Missing data is sorted before all numbers
			return -1, 0
		}
		x, err := strconv.ParseInt(r[column], 0, 64)
		if err != nil {
			// Bad data is sorted after all numbers
			return +1, 0
		}
		return 0, x
	}
	return SortBy(func(a, b string) bool {
		a1, a2 := toNum(a)
		b1, b2 := toNum(b)
		if a1 != b1 {
			return a1 < b1
		}
		return a2 < b2
	})
}

func Reverse(arg Arg) {
	var data []string
	for s := range arg.in {
		data = append(data, s)
	}
	for i := len(data) - 1; i >= 0; i-- {
		arg.out <- data[i]
	}
}

type FindType int

const (
	FILES    FindType = 1
	DIRS     FindType = 2
	SYMLINKS FindType = 4
	ALL      FindType = FILES | DIRS | SYMLINKS
)

func Find(ft FindType, dir string) Filter {
	return func(arg Arg) {
		copydata(arg)
		filepath.Walk(dir, func(f string, s os.FileInfo, e error) error {
			if ft&FILES != 0 && s.Mode().IsRegular() ||
				ft&DIRS != 0 && s.Mode().IsDir() ||
				ft&SYMLINKS != 0 && s.Mode()&os.ModeSymlink != 0 {
				arg.out <- f
			}
			return nil
		})
	}
}

func Cat(filenames ...string) Filter {
	return func(arg Arg) {
		copydata(arg)
		for _, f := range filenames {
			file, err := os.Open(f)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				continue
			}
			scanner := bufio.NewScanner(file)
			for scanner.Scan() {
				arg.out <- scanner.Text()
			}
			file.Close()
		}
	}
}

func First(n int) Filter {
	return func(arg Arg) {
		emitted := 0
		for s := range arg.in {
			if emitted < n {
				arg.out <- s
				emitted++
			}
		}
	}
}

func DropFirst(n int) Filter {
	return func(arg Arg) {
		emitted := 0
		for s := range arg.in {
			if emitted >= n {
				arg.out <- s
			}
			emitted++
		}
	}
}

func Last(n int) Filter {
	return func(arg Arg) {
		var buf []string
		for s := range arg.in {
			buf = append(buf, s)
			if len(buf) > n {
				buf = buf[1:]
			}
		}
		for _, s := range buf {
			arg.out <- s
		}
	}
}

func DropLast(n int) Filter {
	return func(arg Arg) {
		var buf []string
		for s := range arg.in {
			buf = append(buf, s)
			if len(buf) > n {
				arg.out <- buf[0]
				buf = buf[1:]
			}
		}
	}
}

func NumberLines(arg Arg) {
	line := 1
	for s := range arg.in {
		arg.out <- fmt.Sprintf("%5d %s", line, s)
		line++
	}
}

func Cut(start, end int) Filter {
	return Apply(func(s string, out chan<- string) {
		if len(s) > end {
			s = s[:end+1]
		}
		if len(s) < start {
			s = ""
		} else {
			s = s[start:]
		}
		out <- s
	})
}

func main() {
	_ = Apply(func(s string, out chan<- string) {
		out <- s
		out <- s
	})
	dbl := func(arg Arg) {
		for s := range arg.in {
			arg.out <- s
			arg.out <- s
		}
	}

	hash := func(f string, out chan<- string) {
		file, err := os.Open(f)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			return
		}
		hasher := sha1.New()
		_, err = io.Copy(hasher, file)
		file.Close()
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			return
		}
		out <- fmt.Sprintf("%x %s", hasher.Sum(nil), f)
	}

	Print(Numbers(1, 10),
		Grep("3"),
		dbl)

	Print(Sequence())
	Print(Sequence(Echo("1 of 1")))
	Print(Sequence(Echo("1 of 2"), Echo("2 of 2")))

	Print(Numbers(1, 100),
		Grep("3"),
		GrepNot("7"),
		dbl,
		Uniq,
		ReplaceMatch("^(.)$", "x$1"),
		Sort,
		ReplaceMatch("^(.)", "$1 "),
		dbl,
		DeleteMatch(" .$"),
		UniqWithCount,
		SortNumeric(1),
		Reverse,
		Echo("==="))

	Print(Find(FILES, "/home/sanjay/tmp"),
		Grep("/tmp/x"),
		GrepNot("/sub2/"),
		ApplyParallel(4, hash),
		ReplaceMatch(" /home/sanjay/", " HOME/"))

	Print(Echo("a"), Echo("b"), Echo("c"))

	Print(Cat("/home/sanjay/.bashrc"),
		First(10),
		NumberLines,
		Cut(1, 50),
		ReplaceMatch("^", "LINE:"),
		Last(3))

	Print(Numbers(1, 10), DropFirst(8))
	Print(Numbers(1, 10), DropLast(7))

	Print(Echo("=== all ==="), Find(ALL, "/home/sanjay/tmp/x"))
	Print(Echo("=== dirs ==="), Find(FILES, "/home/sanjay/tmp/x"))
	Print(Echo("=== files ==="), Find(DIRS, "/home/sanjay/tmp/x"))

	// Reconcile part 1
	Print(Echo("------------"))
	Print(
		Find(FILES, "/home/sanjay/tmp/y"),
		ApplyParallel(4, hash),
	)
}
<commit_msg>checkpoint<commit_after>// TODO:
//
// Fork and merge filter sequences.
package main

import (
	"bufio"
	"crypto/sha1"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"sync"
)

type Arg struct {
	in  <-chan string
	out chan<- string
	// TODO: add an error channel here?
}

// Filter reads a sequence of strings from a channel and produces a
// sequence on another channel.  Many implementations of Filter are
// provided.
type Filter func(Arg)

// Each() returns a channel that contains all output emitted by a
// sequence of filters. The sequence of filters is fed an empty stream
// as the input.
func Each(filters ...Filter) <-chan string {
	in := make(chan string, 0)
	close(in)
	out := make(chan string, 10000)
	go runAndClose(Sequence(filters...), Arg{in, out})
	return out
}

// Sequence returns a filter that is the concatenation of all filter arguments.
func Sequence(filters ...Filter) Filter {
	return func(arg Arg) {
		in := arg.in
		for _, f := range filters {
			c := make(chan string, 10000)
			go runAndClose(f, Arg{in, c})
			in = c
		}
		copydata(Arg{in, arg.out})
	}
}

// Print prints all output emitted by a sequence of filters. The
// sequence of filters is fed an empty stream as the input.
func Print(filters ...Filter) {
	for s := range Each(filters...) {
		fmt.Println(s)
	}
}

func runAndClose(f Filter, arg Arg) {
	f(arg)
	close(arg.out)
}

// copydata copies all items read from in to out.
func copydata(arg Arg) {
	for s := range arg.in {
		arg.out <- s
	}
}

// Echo copies its input and then emits item.
func Echo(items ...string) Filter {
	return func(arg Arg) {
		copydata(arg)
		for _, s := range items {
			arg.out <- s
		}
	}
}

// Numbers copies its input and then emits the integers x..y
func Numbers(x, y int) Filter {
	return func(arg Arg) {
		copydata(arg)
		for i := x; i <= y; i++ {
			arg.out <- fmt.Sprintf("%d", i)
		}
	}
}

// If emits every input x for which fn(x) is true.
func If(fn func(string) bool) Filter {
	return func(arg Arg) {
		for s := range arg.in {
			if fn(s) {
				arg.out <- s
			}
		}
	}
}

// Grep emits every input x that matches the regular expression r.
func Grep(r string) Filter {
	re := regexp.MustCompile(r)
	return If(re.MatchString)
}

// GrepNot emits every input x that does not match the regular expression r.
func GrepNot(r string) Filter {
	re := regexp.MustCompile(r)
	return If(func(s string) bool { return !re.MatchString(s) })
}

// Uniq squashes adjacent identical items in in into a single output.
func Uniq(arg Arg) {
	first := true
	last := ""
	for s := range arg.in {
		if first || last != s {
			arg.out <- s
		}
		last = s
		first = false
	}
}

// UniqWithCount squashes adjacent identical items in in into a single
// output prefixed with the count of identical items.
func UniqWithCount(arg Arg) {
	current := ""
	count := 0
	for s := range arg.in {
		if s != current {
			if count > 0 {
				arg.out <- fmt.Sprintf("%d %s", count, current)
			}
			count = 0
			current = s
		}
		count++
	}
	if count > 0 {
		arg.out <- fmt.Sprintf("%d %s", count, current)
	}
}

// Parallel calls fn(x, out) for every item x in a pool of n goroutines.
func Parallel(n int, fn func(string, chan<- string)) Filter {
	// TODO: Maintain input order?
	// (a) Input goroutine generates <index, str> pairs
	// (b) n appliers read pairs and produce <index, fn(str)> pairs
	// (c) Output goroutine reads and emits in order
	return func(arg Arg) {
		wg := &sync.WaitGroup{}
		wg.Add(n)
		for i := 0; i < n; i++ {
			go func() {
				for s := range arg.in {
					fn(s, arg.out)
				}
				wg.Done()
			}()
		}
		wg.Wait()
	}
}

func ReplaceMatch(r, replacement string) Filter {
	re := regexp.MustCompile(r)
	return func(arg Arg) {
		for s := range arg.in {
			arg.out <- re.ReplaceAllString(s, replacement)
		}
	}
}

func DeleteMatch(r string) Filter {
	return ReplaceMatch(r, "")
}

func Sort(arg Arg) {
	var data []string
	for s := range arg.in {
		data = append(data, s)
	}
	sort.Strings(data)
	for _, s := range data {
		arg.out <- s
	}
}

type fnStringSlice struct {
	sort.StringSlice
	fn func(string, string) bool
}

func (fs fnStringSlice) Less(i, j int) bool {
	return fs.fn(fs.StringSlice[i], fs.StringSlice[j])
}

func SortBy(fn func(string, string) bool) Filter {
	return func(arg Arg) {
		var data fnStringSlice
		data.fn = fn
		for s := range arg.in {
			data.StringSlice = append(data.StringSlice, s)
		}
		sort.Sort(data)
		for _, s := range data.StringSlice {
			arg.out <- s
		}
	}
}

func SortNumeric(column int) Filter {
	toNum := func(s string) (int, int64) {
		r := regexp.MustCompile(`\s+`).Split(s, -1)
		for len(r) > 0 && r[0] == "" {
			r = r[1:]
		}
		for len(r) > 0 && r[len(r)-1] == "" {
			r = r[:len(r)-1]
		}
		if len(r) <= column {
			// Missing data is sorted before all numbers
			return -1, 0
		}
		x, err := strconv.ParseInt(r[column], 0, 64)
		if err != nil {
			// Bad data is sorted after all numbers
			return +1, 0
		}
		return 0, x
	}
	return SortBy(func(a, b string) bool {
		a1, a2 := toNum(a)
		b1, b2 := toNum(b)
		if a1 != b1 {
			return a1 < b1
		}
		return a2 < b2
	})
}

func Reverse(arg Arg) {
	var data []string
	for s := range arg.in {
		data = append(data, s)
	}
	for i := len(data) - 1; i >= 0; i-- {
		arg.out <- data[i]
	}
}

type FindType int

const (
	FILES    FindType = 1
	DIRS     FindType = 2
	SYMLINKS FindType = 4
	ALL      FindType = FILES | DIRS | SYMLINKS
)

func Find(ft FindType, dir string) Filter {
	return func(arg Arg) {
		copydata(arg)
		filepath.Walk(dir, func(f string, s os.FileInfo, e error) error {
			if ft&FILES != 0 && s.Mode().IsRegular() ||
				ft&DIRS != 0 && s.Mode().IsDir() ||
				ft&SYMLINKS != 0 && s.Mode()&os.ModeSymlink != 0 {
				arg.out <- f
			}
			return nil
		})
	}
}

func Cat(filenames ...string) Filter {
	return func(arg Arg) {
		copydata(arg)
		for _, f := range filenames {
			file, err := os.Open(f)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				continue
			}
			scanner := bufio.NewScanner(file)
			for scanner.Scan() {
				arg.out <- scanner.Text()
			}
			file.Close()
		}
	}
}

func First(n int) Filter {
	return func(arg Arg) {
		emitted := 0
		for s := range arg.in {
			if emitted < n {
				arg.out <- s
				emitted++
			}
		}
	}
}

func DropFirst(n int) Filter {
	return func(arg Arg) {
		emitted := 0
		for s := range arg.in {
			if emitted >= n {
				arg.out <- s
			}
			emitted++
		}
	}
}

func Last(n int) Filter {
	return func(arg Arg) {
		var buf []string
		for s := range arg.in {
			buf = append(buf, s)
			if len(buf) > n {
				buf = buf[1:]
			}
		}
		for _, s := range buf {
			arg.out <- s
		}
	}
}

func DropLast(n int) Filter {
	return func(arg Arg) {
		var buf []string
		for s := range arg.in {
			buf = append(buf, s)
			if len(buf) > n {
				arg.out <- buf[0]
				buf = buf[1:]
			}
		}
	}
}

func NumberLines(arg Arg) {
	line := 1
	for s := range arg.in {
		arg.out <- fmt.Sprintf("%5d %s", line, s)
		line++
	}
}

func Cut(start, end int) Filter {
	return func(arg Arg) {
		for s := range arg.in {
			if len(s) > end {
				s = s[:end+1]
			}
			if len(s) < start {
				s = ""
			} else {
				s = s[start:]
			}
			arg.out <- s
		}
	}
}

func main() {
	dbl := func(arg Arg) {
		for s := range arg.in {
			arg.out <- s
			arg.out <- s
		}
	}

	hash := func(f string, out chan<- string) {
		file, err := os.Open(f)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			return
		}
		hasher := sha1.New()
		_, err = io.Copy(hasher, file)
		file.Close()
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			return
		}
		out <- fmt.Sprintf("%x %s", hasher.Sum(nil), f)
	}

	Print(Numbers(1, 10),
		Grep("3"),
		dbl)

	Print(Sequence())
	Print(Sequence(Echo("1 of 1")))
	Print(Sequence(Echo("1 of 2"), Echo("2 of 2")))

	Print(Numbers(1, 100),
		Grep("3"),
		GrepNot("7"),
		dbl,
		Uniq,
		ReplaceMatch("^(.)$", "x$1"),
		Sort,
		ReplaceMatch("^(.)", "$1 "),
		dbl,
		DeleteMatch(" .$"),
		UniqWithCount,
		SortNumeric(1),
		Reverse,
		Echo("==="))

	Print(Find(FILES, "/home/sanjay/tmp"),
		Grep("/tmp/x"),
		GrepNot("/sub2/"),
		Parallel(4, hash),
		ReplaceMatch(" /home/sanjay/", " HOME/"))

	Print(Echo("a"), Echo("b"), Echo("c"))

	Print(Cat("/home/sanjay/.bashrc"),
		First(10),
		NumberLines,
		Cut(1, 50),
		ReplaceMatch("^", "LINE:"),
		Last(3))

	Print(Numbers(1, 10), DropFirst(8))
	Print(Numbers(1, 10), DropLast(7))

	Print(Echo("=== all ==="), Find(ALL, "/home/sanjay/tmp/x"))
	Print(Echo("=== dirs ==="), Find(FILES, "/home/sanjay/tmp/x"))
	Print(Echo("=== files ==="), Find(DIRS, "/home/sanjay/tmp/x"))

	// Reconcile part 1
	Print(Echo("------------"))
	Print(
		Find(FILES, "/home/sanjay/tmp/y"),
		Parallel(4, hash),
	)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path"
	"text/template"

	"github.com/codegangsta/cli"
	"github.com/spf13/viper"
)

var (
	version string
	prefix  string
	source  string
	output  string
	force   bool
	debug   bool
)

func main() {
	cli.HelpFlag = cli.BoolFlag{
		Name:  "help, h",
		Usage: "Show the help, so what you see now",
	}

	cli.VersionFlag = cli.BoolFlag{
		Name:  "version, v",
		Usage: "Print the current version of that tool",
	}

	app := cli.NewApp()
	app.Name = "templater"
	app.Version = version
	app.Author = "Thomas Boerger <thomas@webhippie.de>"
	app.Usage = "A template processor for environment variables"
	app.ArgsUsage = "<inputfile>"

	app.HideHelp = true

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "prefix, p",
			Usage:       "Prefix of the environemt variables",
			EnvVar:      "TEMPLATER_PREFIX",
			Destination: &prefix,
			Value:       "",
		},
		cli.StringFlag{
			Name:        "output, o",
			Usage:       "Different output than STDOUT",
			EnvVar:      "TEMPLATER_OUTPUT",
			Destination: &output,
			Value:       "",
		},
		cli.BoolFlag{
			Name:        "force, f",
			Usage:       "Force to overwrite the output",
			EnvVar:      "TEMPLATER_FORCE",
			Destination: &force,
		},
		cli.BoolFlag{
			Name:        "debug, d",
			Usage:       "Print a bit more debug output",
			EnvVar:      "TEMPLATER_DEBUG",
			Destination: &debug,
		},
		cli.HelpFlag,
	}

	app.Action = func(c *cli.Context) {
		if len(c.Args()) == 0 {
			cli.ShowAppHelp(c)
			os.Exit(1)
		} else {
			source = c.Args().First()
		}

		if prefix != "" {
			viper.SetEnvPrefix(prefix)
		}

		viper.AutomaticEnv()

		tmpl, err := template.New(
			path.Base(source),
		).Funcs(
			template.FuncMap{
				"get": viper.Get,
			},
		).ParseFiles(
			source,
		)

		if err != nil {
			fmt.Println("Failed to read template")

			if debug {
				fmt.Println(err)
			}

			os.Exit(1)
		}

		exe := tmpl.Execute(
			os.Stdout,
			&Payload{
				Prefix: prefix,
				Source: source,
				Output: output,
				Force:  force,
				Debug:  debug,
			},
		)

		if exe != nil {
			fmt.Println("Failed to parse template")

			if debug {
				fmt.Println(exe)
			}

			os.Exit(2)
		}
	}

	app.Run(os.Args)
}

type Payload struct {
	Prefix string
	Source string
	Output string
	Force  bool
	Debug  bool
}
<commit_msg>Added missing funcmap<commit_after>package main

import (
	"fmt"
	"os"
	"path"
	"text/template"

	"github.com/codegangsta/cli"
	"github.com/spf13/viper"
)

var (
	version string
	prefix  string
	source  string
	output  string
	force   bool
	debug   bool
)

type Payload struct {
	Prefix string
	Source string
	Output string
	Force  bool
	Debug  bool
}

func main() {
	cli.HelpFlag = cli.BoolFlag{
		Name:  "help, h",
		Usage: "Show the help, so what you see now",
	}

	cli.VersionFlag = cli.BoolFlag{
		Name:  "version, v",
		Usage: "Print the current version of that tool",
	}

	app := cli.NewApp()
	app.Name = "templater"
	app.Version = version
	app.Author = "Thomas Boerger <thomas@webhippie.de>"
	app.Usage = "A template processor for environment variables"
	app.ArgsUsage = "<inputfile>"

	app.HideHelp = true

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "prefix, p",
			Usage:       "Prefix of the environemt variables",
			EnvVar:      "TEMPLATER_PREFIX",
			Destination: &prefix,
			Value:       "",
		},
		cli.StringFlag{
			Name:        "output, o",
			Usage:       "Different output than STDOUT",
			EnvVar:      "TEMPLATER_OUTPUT",
			Destination: &output,
			Value:       "",
		},
		cli.BoolFlag{
			Name:        "force, f",
			Usage:       "Force to overwrite the output",
			EnvVar:      "TEMPLATER_FORCE",
			Destination: &force,
		},
		cli.BoolFlag{
			Name:        "debug, d",
			Usage:       "Print a bit more debug output",
			EnvVar:      "TEMPLATER_DEBUG",
			Destination: &debug,
		},
		cli.HelpFlag,
	}

	app.Action = func(c *cli.Context) {
		if len(c.Args()) == 0 {
			cli.ShowAppHelp(c)
			os.Exit(1)
		} else {
			source = c.Args().First()
		}

		if prefix != "" {
			viper.SetEnvPrefix(prefix)
		}

		viper.AutomaticEnv()

		tmpl, err := template.New(
			path.Base(source),
		).Funcs(
			template.FuncMap{
				"Get":                viper.Get,
				"GetBool":            viper.GetBool,
				"GetFloat64":         viper.GetFloat64,
				"GetInt":             viper.GetInt,
				"GetString":          viper.GetString,
				"GetStringMap":       viper.GetStringMap,
				"GetStringMapString": viper.GetStringMapString,
				"GetTime":            viper.GetTime,
				"GetDuration":        viper.GetDuration,
				"IsSet":              viper.IsSet,
			},
		).ParseFiles(
			source,
		)

		if err != nil {
			fmt.Println("Failed to read template")

			if debug {
				fmt.Println(err)
			}

			os.Exit(1)
		}

		exe := tmpl.Execute(
			os.Stdout,
			&Payload{
				Prefix: prefix,
				Source: source,
				Output: output,
				Force:  force,
				Debug:  debug,
			},
		)

		if exe != nil {
			fmt.Println("Failed to parse template")

			if debug {
				fmt.Println(exe)
			}

			os.Exit(2)
		}
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>// Package main initializes a web server.
package main

import (
	"net/http"
	_ "net/http/pprof" // import for side effects

	"github.com/hack4impact/transcribe4all/web"
)

func main() {
	router := web.NewRouter()
	middlewareRouter := web.ApplyMiddleware(router)

	// serve http
	http.Handle("/", middlewareRouter)
	http.Handle("/semantic/", http.FileServer(http.Dir(".")))
	http.ListenAndServe(":8080", nil)

}
<commit_msg>Serving static folder<commit_after>// Package main initializes a web server.
package main

import (
	"net/http"
	_ "net/http/pprof" // import for side effects

	"github.com/hack4impact/transcribe4all/web"
)

func main() {
	router := web.NewRouter()
	middlewareRouter := web.ApplyMiddleware(router)

	// serve http
	http.Handle("/", middlewareRouter)
	http.Handle("/static/", http.FileServer(http.Dir(".")))
	http.ListenAndServe(":8080", nil)

}
<|endoftext|>


<commit_before>package main;
import (
	"net/http"
	ug "github.com/boivie/undergang/app"
	"github.com/codegangsta/cli"
	"log"
	"os"
	"encoding/json"
	"io/ioutil"
)

func main() {
	app := cli.NewApp()
	app.Name = "undergang"
	app.Flags = []cli.Flag {
		cli.StringFlag{
			Name: "listen",
			Value: ":8002",
			Usage: "Listening address (ip and port)",
		},
		cli.StringFlag{
			Name: "pathinfo",
			Usage: "URL for pathinfo service",
		},
		cli.StringFlag{
			Name: "access",
			Usage: "URL for access service",
		},
		cli.StringFlag{
			Name: "sshproxy",
			Usage: "Optional utility for proxying SSH connections",
		},
		cli.StringFlag{
			Name: "config",
			Usage: "Configuration file",
		},

	}
	app.Action = func(c *cli.Context) {
		ug.Init(c.String("pathinfo"), c.String("access"), c.String("sshproxy"))

		if c.String("config") != "" {
			var config struct {
				Paths []ug.PathInfo
			}
			buf, err := ioutil.ReadFile(c.String("config"))
			if err != nil {
				panic(err)
			}
			if err = json.Unmarshal(buf, &config); err != nil {
				panic(err)
			}
			for _, path := range config.Paths {
				log.Printf("Adding path mapping \"%s\"\n", path.Prefix)
				ug.AddPath(path)
			}
		}

		log.Printf("Accepting requests on %s\n", c.String("listen"))
		http.HandleFunc("/", ug.Forward)
		err := http.ListenAndServe(c.String("listen"), nil)
		if err != nil {
			panic(err)
		}
	}

	app.Run(os.Args)
}<commit_msg>gofmt<commit_after>package main;
import (
	"net/http"
	ug "github.com/boivie/undergang/app"
	"github.com/codegangsta/cli"
	"log"
	"os"
	"encoding/json"
	"io/ioutil"
)

func main() {
	app := cli.NewApp()
	app.Name = "undergang"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name: "listen",
			Value: ":8002",
			Usage: "Listening address (ip and port)",
		},
		cli.StringFlag{
			Name: "pathinfo",
			Usage: "URL for pathinfo service",
		},
		cli.StringFlag{
			Name: "access",
			Usage: "URL for access service",
		},
		cli.StringFlag{
			Name: "sshproxy",
			Usage: "Optional utility for proxying SSH connections",
		},
		cli.StringFlag{
			Name: "config",
			Usage: "Configuration file",
		},

	}
	app.Action = func(c *cli.Context) {
		ug.Init(c.String("pathinfo"), c.String("access"), c.String("sshproxy"))

		if c.String("config") != "" {
			var config struct {
				Paths []ug.PathInfo
			}
			buf, err := ioutil.ReadFile(c.String("config"))
			if err != nil {
				panic(err)
			}
			if err = json.Unmarshal(buf, &config); err != nil {
				panic(err)
			}
			for _, path := range config.Paths {
				log.Printf("Adding path mapping \"%s\"\n", path.Prefix)
				ug.AddPath(path)
			}
		}

		log.Printf("Accepting requests on %s\n", c.String("listen"))
		http.HandleFunc("/", ug.Forward)
		err := http.ListenAndServe(c.String("listen"), nil)
		if err != nil {
			panic(err)
		}
	}

	app.Run(os.Args)
}<|endoftext|>