<commit_before>package main

import (
	"babelweb2/parser"
	"babelweb2/ws"
	"bufio"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"sync"
	"time"
)

func flagsInit(node *string, bwPort *string) {
	var host, port, tempPort string
	flag.StringVar(&host, "h", "::1", "hostname (shorthand)")
	flag.StringVar(&port, "p", "33123", "port (shorthand)")
	flag.StringVar(&host, "host", "::1", "hostname")
	flag.StringVar(&port, "port", "33123", "port")
	flag.StringVar(&tempPort, "b", "8080", "babelweb Port (shorthanf)")
	flag.StringVar(&tempPort, "bwport", "8080", "babelweb Port ")
	flag.Parse()
	*node = "[" + host + "]:" + port
	*bwPort = ":" + tempPort
}

func Connection(updates chan parser.BabelUpdate, node string, wg *sync.WaitGroup) {
	var conn net.Conn
	var err error
	wg.Add(1)
	for {
		log.Println("	Trying ", node)
		for {
			conn, err = net.Dial("tcp6", node)
			if err != nil {
				log.Println(err)
				time.Sleep(time.Second * 5)
			} else {
				break
			}
		}
		log.Println("	Connected to", node)
		fmt.Fprintf(conn, "monitor\n")
		r := bufio.NewReader(conn)
		s := parser.NewScanner(r)
		err = ws.Db.Bd.Listen(s, updates)
		conn.Close()
		log.Println("Connection closed")
		if err != nil {
			log.Println(err)
			wg.Done()
			return
		}
		wg.Done()
	}
}

func main() {
	log.Println("	--------launching server--------")
	var node string
	var bwPort string
	flagsInit(&node, &bwPort)
	var wg sync.WaitGroup
	updates := make(chan parser.BabelUpdate, ws.ChanelSize)
	ws.Db.Bd = parser.NewBabelDesc()
	log.Println("connection to initial node:")
	go Connection(updates, node, &wg)
	bcastGrp := ws.NewListenerGroupe()
	go ws.MCUpdates(updates, bcastGrp, &wg)
	ws := ws.Handler(bcastGrp)
	http.Handle("/", http.FileServer(http.Dir("static/")))
	http.Handle("/style.css", http.FileServer(http.Dir("static/css/")))
	http.Handle("/initialize.js", http.FileServer(http.Dir("static/js")))
	http.Handle("/d3/d3.js", http.FileServer(http.Dir("static/js")))
	http.Handle("/ws", ws)
	err := http.ListenAndServe(bwPort, nil)
	if err != nil {
		log.Println(err)
		return
	}

	wg.Wait()
}
<commit_msg>Some minor changes<commit_after>package main

import (
	"babelweb2/parser"
	"babelweb2/ws"
	"bufio"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"sync"
	"time"
)

func flagsInit(node *string, bwPort *string) {
	var host, port, tempPort string
	flag.StringVar(&host, "h", "::1", "hostname (shorthand)")
	flag.StringVar(&port, "p", "33123", "port (shorthand)")
	flag.StringVar(&host, "host", "::1", "hostname")
	flag.StringVar(&port, "port", "33123", "port")
	flag.StringVar(&tempPort, "b", "8080", "babelweb Port (shorthand)")
	flag.StringVar(&tempPort, "bwport", "8080", "babelweb Port")
	flag.Parse()
	*node = "[" + host + "]:" + port
	*bwPort = ":" + tempPort
}

func Connection(updates chan parser.BabelUpdate, node string,
	wg *sync.WaitGroup) {
	var conn net.Conn
	var err error
	wg.Add(1)
	for {
		log.Println("	Trying ", node)
		for {
			conn, err = net.Dial("tcp6", node)
			if err != nil {
				log.Println(err)
				time.Sleep(time.Second * 5)
			} else {
				break
			}
		}
		log.Println("	Connected to", node)
		fmt.Fprintf(conn, "monitor\n")
		r := bufio.NewReader(conn)
		s := parser.NewScanner(r)
		err = ws.Db.Bd.Listen(s, updates)
		conn.Close()
		log.Println("Connection closed")
		if err != nil {
			log.Println(err)
			wg.Done()
			return
		}
	}
	wg.Done()
}

func main() {
	log.Println("	--------launching server--------")
	var node string
	var bwPort string
	flagsInit(&node, &bwPort)
	var wg sync.WaitGroup
	updates := make(chan parser.BabelUpdate, ws.ChanelSize)
	ws.Db.Bd = parser.NewBabelDesc()
	log.Println("connection to initial node:")
	go Connection(updates, node, &wg)
	bcastGrp := ws.NewListenerGroupe()
	go ws.MCUpdates(updates, bcastGrp, &wg)
	ws := ws.Handler(bcastGrp)
	http.Handle("/", http.FileServer(http.Dir("static/")))
	http.Handle("/style.css", http.FileServer(http.Dir("static/css/")))
	http.Handle("/initialize.js", http.FileServer(http.Dir("static/js")))
	http.Handle("/d3/d3.js", http.FileServer(http.Dir("static/js")))
	http.Handle("/ws", ws)
	err := http.ListenAndServe(bwPort, nil)
	if err != nil {
		log.Println(err)
		return
	}

	wg.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"io"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"

	"github.com/michaelmacinnis/oh/internal/common"
	"github.com/michaelmacinnis/oh/internal/common/interface/cell"
	"github.com/michaelmacinnis/oh/internal/common/type/list"
	"github.com/michaelmacinnis/oh/internal/common/type/pair"
	"github.com/michaelmacinnis/oh/internal/common/type/str"
	"github.com/michaelmacinnis/oh/internal/common/type/sym"
	"github.com/michaelmacinnis/oh/internal/engine"
	"github.com/michaelmacinnis/oh/internal/reader"
	"github.com/michaelmacinnis/oh/internal/system/history"
	"github.com/michaelmacinnis/oh/internal/system/job"
	"github.com/michaelmacinnis/oh/internal/system/options"
	"github.com/michaelmacinnis/oh/internal/system/process"
	"github.com/peterh/liner"
)

//nolint:gochecknoglobals
var (
	pathListSeparator = string(os.PathListSeparator)
	pathSeparator     = string(os.PathSeparator)
)

func clean(s string) string {
	if s == "." || s == pathSeparator+"." {
		return s
	}

	head, tail := split(s)
	if tail == s {
		head, tail = tail, head
	}

	return filepath.Clean(head) + tail
}

func command() bool {
	if options.Command() == "" {
		return false
	}

	r := reader.New(os.Args[0])

	c, err := r.Scan(options.Command() + "\n")
	if err != nil {
		println("problem parsing command:", err.Error())
		os.Exit(1)
	}

	if c == nil {
		println("incomplete command")
		os.Exit(1)
	}

	engine.Evaluate(job.New(process.Group()), c)

	return true
}

func completer(r **reader.T) func(s string, n int) (h string, cs []string, t string) {
	return func(s string, n int) (h string, cs []string, t string) {
		h = s[:n]
		t = s[n:]

		completing := h[strings.LastIndex(h, " ")+1:]

		defer func() {
			r := recover()
			if r == nil {
				return
			}

			cs = []string{}
		}()

		lc := (*r).Lexer().Copy()

		lc.Scan(h)

		lp := (*r).Parser().Copy(func(_ cell.I) {}, lc.Token)

		_ = lp.Parse()

		cs = lc.Expected()
		if len(cs) != 0 {
			return
		}

		// Ensure line == prefix + completing + tail
		prefix := h[0 : len(h)-len(completing)]

		cwd := engine.Resolve("PWD")
		home := engine.Resolve("HOME")

		cmd := lp.Current()
		if cmd == pair.Null {
			if completing == "" {
				cs = []string{"    "}

				return
			}

			cs = files(cwd, home, engine.Resolve("PATH"), completing)
		} else {
			cs = files(cwd, home, engine.Resolve("PWD"), completing)
		}

		if len(cs) == 0 {
			return prefix, []string{completing}, t
		}

		unique := make(map[string]bool)
		for _, completion := range cs {
			unique[completion] = true
		}

		cs = make([]string, 0, len(unique))
		for completion := range unique {
			cs = append(cs, completion)
		}

		sort.Strings(cs)

		if len(cs) == 1 && !strings.HasSuffix(cs[0], "/") {
			cs[0] = cs[0] + " "
		}

		return prefix, cs, t
	}
}

func directories(s string) []string {
	dirs := []string{}

	for _, dir := range strings.Split(s, pathListSeparator) {
		if dir == "" {
			dir = "."
		} else {
			dir = filepath.Clean(dir)
		}

		stat, err := os.Stat(dir)
		if err != nil || !stat.IsDir() {
			continue
		}

		dirs = append(dirs, dir)
	}

	return dirs
}

func expand(cwd, home, candidate string) (string, bool) {
	dotdir := false
	prefix := candidate + "   "

	switch {
	case candidate == "":
		// Leave it as is.

	case prefix[0:2] == "./" || prefix[0:3] == "../":
		candidate = join(cwd, candidate)
		dotdir = true

	case prefix[0:1] == "~":
		candidate = join(home, candidate[1:])

	default:
		candidate = clean(candidate)
	}

	return candidate, dotdir
}

func files(cwd, home, paths, word string) []string {
	candidate, dotdir := expand(cwd, home, word)

	candidates := []string{candidate}
	if !path.IsAbs(candidate) && !dotdir {
		candidates = directories(paths)
		for k, v := range candidates {
			candidates[k] = v + pathSeparator + candidate
		}
	}

	return matches(candidates, word)
}

func interactive() bool {
	if !options.Interactive() {
		return false
	}

	name := options.Args()[0]

	err := process.BecomeForegroundGroup()
	if err != nil {
		println(err.Error())

		return false
	}

	// We assume the terminal starts in cooked mode.
	cooked, err := liner.TerminalMode()
	if err != nil {
		println(err.Error())

		return false
	}

	// Restore terminal state when we exit.
	defer func() {
		err := cooked.ApplyMode()
		if err != nil {
			println(err.Error())
		}
	}()

	cli := liner.NewLiner()

	cli.SetCtrlCAborts(true)

	uncooked, err := liner.TerminalMode()
	if err != nil {
		println(err.Error())

		return false
	}

	err = history.Load(cli.ReadHistory)
	if err != nil {
		println(err.Error())
	}

	defer func() {
		err = history.Save(cli.WriteHistory)
		if err != nil {
			println(err.Error())
		}

		_, _ = os.Stdout.Write([]byte{'\n'})
	}()

	err = repl(cli, cooked, uncooked, name)
	if !errors.Is(err, io.EOF) {
		println(err.Error())
	}

	return true
}

func join(s ...string) string {
	last := len(s) - 1
	head, tail := split(s[last])
	s[last] = head

	return filepath.Join(s...) + tail
}

func matches(candidates []string, word string) []string {
	completions := []string{}

	for _, candidate := range candidates {
		dirname, basename := filepath.Split(candidate)

		if skip(dirname, basename) {
			continue
		}

		max := strings.Count(dirname, pathSeparator)

		_ = filepath.Walk(dirname, func(p string, i os.FileInfo, err error) error {
			depth := strings.Count(p, pathSeparator)
			if depth > max {
				if i.IsDir() {
					return filepath.SkipDir
				}

				return nil
			} else if depth < max {
				return nil
			}

			if candidate != pathSeparator && len(basename) == 0 {
				if p == dirname {
					return nil
				}
			} else if !strings.HasPrefix(p, candidate) {
				return nil
			}

			if p != pathSeparator && i.IsDir() {
				p += pathSeparator
			}

			if len(candidate) > len(p) {
				return nil
			}

			s := strings.Index(p, candidate) + len(candidate)
			completion := word + p[s:]
			completions = append(completions, completion)

			return nil
		})
	}

	return completions
}

func skip(dirname, basename string) bool {
	stat, err := os.Stat(dirname)
	if err != nil {
		return true
	} else if len(basename) == 0 && !stat.IsDir() {
		return true
	}

	return false
}

func split(s string) (head, tail string) {
	head = s
	tail = ""

	index := strings.LastIndex(s, pathSeparator)
	if index > -1 {
		head = s[:index]
		tail = s[index:]
	}

	return
}

func repl(cli *liner.State, cooked, uncooked liner.ModeApplier, name string) error {
	j := job.New(0)
	r := reader.New(name)

	cli.SetWordCompleter(completer(&r))
	cli.SetTabCompletionStyle(liner.TabPrints)

	continued := str.New("  ")
	initial := str.New(": ")
	suffix := initial

	for {
		v, _ := engine.System(j, list.New(sym.New("prompt"), suffix))
		p := common.String(v)

		err := uncooked.ApplyMode()
		if err != nil {
			return err
		}

		line, err := cli.Prompt(p)
		if err != nil {
			if errors.Is(err, liner.ErrPromptAborted) {
				r.Close()
				r = reader.New(name)
				suffix = initial

				continue
			} else {
				return err
			}
		}

		err = cooked.ApplyMode()
		if err != nil {
			return err
		}

		if line == "" {
			continue
		}

		cli.AppendHistory(line)
		j.Append(line)

		suffix = continued

		c, err := r.Scan(line + "\n")
		if err != nil {
			println(err.Error())

			r.Close()
			r = reader.New(name)
			suffix = initial

			continue
		}

		if c != nil {
			engine.Evaluate(j, c)

			j = job.New(0)

			process.RestoreForegroundGroup()

			suffix = initial
		}
	}
}

func main() {
	options.Parse()

	engine.Boot(options.Script(), options.Args())

	if !command() && !interactive() {
		println("unexpected error")
	}
}

//go:generate ./oh bin/test.oh
//go:generate ./oh bin/doc.oh manual ../doc/manual.md
//go:generate ./oh bin/doc.oh readme ../README.md
//go:generate go generate github.com/michaelmacinnis/oh/internal/engine/boot
//go:generate go generate github.com/michaelmacinnis/oh/internal/common/type/chn
//go:generate go generate github.com/michaelmacinnis/oh/internal/common/type/env
//go:generate go generate github.com/michaelmacinnis/oh/internal/common/type/num
//go:generate go generate github.com/michaelmacinnis/oh/internal/common/type/obj
//go:generate go generate github.com/michaelmacinnis/oh/internal/common/type/pair
//go:generate go generate github.com/michaelmacinnis/oh/internal/common/type/pipe
//go:generate go generate github.com/michaelmacinnis/oh/internal/common/type/status
//go:generate go generate github.com/michaelmacinnis/oh/internal/common/type/str
<commit_msg>Improving completion.<commit_after>package main

import (
	"errors"
	"io"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"

	"github.com/michaelmacinnis/oh/internal/common"
	"github.com/michaelmacinnis/oh/internal/common/interface/cell"
	"github.com/michaelmacinnis/oh/internal/common/type/list"
	"github.com/michaelmacinnis/oh/internal/common/type/pair"
	"github.com/michaelmacinnis/oh/internal/common/type/str"
	"github.com/michaelmacinnis/oh/internal/common/type/sym"
	"github.com/michaelmacinnis/oh/internal/engine"
	"github.com/michaelmacinnis/oh/internal/reader"
	"github.com/michaelmacinnis/oh/internal/system/history"
	"github.com/michaelmacinnis/oh/internal/system/job"
	"github.com/michaelmacinnis/oh/internal/system/options"
	"github.com/michaelmacinnis/oh/internal/system/process"
	"github.com/peterh/liner"
)

//nolint:gochecknoglobals
var (
	pathListSeparator = string(os.PathListSeparator)
	pathSeparator     = string(os.PathSeparator)
)

func clean(s string) string {
	if s == "." || s == pathSeparator+"." {
		return s
	}

	head, tail := split(s)
	if tail == s {
		head, tail = tail, head
	}

	return filepath.Clean(head) + tail
}

func command() bool {
	if options.Command() == "" {
		return false
	}

	r := reader.New(os.Args[0])

	c, err := r.Scan(options.Command() + "\n")
	if err != nil {
		println("problem parsing command:", err.Error())
		os.Exit(1)
	}

	if c == nil {
		println("incomplete command")
		os.Exit(1)
	}

	engine.Evaluate(job.New(process.Group()), c)

	return true
}

func completer(r **reader.T) func(s string, n int) (h string, cs []string, t string) {
	return func(s string, n int) (h string, cs []string, t string) {
		h = s[:n]
		t = s[n:]

		completing := h[strings.LastIndex(h, " ")+1:]

		defer func() {
			r := recover()
			if r == nil {
				return
			}

			cs = []string{}
		}()

		lc := (*r).Lexer().Copy()

		lc.Scan(h)

		lp := (*r).Parser().Copy(func(_ cell.I) {}, lc.Token)

		_ = lp.Parse()

		cs = lc.Expected()
		if len(cs) != 0 {
			return
		}

		// Ensure line == prefix + completing + tail
		prefix := h[0 : len(h)-len(completing)]

		cwd := engine.Resolve("PWD")
		home := engine.Resolve("HOME")

		cmd := lp.Current()
		if cmd == pair.Null {
			if completing == "" {
				cs = []string{"    "}

				return
			}

			cs = files(cwd, home, engine.Resolve("PATH"), completing)
		} else {
			cs = files(cwd, home, engine.Resolve("PWD"), completing)
		}

		if len(cs) == 0 {
			return prefix, []string{completing}, t
		}

		unique := make(map[string]bool)
		for _, completion := range cs {
			unique[completion] = true
		}

		cs = make([]string, 0, len(unique))
		for completion := range unique {
			cs = append(cs, completion)
		}

		sort.Strings(cs)

		if len(cs) == 1 && !strings.HasSuffix(cs[0], "/") {
			cs[0] = cs[0] + " "
		}

		return prefix, cs, t
	}
}

func directories(s string) []string {
	dirs := []string{}

	for _, dir := range strings.Split(s, pathListSeparator) {
		if dir == "" {
			dir = "."
		} else {
			dir = filepath.Clean(dir)
		}

		stat, err := os.Stat(dir)
		if err != nil || !stat.IsDir() {
			continue
		}

		dirs = append(dirs, dir)
	}

	return dirs
}

func expand(cwd, home, candidate string) (string, bool) {
	dotdir := false
	prefix := candidate + "   "

	switch {
	case candidate == "":
		// Leave it as is.

	case prefix[0:2] == "./" || prefix[0:3] == "../":
		candidate = join(cwd, candidate)
		dotdir = true

	case prefix[0:1] == "~":
		candidate = join(home, candidate[1:])

	default:
		candidate = clean(candidate)
	}

	return candidate, dotdir
}

func files(cwd, home, paths, word string) []string {
	candidate, dotdir := expand(cwd, home, word)

	candidates := []string{candidate}
	if !path.IsAbs(candidate) && !dotdir {
		candidates = directories(paths)
		for k, v := range candidates {
			candidates[k] = v + pathSeparator + candidate
		}
	}

	return matches(candidates, word)
}

func interactive() bool {
	if !options.Interactive() {
		return false
	}

	name := options.Args()[0]

	err := process.BecomeForegroundGroup()
	if err != nil {
		println(err.Error())

		return false
	}

	// We assume the terminal starts in cooked mode.
	cooked, err := liner.TerminalMode()
	if err != nil {
		println(err.Error())

		return false
	}

	// Restore terminal state when we exit.
	defer func() {
		err := cooked.ApplyMode()
		if err != nil {
			println(err.Error())
		}
	}()

	cli := liner.NewLiner()

	cli.SetCtrlCAborts(true)

	uncooked, err := liner.TerminalMode()
	if err != nil {
		println(err.Error())

		return false
	}

	err = history.Load(cli.ReadHistory)
	if err != nil {
		println(err.Error())
	}

	defer func() {
		err = history.Save(cli.WriteHistory)
		if err != nil {
			println(err.Error())
		}

		_, _ = os.Stdout.Write([]byte{'\n'})
	}()

	err = repl(cli, cooked, uncooked, name)
	if !errors.Is(err, io.EOF) {
		println(err.Error())
	}

	return true
}

func join(s ...string) string {
	last := len(s) - 1
	head, tail := split(s[last])
	s[last] = head

	return filepath.Join(s...) + tail
}

func matches(candidates []string, word string) []string {
	completions := []string{}

	for _, candidate := range candidates {
		dirname, basename := filepath.Split(candidate)

		if skip(dirname, basename) {
			continue
		}

		max := strings.Count(dirname, pathSeparator)

		_ = filepath.Walk(dirname, func(p string, i os.FileInfo, err error) error {
			depth := strings.Count(p, pathSeparator)
			if depth > max {
				if i.IsDir() {
					return filepath.SkipDir
				}

				return nil
			} else if depth < max {
				return nil
			}

			if candidate != pathSeparator && len(basename) == 0 {
				if p == dirname {
					return nil
				} else {
					suffix := strings.TrimPrefix(p, dirname)
					if strings.HasPrefix(suffix, ".") {
						return nil
					}
				}
			} else if !strings.HasPrefix(p, candidate) {
				return nil
			}

			if p != pathSeparator && i.IsDir() {
				p += pathSeparator
			}

			if len(candidate) > len(p) {
				return nil
			}

			s := strings.Index(p, candidate) + len(candidate)
			completion := word + p[s:]
			completions = append(completions, completion)

			return nil
		})
	}

	return completions
}

func skip(dirname, basename string) bool {
	stat, err := os.Stat(dirname)
	if err != nil {
		return true
	} else if len(basename) == 0 && !stat.IsDir() {
		return true
	}

	return false
}

func split(s string) (head, tail string) {
	head = s
	tail = ""

	index := strings.LastIndex(s, pathSeparator)
	if index > -1 {
		head = s[:index]
		tail = s[index:]
	}

	return
}

func repl(cli *liner.State, cooked, uncooked liner.ModeApplier, name string) error {
	j := job.New(0)
	r := reader.New(name)

	cli.SetWordCompleter(completer(&r))
	cli.SetTabCompletionStyle(liner.TabPrints)

	continued := str.New("  ")
	initial := str.New(": ")
	suffix := initial

	for {
		v, _ := engine.System(j, list.New(sym.New("prompt"), suffix))
		p := common.String(v)

		err := uncooked.ApplyMode()
		if err != nil {
			return err
		}

		line, err := cli.Prompt(p)
		if err != nil {
			if errors.Is(err, liner.ErrPromptAborted) {
				r.Close()
				r = reader.New(name)
				suffix = initial

				continue
			} else {
				return err
			}
		}

		err = cooked.ApplyMode()
		if err != nil {
			return err
		}

		if line == "" {
			continue
		}

		cli.AppendHistory(line)
		j.Append(line)

		suffix = continued

		c, err := r.Scan(line + "\n")
		if err != nil {
			println(err.Error())

			r.Close()
			r = reader.New(name)
			suffix = initial

			continue
		}

		if c != nil {
			engine.Evaluate(j, c)

			j = job.New(0)

			process.RestoreForegroundGroup()

			suffix = initial
		}
	}
}

func main() {
	options.Parse()

	engine.Boot(options.Script(), options.Args())

	if !command() && !interactive() {
		println("unexpected error")
	}
}

//go:generate ./oh bin/test.oh
//go:generate ./oh bin/doc.oh manual ../doc/manual.md
//go:generate ./oh bin/doc.oh readme ../README.md
//go:generate go generate github.com/michaelmacinnis/oh/internal/engine/boot
//go:generate go generate github.com/michaelmacinnis/oh/internal/common/type/chn
//go:generate go generate github.com/michaelmacinnis/oh/internal/common/type/env
//go:generate go generate github.com/michaelmacinnis/oh/internal/common/type/num
//go:generate go generate github.com/michaelmacinnis/oh/internal/common/type/obj
//go:generate go generate github.com/michaelmacinnis/oh/internal/common/type/pair
//go:generate go generate github.com/michaelmacinnis/oh/internal/common/type/pipe
//go:generate go generate github.com/michaelmacinnis/oh/internal/common/type/status
//go:generate go generate github.com/michaelmacinnis/oh/internal/common/type/str
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"os"
)

func main() {
	orchestrateStreaming()
}

func orchestrateStreaming() {
	from := "wiki-100k.txt"
	to := "words.go"
	err := streamWords(from, to)
	if err != nil {
		fmt.Println("Stream failed:", err)
	}
}

func streamWords(fromFilename, toFilename string) error {

	// Read
	from, err := os.Open(fromFilename)
	defer from.Close()
	if err != nil {
		return errors.New("Error opening " + fromFilename + ": " + err.Error())
	}
	words := make(chan string)
	errCh := make(chan error)
	go func() {
		seen := make(map[string]bool)
		scanner := bufio.NewScanner(from)
		for scanner.Scan() {
			word := scanner.Text()
			if validWord(word) {
				if !seen[word] { // ignore duplicate entries
					seen[word] = true
					words <- word
				}
			}
		}
		close(words)
		if err := scanner.Err(); err != nil {
			errCh <- errors.New("Error reading from " + fromFilename + ": " + err.Error())
		}
		close(errCh)
	}()

	// Write
	to, err := os.Create(toFilename)
	defer to.Close()
	if err != nil {
		return errors.New("Error creating " + toFilename + ": " + err.Error())
	}

	// Write Header
	header := "package main\n\n"
	header += "type alphabet map[string]bool\n\n"
	header += "var englishWords = alphabet{"
	n, err := to.WriteString(header)
	if err != nil {
		return errors.New("Error writing header to " + toFilename + ". Wrote " + fmt.Sprint(n) + " bytes before error: " + err.Error())
	}

	// Write Body
	for {
		var open bool
		var word string

		select {
		case word, open = <-words: // includes word in map literal
			n, err = to.WriteString("\"" + word + "\": true, ")
			if err != nil {
				return errors.New("Error writing " + word + " to " + toFilename + ". Wrote " + fmt.Sprint(n) + " bytes before error: " + err.Error())
			}
			if !open {
				words = nil // mark for no additional recieves
			}
		case err, open = <-errCh: // interrupt
			if err != nil {
				break
			}
			if !open {
				errCh = nil // mark for no additional recieves
			}
		}

		if words == nil && errCh == nil {
			// both channels are nil (therefore they were closed)
			// must exit now before select blocks forever
			break
		}

	}

	if err != nil {
		return errors.New("Error reading from " + fromFilename + ": " + err.Error())
	}

	// Write Footer
	footer := "}"
	n, err = to.WriteString(footer)
	if err != nil {
		return errors.New("Error writing footer to " + toFilename + ". Wrote " + fmt.Sprint(n) + " bytes before error: " + err.Error())
	}

	return nil
}

func validWord(candidate string) bool {
	return len(candidate) > 0 && candidate[0] != '#' && candidate[0] != ' '
}
<commit_msg>Parallelizes the processing step<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
	"sync"
)

type wordMember struct {
	inWord  string
	outWord string
}

func transcribe(inFile, outFile string) error {

	var outWait sync.WaitGroup

	errCh := make(chan error)

	in := make(chan wordMember)

	outWait.Add(1)
	go readFile(inFile, in, errCh, &outWait)

	out := make(chan wordMember)
	for i := 0; i < 1000; i++ {
		outWait.Add(1)
		go func() {
			defer outWait.Done()
			for w := range in {
				w.process()
				out <- w
			}
		}()
	}

	go func() {
		outWait.Wait()
		close(out)
	}()

	go writeFile(outFile, out, errCh)

	// Get Read and Write errors
	for err := range errCh {
		if err != nil {
			return err
		}
	}

	return nil

}

func readFile(filename string, in chan wordMember, errCh chan error, wg *sync.WaitGroup) {

	defer wg.Done()
	defer close(in)

	from, err := os.Open(filename)
	defer from.Close()

	if err != nil {
		errCh <- fmt.Errorf("Error opening " + filename + ": " + err.Error())
		return
	}
	seen := make(map[string]bool)
	scanner := bufio.NewScanner(from)
	for scanner.Scan() {
		word := scanner.Text()
		if !seen[word] { // ignore duplicate entries
			seen[word] = true
			in <- wordMember{inWord: word}
		}
	}
	if err := scanner.Err(); err != nil {
		errCh <- fmt.Errorf("Error reading from " + filename + ": " + err.Error())
	}
}

func (w *wordMember) process() {
	if validWord(w.inWord) {
		w.outWord = "\"" + w.inWord + "\": true, "
	}
}

func validWord(candidate string) bool {
	return len(candidate) > 0 && candidate[0] != '#' && candidate[0] != ' '
}

func writeFile(filename string, out chan wordMember, errCh chan error) {
	// Write
	to, err := os.Create(filename)
	defer to.Close()
	if err != nil {
		errCh <- fmt.Errorf("Error creating " + filename + ": " + err.Error())
		return
	}

	// Write Header
	header := "package main\n\n"
	header += "type alphabet map[string]bool\n\n"
	header += "var englishWords = alphabet{"
	n, err := to.WriteString(header)
	if err != nil {
		errCh <- fmt.Errorf("Error writing header to " + filename + ". Wrote " + fmt.Sprint(n) + " bytes before error: " + err.Error())
		return
	}

	// Write Body
	for word := range out {
		n, err = to.WriteString(word.outWord)
		if err != nil {
			errCh <- fmt.Errorf("Error writing " + word.outWord + " to " + filename + ". Wrote " + fmt.Sprint(n) + " bytes before error: " + err.Error())
			return
		}
	}

	// Write Footer
	footer := "}"
	n, err = to.WriteString(footer)
	if err != nil {
		errCh <- fmt.Errorf("Error writing footer to " + filename + ". Wrote " + fmt.Sprint(n) + " bytes before error: " + err.Error())
		return
	}

	close(errCh)
}

func main() {
	err := transcribe("wiki-100k.txt", "words.go")
	if err != nil {
		fmt.Println("Error transcribing", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	// "fmt"
	// "log"
	// "net"
	// "net/http"
	"crypto/md5"
	"encoding/hex"
	// "net/http/httputil"
)

func main() {
	// empty := md5.Sum([]byte(""))
	// log.Println(empty, hex.EncodeToString([]byte(empty)))
}

func GetMd5String(s string) string {
	h := md5.New()
	h.Write([]byte(s))
	return hex.EncodeToString(h.Sum(nil))
}
<commit_msg>some func~<commit_after>package main

import (
	"fmt"
	"log"
	// "net"
	// "net/http"
	"crypto/md5"
	"encoding/hex"
	"os"
	"path"
	"strconv"
	// "net/http/httputil"
	"github.com/cznic/exp/dbm"
)

const REVERSE_DIR = "reverse"
const SPLIT_SIZE = 2
const PRINTABLE_START = ' '
const PRINTABLE_END = '~'

const ASCII_PRINTABLE_CHARACTERS = ` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_` + "`" + `abcdefghijklmnopqrstuvwxyz{|}~`
const ASCII_PRINTABLE_CHARACTERS_LENGTH = len(ASCII_PRINTABLE_CHARACTERS)

func next(current string) string {
	currentLen := len(current)
	if currentLen == 0 {
		// init
		return string(PRINTABLE_START)
	} else {
		pre, last := current[:currentLen-1], current[currentLen-1]
		if last == PRINTABLE_END {
			return next(pre) + string(PRINTABLE_START)
		} else {
			return pre + string(last+1)
		}
	}
}
func getPath(prefix, hash string) string {
	if len(hash) < SPLIT_SIZE {
		panic("hash string length < " + strconv.Itoa(SPLIT_SIZE))
	} else {
		return path.Join(prefix, hash[:SPLIT_SIZE])
	}
}

func mkdirs(path string) error {
	return os.MkdirAll(path, os.ModeDir)
}

func getMd5String(s string) string {
	h := md5.New()
	h.Write([]byte(s))
	return hex.EncodeToString(h.Sum(nil))
}

func main() {
	_ = log.Fatal
	_ = fmt.Println
	_ = dbm.ACIDNone
	current := ""
	size0 := 0
	size1 := 0
	size2 := 0
	size3 := 0
	for i := 0; i < 1000000; i++ {
		// md5 := getMd5String(current)
		// dir1 := getPath(REVERSE_DIR, md5)
		// log.Println(current, md5, dir1)
		// fmt.Printf(`"%s", `, current)
		size := len(current)
		switch size {
		case 0:
			size0++
		case 1:
			size1++
		case 2:
			size2++
		case 3:
			size3++
		}
		if current == "eggy" {
			fmt.Println(current, i)
		}
		current = next(current)
	}
	log.Println(size0, size1, size2, size3)
}
<|endoftext|>


<commit_before>/*
Quick and simple monitoring system

Usage:
 jsonmon config.yml
 jsonmon -v # Prints version to stdout and exits

Environment:
 HOST
  - defaults to localhost
  - the JSON API network interface
 PORT
  - defaults to 3000
  - the JSON API port
*/
package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"gopkg.in/yaml.v2"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"regexp"
	"runtime"
	"strconv"
	"sync"
	"syscall"
	"time"
)

// Application version.
const Version = "2.0.5"

// This one is for internal use.
type ver struct {
	App  string `json:"jsonmon"`
	Go   string `json:"runtime"`
	Os   string `json:"os"`
	Arch string `json:"arch"`
}

var version ver

// Check details.
type Check struct {
	Name   string      `json:"name,omitempty" yaml:"name"`
	Web    string      `json:"web,omitempty" yaml:"web"`
	Shell  string      `json:"shell,omitempty" yaml:"shell"`
	Match  string      `json:"-" yaml:"match"`
	Return int         `json:"-" yaml:"return"`
	Notify interface{} `json:"-" yaml:"notify"`
	Alert  interface{} `json:"-" yaml:"alert"`
	Tries  int         `json:"-" yaml:"tries"`
	Repeat int         `json:"-" yaml:"repeat"`
	Failed bool        `json:"failed" yaml:"-"`
	Since  string      `json:"since,omitempty" yaml:"-"`
}

// Global checks list. Need to share it with workers and Web UI.
var checks []Check

// Global started and last modified date for HTTP caching.
var modified string
var started string

var mutex *sync.RWMutex

// Construct the last modified string.
func etag(ts time.Time) string {
	return "W/\"" + strconv.FormatInt(ts.UnixNano(), 10) + "\""
}

// The main loop.
func main() {
	// Parse CLI args.
	usage := "Usage: " + path.Base(os.Args[0]) + " config.yml\n" +
		"Docs:  https://github.com/chillum/jsonmon/wiki"
	if len(os.Args) != 2 {
		fmt.Fprintln(os.Stderr, usage)
		os.Exit(1)
	}
	// -v for version.
	version.App = Version
	version.Go = runtime.Version()
	version.Os = runtime.GOOS
	version.Arch = runtime.GOARCH
	switch os.Args[1] {
	case "-h":
		fallthrough
	case "-help":
		fallthrough
	case "--help":
		fmt.Fprintln(os.Stderr, usage)
		os.Exit(0)
	case "-v":
		fallthrough
	case "-version":
		fallthrough
	case "--version":
		json, _ := json.Marshal(&version)
		fmt.Println(string(json))
		os.Exit(0)
	}
	// Tune concurrency.
	if os.Getenv("GOMAXPROCS") == "" {
		runtime.GOMAXPROCS(runtime.NumCPU())
	}
	// Read config file or exit with error.
	config, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		fmt.Fprint(os.Stderr, "<2>", err, "\n")
		os.Exit(3)
	}
	err = yaml.Unmarshal(config, &checks)
	if err != nil {
		fmt.Fprint(os.Stderr, "<2>", "invalid config at ", os.Args[1], "\n")
		os.Exit(3)
	}
	// Exit with return code 0 on kill.
	done := make(chan os.Signal, 1)
	signal.Notify(done, syscall.SIGTERM)
	go func() {
		<-done
		os.Exit(0)
	}()
	// Run checks.
	started = etag(time.Now())
	modified = started
	mutex = &sync.RWMutex{}
	for i := range checks {
		go worker(&checks[i])
	}
	// Launch the JSON API.
	// Probably should do it prior to checks.
	host := os.Getenv("HOST")
	port := os.Getenv("PORT")
	if host == "" {
		host = "localhost"
	}
	if port == "" {
		port = "3000"
	}
	http.HandleFunc("/status", getChecks)
	http.HandleFunc("/version", getVersion)
	http.HandleFunc("/", notFound)
	err = http.ListenAndServe(host+":"+port, nil)
	if err != nil {
		fmt.Fprint(os.Stderr, "<2>", err, "\n")
	}
	os.Exit(4)
}

// Background worker.
func worker(check *Check) {
	mutex.Lock()
	if check.Repeat == 0 { // Set default timeout.
		check.Repeat = 30
	}
	if check.Tries == 0 { // Default to 1 attempt.
		check.Tries = 1
	}
	mutex.Unlock()
	sleep := time.Second * time.Duration(check.Repeat)
	for {
		if check.Web != "" {
			web(check)
		}
		if check.Shell != "" {
			shell(check)
		}
		time.Sleep(sleep)
	}
}

// Shell worker.
func shell(check *Check) {
	// Set check's display name.
	var name string
	if check.Name != "" {
		name = check.Name
	} else {
		name = check.Shell
	}
	// Execute with shell in N attemps.
	var out []byte
	var err error
	for i := 0; i < check.Tries; i++ {
		out, err = exec.Command("/bin/sh", "-c", check.Shell).CombinedOutput()
		if err == nil {
			if check.Match != "" { // Match regexp.
				var regex *regexp.Regexp
				regex, err = regexp.Compile(check.Match)
				if err == nil && !regex.Match(out) {
					err = errors.New("Expected:\n" + check.Match + "\n\nGot:\n" + string(out))
				}
			}
			break
		}
	}
	// Process results.
	if err == nil {
		if check.Failed {
			ts := time.Now()
			mutex.Lock()
			check.Failed = false
			check.Since = ts.Format(time.RFC3339)
			modified = etag(ts)
			mutex.Unlock()
			notify(check.Notify, "Fixed: "+name, nil)
			alert(check, &name, nil)
		}
	} else {
		if !check.Failed {
			ts := time.Now()
			mutex.Lock()
			check.Failed = true
			check.Since = ts.Format(time.RFC3339)
			modified = etag(ts)
			mutex.Unlock()
			msg := string(out) + err.Error()
			notify(check.Notify, "Failed: "+name, &msg)
			alert(check, &name, &msg)
		}
	}
}

// Web worker.
func web(check *Check) {
	// Set check's display name.
	var name string
	if check.Name != "" {
		name = check.Name
	} else {
		name = check.Web
	}
	if check.Return == 0 { // Successful HTTP return code is 200.
		check.Return = 200
	}
	// Get the URL in N attempts.
	var err error
	for i := 0; i < check.Tries; i++ {
		err = fetch(check.Web, check.Match, check.Return)
		if err == nil {
			break
		}
	}
	// Process results.
	if err == nil {
		if check.Failed {
			ts := time.Now()
			mutex.Lock()
			check.Failed = false
			check.Since = ts.Format(time.RFC3339)
			modified = etag(ts)
			mutex.Unlock()
			notify(check.Notify, "Fixed: "+name, nil)
			alert(check, &name, nil)

		}
	} else {
		if !check.Failed {
			ts := time.Now()
			mutex.Lock()
			check.Failed = true
			check.Since = ts.Format(time.RFC3339)
			modified = etag(ts)
			mutex.Unlock()
			msg := err.Error()
			notify(check.Notify, "Failed: "+name, &msg)
			alert(check, &name, &msg)
		}
	}
}

// Check HTTP redirects.
func redirect(req *http.Request, via []*http.Request) error {
	// When redirects number > 10 probably there's a problem.
	if len(via) >= 10 {
		return errors.New("stopped after 10 redirects")
	}
	// Redirects don't get User-Agent.
	req.Header.Set("User-Agent", "jsonmon")
	return nil
}

// The actual HTTP GET.
func fetch(url string, match string, code int) error {
	var err error
	var resp *http.Response
	var req *http.Request
	client := &http.Client{}
	client.CheckRedirect = redirect
	req, err = http.NewRequest("GET", url, nil)
	if err == nil {
		req.Header.Set("User-Agent", "jsonmon")
		resp, err = client.Do(req)
		if err == nil {
			if resp.StatusCode != code { // Check status code.
				err = errors.New(url + " returned " + strconv.Itoa(resp.StatusCode))
			} else { // Match regexp.
				if resp != nil && match != "" {
					var regex *regexp.Regexp
					regex, err = regexp.Compile(match)
					if err == nil {
						var body []byte
						body, _ = ioutil.ReadAll(resp.Body)
						if !regex.Match(body) {
							err = errors.New("Expected:\n" + match + "\n\nGot:\n" + string(body))
						}
					}
				}
			}
		}
	}
	if resp != nil {
		resp.Body.Close()
	}
	return err
}

// Logs and mail alerting.
func notify(mail interface{}, subject string, message *string) {
	// Log the alerts.
	if message == nil {
		fmt.Print("<5>", subject, "\n")
	} else {
		fmt.Print("<5>", subject, "\n", *message, "\n")
	}
	// Mail the alerts.
	if mail != nil {
		// Make the message.
		var rcpt string
		var ok bool
		if rcpt, ok = mail.(string); !ok {
			for i, v := range mail.([]interface{}) {
				if i != 0 {
					rcpt += ", "
				}
				rcpt += v.(string)
			}
		}
		msg := "To: " + rcpt + "\nSubject: " + subject + "\nX-Mailer: jsonmon\n\n"
		if message != nil {
			msg += *message
		}
		msg += "\n.\n"
		// And send it.
		sendmail := exec.Command("/usr/sbin/sendmail", "-t")
		stdin, _ := sendmail.StdinPipe()
		err := sendmail.Start()
		if err != nil {
			fmt.Fprint(os.Stderr, "<3>", err, "\n")
		}
		io.WriteString(stdin, msg)
		sendmail.Wait()
	}
}

// Executes callback. Passes args: true/false, check's name, message.
func alert(check *Check, name *string, msg *string) {
	if check.Alert != nil {
		plugin, ok := check.Alert.(string)
		if ok { // check.Alert is a string.
			out, err := exec.Command(plugin, strconv.FormatBool(check.Failed), *name, *msg).CombinedOutput()
			if err != nil {
				fmt.Fprint(os.Stderr, "<3>", plugin, " failed\n", string(out), err.Error(), "\n")
			}
		} else { // check.Alert is a list.
			for _, i := range check.Alert.([]interface{}) {
				out, err := exec.Command(i.(string), strconv.FormatBool(check.Failed), *name, *msg).CombinedOutput()
				if err != nil {
					fmt.Fprint(os.Stderr, "<3>", i, " failed\n", string(out), err.Error(), "\n")
				}
			}
		}
	}
}

// 404 error.
func notFound(w http.ResponseWriter, r *http.Request) {
	h := w.Header()
	h.Set("Server", "jsonmon")
	h.Set("X-Frame-Options", "DENY")
	h.Set("X-XSS-Protection", "1; mode=block")
	http.NotFound(w, r)
}

// Display checks' details.
func getChecks(w http.ResponseWriter, r *http.Request) {
	displayJSON(w, r, &checks, &modified, true)
}

// Display application version.
func getVersion(w http.ResponseWriter, r *http.Request) {
	displayJSON(w, r, &version, &started, false)
}

// Output JSON.
func displayJSON(w http.ResponseWriter, r *http.Request, data interface{}, cache *string, lock bool) {
	h := w.Header()
	h.Set("Server", "jsonmon")
	h.Set("X-Frame-Options", "DENY")
	h.Set("X-XSS-Protection", "1; mode=block")
	h.Set("X-Content-Type-Options", "nosniff")
	var cached bool
	var result []byte
	if lock {
		mutex.RLock()
	}
	if r.Header.Get("If-None-Match") == *cache {
		cached = true
	} else {
		h.Set("ETag", *cache)
		result, _ = json.Marshal(&data)
	}
	if lock {
		mutex.RUnlock()
	}
	if cached {
		w.WriteHeader(http.StatusNotModified)
		delete(h, "Content-Type")
		delete(h, "Content-Length")
	} else {
		h.Set("Cache-Control", "no-cache")
		h.Set("Access-Control-Allow-Origin", "*")
		h.Set("Content-Type", "application/json; charset=utf-8")
		w.Write(result)
	}
}
<commit_msg>no X-Content-Type-Options when there is no content<commit_after>/*
Quick and simple monitoring system

Usage:
 jsonmon config.yml
 jsonmon -v # Prints version to stdout and exits

Environment:
 HOST
  - defaults to localhost
  - the JSON API network interface
 PORT
  - defaults to 3000
  - the JSON API port
*/
package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"gopkg.in/yaml.v2"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"regexp"
	"runtime"
	"strconv"
	"sync"
	"syscall"
	"time"
)

// Application version.
const Version = "2.0.5"

// This one is for internal use.
type ver struct {
	App  string `json:"jsonmon"`
	Go   string `json:"runtime"`
	Os   string `json:"os"`
	Arch string `json:"arch"`
}

var version ver

// Check details.
type Check struct {
	Name   string      `json:"name,omitempty" yaml:"name"`
	Web    string      `json:"web,omitempty" yaml:"web"`
	Shell  string      `json:"shell,omitempty" yaml:"shell"`
	Match  string      `json:"-" yaml:"match"`
	Return int         `json:"-" yaml:"return"`
	Notify interface{} `json:"-" yaml:"notify"`
	Alert  interface{} `json:"-" yaml:"alert"`
	Tries  int         `json:"-" yaml:"tries"`
	Repeat int         `json:"-" yaml:"repeat"`
	Failed bool        `json:"failed" yaml:"-"`
	Since  string      `json:"since,omitempty" yaml:"-"`
}

// Global checks list. Need to share it with workers and Web UI.
var checks []Check

// Global started and last modified date for HTTP caching.
var modified string
var started string

var mutex *sync.RWMutex

// Construct the last modified string.
func etag(ts time.Time) string {
	return "W/\"" + strconv.FormatInt(ts.UnixNano(), 10) + "\""
}

// The main loop.
func main() {
	// Parse CLI args.
	usage := "Usage: " + path.Base(os.Args[0]) + " config.yml\n" +
		"Docs:  https://github.com/chillum/jsonmon/wiki"
	if len(os.Args) != 2 {
		fmt.Fprintln(os.Stderr, usage)
		os.Exit(1)
	}
	// -v for version.
	version.App = Version
	version.Go = runtime.Version()
	version.Os = runtime.GOOS
	version.Arch = runtime.GOARCH
	switch os.Args[1] {
	case "-h":
		fallthrough
	case "-help":
		fallthrough
	case "--help":
		fmt.Fprintln(os.Stderr, usage)
		os.Exit(0)
	case "-v":
		fallthrough
	case "-version":
		fallthrough
	case "--version":
		json, _ := json.Marshal(&version)
		fmt.Println(string(json))
		os.Exit(0)
	}
	// Tune concurrency.
	if os.Getenv("GOMAXPROCS") == "" {
		runtime.GOMAXPROCS(runtime.NumCPU())
	}
	// Read config file or exit with error.
	config, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		fmt.Fprint(os.Stderr, "<2>", err, "\n")
		os.Exit(3)
	}
	err = yaml.Unmarshal(config, &checks)
	if err != nil {
		fmt.Fprint(os.Stderr, "<2>", "invalid config at ", os.Args[1], "\n")
		os.Exit(3)
	}
	// Exit with return code 0 on kill.
	done := make(chan os.Signal, 1)
	signal.Notify(done, syscall.SIGTERM)
	go func() {
		<-done
		os.Exit(0)
	}()
	// Run checks.
	started = etag(time.Now())
	modified = started
	mutex = &sync.RWMutex{}
	for i := range checks {
		go worker(&checks[i])
	}
	// Launch the JSON API.
	// Probably should do it prior to checks.
	host := os.Getenv("HOST")
	port := os.Getenv("PORT")
	if host == "" {
		host = "localhost"
	}
	if port == "" {
		port = "3000"
	}
	http.HandleFunc("/status", getChecks)
	http.HandleFunc("/version", getVersion)
	http.HandleFunc("/", notFound)
	err = http.ListenAndServe(host+":"+port, nil)
	if err != nil {
		fmt.Fprint(os.Stderr, "<2>", err, "\n")
	}
	os.Exit(4)
}

// Background worker.
func worker(check *Check) {
	mutex.Lock()
	if check.Repeat == 0 { // Set default timeout.
		check.Repeat = 30
	}
	if check.Tries == 0 { // Default to 1 attempt.
		check.Tries = 1
	}
	mutex.Unlock()
	sleep := time.Second * time.Duration(check.Repeat)
	for {
		if check.Web != "" {
			web(check)
		}
		if check.Shell != "" {
			shell(check)
		}
		time.Sleep(sleep)
	}
}

// Shell worker.
func shell(check *Check) {
	// Set check's display name.
	var name string
	if check.Name != "" {
		name = check.Name
	} else {
		name = check.Shell
	}
	// Execute with shell in N attemps.
	var out []byte
	var err error
	for i := 0; i < check.Tries; i++ {
		out, err = exec.Command("/bin/sh", "-c", check.Shell).CombinedOutput()
		if err == nil {
			if check.Match != "" { // Match regexp.
				var regex *regexp.Regexp
				regex, err = regexp.Compile(check.Match)
				if err == nil && !regex.Match(out) {
					err = errors.New("Expected:\n" + check.Match + "\n\nGot:\n" + string(out))
				}
			}
			break
		}
	}
	// Process results.
	if err == nil {
		if check.Failed {
			ts := time.Now()
			mutex.Lock()
			check.Failed = false
			check.Since = ts.Format(time.RFC3339)
			modified = etag(ts)
			mutex.Unlock()
			notify(check.Notify, "Fixed: "+name, nil)
			alert(check, &name, nil)
		}
	} else {
		if !check.Failed {
			ts := time.Now()
			mutex.Lock()
			check.Failed = true
			check.Since = ts.Format(time.RFC3339)
			modified = etag(ts)
			mutex.Unlock()
			msg := string(out) + err.Error()
			notify(check.Notify, "Failed: "+name, &msg)
			alert(check, &name, &msg)
		}
	}
}

// Web worker.
func web(check *Check) {
	// Set check's display name.
	var name string
	if check.Name != "" {
		name = check.Name
	} else {
		name = check.Web
	}
	if check.Return == 0 { // Successful HTTP return code is 200.
		check.Return = 200
	}
	// Get the URL in N attempts.
	var err error
	for i := 0; i < check.Tries; i++ {
		err = fetch(check.Web, check.Match, check.Return)
		if err == nil {
			break
		}
	}
	// Process results.
	if err == nil {
		if check.Failed {
			ts := time.Now()
			mutex.Lock()
			check.Failed = false
			check.Since = ts.Format(time.RFC3339)
			modified = etag(ts)
			mutex.Unlock()
			notify(check.Notify, "Fixed: "+name, nil)
			alert(check, &name, nil)

		}
	} else {
		if !check.Failed {
			ts := time.Now()
			mutex.Lock()
			check.Failed = true
			check.Since = ts.Format(time.RFC3339)
			modified = etag(ts)
			mutex.Unlock()
			msg := err.Error()
			notify(check.Notify, "Failed: "+name, &msg)
			alert(check, &name, &msg)
		}
	}
}

// Check HTTP redirects.
func redirect(req *http.Request, via []*http.Request) error {
	// When redirects number > 10 probably there's a problem.
	if len(via) >= 10 {
		return errors.New("stopped after 10 redirects")
	}
	// Redirects don't get User-Agent.
	req.Header.Set("User-Agent", "jsonmon")
	return nil
}

// The actual HTTP GET.
func fetch(url string, match string, code int) error {
	var err error
	var resp *http.Response
	var req *http.Request
	client := &http.Client{}
	client.CheckRedirect = redirect
	req, err = http.NewRequest("GET", url, nil)
	if err == nil {
		req.Header.Set("User-Agent", "jsonmon")
		resp, err = client.Do(req)
		if err == nil {
			if resp.StatusCode != code { // Check status code.
				err = errors.New(url + " returned " + strconv.Itoa(resp.StatusCode))
			} else { // Match regexp.
				if resp != nil && match != "" {
					var regex *regexp.Regexp
					regex, err = regexp.Compile(match)
					if err == nil {
						var body []byte
						body, _ = ioutil.ReadAll(resp.Body)
						if !regex.Match(body) {
							err = errors.New("Expected:\n" + match + "\n\nGot:\n" + string(body))
						}
					}
				}
			}
		}
	}
	if resp != nil {
		resp.Body.Close()
	}
	return err
}

// Logs and mail alerting.
func notify(mail interface{}, subject string, message *string) {
	// Log the alerts.
	if message == nil {
		fmt.Print("<5>", subject, "\n")
	} else {
		fmt.Print("<5>", subject, "\n", *message, "\n")
	}
	// Mail the alerts.
	if mail != nil {
		// Make the message.
		var rcpt string
		var ok bool
		if rcpt, ok = mail.(string); !ok {
			for i, v := range mail.([]interface{}) {
				if i != 0 {
					rcpt += ", "
				}
				rcpt += v.(string)
			}
		}
		msg := "To: " + rcpt + "\nSubject: " + subject + "\nX-Mailer: jsonmon\n\n"
		if message != nil {
			msg += *message
		}
		msg += "\n.\n"
		// And send it.
		sendmail := exec.Command("/usr/sbin/sendmail", "-t")
		stdin, _ := sendmail.StdinPipe()
		err := sendmail.Start()
		if err != nil {
			fmt.Fprint(os.Stderr, "<3>", err, "\n")
		}
		io.WriteString(stdin, msg)
		sendmail.Wait()
	}
}

// Executes callback. Passes args: true/false, check's name, message.
func alert(check *Check, name *string, msg *string) {
	if check.Alert != nil {
		plugin, ok := check.Alert.(string)
		if ok { // check.Alert is a string.
			out, err := exec.Command(plugin, strconv.FormatBool(check.Failed), *name, *msg).CombinedOutput()
			if err != nil {
				fmt.Fprint(os.Stderr, "<3>", plugin, " failed\n", string(out), err.Error(), "\n")
			}
		} else { // check.Alert is a list.
			for _, i := range check.Alert.([]interface{}) {
				out, err := exec.Command(i.(string), strconv.FormatBool(check.Failed), *name, *msg).CombinedOutput()
				if err != nil {
					fmt.Fprint(os.Stderr, "<3>", i, " failed\n", string(out), err.Error(), "\n")
				}
			}
		}
	}
}

// 404 error.
func notFound(w http.ResponseWriter, r *http.Request) {
	h := w.Header()
	h.Set("Server", "jsonmon")
	h.Set("X-Frame-Options", "DENY")
	h.Set("X-XSS-Protection", "1; mode=block")
	http.NotFound(w, r)
}

// Display checks' details.
func getChecks(w http.ResponseWriter, r *http.Request) {
	displayJSON(w, r, &checks, &modified, true)
}

// Display application version.
func getVersion(w http.ResponseWriter, r *http.Request) {
	displayJSON(w, r, &version, &started, false)
}

// Output JSON.
func displayJSON(w http.ResponseWriter, r *http.Request, data interface{}, cache *string, lock bool) {
	h := w.Header()
	h.Set("Server", "jsonmon")
	h.Set("X-Frame-Options", "DENY")
	h.Set("X-XSS-Protection", "1; mode=block")
	var cached bool
	var result []byte
	if lock {
		mutex.RLock()
	}
	if r.Header.Get("If-None-Match") == *cache {
		cached = true
	} else {
		h.Set("ETag", *cache)
		result, _ = json.Marshal(&data)
	}
	if lock {
		mutex.RUnlock()
	}
	if cached {
		w.WriteHeader(http.StatusNotModified)
	} else {
		h.Set("Cache-Control", "no-cache")
		h.Set("Access-Control-Allow-Origin", "*")
		h.Set("X-Content-Type-Options", "nosniff")
		h.Set("Content-Type", "application/json; charset=utf-8")
		w.Write(result)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"sync"

	termcolor "github.com/fatih/color"
)

// TODO: Perhaps error deserves a better coloring
var (
	errColorP      = termcolor.New(termcolor.FgGreen).Add(termcolor.Underline).PrintlnFunc()
	logSqBracketSP = termcolor.New(termcolor.FgWhite, termcolor.Bold).SprintFunc()
	logSP          = termcolor.New(termcolor.FgYellow, termcolor.Bold).SprintFunc()

	bracketSP = termcolor.New(termcolor.FgBlue).SprintFunc()
	quoteSP   = termcolor.New(termcolor.FgBlue, termcolor.Bold, termcolor.Italic).SprintFunc()
	kvNumSP   = termcolor.New(termcolor.FgYellow, termcolor.Bold).SprintFunc()
	kvSepSP   = termcolor.New(termcolor.FgWhite, termcolor.Bold, termcolor.Italic).SprintFunc()
	strSP     = termcolor.New(termcolor.FgWhite, termcolor.Bold).SprintFunc()
)

func logHeading() string {
	return fmt.Sprintf("%s%s%s", logSqBracketSP("["), logSP("LOG"), logSqBracketSP("]"))
}

type nologArgs struct {
	outToFile   bool
	outFileName string
	color       bool
	verbose     bool
	gocheck     string
	args        []string
}

var nlArgs nologArgs

func init() {
	flag.BoolVar(&nlArgs.outToFile, "f", false, "setting this flag will output the logs to a file.")
	flag.StringVar(&nlArgs.outFileName, "name", "tests.log", "this is an alternative file name for the ouput.")
	flag.BoolVar(&nlArgs.color, "c", false, "setting this flag will color the output logs.")
	flag.BoolVar(&nlArgs.verbose, "v", false, "setting this flag will use -test.v=true on the test run.")
	flag.StringVar(&nlArgs.gocheck, "filter", "", "this will be used to filter tests with -gocheck.f (requires gocheck).")
	flag.Parse()
	nlArgs.args = flag.Args()
}

func main() {
	args := nlArgs.args
	var (
		f   *os.File
		err error
	)
	if nlArgs.outToFile {
		f, err = os.OpenFile(nlArgs.outFileName, os.O_TRUNC|os.O_CREATE|os.O_RDWR, 0644)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error opening file %q: %v", nlArgs.outFileName, err)
			os.Exit(1)
		}
		defer f.Close()

	}
	testFlags := []string{"test"}
	if nlArgs.verbose {
		testFlags = append(testFlags, "-test.v=true")
	}
	if nlArgs.gocheck != "" {
		testFlags = append(testFlags, fmt.Sprintf("-gocheck.f=%q", nlArgs.gocheck))
	}
	args = append(testFlags, args...)
	if len(args) == 1 {
		args = append(args, "./...")
	}

	cmd := exec.Command("go", args...)
	out, err := cmd.StdoutPipe()

	if err != nil {
		fmt.Fprintf(os.Stderr, "Error getting StdoutPipe from command: %v", err)
		os.Exit(1)
	}
	stderr, err := cmd.StderrPipe()

	if err != nil {
		fmt.Fprintf(os.Stderr, "Error getting StderrPipe from command: %v", err)
		os.Exit(1)
	}
	if err := cmd.Start(); err != nil {
		fmt.Fprintf(os.Stderr, "Error running command: %v", err)
		os.Exit(1)
	}
	wg := &sync.WaitGroup{}
	wg.Add(2)
	var colorOut, colorErr chan string
	if nlArgs.color {
		colorOut = make(chan string)
		colorErr = make(chan string)
		wg.Add(2)
		go colorizeOut(colorOut, wg)
		go colorizeErr(colorErr, wg)
	}
	go printOut(f, out, colorOut, wg)
	go printErr(f, stderr, colorErr, wg)
	wg.Wait()
}

func tokenize(s string) string {
	tokenized := []string{}
	bracketCount := 0
	inStr := false
	for k, _ := range s {
		// TODO (perrito666) make this more rune friendly
		c := s[k : k+1] // This will break a lot with unicode chars
		switch c {
		case "{":
			bracketCount += 1
			tokenized = append(tokenized, bracketSP(c))
		case "}":
			bracketCount -= 1
			tokenized = append(tokenized, bracketSP(c))
		case "\"":
			inStr = !inStr
			tokenized = append(tokenized, quoteSP(c))
		case "1", "2", "3", "4", "5", "6", "7", "8", "9", "0":
			// I know, regexes would be better right? NO
			switch {
			case bracketCount > 0 && !inStr:
				tokenized = append(tokenized, kvNumSP(c))
			case inStr:
				tokenized = append(tokenized, strSP(c))
			default:
				tokenized = append(tokenized, c)
			}
		case ":":
			if bracketCount > 0 {
				tokenized = append(tokenized, kvSepSP(c))
			} else {
				tokenized = append(tokenized, c)
			}
		default:
			tokenized = append(tokenized, c)
		}
	}
	return strings.Join(tokenized, "")
}

func colorizeOut(c chan string, wg *sync.WaitGroup) {
	for {
		s, ok := <-c
		if !ok {
			wg.Done()
			return
		}
		var out string
		if strings.HasPrefix(s, "[LOG]") {
			out = logHeading()
			s = s[5:]
		}
		out += tokenize(s)
		fmt.Println(out)
	}

}

func colorizeErr(c chan string, wg *sync.WaitGroup) {
	for {
		s, ok := <-c
		if !ok {
			wg.Done()
			return
		}
		errColorP(s)
	}
}

func printOut(f *os.File, r io.Reader, c chan string, wg *sync.WaitGroup) {
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		s := string(scanner.Bytes())
		if f != nil {
			fmt.Fprintln(f, s)
		}
		if !strings.HasPrefix(s, "[LOG]") && c == nil {
			fmt.Println(s)
		}
		if c != nil {
			c <- s
		}
	}
	if scanner.Err() != nil {
		fmt.Fprintf(os.Stderr, "Error reading output of go test: %v", scanner.Err())
		os.Exit(1)
	}
	close(c)
	wg.Done()
}

func printErr(f *os.File, r io.Reader, c chan string, wg *sync.WaitGroup) {
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		s := string(scanner.Bytes())
		if f != nil {
			fmt.Fprintln(f, s)
		}
		fmt.Fprintln(os.Stderr, s)
		if c != nil {
			c <- s
		}

	}
	if scanner.Err() != nil {
		fmt.Fprintf(os.Stderr, "Error reading output of go test: %v", scanner.Err())
		os.Exit(1)
	}
	close(c)
	wg.Done()
}
<commit_msg>Added proper coloring for strings<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"sync"

	termcolor "github.com/fatih/color"
)

// TODO: Perhaps error deserves a better coloring
var (
	errColorP      = termcolor.New(termcolor.FgGreen).Add(termcolor.Underline).PrintlnFunc()
	logSqBracketSP = termcolor.New(termcolor.FgWhite, termcolor.Bold).SprintFunc()
	logSP          = termcolor.New(termcolor.FgYellow, termcolor.Bold).SprintFunc()

	bracketSP = termcolor.New(termcolor.FgBlue).SprintFunc()
	quoteSP   = termcolor.New(termcolor.FgBlue, termcolor.Bold, termcolor.Italic).SprintFunc()
	kvNumSP   = termcolor.New(termcolor.FgYellow, termcolor.Bold).SprintFunc()
	kvSepSP   = termcolor.New(termcolor.FgWhite, termcolor.Bold, termcolor.Italic).SprintFunc()
	strSP     = termcolor.New(termcolor.FgWhite, termcolor.Bold).SprintFunc()
)

func logHeading() string {
	return fmt.Sprintf("%s%s%s", logSqBracketSP("["), logSP("LOG"), logSqBracketSP("]"))
}

type nologArgs struct {
	outToFile   bool
	outFileName string
	color       bool
	verbose     bool
	gocheck     string
	args        []string
}

var nlArgs nologArgs

func init() {
	flag.BoolVar(&nlArgs.outToFile, "f", false, "setting this flag will output the logs to a file.")
	flag.StringVar(&nlArgs.outFileName, "name", "tests.log", "this is an alternative file name for the ouput.")
	flag.BoolVar(&nlArgs.color, "c", false, "setting this flag will color the output logs.")
	flag.BoolVar(&nlArgs.verbose, "v", false, "setting this flag will use -test.v=true on the test run.")
	flag.StringVar(&nlArgs.gocheck, "filter", "", "this will be used to filter tests with -gocheck.f (requires gocheck).")
	flag.Parse()
	nlArgs.args = flag.Args()
}

func main() {
	args := nlArgs.args
	var (
		f   *os.File
		err error
	)
	if nlArgs.outToFile {
		f, err = os.OpenFile(nlArgs.outFileName, os.O_TRUNC|os.O_CREATE|os.O_RDWR, 0644)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error opening file %q: %v", nlArgs.outFileName, err)
			os.Exit(1)
		}
		defer f.Close()

	}
	testFlags := []string{"test"}
	if nlArgs.verbose {
		testFlags = append(testFlags, "-test.v=true")
	}
	if nlArgs.gocheck != "" {
		testFlags = append(testFlags, "-test.v=true")
		// FIXME (perrito666) This is not working as expected, perhaps quoting
		testFlags = append(testFlags, fmt.Sprintf("-gocheck.f=%q", nlArgs.gocheck))
	}
	args = append(testFlags, args...)
	if len(args) == 1 {
		args = append(args, "./...")
	}

	cmd := exec.Command("go", args...)
	out, err := cmd.StdoutPipe()

	if err != nil {
		fmt.Fprintf(os.Stderr, "Error getting StdoutPipe from command: %v", err)
		os.Exit(1)
	}
	stderr, err := cmd.StderrPipe()

	if err != nil {
		fmt.Fprintf(os.Stderr, "Error getting StderrPipe from command: %v", err)
		os.Exit(1)
	}
	if err := cmd.Start(); err != nil {
		fmt.Fprintf(os.Stderr, "Error running command: %v", err)
		os.Exit(1)
	}
	wg := &sync.WaitGroup{}
	wg.Add(2)
	var colorOut, colorErr chan string
	if nlArgs.color {
		colorOut = make(chan string)
		colorErr = make(chan string)
		wg.Add(2)
		go colorizeOut(colorOut, wg)
		go colorizeErr(colorErr, wg)
	}
	go printOut(f, out, colorOut, wg)
	go printErr(f, stderr, colorErr, wg)
	wg.Wait()
}

func tokenize(s string) string {
	tokenized := []string{}
	bracketCount := 0
	inStr := false
	for k, _ := range s {
		// TODO (perrito666) make this more rune friendly
		c := s[k : k+1] // This will break a lot with unicode chars
		switch c {
		case "{":
			bracketCount += 1
			tokenized = append(tokenized, bracketSP(c))
		case "}":
			bracketCount -= 1
			tokenized = append(tokenized, bracketSP(c))
		case "\"":
			inStr = !inStr
			tokenized = append(tokenized, quoteSP(c))
		case "1", "2", "3", "4", "5", "6", "7", "8", "9", "0":
			// I know, regexes would be better right? NO
			switch {
			case bracketCount > 0 && !inStr:
				tokenized = append(tokenized, kvNumSP(c))
			case inStr:
				tokenized = append(tokenized, strSP(c))
			default:
				tokenized = append(tokenized, c)
			}
		case ":":
			if bracketCount > 0 {
				tokenized = append(tokenized, kvSepSP(c))
			} else {
				tokenized = append(tokenized, c)
			}
		default:
			if inStr {
				tokenized = append(tokenized, strSP(c))
			} else {
				tokenized = append(tokenized, c)
			}

		}
	}
	return strings.Join(tokenized, "")
}

func colorizeOut(c chan string, wg *sync.WaitGroup) {
	for {
		s, ok := <-c
		if !ok {
			wg.Done()
			return
		}
		var out string
		// TODO(perrito666) do the same for the other expected values
		// in a log line: timestamp, loglevel and package
		// [LOG] 0:00.003 INFO a.package
		if strings.HasPrefix(s, "[LOG]") {
			out = logHeading()
			s = s[5:]
		}
		out += tokenize(s)
		fmt.Println(out)
	}

}

func colorizeErr(c chan string, wg *sync.WaitGroup) {
	for {
		s, ok := <-c
		if !ok {
			wg.Done()
			return
		}
		errColorP(s)
	}
}

func printOut(f *os.File, r io.Reader, c chan string, wg *sync.WaitGroup) {
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		s := string(scanner.Bytes())
		if f != nil {
			fmt.Fprintln(f, s)
		}
		if !strings.HasPrefix(s, "[LOG]") && c == nil {
			fmt.Println(s)
		}
		if c != nil {
			c <- s
		}
	}
	if scanner.Err() != nil {
		fmt.Fprintf(os.Stderr, "Error reading output of go test: %v", scanner.Err())
		os.Exit(1)
	}
	close(c)
	wg.Done()
}

func printErr(f *os.File, r io.Reader, c chan string, wg *sync.WaitGroup) {
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		s := string(scanner.Bytes())
		if f != nil {
			fmt.Fprintln(f, s)
		}
		fmt.Fprintln(os.Stderr, s)
		if c != nil {
			c <- s
		}

	}
	if scanner.Err() != nil {
		fmt.Fprintf(os.Stderr, "Error reading output of go test: %v", scanner.Err())
		os.Exit(1)
	}
	close(c)
	wg.Done()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/christoph-k/go-fsevents"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"
)

func main() {
	var pids []int
	var pidslock sync.Mutex
	var command []string
	var exclude []string

	for i, arg := range os.Args {
		if arg == "--" {
			command = os.Args[i+1:]
		}
		if arg == "-exclude" {
			excludestr := os.Args[i+1]
			exclude = strings.Split(excludestr, ",")
			for idx, e := range exclude {
				exclude[idx] = strings.Trim(e, " ")
			}
		}
	}
	if len(command) == 0 {
		usage()
		os.Exit(1)
	}

	// intercept SIGINT and kill childprocesses
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		kill(&pids, &pidslock)
		os.Exit(0)
	}()

	w, err := fsevents.NewWatcher(".", 200)
	if err != nil {
		panic(err)
	}
	go run(&pids, &pidslock, command)
	for {
		e := <-w.EventChan
		excluded := false
		for _, excl := range exclude {
			if strings.Contains(e.Path, excl) {
				excluded = true
			}
		}
		if e.FileInfo == nil || e.FileInfo.IsDir() || excluded {
			continue
		}
		fmt.Printf("%v has changed\n", e.Path)
		kill(&pids, &pidslock)
		w, err = fsevents.NewWatcher(".", 200)
		if err != nil {
			panic(err)
		}
		go run(&pids, &pidslock, command)
	}
}

func run(pids *[]int, pidslock *sync.Mutex, command []string) {
	fmt.Printf("running %v\n", command)
	cmd := exec.Command(command[0], command[1:]...)
	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Start()

	p := cmd.Process
	if p != nil {
		pidslock.Lock()
		*pids = append(*pids, p.Pid)
		pidslock.Unlock()
	}
	cmd.Wait()
}

func kill(pids *[]int, pidslock *sync.Mutex) {
	pidslock.Lock()
	defer pidslock.Unlock()
	for _, pid := range *pids {
		fmt.Printf("killing %v\n", pid)
		syscall.Kill(-pid, syscall.SIGTERM)
		time.Sleep(time.Millisecond * 100)
		syscall.Kill(-pid, syscall.SIGKILL)
	}
	*pids = make([]int, 0)
}

func usage() {
	fmt.Printf("usage: wr [-exclude \"file1, file2\"] -- <command with args>\n")
}
<commit_msg>stop watcher properly<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/christoph-k/go-fsevents"
)

func main() {
	var pids []int
	var pidslock sync.Mutex
	var command []string
	var exclude []string

	for i, arg := range os.Args {
		if arg == "--" {
			command = os.Args[i+1:]
		}
		if arg == "-exclude" {
			excludestr := os.Args[i+1]
			exclude = strings.Split(excludestr, ",")
			for idx, e := range exclude {
				exclude[idx] = strings.Trim(e, " ")
			}
		}
	}
	if len(command) == 0 {
		usage()
		os.Exit(1)
	}

	// intercept SIGINT and kill childprocesses
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		kill(&pids, &pidslock)
		os.Exit(0)
	}()

	w, err := fsevents.NewWatcher(".", 200)
	if err != nil {
		panic(err)
	}
	go run(&pids, &pidslock, command)
	for {
		e := <-w.EventChan
		excluded := false
		for _, excl := range exclude {
			if strings.Contains(e.Path, excl) {
				excluded = true
			}
		}
		if e.FileInfo == nil || e.FileInfo.IsDir() || excluded {
			continue
		}
		fmt.Printf("%v has changed\n", e.Path)
		kill(&pids, &pidslock)
		w.Stop()
		w, err = fsevents.NewWatcher(".", 200)
		if err != nil {
			panic(err)
		}
		go run(&pids, &pidslock, command)
	}
}

func run(pids *[]int, pidslock *sync.Mutex, command []string) {
	fmt.Printf("running %v\n", command)
	cmd := exec.Command(command[0], command[1:]...)
	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Start()

	p := cmd.Process
	if p != nil {
		pidslock.Lock()
		*pids = append(*pids, p.Pid)
		pidslock.Unlock()
	}
	cmd.Wait()
}

func kill(pids *[]int, pidslock *sync.Mutex) {
	pidslock.Lock()
	defer pidslock.Unlock()
	for _, pid := range *pids {
		fmt.Printf("killing %v\n", pid)
		syscall.Kill(-pid, syscall.SIGTERM)
		time.Sleep(time.Millisecond * 100)
		syscall.Kill(-pid, syscall.SIGKILL)
	}
	*pids = make([]int, 0)
}

func usage() {
	fmt.Printf("usage: wr [-exclude \"file1, file2\"] -- <command with args>\n")
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"bytes"
	"encoding/csv"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"path"
	"sort"
	"strconv"
)

var urlString = flag.String("url", "http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country-CSV.zip", "URL for loading the database")

func readUrl(s string) ([]byte, error) {
	u, err := url.Parse(s)
	if err != nil {
		return nil, err
	}

	switch u.Scheme {
	case "file":
		return ioutil.ReadFile(u.Path)
	case "http", "https":
		resp, err := http.Get(s)
		if err != nil {
			return nil, err
		}
		defer resp.Body.Close()

		if resp.StatusCode != http.StatusOK {
			return nil, err
		}

		return ioutil.ReadAll(resp.Body)
	}

	return nil, fmt.Errorf("unknown scheme %q", u.Scheme)
}

type geoNameId uint32

func ParseGeoNameId(n string) (geoNameId, error) {
	i, err := strconv.ParseUint(n, 10, 32)
	if err != nil {
		return 0, err
	}
	return geoNameId(i), nil
}

type continent struct {
	Code string
	Name string
}

type country struct {
	Code string
	Name string
}

type geoNameEntry struct {
	Continent continent
	Country   country
}

type geoNames map[geoNameId]geoNameEntry

func readGeoNames(r io.Reader) (geoNames, error) {
	g := geoNames{}
	c := csv.NewReader(r)
	for i := 0; ; i++ {
		record, err := c.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		if len(record) < 6 {
			return nil, errors.New("unexpected line")
		}
		if i == 0 {
			continue
		}
		id, err := ParseGeoNameId(record[0])
		if err != nil {
			return nil, err
		}
		g[id] = geoNameEntry{
			Continent: continent{
				Code: record[2],
				Name: record[3],
			},
			Country: country{
				Code: record[4],
				Name: record[5],
			},
		}
	}
	return g, nil
}

func (g geoNames) Get(id geoNameId) (geoNameEntry, bool) {
	e, ok := g[id]
	return e, ok
}

func less(left, right net.IP) bool {
	leftLen := len(left)
	rightLen := len(right)
	if leftLen != rightLen {
		return leftLen < rightLen
	}
	for idx, b := range right {
		if left[idx] != b {
			return left[idx] < b
		}
	}
	return false
}

type network struct {
	net.IPNet
	NameId geoNameId
}

type networks []network

func readNetworks(r io.Reader) (networks, error) {
	n := networks{}

	c := csv.NewReader(r)
	for i := 0; ; i++ {
		record, err := c.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		if len(record) < 2 {
			return nil, errors.New("unexpected line")
		}
		if i == 0 || record[1] == "" {
			continue
		}

		_, nw, err := net.ParseCIDR(record[0])
		if err != nil {
			return nil, err
		}
		id, err := ParseGeoNameId(record[1])
		if err != nil {
			return nil, err
		}
		n = append(n, network{IPNet: *nw, NameId: id})
	}

	sort.Sort(n)
	return n, nil
}

func (n networks) Len() int {
	return len(n)
}

func (n networks) Swap(i, j int) {
	n[i], n[j] = n[j], n[i]
}

func (n networks) Less(i, j int) bool {
	return less(n[i].IP, n[j].IP)
}

func (n networks) Get(ip net.IP) (geoNameId, bool) {
	idx := sort.Search(len(n), func(i int) bool {
		return less(ip, n[i].IP)
	})
	if idx == 0 {
		return 0, false
	}

	if idx == -1 {
		idx = len(n)
	}
	idx -= 1
	if idx < len(n) && n[idx].IPNet.Contains(ip) {
		return n[idx].NameId, true
	}
	return 0, false
}

type db struct {
	names geoNames
	ipv4  networks
	ipv6  networks
}

func New(b []byte) (*db, error) {
	z, err := zip.NewReader(bytes.NewReader(b), int64(len(b)))
	if err != nil {
		return nil, err
	}

	var names geoNames
	var ipv4 networks
	var ipv6 networks
	for _, f := range z.File {
		name := path.Base(f.Name)

		switch name {
		case "GeoLite2-Country-Blocks-IPv4.csv":
			r, err := f.Open()
			if err != nil {
				return nil, err
			}
			defer r.Close()
			ipv4, err = readNetworks(r)
			if err != nil {
				return nil, err
			}
		case "GeoLite2-Country-Blocks-IPv6.csv":
			r, err := f.Open()
			if err != nil {
				return nil, err
			}
			defer r.Close()
			ipv6, err = readNetworks(r)
			if err != nil {
				return nil, err
			}
		case "GeoLite2-Country-Locations-en.csv":
			r, err := f.Open()
			if err != nil {
				return nil, err
			}
			defer r.Close()
			names, err = readGeoNames(r)
			if err != nil {
				return nil, err
			}
		}
	}

	if names == nil || ipv4 == nil || ipv6 == nil {
		return nil, errors.New("couldn't find all sections")
	}
	return &db{names: names, ipv4: ipv4, ipv6: ipv6}, nil
}

func (d *db) Get(s string) (geoNameEntry, bool) {
	ip := net.ParseIP(s)

	var id geoNameId
	var ok bool
	if ipv4 := ip.To4(); ipv4 != nil {
		id, ok = d.ipv4.Get(ipv4)
	} else if ipv6 := ip.To16(); ipv6 != nil {
		id, ok = d.ipv6.Get(ipv6)
	}

	if !ok {
		return geoNameEntry{}, false
	}
	e, ok := d.names.Get(id)
	return e, ok
}

func main() {
	flag.Parse()

	b, err := readUrl(*urlString)
	if err != nil {
		log.Fatal(err)
	}

	d, err := New(b)
	if err != nil {
		log.Fatal(err)
	}

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		path := r.URL.Path[1:]
		result, ok := d.Get(path)
		if !ok {
			http.NotFound(w, r)
			return
		}
		b, err := json.Marshal(result)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		w.Write(b)
	})
	http.ListenAndServe("localhost:8080", nil)
}
<commit_msg>Listen on :8080<commit_after>package main

import (
	"archive/zip"
	"bytes"
	"encoding/csv"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"path"
	"sort"
	"strconv"
)

var urlString = flag.String("url", "http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country-CSV.zip", "URL for loading the database")

func readUrl(s string) ([]byte, error) {
	u, err := url.Parse(s)
	if err != nil {
		return nil, err
	}

	switch u.Scheme {
	case "file":
		return ioutil.ReadFile(u.Path)
	case "http", "https":
		resp, err := http.Get(s)
		if err != nil {
			return nil, err
		}
		defer resp.Body.Close()

		if resp.StatusCode != http.StatusOK {
			return nil, err
		}

		return ioutil.ReadAll(resp.Body)
	}

	return nil, fmt.Errorf("unknown scheme %q", u.Scheme)
}

type geoNameId uint32

func ParseGeoNameId(n string) (geoNameId, error) {
	i, err := strconv.ParseUint(n, 10, 32)
	if err != nil {
		return 0, err
	}
	return geoNameId(i), nil
}

type continent struct {
	Code string
	Name string
}

type country struct {
	Code string
	Name string
}

type geoNameEntry struct {
	Continent continent
	Country   country
}

type geoNames map[geoNameId]geoNameEntry

func readGeoNames(r io.Reader) (geoNames, error) {
	g := geoNames{}
	c := csv.NewReader(r)
	for i := 0; ; i++ {
		record, err := c.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		if len(record) < 6 {
			return nil, errors.New("unexpected line")
		}
		if i == 0 {
			continue
		}
		id, err := ParseGeoNameId(record[0])
		if err != nil {
			return nil, err
		}
		g[id] = geoNameEntry{
			Continent: continent{
				Code: record[2],
				Name: record[3],
			},
			Country: country{
				Code: record[4],
				Name: record[5],
			},
		}
	}
	return g, nil
}

func (g geoNames) Get(id geoNameId) (geoNameEntry, bool) {
	e, ok := g[id]
	return e, ok
}

func less(left, right net.IP) bool {
	leftLen := len(left)
	rightLen := len(right)
	if leftLen != rightLen {
		return leftLen < rightLen
	}
	for idx, b := range right {
		if left[idx] != b {
			return left[idx] < b
		}
	}
	return false
}

type network struct {
	net.IPNet
	NameId geoNameId
}

type networks []network

func readNetworks(r io.Reader) (networks, error) {
	n := networks{}

	c := csv.NewReader(r)
	for i := 0; ; i++ {
		record, err := c.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		if len(record) < 2 {
			return nil, errors.New("unexpected line")
		}
		if i == 0 || record[1] == "" {
			continue
		}

		_, nw, err := net.ParseCIDR(record[0])
		if err != nil {
			return nil, err
		}
		id, err := ParseGeoNameId(record[1])
		if err != nil {
			return nil, err
		}
		n = append(n, network{IPNet: *nw, NameId: id})
	}

	sort.Sort(n)
	return n, nil
}

func (n networks) Len() int {
	return len(n)
}

func (n networks) Swap(i, j int) {
	n[i], n[j] = n[j], n[i]
}

func (n networks) Less(i, j int) bool {
	return less(n[i].IP, n[j].IP)
}

func (n networks) Get(ip net.IP) (geoNameId, bool) {
	idx := sort.Search(len(n), func(i int) bool {
		return less(ip, n[i].IP)
	})
	if idx == 0 {
		return 0, false
	}

	if idx == -1 {
		idx = len(n)
	}
	idx -= 1
	if idx < len(n) && n[idx].IPNet.Contains(ip) {
		return n[idx].NameId, true
	}
	return 0, false
}

type db struct {
	names geoNames
	ipv4  networks
	ipv6  networks
}

func New(b []byte) (*db, error) {
	z, err := zip.NewReader(bytes.NewReader(b), int64(len(b)))
	if err != nil {
		return nil, err
	}

	var names geoNames
	var ipv4 networks
	var ipv6 networks
	for _, f := range z.File {
		name := path.Base(f.Name)

		switch name {
		case "GeoLite2-Country-Blocks-IPv4.csv":
			r, err := f.Open()
			if err != nil {
				return nil, err
			}
			defer r.Close()
			ipv4, err = readNetworks(r)
			if err != nil {
				return nil, err
			}
		case "GeoLite2-Country-Blocks-IPv6.csv":
			r, err := f.Open()
			if err != nil {
				return nil, err
			}
			defer r.Close()
			ipv6, err = readNetworks(r)
			if err != nil {
				return nil, err
			}
		case "GeoLite2-Country-Locations-en.csv":
			r, err := f.Open()
			if err != nil {
				return nil, err
			}
			defer r.Close()
			names, err = readGeoNames(r)
			if err != nil {
				return nil, err
			}
		}
	}

	if names == nil || ipv4 == nil || ipv6 == nil {
		return nil, errors.New("couldn't find all sections")
	}
	return &db{names: names, ipv4: ipv4, ipv6: ipv6}, nil
}

func (d *db) Get(s string) (geoNameEntry, bool) {
	ip := net.ParseIP(s)

	var id geoNameId
	var ok bool
	if ipv4 := ip.To4(); ipv4 != nil {
		id, ok = d.ipv4.Get(ipv4)
	} else if ipv6 := ip.To16(); ipv6 != nil {
		id, ok = d.ipv6.Get(ipv6)
	}

	if !ok {
		return geoNameEntry{}, false
	}
	e, ok := d.names.Get(id)
	return e, ok
}

func main() {
	flag.Parse()

	b, err := readUrl(*urlString)
	if err != nil {
		log.Fatal(err)
	}

	d, err := New(b)
	if err != nil {
		log.Fatal(err)
	}

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		path := r.URL.Path[1:]
		result, ok := d.Get(path)
		if !ok {
			w.Write([]byte("{}"))
			return
		}
		b, err := json.Marshal(result)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		w.Write(b)
	})
	http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"strconv"

	"github.com/urfave/cli"
)

var debug bool

func checkFastlyKey(c *cli.Context) *cli.ExitError {
	if c.GlobalString("fastly-key") == "" {
		return cli.NewExitError("Error: Fastly API key must be set.", -1)
	}
	return nil
}

func main() {
	app := cli.NewApp()
	app.Name = "fastlyctl"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config, c",
			Value: "config.toml",
			Usage: "Load Fastly configuration from `FILE`",
		},
		cli.StringFlag{
			Name:   "fastly-key, K",
			Usage:  "Fastly API Key.",
			EnvVar: "FASTLY_KEY",
		},
		cli.BoolFlag{
			Name:  "debug, d",
			Usage: "Print more detailed info for debugging.",
		},
		cli.BoolFlag{
			Name:  "assume-yes, y",
			Usage: "Assume 'yes' to all prompts. USE ONLY IF YOU ARE CERTAIN YOUR COMMANDS WON'T BREAK ANYTHING!",
		},
	}

	app.Commands = []cli.Command{
		cli.Command{
			Name:      "sync",
			Aliases:   []string{"s"},
			Usage:     "Sync remote service configuration with local config file.",
			ArgsUsage: "[<SERVICE_NAME>|<SERVICE_ID>]",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "all, a",
					Usage: "Sync all services listed in config file",
				},
				cli.BoolFlag{
					Name:  "noop, n",
					Usage: "Create new sync'd config versions, but do not activate.",
				},
			},
			Before: func(c *cli.Context) error {
				if err := checkFastlyKey(c); err != nil {
					return err
				}
				if (!c.Bool("all") && !c.Args().Present()) || (c.Bool("all") && c.Args().Present()) {
					return cli.NewExitError("Error: either specify service names to be syncd, or sync all with -a", -1)
				}
				debug = c.GlobalBool("debug")
				return nil
			},
			Action: syncConfig,
		},
		cli.Command{
			Name:    "version",
			Aliases: []string{"v"},
			Usage:   "Manage service versions.",
			Before: func(c *cli.Context) error {
				if err := checkFastlyKey(c); err != nil {
					return err
				}
				// less than 2 here since the subcommand is the first Arg
				if len(c.Args()) < 2 {
					return cli.NewExitError("Please specify service.", -1)
				}
				return nil
			},
			Subcommands: cli.Commands{
				cli.Command{
					Name:      "list",
					Usage:     "List versions associated with a given service",
					Action:    versionList,
					ArgsUsage: "[<SERVICE_NAME>|<SERVICE_ID>]",
				},
				cli.Command{
					Name:      "validate",
					Usage:     "Validate a specified VERSION",
					ArgsUsage: "[<SERVICE_NAME>|<SERVICE_ID>] [<VERSION>]",
					Action:    versionValidate,
					Before: func(c *cli.Context) error {
						if _, err := strconv.Atoi(c.Args().Get(1)); err != nil {
							return cli.NewExitError("Please specify version to validate.", -1)
						}
						return nil
					},
				},
				cli.Command{
					Name:      "activate",
					Usage:     "Activate a specified VERSION",
					ArgsUsage: "[<SERVICE_NAME>|<SERVICE_ID>] [<VERSION>]",
					Action:    versionActivate,
					Before: func(c *cli.Context) error {
						if _, err := strconv.Atoi(c.Args().Get(1)); err != nil {
							return cli.NewExitError("Please specify version to activate.", -1)
						}
						return versionValidate(c)
					},
				},
			},
		},
		cli.Command{
			Name:  "service",
			Usage: "Manage services.",
			Before: func(c *cli.Context) error {
				if err := checkFastlyKey(c); err != nil {
					return err
				}
				return nil
			},
			Subcommands: cli.Commands{
				cli.Command{
					Name:   "list",
					Usage:  "List services associated with account",
					Action: serviceList,
				},
			},
		},
	}

	app.Run(os.Args)
}
<commit_msg>Do an interactive check on some commands.<commit_after>package main

import (
	"os"
	"strconv"
	"syscall"

	"golang.org/x/crypto/ssh/terminal"

	"github.com/urfave/cli"
)

var debug bool

func checkFastlyKey(c *cli.Context) *cli.ExitError {
	if c.GlobalString("fastly-key") == "" {
		return cli.NewExitError("Error: Fastly API key must be set.", -1)
	}
	return nil
}

func checkInteractive(c *cli.Context) *cli.ExitError {
	interactive := terminal.IsTerminal(syscall.Stdin)
	if !interactive && !c.GlobalBool("assume-yes") {
		return cli.NewExitError("In non-interactive shell and --assume-yes not used, exiting.", -1)

	}
	return nil
}

func main() {
	app := cli.NewApp()
	app.Name = "fastlyctl"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config, c",
			Value: "config.toml",
			Usage: "Load Fastly configuration from `FILE`",
		},
		cli.StringFlag{
			Name:   "fastly-key, K",
			Usage:  "Fastly API Key.",
			EnvVar: "FASTLY_KEY",
		},
		cli.BoolFlag{
			Name:  "debug, d",
			Usage: "Print more detailed info for debugging.",
		},
		cli.BoolFlag{
			Name:  "assume-yes, y",
			Usage: "Assume 'yes' to all prompts. USE ONLY IF YOU ARE CERTAIN YOUR COMMANDS WON'T BREAK ANYTHING!",
		},
	}

	app.Commands = []cli.Command{
		cli.Command{
			Name:      "sync",
			Aliases:   []string{"s"},
			Usage:     "Sync remote service configuration with local config file.",
			ArgsUsage: "[<SERVICE_NAME>|<SERVICE_ID>]",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "all, a",
					Usage: "Sync all services listed in config file",
				},
				cli.BoolFlag{
					Name:  "noop, n",
					Usage: "Create new sync'd config versions, but do not activate.",
				},
			},
			Before: func(c *cli.Context) error {
				if err := checkFastlyKey(c); err != nil {
					return err
				}
				if err := checkInteractive(c); err != nil {
					return err
				}
				if (!c.Bool("all") && !c.Args().Present()) || (c.Bool("all") && c.Args().Present()) {
					return cli.NewExitError("Error: either specify service names to be syncd, or sync all with -a", -1)
				}
				debug = c.GlobalBool("debug")
				return nil
			},
			Action: syncConfig,
		},
		cli.Command{
			Name:    "version",
			Aliases: []string{"v"},
			Usage:   "Manage service versions.",
			Before: func(c *cli.Context) error {
				if err := checkFastlyKey(c); err != nil {
					return err
				}
				// less than 2 here since the subcommand is the first Arg
				if len(c.Args()) < 2 {
					return cli.NewExitError("Please specify service.", -1)
				}
				return nil
			},
			Subcommands: cli.Commands{
				cli.Command{
					Name:      "list",
					Usage:     "List versions associated with a given service",
					Action:    versionList,
					ArgsUsage: "[<SERVICE_NAME>|<SERVICE_ID>]",
				},
				cli.Command{
					Name:      "validate",
					Usage:     "Validate a specified VERSION",
					ArgsUsage: "[<SERVICE_NAME>|<SERVICE_ID>] [<VERSION>]",
					Action:    versionValidate,
					Before: func(c *cli.Context) error {
						if _, err := strconv.Atoi(c.Args().Get(1)); err != nil {
							return cli.NewExitError("Please specify version to validate.", -1)
						}
						return nil
					},
				},
				cli.Command{
					Name:      "activate",
					Usage:     "Activate a specified VERSION",
					ArgsUsage: "[<SERVICE_NAME>|<SERVICE_ID>] [<VERSION>]",
					Action:    versionActivate,
					Before: func(c *cli.Context) error {
						if err := checkInteractive(c); err != nil {
							return err
						}
						if _, err := strconv.Atoi(c.Args().Get(1)); err != nil {
							return cli.NewExitError("Please specify version to activate.", -1)
						}
						return versionValidate(c)
					},
				},
			},
		},
		cli.Command{
			Name:  "service",
			Usage: "Manage services.",
			Before: func(c *cli.Context) error {
				if err := checkFastlyKey(c); err != nil {
					return err
				}
				return nil
			},
			Subcommands: cli.Commands{
				cli.Command{
					Name:   "list",
					Usage:  "List services associated with account",
					Action: serviceList,
				},
			},
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"

	"github.com/codegangsta/cli"
)

func format(cc string) {
	fmt.Printf("=> Writing formatted cloud-config to STDOUT...\n")
	file, err := os.Open(cc)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		fmt.Println("\"" + strings.Replace(scanner.Text(), "\"", "\\\"", -1) + "\\n\",")
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}
}

func formatOut(cc, outPath string) {
	fmt.Printf("=> Writing formatted cloud-config to %s...\n", cc)
	file, err := os.Open(cc)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	out, err := os.Create(outPath)
	if err != nil {
		log.Fatal(err)
	}
	defer out.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		out.WriteString("\"" + strings.Replace(scanner.Text(), "\"", "\\\"", -1) + "\\n\",\n")
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}
}

func formatTmp(cc string) *os.File {
	fmt.Printf("=> Writing formatted cloud-config to a temporary file...\n")
	file, err := os.Open(cc)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	tmp, err := ioutil.TempFile("/tmp", "cloudcfg")
	if err != nil {
		log.Fatal(err)
	}

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		tmp.WriteString("\"" + strings.Replace(scanner.Text(), "\"", "\\\"", -1) + "\\n\",\n")
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}

	return tmp
}

func inject(toInject, toReceive string, start, end int64) {
	fmt.Printf("=> Removing lines [%d:%d] in %s...\n", start, end, toReceive)
	// read in the lines to be received
	receiving, err := os.Open(toReceive)
	if err != nil {
		log.Fatal(err)
	}

	receivingScanner := bufio.NewScanner(receiving)
	preLines := ""
	postLines := ""
	var numLines int64 = 0
	for receivingScanner.Scan() {
		numLines++
		if numLines < start {
			preLines = preLines + receivingScanner.Text() + "\n"
		} else if numLines > end {
			postLines = postLines + receivingScanner.Text() + "\n"
		}
	}
	if err := receivingScanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}

	// read in the lines to be injected
	injecting, err := os.Open(toInject)
	if err != nil {
		log.Fatal(err)
	}
	defer injecting.Close()

	injectingScanner := bufio.NewScanner(injecting)
	injectLines := ""
	for injectingScanner.Scan() {
		injectLines = injectLines + injectingScanner.Text() + "\n"
	}
	if err := injectingScanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}

	// combine all parts of the file into one
	fmt.Printf("=> Injecting %s into %s...\n", toInject, toReceive)
	k := strings.LastIndex(injectLines, "\\n\",")
	injectLines = injectLines[:k] + "\""
	injectedLines := preLines + injectLines + postLines

	// write to the receiving file
	receiving.Close()
	receivingWrite, err := os.Create(toReceive)
	if err != nil {
		log.Fatal(err)
	}
	defer receivingWrite.Close()
	receivingWrite.WriteString(injectedLines)
	receivingWrite.Sync()
}

func injectAndFormat(toInject, toReceive string, start, end int64) {
	tmpFile := formatTmp(toInject).Name()
	inject(tmpFile, toReceive, start, end)
	defer os.Remove(tmpFile)
}

func main() {
	app := cli.NewApp()
	app.Name = "cumulus"
	app.Authors = []cli.Author{
		{
			Name:  "Ben Bailey",
			Email: "bennettbailey@gmail.com",
		},
	}
	app.Version = "1.0"
	app.Usage = "inject cloud-configs into cloud-formation templates"
	app.Commands = []cli.Command{
		{
			Name:  "format",
			Usage: "format a cloud-config file into an acceptable JSON structure",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "tmp",
					Usage: "write the cloud-config to a tmp file",
				},
				cli.StringFlag{
					Name:  "out",
					Usage: "write the cloud-config to this file",
				},
			},
			Action: func(c *cli.Context) {
				if c.String("out") != "" {
					formatOut(c.Args().First(), c.String("out"))
				} else if c.Bool("tmp") {
					formatTmp(c.Args().First())
				} else {
					format(c.Args().First())
				}

			},
		},
		{
			Name:  "inject",
			Usage: "inject a file into another, first removing from line1 to line2",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "format",
					Usage: "format the cloud-config before injecting it",
				},
			},
			Action: func(c *cli.Context) {
				start, _ := strconv.ParseInt(c.Args().Get(2), 0, 0)
				end, _ := strconv.ParseInt(c.Args().Get(3), 0, 0)
				if c.Bool("format") == true {
					injectAndFormat(c.Args().First(), c.Args().Get(1), start, end)
				} else {
					inject(c.Args().First(), c.Args().Get(1), start, end)
				}
			},
		},
	}
	app.Run(os.Args)
}
<commit_msg>fix failure when injection is not run with --format<commit_after>package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"

	"github.com/codegangsta/cli"
)

func format(cc string) {
	fmt.Printf("=> Writing formatted cloud-config to STDOUT...\n")
	file, err := os.Open(cc)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		fmt.Println("\"" + strings.Replace(scanner.Text(), "\"", "\\\"", -1) + "\\n\",")
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}
}

func formatOut(cc, outPath string) {
	fmt.Printf("=> Writing formatted cloud-config to %s...\n", cc)
	file, err := os.Open(cc)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	out, err := os.Create(outPath)
	if err != nil {
		log.Fatal(err)
	}
	defer out.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		out.WriteString("\"" + strings.Replace(scanner.Text(), "\"", "\\\"", -1) + "\\n\",\n")
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}
}

func formatTmp(cc string) *os.File {
	fmt.Printf("=> Writing formatted cloud-config to a temporary file...\n")
	file, err := os.Open(cc)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	tmp, err := ioutil.TempFile("/tmp", "cloudcfg")
	if err != nil {
		log.Fatal(err)
	}

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		tmp.WriteString("\"" + strings.Replace(scanner.Text(), "\"", "\\\"", -1) + "\\n\",\n")
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}

	return tmp
}

func inject(toInject, toReceive string, start, end int64) {
	fmt.Printf("=> Removing lines [%d:%d] in %s...\n", start, end, toReceive)

	// read in the lines to be received
	receiving, err := os.Open(toReceive)
	if err != nil {
		log.Fatal(err)
	}

	receivingScanner := bufio.NewScanner(receiving)
	preLines := ""
	postLines := ""
	var numLines int64 = 0
	for receivingScanner.Scan() {
		numLines++
		if numLines < start {
			preLines = preLines + receivingScanner.Text() + "\n"
		} else if numLines > end {
			postLines = postLines + receivingScanner.Text() + "\n"
		}
	}
	if err := receivingScanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}

	// read in the lines to be injected
	injecting, err := os.Open(toInject)
	if err != nil {
		log.Fatal(err)
	}
	defer injecting.Close()

	injectingScanner := bufio.NewScanner(injecting)
	injectLines := ""
	for injectingScanner.Scan() {
		injectLines = injectLines + injectingScanner.Text() + "\n"
	}
	if err := injectingScanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}

	// make sure that the last line gets special treatment
	k := strings.LastIndex(injectLines, "\\n\",")
	if k != -1 {
		injectLines = injectLines[:k] + "\""
	}

	// combine all parts of the file into one
	fmt.Printf("=> Injecting %s into %s...\n", toInject, toReceive)
	injectedLines := preLines + injectLines + postLines

	// write to the receiving file
	receiving.Close()
	receivingWrite, err := os.Create(toReceive)
	if err != nil {
		log.Fatal(err)
	}
	defer receivingWrite.Close()
	receivingWrite.WriteString(injectedLines)
	receivingWrite.Sync()
}

func injectAndFormat(toInject, toReceive string, start, end int64) {
	tmpFile := formatTmp(toInject).Name()
	inject(tmpFile, toReceive, start, end)
	defer os.Remove(tmpFile)
}

func main() {
	app := cli.NewApp()
	app.Name = "cumulus"
	app.Authors = []cli.Author{
		{
			Name:  "Ben Bailey",
			Email: "bennettbailey@gmail.com",
		},
	}
	app.Version = "1.0"
	app.Usage = "inject cloud-configs into cloud-formation templates"
	app.Commands = []cli.Command{
		{
			Name:  "format",
			Usage: "format a cloud-config file into an acceptable JSON structure",
			Flags: []cli.Flag{
				cli.BoolTFlag{
					Name:  "tmp",
					Usage: "write the cloud-config to a tmp file",
				},
				cli.StringFlag{
					Name:  "out",
					Usage: "write the cloud-config to this file",
				},
			},
			Action: func(c *cli.Context) {
				if c.String("out") != "" {
					formatOut(c.Args().First(), c.String("out"))
				} else if c.Bool("tmp") {
					formatTmp(c.Args().First())
				} else {
					format(c.Args().First())
				}

			},
		},
		{
			Name:  "inject",
			Usage: "inject a file into another, first removing from line1 to line2",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "format",
					Usage: "format the cloud-config before injecting it",
				},
			},
			Action: func(c *cli.Context) {
				start, err := strconv.ParseInt(c.Args().Get(2), 0, 0)
				if err != nil {
					log.Fatal("start must be an integer")
				}
				end, err := strconv.ParseInt(c.Args().Get(3), 0, 0)
				if err != nil {
					log.Fatal("start must be an integer")
				}
				if c.Bool("format") == true {
					injectAndFormat(c.Args().First(), c.Args().Get(1), start, end)
				} else {
					inject(c.Args().First(), c.Args().Get(1), start, end)
				}
			},
		},
	}
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/Luzifer/dockerproxy/sni"
	"github.com/fsouza/go-dockerclient"
	"gopkg.in/elazarl/goproxy.v1"
)

func loadConfig(configFile *string) config {
	file, e := ioutil.ReadFile(*configFile)
	if e != nil {
		fmt.Printf("File error: %v\n", e)
		os.Exit(1)
	}
	var cfg config
	err := json.Unmarshal(file, &cfg)
	if err != nil {
		fmt.Printf("JSON error: %v\n", err)
		os.Exit(1)

	}
	return cfg
}

func collectDockerContainer(cfg *config) map[string]string {
	result := make(map[string]string)
	for dockerHostPrivate, dockerHost := range cfg.Docker.Hosts {
		endpoint := fmt.Sprintf("tcp://%s:%d", dockerHostPrivate, cfg.Docker.Port)
		client, _ := docker.NewClient(endpoint)
		containers, _ := client.ListContainers(docker.ListContainersOptions{})
		for _, apiContainer := range containers {
			container, _ := client.InspectContainer(apiContainer.ID)
			currentEnv := make(map[string]string)
			for _, envVar := range container.Config.Env {
				var k, v string
				unpack(strings.Split(envVar, "="), &k, &v)
				currentEnv[k] = v
			}
			if slug, ok := currentEnv["ROUTER_SLUG"]; ok {
				port := currentEnv["ROUTER_PORT"]
				result[slug] = fmt.Sprintf("%s:%s", dockerHost, port)
			}
		}
	}

	return result
}

func unpack(s []string, vars ...*string) {
	for i, str := range s {
		*vars[i] = str
	}
}

func main() {
	var configFile = flag.String("configfile", "./config.json", "Location of the configuration file")
	flag.Parse()

	proxy := goproxy.NewProxyHttpServer()
	cfg := loadConfig(configFile)
	containers := collectDockerContainer(&cfg)

	// We are not really a proxy but act as a HTTP(s) server who delivers remote pages
	proxy.NonproxyHandler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		slug := ""
		// Host is defined and slug has been found
		if host, ok := cfg.Domains[req.Host]; ok {
			slug = host.Slug

			if host.ForceSSL && req.TLS == nil {
				req.URL.Scheme = "https"
				req.URL.Host = req.Host
				http.Redirect(w, req, req.URL.String(), 301)
				return
			}
		}
		// Host is a generic host
		if strings.HasSuffix(req.Host, cfg.Generic) {
			slug = strings.Replace(req.Host, cfg.Generic, "", -1)
		}
		// We found a valid slug before?
		if target, ok := containers[slug]; ok {
			req.URL.Scheme = "http"
			req.URL.Host = target

			proxy.ServeHTTP(w, req)
		} else {
			http.Error(w, "This host is currently not available", 502)
		}
	})

	var certs []sni.Certificates
	for _, domain := range cfg.Domains {
		if domain.SSL.Cert != "" {
			certs = append(certs, sni.Certificates{
				CertFile: domain.SSL.Cert,
				KeyFile:  domain.SSL.Key,
			})
		}
	}

	httpChan := make(chan error)
	httpsChan := make(chan error)
	loaderChan := time.NewTicker(time.Minute).C

	go func(proxy *goproxy.ProxyHttpServer) {
		httpChan <- http.ListenAndServe(cfg.ListenHTTP, proxy)
	}(proxy)

	go func(*goproxy.ProxyHttpServer) {
		httpsServer := &http.Server{
			Handler: proxy,
			Addr:    cfg.ListenHTTPS,
		}

		httpsChan <- sni.ListenAndServeTLSSNI(httpsServer, certs)
	}(proxy)

	for {
		select {
		case httpErr := <-httpChan:
			log.Fatal(httpErr)
		case httpsErr := <-httpsChan:
			log.Fatal(httpsErr)
		case <-loaderChan:
			cfg = loadConfig(configFile)
			containers = collectDockerContainer(&cfg)
		}
	}
}
<commit_msg>Disabled proxy functions as we are not a proxy<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/Luzifer/dockerproxy/sni"
	"github.com/fsouza/go-dockerclient"
	"gopkg.in/elazarl/goproxy.v1"
)

func loadConfig(configFile *string) config {
	file, e := ioutil.ReadFile(*configFile)
	if e != nil {
		fmt.Printf("File error: %v\n", e)
		os.Exit(1)
	}
	var cfg config
	err := json.Unmarshal(file, &cfg)
	if err != nil {
		fmt.Printf("JSON error: %v\n", err)
		os.Exit(1)

	}
	return cfg
}

func collectDockerContainer(cfg *config) map[string]string {
	result := make(map[string]string)
	for dockerHostPrivate, dockerHost := range cfg.Docker.Hosts {
		endpoint := fmt.Sprintf("tcp://%s:%d", dockerHostPrivate, cfg.Docker.Port)
		client, _ := docker.NewClient(endpoint)
		containers, _ := client.ListContainers(docker.ListContainersOptions{})
		for _, apiContainer := range containers {
			container, _ := client.InspectContainer(apiContainer.ID)
			currentEnv := make(map[string]string)
			for _, envVar := range container.Config.Env {
				var k, v string
				unpack(strings.Split(envVar, "="), &k, &v)
				currentEnv[k] = v
			}
			if slug, ok := currentEnv["ROUTER_SLUG"]; ok {
				port := currentEnv["ROUTER_PORT"]
				result[slug] = fmt.Sprintf("%s:%s", dockerHost, port)
			}
		}
	}

	return result
}

func unpack(s []string, vars ...*string) {
	for i, str := range s {
		*vars[i] = str
	}
}

func main() {
	var configFile = flag.String("configfile", "./config.json", "Location of the configuration file")
	flag.Parse()

	proxy := goproxy.NewProxyHttpServer()
	cfg := loadConfig(configFile)
	containers := collectDockerContainer(&cfg)

	proxy.OnRequest().HandleConnect(goproxy.AlwaysReject)

	// We are not really a proxy but act as a HTTP(s) server who delivers remote pages
	proxy.NonproxyHandler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		slug := ""
		// Host is defined and slug has been found
		if host, ok := cfg.Domains[req.Host]; ok {
			slug = host.Slug

			if host.ForceSSL && req.TLS == nil {
				req.URL.Scheme = "https"
				req.URL.Host = req.Host
				http.Redirect(w, req, req.URL.String(), 301)
				return
			}
		}
		// Host is a generic host
		if strings.HasSuffix(req.Host, cfg.Generic) {
			slug = strings.Replace(req.Host, cfg.Generic, "", -1)
		}
		// We found a valid slug before?
		if target, ok := containers[slug]; ok && slug != "" {
			req.URL.Scheme = "http"
			req.URL.Host = target

			proxy.ServeHTTP(w, req)
		} else {
			http.Error(w, "This host is currently not available", 502)
		}
	})

	var certs []sni.Certificates
	for _, domain := range cfg.Domains {
		if domain.SSL.Cert != "" {
			certs = append(certs, sni.Certificates{
				CertFile: domain.SSL.Cert,
				KeyFile:  domain.SSL.Key,
			})
		}
	}

	httpChan := make(chan error)
	httpsChan := make(chan error)
	loaderChan := time.NewTicker(time.Minute).C

	go func(proxy *goproxy.ProxyHttpServer) {
		httpChan <- http.ListenAndServe(cfg.ListenHTTP, proxy)
	}(proxy)

	go func(*goproxy.ProxyHttpServer) {
		httpsServer := &http.Server{
			Handler: proxy,
			Addr:    cfg.ListenHTTPS,
		}

		httpsChan <- sni.ListenAndServeTLSSNI(httpsServer, certs)
	}(proxy)

	for {
		select {
		case httpErr := <-httpChan:
			log.Fatal(httpErr)
		case httpsErr := <-httpsChan:
			log.Fatal(httpsErr)
		case <-loaderChan:
			cfg = loadConfig(configFile)
			containers = collectDockerContainer(&cfg)
		}
	}
}
<|endoftext|>