<commit_before>package main

import (
	"fmt"
	"net/http"
	"os"
	"time"

	"github.com/alienth/fastlyctl/util"
	"github.com/alienth/go-fastly"
	"github.com/urfave/cli"
	"gopkg.in/mcuadros/go-syslog.v2"

	_ "expvar"
	_ "net/http/pprof"
)

var client *fastly.Client
var ipLists IPLists
var hook hookService
var noop bool

var syslogChannel syslog.LogPartsChannel

// How many syslog logs we want to buffer, at max.
const syslogChannelBufferSize = 3000

// TODO pass this along in context to the webserver instead of
// making it global.
var hits = hitMap{m: make(map[string]*ipRate)}

func main() {
	app := cli.NewApp()
	app.Name = "fastly-ratelimit"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config, c",
			Usage: "Read config `FILE`.",
			Value: "config.toml",
		},
		cli.StringFlag{
			Name:  "listen, l",
			Usage: "Specify listen `ADDRESS:PORT`.",
			Value: "0.0.0.0:514",
		},
		cli.StringFlag{
			Name:   "fastly-key, K",
			Usage:  "Fastly API Key. Can be read from 'fastly_key' file in CWD.",
			EnvVar: "FASTLY_KEY",
			Value:  util.GetFastlyKey(),
		},
		cli.BoolFlag{
			Name:  "noop, n",
			Usage: "Noop mode. Print what we'd do, but don't actually do anything.",
		},
	}
	app.Before = func(c *cli.Context) error {
		if len(c.Args()) > 0 {
			return cli.NewExitError("Invalid usage. More arguments received than expected.", -1)
		}
		return nil
	}
	app.Action = runServer

	app.Run(os.Args)
}

func runServer(c *cli.Context) error {
	http.HandleFunc("/", handler)
	go http.ListenAndServe(":80", nil)
	client = fastly.NewClient(nil, c.GlobalString("fastly-key"))
	syslogChannel = make(syslog.LogPartsChannel, syslogChannelBufferSize)
	handler := syslog.NewChannelHandler(syslogChannel)

	noop = c.GlobalBool("noop")

	config, err := readConfig(c.GlobalString("config"))
	if err != nil {
		return cli.NewExitError(fmt.Sprintf("Error reading config file:\n%s\n", err), -1)
	}
	ipLists = config.Lists
	hook = config.HookService
	hook.hookedIPs.m = make(map[string]bool)

	serviceDomains, err := getServiceDomains()
	if err != nil {
		return cli.NewExitError(fmt.Sprintf("Error fetching fasty domains:\n%s\n", err), -1)
	}

	if err := hits.importIPRates(serviceDomains); err != nil {
		return cli.NewExitError(fmt.Sprintf("Error importing existing IP rates: %s", err), -1)
	}
	go hits.expireRecords()
	go hits.expireLimits()
	go queueFanout()
	if hook.SyncIPsUri != "" {
		go hits.syncIPsWithHook()
	}

	server := syslog.NewServer()
	server.SetFormat(syslog.RFC3164)
	server.SetHandler(handler)
	if err := server.ListenUDP(c.GlobalString("listen")); err != nil {
		return cli.NewExitError(fmt.Sprintf("Unable to listen: %s\n", err), -1)
	}
	if err := server.Boot(); err != nil {
		return cli.NewExitError(fmt.Sprintf("Unable to start server: %s\n", err), -1)
	}

	go readLogs(syslogChannel, serviceDomains)

	server.Wait()

	return nil
}

func readLogs(channel syslog.LogPartsChannel, serviceDomains ServiceDomains) {
	for logParts := range channel {
		var line string
		var ok bool
		if line, ok = logParts["content"].(string); !ok || line == "" {
			continue
		}
		log := parseLog(line)
		if log == nil || log.cdnIP == nil || log.clientIP == nil {
			continue
		}
		if time.Now().Sub(log.timestamp) > time.Duration(2)*time.Minute {
			fmt.Printf("Warning: old log line. Log TS: %s, Current time: %s\n", log.timestamp.String(), time.Now().String())
		}
		var ipr *ipRate
		var found bool
		ts := time.Now()
		hits.Lock()
		if d := ts.Sub(time.Now()); d > time.Duration(1)*time.Second {
			fmt.Printf("Blocked for %d seconds waiting for hits lock\n", int(d.Seconds()))
		}
		if ipr, found = hits.m[log.cdnIP.String()]; !found {
			ipr = ipLists.getRate(log.cdnIP)
			hits.m[log.cdnIP.String()] = ipr
		}
		hits.Unlock()
		service, err := serviceDomains.getServiceByHost(log.host.Value)
		if err != nil {
			fmt.Printf("Error while finding fastly service for domain %s: %s\n.", log.host.Value, err)
		}
		if service == nil {
			fmt.Printf("Found request for host %s which is not in fastly. Ignoring\n", log.host.Value)
			continue
		}
		dimension := ipr.list.getDimension(log, service)
		overLimit := ipr.Hit(log.timestamp, dimension)
		if overLimit {
			if err := ipr.Limit(service); err != nil {
				fmt.Printf("Error limiting IP: %s\n", err)
			}
		}

		if len(channel) == syslogChannelBufferSize {
			fmt.Println("Warning: log buffer full. We are dropping logs.")
		}
	}

}
<commit_msg>Allow firing up of multiple workers.<commit_after>package main

import (
	"fmt"
	"net/http"
	"os"
	"time"

	"github.com/alienth/fastlyctl/util"
	"github.com/alienth/go-fastly"
	"github.com/urfave/cli"
	"gopkg.in/mcuadros/go-syslog.v2"

	_ "expvar"
	_ "net/http/pprof"
)

var client *fastly.Client
var ipLists IPLists
var hook hookService
var noop bool

var syslogChannel syslog.LogPartsChannel

// How many syslog logs we want to buffer, at max.
const syslogChannelBufferSize = 3000

// How many log reading workers we fire up
const workers = 1

// TODO pass this along in context to the webserver instead of
// making it global.
var hits = hitMap{m: make(map[string]*ipRate)}

func main() {
	app := cli.NewApp()
	app.Name = "fastly-ratelimit"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config, c",
			Usage: "Read config `FILE`.",
			Value: "config.toml",
		},
		cli.StringFlag{
			Name:  "listen, l",
			Usage: "Specify listen `ADDRESS:PORT`.",
			Value: "0.0.0.0:514",
		},
		cli.StringFlag{
			Name:   "fastly-key, K",
			Usage:  "Fastly API Key. Can be read from 'fastly_key' file in CWD.",
			EnvVar: "FASTLY_KEY",
			Value:  util.GetFastlyKey(),
		},
		cli.BoolFlag{
			Name:  "noop, n",
			Usage: "Noop mode. Print what we'd do, but don't actually do anything.",
		},
	}
	app.Before = func(c *cli.Context) error {
		if len(c.Args()) > 0 {
			return cli.NewExitError("Invalid usage. More arguments received than expected.", -1)
		}
		return nil
	}
	app.Action = runServer

	app.Run(os.Args)
}

func runServer(c *cli.Context) error {
	http.HandleFunc("/", handler)
	go http.ListenAndServe(":80", nil)
	client = fastly.NewClient(nil, c.GlobalString("fastly-key"))
	syslogChannel = make(syslog.LogPartsChannel, syslogChannelBufferSize)
	handler := syslog.NewChannelHandler(syslogChannel)

	noop = c.GlobalBool("noop")

	config, err := readConfig(c.GlobalString("config"))
	if err != nil {
		return cli.NewExitError(fmt.Sprintf("Error reading config file:\n%s\n", err), -1)
	}
	ipLists = config.Lists
	hook = config.HookService
	hook.hookedIPs.m = make(map[string]bool)

	serviceDomains, err := getServiceDomains()
	if err != nil {
		return cli.NewExitError(fmt.Sprintf("Error fetching fasty domains:\n%s\n", err), -1)
	}

	if err := hits.importIPRates(serviceDomains); err != nil {
		return cli.NewExitError(fmt.Sprintf("Error importing existing IP rates: %s", err), -1)
	}
	go hits.expireRecords()
	go hits.expireLimits()
	go queueFanout()
	if hook.SyncIPsUri != "" {
		go hits.syncIPsWithHook()
	}

	server := syslog.NewServer()
	server.SetFormat(syslog.RFC3164)
	server.SetHandler(handler)
	if err := server.ListenUDP(c.GlobalString("listen")); err != nil {
		return cli.NewExitError(fmt.Sprintf("Unable to listen: %s\n", err), -1)
	}
	if err := server.Boot(); err != nil {
		return cli.NewExitError(fmt.Sprintf("Unable to start server: %s\n", err), -1)
	}

	for i := 1; i <= workers; i++ {
		go readLogs(syslogChannel, serviceDomains)
	}

	server.Wait()

	return nil
}

func readLogs(channel syslog.LogPartsChannel, serviceDomains ServiceDomains) {
	for logParts := range channel {
		var line string
		var ok bool
		if line, ok = logParts["content"].(string); !ok || line == "" {
			continue
		}
		log := parseLog(line)
		if log == nil || log.cdnIP == nil || log.clientIP == nil {
			continue
		}
		if time.Now().Sub(log.timestamp) > time.Duration(2)*time.Minute {
			fmt.Printf("Warning: old log line. Log TS: %s, Current time: %s\n", log.timestamp.String(), time.Now().String())
		}
		var ipr *ipRate
		var found bool
		ts := time.Now()
		hits.Lock()
		if d := ts.Sub(time.Now()); d > time.Duration(1)*time.Second {
			fmt.Printf("Blocked for %d seconds waiting for hits lock\n", int(d.Seconds()))
		}
		if ipr, found = hits.m[log.cdnIP.String()]; !found {
			ipr = ipLists.getRate(log.cdnIP)
			hits.m[log.cdnIP.String()] = ipr
		}
		hits.Unlock()
		service, err := serviceDomains.getServiceByHost(log.host.Value)
		if err != nil {
			fmt.Printf("Error while finding fastly service for domain %s: %s\n.", log.host.Value, err)
		}
		if service == nil {
			fmt.Printf("Found request for host %s which is not in fastly. Ignoring\n", log.host.Value)
			continue
		}
		dimension := ipr.list.getDimension(log, service)
		overLimit := ipr.Hit(log.timestamp, dimension)
		if overLimit {
			if err := ipr.Limit(service); err != nil {
				fmt.Printf("Error limiting IP: %s\n", err)
			}
		}

		if len(channel) == syslogChannelBufferSize {
			fmt.Println("Warning: log buffer full. We are dropping logs.")
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"net/url"
	"os"

	"github.com/ChimeraCoder/anaconda"
	"github.com/sirupsen/logrus"
)

var (
	consumerKey       = getenv("TWITTER_CONSUMER_KEY")
	consumerSecret    = getenv("TWITTER_CONSUMER_SECRET")
	accessToken       = getenv("TWITTER_ACCESS_TOKEN")
	accessTokenSecret = getenv("TWITTER_ACCESS_TOKEN_SECRET")
)

func getenv(name string) string {
	v := os.Getenv(name)
	if v == "" {
		panic("Missing required environment variable: " + name)
	}
	return v
}

func main() {
	anaconda.SetConsumerKey(consumerKey)
	anaconda.SetConsumerSecret(consumerSecret)
	api := anaconda.NewTwitterApi(accessToken, accessTokenSecret)

	log := &logger{logrus.New()}
	api.SetLogger(log)

	stream := api.PublicStreamFilter(url.Values{
		"track": []string{"#golang", "#justforfunc"},
	})

	defer stream.Stop()

	for v := range stream.C {
		tweet, ok := v.(anaconda.Tweet)
		if !ok {
			log.Warningf("Received unexpected value of %T", v)
			continue
		}

		if tweet.RetweetedStatus != nil {
			continue
		}

		_, err := api.Retweet(tweet.Id, false)
		if err != nil {
			log.Errorf("Could not retweet %d: %v", tweet.Id, err)
			continue
		}
		log.Infof("Retweeted %d", tweet.Id)
	}
}

type logger struct {
	*logrus.Logger
}

func (log *logger) Critical(args ...interface{})                 { log.Error(args...) }
func (log *logger) Criticalf(format string, args ...interface{}) { log.Errorf(format, args...) }
func (log *logger) Notice(args ...interface{})                   { log.Info(args...) }
func (log *logger) Noticef(format string, args ...interface{})   { log.Infof(format, args...) }
<commit_msg>Check if tweet is possibly sensitive or username is unwanted for a retweet<commit_after>package main

import (
	"net/url"
	"os"

	"github.com/ChimeraCoder/anaconda"
	"github.com/sirupsen/logrus"
	"fmt"
)

var (
	consumerKey       = getenv("TWITTER_CONSUMER_KEY")
	consumerSecret    = getenv("TWITTER_CONSUMER_SECRET")
	accessToken       = getenv("TWITTER_ACCESS_TOKEN")
	accessTokenSecret = getenv("TWITTER_ACCESS_TOKEN_SECRET")
)

func getenv(name string) string {
	v := os.Getenv(name)
	if v == "" {
		panic("Missing required environment variable: " + name)
	}
	return v
}

func main() {
	anaconda.SetConsumerKey(consumerKey)
	anaconda.SetConsumerSecret(consumerSecret)
	api := anaconda.NewTwitterApi(accessToken, accessTokenSecret)

	log := &logger{logrus.New()}
	api.SetLogger(log)

	stream := api.PublicStreamFilter(url.Values{
		"track": []string{"#golang", "#justforfunc"},
	})

	defer stream.Stop()

	for v := range stream.C {
		tweet, ok := v.(anaconda.Tweet)
		if !ok {
			log.Warningf("Received unexpected value of %T", v)
			continue
		}

		if tweet.RetweetedStatus != nil || tweet.User.ScreenName == "tamd87" || tweet.PossiblySensitive {
			continue
		}

		_, err := api.Retweet(tweet.Id, false)
		if err != nil {
			log.Errorf("Could not retweet %d: %v", tweet.Id, err)
			continue
		}
		log.Infof("Retweeted %d", tweet.Id)
	}
}

type logger struct {
	*logrus.Logger
}

func (log *logger) Critical(args ...interface{})                 { log.Error(args...) }
func (log *logger) Criticalf(format string, args ...interface{}) { log.Errorf(format, args...) }
func (log *logger) Notice(args ...interface{})                   { log.Info(args...) }
func (log *logger) Noticef(format string, args ...interface{})   { log.Infof(format, args...) }
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"sync"

	_ "github.com/lib/pq"
)

var workingDir string = path.Join(os.TempDir(), "apollo-audio")
var clipDir string = path.Join(workingDir, "clips")
var AAC string = "aac"
var M4A string = "m4a"
var OGG string = "ogg"

/* container for the request variables */
type RequestVars struct {
	mission  int
	channels []string
	format   string
	start    int
	duration int
}

/* local paths to audio files for all channels belonging to particular time slice */
type TimeSlice struct {
	start    int
	end      int
	location []string
}

type AudioChunk struct {
	start     int
	end       int
	url       string
	localPath string
	channel   int
}

type DatabaseVars struct {
	DB_HOST     string `json:"DB_HOST"`
	DB_PORT     int    `json:"DB_PORT"`
	DB_USER     string `json:"DB_USER"`
	DB_PASSWORD string `json:"DB_PASSWORD"`
	DB_NAME     string `json:"DB_NAME"`
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return true, err
}

func makeDir(dir string) {
	dirExists, err := exists(dir)
	check(err)
	if !dirExists {
		err := os.Mkdir(dir, 0777)
		check(err)
	}
}

func downloadAllAudio(chunkMap map[int][]AudioChunk) {
	// Parallel download
	var wg sync.WaitGroup
	itemQ := make(chan AudioChunk)
	workerCount := 2
	// launch workers
	for a := 0; a < workerCount; a++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for item := range itemQ {
				downloadFromS3AndSave(item.url, item.localPath)
			}
		}()
	}
	// Add items
	for _, a := range chunkMap {
		for _, b := range a {
			itemQ <- b
		}
	}
	close(itemQ)
	wg.Wait()
}

func downloadFromS3AndSave(url string, filename string) string {
	log.Println("Downloading", url, "to", filename)
	clipPath := filename
	if _, err := os.Stat(clipPath); err == nil {
		log.Println("file exists; skipping")
		return clipPath
	}

	out, err := os.Create(clipPath)
	check(err)
	defer out.Close()
	resp, err := http.Get(url)
	check(err)
	defer resp.Body.Close()
	_, err = io.Copy(out, resp.Body)
	check(err)
	return clipPath
}

type flushWriter struct {
	f http.Flusher
	w io.Writer
}

func (fw *flushWriter) Write(p []byte) (n int, err error) {
	n, err = fw.w.Write(p)
	if fw.f != nil {
		fw.f.Flush()
	}
	return
}

func parseParameters(r *http.Request) (RequestVars, error) {
	var rv RequestVars
	r.ParseForm()
	log.Println(r.Form)

	// Handle empty request
	if len(r.Form) == 0 {
		log.Println("Not enough request params")
		return rv, errors.New("bad request")
	}

	missionId := r.PostFormValue("mission")
	tempMission, err := strconv.Atoi(missionId)
	if err != nil {
		log.Println("invalid mission id:", missionId)
		return rv, err
	}
	rv.mission = tempMission
	rv.channels = r.Form["channels"]
	tempFormat := r.PostFormValue("format")
	rv.format = tempFormat

	tempStart, err := strconv.Atoi(r.PostFormValue("start"))
	check(err)
	rv.start = tempStart

	tempDuration, err := strconv.Atoi(r.PostFormValue("duration"))
	check(err)
	rv.duration = tempDuration

	return rv, nil
}

func getLocations(rv RequestVars) map[int][]AudioChunk {

	//var slices []TimeSlice
	chunkMap := make(map[int][]AudioChunk)

	dbjson, err := ioutil.ReadFile("./config.json")
	check(err)

	var dbvars DatabaseVars
	err = json.Unmarshal(dbjson, &dbvars)
	check(err)

	dbcreds := dbvars
	dbinfo := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", dbcreds.DB_HOST, dbcreds.DB_PORT, dbcreds.DB_USER, dbcreds.DB_PASSWORD, dbcreds.DB_NAME)
	db, err := sql.Open("postgres", dbinfo)
	check(err)
	defer db.Close()

	channelString := fmt.Sprintf("{%s}", strings.Join(rv.channels, ","))

	stmt, err := db.Prepare("SELECT met_start, met_end, url, channel FROM channel_chunks WHERE channel = ANY($1::integer[]) AND met_end > $2 AND met_start < $3")
	check(err)

	reqEnd := rv.start + rv.duration
	rows, err := stmt.Query(channelString, rv.start, reqEnd)
	check(err)

	defer rows.Close()

	for rows.Next() {
		var chunk AudioChunk
		err = rows.Scan(&chunk.start, &chunk.end, &chunk.url, &chunk.channel)
		if err == nil {
			// Set local name
			loc := fmt.Sprintf("mission_%d_channel_%d_%d.wav", rv.mission, chunk.channel, chunk.start)
			chunk.localPath = path.Join(clipDir, loc)
			chunkMap[chunk.channel] = append(chunkMap[chunk.channel], chunk)
		}
	}
	return chunkMap
}

func streamHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-type", "audio/mpeg")

	/* PARRRAMETERS */
	rv, err := parseParameters(r)
	// Handle bad params
	if err != nil {
		log.Println("Error processing params:", err)
		http.Error(w, http.StatusText(500), 500)
		return
	}
	// All clear
	log.Println("Handling request for ", rv)

	/* DEEBEE */
	chunkMap := getLocations(rv)
	// Check for no audio
	if len(chunkMap) == 0 {
		http.Error(w, http.StatusText(404), 404)
		return
	}
	downloadAllAudio(chunkMap)

	sox, err := exec.LookPath("sox")
	check(err)
	log.Println("using sox " + sox)
	ffmpeg, err := exec.LookPath("ffmpeg")
	check(err)
	log.Println("using ffmpeg " + ffmpeg)

	var chunkPaths []string
	// concat chunks of the same channel
	for _, a := range chunkMap {
		// Grab first one for now
		chunkPaths = append(chunkPaths, a[0].localPath)
	}

	// Merge the channels
	soxArgs := []string{"-t", "wav"}
	// Only merge if there are multiple files
	if len(chunkPaths) > 1 {
		soxArgs = append(soxArgs, "-m")
	}
	soxArgs = append(soxArgs, chunkPaths...)
	soxArgs = append(soxArgs, "-p")
	log.Println("running sox", strings.Join(soxArgs, " "))
	soxCommand := exec.Command(sox, soxArgs...)

	// Transcode the result
	var ffmpegArgs []string
	if rv.format == AAC || rv.format == M4A {
		ffmpegArgs = []string{"-i", "-", "-c:a", "libfdk_aac", "-b:a", "256k", "-f", M4A, "pipe:"}
		// works, but gotta compile ffmpeg on server with special options
	} else if rv.format == OGG {
		ffmpegArgs = []string{"-i", "-", "-c:a", "libvorbis", "-qscale:a", "6", "-f", OGG, "pipe:"}
	} else {
		log.Println("unsupported output format requested. break some rools.")
		ffmpegArgs = []string{"-i", "-", "-f", "mp3", "-ab", "256k", "pipe:"}
	}
	log.Println("running ffmpeg", strings.Join(ffmpegArgs, " "))
	ffmpegCommand := exec.Command(ffmpeg, ffmpegArgs...)

	fw := flushWriter{w: w}
	if f, ok := w.(http.Flusher); ok {
		fw.f = f
	}

	ffmpegCommand.Stdin, _ = soxCommand.StdoutPipe()
	ffmpegCommand.Stdout = &fw
	ffmpegCommand.Stderr = os.Stdout

	ffmpegCommand.Start()
	soxCommand.Run()
	ffmpegCommand.Wait()

	log.Println("done")
}

func main() {
	makeDir(workingDir)
	makeDir(clipDir)
	http.HandleFunc("/stream", streamHandler)
	ServerPort := "5000" // default port
	if len(os.Getenv("PORT")) > 0 {
		ServerPort = os.Getenv("PORT")
	}
	log.Println("Starting server on " + ServerPort)
	http.ListenAndServe(":"+ServerPort, nil)
}
<commit_msg>Combine with previous approach for sequential chunk streaming.<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"sync"

	_ "github.com/lib/pq"
)

var workingDir string = path.Join(os.TempDir(), "apollo-audio")
var clipDir string = path.Join(workingDir, "clips")
var AAC string = "aac"
var M4A string = "m4a"
var OGG string = "ogg"

/* container for the request variables */
type RequestVars struct {
	mission  int
	channels []string
	format   string
	start    int
	duration int
}

/* local paths to audio files for all channels belonging to particular time slice */
type TimeSlice struct {
	start  int
	end    int
	chunks map[int]AudioChunk
}

type AudioChunk struct {
	start     int
	end       int
	url       string
	localPath string
	channel   int
}

type DatabaseVars struct {
	DB_HOST     string `json:"DB_HOST"`
	DB_PORT     int    `json:"DB_PORT"`
	DB_USER     string `json:"DB_USER"`
	DB_PASSWORD string `json:"DB_PASSWORD"`
	DB_NAME     string `json:"DB_NAME"`
}

func NewTimeSlice(start, end int) TimeSlice {
	chunks := make(map[int]AudioChunk)
	return TimeSlice{start, end, chunks}
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return true, err
}

func makeDir(dir string) {
	dirExists, err := exists(dir)
	check(err)
	if !dirExists {
		err := os.Mkdir(dir, 0777)
		check(err)
	}
}

func downloadAllAudio(timeSlices []TimeSlice) {
	// Parallel download
	var wg sync.WaitGroup
	itemQ := make(chan AudioChunk)
	workerCount := 2
	// launch workers
	for a := 0; a < workerCount; a++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for item := range itemQ {
				downloadFromS3AndSave(item.url, item.localPath)
			}
		}()
	}
	// Add items
	for _, a := range timeSlices {
		for _, b := range a.chunks {
			itemQ <- b
		}
	}
	close(itemQ)
	wg.Wait()
}

func downloadFromS3AndSave(url string, filename string) string {
	log.Println("Downloading", url, "to", filename)
	clipPath := filename
	if _, err := os.Stat(clipPath); err == nil {
		log.Println("file exists; skipping")
		return clipPath
	}

	out, err := os.Create(clipPath)
	check(err)
	defer out.Close()
	resp, err := http.Get(url)
	check(err)
	defer resp.Body.Close()
	_, err = io.Copy(out, resp.Body)
	check(err)
	return clipPath
}

type flushWriter struct {
	f http.Flusher
	w io.Writer
}

func (fw *flushWriter) Write(p []byte) (n int, err error) {
	n, err = fw.w.Write(p)
	if fw.f != nil {
		fw.f.Flush()
	}
	return
}

func parseParameters(r *http.Request) (RequestVars, error) {
	var rv RequestVars
	r.ParseForm()
	log.Println(r.Form)

	// Handle empty request
	if len(r.Form) == 0 {
		log.Println("Not enough request params")
		return rv, errors.New("bad request")
	}

	missionId := r.PostFormValue("mission")
	tempMission, err := strconv.Atoi(missionId)
	if err != nil {
		log.Println("invalid mission id:", missionId)
		return rv, err
	}
	rv.mission = tempMission
	rv.channels = r.Form["channels"]
	tempFormat := r.PostFormValue("format")
	rv.format = tempFormat

	tempStart, err := strconv.Atoi(r.PostFormValue("start"))
	check(err)
	rv.start = tempStart

	tempDuration, err := strconv.Atoi(r.PostFormValue("duration"))
	check(err)
	rv.duration = tempDuration

	return rv, nil
}

func getRequestSlices(rv RequestVars) []TimeSlice {
	var slices []TimeSlice
	var dbvars DatabaseVars

	// Read config
	dbjson, err := ioutil.ReadFile("./config.json")
	check(err)
	err = json.Unmarshal(dbjson, &dbvars)
	check(err)

	// Connect
	dbinfo := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", dbvars.DB_HOST, dbvars.DB_PORT, dbvars.DB_USER, dbvars.DB_PASSWORD, dbvars.DB_NAME)
	db, err := sql.Open("postgres", dbinfo)
	check(err)
	defer db.Close()

	stmt, err := db.Prepare("SELECT met_start, met_end, url, channel FROM channel_chunks WHERE channel = ANY($1::integer[]) AND met_end > $2 AND met_start < $3 ORDER BY met_start")
	check(err)

	// Psql array for ANY query
	channelString := fmt.Sprintf("{%s}", strings.Join(rv.channels, ","))
	reqEnd := rv.start + rv.duration

	rows, err := stmt.Query(channelString, rv.start, reqEnd)
	check(err)
	defer rows.Close()

	//Scan results and build slices
	lastStart := -1
	for rows.Next() {
		var chunk AudioChunk
		err = rows.Scan(&chunk.start, &chunk.end, &chunk.url, &chunk.channel)
		if err != nil {
			log.Println("Error reading row", err)
			continue
		}
		//Check for new timeslice
		if chunk.start > lastStart {
			slices = append(slices, NewTimeSlice(chunk.start, chunk.end))
			lastStart = chunk.start
		}
		// Set local name
		loc := fmt.Sprintf("mission_%d_channel_%d_%d.wav", rv.mission, chunk.channel, chunk.start)
		chunk.localPath = path.Join(clipDir, loc)

		// Add to proper slice
		for i, a := range slices {
			if chunk.start == a.start && chunk.end == a.end {
				slices[i].chunks[chunk.channel] = chunk
				break
			}
		}
	}
	return slices
}

func streamHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-type", "audio/mpeg")

	/* PARRRAMETERS */
	rv, err := parseParameters(r)
	// Handle bad params
	if err != nil {
		log.Println("Error processing params:", err)
		http.Error(w, http.StatusText(500), 500)
		return
	}
	// All clear
	log.Println("Handling request for ", rv)

	/* DEEBEE */
	slices := getRequestSlices(rv)

	// Check for no audio
	if len(slices) == 0 {
		http.Error(w, http.StatusText(404), 404)
		return
	}
	// Download all here. Could be done concurrently while prev slice is streaming
	downloadAllAudio(slices)

	// Build cmds
	sox, err := exec.LookPath("sox")
	check(err)
	log.Println("using sox " + sox)
	ffmpeg, err := exec.LookPath("ffmpeg")
	check(err)
	log.Println("using ffmpeg " + ffmpeg)

	// Process and stream each chunk
	for _, slice := range slices {
		var chunkPaths []string
		// Gather paths
		for _, ch := range slice.chunks {
			chunkPaths = append(chunkPaths, ch.localPath)
		}
		// Merge the channels
		soxArgs := []string{"-t", "wav"}
		// Only merge if there are multiple files
		if len(chunkPaths) > 1 {
			soxArgs = append(soxArgs, "-m")
		}
		soxArgs = append(soxArgs, chunkPaths...)
		soxArgs = append(soxArgs, "-p")
		log.Println("running sox", strings.Join(soxArgs, " "))
		soxCommand := exec.Command(sox, soxArgs...)

		// Transcode the result
		var ffmpegArgs []string
		if rv.format == AAC || rv.format == M4A {
			ffmpegArgs = []string{"-i", "-", "-c:a", "libfdk_aac", "-b:a", "256k", "-f", M4A, "pipe:"}
			// works, but gotta compile ffmpeg on server with special options
		} else if rv.format == OGG {
			ffmpegArgs = []string{"-i", "-", "-c:a", "libvorbis", "-qscale:a", "6", "-f", OGG, "pipe:"}
		} else {
			log.Println("unsupported output format requested. break some rools.")
			ffmpegArgs = []string{"-i", "-", "-f", "mp3", "-ab", "256k", "pipe:"}
		}
		log.Println("running ffmpeg", strings.Join(ffmpegArgs, " "))
		ffmpegCommand := exec.Command(ffmpeg, ffmpegArgs...)

		fw := flushWriter{w: w}
		if f, ok := w.(http.Flusher); ok {
			fw.f = f
		}

		ffmpegCommand.Stdin, _ = soxCommand.StdoutPipe()
		ffmpegCommand.Stdout = &fw
		ffmpegCommand.Stderr = os.Stdout

		ffmpegCommand.Start()
		soxCommand.Run()
		ffmpegCommand.Wait()
	}

	log.Println("done")
}

func main() {
	makeDir(workingDir)
	makeDir(clipDir)
	http.HandleFunc("/stream", streamHandler)
	ServerPort := "5000" // default port
	if len(os.Getenv("PORT")) > 0 {
		ServerPort = os.Getenv("PORT")
	}
	log.Println("Starting server on " + ServerPort)
	http.ListenAndServe(":"+ServerPort, nil)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

func dumpBucket(label string, bucket *ScriptBucket) {
	fmt.Printf("#\n# Script @%s from %s \n#\n", label, bucket.fileName)
	delimFmt := "#" + strings.Repeat("-", 70) + "#  %s %d\n"
	for i, block := range bucket.script {
		fmt.Printf(delimFmt, "Start", i+1)
		fmt.Printf("echo \"Block '%s' (%d/%d in %s) of %s\"\n####\n",
			block.labels[0], i+1, len(bucket.script), label, bucket.fileName)
		fmt.Print(block.codeText)
		fmt.Printf(delimFmt, "End", i+1)
		fmt.Println()
	}
}

func emitStraightScript(label string, scriptBuckets []*ScriptBucket) {
	for _, bucket := range scriptBuckets {
		dumpBucket(label, bucket)
	}
	fmt.Printf("echo \" \"\n")
	fmt.Printf("echo \"All done.  No errors.\"\n")
}

// Emit the first script normally, then emit it again, as well as the
// the remaining scripts, so that they run in a subshell.
//
// This allows the aggregrate script to be structured as 1) a preamble
// initialization script that impacts the environment of the active
// shell, followed by 2) a script that executes as a subshell that
// exits on error.  An exit in (2) won't cause the active shell (most
// likely a terminal) to close.
//
// The first script must be able to complete without exit on error
// because its not running as a subshell.  So it should just set
// environment variables and/or define shell funtions.
func emitPreambledScript(label string, scriptBuckets []*ScriptBucket) {
	dumpBucket(label, scriptBuckets[0])
	delim := "HANDLED_SCRIPT"
	fmt.Printf(" bash -e <<'%s'\n", delim)
	fmt.Printf("function handledTrouble() {\n")
	fmt.Printf("  echo \" \"\n")
	fmt.Printf("  echo \"Unable to continue!\"\n")
	fmt.Printf("  exit 1\n")
	fmt.Printf("}\n")
	fmt.Printf("trap handledTrouble INT TERM\n")
	emitStraightScript(label, scriptBuckets)
	fmt.Printf("%s\n", delim)
}

func usage() {
	fmt.Fprintf(os.Stderr, "\nUsage:  %s {label} {fileName}...\n", os.Args[0])
	flag.PrintDefaults()
	fmt.Fprintf(os.Stderr,
		`
Reads markdown files, extracts code blocks with a given @label, and
either runs them in a subshell or emits them to stdout.

If the markdown file contains

  Blah blah blah.
  <!-- @goHome @foo -->
  '''
  cd $HOME
  '''
  Blah blah blah.
  <!-- @echoApple @apple -->
  '''
  echo "an apple a day keeps the doctor away"
  '''
  Blah blah blah.
  <!-- @echoCloseStar @foo @baz -->
  '''
  echo "Proxima Centauri"
  '''
  Blah blah blah.

then the command '{this} foo {fileName}' emits:

  cd $HOME
  echo "Proxima Centauri"

Pipe output to 'source /dev/stdin' to run it directly.

Use --subshell to run the blocks in a subshell leaving your current
shell env vars and pwd unchanged.  The code blocks can, however, do
anything to your computer that you can.
`)
}

func main() {
	flag.Usage = usage
	preambled := flag.Bool("preambled", false,
		"Place all scripts in a subshell, preambled by the first script.")
	subshell := flag.Bool("subshell", false,
		"Run extracted blocks in subshell (leaves your env vars and pwd unchanged).")
	swallow := flag.Bool("swallow", false,
		"Swallow errors from subshell (non-zero exit only on problems in driver code).")
	flag.Parse()
	if *swallow && !*subshell {
		fmt.Fprintf(os.Stderr, "Makes no sense to specify --swallow but not --subshell.\n")
		usage()
		os.Exit(1)
	}
	if flag.NArg() < 2 {
		usage()
		os.Exit(1)
	}
	label := flag.Arg(0)
	scriptBuckets := make([]*ScriptBucket, flag.NArg()-1)

	for i := 1; i < flag.NArg(); i++ {
		fileName := flag.Arg(i)
		contents, err := ioutil.ReadFile(fileName)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Unable to read %q\n", fileName)
			usage()
			os.Exit(2)
		}
		m := Parse(string(contents))
		script, ok := m[label]
		if !ok {
			fmt.Fprintf(os.Stderr, "No block labelled %q in file %q.\n", label, fileName)
			os.Exit(3)
		}
		scriptBuckets[i-1] = &ScriptBucket{fileName, script}
	}

	if len(scriptBuckets) < 1 {
		return
	}

	if !*subshell {
		if *preambled {
			emitPreambledScript(label, scriptBuckets)
		} else {
			emitStraightScript(label, scriptBuckets)
		}
		return
	}

	result := RunInSubShell(scriptBuckets)
	if result.err != nil {
		Complain(result, label)
		if !*swallow {
			log.Fatal(result.err)
		}
	}
}
<commit_msg>Add monkey flag<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

// If n <= 0, dump everything, else only dump the first n blocks.
// n is 1 relative, i.e., if you want the first two blocks dumped, pass
// n==2, not n==1.
func dumpBucket(label string, bucket *ScriptBucket, n int) {
	fmt.Printf("#\n# Script @%s from %s \n#\n", label, bucket.fileName)
	delimFmt := "#" + strings.Repeat("-", 70) + "#  %s %d\n"
	for i, block := range bucket.script {
		if n > 0 && i >= n {
			break
		}
		fmt.Printf(delimFmt, "Start", i+1)
		fmt.Printf("echo \"Block '%s' (%d/%d in %s) of %s\"\n####\n",
			block.labels[0], i+1, len(bucket.script), label, bucket.fileName)
		fmt.Print(block.codeText)
		fmt.Printf(delimFmt, "End", i+1)
		fmt.Println()
	}
}

func emitStraightScript(label string, scriptBuckets []*ScriptBucket) {
	for _, bucket := range scriptBuckets {
		dumpBucket(label, bucket, 0)
	}
	fmt.Printf("echo \" \"\n")
	fmt.Printf("echo \"All done.  No errors.\"\n")
}

// Emit the first script normally, then emit it again, as well as the
// the remaining scripts, so that they run in a subshell.
//
// This allows the aggregrate script to be structured as 1) a preamble
// initialization script that impacts the environment of the active
// shell, followed by 2) a script that executes as a subshell that
// exits on error.  An exit in (2) won't cause the active shell (most
// likely a terminal) to close.
//
// The first script must be able to complete without exit on error
// because its not running as a subshell.  So it should just set
// environment variables and/or define shell funtions.
func emitPreambledScript(label string, scriptBuckets []*ScriptBucket, n int) {
	dumpBucket(label, scriptBuckets[0], n)
	delim := "HANDLED_SCRIPT"
	fmt.Printf(" bash -e <<'%s'\n", delim)
	fmt.Printf("function handledTrouble() {\n")
	fmt.Printf("  echo \" \"\n")
	fmt.Printf("  echo \"Unable to continue!\"\n")
	fmt.Printf("  exit 1\n")
	fmt.Printf("}\n")
	fmt.Printf("trap handledTrouble INT TERM\n")
	emitStraightScript(label, scriptBuckets)
	fmt.Printf("%s\n", delim)
}

func usage() {
	fmt.Fprintf(os.Stderr, "\nUsage:  %s {label} {fileName}...\n", os.Args[0])
	flag.PrintDefaults()
	fmt.Fprintf(os.Stderr,
		`
Reads markdown files, extracts code blocks with a given @label, and
either runs them in a subshell or emits them to stdout.

If the markdown file contains

  Blah blah blah.
  <!-- @goHome @foo -->
  '''
  cd $HOME
  '''
  Blah blah blah.
  <!-- @echoApple @apple -->
  '''
  echo "an apple a day keeps the doctor away"
  '''
  Blah blah blah.
  <!-- @echoCloseStar @foo @baz -->
  '''
  echo "Proxima Centauri"
  '''
  Blah blah blah.

then the command '{this} foo {fileName}' emits:

  cd $HOME
  echo "Proxima Centauri"

Pipe output to 'source /dev/stdin' to run it directly.

Use --subshell to run the blocks in a subshell leaving your current
shell env vars and pwd unchanged.  The code blocks can, however, do
anything to your computer that you can.
`)
}

func main() {
	flag.Usage = usage
	preambled := flag.Bool("preambled", false,
		"Place all scripts in a subshell, preambled by the first script.")
	monkey := flag.Int("monkey", -1,
		"Place all scripts in a subshell, preambled by the first {n} blocks in the first script.")
	subshell := flag.Bool("subshell", false,
		"Run extracted blocks in subshell (leaves your env vars and pwd unchanged).")
	swallow := flag.Bool("swallow", false,
		"Swallow errors from subshell (non-zero exit only on problems in driver code).")
	flag.Parse()
	if *swallow && !*subshell {
		fmt.Fprintf(os.Stderr, "Makes no sense to specify --swallow but not --subshell.\n")
		usage()
		os.Exit(1)
	}
	if flag.NArg() < 2 {
		usage()
		os.Exit(1)
	}
	label := flag.Arg(0)
	scriptBuckets := make([]*ScriptBucket, flag.NArg()-1)

	for i := 1; i < flag.NArg(); i++ {
		fileName := flag.Arg(i)
		contents, err := ioutil.ReadFile(fileName)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Unable to read %q\n", fileName)
			usage()
			os.Exit(2)
		}
		m := Parse(string(contents))
		script, ok := m[label]
		if !ok {
			fmt.Fprintf(os.Stderr, "No block labelled %q in file %q.\n", label, fileName)
			os.Exit(3)
		}
		scriptBuckets[i-1] = &ScriptBucket{fileName, script}
	}

	if len(scriptBuckets) < 1 {
		return
	}

	// Plan
	// 1) Leave existing behavior, but push a new mdrip binary with a
	//	  new flag "monkey".  Existing Makefile and tests pass.
	// 2) modify Makefile to use the new "monkey" flag
	//    everything still works.  push new website, using the new flag only.
	// 3) Modify the old (unused) "preambled" flag to work just like the new flag.
	// 4) modify Makefile to use the "preambled" flag in the new way.
	//    Everything still works.  push new website.
	// 5) remove the "monkey" flag from mdrip
	if !*subshell {
		n := *monkey
		if *preambled {
			// Trumps for now
			n = 0
		}
		if *preambled || n >= 0 {
			emitPreambledScript(label, scriptBuckets, n)
		} else {
			emitStraightScript(label, scriptBuckets)
		}
		return
	}

	result := RunInSubShell(scriptBuckets)
	if result.err != nil {
		Complain(result, label)
		if !*swallow {
			log.Fatal(result.err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import "net/http"
import "io/ioutil"
import "os"
import "encoding/xml"
import "log"
import "fmt"
import (
	gj "github.com/kpawlik/geojson"
)

//Record is a single update of a single bus location
type Record struct {
	BusID            string  `xml:"Record>BusID"`
	BusName          string  `xml:"Record>BusName"`
	Latitude         float64 `xml:"Record>Latitude"`
	Longitude        float64 `xml:"Record>Longitude"`
	RouteID          string  `xml:"Record>RouteID"`
	TripID           string  `xml:"Record>TripID"`
	Direction        string  `xml:"Record>Direction"`
	ServiceDirection string  `xml:"Record>ServiceDirection"`
	Service          string  `xml:"Record>Service"`
	ServiceName      string  `xml:"Record>ServiceName"`
	TripHeadsign     string  `xml:"Record>TripHeadsign"`
	LocationUpdated  string  `xml:"Record>LocationUpdated"`
}

func parserecord(r **Record, body []byte) {
	err := xml.Unmarshal(body, r)
	if err != nil {
		log.Fatal("Unmarshall failed", err)
	}

}

func makeFeaturefromRecord(r *Record) string {
	var f *gj.Feature
	lon := gj.Coord(r.Longitude)
	lat :=  gj.Coord(r.Latitude)
	p := gj.NewPoint(gj.Coordinate{lon, lat})
	f = gj.NewFeature(p, nil, nil)
	gjstr, err := gj.Marshal(f)
	if err != nil {
		log.Fatal("geojson marshall failed", err)
	}
	return gjstr
}

func init() {
	log.SetOutput(os.Stderr)
}

func main() {
	log.Println("starting")
	resp, err := http.Get("http://www.miamidade.gov/transit/WebServices/Buses/?RouteID=7")
	if err != nil {
		log.Fatal("error fetching url", err)
	}
	log.Println(resp.Status)
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("could not read Response Body")
	}
	resp.Body.Close()
	r := new(Record)
	parserecord(&r, body)
	log.Println(r.BusName)
	log.Printf("POINT(%f %f)", r.Longitude, r.Latitude)
	j := makeFeaturefromRecord(r)
	fmt.Println(j)
}
<commit_msg>features now work<commit_after>package main

import "net/http"
import "io/ioutil"
import "os"
import "encoding/xml"
import "log"
import "fmt"
import (
	gj "github.com/kpawlik/geojson"
)

//Record is a single update of a single bus location
type Record struct {
	BusID            string  `xml:"Record>BusID"`
	BusName          string  `xml:"Record>BusName"`
	Latitude         float64 `xml:"Record>Latitude"`
	Longitude        float64 `xml:"Record>Longitude"`
	RouteID          string  `xml:"Record>RouteID"`
	TripID           string  `xml:"Record>TripID"`
	Direction        string  `xml:"Record>Direction"`
	ServiceDirection string  `xml:"Record>ServiceDirection"`
	Service          string  `xml:"Record>Service"`
	ServiceName      string  `xml:"Record>ServiceName"`
	TripHeadsign     string  `xml:"Record>TripHeadsign"`
	LocationUpdated  string  `xml:"Record>LocationUpdated"`
}

func parserecord(r **Record, body []byte) {
	err := xml.Unmarshal(body, r)
	if err != nil {
		log.Fatal("Unmarshall failed", err)
	}

}

func makeFeaturefromRecord(r *Record) string {
	var f *gj.Feature
	lon := gj.Coord(r.Longitude)
	lat :=  gj.Coord(r.Latitude)
	p := gj.NewPoint(gj.Coordinate{lon, lat})
	id := r.BusID
	props := make(map[string]interface{})
	props["BusName"] = r.BusName
	props["RouteID"] = r.RouteID
	props["TripID"] = r.TripID
	props["TripHeadsign"] = r.TripHeadsign
	props["Service"] = r.Service
	props["LocationUpdated"] = r.LocationUpdated
	f = gj.NewFeature(p, props, id)
	gjstr, err := gj.Marshal(f)
	if err != nil {
		log.Fatal("geojson marshall failed", err)
	}
	return gjstr
}

func init() {
	log.SetOutput(os.Stderr)
}

func main() {
	log.Println("starting")
	resp, err := http.Get("http://www.miamidade.gov/transit/WebServices/Buses/?RouteID=7")
	if err != nil {
		log.Fatal("error fetching url", err)
	}
	log.Println(resp.Status)
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("could not read Response Body")
	}
	resp.Body.Close()
	r := new(Record)
	parserecord(&r, body)
	log.Println(r.BusName)
	log.Printf("POINT(%f %f)", r.Longitude, r.Latitude)
	j := makeFeaturefromRecord(r)
	fmt.Println(j)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"sort"
	"sync"
	"time"

	"github.com/fatih/color"
	"github.com/fatih/structs"
)

var inchan chan Node
var outchan chan string
var toWrite chan string

// Channel to buffer nodes that need closing
var cleanUpNodesChan chan Node
var newNodesChan chan Node
var toSyncNodes chan Node
var toConnectNodes chan string

var nodes map[string]Node
var data DataTable

// Current revision of chat
// Calculated by the hash of all values in map
var dataState string

var nick string
var listen string

type Node struct {
	ConnectionType string
	Connection     net.Conn
	LocalIP        string
	RemoteIP       string
	Listen         string
	DataChecksum   string
	Data           string
}

type Packet struct {
	Type string
	Data map[string]interface{}
}

type Message struct {
	Key  string
	Time string
	Nick string
	Data string
}

type SyncCheck struct {
	Checksum         string
	ListeningAddress string
	KnownHosts       []string
}

type SyncIndex struct {
	Keys []string
}

type SyncPacket struct {
	Key   string
	Value Message
}

type RequestPacket struct {
	Key string
}

type DataTable struct {
	Mutex sync.Mutex
	// Map of chat
	Data_table map[string]Message
}

func init() {
	inchan = make(chan Node)
	outchan = make(chan string)
	toWrite = make(chan string)
	cleanUpNodesChan = make(chan Node)
	newNodesChan = make(chan Node)
	toSyncNodes = make(chan Node)
	toConnectNodes = make(chan string)
	nodes = make(map[string]Node)

	//data_table = make(map[string][]string)
	data.Data_table = make(map[string]Message)
	//data := new(DataTable{}
	dataState = data.getDataCheckSum()
	fmt.Println("Currect DataChecksum: ", dataState)
}

func main() {
	var host string
	var port string
	var newNode string

	fmt.Printf("Enter host (Leave blank for localhost): ")
	fmt.Scanln(&host)

	fmt.Printf("Enter port (Leave blank for 8080): ")
	fmt.Scanln(&port)

	fmt.Print("Enter a nick name:")
	fmt.Scanln(&nick)

	fmt.Print("Enter another node's address:")
	fmt.Scanln(&newNode)

	if len(host) == 0 || host == "" {
		host = "localhost"
	}

	if len(port) == 0 || port == "" {
		port = "8080"
	}

	if len(nick) == 0 || nick == "" {
		nick = "IsItSoHardToGetANick?"
	}

	// Linten to user keystrokes
	go clientInput()
	// Start printing routine
	go handleIncoming()
	// Handle cleanup
	go cleanUpNodes()
	// Sync keep alive
	go syncCheck()
	// Sync index
	go syncIndex()
	// Connect nodes
	go connectNodes()

	go client(newNode)

	server(host, port)
}

// Get checksum
func (data *DataTable) getDataCheckSum() string {
	data.Mutex.Lock()
	defer data.Mutex.Unlock()

	return Calculate_data_checksum(data.Data_table)
}

func Calculate_data_checksum(table map[string]Message) string {
	mk := make([]string, len(table))
	i := 0
	for k := range table {
		mk[i] = k
		i++
	}
	sort.Strings(mk)

	tempValues := ""
	for _, v := range mk {
		tempValues = tempValues + v
	}

	byteValues := []byte(tempValues)
	md5Sum := md5.Sum(byteValues)

	return hex.EncodeToString(md5Sum[:])
}

func GetIndex(table map[string]Message) []string {
	var keys []string

	for k := range table {
		keys = append(keys, k)
	}

	return keys
}

// Compares 2 sets of keys and returns an array of keys that are missing
func CompareKeys(table map[string]Message, other []string) []string {
	var keys []string
	var exists bool

	for _, v := range other {
		exists = false

		for k := range table {
			if k == v {
				exists = true
				//fmt.Println("Exists")
				break
			}
		}

		if exists != true {
			//fmt.Println("Does not exist")
			keys = append(keys, v)
		}
	}

	return keys
}

// Write to table
func (data *DataTable) writeToTable(message Message) {
	data.Mutex.Lock()
	defer data.Mutex.Unlock()

	data.Data_table[message.Key] = message
	//fmt.Println("data updated")
}

// Encode JSON
func Encode_msg(packet Packet) string {
	jsonString, err := json.Marshal(packet)
	if err != nil {
		return "ERROR"
	}
	return string(jsonString[:]) + "\n"
}

// Decode JSON
func Decode_msg(msg string) (Packet, bool) {
	var packet Packet
	err := json.Unmarshal([]byte(msg), &packet)
	if err != nil {
		fmt.Println(msg, err)
		return packet, false
	}
	return packet, true
}

func syncRequest(key string, node Node) {
	message := Encode_msg(Packet{"RequestPacket", structs.Map(RequestPacket{key})})
	unicastMessage(message, node)
}

func syncNode(key string, node Node) {

	msg := data.Data_table[key]

	message := Encode_msg(Packet{"SyncPacket", structs.Map(SyncPacket{key, msg})})
	unicastMessage(message, node)
}

// Send key value pair
func syncIndex() {
	for {
		node := <-toSyncNodes
		message := Encode_msg(Packet{"SyncIndex", structs.Map(SyncIndex{GetIndex(data.Data_table)})})
		unicastMessage(message, node)
	}
}

func connectNodes() {
	for {
		host := <-toConnectNodes
		client(host)
	}
}

// Broadcast current checksum and known hosts
func syncCheck() {
	for {
		var knownHosts []string

		for _, v := range nodes {
			if v.Listen != "nill" {
				knownHosts = append(knownHosts, v.Listen)
			}
		}
		broadCastMessage(Encode_msg(Packet{"SyncCheck", structs.Map(SyncCheck{dataState, listen, knownHosts})}))
		time.Sleep(time.Second * 5)
	}
}

func printReply(message Message) {
	//node := data.Data_table[key]

	timestamp, _ := time.Parse(time.RFC1123, message.Time)
	//fmt.Println(timestamp)
	localTime := timestamp.Local().Format(time.Kitchen)
	//fmt.Println(node[0], node[1], node[2])
	color.Set(color.FgYellow)
	fmt.Printf("<%s> ", localTime)
	color.Set(color.FgGreen)
	fmt.Printf("%s: ", message.Nick)
	color.Set(color.FgCyan)
	fmt.Printf(message.Data)
	color.Unset()
}

// Display incoming messages
func handleIncoming() {
	for {
		node := <-inchan
		//fmt.Println("Got: " ,node.Data)
		packet, success := Decode_msg(node.Data)
		//fmt.Println(key,value)
		if success {
			//printReply(packet)
			//fmt.Println(packet)
			if packet.Type == "Message" {
				//message := packet.Data
				//fmt.Println(message)
				key := packet.Data["Key"].(string)
				timeStamp := packet.Data["Time"].(string)
				nickname := packet.Data["Nick"].(string)
				dataPacket := packet.Data["Data"].(string)
				message := Message{key, timeStamp, nickname, dataPacket}
				data.writeToTable(message)
				go printCheckSum()
				printReply(message)
				// This packet is just a keep alive
			} else if packet.Type == "SyncCheck" {
				node.DataChecksum = packet.Data["Checksum"].(string)
				if node.DataChecksum != dataState {
					toSyncNodes <- node
				}

				if packet.Data["ListeningAddress"] != nil {
					node.Listen = packet.Data["ListeningAddress"].(string)
					nodes[node.RemoteIP] = node
				}

				if packet.Data["KnownHosts"] != nil {
					knownHosts := packet.Data["KnownHosts"].([]interface{})
					var knownHostsString []string

					fmt.Println(packet)

					for _, v := range knownHosts {
						knownHostsString = append(knownHostsString, v.(string))
					}

					for _, v := range knownHostsString {
						found := false
						for _, node := range nodes {
							if node.RemoteIP == v || node.LocalIP == v || node.Listen == v || listen == v || node.Listen == "nill" {
								found = true
								break
							}
						}

						if found != true {
							toConnectNodes <- v
						}
					}

				}
				// If we receive this packet it means there is a mismatch with the data and we need to correct it
			} else if packet.Type == "SyncIndex" {
				if packet.Data["Keys"] != nil {
					index := packet.Data["Keys"].([]interface{})
					var indexString []string
					for _, v := range index {
						indexString = append(indexString, v.(string))
					}

					missingKeys := CompareKeys(data.Data_table, indexString)
					//fmt.Println(missingKeys)

					for _, v := range missingKeys {
						syncRequest(v, node)
					}
				}

				//go unicastMessage(Encode_msg(), node)
				// If we receive this packet it means the node whishes to get a value of a key
			} else if packet.Type == "RequestPacket" {
				key := packet.Data["Key"].(string)
				syncNode(key, node)
				//fmt.Println(packet)
				// If we receive this packet it means we got data from the other node to populate our table
			} else if packet.Type == "SyncPacket" {
				if packet.Data["Key"] != nil && packet.Data["Value"] != nil {
					//key := packet.Data["Key"].(string)
					var message Message
					//fmt.Println(packet)
					value := packet.Data["Value"].(map[string]interface{})
					for k, v := range value {
						if k == "Data" {
							message.Data = v.(string)
						} else if k == "Key" {
							message.Key = v.(string)
						} else if k == "Time" {
							message.Time = v.(string)
						} else if k == "Nick" {
							message.Nick = v.(string)
						}
					}

					data.writeToTable(message)
					printReply(message)
					go printCheckSum()
				}
			}
		}
		nodes[node.RemoteIP] = node
	}
}

// ASYNC update checksum
func printCheckSum() {
	dataState = data.getDataCheckSum()
	fmt.Println(dataState)
}

// Message a single node
func unicastMessage(msg string, node Node) {
	_, err := node.Connection.Write([]byte(msg))
	if err != nil {
		fmt.Println("Error sending message!", err)
		cleanUpNodesChan <- node
	} else {
		//fmt.Println("Sent to node: ", msg)
	}
}

// Broad cast to all Nodes
func broadCastMessage(msg string) {
	for _, node := range nodes {
		_, err := node.Connection.Write([]byte(msg))
		if err != nil {
			fmt.Println("Error sending message!", err)
			cleanUpNodesChan <- node
		} else {
			//fmt.Println("Sent: ", msg)
		}
	}
}

func cleanUpNodes() {
	for {
		node := <-cleanUpNodesChan
		fmt.Println("Cleaning up Connection: " + node.RemoteIP)
		err := node.Connection.Close()
		if err != nil {
			fmt.Println("Failed to close!", err)
		}
		delete(nodes, node.RemoteIP)
	}
}

func processInput(msg string) {

	timestamp := time.Now().Format(time.RFC1123)
	md5SumKey := md5.Sum([]byte(timestamp + msg))
	key := hex.EncodeToString(md5SumKey[:])
	//msg_array := []string{key,timestamp,nick,msg}

	//data_map := make(map[string][]string)
	//data_map["Message"] = msg_array
	message := Message{key, timestamp, nick, msg}

	input := Encode_msg(Packet{"Message", structs.Map(message)})
	//fmt.Println(input)

	go broadCastMessage(input)

	data.writeToTable(message)
	go printCheckSum()

}

// Handle client input
func clientInput() {
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		// Sleep to prevent duplicates
		time.Sleep(time.Millisecond)

		input := scanner.Text()
		// Check for input
		if input != "" || len(input) > 0 {
			processInput(input)
		}
	}
}

// Client
func client(host string) {
	conn, err := net.Dial("tcp", host)
	if err != nil {
		fmt.Println("Failed to connect to host!", err)
		return
	}
	fmt.Println("Connecting to: ", conn.RemoteAddr())
	go handleConnection("Client", conn)
}

// Server function
func server(host string, port string) {
	hostString := host + ":" + port

	// Start listening
	ln, err := net.Listen("tcp", hostString)
	if err != nil {
		log.Fatalf("Failed to listen:", err)
	}

	fmt.Println("Listening on: ", ln.Addr())
	listen = ln.Addr().String()

	// Handle connections
	for {
		if conn, err := ln.Accept(); err == nil {
			fmt.Println("Incomming connection: ", conn.RemoteAddr())
			go handleConnection("Server", conn)
		}
	}
}

func readConnection(node Node) {
	//buf := make([]byte, 4096)
	for {
		//n, err := node.Connection.Read(reader)
		//n, err := reader.ReadLine("\n")
		line, err := bufio.NewReader(node.Connection).ReadBytes('\n')
		if err != nil {
			if err != io.EOF {
				fmt.Printf("Reached EOF")
			}
			cleanUpNodesChan <- node
			break
		}

		node.Data = (string(line))

		inchan <- node
	}
}

func handleConnection(typeOfConnection string, conn net.Conn) {
	nodes[conn.RemoteAddr().String()] = Node{typeOfConnection, conn, conn.LocalAddr().String(), conn.RemoteAddr().String(), "nill", "", ""}
	readConnection(nodes[conn.RemoteAddr().String()])
}
<commit_msg>Started moving over to a better logging library.<commit_after>package main

import (
	"bufio"
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	//"log"
	"net"
	"os"
	"sort"
	"sync"
	"time"

	"github.com/op/go-logging"

	"github.com/fatih/color"
	"github.com/fatih/structs"
)

var inchan chan Node
var outchan chan string
var toWrite chan string

// Channel to buffer nodes that need closing
var cleanUpNodesChan chan Node
var newNodesChan chan Node
var toSyncNodes chan Node
var toConnectNodes chan string

var nodes map[string]Node
var data DataTable

// Current revision of chat
// Calculated by the hash of all values in map
var dataState string

var nick string
var listen string

var log = logging.MustGetLogger("example")
var format = logging.MustStringFormatter(
	`%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
)

type Node struct {
	ConnectionType string
	Connection     net.Conn
	LocalIP        string
	RemoteIP       string
	Listen         string
	DataChecksum   string
	Data           string
}

type Packet struct {
	Type string
	Data map[string]interface{}
}

type Message struct {
	Key  string
	Time string
	Nick string
	Data string
}

type SyncCheck struct {
	Checksum         string
	ListeningAddress string
	KnownHosts       []string
}

type SyncIndex struct {
	Keys []string
}

type SyncPacket struct {
	Key   string
	Value Message
}

type RequestPacket struct {
	Key string
}

type DataTable struct {
	Mutex sync.Mutex
	// Map of chat
	Data_table map[string]Message
}

func init() {
	inchan = make(chan Node)
	outchan = make(chan string)
	toWrite = make(chan string)
	cleanUpNodesChan = make(chan Node)
	newNodesChan = make(chan Node)
	toSyncNodes = make(chan Node)
	toConnectNodes = make(chan string)
	nodes = make(map[string]Node)

	//data_table = make(map[string][]string)
	data.Data_table = make(map[string]Message)
	//data := new(DataTable{}
	dataState = data.getDataCheckSum()
	fmt.Println("Currect DataChecksum: ", dataState)
}

func main() {
	// Setup Logging
	backend1 := logging.NewLogBackend(os.Stderr, "", 0)
	backend2 := logging.NewLogBackend(os.Stderr, "", 0)

	// For messages written to backend2 we want to add some additional
	// information to the output, including the used log level and the name of
	// the function.
	backend2Formatter := logging.NewBackendFormatter(backend2, format)

	// Only errors and more severe messages should be sent to backend1
	backend1Leveled := logging.AddModuleLevel(backend1)
	backend1Leveled.SetLevel(logging.ERROR, "")

	// Set the backends to be used.
	logging.SetBackend(backend1Leveled, backend2Formatter)

	var host string
	var port string
	var newNode string

	fmt.Printf("Enter host (Leave blank for localhost): ")
	fmt.Scanln(&host)

	fmt.Printf("Enter port (Leave blank for 8080): ")
	fmt.Scanln(&port)

	fmt.Print("Enter a nick name:")
	fmt.Scanln(&nick)

	fmt.Print("Enter another node's address:")
	fmt.Scanln(&newNode)

	if len(host) == 0 || host == "" {
		host = "localhost"
	}

	if len(port) == 0 || port == "" {
		port = "8080"
	}

	if len(nick) == 0 || nick == "" {
		nick = "IsItSoHardToGetANick?"
	}

	// Linten to user keystrokes
	go clientInput()
	// Start printing routine
	go handleIncoming()
	// Handle cleanup
	go cleanUpNodes()
	// Sync keep alive
	go syncCheck()
	// Sync index
	go syncIndex()
	// Connect nodes
	go connectNodes()

	go client(newNode)

	server(host, port)
}

// Get checksum
func (data *DataTable) getDataCheckSum() string {
	data.Mutex.Lock()
	defer data.Mutex.Unlock()

	return Calculate_data_checksum(data.Data_table)
}

func Calculate_data_checksum(table map[string]Message) string {
	mk := make([]string, len(table))
	i := 0
	for k := range table {
		mk[i] = k
		i++
	}
	sort.Strings(mk)

	tempValues := ""
	for _, v := range mk {
		tempValues = tempValues + v
	}

	byteValues := []byte(tempValues)
	md5Sum := md5.Sum(byteValues)

	return hex.EncodeToString(md5Sum[:])
}

func GetIndex(table map[string]Message) []string {
	var keys []string

	for k := range table {
		keys = append(keys, k)
	}

	return keys
}

// Compares 2 sets of keys and returns an array of keys that are missing
func CompareKeys(table map[string]Message, other []string) []string {
	var keys []string
	var exists bool

	for _, v := range other {
		exists = false

		for k := range table {
			if k == v {
				exists = true
				//fmt.Println("Exists")
				break
			}
		}

		if exists != true {
			//fmt.Println("Does not exist")
			keys = append(keys, v)
		}
	}

	return keys
}

// Write to table
func (data *DataTable) writeToTable(message Message) {
	data.Mutex.Lock()
	defer data.Mutex.Unlock()

	data.Data_table[message.Key] = message
	//fmt.Println("data updated")
}

// Encode JSON
func Encode_msg(packet Packet) string {
	jsonString, err := json.Marshal(packet)
	if err != nil {
		return "ERROR"
	}
	return string(jsonString[:]) + "\n"
}

// Decode JSON
func Decode_msg(msg string) (Packet, bool) {
	var packet Packet
	err := json.Unmarshal([]byte(msg), &packet)
	if err != nil {
		fmt.Println(msg, err)
		return packet, false
	}
	return packet, true
}

func syncRequest(key string, node Node) {
	message := Encode_msg(Packet{"RequestPacket", structs.Map(RequestPacket{key})})
	unicastMessage(message, node)
}

func syncNode(key string, node Node) {

	msg := data.Data_table[key]

	message := Encode_msg(Packet{"SyncPacket", structs.Map(SyncPacket{key, msg})})
	unicastMessage(message, node)
}

// Send key value pair
func syncIndex() {
	for {
		node := <-toSyncNodes
		message := Encode_msg(Packet{"SyncIndex", structs.Map(SyncIndex{GetIndex(data.Data_table)})})
		unicastMessage(message, node)
	}
}

func connectNodes() {
	for {
		host := <-toConnectNodes
		client(host)
	}
}

// Broadcast current checksum and known hosts
func syncCheck() {
	for {
		var knownHosts []string

		for _, v := range nodes {
			if v.Listen != "nill" {
				knownHosts = append(knownHosts, v.Listen)
			}
		}
		broadCastMessage(Encode_msg(Packet{"SyncCheck", structs.Map(SyncCheck{dataState, listen, knownHosts})}))
		time.Sleep(time.Second * 5)
	}
}

func printReply(message Message) {
	//node := data.Data_table[key]

	timestamp, _ := time.Parse(time.RFC1123, message.Time)
	//fmt.Println(timestamp)
	localTime := timestamp.Local().Format(time.Kitchen)
	//fmt.Println(node[0], node[1], node[2])
	color.Set(color.FgYellow)
	fmt.Printf("<%s> ", localTime)
	color.Set(color.FgGreen)
	fmt.Printf("%s: ", message.Nick)
	color.Set(color.FgCyan)
	fmt.Printf(message.Data)
	color.Unset()
}

// Display incoming messages
func handleIncoming() {
	for {
		node := <-inchan
		//fmt.Println("Got: " ,node.Data)
		packet, success := Decode_msg(node.Data)
		//fmt.Println(key,value)
		if success {
			//printReply(packet)
			//fmt.Println(packet)
			if packet.Type == "Message" {
				//message := packet.Data
				//fmt.Println(message)
				key := packet.Data["Key"].(string)
				timeStamp := packet.Data["Time"].(string)
				nickname := packet.Data["Nick"].(string)
				dataPacket := packet.Data["Data"].(string)
				message := Message{key, timeStamp, nickname, dataPacket}
				data.writeToTable(message)
				go printCheckSum()
				printReply(message)
				// This packet is just a keep alive
			} else if packet.Type == "SyncCheck" {
				node.DataChecksum = packet.Data["Checksum"].(string)
				if node.DataChecksum != dataState {
					toSyncNodes <- node
				}

				if packet.Data["ListeningAddress"] != nil {
					node.Listen = packet.Data["ListeningAddress"].(string)
					nodes[node.RemoteIP] = node
				}

				if packet.Data["KnownHosts"] != nil {
					knownHosts := packet.Data["KnownHosts"].([]interface{})
					var knownHostsString []string

					log.Debug(packet)

					for _, v := range knownHosts {
						knownHostsString = append(knownHostsString, v.(string))
					}

					for _, v := range knownHostsString {
						found := false
						for _, node := range nodes {
							if node.RemoteIP == v || node.LocalIP == v || node.Listen == v || listen == v || node.Listen == "nill" {
								found = true
								break
							}
						}

						if found != true {
							toConnectNodes <- v
						}
					}

				}
				// If we receive this packet it means there is a mismatch with the data and we need to correct it
			} else if packet.Type == "SyncIndex" {
				if packet.Data["Keys"] != nil {
					index := packet.Data["Keys"].([]interface{})
					var indexString []string
					for _, v := range index {
						indexString = append(indexString, v.(string))
					}

					missingKeys := CompareKeys(data.Data_table, indexString)
					//fmt.Println(missingKeys)

					for _, v := range missingKeys {
						syncRequest(v, node)
					}
				}

				//go unicastMessage(Encode_msg(), node)
				// If we receive this packet it means the node whishes to get a value of a key
			} else if packet.Type == "RequestPacket" {
				key := packet.Data["Key"].(string)
				syncNode(key, node)
				//fmt.Println(packet)
				// If we receive this packet it means we got data from the other node to populate our table
			} else if packet.Type == "SyncPacket" {
				if packet.Data["Key"] != nil && packet.Data["Value"] != nil {
					//key := packet.Data["Key"].(string)
					var message Message
					log.Debug(packet)
					value := packet.Data["Value"].(map[string]interface{})
					for k, v := range value {
						if k == "Data" {
							message.Data = v.(string)
						} else if k == "Key" {
							message.Key = v.(string)
						} else if k == "Time" {
							message.Time = v.(string)
						} else if k == "Nick" {
							message.Nick = v.(string)
						}
					}

					data.writeToTable(message)
					printReply(message)
					go printCheckSum()
				}
			}
		}
		nodes[node.RemoteIP] = node
	}
}

// ASYNC update checksum
func printCheckSum() {
	dataState = data.getDataCheckSum()
	fmt.Println(dataState)
}

// Message a single node
func unicastMessage(msg string, node Node) {
	_, err := node.Connection.Write([]byte(msg))
	if err != nil {
		fmt.Println("Error sending message!", err)
		cleanUpNodesChan <- node
	} else {
		//fmt.Println("Sent to node: ", msg)
	}
}

// Broad cast to all Nodes
func broadCastMessage(msg string) {
	for _, node := range nodes {
		_, err := node.Connection.Write([]byte(msg))
		if err != nil {
			log.Error("Error sending message!", err)
			cleanUpNodesChan <- node
		} else {
			//fmt.Println("Sent: ", msg)
		}
	}
}

func cleanUpNodes() {
	for {
		node := <-cleanUpNodesChan
		log.Info("Cleaning up Connection: " + node.RemoteIP)
		err := node.Connection.Close()
		if err != nil {
			log.Error("Failed to close!", err)
		}
		delete(nodes, node.RemoteIP)
	}
}

func processInput(msg string) {

	timestamp := time.Now().Format(time.RFC1123)
	md5SumKey := md5.Sum([]byte(timestamp + msg))
	key := hex.EncodeToString(md5SumKey[:])
	//msg_array := []string{key,timestamp,nick,msg}

	//data_map := make(map[string][]string)
	//data_map["Message"] = msg_array
	message := Message{key, timestamp, nick, msg}

	input := Encode_msg(Packet{"Message", structs.Map(message)})
	//fmt.Println(input)

	go broadCastMessage(input)

	data.writeToTable(message)
	go printCheckSum()

}

// Handle client input
func clientInput() {
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		// Sleep to prevent duplicates
		time.Sleep(time.Millisecond)

		input := scanner.Text()
		// Check for input
		if input != "" || len(input) > 0 {
			processInput(input)
		}
	}
}

// Client
func client(host string) {
	conn, err := net.Dial("tcp", host)
	if err != nil {
		log.Error("Failed to connect to host!", err)
		return
	}
	fmt.Println("Connecting to: ", conn.RemoteAddr())
	go handleConnection("Client", conn)
}

// Server function
func server(host string, port string) {
	hostString := host + ":" + port

	// Start listening
	ln, err := net.Listen("tcp", hostString)
	if err != nil {
		log.Error("Failed to listen:", err)
		os.Exit(1)
	}

	log.Info("Listening on: ", ln.Addr())
	listen = ln.Addr().String()

	// Handle connections
	for {
		if conn, err := ln.Accept(); err == nil {
			log.Info("Incomming connection: ", conn.RemoteAddr())
			go handleConnection("Server", conn)
		}
	}
}

func readConnection(node Node) {
	//buf := make([]byte, 4096)
	for {
		//n, err := node.Connection.Read(reader)
		//n, err := reader.ReadLine("\n")
		line, err := bufio.NewReader(node.Connection).ReadBytes('\n')
		if err != nil {
			if err != io.EOF {
				log.Warning("Reached EOF")
			}
			cleanUpNodesChan <- node
			break
		}

		node.Data = (string(line))

		inchan <- node
	}
}

func handleConnection(typeOfConnection string, conn net.Conn) {
	nodes[conn.RemoteAddr().String()] = Node{typeOfConnection, conn, conn.LocalAddr().String(), conn.RemoteAddr().String(), "nill", "", ""}
	readConnection(nodes[conn.RemoteAddr().String()])
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/mitchellh/go-homedir"
)

func shortUsage() {
	os.Stderr.WriteString(`
usage: vub [option(s)] <repository-uri>
try 'vub --help' for more information.
`[1:])
}

func usage() {
	os.Stderr.WriteString(`
usage: vub [option(s)] <repository-uri>
install Vim plugin to under the management of vim-unbundle.

options:
  -h, --help                show this help message
  -f, --filetype=TYPE       installing under the ftbundle/TYPE
`[1:])
}

var (
	ShortGitHubURI = regexp.MustCompile(`^[\w\-.]+/[\w\-.]+$`)
)

func ToSourceURI(uri string) (string, error) {
	switch {
	case ShortGitHubURI.MatchString(uri):
		return "https://github.com/" + uri, nil
	default:
		return uri, nil
	}
}

var (
	home, errInit = homedir.Dir()
	dotvim        = filepath.Join(home, ".vim")
)

func ToDestinationPath(uri, filetype string) (string, error) {
	name := filepath.Base(uri)
	if filetype == "" {
		return filepath.Join(dotvim, "bundle", name), nil
	}
	return filepath.Join(dotvim, "ftbundle", filetype, name), nil
}

type Package struct {
	verbose bool
	src     string
	dst     string
}

func NewPackage(uri, filetype string) (*Package, error) {
	src, err := ToSourceURI(uri)
	if err != nil {
		return nil, err
	}
	dst, err := ToDestinationPath(uri, filetype)
	if err != nil {
		return nil, err
	}
	return &Package{
		src: src,
		dst: dst,
	}, nil
}

func (p *Package) toCommand() *exec.Cmd {
	return exec.Command("git", "clone", p.src, p.dst)
}

func (p *Package) Install(w io.Writer) error {
	c := p.toCommand()
	if w != nil && p.verbose {
		c.Stdout = w
		c.Stderr = w
		_, err := io.WriteString(w, strings.Join(c.Args, " "))
		if err != nil {
			return err
		}
	}
	return c.Run()
}

func (p *Package) Verbose(enable bool) {
	p.verbose = enable
}

func main() {
	var filetype string
	flag.StringVar(&filetype, "f", "", "")
	flag.StringVar(&filetype, "filetype", "", "")

	var isHelp bool
	flag.BoolVar(&isHelp, "h", false, "")
	flag.BoolVar(&isHelp, "help", false, "")
	flag.Usage = usage
	flag.Parse()
	switch {
	case isHelp:
		usage()
		os.Exit(0)
	case flag.NArg() < 1:
		shortUsage()
		os.Exit(2)
	}
	uri := flag.Arg(0)

	p, err := NewPackage(uri, filetype)
	if err != nil {
		fmt.Fprintln(os.Stderr, "vub:", err)
		os.Exit(2)
	}

	c := p.toCommand()
	if err := c.Run(); err != nil {
		fmt.Fprintln(os.Stderr, "vub:", err)
		os.Exit(1)
	}
}
<commit_msg>Enable use verbose option<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/mitchellh/go-homedir"
)

func shortUsage() {
	os.Stderr.WriteString(`
usage: vub [option(s)] <repository-uri>
try 'vub --help' for more information.
`[1:])
}

func usage() {
	os.Stderr.WriteString(`
usage: vub [option(s)] <repository-uri>
install Vim plugin to under the management of vim-unbundle.

options:
  -h, --help                show this help message
  -f, --filetype=TYPE       installing under the ftbundle/TYPE
  -v, --verbose             display the process
`[1:])
}

var (
	ShortGitHubURI = regexp.MustCompile(`^[\w\-.]+/[\w\-.]+$`)
)

func ToSourceURI(uri string) (string, error) {
	switch {
	case ShortGitHubURI.MatchString(uri):
		return "https://github.com/" + uri, nil
	default:
		return uri, nil
	}
}

var (
	home, errInit = homedir.Dir()
	dotvim        = filepath.Join(home, ".vim")
)

func ToDestinationPath(uri, filetype string) (string, error) {
	name := filepath.Base(uri)
	if filetype == "" {
		return filepath.Join(dotvim, "bundle", name), nil
	}
	return filepath.Join(dotvim, "ftbundle", filetype, name), nil
}

type Package struct {
	verbose bool
	src     string
	dst     string
}

func NewPackage(uri, filetype string) (*Package, error) {
	src, err := ToSourceURI(uri)
	if err != nil {
		return nil, err
	}
	dst, err := ToDestinationPath(uri, filetype)
	if err != nil {
		return nil, err
	}
	return &Package{
		src: src,
		dst: dst,
	}, nil
}

func (p *Package) toCommand() *exec.Cmd {
	return exec.Command("git", "clone", p.src, p.dst)
}

func (p *Package) Install(w io.Writer) error {
	c := p.toCommand()
	if w != nil && p.verbose {
		c.Stdout = w
		c.Stderr = w
		_, err := io.WriteString(w, strings.Join(c.Args, " "))
		if err != nil {
			return err
		}
	}
	return c.Run()
}

func (p *Package) Verbose(enable bool) {
	p.verbose = enable
}

func main() {
	filetype, verbose := "", false
	flag.StringVar(&filetype, "f", "", "")
	flag.StringVar(&filetype, "filetype", "", "")

	isHelp := false
	flag.BoolVar(&isHelp, "h", false, "")
	flag.BoolVar(&isHelp, "help", false, "")
	flag.Usage = usage
	flag.Parse()
	switch {
	case isHelp:
		usage()
		os.Exit(0)
	case flag.NArg() < 1:
		shortUsage()
		os.Exit(2)
	}
	uri := flag.Arg(0)

	p, err := NewPackage(uri, filetype)
	if err != nil {
		fmt.Fprintln(os.Stderr, "vub:", err)
		os.Exit(2)
	}
	p.Verbose(verbose)

	if err := p.Install(os.Stdout); err != nil {
		fmt.Fprintln(os.Stderr, "vub:", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Copyright © 2016 Jacob Dearing
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package main

import (
	"encoding/base64"
	"fmt"
	"log"
	"net/http"
	"os"
	"runtime"
	"strconv"

	_ "net/http/pprof"

	"github.com/dearing/havoc"
	"github.com/julienschmidt/httprouter"
)

var NAME = GetRandomName(0)

func main() {

	procs := runtime.NumCPU()
	runtime.GOMAXPROCS(procs)

	router := httprouter.New()
	router.GET("/", HandleIndex)
	router.GET("/kill", HandleKill)

	router.GET("/data/reset", HandleDataReset)
	router.GET("/data/set/:value", HandleDataSet)
	router.GET("/data/fill", HandleDataFill)
	router.GET("/data/fill/zero", HandleDataFillZero)
	router.GET("/data/fill/crypto", HandleDataFillCrypto)

	router.GET("/procs/:value", HandleProcs)

	log.Printf("%s : starting.\n", NAME)

	go func() {
		log.Println(http.ListenAndServe(":8081", nil))
	}()

	log.Fatal(http.ListenAndServe(":8080", router))
}

// HANDLES

func HandleIndex(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	fmt.Fprintf(w, "My name is %s.\n\n", NAME)
	fmt.Fprint(w, base64.StdEncoding.EncodeToString(havoc.Data))
}

func HandleKill(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	fmt.Fprintf(w, "%s: I hardly knew thee.\n", NAME)
	os.Exit(0)
}

// HANDLES - DATA

func HandleDataSet(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	size, err := strconv.Atoi(ps.ByName("value"))
	if err != nil {
		fmt.Fprintf(w, "%s: error in parse: %s\n", NAME, err)
		return
	}

	go func() {
		havoc.DataSet(size)
	}()

	fmt.Fprintf(w, "%s: %d indices set.\n", NAME, size)

}

func HandleDataReset(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	havoc.DataReset()
	fmt.Fprintf(w, "%s: Reset.\n", NAME)
}

func HandleDataFill(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	havoc.DataFill()
	fmt.Fprintf(w, "%s: Filling Data, (%d) bytes, with ones.\n", NAME, len(havoc.Data))
}

func HandleDataFillZero(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	havoc.DataFillZero()
	fmt.Fprintf(w, "%s: Filling Data, (%d) bytes, with zeroes.\n", NAME, len(havoc.Data))
}

func HandleDataFillCrypto(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	havoc.DataFillCrypto()
	fmt.Fprintf(w, "%s: Filling Data, (%d) bytes, with random data.\n", NAME, len(havoc.Data))
}

// HANDLES - CPU

func HandleProcs(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {

	procs, err := strconv.Atoi(ps.ByName("value"))
	if err != nil {
		fmt.Fprintf(w, "%s: error in parse: %s\n", NAME, err)
		return
	}

	for i := 0; i < procs; i++ {
		go func() {
			havoc.Forever()
		}()
	}

	fmt.Fprintf(w, "%s: %d of %d processors engaged.\n", NAME, procs, runtime.NumCPU())
}
<commit_msg>remove unused function; fixes #3<commit_after>// Copyright © 2016 Jacob Dearing
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package main

import (
	"encoding/base64"
	"fmt"
	"log"
	"net/http"
	"os"
	"runtime"
	"strconv"

	_ "net/http/pprof"

	"github.com/dearing/havoc"
	"github.com/julienschmidt/httprouter"
)

var NAME = GetRandomName(0)

func main() {

	procs := runtime.NumCPU()
	runtime.GOMAXPROCS(procs)

	router := httprouter.New()
	router.GET("/", HandleIndex)
	router.GET("/kill", HandleKill)

	router.GET("/data/reset", HandleDataReset)
	router.GET("/data/set/:value", HandleDataSet)
	router.GET("/data/fill", HandleDataFill)
	router.GET("/data/fill/crypto", HandleDataFillCrypto)

	router.GET("/procs/:value", HandleProcs)

	log.Printf("%s : starting.\n", NAME)

	go func() {
		log.Println(http.ListenAndServe(":8081", nil))
	}()

	log.Fatal(http.ListenAndServe(":8080", router))
}

// HANDLES

func HandleIndex(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	fmt.Fprintf(w, "My name is %s.\n\n", NAME)
	fmt.Fprint(w, base64.StdEncoding.EncodeToString(havoc.Data))
}

func HandleKill(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	fmt.Fprintf(w, "%s: I hardly knew thee.\n", NAME)
	os.Exit(0)
}

// HANDLES - DATA

func HandleDataSet(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	size, err := strconv.Atoi(ps.ByName("value"))
	if err != nil {
		fmt.Fprintf(w, "%s: error in parse: %s\n", NAME, err)
		return
	}

	go func() {
		havoc.DataSet(size)
	}()

	fmt.Fprintf(w, "%s: %d indices set.\n", NAME, size)

}

func HandleDataReset(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	havoc.DataReset()
	fmt.Fprintf(w, "%s: Reset.\n", NAME)
}

func HandleDataFill(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	havoc.DataFill()
	fmt.Fprintf(w, "%s: Filling Data, (%d) bytes, with ones.\n", NAME, len(havoc.Data))
}

func HandleDataFillCrypto(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	havoc.DataFillCrypto()
	fmt.Fprintf(w, "%s: Filling Data, (%d) bytes, with random data.\n", NAME, len(havoc.Data))
}

// HANDLES - CPU

func HandleProcs(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {

	procs, err := strconv.Atoi(ps.ByName("value"))
	if err != nil {
		fmt.Fprintf(w, "%s: error in parse: %s\n", NAME, err)
		return
	}

	for i := 0; i < procs; i++ {
		go func() {
			havoc.Forever()
		}()
	}

	fmt.Fprintf(w, "%s: %d of %d processors engaged.\n", NAME, procs, runtime.NumCPU())
}
<|endoftext|>