<commit_before>package main

import (
	"encoding/json"
	"flag"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"os/user"
	"strconv"
	"syscall"
	"time"

	"github.com/ant0ine/go-json-rest/rest"
	"github.com/boltdb/bolt"
	"github.com/docker/libcontainer/netlink"
	"github.com/docker/libcontainer/network"
	"github.com/rakyll/globalconf"
)

const (
	appName       = "tentacool"
	addressBucket = "address"
)

var (
	flagSocket = flag.String("socket", "/var/run/"+appName, "Path for unix socket")
	flagOwner  = flag.String("owner", "", "Ownership for socket")
	flagGroup  = flag.Int("group", -1, "Group for socket")
	flagDB     = flag.String("db", "/var/lib/"+appName+"/db", "Path for DB")
	// flagMode   = flag.Int("mode", 0640, "FileMode for socket")

	db *bolt.DB
	ln net.Listener
)

type Address struct {
	ID   string
	Link string
	IP   string
}

func main() {
	conf, err := globalconf.New(appName)
	if err != nil {
		log.Fatal(err)
	}
	conf.ParseAll()

	handler := rest.ResourceHandler{}
	err = handler.SetRoutes(
		&rest.Route{"GET", "/interfaces", GetIfaces},
		&rest.Route{"GET", "/interfaces/:iface", GetIface},
		&rest.Route{"POST", "/address", PostAddress},
		&rest.Route{"Get", "/address/:address", GetAddress},
	)
	if err != nil {
		log.Fatal(err)
	}

	ln, err = net.Listen("unix", *flagSocket)
	if nil != err {
		log.Fatal(err)
	}

	if *flagOwner != "" {
		user, err := user.Lookup(*flagOwner)
		if err != nil {
			log.Fatal(err)
		}
		uid, err := strconv.Atoi(user.Uid)
		var gid int
		if *flagGroup != -1 {
			gid = *flagGroup
		} else {
			gid, err = strconv.Atoi(user.Gid)
		}
		if err := os.Chown(*flagSocket, uid, gid); err != nil {
			log.Fatal(err)
		}
	}

	// Handle common process-killing signals so we can gracefully shut down:
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, os.Interrupt, os.Kill, syscall.SIGTERM)
	go func(c chan os.Signal) {
		// Wait for a SIGINT or SIGKILL:
		sig := <-c
		log.Printf("Caught signal %s: shutting down.", sig)
		// Stop listening (and unlink the socket if unix type):
		ln.Close()
		db.Close()
		os.Exit(0)
	}(sigc)

	db, err = bolt.Open(*flagDB, 0600, &bolt.Options{Timeout: 1 * time.Second})
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	db.Update(func(tx *bolt.Tx) (err error) {
		_, err = tx.CreateBucketIfNotExists([]byte(addressBucket))
		if err != nil {
			log.Fatal(err)
		}
		return
	})

	log.Fatal(http.Serve(ln, &handler))
	defer ln.Close()
}

func GetIfaces(w rest.ResponseWriter, req *rest.Request) {
	interfaces, err := net.Interfaces()
	if err != nil {
		log.Printf(err.Error())
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.WriteJson(interfaces)
}

func GetIface(w rest.ResponseWriter, req *rest.Request) {
	iface, err := net.InterfaceByName(req.PathParam("iface"))
	if err != nil {
		log.Printf(err.Error())
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	ip, err := iface.Addrs()
	if err != nil {
		log.Printf(err.Error())
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.WriteJson(ip)
}

func GetAddress(w rest.ResponseWriter, req *rest.Request) {
	id := req.PathParam("address")

	address := Address{}
	db.View(func(tx *bolt.Tx) (err error) {
		tmp := tx.Bucket([]byte(addressBucket)).Get([]byte(id))
		if err = json.Unmarshal(tmp, &address); err != nil {
			log.Printf(err.Error())
			rest.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	})
	w.WriteJson(address)
}

func PostAddress(w rest.ResponseWriter, req *rest.Request) {
	address := Address{}
	if err := req.DecodeJsonPayload(&address); err != nil {
		log.Printf(err.Error())
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if err := network.SetInterfaceIp(address.Link, address.IP); err != nil {
		log.Printf(err.Error())
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	_ = netlink.AddRoute("", address.IP, "", address.Link)

	db.Update(func(tx *bolt.Tx) (err error) {
		b := tx.Bucket([]byte(addressBucket))
		if address.ID == "" {
			int, err := b.NextSequence()
			if err != nil {
				log.Printf(err.Error())
				rest.Error(w, err.Error(), http.StatusInternalServerError)
				return err
			}
			address.ID = strconv.FormatUint(int, 10)
		}
		data, err := json.Marshal(address)
		if err != nil {
			log.Printf(err.Error())
			rest.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		err = b.Put([]byte(address.ID), []byte(data))
		return
	})
	w.WriteJson(address)
}
<commit_msg>Add basic GET /routes<commit_after>package main

import (
	"encoding/json"
	"flag"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"os/user"
	"strconv"
	"syscall"
	"time"

	"github.com/ant0ine/go-json-rest/rest"
	"github.com/boltdb/bolt"
	"github.com/docker/libcontainer/netlink"
	"github.com/docker/libcontainer/network"
	"github.com/rakyll/globalconf"
)

const (
	appName       = "tentacool"
	addressBucket = "address"
)

var (
	flagSocket = flag.String("socket", "/var/run/"+appName, "Path for unix socket")
	flagOwner  = flag.String("owner", "", "Ownership for socket")
	flagGroup  = flag.Int("group", -1, "Group for socket")
	flagDB     = flag.String("db", "/var/lib/"+appName+"/db", "Path for DB")
	// flagMode   = flag.Int("mode", 0640, "FileMode for socket")

	db *bolt.DB
	ln net.Listener
)

type Address struct {
	ID   string
	Link string
	IP   string
}

func main() {
	conf, err := globalconf.New(appName)
	if err != nil {
		log.Fatal(err)
	}
	conf.ParseAll()

	handler := rest.ResourceHandler{}
	err = handler.SetRoutes(
		&rest.Route{"GET", "/interfaces", GetIfaces},
		&rest.Route{"GET", "/interfaces/:iface", GetIface},
		&rest.Route{"POST", "/address", PostAddress},
		&rest.Route{"Get", "/address/:address", GetAddress},
		&rest.Route{"GET", "/routes", GetRoutes},
	)
	if err != nil {
		log.Fatal(err)
	}

	ln, err = net.Listen("unix", *flagSocket)
	if nil != err {
		log.Fatal(err)
	}

	if *flagOwner != "" {
		user, err := user.Lookup(*flagOwner)
		if err != nil {
			log.Fatal(err)
		}
		uid, err := strconv.Atoi(user.Uid)
		var gid int
		if *flagGroup != -1 {
			gid = *flagGroup
		} else {
			gid, err = strconv.Atoi(user.Gid)
		}
		if err := os.Chown(*flagSocket, uid, gid); err != nil {
			log.Fatal(err)
		}
	}

	// Handle common process-killing signals so we can gracefully shut down:
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, os.Interrupt, os.Kill, syscall.SIGTERM)
	go func(c chan os.Signal) {
		// Wait for a SIGINT or SIGKILL:
		sig := <-c
		log.Printf("Caught signal %s: shutting down.", sig)
		// Stop listening (and unlink the socket if unix type):
		ln.Close()
		db.Close()
		os.Exit(0)
	}(sigc)

	db, err = bolt.Open(*flagDB, 0600, &bolt.Options{Timeout: 1 * time.Second})
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	db.Update(func(tx *bolt.Tx) (err error) {
		_, err = tx.CreateBucketIfNotExists([]byte(addressBucket))
		if err != nil {
			log.Fatal(err)
		}
		return
	})

	log.Fatal(http.Serve(ln, &handler))
	defer ln.Close()
}

func GetIfaces(w rest.ResponseWriter, req *rest.Request) {
	interfaces, err := net.Interfaces()
	if err != nil {
		log.Printf(err.Error())
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.WriteJson(interfaces)
}

func GetIface(w rest.ResponseWriter, req *rest.Request) {
	iface, err := net.InterfaceByName(req.PathParam("iface"))
	if err != nil {
		log.Printf(err.Error())
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	ip, err := iface.Addrs()
	if err != nil {
		log.Printf(err.Error())
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.WriteJson(ip)
}

func GetAddress(w rest.ResponseWriter, req *rest.Request) {
	id := req.PathParam("address")

	address := Address{}
	db.View(func(tx *bolt.Tx) (err error) {
		tmp := tx.Bucket([]byte(addressBucket)).Get([]byte(id))
		if err = json.Unmarshal(tmp, &address); err != nil {
			log.Printf(err.Error())
			rest.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	})
	w.WriteJson(address)
}

func PostAddress(w rest.ResponseWriter, req *rest.Request) {
	address := Address{}
	if err := req.DecodeJsonPayload(&address); err != nil {
		log.Printf(err.Error())
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if err := network.SetInterfaceIp(address.Link, address.IP); err != nil {
		log.Printf(err.Error())
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	_ = netlink.AddRoute("", address.IP, "", address.Link)

	db.Update(func(tx *bolt.Tx) (err error) {
		b := tx.Bucket([]byte(addressBucket))
		if address.ID == "" {
			int, err := b.NextSequence()
			if err != nil {
				log.Printf(err.Error())
				rest.Error(w, err.Error(), http.StatusInternalServerError)
				return err
			}
			address.ID = strconv.FormatUint(int, 10)
		}
		data, err := json.Marshal(address)
		if err != nil {
			log.Printf(err.Error())
			rest.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		err = b.Put([]byte(address.ID), []byte(data))
		return
	})
	w.WriteJson(address)
}

func GetRoutes(w rest.ResponseWriter, req *rest.Request) {
	routes, err := netlink.NetworkGetRoutes()
	if err != nil {
		log.Printf(err.Error())
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.WriteJson(routes)
}
<|endoftext|>


<commit_before>package main

import (
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
)

//Puerto donde escuchar
const PORT = "8998"

//TODO: Por el momento suponemos que los pipes estan en /tmp
//Config por defecto, estaría bien usar swig para reusar la clase del daemon
//const CONFIG_FILE = "~/.config/player++/daemon.conf"

//Page contiene datos de la página
type Page struct {
	Title   string
	Request string
}

const templatesPath = "./templates/"

var templates = template.Must(template.ParseGlob(templatesPath + "[a-z]*"))

//RenderTemplate renderiza la template con el texto adecuado
func RenderTemplate(w http.ResponseWriter, tmpl string, title string, r interface{}) {
	var err error
	switch request := r.(type) {
	case string:
		err = templates.ExecuteTemplate(w, tmpl, &Page{title, request})
	case error:
		//TODO: Si es un error se imprima en otro formato
		err = templates.ExecuteTemplate(w, tmpl, &Page{title, request.Error()})
	default:
		log.Print("Strange error ocurred")
	}

	if err != nil {
		log.Print("ExecuteTemplate: ", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

// RootHandler muestra el index
func RootHandler(w http.ResponseWriter, r *http.Request) {
	RenderTemplate(w, "index", "index", "")
}

func NextHandler(w http.ResponseWriter, r *http.Request) {
	err := ioutil.WriteFile("/tmp/dplayer++", []byte{2}, 0666)
	if err != nil {
		panic(err)
	}
	RenderTemplate(w, "index", "index", "")
}

func PrevHandler(w http.ResponseWriter, r *http.Request) {
	err := ioutil.WriteFile("/tmp/dplayer++", []byte{3}, 0666)
	if err != nil {
		panic(err)
	}
	RenderTemplate(w, "index", "index", "")
}

func PauseHandler(w http.ResponseWriter, r *http.Request) {
	err := ioutil.WriteFile("/tmp/dplayer++", []byte{4}, 0666)
	if err != nil {
		panic(err)
	}
	RenderTemplate(w, "index", "index", "")
}

func main() {
	http.Handle("/resources/", http.StripPrefix("/resources/", http.FileServer(http.Dir("resources"))))
	http.HandleFunc("/", RootHandler)
	http.HandleFunc("/next/", NextHandler)
	http.HandleFunc("/prev/", PrevHandler)
	http.HandleFunc("/pause/", PauseHandler)

	log.Print("Escuchando en el puerto " + PORT)
	http.ListenAndServe(":"+PORT, nil)
}
<commit_msg>add commands for volume control<commit_after>package main

import (
	"bufio"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
)

//Puerto donde escuchar
const PORT = "8998"

//TODO: Se debe parsear el enum de los comandos de tal forma que no sea necesario tener los valores harcodeados

//TODO: Por el momento suponemos que los pipes estan en /tmp
//Config por defecto, estaría bien usar swig para reusar la clase del daemon
//const CONFIG_FILE = "~/.config/player++/daemon.conf"

//Page contiene datos de la página
type Page struct {
	Title   string
	Request string
}

const templatesPath = "./templates/"

var templates = template.Must(template.ParseGlob(templatesPath + "[a-z]*"))

//RenderTemplate renderiza la template con el texto adecuado
func RenderTemplate(w http.ResponseWriter, tmpl string, title string, r interface{}) {
	var err error
	switch request := r.(type) {
	case string:
		err = templates.ExecuteTemplate(w, tmpl, &Page{title, request})
	case error:
		//TODO: Si es un error se imprima en otro formato
		err = templates.ExecuteTemplate(w, tmpl, &Page{title, request.Error()})
	default:
		log.Print("Strange error ocurred")
	}

	if err != nil {
		log.Print("ExecuteTemplate: ", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

// RootHandler muestra el index
func RootHandler(w http.ResponseWriter, r *http.Request) {
	RenderTemplate(w, "index", "index", "")
}

func NextHandler(w http.ResponseWriter, r *http.Request) {
	err := ioutil.WriteFile("/tmp/dplayer++", []byte{2}, 0666)
	if err != nil {
		panic(err)
	}
	RenderTemplate(w, "index", "index", "")
}

func PrevHandler(w http.ResponseWriter, r *http.Request) {
	err := ioutil.WriteFile("/tmp/dplayer++", []byte{3}, 0666)
	if err != nil {
		panic(err)
	}
	RenderTemplate(w, "index", "index", "")
}

func PauseHandler(w http.ResponseWriter, r *http.Request) {
	err := ioutil.WriteFile("/tmp/dplayer++", []byte{4}, 0666)
	if err != nil {
		panic(err)
	}
	RenderTemplate(w, "index", "index", "")
}

func GetVolumeHandler(w http.ResponseWriter, r *http.Request) {
	err := ioutil.WriteFile("/tmp/dplayer++", []byte{16}, 0666)
	if err != nil {
		panic(err)
	}

	f, err := os.Open("/tmp/cplayer++")
	if err != nil {
		panic(err)
	}

	reader := bufio.NewReader(f)
	line, err := reader.ReadString('\n')
	if err != nil {
		panic(err)
	}
	f.Close()
	fmt.Fprint(w, line)
}

func SetVolumeHandler(w http.ResponseWriter, r *http.Request) {
	urlPart := strings.Split(r.URL.Path, "/")
	err := ioutil.WriteFile("/tmp/dplayer++", []byte{15}, 0666)
	if err != nil {
		panic(err)
	}
	err = ioutil.WriteFile("/tmp/dplayer++", []byte(urlPart[2]+"\n"), 0666)
	if err != nil {
		panic(err)
	}
}

func main() {
	http.Handle("/resources/", http.StripPrefix("/resources/", http.FileServer(http.Dir("resources"))))
	http.HandleFunc("/", RootHandler)
	http.HandleFunc("/next/", NextHandler)
	http.HandleFunc("/prev/", PrevHandler)
	http.HandleFunc("/pause/", PauseHandler)
	http.HandleFunc("/getvolume/", GetVolumeHandler)
	http.HandleFunc("/setvolume/", SetVolumeHandler)

	log.Print("Escuchando en el puerto " + PORT)
	http.ListenAndServe(":"+PORT, nil)
}
<|endoftext|>


<commit_before>package main

import (
	"sort"
	"time"

	"encoding/json"

	"flag"

	"io"

	"fmt"
	"os"

	"regexp"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/distribution/context"
	schema2 "github.com/docker/distribution/manifest/schema2"
	"github.com/docker/distribution/reference"
	"github.com/docker/distribution/registry/client"
)

var (
	/** CLI flags */
	// Base URL of registry
	registryURL *string
	// Regexps for filtering repositories and tags
	repoRegexp, tagRegexp *string
	// Maximum age of image to consider for deletion
	day, month, year *int
	// Max number of repositories to be fetched from registry
	repoCount *int
	// Number of the latest n matching images of an repository that will be ignored
	latest *int
	// If true, application runs in debug mode
	debug *bool
	// If true, no actual deletion is done
	dry *bool
	// If true, version is shown and program quits
	ver *bool
)

const (
	version string = "0.4.0"
)

func init() {
	/** CLI flags */
	// Max number of repositories to fetch from registry (default = 5)
	repoCount = flag.Int("repos", 5, "number of repositories to garbage collect")
	// Base URL of registry (default = http://localhost:5000)
	registryURL = flag.String("registry", "http://localhost:5000", "URL of registry")
	// Maximum age of iamges to consider for deletion in days (default = 0)
	day = flag.Int("day", 0, "max age in days")
	// Maximum age of months to consider for deletion in days (default = 0)
	month = flag.Int("month", 0, "max age in months")
	// Maximum age of iamges to consider for deletion in years (default = 0)
	year = flag.Int("year", 0, "max age in days")
	// Regexp for images (default = .*)
	repoRegexp = flag.String("repo", ".*", "matching repositories (allows regexp)")
	// Regexp for tags (default = .*)
	tagRegexp = flag.String("tag", ".*", "matching tags (allows regexp)")
	// The number of the latest matching images of an repository that won't be deleted
	latest = flag.Int("latest", 1, "number of the latest matching images of an repository that won't be deleted")
	// Dry run option (doesn't actually delete)
	debug = flag.Bool("debug", false, "run in debug mode")
	// Dry run option (doesn't actually delete)
	dry = flag.Bool("dry", false, "does not actually deletes")
	// Shows version
	ver = flag.Bool("v", false, "shows version and quits")
}

func main() {
	flag.Parse()

	if *ver {
		fmt.Printf("Version: %s\n", version)
		os.Exit(0)
	}

	if *debug {
		log.SetLevel(log.DebugLevel)
	}

	// Create registry object
	r, err := client.NewRegistry(*registryURL, nil)
	if err != nil {
		log.Fatalf("Could not create registry object! (err: %s", err)
	}

	// List of all repositories fetched from the registry. The number
	// of fetched repositories depends on the number provided by the
	// user ('-repos' flag)
	entries := make([]string, *repoCount)

	// Empty context for all requests in sequel
	ctx := context.Background()

	// Fetch all repositories from the registry
	numFilled, err := r.Repositories(ctx, entries, "")
	if err != nil && err != io.EOF {
		log.Fatalf("Error while fetching repositories! (err: %v)", err)
	}
	log.WithFields(log.Fields{"count": numFilled, "entries": entries}).Info("Successfully fetched repositories.")

	// Deadline defines the youngest creation date for an image
	// to be considered for deletion
	deadline := time.Now().AddDate(*year/-1, *month/-1, *day/-1)

	// Fetch information about images belonging to each repository
	for _, entry := range entries[:numFilled] {
		logger := log.WithField("repo", entry)

		matched, err := regexp.MatchString(*repoRegexp, entry)

		if matched == false {
			logger.WithFields(log.Fields{"entry": entry}).Debug("Ignore non matching repository (-repo=", *repoRegexp, ")")
			continue
		}

		// Establish repository object in registry
		repoName, err := reference.WithName(entry)

		if err != nil {
			logger.Fatalf("Could not parse repo from name! (err: %v)", err)
		}

		repo, err := client.NewRepository(repoName, *registryURL, nil)
		if err != nil {
			logger.WithFields(log.Fields{"entry": entry}).Fatalf("Could not create repo from name! (err: %v)", err)
		}
		logger.Debug("Successfully created repository object.")

		tagsService := repo.Tags(ctx)
		blobsService := repo.Blobs(ctx)
		manifestService, err := repo.Manifests(ctx)
		if err != nil {
			logger.Fatalf("Couldn't fetch manifest service! (err: %v)", err)
		}

		tagsData, err := tagsService.All(ctx)
		if err != nil {
			logger.Fatalf("Couldn't fetch tags! (err: %v)", err)
		}

		var tags []Image

		// Fetch information about each tag of a repository
		// This involves fetching the manifest, its details,
		// and the corresponding blob information
		tagFetchDataErrors := false
		for _, tag := range tagsData {
			tagLogger := logger.WithField("tag", tag)

			tagLogger.Debug("Fetching tag...")
			desc, err := tagsService.Get(ctx, tag)
			if err != nil {
				tagLogger.Error("Could not fetch tag!")
				tagFetchDataErrors = true
				break
			}

			tagLogger.Debug("Fetching manifest...")
			mnfst, err := manifestService.Get(ctx, desc.Digest)
			if err != nil {
				tagLogger.Error("Could not fetch manifest!")
				tagFetchDataErrors = true
				break
			}

			tagLogger.Debug("Parsing manifest details...")
			_, p, err := mnfst.Payload()
			if err != nil {
				tagLogger.Error("Could not parse manifest detail!")
				tagFetchDataErrors = true
				break
			}

			m := new(schema2.DeserializedManifest)
			m.UnmarshalJSON(p)

			tagLogger.Debug("Fetching blob")
			b, err := blobsService.Get(ctx, m.Manifest.Config.Digest)
			if err != nil {
				tagLogger.Error("Could not fetch blob!")
				tagFetchDataErrors = true
				break
			}

			var blobInfo BlobInfo
			json.Unmarshal(b, &blobInfo)

			tags = append(tags, Image{entry, tag, blobInfo.Created, desc})
		}

		if tagFetchDataErrors {
			// In case of error at any one tag, skip entire repo
			// (avoid acting on incomplete data, which migth lead to
			// deleting images that are actually in use)
			logger.Error("Error obtaining tag data - skipping this repo")
			continue
		}

		sort.Sort(ImageByDate(tags))

		logger.Debug("Analyzing tags...")

		tagCount := len(tags)

		if tagCount == 0 {
			logger.Debug("Ignore repository with no matching tags")
			continue
		}

		deletableTags := make(map[int]Image)
		nonDeletableTags := make(map[int]Image)

		ignoredTags := 0

		for tagIndex := len(tags) - 1; tagIndex >= 0; tagIndex-- {
			tag := tags[tagIndex]
			markForDeletion := false
			tagLogger := logger.WithField("tag", tag.Tag)

			matched, _ := regexp.MatchString(*tagRegexp, tag.Tag)

			if matched {
				tagLogger.Debug("Tag matches, considering for deletion (-tag=", *tagRegexp, ")")
				if tag.Time.Before(deadline) {
					if ignoredTags < *latest {
						tagLogger.WithField("time", tag.Time).Infof("Ignore %d latest matching tags (-latest=%d)", *latest, *latest)
						ignoredTags++
					} else {
						tagLogger.WithField("tag", tag.Tag).WithField("time", tag.Time).Infof("Marking tag as outdated")
						markForDeletion = true
					}
				} else {
					tagLogger.Info("Tag not outdated")
					ignoredTags++
				}
			} else {
				tagLogger.Info("Ignore non matching tag (-tag=", *tagRegexp, ")")
			}

			if markForDeletion {
				deletableTags[tagIndex] = tag
			} else {
				nonDeletableTags[tagIndex] = tag
			}
		}

		// This approach is actually a workaround for the problem that Docker
		// Distribution doesn't implement TagService.Untag operation at the time of
		// this writing.
		// Actually we have to delete the underlying image (specified via its Digest
		// value), taking care not to delete images that are referenced by tags which
		// we want to preserve
		nonDeletableDigests := make(map[string]string)
		for _, tag := range nonDeletableTags {
			if nonDeletableDigests[tag.Descriptor.Digest.String()] == "" {
				nonDeletableDigests[tag.Descriptor.Digest.String()] = tag.Tag
			} else {
				nonDeletableDigests[tag.Descriptor.Digest.String()] = nonDeletableDigests[tag.Descriptor.Digest.String()] + ", " + tag.Tag
			}
		}

		digestsDeleted := make(map[string]bool)
		for _, tag := range deletableTags {
			if !digestsDeleted[tag.Descriptor.Digest.String()] {
				if nonDeletableDigests[tag.Descriptor.Digest.String()] == "" {
					logger.WithField("tag", tag.Tag).Info("All tags for this image digest marked for deletion")
					if !*dry {
						logger.WithField("tag", tag.Tag).WithField("time", tag.Time).WithField("digest", tag.Descriptor.Digest).Infof("Deleting image (-dry=%v)", *dry)
						err := manifestService.Delete(context.Background(), tag.Descriptor.Digest)
						if err == nil {
							digestsDeleted[tag.Descriptor.Digest.String()] = true
						} else {
							logger.WithField("tag", tag.Tag).Error("Could not delete image!")
						}
					} else {
						logger.WithField("tag", tag.Tag).WithField("time", tag.Time).Infof("Not actually deleting image (-dry=%v)", *dry)
					}
				} else {
					logger.WithField("tag", tag.Tag).WithField("alsoUsedByTags", nonDeletableDigests[tag.Descriptor.Digest.String()]).Infof("The underlying image is also used by non-deletable tags - skipping deletion")
				}
			} else {
				logger.WithField("tag", tag.Tag).Debug("Image under tag already deleted")
			}
		}

	}
}
<commit_msg>Add negative regexp for tags<commit_after>package main

import (
	"sort"
	"strings"
	"time"

	"encoding/json"

	"flag"

	"io"

	"fmt"
	"os"

	"regexp"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/distribution/context"
	schema2 "github.com/docker/distribution/manifest/schema2"
	"github.com/docker/distribution/reference"
	"github.com/docker/distribution/registry/client"
)

var (
	/** CLI flags */
	// Base URL of registry
	registryURL *string
	// Regexps for filtering repositories and tags
	repoRegexp, tagRegexp, negTagRegexp *string
	// Maximum age of image to consider for deletion
	day, month, year *int
	// Max number of repositories to be fetched from registry
	repoCount *int
	// Number of the latest n matching images of an repository that will be ignored
	latest *int
	// If true, application runs in debug mode
	debug *bool
	// If true, no actual deletion is done
	dry *bool
	// If true, version is shown and program quits
	ver *bool
)

const (
	version string = "0.4.0"
)

func init() {
	/** CLI flags */
	// Max number of repositories to fetch from registry (default = 5)
	repoCount = flag.Int("repos", 5, "number of repositories to garbage collect")
	// Base URL of registry (default = http://localhost:5000)
	registryURL = flag.String("registry", "http://localhost:5000", "URL of registry")
	// Maximum age of iamges to consider for deletion in days (default = 0)
	day = flag.Int("day", 0, "max age in days")
	// Maximum age of months to consider for deletion in days (default = 0)
	month = flag.Int("month", 0, "max age in months")
	// Maximum age of iamges to consider for deletion in years (default = 0)
	year = flag.Int("year", 0, "max age in days")
	// Regexp for images (default = .*)
	repoRegexp = flag.String("repo", ".*", "matching repositories (allows regexp)")
	// Regexp for tags (default = .*)
	tagRegexp = flag.String("tag", ".*", "matching tags (allows regexp)")
	// Negative regexp for tags (default = empty)
	negTagRegexp = flag.String("ntag", "", "non matching tags (allows regexp)")
	// The number of the latest matching images of an repository that won't be deleted
	latest = flag.Int("latest", 1, "number of the latest matching images of an repository that won't be deleted")
	// Dry run option (doesn't actually delete)
	debug = flag.Bool("debug", false, "run in debug mode")
	// Dry run option (doesn't actually delete)
	dry = flag.Bool("dry", false, "does not actually deletes")
	// Shows version
	ver = flag.Bool("v", false, "shows version and quits")
}

func main() {
	flag.Parse()

	if *ver {
		fmt.Printf("Version: %s\n", version)
		os.Exit(0)
	}

	if *debug {
		log.SetLevel(log.DebugLevel)
	}

	// Create registry object
	r, err := client.NewRegistry(*registryURL, nil)
	if err != nil {
		log.Fatalf("Could not create registry object! (err: %s", err)
	}

	// List of all repositories fetched from the registry. The number
	// of fetched repositories depends on the number provided by the
	// user ('-repos' flag)
	entries := make([]string, *repoCount)

	// Empty context for all requests in sequel
	ctx := context.Background()

	// Fetch all repositories from the registry
	numFilled, err := r.Repositories(ctx, entries, "")
	if err != nil && err != io.EOF {
		log.Fatalf("Error while fetching repositories! (err: %v)", err)
	}
	log.WithFields(log.Fields{"count": numFilled, "entries": entries}).Info("Successfully fetched repositories.")

	// Deadline defines the youngest creation date for an image
	// to be considered for deletion
	deadline := time.Now().AddDate(*year/-1, *month/-1, *day/-1)

	// Fetch information about images belonging to each repository
	for _, entry := range entries[:numFilled] {
		logger := log.WithField("repo", entry)

		matched, err := regexp.MatchString(*repoRegexp, entry)

		if matched == false {
			logger.WithFields(log.Fields{"entry": entry}).Debug("Ignore non matching repository (-repo=", *repoRegexp, ")")
			continue
		}

		// Establish repository object in registry
		repoName, err := reference.WithName(entry)

		if err != nil {
			logger.Fatalf("Could not parse repo from name! (err: %v)", err)
		}

		repo, err := client.NewRepository(repoName, *registryURL, nil)
		if err != nil {
			logger.WithFields(log.Fields{"entry": entry}).Fatalf("Could not create repo from name! (err: %v)", err)
		}
		logger.Debug("Successfully created repository object.")

		tagsService := repo.Tags(ctx)
		blobsService := repo.Blobs(ctx)
		manifestService, err := repo.Manifests(ctx)
		if err != nil {
			logger.Fatalf("Couldn't fetch manifest service! (err: %v)", err)
		}

		tagsData, err := tagsService.All(ctx)
		if err != nil {
			logger.Fatalf("Couldn't fetch tags! (err: %v)", err)
		}

		var tags []Image

		// Fetch information about each tag of a repository
		// This involves fetching the manifest, its details,
		// and the corresponding blob information
		tagFetchDataErrors := false
		for _, tag := range tagsData {
			tagLogger := logger.WithField("tag", tag)

			tagLogger.Debug("Fetching tag...")
			desc, err := tagsService.Get(ctx, tag)
			if err != nil {
				tagLogger.Error("Could not fetch tag!")
				tagFetchDataErrors = true
				break
			}

			tagLogger.Debug("Fetching manifest...")
			mnfst, err := manifestService.Get(ctx, desc.Digest)
			if err != nil {
				tagLogger.Error("Could not fetch manifest!")
				tagFetchDataErrors = true
				break
			}

			tagLogger.Debug("Parsing manifest details...")
			_, p, err := mnfst.Payload()
			if err != nil {
				tagLogger.Error("Could not parse manifest detail!")
				tagFetchDataErrors = true
				break
			}

			m := new(schema2.DeserializedManifest)
			m.UnmarshalJSON(p)

			tagLogger.Debug("Fetching blob")
			b, err := blobsService.Get(ctx, m.Manifest.Config.Digest)
			if err != nil {
				tagLogger.Error("Could not fetch blob!")
				tagFetchDataErrors = true
				break
			}

			var blobInfo BlobInfo
			json.Unmarshal(b, &blobInfo)

			tags = append(tags, Image{entry, tag, blobInfo.Created, desc})
		}

		if tagFetchDataErrors {
			// In case of error at any one tag, skip entire repo
			// (avoid acting on incomplete data, which migth lead to
			// deleting images that are actually in use)
			logger.Error("Error obtaining tag data - skipping this repo")
			continue
		}

		sort.Sort(ImageByDate(tags))

		logger.Debug("Analyzing tags...")

		tagCount := len(tags)

		if tagCount == 0 {
			logger.Debug("Ignore repository with no matching tags")
			continue
		}

		deletableTags := make(map[int]Image)
		nonDeletableTags := make(map[int]Image)

		ignoredTags := 0

		for tagIndex := len(tags) - 1; tagIndex >= 0; tagIndex-- {
			tag := tags[tagIndex]
			markForDeletion := false
			tagLogger := logger.WithField("tag", tag.Tag)

			// Provides a text which is followed by the tag and ntag flag values. The
			// latter iff defined.
			withTagParens := func(text string) string {
				xs := []string{fmt.Sprintf("-tag=%s", *tagRegexp)}
				if *negTagRegexp != "" {
					xs = append(xs, fmt.Sprintf("-ntag=%s", *negTagRegexp))
				}
				return fmt.Sprintf("%s (%s)", text, strings.Join(xs, ", "))
			}

			// Check whether the tag matches. If that's the case, don't stop there, and
			// check for the negative regexp as well.
			if matched, _ := regexp.MatchString(*tagRegexp, tag.Tag); matched && *negTagRegexp != "" {
				negTagMatch, _ := regexp.MatchString(*negTagRegexp, tag.Tag)
				matched = !negTagMatch
			}

			if matched {
				tagLogger.Debug(withTagParens("Tag matches, considering for deletion"))
				if tag.Time.Before(deadline) {
					if ignoredTags < *latest {
						tagLogger.WithField("time", tag.Time).Infof("Ignore %d latest matching tags (-latest=%d)", *latest, *latest)
						ignoredTags++
					} else {
						tagLogger.WithField("tag", tag.Tag).WithField("time", tag.Time).Infof("Marking tag as outdated")
						markForDeletion = true
					}
				} else {
					tagLogger.Info("Tag not outdated")
					ignoredTags++
				}
			} else {
				tagLogger.Info(withTagParens("Ignore non matching tag"))
			}

			if markForDeletion {
				deletableTags[tagIndex] = tag
			} else {
				nonDeletableTags[tagIndex] = tag
			}
		}

		// This approach is actually a workaround for the problem that Docker
		// Distribution doesn't implement TagService.Untag operation at the time of
		// this writing.
		// Actually we have to delete the underlying image (specified via its Digest
		// value), taking care not to delete images that are referenced by tags which
		// we want to preserve
		nonDeletableDigests := make(map[string]string)
		for _, tag := range nonDeletableTags {
			if nonDeletableDigests[tag.Descriptor.Digest.String()] == "" {
				nonDeletableDigests[tag.Descriptor.Digest.String()] = tag.Tag
			} else {
				nonDeletableDigests[tag.Descriptor.Digest.String()] = nonDeletableDigests[tag.Descriptor.Digest.String()] + ", " + tag.Tag
			}
		}

		digestsDeleted := make(map[string]bool)
		for _, tag := range deletableTags {
			if !digestsDeleted[tag.Descriptor.Digest.String()] {
				if nonDeletableDigests[tag.Descriptor.Digest.String()] == "" {
					logger.WithField("tag", tag.Tag).Info("All tags for this image digest marked for deletion")
					if !*dry {
						logger.WithField("tag", tag.Tag).WithField("time", tag.Time).WithField("digest", tag.Descriptor.Digest).Infof("Deleting image (-dry=%v)", *dry)
						err := manifestService.Delete(context.Background(), tag.Descriptor.Digest)
						if err == nil {
							digestsDeleted[tag.Descriptor.Digest.String()] = true
						} else {
							logger.WithField("tag", tag.Tag).Error("Could not delete image!")
						}
					} else {
						logger.WithField("tag", tag.Tag).WithField("time", tag.Time).Infof("Not actually deleting image (-dry=%v)", *dry)
					}
				} else {
					logger.WithField("tag", tag.Tag).WithField("alsoUsedByTags", nonDeletableDigests[tag.Descriptor.Digest.String()]).Infof("The underlying image is also used by non-deletable tags - skipping deletion")
				}
			} else {
				logger.WithField("tag", tag.Tag).Debug("Image under tag already deleted")
			}
		}

	}
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/go-uuid/uuid"
	"encoding/json"
	"fmt"
	"github.com/gorilla/mux"
	"github.com/ory-am/common/env"
	"github.com/ory-am/common/mgopath"
	"github.com/ory-am/event"
	gde "github.com/ory-am/gitdeploy/event"
	"github.com/ory-am/gitdeploy/job"
	gdLog "github.com/ory-am/gitdeploy/log"
	"github.com/ory-am/gitdeploy/sse"
	"github.com/ory-am/gitdeploy/storage/mongo"
	"github.com/ory-am/google-json-style-response/responder"
	"log"
	"net/http"
	"os/exec"
	"regexp"
	"time"
)

var (
	// API Version
	ApiVersion = "1.0"

	// Generic configuration
	host = env.Getenv("HOST", "")
	port = env.Getenv("PORT", "7654")

	envAppTtl = env.Getenv("APP_TTL", "5m")

	// Configuration for CORS
	corsAllowOrigin = env.Getenv("CORS_ALLOW_ORIGIN", "*")

	// MongoDB
	envMongoPath = env.Getenv("MONGODB", "mongodb://localhost:27017/gitdeploy")
)

type deployRequest struct {
	Repository string `json:"repository"`
}

type appResponse struct {
	App string `json:"app"`
}

func main() {
	checkDependencies()
	eventManager := event.New()

	// mgo
	mongoDatabase, err := mgopath.Connect(envMongoPath)
	if err != nil {
		log.Fatal(err)
	}

	storage := mongo.New(mongoDatabase)
	eventManager.AttachListenerAggregate(storage)

	// SSE broker
	sseBroker := sse.New(storage)

	// Log listener
	eventManager.AttachListenerAggregate(new(gdLog.Listener))

	// Mux router
	r := mux.NewRouter()
	r.HandleFunc("/deployments", deployWrapperAction(sseBroker, eventManager, storage)).Methods("POST")
	r.HandleFunc("/deployments", setCORSHeaders).Methods("OPTIONS")
	r.HandleFunc("/deployments/{app:.+}/events", eventWrapperAction(sseBroker)).Methods("GET")
	r.PathPrefix("/").Handler(http.FileServer(http.Dir("./app/")))
	http.Handle("/", r)

	go job.KillAppsOnHitList(storage)

	listen := fmt.Sprintf("%s:%s", host, port)
	log.Printf("Listening on %s", listen)
	log.Fatal(http.ListenAndServe(listen, nil))
}

func eventWrapperAction(sseBroker *sse.Broker) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		setCORSHeaders(w, r)
		sseBroker.EventHandler(w, r)
	}
}

func deployWrapperAction(sseBroker *sse.Broker, em *event.EventManager, store *mongo.MongoStorage) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		setCORSHeaders(w, r)
		deployAction(w, r, sseBroker, em, store)
	}
}

func deployAction(w http.ResponseWriter, r *http.Request, sseBroker *sse.Broker, em *event.EventManager, store *mongo.MongoStorage) {
	// Parse body
	dr := new(deployRequest)
	decoder := json.NewDecoder(r.Body)
	decoder.Decode(dr)

	// Validate URL
	regExpression := "(https|http):\\/\\/github\\.com\\/[a-zA-Z0-9\\-\\_]+/[a-zA-Z0-9\\-\\_]+\\.git"
	if match, _ := regexp.MatchString(regExpression, dr.Repository); !match {
		responseError(w, http.StatusBadRequest, "I only support GitHub.")
		return
	}

	app := uuid.NewRandom().String()
	if ttl, err := time.ParseDuration(envAppTtl); err != nil {
		log.Println(err.Error())
		responseError(w, http.StatusInternalServerError, err.Error())
		return
	} else {
		appEntity, err := store.AddApp(app, time.Now().Add(ttl))
		if err != nil {
			log.Println(err.Error())
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}
		responseSuccess(w, appEntity)
	}

	go runJobs(w, em, dr, app, sseBroker)
}

func runJobs(w http.ResponseWriter, em *event.EventManager, dr *deployRequest, app string, sseBroker *sse.Broker) {
	sseBroker.OpenChannel(app)
	sseBroker.Start(app)
	defer func() {
		time.Sleep(2 * time.Minute)
		sseBroker.CloseChannel(app)
	}()
	em.Trigger("app.created", gde.New(app, app))

	destination, err := job.Clone(em, app, dr.Repository)
	if err != nil {
		responseError(w, http.StatusInternalServerError, err.Error())
		log.Printf("Error in job.clone %s: %s", app, err.Error())
		return
	}

	if err = job.Parse(em, app, destination); err != nil {
		log.Printf("Error in job.parse %s: %s", app, err.Error())
		return
	}

	if err = job.Deploy(em, app, destination); err != nil {
		log.Printf("Error in job.deploy %s: %s", app, err.Error())
		return
	}

	log.Println("Deployment successful.")
	em.Trigger("app.deployed", gde.New(app, fmt.Sprintf("%s.ew2.flynnhub.com", app)))
}

// Set the different CORS headers required for CORS request
func setCORSHeaders(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Access-Control-Allow-Methods", "POST")
	w.Header().Set("Access-Control-Allow-Origin", corsAllowOrigin)
	w.Header().Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept")
}

func responseError(w http.ResponseWriter, code int, message string) {
	response := responder.New(ApiVersion)
	response.Write(w, response.Error(code, message))
}

func responseSuccess(w http.ResponseWriter, data interface{}) {
	response := responder.New(ApiVersion)
	response.Write(w, response.Success(data))
}

func checkDependencies() {
	go func() {
		_, err := exec.LookPath("git")
		if err != nil {
			log.Fatal("Git CLI is required but not installed or not in path.")
		}
	}()
	go func() {
		_, err := exec.LookPath("flynn")
		if err != nil {
			log.Fatal("Flynn CLI is required but not installed or not in path.")
		}
	}()
	go func() {
		_, err := exec.LookPath("bash")
		if err != nil {
			log.Fatal("Bash is required but not installed or not in path.")
		}
	}()
}
<commit_msg>Refactor<commit_after>package main

import (
	"code.google.com/p/go-uuid/uuid"
	"encoding/json"
	"fmt"
	"github.com/gorilla/mux"
	"github.com/ory-am/common/env"
	"github.com/ory-am/common/mgopath"
	"github.com/ory-am/event"
	gde "github.com/ory-am/gitdeploy/event"
	"github.com/ory-am/gitdeploy/job"
	gdLog "github.com/ory-am/gitdeploy/log"
	"github.com/ory-am/gitdeploy/sse"
	"github.com/ory-am/gitdeploy/storage/mongo"
	"github.com/ory-am/google-json-style-response/responder"
	"log"
	"net/http"
	"os/exec"
	"regexp"
	"time"
)

var (
	// API Version
	ApiVersion = "1.0"

	// Generic configuration
	host = env.Getenv("HOST", "")
	port = env.Getenv("PORT", "7654")

	envAppTtl = env.Getenv("APP_TTL", "5m")

	// Configuration for CORS
	corsAllowOrigin = env.Getenv("CORS_ALLOW_ORIGIN", "*")

	// MongoDB
	envMongoPath = env.Getenv("MONGODB", "mongodb://localhost:27017/gitdeploy")
)

type deployRequest struct {
	Repository string `json:"repository"`
}

type appResponse struct {
	App string `json:"app"`
}

func main() {
	checkDependencies()
	eventManager := event.New()

	// mgo
	mongoDatabase, err := mgopath.Connect(envMongoPath)
	if err != nil {
		log.Fatal(err)
	}

	storage := mongo.New(mongoDatabase)
	eventManager.AttachListenerAggregate(storage)

	// SSE broker
	sseBroker := sse.New(storage)

	// Log listener
	eventManager.AttachListenerAggregate(new(gdLog.Listener))

	// Mux router
	r := mux.NewRouter()
	r.HandleFunc("/deployments", deployWrapperAction(sseBroker, eventManager, storage)).Methods("POST")
	r.HandleFunc("/deployments", setCORSHeaders).Methods("OPTIONS")
	r.HandleFunc("/deployments/{app:.+}/events", eventWrapperAction(sseBroker)).Methods("GET")
	r.PathPrefix("/").Handler(http.FileServer(http.Dir("./app/")))
	http.Handle("/", r)

	go job.KillAppsOnHitList(storage)

	listen := fmt.Sprintf("%s:%s", host, port)
	log.Printf("Listening on %s", listen)
	log.Fatal(http.ListenAndServe(listen, nil))
}

func eventWrapperAction(sseBroker *sse.Broker) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		setCORSHeaders(w, r)
		sseBroker.EventHandler(w, r)
	}
}

func deployWrapperAction(sseBroker *sse.Broker, em *event.EventManager, store *mongo.MongoStorage) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		setCORSHeaders(w, r)
		deployAction(w, r, sseBroker, em, store)
	}
}

func deployAction(w http.ResponseWriter, r *http.Request, sseBroker *sse.Broker, em *event.EventManager, store *mongo.MongoStorage) {
	// Parse body
	dr := new(deployRequest)
	decoder := json.NewDecoder(r.Body)
	decoder.Decode(dr)

	// Validate URL
	regExpression := "(https|http):\\/\\/github\\.com\\/[a-zA-Z0-9\\-\\_]+/[a-zA-Z0-9\\-\\_]+\\.git"
	if match, _ := regexp.MatchString(regExpression, dr.Repository); !match {
		responseError(w, http.StatusBadRequest, "I only support GitHub.")
		return
	}

	app := uuid.NewRandom().String()
	if ttl, err := time.ParseDuration(envAppTtl); err != nil {
		log.Println(err.Error())
		responseError(w, http.StatusInternalServerError, err.Error())
		return
	} else {
		appEntity, err := store.AddApp(app, time.Now().Add(ttl))
		if err != nil {
			log.Println(err.Error())
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}
		responseSuccess(w, appEntity)
	}

	go runJobs(w, em, dr, app, sseBroker)
}

func runJobs(w http.ResponseWriter, em *event.EventManager, dr *deployRequest, app string, sseBroker *sse.Broker) {
	sseBroker.OpenChannel(app)
	sseBroker.Start(app)
	defer func() {
		// Give the client the chance to read the output...
		time.Sleep(2 * time.Minute)
		sseBroker.CloseChannel(app)
	}()

	em.Trigger("app.created", gde.New(app, app))

	destination, err := job.Clone(em, app, dr.Repository)
	if err != nil {
		responseError(w, http.StatusInternalServerError, err.Error())
		log.Printf("Error in job.clone %s: %s", app, err.Error())
		return
	}

	if err = job.Parse(em, app, destination); err != nil {
		log.Printf("Error in job.parse %s: %s", app, err.Error())
		return
	}

	if err = job.Deploy(em, app, destination); err != nil {
		log.Printf("Error in job.deploy %s: %s", app, err.Error())
		return
	}

	log.Println("Deployment successful.")
	em.Trigger("app.deployed", gde.New(app, fmt.Sprintf("%s.ew2.flynnhub.com", app)))
}

// Set the different CORS headers required for CORS request
func setCORSHeaders(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Access-Control-Allow-Methods", "POST")
	w.Header().Set("Access-Control-Allow-Origin", corsAllowOrigin)
	w.Header().Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept")
}

func responseError(w http.ResponseWriter, code int, message string) {
	response := responder.New(ApiVersion)
	response.Write(w, response.Error(code, message))
}

func responseSuccess(w http.ResponseWriter, data interface{}) {
	response := responder.New(ApiVersion)
	response.Write(w, response.Success(data))
}

func checkDependencies() {
	go func() {
		_, err := exec.LookPath("git")
		if err != nil {
			log.Fatal("Git CLI is required but not installed or not in path.")
		}
	}()
	go func() {
		_, err := exec.LookPath("flynn")
		if err != nil {
			log.Fatal("Flynn CLI is required but not installed or not in path.")
		}
	}()
	go func() {
		_, err := exec.LookPath("bash")
		if err != nil {
			log.Fatal("Bash is required but not installed or not in path.")
		}
	}()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/gorilla/mux"
)

func main() {
	fmt.Println("starting")
	redis_client, err := redis.Dial("tcp", ":6379")
	if err != nil {
		panic(err)
	}
	defer redis_client.Close()

	router := mux.NewRouter().StrictSlash(true)
	router.HandleFunc("/api/bins", BinIndexHandler(redis_client))
	router.HandleFunc("/api/bins/{binId}", BinHandler(redis_client))
	router.HandleFunc("/_/{binId}/{rest:.*}", LogHandler(redis_client))
	log.Fatal(http.ListenAndServe(":8080", router))
}

type Request struct {
	Url        string      `json:"url"`
	Method     string      `json:"method"`
	Time       time.Time   `json:"time"`
	Headers    http.Header `json:"headers"`
	Body       string      `json:"body"`
	Host       string      `json:"host"`
	RemoteAddr string      `json:"remote_addr"`
	PostForm   url.Values  `json:"post_form"`
	Form       url.Values  `json:"form"`
	JSON       interface{} `json:"json"`
}

func ParseRequest(r *http.Request) Request {
	// Read the content
	var bodyBytes []byte
	if r.Body != nil {
		bodyBytes, _ = ioutil.ReadAll(r.Body)
	}
	// Restore the io.ReadCloser to its original state
	r.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))

	// Use the content
	body := string(bodyBytes)

	if err := r.ParseForm(); err != nil {
		panic(err)
	}

	var json_content interface{}
	if err := json.NewDecoder(r.Body).Decode(&json_content); err != nil {
		// ignore json parsing errors
	}

	return Request{
		Url:        r.URL.Path,
		Method:     r.Method,
		Host:       r.Host,
		Time:       time.Now(),
		Headers:    r.Header,
		Body:       string(body),
		RemoteAddr: r.RemoteAddr,
		PostForm:   r.PostForm,
		Form:       r.Form,
		JSON:       json_content,
	}

}

func json_response(w http.ResponseWriter, body interface{}) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.WriteHeader(http.StatusOK)
	if err := json.NewEncoder(w).Encode(body); err != nil {
		panic(err)
	}
}

func BinHandler(redis_client redis.Conn) func(http.ResponseWriter, *http.Request) {

	return func(w http.ResponseWriter, r *http.Request) {
		bin := mux.Vars(r)["binId"]

		raw_requests, err := redis.Strings(redis_client.Do("LRANGE", "bins:"+bin, 0, 10))
		if err != nil {
			panic(err)
		}
		var requests = make([]Request, len(raw_requests))
		for i, item := range raw_requests {
			if err = json.Unmarshal([]byte(item), &requests[i]); err != nil {
				panic(err)
			}
		}
		json_response(w, requests)
	}
}

func BinIndexHandler(redis_client redis.Conn) func(http.ResponseWriter, *http.Request) {

	return func(w http.ResponseWriter, r *http.Request) {
		bins, err := redis.Strings(redis_client.Do("SMEMBERS", "bins"))
		if err != nil {
			panic(err)
		}
		json_response(w, bins)
	}
}

func LogHandler(redis_client redis.Conn) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		request := ParseRequest(r)
		bin := mux.Vars(r)["binId"]

		serialised, err := json.Marshal(request)
		if err != nil {
			panic(err)
		}
		redis_client.Do("SADD", "bins", bin)
		redis_client.Do("LPUSH", "bins:"+bin, string(serialised))
		json_response(w, request)
	}
}
<commit_msg>Extracted redis functions<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/gorilla/mux"
)

func main() {
	fmt.Println("starting")
	redis_client, err := redis.Dial("tcp", ":6379")
	if err != nil {
		panic(err)
	}
	defer redis_client.Close()

	router := mux.NewRouter().StrictSlash(true)
	router.HandleFunc("/api/bins", BinIndexHandler(redis_client))
	router.HandleFunc("/api/bins/{binId}", BinHandler(redis_client))
	router.HandleFunc("/_/{binId}", LogHandler(redis_client))
	log.Fatal(http.ListenAndServe(":8080", router))
}

type Request struct {
	Url        string      `json:"url"`
	Method     string      `json:"method"`
	Time       time.Time   `json:"time"`
	Headers    http.Header `json:"headers"`
	Body       string      `json:"body"`
	Host       string      `json:"host"`
	RemoteAddr string      `json:"remote_addr"`
	PostForm   url.Values  `json:"post_form"`
	Form       url.Values  `json:"form"`
	JSON       interface{} `json:"json"`
}

func ParseRequest(r *http.Request) Request {
	// Read the content
	var bodyBytes []byte
	if r.Body != nil {
		bodyBytes, _ = ioutil.ReadAll(r.Body)
	}
	// Restore the io.ReadCloser to its original state
	r.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))

	// Use the content
	body := string(bodyBytes)

	if err := r.ParseForm(); err != nil {
		panic(err)
	}

	var json_content interface{}
	if err := json.NewDecoder(r.Body).Decode(&json_content); err != nil {
		// ignore json parsing errors
	}

	return Request{
		Url:        r.URL.Path,
		Method:     r.Method,
		Host:       r.Host,
		Time:       time.Now(),
		Headers:    r.Header,
		Body:       string(body),
		RemoteAddr: r.RemoteAddr,
		PostForm:   r.PostForm,
		Form:       r.Form,
		JSON:       json_content,
	}
}

func ListBins(redis_client redis.Conn) []string {
	bins, err := redis.Strings(redis_client.Do("SMEMBERS", "bins"))
	if err != nil {
		panic(err)
	}
	return bins
}

func ListRequestsFromBin(redis_client redis.Conn, binId string) []Request {
	raw_requests, err := redis.Strings(redis_client.Do("LRANGE", "bins:"+binId, 0, 10))
	if err != nil {
		panic(err)
	}
	var requests = make([]Request, len(raw_requests))
	for i, item := range raw_requests {
		if err = json.Unmarshal([]byte(item), &requests[i]); err != nil {
			panic(err)
		}
	}
	return requests
}

func StoreRequest(redis_client redis.Conn, binId string, request Request) {
	serialised, err := json.Marshal(request)
	if err != nil {
		panic(err)
	}
	redis_client.Do("SADD", "bins", binId)
	redis_client.Do("LPUSH", "bins:"+binId, string(serialised))
}

func json_response(w http.ResponseWriter, body interface{}) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.WriteHeader(http.StatusOK)
	if err := json.NewEncoder(w).Encode(body); err != nil {
		panic(err)
	}
}

func BinHandler(redis_client redis.Conn) func(http.ResponseWriter, *http.Request) {

	return func(w http.ResponseWriter, r *http.Request) {
		bin := mux.Vars(r)["binId"]
		requests := ListRequestsFromBin(redis_client, bin)
		json_response(w, requests)
	}
}

func BinIndexHandler(redis_client redis.Conn) func(http.ResponseWriter, *http.Request) {

	return func(w http.ResponseWriter, r *http.Request) {
		bins := ListBins(redis_client)
		json_response(w, bins)
	}
}

func LogHandler(redis_client redis.Conn) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		request := ParseRequest(r)
		bin := mux.Vars(r)["binId"]

		StoreRequest(redis_client, bin, request)
		json_response(w, request)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
	"time"
	_ "github.com/go-sql-driver/mysql"
	mailgun "github.com/riobard/go-mailgun"
)

type User struct {
	Id int
	username string
	Email string
}

type GithubUser struct {
	Id int
	Login string
	Url string
}

// db thing.
type GithubRepo struct {
	Id int
	User string // should probably be Login
	RepoName string // should probably be Name
}

func (g *GithubRepo) FullName() string {
	return fmt.Sprintf("%s/%s", g.User, g.RepoName)
}

type GithubApiRepo struct {
	Id int
	Owner GithubUser
	Name NString
	Description NString
}

func (g *GithubApiRepo) FullName() string {
	return fmt.Sprintf("%s/%s", g.Owner.Login, g.Name)
}

type Activity struct {
	Id int
	github_id int
	activity_type string // should be enum
	created_at time.Time
	Username string
	repo GithubRepo // this isn't going to be memory efficient
	Meta string // full payload of json object
}

type Treeish struct {
	Label string
	Sha string
	Repo GithubApiRepo
	Html_url string
	User GithubUser
}

type Commit struct {
	Sha string
	Message string
	Author struct {
		Name string
		Email string
	}
}

func (c *Commit) ShortSha() string {
	return c.Sha[0:6]
}

func (c *Commit) ShortCommit() string {
	msg := strings.Split(c.Message, "\n")[0]
	if len(msg) > 80 {
		return msg[0:77] + "..."
	}
	return msg
}

// Nullable String
type NString string

func (n *NString) UnmarshalJSON(b []byte) (err error) {
	if string(b) == "null" {
		return nil
	}
	return json.Unmarshal(b, (*string)(n))
}

func get_users_repos(db *sql.DB, user_id int) ([]GithubRepo, error) {
	var repo_count int

	// TODO(justinabrahms): Should really alter the schema such
	// that the join from user <-> repo doesn't go through
	// userprofiles.
	row := db.QueryRow("SELECT count(*) FROM streamer_repo r" +
	" JOIN streamer_userprofile_repos upr ON upr.repo_id = r.id " +
	" JOIN streamer_userprofile up ON up.id = upr.userprofile_id" +
	" WHERE up.user_id = ?;", user_id)

	err := row.Scan(&repo_count)
	if (err != nil) {
		return nil, err
	}
	var repos = make([]GithubRepo, repo_count)

	rows, err := db.Query("SELECT r.id, username, project_name FROM streamer_repo r" +
	" JOIN streamer_userprofile_repos upr ON upr.repo_id = r.id " +
	" JOIN streamer_userprofile up ON up.id = upr.userprofile_id" +
	" WHERE up.user_id = ?;", user_id)
	// what's a good way to handle this not working?
	if (err != nil) {
		return nil, err
	}
	defer rows.Close()

	var username, project_name string
	var pk int
	for rows.Next() {
		err = rows.Scan(&pk, &username, &project_name)
		if (err != nil) {
			fmt.Println("Error!: ", err)
		}
		var repo = GithubRepo{pk, username, project_name}
		repos = append(repos, repo)
	}

	return repos, err
}

func get_user(db *sql.DB, uid int) (u User, err error) {
	row := db.QueryRow(
		"SELECT id, username, email" +
		" FROM auth_user" +
		" WHERE id = ?", uid)
	err = row.Scan(&u.Id, &u.username, &u.Email)
	return
}

func get_users(db *sql.DB) (users []User, err error) {
	rows, err := db.Query(
		"SELECT id, username, email" +
		" FROM auth_user")
	if err != nil { return }
	defer rows.Close()

	for rows.Next() {
		var u User
		err = rows.Scan(&u.Id, &u.username, &u.Email)
		if err != nil { return }
		users = append(users, u)
	}
	return
}

func get_repo_activity(db *sql.DB, repo *GithubRepo) (activity_list []Activity, err error){
	activity_list = make([]Activity, 0)
	rows, err := db.Query(
		"SELECT a.id, a.event_id, a.type, a.created_at, ghu.name, r.username, r.project_name, meta" +
		" FROM streamer_activity a" +
		" JOIN streamer_repo r on r.id=a.repo_id" +
		" JOIN streamer_githubuser ghu on ghu.id=a.user_id" +
		" JOIN streamer_userprofile_repos upr on r.id=upr.repo_id" +
		" WHERE r.id = ?" +
		" AND a.created_at > DATE_SUB(NOW(), INTERVAL 5 day)" + // don't send things more than a few days old. Think, new users who subscribe to rails/rails
		" AND (upr.last_sent is null" + // hasn't been sent at all
		"   OR a.created_at > upr.last_sent)",  // or hasn't been sent since we've gotten new stuff
		repo.Id)
	if (err != nil) {
		return
	}
	defer rows.Close()

	var (
		pk, github_id, repo_id int
		activity_type, username, repo_project, repo_user, meta, created_str string
		created_at time.Time
	)

	for rows.Next() {
		err = rows.Scan(&pk, &github_id, &activity_type, &created_str, &username, &repo_user, &repo_project, &meta)
		if (err != nil) {
			fmt.Println("ERR: ", err)
		}

		created_at, err = time.Parse("2006-01-02 15:04:05", created_str)
		if (err != nil) {
			fmt.Println("Can't parse the format of ", created_str)
			return nil, err
		}

		activity_list = append(activity_list,
			Activity{pk, github_id, activity_type, created_at, username,
			GithubRepo{repo_id, repo_user, repo_project}, meta})
	}

	return
}

// basically need to fill these in, likely moving them all to their own files.
func commit_comment_render(activities []Activity, long_template bool) string { return "" }
func pull_request_comment_render(activities []Activity, long_template bool) string { return "" }
func member_render(activities []Activity, long_template bool) string { return "" }
func download_render(activities []Activity, long_template bool) string { return "" }
func fork_apply_render(activities []Activity, long_template bool) string { return "" }
func team_add_render(activities []Activity, long_template bool) string { return "" }
func gist_render(activities []Activity, long_template bool) string { return "" }
func follow_render(activities []Activity, long_template bool) string { return "" }

func repo_to_template(repo GithubRepo, activities []Activity, render_map map[string]func([]Activity, bool)string) string {
	if len(activities) == 0 {
		return ""
	}
	var activity_map = make(map[string][]Activity)
	for _, activity := range activities {
		// This seems like a lot of juggling. Is there a better way?
		arr := activity_map[activity.activity_type]
		if (arr == nil) {
			arr = make([]Activity, 0)
			activity_map[activity.activity_type] = arr
		}
		arr = append(arr, activity)
		activity_map[activity.activity_type] = arr
	}

	// activity_map: activity_type => []activity
	var response = fmt.Sprintf("%s:\n", repo.FullName())
	for activity_type, activities := range activity_map {
		fn, ok := render_map[activity_type]
		if ok {
			response += fn(activities, true)
		} else {
			log.Print("Not sure how to render activites of type ", activity_type)
		}
	}
	return response
}

func repo_to_string(db *sql.DB, repo GithubRepo, response chan string) {
	activities, err := get_repo_activity(db, &repo)
	if (err != nil) {
		fmt.Println("ERR: ", err)
		os.Exit(1)
	}

	activity_type_to_renderer := map[string]func([]Activity, bool)string {
		"P": push_render,
		"PR": pull_request_render,
		"D": delete_render,
		"C": create_render,
		"W": watch_render,
		"F": fork_render,
		"IC": issue_comment_render,
		"Gl": wiki_render, // Gl is for Gollum, Github's wiki thing.
		"I": issue_render,
		"Pb": public_render,
	}

	response <- repo_to_template(repo, activities, activity_type_to_renderer)
}

func mark_user_repo_sent(db *sql.DB, user User, repos []GithubRepo) (err error) {
	// finds the streamer_userprofile_repo row for the repo / user
	// combo, mark its last_sent as now
	ids := make([]string, 0)
	for _, repo := range repos {
		// TODO: why are they 0?
		if repo.Id != 0 {
			ids = append(ids, strconv.FormatInt(int64(repo.Id), 10))
		}
	}

	// There is likely a better way to get parameterization, but
	// it wasn't working for me with ?'s.
	str := fmt.Sprintf(
		"UPDATE streamer_userprofile_repos " +
		"SET last_sent=NOW() " +
		"WHERE repo_id IN (%s)", strings.Join(ids, ","))
	_, err = db.Exec(str)
	return
}

// TODO: Github Users

// TODO: Need to finish the following activity types:
// - TA (team add)
// - FA (fork apply)
// - DO (Download)
// - RC (Pull Request Review Comment)
// - Fl (Follow)
// - G (Gist)
// - M (Member)
// - CC (Commit Comment)

var user_id = flag.Int("user_id", 0, "ID of user to output.")
var all_users = flag.Bool("all_users", false, "Whether to email all users.")
var send_email = flag.Bool("send_email", false, "Whether to send the email to the user.")
var mark_read = flag.Bool("mark_read", true, "Whether to mark activity sent as read. False means subsequent calls will send the same info.")

func main() {
	// Expects the following environment variables:
	// DB_USER, DB_PASS, DB_DB  (database user, password and database)
	// MAILGUN_API_KEY  (api key from mailgun.org)

	flag.Parse()

	db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@/%s?charset=utf8", os.Getenv("DB_USER"), os.Getenv("DB_PASS"), os.Getenv("DB_DB")))
	if (err != nil) {
		log.Fatalf("Unable to connect to mysql. ", err)
	}
	defer db.Close()

	var users []User
	if (*user_id != 0) {
		users = make([]User, 1)
		user, err := get_user(db, *user_id)
		if err != nil { log.Fatalf("Couldn't return user %d. %s", *user_id, err) }
		users[0] = user
	} else if (*all_users) {
		users, err = get_users(db)
		if err != nil { log.Fatal("Couldn't fetch all users.") }
	} else {
		log.Fatal("You must specify either to mail all users or a specific user id.")
	}

	for _, user := range users {
		repos, err := get_users_repos(db, user.Id)
		if err != nil {
			log.Print("Error fetching user's repos. %s", err)
			continue
		}

		response_chan := make(chan string, len(repos))
		for _, repo := range repos {
			repo_to_string(db, repo, response_chan)
		}

		var response string
		for num_responses := 0; num_responses < len(repos); {
			select {
			case r := <-response_chan:
				num_responses++
				response += r
			}
		}

		if *mark_read {
			fmt.Println("Marking read.")
			err = mark_user_repo_sent(db, user, repos)
			if err != nil {
				log.Print("Error updating repositories as sent.")
				continue
			}
		}

		if len(response) == 0 {
			log.Print("No content to email.")
			continue
		}

		if *send_email {
			mg := mailgun.Open(os.Getenv("MAILGUN_API_KEY"))
			e := &Email{
				from: "justin@gitstreams.mailgun.org",
				to: []string{user.Email},
				subject: "Gitstreams Digest Email",
				text: response,
			}

			id, err := mg.Send(e)

			if err != nil {
				log.Print("Unable to send email. ", err)
			}
			log.Printf("MessageId = %s for uid:%s", id, user.Id)
		} else {
			fmt.Println("Would have sent the following email.")
			fmt.Println(response)
		}
	}

	return
}
<commit_msg>doc cleanup, sql reformatting.<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
	"time"
	_ "github.com/go-sql-driver/mysql"
	mailgun "github.com/riobard/go-mailgun"
)

type User struct {
	Id int
	username string
	Email string
}

type GithubUser struct {
	Id int
	Login string
	Url string
}

// db thing.
type GithubRepo struct {
	Id int
	User string // should probably be Login
	RepoName string // should probably be Name
}

func (g *GithubRepo) FullName() string {
	return fmt.Sprintf("%s/%s", g.User, g.RepoName)
}

type GithubApiRepo struct {
	Id int
	Owner GithubUser
	Name NString
	Description NString
}

func (g *GithubApiRepo) FullName() string {
	return fmt.Sprintf("%s/%s", g.Owner.Login, g.Name)
}

type Activity struct {
	Id int
	github_id int
	activity_type string // should be enum
	created_at time.Time
	Username string
	repo GithubRepo // this isn't going to be memory efficient
	Meta string // full payload of json object
}

type Treeish struct {
	Label string
	Sha string
	Repo GithubApiRepo
	Html_url string
	User GithubUser
}

type Commit struct {
	Sha string
	Message string
	Author struct {
		Name string
		Email string
	}
}

func (c *Commit) ShortSha() string {
	return c.Sha[0:6]
}

func (c *Commit) ShortCommit() string {
	msg := strings.Split(c.Message, "\n")[0]
	if len(msg) > 80 {
		return msg[0:77] + "..."
	}
	return msg
}

// Nullable String, per https://groups.google.com/d/msg/golang-nuts/JOFWAqrTbUs/KZIGEPclSpwJ
type NString string

func (n *NString) UnmarshalJSON(b []byte) (err error) {
	if string(b) == "null" {
		return nil
	}
	return json.Unmarshal(b, (*string)(n))
}

func get_users_repos(db *sql.DB, user_id int) ([]GithubRepo, error) {
	var repo_count int

	// TODO(justinabrahms): Should really alter the schema such
	// that the join from user <-> repo doesn't go through
	// userprofiles.
	row := db.QueryRow(
		"SELECT count(*)" +
		" FROM streamer_repo r" +
		" JOIN streamer_userprofile_repos upr ON upr.repo_id = r.id " +
		" JOIN streamer_userprofile up ON up.id = upr.userprofile_id" +
		" WHERE up.user_id = ?;", user_id)
	err := row.Scan(&repo_count)
	if (err != nil) {
		return nil, err
	}
	var repos = make([]GithubRepo, repo_count)

	rows, err := db.Query(
		"SELECT r.id, username, project_name" +
		" FROM streamer_repo r" +
		" JOIN streamer_userprofile_repos upr ON upr.repo_id = r.id " +
		" JOIN streamer_userprofile up ON up.id = upr.userprofile_id" +
		" WHERE up.user_id = ?;", user_id)
	// what's a good way to handle this not working?
	if (err != nil) {
		return nil, err
	}
	defer rows.Close()

	var username, project_name string
	var pk int
	for rows.Next() {
		err = rows.Scan(&pk, &username, &project_name)
		if (err != nil) {
			fmt.Println("Error!: ", err)
		}
		var repo = GithubRepo{pk, username, project_name}
		repos = append(repos, repo)
	}

	return repos, err
}

func get_user(db *sql.DB, uid int) (u User, err error) {
	row := db.QueryRow(
		"SELECT id, username, email" +
		" FROM auth_user" +
		" WHERE id = ?", uid)
	err = row.Scan(&u.Id, &u.username, &u.Email)
	return
}

func get_users(db *sql.DB) (users []User, err error) {
	rows, err := db.Query(
		"SELECT id, username, email" +
		" FROM auth_user")
	if err != nil { return }
	defer rows.Close()

	for rows.Next() {
		var u User
		err = rows.Scan(&u.Id, &u.username, &u.Email)
		if err != nil { return }
		users = append(users, u)
	}
	return
}

func get_repo_activity(db *sql.DB, repo *GithubRepo) (activity_list []Activity, err error){
	activity_list = make([]Activity, 0)
	rows, err := db.Query(
		"SELECT a.id, a.event_id, a.type, a.created_at, ghu.name, r.username, r.project_name, meta" +
		" FROM streamer_activity a" +
		" JOIN streamer_repo r on r.id=a.repo_id" +
		" JOIN streamer_githubuser ghu on ghu.id=a.user_id" +
		" JOIN streamer_userprofile_repos upr on r.id=upr.repo_id" +
		" WHERE r.id = ?" +
		" AND a.created_at > DATE_SUB(NOW(), INTERVAL 5 day)" + // don't send things more than a few days old. Think, new users who subscribe to rails/rails
		" AND (upr.last_sent is null" + // hasn't been sent at all
		"   OR a.created_at > upr.last_sent)",  // or hasn't been sent since we've gotten new stuff
		repo.Id)
	if (err != nil) {
		return
	}
	defer rows.Close()

	var (
		pk, github_id, repo_id int
		activity_type, username, repo_project, repo_user, meta, created_str string
		created_at time.Time
	)

	for rows.Next() {
		err = rows.Scan(&pk, &github_id, &activity_type, &created_str, &username, &repo_user, &repo_project, &meta)
		if (err != nil) {
			fmt.Println("ERR: ", err)
		}

		created_at, err = time.Parse("2006-01-02 15:04:05", created_str)
		if (err != nil) {
			fmt.Println("Can't parse the format of ", created_str)
			return nil, err
		}

		activity_list = append(activity_list,
			Activity{pk, github_id, activity_type, created_at, username,
			GithubRepo{repo_id, repo_user, repo_project}, meta})
	}

	return
}

// basically need to fill these in, likely moving them all to their own files.
func commit_comment_render(activities []Activity, long_template bool) string { return "" }
func pull_request_comment_render(activities []Activity, long_template bool) string { return "" }
func member_render(activities []Activity, long_template bool) string { return "" }
func download_render(activities []Activity, long_template bool) string { return "" }
func fork_apply_render(activities []Activity, long_template bool) string { return "" }
func team_add_render(activities []Activity, long_template bool) string { return "" }
func gist_render(activities []Activity, long_template bool) string { return "" }
func follow_render(activities []Activity, long_template bool) string { return "" }

func repo_to_template(repo GithubRepo, activities []Activity, render_map map[string]func([]Activity, bool)string) string {
	if len(activities) == 0 {
		return ""
	}
	var activity_map = make(map[string][]Activity)
	for _, activity := range activities {
		// This seems like a lot of juggling. Is there a better way?
		arr := activity_map[activity.activity_type]
		if (arr == nil) {
			arr = make([]Activity, 0)
			activity_map[activity.activity_type] = arr
		}
		arr = append(arr, activity)
		activity_map[activity.activity_type] = arr
	}

	// activity_map: activity_type => []activity
	var response = fmt.Sprintf("%s:\n", repo.FullName())
	for activity_type, activities := range activity_map {
		fn, ok := render_map[activity_type]
		if ok {
			response += fn(activities, true)
		} else {
			log.Print("Not sure how to render activites of type ", activity_type)
		}
	}
	return response
}

func repo_to_string(db *sql.DB, repo GithubRepo, response chan string) {
	activities, err := get_repo_activity(db, &repo)
	if (err != nil) {
		fmt.Println("ERR: ", err)
		os.Exit(1)
	}

	activity_type_to_renderer := map[string]func([]Activity, bool)string {
		"P": push_render,
		"PR": pull_request_render,
		"D": delete_render,
		"C": create_render,
		"W": watch_render,
		"F": fork_render,
		"IC": issue_comment_render,
		"Gl": wiki_render, // Gl is for Gollum, Github's wiki thing.
		"I": issue_render,
		"Pb": public_render,
	}

	response <- repo_to_template(repo, activities, activity_type_to_renderer)
}

func mark_user_repo_sent(db *sql.DB, user User, repos []GithubRepo) (err error) {
	// finds the streamer_userprofile_repo row for the repo / user
	// combo, mark its last_sent as now
	ids := make([]string, 0)
	for _, repo := range repos {
		// TODO: why are they 0?
		if repo.Id != 0 {
			ids = append(ids, strconv.FormatInt(int64(repo.Id), 10))
		}
	}

	// There is likely a better way to get parameterization, but
	// it wasn't working for me with ?'s.
	str := fmt.Sprintf(
		"UPDATE streamer_userprofile_repos " +
		"SET last_sent=NOW() " +
		"WHERE repo_id IN (%s)", strings.Join(ids, ","))
	_, err = db.Exec(str)
	return
}

// TODO: Github Users

// TODO: Need to finish the following activity types:
// - TA (team add)
// - FA (fork apply)
// - DO (Download)
// - RC (Pull Request Review Comment)
// - Fl (Follow)
// - G (Gist)
// - M (Member)
// - CC (Commit Comment)

var user_id = flag.Int("user_id", 0, "ID of user to output.")
var all_users = flag.Bool("all_users", false, "Whether to email all users.")
var send_email = flag.Bool("send_email", false, "Whether to send the email to the user.")
var mark_read = flag.Bool("mark_read", true, "Whether to mark activity sent as read. False means subsequent calls will send the same info.")

func main() {
	// Expects the following environment variables:
	// DB_USER, DB_PASS, DB_DB  (database user, password and database)
	// MAILGUN_API_KEY  (api key from mailgun.org)

	flag.Parse()

	db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@/%s?charset=utf8", os.Getenv("DB_USER"), os.Getenv("DB_PASS"), os.Getenv("DB_DB")))
	if (err != nil) {
		log.Fatalf("Unable to connect to mysql. ", err)
	}
	defer db.Close()

	var users []User
	if (*user_id != 0) {
		users = make([]User, 1)
		user, err := get_user(db, *user_id)
		if err != nil { log.Fatalf("Couldn't return user %d. %s", *user_id, err) }
		users[0] = user
	} else if (*all_users) {
		users, err = get_users(db)
		if err != nil { log.Fatal("Couldn't fetch all users.") }
	} else {
		log.Fatal("You must specify either to mail all users or a specific user id.")
	}

	for _, user := range users {
		repos, err := get_users_repos(db, user.Id)
		if err != nil {
			log.Print("Error fetching user's repos. %s", err)
			continue
		}

		response_chan := make(chan string, len(repos))
		for _, repo := range repos {
			repo_to_string(db, repo, response_chan)
		}

		var response string
		for num_responses := 0; num_responses < len(repos); {
			select {
			case r := <-response_chan:
				num_responses++
				response += r
			}
		}

		if *mark_read {
			fmt.Println("Marking read.")
			err = mark_user_repo_sent(db, user, repos)
			if err != nil {
				log.Print("Error updating repositories as sent.")
				continue
			}
		}

		if len(response) == 0 {
			log.Print("No content to email.")
			continue
		}

		if *send_email {
			mg := mailgun.Open(os.Getenv("MAILGUN_API_KEY"))
			e := &Email{
				from: "justin@gitstreams.mailgun.org",
				to: []string{user.Email},
				subject: "Gitstreams Digest Email",
				text: response,
			}

			id, err := mg.Send(e)

			if err != nil {
				log.Print("Unable to send email. ", err)
			}
			log.Printf("MessageId = %s for uid:%s", id, user.Id)
		} else {
			fmt.Println("Would have sent the following email.")
			fmt.Println(response)
		}
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"log"
	"os"

	WardenClient "github.com/cloudfoundry-incubator/garden/client"
	WardenConnection "github.com/cloudfoundry-incubator/garden/client/connection"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
	"github.com/tedsuo/router"

	"github.com/winston-ci/prole/api"
	"github.com/winston-ci/prole/builder"
	"github.com/winston-ci/prole/config"
	"github.com/winston-ci/prole/resource"
	"github.com/winston-ci/prole/routes"
	"github.com/winston-ci/prole/scheduler"
	"github.com/winston-ci/prole/snapshotter"
)

var listenAddr = flag.String(
	"listenAddr",
	"0.0.0.0:4637",
	"listening address",
)

var peerAddr = flag.String(
	"peerAddr",
	"127.0.0.1:4637",
	"external address of the api server, used for callbacks",
)

var wardenNetwork = flag.String(
	"wardenNetwork",
	"unix",
	"warden API connection network (unix or tcp)",
)

var wardenAddr = flag.String(
	"wardenAddr",
	"/tmp/warden.sock",
	"warden API connection address",
)

var resourceTypes = flag.String(
	"resourceTypes",
	`{"git":"winston/git-resource","raw":"winston/raw-resource"}`,
	"map of resource type to its docker image",
)

var snapshotPath = flag.String(
	"snapshotPath",
	"/tmp/prole.json",
	"path to file to store/load snapshots from",
)

func main() {
	flag.Parse()

	wardenClient := WardenClient.New(&WardenConnection.Info{
		Network: *wardenNetwork,
		Addr:    *wardenAddr,
	})

	resourceTypesMap := map[string]string{}
	err := json.Unmarshal([]byte(*resourceTypes), &resourceTypesMap)
	if err != nil {
		log.Fatalln("failed to parse resource types:", err)
	}

	var resourceTypesConfig config.ResourceTypes
	for typ, image := range resourceTypesMap {
		resourceTypesConfig = append(resourceTypesConfig, config.ResourceType{
			Name:  typ,
			Image: image,
		})
	}

	resourceTracker := resource.NewTracker(resourceTypesConfig, wardenClient)

	builder := builder.NewBuilder(resourceTracker, wardenClient)

	scheduler := scheduler.NewScheduler(builder)

	generator := router.NewRequestGenerator("http://"+*peerAddr, routes.Routes)

	drain := make(chan struct{})

	handler, err := api.New(scheduler, resourceTracker, generator, drain)
	if err != nil {
		log.Fatalln("failed to initialize handler:", err)
	}

	group := grouper.EnvokeGroup(grouper.RunGroup{
		"api":         http_server.New(*listenAddr, handler),
		"snapshotter": snapshotter.NewSnapshotter(*snapshotPath, scheduler),
		"drainer": ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error {
			close(ready)
			<-signals
			close(drain)
			return nil
		}),
	})

	running := ifrit.Envoke(sigmon.New(group))

	log.Println("serving api on", *listenAddr)

	err = <-running.Wait()
	if err != nil {
		log.Println("exited with error:", err)
		os.Exit(1)
	}

	log.Println("exited")
}
<commit_msg>add docker-image resource type to flag default<commit_after>package main

import (
	"encoding/json"
	"flag"
	"log"
	"os"

	WardenClient "github.com/cloudfoundry-incubator/garden/client"
	WardenConnection "github.com/cloudfoundry-incubator/garden/client/connection"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
	"github.com/tedsuo/router"

	"github.com/winston-ci/prole/api"
	"github.com/winston-ci/prole/builder"
	"github.com/winston-ci/prole/config"
	"github.com/winston-ci/prole/resource"
	"github.com/winston-ci/prole/routes"
	"github.com/winston-ci/prole/scheduler"
	"github.com/winston-ci/prole/snapshotter"
)

var listenAddr = flag.String(
	"listenAddr",
	"0.0.0.0:4637",
	"listening address",
)

var peerAddr = flag.String(
	"peerAddr",
	"127.0.0.1:4637",
	"external address of the api server, used for callbacks",
)

var wardenNetwork = flag.String(
	"wardenNetwork",
	"unix",
	"warden API connection network (unix or tcp)",
)

var wardenAddr = flag.String(
	"wardenAddr",
	"/tmp/warden.sock",
	"warden API connection address",
)

var resourceTypes = flag.String(
	"resourceTypes",
	`{"git":"winston/git-resource","raw":"winston/raw-resource","docker-image":"concourse/docker-image-resource"}`,
	"map of resource type to its docker image",
)

var snapshotPath = flag.String(
	"snapshotPath",
	"/tmp/prole.json",
	"path to file to store/load snapshots from",
)

func main() {
	flag.Parse()

	wardenClient := WardenClient.New(&WardenConnection.Info{
		Network: *wardenNetwork,
		Addr:    *wardenAddr,
	})

	resourceTypesMap := map[string]string{}
	err := json.Unmarshal([]byte(*resourceTypes), &resourceTypesMap)
	if err != nil {
		log.Fatalln("failed to parse resource types:", err)
	}

	var resourceTypesConfig config.ResourceTypes
	for typ, image := range resourceTypesMap {
		resourceTypesConfig = append(resourceTypesConfig, config.ResourceType{
			Name:  typ,
			Image: image,
		})
	}

	resourceTracker := resource.NewTracker(resourceTypesConfig, wardenClient)

	builder := builder.NewBuilder(resourceTracker, wardenClient)

	scheduler := scheduler.NewScheduler(builder)

	generator := router.NewRequestGenerator("http://"+*peerAddr, routes.Routes)

	drain := make(chan struct{})

	handler, err := api.New(scheduler, resourceTracker, generator, drain)
	if err != nil {
		log.Fatalln("failed to initialize handler:", err)
	}

	group := grouper.EnvokeGroup(grouper.RunGroup{
		"api":         http_server.New(*listenAddr, handler),
		"snapshotter": snapshotter.NewSnapshotter(*snapshotPath, scheduler),
		"drainer": ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error {
			close(ready)
			<-signals
			close(drain)
			return nil
		}),
	})

	running := ifrit.Envoke(sigmon.New(group))

	log.Println("serving api on", *listenAddr)

	err = <-running.Wait()
	if err != nil {
		log.Println("exited with error:", err)
		os.Exit(1)
	}

	log.Println("exited")
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"strings"
)

var textChan = make(chan string)         // channel to pass log lines for processing
var formattedChan = make(chan string)    // channel to pass formatted text back
var signalChan = make(chan os.Signal, 1) // channel to catch ctrl-c
var stopChan = make(chan struct{})       // channel to kill the process thread

type logger interface {
	GoodWords() []string
	GoodLines() []string
	WarnWords() []string
	BadLines() []string
}

func processLine(log logger) {
	for {
		select {
		case t := <-textChan:
			brokenLine := strings.Split(t, " ")
			for _, s := range brokenLine {
				switch {
				case WordExists(s, log.Match.GoodWords):
					formattedChan <- strings.Join(brokenLine, " ")
				case WordExists(s, log.Match.GoodLines):
					formattedChan <- strings.Join(brokenLine, " ")
				case WordExists(s, log.Match.WarnWords):
					formattedChan <- strings.Join(brokenLine, " ")
				case WordExists(s, log.Match.BadWords):
					formattedChan <- strings.Join(brokenLine, " ")
				default:
					formattedChan <- s
				}
			}
		case <-stopChan:
			return
		}
	}
}

// Pointers to hold the contents of the flag args.
var (
	templateFlag = flag.String("t", "", "template to use for log parsing")
)

const USAGE = `Usage: logcolor -t template [-h]`

func main() {
	flag.Parse()

	signal.Notify(signalChan, os.Interrupt)
	// setup go routine to catch a ctrl-c
	go func() {
		for range signalChan {
			stopChan <- struct{}{} // clean up
			os.Exit(1)
		}
	}()

	/*
		if len(*templateFlag) != 1 {
			fmt.Println(USAGE)
			os.Exit(1)
		}
	*/

	var log interface{}
	switch *templateFlag {
	case "http":
		log := logger(&HTTP{})
	case "ftp":
		//f := &FTP{}
		/*
			case "sip":
				s := &SIP{}
			case "mysql":
				m := &MySQL{}
			case "rsync":
				r := &Rsync{}
			case "postgresql":
				p := &Postgresql{}
			case "openstack":
				o := &Openstack{}
		*/
	}

	go processLine(log)

	go func() {
		for {
			reader := bufio.NewReader(os.Stdin)
			text, err := reader.ReadString('\n')
			if err != nil {
				fmt.Println("unable to read line")
				os.Exit(1)
			}
			textChan <- text
		}
	}()

	for {
		select {
		case f := <-formattedChan:
			fmt.Println(f)
		default:
			continue
		}
	}
	os.Exit(0)
}
<commit_msg>fixed it til it was broke...<commit_after>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"strings"
)

var textChan = make(chan string)         // channel to pass log lines for processing
var formattedChan = make(chan string)    // channel to pass formatted text back
var signalChan = make(chan os.Signal, 1) // channel to catch ctrl-c
var stopChan = make(chan struct{})       // channel to kill the process thread

type logger interface {
	GoodWords() []string
	GoodLines() []string
	WarnWords() []string
	BadLines() []string
}

func processLine(log logger) {
	for {
		select {
		case t := <-textChan:
			brokenLine := strings.Split(t, " ")
			for _, s := range brokenLine {
				switch {
				case WordExists(s, log.GoodWords()):
					formattedChan <- strings.Join(brokenLine, " ")
				case WordExists(s, log.GoodLines()):
					formattedChan <- strings.Join(brokenLine, " ")
				case WordExists(s, log.WarnWords()):
					formattedChan <- strings.Join(brokenLine, " ")
				case WordExists(s, log.BadLines()):
					formattedChan <- strings.Join(brokenLine, " ")
				default:
					formattedChan <- s
				}
			}
		case <-stopChan:
			return
		}
	}
}

// Pointers to hold the contents of the flag args.
var (
	templateFlag = flag.String("t", "", "template to use for log parsing")
)

const USAGE = `Usage: logcolor -t template [-h]`

func main() {
	flag.Parse()

	signal.Notify(signalChan, os.Interrupt)
	// setup go routine to catch a ctrl-c
	go func() {
		for range signalChan {
			stopChan <- struct{}{} // clean up
			os.Exit(1)
		}
	}()

	/*
		if len(*templateFlag) != 1 {
			fmt.Println(USAGE)
			os.Exit(1)
		}
	*/

	switch *templateFlag {
	case "http":
		go processLine(logger(&HTTP{}))
	case "ftp":
		//f := &FTP{}
		/*
			case "sip":
				s := &SIP{}
			case "mysql":
				m := &MySQL{}
			case "rsync":
				r := &Rsync{}
			case "postgresql":
				p := &Postgresql{}
			case "openstack":
				o := &Openstack{}
		*/
	}

	go func() {
		for {
			reader := bufio.NewReader(os.Stdin)
			text, err := reader.ReadString('\n')
			if err != nil {
				fmt.Println("unable to read line")
				os.Exit(1)
			}
			textChan <- text
		}
	}()

	for {
		select {
		case f := <-formattedChan:
			fmt.Println(f)
		default:
			continue
		}
	}
	os.Exit(0)
}
<|endoftext|>