<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"net/http"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	gocache "github.com/patrickmn/go-cache"
	"github.com/yhat/scrape"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

const (
	baseUrl = "http://www.auboutdufil.com"
)

var (
	cache = gocache.New(1*time.Hour, 1*time.Minute)
)

type audio struct {
	Title       string    `json:"title"`
	Artist      string    `json:"artist"`
	TrackURL    string    `json:"track_url"`
	Genres      []string  `json:"genres"`
	CoverArtURL string    `json:"cover_art_url"`
	DownloadURL string    `json:"download_url"`
	License     string    `json:"license"`
	Downloads   int       `json:"downloads"`
	Plays       int       `json:"play_count"`
	Rating      float32   `json:"rating"`
	Date        time.Time `json:"published_date"`
}

func parseInfos(parentNode *html.Node, track audio) (error, audio) {

	infosParentDiv := scrape.FindAllNested(parentNode, scrape.ByClass("pure-u-2-3"))
	if len(infosParentDiv) == 0 {
		log.Warn("Incorrect html data, layout may have changed")
		return errors.New("Malformed html"), track
	}

	notPure23Matcher := func(n *html.Node) bool {
		return n.DataAtom == atom.Div && scrape.Attr(n, "class") != "pure-u-2-3"
	}

	divs := scrape.FindAll(infosParentDiv[0], notPure23Matcher)
	if len(divs) != 10 {
		log.WithFields(log.Fields{
			"divsNumber": len(divs),
			"expected":   "10",
		}).Warn("Incorrect html data, layout may have changed")
		return errors.New("Malformed html"), track
	}

	// Parse title infos
	titleTag, ok := scrape.Find(divs[3], scrape.ByTag(atom.B))
	if !ok {
		log.Warn("Incorrect html data while searching for title, layout may have changed")
		return errors.New("Malformed html"), track
	}
	track.Title = scrape.Text(titleTag)

	// Parse artist name and url
	artistTagParent, ok := scrape.Find(divs[4], scrape.ByTag(atom.Strong))
	if !ok {
		log.Warn("Incorrect html data while searching for artist, layout may have changed")
		return errors.New("Malformed html"), track
	}
	artistTag, ok := scrape.Find(artistTagParent, scrape.ByTag(atom.A))
	if !ok {
		log.Warn("Incorrect html data while searching for artist, layout may have changed")
		return errors.New("Malformed html"), track
	}
	track.Artist = scrape.Text(artistTag)
	track.TrackURL = scrape.Attr(artistTag, "href")

	// Parse genres
	genreTags := scrape.FindAll(divs[6], scrape.ByTag(atom.Span))
	for _, genreTag := range genreTags {
		track.Genres = append(track.Genres, scrape.Text(genreTag))
	}

	return nil, track
}

func parseAudioData(node *html.Node) (err error, track audio) {

	err, track = parseInfos(node, track)
	if err != nil {
		return err, track
	}

	// look for cover image
	coverParentDiv := scrape.FindAllNested(node, scrape.ByClass("pure-u-1-3"))
	if len(coverParentDiv) == 0 {
		log.Warn("Incorrect html data while searching for cover url, layout may have changed")
		return errors.New("Malformed html"), track
	}

	notPure13Matcher := func(n *html.Node) bool {
		return n.DataAtom == atom.Div && scrape.Attr(n, "class") != "pure-u-1-3"
	}

	divs := scrape.FindAll(coverParentDiv[0], notPure13Matcher)
	if len(divs) != 6 {
		log.WithFields(log.Fields{
			"divsNumber": len(divs),
			"expected":   "6",
		}).Warn("Incorrect html data while searching for cover url, layout may have changed")
		return errors.New("Malformed html"), track
	}

	coverTag := scrape.FindAllNested(divs[5], scrape.ByTag(atom.Img))
	if len(coverTag) != 1 {
		log.Warn("Incorrect html data while searching for cover url, layout may have changed")
		return errors.New("Malformed html"), track
	}
	track.CoverArtURL = scrape.Attr(coverTag[0], "src")

	// download url
	mp3PlayerDiv, ok := scrape.Find(node.Parent, scrape.ByClass("mp3player"))
	if !ok {
		log.Warn("Incorrect html data while searching for download url, layout may have changed")
		return errors.New("Malformed html"), track
	}
	downloadUrlParent := scrape.FindAllNested(mp3PlayerDiv, scrape.ByClass("sm2-playlist-bd"))
	if len(downloadUrlParent) != 1 {
		log.Warn("Incorrect html data while searching for download url, layout may have changed")
		return errors.New("Malformed html"), track
	}
	downloadUrlTag := scrape.FindAllNested(downloadUrlParent[0], scrape.ByTag(atom.A))
	if len(downloadUrlTag) != 1 {
		log.Warn("Incorrect html data while searching for download url, layout may have changed")
		return errors.New("Malformed html"), track
	}
	track.DownloadURL = strings.Replace(scrape.Attr(downloadUrlTag[0], "href"), "http://www.auboutdufil.com/get.php?fla=", "", 1)

	// additional infos
	additionalInfosParent, ok := scrape.Find(node.Parent, scrape.ByClass("legenddata"))
	if !ok {
		log.Warn("Incorrect html data, layout may have changed")
		return errors.New("Malformed html"), track
	}

	additionalInfosSpans := scrape.FindAll(additionalInfosParent, scrape.ByTag(atom.Span))
	if len(additionalInfosSpans) != 5 {
		log.Warn("Incorrect html data while searching for additional infos, layout may have changed")
		return errors.New("Malformed html"), track
	}

	licenseTag, ok := scrape.Find(additionalInfosSpans[4], scrape.ByTag(atom.A))
	if !ok {
		log.Warn("Incorrect html data while searching for license infos, layout may have changed")
		return errors.New("Malformed html"), track
	}
	track.License = strings.Split(scrape.Attr(licenseTag, "href"), "license=")[1]

	return nil, track
}

func scrapePage(url string) (tracks []audio) {
	resp, err := http.Get(url)

	if err != nil {
		log.WithFields(log.Fields{
			"url": url,
			"err": err,
		}).Error("Failed to get page")
		return
	}

	body := resp.Body
	defer body.Close()

	root, err := html.Parse(resp.Body)
	if err != nil {
		log.WithFields(log.Fields{
			"err": err,
			"url": url,
		}).Error("Unable to parse this web page")
		return
	}

	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Div && n.Parent != nil {
			return strings.Contains(scrape.Attr(n, "class"), "audio-wrapper")
		}
		return false
	}

	audioWrappers := scrape.FindAllNested(root, matcher)
	for _, wrapper := range audioWrappers {
		err, track := parseAudioData(wrapper)
		if err != nil {
			continue
		}

		tracks = append(tracks, track)
	}

	return tracks
}

func HandleLatest(w http.ResponseWriter, r *http.Request) {
	tracks, found := cache.Get("latest")
	if !found {
		log.Info("Cache expired, scraping data...")
		scrapeTracks := scrapePage(baseUrl)
		scrapeTracks = append(scrapeTracks, scrapePage(baseUrl+"/index.php?page=2")...)
		scrapeTracks = append(scrapeTracks, scrapePage(baseUrl+"/index.php?page=3")...)
		cache.Set("latest", scrapeTracks, 0)
		tracks = scrapeTracks
	}

	body, err := json.Marshal(tracks)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	w.Write(body)
}

func server(port string) {
	server := http.NewServeMux()
	server.HandleFunc("/latest", HandleLatest)

	log.WithFields(log.Fields{
		"port": port,
	}).Info("Starting HTTP Server")

	http.ListenAndServe(":"+port, server)

}

func main() {
	var (
		port = flag.String("p", "14000", "Port used for server")
	)
	flag.Parse()

	server(*port)
}
<commit_msg>Resolve redirect for download url<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"net/http"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	gocache "github.com/patrickmn/go-cache"
	"github.com/yhat/scrape"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

const (
	baseUrl = "http://www.auboutdufil.com"
)

var (
	cache = gocache.New(1*time.Hour, 1*time.Minute)
)

type audio struct {
	Title       string    `json:"title"`
	Artist      string    `json:"artist"`
	TrackURL    string    `json:"track_url"`
	Genres      []string  `json:"genres"`
	CoverArtURL string    `json:"cover_art_url"`
	DownloadURL string    `json:"download_url"`
	License     string    `json:"license"`
	Downloads   int       `json:"downloads"`
	Plays       int       `json:"play_count"`
	Rating      float32   `json:"rating"`
	Date        time.Time `json:"published_date"`
}

func parseInfos(parentNode *html.Node, track audio) (error, audio) {

	infosParentDiv := scrape.FindAllNested(parentNode, scrape.ByClass("pure-u-2-3"))
	if len(infosParentDiv) == 0 {
		log.Warn("Incorrect html data, layout may have changed")
		return errors.New("Malformed html"), track
	}

	notPure23Matcher := func(n *html.Node) bool {
		return n.DataAtom == atom.Div && scrape.Attr(n, "class") != "pure-u-2-3"
	}

	divs := scrape.FindAll(infosParentDiv[0], notPure23Matcher)
	if len(divs) != 10 {
		log.WithFields(log.Fields{
			"divsNumber": len(divs),
			"expected":   "10",
		}).Warn("Incorrect html data, layout may have changed")
		return errors.New("Malformed html"), track
	}

	// Parse title infos
	titleTag, ok := scrape.Find(divs[3], scrape.ByTag(atom.B))
	if !ok {
		log.Warn("Incorrect html data while searching for title, layout may have changed")
		return errors.New("Malformed html"), track
	}
	track.Title = scrape.Text(titleTag)

	// Parse artist name and url
	artistTagParent, ok := scrape.Find(divs[4], scrape.ByTag(atom.Strong))
	if !ok {
		log.Warn("Incorrect html data while searching for artist, layout may have changed")
		return errors.New("Malformed html"), track
	}
	artistTag, ok := scrape.Find(artistTagParent, scrape.ByTag(atom.A))
	if !ok {
		log.Warn("Incorrect html data while searching for artist, layout may have changed")
		return errors.New("Malformed html"), track
	}
	track.Artist = scrape.Text(artistTag)
	track.TrackURL = scrape.Attr(artistTag, "href")

	// Parse genres
	genreTags := scrape.FindAll(divs[6], scrape.ByTag(atom.Span))
	for _, genreTag := range genreTags {
		track.Genres = append(track.Genres, scrape.Text(genreTag))
	}

	return nil, track
}

func parseAudioData(node *html.Node) (err error, track audio) {

	err, track = parseInfos(node, track)
	if err != nil {
		return err, track
	}

	// look for cover image
	coverParentDiv := scrape.FindAllNested(node, scrape.ByClass("pure-u-1-3"))
	if len(coverParentDiv) == 0 {
		log.Warn("Incorrect html data while searching for cover url, layout may have changed")
		return errors.New("Malformed html"), track
	}

	notPure13Matcher := func(n *html.Node) bool {
		return n.DataAtom == atom.Div && scrape.Attr(n, "class") != "pure-u-1-3"
	}

	divs := scrape.FindAll(coverParentDiv[0], notPure13Matcher)
	if len(divs) != 6 {
		log.WithFields(log.Fields{
			"divsNumber": len(divs),
			"expected":   "6",
		}).Warn("Incorrect html data while searching for cover url, layout may have changed")
		return errors.New("Malformed html"), track
	}

	coverTag := scrape.FindAllNested(divs[5], scrape.ByTag(atom.Img))
	if len(coverTag) != 1 {
		log.Warn("Incorrect html data while searching for cover url, layout may have changed")
		return errors.New("Malformed html"), track
	}
	track.CoverArtURL = scrape.Attr(coverTag[0], "src")

	// download url
	mp3PlayerDiv, ok := scrape.Find(node.Parent, scrape.ByClass("mp3player"))
	if !ok {
		log.Warn("Incorrect html data while searching for download url, layout may have changed")
		return errors.New("Malformed html"), track
	}
	downloadUrlParent := scrape.FindAllNested(mp3PlayerDiv, scrape.ByClass("sm2-playlist-bd"))
	if len(downloadUrlParent) != 1 {
		log.Warn("Incorrect html data while searching for download url, layout may have changed")
		return errors.New("Malformed html"), track
	}
	downloadUrlTag := scrape.FindAllNested(downloadUrlParent[0], scrape.ByTag(atom.A))
	if len(downloadUrlTag) != 1 {
		log.Warn("Incorrect html data while searching for download url, layout may have changed")
		return errors.New("Malformed html"), track
	}
	tmpUrl := scrape.Attr(downloadUrlTag[0], "href")
	resp, err := http.Head(tmpUrl)
	if err != nil {
		log.Warn("Unable to query download url")
		track.DownloadURL = tmpUrl
	} else {
		track.DownloadURL = resp.Request.URL.String()
	}

	// additional infos
	additionalInfosParent, ok := scrape.Find(node.Parent, scrape.ByClass("legenddata"))
	if !ok {
		log.Warn("Incorrect html data, layout may have changed")
		return errors.New("Malformed html"), track
	}

	additionalInfosSpans := scrape.FindAll(additionalInfosParent, scrape.ByTag(atom.Span))
	if len(additionalInfosSpans) != 5 {
		log.Warn("Incorrect html data while searching for additional infos, layout may have changed")
		return errors.New("Malformed html"), track
	}

	licenseTag, ok := scrape.Find(additionalInfosSpans[4], scrape.ByTag(atom.A))
	if !ok {
		log.Warn("Incorrect html data while searching for license infos, layout may have changed")
		return errors.New("Malformed html"), track
	}
	track.License = strings.Split(scrape.Attr(licenseTag, "href"), "license=")[1]

	return nil, track
}

func scrapePage(url string) (tracks []audio) {
	resp, err := http.Get(url)

	if err != nil {
		log.WithFields(log.Fields{
			"url": url,
			"err": err,
		}).Error("Failed to get page")
		return
	}

	body := resp.Body
	defer body.Close()

	root, err := html.Parse(resp.Body)
	if err != nil {
		log.WithFields(log.Fields{
			"err": err,
			"url": url,
		}).Error("Unable to parse this web page")
		return
	}

	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Div && n.Parent != nil {
			return strings.Contains(scrape.Attr(n, "class"), "audio-wrapper")
		}
		return false
	}

	audioWrappers := scrape.FindAllNested(root, matcher)
	for _, wrapper := range audioWrappers {
		err, track := parseAudioData(wrapper)
		if err != nil {
			continue
		}

		tracks = append(tracks, track)
	}

	return tracks
}

func HandleLatest(w http.ResponseWriter, r *http.Request) {
	tracks, found := cache.Get("latest")
	if !found {
		log.Info("Cache expired, scraping data...")
		scrapeTracks := scrapePage(baseUrl)
		scrapeTracks = append(scrapeTracks, scrapePage(baseUrl+"/index.php?page=2")...)
		scrapeTracks = append(scrapeTracks, scrapePage(baseUrl+"/index.php?page=3")...)
		cache.Set("latest", scrapeTracks, 0)
		tracks = scrapeTracks
	}

	body, err := json.Marshal(tracks)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	w.Write(body)
}

func server(port string) {
	server := http.NewServeMux()
	server.HandleFunc("/latest", HandleLatest)

	log.WithFields(log.Fields{
		"port": port,
	}).Info("Starting HTTP Server")

	http.ListenAndServe(":"+port, server)

}

func main() {
	var (
		port = flag.String("p", "14000", "Port used for server")
	)
	flag.Parse()

	server(*port)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"math/rand"
	"net"
	"os"
	"reflect"
)

var wordList []string

// TODO: A bug exists in that we're only doing wildcard detection on the root domain
// If a subdomain contains a wildcard, it will not be detected during recursive scanning
var wildcard []string
var wildcardDetected bool

func main() {
	// Parse cmdline
	flag_domain := flag.String("domain", "", "The target domain")
	flag_wordlist := flag.String("wordlist", "wordlist.txt", "Path to the wordlist")
	flag_threads := flag.Int("threads", 20, "Number of concurrent threads")

	flag.Parse()

	if *flag_domain == "" {
		fmt.Println("You must specify a domain")
		flag.PrintDefaults()
		os.Exit(1)
	}
	file, err := os.Open(*flag_wordlist)
	if err != nil {
		panic(err)
	}

	// Check for wildcard record(s)
	randomString := randomString(10)
	wildcard, _ = net.LookupHost(randomString + "." + *flag_domain)
	if len(wildcard) > 0 {
		fmt.Println("Detected wildcard record")
		wildcardDetected = true
	}

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		wordList = append(wordList, scanner.Text())
	}

	queue := make(chan string, *flag_threads)
	done := resolveList(queue, *flag_domain)
	<-done
}

func resolveList(queue chan string, apex string) chan bool {
	doneChan := make(chan bool)
	go func() {
		for i := range wordList {
			domainName := fmt.Sprintf("%s.%s", wordList[i], apex)

			// wait for free worker thread
			queue <- domainName
			go func() {
				ips, err := net.LookupHost(domainName)
				// we have looked up the host, so we can remove this item from the queue
				// so that another go routine can give it a go
				<-queue

				// did not resolve
				if err != nil {
					// TODO: find a nicer way of writing this
					errstr := err.Error()
					nsh := "no such host"
					if errstr[len(errstr)-len(nsh):] != nsh {
						fmt.Printf("Unexpected error: %v\n", err)
					}
					return
				}

				// Check if it's a wildcard
				if wildcardDetected && reflect.DeepEqual(ips, wildcard) {
					// Not a real finding -- see note about the bug at wildcard definition
					return
				}

				// we found a non-wildcard sub domain, recurse
				fmt.Printf("%s %v\n", domainName, ips)
				childDone := resolveList(queue, domainName)
				// wait for child to finish
				<-childDone
			}()
		}
		doneChan <- true
	}()
	return doneChan
}

func randomString(length int) string {
	letterRunes := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
	b := make([]rune, length)
	for i := range b {
		b[i] = letterRunes[rand.Intn(len(letterRunes))]
	}
	return string(b)
}
<commit_msg>Support recursive wildcard detection<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"math/rand"
	"net"
	"os"
	"reflect"
	"sync"
)

var wordList []string

// This is a cache of the mapping of domains to their wildcard addresses, e.g.
// *.example.com is stored as ["example.com"] = [127.0.0.1]
var wildcardRegistry map[string][]string
var wildcardRegistryMutex sync.RWMutex

func main() {
	// Parse cmdline
	flag_domain := flag.String("domain", "", "The target domain")
	flag_wordlist := flag.String("wordlist", "wordlist.txt", "Path to the wordlist")
	flag_threads := flag.Int("threads", 20, "Number of concurrent threads")

	flag.Parse()

	if *flag_domain == "" {
		fmt.Println("You must specify a domain")
		flag.PrintDefaults()
		os.Exit(1)
	}
	file, err := os.Open(*flag_wordlist)
	if err != nil {
		panic(err)
	}

	// Make our new wildcard map
	wildcardRegistry = make(map[string][]string)
	// Check for wildcard record(s) before starting
	wildcardDetected := checkWildcard(*flag_domain)

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		wordList = append(wordList, scanner.Text())
	}

	queue := make(chan string, *flag_threads)
	done := resolveList(queue, *flag_domain, wildcardDetected)
	<-done
}

func resolveList(queue chan string, apex string, wildcardDetected bool) chan bool {
	doneChan := make(chan bool)
	go func() {
		for i := range wordList {
			domainName := fmt.Sprintf("%s.%s", wordList[i], apex)

			// wait for free worker thread
			queue <- domainName
			go func() {
				ips, err := net.LookupHost(domainName)
				// we have looked up the host, so we can remove this item from the queue
				// so that another go routine can give it a go
				<-queue

				// did not resolve
				if err != nil {
					// TODO: find a nicer way of writing this
					errstr := err.Error()
					nsh := "no such host"
					if errstr[len(errstr)-len(nsh):] != nsh {
						fmt.Printf("Unexpected error: %v\n", err)
					}
					return
				}

				// Check if it's a wildcard
				if wildcardDetected {
					// Read lock the mutex
					wildcardRegistryMutex.RLock()
					if reflect.DeepEqual(ips, wildcardRegistry[apex]) {
						// Not a real finding -- see note about the bug at wildcard definition
						wildcardRegistryMutex.RUnlock()
						return
					}
					wildcardRegistryMutex.RUnlock()
				}

				// we found a non-wildcard sub domain, recurse
				fmt.Printf("%s %v\n", domainName, ips)
				childDone := resolveList(queue, domainName, checkWildcard(domainName))
				// wait for child to finish
				<-childDone
			}()
		}
		doneChan <- true
	}()
	return doneChan
}

func checkWildcard(domain string) bool {
	// Check for wildcard record(s)
	randomString := randomString(10)
	wildcard, _ := net.LookupHost(randomString + "." + domain)
	if len(wildcard) > 0 {
		fmt.Printf("Detected wildcard record: %s\r\n", domain)
		// Lock for writing
		wildcardRegistryMutex.Lock()
		wildcardRegistry[domain] = wildcard
		wildcardRegistryMutex.Unlock()
		return true
	}
	return false
}

func randomString(length int) string {
	letterRunes := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
	b := make([]rune, length)
	for i := range b {
		b[i] = letterRunes[rand.Intn(len(letterRunes))]
	}
	return string(b)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"path/filepath"
	"runtime"
	"time"

	"github.com/cuigh/auxo/app"
	"github.com/cuigh/auxo/app/flag"
	_ "github.com/cuigh/auxo/cache/memory"
	"github.com/cuigh/auxo/config"
	"github.com/cuigh/auxo/data/valid"
	"github.com/cuigh/auxo/net/web"
	"github.com/cuigh/auxo/net/web/filter"
	"github.com/cuigh/auxo/net/web/filter/auth"
	"github.com/cuigh/auxo/net/web/renderer/jet"
	"github.com/cuigh/swirl/biz"
	"github.com/cuigh/swirl/controller"
	"github.com/cuigh/swirl/misc"
)

func main() {
	misc.BindOptions()

	app.Name = "Swirl"
	app.Version = "0.6.5"
	app.Desc = "A web management UI for Docker, focused on swarm cluster"
	app.Action = func(ctx *app.Context) {
		misc.LoadOptions()
		app.Run(server())
	}
	app.Flags.Register(flag.All)
	app.Start()
}

func server() *web.Server {
	setting, err := biz.Setting.Get()
	if err != nil {
		panic(fmt.Sprintf("Load setting failed: %v", err))
	}

	ws := web.Auto()

	// customize error handler
	ws.ErrorHandler.OnCode(http.StatusNotFound, func(ctx web.Context, err error) {
		if ctx.IsAJAX() {
			ctx.Status(http.StatusNotFound).HTML(http.StatusText(http.StatusNotFound)) // nolint: gas
		} else {
			ctx.Status(http.StatusNotFound).Render("404", nil) // nolint: gas
		}
	})
	ws.ErrorHandler.OnCode(http.StatusForbidden, func(ctx web.Context, err error) {
		if ctx.IsAJAX() {
			ctx.Status(http.StatusForbidden).HTML("You do not have permission to perform this operation") // nolint: gas
		} else {
			ctx.Status(http.StatusForbidden).Render("403", nil) // nolint: gas
		}
	})

	// set render
	ws.Validator = &valid.Validator{Tag: "valid"}
	ws.Renderer = jet.New().SetDebug(config.GetBool("debug")).
		AddFunc("time", misc.FormatTime(setting.TimeZone.Offset)).
		AddFunc("i18n", misc.Message(setting.Language)).
		AddFuncs(misc.Funcs).
		AddVariable("language", setting.Language).
		AddVariable("version", app.Version).
		AddVariable("go_version", runtime.Version())

	// register global filters
	ws.Use(filter.NewRecover())

	// register static handlers
	ws.Static("/assets", filepath.Join(filepath.Dir(app.Path()), "assets"))

	// create biz group
	form := &auth.Form{
		Identifier:        biz.User.Identify,
		Timeout:           time.Minute * 30,
		SlidingExpiration: true,
	}
	g := ws.Group("", form, filter.NewAuthorizer(biz.User.Authorize))

	// register auth handlers
	g.Post("/login", form.LoginJSON(biz.User.Login)).SetAuthorize(web.AuthAnonymous)
	g.Get("/logout", form.Logout).SetAuthorize(web.AuthAuthenticated)

	// register controllers
	g.Handle("", controller.Home())
	g.Handle("/profile", controller.Profile())
	g.Handle("/registry", controller.Registry())
	g.Handle("/node", controller.Node())
	g.Handle("/service", controller.Service(), biz.Perm)
	g.Handle("/service/template", controller.Template())
	g.Handle("/stack", controller.Stack())
	g.Handle("/network", controller.Network())
	g.Handle("/secret", controller.Secret())
	g.Handle("/config", controller.Config())
	g.Handle("/task", controller.Task())
	g.Handle("/container", controller.Container())
	g.Handle("/image", controller.Image())
	g.Handle("/volume", controller.Volume())
	g.Handle("/system/user", controller.User())
	g.Handle("/system/role", controller.Role())
	g.Handle("/system/setting", controller.Setting())
	g.Handle("/system/event", controller.Event())

	return ws
}
<commit_msg>Add name to login & logout handler<commit_after>package main

import (
	"fmt"
	"net/http"
	"path/filepath"
	"runtime"
	"time"

	"github.com/cuigh/auxo/app"
	"github.com/cuigh/auxo/app/flag"
	_ "github.com/cuigh/auxo/cache/memory"
	"github.com/cuigh/auxo/config"
	"github.com/cuigh/auxo/data/valid"
	"github.com/cuigh/auxo/net/web"
	"github.com/cuigh/auxo/net/web/filter"
	"github.com/cuigh/auxo/net/web/filter/auth"
	"github.com/cuigh/auxo/net/web/renderer/jet"
	"github.com/cuigh/swirl/biz"
	"github.com/cuigh/swirl/controller"
	"github.com/cuigh/swirl/misc"
)

func main() {
	misc.BindOptions()

	app.Name = "Swirl"
	app.Version = "0.6.5"
	app.Desc = "A web management UI for Docker, focused on swarm cluster"
	app.Action = func(ctx *app.Context) {
		misc.LoadOptions()
		app.Run(server())
	}
	app.Flags.Register(flag.All)
	app.Start()
}

func server() *web.Server {
	setting, err := biz.Setting.Get()
	if err != nil {
		panic(fmt.Sprintf("Load setting failed: %v", err))
	}

	ws := web.Auto()

	// customize error handler
	ws.ErrorHandler.OnCode(http.StatusNotFound, func(ctx web.Context, err error) {
		if ctx.IsAJAX() {
			ctx.Status(http.StatusNotFound).HTML(http.StatusText(http.StatusNotFound)) // nolint: gas
		} else {
			ctx.Status(http.StatusNotFound).Render("404", nil) // nolint: gas
		}
	})
	ws.ErrorHandler.OnCode(http.StatusForbidden, func(ctx web.Context, err error) {
		if ctx.IsAJAX() {
			ctx.Status(http.StatusForbidden).HTML("You do not have permission to perform this operation") // nolint: gas
		} else {
			ctx.Status(http.StatusForbidden).Render("403", nil) // nolint: gas
		}
	})

	// set render
	ws.Validator = &valid.Validator{Tag: "valid"}
	ws.Renderer = jet.New().SetDebug(config.GetBool("debug")).
		AddFunc("time", misc.FormatTime(setting.TimeZone.Offset)).
		AddFunc("i18n", misc.Message(setting.Language)).
		AddFuncs(misc.Funcs).
		AddVariable("language", setting.Language).
		AddVariable("version", app.Version).
		AddVariable("go_version", runtime.Version())

	// register global filters
	ws.Use(filter.NewRecover())

	// register static handlers
	ws.Static("/assets", filepath.Join(filepath.Dir(app.Path()), "assets"))

	// create biz group
	form := &auth.Form{
		Identifier:        biz.User.Identify,
		Timeout:           time.Minute * 30,
		SlidingExpiration: true,
	}
	g := ws.Group("", form, filter.NewAuthorizer(biz.User.Authorize))

	// register auth handlers
	g.Post("/login", form.LoginJSON(biz.User.Login), web.WithName("login"), web.WithAuthorize(web.AuthAnonymous))
	g.Get("/logout", form.Logout, web.WithName("logout"), web.WithAuthorize(web.AuthAuthenticated))

	// register controllers
	g.Handle("", controller.Home())
	g.Handle("/profile", controller.Profile())
	g.Handle("/registry", controller.Registry())
	g.Handle("/node", controller.Node())
	g.Handle("/service", controller.Service(), biz.Perm)
	g.Handle("/service/template", controller.Template())
	g.Handle("/stack", controller.Stack())
	g.Handle("/network", controller.Network())
	g.Handle("/secret", controller.Secret())
	g.Handle("/config", controller.Config())
	g.Handle("/task", controller.Task())
	g.Handle("/container", controller.Container())
	g.Handle("/image", controller.Image())
	g.Handle("/volume", controller.Volume())
	g.Handle("/system/user", controller.User())
	g.Handle("/system/role", controller.Role())
	g.Handle("/system/setting", controller.Setting())
	g.Handle("/system/event", controller.Event())

	return ws
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/codegangsta/cli"
)

var (
	useUDP bool
	useTLS bool
	doTee  bool

	syslog         bool
	syslogHostname string
	syslogApp      string
)

func main() {
	app := cli.NewApp()
	app.Name = "forward"
	app.Usage = "Transport StdIn lines to a remote destination over UDP, TCP, or TCP+TLS"
	app.UsageText = "forward [global options] [syslog [syslog options]] address:port"
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:        "udp, u",
			Usage:       "Send via UDP (will ignore TLS)",
			Destination: &useUDP,
		},
		cli.BoolTFlag{
			Name:        "tls, s",
			Usage:       "TLS-secured TCP connection",
			Destination: &useTLS,
		},
		cli.BoolTFlag{
			Name:        "tee, t",
			Usage:       "Tee stdin to stdout",
			Destination: &doTee,
		},
	}
	app.Action = func(c *cli.Context) {
		forward(c.Args().First())
	}

	h, _ := os.Hostname()
	app.Commands = []cli.Command{
		{
			Name:      "syslog",
			Aliases:   []string{"log"},
			Usage:     "Wrap lines in RFC-5424 Syslog format",
			ArgsUsage: "address:port",
			Action: func(c *cli.Context) {
				syslog = true
				forward(c.Args().First())
			},
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:        "hostname, n",
					Value:       h,
					Destination: &syslogHostname,
				},
				cli.StringFlag{
					Name:        "app, a",
					Value:       "app",
					Destination: &syslogApp,
				},
			},
		},
	}

	app.Run(os.Args)
}

func forward(destination string) {
	if !validDestination(destination) {
		return
	}

	var conn net.Conn
	var err error
	switch {
	case useUDP:
		conn, err = net.Dial("udp", destination)
	case !useTLS:
		conn, err = net.Dial("tcp", destination)
	default:
		conn, err = tls.Dial("tcp", destination, &tls.Config{})
	}
	connected := false
	if err != nil {
		fmt.Fprintf(os.Stderr, "Could not connect to %s: %v\n", destination, err)
	} else {
		connected = true
		defer conn.Close()
	}

	writers := make([]*io.PipeWriter, 0, 2)
	wg := sync.WaitGroup{}
	if doTee {
		stdOutReader, stdOutWriter := io.Pipe()
		writers = append(writers, stdOutWriter)
		go func() {
			io.Copy(os.Stdout, stdOutReader)
		}()
	}
	if connected {
		netReader, netWriter := io.Pipe()
		writers = append(writers, netWriter)
		go func() {
			wg.Add(1)
			reader := bufio.NewReader(netReader)
			byteBuffer := bytes.NewBuffer([]byte{})
			for {
				data, err := reader.ReadBytes('\n')
				if err != nil {
					break
				}

				if syslog {
					data = toSyslog(byteBuffer, data)
				}

				if _, err = conn.Write(data); err != nil {
					break
				}
				byteBuffer.Reset()
			}
			io.Copy(ioutil.Discard, netReader)
			wg.Done()
		}()
	}

	if len(writers) == 0 {
		io.Copy(ioutil.Discard, os.Stdin)
		return
	}

	writeOnly := make([]io.Writer, 0, len(writers))
	for _, w := range writers {
		writeOnly = append(writeOnly, w)
	}
	mw := io.MultiWriter(writeOnly...)
	io.Copy(mw, os.Stdin)
	for _, w := range writers {
		w.Close()
	}

	wg.Wait()
}

func toSyslog(b *bytes.Buffer, line []byte) []byte {
	//<22>1 2016-06-18T09:56:21Z sendername programname - - - the log message
	b.WriteString("<22>1 ")
	b.WriteString(time.Now().UTC().Format(time.RFC3339) + " ")
	b.WriteString(syslogHostname + " ")
	b.WriteString(syslogApp + " ")
	b.WriteString("- - - ")
	b.Write(line)

	return b.Bytes()
}

// Basic validation test
func validDestination(d string) bool {
	if len(d) == 0 {
		return false
	}

	s := strings.Split(d, ":")
	if len(s) != 2 || len(s[0]) == 0 || len(s[1]) == 0 {
		return false
	}

	if _, err := strconv.Atoi(s[1]); err != nil {
		return false
	}

	return true
}
<commit_msg>Modifiable syslog priority<commit_after>package main

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/codegangsta/cli"
)

var (
	useUDP bool
	useTLS bool
	doTee  bool

	syslog         bool
	syslogHostname string
	syslogApp      string
	syslogPriority int
)

func main() {
	app := cli.NewApp()
	app.Name = "forward"
	app.Usage = "Transport StdIn lines to a remote destination over UDP, TCP, or TCP+TLS"
	app.UsageText = "forward [global options] [syslog [syslog options]] address:port"
	app.Version = "0.1"
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:        "udp, u",
			Usage:       "Send via UDP (will ignore TLS)",
			Destination: &useUDP,
		},
		cli.BoolTFlag{
			Name:        "tls, s",
			Usage:       "TLS-secured TCP connection",
			Destination: &useTLS,
		},
		cli.BoolTFlag{
			Name:        "tee, t",
			Usage:       "Tee stdin to stdout",
			Destination: &doTee,
		},
	}
	app.Action = func(c *cli.Context) {
		forward(c.Args().First())
	}

	h, _ := os.Hostname()
	app.Commands = []cli.Command{
		{
			Name:      "syslog",
			Aliases:   []string{"log"},
			Usage:     "Wrap lines in RFC-5424 Syslog format",
			ArgsUsage: "address:port",
			Action: func(c *cli.Context) {
				syslog = true
				forward(c.Args().First())
			},
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:        "hostname, n",
					Value:       h,
					Destination: &syslogHostname,
				},
				cli.StringFlag{
					Name:        "app, a",
					Value:       "logger",
					Destination: &syslogApp,
				},
				cli.IntFlag{
					Name:        "priority, p",
					Value:       22,
					Destination: &syslogPriority,
				},
			},
		},
	}

	app.Run(os.Args)
}

func forward(destination string) {
	if !validDestination(destination) {
		return
	}

	var conn net.Conn
	var err error
	switch {
	case useUDP:
		conn, err = net.Dial("udp", destination)
	case !useTLS:
		conn, err = net.Dial("tcp", destination)
	default:
		conn, err = tls.Dial("tcp", destination, &tls.Config{})
	}
	connected := false
	if err != nil {
		fmt.Fprintf(os.Stderr, "Could not connect to %s: %v\n", destination, err)
	} else {
		connected = true
		defer conn.Close()
	}

	writers := make([]*io.PipeWriter, 0, 2)
	wg := sync.WaitGroup{}
	if doTee {
		stdOutReader, stdOutWriter := io.Pipe()
		writers = append(writers, stdOutWriter)
		go func() {
			io.Copy(os.Stdout, stdOutReader)
		}()
	}
	if connected {
		netReader, netWriter := io.Pipe()
		writers = append(writers, netWriter)
		go func() {
			wg.Add(1)
			reader := bufio.NewReader(netReader)
			byteBuffer := bytes.NewBuffer([]byte{})
			for {
				data, err := reader.ReadBytes('\n')
				if err != nil {
					break
				}

				if syslog {
					data = toSyslog(byteBuffer, data)
				}

				if _, err = conn.Write(data); err != nil {
					break
				}
				byteBuffer.Reset()
			}
			io.Copy(ioutil.Discard, netReader)
			wg.Done()
		}()
	}

	if len(writers) == 0 {
		io.Copy(ioutil.Discard, os.Stdin)
		return
	}

	writeOnly := make([]io.Writer, 0, len(writers))
	for _, w := range writers {
		writeOnly = append(writeOnly, w)
	}
	mw := io.MultiWriter(writeOnly...)
	io.Copy(mw, os.Stdin)
	for _, w := range writers {
		w.Close()
	}

	wg.Wait()
}

func toSyslog(b *bytes.Buffer, line []byte) []byte {
	//<22>1 2016-06-18T09:56:21Z sendername programname - - - the log message
	b.WriteString("<")
	b.WriteString(strconv.Itoa(syslogPriority))
	b.WriteString(">1 ")
	b.WriteString(time.Now().UTC().Format(time.RFC3339) + " ")
	b.WriteString(syslogHostname + " ")
	b.WriteString(syslogApp + " ")
	b.WriteString("- - - ")
	b.Write(line)

	return b.Bytes()
}

// Basic validation test
func validDestination(d string) bool {
	if len(d) == 0 {
		return false
	}

	s := strings.Split(d, ":")
	if len(s) != 2 || len(s[0]) == 0 || len(s[1]) == 0 {
		return false
	}

	if _, err := strconv.Atoi(s[1]); err != nil {
		return false
	}

	return true
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"regexp"
	"strconv"
	"time"

	"github.com/fatih/color"
	"github.com/lucagrulla/cw/cloudwatch"
	kingpin "gopkg.in/alecthomas/kingpin.v2"
)

var (
	timeFormat = "2006-01-02T15:04:05"
	version    = "2.1.3"

	kp = kingpin.New("cw", "The best way to tail AWS Cloudwatch Logs from your terminal.")

	awsProfile = kp.Flag("profile", "The target AWS profile. By default cw will use the default profile defined in the .aws/credentials file.").Short('p').String()
	awsRegion  = kp.Flag("region", "The target AWS region. By default cw will use the default region defined in the .aws/credentials file.").Short('r').String()
	noColor    = kp.Flag("no-color", "Disable coloured output.").Short('c').Default("false").Bool()
	debug      = kp.Flag("debug", "Enable debug logging.").Short('d').Default("false").Hidden().Bool()

	lsCommand      = kp.Command("ls", "Show an entity.")
	lsGroups       = lsCommand.Command("groups", "Show all groups.")
	lsStreams      = lsCommand.Command("streams", "Show all streams in a given log group.")
	lsLogGroupName = lsStreams.Arg("group", "The group name.").HintAction(groupsCompletion).Required().String()

	tailCommand = kp.Command("tail", "Tail a log group.")
	follow      = tailCommand.Flag("follow", "Don't stop when the end of stream is reached, but rather wait for additional data to be appended.").Short('f').Default("false").Bool()

	printTimestamp  = tailCommand.Flag("timestamp", "Print the event timestamp.").Short('t').Default("false").Bool()
	printEventID    = tailCommand.Flag("event-id", "Print the event Id.").Short('i').Default("false").Bool()
	printStreamName = tailCommand.Flag("stream-name", "Print the log stream name this event belongs to.").Short('s').Default("false").Bool()

	grep = tailCommand.Flag("grep", "Pattern to filter logs by. See http://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/FilterAndPatternSyntax.html for syntax.").
		Short('g').Default("").String()
	grepv = tailCommand.Flag("grepv", "Equivalent of grep --invert-match. Invert match pattern to filter logs by.").Short('v').Default("").String()

	startTime = tailCommand.Flag("start", `The UTC start time. Passed as either date/time or human-friendly format.
											The human-friendly format accepts the number of hours and minutes prior to the present.
											Denote hours with 'h' and minutes with 'm' i.e. 80m, 4h30m.
											If time is used (format: hh[:mm]) it is expanded to today at the given time. Full available date/time format: 2017-02-27[T09:00[:00]].`).
		Short('b').Default(time.Now().UTC().Add(-30 * time.Second).Format(timeFormat)).String()
	endTime = tailCommand.Flag("end", `The UTC start time. Passed as either date/time or human-friendly format.
										The human-friendly format accepts the number of hours and minutes prior to the present.
										Denote hours with 'h' and minutes with 'm' i.e. 80m, 4h30m.
										If time is used (format: hh[:mm]) it is expanded to today at the given time. Full available date/time format: 2017-02-27[T09:00[:00]].`).
		Short('e').Default("").String()
	local = tailCommand.Flag("local", "Treat date and time in Local zone.").Short('l').Default("false").Bool()

	logGroupName  = tailCommand.Arg("group", "The log group name.").Required().HintAction(groupsCompletion).String()
	logStreamName = tailCommand.Arg("stream", "The log stream name. If not specified all stream names in the given group will be tailed.").HintAction(streamsCompletion).String()
)

func groupsCompletion() []string {
	var groups []string
	kingpin.MustParse(kp.Parse(os.Args[1:]))

	for msg := range cloudwatch.New(awsProfile, awsRegion, debug).LsGroups() {
		groups = append(groups, *msg)
	}
	return groups
}

func streamsCompletion() []string {
	var streams []string
	kingpin.MustParse(kp.Parse(os.Args[1:]))

	for msg := range cloudwatch.New(awsProfile, awsRegion, debug).LsStreams(logGroupName, nil) {
		streams = append(streams, *msg)
	}
	return streams
}

func timestampToTime(timeStamp *string) time.Time {
	var zone *time.Location
	if *local {
		zone = time.Local
	} else {
		zone = time.UTC
	}
	if regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`).MatchString(*timeStamp) {
		t, _ := time.ParseInLocation("2006-01-02", *timeStamp, zone)
		return t
	} else if regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}$`).MatchString(*timeStamp) {
		t, _ := time.ParseInLocation("2006-01-02T15", *timeStamp, zone)
		return t
	} else if regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}$`).MatchString(*timeStamp) {
		t, _ := time.ParseInLocation("2006-01-02T15:04", *timeStamp, zone)
		return t
	} else if regexp.MustCompile(`^\d{1,2}$`).MatchString(*timeStamp) {
		y, m, d := time.Now().In(zone).Date()
		t, _ := strconv.Atoi(*timeStamp)
		return time.Date(y, m, d, t, 0, 0, 0, zone)
	} else if res := regexp.MustCompile(`^(?P<Hour>\d{1,2}):(?P<Minute>\d{2})$`).FindStringSubmatch(*timeStamp); res != nil {
		y, m, d := time.Now().Date()

		t, _ := strconv.Atoi(res[1])
		mm, _ := strconv.Atoi(res[2])

		return time.Date(y, m, d, t, mm, 0, 0, zone)
	} else if regexp.MustCompile(`^\d{1,}h$|^\d{1,}m$|^\d{1,}h\d{1,}m$`).MatchString(*timeStamp) {
		d, _ := time.ParseDuration(*timeStamp)

		t := time.Now().In(zone).Add(-d)
		y, m, dd := t.Date()
		return time.Date(y, m, dd, t.Hour(), t.Minute(), 0, 0, zone)
	}

	//TODO check even last scenario and if it's not a recognized pattern throw an error
	t, _ := time.ParseInLocation("2006-01-02T15:04:05", *timeStamp, zone)
	return t
}

func main() {
	kp.Version(version).Author("Luca Grulla")

	defer newVersionMsg(version, fetchLatestVersion(), *noColor)
	go versionCheckOnSigterm()

	cmd := kingpin.MustParse(kp.Parse(os.Args[1:]))
	c := cloudwatch.New(awsProfile, awsRegion, debug)
	switch cmd {
	case "ls groups":

		for msg := range c.LsGroups() {
			fmt.Println(*msg)
		}
	case "ls streams":
		for msg := range c.LsStreams(lsLogGroupName, nil) {
			fmt.Println(*msg)
		}
	case "tail":
		st := timestampToTime(startTime)
		var et time.Time
		if *endTime != "" {
			et = timestampToTime(endTime)
		}
		for event := range c.Tail(logGroupName, logStreamName, follow, &st, &et, grep, grepv) {
			msg := *event.Message
			if *printEventID {
				if *noColor {
					msg = fmt.Sprintf("%s - %s", *event.EventId, msg)
				} else {
					msg = fmt.Sprintf("%s - %s", color.YellowString(*event.EventId), msg)
				}
			}
			if *printStreamName {
				if *noColor {
					msg = fmt.Sprintf("%s - %s", *event.LogStreamName, msg)
				} else {
					msg = fmt.Sprintf("%s - %s", color.BlueString(*event.LogStreamName), msg)
				}
			}
			if *printTimestamp {
				eventTimestamp := *event.Timestamp / 1000
				ts := time.Unix(eventTimestamp, 0).Format(timeFormat)
				if *noColor {
					msg = fmt.Sprintf("%s - %s", ts, msg)
				} else {
					msg = fmt.Sprintf("%s - %s", color.GreenString(ts), msg)
				}
			}
			fmt.Println(msg)
		}
	}
}
<commit_msg>Fix command help<commit_after>package main

import (
	"fmt"
	"os"
	"regexp"
	"strconv"
	"time"

	"github.com/fatih/color"
	"github.com/lucagrulla/cw/cloudwatch"
	kingpin "gopkg.in/alecthomas/kingpin.v2"
)

var (
	timeFormat = "2006-01-02T15:04:05"
	version    = "2.1.3"

	kp = kingpin.New("cw", "The best way to tail AWS Cloudwatch Logs from your terminal.")

	awsProfile = kp.Flag("profile", "The target AWS profile. By default cw will use the default profile defined in the .aws/credentials file.").Short('p').String()
	awsRegion  = kp.Flag("region", "The target AWS region. By default cw will use the default region defined in the .aws/credentials file.").Short('r').String()
	noColor    = kp.Flag("no-color", "Disable coloured output.").Short('c').Default("false").Bool()
	debug      = kp.Flag("debug", "Enable debug logging.").Short('d').Default("false").Hidden().Bool()

	lsCommand      = kp.Command("ls", "Show an entity.")
	lsGroups       = lsCommand.Command("groups", "Show all groups.")
	lsStreams      = lsCommand.Command("streams", "Show all streams in a given log group.")
	lsLogGroupName = lsStreams.Arg("group", "The group name.").HintAction(groupsCompletion).Required().String()

	tailCommand = kp.Command("tail", "Tail a log group.")
	follow      = tailCommand.Flag("follow", "Don't stop when the end of stream is reached, but rather wait for additional data to be appended.").Short('f').Default("false").Bool()

	printTimestamp  = tailCommand.Flag("timestamp", "Print the event timestamp.").Short('t').Default("false").Bool()
	printEventID    = tailCommand.Flag("event-id", "Print the event Id.").Short('i').Default("false").Bool()
	printStreamName = tailCommand.Flag("stream-name", "Print the log stream name this event belongs to.").Short('s').Default("false").Bool()

	grep = tailCommand.Flag("grep", "Pattern to filter logs by. See http://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/FilterAndPatternSyntax.html for syntax.").
		Short('g').Default("").String()
	grepv = tailCommand.Flag("grepv", "Equivalent of grep --invert-match. Invert match pattern to filter logs by.").Short('v').Default("").String()

	startTime = tailCommand.Flag("start", `The UTC start time. Passed as either date/time or human-friendly format.
											The human-friendly format accepts the number of hours and minutes prior to the present.
											Denote hours with 'h' and minutes with 'm' i.e. 80m, 4h30m.
											If time is used (format: hh[:mm]) it is expanded to today at the given time. Full available date/time format: 2017-02-27[T09:00[:00]].`).
		Short('b').Default(time.Now().UTC().Add(-30 * time.Second).Format(timeFormat)).String()
	endTime = tailCommand.Flag("end", `The UTC end time. Passed as either date/time or human-friendly format.
										The human-friendly format accepts the number of hours and minutes prior to the present.
										Denote hours with 'h' and minutes with 'm' i.e. 80m, 4h30m.
										If time is used (format: hh[:mm]) it is expanded to today at the given time. Full available date/time format: 2017-02-27[T09:00[:00]].`).
		Short('e').Default("").String()
	local = tailCommand.Flag("local", "Treat date and time in Local timezone.").Short('l').Default("false").Bool()

	logGroupName  = tailCommand.Arg("group", "The log group name.").Required().HintAction(groupsCompletion).String()
	logStreamName = tailCommand.Arg("stream", "The log stream name. If not specified all stream names in the given group will be tailed.").HintAction(streamsCompletion).String()
)

func groupsCompletion() []string {
	var groups []string
	kingpin.MustParse(kp.Parse(os.Args[1:]))

	for msg := range cloudwatch.New(awsProfile, awsRegion, debug).LsGroups() {
		groups = append(groups, *msg)
	}
	return groups
}

func streamsCompletion() []string {
	var streams []string
	kingpin.MustParse(kp.Parse(os.Args[1:]))

	for msg := range cloudwatch.New(awsProfile, awsRegion, debug).LsStreams(logGroupName, nil) {
		streams = append(streams, *msg)
	}
	return streams
}

func timestampToTime(timeStamp *string) time.Time {
	var zone *time.Location
	if *local {
		zone = time.Local
	} else {
		zone = time.UTC
	}
	if regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`).MatchString(*timeStamp) {
		t, _ := time.ParseInLocation("2006-01-02", *timeStamp, zone)
		return t
	} else if regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}$`).MatchString(*timeStamp) {
		t, _ := time.ParseInLocation("2006-01-02T15", *timeStamp, zone)
		return t
	} else if regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}$`).MatchString(*timeStamp) {
		t, _ := time.ParseInLocation("2006-01-02T15:04", *timeStamp, zone)
		return t
	} else if regexp.MustCompile(`^\d{1,2}$`).MatchString(*timeStamp) {
		y, m, d := time.Now().In(zone).Date()
		t, _ := strconv.Atoi(*timeStamp)
		return time.Date(y, m, d, t, 0, 0, 0, zone)
	} else if res := regexp.MustCompile(`^(?P<Hour>\d{1,2}):(?P<Minute>\d{2})$`).FindStringSubmatch(*timeStamp); res != nil {
		y, m, d := time.Now().Date()

		t, _ := strconv.Atoi(res[1])
		mm, _ := strconv.Atoi(res[2])

		return time.Date(y, m, d, t, mm, 0, 0, zone)
	} else if regexp.MustCompile(`^\d{1,}h$|^\d{1,}m$|^\d{1,}h\d{1,}m$`).MatchString(*timeStamp) {
		d, _ := time.ParseDuration(*timeStamp)

		t := time.Now().In(zone).Add(-d)
		y, m, dd := t.Date()
		return time.Date(y, m, dd, t.Hour(), t.Minute(), 0, 0, zone)
	}

	//TODO check even last scenario and if it's not a recognized pattern throw an error
	t, _ := time.ParseInLocation("2006-01-02T15:04:05", *timeStamp, zone)
	return t
}

func main() {
	kp.Version(version).Author("Luca Grulla")

	defer newVersionMsg(version, fetchLatestVersion(), *noColor)
	go versionCheckOnSigterm()

	cmd := kingpin.MustParse(kp.Parse(os.Args[1:]))
	c := cloudwatch.New(awsProfile, awsRegion, debug)
	switch cmd {
	case "ls groups":

		for msg := range c.LsGroups() {
			fmt.Println(*msg)
		}
	case "ls streams":
		for msg := range c.LsStreams(lsLogGroupName, nil) {
			fmt.Println(*msg)
		}
	case "tail":
		st := timestampToTime(startTime)
		var et time.Time
		if *endTime != "" {
			et = timestampToTime(endTime)
		}
		for event := range c.Tail(logGroupName, logStreamName, follow, &st, &et, grep, grepv) {
			msg := *event.Message
			if *printEventID {
				if *noColor {
					msg = fmt.Sprintf("%s - %s", *event.EventId, msg)
				} else {
					msg = fmt.Sprintf("%s - %s", color.YellowString(*event.EventId), msg)
				}
			}
			if *printStreamName {
				if *noColor {
					msg = fmt.Sprintf("%s - %s", *event.LogStreamName, msg)
				} else {
					msg = fmt.Sprintf("%s - %s", color.BlueString(*event.LogStreamName), msg)
				}
			}
			if *printTimestamp {
				eventTimestamp := *event.Timestamp / 1000
				ts := time.Unix(eventTimestamp, 0).Format(timeFormat)
				if *noColor {
					msg = fmt.Sprintf("%s - %s", ts, msg)
				} else {
					msg = fmt.Sprintf("%s - %s", color.GreenString(ts), msg)
				}
			}
			fmt.Println(msg)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
       "os"

       "github.com/ninjasphere/go-ninja"
       "github.com/ninjasphere/go-ninja/logger"
       "github.com/ninjasphere/go-openzwave"
       "github.com/ninjasphere/go-openzwave/LOG_LEVEL"
)

const driverName = "driver-zwave"

var log = logger.GetLogger(driverName)

func main() {

	log.Infof("Starting " + driverName)

	conn, err := ninja.Connect("com.ninjablocks.zwave")
	if err != nil {
		log.FatalError(err, "Could not connect to MQTT")
	}

	pwd, _ := os.Getwd()

	bus, err := conn.AnnounceDriver("com.ninjablocks.zwave", driverName, pwd)
	if err != nil {
		log.FatalError(err, "Could not get driver bus")
	}

	statusJob, err := ninja.CreateStatusJob(conn, driverName)

	if err != nil {
		log.FatalError(err, "Could not setup status job")
	}

	statusJob.Start()

	ipAddr, err := ninja.GetNetAddress()
	if err != nil {
		log.FatalError(err, "Could not get net address")
	}

	_ = bus
	_ = ipAddr

	loop := func(api openzwave.API) {
		for {
		    select {
		    	case notification := <- api.Notifications():
			     log.Infof("notification received <- %v\n", notification);
			     api.FreeNotification(notification);
			case quitReceived := <- api.QuitSignal():
			     _ = quitReceived
			     return;
		    }
		}
	}

	os.Exit(openzwave.
		BuildAPI("/usr/local/etc/openzwave", "", "").
		AddIntOption("SaveLogLevel", LOG_LEVEL.NONE).
		AddIntOption("QueueLogLevel", LOG_LEVEL.NONE).
		AddIntOption("DumpTrigger", LOG_LEVEL.NONE).
		AddIntOption("PollInterval", 360).	// a 6 minute interval
		AddBoolOption("IntervalBetweenPolls", true).
		AddBoolOption("ValidateValueChanges", true).
		Run(loop));

}
<commit_msg>Configure the logger.<commit_after>package main

import (
       "os"

       "github.com/ninjasphere/go-ninja"
       "github.com/ninjasphere/go-ninja/logger"
       "github.com/ninjasphere/go-openzwave"
       "github.com/ninjasphere/go-openzwave/LOG_LEVEL"
)

const driverName = "driver-zwave"

var log = logger.GetLogger(driverName)

func main() {

	log.Infof("Starting " + driverName)

	conn, err := ninja.Connect("com.ninjablocks.zwave")
	if err != nil {
		log.FatalError(err, "Could not connect to MQTT")
	}

	pwd, _ := os.Getwd()

	bus, err := conn.AnnounceDriver("com.ninjablocks.zwave", driverName, pwd)
	if err != nil {
		log.FatalError(err, "Could not get driver bus")
	}

	statusJob, err := ninja.CreateStatusJob(conn, driverName)

	if err != nil {
		log.FatalError(err, "Could not setup status job")
	}

	statusJob.Start()

	ipAddr, err := ninja.GetNetAddress()
	if err != nil {
		log.FatalError(err, "Could not get net address")
	}

	_ = bus
	_ = ipAddr

	loop := func(api openzwave.API) {
		for {
		    select {
		    	case notification := <- api.Notifications():
			     log.Infof("notification received <- %v\n", notification);
			     api.FreeNotification(notification);
			case quitReceived := <- api.QuitSignal():
			     _ = quitReceived
			     return;
		    }
		}
	}

	os.Exit(openzwave.
		BuildAPI("/usr/local/etc/openzwave", "", "").
		SetLogger(log).
		AddIntOption("SaveLogLevel", LOG_LEVEL.NONE).
		AddIntOption("QueueLogLevel", LOG_LEVEL.NONE).
		AddIntOption("DumpTrigger", LOG_LEVEL.NONE).
		AddIntOption("PollInterval", 360).	// a 6 minute interval
		AddBoolOption("IntervalBetweenPolls", true).
		AddBoolOption("ValidateValueChanges", true).
		Run(loop));

}
<|endoftext|>


<commit_before>package main

import "github.com/gin-gonic/gin"

func main() {
	router := gin.Default()

	router.GET("/ping", func(c *gin.Context) {
		c.String(200, "pong");
	})
	router.Run(":5000")

	return
}
<commit_msg>:rocket: Listening on environment-defined address<commit_after>package main

import (
	"github.com/gin-gonic/gin"
	"os"
)

func main() {
	router := gin.Default()

	router.GET("/ping", func(c *gin.Context) {
		c.String(200, "pong");
	})
	router.Run(os.Getenv("LISTEN"))

	return
}
<|endoftext|>


<commit_before>package main

import (
    "io/ioutil"
    "os"
    "os/exec"
    "strconv"
    "strings"
    "github.com/fatih/color"
)

var (
    progPath string
    progName string
    testDir string
    ioStyle int
    isPython bool
)

func main() {
    initVars()

    // chdir to directory containg test cases
    os.Chdir(testDir)

    setIOStyle()

    i := 1

    if ioStyle == 1 {
        for {
            if _, err := os.Stat(strconv.Itoa(i) + ".in"); os.IsNotExist(err) {
                // TODO: Add number of total test cases passed
                color.White("Finished testing")
                os.Exit(0)
            }

            copyTestData(strconv.Itoa(i) + ".in")

            runProgram()

            a := readOutput(progName + ".out")
            b := readOutput(strconv.Itoa(i) + ".out")

            if compareOutput(a, b) != -1 {
                color.Red("Wrong output for test case " + strconv.Itoa(i) + ":\nExpected:\n" + string(b[:]) + "Found:\n" + string(a[:]) + "\n")
            } else {
                color.Green("Test case " + strconv.Itoa(i) + " passed");
            }
            i++
        }
    } else if ioStyle == 2 {
        for {
            if _, err := os.Stat("I." + strconv.Itoa(i)); os.IsNotExist(err) {
                // TODO: Add number of total test cases passed
                color.White("Finished testing")
                os.Exit(0)
            }

            copyTestData("I." + strconv.Itoa(i))

            runProgram()

            a := readOutput(progName + ".out")
            b := readOutput("O." + strconv.Itoa(i))

            if compareOutput(a, b) != -1 {
                color.Red("Wrong output for test case " + strconv.Itoa(i) + ":\nExpected:\n" + string(b[:]) + "Found:\n" + string(a[:]) + "\n")
            } else {
                color.Green("Test case " + strconv.Itoa(i) + " passed");
            }
            i++
        }

    }
}

func initVars() {
    numArgs := len(os.Args)
    if numArgs >= 2 {
        progPath = os.Args[1]
    } else {
        color.Red("Please specify exectable as first argument")
        os.Exit(0)
    }

    if numArgs >= 3 {
        testDir = os.Args[2]
    } else {
        x, err := os.Getwd()
        if err != nil {
            color.Red("Error getting current working directory:\n" + err.Error())
            os.Exit(0)
        }
        testDir = x
    }

    progName = getProgName()

    if progName[len(progName) - 3:] == ".py" {
        isPython = true
        progName = progName[:len(progName) - 3]
    }
}

func getProgName() string {
    pos := strings.LastIndex(progPath, "/")
    if pos == -1 {
        pos = strings.LastIndex(progPath, "\\")
    }
    return progPath[pos + 1:]
}

func copyTestData(fileName string) {
    b, err := ioutil.ReadFile(fileName)
    if err != nil {
        panic(err)
    }

    err = ioutil.WriteFile(progName + ".in", b, 0644)
    if err != nil {
        panic(err)
    }
}

// if outputs are not same, return position of nonequivalence; otherwise return -1
func compareOutput(a, b []byte) int {
    fileLen := len(a)
    bLen := len(b)
    if fileLen != bLen {
        if fileLen > bLen {
            return fileLen - bLen
        }
        return bLen - fileLen
    }
    for i := 0; i < fileLen; i++ {
        if a[i] != b[i] {
            return i
        }
    }
    return -1
}

func readOutput(fileName string) []byte {
    b, err := ioutil.ReadFile(fileName)
    if err != nil {
        panic(err)
    }
    return b
}

// ioStyle 1 is of form "X.in"; ioStyle 2 is of form "I.X";
// TODO: Change function to check if list of files in "testDir" contains "X.in" or "I.X" files using regex
func setIOStyle() {
    if _, err := os.Stat("1.in"); err == nil {
        ioStyle = 1
        return
    } else if _, err = os.Stat("I.1"); err == nil {
        ioStyle = 2
        return
    }
    color.Red("Could not find test input/ouput files in " + testDir)
    os.Exit(0)
}

func runProgram() {
    cmd := exec.Command(progPath)
    if isPython {
        cmd = exec.Command("python", progPath)
    }
    err := cmd.Run()
    if err != nil {
        color.Red("Error running program:\n" + err.Error())
        os.Exit(0)
    }
}
<commit_msg>change error text<commit_after>package main

import (
    "io/ioutil"
    "os"
    "os/exec"
    "strconv"
    "strings"
    "github.com/fatih/color"
)

var (
    progPath string
    progName string
    testDir string
    ioStyle int
    isPython bool
)

func main() {
    initVars()

    // chdir to directory containg test cases
    os.Chdir(testDir)

    setIOStyle()

    i := 1

    if ioStyle == 1 {
        for {
            if _, err := os.Stat(strconv.Itoa(i) + ".in"); os.IsNotExist(err) {
                // TODO: Add number of total test cases passed
                color.White("Finished testing")
                os.Exit(0)
            }

            copyTestData(strconv.Itoa(i) + ".in")

            runProgram()

            a := readOutput(progName + ".out")
            b := readOutput(strconv.Itoa(i) + ".out")

            if compareOutput(a, b) != -1 {
                color.Red("Wrong output for test case " + strconv.Itoa(i) + ":\nExpected:\n" + string(b[:]) + "Found:\n" + string(a[:]) + "\n")
            } else {
                color.Green("Test case " + strconv.Itoa(i) + " passed");
            }
            i++
        }
    } else if ioStyle == 2 {
        for {
            if _, err := os.Stat("I." + strconv.Itoa(i)); os.IsNotExist(err) {
                // TODO: Add number of total test cases passed
                color.White("Finished testing")
                os.Exit(0)
            }

            copyTestData("I." + strconv.Itoa(i))

            runProgram()

            a := readOutput(progName + ".out")
            b := readOutput("O." + strconv.Itoa(i))

            if compareOutput(a, b) != -1 {
                color.Red("Wrong output for test case " + strconv.Itoa(i) + ":\nExpected:\n" + string(b[:]) + "Found:\n" + string(a[:]) + "\n")
            } else {
                color.Green("Test case " + strconv.Itoa(i) + " passed");
            }
            i++
        }

    }
}

func initVars() {
    numArgs := len(os.Args)
    if numArgs >= 2 {
        progPath = os.Args[1]
    } else {
        color.Red("Please specify program path as first argument")
        os.Exit(0)
    }

    if numArgs >= 3 {
        testDir = os.Args[2]
    } else {
        x, err := os.Getwd()
        if err != nil {
            color.Red("Error getting current working directory:\n" + err.Error())
            os.Exit(0)
        }
        testDir = x
    }

    progName = getProgName()

    if progName[len(progName) - 3:] == ".py" {
        isPython = true
        progName = progName[:len(progName) - 3]
    }
}

func getProgName() string {
    pos := strings.LastIndex(progPath, "/")
    if pos == -1 {
        pos = strings.LastIndex(progPath, "\\")
    }
    return progPath[pos + 1:]
}

func copyTestData(fileName string) {
    b, err := ioutil.ReadFile(fileName)
    if err != nil {
        panic(err)
    }

    err = ioutil.WriteFile(progName + ".in", b, 0644)
    if err != nil {
        panic(err)
    }
}

// if outputs are not same, return position of nonequivalence; otherwise return -1
func compareOutput(a, b []byte) int {
    fileLen := len(a)
    bLen := len(b)
    if fileLen != bLen {
        if fileLen > bLen {
            return fileLen - bLen
        }
        return bLen - fileLen
    }
    for i := 0; i < fileLen; i++ {
        if a[i] != b[i] {
            return i
        }
    }
    return -1
}

func readOutput(fileName string) []byte {
    b, err := ioutil.ReadFile(fileName)
    if err != nil {
        panic(err)
    }
    return b
}

// ioStyle 1 is of form "X.in"; ioStyle 2 is of form "I.X";
// TODO: Change function to check if list of files in "testDir" contains "X.in" or "I.X" files using regex
func setIOStyle() {
    if _, err := os.Stat("1.in"); err == nil {
        ioStyle = 1
        return
    } else if _, err = os.Stat("I.1"); err == nil {
        ioStyle = 2
        return
    }
    color.Red("Could not find test input/ouput files in " + testDir)
    os.Exit(0)
}

func runProgram() {
    cmd := exec.Command(progPath)
    if isPython {
        cmd = exec.Command("python", progPath)
    }
    err := cmd.Run()
    if err != nil {
        color.Red("Error running program:\n" + err.Error())
        os.Exit(0)
    }
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/binary"
	"encoding/hex"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"sync"
	"time"

	"github.com/calmh/syncthing/discover"
	"github.com/golang/groupcache/lru"
	"github.com/juju/ratelimit"
)

type Node struct {
	Addresses []Address
	Updated   time.Time
}

type Address struct {
	IP   []byte
	Port uint16
}

var (
	nodes     = make(map[string]Node)
	lock      sync.Mutex
	queries   = 0
	announces = 0
	answered  = 0
	limited   = 0
	unknowns  = 0
	debug     = false
	limiter   = lru.New(1024)
)

func main() {
	var listen string
	var timestamp bool
	var statsIntv int
	var statsFile string

	flag.StringVar(&listen, "listen", ":22025", "Listen address")
	flag.BoolVar(&debug, "debug", false, "Enable debug output")
	flag.BoolVar(&timestamp, "timestamp", true, "Timestamp the log output")
	flag.IntVar(&statsIntv, "stats-intv", 0, "Statistics output interval (s)")
	flag.StringVar(&statsFile, "stats-file", "/var/log/discosrv.stats", "Statistics file name")
	flag.Parse()

	log.SetOutput(os.Stdout)
	if !timestamp {
		log.SetFlags(0)
	}

	addr, _ := net.ResolveUDPAddr("udp", listen)
	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		log.Fatal(err)
	}

	if statsIntv > 0 {
		go logStats(statsFile, statsIntv)
	}

	var buf = make([]byte, 1024)
	for {
		buf = buf[:cap(buf)]
		n, addr, err := conn.ReadFromUDP(buf)

		if limit(addr) {
			// Rate limit in effect for source
			continue
		}

		if err != nil {
			log.Fatal(err)
		}

		if n < 4 {
			log.Printf("Received short packet (%d bytes)", n)
			continue
		}

		buf = buf[:n]
		magic := binary.BigEndian.Uint32(buf)

		switch magic {
		case discover.AnnouncementMagicV2:
			handleAnnounceV2(addr, buf)

		case discover.QueryMagicV2:
			handleQueryV2(conn, addr, buf)

		default:
			lock.Lock()
			unknowns++
			lock.Unlock()
		}
	}
}

func limit(addr *net.UDPAddr) bool {
	key := addr.IP.String()

	lock.Lock()
	defer lock.Unlock()

	bkt, ok := limiter.Get(key)
	if ok {
		bkt := bkt.(*ratelimit.Bucket)
		if bkt.TakeAvailable(1) != 1 {
			// Rate limit exceeded; ignore packet
			if debug {
				log.Println("Rate limit exceeded for", key)
			}
			limited++
			return true
		}
	} else {
		if debug {
			log.Println("New limiter for", key)
		}
		// One packet per ten seconds average rate, burst ten packets
		limiter.Add(key, ratelimit.NewBucket(10*time.Second, 10))
	}

	return false
}

func handleAnnounceV2(addr *net.UDPAddr, buf []byte) {
	var pkt discover.AnnounceV2
	err := pkt.UnmarshalXDR(buf)
	if err != nil {
		log.Println("AnnounceV2 Unmarshal:", err)
		log.Println(hex.Dump(buf))
		return
	}
	if debug {
		log.Printf("<- %v %#v", addr, pkt)
	}

	lock.Lock()
	announces++
	lock.Unlock()

	ip := addr.IP.To4()
	if ip == nil {
		ip = addr.IP.To16()
	}

	var addrs []Address
	for _, addr := range pkt.Addresses {
		tip := addr.IP
		if len(tip) == 0 {
			tip = ip
		}
		addrs = append(addrs, Address{
			IP:   tip,
			Port: addr.Port,
		})
	}

	node := Node{
		Addresses: addrs,
		Updated:   time.Now(),
	}

	lock.Lock()
	nodes[pkt.NodeID] = node
	lock.Unlock()
}

func handleQueryV2(conn *net.UDPConn, addr *net.UDPAddr, buf []byte) {
	var pkt discover.QueryV2
	err := pkt.UnmarshalXDR(buf)
	if err != nil {
		log.Println("QueryV2 Unmarshal:", err)
		log.Println(hex.Dump(buf))
		return
	}
	if debug {
		log.Printf("<- %v %#v", addr, pkt)
	}

	lock.Lock()
	node, ok := nodes[pkt.NodeID]
	queries++
	lock.Unlock()

	if ok && len(node.Addresses) > 0 {
		pkt := discover.AnnounceV2{
			Magic:  discover.AnnouncementMagicV2,
			NodeID: pkt.NodeID,
		}
		for _, addr := range node.Addresses {
			pkt.Addresses = append(pkt.Addresses, discover.Address{IP: addr.IP, Port: addr.Port})
		}
		if debug {
			log.Printf("-> %v %#v", addr, pkt)
		}

		tb := pkt.MarshalXDR()
		_, _, err = conn.WriteMsgUDP(tb, nil, addr)
		if err != nil {
			log.Println("QueryV2 response write:", err)
		}

		lock.Lock()
		answered++
		lock.Unlock()
	}
}

func next(intv int) time.Time {
	d := time.Duration(intv) * time.Second
	t0 := time.Now()
	t1 := t0.Add(d).Truncate(d)
	time.Sleep(t1.Sub(t0))
	return t1
}

func logStats(file string, intv int) {
	f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		log.Fatal(err)
	}
	for {
		t := next(intv)

		lock.Lock()

		var deleted = 0
		for id, node := range nodes {
			if time.Since(node.Updated) > 60*time.Minute {
				delete(nodes, id)
				deleted++
			}
		}

		fmt.Fprintf(f, "%d Nr:%d Ne:%d Qt:%d Qa:%d A:%d U:%d Lq:%d Lc:%d\n",
			t.Unix(), len(nodes), deleted, queries, answered, announces, unknowns, limited, limiter.Len())
		f.Sync()

		queries = 0
		announces = 0
		answered = 0
		limited = 0
		unknowns = 0

		lock.Unlock()
	}
}
<commit_msg>Fix discosrv build, build as part of all (fixes #257)<commit_after>package main

import (
	"encoding/binary"
	"encoding/hex"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"sync"
	"time"

	"github.com/calmh/syncthing/discover"
	"github.com/golang/groupcache/lru"
	"github.com/juju/ratelimit"
)

type node struct {
	addresses []address
	updated   time.Time
}

type address struct {
	ip   []byte
	port uint16
}

var (
	nodes     = make(map[string]node)
	lock      sync.Mutex
	queries   = 0
	announces = 0
	answered  = 0
	limited   = 0
	unknowns  = 0
	debug     = false
	limiter   = lru.New(1024)
)

func main() {
	var listen string
	var timestamp bool
	var statsIntv int
	var statsFile string

	flag.StringVar(&listen, "listen", ":22025", "Listen address")
	flag.BoolVar(&debug, "debug", false, "Enable debug output")
	flag.BoolVar(&timestamp, "timestamp", true, "Timestamp the log output")
	flag.IntVar(&statsIntv, "stats-intv", 0, "Statistics output interval (s)")
	flag.StringVar(&statsFile, "stats-file", "/var/log/discosrv.stats", "Statistics file name")
	flag.Parse()

	log.SetOutput(os.Stdout)
	if !timestamp {
		log.SetFlags(0)
	}

	addr, _ := net.ResolveUDPAddr("udp", listen)
	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		log.Fatal(err)
	}

	if statsIntv > 0 {
		go logStats(statsFile, statsIntv)
	}

	var buf = make([]byte, 1024)
	for {
		buf = buf[:cap(buf)]
		n, addr, err := conn.ReadFromUDP(buf)

		if limit(addr) {
			// Rate limit in effect for source
			continue
		}

		if err != nil {
			log.Fatal(err)
		}

		if n < 4 {
			log.Printf("Received short packet (%d bytes)", n)
			continue
		}

		buf = buf[:n]
		magic := binary.BigEndian.Uint32(buf)

		switch magic {
		case discover.AnnouncementMagicV2:
			handleAnnounceV2(addr, buf)

		case discover.QueryMagicV2:
			handleQueryV2(conn, addr, buf)

		default:
			lock.Lock()
			unknowns++
			lock.Unlock()
		}
	}
}

func limit(addr *net.UDPAddr) bool {
	key := addr.IP.String()

	lock.Lock()
	defer lock.Unlock()

	bkt, ok := limiter.Get(key)
	if ok {
		bkt := bkt.(*ratelimit.Bucket)
		if bkt.TakeAvailable(1) != 1 {
			// Rate limit exceeded; ignore packet
			if debug {
				log.Println("Rate limit exceeded for", key)
			}
			limited++
			return true
		}
	} else {
		if debug {
			log.Println("New limiter for", key)
		}
		// One packet per ten seconds average rate, burst ten packets
		limiter.Add(key, ratelimit.NewBucket(10*time.Second, 10))
	}

	return false
}

func handleAnnounceV2(addr *net.UDPAddr, buf []byte) {
	var pkt discover.AnnounceV2
	err := pkt.UnmarshalXDR(buf)
	if err != nil && err != io.EOF {
		log.Println("AnnounceV2 Unmarshal:", err)
		log.Println(hex.Dump(buf))
		return
	}
	if debug {
		log.Printf("<- %v %#v", addr, pkt)
	}

	lock.Lock()
	announces++
	lock.Unlock()

	ip := addr.IP.To4()
	if ip == nil {
		ip = addr.IP.To16()
	}

	var addrs []address
	for _, addr := range pkt.This.Addresses {
		tip := addr.IP
		if len(tip) == 0 {
			tip = ip
		}
		addrs = append(addrs, address{
			ip:   tip,
			port: addr.Port,
		})
	}

	node := node{
		addresses: addrs,
		updated:   time.Now(),
	}

	lock.Lock()
	nodes[pkt.This.ID] = node
	lock.Unlock()
}

func handleQueryV2(conn *net.UDPConn, addr *net.UDPAddr, buf []byte) {
	var pkt discover.QueryV2
	err := pkt.UnmarshalXDR(buf)
	if err != nil {
		log.Println("QueryV2 Unmarshal:", err)
		log.Println(hex.Dump(buf))
		return
	}
	if debug {
		log.Printf("<- %v %#v", addr, pkt)
	}

	lock.Lock()
	node, ok := nodes[pkt.NodeID]
	queries++
	lock.Unlock()

	if ok && len(node.addresses) > 0 {
		ann := discover.AnnounceV2{
			Magic: discover.AnnouncementMagicV2,
			This: discover.Node{
				ID: pkt.NodeID,
			},
		}
		for _, addr := range node.addresses {
			ann.This.Addresses = append(ann.This.Addresses, discover.Address{IP: addr.ip, Port: addr.port})
		}
		if debug {
			log.Printf("-> %v %#v", addr, pkt)
		}

		tb := ann.MarshalXDR()
		_, _, err = conn.WriteMsgUDP(tb, nil, addr)
		if err != nil {
			log.Println("QueryV2 response write:", err)
		}

		lock.Lock()
		answered++
		lock.Unlock()
	}
}

func next(intv int) time.Time {
	d := time.Duration(intv) * time.Second
	t0 := time.Now()
	t1 := t0.Add(d).Truncate(d)
	time.Sleep(t1.Sub(t0))
	return t1
}

func logStats(file string, intv int) {
	f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		log.Fatal(err)
	}
	for {
		t := next(intv)

		lock.Lock()

		var deleted = 0
		for id, node := range nodes {
			if time.Since(node.updated) > 60*time.Minute {
				delete(nodes, id)
				deleted++
			}
		}

		fmt.Fprintf(f, "%d Nr:%d Ne:%d Qt:%d Qa:%d A:%d U:%d Lq:%d Lc:%d\n",
			t.Unix(), len(nodes), deleted, queries, answered, announces, unknowns, limited, limiter.Len())
		f.Sync()

		queries = 0
		announces = 0
		answered = 0
		limited = 0
		unknowns = 0

		lock.Unlock()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/codegangsta/cli"
	"github.com/mitsuse/arcus/application"
)

const (
	NAME         = `arcus`
	VERSION      = "0.1.3"
	DESCRIPTION  = "A command-line tool to send a message to devices via Pushbullet."
	AUTHOR       = "Tomoya Kose (mitsuse)"
	AUTHOR_EMAIL = "tomoya@mitsuse.jp"
)

func main() {
	app := cli.NewApp()

	app.Name = NAME
	app.Version = VERSION
	app.Usage = DESCRIPTION
	app.Author = AUTHOR
	app.Email = AUTHOR_EMAIL

	app.Commands = []cli.Command{
		newSendCommand(),
		newListCommand(),
	}

	app.Run(os.Args)
}

func newListCommand() cli.Command {
	command := cli.Command{
		Name:      "list",
		ShortName: "l",
		Usage:     "List devices that can be pushed to",

		Action: func(c *cli.Context) {
			token := os.Getenv("ARCUS_ACCESS_TOKEN")

			devices, err := application.ListDevices(token)
			if err != nil {
				printError(err)
				return
			}

			for _, d := range devices {
				if !d.Pushable {
					continue
				}

				fmt.Println(d.Nickname)
			}
		},
	}

	return command
}

func newSendCommand() cli.Command {
	command := cli.Command{
		Name:      "send",
		ShortName: "s",
		Usage:     "Send a message or a file",

		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "device,d",
				Value: "",
				Usage: "The name of target device",
			},

			cli.StringFlag{
				Name:  "title,t",
				Value: "",
				Usage: "The title of the message or file to be sent",
			},

			cli.StringFlag{
				Name:  "message,m",
				Value: "",
				Usage: "The message to be sent",
			},

			cli.StringFlag{
				Name:  "location,l",
				Value: "",
				Usage: "The path of file or link to be sent",
			},
		},

		Action: func(c *cli.Context) {
			token := os.Getenv("ARCUS_ACCESS_TOKEN")
			title := c.String("title")
			message := c.String("message")
			location := c.String("location")
			device := c.String("device")

			if err := application.Send(token, title, message, location, device); err != nil {
				printError(err)
				return
			}
		},
	}

	return command
}

func printError(err error) {
	fmt.Fprintf(os.Stderr, "%s: %s\n", NAME, err)
}
<commit_msg>Manage the name of environment variable as constant.<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/codegangsta/cli"
	"github.com/mitsuse/arcus/application"
)

const (
	NAME         = `arcus`
	VERSION      = "0.1.3"
	DESCRIPTION  = "A command-line tool to send a message to devices via Pushbullet."
	AUTHOR       = "Tomoya Kose (mitsuse)"
	AUTHOR_EMAIL = "tomoya@mitsuse.jp"

	variableToken = "ARCUS_ACCESS_TOKEN"
)

func main() {
	app := cli.NewApp()

	app.Name = NAME
	app.Version = VERSION
	app.Usage = DESCRIPTION
	app.Author = AUTHOR
	app.Email = AUTHOR_EMAIL

	app.Commands = []cli.Command{
		newSendCommand(),
		newListCommand(),
	}

	app.Run(os.Args)
}

func newListCommand() cli.Command {
	command := cli.Command{
		Name:      "list",
		ShortName: "l",
		Usage:     "List devices that can be pushed to",

		Action: func(c *cli.Context) {
			token := getToken()

			devices, err := application.ListDevices(token)
			if err != nil {
				printError(err)
				return
			}

			for _, d := range devices {
				if !d.Pushable {
					continue
				}

				fmt.Println(d.Nickname)
			}
		},
	}

	return command
}

func newSendCommand() cli.Command {
	command := cli.Command{
		Name:      "send",
		ShortName: "s",
		Usage:     "Send a message or a file",

		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "device,d",
				Value: "",
				Usage: "The name of target device",
			},

			cli.StringFlag{
				Name:  "title,t",
				Value: "",
				Usage: "The title of the message or file to be sent",
			},

			cli.StringFlag{
				Name:  "message,m",
				Value: "",
				Usage: "The message to be sent",
			},

			cli.StringFlag{
				Name:  "location,l",
				Value: "",
				Usage: "The path of file or link to be sent",
			},
		},

		Action: func(c *cli.Context) {
			token := getToken()
			title := c.String("title")
			message := c.String("message")
			location := c.String("location")
			device := c.String("device")

			if err := application.Send(token, title, message, location, device); err != nil {
				printError(err)
				return
			}
		},
	}

	return command
}

func getToken() string {
	return os.Getenv(variableToken)
}

func printError(err error) {
	fmt.Fprintf(os.Stderr, "%s: %s\n", NAME, err)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/oschwald/geoip2-golang"
	"log"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"
)

// The GeoIP database containing data on what IP match to what city/country blah
// blah.
var db *geoip2.Reader

func main() {
	// Initialize the database.
	var err error
	db, err = geoip2.Open("GeoLite2-City.mmdb")
	if err != nil {
		log.Fatal(err)
	}

	// Get the HTTP server rollin'
	http.HandleFunc("/", HTTPRequestHandler)
	log.Println("Server listening!")
	http.ListenAndServe(":61430", nil)
}

// Standard request handler if there's no static file to be served.
func HTTPRequestHandler(w http.ResponseWriter, r *http.Request) {
	// Get the current time, so that we can then calculate the execution time.
	start := time.Now()

	var requestIP string
	// The request is most likely being done through a reverse proxy.
	if realIP, ok := r.Header["X-Real-Ip"]; ok && len(r.Header["X-Real-Ip"]) > 0 {
		requestIP = realIP[0]
	} else {
		// Get the real actual request IP without the trolls
		requestIP = UnfuckRequestIP(r.RemoteAddr)
	}

	// Log how much time it took to respond to the request, when we're done.
	defer log.Printf(
		"[rq] %s %s %s %dns",
		requestIP,
		r.Method,
		r.URL.Path,
		time.Since(start).Nanoseconds())

	// Index, redirect to github.com page if the request is sent from a browser.
	// There's a very good reason for which we aren't using regexes.
	// http://ideone.com/jNEMob
	// (tl;dr: regex is holy shit fucking slow)
	if r.URL.Path == "/" && (strings.Index(r.UserAgent(), "mozilla") != -1 ||
		strings.Index(r.UserAgent(), "webkit") != -1 ||
		strings.Index(r.UserAgent(), "opera") != -1) {
		http.Redirect(w, r, "https://github.com/TheHowl/ip.zxq.co/blob/master/README.md", 301)
		return
	}

	// Separate two strings when there is a / in the URL requested.
	requestedThings := strings.Split(r.URL.Path, "/")

	var IPAddress string
	var Which string
	// How in the world the user would manage to even send a request to
	// something without even having Path = "/"?
	// I... have no idea. But I'm paranoid. So let's just do it anyway.
	if len(requestedThings) < 2 {
		IPAddress = ""
	} else {
		IPAddress = requestedThings[1]
	}
	// In case the user didn't write a specific index, let's specify it for
	// them.
	if len(requestedThings) < 3 {
		Which = ""
	} else {
		Which = requestedThings[2]
	}

	// Set the requested IP to the user's request request IP, if we got no address.
	if IPAddress == "" || IPAddress == "self" {
		IPAddress = requestIP
	}

	// Query parameters array making
	queryParamsRaw, _ := url.ParseQuery(r.URL.RawQuery)
	queryParams := SimplifyQueryMap(queryParamsRaw)
	queryParams = AppendDefaultIfNotSet(queryParams, "callback", "#none#")
	queryParams = AppendDefaultIfNotSet(queryParams, "pretty", "0")

	// Get the geodata of the requested IP.
	o, contentType := IPToResponse(IPAddress, Which, queryParams)

	// Set the content type as the one given by IPToResponse.
	w.Header().Set("Content-Type", contentType+"; charset=utf-8")
	// Write the data out to the response.
	fmt.Fprint(w, o)
}

// Appends a default value to a map only if the key, defined as k, doesn't
// already exist in the array.
func AppendDefaultIfNotSet(sl map[string]string, k string, dv string) map[string]string {
	if _, ok := sl[k]; !ok {
		sl[k] = dv
	}
	return sl
}

// url.ParseQuery returns a map containing as a value a slice with often just
// one value. We're fixing that.
func SimplifyQueryMap(sl url.Values) map[string]string {
	var ret map[string]string = map[string]string{}
	for k, v := range sl {
		// We're getting only the last element, because we take as granted that
		// what the use actually means is the last element, if he has provided
		// multiple values for the same key.
		if len(v) > 0 {
			ret[k] = v[len(v)-1]
		}
	}
	return ret
}

// Remove from the IP eventual [ or ], and remove the port part of the IP.
func UnfuckRequestIP(ip string) string {
	ip = strings.Replace(ip, "[", "", 1)
	ip = strings.Replace(ip, "]", "", 1)
	ss := strings.Split(ip, ":")
	ip = strings.Join(ss[:len(ss)-1], ":")
	return ip
}

// Turn the IP into a JSON string containing geodata.
//
// * i: the raw IP string.
// * specific: the specific value to get from the geodata array. Default is ""
// * params: Set callback in the map to a non-"#none#" value to use it as a
//   JSONP callback. Set "pretty" to 1 if you want a 2-space indented JSON
//   output.
func IPToResponse(i string, specific string, params map[string]string) (string, string) {
	ip := net.ParseIP(i)
	if ip == nil {
		return "Please provide a valid IP address", "text/html"
	}

	// Query the maxmind database for that IP address.
	record, err := db.City(ip)
	if err != nil {
		log.Fatal(err)
	}

	// String containing the region/subdivision of the IP. (E.g.: Scotland, or
	// California).
	var sd string
	// If there are subdivisions for this IP, set sd as the first element in the
	// array's name.
	if record.Subdivisions != nil {
		sd = record.Subdivisions[0].Names["en"]
	}

	// Create a new instance of all the data to be returned to the user.
	data := map[string]string{}
	// Fill up the data array with the geoip data.
	data["ip"] = ip.String()
	data["country"] = record.Country.IsoCode
	data["country_full"] = record.Country.Names["en"]
	data["city"] = record.City.Names["en"]
	data["region"] = sd
	data["continent"] = record.Continent.Code
	data["continent_full"] = record.Continent.Names["en"]
	data["postal"] = record.Postal.Code
	// precision of latitude/longitude is up to 4 decimal places (even on
	// ipinfo.io).
	data["loc"] = fmt.Sprintf("%.4f,%.4f", record.Location.Latitude, record.Location.Longitude)

	// Since we don't have HTML output, nor other data from geo data,
	// everything is the same if you do /8.8.8.8, /8.8.8.8/json or /8.8.8.8/geo.
	if specific == "" || specific == "json" || specific == "geo" {
		var bytes_output []byte
		if params["pretty"] == "1" {
			bytes_output, _ = json.MarshalIndent(data, "", "  ")
		} else {
			bytes_output, _ = json.Marshal(data)
		}
		return JSONPify(params["callback"], string(bytes_output[:])),
			"application/json"
	} else if val, ok := data[specific]; ok {
		// If we got a specific value for what the user requested, return only
		// that specific value.
		return val, "text/html"
	} else {
		// We got nothing to show to the user.
		return "undefined", "text/html"
	}
}

// Wraps wrapData into a JSONP callback, if the callback name is valid.
func JSONPify(callback string, wrapData string) string {
	// If you have a callback name longer than 2000 characters, I gotta say, you
	// really should learn to minify your javascript code!
	if callback != "#none#" && callback != "" && len(callback) < 2000 {
		// In case you're wondering, yes, there is a reason for the empty
		// comment! http://stackoverflow.com/a/16048976/5328069
		wrapData = fmt.Sprintf("/**/ typeof %s === 'function' "+
			"&& %s(%s);", callback, callback, wrapData)
	}
	return wrapData
}
<commit_msg>rewrite most of the code<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"regexp"
	"strings"
	"syscall"
	"time"

	"github.com/oschwald/geoip2-golang"
)

// The GeoIP database containing data on what IP match to what city/country blah
// blah.
var db *geoip2.Reader

func main() {
	// Initialize the database.
	var err error
	db, err = geoip2.Open("GeoLite2-City.mmdb")
	if err != nil {
		log.Fatal(err)
	}

	// Get the HTTP server rollin'
	log.Println("Server listening!")
	ln, err := net.Listen("unix", "/tmp/ip.zxq.co.sock")
	if err != nil {
		log.Fatal(err)
	}
	os.Chmod("/tmp/ip.zxq.co.sock", 0777)
	var stop = make(chan os.Signal)
	signal.Notify(stop, syscall.SIGTERM)
	signal.Notify(stop, syscall.SIGINT)
	go func() {
		<-stop
		ln.Close()
		os.Exit(0)
	}()
	http.Serve(ln, http.HandlerFunc(handler))
}

var invalidIPBytes = []byte("Please provide a valid IP address.")

type dataStruct struct {
	IP            string `json:"ip"`
	City          string `json:"city"`
	Region        string `json:"region"`
	Country       string `json:"country"`
	CountryFull   string `json:"country_full"`
	Continent     string `json:"continent"`
	ContinentFull string `json:"continent_full"`
	Loc           string `json:"loc"`
	Postal        string `json:"postal"`
}

var nameToField = map[string]func(dataStruct) string{
	"ip":             func(d dataStruct) string { return d.IP },
	"city":           func(d dataStruct) string { return d.City },
	"region":         func(d dataStruct) string { return d.Region },
	"country":        func(d dataStruct) string { return d.Country },
	"country_full":   func(d dataStruct) string { return d.CountryFull },
	"continent":      func(d dataStruct) string { return d.Continent },
	"continent_full": func(d dataStruct) string { return d.ContinentFull },
	"loc":            func(d dataStruct) string { return d.Loc },
	"postal":         func(d dataStruct) string { return d.Postal },
}

func handler(w http.ResponseWriter, r *http.Request) {
	// Get the current time, so that we can then calculate the execution time.
	start := time.Now()

	// Log how much time it took to respond to the request, when we're done.
	defer log.Printf(
		"[rq] %s %s %dns",
		r.Method,
		r.URL.Path,
		time.Since(start).Nanoseconds())

	// Separate two strings when there is a / in the URL requested.
	requestedThings := strings.Split(r.URL.Path, "/")

	var IPAddress string
	var Which string
	switch len(requestedThings) {
	case 3:
		Which = requestedThings[2]
		fallthrough
	case 2:
		IPAddress = requestedThings[1]
	}

	// Set the requested IP to the user's request request IP, if we got no address.
	if IPAddress == "" || IPAddress == "self" {
		// The request is most likely being done through a reverse proxy.
		if realIP, ok := r.Header["X-Real-Ip"]; ok && len(r.Header["X-Real-Ip"]) > 0 {
			IPAddress = realIP[0]
		} else {
			// Get the real actual request IP without the trolls
			IPAddress = UnfuckRequestIP(r.RemoteAddr)
		}
	}

	ip := net.ParseIP(IPAddress)
	if ip == nil {
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.Write(invalidIPBytes)
		return
	}

	// Query the maxmind database for that IP address.
	record, err := db.City(ip)
	if err != nil {
		log.Fatal(err)
	}

	// String containing the region/subdivision of the IP. (E.g.: Scotland, or
	// California).
	var sd string
	// If there are subdivisions for this IP, set sd as the first element in the
	// array's name.
	if record.Subdivisions != nil {
		sd = record.Subdivisions[0].Names["en"]
	}

	// Fill up the data array with the geoip data.
	d := dataStruct{
		IP:            ip.String(),
		Country:       record.Country.IsoCode,
		CountryFull:   record.Country.Names["en"],
		City:          record.City.Names["en"],
		Region:        sd,
		Continent:     record.Continent.Code,
		ContinentFull: record.Continent.Names["en"],
		Postal:        record.Postal.Code,
		Loc:           fmt.Sprintf("%.4f,%.4f", record.Location.Latitude, record.Location.Longitude),
	}

	// Since we don't have HTML output, nor other data from geo data,
	// everything is the same if you do /8.8.8.8, /8.8.8.8/json or /8.8.8.8/geo.
	if Which == "" || Which == "json" || Which == "geo" {
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		callback := r.URL.Query().Get("callback")
		enableJSONP := callback != "" && len(callback) < 2000 && callbackJSONP.MatchString(callback)
		if enableJSONP {
			_, err = w.Write([]byte("/**/ typeof " + callback + " === 'function' " +
				"&& " + callback + "("))
			if err != nil {
				return
			}
		}
		enc := json.NewEncoder(w)
		if r.URL.Query().Get("pretty") == "1" {
			enc.SetIndent("", "  ")
		}
		enc.Encode(d)
		if enableJSONP {
			w.Write([]byte(");"))
		}
	} else {
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		if val := nameToField[Which]; val != nil {
			w.Write([]byte(val(d)))
		} else {
			w.Write([]byte("undefined"))
		}
	}
}

// Very restrictive, but this way it shouldn't completely fuck up.
var callbackJSONP = regexp.MustCompile(`^[a-zA-Z_\$][a-zA-Z0-9_\$]*$`)

// Remove from the IP eventual [ or ], and remove the port part of the IP.
func UnfuckRequestIP(ip string) string {
	ip = strings.Replace(ip, "[", "", 1)
	ip = strings.Replace(ip, "]", "", 1)
	ss := strings.Split(ip, ":")
	ip = strings.Join(ss[:len(ss)-1], ":")
	return ip
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"flag"
	"hash/crc32"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	"qiniupkg.com/http/httputil.v2"
	"qiniupkg.com/x/log.v7"
)

const (
	mutexCount = 9973
)

var (
	ErrUnmatchedInodeType    = errors.New("unmatched inode type(file or dir)")
	ErrRefreshWithoutPath    = httputil.NewError(400, "refresh without path")
	ErrInvalidPkgPath        = httputil.NewError(400, "invalid package path")
	ErrInvalidGithubMarkdown = httputil.NewError(400, "invalid github markdown")
)

var (
	doxygenApp string

	dataRootDir string
	srcRootDir  string
	tmpRootDir  string

	refreshRootDir string

	mutexs [mutexCount]sync.Mutex
)

func handleHome(w http.ResponseWriter, req *http.Request) {

}

func handleUnknown(w http.ResponseWriter, req *http.Request) {

}

// ---------------------------------------------------

func handleRefresh(w http.ResponseWriter, req *http.Request) {

	pkg := req.PostFormValue("path")
	if pkg == "" {
		httputil.Error(w, ErrRefreshWithoutPath)
		return
	}

	log.Info("Refresh", pkg)

	err := refresh(pkg)
	if err != nil {
		httputil.Error(w, err)
		return
	}

	http.Redirect(w, req, "/" + pkg + "/", 301)
}

func refresh(pkg string) (err error) {

	if strings.Index(pkg, "..") >= 0 {
		return ErrInvalidPkgPath
	}

	parts := strings.SplitN(pkg, "/", 4)
	if len(parts) != 3 {
		return ErrInvalidPkgPath
	}

	dataDir := dataRootDir + pkg
	indexFile := dataDir + "/html/index.html"
	if isRefreshed(indexFile) {
		return nil
	}

	refreshDir := refreshRootDir + pkg
	refreshHtmlDir := refreshDir + "/html/"
	os.RemoveAll(refreshDir)
	err = genDoc(parts, pkg, refreshDir, refreshHtmlDir)
	if err != nil {
		return
	}

	os.RemoveAll(dataDir)
	return os.Rename(refreshDir, dataDir)
}

func isRefreshed(indexFile string) bool {

	fi, err := os.Stat(indexFile)
	if err != nil {
		return false
	}

	return time.Now().Sub(fi.ModTime()) < 10*time.Second
}

// ---------------------------------------------------

func handleMain(w http.ResponseWriter, req *http.Request) {

	path := req.URL.Path

	if path == "/" {
		handleHome(w, req)
		return
	}

	log.Info("View", path, req.URL.RawQuery)

	if strings.Index(path, "..") >= 0 {
		handleUnknown(w, req)
		return
	}

	parts := strings.SplitN(path[1:], "/", 4)
	if parts[0] != "github.com" || len(parts) < 3 {
		handleUnknown(w, req)
		return
	}

	req.ParseForm()
	if _, ok := req.Form["status.svg"]; ok {
		handleBadge(w, req)
		return
	}

	pkg := strings.Join(parts[:3], "/")

	if _, ok := req.Form["tools"]; ok {
		log.Info("handleTools")
		handleTools(w, req, pkg)
		return
	}

	dataDir := dataRootDir + pkg
	htmlDir := dataDir + "/html/"
	err := isEntryExists(htmlDir, true)
	if err != nil {
		err = genDoc(parts, pkg, dataDir, htmlDir)
		if err != nil {
			httputil.Error(w, err)
			return
		}
	}

	if len(parts) > 3 {
		file := htmlDir + parts[3]
		if strings.HasSuffix(file, "/") {
			file += "index.html"
		}
		f, err := os.Open(file)
		if err != nil {
			httputil.Error(w, err)
			return
		}
		defer f.Close()
		fi, err := f.Stat()
		if err != nil {
			httputil.Error(w, err)
			return
		}
		serveContent(w, req, pkg, fi.Name(), fi.ModTime(), f)
	} else {
		http.Redirect(w, req, path + "/", 301)
	}
}

func genDoc(parts []string, pkg, dataDir, htmlDir string) (err error) {

	srcDir := srcRootDir + pkg
	repo := "https://github.com/" + parts[1] + "/" + parts[2] + ".git"

	crc := crc32.ChecksumIEEE([]byte(pkg))
	mutex := &mutexs[crc % mutexCount]
	mutex.Lock()
	defer mutex.Unlock()

	err2 := isEntryExists(htmlDir, true)
	if err2 != nil {
		err = cloneRepo(srcDir, repo)
		if err != nil {
			return
		}

		err = os.MkdirAll(dataDir, 0755)
		if err != nil {
			return
		}

		doxyfile := tmpRootDir + "github.com!" + parts[1] + "!" + parts[2] + ".doxyfile"
		err = genDoxyfileFile(doxyfile, &doxyfileConf{
			ProjectName:  parts[2],
			OutputDir:    dataDir,
			InputDir:     srcDir,
			FilePatterns: "*.md *.dox *.java *.h *.hpp *.hxx *.py *.php *.rb *.cs *.js *.scala *.go *.lua *.asp",
		})
		if err != nil {
			return
		}

		err = runCmd(doxygenApp, doxyfile)
		if err != nil {
			return
		}

		makeMainPage(htmlDir + "index.html", pkg)
	}
	return
}

// ---------------------------------------------------

func cloneRepo(srcDir string, repo string) (err error) {

	os.RemoveAll(srcDir)
	err = os.MkdirAll(srcDir, 0755)
	if err != nil {
		return
	}

	err = runCmd("git", "clone", "--depth=50", repo, srcDir)
	if err != nil {
		return
	}
	return checkoutBranch(srcDir, "master")
}

func checkoutBranch(srcDir string, branch string) (err error) {

	branchMutex.Lock()
	defer branchMutex.Unlock()

	workDir, _ := os.Getwd()
	err = os.Chdir(srcDir)
	if err != nil {
		return
	}
	err = runCmd("git", "checkout", branch)
	os.Chdir(workDir)
	return
}

var (
	branchMutex sync.Mutex
)

// ---------------------------------------------------

func runCmd(command string, args ...string) (err error) {

	cmd := exec.Command(command, args...)

	var out bytes.Buffer
	cmd.Stderr = &out

	err = cmd.Run()
	if err == nil {
		return
	}

	emsg := out.String()
	if emsg != "" {
		return errors.New(emsg)
	}
	return err
}

// ---------------------------------------------------

func isEntryExists(entryPath string, isDir bool) (err error) {

	fi, err := os.Stat(entryPath)
	if err != nil {
		return
	}

	if fi.IsDir() != isDir {
		err = ErrUnmatchedInodeType
		return
	}
	return nil
}

// ---------------------------------------------------

var (
	bindHost = flag.String("http", ":8888", "address that doxygen.io server listen")
)

func main() {

	flag.Parse()

	rootDir := os.Getenv("HOME") + "/.doxygen.io/"
	doxygenApp = os.Getenv("DOXYGEN")
	if doxygenApp == "" {
		doxygenApp = "doxygen"
	}

	dataRootDir = rootDir + "data/"
	refreshRootDir = rootDir + "refresh/"
	srcRootDir = rootDir + "src/"
	tmpRootDir = rootDir + "tmp/"
	os.MkdirAll(tmpRootDir, 0755)

	http.HandleFunc("/-/refresh", handleRefresh)
	http.HandleFunc("/", handleMain)
	err := http.ListenAndServe(*bindHost, nil)
	if err != nil {
		log.Fatal(err)
	}
}

// ---------------------------------------------------

<commit_msg>git clone => git pull<commit_after>package main

import (
	"bytes"
	"errors"
	"flag"
	"hash/crc32"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	"qiniupkg.com/http/httputil.v2"
	"qiniupkg.com/x/log.v7"
)

const (
	mutexCount = 9973
)

var (
	ErrUnmatchedInodeType    = errors.New("unmatched inode type(file or dir)")
	ErrRefreshWithoutPath    = httputil.NewError(400, "refresh without path")
	ErrInvalidPkgPath        = httputil.NewError(400, "invalid package path")
	ErrInvalidGithubMarkdown = httputil.NewError(400, "invalid github markdown")
)

var (
	doxygenApp string

	dataRootDir string
	srcRootDir  string
	tmpRootDir  string

	refreshRootDir string

	mutexs [mutexCount]sync.Mutex
)

func handleHome(w http.ResponseWriter, req *http.Request) {

}

func handleUnknown(w http.ResponseWriter, req *http.Request) {

}

// ---------------------------------------------------

func handleRefresh(w http.ResponseWriter, req *http.Request) {

	pkg := req.PostFormValue("path")
	if pkg == "" {
		httputil.Error(w, ErrRefreshWithoutPath)
		return
	}

	log.Info("Refresh", pkg)

	err := refresh(pkg)
	if err != nil {
		httputil.Error(w, err)
		return
	}

	http.Redirect(w, req, "/" + pkg + "/", 301)
}

func refresh(pkg string) (err error) {

	if strings.Index(pkg, "..") >= 0 {
		return ErrInvalidPkgPath
	}

	parts := strings.SplitN(pkg, "/", 4)
	if len(parts) != 3 {
		return ErrInvalidPkgPath
	}

	dataDir := dataRootDir + pkg
	indexFile := dataDir + "/html/index.html"
	if isRefreshed(indexFile) {
		return nil
	}

	refreshDir := refreshRootDir + pkg
	refreshHtmlDir := refreshDir + "/html/"
	os.RemoveAll(refreshDir)
	err = genDoc(parts, pkg, refreshDir, refreshHtmlDir)
	if err != nil {
		return
	}

	os.RemoveAll(dataDir)
	return os.Rename(refreshDir, dataDir)
}

func isRefreshed(indexFile string) bool {

	fi, err := os.Stat(indexFile)
	if err != nil {
		return false
	}

	return time.Now().Sub(fi.ModTime()) < 10*time.Second
}

// ---------------------------------------------------

func handleMain(w http.ResponseWriter, req *http.Request) {

	path := req.URL.Path

	if path == "/" {
		handleHome(w, req)
		return
	}

	if strings.Index(path, "..") >= 0 {
		handleUnknown(w, req)
		return
	}

	parts := strings.SplitN(path[1:], "/", 4)
	if parts[0] != "github.com" || len(parts) < 3 {
		handleUnknown(w, req)
		return
	}

	req.ParseForm()
	if _, ok := req.Form["status.svg"]; ok {
		handleBadge(w, req)
		return
	}

	pkg := strings.Join(parts[:3], "/")

	if _, ok := req.Form["tools"]; ok {
		log.Info("handleTools")
		handleTools(w, req, pkg)
		return
	}

	dataDir := dataRootDir + pkg
	htmlDir := dataDir + "/html/"
	err := isEntryExists(htmlDir, true)
	if err != nil {
		err = genDoc(parts, pkg, dataDir, htmlDir)
		if err != nil {
			httputil.Error(w, err)
			return
		}
	}

	if len(parts) > 3 {
		file := htmlDir + parts[3]
		if strings.HasSuffix(file, "/") {
			file += "index.html"
		}
		f, err := os.Open(file)
		if err != nil {
			httputil.Error(w, err)
			return
		}
		defer f.Close()
		fi, err := f.Stat()
		if err != nil {
			httputil.Error(w, err)
			return
		}
		serveContent(w, req, pkg, fi.Name(), fi.ModTime(), f)
	} else {
		http.Redirect(w, req, path + "/", 301)
	}
}

func genDoc(parts []string, pkg, dataDir, htmlDir string) (err error) {

	srcDir := srcRootDir + pkg
	repo := "https://github.com/" + parts[1] + "/" + parts[2] + ".git"

	crc := crc32.ChecksumIEEE([]byte(pkg))
	mutex := &mutexs[crc % mutexCount]
	mutex.Lock()
	defer mutex.Unlock()

	err2 := isEntryExists(htmlDir, true)
	if err2 != nil {
		err = cloneRepo(srcDir, repo)
		if err != nil {
			return
		}

		err = os.MkdirAll(dataDir, 0755)
		if err != nil {
			return
		}

		doxyfile := tmpRootDir + "github.com!" + parts[1] + "!" + parts[2] + ".doxyfile"
		err = genDoxyfileFile(doxyfile, &doxyfileConf{
			ProjectName:  parts[2],
			OutputDir:    dataDir,
			InputDir:     srcDir,
			FilePatterns: "*.md *.dox *.java *.h *.hpp *.hxx *.py *.php *.rb *.cs *.js *.scala *.go *.lua *.asp",
		})
		if err != nil {
			return
		}

		err = runCmd(doxygenApp, doxyfile)
		if err != nil {
			return
		}

		makeMainPage(htmlDir + "index.html", pkg)
	}
	return
}

// ---------------------------------------------------

func cloneRepo(srcDir string, repo string) (err error) {

	err = pullRepo(srcDir)
	log.Info("pullRepo", srcDir, "-", err)

	if err != nil {
		os.RemoveAll(srcDir)
		err = os.MkdirAll(srcDir, 0755)
		if err != nil {
			return
		}
		err = runCmd("git", "clone", repo, srcDir)
		if err != nil {
			return
		}
	}
	return checkoutBranch(srcDir, "master")
}

func pullRepo(srcDir string) (err error) {

	gitMutex.Lock()
	defer gitMutex.Unlock()

	workDir, _ := os.Getwd()
	err = os.Chdir(srcDir)
	if err != nil {
		return
	}
	err = runCmd("git", "pull")
	os.Chdir(workDir)
	return
}

func checkoutBranch(srcDir string, branch string) (err error) {

	gitMutex.Lock()
	defer gitMutex.Unlock()

	workDir, _ := os.Getwd()
	err = os.Chdir(srcDir)
	if err != nil {
		return
	}
	err = runCmd("git", "checkout", branch)
	os.Chdir(workDir)
	return
}

var (
	gitMutex sync.Mutex
)

// ---------------------------------------------------

func runCmd(command string, args ...string) (err error) {

	cmd := exec.Command(command, args...)

	var out bytes.Buffer
	cmd.Stderr = &out

	err = cmd.Run()
	if err == nil {
		return
	}

	emsg := out.String()
	if emsg != "" {
		return errors.New(emsg)
	}
	return err
}

// ---------------------------------------------------

func isEntryExists(entryPath string, isDir bool) (err error) {

	fi, err := os.Stat(entryPath)
	if err != nil {
		return
	}

	if fi.IsDir() != isDir {
		err = ErrUnmatchedInodeType
		return
	}
	return nil
}

// ---------------------------------------------------

var (
	bindHost = flag.String("http", ":8888", "address that doxygen.io server listen")
)

func main() {

	flag.Parse()

	rootDir := os.Getenv("HOME") + "/.doxygen.io/"
	doxygenApp = os.Getenv("DOXYGEN")
	if doxygenApp == "" {
		doxygenApp = "doxygen"
	}

	dataRootDir = rootDir + "data/"
	refreshRootDir = rootDir + "refresh/"
	srcRootDir = rootDir + "src/"
	tmpRootDir = rootDir + "tmp/"
	os.MkdirAll(tmpRootDir, 0755)

	http.HandleFunc("/-/refresh", handleRefresh)
	http.HandleFunc("/", handleMain)
	err := http.ListenAndServe(*bindHost, nil)
	if err != nil {
		log.Fatal(err)
	}
}

// ---------------------------------------------------

<|endoftext|>